Projects
Eulaceura:Factory
rocketmq
_service:obs_scm:patch020-backport-add-goaway-m...
Sign Up
Log In
Username
Password
Overview
Repositories
Revisions
Requests
Users
Attributes
Meta
File _service:obs_scm:patch020-backport-add-goaway-mechanism.patch of Package rocketmq
From 84156084a4c5228e1d2fe21e068fff330bbc40d1 Mon Sep 17 00:00:00 2001 From: Zhouxiang Zhan <zhouxzhan@apache.org> Date: Sun, 8 Oct 2023 11:13:25 +0800 Subject: [PATCH 1/7] [ISSUE #7321] Refector NettyRemotingAbstract with unify future implementation (#7322) * Refector NettyRemotingAbstract * Add invoke with future method * Deprecate InvokeCallback#operationComplete * Add operationSuccess and operationException for InvokeCallback * fix unit test * fix unit test * Keep InvokeCallback#operationComplete * Optimize invokeAsyncImpl operationComplete * Add unit test for NettyRemotingClient * fix checkstyle --- .../rocketmq/broker/out/BrokerOuterAPI.java | 147 +++++---- .../rocketmq/client/impl/MQAdminImpl.java | 71 ++-- .../rocketmq/client/impl/MQClientAPIImpl.java | 239 +++++++------- .../client/impl/mqclient/MQClientAPIExt.java | 309 ++++++++---------- .../client/impl/MQClientAPIImplTest.java | 12 +- .../remoting/RemotingProtocolServer.java | 22 +- .../service/mqclient/MQClientAPIExtTest.java | 97 +++--- .../rocketmq/remoting/InvokeCallback.java | 15 + .../rocketmq/remoting/RemotingClient.java | 27 +- .../remoting/netty/NettyRemotingAbstract.java | 123 ++++--- .../remoting/netty/NettyRemotingClient.java | 33 +- .../remoting/netty/ResponseFuture.java | 15 + .../rocketmq/remoting/rpc/RpcClientImpl.java | 29 +- .../rocketmq/remoting/RemotingServerTest.java | 22 +- .../rocketmq/remoting/netty/MockChannel.java | 21 +- .../remoting/netty/MockChannelPromise.java | 191 +++++++++++ .../netty/NettyRemotingAbstractTest.java | 54 ++- .../netty/NettyRemotingClientTest.java | 185 ++++++++++- 18 files changed, 1029 insertions(+), 583 deletions(-) rename client/src/main/java/org/apache/rocketmq/client/impl/BaseInvokeCallback.java => remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannel.java (57%) create mode 100644 remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannelPromise.java diff --git a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java index 9dfb8127d..6fde48dd9 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java @@ -73,6 +73,7 @@ import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; import org.apache.rocketmq.remoting.netty.NettyClientConfig; import org.apache.rocketmq.remoting.netty.NettyRemotingClient; +import org.apache.rocketmq.remoting.netty.ResponseFuture; import org.apache.rocketmq.remoting.protocol.BrokerSyncInfo; import org.apache.rocketmq.remoting.protocol.DataVersion; import org.apache.rocketmq.remoting.protocol.RemotingCommand; @@ -107,6 +108,8 @@ import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader; import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeaderV2; import org.apache.rocketmq.remoting.protocol.header.SendMessageResponseHeader; import org.apache.rocketmq.remoting.protocol.header.controller.AlterSyncStateSetRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterRequestHeader; +import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader; import org.apache.rocketmq.remoting.protocol.header.controller.GetMetaDataResponseHeader; import org.apache.rocketmq.remoting.protocol.header.controller.GetReplicaInfoRequestHeader; import org.apache.rocketmq.remoting.protocol.header.controller.GetReplicaInfoResponseHeader; @@ -124,8 +127,6 @@ import org.apache.rocketmq.remoting.protocol.header.namesrv.RegisterBrokerReques import org.apache.rocketmq.remoting.protocol.header.namesrv.RegisterBrokerResponseHeader; import org.apache.rocketmq.remoting.protocol.header.namesrv.RegisterTopicRequestHeader; import org.apache.rocketmq.remoting.protocol.header.namesrv.UnRegisterBrokerRequestHeader; -import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterRequestHeader; -import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader; import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData; import org.apache.rocketmq.remoting.protocol.namesrv.RegisterBrokerResult; import org.apache.rocketmq.remoting.protocol.route.BrokerData; @@ -151,7 +152,6 @@ public class BrokerOuterAPI { private final RpcClient rpcClient; private String nameSrvAddr = null; - public BrokerOuterAPI(final NettyClientConfig nettyClientConfig) { this(nettyClientConfig, new DynamicalExtFieldRPCHook(), new ClientMetadata()); } @@ -459,7 +459,7 @@ public class BrokerOuterAPI { * @param filterServerList * @param oneway * @param timeoutMills - * @param compressed default false + * @param compressed default false * @return */ public List<RegisterBrokerResult> registerBrokerAll( @@ -643,7 +643,6 @@ public class BrokerOuterAPI { queueDatas.add(queueData); final byte[] topicRouteBody = topicRouteData.encode(); - List<String> nameServerAddressList = this.remotingClient.getNameServerAddressList(); final CountDownLatch countDownLatch = new CountDownLatch(nameServerAddressList.size()); for (final String namesrvAddr : nameServerAddressList) { @@ -910,25 +909,33 @@ public class BrokerOuterAPI { RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.LOCK_BATCH_MQ, null); request.setBody(requestBody.encode()); - this.remotingClient.invokeAsync(addr, request, timeoutMillis, responseFuture -> { - if (callback == null) { - return; + this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() { + @Override + public void operationComplete(ResponseFuture responseFuture) { + } - try { - RemotingCommand response = responseFuture.getResponseCommand(); - if (response != null) { - if (response.getCode() == ResponseCode.SUCCESS) { - LockBatchResponseBody responseBody = LockBatchResponseBody.decode(response.getBody(), - LockBatchResponseBody.class); - Set<MessageQueue> messageQueues = responseBody.getLockOKMQSet(); - callback.onSuccess(messageQueues); - } else { - callback.onException(new MQBrokerException(response.getCode(), response.getRemark())); - } + @Override + public void operationSucceed(RemotingCommand response) { + if (callback == null) { + return; } - } catch (Throwable ignored) { + if (response.getCode() == ResponseCode.SUCCESS) { + LockBatchResponseBody responseBody = LockBatchResponseBody.decode(response.getBody(), + LockBatchResponseBody.class); + Set<MessageQueue> messageQueues = responseBody.getLockOKMQSet(); + callback.onSuccess(messageQueues); + } else { + callback.onException(new MQBrokerException(response.getCode(), response.getRemark())); + } + } + @Override + public void operationFail(Throwable throwable) { + if (callback == null) { + return; + } + callback.onException(throwable); } }); } @@ -942,22 +949,30 @@ public class BrokerOuterAPI { request.setBody(requestBody.encode()); - this.remotingClient.invokeAsync(addr, request, timeoutMillis, responseFuture -> { - if (callback == null) { - return; + this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() { + @Override + public void operationComplete(ResponseFuture responseFuture) { + } - try { - RemotingCommand response = responseFuture.getResponseCommand(); - if (response != null) { - if (response.getCode() == ResponseCode.SUCCESS) { - callback.onSuccess(); - } else { - callback.onException(new MQBrokerException(response.getCode(), response.getRemark())); - } + @Override + public void operationSucceed(RemotingCommand response) { + if (callback == null) { + return; } - } catch (Throwable ignored) { + if (response.getCode() == ResponseCode.SUCCESS) { + callback.onSuccess(); + } else { + callback.onException(new MQBrokerException(response.getCode(), response.getRemark())); + } + } + @Override + public void operationFail(Throwable throwable) { + if (callback == null) { + return; + } + callback.onException(throwable); } }); } @@ -983,21 +998,27 @@ public class BrokerOuterAPI { CompletableFuture<SendResult> cf = new CompletableFuture<>(); final String msgId = msg.getMsgId(); try { - this.remotingClient.invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { - RemotingCommand response = responseFuture.getResponseCommand(); - if (null != response) { - SendResult sendResult = null; + this.remotingClient.invokeAsync(brokerAddr, request, timeoutMillis, new InvokeCallback() { + @Override + public void operationComplete(ResponseFuture responseFuture) { + + } + + @Override + public void operationSucceed(RemotingCommand response) { try { - sendResult = this.processSendResponse(brokerName, msg, response); + SendResult sendResult = processSendResponse(brokerName, msg, response); cf.complete(sendResult); } catch (MQBrokerException | RemotingCommandException e) { LOGGER.error("processSendResponse in sendMessageToSpecificBrokerAsync failed, msgId=" + msgId, e); cf.completeExceptionally(e); } - } else { - cf.complete(null); } + @Override + public void operationFail(Throwable throwable) { + cf.completeExceptionally(throwable); + } }); } catch (Throwable t) { LOGGER.error("invokeAsync failed in sendMessageToSpecificBrokerAsync, msgId=" + msgId, t); @@ -1057,7 +1078,7 @@ public class BrokerOuterAPI { } if (sendStatus != null) { SendMessageResponseHeader responseHeader = - (SendMessageResponseHeader) response.decodeCommandCustomHeader(SendMessageResponseHeader.class); + (SendMessageResponseHeader) response.decodeCommandCustomHeader(SendMessageResponseHeader.class); //If namespace not null , reset Topic without namespace. String topic = msg.getTopic(); @@ -1073,8 +1094,8 @@ public class BrokerOuterAPI { uniqMsgId = sb.toString(); } SendResult sendResult = new SendResult(sendStatus, - uniqMsgId, - responseHeader.getMsgId(), messageQueue, responseHeader.getQueueOffset()); + uniqMsgId, + responseHeader.getMsgId(), messageQueue, responseHeader.getQueueOffset()); sendResult.setTransactionId(responseHeader.getTransactionId()); String regionId = response.getExtFields().get(MessageConst.PROPERTY_MSG_REGION); String traceOn = response.getExtFields().get(MessageConst.PROPERTY_TRACE_SWITCH); @@ -1218,8 +1239,9 @@ public class BrokerOuterAPI { /** * Broker try to elect itself as a master in broker set */ - public Pair<ElectMasterResponseHeader, Set<Long>> brokerElect(String controllerAddress, String clusterName, String brokerName, - Long brokerId) throws Exception { + public Pair<ElectMasterResponseHeader, Set<Long>> brokerElect(String controllerAddress, String clusterName, + String brokerName, + Long brokerId) throws Exception { final ElectMasterRequestHeader requestHeader = ElectMasterRequestHeader.ofBrokerTrigger(clusterName, brokerName, brokerId); RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_ELECT_MASTER, requestHeader); @@ -1237,7 +1259,8 @@ public class BrokerOuterAPI { throw new MQBrokerException(response.getCode(), response.getRemark()); } - public GetNextBrokerIdResponseHeader getNextBrokerId(final String clusterName, final String brokerName, final String controllerAddress) throws Exception { + public GetNextBrokerIdResponseHeader getNextBrokerId(final String clusterName, final String brokerName, + final String controllerAddress) throws Exception { final GetNextBrokerIdRequestHeader requestHeader = new GetNextBrokerIdRequestHeader(clusterName, brokerName); final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_GET_NEXT_BROKER_ID, requestHeader); final RemotingCommand response = this.remotingClient.invokeSync(controllerAddress, request, 3000); @@ -1248,7 +1271,8 @@ public class BrokerOuterAPI { throw new MQBrokerException(response.getCode(), response.getRemark()); } - public ApplyBrokerIdResponseHeader applyBrokerId(final String clusterName, final String brokerName, final Long brokerId, final String registerCheckCode, final String controllerAddress) throws Exception { + public ApplyBrokerIdResponseHeader applyBrokerId(final String clusterName, final String brokerName, + final Long brokerId, final String registerCheckCode, final String controllerAddress) throws Exception { final ApplyBrokerIdRequestHeader requestHeader = new ApplyBrokerIdRequestHeader(clusterName, brokerName, brokerId, registerCheckCode); final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_APPLY_BROKER_ID, requestHeader); final RemotingCommand response = this.remotingClient.invokeSync(controllerAddress, request, 3000); @@ -1259,7 +1283,9 @@ public class BrokerOuterAPI { throw new MQBrokerException(response.getCode(), response.getRemark()); } - public Pair<RegisterBrokerToControllerResponseHeader, Set<Long>> registerBrokerToController(final String clusterName, final String brokerName, final Long brokerId, final String brokerAddress, final String controllerAddress) throws Exception { + public Pair<RegisterBrokerToControllerResponseHeader, Set<Long>> registerBrokerToController( + final String clusterName, final String brokerName, final Long brokerId, final String brokerAddress, + final String controllerAddress) throws Exception { final RegisterBrokerToControllerRequestHeader requestHeader = new RegisterBrokerToControllerRequestHeader(clusterName, brokerName, brokerId, brokerAddress); final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_REGISTER_BROKER, requestHeader); final RemotingCommand response = this.remotingClient.invokeSync(controllerAddress, request, 3000); @@ -1355,16 +1381,25 @@ public class BrokerOuterAPI { RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, requestHeader); CompletableFuture<PullResult> pullResultFuture = new CompletableFuture<>(); - this.remotingClient.invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { - if (responseFuture.getCause() != null) { - pullResultFuture.complete(new PullResult(PullStatus.NO_MATCHED_MSG, -1, -1, -1, new ArrayList<>())); - return; + this.remotingClient.invokeAsync(brokerAddr, request, timeoutMillis, new InvokeCallback() { + @Override + public void operationComplete(ResponseFuture responseFuture) { + } - try { - PullResultExt pullResultExt = this.processPullResponse(responseFuture.getResponseCommand(), brokerAddr); - this.processPullResult(pullResultExt, brokerName, queueId); - pullResultFuture.complete(pullResultExt); - } catch (Exception e) { + + @Override + public void operationSucceed(RemotingCommand response) { + try { + PullResultExt pullResultExt = processPullResponse(response, brokerAddr); + processPullResult(pullResultExt, brokerName, queueId); + pullResultFuture.complete(pullResultExt); + } catch (Exception e) { + pullResultFuture.complete(new PullResult(PullStatus.NO_MATCHED_MSG, -1, -1, -1, new ArrayList<>())); + } + } + + @Override + public void operationFail(Throwable throwable) { pullResultFuture.complete(new PullResult(PullStatus.NO_MATCHED_MSG, -1, -1, -1, new ArrayList<>())); } }); diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java index 1ef3a9483..83835bd3d 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java @@ -44,6 +44,8 @@ import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageId; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.utils.NetworkUtil; +import org.apache.rocketmq.logging.org.slf4j.Logger; +import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.remoting.InvokeCallback; import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.exception.RemotingException; @@ -55,8 +57,6 @@ import org.apache.rocketmq.remoting.protocol.header.QueryMessageRequestHeader; import org.apache.rocketmq.remoting.protocol.header.QueryMessageResponseHeader; import org.apache.rocketmq.remoting.protocol.route.BrokerData; import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; -import org.apache.rocketmq.logging.org.slf4j.Logger; -import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; public class MQAdminImpl { @@ -357,44 +357,51 @@ public class MQAdminImpl { new InvokeCallback() { @Override public void operationComplete(ResponseFuture responseFuture) { + + } + + @Override + public void operationSucceed(RemotingCommand response) { try { - RemotingCommand response = responseFuture.getResponseCommand(); - if (response != null) { - switch (response.getCode()) { - case ResponseCode.SUCCESS: { - QueryMessageResponseHeader responseHeader = null; - try { - responseHeader = - (QueryMessageResponseHeader) response - .decodeCommandCustomHeader(QueryMessageResponseHeader.class); - } catch (RemotingCommandException e) { - log.error("decodeCommandCustomHeader exception", e); - return; - } - - List<MessageExt> wrappers = - MessageDecoder.decodes(ByteBuffer.wrap(response.getBody()), true); - - QueryResult qr = new QueryResult(responseHeader.getIndexLastUpdateTimestamp(), wrappers); - try { - lock.writeLock().lock(); - queryResultList.add(qr); - } finally { - lock.writeLock().unlock(); - } - break; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + QueryMessageResponseHeader responseHeader = null; + try { + responseHeader = + (QueryMessageResponseHeader) response + .decodeCommandCustomHeader(QueryMessageResponseHeader.class); + } catch (RemotingCommandException e) { + log.error("decodeCommandCustomHeader exception", e); + return; } - default: - log.warn("getResponseCommand failed, {} {}", response.getCode(), response.getRemark()); - break; + + List<MessageExt> wrappers = + MessageDecoder.decodes(ByteBuffer.wrap(response.getBody()), true); + + QueryResult qr = new QueryResult(responseHeader.getIndexLastUpdateTimestamp(), wrappers); + try { + lock.writeLock().lock(); + queryResultList.add(qr); + } finally { + lock.writeLock().unlock(); + } + break; } - } else { - log.warn("getResponseCommand return null"); + default: + log.warn("getResponseCommand failed, {} {}", response.getCode(), response.getRemark()); + break; } + } finally { countDownLatch.countDown(); } } + + @Override + public void operationFail(Throwable throwable) { + log.error("queryMessage error, requestHeader={}", requestHeader); + countDownLatch.countDown(); + } }, isUniqKey); } catch (Exception e) { log.warn("queryMessage exception", e); diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java index 3201a493f..2407e5737 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java @@ -33,7 +33,6 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.Validators; -import org.apache.rocketmq.client.common.ClientErrorCode; import org.apache.rocketmq.client.consumer.AckCallback; import org.apache.rocketmq.client.consumer.AckResult; import org.apache.rocketmq.client.consumer.AckStatus; @@ -653,10 +652,13 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() { @Override public void operationComplete(ResponseFuture responseFuture) { - long cost = System.currentTimeMillis() - beginStartTime; - RemotingCommand response = responseFuture.getResponseCommand(); - if (null == sendCallback && response != null) { + } + + @Override + public void operationSucceed(RemotingCommand response) { + long cost = System.currentTimeMillis() - beginStartTime; + if (null == sendCallback) { try { SendResult sendResult = MQClientAPIImpl.this.processSendResponse(brokerName, msg, response, addr); if (context != null && sendResult != null) { @@ -666,46 +668,47 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { } catch (Throwable e) { } - producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), false, true); + producer.updateFaultItem(brokerName, System.currentTimeMillis() - beginStartTime, false, true); return; } - if (response != null) { + try { + SendResult sendResult = MQClientAPIImpl.this.processSendResponse(brokerName, msg, response, addr); + assert sendResult != null; + if (context != null) { + context.setSendResult(sendResult); + context.getProducer().executeSendMessageHookAfter(context); + } + try { - SendResult sendResult = MQClientAPIImpl.this.processSendResponse(brokerName, msg, response, addr); - assert sendResult != null; - if (context != null) { - context.setSendResult(sendResult); - context.getProducer().executeSendMessageHookAfter(context); - } + sendCallback.onSuccess(sendResult); + } catch (Throwable e) { + } - try { - sendCallback.onSuccess(sendResult); - } catch (Throwable e) { - } + producer.updateFaultItem(brokerName, System.currentTimeMillis() - beginStartTime, false, true); + } catch (Exception e) { + producer.updateFaultItem(brokerName, System.currentTimeMillis() - beginStartTime, true, true); + onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, + retryTimesWhenSendFailed, times, e, context, false, producer); + } + } - producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), false, true); - } catch (Exception e) { - producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), true, true); - onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, - retryTimesWhenSendFailed, times, e, context, false, producer); - } + @Override + public void operationFail(Throwable throwable) { + producer.updateFaultItem(brokerName, System.currentTimeMillis() - beginStartTime, true, true); + long cost = System.currentTimeMillis() - beginStartTime; + if (throwable instanceof RemotingSendRequestException) { + MQClientException ex = new MQClientException("send request failed", throwable); + onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, + retryTimesWhenSendFailed, times, ex, context, true, producer); + } else if (throwable instanceof RemotingTimeoutException) { + MQClientException ex = new MQClientException("wait response timeout, cost=" + cost, throwable); + onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, + retryTimesWhenSendFailed, times, ex, context, true, producer); } else { - producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), true, true); - if (!responseFuture.isSendRequestOK()) { - MQClientException ex = new MQClientException("send request failed", responseFuture.getCause()); - onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, - retryTimesWhenSendFailed, times, ex, context, true, producer); - } else if (responseFuture.isTimeout()) { - MQClientException ex = new MQClientException("wait response timeout " + responseFuture.getTimeoutMillis() + "ms", - responseFuture.getCause()); - onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, - retryTimesWhenSendFailed, times, ex, context, true, producer); - } else { - MQClientException ex = new MQClientException("unknow reseaon", responseFuture.getCause()); - onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, - retryTimesWhenSendFailed, times, ex, context, true, producer); - } + MQClientException ex = new MQClientException("unknow reseaon", throwable); + onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, + retryTimesWhenSendFailed, times, ex, context, true, producer); } } }); @@ -857,30 +860,25 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { final long timeoutMillis, final PopCallback popCallback ) throws RemotingException, InterruptedException { final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.POP_MESSAGE, requestHeader); - this.remotingClient.invokeAsync(addr, request, timeoutMillis, new BaseInvokeCallback(MQClientAPIImpl.this) { + this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() { @Override - public void onComplete(ResponseFuture responseFuture) { - RemotingCommand response = responseFuture.getResponseCommand(); - if (response != null) { - try { - PopResult - popResult = MQClientAPIImpl.this.processPopResponse(brokerName, response, requestHeader.getTopic(), requestHeader); - assert popResult != null; - popCallback.onSuccess(popResult); - } catch (Exception e) { - popCallback.onException(e); - } - } else { - if (!responseFuture.isSendRequestOK()) { - popCallback.onException(new MQClientException(ClientErrorCode.CONNECT_BROKER_EXCEPTION, "send request failed to " + addr + ". Request: " + request, responseFuture.getCause())); - } else if (responseFuture.isTimeout()) { - popCallback.onException(new MQClientException(ClientErrorCode.ACCESS_BROKER_TIMEOUT, "wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request, - responseFuture.getCause())); - } else { - popCallback.onException(new MQClientException("unknown reason. addr: " + addr + ", timeoutMillis: " + timeoutMillis + ". Request: " + request, responseFuture.getCause())); - } + public void operationComplete(ResponseFuture responseFuture) { + + } + + @Override + public void operationSucceed(RemotingCommand response) { + try { + PopResult popResult = MQClientAPIImpl.this.processPopResponse(brokerName, response, requestHeader.getTopic(), requestHeader); + popCallback.onSuccess(popResult); + } catch (Exception e) { + popCallback.onException(e); } } + @Override + public void operationFail(Throwable throwable) { + popCallback.onException(throwable); + } }); } @@ -959,34 +957,26 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { request.setBody(requestBody.encode()); } } - this.remotingClient.invokeAsync(addr, request, timeOut, new BaseInvokeCallback(MQClientAPIImpl.this) { + this.remotingClient.invokeAsync(addr, request, timeOut, new InvokeCallback() { + @Override + public void operationComplete(ResponseFuture responseFuture) { + + } @Override - public void onComplete(ResponseFuture responseFuture) { - RemotingCommand response = responseFuture.getResponseCommand(); - if (response != null) { - try { - AckResult ackResult = new AckResult(); - if (ResponseCode.SUCCESS == response.getCode()) { - ackResult.setStatus(AckStatus.OK); - } else { - ackResult.setStatus(AckStatus.NO_EXIST); - } - ackCallback.onSuccess(ackResult); - } catch (Exception e) { - ackCallback.onException(e); - } + public void operationSucceed(RemotingCommand response) { + AckResult ackResult = new AckResult(); + if (ResponseCode.SUCCESS == response.getCode()) { + ackResult.setStatus(AckStatus.OK); } else { - if (!responseFuture.isSendRequestOK()) { - ackCallback.onException(new MQClientException(ClientErrorCode.CONNECT_BROKER_EXCEPTION, "send request failed to " + addr + ". Request: " + request, responseFuture.getCause())); - } else if (responseFuture.isTimeout()) { - ackCallback.onException(new MQClientException(ClientErrorCode.ACCESS_BROKER_TIMEOUT, "wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request, - responseFuture.getCause())); - } else { - ackCallback.onException(new MQClientException("unknown reason. addr: " + addr + ", timeoutMillis: " + timeOut + ". Request: " + request, responseFuture.getCause())); - } + ackResult.setStatus(AckStatus.NO_EXIST); } + ackCallback.onSuccess(ackResult); + } + @Override + public void operationFail(Throwable throwable) { + ackCallback.onException(throwable); } }); } @@ -999,39 +989,37 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { final AckCallback ackCallback ) throws RemotingException, MQBrokerException, InterruptedException { final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CHANGE_MESSAGE_INVISIBLETIME, requestHeader); - this.remotingClient.invokeAsync(addr, request, timeoutMillis, new BaseInvokeCallback(MQClientAPIImpl.this) { + this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() { @Override - public void onComplete(ResponseFuture responseFuture) { - RemotingCommand response = responseFuture.getResponseCommand(); - if (response != null) { - try { - ChangeInvisibleTimeResponseHeader responseHeader = (ChangeInvisibleTimeResponseHeader) response.decodeCommandCustomHeader(ChangeInvisibleTimeResponseHeader.class); - AckResult ackResult = new AckResult(); - if (ResponseCode.SUCCESS == response.getCode()) { - ackResult.setStatus(AckStatus.OK); - ackResult.setPopTime(responseHeader.getPopTime()); - ackResult.setExtraInfo(ExtraInfoUtil - .buildExtraInfo(requestHeader.getOffset(), responseHeader.getPopTime(), responseHeader.getInvisibleTime(), - responseHeader.getReviveQid(), requestHeader.getTopic(), brokerName, requestHeader.getQueueId()) + MessageConst.KEY_SEPARATOR - + requestHeader.getOffset()); - } else { - ackResult.setStatus(AckStatus.NO_EXIST); - } - ackCallback.onSuccess(ackResult); - } catch (Exception e) { - ackCallback.onException(e); - } - } else { - if (!responseFuture.isSendRequestOK()) { - ackCallback.onException(new MQClientException(ClientErrorCode.CONNECT_BROKER_EXCEPTION, "send request failed to " + addr + ". Request: " + request, responseFuture.getCause())); - } else if (responseFuture.isTimeout()) { - ackCallback.onException(new MQClientException(ClientErrorCode.ACCESS_BROKER_TIMEOUT, "wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request, - responseFuture.getCause())); + public void operationComplete(ResponseFuture responseFuture) { + + } + + @Override + public void operationSucceed(RemotingCommand response) { + try { + ChangeInvisibleTimeResponseHeader responseHeader = (ChangeInvisibleTimeResponseHeader) response.decodeCommandCustomHeader(ChangeInvisibleTimeResponseHeader.class); + AckResult ackResult = new AckResult(); + if (ResponseCode.SUCCESS == response.getCode()) { + ackResult.setStatus(AckStatus.OK); + ackResult.setPopTime(responseHeader.getPopTime()); + ackResult.setExtraInfo(ExtraInfoUtil + .buildExtraInfo(requestHeader.getOffset(), responseHeader.getPopTime(), responseHeader.getInvisibleTime(), + responseHeader.getReviveQid(), requestHeader.getTopic(), brokerName, requestHeader.getQueueId()) + MessageConst.KEY_SEPARATOR + + requestHeader.getOffset()); } else { - ackCallback.onException(new MQClientException("unknown reason. addr: " + addr + ", timeoutMillis: " + timeoutMillis + ". Request: " + request, responseFuture.getCause())); + ackResult.setStatus(AckStatus.NO_EXIST); } + ackCallback.onSuccess(ackResult); + } catch (Exception e) { + ackCallback.onException(e); } } + + @Override + public void operationFail(Throwable throwable) { + ackCallback.onException(throwable); + } }); } @@ -1044,26 +1032,23 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() { @Override public void operationComplete(ResponseFuture responseFuture) { - RemotingCommand response = responseFuture.getResponseCommand(); - if (response != null) { - try { - PullResult pullResult = MQClientAPIImpl.this.processPullResponse(response, addr); - assert pullResult != null; - pullCallback.onSuccess(pullResult); - } catch (Exception e) { - pullCallback.onException(e); - } - } else { - if (!responseFuture.isSendRequestOK()) { - pullCallback.onException(new MQClientException(ClientErrorCode.CONNECT_BROKER_EXCEPTION, "send request failed to " + addr + ". Request: " + request, responseFuture.getCause())); - } else if (responseFuture.isTimeout()) { - pullCallback.onException(new MQClientException(ClientErrorCode.ACCESS_BROKER_TIMEOUT, "wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request, - responseFuture.getCause())); - } else { - pullCallback.onException(new MQClientException("unknown reason. addr: " + addr + ", timeoutMillis: " + timeoutMillis + ". Request: " + request, responseFuture.getCause())); - } + + } + + @Override + public void operationSucceed(RemotingCommand response) { + try { + PullResult pullResult = MQClientAPIImpl.this.processPullResponse(response, addr); + pullCallback.onSuccess(pullResult); + } catch (Exception e) { + pullCallback.onException(e); } } + + @Override + public void operationFail(Throwable throwable) { + pullCallback.onException(throwable); + } }); } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java b/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java index d7c8ef8d9..f3102e175 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java @@ -30,7 +30,6 @@ import org.apache.rocketmq.client.consumer.PullCallback; import org.apache.rocketmq.client.consumer.PullResult; import org.apache.rocketmq.client.consumer.PullStatus; import org.apache.rocketmq.client.exception.MQBrokerException; -import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.exception.OffsetNotFoundException; import org.apache.rocketmq.client.impl.ClientRemotingProcessor; import org.apache.rocketmq.client.impl.CommunicationMode; @@ -47,6 +46,7 @@ import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +import org.apache.rocketmq.remoting.InvokeCallback; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.netty.NettyClientConfig; @@ -106,19 +106,6 @@ public class MQClientAPIExt extends MQClientAPIImpl { return false; } - protected static MQClientException processNullResponseErr(ResponseFuture responseFuture) { - MQClientException ex; - if (!responseFuture.isSendRequestOK()) { - ex = new MQClientException("send request failed", responseFuture.getCause()); - } else if (responseFuture.isTimeout()) { - ex = new MQClientException("wait response timeout " + responseFuture.getTimeoutMillis() + "ms", - responseFuture.getCause()); - } else { - ex = new MQClientException("unknown reason", responseFuture.getCause()); - } - return ex; - } - public CompletableFuture<Void> sendHeartbeatOneway( String brokerAddr, HeartbeatData heartbeatData, @@ -146,24 +133,15 @@ public class MQClientAPIExt extends MQClientAPIImpl { request.setLanguage(clientConfig.getLanguage()); request.setBody(heartbeatData.encode()); - CompletableFuture<Integer> future = new CompletableFuture<>(); - try { - this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { - RemotingCommand response = responseFuture.getResponseCommand(); - if (response != null) { - if (ResponseCode.SUCCESS == response.getCode()) { - future.complete(response.getVersion()); - } else { - future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark(), brokerAddr)); - } - } else { - future.completeExceptionally(processNullResponseErr(responseFuture)); - } - }); - } catch (Throwable t) { - future.completeExceptionally(t); - } - return future; + return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> { + CompletableFuture<Integer> future0 = new CompletableFuture<>(); + if (ResponseCode.SUCCESS == response.getCode()) { + future0.complete(response.getVersion()); + } else { + future0.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark(), brokerAddr)); + } + return future0; + }); } public CompletableFuture<SendResult> sendMessageAsync( @@ -177,24 +155,15 @@ public class MQClientAPIExt extends MQClientAPIImpl { RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE_V2, requestHeaderV2); request.setBody(msg.getBody()); - CompletableFuture<SendResult> future = new CompletableFuture<>(); - try { - this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { - RemotingCommand response = responseFuture.getResponseCommand(); - if (response != null) { - try { - future.complete(this.processSendResponse(brokerName, msg, response, brokerAddr)); - } catch (Exception e) { - future.completeExceptionally(e); - } - } else { - future.completeExceptionally(processNullResponseErr(responseFuture)); - } - }); - } catch (Throwable t) { - future.completeExceptionally(t); - } - return future; + return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> { + CompletableFuture<SendResult> future0 = new CompletableFuture<>(); + try { + future0.complete(this.processSendResponse(brokerName, msg, response, brokerAddr)); + } catch (Exception e) { + future0.completeExceptionally(e); + } + return future0; + }); } public CompletableFuture<SendResult> sendMessageAsync( @@ -216,17 +185,14 @@ public class MQClientAPIExt extends MQClientAPIImpl { msgBatch.setBody(body); request.setBody(body); - this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { - RemotingCommand response = responseFuture.getResponseCommand(); - if (response != null) { - try { - future.complete(this.processSendResponse(brokerName, msgBatch, response, brokerAddr)); - } catch (Exception e) { - future.completeExceptionally(e); - } - } else { - future.completeExceptionally(processNullResponseErr(responseFuture)); + return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> { + CompletableFuture<SendResult> future0 = new CompletableFuture<>(); + try { + future0.complete(processSendResponse(brokerName, msgBatch, response, brokerAddr)); + } catch (Exception e) { + future0.completeExceptionally(e); } + return future0; }); } catch (Throwable t) { future.completeExceptionally(t); @@ -240,21 +206,7 @@ public class MQClientAPIExt extends MQClientAPIImpl { long timeoutMillis ) { RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONSUMER_SEND_MSG_BACK, requestHeader); - - CompletableFuture<RemotingCommand> future = new CompletableFuture<>(); - try { - this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { - RemotingCommand response = responseFuture.getResponseCommand(); - if (response != null) { - future.complete(response); - } else { - future.completeExceptionally(processNullResponseErr(responseFuture)); - } - }); - } catch (Throwable t) { - future.completeExceptionally(t); - } - return future; + return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis); } public CompletableFuture<PopResult> popMessageAsync( @@ -402,38 +354,31 @@ public class MQClientAPIExt extends MQClientAPIImpl { QueryConsumerOffsetRequestHeader requestHeader, long timeoutMillis ) { - CompletableFuture<Long> future = new CompletableFuture<>(); - try { - RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_CONSUMER_OFFSET, requestHeader); - this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { - RemotingCommand response = responseFuture.getResponseCommand(); - if (response != null) { - switch (response.getCode()) { - case ResponseCode.SUCCESS: { - try { - QueryConsumerOffsetResponseHeader responseHeader = - (QueryConsumerOffsetResponseHeader) response.decodeCommandCustomHeader(QueryConsumerOffsetResponseHeader.class); - future.complete(responseHeader.getOffset()); - } catch (RemotingCommandException e) { - future.completeExceptionally(e); - } - break; - } - case ResponseCode.QUERY_NOT_FOUND: { - future.completeExceptionally(new OffsetNotFoundException(response.getCode(), response.getRemark(), brokerAddr)); - break; - } - default: - break; + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_CONSUMER_OFFSET, requestHeader); + return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> { + CompletableFuture<Long> future0 = new CompletableFuture<>(); + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + try { + QueryConsumerOffsetResponseHeader responseHeader = + (QueryConsumerOffsetResponseHeader) response.decodeCommandCustomHeader(QueryConsumerOffsetResponseHeader.class); + future0.complete(responseHeader.getOffset()); + } catch (RemotingCommandException e) { + future0.completeExceptionally(e); } - } else { - future.completeExceptionally(processNullResponseErr(responseFuture)); + break; } - }); - } catch (Throwable t) { - future.completeExceptionally(t); - } - return future; + case ResponseCode.QUERY_NOT_FOUND: { + future0.completeExceptionally(new OffsetNotFoundException(response.getCode(), response.getRemark(), brokerAddr)); + break; + } + default: { + future0.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); + break; + } + } + return future0; + }); } public CompletableFuture<Void> updateConsumerOffsetOneWay( @@ -461,9 +406,14 @@ public class MQClientAPIExt extends MQClientAPIImpl { CompletableFuture<List<String>> future = new CompletableFuture<>(); try { - this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { - RemotingCommand response = responseFuture.getResponseCommand(); - if (response != null) { + this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, new InvokeCallback() { + @Override + public void operationComplete(ResponseFuture responseFuture) { + + } + + @Override + public void operationSucceed(RemotingCommand response) { switch (response.getCode()) { case ResponseCode.SUCCESS: { if (response.getBody() != null) { @@ -485,8 +435,11 @@ public class MQClientAPIExt extends MQClientAPIImpl { break; } future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); - } else { - future.completeExceptionally(processNullResponseErr(responseFuture)); + } + + @Override + public void operationFail(Throwable throwable) { + future.completeExceptionally(throwable); } }); } catch (Throwable t) { @@ -501,9 +454,14 @@ public class MQClientAPIExt extends MQClientAPIImpl { CompletableFuture<Long> future = new CompletableFuture<>(); try { - this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { - RemotingCommand response = responseFuture.getResponseCommand(); - if (response != null) { + this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, new InvokeCallback() { + @Override + public void operationComplete(ResponseFuture responseFuture) { + + } + + @Override + public void operationSucceed(RemotingCommand response) { if (ResponseCode.SUCCESS == response.getCode()) { try { GetMaxOffsetResponseHeader responseHeader = (GetMaxOffsetResponseHeader) response.decodeCommandCustomHeader(GetMaxOffsetResponseHeader.class); @@ -513,8 +471,11 @@ public class MQClientAPIExt extends MQClientAPIImpl { } } future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); - } else { - future.completeExceptionally(processNullResponseErr(responseFuture)); + } + + @Override + public void operationFail(Throwable throwable) { + future.completeExceptionally(throwable); } }); } catch (Throwable t) { @@ -529,9 +490,14 @@ public class MQClientAPIExt extends MQClientAPIImpl { CompletableFuture<Long> future = new CompletableFuture<>(); try { - this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { - RemotingCommand response = responseFuture.getResponseCommand(); - if (response != null) { + this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, new InvokeCallback() { + @Override + public void operationComplete(ResponseFuture responseFuture) { + + } + + @Override + public void operationSucceed(RemotingCommand response) { if (ResponseCode.SUCCESS == response.getCode()) { try { GetMinOffsetResponseHeader responseHeader = (GetMinOffsetResponseHeader) response.decodeCommandCustomHeader(GetMinOffsetResponseHeader.class); @@ -541,8 +507,11 @@ public class MQClientAPIExt extends MQClientAPIImpl { } } future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); - } else { - future.completeExceptionally(processNullResponseErr(responseFuture)); + } + + @Override + public void operationFail(Throwable throwable) { + future.completeExceptionally(throwable); } }); } catch (Throwable t) { @@ -555,57 +524,41 @@ public class MQClientAPIExt extends MQClientAPIImpl { long timeoutMillis) { RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEARCH_OFFSET_BY_TIMESTAMP, requestHeader); - CompletableFuture<Long> future = new CompletableFuture<>(); - try { - this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { - RemotingCommand response = responseFuture.getResponseCommand(); - if (response != null) { - if (response.getCode() == ResponseCode.SUCCESS) { - try { - SearchOffsetResponseHeader responseHeader = (SearchOffsetResponseHeader) response.decodeCommandCustomHeader(SearchOffsetResponseHeader.class); - future.complete(responseHeader.getOffset()); - } catch (Throwable t) { - future.completeExceptionally(t); - } - } - future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); - } else { - future.completeExceptionally(processNullResponseErr(responseFuture)); + return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> { + CompletableFuture<Long> future0 = new CompletableFuture<>(); + if (response.getCode() == ResponseCode.SUCCESS) { + try { + SearchOffsetResponseHeader responseHeader = (SearchOffsetResponseHeader) response.decodeCommandCustomHeader(SearchOffsetResponseHeader.class); + future0.complete(responseHeader.getOffset()); + } catch (Throwable t) { + future0.completeExceptionally(t); } - }); - } catch (Throwable t) { - future.completeExceptionally(t); - } - return future; + } else { + future0.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); + } + return future0; + }); } public CompletableFuture<Set<MessageQueue>> lockBatchMQWithFuture(String brokerAddr, LockBatchRequestBody requestBody, long timeoutMillis) { - CompletableFuture<Set<MessageQueue>> future = new CompletableFuture<>(); RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.LOCK_BATCH_MQ, null); request.setBody(requestBody.encode()); - try { - this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { - RemotingCommand response = responseFuture.getResponseCommand(); - if (response != null) { - if (response.getCode() == ResponseCode.SUCCESS) { - try { - LockBatchResponseBody responseBody = LockBatchResponseBody.decode(response.getBody(), LockBatchResponseBody.class); - Set<MessageQueue> messageQueues = responseBody.getLockOKMQSet(); - future.complete(messageQueues); - } catch (Throwable t) { - future.completeExceptionally(t); - } - } - future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); - } else { - future.completeExceptionally(processNullResponseErr(responseFuture)); + return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> { + CompletableFuture<Set<MessageQueue>> future0 = new CompletableFuture<>(); + if (response.getCode() == ResponseCode.SUCCESS) { + try { + LockBatchResponseBody responseBody = LockBatchResponseBody.decode(response.getBody(), LockBatchResponseBody.class); + Set<MessageQueue> messageQueues = responseBody.getLockOKMQSet(); + future0.complete(messageQueues); + } catch (Throwable t) { + future0.completeExceptionally(t); } - }); - } catch (Exception e) { - future.completeExceptionally(e); - } - return future; + } else { + future0.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); + } + return future0; + }); } public CompletableFuture<Void> unlockBatchMQOneway(String brokerAddr, @@ -624,25 +577,21 @@ public class MQClientAPIExt extends MQClientAPIImpl { public CompletableFuture<Boolean> notification(String brokerAddr, NotificationRequestHeader requestHeader, long timeoutMillis) { - CompletableFuture<Boolean> future = new CompletableFuture<>(); RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.NOTIFICATION, requestHeader); - try { - this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenAccept(response -> { - if (response.getCode() == ResponseCode.SUCCESS) { - try { - NotificationResponseHeader responseHeader = (NotificationResponseHeader) response.decodeCommandCustomHeader(NotificationResponseHeader.class); - future.complete(responseHeader.isHasMsg()); - } catch (Throwable t) { - future.completeExceptionally(t); - } - } else { - future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); + return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> { + CompletableFuture<Boolean> future0 = new CompletableFuture<>(); + if (response.getCode() == ResponseCode.SUCCESS) { + try { + NotificationResponseHeader responseHeader = (NotificationResponseHeader) response.decodeCommandCustomHeader(NotificationResponseHeader.class); + future0.complete(responseHeader.isHasMsg()); + } catch (Throwable t) { + future0.completeExceptionally(t); } - }); - } catch (Throwable t) { - future.completeExceptionally(t); - } - return future; + } else { + future0.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); + } + return future0; + }); } public CompletableFuture<RemotingCommand> invoke(String brokerAddr, RemotingCommand request, long timeoutMillis) { diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java index d13f2cfe4..c152d38ea 100644 --- a/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java @@ -212,7 +212,7 @@ public class MQClientAPIImplTest { RemotingCommand request = mock.getArgument(1); ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); responseFuture.setResponseCommand(createSendMessageSuccessResponse(request)); - callback.operationComplete(responseFuture); + callback.operationSucceed(responseFuture.getResponseCommand()); return null; } }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); @@ -386,7 +386,7 @@ public class MQClientAPIImplTest { RemotingCommand request = mock.getArgument(1); ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); responseFuture.setResponseCommand(createSendMessageSuccessResponse(request)); - callback.operationComplete(responseFuture); + callback.operationSucceed(responseFuture.getResponseCommand()); return null; } }).when(remotingClient).invokeAsync(Matchers.anyString(), Matchers.any(RemotingCommand.class), Matchers.anyLong(), Matchers.any(InvokeCallback.class)); @@ -472,7 +472,7 @@ public class MQClientAPIImplTest { message.putUserProperty("key", "value"); response.setBody(MessageDecoder.encode(message, false)); responseFuture.setResponseCommand(response); - callback.operationComplete(responseFuture); + callback.operationSucceed(responseFuture.getResponseCommand()); return null; } }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); @@ -543,7 +543,7 @@ public class MQClientAPIImplTest { message.getProperties().put(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET, String.valueOf(0)); response.setBody(MessageDecoder.encode(message, false)); responseFuture.setResponseCommand(response); - callback.operationComplete(responseFuture); + callback.operationSucceed(responseFuture.getResponseCommand()); return null; } }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); @@ -585,7 +585,7 @@ public class MQClientAPIImplTest { response.setOpaque(request.getOpaque()); response.setCode(ResponseCode.SUCCESS); responseFuture.setResponseCommand(response); - callback.operationComplete(responseFuture); + callback.operationSucceed(responseFuture.getResponseCommand()); return null; } }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); @@ -622,7 +622,7 @@ public class MQClientAPIImplTest { responseHeader.setPopTime(System.currentTimeMillis()); responseHeader.setInvisibleTime(10 * 1000L); responseFuture.setResponseCommand(response); - callback.operationComplete(responseFuture); + callback.operationSucceed(responseFuture.getResponseCommand()); return null; } }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java index fe07090d5..3227d1e1c 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java @@ -26,7 +26,6 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import org.apache.rocketmq.acl.AccessValidator; -import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.future.FutureTaskExt; import org.apache.rocketmq.common.thread.ThreadPoolMonitor; @@ -51,10 +50,12 @@ import org.apache.rocketmq.proxy.remoting.channel.RemotingChannelManager; import org.apache.rocketmq.proxy.remoting.pipeline.AuthenticationPipeline; import org.apache.rocketmq.proxy.remoting.pipeline.RequestPipeline; import org.apache.rocketmq.remoting.ChannelEventListener; +import org.apache.rocketmq.remoting.InvokeCallback; import org.apache.rocketmq.remoting.RemotingServer; import org.apache.rocketmq.remoting.netty.NettyRemotingServer; import org.apache.rocketmq.remoting.netty.NettyServerConfig; import org.apache.rocketmq.remoting.netty.RequestTask; +import org.apache.rocketmq.remoting.netty.ResponseFuture; import org.apache.rocketmq.remoting.netty.TlsSystemConfig; import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.remoting.protocol.RequestCode; @@ -239,12 +240,21 @@ public class RemotingProtocolServer implements StartAndShutdown, RemotingProxyOu long timeoutMillis) { CompletableFuture<RemotingCommand> future = new CompletableFuture<>(); try { - this.defaultRemotingServer.invokeAsync(channel, request, timeoutMillis, responseFuture -> { - if (responseFuture.getResponseCommand() == null) { - future.completeExceptionally(new MQClientException("response is null after send request to client", responseFuture.getCause())); - return; + this.defaultRemotingServer.invokeAsync(channel, request, timeoutMillis, new InvokeCallback() { + @Override + public void operationComplete(ResponseFuture responseFuture) { + + } + + @Override + public void operationSucceed(RemotingCommand response) { + future.complete(response); + } + + @Override + public void operationFail(Throwable throwable) { + future.completeExceptionally(throwable); } - future.complete(responseFuture.getResponseCommand()); }); } catch (Throwable t) { future.completeExceptionally(t); diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java index 3f3a4ae40..e2d05b0f5 100644 --- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java @@ -24,6 +24,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Set; import java.util.UUID; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; @@ -85,6 +86,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; @RunWith(MockitoJUnitRunner.class) public class MQClientAPIExtTest { @@ -109,13 +111,9 @@ public class MQClientAPIExtTest { @Test public void testSendHeartbeatAsync() throws Exception { - doAnswer((Answer<Void>) mock -> { - InvokeCallback invokeCallback = mock.getArgument(3); - ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null); - responseFuture.putResponse(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, "")); - invokeCallback.operationComplete(responseFuture); - return null; - }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); + CompletableFuture<RemotingCommand> future = new CompletableFuture<>(); + future.complete(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, "")); + doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); assertNotNull(mqClientAPI.sendHeartbeatAsync(BROKER_ADDR, new HeartbeatData(), TIMEOUT).get()); } @@ -123,20 +121,16 @@ public class MQClientAPIExtTest { @Test public void testSendMessageAsync() throws Exception { AtomicReference<String> msgIdRef = new AtomicReference<>(); - doAnswer((Answer<Void>) mock -> { - InvokeCallback invokeCallback = mock.getArgument(3); - ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null); - RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); - SendMessageResponseHeader sendMessageResponseHeader = (SendMessageResponseHeader) response.readCustomHeader(); - sendMessageResponseHeader.setMsgId(msgIdRef.get()); - sendMessageResponseHeader.setQueueId(0); - sendMessageResponseHeader.setQueueOffset(1L); - response.setCode(ResponseCode.SUCCESS); - response.makeCustomHeaderToNet(); - responseFuture.putResponse(response); - invokeCallback.operationComplete(responseFuture); - return null; - }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); + CompletableFuture<RemotingCommand> future = new CompletableFuture<>(); + RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); + SendMessageResponseHeader sendMessageResponseHeader = (SendMessageResponseHeader) response.readCustomHeader(); + sendMessageResponseHeader.setMsgId(msgIdRef.get()); + sendMessageResponseHeader.setQueueId(0); + sendMessageResponseHeader.setQueueOffset(1L); + response.setCode(ResponseCode.SUCCESS); + response.makeCustomHeaderToNet(); + future.complete(response); + doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); MessageExt messageExt = createMessage(); msgIdRef.set(MessageClientIDSetter.getUniqID(messageExt)); @@ -150,20 +144,16 @@ public class MQClientAPIExtTest { @Test public void testSendMessageListAsync() throws Exception { - doAnswer((Answer<Void>) mock -> { - InvokeCallback invokeCallback = mock.getArgument(3); - ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null); - RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); - SendMessageResponseHeader sendMessageResponseHeader = (SendMessageResponseHeader) response.readCustomHeader(); - sendMessageResponseHeader.setMsgId(""); - sendMessageResponseHeader.setQueueId(0); - sendMessageResponseHeader.setQueueOffset(1L); - response.setCode(ResponseCode.SUCCESS); - response.makeCustomHeaderToNet(); - responseFuture.putResponse(response); - invokeCallback.operationComplete(responseFuture); - return null; - }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); + CompletableFuture<RemotingCommand> future = new CompletableFuture<>(); + RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); + SendMessageResponseHeader sendMessageResponseHeader = (SendMessageResponseHeader) response.readCustomHeader(); + sendMessageResponseHeader.setMsgId(""); + sendMessageResponseHeader.setQueueId(0); + sendMessageResponseHeader.setQueueOffset(1L); + response.setCode(ResponseCode.SUCCESS); + response.makeCustomHeaderToNet(); + future.complete(response); + doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); List<MessageExt> messageExtList = new ArrayList<>(); StringBuilder sb = new StringBuilder(); @@ -182,13 +172,9 @@ public class MQClientAPIExtTest { @Test public void testSendMessageBackAsync() throws Exception { - doAnswer((Answer<Void>) mock -> { - InvokeCallback invokeCallback = mock.getArgument(3); - ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null); - responseFuture.putResponse(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, "")); - invokeCallback.operationComplete(responseFuture); - return null; - }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); + CompletableFuture<RemotingCommand> future = new CompletableFuture<>(); + future.complete(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, "")); + doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); RemotingCommand remotingCommand = mqClientAPI.sendMessageBackAsync(BROKER_ADDR, new ConsumerSendMsgBackRequestHeader(), TIMEOUT) .get(); @@ -285,7 +271,7 @@ public class MQClientAPIExtTest { body.setConsumerIdList(clientIds); response.setBody(body.encode()); responseFuture.putResponse(response); - invokeCallback.operationComplete(responseFuture); + invokeCallback.operationSucceed(responseFuture.getResponseCommand()); return null; }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); @@ -302,7 +288,7 @@ public class MQClientAPIExtTest { response.setCode(ResponseCode.SYSTEM_ERROR); response.makeCustomHeaderToNet(); responseFuture.putResponse(response); - invokeCallback.operationComplete(responseFuture); + invokeCallback.operationSucceed(responseFuture.getResponseCommand()); return null; }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); @@ -322,7 +308,7 @@ public class MQClientAPIExtTest { response.setCode(ResponseCode.SUCCESS); response.makeCustomHeaderToNet(); responseFuture.putResponse(response); - invokeCallback.operationComplete(responseFuture); + invokeCallback.operationSucceed(responseFuture.getResponseCommand()); return null; }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); @@ -335,18 +321,15 @@ public class MQClientAPIExtTest { @Test public void testSearchOffsetAsync() throws Exception { long offset = ThreadLocalRandom.current().nextLong(); - doAnswer((Answer<Void>) mock -> { - InvokeCallback invokeCallback = mock.getArgument(3); - ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null); - RemotingCommand response = RemotingCommand.createResponseCommand(SearchOffsetResponseHeader.class); - SearchOffsetResponseHeader responseHeader = (SearchOffsetResponseHeader) response.readCustomHeader(); - responseHeader.setOffset(offset); - response.setCode(ResponseCode.SUCCESS); - response.makeCustomHeaderToNet(); - responseFuture.putResponse(response); - invokeCallback.operationComplete(responseFuture); - return null; - }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); + CompletableFuture<RemotingCommand> future = new CompletableFuture<>(); + RemotingCommand response = RemotingCommand.createResponseCommand(SearchOffsetResponseHeader.class); + SearchOffsetResponseHeader responseHeader = (SearchOffsetResponseHeader) response.readCustomHeader(); + responseHeader.setOffset(offset); + response.setCode(ResponseCode.SUCCESS); + response.makeCustomHeaderToNet(); + future.complete(response); + + doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); SearchOffsetRequestHeader requestHeader = new SearchOffsetRequestHeader(); requestHeader.setTopic(TOPIC); diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/InvokeCallback.java b/remoting/src/main/java/org/apache/rocketmq/remoting/InvokeCallback.java index ce78fa923..6be491745 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/InvokeCallback.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/InvokeCallback.java @@ -17,7 +17,22 @@ package org.apache.rocketmq.remoting; import org.apache.rocketmq.remoting.netty.ResponseFuture; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; public interface InvokeCallback { + /** + * This method is expected to be invoked after {@link #operationSucceed(RemotingCommand)} + * or {@link #operationFail(Throwable)} + * + * @param responseFuture the returned object contains response or exception + */ void operationComplete(final ResponseFuture responseFuture); + + default void operationSucceed(final RemotingCommand response) { + + } + + default void operationFail(final Throwable throwable) { + + } } diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/RemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/RemotingClient.java index ff0b3df95..c8389eedb 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/RemotingClient.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/RemotingClient.java @@ -20,11 +20,11 @@ import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import org.apache.rocketmq.remoting.exception.RemotingConnectException; -import org.apache.rocketmq.remoting.exception.RemotingException; import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; +import org.apache.rocketmq.remoting.netty.ResponseFuture; import org.apache.rocketmq.remoting.protocol.RemotingCommand; public interface RemotingClient extends RemotingService { @@ -51,18 +51,21 @@ public interface RemotingClient extends RemotingService { final long timeoutMillis) { CompletableFuture<RemotingCommand> future = new CompletableFuture<>(); try { - invokeAsync(addr, request, timeoutMillis, responseFuture -> { - RemotingCommand response = responseFuture.getResponseCommand(); - if (response != null) { + invokeAsync(addr, request, timeoutMillis, new InvokeCallback() { + + @Override + public void operationComplete(ResponseFuture responseFuture) { + + } + + @Override + public void operationSucceed(RemotingCommand response) { future.complete(response); - } else { - if (!responseFuture.isSendRequestOK()) { - future.completeExceptionally(new RemotingSendRequestException(addr, responseFuture.getCause())); - } else if (responseFuture.isTimeout()) { - future.completeExceptionally(new RemotingTimeoutException(addr, timeoutMillis, responseFuture.getCause())); - } else { - future.completeExceptionally(new RemotingException(request.toString(), responseFuture.getCause())); - } + } + + @Override + public void operationFail(Throwable throwable) { + future.completeExceptionally(throwable); } }); } catch (Throwable t) { diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java index fce2de267..12e66f913 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java @@ -23,20 +23,23 @@ import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslHandler; import io.netty.util.concurrent.Future; import io.opentelemetry.api.common.AttributesBuilder; -import java.net.SocketAddress; import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map.Entry; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import javax.annotation.Nullable; import org.apache.rocketmq.common.AbortProcessException; @@ -125,7 +128,7 @@ public abstract class NettyRemotingAbstract { * Constructor, specifying capacity of one-way and asynchronous semaphores. * * @param permitsOneway Number of permits for one-way requests. - * @param permitsAsync Number of permits for asynchronous requests. + * @param permitsAsync Number of permits for asynchronous requests. */ public NettyRemotingAbstract(final int permitsOneway, final int permitsAsync) { this.semaphoreOneway = new Semaphore(permitsOneway, true); @@ -367,8 +370,7 @@ public abstract class NettyRemotingAbstract { responseFuture.release(); } } else { - log.warn("receive response, but not matched any request, " + RemotingHelper.parseChannelRemoteAddr(ctx.channel())); - log.warn(cmd.toString()); + log.warn("receive response, cmd={}, but not matched any request, address={}", cmd, RemotingHelper.parseChannelRemoteAddr(ctx.channel())); } } @@ -467,57 +469,68 @@ public abstract class NettyRemotingAbstract { public RemotingCommand invokeSyncImpl(final Channel channel, final RemotingCommand request, final long timeoutMillis) throws InterruptedException, RemotingSendRequestException, RemotingTimeoutException { - //get the request id - final int opaque = request.getOpaque(); - try { - final ResponseFuture responseFuture = new ResponseFuture(channel, opaque, timeoutMillis, null, null); - this.responseTable.put(opaque, responseFuture); - final SocketAddress addr = channel.remoteAddress(); - channel.writeAndFlush(request).addListener((ChannelFutureListener) f -> { - if (f.isSuccess()) { - responseFuture.setSendRequestOK(true); - return; - } - - responseFuture.setSendRequestOK(false); - responseTable.remove(opaque); - responseFuture.setCause(f.cause()); - responseFuture.putResponse(null); - log.warn("Failed to write a request command to {}, caused by underlying I/O operation failure", addr); - }); + return invokeImpl(channel, request, timeoutMillis).thenApply(ResponseFuture::getResponseCommand) + .get(timeoutMillis, TimeUnit.MILLISECONDS); + } catch (ExecutionException e) { + throw new RemotingSendRequestException(channel.remoteAddress().toString(), e.getCause()); + } catch (TimeoutException e) { + throw new RemotingTimeoutException(channel.remoteAddress().toString(), timeoutMillis, e.getCause()); + } + } - RemotingCommand responseCommand = responseFuture.waitResponse(timeoutMillis); - if (null == responseCommand) { - if (responseFuture.isSendRequestOK()) { - throw new RemotingTimeoutException(RemotingHelper.parseSocketAddressAddr(addr), timeoutMillis, - responseFuture.getCause()); - } else { - throw new RemotingSendRequestException(RemotingHelper.parseSocketAddressAddr(addr), responseFuture.getCause()); - } + public CompletableFuture<ResponseFuture> invokeImpl(final Channel channel, final RemotingCommand request, + final long timeoutMillis) { + String channelRemoteAddr = RemotingHelper.parseChannelRemoteAddr(channel); + doBeforeRpcHooks(channelRemoteAddr, request); + return invoke0(channel, request, timeoutMillis).whenComplete((v, t) -> { + if (t == null) { + doAfterRpcHooks(channelRemoteAddr, request, v.getResponseCommand()); } - - return responseCommand; - } finally { - this.responseTable.remove(opaque); - } + }); } - public void invokeAsyncImpl(final Channel channel, final RemotingCommand request, final long timeoutMillis, - final InvokeCallback invokeCallback) - throws InterruptedException, RemotingTooMuchRequestException, RemotingTimeoutException, RemotingSendRequestException { + protected CompletableFuture<ResponseFuture> invoke0(final Channel channel, final RemotingCommand request, + final long timeoutMillis) { + CompletableFuture<ResponseFuture> future = new CompletableFuture<>(); long beginStartTime = System.currentTimeMillis(); final int opaque = request.getOpaque(); - boolean acquired = this.semaphoreAsync.tryAcquire(timeoutMillis, TimeUnit.MILLISECONDS); + + boolean acquired; + try { + acquired = this.semaphoreAsync.tryAcquire(timeoutMillis, TimeUnit.MILLISECONDS); + } catch (Throwable t) { + future.completeExceptionally(t); + return future; + } if (acquired) { final SemaphoreReleaseOnlyOnce once = new SemaphoreReleaseOnlyOnce(this.semaphoreAsync); long costTime = System.currentTimeMillis() - beginStartTime; if (timeoutMillis < costTime) { once.release(); - throw new RemotingTimeoutException("invokeAsyncImpl call timeout"); + future.completeExceptionally(new RemotingTimeoutException("invokeAsyncImpl call timeout")); + return future; } - final ResponseFuture responseFuture = new ResponseFuture(channel, opaque, timeoutMillis - costTime, invokeCallback, once); + AtomicReference<ResponseFuture> responseFutureReference = new AtomicReference<>(); + final ResponseFuture responseFuture = new ResponseFuture(channel, opaque, request, timeoutMillis - costTime, + new InvokeCallback() { + @Override + public void operationComplete(ResponseFuture responseFuture) { + + } + + @Override + public void operationSucceed(RemotingCommand response) { + future.complete(responseFutureReference.get()); + } + + @Override + public void operationFail(Throwable throwable) { + future.completeExceptionally(throwable); + } + }, once); + responseFutureReference.set(responseFuture); this.responseTable.put(opaque, responseFuture); try { channel.writeAndFlush(request).addListener((ChannelFutureListener) f -> { @@ -528,15 +541,17 @@ public abstract class NettyRemotingAbstract { requestFail(opaque); log.warn("send a request command to channel <{}> failed.", RemotingHelper.parseChannelRemoteAddr(channel)); }); + return future; } catch (Exception e) { responseTable.remove(opaque); responseFuture.release(); log.warn("send a request command to channel <" + RemotingHelper.parseChannelRemoteAddr(channel) + "> Exception", e); - throw new RemotingSendRequestException(RemotingHelper.parseChannelRemoteAddr(channel), e); + future.completeExceptionally(new RemotingSendRequestException(RemotingHelper.parseChannelRemoteAddr(channel), e)); + return future; } } else { if (timeoutMillis <= 0) { - throw new RemotingTooMuchRequestException("invokeAsyncImpl invoke too fast"); + future.completeExceptionally(new RemotingTooMuchRequestException("invokeAsyncImpl invoke too fast")); } else { String info = String.format("invokeAsyncImpl tryAcquire semaphore timeout, %dms, waiting thread nums: %d semaphoreAsyncValue: %d", @@ -545,11 +560,31 @@ public abstract class NettyRemotingAbstract { this.semaphoreAsync.availablePermits() ); log.warn(info); - throw new RemotingTimeoutException(info); + future.completeExceptionally(new RemotingTimeoutException(info)); } + return future; } } + public void invokeAsyncImpl(final Channel channel, final RemotingCommand request, final long timeoutMillis, + final InvokeCallback invokeCallback) { + invokeImpl(channel, request, timeoutMillis) + .whenComplete((v, t) -> { + if (t == null) { + invokeCallback.operationComplete(v); + } else { + ResponseFuture responseFuture = new ResponseFuture(channel, request.getOpaque(), request, timeoutMillis, null, null); + responseFuture.setCause(t); + invokeCallback.operationComplete(responseFuture); + } + }) + .thenAccept(responseFuture -> invokeCallback.operationSucceed(responseFuture.getResponseCommand())) + .exceptionally(t -> { + invokeCallback.operationFail(t); + return null; + }); + } + private void requestFail(final int opaque) { ResponseFuture responseFuture = responseTable.remove(opaque); if (responseFuture != null) { diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java index 64621dd6c..d784351a5 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java @@ -527,15 +527,13 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti if (channel != null && channel.isActive()) { long left = timeoutMillis; try { - doBeforeRpcHooks(channelRemoteAddr, request); long costTime = System.currentTimeMillis() - beginStartTime; left -= costTime; if (left <= 0) { throw new RemotingTimeoutException("invokeSync call the addr[" + channelRemoteAddr + "] timeout"); } RemotingCommand response = this.invokeSyncImpl(channel, request, left); - doAfterRpcHooks(channelRemoteAddr, request, response); - this.updateChannelLastResponseTime(addr); + updateChannelLastResponseTime(addr); return response; } catch (RemotingSendRequestException e) { LOGGER.warn("invokeSync: send request exception, so close the channel[{}]", channelRemoteAddr); @@ -727,18 +725,11 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti final Channel channel = this.getAndCreateChannel(addr); String channelRemoteAddr = RemotingHelper.parseChannelRemoteAddr(channel); if (channel != null && channel.isActive()) { - try { - doBeforeRpcHooks(channelRemoteAddr, request); - long costTime = System.currentTimeMillis() - beginStartTime; - if (timeoutMillis < costTime) { - throw new RemotingTooMuchRequestException("invokeAsync call the addr[" + channelRemoteAddr + "] timeout"); - } - this.invokeAsyncImpl(channel, request, timeoutMillis - costTime, new InvokeCallbackWrapper(invokeCallback, addr)); - } catch (RemotingSendRequestException e) { - LOGGER.warn("invokeAsync: send request exception, so close the channel[{}]", channelRemoteAddr); - this.closeChannel(addr, channel); - throw e; + long costTime = System.currentTimeMillis() - beginStartTime; + if (timeoutMillis < costTime) { + throw new RemotingTooMuchRequestException("invokeAsync call the addr[" + channelRemoteAddr + "] timeout"); } + this.invokeAsyncImpl(channel, request, timeoutMillis - costTime, new InvokeCallbackWrapper(invokeCallback, addr)); } else { this.closeChannel(addr, channel); throw new RemotingConnectException(addr); @@ -931,11 +922,19 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti @Override public void operationComplete(ResponseFuture responseFuture) { - if (responseFuture != null && responseFuture.isSendRequestOK() && responseFuture.getResponseCommand() != null) { - NettyRemotingClient.this.updateChannelLastResponseTime(addr); - } this.invokeCallback.operationComplete(responseFuture); } + + @Override + public void operationSucceed(RemotingCommand response) { + updateChannelLastResponseTime(addr); + this.invokeCallback.operationSucceed(response); + } + + @Override + public void operationFail(final Throwable throwable) { + this.invokeCallback.operationFail(throwable); + } } class NettyClientHandler extends SimpleChannelInboundHandler<RemotingCommand> { diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/ResponseFuture.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/ResponseFuture.java index 19f705d74..0882818fe 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/ResponseFuture.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/ResponseFuture.java @@ -22,6 +22,9 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.rocketmq.remoting.InvokeCallback; import org.apache.rocketmq.remoting.common.SemaphoreReleaseOnlyOnce; +import org.apache.rocketmq.remoting.exception.RemotingException; +import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; +import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; import org.apache.rocketmq.remoting.protocol.RemotingCommand; public class ResponseFuture { @@ -59,6 +62,18 @@ public class ResponseFuture { public void executeInvokeCallback() { if (invokeCallback != null) { if (this.executeCallbackOnlyOnce.compareAndSet(false, true)) { + RemotingCommand response = getResponseCommand(); + if (response != null) { + invokeCallback.operationSucceed(response); + } else { + if (!isSendRequestOK()) { + invokeCallback.operationFail(new RemotingSendRequestException(channel.remoteAddress().toString(), getCause())); + } else if (isTimeout()) { + invokeCallback.operationFail(new RemotingTimeoutException(channel.remoteAddress().toString(), getTimeoutMillis(), getCause())); + } else { + invokeCallback.operationFail(new RemotingException(getRequestCommand().toString(), getCause())); + } + } invokeCallback.operationComplete(this); } } diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcClientImpl.java b/remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcClientImpl.java index 133e0ed31..5328e8845 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcClientImpl.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcClientImpl.java @@ -160,31 +160,38 @@ public class RpcClientImpl implements RpcClient { InvokeCallback callback = new InvokeCallback() { @Override public void operationComplete(ResponseFuture responseFuture) { - RemotingCommand responseCommand = responseFuture.getResponseCommand(); - if (responseCommand == null) { - processFailedResponse(addr, requestCommand, responseFuture, rpcResponsePromise); - return; - } + + } + + @Override + public void operationSucceed(RemotingCommand response) { try { - switch (responseCommand.getCode()) { + switch (response.getCode()) { case ResponseCode.SUCCESS: case ResponseCode.PULL_NOT_FOUND: case ResponseCode.PULL_RETRY_IMMEDIATELY: case ResponseCode.PULL_OFFSET_MOVED: PullMessageResponseHeader responseHeader = - (PullMessageResponseHeader) responseCommand.decodeCommandCustomHeader(PullMessageResponseHeader.class); - rpcResponsePromise.setSuccess(new RpcResponse(responseCommand.getCode(), responseHeader, responseCommand.getBody())); + (PullMessageResponseHeader) response.decodeCommandCustomHeader(PullMessageResponseHeader.class); + rpcResponsePromise.setSuccess(new RpcResponse(response.getCode(), responseHeader, response.getBody())); default: - RpcResponse rpcResponse = new RpcResponse(new RpcException(responseCommand.getCode(), "unexpected remote response code")); + RpcResponse rpcResponse = new RpcResponse(new RpcException(response.getCode(), "unexpected remote response code")); rpcResponsePromise.setSuccess(rpcResponse); } } catch (Exception e) { - String errorMessage = "process failed. addr: " + addr + ", timeoutMillis: " + responseFuture.getTimeoutMillis() + ". Request: " + requestCommand; - RpcResponse rpcResponse = new RpcResponse(new RpcException(ResponseCode.RPC_UNKNOWN, errorMessage, e)); + String errorMessage = "process failed. addr: " + addr + ", timeoutMillis: " + timeoutMillis + ". Request: " + requestCommand; + RpcResponse rpcResponse = new RpcResponse(new RpcException(ResponseCode.RPC_UNKNOWN, errorMessage, e)); rpcResponsePromise.setSuccess(rpcResponse); } } + + @Override + public void operationFail(Throwable throwable) { + String errorMessage = "process failed. addr: " + addr + ". Request: " + requestCommand; + RpcResponse rpcResponse = new RpcResponse(new RpcException(ResponseCode.RPC_UNKNOWN, errorMessage, throwable)); + rpcResponsePromise.setSuccess(rpcResponse); + } }; this.remotingClient.invokeAsync(addr, requestCommand, timeoutMillis, callback); diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/RemotingServerTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/RemotingServerTest.java index 90072960b..d0da0eb2e 100644 --- a/remoting/src/test/java/org/apache/rocketmq/remoting/RemotingServerTest.java +++ b/remoting/src/test/java/org/apache/rocketmq/remoting/RemotingServerTest.java @@ -26,12 +26,12 @@ import org.apache.rocketmq.remoting.exception.RemotingConnectException; import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; -import org.apache.rocketmq.remoting.netty.ResponseFuture; -import org.apache.rocketmq.remoting.netty.NettyServerConfig; import org.apache.rocketmq.remoting.netty.NettyClientConfig; -import org.apache.rocketmq.remoting.netty.NettyRemotingServer; import org.apache.rocketmq.remoting.netty.NettyRemotingClient; +import org.apache.rocketmq.remoting.netty.NettyRemotingServer; import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; +import org.apache.rocketmq.remoting.netty.NettyServerConfig; +import org.apache.rocketmq.remoting.netty.ResponseFuture; import org.apache.rocketmq.remoting.protocol.LanguageCode; import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.junit.AfterClass; @@ -40,7 +40,6 @@ import org.junit.Test; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; public class RemotingServerTest { private static RemotingServer remotingServer; @@ -122,10 +121,19 @@ public class RemotingServerTest { remotingClient.invokeAsync("localhost:" + remotingServer.localListenPort(), request, 1000 * 3, new InvokeCallback() { @Override public void operationComplete(ResponseFuture responseFuture) { + + } + + @Override + public void operationSucceed(RemotingCommand response) { latch.countDown(); - assertTrue(responseFuture != null); - assertThat(responseFuture.getResponseCommand().getLanguage()).isEqualTo(LanguageCode.JAVA); - assertThat(responseFuture.getResponseCommand().getExtFields()).hasSize(2); + assertThat(response.getLanguage()).isEqualTo(LanguageCode.JAVA); + assertThat(response.getExtFields()).hasSize(2); + } + + @Override + public void operationFail(Throwable throwable) { + } }); latch.await(); diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/BaseInvokeCallback.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannel.java similarity index 57% rename from client/src/main/java/org/apache/rocketmq/client/impl/BaseInvokeCallback.java rename to remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannel.java index 80188832e..8ddcdf35d 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/BaseInvokeCallback.java +++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannel.java @@ -15,23 +15,14 @@ * limitations under the License. */ -package org.apache.rocketmq.client.impl; +package org.apache.rocketmq.remoting.netty; -import org.apache.rocketmq.remoting.InvokeCallback; -import org.apache.rocketmq.remoting.netty.ResponseFuture; - -public abstract class BaseInvokeCallback implements InvokeCallback { - private final MQClientAPIImpl mqClientAPI; - - public BaseInvokeCallback(MQClientAPIImpl mqClientAPI) { - this.mqClientAPI = mqClientAPI; - } +import io.netty.channel.ChannelFuture; +import io.netty.channel.local.LocalChannel; +public class MockChannel extends LocalChannel { @Override - public void operationComplete(final ResponseFuture responseFuture) { - mqClientAPI.execRpcHooksAfterRequest(responseFuture); - onComplete(responseFuture); + public ChannelFuture writeAndFlush(Object msg) { + return new MockChannelPromise(MockChannel.this); } - - public abstract void onComplete(final ResponseFuture responseFuture); } diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannelPromise.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannelPromise.java new file mode 100644 index 000000000..9c3a35487 --- /dev/null +++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannelPromise.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.remoting.netty; + +import io.netty.channel.Channel; +import io.netty.channel.ChannelPromise; +import io.netty.util.concurrent.Future; +import io.netty.util.concurrent.GenericFutureListener; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.jetbrains.annotations.NotNull; + +public class MockChannelPromise implements ChannelPromise { + protected Channel channel; + + public MockChannelPromise(Channel channel) { + this.channel = channel; + } + + @Override + public Channel channel() { + return channel; + } + + @Override + public ChannelPromise setSuccess(Void result) { + return this; + } + + @Override + public ChannelPromise setSuccess() { + return this; + } + + @Override + public boolean trySuccess() { + return false; + } + + @Override + public ChannelPromise setFailure(Throwable cause) { + return this; + } + + @Override + public ChannelPromise addListener(GenericFutureListener<? extends Future<? super Void>> listener) { + return this; + } + + @Override + public ChannelPromise addListeners(GenericFutureListener<? extends Future<? super Void>>... listeners) { + return this; + } + + @Override + public ChannelPromise removeListener(GenericFutureListener<? extends Future<? super Void>> listener) { + return this; + } + + @Override + public ChannelPromise removeListeners(GenericFutureListener<? extends Future<? super Void>>... listeners) { + return this; + } + + @Override + public ChannelPromise sync() throws InterruptedException { + return this; + } + + @Override + public ChannelPromise syncUninterruptibly() { + return this; + } + + @Override + public ChannelPromise await() throws InterruptedException { + return this; + } + + @Override + public ChannelPromise awaitUninterruptibly() { + return this; + } + + @Override + public ChannelPromise unvoid() { + return this; + } + + @Override + public boolean isVoid() { + return false; + } + + @Override + public boolean trySuccess(Void result) { + return false; + } + + @Override + public boolean tryFailure(Throwable cause) { + return false; + } + + @Override + public boolean setUncancellable() { + return false; + } + + @Override + public boolean isSuccess() { + return false; + } + + @Override + public boolean isCancellable() { + return false; + } + + @Override + public Throwable cause() { + return null; + } + + @Override + public boolean await(long timeout, TimeUnit unit) throws InterruptedException { + return false; + } + + @Override + public boolean await(long timeoutMillis) throws InterruptedException { + return false; + } + + @Override + public boolean awaitUninterruptibly(long timeout, TimeUnit unit) { + return false; + } + + @Override + public boolean awaitUninterruptibly(long timeoutMillis) { + return false; + } + + @Override + public Void getNow() { + return null; + } + + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + return false; + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public boolean isDone() { + return false; + } + + @Override + public Void get() throws InterruptedException, ExecutionException { + return null; + } + + @Override + public Void get(long timeout, + @NotNull java.util.concurrent.TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { + return null; + } +} diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstractTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstractTest.java index 8381c132b..dbbea86ea 100644 --- a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstractTest.java +++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstractTest.java @@ -39,9 +39,19 @@ public class NettyRemotingAbstractTest { final Semaphore semaphore = new Semaphore(0); ResponseFuture responseFuture = new ResponseFuture(null, 1, 3000, new InvokeCallback() { @Override - public void operationComplete(final ResponseFuture responseFuture) { + public void operationComplete(ResponseFuture responseFuture) { + + } + + @Override + public void operationSucceed(RemotingCommand response) { assertThat(semaphore.availablePermits()).isEqualTo(0); } + + @Override + public void operationFail(Throwable throwable) { + + } }, new SemaphoreReleaseOnlyOnce(semaphore)); remotingAbstract.responseTable.putIfAbsent(1, responseFuture); @@ -75,9 +85,19 @@ public class NettyRemotingAbstractTest { final Semaphore semaphore = new Semaphore(0); ResponseFuture responseFuture = new ResponseFuture(null, 1, 3000, new InvokeCallback() { @Override - public void operationComplete(final ResponseFuture responseFuture) { + public void operationComplete(ResponseFuture responseFuture) { + + } + + @Override + public void operationSucceed(RemotingCommand response) { assertThat(semaphore.availablePermits()).isEqualTo(0); } + + @Override + public void operationFail(Throwable throwable) { + + } }, new SemaphoreReleaseOnlyOnce(semaphore)); remotingAbstract.responseTable.putIfAbsent(1, responseFuture); @@ -98,7 +118,18 @@ public class NettyRemotingAbstractTest { // mock timeout ResponseFuture responseFuture = new ResponseFuture(null, dummyId, -1000, new InvokeCallback() { @Override - public void operationComplete(final ResponseFuture responseFuture) { + public void operationComplete(ResponseFuture responseFuture) { + + } + + @Override + public void operationSucceed(RemotingCommand response) { + + } + + @Override + public void operationFail(Throwable throwable) { + } }, null); remotingAbstract.responseTable.putIfAbsent(dummyId, responseFuture); @@ -111,7 +142,22 @@ public class NettyRemotingAbstractTest { final Semaphore semaphore = new Semaphore(0); RemotingCommand request = RemotingCommand.createRequestCommand(1, null); ResponseFuture responseFuture = new ResponseFuture(null, 1, request, 3000, - responseFuture1 -> assertThat(semaphore.availablePermits()).isEqualTo(0), new SemaphoreReleaseOnlyOnce(semaphore)); + new InvokeCallback() { + @Override + public void operationComplete(ResponseFuture responseFuture) { + + } + + @Override + public void operationSucceed(RemotingCommand response) { + assertThat(semaphore.availablePermits()).isEqualTo(0); + } + + @Override + public void operationFail(Throwable throwable) { + + } + }, new SemaphoreReleaseOnlyOnce(semaphore)); remotingAbstract.responseTable.putIfAbsent(1, responseFuture); RemotingCommand response = RemotingCommand.createResponseCommand(0, "Foo"); diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java index 8fabbb21d..e72e7bd53 100644 --- a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java +++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java @@ -16,10 +16,17 @@ */ package org.apache.rocketmq.remoting.netty; +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.channel.local.LocalChannel; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Semaphore; import org.apache.rocketmq.remoting.InvokeCallback; +import org.apache.rocketmq.remoting.RPCHook; +import org.apache.rocketmq.remoting.common.SemaphoreReleaseOnlyOnce; import org.apache.rocketmq.remoting.exception.RemotingConnectException; import org.apache.rocketmq.remoting.exception.RemotingException; import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; @@ -29,23 +36,33 @@ import org.apache.rocketmq.remoting.protocol.RequestCode; import org.apache.rocketmq.remoting.protocol.ResponseCode; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.Mock; import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.AssertionsForClassTypes.catchThrowable; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; @RunWith(MockitoJUnitRunner.class) public class NettyRemotingClientTest { @Spy private NettyRemotingClient remotingClient = new NettyRemotingClient(new NettyClientConfig()); + @Mock + private RPCHook rpcHookMock; @Test - public void testSetCallbackExecutor() throws NoSuchFieldException, IllegalAccessException { + public void testSetCallbackExecutor() { ExecutorService customized = Executors.newCachedThreadPool(); remotingClient.setCallbackExecutor(customized); assertThat(remotingClient.getCallbackExecutor()).isEqualTo(customized); @@ -61,7 +78,7 @@ public class NettyRemotingClientTest { InvokeCallback callback = invocation.getArgument(3); ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); responseFuture.setResponseCommand(response); - callback.operationComplete(responseFuture); + callback.operationSucceed(responseFuture.getResponseCommand()); return null; }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); @@ -78,9 +95,7 @@ public class NettyRemotingClientTest { response.setCode(ResponseCode.SUCCESS); doAnswer(invocation -> { InvokeCallback callback = invocation.getArgument(3); - ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); - responseFuture.setSendRequestOK(false); - callback.operationComplete(responseFuture); + callback.operationFail(new RemotingSendRequestException(null)); return null; }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); @@ -97,8 +112,7 @@ public class NettyRemotingClientTest { response.setCode(ResponseCode.SUCCESS); doAnswer(invocation -> { InvokeCallback callback = invocation.getArgument(3); - ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), -1L, null, null); - callback.operationComplete(responseFuture); + callback.operationFail(new RemotingTimeoutException("")); return null; }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); @@ -115,8 +129,7 @@ public class NettyRemotingClientTest { response.setCode(ResponseCode.SUCCESS); doAnswer(invocation -> { InvokeCallback callback = invocation.getArgument(3); - ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); - callback.operationComplete(responseFuture); + callback.operationFail(new RemotingException(null)); return null; }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); @@ -134,4 +147,158 @@ public class NettyRemotingClientTest { assertThat(e.getMessage()).contains(addr); } } + + @Test + public void testInvoke0() throws ExecutionException, InterruptedException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null); + RemotingCommand response = RemotingCommand.createResponseCommand(null); + response.setCode(ResponseCode.SUCCESS); + Channel channel = new MockChannel() { + @Override + public ChannelFuture writeAndFlush(Object msg) { + ResponseFuture responseFuture = remotingClient.responseTable.get(request.getOpaque()); + responseFuture.setResponseCommand(response); + responseFuture.executeInvokeCallback(); + return super.writeAndFlush(msg); + } + }; + CompletableFuture<ResponseFuture> future = remotingClient.invoke0(channel, request, 1000L); + assertThat(future.get().getResponseCommand()).isEqualTo(response); + } + + @Test + public void testInvoke0WithException() { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null); + RemotingCommand response = RemotingCommand.createResponseCommand(null); + response.setCode(ResponseCode.SUCCESS); + Channel channel = new MockChannel() { + @Override + public ChannelFuture writeAndFlush(Object msg) { + ResponseFuture responseFuture = remotingClient.responseTable.get(request.getOpaque()); + responseFuture.executeInvokeCallback(); + return super.writeAndFlush(msg); + } + }; + CompletableFuture<ResponseFuture> future = remotingClient.invoke0(channel, request, 1000L); + assertThatThrownBy(future::get).getCause().isInstanceOf(RemotingException.class); + } + + @Test + public void testInvokeSync() throws RemotingSendRequestException, RemotingTimeoutException, InterruptedException { + remotingClient.registerRPCHook(rpcHookMock); + + Channel channel = new LocalChannel(); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null); + RemotingCommand response = RemotingCommand.createResponseCommand(null); + response.setCode(ResponseCode.SUCCESS); + ResponseFuture responseFuture = new ResponseFuture(channel, request.getOpaque(), request, 1000, new InvokeCallback() { + @Override + public void operationComplete(ResponseFuture responseFuture) { + + } + }, new SemaphoreReleaseOnlyOnce(new Semaphore(1))); + responseFuture.setResponseCommand(response); + CompletableFuture<ResponseFuture> future = new CompletableFuture<>(); + future.complete(responseFuture); + + doReturn(future).when(remotingClient).invoke0(any(Channel.class), any(RemotingCommand.class), anyLong()); + RemotingCommand actual = remotingClient.invokeSyncImpl(channel, request, 1000); + assertThat(actual).isEqualTo(response); + + verify(rpcHookMock).doBeforeRequest(anyString(), eq(request)); + verify(rpcHookMock).doAfterResponse(anyString(), eq(request), eq(response)); + } + + @Test + public void testInvokeAsync() { + remotingClient.registerRPCHook(rpcHookMock); + Channel channel = new LocalChannel(); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null); + RemotingCommand response = RemotingCommand.createResponseCommand(null); + response.setCode(ResponseCode.SUCCESS); + ResponseFuture responseFuture = new ResponseFuture(channel, request.getOpaque(), request, 1000, new InvokeCallback() { + @Override + public void operationComplete(ResponseFuture responseFuture) { + + } + }, new SemaphoreReleaseOnlyOnce(new Semaphore(1))); + responseFuture.setResponseCommand(response); + CompletableFuture<ResponseFuture> future = new CompletableFuture<>(); + future.complete(responseFuture); + + doReturn(future).when(remotingClient).invoke0(any(Channel.class), any(RemotingCommand.class), anyLong()); + + InvokeCallback callback = mock(InvokeCallback.class); + remotingClient.invokeAsyncImpl(channel, request, 1000, callback); + verify(callback, times(1)).operationSucceed(eq(response)); + verify(callback, times(1)).operationComplete(eq(responseFuture)); + verify(callback, never()).operationFail(any()); + + verify(rpcHookMock).doBeforeRequest(anyString(), eq(request)); + verify(rpcHookMock).doAfterResponse(anyString(), eq(request), eq(response)); + } + + @Test + public void testInvokeAsyncFail() { + remotingClient.registerRPCHook(rpcHookMock); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null); + + Channel channel = new LocalChannel(); + CompletableFuture<ResponseFuture> future = new CompletableFuture<>(); + future.completeExceptionally(new RemotingException(null)); + + doReturn(future).when(remotingClient).invoke0(any(Channel.class), any(RemotingCommand.class), anyLong()); + + InvokeCallback callback = mock(InvokeCallback.class); + remotingClient.invokeAsyncImpl(channel, request, 1000, callback); + verify(callback, never()).operationSucceed(any()); + verify(callback, times(1)).operationComplete(any()); + verify(callback, times(1)).operationFail(any()); + + verify(rpcHookMock).doBeforeRequest(anyString(), eq(request)); + verify(rpcHookMock, never()).doAfterResponse(anyString(), eq(request), any()); + } + + @Test + public void testInvokeImpl() throws ExecutionException, InterruptedException { + remotingClient.registerRPCHook(rpcHookMock); + Channel channel = new LocalChannel(); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null); + RemotingCommand response = RemotingCommand.createResponseCommand(null); + response.setCode(ResponseCode.SUCCESS); + ResponseFuture responseFuture = new ResponseFuture(channel, request.getOpaque(), request, 1000, new InvokeCallback() { + @Override + public void operationComplete(ResponseFuture responseFuture) { + + } + }, new SemaphoreReleaseOnlyOnce(new Semaphore(1))); + responseFuture.setResponseCommand(response); + CompletableFuture<ResponseFuture> future = new CompletableFuture<>(); + future.complete(responseFuture); + + doReturn(future).when(remotingClient).invoke0(any(Channel.class), any(RemotingCommand.class), anyLong()); + + CompletableFuture<ResponseFuture> future0 = remotingClient.invokeImpl(channel, request, 1000); + assertThat(future0.get()).isEqualTo(responseFuture); + + verify(rpcHookMock).doBeforeRequest(anyString(), eq(request)); + verify(rpcHookMock).doAfterResponse(anyString(), eq(request), eq(response)); + } + + @Test + public void testInvokeImplFail() { + remotingClient.registerRPCHook(rpcHookMock); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null); + + Channel channel = new LocalChannel(); + CompletableFuture<ResponseFuture> future = new CompletableFuture<>(); + future.completeExceptionally(new RemotingException(null)); + + doReturn(future).when(remotingClient).invoke0(any(Channel.class), any(RemotingCommand.class), anyLong()); + + assertThatThrownBy(() -> remotingClient.invokeImpl(channel, request, 1000).get()).getCause().isInstanceOf(RemotingException.class); + + verify(rpcHookMock).doBeforeRequest(anyString(), eq(request)); + verify(rpcHookMock, never()).doAfterResponse(anyString(), eq(request), any()); + } } -- 2.32.0.windows.2 From b9ffe0f9576f68b8a37cf3e2f68051658ae5a9a2 Mon Sep 17 00:00:00 2001 From: Zhouxiang Zhan <zhouxzhan@apache.org> Date: Sun, 8 Oct 2023 16:33:44 +0800 Subject: [PATCH 2/7] [ISSUE #7296] Add ChannelEventListener for MQClientAPIImpl (#7324) * Add ChannelEventListener for MQClientAPIImpl * add heartbeat when channel connect * remove log * Add enableHeartbeatChannelEventListener for ClientConfig --- .../apache/rocketmq/client/ClientConfig.java | 55 ++++++++++++++----- .../rocketmq/client/impl/MQClientAPIImpl.java | 9 ++- .../client/impl/factory/MQClientInstance.java | 35 +++++++++++- .../remoting/netty/NettyRemotingClient.java | 2 + 4 files changed, 85 insertions(+), 16 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java b/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java index bb0fe3522..f9843cc02 100644 --- a/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java +++ b/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java @@ -94,6 +94,8 @@ public class ClientConfig { private boolean sendLatencyEnable = Boolean.parseBoolean(System.getProperty(SEND_LATENCY_ENABLE, "false")); private boolean startDetectorEnable = Boolean.parseBoolean(System.getProperty(START_DETECTOR_ENABLE, "false")); + private boolean enableHeartbeatChannelEventListener = true; + public String buildMQClientId() { StringBuilder sb = new StringBuilder(); sb.append(this.getClientIP()); @@ -201,6 +203,7 @@ public class ClientConfig { this.useHeartbeatV2 = cc.useHeartbeatV2; this.startDetectorEnable = cc.startDetectorEnable; this.sendLatencyEnable = cc.sendLatencyEnable; + this.enableHeartbeatChannelEventListener = cc.enableHeartbeatChannelEventListener; this.detectInterval = cc.detectInterval; this.detectTimeout = cc.detectTimeout; } @@ -228,6 +231,7 @@ public class ClientConfig { cc.enableStreamRequestType = enableStreamRequestType; cc.useHeartbeatV2 = useHeartbeatV2; cc.startDetectorEnable = startDetectorEnable; + cc.enableHeartbeatChannelEventListener = enableHeartbeatChannelEventListener; cc.sendLatencyEnable = sendLatencyEnable; cc.detectInterval = detectInterval; cc.detectTimeout = detectTimeout; @@ -418,6 +422,14 @@ public class ClientConfig { this.startDetectorEnable = startDetectorEnable; } + public boolean isEnableHeartbeatChannelEventListener() { + return enableHeartbeatChannelEventListener; + } + + public void setEnableHeartbeatChannelEventListener(boolean enableHeartbeatChannelEventListener) { + this.enableHeartbeatChannelEventListener = enableHeartbeatChannelEventListener; + } + public int getDetectTimeout() { return this.detectTimeout; } @@ -444,19 +456,34 @@ public class ClientConfig { @Override public String toString() { - return "ClientConfig [namesrvAddr=" + namesrvAddr - + ", clientIP=" + clientIP + ", instanceName=" + instanceName - + ", clientCallbackExecutorThreads=" + clientCallbackExecutorThreads - + ", pollNameServerInterval=" + pollNameServerInterval - + ", heartbeatBrokerInterval=" + heartbeatBrokerInterval - + ", persistConsumerOffsetInterval=" + persistConsumerOffsetInterval - + ", pullTimeDelayMillsWhenException=" + pullTimeDelayMillsWhenException - + ", unitMode=" + unitMode + ", unitName=" + unitName - + ", vipChannelEnabled=" + vipChannelEnabled + ", useTLS=" + useTLS - + ", socksProxyConfig=" + socksProxyConfig + ", language=" + language.name() - + ", namespace=" + namespace + ", mqClientApiTimeout=" + mqClientApiTimeout - + ", decodeReadBody=" + decodeReadBody + ", decodeDecompressBody=" + decodeDecompressBody - + ", sendLatencyEnable=" + sendLatencyEnable + ", startDetectorEnable=" + startDetectorEnable - + ", enableStreamRequestType=" + enableStreamRequestType + ", useHeartbeatV2=" + useHeartbeatV2 + "]"; + return "ClientConfig{" + + "namesrvAddr='" + namesrvAddr + '\'' + + ", clientIP='" + clientIP + '\'' + + ", instanceName='" + instanceName + '\'' + + ", clientCallbackExecutorThreads=" + clientCallbackExecutorThreads + + ", namespace='" + namespace + '\'' + + ", namespaceInitialized=" + namespaceInitialized + + ", accessChannel=" + accessChannel + + ", pollNameServerInterval=" + pollNameServerInterval + + ", heartbeatBrokerInterval=" + heartbeatBrokerInterval + + ", persistConsumerOffsetInterval=" + persistConsumerOffsetInterval + + ", pullTimeDelayMillsWhenException=" + pullTimeDelayMillsWhenException + + ", unitMode=" + unitMode + + ", unitName='" + unitName + '\'' + + ", decodeReadBody=" + decodeReadBody + + ", decodeDecompressBody=" + decodeDecompressBody + + ", vipChannelEnabled=" + vipChannelEnabled + + ", useHeartbeatV2=" + useHeartbeatV2 + + ", useTLS=" + useTLS + + ", socksProxyConfig='" + socksProxyConfig + '\'' + + ", mqClientApiTimeout=" + mqClientApiTimeout + + ", detectTimeout=" + detectTimeout + + ", detectInterval=" + detectInterval + + ", language=" + language + + ", enableStreamRequestType=" + enableStreamRequestType + + ", sendLatencyEnable=" + sendLatencyEnable + + ", startDetectorEnable=" + startDetectorEnable + + ", enableHeartbeatChannelEventListener=" + enableHeartbeatChannelEventListener + + '}'; } } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java index 2407e5737..e152be811 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java @@ -79,6 +79,7 @@ import org.apache.rocketmq.common.sysflag.PullSysFlag; import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +import org.apache.rocketmq.remoting.ChannelEventListener; import org.apache.rocketmq.remoting.CommandCustomHeader; import org.apache.rocketmq.remoting.InvokeCallback; import org.apache.rocketmq.remoting.RPCHook; @@ -246,10 +247,16 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { public MQClientAPIImpl(final NettyClientConfig nettyClientConfig, final ClientRemotingProcessor clientRemotingProcessor, RPCHook rpcHook, final ClientConfig clientConfig) { + this(nettyClientConfig, clientRemotingProcessor, rpcHook, clientConfig, null); + } + + public MQClientAPIImpl(final NettyClientConfig nettyClientConfig, + final ClientRemotingProcessor clientRemotingProcessor, + RPCHook rpcHook, final ClientConfig clientConfig, final ChannelEventListener channelEventListener) { this.clientConfig = clientConfig; topAddressing = new DefaultTopAddressing(MixAll.getWSAddr(), clientConfig.getUnitName()); topAddressing.registerChangeCallBack(this); - this.remotingClient = new NettyRemotingClient(nettyClientConfig, null); + this.remotingClient = new NettyRemotingClient(nettyClientConfig, channelEventListener); this.clientRemotingProcessor = clientRemotingProcessor; // Inject stream rpc hook first to make reserve field signature diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java index 9484b26f8..09534a176 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java @@ -16,6 +16,7 @@ */ package org.apache.rocketmq.client.impl.factory; +import io.netty.channel.Channel; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -65,6 +66,7 @@ import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.message.MessageQueueAssignment; import org.apache.rocketmq.common.topic.TopicValidator; +import org.apache.rocketmq.remoting.ChannelEventListener; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.common.HeartbeatV2Result; import org.apache.rocketmq.remoting.exception.RemotingException; @@ -151,7 +153,38 @@ public class MQClientInstance { this.nettyClientConfig.setUseTLS(clientConfig.isUseTLS()); this.nettyClientConfig.setSocksProxyConfig(clientConfig.getSocksProxyConfig()); ClientRemotingProcessor clientRemotingProcessor = new ClientRemotingProcessor(this); - this.mQClientAPIImpl = new MQClientAPIImpl(this.nettyClientConfig, clientRemotingProcessor, rpcHook, clientConfig); + ChannelEventListener channelEventListener; + if (clientConfig.isEnableHeartbeatChannelEventListener()) { + channelEventListener = new ChannelEventListener() { + private final ConcurrentMap<String, HashMap<Long, String>> brokerAddrTable = MQClientInstance.this.brokerAddrTable; + @Override + public void onChannelConnect(String remoteAddr, Channel channel) { + for (Map.Entry<String, HashMap<Long, String>> addressEntry : brokerAddrTable.entrySet()) { + for (String address : addressEntry.getValue().values()) { + if (address.equals(remoteAddr)) { + sendHeartbeatToAllBrokerWithLockV2(false); + break; + } + } + } + } + + @Override + public void onChannelClose(String remoteAddr, Channel channel) { + } + + @Override + public void onChannelException(String remoteAddr, Channel channel) { + } + + @Override + public void onChannelIdle(String remoteAddr, Channel channel) { + } + }; + } else { + channelEventListener = null; + } + this.mQClientAPIImpl = new MQClientAPIImpl(this.nettyClientConfig, clientRemotingProcessor, rpcHook, clientConfig, channelEventListener); if (this.clientConfig.getNamesrvAddr() != null) { this.mQClientAPIImpl.updateNameServerAddressList(this.clientConfig.getNamesrvAddr()); diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java index d784351a5..8631d0447 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java @@ -229,6 +229,8 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti handler.option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT); } + nettyEventExecutor.start(); + TimerTask timerTaskScanResponseTable = new TimerTask() { @Override public void run(Timeout timeout) { -- 2.32.0.windows.2 From 3808387e1389278edbe4ef023d200ecb3015622b Mon Sep 17 00:00:00 2001 From: lk <xdkxlk@outlook.com> Date: Mon, 9 Oct 2023 16:07:56 +0800 Subject: [PATCH 3/7] [ISSUE #7429] clean channel map when CLIENT_UNREGISTER in proxy --- .../service/sysmessage/HeartbeatSyncer.java | 31 ++++++--- .../sysmessage/HeartbeatSyncerTest.java | 68 +++++++++++++++++++ 2 files changed, 88 insertions(+), 11 deletions(-) diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java index f70c06b8f..fee3ea87d 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java @@ -18,6 +18,7 @@ package org.apache.rocketmq.proxy.service.sysmessage; import com.alibaba.fastjson.JSON; +import io.netty.channel.Channel; import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; @@ -73,16 +74,8 @@ public class HeartbeatSyncer extends AbstractSystemMessageSyncer { ); this.consumerManager.appendConsumerIdsChangeListener(new ConsumerIdsChangeListener() { @Override - public void handle(ConsumerGroupEvent event, String s, Object... args) { - if (event == ConsumerGroupEvent.CLIENT_UNREGISTER) { - if (args == null || args.length < 1) { - return; - } - if (args[0] instanceof ClientChannelInfo) { - ClientChannelInfo clientChannelInfo = (ClientChannelInfo) args[0]; - remoteChannelMap.remove(clientChannelInfo.getChannel().id().asLongText()); - } - } + public void handle(ConsumerGroupEvent event, String group, Object... args) { + processConsumerGroupEvent(event, group, args); } @Override @@ -98,6 +91,18 @@ public class HeartbeatSyncer extends AbstractSystemMessageSyncer { super.shutdown(); } + protected void processConsumerGroupEvent(ConsumerGroupEvent event, String group, Object... args) { + if (event == ConsumerGroupEvent.CLIENT_UNREGISTER) { + if (args == null || args.length < 1) { + return; + } + if (args[0] instanceof ClientChannelInfo) { + ClientChannelInfo clientChannelInfo = (ClientChannelInfo) args[0]; + remoteChannelMap.remove(buildKey(group, clientChannelInfo.getChannel())); + } + } + } + public void onConsumerRegister(String consumerGroup, ClientChannelInfo clientChannelInfo, ConsumeType consumeType, MessageModel messageModel, ConsumeFromWhere consumeFromWhere, Set<SubscriptionData> subList) { @@ -189,7 +194,7 @@ public class HeartbeatSyncer extends AbstractSystemMessageSyncer { } RemoteChannel decodedChannel = RemoteChannel.decode(data.getChannelData()); - RemoteChannel channel = remoteChannelMap.computeIfAbsent(data.getGroup() + "@" + decodedChannel.id().asLongText(), key -> decodedChannel); + RemoteChannel channel = remoteChannelMap.computeIfAbsent(buildKey(data.getGroup(), decodedChannel), key -> decodedChannel); channel.setExtendAttribute(decodedChannel.getChannelExtendAttribute()); ClientChannelInfo clientChannelInfo = new ClientChannelInfo( channel, @@ -228,4 +233,8 @@ public class HeartbeatSyncer extends AbstractSystemMessageSyncer { // use local address, remoting port and grpc port to build unique local proxy Id return proxyConfig.getLocalServeAddr() + "%" + proxyConfig.getRemotingListenPort() + "%" + proxyConfig.getGrpcServerPort(); } + + private static String buildKey(String group, Channel channel) { + return group + "@" + channel.id().asLongText(); + } } diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java index 43fba3d03..9a2c5e343 100644 --- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java @@ -27,6 +27,7 @@ import com.google.common.collect.Sets; import io.netty.channel.Channel; import io.netty.channel.ChannelId; import java.time.Duration; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -35,6 +36,7 @@ import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; import org.apache.commons.lang3.RandomStringUtils; import org.apache.rocketmq.broker.client.ClientChannelInfo; +import org.apache.rocketmq.broker.client.ConsumerGroupEvent; import org.apache.rocketmq.broker.client.ConsumerManager; import org.apache.rocketmq.client.impl.mqclient.MQClientAPIExt; import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; @@ -320,6 +322,72 @@ public class HeartbeatSyncerTest extends InitConfigTest { } } + @Test + public void testProcessConsumerGroupEventForRemoting() { + String consumerGroup = "consumerGroup"; + Channel channel = createMockChannel(); + RemotingProxyOutClient remotingProxyOutClient = mock(RemotingProxyOutClient.class); + RemotingChannel remotingChannel = new RemotingChannel(remotingProxyOutClient, proxyRelayService, channel, clientId, Collections.emptySet()); + ClientChannelInfo clientChannelInfo = new ClientChannelInfo( + remotingChannel, + clientId, + LanguageCode.JAVA, + 4 + ); + + testProcessConsumerGroupEvent(consumerGroup, clientChannelInfo); + } + + @Test + public void testProcessConsumerGroupEventForGrpcV2() { + String consumerGroup = "consumerGroup"; + GrpcClientSettingsManager grpcClientSettingsManager = mock(GrpcClientSettingsManager.class); + GrpcChannelManager grpcChannelManager = mock(GrpcChannelManager.class); + GrpcClientChannel grpcClientChannel = new GrpcClientChannel( + proxyRelayService, grpcClientSettingsManager, grpcChannelManager, + ProxyContext.create().setRemoteAddress(remoteAddress).setLocalAddress(localAddress), + clientId); + ClientChannelInfo clientChannelInfo = new ClientChannelInfo( + grpcClientChannel, + clientId, + LanguageCode.JAVA, + 5 + ); + + testProcessConsumerGroupEvent(consumerGroup, clientChannelInfo); + } + + private void testProcessConsumerGroupEvent(String consumerGroup, ClientChannelInfo clientChannelInfo) { + HeartbeatSyncer heartbeatSyncer = new HeartbeatSyncer(topicRouteService, adminService, consumerManager, mqClientAPIFactory, null); + SendResult okSendResult = new SendResult(); + okSendResult.setSendStatus(SendStatus.SEND_OK); + + ArgumentCaptor<Message> messageArgumentCaptor = ArgumentCaptor.forClass(Message.class); + doReturn(CompletableFuture.completedFuture(okSendResult)).when(this.mqClientAPIExt) + .sendMessageAsync(anyString(), anyString(), messageArgumentCaptor.capture(), any(), anyLong()); + + heartbeatSyncer.onConsumerRegister( + consumerGroup, + clientChannelInfo, + ConsumeType.CONSUME_PASSIVELY, + MessageModel.CLUSTERING, + ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET, + Collections.emptySet() + ); + await().atMost(Duration.ofSeconds(3)).until(() -> messageArgumentCaptor.getAllValues().size() == 1); + + // change local serve addr, to simulate other proxy receive messages + heartbeatSyncer.localProxyId = RandomStringUtils.randomAlphabetic(10); + ArgumentCaptor<ClientChannelInfo> channelInfoArgumentCaptor = ArgumentCaptor.forClass(ClientChannelInfo.class); + doReturn(true).when(consumerManager).registerConsumer(anyString(), channelInfoArgumentCaptor.capture(), any(), any(), any(), any(), anyBoolean()); + + heartbeatSyncer.consumeMessage(convertFromMessage(messageArgumentCaptor.getAllValues()), null); + assertEquals(1, heartbeatSyncer.remoteChannelMap.size()); + + heartbeatSyncer.processConsumerGroupEvent(ConsumerGroupEvent.CLIENT_UNREGISTER, consumerGroup, channelInfoArgumentCaptor.getValue()); + assertTrue(heartbeatSyncer.remoteChannelMap.isEmpty()); + } + private MessageExt convertFromMessage(Message message) { MessageExt messageExt = new MessageExt(); messageExt.setTopic(message.getTopic()); -- 2.32.0.windows.2 From 0027a1486d4f2d6f7dce3010751167e883783945 Mon Sep 17 00:00:00 2001 From: redlsz <szliu0927@gmail.com> Date: Mon, 9 Oct 2023 16:52:10 +0800 Subject: [PATCH 4/7] [ISSUE #7412] Fix pop revive message error when reput checkpoint --- .../org/apache/rocketmq/broker/processor/PopReviveService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java index 93167db37..d5174d3d1 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java @@ -595,6 +595,7 @@ public class PopReviveService extends ServiceThread { newCk.setCId(oldCK.getCId()); newCk.setTopic(oldCK.getTopic()); newCk.setQueueId(oldCK.getQueueId()); + newCk.setBrokerName(oldCK.getBrokerName()); newCk.addDiff(0); MessageExtBrokerInner ckMsg = brokerController.getPopMessageProcessor().buildCkMsg(newCk, queueId); brokerController.getMessageStore().putMessage(ckMsg); -- 2.32.0.windows.2 From b18e564addbcff50165a5e1d9d4ab7db789d901b Mon Sep 17 00:00:00 2001 From: rongtong <jinrongtong5@163.com> Date: Mon, 9 Oct 2023 21:43:01 +0800 Subject: [PATCH 5/7] [ISSUE #7431] Fix flaky test of DLedgerControllerTest#testBrokerLifecycleListener (#7432) * Fix flaky test of DLedgerControllerTest#testBrokerLifecycleListener --- .../impl/DLedgerControllerTest.java | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/controller/src/test/java/org/apache/rocketmq/controller/impl/DLedgerControllerTest.java b/controller/src/test/java/org/apache/rocketmq/controller/impl/DLedgerControllerTest.java index 595a5cb65..d6e5449c5 100644 --- a/controller/src/test/java/org/apache/rocketmq/controller/impl/DLedgerControllerTest.java +++ b/controller/src/test/java/org/apache/rocketmq/controller/impl/DLedgerControllerTest.java @@ -63,7 +63,8 @@ public class DLedgerControllerTest { private List<String> baseDirs; private List<DLedgerController> controllers; - public DLedgerController launchController(final String group, final String peers, final String selfId, final boolean isEnableElectUncleanMaster) { + public DLedgerController launchController(final String group, final String peers, final String selfId, + final boolean isEnableElectUncleanMaster) { String tmpdir = System.getProperty("java.io.tmpdir"); final String path = (StringUtils.endsWith(tmpdir, File.separator) ? tmpdir : tmpdir + File.separator) + group + File.separator + selfId; baseDirs.add(path); @@ -121,11 +122,11 @@ public class DLedgerControllerTest { final RegisterBrokerToControllerRequestHeader registerBrokerToControllerRequestHeader = new RegisterBrokerToControllerRequestHeader(clusterName, brokerName, nextBrokerId, brokerAddress); RemotingCommand remotingCommand2 = leader.registerBroker(registerBrokerToControllerRequestHeader).get(2, TimeUnit.SECONDS); - assertEquals(ResponseCode.SUCCESS, remotingCommand2.getCode()); } - public void brokerTryElectMaster(Controller leader, String clusterName, String brokerName, String brokerAddress, Long brokerId, + public void brokerTryElectMaster(Controller leader, String clusterName, String brokerName, String brokerAddress, + Long brokerId, boolean exceptSuccess) throws Exception { final ElectMasterRequestHeader electMasterRequestHeader = ElectMasterRequestHeader.ofBrokerTrigger(clusterName, brokerName, brokerId); RemotingCommand command = leader.electMaster(electMasterRequestHeader).get(2, TimeUnit.SECONDS); @@ -186,9 +187,9 @@ public class DLedgerControllerTest { registerNewBroker(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[1], 2L); registerNewBroker(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[2], 3L); // try elect - brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[0], 1L,true); - brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[1], 2L, false); - brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[2], 3L,false); + brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[0], 1L, true); + brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[1], 2L, false); + brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[2], 3L, false); final RemotingCommand getInfoResponse = leader.getReplicaInfo(new GetReplicaInfoRequestHeader(DEFAULT_BROKER_NAME)).get(10, TimeUnit.SECONDS); final GetReplicaInfoResponseHeader replicaInfo = (GetReplicaInfoResponseHeader) getInfoResponse.readCustomHeader(); assertEquals(1, replicaInfo.getMasterEpoch().intValue()); @@ -239,6 +240,8 @@ public class DLedgerControllerTest { @Test public void testBrokerLifecycleListener() throws Exception { final DLedgerController leader = mockMetaData(false); + + assertTrue(leader.isLeaderState()); // Mock that master broker has been inactive, and try to elect a new master from sync-state-set // But we shut down two controller, so the ElectMasterEvent will be appended to DLedger failed. // So the statemachine still keep the stale master's information @@ -247,15 +250,20 @@ public class DLedgerControllerTest { dLedgerController.shutdown(); controllers.remove(dLedgerController); } + final ElectMasterRequestHeader request = ElectMasterRequestHeader.ofControllerTrigger(DEFAULT_BROKER_NAME); setBrokerElectPolicy(leader, 1L); Exception exception = null; + RemotingCommand remotingCommand = null; try { - leader.electMaster(request).get(5, TimeUnit.SECONDS); + remotingCommand = leader.electMaster(request).get(5, TimeUnit.SECONDS); } catch (Exception e) { exception = e; } - assertNotNull(exception); + + assertTrue(exception != null || + remotingCommand != null && remotingCommand.getCode() == ResponseCode.CONTROLLER_NOT_LEADER); + // Shut down leader controller leader.shutdown(); controllers.remove(leader); @@ -272,7 +280,7 @@ public class DLedgerControllerTest { setBrokerAlivePredicate(newLeader, 1L); // Check if the statemachine is stale final RemotingCommand resp = newLeader.getReplicaInfo(new GetReplicaInfoRequestHeader(DEFAULT_BROKER_NAME)). - get(10, TimeUnit.SECONDS); + get(10, TimeUnit.SECONDS); final GetReplicaInfoResponseHeader replicaInfo = (GetReplicaInfoResponseHeader) resp.readCustomHeader(); assertEquals(1, replicaInfo.getMasterBrokerId().longValue()); assertEquals(1, replicaInfo.getMasterEpoch().intValue()); -- 2.32.0.windows.2 From 38d3d5d95d371ac89f7d491a4c8719b4a22c60e1 Mon Sep 17 00:00:00 2001 From: mxsm <ljbmxsm@gmail.com> Date: Tue, 10 Oct 2023 09:37:04 +0800 Subject: [PATCH 6/7] [ISSUE #7433]Update the version in the README.md document to 5.1.4 (#7434) --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 56d253ce1..5aaa2ba73 100644 --- a/README.md +++ b/README.md @@ -49,21 +49,21 @@ $ java -version java version "1.8.0_121" ``` -For Windows users, click [here](https://dist.apache.org/repos/dist/release/rocketmq/5.1.3/rocketmq-all-5.1.3-bin-release.zip) to download the 5.1.3 RocketMQ binary release, +For Windows users, click [here](https://dist.apache.org/repos/dist/release/rocketmq/5.1.4/rocketmq-all-5.1.4-bin-release.zip) to download the 5.1.4 RocketMQ binary release, unpack it to your local disk, such as `D:\rocketmq`. For macOS and Linux users, execute following commands: ```shell # Download release from the Apache mirror -$ wget https://dist.apache.org/repos/dist/release/rocketmq/5.1.3/rocketmq-all-5.1.3-bin-release.zip +$ wget https://dist.apache.org/repos/dist/release/rocketmq/5.1.4/rocketmq-all-5.1.4-bin-release.zip # Unpack the release -$ unzip rocketmq-all-5.1.3-bin-release.zip +$ unzip rocketmq-all-5.1.4-bin-release.zip ``` Prepare a terminal and change to the extracted `bin` directory: ```shell -$ cd rocketmq-all-5.1.3-bin-release/bin +$ cd rocketmq-all-5.1.4-bin-release/bin ``` **1) Start NameServer** -- 2.32.0.windows.2 From 4acb43ecee03e429d036e3ff4c28bd402d1b30c7 Mon Sep 17 00:00:00 2001 From: Zhouxiang Zhan <zhouxzhan@apache.org> Date: Tue, 10 Oct 2023 13:54:01 +0800 Subject: [PATCH 7/7] [ISSUE #7330] Add goaway and reconnection mechanism (#7331) * Add shutdown wait for NettyRemotingServer * Add goaway and reconnection mechanism * Add client version check * Add enableTransparentRetry for NettyClientConfig * Add enableReconnectForGoAway for NettyClientConfig * fix unit test * fix client version check --- .../remoting/netty/NettyClientConfig.java | 30 ++++ .../remoting/netty/NettyRemotingAbstract.java | 15 ++ .../remoting/netty/NettyRemotingClient.java | 153 ++++++++++++++++-- .../remoting/netty/NettyRemotingServer.java | 31 ++-- .../remoting/netty/NettyServerConfig.java | 19 +++ .../remoting/protocol/ResponseCode.java | 2 + .../netty/NettyRemotingClientTest.java | 39 ++--- 7 files changed, 239 insertions(+), 50 deletions(-) diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java index b2e7df754..c28288786 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java @@ -53,6 +53,12 @@ public class NettyClientConfig { private boolean disableCallbackExecutor = false; private boolean disableNettyWorkerGroup = false; + private long maxReconnectIntervalTimeSeconds = 60; + + private boolean enableReconnectForGoAway = true; + + private boolean enableTransparentRetry = true; + public boolean isClientCloseSocketIfTimeout() { return clientCloseSocketIfTimeout; } @@ -181,6 +187,30 @@ public class NettyClientConfig { this.disableNettyWorkerGroup = disableNettyWorkerGroup; } + public long getMaxReconnectIntervalTimeSeconds() { + return maxReconnectIntervalTimeSeconds; + } + + public void setMaxReconnectIntervalTimeSeconds(long maxReconnectIntervalTimeSeconds) { + this.maxReconnectIntervalTimeSeconds = maxReconnectIntervalTimeSeconds; + } + + public boolean isEnableReconnectForGoAway() { + return enableReconnectForGoAway; + } + + public void setEnableReconnectForGoAway(boolean enableReconnectForGoAway) { + this.enableReconnectForGoAway = enableReconnectForGoAway; + } + + public boolean isEnableTransparentRetry() { + return enableTransparentRetry; + } + + public void setEnableTransparentRetry(boolean enableTransparentRetry) { + this.enableTransparentRetry = enableTransparentRetry; + } + public String getSocksProxyConfig() { return socksProxyConfig; } diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java index 12e66f913..07ace28ea 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java @@ -40,9 +40,11 @@ import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; import javax.annotation.Nullable; import org.apache.rocketmq.common.AbortProcessException; +import org.apache.rocketmq.common.MQVersion; import org.apache.rocketmq.common.Pair; import org.apache.rocketmq.common.ServiceThread; import org.apache.rocketmq.common.UtilAll; @@ -60,6 +62,7 @@ import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; import org.apache.rocketmq.remoting.metrics.RemotingMetricsManager; import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.remoting.protocol.RemotingSysResponseCode; +import org.apache.rocketmq.remoting.protocol.ResponseCode; import static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.LABEL_IS_LONG_POLLING; import static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.LABEL_REQUEST_CODE; @@ -120,6 +123,8 @@ public abstract class NettyRemotingAbstract { */ protected List<RPCHook> rpcHooks = new ArrayList<>(); + protected AtomicBoolean isShuttingDown = new AtomicBoolean(false); + static { NettyLogger.initNettyLogger(); } @@ -264,6 +269,16 @@ public abstract class NettyRemotingAbstract { Runnable run = buildProcessRequestHandler(ctx, cmd, pair, opaque); + if (isShuttingDown.get()) { + if (cmd.getVersion() > MQVersion.Version.V5_1_4.ordinal()) { + final RemotingCommand response = RemotingCommand.createResponseCommand(ResponseCode.GO_AWAY, + "please go away"); + response.setOpaque(opaque); + writeResponse(ctx.channel(), cmd, response); + return; + } + } + if (pair.getObject1().rejectRequest()) { final RemotingCommand response = RemotingCommand.createResponseCommand(RemotingSysResponseCode.SYSTEM_BUSY, "[REJECTREQUEST]system busy, start flow control for a while"); diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java index 8631d0447..4bc51bd83 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java @@ -18,6 +18,7 @@ package org.apache.rocketmq.remoting.netty; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.TypeReference; +import com.google.common.base.Stopwatch; import io.netty.bootstrap.Bootstrap; import io.netty.buffer.PooledByteBufAllocator; import io.netty.channel.Channel; @@ -48,6 +49,7 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.net.SocketAddress; import java.security.cert.CertificateException; +import java.time.Duration; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -57,6 +59,7 @@ import java.util.Map; import java.util.Random; import java.util.Set; import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutorService; @@ -66,6 +69,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.common.Pair; import org.apache.rocketmq.common.ThreadFactoryImpl; @@ -82,6 +86,7 @@ import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.remoting.protocol.ResponseCode; import org.apache.rocketmq.remoting.proxy.SocksProxyConfig; public class NettyRemotingClient extends NettyRemotingAbstract implements RemotingClient { @@ -97,6 +102,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti private final Map<String /* cidr */, SocksProxyConfig /* proxy */> proxyMap = new HashMap<>(); private final ConcurrentHashMap<String /* cidr */, Bootstrap> bootstrapMap = new ConcurrentHashMap<>(); private final ConcurrentMap<String /* addr */, ChannelWrapper> channelTables = new ConcurrentHashMap<>(); + private final ConcurrentMap<Channel, ChannelWrapper> channelWrapperTables = new ConcurrentHashMap<>(); private final HashedWheelTimer timer = new HashedWheelTimer(r -> new Thread(r, "ClientHouseKeepingService")); @@ -356,9 +362,10 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti this.timer.stop(); for (String addr : this.channelTables.keySet()) { - this.closeChannel(addr, this.channelTables.get(addr).getChannel()); + this.channelTables.get(addr).close(); } + this.channelWrapperTables.clear(); this.channelTables.clear(); this.eventLoopGroupWorker.shutdownGracefully(); @@ -416,7 +423,10 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti } if (removeItemFromTable) { - this.channelTables.remove(addrRemote); + ChannelWrapper channelWrapper = this.channelWrapperTables.remove(channel); + if (channelWrapper != null && channelWrapper.tryClose(channel)) { + this.channelTables.remove(addrRemote); + } LOGGER.info("closeChannel: the channel[{}] was removed from channel table", addrRemote); } @@ -463,7 +473,10 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti } if (removeItemFromTable) { - this.channelTables.remove(addrRemote); + ChannelWrapper channelWrapper = this.channelWrapperTables.remove(channel); + if (channelWrapper != null && channelWrapper.tryClose(channel)) { + this.channelTables.remove(addrRemote); + } LOGGER.info("closeChannel: the channel[{}] was removed from channel table", addrRemote); RemotingHelper.closeChannel(channel); } @@ -511,7 +524,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti if (addr.contains(namesrvAddr)) { ChannelWrapper channelWrapper = this.channelTables.get(addr); if (channelWrapper != null) { - closeChannel(channelWrapper.getChannel()); + channelWrapper.close(); } } } @@ -689,8 +702,9 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti ChannelFuture channelFuture = fetchBootstrap(addr) .connect(hostAndPort[0], Integer.parseInt(hostAndPort[1])); LOGGER.info("createChannel: begin to connect remote host[{}] asynchronously", addr); - cw = new ChannelWrapper(channelFuture); + cw = new ChannelWrapper(addr, channelFuture); this.channelTables.put(addr, cw); + this.channelWrapperTables.put(channelFuture.channel(), cw); } } catch (Exception e) { LOGGER.error("createChannel: create channel exception", e); @@ -758,6 +772,64 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti } } + @Override + public CompletableFuture<RemotingCommand> invoke(String addr, RemotingCommand request, + long timeoutMillis) { + CompletableFuture<RemotingCommand> future = new CompletableFuture<>(); + try { + final Channel channel = this.getAndCreateChannel(addr); + if (channel != null && channel.isActive()) { + return invokeImpl(channel, request, timeoutMillis).whenComplete((v, t) -> { + if (t == null) { + updateChannelLastResponseTime(addr); + } + }).thenApply(ResponseFuture::getResponseCommand); + } else { + this.closeChannel(addr, channel); + future.completeExceptionally(new RemotingConnectException(addr)); + } + } catch (Throwable t) { + future.completeExceptionally(t); + } + return future; + } + + @Override + public CompletableFuture<ResponseFuture> invokeImpl(final Channel channel, final RemotingCommand request, + final long timeoutMillis) { + Stopwatch stopwatch = Stopwatch.createStarted(); + return super.invokeImpl(channel, request, timeoutMillis).thenCompose(responseFuture -> { + RemotingCommand response = responseFuture.getResponseCommand(); + if (response.getCode() == ResponseCode.GO_AWAY) { + if (nettyClientConfig.isEnableReconnectForGoAway()) { + ChannelWrapper channelWrapper = channelWrapperTables.computeIfPresent(channel, (channel0, channelWrapper0) -> { + try { + if (channelWrapper0.reconnect()) { + LOGGER.info("Receive go away from channel {}, recreate the channel", channel0); + channelWrapperTables.put(channelWrapper0.getChannel(), channelWrapper0); + } + } catch (Throwable t) { + LOGGER.error("Channel {} reconnect error", channelWrapper0, t); + } + return channelWrapper0; + }); + if (channelWrapper != null) { + if (nettyClientConfig.isEnableTransparentRetry()) { + long duration = stopwatch.elapsed(TimeUnit.MILLISECONDS); + stopwatch.stop(); + RemotingCommand retryRequest = RemotingCommand.createRequestCommand(request.getCode(), request.readCustomHeader()); + Channel retryChannel = channelWrapper.getChannel(); + if (channel != retryChannel) { + return super.invokeImpl(retryChannel, retryRequest, timeoutMillis - duration); + } + } + } + } + } + return CompletableFuture.completedFuture(responseFuture); + }); + } + @Override public void registerProcessor(int requestCode, NettyRequestProcessor processor, ExecutorService executor) { ExecutorService executorThis = executor; @@ -877,30 +949,41 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti } } - static class ChannelWrapper { - private final ChannelFuture channelFuture; + class ChannelWrapper { + private final ReentrantReadWriteLock lock; + private ChannelFuture channelFuture; // only affected by sync or async request, oneway is not included. + private ChannelFuture channelToClose; private long lastResponseTime; + private volatile long lastReconnectTimestamp = 0L; + private final String channelAddress; - public ChannelWrapper(ChannelFuture channelFuture) { + public ChannelWrapper(String address, ChannelFuture channelFuture) { + this.lock = new ReentrantReadWriteLock(); this.channelFuture = channelFuture; this.lastResponseTime = System.currentTimeMillis(); + this.channelAddress = address; } public boolean isOK() { - return this.channelFuture.channel() != null && this.channelFuture.channel().isActive(); + return getChannel() != null && getChannel().isActive(); } public boolean isWritable() { - return this.channelFuture.channel().isWritable(); + return getChannel().isWritable(); } private Channel getChannel() { - return this.channelFuture.channel(); + return getChannelFuture().channel(); } public ChannelFuture getChannelFuture() { - return channelFuture; + lock.readLock().lock(); + try { + return this.channelFuture; + } finally { + lock.readLock().unlock(); + } } public long getLastResponseTime() { @@ -910,6 +993,52 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti public void updateLastResponseTime() { this.lastResponseTime = System.currentTimeMillis(); } + + public boolean reconnect() { + if (lock.writeLock().tryLock()) { + try { + if (lastReconnectTimestamp == 0L || System.currentTimeMillis() - lastReconnectTimestamp > Duration.ofSeconds(nettyClientConfig.getMaxReconnectIntervalTimeSeconds()).toMillis()) { + channelToClose = channelFuture; + String[] hostAndPort = getHostAndPort(channelAddress); + channelFuture = fetchBootstrap(channelAddress) + .connect(hostAndPort[0], Integer.parseInt(hostAndPort[1])); + lastReconnectTimestamp = System.currentTimeMillis(); + return true; + } + } finally { + lock.writeLock().unlock(); + } + } + return false; + } + + public boolean tryClose(Channel channel) { + try { + lock.readLock().lock(); + if (channelFuture != null) { + if (channelFuture.channel().equals(channel)) { + return true; + } + } + } finally { + lock.readLock().unlock(); + } + return false; + } + + public void close() { + try { + lock.writeLock().lock(); + if (channelFuture != null) { + closeChannel(channelFuture.channel()); + } + if (channelToClose != null) { + closeChannel(channelToClose.channel()); + } + } finally { + lock.writeLock().unlock(); + } + } } class InvokeCallbackWrapper implements InvokeCallback { diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java index aa0d46542..735d36168 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java @@ -53,6 +53,19 @@ import io.netty.util.HashedWheelTimer; import io.netty.util.Timeout; import io.netty.util.TimerTask; import io.netty.util.concurrent.DefaultEventExecutorGroup; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.security.cert.CertificateException; +import java.time.Duration; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.common.Pair; @@ -74,19 +87,6 @@ import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; import org.apache.rocketmq.remoting.protocol.RemotingCommand; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.security.cert.CertificateException; -import java.util.List; -import java.util.NoSuchElementException; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - @SuppressWarnings("NullableProblems") public class NettyRemotingServer extends NettyRemotingAbstract implements RemotingServer { private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_REMOTING_NAME); @@ -305,6 +305,10 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti @Override public void shutdown() { try { + if (nettyServerConfig.isEnableShutdownGracefully() && isShuttingDown.compareAndSet(false, true)) { + Thread.sleep(Duration.ofSeconds(nettyServerConfig.getShutdownWaitTimeSeconds()).toMillis()); + } + this.timer.stop(); this.eventLoopGroupBoss.shutdownGracefully(); @@ -736,6 +740,7 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti @Override public void shutdown() { + isShuttingDown.set(true); if (this.serverChannel != null) { try { this.serverChannel.close().await(5, TimeUnit.SECONDS); diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyServerConfig.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyServerConfig.java index 59ef2c84f..756661f62 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyServerConfig.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyServerConfig.java @@ -38,6 +38,9 @@ public class NettyServerConfig implements Cloneable { private int serverSocketBacklog = NettySystemConfig.socketBacklog; private boolean serverPooledByteBufAllocatorEnable = true; + private boolean enableShutdownGracefully = false; + private int shutdownWaitTimeSeconds = 30; + /** * make install * @@ -171,4 +174,20 @@ public class NettyServerConfig implements Cloneable { public void setWriteBufferHighWaterMark(int writeBufferHighWaterMark) { this.writeBufferHighWaterMark = writeBufferHighWaterMark; } + + public boolean isEnableShutdownGracefully() { + return enableShutdownGracefully; + } + + public void setEnableShutdownGracefully(boolean enableShutdownGracefully) { + this.enableShutdownGracefully = enableShutdownGracefully; + } + + public int getShutdownWaitTimeSeconds() { + return shutdownWaitTimeSeconds; + } + + public void setShutdownWaitTimeSeconds(int shutdownWaitTimeSeconds) { + this.shutdownWaitTimeSeconds = shutdownWaitTimeSeconds; + } } diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ResponseCode.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ResponseCode.java index e81dadf2e..be945c48f 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ResponseCode.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ResponseCode.java @@ -99,6 +99,8 @@ public class ResponseCode extends RemotingSysResponseCode { public static final int RPC_SEND_TO_CHANNEL_FAILED = -1004; public static final int RPC_TIME_OUT = -1006; + public static final int GO_AWAY = 1500; + /** * Controller response code */ diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java index e72e7bd53..1cc6b4f46 100644 --- a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java +++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java @@ -47,7 +47,6 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -74,13 +73,11 @@ public class NettyRemotingClientTest { RemotingCommand response = RemotingCommand.createResponseCommand(null); response.setCode(ResponseCode.SUCCESS); - doAnswer(invocation -> { - InvokeCallback callback = invocation.getArgument(3); - ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); - responseFuture.setResponseCommand(response); - callback.operationSucceed(responseFuture.getResponseCommand()); - return null; - }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); + ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); + responseFuture.setResponseCommand(response); + CompletableFuture<RemotingCommand> future0 = new CompletableFuture<>(); + future0.complete(responseFuture.getResponseCommand()); + doReturn(future0).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); CompletableFuture<RemotingCommand> future = remotingClient.invoke("0.0.0.0", request, 1000); RemotingCommand actual = future.get(); @@ -93,11 +90,9 @@ public class NettyRemotingClientTest { RemotingCommand response = RemotingCommand.createResponseCommand(null); response.setCode(ResponseCode.SUCCESS); - doAnswer(invocation -> { - InvokeCallback callback = invocation.getArgument(3); - callback.operationFail(new RemotingSendRequestException(null)); - return null; - }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); + CompletableFuture<RemotingCommand> future0 = new CompletableFuture<>(); + future0.completeExceptionally(new RemotingSendRequestException(null)); + doReturn(future0).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); CompletableFuture<RemotingCommand> future = remotingClient.invoke("0.0.0.0", request, 1000); Throwable thrown = catchThrowable(future::get); @@ -110,11 +105,9 @@ public class NettyRemotingClientTest { RemotingCommand response = RemotingCommand.createResponseCommand(null); response.setCode(ResponseCode.SUCCESS); - doAnswer(invocation -> { - InvokeCallback callback = invocation.getArgument(3); - callback.operationFail(new RemotingTimeoutException("")); - return null; - }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); + CompletableFuture<RemotingCommand> future0 = new CompletableFuture<>(); + future0.completeExceptionally(new RemotingTimeoutException("")); + doReturn(future0).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); CompletableFuture<RemotingCommand> future = remotingClient.invoke("0.0.0.0", request, 1000); Throwable thrown = catchThrowable(future::get); @@ -125,13 +118,9 @@ public class NettyRemotingClientTest { public void testRemotingException() throws Exception { RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null); - RemotingCommand response = RemotingCommand.createResponseCommand(null); - response.setCode(ResponseCode.SUCCESS); - doAnswer(invocation -> { - InvokeCallback callback = invocation.getArgument(3); - callback.operationFail(new RemotingException(null)); - return null; - }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); + CompletableFuture<RemotingCommand> future0 = new CompletableFuture<>(); + future0.completeExceptionally(new RemotingException("")); + doReturn(future0).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); CompletableFuture<RemotingCommand> future = remotingClient.invoke("0.0.0.0", request, 1000); Throwable thrown = catchThrowable(future::get); -- 2.32.0.windows.2
Locations
Projects
Search
Status Monitor
Help
Open Build Service
OBS Manuals
API Documentation
OBS Portal
Reporting a Bug
Contact
Mailing List
Forums
Chat (IRC)
Twitter
Open Build Service (OBS)
is an
openSUSE project
.
浙ICP备2022010568号-2