From 2bf80b3502415d575c22410e9ee71606023d5685 Mon Sep 17 00:00:00 2001 From: Oleg Ovcharuk Date: Wed, 11 Nov 2026 19:14:33 +0300 Subject: [PATCH] Add opt-in primary Bridge pile balancing --- CHANGELOG.md | 2 + README.md | 16 +++++++ .../tech/ydb/core/grpc/BalancingSettings.java | 13 ++++++ .../java/tech/ydb/core/impl/YdbDiscovery.java | 11 ++++- .../tech/ydb/core/impl/pool/EndpointPool.java | 5 ++- .../ydb/core/impl/pool/EndpointRecord.java | 16 +++++++ .../ydb/core/impl/pool/PriorityPicker.java | 26 +++++++++++ .../java/tech/ydb/core/impl/MockedCall.java | 11 ++++- .../tech/ydb/core/impl/YdbDiscoveryTest.java | 40 +++++++++++++++++ .../ydb/core/impl/YdbTransportImplTest.java | 42 +++++++++++++++++- .../ydb/core/impl/pool/EndpointPoolTest.java | 43 +++++++++++++++++++ .../core/impl/pool/PriorityPickerTest.java | 19 ++++++++ 12 files changed, 240 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a1b0ee4e..1fe577180 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +* Core: added opt-in PRIMARY/PROMOTED Bridge pile preference with synchronized-pile fallback + ## 2.5.0 ## * Core: upgraded to grpc-java v1.82.4 * Core: removed deprecated methods from GrpcTransportBuilder diff --git a/README.md b/README.md index 69f9a2a11..8afed025b 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,22 @@ After that you can specify the SDK modules that your project needs in the depend ``` +### Prefer the primary Bridge pile + +Enable PRIMARY/PROMOTED pile preference explicitly when building the transport: + +```java +GrpcTransport transport = GrpcTransport.forConnectionString(connectionString) + .withBalancingSettings(BalancingSettings.preferPrimaryPile()) + .build(); +``` + +Use `tech.ydb.core.grpc.BalancingSettings` and `tech.ydb.core.grpc.GrpcTransport`. +The policy follows pile roles reported by discovery, including role changes at unchanged endpoint addresses. +SYNCHRONIZED endpoints remain available as fallback; other or unknown states are last-resort candidates. +On a cluster without Bridge metadata the policy uses ordinary balancing. The default policy is unchanged. +Requests bound to an existing session or a specific node retain their node affinity. + ## Versioning Policy YDB Java SDK comply to guidelines from [SemVer2.0.0](https://semver.org/) with several [exceptions](VERSIONING.md). diff --git a/core/src/main/java/tech/ydb/core/grpc/BalancingSettings.java b/core/src/main/java/tech/ydb/core/grpc/BalancingSettings.java index c6430ce55..dd232cedd 100644 --- a/core/src/main/java/tech/ydb/core/grpc/BalancingSettings.java +++ b/core/src/main/java/tech/ydb/core/grpc/BalancingSettings.java @@ -21,6 +21,9 @@ public enum Policy { * Detecting of local DC by the latency measuring */ USE_DETECT_LOCAL_DC, + + /** Prefer PRIMARY/PROMOTED Bridge piles, with fallback to synchronized piles. */ + USE_PREFERABLE_PRIMARY_PILE, } private final Policy policy; @@ -59,6 +62,16 @@ public static BalancingSettings detectLocalDs() { return new BalancingSettings(Policy.USE_DETECT_LOCAL_DC); } + /** + * Prefer PRIMARY/PROMOTED Bridge endpoints, retaining synchronized endpoints as fallback. + * On an ordinary cluster all endpoints remain equally preferred. Node-bound requests retain their affinity. + * + * @return primary-pile balancing settings + */ + public static BalancingSettings preferPrimaryPile() { + return new BalancingSettings(Policy.USE_PREFERABLE_PRIMARY_PILE); + } + @Override public String toString() { return "BalancingSettings{" + diff --git a/core/src/main/java/tech/ydb/core/impl/YdbDiscovery.java b/core/src/main/java/tech/ydb/core/impl/YdbDiscovery.java index 4b53a12f0..2ac41ed24 100644 --- a/core/src/main/java/tech/ydb/core/impl/YdbDiscovery.java +++ b/core/src/main/java/tech/ydb/core/impl/YdbDiscovery.java @@ -2,7 +2,9 @@ import java.time.Duration; import java.time.Instant; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; @@ -25,6 +27,8 @@ import tech.ydb.core.impl.pool.EndpointRecord; import tech.ydb.core.operation.OperationBinder; import tech.ydb.core.utils.FutureTools; +import tech.ydb.proto.bridge.YdbBridgeCommon.PileState; +import tech.ydb.proto.bridge.YdbBridgeCommon.PileState.State; import tech.ydb.proto.discovery.DiscoveryProtos; import tech.ydb.proto.discovery.DiscoveryProtos.EndpointInfo; import tech.ydb.proto.discovery.v1.DiscoveryServiceGrpc; @@ -237,9 +241,14 @@ private void handleDiscoveryResult(Result r return; } + Map piles = new HashMap<>(); + for (PileState pile : result.getPileStatesList()) { + piles.put(pile.getPileName(), pile.getState()); + } List records = result.getEndpointsList().stream() .map(e -> new EndpointRecord(createAddress(e), e.getPort(), e.getNodeId(), e.getLocation(), - e.getSslTargetNameOverride())) + e.getSslTargetNameOverride(), piles.isEmpty() ? null + : piles.getOrDefault(e.getBridgePileName(), State.UNSPECIFIED))) .collect(Collectors.toList()); logger.debug("successfully received ListEndpoints result with {} endpoints", records.size()); diff --git a/core/src/main/java/tech/ydb/core/impl/pool/EndpointPool.java b/core/src/main/java/tech/ydb/core/impl/pool/EndpointPool.java index 9a1f07803..6b0477851 100644 --- a/core/src/main/java/tech/ydb/core/impl/pool/EndpointPool.java +++ b/core/src/main/java/tech/ydb/core/impl/pool/EndpointPool.java @@ -83,6 +83,9 @@ public EndpointRecord getEndpoint(Set readyEnpoints, GrpcRequestSettings List ready = readyEnpoints.stream() .map(recordsByEndpoint::get) .filter(pr -> pr != null && !pr.isPessimized()) + .filter(pr -> balancingSettings.getPolicy() + != BalancingSettings.Policy.USE_PREFERABLE_PRIMARY_PILE + || pr.getPriority() == records.get(0).getPriority()) .collect(Collectors.toList()); if (!ready.isEmpty()) { @@ -112,7 +115,7 @@ public List setNewState(String selfLocation, List endpoints; @Override public Instant instant() { @@ -306,6 +345,7 @@ public boolean needToForceDiscovery() { @Override public CompletableFuture handleEndpoints(List endpoints, String selfLocation) { + this.endpoints = endpoints; return CompletableFuture.completedFuture(Boolean.TRUE); } } diff --git a/core/src/test/java/tech/ydb/core/impl/YdbTransportImplTest.java b/core/src/test/java/tech/ydb/core/impl/YdbTransportImplTest.java index 087566642..1bca77711 100644 --- a/core/src/test/java/tech/ydb/core/impl/YdbTransportImplTest.java +++ b/core/src/test/java/tech/ydb/core/impl/YdbTransportImplTest.java @@ -26,6 +26,7 @@ import tech.ydb.core.Result; import tech.ydb.core.StatusCode; import tech.ydb.core.UnexpectedResultException; +import tech.ydb.core.grpc.BalancingSettings; import tech.ydb.core.grpc.GrpcRequestSettings; import tech.ydb.core.grpc.GrpcTransport; import tech.ydb.core.grpc.GrpcTransportBuilder; @@ -38,6 +39,7 @@ import tech.ydb.core.tracing.SpanKind; import tech.ydb.core.tracing.Tracer; import tech.ydb.core.utils.Version; +import tech.ydb.proto.bridge.YdbBridgeCommon.PileState; import tech.ydb.proto.discovery.DiscoveryProtos; import tech.ydb.proto.discovery.v1.DiscoveryServiceGrpc; @@ -76,7 +78,11 @@ public void shutdown() { } private CompletableFuture> whoAmI(GrpcTransport transport) { - GrpcRequestSettings settings = GrpcRequestSettings.newBuilder().build(); + return whoAmI(transport, GrpcRequestSettings.newBuilder().build()); + } + + private CompletableFuture> whoAmI( + GrpcTransport transport, GrpcRequestSettings settings) { DiscoveryProtos.WhoAmIRequest request = DiscoveryProtos.WhoAmIRequest.newBuilder().build(); return transport.unaryCall(DiscoveryServiceGrpc.getWhoAmIMethod(), settings, request) .thenApply(OperationBinder.bindSync( @@ -85,6 +91,40 @@ private CompletableFuture> whoAmI(GrpcTrans )); } + @Test + public void primaryPilePolicyUsesDiscoveryMetadataAndPreservesAffinity() throws InterruptedException { + ManagedChannel secondary = Mockito.mock(ManagedChannel.class); + Mockito.when(secondary.getState(Mockito.anyBoolean())).thenReturn(ConnectivityState.READY); + Mockito.when(secondary.shutdown()).thenReturn(secondary); + Mockito.when(secondary.shutdownNow()).thenReturn(secondary); + Mockito.when(secondary.awaitTermination(Mockito.anyLong(), Mockito.any())).thenReturn(true); + Mockito.when(channelFactory.newManagedChannel("secondary", 2136, null)).thenReturn(secondary); + DiscoveryProtos.ListEndpointsResult result = DiscoveryProtos.ListEndpointsResult.newBuilder() + .addEndpoints(DiscoveryProtos.EndpointInfo.newBuilder().setAddress("node").setPort(2136) + .setNodeId(1).setBridgePileName("A")) + .addEndpoints(DiscoveryProtos.EndpointInfo.newBuilder().setAddress("secondary").setPort(2136) + .setNodeId(2).setBridgePileName("B")) + .addPileStates(PileState.newBuilder().setPileName("A").setState(PileState.State.PRIMARY)) + .addPileStates(PileState.newBuilder().setPileName("B").setState(PileState.State.SYNCHRONIZED)) + .build(); + Mockito.when(discoveryChannel.newCall(Mockito.eq(DiscoveryServiceGrpc.getListEndpointsMethod()), Mockito.any())) + .thenAnswer(call -> MockedCall.discovery(result)); + Mockito.when(transportChannel.newCall(Mockito.eq(DiscoveryServiceGrpc.getWhoAmIMethod()), Mockito.any())) + .thenAnswer(call -> MockedCall.whoAmICall("primary")); + Mockito.when(secondary.newCall(Mockito.eq(DiscoveryServiceGrpc.getWhoAmIMethod()), Mockito.any())) + .thenAnswer(call -> MockedCall.whoAmICall("secondary")); + try (GrpcTransport transport = GrpcTransport.forConnectionString("grpc://mocked:2136/local") + .withBalancingSettings(BalancingSettings.preferPrimaryPile()) + .withChannelFactoryBuilder(builder -> channelFactory) + .build()) { + Assert.assertEquals("secondary", whoAmI(transport, GrpcRequestSettings.newBuilder() + .withPreferredNodeID(2).build()).join().getValue().getUser()); + Assert.assertEquals("primary", whoAmI(transport, GrpcRequestSettings.newBuilder() + .withPreferReadyChannel(true).build()).join().getValue().getUser()); + Assert.assertEquals("primary", whoAmI(transport).join().getValue().getUser()); + } + } + @Test public void defaultBuildGoodTest() { Mockito.when(discoveryChannel.newCall(Mockito.eq(DiscoveryServiceGrpc.getListEndpointsMethod()), Mockito.any())) diff --git a/core/src/test/java/tech/ydb/core/impl/pool/EndpointPoolTest.java b/core/src/test/java/tech/ydb/core/impl/pool/EndpointPoolTest.java index 0af168638..ceeb537c4 100644 --- a/core/src/test/java/tech/ydb/core/impl/pool/EndpointPoolTest.java +++ b/core/src/test/java/tech/ydb/core/impl/pool/EndpointPoolTest.java @@ -23,6 +23,7 @@ import tech.ydb.core.UnexpectedResultException; import tech.ydb.core.grpc.BalancingSettings; import tech.ydb.core.grpc.GrpcRequestSettings; +import tech.ydb.proto.bridge.YdbBridgeCommon.PileState.State; /** * @author Aleksandr Gorshenin @@ -67,6 +68,48 @@ private GrpcRequestSettings preferReady() { return GrpcRequestSettings.newBuilder().withPreferReadyChannel(true).build(); } + @Test + public void primaryPileRoleOnlySwitchoverTest() { + EndpointPool pool = new EndpointPool(BalancingSettings.preferPrimaryPile()); + EndpointRecord a = new EndpointRecord("a", 2135, 1, "DC1", null, State.PRIMARY); + EndpointRecord b = new EndpointRecord("b", 2135, 2, "DC2", null, State.SYNCHRONIZED); + Assert.assertTrue(pool.setNewState("DC1", list(a, b)).isEmpty()); + Assert.assertEquals(1, pool.getEndpoint(EMPTY, empty()).getNodeId()); + + a = new EndpointRecord("a", 2135, 1, "DC1", null, State.SYNCHRONIZED); + b = new EndpointRecord("b", 2135, 2, "DC2", null, State.PRIMARY); + Assert.assertTrue(pool.setNewState("DC1", list(a, b)).isEmpty()); + Assert.assertEquals(2, pool.getEndpoint(EMPTY, empty()).getNodeId()); + Assert.assertEquals(1, pool.getEndpoint(EMPTY, nodeId(1)).getNodeId()); + Assert.assertEquals(1, pool.getEndpoint(EMPTY, direct(1)).getNodeId()); + } + + @Test + public void primaryPilePreferReadyAndFallbackTest() { + EndpointPool pool = new EndpointPool(BalancingSettings.preferPrimaryPile()); + EndpointRecord a = new EndpointRecord("a", 2135, 1, "DC1", null, State.PRIMARY); + EndpointRecord b = new EndpointRecord("b", 2135, 2, "DC2", null, State.SYNCHRONIZED); + EndpointRecord c = new EndpointRecord("c", 2135, 3, "DC1", null, State.UNRECOGNIZED); + pool.setNewState("DC1", list(a, b, c)); + Assert.assertEquals(1, pool.getEndpoint(Collections.singleton("b:2135"), preferReady()).getNodeId()); + pool.pessimizeEndpoint(a, "primary unavailable"); + Assert.assertEquals(2, pool.getEndpoint(Collections.singleton("c:2135"), preferReady()).getNodeId()); + pool.pessimizeEndpoint(b, "synchronized unavailable"); + Assert.assertEquals(3, pool.getEndpoint(EMPTY, empty()).getNodeId()); + } + + @Test + public void primaryPileOrdinaryClusterAndPromotedTest() { + EndpointPool pool = new EndpointPool(BalancingSettings.preferPrimaryPile()); + pool.setNewState("DC1", list(endpoint(1, "a", 2135, "DC1"), endpoint(2, "b", 2135, "DC2"))); + check(pool).bestEndpointsCount(2); + pool.setNewState("DC1", list( + new EndpointRecord("a", 2135, 1, "DC1", null, State.PRIMARY), + new EndpointRecord("b", 2135, 2, "DC2", null, State.PROMOTED), + new EndpointRecord("c", 2135, 3, "DC1", null, State.SYNCHRONIZED))); + check(pool).bestEndpointsCount(2); + } + @Test public void directWithoutNodeIdTest() { EndpointPool pool = new EndpointPool(useAllNodes()); diff --git a/core/src/test/java/tech/ydb/core/impl/pool/PriorityPickerTest.java b/core/src/test/java/tech/ydb/core/impl/pool/PriorityPickerTest.java index c11cfdb8c..16d4612a1 100644 --- a/core/src/test/java/tech/ydb/core/impl/pool/PriorityPickerTest.java +++ b/core/src/test/java/tech/ydb/core/impl/pool/PriorityPickerTest.java @@ -15,12 +15,31 @@ import org.junit.Test; import tech.ydb.core.grpc.BalancingSettings; +import tech.ydb.proto.bridge.YdbBridgeCommon.PileState.State; /** * @author Kirill */ public class PriorityPickerTest { + @Test + public void primaryPilePrioritiesTest() { + PriorityPicker picker = PriorityPicker.from(BalancingSettings.preferPrimaryPile(), "DC1", null); + Assert.assertEquals(0, picker.getPriority(new EndpointRecord("ordinary", 2135))); + for (State state : State.values()) { + EndpointRecord endpoint = new EndpointRecord("host", 2135, 1, "DC2", null, state); + int expected = state == State.PRIMARY || state == State.PROMOTED ? 0 + : state == State.SYNCHRONIZED ? 1000 : 2000; + Assert.assertEquals(state.name(), expected, picker.getPriority(endpoint)); + } + PriorityPicker ordinary = PriorityPicker.from(BalancingSettings.defaultInstance(), "DC1", null); + Assert.assertEquals(0, ordinary.getPriority( + new EndpointRecord("host", 2135, 1, "DC2", null, State.DISCONNECTED))); + PriorityPicker locality = PriorityPicker.from(BalancingSettings.fromLocation("DC1"), "DC1", null); + Assert.assertEquals(1000, locality.getPriority( + new EndpointRecord("host", 2135, 1, "DC2", null, State.PRIMARY))); + } + @Test public void randomEvaluatorTest() { PriorityPicker picker = PriorityPicker.from(BalancingSettings.defaultInstance(), null, null);