Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
* Core: added opt-in PRIMARY/PROMOTED Bridge pile preference with synchronized-pile fallback

## 2.5.0 ##
* Core: upgraded to grpc-java v1.82.4
* Core: removed deprecated methods from GrpcTransportBuilder
Expand Down
16 changes: 16 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,22 @@ After that you can specify the SDK modules that your project needs in the depend
</dependencies>
```

### Prefer the primary Bridge pile

Enable PRIMARY/PROMOTED pile preference explicitly when building the transport:

```java
GrpcTransport transport = GrpcTransport.forConnectionString(connectionString)
.withBalancingSettings(BalancingSettings.preferPrimaryPile())
.build();
```

Use `tech.ydb.core.grpc.BalancingSettings` and `tech.ydb.core.grpc.GrpcTransport`.
The policy follows pile roles reported by discovery, including role changes at unchanged endpoint addresses.
SYNCHRONIZED endpoints remain available as fallback; other or unknown states are last-resort candidates.
On a cluster without Bridge metadata the policy uses ordinary balancing. The default policy is unchanged.
Requests bound to an existing session or a specific node retain their node affinity.

## Versioning Policy

YDB Java SDK comply to guidelines from [SemVer2.0.0](https://semver.org/) with several [exceptions](VERSIONING.md).
Expand Down
13 changes: 13 additions & 0 deletions core/src/main/java/tech/ydb/core/grpc/BalancingSettings.java
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ public enum Policy {
* Detecting of local DC by the latency measuring
*/
USE_DETECT_LOCAL_DC,

/** Prefer PRIMARY/PROMOTED Bridge piles, with fallback to synchronized piles. */
USE_PREFERABLE_PRIMARY_PILE,
}

private final Policy policy;
Expand Down Expand Up @@ -59,6 +62,16 @@ public static BalancingSettings detectLocalDs() {
return new BalancingSettings(Policy.USE_DETECT_LOCAL_DC);
}

/**
* Prefer PRIMARY/PROMOTED Bridge endpoints, retaining synchronized endpoints as fallback.
* On an ordinary cluster all endpoints remain equally preferred. Node-bound requests retain their affinity.
*
* @return primary-pile balancing settings
*/
public static BalancingSettings preferPrimaryPile() {
return new BalancingSettings(Policy.USE_PREFERABLE_PRIMARY_PILE);
}

@Override
public String toString() {
return "BalancingSettings{" +
Expand Down
11 changes: 10 additions & 1 deletion core/src/main/java/tech/ydb/core/impl/YdbDiscovery.java
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@

import java.time.Duration;
import java.time.Instant;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Future;
import java.util.concurrent.ScheduledExecutorService;
Expand All @@ -25,6 +27,8 @@
import tech.ydb.core.impl.pool.EndpointRecord;
import tech.ydb.core.operation.OperationBinder;
import tech.ydb.core.utils.FutureTools;
import tech.ydb.proto.bridge.YdbBridgeCommon.PileState;
import tech.ydb.proto.bridge.YdbBridgeCommon.PileState.State;
import tech.ydb.proto.discovery.DiscoveryProtos;
import tech.ydb.proto.discovery.DiscoveryProtos.EndpointInfo;
import tech.ydb.proto.discovery.v1.DiscoveryServiceGrpc;
Expand Down Expand Up @@ -237,9 +241,14 @@ private void handleDiscoveryResult(Result<DiscoveryProtos.ListEndpointsResult> r
return;
}

Map<String, State> piles = new HashMap<>();
for (PileState pile : result.getPileStatesList()) {
piles.put(pile.getPileName(), pile.getState());
}
List<EndpointRecord> records = result.getEndpointsList().stream()
.map(e -> new EndpointRecord(createAddress(e), e.getPort(), e.getNodeId(), e.getLocation(),
e.getSslTargetNameOverride()))
e.getSslTargetNameOverride(), piles.isEmpty() ? null
: piles.getOrDefault(e.getBridgePileName(), State.UNSPECIFIED)))
.collect(Collectors.toList());

logger.debug("successfully received ListEndpoints result with {} endpoints", records.size());
Expand Down
5 changes: 4 additions & 1 deletion core/src/main/java/tech/ydb/core/impl/pool/EndpointPool.java
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,9 @@ public EndpointRecord getEndpoint(Set<String> readyEnpoints, GrpcRequestSettings
List<PriorityEndpoint> ready = readyEnpoints.stream()
.map(recordsByEndpoint::get)
.filter(pr -> pr != null && !pr.isPessimized())
.filter(pr -> balancingSettings.getPolicy()
!= BalancingSettings.Policy.USE_PREFERABLE_PRIMARY_PILE
|| pr.getPriority() == records.get(0).getPriority())
.collect(Collectors.toList());

if (!ready.isEmpty()) {
Expand Down Expand Up @@ -112,7 +115,7 @@ public List<EndpointRecord> setNewState(String selfLocation, List<EndpointRecord

logger.debug("init new state with {} endpoints", endpoints.size());
for (EndpointRecord endpoint : endpoints) {
int priority = picker.getEndpointPriority(endpoint.getLocation());
int priority = picker.getPriority(endpoint);

PriorityEndpoint entry = new PriorityEndpoint(endpoint, priority);
String hostAndPort = endpoint.getHostAndPort();
Expand Down
16 changes: 16 additions & 0 deletions core/src/main/java/tech/ydb/core/impl/pool/EndpointRecord.java
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@

import java.util.Objects;

import javax.annotation.Nullable;

import tech.ydb.proto.bridge.YdbBridgeCommon.PileState.State;

/**
* @author Nikolay Perfilov
*/
Expand All @@ -12,12 +16,19 @@ public class EndpointRecord {
private final String authority;
private final int port;
private final int nodeId;
private final State pileState;

public EndpointRecord(String host, int port, int nodeId, String locationDC, String authority) {
this(host, port, nodeId, locationDC, authority, null);
}

public EndpointRecord(String host, int port, int nodeId, String locationDC, String authority,
@Nullable State pileState) {
this.host = Objects.requireNonNull(host);
this.port = port;
this.hostAndPort = host + ":" + port;
this.nodeId = nodeId;
this.pileState = pileState;
this.locationDC = locationDC;
if (authority != null && !authority.isEmpty()) {
this.authority = authority;
Expand Down Expand Up @@ -54,6 +65,11 @@ public String getLocation() {
return locationDC;
}

@Nullable
public State getPileState() {
return pileState;
}

@Override
public String toString() {
return "Endpoint{host=" + host + ", port=" + port + ", node=" + nodeId +
Expand Down
26 changes: 26 additions & 0 deletions core/src/main/java/tech/ydb/core/impl/pool/PriorityPicker.java
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,33 @@ public class PriorityPicker {
private static final int DETECT_DC_TCP_PING_TIMEOUT_MS = 5000;

private final String preferredLocation;
private final boolean preferPrimaryPile;

private PriorityPicker(String location) {
this(location, false);
}

private PriorityPicker(String location, boolean preferPrimaryPile) {
this.preferredLocation = location;
this.preferPrimaryPile = preferPrimaryPile;
}

int getPriority(EndpointRecord endpoint) {
if (!preferPrimaryPile) {
return getEndpointPriority(endpoint.getLocation());
}
if (endpoint.getPileState() == null) {
return 0; // Discovery has no Bridge roster.
}
switch (endpoint.getPileState()) {
case PRIMARY:
case PROMOTED:
return 0;
case SYNCHRONIZED:
return LOCALITY_SHIFT;
default:
return 2 * LOCALITY_SHIFT;
}
}

public int getEndpointPriority(String location) {
Expand All @@ -49,6 +73,8 @@ public static PriorityPicker from(BalancingSettings settings, String selfLocatio
return new PriorityPicker(getLocationFromConfig(settings.getPreferableLocation(), selfLocation));
case USE_DETECT_LOCAL_DC:
return new PriorityPicker(detectLocalDC(endpoints, Ticker.systemTicker()));
case USE_PREFERABLE_PRIMARY_PILE:
return new PriorityPicker(null, true);
default:
throw new RuntimeException("Not implemented balancing policy: " + settings.getPolicy().name());
}
Expand Down
11 changes: 10 additions & 1 deletion core/src/test/java/tech/ydb/core/impl/MockedCall.java
Original file line number Diff line number Diff line change
Expand Up @@ -132,9 +132,18 @@ public static DiscoveryCall discovery(Executor executor, String selfLocation, En
}
builder.setSelfLocation(selfLocation);

return discovery(executor, builder.build());
}

public static DiscoveryCall discovery(DiscoveryProtos.ListEndpointsResult result) {
return discovery(Runnable::run, result);
}

public static DiscoveryCall discovery(Executor executor, DiscoveryProtos.ListEndpointsResult result) {

OperationProtos.Operation operation = OperationProtos.Operation.newBuilder()
.setReady(true)
.setResult(Any.pack(builder.build()))
.setResult(Any.pack(result))
.setId("discovery-id")
.setStatus(StatusCodesProtos.StatusIds.StatusCode.SUCCESS)
.build();
Expand Down
40 changes: 40 additions & 0 deletions core/src/test/java/tech/ydb/core/impl/YdbDiscoveryTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@
import tech.ydb.core.impl.auth.AuthCallOptions;
import tech.ydb.core.impl.pool.EndpointRecord;
import tech.ydb.core.impl.pool.ManagedChannelFactory;
import tech.ydb.proto.bridge.YdbBridgeCommon.PileState;
import tech.ydb.proto.bridge.YdbBridgeCommon.PileState.State;
import tech.ydb.proto.discovery.DiscoveryProtos;
import tech.ydb.proto.discovery.v1.DiscoveryServiceGrpc;


Expand Down Expand Up @@ -96,6 +99,41 @@ public void baseTest() {
discovery.stop();
}

@Test
public void bridgeMetadataAndRoleChangesTest() {
DiscoveryProtos.ListEndpointsResult.Builder result = DiscoveryProtos.ListEndpointsResult.newBuilder()
.addEndpoints(DiscoveryProtos.EndpointInfo.newBuilder().setAddress("a").setPort(2135)
.setNodeId(1).setBridgePileName("A"))
.addEndpoints(DiscoveryProtos.EndpointInfo.newBuilder().setAddress("b").setPort(2135)
.setNodeId(2).setBridgePileName("B"))
.addEndpoints(DiscoveryProtos.EndpointInfo.newBuilder().setAddress("unknown").setPort(2135)
.setNodeId(3).setBridgePileName("missing"));
DiscoveryProtos.ListEndpointsResult ordinary = result.build();
result.addPileStates(PileState.newBuilder().setPileName("A").setState(State.PRIMARY))
.addPileStates(PileState.newBuilder().setPileName("B").setState(State.SYNCHRONIZED));
DiscoveryProtos.ListEndpointsResult bridged = result.build();
result.setPileStates(0, PileState.newBuilder().setPileName("A").setState(State.SYNCHRONIZED))
.setPileStates(1, PileState.newBuilder().setPileName("B").setState(State.PRIMARY));
Mockito.when(channel.newCall(Mockito.eq(DiscoveryServiceGrpc.getListEndpointsMethod()), Mockito.any()))
.thenReturn(MockedCall.discovery(ordinary))
.thenReturn(MockedCall.discovery(bridged))
.thenReturn(MockedCall.discovery(result.build()));
TestHandler handler = new TestHandler();
YdbDiscovery discovery = new YdbDiscovery(handler, scheduler, "/bridge", Duration.ofMillis(100));
discovery.start();
scheduler.hasTasksCount(1).runNextTask();
Assert.assertNull(handler.endpoints.get(0).getPileState());
handler.force = true;
scheduler.hasTasksCount(1).runNextTask();
Assert.assertEquals(State.PRIMARY, handler.endpoints.get(0).getPileState());
Assert.assertEquals(State.SYNCHRONIZED, handler.endpoints.get(1).getPileState());
Assert.assertEquals(State.UNSPECIFIED, handler.endpoints.get(2).getPileState());
scheduler.hasTasksCount(1).runNextTask();
Assert.assertEquals(State.SYNCHRONIZED, handler.endpoints.get(0).getPileState());
Assert.assertEquals(State.PRIMARY, handler.endpoints.get(1).getPileState());
discovery.stop();
}

@Test
public void baseDiscoveryForcing() {
Mockito.when(channel.newCall(Mockito.eq(DiscoveryServiceGrpc.getListEndpointsMethod()), Mockito.any()))
Expand Down Expand Up @@ -286,6 +324,7 @@ public void failedDiscoveryTest() {

private class TestHandler implements YdbDiscovery.Handler {
private volatile boolean force = false;
private List<EndpointRecord> endpoints;

@Override
public Instant instant() {
Expand All @@ -306,6 +345,7 @@ public boolean needToForceDiscovery() {

@Override
public CompletableFuture<Boolean> handleEndpoints(List<EndpointRecord> endpoints, String selfLocation) {
this.endpoints = endpoints;
return CompletableFuture.completedFuture(Boolean.TRUE);
}
}
Expand Down
42 changes: 41 additions & 1 deletion core/src/test/java/tech/ydb/core/impl/YdbTransportImplTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import tech.ydb.core.Result;
import tech.ydb.core.StatusCode;
import tech.ydb.core.UnexpectedResultException;
import tech.ydb.core.grpc.BalancingSettings;
import tech.ydb.core.grpc.GrpcRequestSettings;
import tech.ydb.core.grpc.GrpcTransport;
import tech.ydb.core.grpc.GrpcTransportBuilder;
Expand All @@ -38,6 +39,7 @@
import tech.ydb.core.tracing.SpanKind;
import tech.ydb.core.tracing.Tracer;
import tech.ydb.core.utils.Version;
import tech.ydb.proto.bridge.YdbBridgeCommon.PileState;
import tech.ydb.proto.discovery.DiscoveryProtos;
import tech.ydb.proto.discovery.v1.DiscoveryServiceGrpc;

Expand Down Expand Up @@ -76,7 +78,11 @@ public void shutdown() {
}

private CompletableFuture<Result<DiscoveryProtos.WhoAmIResult>> whoAmI(GrpcTransport transport) {
GrpcRequestSettings settings = GrpcRequestSettings.newBuilder().build();
return whoAmI(transport, GrpcRequestSettings.newBuilder().build());
}

private CompletableFuture<Result<DiscoveryProtos.WhoAmIResult>> whoAmI(
GrpcTransport transport, GrpcRequestSettings settings) {
DiscoveryProtos.WhoAmIRequest request = DiscoveryProtos.WhoAmIRequest.newBuilder().build();
return transport.unaryCall(DiscoveryServiceGrpc.getWhoAmIMethod(), settings, request)
.thenApply(OperationBinder.bindSync(
Expand All @@ -85,6 +91,40 @@ private CompletableFuture<Result<DiscoveryProtos.WhoAmIResult>> whoAmI(GrpcTrans
));
}

@Test
public void primaryPilePolicyUsesDiscoveryMetadataAndPreservesAffinity() throws InterruptedException {
ManagedChannel secondary = Mockito.mock(ManagedChannel.class);
Mockito.when(secondary.getState(Mockito.anyBoolean())).thenReturn(ConnectivityState.READY);
Mockito.when(secondary.shutdown()).thenReturn(secondary);
Mockito.when(secondary.shutdownNow()).thenReturn(secondary);
Mockito.when(secondary.awaitTermination(Mockito.anyLong(), Mockito.any())).thenReturn(true);
Mockito.when(channelFactory.newManagedChannel("secondary", 2136, null)).thenReturn(secondary);
DiscoveryProtos.ListEndpointsResult result = DiscoveryProtos.ListEndpointsResult.newBuilder()
.addEndpoints(DiscoveryProtos.EndpointInfo.newBuilder().setAddress("node").setPort(2136)
.setNodeId(1).setBridgePileName("A"))
.addEndpoints(DiscoveryProtos.EndpointInfo.newBuilder().setAddress("secondary").setPort(2136)
.setNodeId(2).setBridgePileName("B"))
.addPileStates(PileState.newBuilder().setPileName("A").setState(PileState.State.PRIMARY))
.addPileStates(PileState.newBuilder().setPileName("B").setState(PileState.State.SYNCHRONIZED))
.build();
Mockito.when(discoveryChannel.newCall(Mockito.eq(DiscoveryServiceGrpc.getListEndpointsMethod()), Mockito.any()))
.thenAnswer(call -> MockedCall.discovery(result));
Mockito.when(transportChannel.newCall(Mockito.eq(DiscoveryServiceGrpc.getWhoAmIMethod()), Mockito.any()))
.thenAnswer(call -> MockedCall.whoAmICall("primary"));
Mockito.when(secondary.newCall(Mockito.eq(DiscoveryServiceGrpc.getWhoAmIMethod()), Mockito.any()))
.thenAnswer(call -> MockedCall.whoAmICall("secondary"));
try (GrpcTransport transport = GrpcTransport.forConnectionString("grpc://mocked:2136/local")
.withBalancingSettings(BalancingSettings.preferPrimaryPile())
.withChannelFactoryBuilder(builder -> channelFactory)
.build()) {
Assert.assertEquals("secondary", whoAmI(transport, GrpcRequestSettings.newBuilder()
.withPreferredNodeID(2).build()).join().getValue().getUser());
Assert.assertEquals("primary", whoAmI(transport, GrpcRequestSettings.newBuilder()
.withPreferReadyChannel(true).build()).join().getValue().getUser());
Assert.assertEquals("primary", whoAmI(transport).join().getValue().getUser());
}
}

@Test
public void defaultBuildGoodTest() {
Mockito.when(discoveryChannel.newCall(Mockito.eq(DiscoveryServiceGrpc.getListEndpointsMethod()), Mockito.any()))
Expand Down
Loading