Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/import_generation.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
54
55
2 changes: 1 addition & 1 deletion .github/last_commit.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
5c5df6a4773b1dd84742c1f0a4867be9fa284c7e
9dcf0822149e3d827787f80bbf63dfa9245863c6
14 changes: 14 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,8 +1,20 @@
## v3.27.0

* Fixed static login credentials failing permanently when the initial Login operation returns a transient service error.

* Added draft `TBridgeClient::Rollback()` to roll a cluster converted into a bridged one back to ordinary mode, and `TBridgeClient::GetRollbackStatus()` to show the progress of the rollback.

* Topic and PersQueue writers now schedule RAW and pre-encoded messages on the configured compression executor, reducing request construction on the caller thread.

* Added optional `TReplicationDescription::GetTransactions()` to report assigned global-consistency replication transactions and versions pending transaction-ID assignment.

* Added `TGetClusterStateSettings::IncludePileConnectivity()` and `TGetClusterStateResult::GetPileConnectivity()` for optional bridge static-node connectivity reports.
* Added `TGetClusterStateSettings::IncludeUnbridgedStatus()` and `TGetClusterStateResult::GetUnbridgedStatus()` to report node configuration when the cluster has no bridge state.

* Added `TUpdateClusterStateSettings::ExpectedGeneration()` to apply a bridge cluster state update only if the cluster state still has the given generation.
* Added draft bridge synchronization APIs for pile, database, and group diagnostics.

* Fixed Topic write-session path handling: paths starting with `/` remain absolute, and relative database names are left for server-side resolution.

* Added draft `TBridgeClient::SetDatabaseBridgeMode()` to atomically change a database's bridge mode. `TSetDatabaseBridgeModeSettings::RetainPileName()` selects the pile to retain when disabling bridge mode.

Expand All @@ -12,6 +24,8 @@

* Supported Topic read sessions counting replication control messages as data offsets, which could block commits after an offset gap.

* Fixed a Topic read session crash (Panic in `OnUserRetrievedEvent`) when an unrecoverable stream read error tore down the session while data events were still queued for delivery; the teardown now releases the accounting for undelivered events exactly once.

## v3.26.0

* Federated topic read sessions now follow federation membership. After the initial discovery, databases that join the federation are read and databases that leave it are dropped; partition sessions of a dropped database get `TPartitionSessionClosedEvent` with the `ConnectionLost` reason. A listed database that becomes unavailable keeps its subsession. The refresh period is `TFederatedTopicClientSettings::FederationDiscoveryInterval` (30 seconds by default).
Expand Down
45 changes: 45 additions & 0 deletions include/ydb-cpp-sdk/client/table/table.h
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,16 @@ struct TSequenceDescription {
bool NextUsed;
};
std::optional<TSetVal> SetVal;
// Name of the sequence: serial columns use `_serial_column_<column>` by default, while
// sequences provisioned for index columns have their own names.
std::string Name;
std::optional<int64_t> MinValue;
std::optional<int64_t> MaxValue;
std::optional<int64_t> StartValue;
std::optional<uint64_t> Cache;
std::optional<int64_t> Increment;
std::optional<bool> Cycle;
std::optional<TType> DataType;
};

struct TTableColumn {
Expand Down Expand Up @@ -324,6 +334,11 @@ struct TGlobalIndexSettings {

struct TVectorIndexSettings {
public:
static constexpr uint64_t DefaultMinRows = 10000;
static constexpr uint32_t DefaultM = 16;
static constexpr uint32_t DefaultEfConstruction = 200;
static constexpr uint64_t DefaultDeltaRows = 10000;

enum class EMetric {
Unspecified = 0,
InnerProduct,
Expand All @@ -350,6 +365,11 @@ struct TVectorIndexSettings {
EMetric Metric = EMetric::Unspecified;
EVectorType VectorType = EVectorType::Unspecified;
uint32_t VectorDimension = 0;
uint64_t MinRows = DefaultMinRows;
uint32_t M = DefaultM;
uint32_t EfConstruction = DefaultEfConstruction;
// Rebuild after more than this many distinct rows change since the graph snapshot.
uint64_t DeltaRows = DefaultDeltaRows;

static TVectorIndexSettings FromProto(const Ydb::Table::VectorIndexSettings& proto);

Expand Down Expand Up @@ -400,6 +420,15 @@ struct TKMeansTreeSettings {
void Out(IOutputStream &o) const;
};

struct THnswIndexSettings {
TKMeansTreeSettings VectorSettings;
std::vector<std::string> DataColumns;
TGlobalIndexSettings LevelTableSettings;
TGlobalIndexSettings PostingTableSettings;
// Used only by CreatePrefixedHnswIndex.
TGlobalIndexSettings PrefixTableSettings;
};

struct TFulltextIndexSettings {
public:
enum class ETokenizer {
Expand Down Expand Up @@ -533,6 +562,18 @@ class TIndexDescription {
const TGlobalIndexSettings& prefixTableSettings = {}
);

static TIndexDescription CreateHnswIndex(
const std::string& name,
const std::string& vectorColumn,
const THnswIndexSettings& settings
);

static TIndexDescription CreatePrefixedHnswIndex(
const std::string& name,
const std::vector<std::string>& indexColumns,
const THnswIndexSettings& settings
);

static TIndexDescription CreateFulltextPlainIndex(
const std::string& name,
const std::vector<std::string>& indexColumns,
Expand Down Expand Up @@ -1200,6 +1241,8 @@ class TTableDescription {
// vector KMeansTree
void AddVectorKMeansTreeIndex(const std::string& indexName, const std::vector<std::string>& indexColumns, const TKMeansTreeSettings& indexSettings);
void AddVectorKMeansTreeIndex(const std::string& indexName, const std::vector<std::string>& indexColumns, const std::vector<std::string>& dataColumns, const TKMeansTreeSettings& indexSettings);
void AddHnswIndex(const std::string& indexName, const std::vector<std::string>& indexColumns, const TKMeansTreeSettings& indexSettings);
void AddHnswIndex(const std::string& indexName, const std::vector<std::string>& indexColumns, const std::vector<std::string>& dataColumns, const TKMeansTreeSettings& indexSettings);
// fulltext
void AddFulltextIndex(const std::string& indexName, EIndexType type, const std::vector<std::string>& indexColumns, const TFulltextIndexSettings& indexSettings);
void AddFulltextIndex(const std::string& indexName, EIndexType type, const std::vector<std::string>& indexColumns, const std::vector<std::string>& dataColumns, const TFulltextIndexSettings& indexSettings);
Expand Down Expand Up @@ -1457,6 +1500,8 @@ class TTableBuilder {
// vector KMeansTree
TTableBuilder& AddVectorKMeansTreeIndex(const std::string& indexName, const std::vector<std::string>& indexColumns, const TKMeansTreeSettings& indexSettings);
TTableBuilder& AddVectorKMeansTreeIndex(const std::string& indexName, const std::vector<std::string>& indexColumns, const std::vector<std::string>& dataColumns, const TKMeansTreeSettings& indexSettings);
TTableBuilder& AddHnswIndex(const std::string& indexName, const std::vector<std::string>& indexColumns, const TKMeansTreeSettings& indexSettings);
TTableBuilder& AddHnswIndex(const std::string& indexName, const std::vector<std::string>& indexColumns, const std::vector<std::string>& dataColumns, const TKMeansTreeSettings& indexSettings);

// fulltext
TTableBuilder& AddFulltextIndex(const std::string& indexName, const std::vector<std::string>& indexColumns, const TFulltextIndexSettings& indexSettings);
Expand Down
1 change: 1 addition & 0 deletions include/ydb-cpp-sdk/client/table/table_enum.h
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ enum class EIndexType {
LocalBloomFilter,
LocalBloomNgramFilter,
LocalMinMax,
GlobalHnsw,

Unknown = std::numeric_limits<int>::max()
};
Expand Down
2 changes: 1 addition & 1 deletion include/ydb-cpp-sdk/client/topic/write_session.h
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ struct TWriteSessionSettings : public TRequestSettings<TWriteSessionSettings> {

FLUENT_SETTING_OPTIONAL(TWriterCounters::TPtr, Counters);

//! Executor for compression tasks.
//! Executor for compression tasks and sending messages.
//! If not set, default executor will be used.
FLUENT_SETTING(IExecutor::TPtr, CompressionExecutor);

Expand Down
2 changes: 1 addition & 1 deletion library/cpp/monlib/counters/counters.h
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ namespace NMonitoring {
}

TValueBase operator=(TValueBase rhs) {
AtomicSwap(&Value, rhs);
Set(rhs);
return rhs;
}

Expand Down
12 changes: 9 additions & 3 deletions library/cpp/monlib/encode/spack/spack_v1_decoder.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -67,9 +67,13 @@ namespace NMonitoring {
TVector<char> valuesBuf;

if (Header_.Version == SV1_03 || Header_.Version == SV1_04) {
auto namesResult = ReadLengthDelimitedStringPool(Header_.LabelNamesSize);
ui64 labelSizeTotal = (ui64(Header_.LabelNamesSize) + Header_.LabelValuesSize)
* (sizeof(std::pair<ui32, ui32>) + sizeof(TStringBuf));
DECODE_ENSURE(labelSizeTotal <= LABEL_SIZE_LIMIT, "label pool limit exceeded by segment metadata");

auto namesResult = ReadLengthDelimitedStringPool(Header_.LabelNamesSize, labelSizeTotal);
namesBuf = std::move(namesResult.first);
auto valuesResult = ReadLengthDelimitedStringPool(Header_.LabelValuesSize);
auto valuesResult = ReadLengthDelimitedStringPool(Header_.LabelValuesSize, labelSizeTotal);
valuesBuf = std::move(valuesResult.first);

TStringPool labelNames(namesBuf.data(), namesResult.second);
Expand Down Expand Up @@ -306,13 +310,15 @@ namespace NMonitoring {
return ReadVarUInt32(In_);
}

std::pair<TVector<char>, TVector<std::pair<ui32, ui32>>> ReadLengthDelimitedStringPool(ui32 count) {
std::pair<TVector<char>, TVector<std::pair<ui32, ui32>>> ReadLengthDelimitedStringPool(ui32 count, ui64& labelSizeTotal) {
TVector<char> storage;
TVector<std::pair<ui32, ui32>> segments;
segments.reserve(count);

for (ui32 i = 0; i < count; i++) {
ui32 len = ReadVarint();
DECODE_ENSURE(len <= LABEL_SIZE_LIMIT - labelSizeTotal, "label pool limit exceeded by string #" << i);
labelSizeTotal += len;
ui32 offset = static_cast<ui32>(storage.size());
storage.resize(offset + len);
if (len > 0) {
Expand Down
61 changes: 61 additions & 0 deletions library/cpp/monlib/encode/spack/spack_v1_ut.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1402,6 +1402,67 @@ Y_UNIT_TEST_SUITE(TSpackTest) {
UNIT_ASSERT_VALUES_EQUAL(consumer.Metrics[0].StartTimeSeconds, startTimeSeconds);
}

Y_UNIT_TEST(V13AndV14RejectOversizedLabelCount) {
// EzoEARgAAAD///8tAQAAAFsAAAAIABGrAPY=
ui8 corpus[] = {
0x13, 0x3a, 0x04, 0x01, 0x18, 0x00, 0x00, 0x00,
0xff, 0xff, 0xff, 0x2d, 0x01, 0x00, 0x00, 0x00,
0x5b, 0x00, 0x00, 0x00, 0x08, 0x00, 0x11, 0xab,
0x00, 0xf6,
};
for (ui8 version : {ui8{3}, ui8{4}}) {
corpus[2] = version;
TBuffer buffer;
buffer.Append(reinterpret_cast<const char*>(corpus), sizeof(corpus));
TBufferInput in(buffer);
TCollectingConsumer consumer;
UNIT_ASSERT_EXCEPTION_CONTAINS(DecodeSpackV1(&in, &consumer), TSpackDecodeError, "limit");
}
}

Y_UNIT_TEST(V13AndV14RejectCombinedLabelCount) {
for (auto version : {SV1_03, SV1_04}) {
TSpackHeader header;
header.Version = version;
header.TimePrecision = EncodeTimePrecision(ETimePrecision::SECONDS);
header.Compression = EncodeCompression(ECompression::IDENTITY);
header.LabelNamesSize = 3'000'000;
header.LabelValuesSize = 3'000'000;
header.MetricCount = 1;
header.PointsCount = 0;

TBuffer buffer;
TBufferOutput out(buffer);
out.Write(&header, sizeof(header));
TBufferInput in(buffer);
TCollectingConsumer consumer;
UNIT_ASSERT_EXCEPTION_CONTAINS(DecodeSpackV1(&in, &consumer), TSpackDecodeError, "limit");
}
}

Y_UNIT_TEST(V13AndV14RejectOversizedLabelString) {
for (auto version : {SV1_03, SV1_04}) {
TSpackHeader header;
header.Version = version;
header.TimePrecision = EncodeTimePrecision(ETimePrecision::SECONDS);
header.Compression = EncodeCompression(ECompression::IDENTITY);
header.LabelNamesSize = 1;
header.LabelValuesSize = 0;
header.MetricCount = 1;
header.PointsCount = 0;

TBuffer buffer;
TBufferOutput out(buffer);
out.Write(&header, sizeof(header));
const ui8 oversizedLength[] = {0x80, 0x80, 0x80, 0x80, 0x01};
out.Write(oversizedLength, sizeof(oversizedLength));

TBufferInput in(buffer);
TCollectingConsumer consumer;
UNIT_ASSERT_EXCEPTION_CONTAINS(DecodeSpackV1(&in, &consumer), TSpackDecodeError, "limit");
}
}

Y_UNIT_TEST(V12MissingNameForOneMetric) {
TBuffer b;
TBufferOutput out(b);
Expand Down
1 change: 1 addition & 0 deletions library/cpp/resource/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ target_sources(resource
PRIVATE
registry.cpp
resource.cpp
static_blob_default.cpp
)

_ydb_sdk_install_targets(TARGETS resource)
Loading
Loading