Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/import_generation.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
52
53
2 changes: 1 addition & 1 deletion .github/last_commit.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
27b9404fde564e5a88652ef9ad5796d71206fa81
e298c2650fafe1aab48d99022912303996c9e710
10 changes: 10 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,15 @@
## v3.26.0

* Federated topic read sessions now follow federation membership. After the initial discovery, databases that join the federation are read and databases that leave it are dropped; partition sessions of a dropped database get `TPartitionSessionClosedEvent` with the `ConnectionLost` reason. A listed database that becomes unavailable keeps its subsession. The refresh period is `TFederatedTopicClientSettings::FederationDiscoveryInterval` (30 seconds by default).

* Fixed Query session shutdown racing with session pool return.

* Added `TConvertToBridgeSettings::SkipStoragePoolConversion()` to leave existing storage pools and their groups non-bridged when converting a cluster into a bridged one.

## v3.25.0

* Added recognition of Solomon volumes in Scheme API results and a draft `NKeyValue::TKeyValueClient::DropVolume()` method.

* Added `TExecuteQueryIterator::Cancel()` to explicitly cancel a query response stream, including a pending read.

* Added `TBridgeClient::RemovePile()` to remove a DISCONNECTED pile from a bridged cluster along with its data.
Expand Down
11 changes: 11 additions & 0 deletions include/ydb-cpp-sdk/client/federated_topic/federated_topic.h
Original file line number Diff line number Diff line change
Expand Up @@ -407,6 +407,11 @@ struct TFederatedReadSessionSettings: public NTopic::TReadSessionSettings {

//! Databases to read from.
//! Default (empty) value means reading from all available databases.
//! The session tracks federation membership for its whole lifetime: a database that
//! appears and is readable is subscribed, a database that disappears from the federation
//! is unsubscribed and its partition sessions are closed with the ConnectionLost reason.
//! A listed database that becomes unavailable stays subscribed.
//! Refresh period is TFederatedTopicClientSettings::FederationDiscoveryInterval.
//! Adding duplicates or unavailable databases is okay, they will be ignored.
struct TReadOriginalSettings {
//! Add reading from specified database if it's available.
Expand Down Expand Up @@ -500,6 +505,12 @@ struct TFederatedTopicClientSettings : public TCommonClientSettingsBase<TFederat
//! Connection timeoout for federation discovery.
FLUENT_SETTING_DEFAULT(TDuration, ConnectionTimeout, TDuration::Seconds(30));

//! How often federation membership is rediscovered.
//! Read sessions open subsessions for databases that joined the federation
//! and close subsessions for databases that left it on this cadence.
//! Zero keeps the default of 30 seconds.
FLUENT_SETTING_DEFAULT(TDuration, FederationDiscoveryInterval, TDuration::Seconds(30));

//! Retry policy enables automatic retries for non-fatal errors.
FLUENT_SETTING_DEFAULT(NTopic::IRetryPolicy::TPtr, RetryPolicy, NTopic::IRetryPolicy::GetDefaultPolicy());
};
Expand Down
1 change: 1 addition & 0 deletions include/ydb-cpp-sdk/client/scheme/scheme.h
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ enum class ESchemeEntryType : i32 {
RtmrVolume = 5,
BlockStoreVolume = 6,
CoordinationNode = 7,
SolomonVolume = 8,
ColumnStore = 12,
ColumnTable = 13,
Sequence = 15,
Expand Down
4 changes: 4 additions & 0 deletions include/ydb-cpp-sdk/client/table/table.h
Original file line number Diff line number Diff line change
Expand Up @@ -381,6 +381,10 @@ struct TKMeansTreeSettings {
uint32_t OverlapClusters = 0;
double OverlapRatio = 0;

// Maintain the index on table writes. Enabled when unset.
// When false, rebuild the index to reflect table changes.
std::optional<bool> AutoUpdate = std::nullopt;

static TKMeansTreeSettings FromProto(const Ydb::Table::KMeansTreeSettings& proto);

void SerializeTo(Ydb::Table::KMeansTreeSettings& settings) const;
Expand Down
62 changes: 54 additions & 8 deletions library/cpp/http/simple/http_client.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,32 @@
TSpinLock TKeepAliveHttpClient::ConnectionQuarantineMutex;
TQueue<THolder<NPrivate::THttpConnection>> TKeepAliveHttpClient::ConnectionQuarantine;

namespace {
TKeepAliveHttpClient::THeaders GetRedirectHeaders(const TKeepAliveHttpClient::THeaders& headers) {
TKeepAliveHttpClient::THeaders redirectHeaders;
for (const auto& header : headers) {
if (AsciiEqualsIgnoreCase(header.first, "User-Agent") || AsciiEqualsIgnoreCase(header.first, "X-Request-ID")) {
redirectHeaders.emplace(header.first, header.second);
}
}
return redirectHeaders;
}

TString GetResponseRequestIds(const THttpInput& input) {
TString requestIds;
// S3 uses X-Amz-Request-Id instead of X-Request-ID in responses.
for (TStringBuf name : {TStringBuf("X-Request-ID"), TStringBuf("X-Amz-Request-Id")}) {
if (const auto* header = input.Headers().FindHeader(name)) {
requestIds += "\n";
requestIds += name;
requestIds += ": ";
requestIds += header->Value();
}
}
return requestIds;
}
} // namespace

TKeepAliveHttpClient::TKeepAliveHttpClient(const TString& host,
ui32 port,
TDuration socketTimeout,
Expand Down Expand Up @@ -243,7 +269,7 @@ void TSimpleHttpClient::DoGet(const TStringBuf relativeUrl, IOutputStream* outpu
TKeepAliveHttpClient::THttpCode code = cl.DoGet(relativeUrl, output, headers, outHeaders, std::move(cancellation));

Y_ENSURE(cl.GetHttpInput());
ProcessResponse(relativeUrl, *cl.GetHttpInput(), output, code);
ProcessResponseWithHeaders(relativeUrl, *cl.GetHttpInput(), output, code, headers);
}

void TSimpleHttpClient::DoPost(const TStringBuf relativeUrl, TStringBuf body, IOutputStream* output, const THashMap<TString, TString>& headers, THttpHeaders* outHeaders, NThreading::TCancellationToken cancellation) const {
Expand All @@ -252,7 +278,7 @@ void TSimpleHttpClient::DoPost(const TStringBuf relativeUrl, TStringBuf body, IO
TKeepAliveHttpClient::THttpCode code = cl.DoPost(relativeUrl, body, output, headers, outHeaders, std::move(cancellation));

Y_ENSURE(cl.GetHttpInput());
ProcessResponse(relativeUrl, *cl.GetHttpInput(), output, code);
ProcessResponseWithHeaders(relativeUrl, *cl.GetHttpInput(), output, code, headers);
}

void TSimpleHttpClient::DoPostRaw(const TStringBuf relativeUrl, const TStringBuf rawRequest, IOutputStream* output, THttpHeaders* outHeaders, NThreading::TCancellationToken cancellation) const {
Expand All @@ -261,7 +287,7 @@ void TSimpleHttpClient::DoPostRaw(const TStringBuf relativeUrl, const TStringBuf
TKeepAliveHttpClient::THttpCode code = cl.DoRequestRaw(rawRequest, output, outHeaders, std::move(cancellation));

Y_ENSURE(cl.GetHttpInput());
ProcessResponse(relativeUrl, *cl.GetHttpInput(), output, code);
ProcessResponseWithHeaders(relativeUrl, *cl.GetHttpInput(), output, code, {});
}

namespace NPrivate {
Expand Down Expand Up @@ -326,14 +352,28 @@ namespace NPrivate {
}
}

void TSimpleHttpClient::ProcessResponse(const TStringBuf relativeUrl, THttpInput& input, IOutputStream*, const unsigned statusCode) const {
void TSimpleHttpClient::ProcessResponse(
const TStringBuf relativeUrl,
THttpInput& input,
IOutputStream*,
const unsigned statusCode) const {
if (!(statusCode >= 200 && statusCode < 300)) {
TString responseRequestIds = GetResponseRequestIds(input);
TString rest = ReadDiagnosticBody(input, statusCode);
ythrow THttpRequestException(statusCode) << "Got " << statusCode << " at " << Host << relativeUrl << "\nFull http response:\n"
<< rest;
<< rest << responseRequestIds;
}
}

void TSimpleHttpClient::ProcessResponseWithHeaders(
const TStringBuf relativeUrl,
THttpInput& input,
IOutputStream* output,
const unsigned statusCode,
const THeaders&) const {
ProcessResponse(relativeUrl, input, output, statusCode);
}

TSimpleHttpClient::~TSimpleHttpClient() {
}

Expand Down Expand Up @@ -376,7 +416,12 @@ void TRedirectableHttpClient::PrepareClient(TKeepAliveHttpClient& cl) const {
};
}

void TRedirectableHttpClient::ProcessResponse(const TStringBuf relativeUrl, THttpInput& input, IOutputStream* output, const unsigned statusCode) const {
void TRedirectableHttpClient::ProcessResponseWithHeaders(
const TStringBuf relativeUrl,
THttpInput& input,
IOutputStream* output,
const unsigned statusCode,
const THeaders& headers) const {
for (auto i = input.Headers().Begin(), e = input.Headers().End(); i != e; ++i) {
if (0 == TString::compare(i->Name(), TStringBuf("Location"))) {
if (Opts.MaxRedirectCount() == 0) {
Expand Down Expand Up @@ -408,14 +453,15 @@ void TRedirectableHttpClient::ProcessResponse(const TStringBuf relativeUrl, THtt
if (HttpsVerification) {
cl.EnableVerificationForHttps();
}
cl.DoGet(body, output);
cl.DoGet(body, output, GetRedirectHeaders(headers));
return;
}
}
if (!(statusCode >= 200 && statusCode < 300)) {
TString responseRequestIds = GetResponseRequestIds(input);
TString rest = ReadDiagnosticBody(input, statusCode);
ythrow THttpRequestException(statusCode) << "Got " << statusCode << " at " << Host << relativeUrl << "\nFull http response:\n"
<< rest;
<< rest << responseRequestIds;
}
TransferData(&input, output);
}
35 changes: 26 additions & 9 deletions library/cpp/http/simple/http_client.h
Original file line number Diff line number Diff line change
Expand Up @@ -212,7 +212,17 @@ class TSimpleHttpClient {
TKeepAliveHttpClient CreateClient() const;

virtual void PrepareClient(TKeepAliveHttpClient& cl) const;
virtual void ProcessResponse(const TStringBuf relativeUrl, THttpInput& input, IOutputStream* output, const unsigned statusCode) const;
virtual void ProcessResponse(
const TStringBuf relativeUrl,
THttpInput& input,
IOutputStream* output,
const unsigned statusCode) const;
virtual void ProcessResponseWithHeaders(
const TStringBuf relativeUrl,
THttpInput& input,
IOutputStream* output,
const unsigned statusCode,
const THeaders& headers) const;
};

class TRedirectableHttpClient: public TSimpleHttpClient {
Expand All @@ -226,7 +236,12 @@ class TRedirectableHttpClient: public TSimpleHttpClient {

private:
void PrepareClient(TKeepAliveHttpClient& cl) const override;
void ProcessResponse(const TStringBuf relativeUrl, THttpInput& input, IOutputStream* output, const unsigned statusCode) const override;
void ProcessResponseWithHeaders(
const TStringBuf relativeUrl,
THttpInput& input,
IOutputStream* output,
const unsigned statusCode,
const THeaders& headers) const override;

private:
TOptions Opts;
Expand Down Expand Up @@ -299,14 +314,16 @@ TKeepAliveHttpClient::THttpCode TKeepAliveHttpClient::DoRequestReliable(const T&
const bool couldRetry = !haveNewConnection && i == 0; // Actually old connection could be already closed by server,
// so we should try one more time in this case.
TAtomicSharedPtr<TManualEvent> cancellationEndEvent = MakeAtomicShared<TManualEvent>();
auto cancelSub = cancellation.Future().Subscribe([this, cancellationEndEvent](auto&) {
if (cancellationEndEvent.RefCount() > 1) {
if (Connection && Connection->IsOk()) {
Connection->Shutdown();
if (!cancellation.IsDefault()) {
cancellation.Future().Subscribe([this, cancellationEndEvent](auto&) {
if (cancellationEndEvent.RefCount() > 1) {
if (Connection && Connection->IsOk()) {
Connection->Shutdown();
}
cancellationEndEvent->Signal();
}
cancellationEndEvent->Signal();
}
});
});
}

try {
Connection->Write(raw);
Expand Down
Loading
Loading