Skip to content

nvmf/rdma: apply TOS on listen and accept #4

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jleeh merged 2 commits into from
Aug 28, 2026
Merged

nvmf/rdma: apply TOS on listen and accept #4

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5c1cd3d
nvmf/rdma: apply TOS on listen and accept.
jleeh Aug 28, 2026
75b641c
test/nvmf/rdma: cover transport TOS option.
jleeh Aug 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 38 additions & 2 deletions lib/nvmf/rdma.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -501,6 +501,8 @@ struct rdma_transport_opts {
bool no_srq;
bool no_wr_batching;
int acceptor_backlog;
/* IPv4 TOS / IPv6 TCLASS. 0 keeps the rdma-core default (untagged). */
uint8_t tos;
};

struct spdk_nvmf_rdma_transport {
Expand Down Expand Up @@ -559,8 +561,14 @@ static const struct spdk_json_object_decoder rdma_transport_opts_decoder[] = {
"acceptor_backlog", offsetof(struct rdma_transport_opts, acceptor_backlog),
spdk_json_decode_int32, true
},
{
"tos", offsetof(struct rdma_transport_opts, tos),
spdk_json_decode_uint8, true
},
};

static int nvmf_rdma_apply_tos(struct rdma_cm_id *id, uint8_t tos);

static int
nvmf_rdma_qpair_compare(struct spdk_nvmf_rdma_qpair *rqpair1, struct spdk_nvmf_rdma_qpair *rqpair2)
{
Expand Down Expand Up @@ -1434,6 +1442,8 @@ nvmf_rdma_connect(struct spdk_nvmf_transport *transport, struct rdma_cm_event *e
rqpair->cm_id = event->id;
rqpair->listen_id = event->listen_id;
rqpair->qpair.transport = transport;
/* Passive side: TOS must be set on the accepted ID before rdma_accept(). */
nvmf_rdma_apply_tos(rqpair->cm_id, rtransport->rdma_opts.tos);
/* use qid from the private data to determine the qpair type
qid will be set to the appropriate value when the controller is created */
rqpair->qpair.qid = private_data->qid;
Expand Down Expand Up @@ -2813,7 +2823,7 @@ nvmf_rdma_create(struct spdk_nvmf_transport_opts *opts)
" max_io_qpairs_per_ctrlr=%d,\n"
" in_capsule_data_size=%d, max_aq_depth=%d,\n"
" num_cqe=%d, max_srq_depth=%d, no_srq=%d,"
" acceptor_backlog=%d, no_wr_batching=%d abort_timeout_sec=%d\n",
" acceptor_backlog=%d, no_wr_batching=%d abort_timeout_sec=%d tos=%u\n",
opts->max_queue_depth,
opts->max_io_size,
opts->max_qpairs_per_ctrlr - 1,
Expand All @@ -2824,7 +2834,8 @@ nvmf_rdma_create(struct spdk_nvmf_transport_opts *opts)
rtransport->rdma_opts.no_srq,
rtransport->rdma_opts.acceptor_backlog,
rtransport->rdma_opts.no_wr_batching,
opts->abort_timeout_sec);
opts->abort_timeout_sec,
rtransport->rdma_opts.tos);

if (rtransport->rdma_opts.acceptor_backlog <= 0) {
SPDK_ERRLOG("The acceptor backlog cannot be less than 1, setting to the default value of (%d).\n",
Expand Down Expand Up @@ -2963,6 +2974,7 @@ nvmf_rdma_dump_opts(struct spdk_nvmf_transport *transport, struct spdk_json_writ
}
spdk_json_write_named_int32(w, "acceptor_backlog", rtransport->rdma_opts.acceptor_backlog);
spdk_json_write_named_bool(w, "no_wr_batching", rtransport->rdma_opts.no_wr_batching);
spdk_json_write_named_uint32(w, "tos", rtransport->rdma_opts.tos);
}

static void
Expand Down Expand Up @@ -3019,6 +3031,27 @@ static void nvmf_rdma_trid_from_cm_id(struct rdma_cm_id *id,

static bool nvmf_rdma_rescan_devices(struct spdk_nvmf_rdma_transport *rtransport);

static int
nvmf_rdma_apply_tos(struct rdma_cm_id *id, uint8_t tos)
{
int rc;

if (id == NULL || tos == 0) {
return 0;
}

#ifdef SPDK_CONFIG_RDMA_SET_TOS
rc = rdma_set_option(id, RDMA_OPTION_ID, RDMA_OPTION_ID_TOS, &tos, sizeof(tos));
if (rc) {
SPDK_NOTICELOG("Can't apply RDMA_OPTION_ID_TOS %u, ret %d\n", tos, rc);
}
return rc;
#else
SPDK_DEBUGLOG(rdma, "transport tos is not supported\n");
return 0;
#endif
}

static int
nvmf_rdma_listen(struct spdk_nvmf_transport *transport, const struct spdk_nvme_transport_id *trid,
struct spdk_nvmf_listen_opts *listen_opts)
Expand Down Expand Up @@ -3120,6 +3153,9 @@ nvmf_rdma_listen(struct spdk_nvmf_transport *transport, const struct spdk_nvme_t
return -1;
}

/* Must be set before rdma_listen() so accepted QPs inherit the TOS. */
nvmf_rdma_apply_tos(port->id, rtransport->rdma_opts.tos);

rc = rdma_listen(port->id, rtransport->rdma_opts.acceptor_backlog);
if (rc < 0) {
SPDK_ERRLOG("rdma_listen() failed\n");
Expand Down
149 changes: 149 additions & 0 deletions test/unit/lib/nvmf/rdma.c/rdma_ut.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,42 @@ DEFINE_STUB(spdk_rdma_cm_id_get_numa_id, int32_t, (struct rdma_cm_id *cm_id), 0)
DEFINE_STUB(spdk_rdma_utils_poll_cq, int, (struct ibv_cq *cq, int num_entries, struct ibv_wc *wc),
0);

#ifdef SPDK_CONFIG_RDMA_SET_TOS
static int g_rdma_set_option_calls;
static struct rdma_cm_id *g_rdma_set_option_id;
static int g_rdma_set_option_level;
static int g_rdma_set_option_optname;
static uint8_t g_rdma_set_option_tos;
static size_t g_rdma_set_option_optlen;
static int g_rdma_set_option_rc;

static void
reset_rdma_set_option_mock(void)
{
g_rdma_set_option_calls = 0;
g_rdma_set_option_id = NULL;
g_rdma_set_option_level = 0;
g_rdma_set_option_optname = 0;
g_rdma_set_option_tos = 0;
g_rdma_set_option_optlen = 0;
g_rdma_set_option_rc = 0;
}

int
rdma_set_option(struct rdma_cm_id *id, int level, int optname, void *optval, size_t optlen)
{
g_rdma_set_option_calls++;
g_rdma_set_option_id = id;
g_rdma_set_option_level = level;
g_rdma_set_option_optname = optname;
g_rdma_set_option_optlen = optlen;
if (optval != NULL && optlen >= sizeof(uint8_t)) {
g_rdma_set_option_tos = *(uint8_t *)optval;
}
return g_rdma_set_option_rc;
}
#endif

/* ibv_reg_mr can be a macro, need to undefine it */
#ifdef ibv_reg_mr
#undef ibv_reg_mr
Expand Down Expand Up @@ -1872,6 +1908,116 @@ test_nvmf_rdma_resize_cq(void)
CU_ASSERT(rpoller.num_cqe > tnum_cqe);
}

static void
test_nvmf_rdma_opts_tos(void)
{
struct rdma_transport_opts opts = {};
struct spdk_json_val tos_obj[] = {
{"", 2, SPDK_JSON_VAL_OBJECT_BEGIN},
{"tos", 3, SPDK_JSON_VAL_NAME},
{"96", 2, SPDK_JSON_VAL_NUMBER},
{"", 0, SPDK_JSON_VAL_OBJECT_END},
};
struct spdk_json_val empty_obj[] = {
{"", 0, SPDK_JSON_VAL_OBJECT_BEGIN},
{"", 0, SPDK_JSON_VAL_OBJECT_END},
};
struct spdk_json_val zero_obj[] = {
{"", 2, SPDK_JSON_VAL_OBJECT_BEGIN},
{"tos", 3, SPDK_JSON_VAL_NAME},
{"0", 1, SPDK_JSON_VAL_NUMBER},
{"", 0, SPDK_JSON_VAL_OBJECT_END},
};

CU_ASSERT(spdk_json_decode_object_relaxed(tos_obj, rdma_transport_opts_decoder,
SPDK_COUNTOF(rdma_transport_opts_decoder), &opts) == 0);
CU_ASSERT(opts.tos == 96);

memset(&opts, 0, sizeof(opts));
CU_ASSERT(spdk_json_decode_object_relaxed(empty_obj, rdma_transport_opts_decoder,
SPDK_COUNTOF(rdma_transport_opts_decoder), &opts) == 0);
CU_ASSERT(opts.tos == 0);

opts.tos = 99;
CU_ASSERT(spdk_json_decode_object_relaxed(zero_obj, rdma_transport_opts_decoder,
SPDK_COUNTOF(rdma_transport_opts_decoder), &opts) == 0);
CU_ASSERT(opts.tos == 0);
}

static void
test_nvmf_rdma_apply_tos(void)
{
struct rdma_cm_id id = {};
int rc;

#ifdef SPDK_CONFIG_RDMA_SET_TOS
reset_rdma_set_option_mock();
#endif
rc = nvmf_rdma_apply_tos(&id, 0);
CU_ASSERT(rc == 0);
#ifdef SPDK_CONFIG_RDMA_SET_TOS
CU_ASSERT(g_rdma_set_option_calls == 0);
#endif

rc = nvmf_rdma_apply_tos(NULL, 96);
CU_ASSERT(rc == 0);
#ifdef SPDK_CONFIG_RDMA_SET_TOS
CU_ASSERT(g_rdma_set_option_calls == 0);

reset_rdma_set_option_mock();
rc = nvmf_rdma_apply_tos(&id, 96);
CU_ASSERT(rc == 0);
CU_ASSERT(g_rdma_set_option_calls == 1);
CU_ASSERT(g_rdma_set_option_id == &id);
CU_ASSERT(g_rdma_set_option_level == RDMA_OPTION_ID);
CU_ASSERT(g_rdma_set_option_optname == RDMA_OPTION_ID_TOS);
CU_ASSERT(g_rdma_set_option_tos == 96);
CU_ASSERT(g_rdma_set_option_optlen == sizeof(uint8_t));

reset_rdma_set_option_mock();
g_rdma_set_option_rc = -1;
rc = nvmf_rdma_apply_tos(&id, 32);
CU_ASSERT(rc == -1);
CU_ASSERT(g_rdma_set_option_calls == 1);
CU_ASSERT(g_rdma_set_option_tos == 32);
#else
rc = nvmf_rdma_apply_tos(&id, 96);
CU_ASSERT(rc == 0);
#endif
}

static char g_tos_json_buf[512];
static size_t g_tos_json_off;

static int
tos_json_write_cb(void *cb_ctx, const void *data, size_t size)
{
if (g_tos_json_off + size >= sizeof(g_tos_json_buf)) {
return -1;
}
memcpy(g_tos_json_buf + g_tos_json_off, data, size);
g_tos_json_off += size;
return 0;
}

static void
test_nvmf_rdma_dump_opts_tos(void)
{
struct spdk_nvmf_rdma_transport rtransport = {};
struct spdk_json_write_ctx *w;

rtransport.rdma_opts.tos = 96;
memset(g_tos_json_buf, 0, sizeof(g_tos_json_buf));
g_tos_json_off = 0;
w = spdk_json_write_begin(tos_json_write_cb, NULL, 0);
SPDK_CU_ASSERT_FATAL(w != NULL);
CU_ASSERT(spdk_json_write_object_begin(w) == 0);
nvmf_rdma_dump_opts(&rtransport.transport, w);
CU_ASSERT(spdk_json_write_object_end(w) == 0);
CU_ASSERT(spdk_json_write_end(w) == 0);
CU_ASSERT(strstr(g_tos_json_buf, "\"tos\":96") != NULL);
}

int
main(int argc, char **argv)
{
Expand All @@ -1891,6 +2037,9 @@ main(int argc, char **argv)
CU_ADD_TEST(suite, test_nvmf_rdma_resources_create);
CU_ADD_TEST(suite, test_nvmf_rdma_qpair_compare);
CU_ADD_TEST(suite, test_nvmf_rdma_resize_cq);
CU_ADD_TEST(suite, test_nvmf_rdma_opts_tos);
CU_ADD_TEST(suite, test_nvmf_rdma_apply_tos);
CU_ADD_TEST(suite, test_nvmf_rdma_dump_opts_tos);

num_failures = spdk_ut_run_tests(argc, argv, NULL);
CU_cleanup_registry();
Expand Down
Loading