From 540cc82e43c2c5dcfc11b38aaac9b97d9fb40f85 Mon Sep 17 00:00:00 2001 From: Denis Leshchev Date: Tue, 22 Sep 2026 13:27:38 -0400 Subject: [PATCH 1/7] #315 - Add hardware-free configuration validator Signed-off-by: Denis Leshchev --- include/daqiri/common.h | 428 ++++++++++++++++++++++++------- src/common.cpp | 525 ++++++++++++++++++++++++++++++-------- src/engine.cpp | 72 ++++-- src/engine.h | 4 + tools/CMakeLists.txt | 61 +++++ tools/config_validate.cpp | 31 +++ 6 files changed, 890 insertions(+), 231 deletions(-) create mode 100644 tools/CMakeLists.txt create mode 100644 tools/config_validate.cpp diff --git a/include/daqiri/common.h b/include/daqiri/common.h index bbcd6b58..3ba63f66 100644 --- a/include/daqiri/common.h +++ b/include/daqiri/common.h @@ -18,12 +18,17 @@ #pragma once #include #include +#include #include +#include +#include #include #include #include #include +#include #include +#include #include #include #include @@ -72,6 +77,159 @@ struct S3WriteStatus { static constexpr uint32_t DEFAULT_TX_META_BUFFERS = 1UL << 8; static constexpr uint32_t DEFAULT_RX_META_BUFFERS = 1UL << 8; namespace detail { +inline bool validate_yaml_mapping_keys(const YAML::Node& node, + std::initializer_list allowed_keys, + std::string_view context) { + if (!node.IsMap()) { + DAQIRI_LOG_ERROR("{} must be a mapping", context); + return false; + } + for (const auto& entry : node) { + if (!entry.first.IsScalar()) { + DAQIRI_LOG_ERROR("{} contains a non-scalar key", context); + return false; + } + const std::string key = entry.first.as(); + bool known = false; + for (const char* allowed : allowed_keys) { + if (key == allowed) { + known = true; + break; + } + } + if (!known) { + DAQIRI_LOG_ERROR("Unknown key '{}.{}'", context, key); + return false; + } + } + return true; +} + +template +inline bool parse_optional_yaml_scalar(const YAML::Node& node, const char* key, T default_value, + T& value, std::string_view context) { + if (!node[key].IsDefined()) { + value = default_value; + return true; + } + const YAML::Node value_node = node[key]; + if (!value_node.IsScalar()) { + DAQIRI_LOG_ERROR("Invalid value for '{}.{}': expected a scalar", context, key); + return false; + } + if constexpr (std::is_same_v) { + // yaml-cpp deliberately accepts a quoted string such as "false" as a + // boolean. Configuration booleans must retain their YAML scalar type so a + // typo cannot silently turn a string into a flag. + if (value_node.Tag() == "!" || value_node.Tag() == "tag:yaml.org,2002:str") { + DAQIRI_LOG_ERROR("Invalid boolean for '{}.{}': quoted strings are not booleans", context, + key); + return false; + } + } + try { + value = value_node.as(); + return true; + } catch (const std::exception& e) { + DAQIRI_LOG_ERROR("Invalid value for '{}.{}': {}", context, key, e.what()); + return false; + } +} + +template +inline bool parse_yaml_integer(const YAML::Node& node, T& value) { + static_assert(std::is_integral_v && !std::is_same_v); + static_assert(sizeof(T) <= sizeof(unsigned long long)); + if (!node.IsScalar() || (node.Tag() != "?" && node.Tag() != "tag:yaml.org,2002:int")) { + return false; + } + try { + const std::string text = node.as(); + if (text.empty()) { + return false; + } + + size_t offset = 0; + bool negative = false; + if (text[offset] == '+' || text[offset] == '-') { + negative = text[offset] == '-'; + if (++offset == text.size()) { + return false; + } + } + if constexpr (std::is_unsigned_v) { + if (negative) { + return false; + } + } + + int base = 10; + if (text.compare(offset, 2, "0o") == 0) { + base = 8; + offset += 2; + } else if (text.compare(offset, 2, "0x") == 0) { + base = 16; + offset += 2; + } else if (text[offset] == '0' && offset + 1 != text.size()) { + // YAML 1.2 requires 0o for octal. Reject ambiguous legacy YAML 1.1 + // spellings instead of letting C's base-0 conversion reinterpret them. + return false; + } + if (offset == text.size()) { + return false; + } + + unsigned long long magnitude = 0; + const char* begin = text.data() + offset; + const char* end = text.data() + text.size(); + const auto result = std::from_chars(begin, end, magnitude, base); + if (result.ec != std::errc{} || result.ptr != end) { + return false; + } + + if constexpr (std::is_signed_v) { + using U = std::make_unsigned_t; + const auto positive_limit = + static_cast(static_cast(std::numeric_limits::max())); + const auto negative_limit = positive_limit + 1; + if (negative) { + if (magnitude > negative_limit) { + return false; + } + value = magnitude == negative_limit ? std::numeric_limits::min() + : static_cast(-static_cast(magnitude)); + } else { + if (magnitude > positive_limit) { + return false; + } + value = static_cast(magnitude); + } + } else { + if (magnitude > static_cast(std::numeric_limits::max())) { + return false; + } + value = static_cast(magnitude); + } + return true; + } catch (const std::exception&) { + return false; + } +} + +template +inline bool parse_optional_yaml_integer(const YAML::Node& node, const char* key, T default_value, + T& value, std::string_view context) { + if (!node[key].IsDefined()) { + value = default_value; + return true; + } + if (!parse_yaml_integer(node[key], value)) { + DAQIRI_LOG_ERROR("Invalid integer for '{}.{}'", context, key); + return false; + } + return true; +} + inline Direction DirectionStringToType(const std::string &dir) { if (dir == "rx") { return Direction::RX; @@ -1157,6 +1315,14 @@ template <> struct YAML::convert { DAQIRI_LOG_ERROR("InputSpec: expected a map"); return false; } + if (!daqiri::detail::validate_yaml_mapping_keys( + node, + {"version", "stream_type", "engine", "master_core", "debug", "log_level", "loopback", + "tx_meta_buffers", "rx_meta_buffers", "memory_regions", "interfaces", "manager", + "protocol"}, + "daqiri.cfg")) { + return false; + } // YAML is using exceptions, catch them try { @@ -1167,8 +1333,20 @@ template <> struct YAML::convert { return false; } - input_spec.common_.version = node["version"].as(); - input_spec.common_.master_core_ = node["master_core"].as(); + if (!daqiri::detail::parse_yaml_integer(node["version"], input_spec.common_.version) || + !daqiri::detail::parse_yaml_integer(node["master_core"], + input_spec.common_.master_core_)) { + DAQIRI_LOG_ERROR("version and master_core must be 32-bit integers"); + return false; + } + if (input_spec.common_.version != 1) { + DAQIRI_LOG_ERROR("Unsupported config version {}; expected 1", input_spec.common_.version); + return false; + } + if (input_spec.common_.master_core_ < -1) { + DAQIRI_LOG_ERROR("master_core must be -1 or a non-negative CPU index"); + return false; + } if (!node["stream_type"].IsDefined()) { DAQIRI_LOG_ERROR( @@ -1256,17 +1434,18 @@ template <> struct YAML::convert { } input_spec.common_.loopback_ = daqiri::LoopbackType::DISABLED; - try { - const auto lbstr = node["loopback"].as(); - if (lbstr == "sw") { - input_spec.common_.loopback_ = daqiri::LoopbackType::LOOPBACK_TYPE_SW; - } else if (lbstr == "hw") { - input_spec.common_.loopback_ = daqiri::LoopbackType::LOOPBACK_TYPE_HW; - } else if (!lbstr.empty()) { - DAQIRI_LOG_ERROR("Invalid loopback type: {}. Use 'sw', 'hw', or empty string ''", lbstr); - return false; - } - } catch (const std::exception &e) { + std::string loopback; + if (!daqiri::detail::parse_optional_yaml_scalar(node, "loopback", std::string{}, loopback, + "daqiri.cfg")) { + return false; + } + if (loopback == "sw") { + input_spec.common_.loopback_ = daqiri::LoopbackType::LOOPBACK_TYPE_SW; + } else if (loopback == "hw") { + input_spec.common_.loopback_ = daqiri::LoopbackType::LOOPBACK_TYPE_HW; + } else if (!loopback.empty()) { + DAQIRI_LOG_ERROR("Invalid loopback type: {}. Use 'sw', 'hw', or empty string ''", loopback); + return false; } if (input_spec.common_.loopback_ == daqiri::LoopbackType::LOOPBACK_TYPE_HW && (input_spec.common_.stream_type != daqiri::StreamType::RAW || @@ -1275,58 +1454,70 @@ template <> struct YAML::convert { return false; } - try { - input_spec.debug_ = node["debug"].as(false); - } catch (const std::exception &e) { - input_spec.debug_ = false; + bool debug = false; + if (!daqiri::detail::parse_optional_yaml_scalar(node, "debug", false, debug, "daqiri.cfg")) { + return false; } + input_spec.debug_ = debug; - try { - input_spec.log_level_ = - daqiri::LogLevel::from_string(node["log_level"].as( - daqiri::LogLevel::to_string(daqiri::LogLevel::WARN))); - } catch (const std::exception &e) { - input_spec.log_level_ = daqiri::LogLevel::WARN; + std::string log_level; + if (!daqiri::detail::parse_optional_yaml_scalar( + node, "log_level", daqiri::LogLevel::to_string(daqiri::LogLevel::WARN), log_level, + "daqiri.cfg")) { + return false; } + input_spec.log_level_ = daqiri::LogLevel::from_string(log_level); - try { - input_spec.tx_meta_buffers_ = node["tx_meta_buffers"].as( - daqiri::DEFAULT_TX_META_BUFFERS); - } catch (const std::exception &e) { - input_spec.tx_meta_buffers_ = daqiri::DEFAULT_TX_META_BUFFERS; + if (!daqiri::detail::parse_optional_yaml_integer(node, "tx_meta_buffers", + daqiri::DEFAULT_TX_META_BUFFERS, + input_spec.tx_meta_buffers_, "daqiri.cfg") || + input_spec.tx_meta_buffers_ == 0) { + DAQIRI_LOG_ERROR("tx_meta_buffers must be greater than zero"); + return false; } - try { - input_spec.rx_meta_buffers_ = node["rx_meta_buffers"].as( - daqiri::DEFAULT_RX_META_BUFFERS); - } catch (const std::exception &e) { - input_spec.rx_meta_buffers_ = daqiri::DEFAULT_RX_META_BUFFERS; + if (!daqiri::detail::parse_optional_yaml_integer(node, "rx_meta_buffers", + daqiri::DEFAULT_RX_META_BUFFERS, + input_spec.rx_meta_buffers_, "daqiri.cfg") || + input_spec.rx_meta_buffers_ == 0) { + DAQIRI_LOG_ERROR("rx_meta_buffers must be greater than zero"); + return false; } - try { - const auto &mrs = node["memory_regions"]; - for (const auto &mr : mrs) { - daqiri::MemoryRegionConfig tmr; - if (!parse_memory_region_config(mr, tmr)) { - DAQIRI_LOG_ERROR("Failed to parse memory region config"); - return false; - } - if (input_spec.mrs_.find(tmr.name_) != input_spec.mrs_.end()) { - DAQIRI_LOG_CRITICAL("Duplicate memory region names: {}", tmr.name_); - return false; - } - input_spec.mrs_[tmr.name_] = tmr; - } - } catch (const std::exception &e) { - DAQIRI_LOG_ERROR("Must define at least one memory type"); + const auto& mrs = node["memory_regions"]; + if (!mrs.IsSequence() || mrs.size() == 0) { + DAQIRI_LOG_ERROR("memory_regions must be a non-empty sequence"); return false; } + for (const auto& mr : mrs) { + daqiri::MemoryRegionConfig tmr; + if (!parse_memory_region_config(mr, tmr)) { + DAQIRI_LOG_ERROR("Failed to parse memory region config"); + return false; + } + if (input_spec.mrs_.find(tmr.name_) != input_spec.mrs_.end()) { + DAQIRI_LOG_CRITICAL("Duplicate memory region names: {}", tmr.name_); + return false; + } + input_spec.mrs_[tmr.name_] = tmr; + } try { - const auto &intfs = node["interfaces"]; - for (const auto &intf : intfs) { + const auto& intfs = node["interfaces"]; + if (!intfs.IsSequence() || intfs.size() == 0) { + DAQIRI_LOG_ERROR("interfaces must be a non-empty sequence"); + return false; + } + for (const auto& intf : intfs) { daqiri::InterfaceConfig ifcfg; + if (!daqiri::detail::validate_yaml_mapping_keys( + intf, + {"name", "address", "socket_config", "roce_config", "rx", "tx", "rdma_config"}, + "daqiri.cfg.interfaces[]")) { + return false; + } + ifcfg.name_ = intf["name"].as(); ifcfg.address_ = intf["address"].as(); @@ -1402,50 +1593,69 @@ template <> struct YAML::convert { return false; } } else { - if (intf["socket_config"].IsDefined() || - intf["roce_config"].IsDefined()) { - DAQIRI_LOG_ERROR("'socket_config'/'roce_config' are only valid " - "for stream_type 'socket' " - "(interface '{}')", - ifcfg.name_); + if (intf["socket_config"].IsDefined() || intf["roce_config"].IsDefined()) { + DAQIRI_LOG_ERROR( + "'socket_config'/'roce_config' are only valid " + "for stream_type 'socket' " + "(interface '{}')", + ifcfg.name_); return false; } } - try { - const auto &rx = intf["rx"]; + const bool has_rx = intf["rx"].IsDefined(); + const bool has_tx = intf["tx"].IsDefined(); + if (!has_rx && !has_tx) { + DAQIRI_LOG_ERROR("Interface '{}' requires an rx or tx section", ifcfg.name_); + return false; + } + + if (has_rx) { + const auto& rx = intf["rx"]; + if (!daqiri::detail::validate_yaml_mapping_keys( + rx, + {"flow_isolation", "dynamic_flow_capacity", "hardware_timestamps", "queues", + "flows", "flex_items", "reorder_configs"}, + "daqiri.cfg.interfaces[].rx")) { + return false; + } + if (!rx["queues"].IsSequence() || rx["queues"].size() == 0) { + DAQIRI_LOG_ERROR("Interface '{}' rx.queues must be a non-empty sequence", + ifcfg.name_); + return false; + } + daqiri::RxConfig rx_cfg; - try { - rx_cfg.flow_isolation_ = rx["flow_isolation"].as(); - } catch (const std::exception& e) { - rx_cfg.flow_isolation_ = false; + if (!daqiri::detail::parse_optional_yaml_scalar(rx, "flow_isolation", false, + rx_cfg.flow_isolation_, + "daqiri.cfg.interfaces[].rx")) { + return false; } - try { - rx_cfg.hardware_timestamps_ = rx["hardware_timestamps"].as(); - } catch (const std::exception& e) { - rx_cfg.hardware_timestamps_ = false; + if (!daqiri::detail::parse_optional_yaml_scalar(rx, "hardware_timestamps", false, + rx_cfg.hardware_timestamps_, + "daqiri.cfg.interfaces[].rx")) { + return false; } - try { - rx_cfg.dynamic_flow_capacity_ = rx["dynamic_flow_capacity"].as(); - } catch (const std::exception& e) { - rx_cfg.dynamic_flow_capacity_ = daqiri::DEFAULT_DYNAMIC_FLOW_CAPACITY; + if (!daqiri::detail::parse_optional_yaml_integer( + rx, "dynamic_flow_capacity", daqiri::DEFAULT_DYNAMIC_FLOW_CAPACITY, + rx_cfg.dynamic_flow_capacity_, "daqiri.cfg.interfaces[].rx")) { + return false; } - for (const auto &q_item : rx["queues"]) { + for (const auto& q_item : rx["queues"]) { daqiri::RxQueueConfig q; - if (!parse_rx_queue_config( - q_item, input_spec.common_.engine_type, q, !roce_used)) { + if (!parse_rx_queue_config(q_item, input_spec.common_.engine_type, q, !roce_used)) { DAQIRI_LOG_ERROR("Failed to parse RxQueueConfig"); return false; } - try { - q.timeout_us_ = q_item["timeout_us"].as(); - } catch (const std::exception &e) { - q.timeout_us_ = 0; + if (!daqiri::detail::parse_optional_yaml_integer( + q_item, "timeout_us", uint64_t{0}, q.timeout_us_, + "daqiri.cfg.interfaces[].rx.queues[]")) { + return false; } rx_cfg.queues_.emplace_back(std::move(q)); @@ -1467,7 +1677,12 @@ template <> struct YAML::convert { } } - try { + if (rx["flex_items"].IsDefined()) { + if (!rx["flex_items"].IsSequence()) { + DAQIRI_LOG_ERROR("'rx.flex_items' must be a sequence for interface '{}'", + ifcfg.name_); + return false; + } for (const auto &flex_item : rx["flex_items"]) { daqiri::FlexItemConfig flex_item_config; if (!parse_flex_item_config(flex_item, flex_item_config)) { @@ -1476,10 +1691,14 @@ template <> struct YAML::convert { } rx_cfg.flex_items_.emplace_back(std::move(flex_item_config)); } - } catch (const std::exception &e) { - } // No flex_items defined for this interface. + } - try { + if (rx["reorder_configs"].IsDefined()) { + if (!rx["reorder_configs"].IsSequence()) { + DAQIRI_LOG_ERROR("'rx.reorder_configs' must be a sequence for interface '{}'", + ifcfg.name_); + return false; + } std::unordered_set reorder_names; for (const auto &reorder_item : rx["reorder_configs"]) { daqiri::ReorderConfig reorder_cfg; @@ -1497,21 +1716,29 @@ template <> struct YAML::convert { reorder_names.insert(reorder_cfg.name_); rx_cfg.reorder_configs_.emplace_back(std::move(reorder_cfg)); } - } catch (const std::exception &e) { - } // No reorder_configs defined for this interface. + } ifcfg.rx_ = rx_cfg; - } catch (const std::exception &e) { - } // No RX queues defined for this interface. + } - try { + if (has_tx) { const auto &tx = intf["tx"]; + if (!daqiri::detail::validate_yaml_mapping_keys( + tx, {"accurate_send", "queues", "flows"}, "daqiri.cfg.interfaces[].tx")) { + return false; + } + if (!tx["queues"].IsSequence() || tx["queues"].size() == 0) { + DAQIRI_LOG_ERROR("Interface '{}' tx.queues must be a non-empty sequence", + ifcfg.name_); + return false; + } + daqiri::TxConfig tx_cfg; - try { - tx_cfg.accurate_send_ = tx["accurate_send"].as(); - } catch (const std::exception &e) { - tx_cfg.accurate_send_ = false; + if (!daqiri::detail::parse_optional_yaml_scalar(tx, "accurate_send", false, + tx_cfg.accurate_send_, + "daqiri.cfg.interfaces[].tx")) { + return false; } for (const auto &q_item : tx["queues"]) { @@ -1524,18 +1751,23 @@ template <> struct YAML::convert { tx_cfg.queues_.emplace_back(std::move(q)); } - for (const auto &flow_item : tx["flows"]) { - daqiri::FlowConfig flow; - if (!parse_flow_config(flow_item, flow)) { - DAQIRI_LOG_ERROR("Failed to parse TX FlowConfig"); + if (tx["flows"].IsDefined()) { + if (!tx["flows"].IsSequence()) { + DAQIRI_LOG_ERROR("'tx.flows' must be a sequence for interface '{}'", ifcfg.name_); return false; } - tx_cfg.flows_.emplace_back(std::move(flow)); + for (const auto& flow_item : tx["flows"]) { + daqiri::FlowConfig flow; + if (!parse_flow_config(flow_item, flow)) { + DAQIRI_LOG_ERROR("Failed to parse TX FlowConfig"); + return false; + } + tx_cfg.flows_.emplace_back(std::move(flow)); + } } ifcfg.tx_ = tx_cfg; - } catch (const std::exception &e) { - } // No TX queues defined for this interface. + } input_spec.ifs_.push_back(ifcfg); } diff --git a/src/common.cpp b/src/common.cpp index f9fc6f95..407be4e3 100644 --- a/src/common.cpp +++ b/src/common.cpp @@ -102,20 +102,7 @@ Status parse_eth_addr(std::array* dst, } bool parse_u64_scalar(const YAML::Node& node, uint64_t* value) { - if (!node || value == nullptr) { return false; } - try { - const std::string text = node.as(); - char* end = nullptr; - errno = 0; - const unsigned long long parsed = std::strtoull(text.c_str(), &end, 0); - if (errno != 0 || end == text.c_str() || (end != nullptr && *end != '\0')) { - return false; - } - *value = static_cast(parsed); - return true; - } catch (const std::exception&) { - return false; - } + return node && value != nullptr && detail::parse_yaml_integer(node, *value); } bool parse_u16_field(const YAML::Node& node, const char* key, uint16_t* value) { @@ -164,6 +151,9 @@ bool parse_flow_action_config(const YAML::Node& action_node, FlowAction& action) } if (action.type_ == FlowType::QUEUE) { + if (!detail::validate_yaml_mapping_keys(action_node, {"type", "id", "ids"}, "flow action")) { + return false; + } if (action_node["id"] && action_node["ids"]) { DAQIRI_LOG_ERROR("Queue flow action must use either 'id' or 'ids', not both"); return false; @@ -193,6 +183,22 @@ bool parse_flow_action_config(const YAML::Node& action_node, FlowAction& action) if (action.type_ == FlowType::VLAN_PUSH || action.type_ == FlowType::VLAN_POP) { const YAML::Node vlan = nested_or_self(action_node, "vlan"); + if (action_node["vlan"].IsDefined()) { + if (!detail::validate_yaml_mapping_keys(action_node, {"type", "vlan"}, "VLAN flow action") || + !detail::validate_yaml_mapping_keys(vlan, {"vlan_id", "pcp", "dei", "ethertype"}, + "VLAN flow action.vlan")) { + return false; + } + } else { + const bool valid_keys = + action.type_ == FlowType::VLAN_PUSH + ? detail::validate_yaml_mapping_keys( + action_node, {"type", "vlan_id", "pcp", "dei", "ethertype"}, "VLAN flow action") + : detail::validate_yaml_mapping_keys(action_node, {"type"}, "VLAN flow action"); + if (!valid_keys) { + return false; + } + } if (action.type_ == FlowType::VLAN_PUSH) { if (!parse_u16_field(vlan, "vlan_id", &action.vlan_.vlan_id_)) { DAQIRI_LOG_ERROR("vlan_push action requires integer 'vlan_id'"); @@ -215,6 +221,27 @@ bool parse_flow_action_config(const YAML::Node& action_node, FlowAction& action) } const YAML::Node tunnel = nested_or_self(action_node, "tunnel"); + const std::initializer_list tunnel_keys = {"type", "outer_eth_src", + "outer_eth_dst", "outer_ipv4_src", + "outer_ipv4_dst", "outer_ipv4_ttl", + "outer_ipv4_tos", "outer_udp_src", + "outer_udp_dst", "vni", + "gre_protocol", "tni", + "flow_id"}; + if (action_node["tunnel"].IsDefined()) { + if (!detail::validate_yaml_mapping_keys(action_node, {"type", "tunnel"}, + "tunnel flow action") || + !detail::validate_yaml_mapping_keys(tunnel, tunnel_keys, "tunnel flow action.tunnel")) { + return false; + } + } else if (!detail::validate_yaml_mapping_keys( + action_node, + {"type", "tunnel_type", "outer_eth_src", "outer_eth_dst", "outer_ipv4_src", + "outer_ipv4_dst", "outer_ipv4_ttl", "outer_ipv4_tos", "outer_udp_src", + "outer_udp_dst", "vni", "gre_protocol", "tni", "flow_id"}, + "tunnel flow action")) { + return false; + } try { const YAML::Node type_node = tunnel["type"] ? tunnel["type"] : action_node["tunnel_type"]; action.tunnel_.type_ = tunnel_type_from_string(type_node.as()); @@ -223,10 +250,16 @@ bool parse_flow_action_config(const YAML::Node& action_node, FlowAction& action) return false; } - action.tunnel_.outer_eth_src_ = tunnel["outer_eth_src"].as(""); - action.tunnel_.outer_eth_dst_ = tunnel["outer_eth_dst"].as(""); - action.tunnel_.outer_ipv4_src_ = tunnel["outer_ipv4_src"].as(""); - action.tunnel_.outer_ipv4_dst_ = tunnel["outer_ipv4_dst"].as(""); + if (!detail::parse_optional_yaml_scalar(tunnel, "outer_eth_src", std::string{}, + action.tunnel_.outer_eth_src_, "tunnel") || + !detail::parse_optional_yaml_scalar(tunnel, "outer_eth_dst", std::string{}, + action.tunnel_.outer_eth_dst_, "tunnel") || + !detail::parse_optional_yaml_scalar(tunnel, "outer_ipv4_src", std::string{}, + action.tunnel_.outer_ipv4_src_, "tunnel") || + !detail::parse_optional_yaml_scalar(tunnel, "outer_ipv4_dst", std::string{}, + action.tunnel_.outer_ipv4_dst_, "tunnel")) { + return false; + } if (tunnel["outer_ipv4_ttl"] && !parse_u8_field(tunnel, "outer_ipv4_ttl", &action.tunnel_.outer_ipv4_ttl_)) { DAQIRI_LOG_ERROR("tunnel action has invalid 'outer_ipv4_ttl'"); @@ -795,6 +828,16 @@ YAML::Node get_network_node(const YAML::Node& root) { Status parse_network_config_node(const YAML::Node& root, NetworkConfig& config) { try { + if (root["daqiri"].IsDefined()) { + const YAML::Node daqiri_node = root["daqiri"]; + if (!detail::validate_yaml_mapping_keys(daqiri_node, {"cfg"}, "daqiri")) { + return Status::INVALID_PARAMETER; + } + if (!daqiri_node["cfg"].IsDefined()) { + DAQIRI_LOG_ERROR("daqiri.cfg is required"); + return Status::INVALID_PARAMETER; + } + } const YAML::Node network_node = get_network_node(root); if (!network_node || !network_node.IsMap()) { DAQIRI_LOG_ERROR("Invalid YAML: expected top-level map for network configuration"); @@ -1071,9 +1114,16 @@ RDMAOpCode rdma_get_opcode(BurstParams* burst) { bool YAML::convert::parse_flow_config( const YAML::Node& flow_item, daqiri::FlowConfig& flow) { struct in_addr addr; + if (!daqiri::detail::validate_yaml_mapping_keys( + flow_item, {"name", "id", "match", "action", "actions"}, "flow")) { + return false; + } try { flow.name_ = flow_item["name"].as(); - flow.id_ = flow_item["id"].as(); + if (!daqiri::detail::parse_yaml_integer(flow_item["id"], flow.id_)) { + DAQIRI_LOG_ERROR("Flow ID must be a 32-bit unsigned integer"); + return false; + } } catch (const std::exception& e) { DAQIRI_LOG_ERROR("Error parsing FlowConfig: {}", e.what()); return false; @@ -1121,14 +1171,25 @@ bool YAML::convert::parse_flow_config( flow.match_.type_ = daqiri::FlowMatchType::IPV4_UDP; const YAML::Node match = flow_item["match"]; + if (match.IsDefined() && !daqiri::detail::validate_yaml_mapping_keys( + match, + {"udp_src", "udp_dst", "ipv4_len", "ipv4_src", "ipv4_dst", + "flex_item_id", "val", "mask", "ethernet", "ecpri"}, + "flow.match")) { + return false; + } - const YAML::Node ecpri_node = match["ecpri"]; - const YAML::Node ethernet_node = match["ethernet"]; - if (ethernet_node) { + const YAML::Node ecpri_node = match.IsDefined() ? match["ecpri"] : YAML::Node{}; + const YAML::Node ethernet_node = match.IsDefined() ? match["ethernet"] : YAML::Node{}; + if (match.IsDefined() && ethernet_node.IsDefined()) { if (!ethernet_node.IsMap()) { DAQIRI_LOG_ERROR("Flow '{}' field 'match.ethernet' must be a map", flow.name_); return false; } + if (!daqiri::detail::validate_yaml_mapping_keys(ethernet_node, {"src", "dst"}, + "flow.match.ethernet")) { + return false; + } auto parse_address = [&](const char* key, bool& enabled, std::array& address) { @@ -1166,17 +1227,30 @@ bool YAML::convert::parse_flow_config( // Matches the eCPRI EtherType (0xAEFE) plus an optional common-header message // type and message identifier (pc_id/rtc_id). Detected before the UDP/IP and // flex-item paths because it is a distinct, mutually exclusive match class. - if (ecpri_node && ecpri_node.IsMap()) { + if (match.IsDefined() && ecpri_node.IsDefined() && !ecpri_node.IsMap()) { + DAQIRI_LOG_ERROR("Flow '{}' field 'match.ecpri' must be a map", flow.name_); + return false; + } + if (match.IsDefined() && ecpri_node.IsDefined()) { + if (!daqiri::detail::validate_yaml_mapping_keys(ecpri_node, {"msg_type", "pc_id", "rtc_id"}, + "flow.match.ecpri")) { + return false; + } flow.match_.type_ = daqiri::FlowMatchType::ECPRI; if (ecpri_node["msg_type"]) { - flow.match_.ecpri_match_.msg_type_ = - static_cast(ecpri_node["msg_type"].as() & 0xff); + if (!daqiri::parse_u8_field(ecpri_node, "msg_type", &flow.match_.ecpri_match_.msg_type_)) { + DAQIRI_LOG_ERROR("eCPRI flow '{}' has invalid msg_type", flow.name_); + return false; + } flow.match_.ecpri_match_.match_msg_type_ = true; } // pc_id (msg type 0/1) and rtc_id (msg type 2) name the same 16-bit field. const YAML::Node id_node = ecpri_node["pc_id"] ? ecpri_node["pc_id"] : ecpri_node["rtc_id"]; if (id_node) { - flow.match_.ecpri_match_.id_ = id_node.as(); + if (!daqiri::detail::parse_yaml_integer(id_node, flow.match_.ecpri_match_.id_)) { + DAQIRI_LOG_ERROR("eCPRI flow '{}' has invalid pc_id/rtc_id", flow.name_); + return false; + } flow.match_.ecpri_match_.match_id_ = true; } if (flow.match_.ecpri_match_.match_id_ && !flow.match_.ecpri_match_.match_msg_type_) { @@ -1192,66 +1266,80 @@ bool YAML::convert::parse_flow_config( return true; } - try { - flow.match_.udp_src_ = match["udp_src"].as(); - } catch (const std::exception& e) { - flow.match_.udp_src_ = 0; + flow.match_.udp_src_ = 0; + if (match.IsDefined() && match["udp_src"] && + !daqiri::parse_u16_field(match, "udp_src", &flow.match_.udp_src_)) { + DAQIRI_LOG_ERROR("Flow '{}' has invalid udp_src", flow.name_); + return false; } - try { - flow.match_.udp_dst_ = match["udp_dst"].as(); - } catch (const std::exception& e) { - flow.match_.udp_dst_ = 0; + flow.match_.udp_dst_ = 0; + if (match.IsDefined() && match["udp_dst"] && + !daqiri::parse_u16_field(match, "udp_dst", &flow.match_.udp_dst_)) { + DAQIRI_LOG_ERROR("Flow '{}' has invalid udp_dst", flow.name_); + return false; } - try { - flow.match_.ipv4_len_ = match["ipv4_len"].as(); - } catch (const std::exception& e) { - flow.match_.ipv4_len_ = 0; + flow.match_.ipv4_len_ = 0; + if (match.IsDefined() && match["ipv4_len"] && + !daqiri::parse_u16_field(match, "ipv4_len", &flow.match_.ipv4_len_)) { + DAQIRI_LOG_ERROR("Flow '{}' has invalid ipv4_len", flow.name_); + return false; } - try { - std::string ipv4_src = match["ipv4_src"].as(); + flow.match_.ipv4_src_ = INADDR_ANY; + if (match.IsDefined() && match["ipv4_src"].IsDefined()) { + std::string ipv4_src; + try { + ipv4_src = match["ipv4_src"].as(); + } catch (const std::exception& e) { + DAQIRI_LOG_ERROR("Flow '{}' has invalid ipv4_src: {}", flow.name_, e.what()); + return false; + } if (inet_pton(AF_INET, ipv4_src.c_str(), &addr) != 1) { DAQIRI_LOG_ERROR("Error parsing ipv4_src : {}", ipv4_src); return false; } else { flow.match_.ipv4_src_ = addr.s_addr; } - } catch (const std::exception& e) { - flow.match_.ipv4_src_ = INADDR_ANY; } - try { - std::string ipv4_dst = match["ipv4_dst"].as(); + flow.match_.ipv4_dst_ = INADDR_ANY; + if (match.IsDefined() && match["ipv4_dst"].IsDefined()) { + std::string ipv4_dst; + try { + ipv4_dst = match["ipv4_dst"].as(); + } catch (const std::exception& e) { + DAQIRI_LOG_ERROR("Flow '{}' has invalid ipv4_dst: {}", flow.name_, e.what()); + return false; + } if (inet_pton(AF_INET, ipv4_dst.c_str(), &addr) != 1) { DAQIRI_LOG_ERROR("Error parsing ipv4_dst : {}", ipv4_dst); return false; } else { flow.match_.ipv4_dst_ = addr.s_addr; } - } catch (const std::exception& e) { - flow.match_.ipv4_dst_ = INADDR_ANY; } // if none of the normal match criteria are defined, use flex item match - if ( flow.match_.udp_src_ == 0 - && flow.match_.udp_dst_ == 0 - && flow.match_.ipv4_len_ == 0 - && flow.match_.ipv4_src_ == INADDR_ANY - && flow.match_.ipv4_dst_ == INADDR_ANY - && match["flex_item_id"] - ) { + if (flow.match_.udp_src_ == 0 && flow.match_.udp_dst_ == 0 && flow.match_.ipv4_len_ == 0 && + flow.match_.ipv4_src_ == INADDR_ANY && flow.match_.ipv4_dst_ == INADDR_ANY && + match.IsDefined() && match["flex_item_id"]) { // No normal match criteria defined, use flex item match - flow.match_.flex_item_match_.flex_item_id_ = match["flex_item_id"].as(); - flow.match_.flex_item_match_.val_ = match["val"].as(); - flow.match_.flex_item_match_.mask_ = match["mask"].as(); + if (!daqiri::detail::parse_yaml_integer(match["flex_item_id"], + flow.match_.flex_item_match_.flex_item_id_) || + !daqiri::detail::parse_yaml_integer(match["val"], flow.match_.flex_item_match_.val_) || + !daqiri::detail::parse_yaml_integer(match["mask"], flow.match_.flex_item_match_.mask_)) { + DAQIRI_LOG_ERROR("Flow '{}' has invalid flex-item numeric fields", flow.name_); + return false; + } flow.match_.type_ = daqiri::FlowMatchType::FLEX_ITEM; DAQIRI_LOG_INFO("Using flex item match: flex_item_id={}, val={}, mask={}", flow.match_.flex_item_match_.flex_item_id_, flow.match_.flex_item_match_.val_, flow.match_.flex_item_match_.mask_); - } else if (ethernet_node && flow.match_.udp_src_ == 0 && flow.match_.udp_dst_ == 0 && + } else if (match.IsDefined() && ethernet_node.IsDefined() && flow.match_.udp_src_ == 0 && + flow.match_.udp_dst_ == 0 && flow.match_.ipv4_len_ == 0 && flow.match_.ipv4_src_ == INADDR_ANY && flow.match_.ipv4_dst_ == INADDR_ANY) { flow.match_.type_ = daqiri::FlowMatchType::ETHERNET; @@ -1269,11 +1357,19 @@ bool YAML::convert::parse_flow_config( */ bool YAML::convert::parse_flex_item_config( const YAML::Node& flex_item, daqiri::FlexItemConfig& flex_item_config) { + if (!daqiri::detail::validate_yaml_mapping_keys( + flex_item, {"name", "id", "udp_dst_port", "offset"}, "flex item")) { + return false; + } try { flex_item_config.name_ = flex_item["name"].as(); - flex_item_config.id_ = flex_item["id"].as(); - flex_item_config.udp_dst_port_ = flex_item["udp_dst_port"].as(); - flex_item_config.offset_ = flex_item["offset"].as(); + if (!daqiri::detail::parse_yaml_integer(flex_item["id"], flex_item_config.id_) || + !daqiri::detail::parse_yaml_integer(flex_item["udp_dst_port"], + flex_item_config.udp_dst_port_) || + !daqiri::detail::parse_yaml_integer(flex_item["offset"], flex_item_config.offset_)) { + DAQIRI_LOG_ERROR("Flex-item numeric fields must be 16-bit unsigned integers"); + return false; + } if ((flex_item_config.offset_ % 4) != 0 || flex_item_config.offset_ > 28) { DAQIRI_LOG_CRITICAL("Flex item offset (in bytes) must be a multiple of 4 and less than 28"); return false; @@ -1294,6 +1390,14 @@ bool YAML::convert::parse_flex_item_config( */ bool YAML::convert::parse_reorder_config( const YAML::Node& reorder_item, daqiri::ReorderConfig& reorder_config) { + if (!daqiri::detail::validate_yaml_mapping_keys( + reorder_item, + {"name", "reorder_engine", "cyclic_sequence", "missing_action", "reorder_type", + "memory_region", "payload_byte_offset", "packet_size", "flow_ids", "data_types", + "method"}, + "reorder config")) { + return false; + } auto parse_bit_field = [](const YAML::Node& node, const char* field_name, daqiri::ReorderBitFieldConfig& field_cfg) -> bool { @@ -1304,8 +1408,15 @@ bool YAML::convert::parse_reorder_config( try { const auto& bit_field = node[field_name]; - field_cfg.bit_offset_ = bit_field["bit_offset"].as(); - field_cfg.bit_width_ = bit_field["bit_width"].as(); + if (!daqiri::detail::validate_yaml_mapping_keys(bit_field, {"bit_offset", "bit_width"}, + field_name)) { + return false; + } + if (!daqiri::detail::parse_yaml_integer(bit_field["bit_offset"], field_cfg.bit_offset_) || + !daqiri::detail::parse_yaml_integer(bit_field["bit_width"], field_cfg.bit_width_)) { + DAQIRI_LOG_ERROR("Bit-field offsets and widths must be unsigned integers"); + return false; + } } catch (const std::exception& e) { DAQIRI_LOG_ERROR("Failed to parse bit field '{}': {}", field_name, e.what()); return false; @@ -1329,14 +1440,31 @@ bool YAML::convert::parse_reorder_config( try { reorder_config.name_ = reorder_item["name"].as(); - reorder_config.reorder_engine_ = reorder_item["reorder_engine"].as("sw"); - reorder_config.cyclic_sequence_ = reorder_item["cyclic_sequence"].as(false); - reorder_config.missing_action_ = daqiri::reorder_missing_action_from_string( - reorder_item["missing_action"].as("passthrough")); + std::string missing_action; + if (!daqiri::detail::parse_optional_yaml_scalar( + reorder_item, "reorder_engine", std::string{"sw"}, reorder_config.reorder_engine_, + "reorder config") || + !daqiri::detail::parse_optional_yaml_scalar(reorder_item, "cyclic_sequence", false, + reorder_config.cyclic_sequence_, + "reorder config") || + !daqiri::detail::parse_optional_yaml_scalar(reorder_item, "missing_action", + std::string{"passthrough"}, missing_action, + "reorder config")) { + return false; + } + reorder_config.missing_action_ = daqiri::reorder_missing_action_from_string(missing_action); reorder_config.reorder_type_ = reorder_item["reorder_type"].as(); reorder_config.memory_region_ = reorder_item["memory_region"].as(); - reorder_config.payload_byte_offset_ = reorder_item["payload_byte_offset"].as(); - reorder_config.packet_size_ = reorder_item["packet_size"].as(0); + if (!daqiri::detail::parse_yaml_integer(reorder_item["payload_byte_offset"], + reorder_config.payload_byte_offset_)) { + DAQIRI_LOG_ERROR("payload_byte_offset must be a 32-bit unsigned integer"); + return false; + } + if (!daqiri::detail::parse_optional_yaml_integer(reorder_item, "packet_size", uint32_t{0}, + reorder_config.packet_size_, + "reorder config")) { + return false; + } if (!reorder_item["flow_ids"] || !reorder_item["flow_ids"].IsSequence()) { DAQIRI_LOG_ERROR("Reorder config '{}' requires a non-empty flow_ids sequence", @@ -1345,7 +1473,12 @@ bool YAML::convert::parse_reorder_config( } for (const auto& flow_id_node : reorder_item["flow_ids"]) { - reorder_config.flow_ids_.push_back(flow_id_node.as()); + daqiri::FlowId flow_id = 0; + if (!daqiri::detail::parse_yaml_integer(flow_id_node, flow_id)) { + DAQIRI_LOG_ERROR("Reorder flow IDs must be 32-bit unsigned integers"); + return false; + } + reorder_config.flow_ids_.push_back(flow_id); } if (reorder_config.flow_ids_.empty()) { DAQIRI_LOG_ERROR("Reorder config '{}' requires at least one flow ID", @@ -1385,6 +1518,12 @@ bool YAML::convert::parse_reorder_config( DAQIRI_LOG_ERROR("Reorder config '{}' data_types must be a map", reorder_config.name_); return false; } + if (!daqiri::detail::validate_yaml_mapping_keys( + data_types_node, + {"input_type", "output_type", "endianness", "input", "output", "input_endianness"}, + "reorder config.data_types")) { + return false; + } const auto input_node = data_types_node["input_type"] ? data_types_node["input_type"] : data_types_node["input"]; @@ -1447,6 +1586,10 @@ bool YAML::convert::parse_reorder_config( } const auto& method_node = reorder_item["method"]; + if (!daqiri::detail::validate_yaml_mapping_keys( + method_node, {"seq_batch_number", "seq_packets_per_batch"}, "reorder config.method")) { + return false; + } const bool has_seq_batch_number = method_node["seq_batch_number"].IsDefined(); const bool has_seq_packets_per_batch = method_node["seq_packets_per_batch"].IsDefined(); if (has_seq_batch_number == has_seq_packets_per_batch) { @@ -1459,6 +1602,11 @@ bool YAML::convert::parse_reorder_config( if (has_seq_batch_number) { const auto& seq_batch_node = method_node["seq_batch_number"]; + if (!daqiri::detail::validate_yaml_mapping_keys(seq_batch_node, + {"sequence_number", "batch_number"}, + "reorder config.method.seq_batch_number")) { + return false; + } reorder_config.method_ = daqiri::ReorderMethod::SEQ_BATCH_NUMBER; if (!parse_bit_field(seq_batch_node, "sequence_number", reorder_config.seq_batch_number_.sequence_number_)) { @@ -1497,6 +1645,11 @@ bool YAML::convert::parse_reorder_config( static_cast(derived_packets_per_batch); } else { const auto& seq_ppb_node = method_node["seq_packets_per_batch"]; + if (!daqiri::detail::validate_yaml_mapping_keys( + seq_ppb_node, {"sequence_number", "packets_per_batch"}, + "reorder config.method.seq_packets_per_batch")) { + return false; + } reorder_config.method_ = daqiri::ReorderMethod::SEQ_PACKETS_PER_BATCH; if (!parse_bit_field(seq_ppb_node, "sequence_number", reorder_config.seq_packets_per_batch_.sequence_number_)) { @@ -1504,8 +1657,12 @@ bool YAML::convert::parse_reorder_config( } try { - reorder_config.seq_packets_per_batch_.packets_per_batch_ = - seq_ppb_node["packets_per_batch"].as(); + if (!daqiri::detail::parse_yaml_integer( + seq_ppb_node["packets_per_batch"], + reorder_config.seq_packets_per_batch_.packets_per_batch_)) { + DAQIRI_LOG_ERROR("packets_per_batch must be a 32-bit unsigned integer"); + return false; + } } catch (const std::exception& e) { DAQIRI_LOG_ERROR("Failed to parse packets_per_batch in reorder config '{}': {}", reorder_config.name_, @@ -1550,19 +1707,65 @@ bool YAML::convert::parse_reorder_config( */ bool YAML::convert::parse_memory_region_config( const YAML::Node& mr, daqiri::MemoryRegionConfig& tmr) { + if (!daqiri::detail::validate_yaml_mapping_keys( + mr, {"name", "kind", "affinity", "access", "num_bufs", "buf_size", "owned"}, + "memory region")) { + return false; + } try { tmr.name_ = mr["name"].as(); tmr.kind_ = daqiri::GetMemoryKindFromString(mr["kind"].template as()); - tmr.buf_size_ = mr["buf_size"].as(); - tmr.num_bufs_ = mr["num_bufs"].as(); - tmr.affinity_ = mr["affinity"].as(); + if (tmr.name_.empty()) { + DAQIRI_LOG_ERROR("Memory-region name must not be empty"); + return false; + } + if (tmr.kind_ == daqiri::MemoryKind::INVALID) { + DAQIRI_LOG_ERROR( + "Invalid memory-region kind; valid values are huge, device, host_pinned, and host"); + return false; + } + if (!daqiri::detail::parse_yaml_integer(mr["buf_size"], tmr.buf_size_) || + !daqiri::detail::parse_yaml_integer(mr["num_bufs"], tmr.num_bufs_) || + !daqiri::detail::parse_yaml_integer(mr["affinity"], tmr.affinity_)) { + DAQIRI_LOG_ERROR("Memory-region sizes and affinity are out of range"); + return false; + } + if (tmr.buf_size_ == 0 || tmr.num_bufs_ == 0) { + DAQIRI_LOG_ERROR("Memory-region buf_size and num_bufs must be greater than zero"); + return false; + } if (mr["access"].IsDefined()) { - tmr.access_ = daqiri::GetMemoryAccessPropertiesFromList(mr["access"]); + if (!mr["access"].IsSequence()) { + DAQIRI_LOG_ERROR("Memory-region access must be a sequence"); + return false; + } + tmr.access_ = 0; + std::unordered_set seen_access; + for (const auto& access_node : mr["access"]) { + const std::string access = access_node.as(); + if (!seen_access.insert(access).second) { + DAQIRI_LOG_ERROR("Duplicate memory-region access value '{}'", access); + return false; + } + if (access == "local") { + tmr.access_ |= daqiri::MEM_ACCESS_LOCAL; + } else if (access == "rdma_write") { + tmr.access_ |= daqiri::MEM_ACCESS_RDMA_WRITE; + } else if (access == "rdma_read") { + tmr.access_ |= daqiri::MEM_ACCESS_RDMA_READ; + } else { + DAQIRI_LOG_ERROR("Unknown memory-region access value '{}'", access); + return false; + } + } } else { tmr.access_ = daqiri::MEM_ACCESS_LOCAL; } - tmr.owned_ = mr["owned"].template as(true); + if (!daqiri::detail::parse_optional_yaml_scalar(mr, "owned", true, tmr.owned_, + "memory region")) { + return false; + } } catch (const std::exception& e) { DAQIRI_LOG_ERROR("Error parsing MemoryRegionConfig: {}", e.what()); return false; @@ -1678,20 +1881,25 @@ bool parse_endpoint_addr(const std::string& value, if (allow_missing_roce_port && parsed.protocol == daqiri::SocketProtocol::ROCE) { parsed.host = authority; - return true; + } else { + DAQIRI_LOG_ERROR("{} must include an IPv4 address and port", field_name); + return false; + } + } else { + if (port_sep == 0 || port_sep + 1 >= authority.size()) { + DAQIRI_LOG_ERROR("{} must include an IPv4 address and port", field_name); + return false; } - DAQIRI_LOG_ERROR("{} must include an IPv4 address and port", field_name); - return false; - } - if (port_sep == 0 || port_sep + 1 >= authority.size()) { - DAQIRI_LOG_ERROR("{} must include an IPv4 address and port", field_name); - return false; - } - parsed.host = authority.substr(0, port_sep); - const std::string port_str = authority.substr(port_sep + 1); - try { - const unsigned long port = std::stoul(port_str); + parsed.host = authority.substr(0, port_sep); + const std::string port_str = authority.substr(port_sep + 1); + uint32_t port = 0; + const auto parse_result = + std::from_chars(port_str.data(), port_str.data() + port_str.size(), port); + if (parse_result.ec != std::errc{} || parse_result.ptr != port_str.data() + port_str.size()) { + DAQIRI_LOG_ERROR("{} port '{}' is not a valid integer", field_name, port_str); + return false; + } if (port > std::numeric_limits::max() || (port == 0 && !(allow_missing_roce_port && @@ -1700,8 +1908,11 @@ bool parse_endpoint_addr(const std::string& value, return false; } parsed.port = static_cast(port); - } catch (const std::exception&) { - DAQIRI_LOG_ERROR("{} port '{}' is not a valid integer", field_name, port_str); + } + + struct in_addr ipv4 {}; + if (inet_pton(AF_INET, parsed.host.c_str(), &ipv4) != 1) { + DAQIRI_LOG_ERROR("{} host '{}' is not a valid IPv4 address", field_name, parsed.host); return false; } @@ -1752,6 +1963,13 @@ bool YAML::convert::parse_socket_config( const YAML::Node& socket_item, daqiri::SocketConfig& socket_cfg, daqiri::SocketProtocol& protocol) { + if (!daqiri::detail::validate_yaml_mapping_keys( + socket_item, + {"mode", "local_addr", "remote_addr", "max_payload_size", "max_burst_interval_ms", + "min_ipg_ns", "retry_connect_s", "local_ip", "local_port", "remote_ip", "remote_port"}, + "socket_config")) { + return false; + } try { socket_cfg.mode_ = daqiri::GetSocketModeFromString( socket_item["mode"].template as()); @@ -1761,8 +1979,12 @@ bool YAML::convert::parse_socket_config( return false; } - socket_cfg.local_addr_ = socket_item["local_addr"].template as(""); - socket_cfg.remote_addr_ = socket_item["remote_addr"].template as(""); + if (!daqiri::detail::parse_optional_yaml_scalar(socket_item, "local_addr", std::string{}, + socket_cfg.local_addr_, "socket_config") || + !daqiri::detail::parse_optional_yaml_scalar(socket_item, "remote_addr", std::string{}, + socket_cfg.remote_addr_, "socket_config")) { + return false; + } const bool has_local_addr = !socket_cfg.local_addr_.empty(); const bool has_remote_addr = !socket_cfg.remote_addr_.empty(); const bool has_legacy_local = socket_item["local_ip"].IsDefined() || @@ -1781,10 +2003,18 @@ bool YAML::convert::parse_socket_config( return false; } - socket_cfg.local_ip_ = socket_item["local_ip"].template as(""); - socket_cfg.remote_ip_ = socket_item["remote_ip"].template as(""); - socket_cfg.local_port_ = socket_item["local_port"].as(0); - socket_cfg.remote_port_ = socket_item["remote_port"].as(0); + if (!daqiri::detail::parse_optional_yaml_scalar(socket_item, "local_ip", std::string{}, + socket_cfg.local_ip_, "socket_config") || + !daqiri::detail::parse_optional_yaml_scalar(socket_item, "remote_ip", std::string{}, + socket_cfg.remote_ip_, "socket_config")) { + return false; + } + if (!daqiri::detail::parse_optional_yaml_integer(socket_item, "local_port", uint16_t{0}, + socket_cfg.local_port_, "socket_config") || + !daqiri::detail::parse_optional_yaml_integer(socket_item, "remote_port", uint16_t{0}, + socket_cfg.remote_port_, "socket_config")) { + return false; + } if (has_local_addr && !apply_endpoint_addr(socket_cfg.local_addr_, @@ -1821,10 +2051,27 @@ bool YAML::convert::parse_socket_config( protocol, socket_cfg.remote_ip_, socket_cfg.remote_port_); } - socket_cfg.max_payload_size_ = socket_item["max_payload_size"].as(0); - socket_cfg.max_burst_interval_ms_ = socket_item["max_burst_interval_ms"].as(0); - socket_cfg.min_ipg_ns_ = socket_item["min_ipg_ns"].as(0); - socket_cfg.retry_connect_s_ = socket_item["retry_connect_s"].as(1); + if (!daqiri::detail::parse_optional_yaml_integer(socket_item, "max_payload_size", uint16_t{0}, + socket_cfg.max_payload_size_, + "socket_config") || + !daqiri::detail::parse_optional_yaml_integer(socket_item, "max_burst_interval_ms", + uint64_t{0}, socket_cfg.max_burst_interval_ms_, + "socket_config") || + !daqiri::detail::parse_optional_yaml_integer(socket_item, "min_ipg_ns", uint32_t{0}, + socket_cfg.min_ipg_ns_, "socket_config") || + !daqiri::detail::parse_optional_yaml_integer(socket_item, "retry_connect_s", int32_t{1}, + socket_cfg.retry_connect_s_, + "socket_config")) { + return false; + } + if (socket_item["max_payload_size"].IsDefined() && socket_cfg.max_payload_size_ == 0) { + DAQIRI_LOG_ERROR("socket_config.max_payload_size must be greater than zero"); + return false; + } + if (socket_cfg.retry_connect_s_ < 0) { + DAQIRI_LOG_ERROR("socket_config.retry_connect_s must not be negative"); + return false; + } const bool roce_client = socket_cfg.mode_ == daqiri::SocketMode::CLIENT && protocol == daqiri::SocketProtocol::ROCE; @@ -1867,6 +2114,9 @@ bool YAML::convert::parse_socket_config( bool YAML::convert::parse_roce_config( const YAML::Node& roce_item, daqiri::RoCEConfig& roce_cfg) { + if (!daqiri::detail::validate_yaml_mapping_keys(roce_item, {"transport_mode"}, "roce_config")) { + return false; + } try { roce_cfg.transport_mode_ = daqiri::GetRDMATransportModeFromString( roce_item["transport_mode"].template as()); @@ -1894,28 +2144,54 @@ bool parse_common_queue_config(const YAML::Node& q_item, daqiri::CommonQueueConf bool parse_memory_regions, bool require_worker_fields = true) { try { common.name_ = q_item["name"].as(); - common.id_ = q_item["id"].as(); + uint16_t queue_id = 0; + if (!daqiri::detail::parse_yaml_integer(q_item["id"], queue_id)) { + DAQIRI_LOG_ERROR("Queue ID must be a 16-bit unsigned integer"); + return false; + } + common.id_ = queue_id; if (require_worker_fields) { if (!q_item["cpu_core"].IsDefined() || !q_item["batch_size"].IsDefined()) { DAQIRI_LOG_ERROR("Queue '{}' requires cpu_core and batch_size in indirect mode", common.name_); return false; } - common.cpu_core_ = q_item["cpu_core"].as(); - common.batch_size_ = q_item["batch_size"].as(); + int32_t cpu_core = 0; + if (!daqiri::detail::parse_yaml_integer(q_item["cpu_core"], cpu_core)) { + DAQIRI_LOG_ERROR("Queue '{}' cpu_core must be a 32-bit integer", common.name_); + return false; + } + if (cpu_core < -1) { + DAQIRI_LOG_ERROR("Queue '{}' cpu_core must be -1 or a non-negative CPU index", + common.name_); + return false; + } + common.cpu_core_ = std::to_string(cpu_core); + if (!daqiri::detail::parse_yaml_integer(q_item["batch_size"], common.batch_size_)) { + DAQIRI_LOG_ERROR("Queue '{}' batch_size is out of range", common.name_); + return false; + } + if (common.batch_size_ <= 0) { + DAQIRI_LOG_ERROR("Queue '{}' batch_size must be greater than zero", common.name_); + return false; + } } else { common.cpu_core_.clear(); common.batch_size_ = 0; } common.extra_queue_config_ = nullptr; if (q_item["memory_regions"].IsDefined()) { + const auto& mrs = q_item["memory_regions"]; + if (!mrs.IsSequence() || mrs.size() == 0) { + DAQIRI_LOG_ERROR("Queue '{}' memory_regions must be a non-empty sequence", common.name_); + return false; + } if (!parse_memory_regions) { DAQIRI_LOG_WARN("Memory regions in queue section not used in RoCE engine for queue: {}", common.name_); } else { - const auto& mrs = q_item["memory_regions"]; - if (mrs.size() > 0) { common.mrs_.reserve(mrs.size()); } + common.mrs_.reserve(mrs.size()); for (const auto& mr : mrs) { common.mrs_.push_back(mr.as()); } } } @@ -1956,6 +2232,12 @@ bool YAML::convert::parse_rx_queue_common_config( bool YAML::convert::parse_rx_queue_config( const YAML::Node& q_item, const daqiri::EngineType& engine_type, daqiri::RxQueueConfig& q, bool parse_memory_regions) { + if (!daqiri::detail::validate_yaml_mapping_keys( + q_item, + {"name", "id", "poll_mode", "cpu_core", "batch_size", "memory_regions", "timeout_us"}, + "RX queue")) { + return false; + } try { daqiri::EngineType _engine_type = engine_type; if (engine_type == daqiri::EngineType::DEFAULT) { @@ -2019,14 +2301,36 @@ bool YAML::convert::parse_tx_queue_common_config( try { if (q_item["offloads"].IsDefined()) { const auto& offload = q_item["offloads"]; + if (!offload.IsSequence()) { + DAQIRI_LOG_ERROR("TX queue offloads must be a sequence"); + return false; + } + std::unordered_set seen; q.common_.offloads_.reserve(offload.size()); for (const auto& off : offload) { - q.common_.offloads_.push_back(off.as()); + const std::string value = off.as(); + if (value != "tx_eth_src") { + DAQIRI_LOG_ERROR("Unknown TX queue offload '{}'", value); + return false; + } + if (!seen.insert(value).second) { + DAQIRI_LOG_ERROR("Duplicate TX queue offload '{}'", value); + return false; + } + q.common_.offloads_.push_back(value); } } // Optional per-queue packet-pacing rate in Mbps (0/absent = pacing off). if (q_item["pacing_mbps"].IsDefined()) { - q.pacing_mbps_ = q_item["pacing_mbps"].as(); + if (!daqiri::detail::parse_yaml_integer(q_item["pacing_mbps"], q.pacing_mbps_)) { + DAQIRI_LOG_ERROR("TX queue pacing_mbps is out of range"); + return false; + } + } + uint64_t ignored_timeout_us = 0; + if (!daqiri::detail::parse_optional_yaml_integer(q_item, "timeout_us", uint64_t{0}, + ignored_timeout_us, "TX queue")) { + return false; } } catch (const std::exception& e) { DAQIRI_LOG_ERROR("Error parsing TxQueueConfig: {}", e.what()); @@ -2046,6 +2350,13 @@ bool YAML::convert::parse_tx_queue_common_config( bool YAML::convert::parse_tx_queue_config( const YAML::Node& q_item, const daqiri::EngineType& engine_type, daqiri::TxQueueConfig& q, bool parse_memory_regions) { + if (!daqiri::detail::validate_yaml_mapping_keys( + q_item, + {"name", "id", "poll_mode", "cpu_core", "batch_size", "memory_regions", "offloads", + "pacing_mbps", "timeout_us"}, + "TX queue")) { + return false; + } try { daqiri::EngineType _engine_type = engine_type; diff --git a/src/engine.cpp b/src/engine.cpp index 2f4bf7bb..e4977130 100644 --- a/src/engine.cpp +++ b/src/engine.cpp @@ -1070,33 +1070,37 @@ int Engine::get_port_id(const std::string& key) { return -1; } -bool Engine::validate_config() const { +bool validate_network_config(const NetworkConfig& config) { bool pass = true; std::set mr_names; std::set q_mr_names; - const bool tunnel_supported_engine = - cfg_.common_.engine_type == EngineType::DPDK || cfg_.common_.engine_type == EngineType::IBVERBS; + std::unordered_set static_rx_flow_ids; + const bool tunnel_supported_engine = config.common_.engine_type == EngineType::DPDK || + config.common_.engine_type == EngineType::IBVERBS; - if (cfg_.common_.loopback_ == LoopbackType::LOOPBACK_TYPE_HW && - (cfg_.common_.stream_type != StreamType::RAW || - cfg_.common_.engine_type != EngineType::IBVERBS)) { + if (config.common_.loopback_ == LoopbackType::LOOPBACK_TYPE_HW && + (config.common_.stream_type != StreamType::RAW || + config.common_.engine_type != EngineType::IBVERBS)) { DAQIRI_LOG_ERROR( "Hardware loopback is supported only for stream_type 'raw' with engine 'ibverbs'"); pass = false; } // Verify all memory regions are used in queues and all queue MRs are listed in the MR section - for (const auto& mr : cfg_.mrs_) { mr_names.emplace(mr.second.name_); } + for (const auto& mr : config.mrs_) { + mr_names.emplace(mr.second.name_); + } - for (const auto& intf : cfg_.ifs_) { + for (const auto& intf : config.ifs_) { std::set rx_queue_ids; std::set direct_rx_queue_ids; + std::unordered_set interface_rx_flow_ids; for (const auto& rxq : intf.rx_.queues_) { rx_queue_ids.insert(rxq.common_.id_); if (rxq.poll_mode_ == QueuePollMode::DIRECT) { direct_rx_queue_ids.insert(rxq.common_.id_); - if (cfg_.common_.stream_type != StreamType::RAW || - cfg_.common_.engine_type != EngineType::IBVERBS) { + if (config.common_.stream_type != StreamType::RAW || + config.common_.engine_type != EngineType::IBVERBS) { DAQIRI_LOG_WARN( "RX queue '{}' requests direct polling, which is supported only by the raw " "ibverbs engine", @@ -1124,8 +1128,8 @@ bool Engine::validate_config() const { } for (const auto& txq : intf.tx_.queues_) { if (txq.poll_mode_ == QueuePollMode::DIRECT) { - if (cfg_.common_.stream_type != StreamType::RAW || - cfg_.common_.engine_type != EngineType::IBVERBS) { + if (config.common_.stream_type != StreamType::RAW || + config.common_.engine_type != EngineType::IBVERBS) { DAQIRI_LOG_WARN( "TX queue '{}' requests direct polling, which is supported only by the raw " "ibverbs engine", @@ -1156,8 +1160,8 @@ bool Engine::validate_config() const { auto queue_frame_size = [&](const CommonQueueConfig& queue) { size_t total = 0; for (const auto& mr_name : queue.mrs_) { - auto it = cfg_.mrs_.find(mr_name); - if (it != cfg_.mrs_.end()) { + auto it = config.mrs_.find(mr_name); + if (it != config.mrs_.end()) { total += it->second.buf_size_; } } @@ -1176,6 +1180,13 @@ bool Engine::validate_config() const { } for (const auto& flow : intf.rx_.flows_) { + if (!interface_rx_flow_ids.insert(flow.id_).second) { + DAQIRI_LOG_ERROR("Duplicate flow ID {} in interface '{}'", flow.id_, intf.name_); + pass = false; + } else if (flow.id_ != 0 && !static_rx_flow_ids.insert(flow.id_).second) { + DAQIRI_LOG_ERROR("Duplicate static flow ID {}", flow.id_); + pass = false; + } const auto actions = flow_config_actions(flow); if (actions.size() > kMaxFlowActions) { DAQIRI_LOG_ERROR("RX flow '{}' on interface '{}' has {} actions; maximum supported is {}", @@ -1189,7 +1200,7 @@ bool Engine::validate_config() const { } else { const FlowAction& queue_action = actions.back(); const auto queue_ids = flow_queue_ids(queue_action); - flow_rx_queue_ids[flow.id_] = queue_ids; + flow_rx_queue_ids.emplace(flow.id_, queue_ids); std::set unique_ids; for (const uint16_t queue_id : queue_ids) { if (!unique_ids.insert(queue_id).second) { @@ -1219,7 +1230,8 @@ bool Engine::validate_config() const { flow.name_, intf.name_); pass = false; } - if (has_transform && (cfg_.common_.stream_type != StreamType::RAW || !tunnel_supported_engine)) { + if (has_transform && + (config.common_.stream_type != StreamType::RAW || !tunnel_supported_engine)) { DAQIRI_LOG_ERROR("RX flow '{}' uses tunnel/VLAN actions, which are supported only for raw " "DPDK or raw ibverbs engines", flow.name_); @@ -1253,15 +1265,19 @@ bool Engine::validate_config() const { pass = false; } const auto queues_it = flow_rx_queue_ids.find(flow_id); - if (queues_it != flow_rx_queue_ids.end()) { - for (const uint16_t queue_id : queues_it->second) { - if (direct_rx_queue_ids.find(queue_id) != direct_rx_queue_ids.end()) { - DAQIRI_LOG_WARN( - "Reorder config '{}' targets direct RX queue {} on interface '{}'; direct " - "polling does not support reorder", - reorder.name_, queue_id, intf.name_); - pass = false; - } + if (queues_it == flow_rx_queue_ids.end()) { + DAQIRI_LOG_ERROR("Reorder config '{}' references unknown flow ID {} on interface '{}'", + reorder.name_, flow_id, intf.name_); + pass = false; + continue; + } + for (const uint16_t queue_id : queues_it->second) { + if (direct_rx_queue_ids.find(queue_id) != direct_rx_queue_ids.end()) { + DAQIRI_LOG_WARN( + "Reorder config '{}' targets direct RX queue {} on interface '{}'; direct " + "polling does not support reorder", + reorder.name_, queue_id, intf.name_); + pass = false; } } } @@ -1304,7 +1320,7 @@ bool Engine::validate_config() const { pass = false; } if (has_transform && - (cfg_.common_.stream_type != StreamType::RAW || !tunnel_supported_engine)) { + (config.common_.stream_type != StreamType::RAW || !tunnel_supported_engine)) { DAQIRI_LOG_ERROR("TX flow '{}' uses tunnel/VLAN actions, which are supported only for raw " "DPDK or raw ibverbs engines", flow.name_); @@ -1337,6 +1353,10 @@ bool Engine::validate_config() const { return pass; } +bool Engine::validate_config() const { + return validate_network_config(cfg_); +} + void Engine::init_rx_core_q_map() { for (const auto& intf : cfg_.ifs_) { // Initialize the round-robin index for this port diff --git a/src/engine.h b/src/engine.h index 127307e4..98c455e6 100644 --- a/src/engine.h +++ b/src/engine.h @@ -35,6 +35,10 @@ struct rte_pktmbuf_extmem; namespace daqiri { +// Run the hardware-independent semantic checks shared by every engine. +// This does not allocate memory, initialize a runtime, or touch a NIC. +bool validate_network_config(const NetworkConfig& config); + struct AllocRegion { enum class Deallocator { NONE, diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt new file mode 100644 index 00000000..357f0d1a --- /dev/null +++ b/tools/CMakeLists.txt @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.20) + +include(GNUInstallDirs) + +add_executable(daqiri_config_validate config_validate.cpp) +target_compile_features(daqiri_config_validate PRIVATE cxx_std_17) +target_include_directories(daqiri_config_validate PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/..) + +if(NOT DAQIRI_YAML_TARGET) + find_package(yaml-cpp QUIET) + if(TARGET yaml-cpp::yaml-cpp) + set(DAQIRI_YAML_TARGET yaml-cpp::yaml-cpp) + elseif(TARGET yaml-cpp) + set(DAQIRI_YAML_TARGET yaml-cpp) + elseif(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../third_party/yaml-cpp/CMakeLists.txt") + add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../third_party/yaml-cpp + ${CMAKE_CURRENT_BINARY_DIR}/third_party/yaml-cpp) + set(DAQIRI_YAML_TARGET yaml-cpp) + else() + message(FATAL_ERROR + "yaml-cpp not found; install yaml-cpp-dev or run " + "`git submodule update --init third_party/yaml-cpp`") + endif() +endif() + +if(NOT BUILD_SHARED_LIBS AND UNIX AND NOT APPLE) + set(DAQIRI_TOOL_STATIC_LIBS daqiri::daqiri) + foreach(engine IN ITEMS dpdk socket rdma ibverbs) + if(TARGET daqiri_${engine}) + list(APPEND DAQIRI_TOOL_STATIC_LIBS daqiri_${engine}) + endif() + endforeach() + target_link_libraries(daqiri_config_validate PRIVATE + "-Wl,--start-group" + ${DAQIRI_TOOL_STATIC_LIBS} + "-Wl,--end-group" + ${DAQIRI_YAML_TARGET}) +else() + target_link_libraries(daqiri_config_validate PRIVATE daqiri::daqiri ${DAQIRI_YAML_TARGET}) +endif() + +set_target_properties(daqiri_config_validate PROPERTIES + BUILD_RPATH "$ORIGIN/../src;$ORIGIN/../src/third_party/yaml-cpp" + INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}") + +install(TARGETS daqiri_config_validate RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) diff --git a/tools/config_validate.cpp b/tools/config_validate.cpp new file mode 100644 index 00000000..49aaaa27 --- /dev/null +++ b/tools/config_validate.cpp @@ -0,0 +1,31 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. + * All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +#include "src/engine.h" + +#include + +int main(int argc, char** argv) { + if (argc < 2) { + std::cerr << "Usage: daqiri_config_validate [...]\n"; + return 2; + } + + bool valid = true; + for (int index = 1; index < argc; ++index) { + daqiri::NetworkConfig config; + const auto status = daqiri::parse_network_config_from_yaml_file(argv[index], config); + if (status != daqiri::Status::SUCCESS || !daqiri::validate_network_config(config)) { + std::cerr << argv[index] << ": invalid\n"; + valid = false; + continue; + } + std::cout << argv[index] << ": valid\n"; + } + return valid ? 0 : 1; +} From a4ed9178dc86f01ba4933ca1006ef09e099fe14b Mon Sep 17 00:00:00 2001 From: Denis Leshchev Date: Tue, 22 Sep 2026 13:28:01 -0400 Subject: [PATCH 2/7] #315 - Integrate configuration validation Signed-off-by: Denis Leshchev --- AGENTS.md | 10 +- CMakeLists.txt | 1 + CONTRIBUTING.md | 5 + Dockerfile | 2 + README.md | 3 + docs/api-reference/configuration.md | 14 ++ docs/getting-started.md | 2 +- scripts/check_daqiri_configs.py | 286 ++++++++++++++++++++++++++++ scripts/check_pr.sh | 11 ++ src/engine.h | 4 +- 10 files changed, 334 insertions(+), 4 deletions(-) create mode 100755 scripts/check_daqiri_configs.py diff --git a/AGENTS.md b/AGENTS.md index b19efee5..449d7dcc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -22,7 +22,8 @@ canonical multi-architecture version tag. CMake options (full table in `docs/getting-started.md`): - `DAQIRI_ENGINE` — space-separated list of optional engines to compile. Valid values: `dpdk` (raw Ethernet) and `ibverbs` (RDMA/RoCE). Linux sockets (UDP/TCP) are always built in, so there is no `socket` value. Default is `"dpdk ibverbs"`. - `DAQIRI_BUILD_PYTHON` — builds `pybind11` bindings from `python/`. -- `DAQIRI_BUILD_EXAMPLES` — builds the benchmark executables (default `ON`). +- `DAQIRI_BUILD_EXAMPLES` — builds the benchmark executables (default `ON`). The + hardware-free `daqiri_config_validate` tool is always built and installed. - `DAQIRI_BUILD_APPLICATIONS` — builds the end-to-end example applications under `applications/` (default `OFF`; requires TensorRT, e.g. the `BASE_IMAGE=torch` container). Currently builds `applications/resnet50_inference/` (DAQIRI → TensorRT ResNet inference). - `DAQIRI_ENABLE_OTEL_METRICS` — enables OpenTelemetry metrics instrumentation (default `OFF`). - `DAQIRI_REORDER_GPU_PROFILE` — enable CUDA event timing in the DPDK reorder kernels (off by default). @@ -45,6 +46,13 @@ python3 -m venv .venv .venv/bin/python -m pytest ``` +Validate checked-in configurations through the production parser and common semantic checks +without initializing hardware: + +```bash +python3 scripts/check_daqiri_configs.py --validator build/tools/daqiri_config_validate +``` + The default suite collects only `tests/portable/`. Future build-backed C++ tests live under `tests/cpp/`; Python-binding tests live under `tests/bindings/` and require a container built with `DAQIRI_BUILD_PYTHON=ON`. Platform tests live under `tests/platform/` and are selected by CI/CD jobs running on provisioned GPU/NIC systems; they are never part of the default pytest collection. The project container already includes the current test packages; use the container-specific dependency command in `tests/README.md` when `tests/requirements.txt` changes. Integration and performance verification is done via the benchmark executables in `examples/`, driven by YAML configs. Build outputs (`examples/CMakeLists.txt:59-71`): diff --git a/CMakeLists.txt b/CMakeLists.txt index 18ca2d7f..d3a7b44b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -55,6 +55,7 @@ option(DAQIRI_ENABLE_OTEL_METRICS "Enable OpenTelemetry metrics instrumentation" set(DAQIRI_ENGINE "dpdk ibverbs" CACHE STRING "Optional engine implementations to build: dpdk, ibverbs (Linux sockets are always built in)") add_subdirectory(src) +add_subdirectory(tools) if(TARGET yaml-cpp) install( diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ee0e5998..4e334061 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -87,6 +87,11 @@ a GPU, NIC, hugepages, privileged access, or a particular host topology belong u runners. See `tests/README.md` for the container dependency command, supported invocations, and marker policy. +Build `daqiri_config_validate` in the required project container before running +`scripts/check_pr.sh`. The check script validates representative checked-in configurations +through the production C++ parser and hardware-independent semantic checks. Set +`DAQIRI_CONFIG_VALIDATOR` when the executable is not at `build/tools/daqiri_config_validate`. + #### Pull Requests Developer workflow for code contributions is as follows: diff --git a/Dockerfile b/Dockerfile index f1b5b2b2..17d6f9a5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -321,6 +321,8 @@ RUN cmake -S . -B build \ -DDAQIRI_ENABLE_S3=${DAQIRI_ENABLE_S3} \ -DDAQIRI_ENGINE="${DAQIRI_ENGINE}" \ && cmake --build build -j "$(nproc)" \ + && python3 scripts/check_daqiri_configs.py \ + --validator build/tools/daqiri_config_validate \ && cmake --install build # ============================== diff --git a/README.md b/README.md index 2117ef1f..9d120d5e 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,9 @@ DAQIRI provides direct NIC hardware access in userspace, bypassing the Linux ker ## Features +- **Hardware-free configuration validation** — The installed + `daqiri_config_validate` tool uses DAQIRI's production parser and common semantic checks + without allocating packet memory, initializing an engine, or accessing a NIC. - **High Throughput** — Sustained line rate with proper hardware and tuning. - **Low Latency** — Direct access to NIC ring buffers; most latency is PCIe transit only. - **Explicit hugepage allocation** — `kind: huge` always means hugetlb-backed memory for diff --git a/docs/api-reference/configuration.md b/docs/api-reference/configuration.md index d09632b6..f83f04e2 100644 --- a/docs/api-reference/configuration.md +++ b/docs/api-reference/configuration.md @@ -12,6 +12,20 @@ want to interoperate with existing configuration code. See `examples/daqiri_bench_*.yaml` for complete working examples. +## Validate without hardware initialization + +`daqiri_config_validate` parses one or more YAML files and applies the same common semantic +checks used before `daqiri_init()`. It does not allocate packet memory, initialize CUDA or a +network engine, or access a NIC: + +```bash +daqiri_config_validate config.yaml another-config.yaml +``` + +The command exits with status `0` when every file is valid, `1` when any file is invalid, and +`2` when no file was provided. It is built and installed even when +`DAQIRI_BUILD_EXAMPLES=OFF`. + OpenTelemetry metrics do not add YAML fields. Metrics-enabled builds use the same interface, queue, and flow names from the active configuration as metric labels, and applications are still responsible for configuring the OpenTelemetry diff --git a/docs/getting-started.md b/docs/getting-started.md index 47810c41..50378566 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -213,7 +213,7 @@ DAQIRI's shared-library ABI version is tracked separately through |--------|---------|-------------| | `DAQIRI_ENGINE` | `"dpdk ibverbs"` | Space-separated list of optional engine implementations to compile in. Valid values: `dpdk` (Raw Ethernet) and `ibverbs`. `ibverbs` builds two libibverbs-based engines: RDMA/RoCE (for `stream_type: "socket"` with `roce://` endpoints) and the default Mellanox/mlx5 Multi-Packet (striding) Receive Queue engine for `stream_type: "raw"`. Set `engine: "dpdk"` on a raw stream to select the compiled DPDK implementation instead. Linux UDP/TCP sockets are always built in, so there is no `socket` value. | | `DAQIRI_BUILD_PYTHON` | `OFF` | Build pybind11 Python bindings. | -| `DAQIRI_BUILD_EXAMPLES` | `ON` | Build benchmark executables. | +| `DAQIRI_BUILD_EXAMPLES` | `ON` | Build benchmark executables. The `daqiri_config_validate` tool is always built and installed, including when this option is `OFF`. | | `DAQIRI_ENABLE_GDS` | `OFF` | Enable cuFile-backed burst file writes from CUDA device memory. Host-memory writes use POSIX APIs without GDS. | | `DAQIRI_ENABLE_OTEL_METRICS` | `OFF` | Enable OpenTelemetry C++ metrics instrumentation. When enabled, OpenTelemetry C++ API package metadata must be available to CMake. | | `DAQIRI_ENABLE_S3` | `OFF` | Enable AWS SDK-backed asynchronous raw packet writes to S3. | diff --git a/scripts/check_daqiri_configs.py b/scripts/check_daqiri_configs.py new file mode 100755 index 00000000..2bf045ad --- /dev/null +++ b/scripts/check_daqiri_configs.py @@ -0,0 +1,286 @@ +#!/usr/bin/env python3 +# +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Validate checked-in DAQIRI configurations with the production C++ validator.""" + +from __future__ import annotations + +import argparse +import re +import subprocess +import sys +import tempfile +from pathlib import Path + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +INTEGER_PLACEHOLDER = re.compile( + r"^(?P\s*)(?Pmaster_core|cpu_core|affinity):" + r"\s*<[^>]+>(?P\s*(?:#.*)?)$" +) +STRING_PLACEHOLDERS = { + "": "192.0.2.1", + "": "192.0.2.2", +} +DEFAULT_CONFIGS = ( + "examples/daqiri_bench_raw_sw_loopback.yaml", + "examples/daqiri_bench_raw_tx_rx.yaml", + "examples/daqiri_bench_raw_tx_rx_hds.yaml", + "examples/daqiri_bench_rdma_tx_rx.yaml", + "examples/daqiri_bench_socket_udp_tx_rx.yaml", + "examples/daqiri_bench_socket_tcp_tx_rx.yaml", + "examples/daqiri_example_dynamic_rx_flow.yaml", +) +SEMANTIC_FIXTURE = "examples/daqiri_bench_raw_rx_reorder_seq_batch.yaml" +ZERO_ID_FIXTURE = "examples/daqiri_bench_raw_tx_rx.yaml" +PARSER_FIXTURE = "examples/daqiri_bench_socket_udp_tx_rx.yaml" +FLOW_ID_DEFINITION = " id: 100\n" +REORDER_SECTION = " reorder_configs:\n" +DUPLICATE_FLOW = """\ + - name: "duplicate_flow_id" + id: 100 + action: + type: queue + id: 0 + match: + udp_dst: 4097 +""" + + +def materialize_integer_placeholders(text: str) -> str: + """Materialize typed values that the production parser validates syntactically.""" + + lines: list[str] = [] + for line in text.splitlines(keepends=True): + newline = "\n" if line.endswith("\n") else "" + content = line[:-1] if newline else line + match = INTEGER_PLACEHOLDER.fullmatch(content) + if match: + content = ( + f"{match.group('indent')}{match.group('key')}: 0" + f"{match.group('suffix')}" + ) + lines.append(content + newline) + materialized = "".join(lines) + for placeholder, replacement in STRING_PLACEHOLDERS.items(): + materialized = materialized.replace(placeholder, replacement) + return materialized + + +def checked_in_paths() -> list[Path]: + return [REPOSITORY_ROOT / relative_path for relative_path in DEFAULT_CONFIGS] + + +def zero_id_multi_interface_case() -> str: + source = materialize_integer_placeholders( + (REPOSITORY_ROOT / ZERO_ID_FIXTURE).read_text(encoding="utf-8") + ) + stream_type = ' stream_type: "raw"\n' + rx_interface = ' - name: "rx_port"\n' + bench_rx = "\nbench_rx:" + if source.count(stream_type) != 1 or source.count(rx_interface) != 1: + raise ValueError(f"{ZERO_ID_FIXTURE}: compatibility fixture markers changed") + start = source.index(rx_interface) + end = source.index(bench_rx) + first = source[start:end].replace('"rx_port"', '"rx_port_0"') + second = ( + first.replace('"rx_port_0"', '"rx_port_1"') + .replace('"rq_q_0"', '"rq_q_1"') + .replace('"flow_0"', '"flow_1"') + .replace("<0000:00:00.0>", "<0000:00:00.1>") + ) + source = source[:start] + first + second + source[end:] + return source.replace(stream_type, stream_type + ' engine: "dpdk"\n', 1) + + +def semantic_invalid_cases() -> dict[str, str]: + source = materialize_integer_placeholders( + (REPOSITORY_ROOT / SEMANTIC_FIXTURE).read_text(encoding="utf-8") + ) + if source.count(FLOW_ID_DEFINITION) != 1 or source.count(REORDER_SECTION) != 1: + raise ValueError(f"{SEMANTIC_FIXTURE}: semantic fixture markers changed") + return { + "unknown-reorder-flow.yaml": source.replace( + FLOW_ID_DEFINITION, " id: 999\n", 1 + ), + "duplicate-static-flow-id.yaml": source.replace( + REORDER_SECTION, DUPLICATE_FLOW + REORDER_SECTION, 1 + ), + } + + +def parser_invalid_cases() -> dict[str, str]: + """Create focused malformed inputs from one known-good socket configuration.""" + + source = (REPOSITORY_ROOT / PARSER_FIXTURE).read_text(encoding="utf-8") + + def replace(name: str, old: str, new: str) -> str: + if old not in source: + raise ValueError(f"{PARSER_FIXTURE}: marker for {name} changed") + return source.replace(old, new, 1) + + tx_flow_prefix = """\ + tx: + flows: + - name: invalid + id: 1 + match: +""" + tx_flow_suffix = """\ + action: + type: queue + id: 0 + queues: +""" + + return { + "unknown-queue-key.yaml": replace( + "unknown queue key", + " batch_size: 32\n", + " batch_sise: 32\n", + ), + "integer-overflow.yaml": replace( + "integer overflow", + " rx:\n queues:\n", + " rx:\n dynamic_flow_capacity: 4294967296\n queues:\n", + ), + "unknown-offload.yaml": replace( + "unknown offload", + " batch_size: 1\n memory_regions:\n", + " batch_size: 1\n offloads:\n" + " - tx_eth_scr\n memory_regions:\n", + ), + "malformed-tx-flows.yaml": replace( + "malformed tx flows", + " tx:\n queues:\n", + " tx:\n flows: typo\n queues:\n", + ), + "malformed-ecpri.yaml": replace( + "malformed eCPRI match", + " tx:\n queues:\n", + tx_flow_prefix + " ecpri: ecpri_typo\n" + tx_flow_suffix, + ), + "malformed-ipv4.yaml": replace( + "malformed IPv4 match", + " tx:\n queues:\n", + tx_flow_prefix + + " ipv4_src:\n - 10.0.0.1\n" + + tx_flow_suffix, + ), + "invalid-memory-kind.yaml": replace( + "invalid memory kind", ' kind: "host"\n', ' kind: "devcie"\n' + ), + "invalid-memory-access.yaml": replace( + "invalid memory access", + ' kind: "host"\n', + ' kind: "host"\n access:\n - locla\n', + ), + "invalid-log-level.yaml": replace( + "invalid log level", + ' log_level: "info"\n', + ' log_level: "verbose"\n', + ), + "malformed-endpoint-host.yaml": replace( + "malformed endpoint host", + ' local_addr: "udp://127.0.0.1:5001"\n', + ' local_addr: "udp://not-an-ip:5001"\n', + ), + "malformed-endpoint-port.yaml": replace( + "malformed endpoint port", + ' local_addr: "udp://127.0.0.1:5001"\n', + ' local_addr: "udp://127.0.0.1:5001junk"\n', + ), + "missing-memory-region.yaml": replace( + "missing memory region", + ' - "DATA_SOCKET_SERVER"\n\n - name: udp_client\n', + ' - "missing-region"\n\n - name: udp_client\n', + ), + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--validator", + type=Path, + required=True, + help="path to the built daqiri_config_validate executable", + ) + parser.add_argument("paths", nargs="*", type=Path) + args = parser.parse_args(argv) + + paths = args.paths or checked_in_paths() + if not paths: + parser.error("no configuration files were found") + + compatibility_count = 0 + invalid_count = 0 + with tempfile.TemporaryDirectory(prefix="daqiri-checked-configs-") as temp_dir: + materialized_paths: list[Path] = [] + for index, path in enumerate(paths): + try: + text = path.read_text(encoding="utf-8") + except OSError as error: + print(f"{path}: {error}", file=sys.stderr) + return 1 + output_path = Path(temp_dir) / f"{index}-{path.name}" + output_path.write_text( + materialize_integer_placeholders(text), encoding="utf-8" + ) + materialized_paths.append(output_path) + + result = subprocess.run( + [str(args.validator), *(str(path) for path in materialized_paths)], + check=False, + ) + if result.returncode != 0: + return result.returncode + + if not args.paths: + try: + compatibility_case = zero_id_multi_interface_case() + invalid_cases = {**semantic_invalid_cases(), **parser_invalid_cases()} + except (OSError, ValueError) as error: + print(error, file=sys.stderr) + return 1 + compatibility_path = Path(temp_dir) / "zero-flow-id-per-interface.yaml" + compatibility_path.write_text(compatibility_case, encoding="utf-8") + result = subprocess.run( + [str(args.validator), str(compatibility_path)], check=False + ) + if result.returncode != 0: + print( + "zero-flow-id-per-interface.yaml: expected validator exit status 0, " + f"got {result.returncode}", + file=sys.stderr, + ) + return 1 + compatibility_count = 1 + invalid_count = len(invalid_cases) + for name, text in invalid_cases.items(): + invalid_path = Path(temp_dir) / name + invalid_path.write_text(text, encoding="utf-8") + result = subprocess.run( + [str(args.validator), str(invalid_path)], check=False + ) + if result.returncode != 1: + print( + f"{name}: expected validator exit status 1, " + f"got {result.returncode}", + file=sys.stderr, + ) + return 1 + + compatibility_label = "case" if compatibility_count == 1 else "cases" + print( + f"Validated {len(materialized_paths)} checked-in configurations and " + f"{compatibility_count} compatibility {compatibility_label}; rejected " + f"{invalid_count} invalid configurations without hardware initialization." + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/check_pr.sh b/scripts/check_pr.sh index f6474b08..1a3c6a62 100755 --- a/scripts/check_pr.sh +++ b/scripts/check_pr.sh @@ -7,6 +7,7 @@ run_docker_base=0 run_diagrams=0 PYTHON="${PYTHON:-python3}" VENV="${VENV:-.venv}" +DAQIRI_CONFIG_VALIDATOR="${DAQIRI_CONFIG_VALIDATOR:-build/tools/daqiri_config_validate}" usage() { cat <<'USAGE' @@ -14,8 +15,11 @@ Usage: scripts/check_pr.sh [--diagrams] [--docker-base] Runs the standard local checks before opening a PR: - portable Python tests + - checked-in configuration validation - documentation build and documentation reference checks +Set DAQIRI_CONFIG_VALIDATOR to the validator built in the project container. + Options: --diagrams Force regeneration of packet diagram assets before checking docs. --docker-base Also build the Docker base stage. @@ -54,6 +58,13 @@ export PYTHON VENV scripts/check_portable_tests.sh +if [ ! -x "${DAQIRI_CONFIG_VALIDATOR}" ]; then + echo "ERROR: configuration validator not found at '${DAQIRI_CONFIG_VALIDATOR}'" >&2 + echo "Build daqiri_config_validate in the project container or set DAQIRI_CONFIG_VALIDATOR." >&2 + exit 1 +fi +"${PYTHON}" scripts/check_daqiri_configs.py --validator "${DAQIRI_CONFIG_VALIDATOR}" + docs_args=() if [ "${run_diagrams}" -eq 1 ]; then docs_args+=(--diagrams) diff --git a/src/engine.h b/src/engine.h index 98c455e6..b2e970c9 100644 --- a/src/engine.h +++ b/src/engine.h @@ -35,8 +35,8 @@ struct rte_pktmbuf_extmem; namespace daqiri { -// Run the hardware-independent semantic checks shared by every engine. -// This does not allocate memory, initialize a runtime, or touch a NIC. +// Apply the common semantic checks used before engine initialization. This path +// does not allocate packet memory, initialize an engine, or access a NIC. bool validate_network_config(const NetworkConfig& config); struct AllocRegion { From 308a187475c500a69d7d526e72d69e60bba0c339 Mon Sep 17 00:00:00 2001 From: Denis Leshchev Date: Tue, 22 Sep 2026 15:07:02 -0400 Subject: [PATCH 3/7] #315 - Enforce common initialization validation Signed-off-by: Denis Leshchev --- CMakeLists.txt | 5 ++ src/engines/rdma/daqiri_rdma_engine.cpp | 4 ++ src/engines/rdma/daqiri_rdma_engine.h | 1 - src/engines/socket/daqiri_socket_engine.cpp | 5 ++ tests/cpp/CMakeLists.txt | 25 +++++++++ tests/cpp/daqiri_init_validation_test.cpp | 59 +++++++++++++++++++++ 6 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 tests/cpp/CMakeLists.txt create mode 100644 tests/cpp/daqiri_init_validation_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index d3a7b44b..4b589f0e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,6 +29,7 @@ project(daqiri VERSION ${DAQIRI_PROJECT_VERSION} LANGUAGES C CXX CUDA) include(GNUInstallDirs) include(CMakePackageConfigHelpers) +include(CTest) find_package(CUDAToolkit REQUIRED) set(DAQIRI_ABI_VERSION "2" CACHE STRING "DAQIRI shared library ABI version") @@ -57,6 +58,10 @@ set(DAQIRI_ENGINE "dpdk ibverbs" CACHE STRING "Optional engine implementations t add_subdirectory(src) add_subdirectory(tools) +if(BUILD_TESTING) + add_subdirectory(tests/cpp) +endif() + if(TARGET yaml-cpp) install( TARGETS yaml-cpp diff --git a/src/engines/rdma/daqiri_rdma_engine.cpp b/src/engines/rdma/daqiri_rdma_engine.cpp index a9168cc1..deb88552 100644 --- a/src/engines/rdma/daqiri_rdma_engine.cpp +++ b/src/engines/rdma/daqiri_rdma_engine.cpp @@ -112,6 +112,10 @@ bool RdmaEngine::set_config_and_initialize(const NetworkConfig& cfg) { DAQIRI_LOG_INFO("Setting up RDMA engine"); rdma_force_quit.store(false, std::memory_order_relaxed); cfg_ = cfg; + if (!validate_config()) { + DAQIRI_LOG_CRITICAL("Config validation failed"); + return false; + } initialize(); return initialized_; diff --git a/src/engines/rdma/daqiri_rdma_engine.h b/src/engines/rdma/daqiri_rdma_engine.h index 9689c6aa..fa9bfeaf 100644 --- a/src/engines/rdma/daqiri_rdma_engine.h +++ b/src/engines/rdma/daqiri_rdma_engine.h @@ -157,7 +157,6 @@ class RdmaEngine : public Engine { Status get_mac_addr(int port, char* mac) override { return Status::SUCCESS; } void shutdown() override; void print_stats() override; - bool validate_config() const override { return true; } // RDMA-specific functions Status rdma_connect_to_server(const std::string& dst_addr, uint16_t dst_port, diff --git a/src/engines/socket/daqiri_socket_engine.cpp b/src/engines/socket/daqiri_socket_engine.cpp index 2fa079f5..a8f1e127 100644 --- a/src/engines/socket/daqiri_socket_engine.cpp +++ b/src/engines/socket/daqiri_socket_engine.cpp @@ -142,6 +142,11 @@ bool SocketEngine::apply_socket_int_option(int fd, bool SocketEngine::set_config_and_initialize(const NetworkConfig& cfg) { cfg_ = cfg; + if (!Engine::validate_config()) { + DAQIRI_LOG_CRITICAL("Config validation failed"); + return false; + } + for (size_t i = 0; i < cfg_.ifs_.size(); ++i) { cfg_.ifs_[i].port_id_ = static_cast(i); } diff --git a/tests/cpp/CMakeLists.txt b/tests/cpp/CMakeLists.txt new file mode 100644 index 00000000..e43ba454 --- /dev/null +++ b/tests/cpp/CMakeLists.txt @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +add_executable(daqiri_init_validation_test daqiri_init_validation_test.cpp) +target_compile_features(daqiri_init_validation_test PRIVATE cxx_std_17) +target_include_directories(daqiri_init_validation_test PRIVATE ${PROJECT_SOURCE_DIR}) + +if(NOT BUILD_SHARED_LIBS AND UNIX AND NOT APPLE) + set(DAQIRI_TEST_STATIC_LIBS daqiri::daqiri) + foreach(engine IN ITEMS dpdk socket rdma ibverbs) + if(TARGET daqiri_${engine}) + list(APPEND DAQIRI_TEST_STATIC_LIBS daqiri_${engine}) + endif() + endforeach() + target_link_libraries(daqiri_init_validation_test PRIVATE + "-Wl,--start-group" + ${DAQIRI_TEST_STATIC_LIBS} + "-Wl,--end-group" + ${DAQIRI_YAML_TARGET}) +else() + target_link_libraries(daqiri_init_validation_test PRIVATE daqiri::daqiri + ${DAQIRI_YAML_TARGET}) +endif() + +add_test(NAME daqiri_init_validation_test COMMAND daqiri_init_validation_test) diff --git a/tests/cpp/daqiri_init_validation_test.cpp b/tests/cpp/daqiri_init_validation_test.cpp new file mode 100644 index 00000000..d9e313de --- /dev/null +++ b/tests/cpp/daqiri_init_validation_test.cpp @@ -0,0 +1,59 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include + +int main() { + const std::string config = R"yaml(%YAML 1.2 +--- +daqiri: + cfg: + version: 1 + stream_type: socket + master_core: 0 + memory_regions: + - name: DATA + kind: host + affinity: 0 + num_bufs: 1 + buf_size: 2048 + interfaces: + - name: udp + address: 127.0.0.1 + socket_config: + mode: client + remote_addr: udp://127.0.0.1:9 + max_payload_size: 2048 + rx: + queues: + - name: RX + id: 0 + cpu_core: -1 + batch_size: 1 + memory_regions: + - MISSING +)yaml"; + + const daqiri::Status status = daqiri::daqiri_init_from_yaml_string(config); + if (status == daqiri::Status::SUCCESS) { + daqiri::shutdown(); + return 1; + } + return status == daqiri::Status::INTERNAL_ERROR ? 0 : 1; +} From 855044d3fe9a6c1f82b07fdb47078107f2ec4fe6 Mon Sep 17 00:00:00 2001 From: Denis Leshchev Date: Tue, 22 Sep 2026 16:20:41 -0400 Subject: [PATCH 4/7] #315 - Document configuration validation tests Signed-off-by: Denis Leshchev --- AGENTS.md | 10 +++++++++- CONTRIBUTING.md | 3 ++- docs/getting-started.md | 1 + tests/README.md | 17 +++++++++++++---- 4 files changed, 25 insertions(+), 6 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 449d7dcc..e8783e3c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,6 +24,8 @@ CMake options (full table in `docs/getting-started.md`): - `DAQIRI_BUILD_PYTHON` — builds `pybind11` bindings from `python/`. - `DAQIRI_BUILD_EXAMPLES` — builds the benchmark executables (default `ON`). The hardware-free `daqiri_config_validate` tool is always built and installed. +- `BUILD_TESTING` — builds and registers the hardware-free C++ tests under + `tests/cpp/` with CTest (default `ON`). Set it to `OFF` to omit test targets. - `DAQIRI_BUILD_APPLICATIONS` — builds the end-to-end example applications under `applications/` (default `OFF`; requires TensorRT, e.g. the `BASE_IMAGE=torch` container). Currently builds `applications/resnet50_inference/` (DAQIRI → TensorRT ResNet inference). - `DAQIRI_ENABLE_OTEL_METRICS` — enables OpenTelemetry metrics instrumentation (default `OFF`). - `DAQIRI_REORDER_GPU_PROFILE` — enable CUDA event timing in the DPDK reorder kernels (off by default). @@ -53,7 +55,13 @@ without initializing hardware: python3 scripts/check_daqiri_configs.py --validator build/tools/daqiri_config_validate ``` -The default suite collects only `tests/portable/`. Future build-backed C++ tests live under `tests/cpp/`; Python-binding tests live under `tests/bindings/` and require a container built with `DAQIRI_BUILD_PYTHON=ON`. Platform tests live under `tests/platform/` and are selected by CI/CD jobs running on provisioned GPU/NIC systems; they are never part of the default pytest collection. The project container already includes the current test packages; use the container-specific dependency command in `tests/README.md` when `tests/requirements.txt` changes. +The default pytest suite collects only `tests/portable/`. Build-backed C++ tests live under +`tests/cpp/` and run through CTest; Python-binding tests live under `tests/bindings/` and +require a container built with `DAQIRI_BUILD_PYTHON=ON`. Platform tests live under +`tests/platform/` and are selected by CI/CD jobs running on provisioned GPU/NIC systems; +they are never part of the default pytest collection. The project container already includes +the current test packages; use the container-specific dependency command in +`tests/README.md` when `tests/requirements.txt` changes. Integration and performance verification is done via the benchmark executables in `examples/`, driven by YAML configs. Build outputs (`examples/CMakeLists.txt:59-71`): diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4e334061..3f0fae86 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -81,7 +81,8 @@ python3 -m venv .venv The default invocation collects only `tests/portable/`. Tests importing the compiled `daqiri` module belong under `tests/bindings/` and require a container built with -`DAQIRI_BUILD_PYTHON=ON`. Future C++ tests belong under `tests/cpp/`. Tests requiring +`DAQIRI_BUILD_PYTHON=ON`. Build-backed C++ tests belong under `tests/cpp/` and run +through CTest. Tests requiring a GPU, NIC, hugepages, privileged access, or a particular host topology belong under `tests/platform/` and are selected by dedicated CI/CD jobs on compatible provisioned runners. See `tests/README.md` for the container dependency command, supported diff --git a/docs/getting-started.md b/docs/getting-started.md index 50378566..8640ad50 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -214,6 +214,7 @@ DAQIRI's shared-library ABI version is tracked separately through | `DAQIRI_ENGINE` | `"dpdk ibverbs"` | Space-separated list of optional engine implementations to compile in. Valid values: `dpdk` (Raw Ethernet) and `ibverbs`. `ibverbs` builds two libibverbs-based engines: RDMA/RoCE (for `stream_type: "socket"` with `roce://` endpoints) and the default Mellanox/mlx5 Multi-Packet (striding) Receive Queue engine for `stream_type: "raw"`. Set `engine: "dpdk"` on a raw stream to select the compiled DPDK implementation instead. Linux UDP/TCP sockets are always built in, so there is no `socket` value. | | `DAQIRI_BUILD_PYTHON` | `OFF` | Build pybind11 Python bindings. | | `DAQIRI_BUILD_EXAMPLES` | `ON` | Build benchmark executables. The `daqiri_config_validate` tool is always built and installed, including when this option is `OFF`. | +| `BUILD_TESTING` | `ON` | Build and register the hardware-free C++ tests under `tests/cpp/` with CTest. Set this to `OFF` to omit test targets from a production-only build. | | `DAQIRI_ENABLE_GDS` | `OFF` | Enable cuFile-backed burst file writes from CUDA device memory. Host-memory writes use POSIX APIs without GDS. | | `DAQIRI_ENABLE_OTEL_METRICS` | `OFF` | Enable OpenTelemetry C++ metrics instrumentation. When enabled, OpenTelemetry C++ API package metadata must be available to CMake. | | `DAQIRI_ENABLE_S3` | `OFF` | Enable AWS SDK-backed asynchronous raw packet writes to S3. | diff --git a/tests/README.md b/tests/README.md index b35e2cb4..e33655e7 100644 --- a/tests/README.md +++ b/tests/README.md @@ -40,10 +40,19 @@ added by the feature that needs them. ## C++ tests -Future C++ unit and build-backed integration tests belong under `tests/cpp/`. They -should be registered with CTest and use a native C++ test framework where appropriate. -CTest can also register the binding and platform pytest commands so a configured build -has one test entry point without making pytest the C++ unit-test framework. +C++ unit and build-backed integration tests live under `tests/cpp/` and are registered +with CTest when `BUILD_TESTING=ON` (the default). The current +`daqiri_init_validation_test` exercises common semantic validation through the production +socket initialization path and verifies rejection before socket resources are initialized. +After building in the required project container, run: + +```bash +ctest --test-dir build --output-on-failure +``` + +Use a native C++ test framework where appropriate. CTest can also register binding and +platform pytest commands so a configured build has one test entry point without making +pytest the C++ unit-test framework. ## Python-binding tests From 188886bff4c0caebbc9d51deb61fc8e7ff850562 Mon Sep 17 00:00:00 2001 From: Denis Leshchev Date: Tue, 22 Sep 2026 16:29:01 -0400 Subject: [PATCH 5/7] #315 - Complete validator documentation mapping Signed-off-by: Denis Leshchev --- AGENTS.md | 2 +- docs/api-reference/cpp.md | 7 +++++++ docs/api-reference/index.md | 7 +++++++ docs/api-reference/python.md | 6 ++++++ docs/concepts.md | 13 +++++++++++++ docs/tutorials/configuration-walkthrough.md | 6 ++++++ 6 files changed, 40 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index e8783e3c..c98b60cd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -128,7 +128,7 @@ clang-format -style=file -i -fallback-style=none ### Engine abstraction `src/engine.h` defines `daqiri::Engine` — an (almost) ABC with ~50 virtual methods covering init, RX/TX burst dequeue/enqueue, header-fill helpers, buffer free, socket connection helpers, runtime TCP/UDP `setsockopt` passthrough, and RDMA connection setup. Engines live in `src/engines//` (`dpdk/`, `rdma/`, `socket/`, `ibverbs/`). `DAQIRI_ENGINE` selects the optional `dpdk` and `ibverbs` engines at CMake configure time; the `socket` engine is always built. The user-facing value `ibverbs` builds **two** internal engines that both use libibverbs: `rdma` (`src/engines/rdma/`, `DAQIRI_ENGINE_RDMA`, RoCE/InfiniBand for socket `roce://`) and `ibverbs` (`src/engines/ibverbs/`, `DAQIRI_ENGINE_IBVERBS`, the pure-DevX MPRQ raw-Ethernet engine). Each engine produces its own static library (`daqiri_dpdk`, `daqiri_rdma`, `daqiri_socket`, `daqiri_ibverbs`) linked into `daqiri_common`, and each adds a `DAQIRI_ENGINE_=1` compile definition. -`EngineType` (`include/daqiri/types.h`) is resolved from `(stream_type, engine)`: `raw` defaults to `EngineType::IBVERBS` when that engine is built (falling back to `EngineType::DPDK` in DPDK-only builds); `raw` + `engine: "dpdk"` explicitly selects `EngineType::DPDK`; `socket` + a `roce://` endpoint (or `engine: "ibverbs"`) selects `EngineType::RDMA`. The stream-aware `config_engine_from_string(str, stream_type)` overload encodes the `ibverbs`→`{IBVERBS for raw, RDMA for socket}` split. `EngineFactory` (in `engine.h`) is a singleton that instantiates the active engine. `daqiri_init(...)` resolves which engine to use from the `NetworkConfig` and then delegates everything through the `Engine` vtable. There is only ever **one** active `Engine` per process. +`EngineType` (`include/daqiri/types.h`) is resolved from `(stream_type, engine)`: `raw` defaults to `EngineType::IBVERBS` when that engine is built (falling back to `EngineType::DPDK` in DPDK-only builds); `raw` + `engine: "dpdk"` explicitly selects `EngineType::DPDK`; `socket` + a `roce://` endpoint (or `engine: "ibverbs"`) selects `EngineType::RDMA`. The stream-aware `config_engine_from_string(str, stream_type)` overload encodes the `ibverbs`→`{IBVERBS for raw, RDMA for socket}` split. `EngineFactory` (in `engine.h`) is a singleton that instantiates the active engine. `daqiri_init(...)` resolves which engine to use from the `NetworkConfig`, runs the shared hardware-independent semantic validation, and only then delegates initialization through the `Engine` vtable. The standalone `daqiri_config_validate` tool calls the same parser and shared checks without creating an engine. There is only ever **one** active `Engine` per process. The always-built socket engine implements Linux UDP/TCP streams directly. Applications that need kernel socket tuning call `socket_setsockopt(conn_id, level, optname, optval, optlen)` after resolving a TCP/UDP connection ID; DAQIRI passes the numeric Linux constants through without maintaining a symbolic option map. `socket_setsockopt` is not supported for `roce://` connections, which delegate to the RDMA/ibverbs path. diff --git a/docs/api-reference/cpp.md b/docs/api-reference/cpp.md index b1296add..a0f153cc 100644 --- a/docs/api-reference/cpp.md +++ b/docs/api-reference/cpp.md @@ -48,6 +48,13 @@ daqiri::NetworkConfig config; auto status = daqiri::daqiri_init(config); ``` +The YAML overloads use the production parser, and every initialization overload applies the shared +common semantic checks before allocating packet memory or initializing a transport engine. A +failed check returns a non-success status before hardware resources are touched. Application +startup performs this validation automatically; use the installed `daqiri_config_validate` +command only when a hardware-free preflight is useful. See +[Validate without hardware initialization](configuration.md#validate-without-hardware-initialization). + After `daqiri_init()` returns `Status::SUCCESS`, all memory regions are allocated, NIC queues are configured, and worker threads are running. diff --git a/docs/api-reference/index.md b/docs/api-reference/index.md index 24975f41..1b307f32 100644 --- a/docs/api-reference/index.md +++ b/docs/api-reference/index.md @@ -25,6 +25,13 @@ queues, memory regions, flow steering rules, flow isolation, hardware flow transform actions, header-data split, and optional reorder plans. After initialization, the language API operates on that topology and, where supported, can extend it explicitly at runtime. +The YAML `daqiri_init` entry points use the production parser, and every initialization path +applies the shared common semantic checks before allocating memory or initializing a transport +engine. Application startup remains the normal validation path. For CI, batch checks, or +development without target hardware, the installed `daqiri_config_validate` command runs the same +parser and common checks without creating an engine; see the +[configuration reference](configuration.md#validate-without-hardware-initialization). + The APIs do **not** discover queues, memory, or flow steering rules on their own. The startup configuration remains the source of truth for stream-type, engine, endpoint selection, and immutable static flows. Applications may then diff --git a/docs/api-reference/python.md b/docs/api-reference/python.md index 98db538d..1b034b41 100644 --- a/docs/api-reference/python.md +++ b/docs/api-reference/python.md @@ -83,6 +83,12 @@ or a config-like object that provides `as_dict()`. - `daqiri.NetworkConfig` instance - config-like object with a `value` attribute or `as_dict()` method +The YAML and dictionary forms use the production parser, and every form is checked for common +semantic errors before packet memory or transport resources are initialized. This happens +automatically during normal application startup. For a hardware-free preflight outside the Python +process, use the installed `daqiri_config_validate` command described in the +[configuration reference](configuration.md#validate-without-hardware-initialization). + ```python import daqiri diff --git a/docs/concepts.md b/docs/concepts.md index 14e81646..3075504f 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -110,6 +110,19 @@ sockets), see [Choosing an example config](tutorials/configuration-walkthrough.md#choosing-an-example-config) in the configuration walkthrough. +### Configuration validation + +Application startup is the authority for configuration acceptance. The YAML `daqiri_init()` entry +points use the production parser, and every initialization path applies hardware-independent +common semantic checks before it allocates packet memory or initializes the selected engine. +Engine- and device-specific capability checks still occur during initialization because they +depend on the compiled engines and available hardware. + +The installed `daqiri_config_validate` command is an optional hardware-free preflight for CI, +batch validation, or development away from the target system. It runs the same parser and common +checks as initialization, but it does not claim that a particular NIC can program every requested +feature. See the [Configuration YAML Reference](api-reference/configuration.md#validate-without-hardware-initialization). + ??? example "Support and testing" The DAQIRI library integration testing infrastructure is under active diff --git a/docs/tutorials/configuration-walkthrough.md b/docs/tutorials/configuration-walkthrough.md index c5dc88ff..8e3a9d63 100644 --- a/docs/tutorials/configuration-walkthrough.md +++ b/docs/tutorials/configuration-walkthrough.md @@ -19,6 +19,12 @@ If you don't have any NIC at all, the `*_sw_loopback*` variants of the Raw Ether (`DAQIRI_ENGINE` at the CMake layer selects which optional engine implementations to compile in. `dpdk` enables the default raw engine, while `ibverbs` enables both the pure-DevX raw engine and `roce://` endpoints. Linux UDP/TCP sockets are always built in. The default build is `dpdk ibverbs`.) +The checked-in examples show how the pieces fit together, and `daqiri_init()` validates the chosen +configuration automatically when the application starts. You do not need a separate validation +step for normal use. The installed `daqiri_config_validate` command is available when CI or an +offline workflow needs to run the same parser and common semantic checks without initializing +hardware; see [Validate without hardware initialization](../api-reference/configuration.md#validate-without-hardware-initialization). + For a shorter selection guide, start with the [Benchmarking overview](../benchmarks/index.md). With a stream type in mind, read down the questions below and stop at the first one that matches what you're trying to do. Each section names the YAML, the binary that consumes it, and any platform-specific notes. ??? question "1. I want to measure baseline throughput" From 6be5a13266cbace91765b10270721110d5bf8b56 Mon Sep 17 00:00:00 2001 From: Denis Leshchev Date: Mon, 28 Sep 2026 17:54:20 +0000 Subject: [PATCH 6/7] #315 - Simplify configuration validation documentation Signed-off-by: Denis Leshchev --- README.md | 3 --- docs/api-reference/configuration.md | 7 ++++--- docs/api-reference/cpp.md | 11 ++--------- docs/api-reference/index.md | 7 ------- docs/api-reference/python.md | 6 ------ docs/concepts.md | 13 ------------- docs/tutorials/configuration-walkthrough.md | 7 ++----- 7 files changed, 8 insertions(+), 46 deletions(-) diff --git a/README.md b/README.md index 9d120d5e..2117ef1f 100644 --- a/README.md +++ b/README.md @@ -27,9 +27,6 @@ DAQIRI provides direct NIC hardware access in userspace, bypassing the Linux ker ## Features -- **Hardware-free configuration validation** — The installed - `daqiri_config_validate` tool uses DAQIRI's production parser and common semantic checks - without allocating packet memory, initializing an engine, or accessing a NIC. - **High Throughput** — Sustained line rate with proper hardware and tuning. - **Low Latency** — Direct access to NIC ring buffers; most latency is PCIe transit only. - **Explicit hugepage allocation** — `kind: huge` always means hugetlb-backed memory for diff --git a/docs/api-reference/configuration.md b/docs/api-reference/configuration.md index f83f04e2..f042bb3d 100644 --- a/docs/api-reference/configuration.md +++ b/docs/api-reference/configuration.md @@ -14,9 +14,10 @@ See `examples/daqiri_bench_*.yaml` for complete working examples. ## Validate without hardware initialization -`daqiri_config_validate` parses one or more YAML files and applies the same common semantic -checks used before `daqiri_init()`. It does not allocate packet memory, initialize CUDA or a -network engine, or access a NIC: +Use `daqiri_config_validate` to check YAML files before running an application, such as in CI or +on a machine without the target NIC. `daqiri_init()` performs these checks during startup. The +command cannot determine whether the selected engine and hardware support every requested +setting. ```bash daqiri_config_validate config.yaml another-config.yaml diff --git a/docs/api-reference/cpp.md b/docs/api-reference/cpp.md index a0f153cc..d09c610b 100644 --- a/docs/api-reference/cpp.md +++ b/docs/api-reference/cpp.md @@ -48,15 +48,8 @@ daqiri::NetworkConfig config; auto status = daqiri::daqiri_init(config); ``` -The YAML overloads use the production parser, and every initialization overload applies the shared -common semantic checks before allocating packet memory or initializing a transport engine. A -failed check returns a non-success status before hardware resources are touched. Application -startup performs this validation automatically; use the installed `daqiri_config_validate` -command only when a hardware-free preflight is useful. See -[Validate without hardware initialization](configuration.md#validate-without-hardware-initialization). - -After `daqiri_init()` returns `Status::SUCCESS`, all memory regions are allocated, NIC -queues are configured, and worker threads are running. +`daqiri_init()` checks the configuration, prepares packet memory, and starts the selected engine. +It returns `Status::SUCCESS` when DAQIRI is ready. `MemoryKind::HUGE` is an explicit hugetlb request for DAQIRI-owned memory. DAQIRI does not substitute regular or transparent-hugepage memory when the requested hugetlb allocation is diff --git a/docs/api-reference/index.md b/docs/api-reference/index.md index 1b307f32..24975f41 100644 --- a/docs/api-reference/index.md +++ b/docs/api-reference/index.md @@ -25,13 +25,6 @@ queues, memory regions, flow steering rules, flow isolation, hardware flow transform actions, header-data split, and optional reorder plans. After initialization, the language API operates on that topology and, where supported, can extend it explicitly at runtime. -The YAML `daqiri_init` entry points use the production parser, and every initialization path -applies the shared common semantic checks before allocating memory or initializing a transport -engine. Application startup remains the normal validation path. For CI, batch checks, or -development without target hardware, the installed `daqiri_config_validate` command runs the same -parser and common checks without creating an engine; see the -[configuration reference](configuration.md#validate-without-hardware-initialization). - The APIs do **not** discover queues, memory, or flow steering rules on their own. The startup configuration remains the source of truth for stream-type, engine, endpoint selection, and immutable static flows. Applications may then diff --git a/docs/api-reference/python.md b/docs/api-reference/python.md index 1b034b41..98db538d 100644 --- a/docs/api-reference/python.md +++ b/docs/api-reference/python.md @@ -83,12 +83,6 @@ or a config-like object that provides `as_dict()`. - `daqiri.NetworkConfig` instance - config-like object with a `value` attribute or `as_dict()` method -The YAML and dictionary forms use the production parser, and every form is checked for common -semantic errors before packet memory or transport resources are initialized. This happens -automatically during normal application startup. For a hardware-free preflight outside the Python -process, use the installed `daqiri_config_validate` command described in the -[configuration reference](configuration.md#validate-without-hardware-initialization). - ```python import daqiri diff --git a/docs/concepts.md b/docs/concepts.md index 3075504f..14e81646 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -110,19 +110,6 @@ sockets), see [Choosing an example config](tutorials/configuration-walkthrough.md#choosing-an-example-config) in the configuration walkthrough. -### Configuration validation - -Application startup is the authority for configuration acceptance. The YAML `daqiri_init()` entry -points use the production parser, and every initialization path applies hardware-independent -common semantic checks before it allocates packet memory or initializes the selected engine. -Engine- and device-specific capability checks still occur during initialization because they -depend on the compiled engines and available hardware. - -The installed `daqiri_config_validate` command is an optional hardware-free preflight for CI, -batch validation, or development away from the target system. It runs the same parser and common -checks as initialization, but it does not claim that a particular NIC can program every requested -feature. See the [Configuration YAML Reference](api-reference/configuration.md#validate-without-hardware-initialization). - ??? example "Support and testing" The DAQIRI library integration testing infrastructure is under active diff --git a/docs/tutorials/configuration-walkthrough.md b/docs/tutorials/configuration-walkthrough.md index 8e3a9d63..0db2f3c2 100644 --- a/docs/tutorials/configuration-walkthrough.md +++ b/docs/tutorials/configuration-walkthrough.md @@ -19,11 +19,8 @@ If you don't have any NIC at all, the `*_sw_loopback*` variants of the Raw Ether (`DAQIRI_ENGINE` at the CMake layer selects which optional engine implementations to compile in. `dpdk` enables the default raw engine, while `ibverbs` enables both the pure-DevX raw engine and `roce://` endpoints. Linux UDP/TCP sockets are always built in. The default build is `dpdk ibverbs`.) -The checked-in examples show how the pieces fit together, and `daqiri_init()` validates the chosen -configuration automatically when the application starts. You do not need a separate validation -step for normal use. The installed `daqiri_config_validate` command is available when CI or an -offline workflow needs to run the same parser and common semantic checks without initializing -hardware; see [Validate without hardware initialization](../api-reference/configuration.md#validate-without-hardware-initialization). +The example configs show the DAQIRI fields and application settings used for different transport +and hardware setups. For a shorter selection guide, start with the [Benchmarking overview](../benchmarks/index.md). With a stream type in mind, read down the questions below and stop at the first one that matches what you're trying to do. Each section names the YAML, the binary that consumes it, and any platform-specific notes. From 703fc352f1298fe6100124c0fedde2787dbfe39b Mon Sep 17 00:00:00 2001 From: Denis Leshchev Date: Mon, 28 Sep 2026 20:14:58 +0000 Subject: [PATCH 7/7] #315 - Document configuration validation coverage Signed-off-by: Denis Leshchev --- CONTRIBUTING.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3f0fae86..f648ed49 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -93,6 +93,10 @@ Build `daqiri_config_validate` in the required project container before running through the production C++ parser and hardware-independent semantic checks. Set `DAQIRI_CONFIG_VALIDATOR` when the executable is not at `build/tools/daqiri_config_validate`. +When a new example config exercises a configuration form these cases do not cover, add a +representative case to `scripts/check_daqiri_configs.py`. Add a focused fixture for any new +validation rule. + #### Pull Requests Developer workflow for code contributions is as follows: