feat: experimental MVP of Comet Native Executors

.github/actions/rust-test/action.yaml

@@ -45,6 +45,13 @@ runs:
         cd native
         cargo clippy --color=never -p datafusion-comet --all-targets --features python-udf -- -D warnings
 
+    # The JNI client for Comet Native Executors is off by default, so nothing above compiles it.
+    - name: Check native executors feature
+      shell: bash
+      run: |
+        cd native
+        cargo clippy --color=never -p datafusion-comet --all-targets --features native-executors -- -D warnings
+
     - name: Check compilation
       shell: bash
       run: |
@@ -106,6 +113,13 @@ runs:
         export LD_LIBRARY_PATH=${JAVA_HOME}/lib/server:${LD_LIBRARY_PATH}
         RUST_BACKTRACE=1 cargo nextest run -p datafusion-comet --lib --features python-udf python_udf
 
+    - name: Run native executors client tests
+      shell: bash
+      run: |
+        cd native
+        export LD_LIBRARY_PATH=${JAVA_HOME}/lib/server:${LD_LIBRARY_PATH}
+        RUST_BACKTRACE=1 cargo nextest run -p datafusion-comet --lib --features native-executors native_executors
+
     # The steps above lint and test the accounting allocator over the system allocator. Lint and
     # test it over jemalloc too, since that is a different allocator backend and nothing else in CI
     # builds it.

dev/ci/check-suites.py

@@ -42,6 +42,7 @@ def file_to_class_name(path: Path) -> str | None:
         "org.apache.comet.CometIcebergSchedulerFailureSuite", # manual, multiple hosts + SIGKILL
         "org.apache.comet.cloud.s3.CometS3CredentialBridgeSuite", # manual test suite
         "org.apache.comet.cloud.s3.HadoopS3ACredentialProviderAdapterBridgeSuite", # manual test suite
+        "org.apache.comet.exec.CometNativeExecutorsSuite", # manual, needs comet-scheduler and comet-executor binaries
         "org.apache.comet.shuffle.CelebornReflectionCompatibilitySuite", # dedicated version matrix
         "org.apache.spark.sql.comet.CometPlanStabilitySuite", # abstract
         "org.apache.spark.sql.comet.ParquetDatetimeRebaseSuite", # abstract

docs/source/contributor-guide/index.md

@@ -51,6 +51,7 @@ Floating-Point Semantics <floating_point>
 JVM Shuffle <jvm_shuffle>
 Native Shuffle <native_shuffle>
 Memory Management <memory_management>
+Comet Native Executors <native_executors>
 Iceberg Writes <iceberg-writes>
 ANSI Error Propagation <sql_error_propagation>
 S3 Credential Provider Design <s3-credential-provider-design>