From 60351badf30c83f5b8e07e731fda1d51470b27f8 Mon Sep 17 00:00:00 2001 From: devoopsman45 Date: Tue, 9 Jun 2026 17:39:25 -0400 Subject: [PATCH 1/4] feat(session): add tableExists and deregisterTable to SessionContext (#102) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? - Closes #101 . ## Rationale for this change SessionContext can let a caller register tables but there is no way to Check whether a table name is already registered Remove a registered table from the session This makes it tough to write safer code while registering ## What changes are included in this PR? - `SessionContext.tableExists(String name)` — returns true if a table with that name is registered in the session - `SessionContext.deregisterTable(String name)` — removes a registered table; no-op if the name is not found Both are thin JNI wrappers over DataFusion's existing `SessionContext::table_exist` and `SessionContext::deregister_table` on the Rust side. ## Are these changes tested? Yes. `SessionContextTableRegistrationTest` covers: - tableExists returns false for an unregistered table - tableExists returns true after registerCsv - deregisterTable removes a registered table - deregisterTable is a no-op for an unregistered table - Both methods throw IllegalStateException on a closed context --> ## Are there any user-facing changes? Yes — two new public methods on `SessionContext`. Additive only, no breaking changes. --- .../org/apache/datafusion/SessionContext.java | 41 +++++++++++ .../apache/datafusion/SessionContextTest.java | 69 +++++++++++++++++++ native/src/lib.rs | 43 ++++++++++++ 3 files changed, 153 insertions(+) diff --git a/core/src/main/java/org/apache/datafusion/SessionContext.java b/core/src/main/java/org/apache/datafusion/SessionContext.java index ffc58dd..ec0bd85 100644 --- a/core/src/main/java/org/apache/datafusion/SessionContext.java +++ b/core/src/main/java/org/apache/datafusion/SessionContext.java @@ -601,6 +601,43 @@ private static byte[] serializeSchemaIpc(Schema schema) { return baos.toByteArray(); } + /** + * Returns {@code true} if a table with the given name is registered in this session. + * + *

This is the Java counterpart to DataFusion's Rust {@code SessionContext::table_exist}. + * + * @throws IllegalStateException if this context is closed. + */ + public boolean tableExists(String name) { + checkOpenSessionContext(); + if (name == null) { + throw new IllegalArgumentException("tableExists name must be non-null"); + } + return tableExists(nativeHandle, name); + } + + /** + * Removes the table with the given name from this session. Does nothing if no table with that + * name is registered. + * + *

This is the Java counterpart to DataFusion's Rust {@code SessionContext::deregister_table}. + * + * @throws IllegalStateException if this context is closed. + */ + public void deregisterTable(String name) { + checkOpenSessionContext(); + if (name == null) { + throw new IllegalArgumentException("deregisterTable name must be non-null"); + } + deregisterTable(nativeHandle, name); + } + + private void checkOpenSessionContext() { + if (nativeHandle == 0) { + throw new IllegalStateException("SessionContext is closed"); + } + } + @Override public void close() { if (nativeHandle != 0) { @@ -664,4 +701,8 @@ private static native void registerScalarUdf( private static native void registerTableNative( long handle, String name, byte[] schemaIpcBytes, TableProvider provider); + + private static native boolean tableExists(long handle, String name); + + private static native void deregisterTable(long handle, String name); } diff --git a/core/src/test/java/org/apache/datafusion/SessionContextTest.java b/core/src/test/java/org/apache/datafusion/SessionContextTest.java index 4465546..8c80e42 100644 --- a/core/src/test/java/org/apache/datafusion/SessionContextTest.java +++ b/core/src/test/java/org/apache/datafusion/SessionContextTest.java @@ -20,6 +20,7 @@ package org.apache.datafusion; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -33,6 +34,7 @@ import org.apache.arrow.vector.ipc.ArrowReader; import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; class SessionContextTest { @Test @@ -90,4 +92,71 @@ void dataFrameCollectTwiceFails() { assertThrows(IllegalStateException.class, () -> df.collect(allocator)); } } + + @Test + void tableExistsReturnsFalseForUnregisteredTable() { + try (SessionContext ctx = new SessionContext()) { + assertFalse(ctx.tableExists("orders")); + } + } + + @Test + void tableExistsReturnsTrueAfterRegisterCsv(@TempDir Path tempDir) throws Exception { + Path csv = tempDir.resolve("orders.csv"); + Files.writeString(csv, "id,amount\n1,100\n2,200\n"); + + try (SessionContext ctx = new SessionContext()) { + assertFalse(ctx.tableExists("orders")); + ctx.registerCsv("orders", csv.toAbsolutePath().toString()); + assertTrue(ctx.tableExists("orders")); + } + } + + @Test + void deregisterTableRemovesRegisteredTable(@TempDir Path tempDir) throws Exception { + Path csv = tempDir.resolve("orders.csv"); + Files.writeString(csv, "id,amount\n1,100\n2,200\n"); + + try (SessionContext ctx = new SessionContext()) { + ctx.registerCsv("orders", csv.toAbsolutePath().toString()); + assertTrue(ctx.tableExists("orders")); + ctx.deregisterTable("orders"); + assertFalse(ctx.tableExists("orders")); + } + } + + @Test + void deregisterTableIsNoOpForUnregisteredTable() { + try (SessionContext ctx = new SessionContext()) { + ctx.deregisterTable("nonexistent"); + } + } + + @Test + void tableExistsThrowsWhenContextIsClosed() { + SessionContext ctx = new SessionContext(); + ctx.close(); + assertThrows(IllegalStateException.class, () -> ctx.tableExists("orders")); + } + + @Test + void deregisterTableThrowsWhenContextIsClosed() { + SessionContext ctx = new SessionContext(); + ctx.close(); + assertThrows(IllegalStateException.class, () -> ctx.deregisterTable("orders")); + } + + @Test + void tableExistsThrowsForNullName() { + try (SessionContext ctx = new SessionContext()) { + assertThrows(IllegalArgumentException.class, () -> ctx.tableExists(null)); + } + } + + @Test + void deregisterTableThrowsForNullName() { + try (SessionContext ctx = new SessionContext()) { + assertThrows(IllegalArgumentException.class, () -> ctx.deregisterTable(null)); + } + } } diff --git a/native/src/lib.rs b/native/src/lib.rs index 4fd7a8a..0b2d89b 100644 --- a/native/src/lib.rs +++ b/native/src/lib.rs @@ -1042,6 +1042,49 @@ pub extern "system" fn Java_org_apache_datafusion_SessionContext_getOptionNative ) } +/// Converts a raw JNI handle into a shared reference to a [`SessionContext`]. +/// +/// # Safety for the unsafe usage +/// The caller must ensure `handle` was produced by `Box::into_raw(Box::new(ctx))` +/// in `createSessionContext` and has not yet been freed. The Java side zeroes +/// `nativeHandle` on `close()`, so the `handle == 0` guard in every JNI handler +/// ensures this invariant holds before `unwrap_context` is called. +fn unwrap_context(handle: jlong) -> JniResult<&'static SessionContext> { + if handle == 0 { + return Err("SessionContext handle is null".into()); + } + Ok(unsafe { &*(handle as *const SessionContext) }) +} + +#[no_mangle] +pub extern "system" fn Java_org_apache_datafusion_SessionContext_tableExists<'local>( + mut env: JNIEnv<'local>, + _class: JClass<'local>, + handle: jlong, + name: JString<'local>, +) -> jboolean { + try_unwrap_or_throw(&mut env, 0, |env| -> JniResult { + let ctx = unwrap_context(handle)?; + let name: String = env.get_string(&name)?.into(); + Ok(ctx.table_exist(&name)? as jboolean) + }) +} + +#[no_mangle] +pub extern "system" fn Java_org_apache_datafusion_SessionContext_deregisterTable<'local>( + mut env: JNIEnv<'local>, + _class: JClass<'local>, + handle: jlong, + name: JString<'local>, +) { + try_unwrap_or_throw(&mut env, (), |env| -> JniResult<()> { + let ctx = unwrap_context(handle)?; + let name: String = env.get_string(&name)?.into(); + ctx.deregister_table(&name)?; + Ok(()) + }) +} + #[no_mangle] pub extern "system" fn Java_org_apache_datafusion_SessionContext_closeSessionContext<'local>( mut env: JNIEnv<'local>, From 051b00b528691691103891f335885afe2c506760 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 10 Jun 2026 13:47:45 -0600 Subject: [PATCH 2/4] feat: add SessionContextBuilder.withSparkFunctions for Spark-compatible functions (#100) --- .../datafusion/SessionContextBuilder.java | 18 +++++ .../datafusion/SessionContextBuilderTest.java | 15 +++++ .../SessionContextSparkFunctionsTest.java | 62 +++++++++++++++++ docs/source/user-guide/sessioncontext.md | 21 ++++++ native/Cargo.lock | 39 +++++++++++ native/Cargo.toml | 10 +++ native/src/lib.rs | 66 ++++++++++++++++++- proto/session_options.proto | 6 ++ 8 files changed, 234 insertions(+), 3 deletions(-) create mode 100644 core/src/test/java/org/apache/datafusion/SessionContextSparkFunctionsTest.java diff --git a/core/src/main/java/org/apache/datafusion/SessionContextBuilder.java b/core/src/main/java/org/apache/datafusion/SessionContextBuilder.java index 81c59de..ef6fd4c 100644 --- a/core/src/main/java/org/apache/datafusion/SessionContextBuilder.java +++ b/core/src/main/java/org/apache/datafusion/SessionContextBuilder.java @@ -44,6 +44,7 @@ public final class SessionContextBuilder { private boolean spillDisabled; private Long maxTempDirectorySize; private CacheManagerOptions cacheManager; + private boolean sparkFunctions; private final LinkedHashMap options = new LinkedHashMap<>(); private final List objectStores = new ArrayList<>(); @@ -227,6 +228,20 @@ public SessionContextBuilder cacheManager(CacheManagerOptions options) { return this; } + /** + * Register Apache Spark-compatible functions and expression planners on the new context, using + * the {@code datafusion-spark} crate. Once enabled, Spark-compatible functions (e.g. {@code + * crc32}) are callable from SQL and override any DataFusion built-in of the same name. + * + *

Requires the native library to be built with the {@code spark} Cargo feature, which is + * enabled in the default build. If it is not, {@link #build()} throws a {@link RuntimeException} + * explaining the feature is missing. + */ + public SessionContextBuilder withSparkFunctions() { + this.sparkFunctions = true; + return this; + } + /** * Register an {@code object_store::ObjectStore} backend on the new context's {@code RuntimeEnv}. * Build {@link ObjectStoreOptions} via the per-backend factories ({@link ObjectStoreOptions#s3}, @@ -309,6 +324,9 @@ byte[] toBytes() { for (Map.Entry e : options.entrySet()) { b.addOptions(ConfigOption.newBuilder().setKey(e.getKey()).setValue(e.getValue()).build()); } + if (sparkFunctions) { + b.setSparkFunctions(true); + } for (ObjectStoreOptions os : objectStores) { b.addObjectStores(os.toRegistration()); } diff --git a/core/src/test/java/org/apache/datafusion/SessionContextBuilderTest.java b/core/src/test/java/org/apache/datafusion/SessionContextBuilderTest.java index b7451ab..93eeac1 100644 --- a/core/src/test/java/org/apache/datafusion/SessionContextBuilderTest.java +++ b/core/src/test/java/org/apache/datafusion/SessionContextBuilderTest.java @@ -426,6 +426,21 @@ void diskManagerFieldIsAbsentWhenNothingSet() throws Exception { assertFalse(parsed.hasDiskManager()); } + @Test + void withSparkFunctionsRoundTripsThroughProto() throws Exception { + byte[] bytes = SessionContext.builder().withSparkFunctions().toBytes(); + SessionOptions parsed = SessionOptions.parseFrom(bytes); + assertTrue(parsed.hasSparkFunctions()); + assertTrue(parsed.getSparkFunctions()); + } + + @Test + void sparkFunctionsAbsentWhenNotRequested() throws Exception { + byte[] bytes = SessionContext.builder().batchSize(8192).toBytes(); + SessionOptions parsed = SessionOptions.parseFrom(bytes); + assertFalse(parsed.hasSparkFunctions()); + } + @Test void tempDirectoryStaysOnLegacyField() throws Exception { // tempDirectory(String) writes the existing SessionOptions.temp_directory diff --git a/core/src/test/java/org/apache/datafusion/SessionContextSparkFunctionsTest.java b/core/src/test/java/org/apache/datafusion/SessionContextSparkFunctionsTest.java new file mode 100644 index 0000000..6359c14 --- /dev/null +++ b/core/src/test/java/org/apache/datafusion/SessionContextSparkFunctionsTest.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.datafusion; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.BigIntVector; +import org.apache.arrow.vector.ipc.ArrowReader; +import org.junit.jupiter.api.Test; + +class SessionContextSparkFunctionsTest { + + // crc32 is a Spark function provided by datafusion-spark and is NOT a core + // DataFusion built-in. Spark's crc32('Spark') == 1557323817 (standard + // CRC-32 of the ASCII bytes), so it doubles as a correctness check. + @Test + void sparkFunctionIsCallableWhenEnabled() throws Exception { + try (BufferAllocator allocator = new RootAllocator(); + SessionContext ctx = SessionContext.builder().withSparkFunctions().build(); + DataFrame df = ctx.sql("SELECT crc32('Spark') AS c"); + ArrowReader reader = df.collect(allocator)) { + assertTrue(reader.loadNextBatch()); + BigIntVector v = (BigIntVector) reader.getVectorSchemaRoot().getVector("c"); + assertEquals(1, v.getValueCount()); + assertEquals(1557323817L, v.get(0)); + } + } + + // Without the flag, crc32 is unregistered, so planning the same SQL must + // fail. This is the proof that withSparkFunctions() is what enabled it. + @Test + void sparkFunctionIsAbsentByDefault() { + try (SessionContext ctx = SessionContext.builder().build()) { + RuntimeException thrown = + assertThrows(RuntimeException.class, () -> ctx.sql("SELECT crc32('Spark')")); + assertTrue( + thrown.getMessage() != null && thrown.getMessage().toLowerCase().contains("crc32"), + "expected error to mention the unknown function, got: " + thrown.getMessage()); + } + } +} diff --git a/docs/source/user-guide/sessioncontext.md b/docs/source/user-guide/sessioncontext.md index 818ee0e..f2408c9 100644 --- a/docs/source/user-guide/sessioncontext.md +++ b/docs/source/user-guide/sessioncontext.md @@ -60,3 +60,24 @@ try (SessionContext ctx = SessionContext.builder() // ... } ``` + +## Spark-compatible functions + +`withSparkFunctions()` registers Apache Spark–compatible functions and +expression planners (from the +[`datafusion-spark`](https://crates.io/crates/datafusion-spark) crate) on the +context: + +```java +try (SessionContext ctx = SessionContext.builder() + .withSparkFunctions() + .build(); + DataFrame df = ctx.sql("SELECT crc32('Spark')")) { + // ... +} +``` + +When enabled, Spark-compatible functions override any DataFusion built-in of +the same name. This requires the native library to be built with the `spark` +Cargo feature, which is enabled in the default build; otherwise `build()` +throws explaining the feature is missing. diff --git a/native/Cargo.lock b/native/Cargo.lock index 8c56280..96d2f9d 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -1307,6 +1307,7 @@ dependencies = [ "async-trait", "datafusion", "datafusion-proto", + "datafusion-spark", "datafusion-substrait", "futures", "jni", @@ -1527,6 +1528,33 @@ dependencies = [ "parking_lot", ] +[[package]] +name = "datafusion-spark" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e059dcf8544da0d6598d0235be3cc29c209094a5976b2e4822e4a2cf91c2b5c5" +dependencies = [ + "arrow", + "bigdecimal", + "chrono", + "crc32fast", + "datafusion", + "datafusion-catalog", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-nested", + "log", + "percent-encoding", + "rand 0.9.4", + "serde_json", + "sha1", + "sha2", + "url", +] + [[package]] name = "datafusion-sql" version = "53.1.0" @@ -3409,6 +3437,17 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + [[package]] name = "sha2" version = "0.10.9" diff --git a/native/Cargo.toml b/native/Cargo.toml index c462408..0362ae6 100644 --- a/native/Cargo.toml +++ b/native/Cargo.toml @@ -38,6 +38,7 @@ default = [ "object-store-aws", "object-store-gcp", "object-store-http", + "spark", ] object-store-aws = ["object_store/aws"] object-store-gcp = ["object_store/gcp"] @@ -67,12 +68,21 @@ protoc = ["datafusion-substrait?/protoc"] # invoked in a build that compiled the feature off, so the Java surface is # unchanged either way. runtime-metrics = ["dep:tokio-metrics"] +# Apache Spark-compatible functions, registered when a caller builds a +# context with `SessionContextBuilder.withSparkFunctions()`. On by default so +# Java callers don't have to think about features; builds that strip it get a +# clear runtime error from the JVM if the flag is requested. +spark = ["dep:datafusion-spark"] [dependencies] arrow = { version = "58", features = ["ffi"] } async-trait = "0.1" datafusion = { version = "53.1.0", features = ["avro"] } datafusion-proto = "53.1.0" +# Apache Spark-compatible functions + expression planners. Optional and +# gated behind the `spark` feature (in the default set). The `core` feature +# of the crate is what exposes `SessionStateBuilderSpark`. +datafusion-spark = { version = "53.1.0", features = ["core"], optional = true } datafusion-substrait = { version = "53.1.0", optional = true } futures = "0.3" jni = "0.21" diff --git a/native/src/lib.rs b/native/src/lib.rs index 0b2d89b..43161d2 100644 --- a/native/src/lib.rs +++ b/native/src/lib.rs @@ -50,7 +50,7 @@ use datafusion::dataframe::DataFrame; use datafusion::dataframe::DataFrameWriteOptions; use datafusion::error::DataFusionError; use datafusion::execution::disk_manager::{DiskManagerBuilder, DiskManagerMode}; -use datafusion::execution::runtime_env::RuntimeEnvBuilder; +use datafusion::execution::runtime_env::{RuntimeEnv, RuntimeEnvBuilder}; use datafusion::execution::SendableRecordBatchStream; use datafusion::logical_expr::Expr; use datafusion::logical_expr::{col, Partitioning, ScalarUDF, Signature, SortExpr}; @@ -117,6 +117,40 @@ fn install_memory_tracker( tracker } +/// Build a `SessionContext` from a prepared config + runtime env. When +/// `spark_functions` is set, register Apache Spark-compatible functions and +/// expression planners via the `datafusion-spark` crate (requires the `spark` +/// Cargo feature); otherwise build the plain context. Returns an error if the +/// caller asked for Spark functions in a build that compiled the feature off. +fn build_session_context( + config: SessionConfig, + runtime_env: Arc, + spark_functions: bool, +) -> JniResult { + if spark_functions { + #[cfg(feature = "spark")] + { + use datafusion::execution::SessionStateBuilder; + use datafusion_spark::SessionStateBuilderSpark; + // `with_spark_features` runs after the default features so Spark + // implementations override the built-ins of the same name. + let state = SessionStateBuilder::new_with_default_features() + .with_config(config) + .with_runtime_env(runtime_env) + .with_spark_features() + .build(); + Ok(SessionContext::new_with_state(state)) + } + #[cfg(not(feature = "spark"))] + { + let _ = (config, runtime_env); + Err("spark Cargo feature is not enabled in this build of datafusion-jni".into()) + } + } else { + Ok(SessionContext::new_with_config_rt(config, runtime_env)) + } +} + #[no_mangle] pub extern "system" fn Java_org_apache_datafusion_SessionContext_createSessionContext<'local>( mut env: JNIEnv<'local>, @@ -225,8 +259,9 @@ pub extern "system" fn Java_org_apache_datafusion_SessionContext_createSessionCo // update two atomics. let tracker = install_memory_tracker(&mut runtime_builder); - let runtime_env = runtime_builder.build()?; - let ctx = SessionContext::new_with_config_rt(config, Arc::new(runtime_env)); + let runtime_env = Arc::new(runtime_builder.build()?); + let ctx = + build_session_context(config, runtime_env, opts.spark_functions.unwrap_or(false))?; // Object-store registrations come last because they need a built // RuntimeEnv to register against. A failure here drops `ctx` (and its @@ -1337,3 +1372,28 @@ pub extern "system" fn Java_org_apache_datafusion_SessionContext_registerTableNa Ok(()) }) } + +#[cfg(all(test, feature = "spark"))] +mod spark_tests { + use super::*; + + #[test] + fn spark_functions_register_crc32() { + let rt = Arc::new(RuntimeEnvBuilder::new().build().unwrap()); + let ctx = build_session_context(SessionConfig::new(), rt, true).unwrap(); + assert!( + ctx.state().scalar_functions().contains_key("crc32"), + "expected Spark crc32 to be registered when spark_functions = true" + ); + } + + #[test] + fn plain_context_has_no_crc32() { + let rt = Arc::new(RuntimeEnvBuilder::new().build().unwrap()); + let ctx = build_session_context(SessionConfig::new(), rt, false).unwrap(); + assert!( + !ctx.state().scalar_functions().contains_key("crc32"), + "crc32 must not be registered without spark_functions" + ); + } +} diff --git a/proto/session_options.proto b/proto/session_options.proto index 516e3c4..cfed21e 100644 --- a/proto/session_options.proto +++ b/proto/session_options.proto @@ -64,6 +64,12 @@ message SessionOptions { // Disk-manager configuration (disable spill, size cap). Unset fields // leave the upstream `RuntimeEnvBuilder` default in place. optional DiskManagerOptions disk_manager = 10; + // Register Apache Spark-compatible functions and expression planners on the + // new context (via the datafusion-spark crate). Requires the native library + // to be built with the `spark` Cargo feature (on in the default build); + // otherwise construction fails with a clear error. Spark functions override + // DataFusion built-ins of the same name. + optional bool spark_functions = 11; } // Disk-manager configuration. The two fields are independent on the wire; From ab57cb91cb64037a6d1e7319e9dd3d40849ed7f9 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Sat, 13 Jun 2026 22:43:23 +0200 Subject: [PATCH 3/4] build: Cargo workspace + native-common extraction (1/6) (#104) --- .cargo/config.toml | 21 ++ .github/workflows/build.yml | 4 +- .github/workflows/lint.yml | 8 +- .gitignore | 1 + native/Cargo.lock => Cargo.lock | 244 +++++++++--------- Cargo.toml | 57 ++++ Makefile | 10 +- core/pom.xml | 4 +- .../org/apache/datafusion/SessionContext.java | 11 +- .../SessionContextRuntimeStatsTest.java | 2 +- .../SessionContextSubstraitTest.java | 2 +- dev/release/build-release.sh | 14 +- .../datafusion-java-rm/build-native-libs.sh | 9 +- dev/release/rat_exclude_files.txt | 2 +- dev/release/verify-release-candidate.sh | 3 +- docs/source/contributor-guide/development.md | 12 +- .../updating-datafusion-version.md | 10 +- native-common/Cargo.toml | 41 +++ native-common/README.md | 37 +++ {native => native-common}/src/errors.rs | 12 +- native-common/src/lib.rs | 98 +++++++ native/Cargo.toml | 50 ++-- native/src/arrow.rs | 2 +- native/src/avro.rs | 2 +- native/src/cache_manager.rs | 2 +- native/src/csv.rs | 2 +- native/src/json.rs | 2 +- native/src/lib.rs | 78 +----- native/src/object_store.rs | 2 +- native/src/proto.rs | 2 +- native/src/runtime_metrics.rs | 6 +- native/src/schema.rs | 2 +- pom.xml | 11 +- 33 files changed, 503 insertions(+), 260 deletions(-) create mode 100644 .cargo/config.toml rename native/Cargo.lock => Cargo.lock (95%) create mode 100644 Cargo.toml create mode 100644 native-common/Cargo.toml create mode 100644 native-common/README.md rename {native => native-common}/src/errors.rs (95%) create mode 100644 native-common/src/lib.rs diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000..d7e0ee2 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Keep Cargo's workspace output out of `target/` so `mvn clean` (which deletes +# the root `target/`) does not nuke the Rust build cache. +[build] +target-dir = "rust-target" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c5db936..da8e65a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -83,8 +83,8 @@ jobs: path: | ~/.cargo/registry ~/.cargo/git - native/target - key: ${{ runner.os }}-cargo-${{ hashFiles('native/Cargo.lock') }} + rust-target + key: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.lock') }} restore-keys: ${{ runner.os }}-cargo- - name: Build native and run tests diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 4cf628f..952bf34 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -54,7 +54,7 @@ jobs: run: ./mvnw -q spotless:check - name: Check Rust formatting - run: cd native && cargo fmt --all -- --check + run: cargo fmt --all -- --check clippy: name: Clippy @@ -81,9 +81,9 @@ jobs: path: | ~/.cargo/registry ~/.cargo/git - native/target - key: ${{ runner.os }}-clippy-${{ hashFiles('native/Cargo.lock') }} + rust-target + key: ${{ runner.os }}-clippy-${{ hashFiles('Cargo.lock') }} restore-keys: ${{ runner.os }}-clippy- - name: Run clippy - run: cd native && cargo clippy --all-targets -- -D warnings + run: cargo clippy --workspace --all-targets -- -D warnings diff --git a/.gitignore b/.gitignore index 719a2a4..25c9216 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ target/ +rust-target/ *.class .idea/ .vscode/ diff --git a/native/Cargo.lock b/Cargo.lock similarity index 95% rename from native/Cargo.lock rename to Cargo.lock index 96d2f9d..dbbfcde 100644 --- a/native/Cargo.lock +++ b/Cargo.lock @@ -98,9 +98,9 @@ dependencies = [ [[package]] name = "ar_archive_writer" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eb93bbb63b9c227414f6eb3a0adfddca591a8ce1e9b60661bb08969b87e340b" +checksum = "4087686b4b0a3427190bae57a1d9a478dbb2d40c5dc1bd6e2b6d797913bdd348" dependencies = [ "object", ] @@ -119,9 +119,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "58.2.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "607e64bb911ee4f90483e044fe78f175989148c2892e659a2cd25429e782ec54" +checksum = "378530e55cd479eda3c14eb345310799717e6f76d0c332041e8487022166b471" dependencies = [ "arrow-arith", "arrow-array", @@ -140,9 +140,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "58.2.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e754319ed8a85d817fe7adf183227e0b5308b82790a737b426c1124626b48118" +checksum = "a0ab212d2c1886e802f51c5212d78ebbcbb0bec980fff9dadc1eb8d45cd0b738" dependencies = [ "arrow-array", "arrow-buffer", @@ -154,9 +154,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "58.2.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841321891f247aa86c6112c80d83d89cb36e0addd020fa2425085b8eb6c3f579" +checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e" dependencies = [ "ahash", "arrow-buffer", @@ -173,9 +173,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "58.2.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f955dfb73fae000425f49c8226d2044dab60fb7ad4af1e24f961756354d996c9" +checksum = "0c6cd424c2693bcdbc150d843dc9d4d137dd2de4782ce6df491ad11a3a0416c0" dependencies = [ "bytes", "half", @@ -185,9 +185,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "58.2.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca5e686972523798f76bef355145bc1ae25a84c731e650268d31ab763c701663" +checksum = "4c5aefb56a2c02e9e2b30746241058b85f8983f0fcff2ba0c6d09006e1cded7f" dependencies = [ "arrow-array", "arrow-buffer", @@ -207,9 +207,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "58.2.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86c276756867fc8186ec380c72c290e6e3b23a1d4fb05df6b1d62d2e62666d48" +checksum = "e94e8cf7e517657a52b91ea1263acf38c4ca62a84655d72458a3359b12ab97de" dependencies = [ "arrow-array", "arrow-cast", @@ -222,9 +222,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "58.2.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3b5846209775b6dc8056d77ff9a032b27043383dd5488abd0b663e265b9373" +checksum = "3c88210023a2bfee1896af366309a3028fc3bcbd6515fa29a7990ee1baa08ee0" dependencies = [ "arrow-buffer", "arrow-schema", @@ -235,9 +235,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "58.2.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd8907ddd8f9fbabf91ec2c85c1d81fe2874e336d2443eb36373595e28b98dd5" +checksum = "238438f0834483703d88896db6fe5a7138b2230debc31b34c0336c2996e3c64f" dependencies = [ "arrow-array", "arrow-buffer", @@ -251,9 +251,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "58.2.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4518c59acc501f10d7dcae397fe12b8db3d81bc7de94456f8a58f9165d6f502" +checksum = "205ca2119e6d679d5c133c6f30e68f027738d95ed948cf77677ea69c7800036b" dependencies = [ "arrow-array", "arrow-buffer", @@ -276,9 +276,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "58.2.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efa70d9d6b1356f1fb9f1f651b84a725b7e0abb93f188cf7d31f14abfa2f2e6f" +checksum = "1bffd8fd2579286a5d63bac898159873e5094a79009940bcb42bbfce4f19f1d0" dependencies = [ "arrow-array", "arrow-buffer", @@ -289,9 +289,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "58.2.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faec88a945338192beffbbd4be0def70135422930caa244ac3cec0cd213b26b4" +checksum = "bab5994731204603c73ba69267616c50f80780774c6bb0476f1f830625115e0c" dependencies = [ "arrow-array", "arrow-buffer", @@ -302,9 +302,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "58.2.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18aa020f6bc8e5201dcd2d4b7f98c68f8a410ef37128263243e6ff2a47a67d4f" +checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed" dependencies = [ "bitflags", "serde_core", @@ -313,9 +313,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "58.2.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a657ab5132e9c8ca3b24eb15a823d0ced38017fe3930ff50167466b02e2d592c" +checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222" dependencies = [ "ahash", "arrow-array", @@ -327,9 +327,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "58.2.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6de2efbbd1a9f9780ceb8d1ff5d20421b35863b361e3386b4f571f1fc69fcb8" +checksum = "29dd7cda3ab9692f43a2e4acc444d760cc17b12bb6d8232ddf64e9bab7c06b42" dependencies = [ "arrow-array", "arrow-buffer", @@ -393,9 +393,9 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "autocfg" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "base64" @@ -419,9 +419,9 @@ dependencies = [ [[package]] name = "bitflags" -version = "2.11.1" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" [[package]] name = "blake2" @@ -457,9 +457,9 @@ dependencies = [ [[package]] name = "bon" -version = "3.9.1" +version = "3.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f47dbe92550676ee653353c310dfb9cf6ba17ee70396e1f7cf0a2020ad49b2fe" +checksum = "b2f04f6fef12d70d42a77b1433c9e0f065238479a6cefc4f5bab105e9873a3c3" dependencies = [ "bon-macros", "rustversion", @@ -467,9 +467,9 @@ dependencies = [ [[package]] name = "bon-macros" -version = "3.9.1" +version = "3.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "519bd3116aeeb42d5372c29d982d16d0170d3d4a5ed85fc7dd91642ffff3c67c" +checksum = "7d0bd4c2f75335ad98052a37efb54f428b492f64340257143b3429c8a508fa7b" dependencies = [ "darling", "ident_case", @@ -482,9 +482,9 @@ dependencies = [ [[package]] name = "brotli" -version = "8.0.2" +version = "8.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bd8b9603c7aa97359dbd97ecf258968c95f3adddd6db2f7e7a5bef101c84560" +checksum = "8119e4516436f5708bbc474a9d395bf12f1b5395e93a92a56e647ac3388c8610" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -493,9 +493,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "5.0.0" +version = "5.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03" +checksum = "5962523e1b92ce1b5e793d9169b9943eece10d39f62550bc04bb605d75b94924" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -503,9 +503,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.20.2" +version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] name = "byteorder" @@ -530,9 +530,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.62" +version = "1.2.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98" +checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f" dependencies = [ "find-msvc-tools", "jobserver", @@ -571,9 +571,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.44" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", "num-traits", @@ -789,9 +789,9 @@ dependencies = [ [[package]] name = "dashmap" -version = "6.1.0" +version = "6.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" dependencies = [ "cfg-if", "crossbeam-utils", @@ -1306,6 +1306,7 @@ dependencies = [ "arrow", "async-trait", "datafusion", + "datafusion-jni-common", "datafusion-proto", "datafusion-spark", "datafusion-substrait", @@ -1320,6 +1321,16 @@ dependencies = [ "url", ] +[[package]] +name = "datafusion-jni-common" +version = "0.1.0" +dependencies = [ + "datafusion", + "futures", + "jni", + "tokio", +] + [[package]] name = "datafusion-macros" version = "53.1.0" @@ -1607,9 +1618,9 @@ dependencies = [ [[package]] name = "displaydoc" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", @@ -1624,9 +1635,9 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" [[package]] name = "either" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" [[package]] name = "equivalent" @@ -1932,9 +1943,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "http" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" dependencies = [ "bytes", "itoa", @@ -1977,9 +1988,9 @@ checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" [[package]] name = "hyper" -version = "1.9.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" dependencies = [ "atomic-waker", "bytes", @@ -2269,13 +2280,12 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.98" +version = "0.3.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08" +checksum = "f2025f20d7a4fa7785846e7b63d10a76d3f1cee98ee5cb79ea59703f95e42162" dependencies = [ "cfg-if", "futures-util", - "once_cell", "wasm-bindgen", ] @@ -2344,9 +2354,9 @@ dependencies = [ [[package]] name = "libbz2-rs-sys" -version = "0.2.3" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3a6a8c165077efc8f3a971534c50ea6a1a18b329ef4a66e897a7e3a1494565f" +checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c" [[package]] name = "libc" @@ -2403,9 +2413,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.29" +version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" [[package]] name = "lru-slab" @@ -2434,9 +2444,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.8.0" +version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" [[package]] name = "miniz_oxide" @@ -2450,9 +2460,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" dependencies = [ "libc", "wasi", @@ -2598,9 +2608,9 @@ dependencies = [ [[package]] name = "parquet" -version = "58.2.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43d7efd3052f7d6ef601085559a246bc991e9a8cc77e02753737df6322ce35f1" +checksum = "5dafa7d01085b62a47dd0c1829550a0a36710ea9c4fe358a05a85477cec8a908" dependencies = [ "ahash", "arrow-array", @@ -2762,9 +2772,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" dependencies = [ "bytes", "prost-derive", @@ -2772,9 +2782,9 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" +checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ "heck", "itertools", @@ -2791,9 +2801,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", "itertools", @@ -2804,9 +2814,9 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" dependencies = [ "prost", ] @@ -3063,9 +3073,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.12.3" +version = "1.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" dependencies = [ "aho-corasick", "memchr", @@ -3092,9 +3102,9 @@ checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" [[package]] name = "regex-syntax" -version = "0.8.10" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "regress" @@ -3206,9 +3216,9 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" dependencies = [ "openssl-probe", "rustls-pki-types", @@ -3389,9 +3399,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.149" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ "itoa", "memchr", @@ -3461,9 +3471,9 @@ dependencies = [ [[package]] name = "shlex" -version = "1.3.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" [[package]] name = "simd-adler32" @@ -3503,9 +3513,9 @@ checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" [[package]] name = "socket2" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", "windows-sys 0.61.2", @@ -3900,9 +3910,9 @@ checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" [[package]] name = "typenum" -version = "1.20.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "typify" @@ -3959,9 +3969,9 @@ checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] name = "unicode-segmentation" -version = "1.13.2" +version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" [[package]] name = "unicode-width" @@ -4007,9 +4017,9 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "uuid" -version = "1.23.1" +version = "1.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" +checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7" dependencies = [ "getrandom 0.4.2", "js-sys", @@ -4068,9 +4078,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.121" +version = "0.2.123" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790" +checksum = "a254a4b10c19a76f09a27640e7ffbf9bc30bf67e16a3bf28aaefa4920fe81563" dependencies = [ "cfg-if", "once_cell", @@ -4081,9 +4091,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.71" +version = "0.4.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96492d0d3ffba25305a7dc88720d250b1401d7edca02cc3bcd50633b424673b8" +checksum = "54568702fabf5d4849ce2b90fadfa64168a097eaf4b351ce9df8b687a0086aaf" dependencies = [ "js-sys", "wasm-bindgen", @@ -4091,9 +4101,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.121" +version = "0.2.123" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578" +checksum = "24a40fc75b0ec6f3746ceb10d36f53a93dcd68a93b11b6445983945d79eba0dc" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4101,9 +4111,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.121" +version = "0.2.123" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2" +checksum = "908f34bd9b9ce3d4caf07b72dfab63d61504d156856c6bd3cd87fa350cf3985b" dependencies = [ "bumpalo", "proc-macro2", @@ -4114,9 +4124,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.121" +version = "0.2.123" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441" +checksum = "7acbf7616c27b194bbb550bf77ed0c2c3e5b7fd1260a93082b95fb7f47959b92" dependencies = [ "unicode-ident", ] @@ -4170,9 +4180,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.98" +version = "0.3.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b572dff8bcf38bad0fa19729c89bb5748b2b9b1d8be70cf90df697e3a8f32aa" +checksum = "6e0871acf327f283dc6da28a1696cdc64fb355ba9f935d052021fa77f35cce69" dependencies = [ "js-sys", "wasm-bindgen", @@ -4580,9 +4590,9 @@ checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" [[package]] name = "yoke" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" dependencies = [ "stable_deref_trait", "yoke-derive", @@ -4603,18 +4613,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.48" +version = "0.8.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.48" +version = "0.8.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" dependencies = [ "proc-macro2", "quote", @@ -4623,9 +4633,9 @@ dependencies = [ [[package]] name = "zerofrom" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" dependencies = [ "zerofrom-derive", ] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..fd1971a --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,57 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[workspace] +resolver = "2" +members = [ + "native", + "native-common", +] + +# Shared package metadata so every crate moves in lock step. Members inherit +# via `version.workspace = true` / `edition.workspace = true` etc.; a single +# bump here re-versions the whole workspace. +[workspace.package] +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" +repository = "https://github.com/apache/datafusion-java" + +# Every dependency used by any workspace member is declared here so version +# bumps live in one place and the resolver picks a single version of each +# crate across the workspace. Members reference these via `{ workspace = true }` +# and add per-crate flags (optional, features, default-features) at the use +# site. +[workspace.dependencies] +arrow = { version = "58", features = ["ffi"] } +async-trait = "0.1" +datafusion = { version = "53.1.0" } +datafusion-proto = "53.1.0" +datafusion-spark = "53.1.0" +datafusion-substrait = "53.1.0" +futures = "0.3" +jni = "0.21" +# Pinned to the major DataFusion 53.1 pulls in transitively (0.13.x) so we +# share the same `dyn ObjectStore` vtable and don't double-link. +object_store = { version = "0.13", default-features = false } +prost = "0.14" +prost-build = "0.14" +protoc-bin-vendored = "3" +tokio = { version = "1", features = ["rt-multi-thread"] } +# Optional, cfg-gated. See `native/Cargo.toml` for the build-flag dance. +tokio-metrics = "0.5" +url = "2" diff --git a/Makefile b/Makefile index 6d9b0ae..d6bcf2c 100644 --- a/Makefile +++ b/Makefile @@ -20,14 +20,14 @@ all: native jvm native: - cd native && cargo build + cargo build --workspace -# Build the native crate with the `runtime-metrics` Cargo feature enabled. +# Build the JNI crate with the `runtime-metrics` Cargo feature enabled. # Requires `--cfg tokio_unstable` because tokio-metrics gates its API there. # Default `make native` does not pull this in; callers who need # SessionContext.runtimeStats() pick this target explicitly. native-runtime-metrics: - cd native && RUSTFLAGS="--cfg tokio_unstable" cargo build --features runtime-metrics + RUSTFLAGS="--cfg tokio_unstable" cargo build -p datafusion-jni --features runtime-metrics jvm: ./mvnw package -DskipTests @@ -39,10 +39,10 @@ test: native # `:check` form inline in .github/workflows/lint.yml. format: ./mvnw -q spotless:apply - cd native && cargo fmt --all + cargo fmt --all clean: - cd native && cargo clean + cargo clean ./mvnw clean tpch-data: diff --git a/core/pom.xml b/core/pom.xml index 5ddf107..1e25736 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -102,8 +102,8 @@ under the License. - + value="${maven.multiModuleProjectDirectory}/rust-target/${datafusion.native.profile}/${datafusion.lib.filename}"/> + diff --git a/core/src/main/java/org/apache/datafusion/SessionContext.java b/core/src/main/java/org/apache/datafusion/SessionContext.java index ec0bd85..b68cda5 100644 --- a/core/src/main/java/org/apache/datafusion/SessionContext.java +++ b/core/src/main/java/org/apache/datafusion/SessionContext.java @@ -113,10 +113,11 @@ public DataFrame fromProto(byte[] planBytes) { * other Substrait-emitting tool — and hand them to DataFusion without round-tripping through SQL. * *

Substrait support is gated behind the {@code substrait} Cargo feature on the native crate - * and is off by default. Rebuild the native crate with {@code cargo build - * --features substrait} (or {@code cargo build --features substrait,protoc} for hermetic builds - * that vendor {@code protoc} via {@code cmake}) to enable it. If invoked against a native binary - * built without the feature, this method throws {@link RuntimeException} pointing at the flag. + * and is off by default. Rebuild the native crate with {@code cargo build -p + * datafusion-jni --features substrait} (or {@code ... --features substrait,protoc} for hermetic + * builds that vendor {@code protoc} via {@code cmake}) to enable it. If invoked against a native + * binary built without the feature, this method throws {@link RuntimeException} pointing at the + * flag. * * @throws IllegalArgumentException if {@code planBytes} is {@code null}. * @throws IllegalStateException if this context is closed. @@ -183,7 +184,7 @@ public MemoryUsage memoryUsage() { * Rebuild with: * * 

{@code
-   * RUSTFLAGS="--cfg tokio_unstable" cargo build --features runtime-metrics
+   * RUSTFLAGS="--cfg tokio_unstable" cargo build -p datafusion-jni --features runtime-metrics
    * }
* *

If invoked against a native binary built without the feature, this method throws {@link diff --git a/core/src/test/java/org/apache/datafusion/SessionContextRuntimeStatsTest.java b/core/src/test/java/org/apache/datafusion/SessionContextRuntimeStatsTest.java index 120d179..d567275 100644 --- a/core/src/test/java/org/apache/datafusion/SessionContextRuntimeStatsTest.java +++ b/core/src/test/java/org/apache/datafusion/SessionContextRuntimeStatsTest.java @@ -37,7 +37,7 @@ * #checkFeatureEnabled}. Run * * 

{@code
- * (cd native && RUSTFLAGS="--cfg tokio_unstable" cargo build --features runtime-metrics)
+ * RUSTFLAGS="--cfg tokio_unstable" cargo build -p datafusion-jni --features runtime-metrics
  * }
* * before {@code ./mvnw test} to exercise this class. diff --git a/core/src/test/java/org/apache/datafusion/SessionContextSubstraitTest.java b/core/src/test/java/org/apache/datafusion/SessionContextSubstraitTest.java index 34db3b5..a2cfb0a 100644 --- a/core/src/test/java/org/apache/datafusion/SessionContextSubstraitTest.java +++ b/core/src/test/java/org/apache/datafusion/SessionContextSubstraitTest.java @@ -50,7 +50,7 @@ * *

The {@code substrait} Cargo feature is off by default in {@code native/Cargo.toml}; if the * native crate was built without it, every test here is skipped (see {@link #checkFeatureEnabled}). - * Run {@code (cd native && cargo build --features substrait)} before {@code ./mvnw test} to + * Run {@code cargo build -p datafusion-jni --features substrait} before {@code ./mvnw test} to * exercise this class. */ class SessionContextSubstraitTest { diff --git a/dev/release/build-release.sh b/dev/release/build-release.sh index 2b033bb..4d4ab13 100755 --- a/dev/release/build-release.sh +++ b/dev/release/build-release.sh @@ -135,26 +135,28 @@ JVM_TARGET_DIR="$PROJECT_HOME/core/target/classes/org/apache/datafusion" mkdir -p "$JVM_TARGET_DIR/linux/amd64" docker cp \ - "$CONTAINER_AMD64:/opt/datafusion-java-rm/datafusion-java/native/target/release/libdatafusion_jni.so" \ + "$CONTAINER_AMD64:/opt/datafusion-java-rm/datafusion-java/rust-target/release/libdatafusion_jni.so" \ "$JVM_TARGET_DIR/linux/amd64/" mkdir -p "$JVM_TARGET_DIR/linux/aarch64" docker cp \ - "$CONTAINER_ARM64:/opt/datafusion-java-rm/datafusion-java/native/target/release/libdatafusion_jni.so" \ + "$CONTAINER_ARM64:/opt/datafusion-java-rm/datafusion-java/rust-target/release/libdatafusion_jni.so" \ "$JVM_TARGET_DIR/linux/aarch64/" echo "Building macOS native libs on the host (host=$HOST_ARCH)" rustup target add "$OTHER_DARWIN_TARGET" -(cd "$PROJECT_HOME/native" && cargo build --release) -(cd "$PROJECT_HOME/native" && cargo build --release --target "$OTHER_DARWIN_TARGET") +# Cargo writes to the workspace `rust-target/` dir (set in .cargo/config.toml), +# not the per-crate `native/target/`, so build from the repo root. +(cd "$PROJECT_HOME" && cargo build --release -p datafusion-jni) +(cd "$PROJECT_HOME" && cargo build --release -p datafusion-jni --target "$OTHER_DARWIN_TARGET") mkdir -p "$JVM_TARGET_DIR/darwin/$HOST_DARWIN_DIR" -cp "$PROJECT_HOME/native/target/release/libdatafusion_jni.dylib" \ +cp "$PROJECT_HOME/rust-target/release/libdatafusion_jni.dylib" \ "$JVM_TARGET_DIR/darwin/$HOST_DARWIN_DIR/" mkdir -p "$JVM_TARGET_DIR/darwin/$OTHER_DARWIN_DIR" -cp "$PROJECT_HOME/native/target/$OTHER_DARWIN_TARGET/release/libdatafusion_jni.dylib" \ +cp "$PROJECT_HOME/rust-target/$OTHER_DARWIN_TARGET/release/libdatafusion_jni.dylib" \ "$JVM_TARGET_DIR/darwin/$OTHER_DARWIN_DIR/" echo "Installing JAR into local Maven repo" diff --git a/dev/release/datafusion-java-rm/build-native-libs.sh b/dev/release/datafusion-java-rm/build-native-libs.sh index 5f273cc..79f8ae0 100755 --- a/dev/release/datafusion-java-rm/build-native-libs.sh +++ b/dev/release/datafusion-java-rm/build-native-libs.sh @@ -38,8 +38,9 @@ git clone "$REPO" datafusion-java cd datafusion-java git checkout "$BRANCH" -cd native -cargo build --release +# Cargo writes to the workspace `rust-target/` dir (set in .cargo/config.toml), +# not the per-crate `native/target/`, so build from the repo root. +cargo build --release -p datafusion-jni -echo "Built $(pwd)/target/release/libdatafusion_jni.so" -ls -l target/release/libdatafusion_jni.so +echo "Built $(pwd)/rust-target/release/libdatafusion_jni.so" +ls -l rust-target/release/libdatafusion_jni.so diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index 81d83e8..3dbd90f 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -7,7 +7,7 @@ .mvn/wrapper/maven-wrapper.properties mvnw mvnw.cmd -native/Cargo.lock +Cargo.lock dev/release/rat_exclude_files.txt docs/source/_static/** docs/source/conf.py diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index e486adc..c7767bf 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -150,7 +150,8 @@ test_source_distribution() { # raises on any formatting errors rustup component add rustfmt - (cd native && cargo fmt --all -- --check) + # Workspace-wide: covers native, native-common, and any future members. + cargo fmt --all -- --check # build native + JVM and run the full test suite make test diff --git a/docs/source/contributor-guide/development.md b/docs/source/contributor-guide/development.md index 984d77c..61d4fb0 100644 --- a/docs/source/contributor-guide/development.md +++ b/docs/source/contributor-guide/development.md @@ -42,7 +42,7 @@ This builds the native Rust crate and runs the JUnit tests. The steps can be run individually: ```sh -cd native && cargo build +cargo build --workspace ./mvnw test ``` @@ -74,6 +74,11 @@ disk space. The repository is a multi-module Maven build: +- `Cargo.toml` — Rust workspace root declaring the crate members + (`native`, `native-common`) and `[workspace.dependencies]` that pin + shared versions in one place. Cargo writes artifacts to `rust-target/` + (overridden in `.cargo/config.toml`) so `mvn clean` at the repo root does + not nuke the Rust build cache. - `pom.xml` — parent POM declaring the `core` and `examples` modules and shared plugin/dependency versions. - `core/` — `datafusion-java` library module (Java sources, tests, and @@ -81,7 +86,10 @@ The repository is a multi-module Maven build: - `examples/` — `datafusion-java-examples` module containing runnable examples that depend on the library; built alongside the library so they cannot fall out of sync with the API. -- `native/` — Rust crate (JNI + Arrow C Data Interface). +- `native/` — `datafusion-jni` Rust crate (JNI + Arrow C Data Interface). +- `native-common/` — `datafusion-jni-common` Rust crate: JNI plumbing + shared across native crates (error→exception mapping, the per-cdylib + Tokio runtime singleton, the async-stream→`FFI_ArrowArrayStream` bridge). - `proto/` — Protobuf definitions shared between Java and Rust. - `Makefile` — top-level build orchestration (`make test`, `make format`, `make tpch-data`). diff --git a/docs/source/contributor-guide/updating-datafusion-version.md b/docs/source/contributor-guide/updating-datafusion-version.md index 56d50dc..6e3b90b 100644 --- a/docs/source/contributor-guide/updating-datafusion-version.md +++ b/docs/source/contributor-guide/updating-datafusion-version.md @@ -21,7 +21,9 @@ under the License. Three things must move together when bumping DataFusion: -1. `native/Cargo.toml` — the `datafusion` crate dependency. +1. `Cargo.toml` (workspace root) — the `datafusion`, `datafusion-proto`, + `datafusion-spark`, and `datafusion-substrait` entries in + `[workspace.dependencies]`. Members inherit from there. 2. `pom.xml` — the `` Maven property. **Must equal the Cargo version**; a mismatch means JVM-built protobuf plans won't deserialize on the native side. @@ -32,9 +34,9 @@ Three things must move together when bumping DataFusion: ## Recipe ```sh -# 1. Bump the Cargo dep -$EDITOR native/Cargo.toml # set datafusion = "" -(cd native && cargo update -p datafusion) +# 1. Bump the workspace dep +$EDITOR Cargo.toml # set datafusion = "" in [workspace.dependencies] +cargo update -p datafusion # 2. Bump the Maven property to match $EDITOR pom.xml # set diff --git a/native-common/Cargo.toml b/native-common/Cargo.toml new file mode 100644 index 0000000..21a2296 --- /dev/null +++ b/native-common/Cargo.toml @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "datafusion-jni-common" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +# Implementation detail of datafusion-java's native crates, not a standalone +# crates.io library. Matches `publish = false` on the `datafusion-jni` crate. +publish = false +readme = "README.md" +description = "Shared JNI plumbing for DataFusion Java native crates: error-to-exception mapping, the per-cdylib Tokio runtime singleton, and the async-stream-to-FFI_ArrowArrayStream bridge." + +[features] +# `datafusion-jni` builds DataFusion with `avro`, which adds the +# `DataFusionError::AvroError` variant our classifier maps to IoException. +# Feature-forwarded so consumers that don't read Avro (the Spark helper) +# don't pull the apache-avro stack into their cdylib. +avro = ["datafusion/avro"] + +[dependencies] +datafusion = { workspace = true } +futures = { workspace = true } +jni = { workspace = true } +tokio = { workspace = true } diff --git a/native-common/README.md b/native-common/README.md new file mode 100644 index 0000000..aadf877 --- /dev/null +++ b/native-common/README.md @@ -0,0 +1,37 @@ + + +# datafusion-jni-common + +Shared JNI plumbing for the [Apache DataFusion Java](https://github.com/apache/datafusion-java) +native crates. It holds the pieces every DataFusion-backed `cdylib` loaded into a +JVM needs, factored out so they live in one place. + +## Linking model + +Each consuming `cdylib` statically links its own copy of this crate, so the +runtime singleton is per-library, not per-process. Nothing here is exported with +`#[no_mangle]`, so linking it into several `cdylib`s loaded in one JVM cannot +collide. + +## Status + +This crate is an implementation detail of Apache DataFusion Java. Its API may +change between releases to track the needs of the native crates that depend on +it. diff --git a/native/src/errors.rs b/native-common/src/errors.rs similarity index 95% rename from native/src/errors.rs rename to native-common/src/errors.rs index d926544..f9dbb03 100644 --- a/native/src/errors.rs +++ b/native-common/src/errors.rs @@ -96,8 +96,11 @@ fn classify(err: &DataFusionError) -> &'static str { } DataFusionError::IoError(_) | DataFusionError::ObjectStore(_) - | DataFusionError::ParquetError(_) - | DataFusionError::AvroError(_) => "org/apache/datafusion/IoException", + | DataFusionError::ParquetError(_) => "org/apache/datafusion/IoException", + // The AvroError variant only exists when DataFusion is built with its + // `avro` feature, forwarded by this crate's own `avro` feature. + #[cfg(feature = "avro")] + DataFusionError::AvroError(_) => "org/apache/datafusion/IoException", // ArrowError is a 21-variant grab bag -- only some of those variants // are actually IO-shaped. DivideByZero / ArithmeticOverflow / Compute // / Cast / InvalidArgument / Memory etc. are execution-time failures @@ -161,7 +164,10 @@ fn throw(env: &mut JNIEnv, class: &str, message: &str) { let _ = env.throw_new(class, message); } -fn panic_message(panic: &Box) -> String { +/// Best-effort extraction of a panic payload's message. `catch_unwind` hands +/// back a `Box`; the payload is a `String` or `&str` for ordinary +/// `panic!`/`unwrap` sites, anything else is opaque. +pub fn panic_message(panic: &Box) -> String { if let Some(s) = panic.downcast_ref::() { s.clone() } else if let Some(s) = panic.downcast_ref::<&str>() { diff --git a/native-common/src/lib.rs b/native-common/src/lib.rs new file mode 100644 index 0000000..ba47004 --- /dev/null +++ b/native-common/src/lib.rs @@ -0,0 +1,98 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! JNI plumbing shared by this workspace's native crates (`datafusion-jni` +//! and `datafusion-spark-bridge`, and through the latter every bridge +//! cdylib): the error-to-Java-exception mapping, the per-cdylib Tokio +//! runtime singleton, and the async-stream-to-`FFI_ArrowArrayStream` +//! bridge. +//! +//! Each cdylib statically links its own copy of this rlib, so [`runtime`] is +//! a per-cdylib singleton -- exactly the behaviour each crate had when this +//! code lived inline. Nothing here is exported with `#[no_mangle]`, so +//! linking this crate into several cdylibs loaded in one JVM cannot collide. + +pub mod errors; + +use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::sync::OnceLock; + +use datafusion::arrow::array::RecordBatch; +use datafusion::arrow::datatypes::SchemaRef; +use datafusion::arrow::error::ArrowError; +use datafusion::arrow::record_batch::RecordBatchReader; +use datafusion::execution::SendableRecordBatchStream; +use futures::StreamExt; +use tokio::runtime::{Handle, Runtime}; + +static RT: OnceLock = OnceLock::new(); + +/// The cdylib-wide Tokio runtime. +pub fn runtime() -> &'static Runtime { + runtime_with_init(|_| {}) +} + +/// Same singleton as [`runtime`], with a hook that runs exactly once, when +/// the runtime is created. `datafusion-jni` uses it to install its +/// runtime-metrics accumulator so the sampling baseline coincides with +/// runtime start; every later call (either entry point) returns the existing +/// runtime without invoking the hook. +pub fn runtime_with_init(init: impl FnOnce(&Handle)) -> &'static Runtime { + RT.get_or_init(|| { + let rt = Runtime::new().expect("failed to create Tokio runtime"); + init(rt.handle()); + rt + }) +} + +/// Bridges DataFusion's async [`SendableRecordBatchStream`] to the synchronous +/// [`RecordBatchReader`] interface that `FFI_ArrowArrayStream` (and therefore +/// the Java `ArrowReader`) consumes. Each call to `next()` drives one +/// `runtime().block_on(stream.next())`, so memory pressure stays bounded by the +/// executor pipeline plus a single in-flight batch. +pub struct StreamingReader { + pub schema: SchemaRef, + pub stream: SendableRecordBatchStream, +} + +impl Iterator for StreamingReader { + type Item = Result; + + fn next(&mut self) -> Option { + // Arrow's C ABI invokes this iterator through FFI_ArrowArrayStream's + // vtable, outside the JNI handler's try_unwrap_or_throw guard. A panic + // here (buggy UDF, arrow cast that panics, runtime poison) would + // unwind across C/FFI -- undefined behaviour. Catch it and surface as + // an ArrowError so the Java side sees a normal exception instead. + let next = catch_unwind(AssertUnwindSafe(|| runtime().block_on(self.stream.next()))); + match next { + Ok(item) => item.map(|r| r.map_err(|e| ArrowError::ExternalError(Box::new(e)))), + Err(panic) => { + let msg = errors::panic_message(&panic); + Some(Err(ArrowError::ExternalError( + format!("panic in DataFrame stream: {msg}").into(), + ))) + } + } + } +} + +impl RecordBatchReader for StreamingReader { + fn schema(&self) -> SchemaRef { + self.schema.clone() + } +} diff --git a/native/Cargo.toml b/native/Cargo.toml index 0362ae6..c040448 100644 --- a/native/Cargo.toml +++ b/native/Cargo.toml @@ -17,14 +17,17 @@ [package] name = "datafusion-jni" -version = "0.1.0" -edition = "2021" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +# cdylib JNI artifact loaded by the JVM, not a crates.io library. publish = false [lib] # `rlib` alongside `cdylib` so `cargo test` has a Rust-level harness for -# native-only invariants (e.g. error-classification routing through wrapped -# DataFusionError chains). The `cdylib` is still the artifact the JVM loads. +# native-only invariants (the error-classification tests now live in +# `datafusion-jni-common`). The `cdylib` is still the artifact the JVM loads. crate-type = ["cdylib", "rlib"] [features] @@ -75,28 +78,27 @@ runtime-metrics = ["dep:tokio-metrics"] spark = ["dep:datafusion-spark"] [dependencies] -arrow = { version = "58", features = ["ffi"] } -async-trait = "0.1" -datafusion = { version = "53.1.0", features = ["avro"] } -datafusion-proto = "53.1.0" +arrow = { workspace = true } +async-trait = { workspace = true } +datafusion = { workspace = true, features = ["avro"] } +# Shared JNI plumbing (error->exception mapping, runtime singleton, +# StreamingReader). `avro` keeps the classifier's AvroError->IoException arm +# in sync with the `avro` feature on `datafusion` above. +datafusion-jni-common = { path = "../native-common", features = ["avro"] } +datafusion-proto = { workspace = true } # Apache Spark-compatible functions + expression planners. Optional and # gated behind the `spark` feature (in the default set). The `core` feature # of the crate is what exposes `SessionStateBuilderSpark`. -datafusion-spark = { version = "53.1.0", features = ["core"], optional = true } -datafusion-substrait = { version = "53.1.0", optional = true } -futures = "0.3" -jni = "0.21" -# Pin to the same major as DataFusion 53.1 pulls in transitively (0.13.x) -# so we share the same `dyn ObjectStore` vtable and don't double-link. -object_store = { version = "0.13", default-features = false } -prost = "0.14" -tokio = { version = "1", features = ["rt-multi-thread"] } -# Tokio runtime metrics. Optional + cfg-gated: this crate's API surface lives -# behind `--cfg tokio_unstable`, so enabling the `runtime-metrics` feature also -# requires the caller to set `RUSTFLAGS="--cfg tokio_unstable"` at build time. -tokio-metrics = { version = "0.5", optional = true } -url = "2" +datafusion-spark = { workspace = true, features = ["core"], optional = true } +datafusion-substrait = { workspace = true, optional = true } +futures = { workspace = true } +jni = { workspace = true } +object_store = { workspace = true } +prost = { workspace = true } +tokio = { workspace = true } +tokio-metrics = { workspace = true, optional = true } +url = { workspace = true } [build-dependencies] -prost-build = "0.14" -protoc-bin-vendored = "3" +prost-build = { workspace = true } +protoc-bin-vendored = { workspace = true } diff --git a/native/src/arrow.rs b/native/src/arrow.rs index 2bbe7b0..67e5caf 100644 --- a/native/src/arrow.rs +++ b/native/src/arrow.rs @@ -23,10 +23,10 @@ use jni::sys::jlong; use jni::JNIEnv; use prost::Message; -use crate::errors::{try_unwrap_or_throw, JniResult}; use crate::proto_gen::ArrowReadOptionsProto; use crate::runtime; use crate::schema::decode_optional_schema; +use datafusion_jni_common::errors::{try_unwrap_or_throw, JniResult}; fn with_arrow_options( env: &mut JNIEnv, diff --git a/native/src/avro.rs b/native/src/avro.rs index 85d4a07..257ae32 100644 --- a/native/src/avro.rs +++ b/native/src/avro.rs @@ -23,10 +23,10 @@ use jni::sys::jlong; use jni::JNIEnv; use prost::Message; -use crate::errors::{try_unwrap_or_throw, JniResult}; use crate::proto_gen::AvroReadOptionsProto; use crate::runtime; use crate::schema::decode_optional_schema; +use datafusion_jni_common::errors::{try_unwrap_or_throw, JniResult}; fn with_avro_options( env: &mut JNIEnv, diff --git a/native/src/cache_manager.rs b/native/src/cache_manager.rs index 3b9e286..ec38dc8 100644 --- a/native/src/cache_manager.rs +++ b/native/src/cache_manager.rs @@ -34,8 +34,8 @@ use datafusion::execution::cache::cache_unit::{ }; use datafusion::execution::cache::DefaultListFilesCache; -use crate::errors::JniResult; use crate::proto_gen::CacheManagerOptionsProto; +use datafusion_jni_common::errors::JniResult; /// Build a [`CacheManagerConfig`] from the proto. Returns `Ok(None)` if the /// caller did not set any cache-manager field, so the JNI layer can skip the diff --git a/native/src/csv.rs b/native/src/csv.rs index 3ae4627..b79ed59 100644 --- a/native/src/csv.rs +++ b/native/src/csv.rs @@ -26,12 +26,12 @@ use jni::sys::jlong; use jni::JNIEnv; use prost::Message; -use crate::errors::{try_unwrap_or_throw, JniResult}; use crate::proto_gen::{ CsvReadOptionsProto, CsvWriteOptionsProto, FileCompressionType as ProtoFileCompressionType, }; use crate::runtime; use crate::schema::decode_optional_schema; +use datafusion_jni_common::errors::{try_unwrap_or_throw, JniResult}; fn with_csv_options( env: &mut JNIEnv, diff --git a/native/src/json.rs b/native/src/json.rs index 8eea32f..b87be78 100644 --- a/native/src/json.rs +++ b/native/src/json.rs @@ -27,12 +27,12 @@ use jni::sys::jlong; use jni::JNIEnv; use prost::Message; -use crate::errors::{try_unwrap_or_throw, JniResult}; use crate::proto_gen::{ FileCompressionType as ProtoFileCompressionType, JsonWriteOptionsProto, NdJsonReadOptionsProto, }; use crate::runtime; use crate::schema::decode_optional_schema; +use datafusion_jni_common::errors::{try_unwrap_or_throw, JniResult}; fn with_json_options( env: &mut JNIEnv, diff --git a/native/src/lib.rs b/native/src/lib.rs index 43161d2..56bef5d 100644 --- a/native/src/lib.rs +++ b/native/src/lib.rs @@ -19,7 +19,6 @@ mod arrow; mod avro; mod cache_manager; mod csv; -mod errors; mod jni_util; mod json; mod memory; @@ -34,16 +33,13 @@ pub(crate) mod proto_gen { include!(concat!(env!("OUT_DIR"), "/datafusion_java.rs")); } -use std::panic::{catch_unwind, AssertUnwindSafe}; use std::path::PathBuf; use std::sync::{Arc, OnceLock}; -use datafusion::arrow::array::RecordBatch; use datafusion::arrow::datatypes::SchemaRef; -use datafusion::arrow::error::ArrowError; use datafusion::arrow::ffi_stream::FFI_ArrowArrayStream; use datafusion::arrow::ipc::writer::StreamWriter; -use datafusion::arrow::record_batch::{RecordBatchIterator, RecordBatchReader}; +use datafusion::arrow::record_batch::RecordBatchIterator; use datafusion::common::{JoinType, UnnestOptions}; use datafusion::config::TableParquetOptions; use datafusion::dataframe::DataFrame; @@ -51,11 +47,9 @@ use datafusion::dataframe::DataFrameWriteOptions; use datafusion::error::DataFusionError; use datafusion::execution::disk_manager::{DiskManagerBuilder, DiskManagerMode}; use datafusion::execution::runtime_env::{RuntimeEnv, RuntimeEnvBuilder}; -use datafusion::execution::SendableRecordBatchStream; use datafusion::logical_expr::Expr; use datafusion::logical_expr::{col, Partitioning, ScalarUDF, Signature, SortExpr}; use datafusion::prelude::{ParquetReadOptions, SessionConfig, SessionContext}; -use futures::StreamExt; use jni::objects::{JBooleanArray, JByteArray, JClass, JObject, JObjectArray, JString}; use jni::sys::{jboolean, jbyte, jbyteArray, jint, jlong}; use jni::JNIEnv; @@ -63,7 +57,10 @@ use jni::JavaVM; use prost::Message; use tokio::runtime::Runtime; -use crate::errors::{try_unwrap_or_throw, JniResult}; +use datafusion_jni_common::errors::{try_unwrap_or_throw, JniResult}; +// Re-exported so sibling modules keep their crate-local `crate::StreamingReader` path. +pub(crate) use datafusion_jni_common::StreamingReader; + use crate::proto_gen::ParquetReadOptionsProto; use crate::proto_gen::SessionOptions; use crate::schema::decode_optional_schema; @@ -84,18 +81,15 @@ pub(crate) fn jvm() -> &'static JavaVM { } pub(crate) fn runtime() -> &'static Runtime { - static RT: OnceLock = OnceLock::new(); - RT.get_or_init(|| { - let rt = Runtime::new().expect("failed to create Tokio runtime"); - // Eagerly install the runtime-metrics accumulator (no-op when the - // `runtime-metrics` Cargo feature is off). Initialising here -- not - // lazily on the first `runtimeStats()` call -- means the - // RuntimeMonitor's sampling baseline coincides with runtime start, so - // poll/park/busy totals reflect activity from the first query onward - // rather than from the first observation. - crate::runtime_metrics::init(rt.handle()); - rt - }) + // The singleton itself lives in datafusion-jni-common (shared with the + // datafusion-spark-bridge SDK; each cdylib statically links its own + // copy, so the runtime stays per-library). The init hook eagerly installs the + // runtime-metrics accumulator (no-op when the `runtime-metrics` Cargo + // feature is off). Initialising here -- not lazily on the first + // `runtimeStats()` call -- means the RuntimeMonitor's sampling baseline + // coincides with runtime start, so poll/park/busy totals reflect activity + // from the first query onward rather than from the first observation. + datafusion_jni_common::runtime_with_init(crate::runtime_metrics::init) } /// Wrap the (already-built) `RuntimeEnvBuilder`'s memory pool with a @@ -324,50 +318,6 @@ pub extern "system" fn Java_org_apache_datafusion_DataFrame_collectDataFrame<'lo }) } -/// Bridges DataFusion's async [`SendableRecordBatchStream`] to the synchronous -/// [`RecordBatchReader`] interface that `FFI_ArrowArrayStream` (and therefore -/// the Java `ArrowReader`) consumes. Each call to `next()` drives one -/// `runtime().block_on(stream.next())`, so memory pressure stays bounded by the -/// executor pipeline plus a single in-flight batch. -struct StreamingReader { - schema: SchemaRef, - stream: SendableRecordBatchStream, -} - -impl Iterator for StreamingReader { - type Item = Result; - - fn next(&mut self) -> Option { - // Arrow's C ABI invokes this iterator through FFI_ArrowArrayStream's - // vtable, outside the JNI handler's try_unwrap_or_throw guard. A panic - // here (buggy UDF, arrow cast that panics, runtime poison) would - // unwind across C/FFI -- undefined behaviour. Catch it and surface as - // an ArrowError so the Java side sees a normal exception instead. - let next = catch_unwind(AssertUnwindSafe(|| runtime().block_on(self.stream.next()))); - match next { - Ok(item) => item.map(|r| r.map_err(|e| ArrowError::ExternalError(Box::new(e)))), - Err(panic) => { - let msg = if let Some(s) = panic.downcast_ref::() { - s.clone() - } else if let Some(s) = panic.downcast_ref::<&str>() { - (*s).to_string() - } else { - "rust panic with non-string payload".to_string() - }; - Some(Err(ArrowError::ExternalError( - format!("panic in DataFrame stream: {msg}").into(), - ))) - } - } - } -} - -impl RecordBatchReader for StreamingReader { - fn schema(&self) -> SchemaRef { - self.schema.clone() - } -} - #[no_mangle] pub extern "system" fn Java_org_apache_datafusion_DataFrame_executeStreamDataFrame<'local>( mut env: JNIEnv<'local>, diff --git a/native/src/object_store.rs b/native/src/object_store.rs index eefccf2..985d721 100644 --- a/native/src/object_store.rs +++ b/native/src/object_store.rs @@ -28,9 +28,9 @@ use std::sync::Arc; use datafusion::prelude::SessionContext; use url::Url; -use crate::errors::JniResult; use crate::proto_gen::object_store_registration::Backend; use crate::proto_gen::ObjectStoreRegistration; +use datafusion_jni_common::errors::JniResult; #[cfg(feature = "object-store-gcp")] use crate::proto_gen::GcsOptions; diff --git a/native/src/proto.rs b/native/src/proto.rs index 4f187bc..c1315f9 100644 --- a/native/src/proto.rs +++ b/native/src/proto.rs @@ -28,8 +28,8 @@ use jni::sys::{jbyteArray, jlong}; use jni::JNIEnv; use prost::Message; -use crate::errors::{try_unwrap_or_throw, JniResult}; use crate::runtime; +use datafusion_jni_common::errors::{try_unwrap_or_throw, JniResult}; #[no_mangle] pub extern "system" fn Java_org_apache_datafusion_SessionContext_createDataFrameFromProto< diff --git a/native/src/runtime_metrics.rs b/native/src/runtime_metrics.rs index e69410e..dd60dcb 100644 --- a/native/src/runtime_metrics.rs +++ b/native/src/runtime_metrics.rs @@ -38,7 +38,7 @@ //! 10 totalOverflowCount #[cfg(not(feature = "runtime-metrics"))] -use crate::errors::JniResult; +use datafusion_jni_common::errors::JniResult; /// Number of i64 values in the snapshot array; kept here so the Java side and /// the feature-off stub agree on the layout. @@ -51,7 +51,7 @@ mod imp { use tokio_metrics::{RuntimeIntervals, RuntimeMonitor}; use super::STATS_FIELD_COUNT; - use crate::errors::JniResult; + use datafusion_jni_common::errors::JniResult; /// `RuntimeMonitor::intervals().next()` returns *delta* metrics covering /// the period since the previous call (or, on the very first call, since @@ -196,7 +196,7 @@ pub fn runtime_stats() -> JniResult<[i64; STATS_FIELD_COUNT]> { Err( "datafusion-jni was built without the `runtime-metrics` Cargo feature; \ rebuild the native crate with \ - `RUSTFLAGS=\"--cfg tokio_unstable\" cargo build --features runtime-metrics` \ + `RUSTFLAGS=\"--cfg tokio_unstable\" cargo build -p datafusion-jni --features runtime-metrics` \ to enable SessionContext.runtimeStats" .into(), ) diff --git a/native/src/schema.rs b/native/src/schema.rs index 968a73a..0c3c7ab 100644 --- a/native/src/schema.rs +++ b/native/src/schema.rs @@ -20,7 +20,7 @@ use datafusion::arrow::ipc::reader::StreamReader; use jni::objects::JByteArray; use jni::JNIEnv; -use crate::errors::JniResult; +use datafusion_jni_common::errors::JniResult; /// Decode an optional Arrow-IPC schema byte array passed in from Java. /// Returns `None` if the byte-array reference is null. diff --git a/pom.xml b/pom.xml index 6210841..7ceec07 100644 --- a/pom.xml +++ b/pom.xml @@ -95,6 +95,11 @@ under the License. + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + org.apache.maven.plugins maven-surefire-plugin @@ -173,10 +178,10 @@ under the License. .mvn/** **/target/** - native/target/** + rust-target/** tpch-data/** - - native/Cargo.lock + + Cargo.lock dev/release/rat_exclude_files.txt From 0d2c72b56b65961e1b6a040f172af534725f600f Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 6 Aug 2026 08:22:08 -0600 Subject: [PATCH 4/4] chore(deps): bump DataFusion to 54.1.0 (#114) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? N/A — routine dependency bump; no tracking issue was filed. ## Rationale for this change Keeps the binding current with upstream DataFusion. 54.1.0 is the latest release line, and staying close to it keeps the next bump small and makes upstream fixes available to Java callers. ## What changes are included in this PR? Bumps `datafusion`, `datafusion-proto`, `datafusion-spark` and `datafusion-substrait` from 53.1.0 to 54.1.0. The `datafusion.version` Maven property moves in lock step, since it selects the upstream tag the `datafusion.proto` / `datafusion_common.proto` definitions are downloaded from — the generated Java protobuf classes must match what `datafusion-proto` 54.1.0 decodes. The pinned sha512 digests for both protos are updated accordingly; each was verified to match the copy vendored in the published `datafusion-proto` / `datafusion-proto-common` 54.1.0 crates, independently of the GitHub download. `arrow` (58) and `object_store` (0.13) are unchanged — 54.1.0 resolves to the same majors, so the `object_store` pin comment still holds. Adapting to the upstream API changes the bump requires: - `TableProvider`, `ExecutionPlan` and `ScalarUDFImpl` now take `Any` as a supertrait, so the manual `as_any` overrides are no longer trait members and are removed. - `MemoryPool` gained a `name()` method and a `Display` supertrait. `TrackingMemoryPool` implements both, following upstream's wrapper convention: name the wrapper, add the counters it exists to expose, and defer to the inner pool for the usage detail. - `CacheManagerConfig::table_files_statistics_cache` is renamed to `file_statistics_cache`, and the accompanying limit is now the on/off switch — `CacheManager::try_new` installs a default statistics cache whenever `file_statistics_cache_limit > 0`, even when the cache slot is `None`, and the default limit is non-zero. An explicit `fileStatisticsCache(false)` from the Java surface therefore has to zero the limit as well; otherwise upstream would install the very cache the caller asked us to skip. This is the one place the bump would have silently changed observable Java behavior. - The `cache_unit` module is gone; the default cache impls now live in `cache::file_statistics_cache` and `cache`. - `DataFusionError::AvroError` is gone — DataFusion 54 reads Avro through `arrow-avro` rather than `apache-avro`. Avro decode failures now arrive as `ArrowError::AvroError`, which the exception classifier already routed to `ExecutionException` alongside the `CsvError` / `JsonError` decoder variants, so the mapping stays coherent. The dead arm is dropped, along with the `avro` feature on `datafusion-jni-common` that existed only to gate it. One test fixture is also corrected. `SessionContextSubstraitTest` built plans whose base schema declared both columns `NULLABILITY_REQUIRED`, while the tests register a CSV — whose inferred schema is always nullable. DataFusion 54's Substrait consumer now validates that a field a plan declares non-nullable really is non-nullable in the table, and rejects the mismatch. That check is correct: a plan built around a "never null" assumption must not run against data that can contain nulls. The fixture is fixed to declare nullable columns; it only passed before because 53 did not check. ## Are these changes tested? Covered by the existing suites — this is a dependency bump, so the value is in the current tests continuing to pass against the new version rather than in new assertions. - `./mvnw test` — 349 tests, 0 failures. Run with the `substrait` Cargo feature enabled (`cargo build -p datafusion-jni --features substrait`) so the Substrait tests execute rather than skip. - `cargo test --workspace` — all passing. - `cargo clippy --workspace --all-targets -- -D warnings` — clean. - `cargo fmt --all -- --check` and `./mvnw -q spotless:check` — clean. - `cargo build --workspace --all-features` with `RUSTFLAGS="--cfg tokio_unstable"`, to cover the optional `substrait` and `runtime-metrics` features that the default build does not compile. ## Are there any user-facing changes? No API changes. Two behavioral notes, both inherited from upstream: - Avro decode failures now surface as `ExecutionException` rather than `IoException`, following the move to `arrow-avro`. The `IoException` javadoc is updated to match. - DataFusion 54 enables the file statistics and list files caches by default. Callers who never configured `CacheManagerOptions` pick up upstream's new defaults; an explicit `fileStatisticsCache(false)` continues to disable the cache, as described above. --- Cargo.lock | 490 +++++++++--------- Cargo.toml | 10 +- core/pom.xml | 4 +- .../org/apache/datafusion/IoException.java | 5 +- .../DataFrameIntrospectionTest.java | 2 +- .../SessionContextSubstraitTest.java | 13 +- native-common/Cargo.toml | 7 - native-common/src/errors.rs | 16 +- native/Cargo.toml | 5 +- native/src/cache_manager.rs | 22 +- native/src/memory.rs | 20 + native/src/table_provider.rs | 9 - native/src/udf.rs | 5 - pom.xml | 2 +- 14 files changed, 301 insertions(+), 309 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dbbfcde..ab96620 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -67,35 +67,6 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" -[[package]] -name = "apache-avro" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36fa98bc79671c7981272d91a8753a928ff6a1cd8e4f20a44c45bd5d313840bf" -dependencies = [ - "bigdecimal", - "bon", - "bzip2", - "crc32fast", - "digest", - "liblzma", - "log", - "miniz_oxide", - "num-bigint", - "quad-rand", - "rand 0.9.4", - "regex-lite", - "serde", - "serde_bytes", - "serde_json", - "snap", - "strum", - "strum_macros", - "thiserror 2.0.18", - "uuid", - "zstd", -] - [[package]] name = "ar_archive_writer" version = "0.5.2" @@ -171,6 +142,30 @@ dependencies = [ "num-traits", ] +[[package]] +name = "arrow-avro" +version = "58.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "049230728cd6e093088c8d231b4beede184e35cad7777c1505c0d5a8571f4376" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-schema", + "bytes", + "bzip2", + "crc", + "flate2", + "indexmap", + "liblzma", + "rand 0.9.4", + "serde", + "serde_json", + "snap", + "strum_macros", + "uuid", + "zstd", +] + [[package]] name = "arrow-buffer" version = "58.3.0" @@ -414,7 +409,6 @@ dependencies = [ "num-bigint", "num-integer", "num-traits", - "serde", ] [[package]] @@ -429,7 +423,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" dependencies = [ - "digest", + "digest 0.10.7", ] [[package]] @@ -443,7 +437,7 @@ dependencies = [ "cc", "cfg-if", "constant_time_eq", - "cpufeatures 0.3.0", + "cpufeatures", ] [[package]] @@ -456,28 +450,12 @@ dependencies = [ ] [[package]] -name = "bon" -version = "3.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2f04f6fef12d70d42a77b1433c9e0f065238479a6cefc4f5bab105e9873a3c3" -dependencies = [ - "bon-macros", - "rustversion", -] - -[[package]] -name = "bon-macros" -version = "3.9.2" +name = "block-buffer" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d0bd4c2f75335ad98052a37efb54f428b492f64340257143b3429c8a508fa7b" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" dependencies = [ - "darling", - "ident_case", - "prettyplease", - "proc-macro2", - "quote", - "rustversion", - "syn", + "hybrid-array", ] [[package]] @@ -565,7 +543,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" dependencies = [ "cfg-if", - "cpufeatures 0.3.0", + "cpufeatures", "rand_core 0.10.1", ] @@ -641,6 +619,12 @@ version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "const-random" version = "0.1.18" @@ -685,22 +669,28 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.17" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" dependencies = [ "libc", ] [[package]] -name = "cpufeatures" -version = "0.3.0" +name = "crc" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" dependencies = [ - "libc", + "crc-catalog", ] +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + [[package]] name = "crc32fast" version = "1.5.0" @@ -732,6 +722,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "csv" version = "1.4.0" @@ -753,40 +752,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "darling" -version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" -dependencies = [ - "darling_core", - "darling_macro", -] - -[[package]] -name = "darling_core" -version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" -dependencies = [ - "ident_case", - "proc-macro2", - "quote", - "strsim", - "syn", -] - -[[package]] -name = "darling_macro" -version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" -dependencies = [ - "darling_core", - "quote", - "syn", -] - [[package]] name = "dashmap" version = "6.2.1" @@ -803,14 +768,13 @@ dependencies = [ [[package]] name = "datafusion" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93db0e623840612f7f2cd757f7e8a8922064192363732c88692e0870016e141b" +checksum = "754ef4e8f073922a26f5b23133b9db4829342362b09be0bc94309cf261c2f098" dependencies = [ "arrow", "arrow-schema", "async-trait", - "bytes", "bzip2", "chrono", "datafusion-catalog", @@ -841,14 +805,13 @@ dependencies = [ "datafusion-sql", "flate2", "futures", + "indexmap", "itertools", "liblzma", "log", "object_store", "parking_lot", "parquet", - "rand 0.9.4", - "regex", "sqlparser", "tempfile", "tokio", @@ -859,9 +822,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37cefde60b26a7f4ff61e9d2ff2833322f91df2b568d7238afe67bde5bdffb66" +checksum = "06afd1e38dd27bbb1258685a1fc6524df6aff4e07b25b393a47de59635178d99" dependencies = [ "arrow", "async-trait", @@ -884,9 +847,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17e112307715d6a7a331111a4c2330ff54bc237183511c319e3708a4cff431fb" +checksum = "f0668fb32c12065ec242be0e5b4bc62bd7a06a0be3ecd83791ef877e4be67e02" dependencies = [ "arrow", "async-trait", @@ -907,35 +870,35 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d72a11ca44a95e1081870d3abb80c717496e8a7acb467a1d3e932bb636af5cc2" +checksum = "ca43b263cdff57042cfa8fb817fb3469f4878933380dccff25f5e793580abbf9" dependencies = [ - "ahash", - "apache-avro", "arrow", "arrow-ipc", + "arrow-schema", "chrono", + "foldhash 0.2.0", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap", "itertools", "libc", "log", "object_store", "parquet", - "paste", "recursive", "sqlparser", "tokio", + "uuid", "web-time", ] [[package]] name = "datafusion-common-runtime" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89f4afaed29670ec4fd6053643adc749fe3f4bc9d1ce1b8c5679b22c67d12def" +checksum = "05f0ba2b864792bdca4d76c59a1de0ab6e1b61946596b9936888dbd6360035f2" dependencies = [ "futures", "log", @@ -944,9 +907,9 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9fb386e1691355355a96419978a0022b7947b44d4a24a6ea99f00b6b485cbb6" +checksum = "b840a8bce0bcbf5afad02946d438591e7c373f7afccaf3d874c04485772514dd" dependencies = [ "arrow", "async-compression", @@ -970,6 +933,7 @@ dependencies = [ "liblzma", "log", "object_store", + "parking_lot", "rand 0.9.4", "tokio", "tokio-util", @@ -979,9 +943,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffa6c52cfed0734c5f93754d1c0175f558175248bf686c944fb05c373e5fc096" +checksum = "a24cc0b9cf6e367f27f27406eff13abf48a11b72446aaa40b3105c0ded5c17d9" dependencies = [ "arrow", "arrow-ipc", @@ -1003,29 +967,28 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a579c3bd290c66ea4b269493e75e8a3ed42c9c895a651f10210a29538aee50c4" +checksum = "f26a312bb06528c17b4bb86b798b0c4321e7656d85e0ef65c2e9adf07b0a518d" dependencies = [ - "apache-avro", "arrow", + "arrow-avro", "async-trait", "bytes", "datafusion-common", "datafusion-datasource", - "datafusion-physical-expr-common", + "datafusion-physical-expr-adapter", "datafusion-physical-plan", "datafusion-session", "futures", - "num-traits", "object_store", ] [[package]] name = "datafusion-datasource-csv" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "503f29e0582c1fc189578d665ff57d9300da1f80c282777d7eb67bb79fb8cdca" +checksum = "e1abe56b2a7a2d1d6de5117dd1a203181e267f28529faa5da546947621b697d7" dependencies = [ "arrow", "async-trait", @@ -1046,9 +1009,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e33804749abc8d0c8cb7473228483cb8070e524c6f6086ee1b85a64debe2b3d2" +checksum = "9c3e467f0611ad7bdd5aad17c63c9bb6182d04e5282e5496d897ea2b49c024ba" dependencies = [ "arrow", "async-trait", @@ -1063,16 +1026,15 @@ dependencies = [ "datafusion-session", "futures", "object_store", - "serde_json", "tokio", "tokio-stream", ] [[package]] name = "datafusion-datasource-parquet" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a8e0365e0e08e8ff94d912f0ababcf9065a1a304018ba90b1fc83c855b4997" +checksum = "4cc35b92cd560082155e80d9c826929c852d3c51543f4affd3a51c464a0aab3a" dependencies = [ "arrow", "async-trait", @@ -1082,6 +1044,7 @@ dependencies = [ "datafusion-datasource", "datafusion-execution", "datafusion-expr", + "datafusion-functions", "datafusion-functions-aggregate-common", "datafusion-physical-expr", "datafusion-physical-expr-adapter", @@ -1100,20 +1063,19 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de6ac0df1662b9148ad3c987978b32cbec7c772f199b1d53520c8fa764a87ee" +checksum = "d69bb69d8769e34f76839c960dbde24c1ac0c885a79b6c3c2287bdc56ec67891" [[package]] name = "datafusion-execution" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c03c7fbdaefcca4ef6ffe425a5fc2325763bfb426599bb0bf4536466efabe709" +checksum = "d8eac0a09bc8d263f52025cad9e001da4d8138d633fa288edda4d06b1772eae6" dependencies = [ "arrow", "arrow-buffer", "async-trait", - "chrono", "dashmap", "datafusion-common", "datafusion-expr", @@ -1129,11 +1091,12 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "574b9b6977fedbd2a611cbff12e5caf90f31640ad9dc5870f152836d94bad0dd" +checksum = "eeb14d374767ee0fc62dc79a5ba8bcf8a63c14e993c7d992d0e63adfa23d77d3" dependencies = [ "arrow", + "arrow-schema", "async-trait", "chrono", "datafusion-common", @@ -1144,7 +1107,6 @@ dependencies = [ "datafusion-physical-expr-common", "indexmap", "itertools", - "paste", "recursive", "serde_json", "sqlparser", @@ -1152,22 +1114,21 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d7c3adf3db8bf61e92eb90cb659c8e8b734593a8f7c8e12a843c7ddba24b87e" +checksum = "b7b19a8c95522bee8cbb313d74263b85e355d2b52f42e67ef5694bf5de9e9356" dependencies = [ "arrow", "datafusion-common", "indexmap", "itertools", - "paste", ] [[package]] name = "datafusion-functions" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28aa4e10384e782774b10e72aca4d93ef7b31aa653095d9d4536b0a3dbc51b6" +checksum = "5f64c983bbbdcb729d921a2b2ac3375598719b5cc0c30345ad664936f3176fc7" dependencies = [ "arrow", "arrow-buffer", @@ -1182,26 +1143,25 @@ dependencies = [ "datafusion-expr", "datafusion-expr-common", "datafusion-macros", + "datafusion-physical-expr-common", "hex", "itertools", "log", - "md-5", + "md-5 0.11.0", "memchr", "num-traits", "rand 0.9.4", "regex", "sha2", - "unicode-segmentation", "uuid", ] [[package]] name = "datafusion-functions-aggregate" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00aa6217e56098ba84e0a338176fe52f0a84cca398021512c6c8c5eff806d0ad" +checksum = "89bc17041e424a47ed062f43df24d84aab8b57c4c3221e5c1a5eef46d6c5718b" dependencies = [ - "ahash", "arrow", "datafusion-common", "datafusion-doc", @@ -1211,19 +1171,18 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr", "datafusion-physical-expr-common", + "foldhash 0.2.0", "half", "log", "num-traits", - "paste", ] [[package]] name = "datafusion-functions-aggregate-common" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b511250349407db7c43832ab2de63f5557b19a20dfd236b39ca2c04468b50d47" +checksum = "97dd2a9e865c6108059f5b37b77934f84b50bfb108f837bd0e5c9536e03f0545" dependencies = [ - "ahash", "arrow", "datafusion-common", "datafusion-expr-common", @@ -1232,9 +1191,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef13a858e20d50f0a9bb5e96e7ac82b4e7597f247515bccca4fdd2992df0212a" +checksum = "75f0bdfeef16d96417b9632ef855645376b242e9006a126dfd0bedfc54a93f5f" dependencies = [ "arrow", "arrow-ord", @@ -1248,34 +1207,34 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-macros", "datafusion-physical-expr-common", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "itertools", "itoa", "log", - "paste", + "memchr", ] [[package]] name = "datafusion-functions-table" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b40d3f5bbb3905f9ccb1ce9485a9595c77b69758a7c24d3ba79e334ff51e7e" +checksum = "f4e4941673c917819616877e9993da4503e4f4739812be0bc32c5356184c6383" dependencies = [ "arrow", "async-trait", "datafusion-catalog", "datafusion-common", "datafusion-expr", + "datafusion-physical-expr", "datafusion-physical-plan", "parking_lot", - "paste", ] [[package]] name = "datafusion-functions-window" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e88ec9d57c9b685d02f58bfee7be62d72610430ddcedb82a08e5d9925dbfb6" +checksum = "12dd2e16c12b84b6f6b41b19f55b366dd1c46876bb35b86896c6349067379e8d" dependencies = [ "arrow", "datafusion-common", @@ -1286,14 +1245,13 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "log", - "paste", ] [[package]] name = "datafusion-functions-window-common" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8307bb93519b1a91913723a1130cfafeee3f72200d870d88e91a6fc5470ede5c" +checksum = "4cdc5e4b6f8b6ef823cc1c761f85088ad4c884fe8df64df3cbcc6b2b84698441" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -1333,9 +1291,9 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e367e6a71051d0ebdd29b2f85d12059b38b1d1f172c6906e80016da662226bd" +checksum = "1a3614234dd93578c92428cb4f408e020874f0d2b7e6c90c928d9d28b5df2ceb" dependencies = [ "datafusion-doc", "quote", @@ -1344,9 +1302,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e929015451a67f77d9d8b727b2bf3a40c4445fdef6cdc53281d7d97c76888ace" +checksum = "a0635620b050b81bb92764e99250868f654e2cd5ad1bece413283b3f73c83179" dependencies = [ "arrow", "chrono", @@ -1364,11 +1322,10 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b1e68aba7a4b350401cfdf25a3d6f989ad898a7410164afe9ca52080244cb59" +checksum = "8cabf7a86eb70b816729e33c81bf7767c936ee1226f607a114f5dac2decac8d0" dependencies = [ - "ahash", "arrow", "datafusion-common", "datafusion-expr", @@ -1376,11 +1333,10 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-physical-expr-common", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap", "itertools", "parking_lot", - "paste", "petgraph", "recursive", "tokio", @@ -1388,9 +1344,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea22315f33cf2e0adc104e8ec42e285f6ed93998d565c65e82fec6a9ee9f9db4" +checksum = "de222e04f7e6744501555a54ab0abe26bfdfebee380af79a9bdc175704246859" dependencies = [ "arrow", "datafusion-common", @@ -1403,26 +1359,26 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b04b45ea8ad3ac2d78f2ea2a76053e06591c9629c7a603eda16c10649ecf4362" +checksum = "72d0d0057fc5a502d45c870cb6d47c66eb7bdd5edb1bd71ad6f3f724975ac2a8" dependencies = [ - "ahash", "arrow", "chrono", "datafusion-common", "datafusion-expr-common", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap", "itertools", "parking_lot", + "pin-project", ] [[package]] name = "datafusion-physical-optimizer" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cb13397809a425918f608dfe8653f332015a3e330004ab191b4404187238b95" +checksum = "86046eed10950c5f9aaed9acfd148e9bd2e1dfdfe4f9aef607d1447b271e4183" dependencies = [ "arrow", "datafusion-common", @@ -1439,12 +1395,13 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5edc023675791af9d5fb4cc4c24abf5f7bd3bd4dcf9e5bd90ea1eff6976dcc79" +checksum = "9bc84da934c903407ba297971ebcc020c4c1a38aafd765d6c144c76eff3fa6a1" dependencies = [ - "ahash", "arrow", + "arrow-data", + "arrow-ipc", "arrow-ord", "arrow-schema", "async-trait", @@ -1459,7 +1416,7 @@ dependencies = [ "datafusion-physical-expr-common", "futures", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap", "itertools", "log", @@ -1471,9 +1428,9 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a387aaef949dc16bb6abc81bd1af850ec7449183aef011214f9724957495738" +checksum = "67791dcfacd142a9d95f8f73c2a161094cc936d231d80c235f5017dacf24e84e" dependencies = [ "arrow", "chrono", @@ -1494,14 +1451,13 @@ dependencies = [ "datafusion-proto-common", "object_store", "prost", - "rand 0.9.4", ] [[package]] name = "datafusion-proto-common" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16e614c7c53a9c304c6a850b821010bb492e57300311835f1180613f9d2c63d9" +checksum = "b8cd9e80d637891645d074db0f6c650b591117367247deb313fdfb78dff559cb" dependencies = [ "arrow", "datafusion-common", @@ -1510,9 +1466,9 @@ dependencies = [ [[package]] name = "datafusion-pruning" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac8c76860e355616555081cab5968cec1af7a80701ff374510860bcd567e365a" +checksum = "eb63eeac6de19be40f487b65dd84e546195f783c5a9928618e0c4f2a3569b0d7" dependencies = [ "arrow", "datafusion-common", @@ -1521,15 +1477,14 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", - "itertools", "log", ] [[package]] name = "datafusion-session" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5412111aa48e2424ba926112e192f7a6b7e4ccb450145d25ce5ede9f19dc491e" +checksum = "5f961d209177f91bd014db5cbb2c33b7d28a2597b9003e77f17aeb712964315a" dependencies = [ "async-trait", "datafusion-common", @@ -1541,9 +1496,9 @@ dependencies = [ [[package]] name = "datafusion-spark" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e059dcf8544da0d6598d0235be3cc29c209094a5976b2e4822e4a2cf91c2b5c5" +checksum = "a1ccd16a6949503e56c084df1b90c8889db826ec9347d2f0f51a7837d6fa011e" dependencies = [ "arrow", "bigdecimal", @@ -1556,21 +1511,24 @@ dependencies = [ "datafusion-expr", "datafusion-functions", "datafusion-functions-aggregate", + "datafusion-functions-aggregate-common", "datafusion-functions-nested", "log", + "num-traits", "percent-encoding", "rand 0.9.4", "serde_json", "sha1", "sha2", + "twox-hash", "url", ] [[package]] name = "datafusion-sql" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa0d133ddf8b9b3b872acac900157f783e7b879fe9a6bccf389abebbfac45ec1" +checksum = "b1d71cb454da682b2af7488e1fc1ddd72ee1b28f19297b8ccad73f0a21ee9a69" dependencies = [ "arrow", "bigdecimal", @@ -1587,9 +1545,9 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98494539a5468979cc42d86c7bc5f0f8cb71ee5c742694c26fc34efdd29dd2e5" +checksum = "f047a6fbf967b6b523758a48d2377bb5f9373a20e7ae4c4326d3c569f80ba3d7" dependencies = [ "async-recursion", "async-trait", @@ -1611,11 +1569,22 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.10.4", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid", + "crypto-common 0.2.2", +] + [[package]] name = "displaydoc" version = "0.2.6" @@ -1928,6 +1897,11 @@ name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] [[package]] name = "heck" @@ -1986,6 +1960,15 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" +[[package]] +name = "hybrid-array" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.10.1" @@ -2158,12 +2141,6 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" -[[package]] -name = "ident_case" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" - [[package]] name = "idna" version = "1.1.0" @@ -2439,7 +2416,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ "cfg-if", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", ] [[package]] @@ -2483,7 +2470,6 @@ checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ "num-integer", "num-traits", - "serde", ] [[package]] @@ -2542,7 +2528,7 @@ dependencies = [ "humantime", "hyper", "itertools", - "md-5", + "md-5 0.10.6", "parking_lot", "percent-encoding", "quick-xml", @@ -2721,6 +2707,26 @@ dependencies = [ "siphasher", ] +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -2904,12 +2910,6 @@ dependencies = [ "cc", ] -[[package]] -name = "quad-rand" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40" - [[package]] name = "quick-xml" version = "0.39.4" @@ -3094,12 +3094,6 @@ dependencies = [ "regex-syntax", ] -[[package]] -name = "regex-lite" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" - [[package]] name = "regex-syntax" version = "0.8.11" @@ -3356,16 +3350,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "serde_bytes" -version = "0.11.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" -dependencies = [ - "serde", - "serde_core", -] - [[package]] name = "serde_core" version = "1.0.228" @@ -3403,6 +3387,7 @@ version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ + "indexmap", "itoa", "memchr", "serde", @@ -3449,24 +3434,24 @@ dependencies = [ [[package]] name = "sha1" -version = "0.10.6" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" dependencies = [ "cfg-if", - "cpufeatures 0.2.17", - "digest", + "cpufeatures", + "digest 0.11.3", ] [[package]] name = "sha2" -version = "0.10.9" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" dependencies = [ "cfg-if", - "cpufeatures 0.2.17", - "digest", + "cpufeatures", + "digest 0.11.3", ] [[package]] @@ -3523,9 +3508,9 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.61.0" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf5ea8d4d7c808e1af1cbabebca9a2abe603bcefc22294c5b95018d53200cb7" +checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec" dependencies = [ "log", "recursive", @@ -3562,23 +3547,11 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "strsim" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" - -[[package]] -name = "strum" -version = "0.27.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" - [[package]] name = "strum_macros" -version = "0.27.2" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" dependencies = [ "heck", "proc-macro2", @@ -3588,11 +3561,12 @@ dependencies = [ [[package]] name = "substrait" -version = "0.62.2" +version = "0.63.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62fc4b483a129b9772ccb9c3f7945a472112fdd9140da87f8a4e7f1d44e045d0" +checksum = "e620ff4d5c02fd6f7752931aa74b16a26af66a63022cc1ad412c77edbe0bab47" dependencies = [ "heck", + "indexmap", "pbjson", "pbjson-build", "pbjson-types", @@ -3907,6 +3881,9 @@ name = "twox-hash" version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" +dependencies = [ + "rand 0.9.4", +] [[package]] name = "typenum" @@ -4023,7 +4000,6 @@ checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7" dependencies = [ "getrandom 0.4.2", "js-sys", - "serde_core", "wasm-bindgen", ] diff --git a/Cargo.toml b/Cargo.toml index fd1971a..57256e6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,13 +39,13 @@ repository = "https://github.com/apache/datafusion-java" [workspace.dependencies] arrow = { version = "58", features = ["ffi"] } async-trait = "0.1" -datafusion = { version = "53.1.0" } -datafusion-proto = "53.1.0" -datafusion-spark = "53.1.0" -datafusion-substrait = "53.1.0" +datafusion = { version = "54.1.0" } +datafusion-proto = "54.1.0" +datafusion-spark = "54.1.0" +datafusion-substrait = "54.1.0" futures = "0.3" jni = "0.21" -# Pinned to the major DataFusion 53.1 pulls in transitively (0.13.x) so we +# Pinned to the major DataFusion 54.1 pulls in transitively (0.13.x) so we # share the same `dyn ObjectStore` vtable and don't double-link. object_store = { version = "0.13", default-features = false } prost = "0.14" diff --git a/core/pom.xml b/core/pom.xml index 1e25736..ee2cb65 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -129,7 +129,7 @@ under the License. https://raw.githubusercontent.com/apache/datafusion/${datafusion.version}/datafusion/proto-common/proto/datafusion_common.proto ${project.build.directory}/proto/datafusion/proto-common/proto datafusion_common.proto - d6f3368372ea277cc23e26f196994b81616d38599357bb374cbd7eb1760e649a789e4c133d86a395ac701049a500348da2ec039d3f978ac5d8112c2876dded1f + e6a20115badec14f641b9237b0faa3f553162992444208322558ea3fcfaa24ca15334e4945827259dcb7c1de7f9aed659d496974eb8eeb1cfcf54ad4d7f6d20f @@ -140,7 +140,7 @@ under the License. https://raw.githubusercontent.com/apache/datafusion/${datafusion.version}/datafusion/proto/proto/datafusion.proto ${project.build.directory}/proto/datafusion/proto/proto datafusion.proto - c3d162b8e2a418e03f74caceaccfd934af89bb95a12ede13d4cc1701d24c734d74b1e96372142b173db05938dab7f965ad60d476363308c441677a63ea5fbcf7 + bdae5361d32b4d4f0418677ce650ee8cc212fddfe9390b2c033ee4ab3bb3c969a6148b7b318b9bb268ee9384f18d4254ebe6e0701f15639059a8f2800d6e7847 diff --git a/core/src/main/java/org/apache/datafusion/IoException.java b/core/src/main/java/org/apache/datafusion/IoException.java index 79f196e..42cdad7 100644 --- a/core/src/main/java/org/apache/datafusion/IoException.java +++ b/core/src/main/java/org/apache/datafusion/IoException.java @@ -21,9 +21,8 @@ /** * IO-shaped failure: a local filesystem read failed, an object store request failed, or a parquet / - * arrow / avro decoder reported a malformed file. Surfaces upstream {@code - * DataFusionError::IoError}, {@code ObjectStore}, {@code ArrowError}, {@code ParquetError}, and - * {@code AvroError}. + * arrow decoder reported a malformed file. Surfaces upstream {@code DataFusionError::IoError}, + * {@code ObjectStore}, {@code ParquetError}, and the IO-shaped {@code ArrowError} variants. * *

Note: this is {@code org.apache.datafusion.IoException} (lowercase {@code o}), distinct from * {@code java.io.IOException}. The {@code IoException} spelling matches the orthography of the diff --git a/core/src/test/java/org/apache/datafusion/DataFrameIntrospectionTest.java b/core/src/test/java/org/apache/datafusion/DataFrameIntrospectionTest.java index 91cf1e7..ede96d9 100644 --- a/core/src/test/java/org/apache/datafusion/DataFrameIntrospectionTest.java +++ b/core/src/test/java/org/apache/datafusion/DataFrameIntrospectionTest.java @@ -195,7 +195,7 @@ void describeReturnsSummaryStats() throws Exception { } } } - // DataFusion 53.1 reports these seven labels. + // DataFusion 54.1 reports these seven labels. assertTrue(seen.contains("count"), () -> "labels=" + seen); assertTrue(seen.contains("null_count"), () -> "labels=" + seen); assertTrue(seen.contains("mean"), () -> "labels=" + seen); diff --git a/core/src/test/java/org/apache/datafusion/SessionContextSubstraitTest.java b/core/src/test/java/org/apache/datafusion/SessionContextSubstraitTest.java index a2cfb0a..2877fcf 100644 --- a/core/src/test/java/org/apache/datafusion/SessionContextSubstraitTest.java +++ b/core/src/test/java/org/apache/datafusion/SessionContextSubstraitTest.java @@ -73,7 +73,14 @@ static void checkFeatureEnabled() { /** * Build a minimal Substrait {@code Plan} that scans a registered named table {@code tableName} - * with two columns {@code (id int32, v int32)} and projects them through unchanged. + * with two columns {@code (id int64, v int64)} and projects them through unchanged. + * + *

The columns are declared {@code NULLABILITY_NULLABLE} to match the schema DataFusion infers + * for the CSV these tests register — CSV inference always yields nullable fields. DataFusion's + * Substrait consumer verifies that a field the plan declares non-nullable really is non-nullable + * in the table, and rejects the plan otherwise: a plan built around a "never null" assumption + * must not silently run against data that can contain nulls. The enclosing struct stays {@code + * REQUIRED} — the row itself is always present. */ private static Plan namedTableScanPlan(String tableName) { NamedStruct schema = @@ -87,13 +94,13 @@ private static Plan namedTableScanPlan(String tableName) { .setI64( Type.I64 .newBuilder() - .setNullability(Type.Nullability.NULLABILITY_REQUIRED))) + .setNullability(Type.Nullability.NULLABILITY_NULLABLE))) .addTypes( Type.newBuilder() .setI64( Type.I64 .newBuilder() - .setNullability(Type.Nullability.NULLABILITY_REQUIRED))) + .setNullability(Type.Nullability.NULLABILITY_NULLABLE))) .setNullability(Type.Nullability.NULLABILITY_REQUIRED)) .build(); ReadRel read = diff --git a/native-common/Cargo.toml b/native-common/Cargo.toml index 21a2296..d136dcd 100644 --- a/native-common/Cargo.toml +++ b/native-common/Cargo.toml @@ -27,13 +27,6 @@ publish = false readme = "README.md" description = "Shared JNI plumbing for DataFusion Java native crates: error-to-exception mapping, the per-cdylib Tokio runtime singleton, and the async-stream-to-FFI_ArrowArrayStream bridge." -[features] -# `datafusion-jni` builds DataFusion with `avro`, which adds the -# `DataFusionError::AvroError` variant our classifier maps to IoException. -# Feature-forwarded so consumers that don't read Avro (the Spark helper) -# don't pull the apache-avro stack into their cdylib. -avro = ["datafusion/avro"] - [dependencies] datafusion = { workspace = true } futures = { workspace = true } diff --git a/native-common/src/errors.rs b/native-common/src/errors.rs index f9dbb03..b3e438d 100644 --- a/native-common/src/errors.rs +++ b/native-common/src/errors.rs @@ -97,10 +97,6 @@ fn classify(err: &DataFusionError) -> &'static str { DataFusionError::IoError(_) | DataFusionError::ObjectStore(_) | DataFusionError::ParquetError(_) => "org/apache/datafusion/IoException", - // The AvroError variant only exists when DataFusion is built with its - // `avro` feature, forwarded by this crate's own `avro` feature. - #[cfg(feature = "avro")] - DataFusionError::AvroError(_) => "org/apache/datafusion/IoException", // ArrowError is a 21-variant grab bag -- only some of those variants // are actually IO-shaped. DivideByZero / ArithmeticOverflow / Compute // / Cast / InvalidArgument / Memory etc. are execution-time failures @@ -125,9 +121,15 @@ fn classify(err: &DataFusionError) -> &'static str { /// Map an [`ArrowError`] variant onto the Java exception class to throw. /// Only the genuinely IO-shaped variants (`IoError`, `IpcError`) land on /// `IoException`; everything else is execution-time and routes through -/// `ExecutionException`. Schema/parse-shaped variants route through -/// `PlanException` so a malformed IPC schema or a parse error surfaces as a -/// query problem rather than an execution failure. +/// `ExecutionException` -- including `AvroError`, which is where Avro decode +/// failures land now that DataFusion 54 reads Avro through `arrow-avro` +/// instead of `apache-avro` (the old `DataFusionError::AvroError` variant is +/// gone). That puts Avro alongside the `CsvError` / `JsonError` decoder +/// variants, which this arm already classified as execution failures. +/// +/// Schema/parse-shaped variants route through `PlanException` so a malformed +/// IPC schema or a parse error surfaces as a query problem rather than an +/// execution failure. /// /// Variants without a clean caller-facing category (`CDataInterface`, the /// various overflow/index-overflow markers) fall through to the parent. diff --git a/native/Cargo.toml b/native/Cargo.toml index c040448..8f4add5 100644 --- a/native/Cargo.toml +++ b/native/Cargo.toml @@ -82,9 +82,8 @@ arrow = { workspace = true } async-trait = { workspace = true } datafusion = { workspace = true, features = ["avro"] } # Shared JNI plumbing (error->exception mapping, runtime singleton, -# StreamingReader). `avro` keeps the classifier's AvroError->IoException arm -# in sync with the `avro` feature on `datafusion` above. -datafusion-jni-common = { path = "../native-common", features = ["avro"] } +# StreamingReader). +datafusion-jni-common = { path = "../native-common" } datafusion-proto = { workspace = true } # Apache Spark-compatible functions + expression planners. Optional and # gated behind the `spark` feature (in the default set). The `core` feature diff --git a/native/src/cache_manager.rs b/native/src/cache_manager.rs index ec38dc8..4be7070 100644 --- a/native/src/cache_manager.rs +++ b/native/src/cache_manager.rs @@ -29,10 +29,8 @@ use std::sync::Arc; use std::time::Duration; use datafusion::execution::cache::cache_manager::CacheManagerConfig; -use datafusion::execution::cache::cache_unit::{ - DefaultFileStatisticsCache, DefaultFilesMetadataCache, -}; -use datafusion::execution::cache::DefaultListFilesCache; +use datafusion::execution::cache::file_statistics_cache::DefaultFileStatisticsCache; +use datafusion::execution::cache::{DefaultFilesMetadataCache, DefaultListFilesCache}; use crate::proto_gen::CacheManagerOptionsProto; use datafusion_jni_common::errors::JniResult; @@ -76,8 +74,20 @@ pub(crate) fn build_config( config.list_files_cache_ttl = ttl; } - if opts.file_statistics_cache_enabled.unwrap_or(false) { - config.table_files_statistics_cache = Some(Arc::new(DefaultFileStatisticsCache::default())); + // DataFusion 54 renamed `table_files_statistics_cache` to + // `file_statistics_cache` and made the *limit* the on/off switch: + // `CacheManager::try_new` now builds a `DefaultFileStatisticsCache` + // whenever `file_statistics_cache_limit > 0`, even when the cache slot is + // `None`. The default limit is non-zero, so an explicit `false` from the + // Java surface has to zero the limit -- otherwise upstream would install a + // stats cache the caller just asked us not to. + if let Some(enabled) = opts.file_statistics_cache_enabled { + if enabled { + config.file_statistics_cache = Some(Arc::new(DefaultFileStatisticsCache::default())); + } else { + config.file_statistics_cache = None; + config.file_statistics_cache_limit = 0; + } } Ok(Some(config)) diff --git a/native/src/memory.rs b/native/src/memory.rs index e105299..8068aba 100644 --- a/native/src/memory.rs +++ b/native/src/memory.rs @@ -120,7 +120,27 @@ impl TrackingMemoryPool { } } +/// DataFusion 54 added `Display` as a supertrait of [`MemoryPool`] so pools +/// can render themselves in resource-exhausted messages. Mirror upstream's +/// wrapper convention: name the wrapper, then defer to the inner pool for the +/// limit/usage detail, and add the counters this wrapper exists to expose. +impl std::fmt::Display for TrackingMemoryPool { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let (current, peak) = self.snapshot(); + write!( + f, + "{}(current: {current}, peak: {peak}, inner: {})", + self.name(), + self.inner + ) + } +} + impl MemoryPool for TrackingMemoryPool { + fn name(&self) -> &str { + "tracking" + } + fn register(&self, consumer: &datafusion::execution::memory_pool::MemoryConsumer) { self.inner.register(consumer); } diff --git a/native/src/table_provider.rs b/native/src/table_provider.rs index 70eaac2..51c892b 100644 --- a/native/src/table_provider.rs +++ b/native/src/table_provider.rs @@ -22,7 +22,6 @@ //! `TableProvider` trait; it currently only supports a single-partition, no-pushdown scan, //! with future pushdown and partitioning support tracked as follow-ups. -use std::any::Any; use std::fmt; use std::sync::Arc; @@ -74,10 +73,6 @@ impl fmt::Debug for JavaTableProvider { #[async_trait] impl TableProvider for JavaTableProvider { - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { Arc::clone(&self.schema) } @@ -152,10 +147,6 @@ impl ExecutionPlan for JavaScanExec { "JavaScanExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn properties(&self) -> &Arc { &self.plan_properties } diff --git a/native/src/udf.rs b/native/src/udf.rs index da7b260..3c3c7da 100644 --- a/native/src/udf.rs +++ b/native/src/udf.rs @@ -17,7 +17,6 @@ //! Java-backed scalar UDF support. -use std::any::Any; use std::fmt; use datafusion::arrow::array::{make_array, Array, ArrayRef, StructArray}; @@ -80,10 +79,6 @@ impl std::hash::Hash for JavaScalarUdf { } impl ScalarUDFImpl for JavaScalarUdf { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { &self.name } diff --git a/pom.xml b/pom.xml index 7ceec07..d9f3212 100644 --- a/pom.xml +++ b/pom.xml @@ -40,7 +40,7 @@ under the License. 17 UTF-8 5.11.3 - 53.1.0 + 54.1.0 3.25.5 19.0.0 1.12.0