diff --git a/.cargo/config.toml b/.cargo/config.toml
new file mode 100644
index 0000000..d7e0ee2
--- /dev/null
+++ b/.cargo/config.toml
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Keep Cargo's workspace output out of `target/` so `mvn clean` (which deletes
+# the root `target/`) does not nuke the Rust build cache.
+[build]
+target-dir = "rust-target"
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index c5db936..da8e65a 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -83,8 +83,8 @@ jobs:
path: |
~/.cargo/registry
~/.cargo/git
- native/target
- key: ${{ runner.os }}-cargo-${{ hashFiles('native/Cargo.lock') }}
+ rust-target
+ key: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.lock') }}
restore-keys: ${{ runner.os }}-cargo-
- name: Build native and run tests
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
index 4cf628f..952bf34 100644
--- a/.github/workflows/lint.yml
+++ b/.github/workflows/lint.yml
@@ -54,7 +54,7 @@ jobs:
run: ./mvnw -q spotless:check
- name: Check Rust formatting
- run: cd native && cargo fmt --all -- --check
+ run: cargo fmt --all -- --check
clippy:
name: Clippy
@@ -81,9 +81,9 @@ jobs:
path: |
~/.cargo/registry
~/.cargo/git
- native/target
- key: ${{ runner.os }}-clippy-${{ hashFiles('native/Cargo.lock') }}
+ rust-target
+ key: ${{ runner.os }}-clippy-${{ hashFiles('Cargo.lock') }}
restore-keys: ${{ runner.os }}-clippy-
- name: Run clippy
- run: cd native && cargo clippy --all-targets -- -D warnings
+ run: cargo clippy --workspace --all-targets -- -D warnings
diff --git a/.gitignore b/.gitignore
index 719a2a4..25c9216 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,5 @@
target/
+rust-target/
*.class
.idea/
.vscode/
diff --git a/native/Cargo.lock b/Cargo.lock
similarity index 89%
rename from native/Cargo.lock
rename to Cargo.lock
index 8c56280..ab96620 100644
--- a/native/Cargo.lock
+++ b/Cargo.lock
@@ -67,40 +67,11 @@ version = "1.0.102"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"
-[[package]]
-name = "apache-avro"
-version = "0.21.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "36fa98bc79671c7981272d91a8753a928ff6a1cd8e4f20a44c45bd5d313840bf"
-dependencies = [
- "bigdecimal",
- "bon",
- "bzip2",
- "crc32fast",
- "digest",
- "liblzma",
- "log",
- "miniz_oxide",
- "num-bigint",
- "quad-rand",
- "rand 0.9.4",
- "regex-lite",
- "serde",
- "serde_bytes",
- "serde_json",
- "snap",
- "strum",
- "strum_macros",
- "thiserror 2.0.18",
- "uuid",
- "zstd",
-]
-
[[package]]
name = "ar_archive_writer"
-version = "0.5.1"
+version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7eb93bbb63b9c227414f6eb3a0adfddca591a8ce1e9b60661bb08969b87e340b"
+checksum = "4087686b4b0a3427190bae57a1d9a478dbb2d40c5dc1bd6e2b6d797913bdd348"
dependencies = [
"object",
]
@@ -119,9 +90,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
[[package]]
name = "arrow"
-version = "58.2.0"
+version = "58.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "607e64bb911ee4f90483e044fe78f175989148c2892e659a2cd25429e782ec54"
+checksum = "378530e55cd479eda3c14eb345310799717e6f76d0c332041e8487022166b471"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -140,9 +111,9 @@ dependencies = [
[[package]]
name = "arrow-arith"
-version = "58.2.0"
+version = "58.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e754319ed8a85d817fe7adf183227e0b5308b82790a737b426c1124626b48118"
+checksum = "a0ab212d2c1886e802f51c5212d78ebbcbb0bec980fff9dadc1eb8d45cd0b738"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -154,9 +125,9 @@ dependencies = [
[[package]]
name = "arrow-array"
-version = "58.2.0"
+version = "58.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "841321891f247aa86c6112c80d83d89cb36e0addd020fa2425085b8eb6c3f579"
+checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e"
dependencies = [
"ahash",
"arrow-buffer",
@@ -171,11 +142,35 @@ dependencies = [
"num-traits",
]
+[[package]]
+name = "arrow-avro"
+version = "58.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "049230728cd6e093088c8d231b4beede184e35cad7777c1505c0d5a8571f4376"
+dependencies = [
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-schema",
+ "bytes",
+ "bzip2",
+ "crc",
+ "flate2",
+ "indexmap",
+ "liblzma",
+ "rand 0.9.4",
+ "serde",
+ "serde_json",
+ "snap",
+ "strum_macros",
+ "uuid",
+ "zstd",
+]
+
[[package]]
name = "arrow-buffer"
-version = "58.2.0"
+version = "58.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f955dfb73fae000425f49c8226d2044dab60fb7ad4af1e24f961756354d996c9"
+checksum = "0c6cd424c2693bcdbc150d843dc9d4d137dd2de4782ce6df491ad11a3a0416c0"
dependencies = [
"bytes",
"half",
@@ -185,9 +180,9 @@ dependencies = [
[[package]]
name = "arrow-cast"
-version = "58.2.0"
+version = "58.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ca5e686972523798f76bef355145bc1ae25a84c731e650268d31ab763c701663"
+checksum = "4c5aefb56a2c02e9e2b30746241058b85f8983f0fcff2ba0c6d09006e1cded7f"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -207,9 +202,9 @@ dependencies = [
[[package]]
name = "arrow-csv"
-version = "58.2.0"
+version = "58.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "86c276756867fc8186ec380c72c290e6e3b23a1d4fb05df6b1d62d2e62666d48"
+checksum = "e94e8cf7e517657a52b91ea1263acf38c4ca62a84655d72458a3359b12ab97de"
dependencies = [
"arrow-array",
"arrow-cast",
@@ -222,9 +217,9 @@ dependencies = [
[[package]]
name = "arrow-data"
-version = "58.2.0"
+version = "58.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "db3b5846209775b6dc8056d77ff9a032b27043383dd5488abd0b663e265b9373"
+checksum = "3c88210023a2bfee1896af366309a3028fc3bcbd6515fa29a7990ee1baa08ee0"
dependencies = [
"arrow-buffer",
"arrow-schema",
@@ -235,9 +230,9 @@ dependencies = [
[[package]]
name = "arrow-ipc"
-version = "58.2.0"
+version = "58.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd8907ddd8f9fbabf91ec2c85c1d81fe2874e336d2443eb36373595e28b98dd5"
+checksum = "238438f0834483703d88896db6fe5a7138b2230debc31b34c0336c2996e3c64f"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -251,9 +246,9 @@ dependencies = [
[[package]]
name = "arrow-json"
-version = "58.2.0"
+version = "58.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f4518c59acc501f10d7dcae397fe12b8db3d81bc7de94456f8a58f9165d6f502"
+checksum = "205ca2119e6d679d5c133c6f30e68f027738d95ed948cf77677ea69c7800036b"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -276,9 +271,9 @@ dependencies = [
[[package]]
name = "arrow-ord"
-version = "58.2.0"
+version = "58.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "efa70d9d6b1356f1fb9f1f651b84a725b7e0abb93f188cf7d31f14abfa2f2e6f"
+checksum = "1bffd8fd2579286a5d63bac898159873e5094a79009940bcb42bbfce4f19f1d0"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -289,9 +284,9 @@ dependencies = [
[[package]]
name = "arrow-row"
-version = "58.2.0"
+version = "58.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "faec88a945338192beffbbd4be0def70135422930caa244ac3cec0cd213b26b4"
+checksum = "bab5994731204603c73ba69267616c50f80780774c6bb0476f1f830625115e0c"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -302,9 +297,9 @@ dependencies = [
[[package]]
name = "arrow-schema"
-version = "58.2.0"
+version = "58.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "18aa020f6bc8e5201dcd2d4b7f98c68f8a410ef37128263243e6ff2a47a67d4f"
+checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed"
dependencies = [
"bitflags",
"serde_core",
@@ -313,9 +308,9 @@ dependencies = [
[[package]]
name = "arrow-select"
-version = "58.2.0"
+version = "58.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a657ab5132e9c8ca3b24eb15a823d0ced38017fe3930ff50167466b02e2d592c"
+checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222"
dependencies = [
"ahash",
"arrow-array",
@@ -327,9 +322,9 @@ dependencies = [
[[package]]
name = "arrow-string"
-version = "58.2.0"
+version = "58.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f6de2efbbd1a9f9780ceb8d1ff5d20421b35863b361e3386b4f571f1fc69fcb8"
+checksum = "29dd7cda3ab9692f43a2e4acc444d760cc17b12bb6d8232ddf64e9bab7c06b42"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -393,9 +388,9 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
[[package]]
name = "autocfg"
-version = "1.5.0"
+version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
+checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
[[package]]
name = "base64"
@@ -414,14 +409,13 @@ dependencies = [
"num-bigint",
"num-integer",
"num-traits",
- "serde",
]
[[package]]
name = "bitflags"
-version = "2.11.1"
+version = "2.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3"
+checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8"
[[package]]
name = "blake2"
@@ -429,7 +423,7 @@ version = "0.10.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe"
dependencies = [
- "digest",
+ "digest 0.10.7",
]
[[package]]
@@ -443,7 +437,7 @@ dependencies = [
"cc",
"cfg-if",
"constant_time_eq",
- "cpufeatures 0.3.0",
+ "cpufeatures",
]
[[package]]
@@ -456,35 +450,19 @@ dependencies = [
]
[[package]]
-name = "bon"
-version = "3.9.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f47dbe92550676ee653353c310dfb9cf6ba17ee70396e1f7cf0a2020ad49b2fe"
-dependencies = [
- "bon-macros",
- "rustversion",
-]
-
-[[package]]
-name = "bon-macros"
-version = "3.9.1"
+name = "block-buffer"
+version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "519bd3116aeeb42d5372c29d982d16d0170d3d4a5ed85fc7dd91642ffff3c67c"
+checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa"
dependencies = [
- "darling",
- "ident_case",
- "prettyplease",
- "proc-macro2",
- "quote",
- "rustversion",
- "syn",
+ "hybrid-array",
]
[[package]]
name = "brotli"
-version = "8.0.2"
+version = "8.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4bd8b9603c7aa97359dbd97ecf258968c95f3adddd6db2f7e7a5bef101c84560"
+checksum = "8119e4516436f5708bbc474a9d395bf12f1b5395e93a92a56e647ac3388c8610"
dependencies = [
"alloc-no-stdlib",
"alloc-stdlib",
@@ -493,9 +471,9 @@ dependencies = [
[[package]]
name = "brotli-decompressor"
-version = "5.0.0"
+version = "5.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03"
+checksum = "5962523e1b92ce1b5e793d9169b9943eece10d39f62550bc04bb605d75b94924"
dependencies = [
"alloc-no-stdlib",
"alloc-stdlib",
@@ -503,9 +481,9 @@ dependencies = [
[[package]]
name = "bumpalo"
-version = "3.20.2"
+version = "3.20.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb"
+checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
[[package]]
name = "byteorder"
@@ -530,9 +508,9 @@ dependencies = [
[[package]]
name = "cc"
-version = "1.2.62"
+version = "1.2.63"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98"
+checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f"
dependencies = [
"find-msvc-tools",
"jobserver",
@@ -565,15 +543,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601"
dependencies = [
"cfg-if",
- "cpufeatures 0.3.0",
+ "cpufeatures",
"rand_core 0.10.1",
]
[[package]]
name = "chrono"
-version = "0.4.44"
+version = "0.4.45"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0"
+checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327"
dependencies = [
"iana-time-zone",
"num-traits",
@@ -641,6 +619,12 @@ version = "0.4.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789"
+[[package]]
+name = "const-oid"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c"
+
[[package]]
name = "const-random"
version = "0.1.18"
@@ -685,22 +669,28 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
[[package]]
name = "cpufeatures"
-version = "0.2.17"
+version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
+checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201"
dependencies = [
"libc",
]
[[package]]
-name = "cpufeatures"
-version = "0.3.0"
+name = "crc"
+version = "3.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201"
+checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d"
dependencies = [
- "libc",
+ "crc-catalog",
]
+[[package]]
+name = "crc-catalog"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853"
+
[[package]]
name = "crc32fast"
version = "1.5.0"
@@ -732,6 +722,15 @@ dependencies = [
"typenum",
]
+[[package]]
+name = "crypto-common"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453"
+dependencies = [
+ "hybrid-array",
+]
+
[[package]]
name = "csv"
version = "1.4.0"
@@ -753,45 +752,11 @@ dependencies = [
"memchr",
]
-[[package]]
-name = "darling"
-version = "0.23.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d"
-dependencies = [
- "darling_core",
- "darling_macro",
-]
-
-[[package]]
-name = "darling_core"
-version = "0.23.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0"
-dependencies = [
- "ident_case",
- "proc-macro2",
- "quote",
- "strsim",
- "syn",
-]
-
-[[package]]
-name = "darling_macro"
-version = "0.23.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d"
-dependencies = [
- "darling_core",
- "quote",
- "syn",
-]
-
[[package]]
name = "dashmap"
-version = "6.1.0"
+version = "6.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf"
+checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c"
dependencies = [
"cfg-if",
"crossbeam-utils",
@@ -803,14 +768,13 @@ dependencies = [
[[package]]
name = "datafusion"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "93db0e623840612f7f2cd757f7e8a8922064192363732c88692e0870016e141b"
+checksum = "754ef4e8f073922a26f5b23133b9db4829342362b09be0bc94309cf261c2f098"
dependencies = [
"arrow",
"arrow-schema",
"async-trait",
- "bytes",
"bzip2",
"chrono",
"datafusion-catalog",
@@ -841,14 +805,13 @@ dependencies = [
"datafusion-sql",
"flate2",
"futures",
+ "indexmap",
"itertools",
"liblzma",
"log",
"object_store",
"parking_lot",
"parquet",
- "rand 0.9.4",
- "regex",
"sqlparser",
"tempfile",
"tokio",
@@ -859,9 +822,9 @@ dependencies = [
[[package]]
name = "datafusion-catalog"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "37cefde60b26a7f4ff61e9d2ff2833322f91df2b568d7238afe67bde5bdffb66"
+checksum = "06afd1e38dd27bbb1258685a1fc6524df6aff4e07b25b393a47de59635178d99"
dependencies = [
"arrow",
"async-trait",
@@ -884,9 +847,9 @@ dependencies = [
[[package]]
name = "datafusion-catalog-listing"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "17e112307715d6a7a331111a4c2330ff54bc237183511c319e3708a4cff431fb"
+checksum = "f0668fb32c12065ec242be0e5b4bc62bd7a06a0be3ecd83791ef877e4be67e02"
dependencies = [
"arrow",
"async-trait",
@@ -907,35 +870,35 @@ dependencies = [
[[package]]
name = "datafusion-common"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d72a11ca44a95e1081870d3abb80c717496e8a7acb467a1d3e932bb636af5cc2"
+checksum = "ca43b263cdff57042cfa8fb817fb3469f4878933380dccff25f5e793580abbf9"
dependencies = [
- "ahash",
- "apache-avro",
"arrow",
"arrow-ipc",
+ "arrow-schema",
"chrono",
+ "foldhash 0.2.0",
"half",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
"indexmap",
"itertools",
"libc",
"log",
"object_store",
"parquet",
- "paste",
"recursive",
"sqlparser",
"tokio",
+ "uuid",
"web-time",
]
[[package]]
name = "datafusion-common-runtime"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89f4afaed29670ec4fd6053643adc749fe3f4bc9d1ce1b8c5679b22c67d12def"
+checksum = "05f0ba2b864792bdca4d76c59a1de0ab6e1b61946596b9936888dbd6360035f2"
dependencies = [
"futures",
"log",
@@ -944,9 +907,9 @@ dependencies = [
[[package]]
name = "datafusion-datasource"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9fb386e1691355355a96419978a0022b7947b44d4a24a6ea99f00b6b485cbb6"
+checksum = "b840a8bce0bcbf5afad02946d438591e7c373f7afccaf3d874c04485772514dd"
dependencies = [
"arrow",
"async-compression",
@@ -970,6 +933,7 @@ dependencies = [
"liblzma",
"log",
"object_store",
+ "parking_lot",
"rand 0.9.4",
"tokio",
"tokio-util",
@@ -979,9 +943,9 @@ dependencies = [
[[package]]
name = "datafusion-datasource-arrow"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ffa6c52cfed0734c5f93754d1c0175f558175248bf686c944fb05c373e5fc096"
+checksum = "a24cc0b9cf6e367f27f27406eff13abf48a11b72446aaa40b3105c0ded5c17d9"
dependencies = [
"arrow",
"arrow-ipc",
@@ -1003,29 +967,28 @@ dependencies = [
[[package]]
name = "datafusion-datasource-avro"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a579c3bd290c66ea4b269493e75e8a3ed42c9c895a651f10210a29538aee50c4"
+checksum = "f26a312bb06528c17b4bb86b798b0c4321e7656d85e0ef65c2e9adf07b0a518d"
dependencies = [
- "apache-avro",
"arrow",
+ "arrow-avro",
"async-trait",
"bytes",
"datafusion-common",
"datafusion-datasource",
- "datafusion-physical-expr-common",
+ "datafusion-physical-expr-adapter",
"datafusion-physical-plan",
"datafusion-session",
"futures",
- "num-traits",
"object_store",
]
[[package]]
name = "datafusion-datasource-csv"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "503f29e0582c1fc189578d665ff57d9300da1f80c282777d7eb67bb79fb8cdca"
+checksum = "e1abe56b2a7a2d1d6de5117dd1a203181e267f28529faa5da546947621b697d7"
dependencies = [
"arrow",
"async-trait",
@@ -1046,9 +1009,9 @@ dependencies = [
[[package]]
name = "datafusion-datasource-json"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e33804749abc8d0c8cb7473228483cb8070e524c6f6086ee1b85a64debe2b3d2"
+checksum = "9c3e467f0611ad7bdd5aad17c63c9bb6182d04e5282e5496d897ea2b49c024ba"
dependencies = [
"arrow",
"async-trait",
@@ -1063,16 +1026,15 @@ dependencies = [
"datafusion-session",
"futures",
"object_store",
- "serde_json",
"tokio",
"tokio-stream",
]
[[package]]
name = "datafusion-datasource-parquet"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32a8e0365e0e08e8ff94d912f0ababcf9065a1a304018ba90b1fc83c855b4997"
+checksum = "4cc35b92cd560082155e80d9c826929c852d3c51543f4affd3a51c464a0aab3a"
dependencies = [
"arrow",
"async-trait",
@@ -1082,6 +1044,7 @@ dependencies = [
"datafusion-datasource",
"datafusion-execution",
"datafusion-expr",
+ "datafusion-functions",
"datafusion-functions-aggregate-common",
"datafusion-physical-expr",
"datafusion-physical-expr-adapter",
@@ -1100,20 +1063,19 @@ dependencies = [
[[package]]
name = "datafusion-doc"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8de6ac0df1662b9148ad3c987978b32cbec7c772f199b1d53520c8fa764a87ee"
+checksum = "d69bb69d8769e34f76839c960dbde24c1ac0c885a79b6c3c2287bdc56ec67891"
[[package]]
name = "datafusion-execution"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c03c7fbdaefcca4ef6ffe425a5fc2325763bfb426599bb0bf4536466efabe709"
+checksum = "d8eac0a09bc8d263f52025cad9e001da4d8138d633fa288edda4d06b1772eae6"
dependencies = [
"arrow",
"arrow-buffer",
"async-trait",
- "chrono",
"dashmap",
"datafusion-common",
"datafusion-expr",
@@ -1129,11 +1091,12 @@ dependencies = [
[[package]]
name = "datafusion-expr"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "574b9b6977fedbd2a611cbff12e5caf90f31640ad9dc5870f152836d94bad0dd"
+checksum = "eeb14d374767ee0fc62dc79a5ba8bcf8a63c14e993c7d992d0e63adfa23d77d3"
dependencies = [
"arrow",
+ "arrow-schema",
"async-trait",
"chrono",
"datafusion-common",
@@ -1144,7 +1107,6 @@ dependencies = [
"datafusion-physical-expr-common",
"indexmap",
"itertools",
- "paste",
"recursive",
"serde_json",
"sqlparser",
@@ -1152,22 +1114,21 @@ dependencies = [
[[package]]
name = "datafusion-expr-common"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d7c3adf3db8bf61e92eb90cb659c8e8b734593a8f7c8e12a843c7ddba24b87e"
+checksum = "b7b19a8c95522bee8cbb313d74263b85e355d2b52f42e67ef5694bf5de9e9356"
dependencies = [
"arrow",
"datafusion-common",
"indexmap",
"itertools",
- "paste",
]
[[package]]
name = "datafusion-functions"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f28aa4e10384e782774b10e72aca4d93ef7b31aa653095d9d4536b0a3dbc51b6"
+checksum = "5f64c983bbbdcb729d921a2b2ac3375598719b5cc0c30345ad664936f3176fc7"
dependencies = [
"arrow",
"arrow-buffer",
@@ -1182,26 +1143,25 @@ dependencies = [
"datafusion-expr",
"datafusion-expr-common",
"datafusion-macros",
+ "datafusion-physical-expr-common",
"hex",
"itertools",
"log",
- "md-5",
+ "md-5 0.11.0",
"memchr",
"num-traits",
"rand 0.9.4",
"regex",
"sha2",
- "unicode-segmentation",
"uuid",
]
[[package]]
name = "datafusion-functions-aggregate"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "00aa6217e56098ba84e0a338176fe52f0a84cca398021512c6c8c5eff806d0ad"
+checksum = "89bc17041e424a47ed062f43df24d84aab8b57c4c3221e5c1a5eef46d6c5718b"
dependencies = [
- "ahash",
"arrow",
"datafusion-common",
"datafusion-doc",
@@ -1211,19 +1171,18 @@ dependencies = [
"datafusion-macros",
"datafusion-physical-expr",
"datafusion-physical-expr-common",
+ "foldhash 0.2.0",
"half",
"log",
"num-traits",
- "paste",
]
[[package]]
name = "datafusion-functions-aggregate-common"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b511250349407db7c43832ab2de63f5557b19a20dfd236b39ca2c04468b50d47"
+checksum = "97dd2a9e865c6108059f5b37b77934f84b50bfb108f837bd0e5c9536e03f0545"
dependencies = [
- "ahash",
"arrow",
"datafusion-common",
"datafusion-expr-common",
@@ -1232,9 +1191,9 @@ dependencies = [
[[package]]
name = "datafusion-functions-nested"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef13a858e20d50f0a9bb5e96e7ac82b4e7597f247515bccca4fdd2992df0212a"
+checksum = "75f0bdfeef16d96417b9632ef855645376b242e9006a126dfd0bedfc54a93f5f"
dependencies = [
"arrow",
"arrow-ord",
@@ -1248,34 +1207,34 @@ dependencies = [
"datafusion-functions-aggregate-common",
"datafusion-macros",
"datafusion-physical-expr-common",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
"itertools",
"itoa",
"log",
- "paste",
+ "memchr",
]
[[package]]
name = "datafusion-functions-table"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72b40d3f5bbb3905f9ccb1ce9485a9595c77b69758a7c24d3ba79e334ff51e7e"
+checksum = "f4e4941673c917819616877e9993da4503e4f4739812be0bc32c5356184c6383"
dependencies = [
"arrow",
"async-trait",
"datafusion-catalog",
"datafusion-common",
"datafusion-expr",
+ "datafusion-physical-expr",
"datafusion-physical-plan",
"parking_lot",
- "paste",
]
[[package]]
name = "datafusion-functions-window"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d4e88ec9d57c9b685d02f58bfee7be62d72610430ddcedb82a08e5d9925dbfb6"
+checksum = "12dd2e16c12b84b6f6b41b19f55b366dd1c46876bb35b86896c6349067379e8d"
dependencies = [
"arrow",
"datafusion-common",
@@ -1286,14 +1245,13 @@ dependencies = [
"datafusion-physical-expr",
"datafusion-physical-expr-common",
"log",
- "paste",
]
[[package]]
name = "datafusion-functions-window-common"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8307bb93519b1a91913723a1130cfafeee3f72200d870d88e91a6fc5470ede5c"
+checksum = "4cdc5e4b6f8b6ef823cc1c761f85088ad4c884fe8df64df3cbcc6b2b84698441"
dependencies = [
"datafusion-common",
"datafusion-physical-expr-common",
@@ -1306,7 +1264,9 @@ dependencies = [
"arrow",
"async-trait",
"datafusion",
+ "datafusion-jni-common",
"datafusion-proto",
+ "datafusion-spark",
"datafusion-substrait",
"futures",
"jni",
@@ -1319,11 +1279,21 @@ dependencies = [
"url",
]
+[[package]]
+name = "datafusion-jni-common"
+version = "0.1.0"
+dependencies = [
+ "datafusion",
+ "futures",
+ "jni",
+ "tokio",
+]
+
[[package]]
name = "datafusion-macros"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2e367e6a71051d0ebdd29b2f85d12059b38b1d1f172c6906e80016da662226bd"
+checksum = "1a3614234dd93578c92428cb4f408e020874f0d2b7e6c90c928d9d28b5df2ceb"
dependencies = [
"datafusion-doc",
"quote",
@@ -1332,9 +1302,9 @@ dependencies = [
[[package]]
name = "datafusion-optimizer"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e929015451a67f77d9d8b727b2bf3a40c4445fdef6cdc53281d7d97c76888ace"
+checksum = "a0635620b050b81bb92764e99250868f654e2cd5ad1bece413283b3f73c83179"
dependencies = [
"arrow",
"chrono",
@@ -1352,11 +1322,10 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4b1e68aba7a4b350401cfdf25a3d6f989ad898a7410164afe9ca52080244cb59"
+checksum = "8cabf7a86eb70b816729e33c81bf7767c936ee1226f607a114f5dac2decac8d0"
dependencies = [
- "ahash",
"arrow",
"datafusion-common",
"datafusion-expr",
@@ -1364,11 +1333,10 @@ dependencies = [
"datafusion-functions-aggregate-common",
"datafusion-physical-expr-common",
"half",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
"indexmap",
"itertools",
"parking_lot",
- "paste",
"petgraph",
"recursive",
"tokio",
@@ -1376,9 +1344,9 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr-adapter"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea22315f33cf2e0adc104e8ec42e285f6ed93998d565c65e82fec6a9ee9f9db4"
+checksum = "de222e04f7e6744501555a54ab0abe26bfdfebee380af79a9bdc175704246859"
dependencies = [
"arrow",
"datafusion-common",
@@ -1391,26 +1359,26 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr-common"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b04b45ea8ad3ac2d78f2ea2a76053e06591c9629c7a603eda16c10649ecf4362"
+checksum = "72d0d0057fc5a502d45c870cb6d47c66eb7bdd5edb1bd71ad6f3f724975ac2a8"
dependencies = [
- "ahash",
"arrow",
"chrono",
"datafusion-common",
"datafusion-expr-common",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
"indexmap",
"itertools",
"parking_lot",
+ "pin-project",
]
[[package]]
name = "datafusion-physical-optimizer"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7cb13397809a425918f608dfe8653f332015a3e330004ab191b4404187238b95"
+checksum = "86046eed10950c5f9aaed9acfd148e9bd2e1dfdfe4f9aef607d1447b271e4183"
dependencies = [
"arrow",
"datafusion-common",
@@ -1427,12 +1395,13 @@ dependencies = [
[[package]]
name = "datafusion-physical-plan"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5edc023675791af9d5fb4cc4c24abf5f7bd3bd4dcf9e5bd90ea1eff6976dcc79"
+checksum = "9bc84da934c903407ba297971ebcc020c4c1a38aafd765d6c144c76eff3fa6a1"
dependencies = [
- "ahash",
"arrow",
+ "arrow-data",
+ "arrow-ipc",
"arrow-ord",
"arrow-schema",
"async-trait",
@@ -1447,7 +1416,7 @@ dependencies = [
"datafusion-physical-expr-common",
"futures",
"half",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
"indexmap",
"itertools",
"log",
@@ -1459,9 +1428,9 @@ dependencies = [
[[package]]
name = "datafusion-proto"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a387aaef949dc16bb6abc81bd1af850ec7449183aef011214f9724957495738"
+checksum = "67791dcfacd142a9d95f8f73c2a161094cc936d231d80c235f5017dacf24e84e"
dependencies = [
"arrow",
"chrono",
@@ -1482,14 +1451,13 @@ dependencies = [
"datafusion-proto-common",
"object_store",
"prost",
- "rand 0.9.4",
]
[[package]]
name = "datafusion-proto-common"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "16e614c7c53a9c304c6a850b821010bb492e57300311835f1180613f9d2c63d9"
+checksum = "b8cd9e80d637891645d074db0f6c650b591117367247deb313fdfb78dff559cb"
dependencies = [
"arrow",
"datafusion-common",
@@ -1498,9 +1466,9 @@ dependencies = [
[[package]]
name = "datafusion-pruning"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac8c76860e355616555081cab5968cec1af7a80701ff374510860bcd567e365a"
+checksum = "eb63eeac6de19be40f487b65dd84e546195f783c5a9928618e0c4f2a3569b0d7"
dependencies = [
"arrow",
"datafusion-common",
@@ -1509,15 +1477,14 @@ dependencies = [
"datafusion-physical-expr",
"datafusion-physical-expr-common",
"datafusion-physical-plan",
- "itertools",
"log",
]
[[package]]
name = "datafusion-session"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5412111aa48e2424ba926112e192f7a6b7e4ccb450145d25ce5ede9f19dc491e"
+checksum = "5f961d209177f91bd014db5cbb2c33b7d28a2597b9003e77f17aeb712964315a"
dependencies = [
"async-trait",
"datafusion-common",
@@ -1527,11 +1494,41 @@ dependencies = [
"parking_lot",
]
+[[package]]
+name = "datafusion-spark"
+version = "54.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1ccd16a6949503e56c084df1b90c8889db826ec9347d2f0f51a7837d6fa011e"
+dependencies = [
+ "arrow",
+ "bigdecimal",
+ "chrono",
+ "crc32fast",
+ "datafusion",
+ "datafusion-catalog",
+ "datafusion-common",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-functions",
+ "datafusion-functions-aggregate",
+ "datafusion-functions-aggregate-common",
+ "datafusion-functions-nested",
+ "log",
+ "num-traits",
+ "percent-encoding",
+ "rand 0.9.4",
+ "serde_json",
+ "sha1",
+ "sha2",
+ "twox-hash",
+ "url",
+]
+
[[package]]
name = "datafusion-sql"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa0d133ddf8b9b3b872acac900157f783e7b879fe9a6bccf389abebbfac45ec1"
+checksum = "b1d71cb454da682b2af7488e1fc1ddd72ee1b28f19297b8ccad73f0a21ee9a69"
dependencies = [
"arrow",
"bigdecimal",
@@ -1548,9 +1545,9 @@ dependencies = [
[[package]]
name = "datafusion-substrait"
-version = "53.1.0"
+version = "54.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "98494539a5468979cc42d86c7bc5f0f8cb71ee5c742694c26fc34efdd29dd2e5"
+checksum = "f047a6fbf967b6b523758a48d2377bb5f9373a20e7ae4c4326d3c569f80ba3d7"
dependencies = [
"async-recursion",
"async-trait",
@@ -1572,16 +1569,27 @@ version = "0.10.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
dependencies = [
- "block-buffer",
- "crypto-common",
+ "block-buffer 0.10.4",
+ "crypto-common 0.1.7",
"subtle",
]
+[[package]]
+name = "digest"
+version = "0.11.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2"
+dependencies = [
+ "block-buffer 0.12.1",
+ "const-oid",
+ "crypto-common 0.2.2",
+]
+
[[package]]
name = "displaydoc"
-version = "0.2.5"
+version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"
+checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f"
dependencies = [
"proc-macro2",
"quote",
@@ -1596,9 +1604,9 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
[[package]]
name = "either"
-version = "1.15.0"
+version = "1.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
+checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e"
[[package]]
name = "equivalent"
@@ -1889,6 +1897,11 @@ name = "hashbrown"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
+dependencies = [
+ "allocator-api2",
+ "equivalent",
+ "foldhash 0.2.0",
+]
[[package]]
name = "heck"
@@ -1904,9 +1917,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
[[package]]
name = "http"
-version = "1.4.0"
+version = "1.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a"
+checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425"
dependencies = [
"bytes",
"itoa",
@@ -1947,11 +1960,20 @@ version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424"
+[[package]]
+name = "hybrid-array"
+version = "0.4.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b"
+dependencies = [
+ "typenum",
+]
+
[[package]]
name = "hyper"
-version = "1.9.0"
+version = "1.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca"
+checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498"
dependencies = [
"atomic-waker",
"bytes",
@@ -2119,12 +2141,6 @@ version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954"
-[[package]]
-name = "ident_case"
-version = "1.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
-
[[package]]
name = "idna"
version = "1.1.0"
@@ -2241,13 +2257,12 @@ dependencies = [
[[package]]
name = "js-sys"
-version = "0.3.98"
+version = "0.3.100"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08"
+checksum = "f2025f20d7a4fa7785846e7b63d10a76d3f1cee98ee5cb79ea59703f95e42162"
dependencies = [
"cfg-if",
"futures-util",
- "once_cell",
"wasm-bindgen",
]
@@ -2316,9 +2331,9 @@ dependencies = [
[[package]]
name = "libbz2-rs-sys"
-version = "0.2.3"
+version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b3a6a8c165077efc8f3a971534c50ea6a1a18b329ef4a66e897a7e3a1494565f"
+checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c"
[[package]]
name = "libc"
@@ -2375,9 +2390,9 @@ dependencies = [
[[package]]
name = "log"
-version = "0.4.29"
+version = "0.4.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
+checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a"
[[package]]
name = "lru-slab"
@@ -2401,14 +2416,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf"
dependencies = [
"cfg-if",
- "digest",
+ "digest 0.10.7",
+]
+
+[[package]]
+name = "md-5"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98"
+dependencies = [
+ "cfg-if",
+ "digest 0.11.3",
]
[[package]]
name = "memchr"
-version = "2.8.0"
+version = "2.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
+checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8"
[[package]]
name = "miniz_oxide"
@@ -2422,9 +2447,9 @@ dependencies = [
[[package]]
name = "mio"
-version = "1.2.0"
+version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1"
+checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda"
dependencies = [
"libc",
"wasi",
@@ -2445,7 +2470,6 @@ checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9"
dependencies = [
"num-integer",
"num-traits",
- "serde",
]
[[package]]
@@ -2504,7 +2528,7 @@ dependencies = [
"humantime",
"hyper",
"itertools",
- "md-5",
+ "md-5 0.10.6",
"parking_lot",
"percent-encoding",
"quick-xml",
@@ -2570,9 +2594,9 @@ dependencies = [
[[package]]
name = "parquet"
-version = "58.2.0"
+version = "58.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43d7efd3052f7d6ef601085559a246bc991e9a8cc77e02753737df6322ce35f1"
+checksum = "5dafa7d01085b62a47dd0c1829550a0a36710ea9c4fe358a05a85477cec8a908"
dependencies = [
"ahash",
"arrow-array",
@@ -2683,6 +2707,26 @@ dependencies = [
"siphasher",
]
+[[package]]
+name = "pin-project"
+version = "1.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924"
+dependencies = [
+ "pin-project-internal",
+]
+
+[[package]]
+name = "pin-project-internal"
+version = "1.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
[[package]]
name = "pin-project-lite"
version = "0.2.17"
@@ -2734,9 +2778,9 @@ dependencies = [
[[package]]
name = "prost"
-version = "0.14.3"
+version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568"
+checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1"
dependencies = [
"bytes",
"prost-derive",
@@ -2744,9 +2788,9 @@ dependencies = [
[[package]]
name = "prost-build"
-version = "0.14.3"
+version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7"
+checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042"
dependencies = [
"heck",
"itertools",
@@ -2763,9 +2807,9 @@ dependencies = [
[[package]]
name = "prost-derive"
-version = "0.14.3"
+version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b"
+checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf"
dependencies = [
"anyhow",
"itertools",
@@ -2776,9 +2820,9 @@ dependencies = [
[[package]]
name = "prost-types"
-version = "0.14.3"
+version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7"
+checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a"
dependencies = [
"prost",
]
@@ -2866,12 +2910,6 @@ dependencies = [
"cc",
]
-[[package]]
-name = "quad-rand"
-version = "0.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40"
-
[[package]]
name = "quick-xml"
version = "0.39.4"
@@ -3035,9 +3073,9 @@ dependencies = [
[[package]]
name = "regex"
-version = "1.12.3"
+version = "1.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276"
+checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba"
dependencies = [
"aho-corasick",
"memchr",
@@ -3056,17 +3094,11 @@ dependencies = [
"regex-syntax",
]
-[[package]]
-name = "regex-lite"
-version = "0.1.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973"
-
[[package]]
name = "regex-syntax"
-version = "0.8.10"
+version = "0.8.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
+checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
[[package]]
name = "regress"
@@ -3178,9 +3210,9 @@ dependencies = [
[[package]]
name = "rustls-native-certs"
-version = "0.8.3"
+version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63"
+checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d"
dependencies = [
"openssl-probe",
"rustls-pki-types",
@@ -3318,16 +3350,6 @@ dependencies = [
"serde_derive",
]
-[[package]]
-name = "serde_bytes"
-version = "0.11.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8"
-dependencies = [
- "serde",
- "serde_core",
-]
-
[[package]]
name = "serde_core"
version = "1.0.228"
@@ -3361,10 +3383,11 @@ dependencies = [
[[package]]
name = "serde_json"
-version = "1.0.149"
+version = "1.0.150"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86"
+checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
dependencies = [
+ "indexmap",
"itoa",
"memchr",
"serde",
@@ -3409,22 +3432,33 @@ dependencies = [
"unsafe-libyaml",
]
+[[package]]
+name = "sha1"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214"
+dependencies = [
+ "cfg-if",
+ "cpufeatures",
+ "digest 0.11.3",
+]
+
[[package]]
name = "sha2"
-version = "0.10.9"
+version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
+checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4"
dependencies = [
"cfg-if",
- "cpufeatures 0.2.17",
- "digest",
+ "cpufeatures",
+ "digest 0.11.3",
]
[[package]]
name = "shlex"
-version = "1.3.0"
+version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
+checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
[[package]]
name = "simd-adler32"
@@ -3464,9 +3498,9 @@ checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b"
[[package]]
name = "socket2"
-version = "0.6.3"
+version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"
+checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51"
dependencies = [
"libc",
"windows-sys 0.61.2",
@@ -3474,9 +3508,9 @@ dependencies = [
[[package]]
name = "sqlparser"
-version = "0.61.0"
+version = "0.62.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dbf5ea8d4d7c808e1af1cbabebca9a2abe603bcefc22294c5b95018d53200cb7"
+checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec"
dependencies = [
"log",
"recursive",
@@ -3513,23 +3547,11 @@ dependencies = [
"windows-sys 0.61.2",
]
-[[package]]
-name = "strsim"
-version = "0.11.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
-
-[[package]]
-name = "strum"
-version = "0.27.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf"
-
[[package]]
name = "strum_macros"
-version = "0.27.2"
+version = "0.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7"
+checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664"
dependencies = [
"heck",
"proc-macro2",
@@ -3539,11 +3561,12 @@ dependencies = [
[[package]]
name = "substrait"
-version = "0.62.2"
+version = "0.63.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62fc4b483a129b9772ccb9c3f7945a472112fdd9140da87f8a4e7f1d44e045d0"
+checksum = "e620ff4d5c02fd6f7752931aa74b16a26af66a63022cc1ad412c77edbe0bab47"
dependencies = [
"heck",
+ "indexmap",
"pbjson",
"pbjson-build",
"pbjson-types",
@@ -3858,12 +3881,15 @@ name = "twox-hash"
version = "2.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c"
+dependencies = [
+ "rand 0.9.4",
+]
[[package]]
name = "typenum"
-version = "1.20.0"
+version = "1.20.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de"
+checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
[[package]]
name = "typify"
@@ -3920,9 +3946,9 @@ checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
[[package]]
name = "unicode-segmentation"
-version = "1.13.2"
+version = "1.13.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c"
+checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8"
[[package]]
name = "unicode-width"
@@ -3968,13 +3994,12 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
[[package]]
name = "uuid"
-version = "1.23.1"
+version = "1.23.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76"
+checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7"
dependencies = [
"getrandom 0.4.2",
"js-sys",
- "serde_core",
"wasm-bindgen",
]
@@ -4029,9 +4054,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen"
-version = "0.2.121"
+version = "0.2.123"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790"
+checksum = "a254a4b10c19a76f09a27640e7ffbf9bc30bf67e16a3bf28aaefa4920fe81563"
dependencies = [
"cfg-if",
"once_cell",
@@ -4042,9 +4067,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-futures"
-version = "0.4.71"
+version = "0.4.73"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96492d0d3ffba25305a7dc88720d250b1401d7edca02cc3bcd50633b424673b8"
+checksum = "54568702fabf5d4849ce2b90fadfa64168a097eaf4b351ce9df8b687a0086aaf"
dependencies = [
"js-sys",
"wasm-bindgen",
@@ -4052,9 +4077,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro"
-version = "0.2.121"
+version = "0.2.123"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578"
+checksum = "24a40fc75b0ec6f3746ceb10d36f53a93dcd68a93b11b6445983945d79eba0dc"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@@ -4062,9 +4087,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
-version = "0.2.121"
+version = "0.2.123"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2"
+checksum = "908f34bd9b9ce3d4caf07b72dfab63d61504d156856c6bd3cd87fa350cf3985b"
dependencies = [
"bumpalo",
"proc-macro2",
@@ -4075,9 +4100,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-shared"
-version = "0.2.121"
+version = "0.2.123"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441"
+checksum = "7acbf7616c27b194bbb550bf77ed0c2c3e5b7fd1260a93082b95fb7f47959b92"
dependencies = [
"unicode-ident",
]
@@ -4131,9 +4156,9 @@ dependencies = [
[[package]]
name = "web-sys"
-version = "0.3.98"
+version = "0.3.100"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4b572dff8bcf38bad0fa19729c89bb5748b2b9b1d8be70cf90df697e3a8f32aa"
+checksum = "6e0871acf327f283dc6da28a1696cdc64fb355ba9f935d052021fa77f35cce69"
dependencies = [
"js-sys",
"wasm-bindgen",
@@ -4541,9 +4566,9 @@ checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4"
[[package]]
name = "yoke"
-version = "0.8.2"
+version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca"
+checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5"
dependencies = [
"stable_deref_trait",
"yoke-derive",
@@ -4564,18 +4589,18 @@ dependencies = [
[[package]]
name = "zerocopy"
-version = "0.8.48"
+version = "0.8.52"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9"
+checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f"
dependencies = [
"zerocopy-derive",
]
[[package]]
name = "zerocopy-derive"
-version = "0.8.48"
+version = "0.8.52"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4"
+checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930"
dependencies = [
"proc-macro2",
"quote",
@@ -4584,9 +4609,9 @@ dependencies = [
[[package]]
name = "zerofrom"
-version = "0.1.7"
+version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df"
+checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272"
dependencies = [
"zerofrom-derive",
]
diff --git a/Cargo.toml b/Cargo.toml
new file mode 100644
index 0000000..57256e6
--- /dev/null
+++ b/Cargo.toml
@@ -0,0 +1,57 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[workspace]
+resolver = "2"
+members = [
+ "native",
+ "native-common",
+]
+
+# Shared package metadata so every crate moves in lock step. Members inherit
+# via `version.workspace = true` / `edition.workspace = true` etc.; a single
+# bump here re-versions the whole workspace.
+[workspace.package]
+version = "0.1.0"
+edition = "2021"
+license = "Apache-2.0"
+repository = "https://github.com/apache/datafusion-java"
+
+# Every dependency used by any workspace member is declared here so version
+# bumps live in one place and the resolver picks a single version of each
+# crate across the workspace. Members reference these via `{ workspace = true }`
+# and add per-crate flags (optional, features, default-features) at the use
+# site.
+[workspace.dependencies]
+arrow = { version = "58", features = ["ffi"] }
+async-trait = "0.1"
+datafusion = { version = "54.1.0" }
+datafusion-proto = "54.1.0"
+datafusion-spark = "54.1.0"
+datafusion-substrait = "54.1.0"
+futures = "0.3"
+jni = "0.21"
+# Pinned to the major DataFusion 54.1 pulls in transitively (0.13.x) so we
+# share the same `dyn ObjectStore` vtable and don't double-link.
+object_store = { version = "0.13", default-features = false }
+prost = "0.14"
+prost-build = "0.14"
+protoc-bin-vendored = "3"
+tokio = { version = "1", features = ["rt-multi-thread"] }
+# Optional, cfg-gated. See `native/Cargo.toml` for the build-flag dance.
+tokio-metrics = "0.5"
+url = "2"
diff --git a/Makefile b/Makefile
index 6d9b0ae..d6bcf2c 100644
--- a/Makefile
+++ b/Makefile
@@ -20,14 +20,14 @@
all: native jvm
native:
- cd native && cargo build
+ cargo build --workspace
-# Build the native crate with the `runtime-metrics` Cargo feature enabled.
+# Build the JNI crate with the `runtime-metrics` Cargo feature enabled.
# Requires `--cfg tokio_unstable` because tokio-metrics gates its API there.
# Default `make native` does not pull this in; callers who need
# SessionContext.runtimeStats() pick this target explicitly.
native-runtime-metrics:
- cd native && RUSTFLAGS="--cfg tokio_unstable" cargo build --features runtime-metrics
+ RUSTFLAGS="--cfg tokio_unstable" cargo build -p datafusion-jni --features runtime-metrics
jvm:
./mvnw package -DskipTests
@@ -39,10 +39,10 @@ test: native
# `:check` form inline in .github/workflows/lint.yml.
format:
./mvnw -q spotless:apply
- cd native && cargo fmt --all
+ cargo fmt --all
clean:
- cd native && cargo clean
+ cargo clean
./mvnw clean
tpch-data:
diff --git a/core/pom.xml b/core/pom.xml
index 5ddf107..ee2cb65 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -102,8 +102,8 @@ under the License.
-
+ value="${maven.multiModuleProjectDirectory}/rust-target/${datafusion.native.profile}/${datafusion.lib.filename}"/>
+
@@ -129,7 +129,7 @@ under the License.
https://raw.githubusercontent.com/apache/datafusion/${datafusion.version}/datafusion/proto-common/proto/datafusion_common.proto
${project.build.directory}/proto/datafusion/proto-common/proto
datafusion_common.proto
- d6f3368372ea277cc23e26f196994b81616d38599357bb374cbd7eb1760e649a789e4c133d86a395ac701049a500348da2ec039d3f978ac5d8112c2876dded1f
+ e6a20115badec14f641b9237b0faa3f553162992444208322558ea3fcfaa24ca15334e4945827259dcb7c1de7f9aed659d496974eb8eeb1cfcf54ad4d7f6d20f
@@ -140,7 +140,7 @@ under the License.
https://raw.githubusercontent.com/apache/datafusion/${datafusion.version}/datafusion/proto/proto/datafusion.proto
${project.build.directory}/proto/datafusion/proto/proto
datafusion.proto
- c3d162b8e2a418e03f74caceaccfd934af89bb95a12ede13d4cc1701d24c734d74b1e96372142b173db05938dab7f965ad60d476363308c441677a63ea5fbcf7
+ bdae5361d32b4d4f0418677ce650ee8cc212fddfe9390b2c033ee4ab3bb3c969a6148b7b318b9bb268ee9384f18d4254ebe6e0701f15639059a8f2800d6e7847
diff --git a/core/src/main/java/org/apache/datafusion/IoException.java b/core/src/main/java/org/apache/datafusion/IoException.java
index 79f196e..42cdad7 100644
--- a/core/src/main/java/org/apache/datafusion/IoException.java
+++ b/core/src/main/java/org/apache/datafusion/IoException.java
@@ -21,9 +21,8 @@
/**
* IO-shaped failure: a local filesystem read failed, an object store request failed, or a parquet /
- * arrow / avro decoder reported a malformed file. Surfaces upstream {@code
- * DataFusionError::IoError}, {@code ObjectStore}, {@code ArrowError}, {@code ParquetError}, and
- * {@code AvroError}.
+ * arrow decoder reported a malformed file. Surfaces upstream {@code DataFusionError::IoError},
+ * {@code ObjectStore}, {@code ParquetError}, and the IO-shaped {@code ArrowError} variants.
*
* Note: this is {@code org.apache.datafusion.IoException} (lowercase {@code o}), distinct from
* {@code java.io.IOException}. The {@code IoException} spelling matches the orthography of the
diff --git a/core/src/main/java/org/apache/datafusion/SessionContext.java b/core/src/main/java/org/apache/datafusion/SessionContext.java
index ffc58dd..b68cda5 100644
--- a/core/src/main/java/org/apache/datafusion/SessionContext.java
+++ b/core/src/main/java/org/apache/datafusion/SessionContext.java
@@ -113,10 +113,11 @@ public DataFrame fromProto(byte[] planBytes) {
* other Substrait-emitting tool — and hand them to DataFusion without round-tripping through SQL.
*
*
Substrait support is gated behind the {@code substrait} Cargo feature on the native crate
- * and is off by default. Rebuild the native crate with {@code cargo build
- * --features substrait} (or {@code cargo build --features substrait,protoc} for hermetic builds
- * that vendor {@code protoc} via {@code cmake}) to enable it. If invoked against a native binary
- * built without the feature, this method throws {@link RuntimeException} pointing at the flag.
+ * and is off by default. Rebuild the native crate with {@code cargo build -p
+ * datafusion-jni --features substrait} (or {@code ... --features substrait,protoc} for hermetic
+ * builds that vendor {@code protoc} via {@code cmake}) to enable it. If invoked against a native
+ * binary built without the feature, this method throws {@link RuntimeException} pointing at the
+ * flag.
*
* @throws IllegalArgumentException if {@code planBytes} is {@code null}.
* @throws IllegalStateException if this context is closed.
@@ -183,7 +184,7 @@ public MemoryUsage memoryUsage() {
* Rebuild with:
*
*
{@code
- * RUSTFLAGS="--cfg tokio_unstable" cargo build --features runtime-metrics
+ * RUSTFLAGS="--cfg tokio_unstable" cargo build -p datafusion-jni --features runtime-metrics
* }
*
* If invoked against a native binary built without the feature, this method throws {@link
@@ -601,6 +602,43 @@ private static byte[] serializeSchemaIpc(Schema schema) {
return baos.toByteArray();
}
+ /**
+ * Returns {@code true} if a table with the given name is registered in this session.
+ *
+ *
This is the Java counterpart to DataFusion's Rust {@code SessionContext::table_exist}.
+ *
+ * @throws IllegalStateException if this context is closed.
+ */
+ public boolean tableExists(String name) {
+ checkOpenSessionContext();
+ if (name == null) {
+ throw new IllegalArgumentException("tableExists name must be non-null");
+ }
+ return tableExists(nativeHandle, name);
+ }
+
+ /**
+ * Removes the table with the given name from this session. Does nothing if no table with that
+ * name is registered.
+ *
+ *
This is the Java counterpart to DataFusion's Rust {@code SessionContext::deregister_table}.
+ *
+ * @throws IllegalStateException if this context is closed.
+ */
+ public void deregisterTable(String name) {
+ checkOpenSessionContext();
+ if (name == null) {
+ throw new IllegalArgumentException("deregisterTable name must be non-null");
+ }
+ deregisterTable(nativeHandle, name);
+ }
+
+ private void checkOpenSessionContext() {
+ if (nativeHandle == 0) {
+ throw new IllegalStateException("SessionContext is closed");
+ }
+ }
+
@Override
public void close() {
if (nativeHandle != 0) {
@@ -664,4 +702,8 @@ private static native void registerScalarUdf(
private static native void registerTableNative(
long handle, String name, byte[] schemaIpcBytes, TableProvider provider);
+
+ private static native boolean tableExists(long handle, String name);
+
+ private static native void deregisterTable(long handle, String name);
}
diff --git a/core/src/main/java/org/apache/datafusion/SessionContextBuilder.java b/core/src/main/java/org/apache/datafusion/SessionContextBuilder.java
index 81c59de..ef6fd4c 100644
--- a/core/src/main/java/org/apache/datafusion/SessionContextBuilder.java
+++ b/core/src/main/java/org/apache/datafusion/SessionContextBuilder.java
@@ -44,6 +44,7 @@ public final class SessionContextBuilder {
private boolean spillDisabled;
private Long maxTempDirectorySize;
private CacheManagerOptions cacheManager;
+ private boolean sparkFunctions;
private final LinkedHashMap options = new LinkedHashMap<>();
private final List objectStores = new ArrayList<>();
@@ -227,6 +228,20 @@ public SessionContextBuilder cacheManager(CacheManagerOptions options) {
return this;
}
+ /**
+ * Register Apache Spark-compatible functions and expression planners on the new context, using
+ * the {@code datafusion-spark} crate. Once enabled, Spark-compatible functions (e.g. {@code
+ * crc32}) are callable from SQL and override any DataFusion built-in of the same name.
+ *
+ * Requires the native library to be built with the {@code spark} Cargo feature, which is
+ * enabled in the default build. If it is not, {@link #build()} throws a {@link RuntimeException}
+ * explaining the feature is missing.
+ */
+ public SessionContextBuilder withSparkFunctions() {
+ this.sparkFunctions = true;
+ return this;
+ }
+
/**
* Register an {@code object_store::ObjectStore} backend on the new context's {@code RuntimeEnv}.
* Build {@link ObjectStoreOptions} via the per-backend factories ({@link ObjectStoreOptions#s3},
@@ -309,6 +324,9 @@ byte[] toBytes() {
for (Map.Entry e : options.entrySet()) {
b.addOptions(ConfigOption.newBuilder().setKey(e.getKey()).setValue(e.getValue()).build());
}
+ if (sparkFunctions) {
+ b.setSparkFunctions(true);
+ }
for (ObjectStoreOptions os : objectStores) {
b.addObjectStores(os.toRegistration());
}
diff --git a/core/src/test/java/org/apache/datafusion/DataFrameIntrospectionTest.java b/core/src/test/java/org/apache/datafusion/DataFrameIntrospectionTest.java
index 91cf1e7..ede96d9 100644
--- a/core/src/test/java/org/apache/datafusion/DataFrameIntrospectionTest.java
+++ b/core/src/test/java/org/apache/datafusion/DataFrameIntrospectionTest.java
@@ -195,7 +195,7 @@ void describeReturnsSummaryStats() throws Exception {
}
}
}
- // DataFusion 53.1 reports these seven labels.
+ // DataFusion 54.1 reports these seven labels.
assertTrue(seen.contains("count"), () -> "labels=" + seen);
assertTrue(seen.contains("null_count"), () -> "labels=" + seen);
assertTrue(seen.contains("mean"), () -> "labels=" + seen);
diff --git a/core/src/test/java/org/apache/datafusion/SessionContextBuilderTest.java b/core/src/test/java/org/apache/datafusion/SessionContextBuilderTest.java
index b7451ab..93eeac1 100644
--- a/core/src/test/java/org/apache/datafusion/SessionContextBuilderTest.java
+++ b/core/src/test/java/org/apache/datafusion/SessionContextBuilderTest.java
@@ -426,6 +426,21 @@ void diskManagerFieldIsAbsentWhenNothingSet() throws Exception {
assertFalse(parsed.hasDiskManager());
}
+ @Test
+ void withSparkFunctionsRoundTripsThroughProto() throws Exception {
+ byte[] bytes = SessionContext.builder().withSparkFunctions().toBytes();
+ SessionOptions parsed = SessionOptions.parseFrom(bytes);
+ assertTrue(parsed.hasSparkFunctions());
+ assertTrue(parsed.getSparkFunctions());
+ }
+
+ @Test
+ void sparkFunctionsAbsentWhenNotRequested() throws Exception {
+ byte[] bytes = SessionContext.builder().batchSize(8192).toBytes();
+ SessionOptions parsed = SessionOptions.parseFrom(bytes);
+ assertFalse(parsed.hasSparkFunctions());
+ }
+
@Test
void tempDirectoryStaysOnLegacyField() throws Exception {
// tempDirectory(String) writes the existing SessionOptions.temp_directory
diff --git a/core/src/test/java/org/apache/datafusion/SessionContextRuntimeStatsTest.java b/core/src/test/java/org/apache/datafusion/SessionContextRuntimeStatsTest.java
index 120d179..d567275 100644
--- a/core/src/test/java/org/apache/datafusion/SessionContextRuntimeStatsTest.java
+++ b/core/src/test/java/org/apache/datafusion/SessionContextRuntimeStatsTest.java
@@ -37,7 +37,7 @@
* #checkFeatureEnabled}. Run
*
* {@code
- * (cd native && RUSTFLAGS="--cfg tokio_unstable" cargo build --features runtime-metrics)
+ * RUSTFLAGS="--cfg tokio_unstable" cargo build -p datafusion-jni --features runtime-metrics
* }
*
* before {@code ./mvnw test} to exercise this class.
diff --git a/core/src/test/java/org/apache/datafusion/SessionContextSparkFunctionsTest.java b/core/src/test/java/org/apache/datafusion/SessionContextSparkFunctionsTest.java
new file mode 100644
index 0000000..6359c14
--- /dev/null
+++ b/core/src/test/java/org/apache/datafusion/SessionContextSparkFunctionsTest.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datafusion;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.BigIntVector;
+import org.apache.arrow.vector.ipc.ArrowReader;
+import org.junit.jupiter.api.Test;
+
+class SessionContextSparkFunctionsTest {
+
+ // crc32 is a Spark function provided by datafusion-spark and is NOT a core
+ // DataFusion built-in. Spark's crc32('Spark') == 1557323817 (standard
+ // CRC-32 of the ASCII bytes), so it doubles as a correctness check.
+ @Test
+ void sparkFunctionIsCallableWhenEnabled() throws Exception {
+ try (BufferAllocator allocator = new RootAllocator();
+ SessionContext ctx = SessionContext.builder().withSparkFunctions().build();
+ DataFrame df = ctx.sql("SELECT crc32('Spark') AS c");
+ ArrowReader reader = df.collect(allocator)) {
+ assertTrue(reader.loadNextBatch());
+ BigIntVector v = (BigIntVector) reader.getVectorSchemaRoot().getVector("c");
+ assertEquals(1, v.getValueCount());
+ assertEquals(1557323817L, v.get(0));
+ }
+ }
+
+ // Without the flag, crc32 is unregistered, so planning the same SQL must
+ // fail. This is the proof that withSparkFunctions() is what enabled it.
+ @Test
+ void sparkFunctionIsAbsentByDefault() {
+ try (SessionContext ctx = SessionContext.builder().build()) {
+ RuntimeException thrown =
+ assertThrows(RuntimeException.class, () -> ctx.sql("SELECT crc32('Spark')"));
+ assertTrue(
+ thrown.getMessage() != null && thrown.getMessage().toLowerCase().contains("crc32"),
+ "expected error to mention the unknown function, got: " + thrown.getMessage());
+ }
+ }
+}
diff --git a/core/src/test/java/org/apache/datafusion/SessionContextSubstraitTest.java b/core/src/test/java/org/apache/datafusion/SessionContextSubstraitTest.java
index 34db3b5..2877fcf 100644
--- a/core/src/test/java/org/apache/datafusion/SessionContextSubstraitTest.java
+++ b/core/src/test/java/org/apache/datafusion/SessionContextSubstraitTest.java
@@ -50,7 +50,7 @@
*
* The {@code substrait} Cargo feature is off by default in {@code native/Cargo.toml}; if the
* native crate was built without it, every test here is skipped (see {@link #checkFeatureEnabled}).
- * Run {@code (cd native && cargo build --features substrait)} before {@code ./mvnw test} to
+ * Run {@code cargo build -p datafusion-jni --features substrait} before {@code ./mvnw test} to
* exercise this class.
*/
class SessionContextSubstraitTest {
@@ -73,7 +73,14 @@ static void checkFeatureEnabled() {
/**
* Build a minimal Substrait {@code Plan} that scans a registered named table {@code tableName}
- * with two columns {@code (id int32, v int32)} and projects them through unchanged.
+ * with two columns {@code (id int64, v int64)} and projects them through unchanged.
+ *
+ *
The columns are declared {@code NULLABILITY_NULLABLE} to match the schema DataFusion infers
+ * for the CSV these tests register — CSV inference always yields nullable fields. DataFusion's
+ * Substrait consumer verifies that a field the plan declares non-nullable really is non-nullable
+ * in the table, and rejects the plan otherwise: a plan built around a "never null" assumption
+ * must not silently run against data that can contain nulls. The enclosing struct stays {@code
+ * REQUIRED} — the row itself is always present.
*/
private static Plan namedTableScanPlan(String tableName) {
NamedStruct schema =
@@ -87,13 +94,13 @@ private static Plan namedTableScanPlan(String tableName) {
.setI64(
Type.I64
.newBuilder()
- .setNullability(Type.Nullability.NULLABILITY_REQUIRED)))
+ .setNullability(Type.Nullability.NULLABILITY_NULLABLE)))
.addTypes(
Type.newBuilder()
.setI64(
Type.I64
.newBuilder()
- .setNullability(Type.Nullability.NULLABILITY_REQUIRED)))
+ .setNullability(Type.Nullability.NULLABILITY_NULLABLE)))
.setNullability(Type.Nullability.NULLABILITY_REQUIRED))
.build();
ReadRel read =
diff --git a/core/src/test/java/org/apache/datafusion/SessionContextTest.java b/core/src/test/java/org/apache/datafusion/SessionContextTest.java
index 4465546..8c80e42 100644
--- a/core/src/test/java/org/apache/datafusion/SessionContextTest.java
+++ b/core/src/test/java/org/apache/datafusion/SessionContextTest.java
@@ -20,6 +20,7 @@
package org.apache.datafusion;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -33,6 +34,7 @@
import org.apache.arrow.vector.ipc.ArrowReader;
import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
class SessionContextTest {
@Test
@@ -90,4 +92,71 @@ void dataFrameCollectTwiceFails() {
assertThrows(IllegalStateException.class, () -> df.collect(allocator));
}
}
+
+ @Test
+ void tableExistsReturnsFalseForUnregisteredTable() {
+ try (SessionContext ctx = new SessionContext()) {
+ assertFalse(ctx.tableExists("orders"));
+ }
+ }
+
+ @Test
+ void tableExistsReturnsTrueAfterRegisterCsv(@TempDir Path tempDir) throws Exception {
+ Path csv = tempDir.resolve("orders.csv");
+ Files.writeString(csv, "id,amount\n1,100\n2,200\n");
+
+ try (SessionContext ctx = new SessionContext()) {
+ assertFalse(ctx.tableExists("orders"));
+ ctx.registerCsv("orders", csv.toAbsolutePath().toString());
+ assertTrue(ctx.tableExists("orders"));
+ }
+ }
+
+ @Test
+ void deregisterTableRemovesRegisteredTable(@TempDir Path tempDir) throws Exception {
+ Path csv = tempDir.resolve("orders.csv");
+ Files.writeString(csv, "id,amount\n1,100\n2,200\n");
+
+ try (SessionContext ctx = new SessionContext()) {
+ ctx.registerCsv("orders", csv.toAbsolutePath().toString());
+ assertTrue(ctx.tableExists("orders"));
+ ctx.deregisterTable("orders");
+ assertFalse(ctx.tableExists("orders"));
+ }
+ }
+
+ @Test
+ void deregisterTableIsNoOpForUnregisteredTable() {
+ try (SessionContext ctx = new SessionContext()) {
+ ctx.deregisterTable("nonexistent");
+ }
+ }
+
+ @Test
+ void tableExistsThrowsWhenContextIsClosed() {
+ SessionContext ctx = new SessionContext();
+ ctx.close();
+ assertThrows(IllegalStateException.class, () -> ctx.tableExists("orders"));
+ }
+
+ @Test
+ void deregisterTableThrowsWhenContextIsClosed() {
+ SessionContext ctx = new SessionContext();
+ ctx.close();
+ assertThrows(IllegalStateException.class, () -> ctx.deregisterTable("orders"));
+ }
+
+ @Test
+ void tableExistsThrowsForNullName() {
+ try (SessionContext ctx = new SessionContext()) {
+ assertThrows(IllegalArgumentException.class, () -> ctx.tableExists(null));
+ }
+ }
+
+ @Test
+ void deregisterTableThrowsForNullName() {
+ try (SessionContext ctx = new SessionContext()) {
+ assertThrows(IllegalArgumentException.class, () -> ctx.deregisterTable(null));
+ }
+ }
}
diff --git a/dev/release/build-release.sh b/dev/release/build-release.sh
index 2b033bb..4d4ab13 100755
--- a/dev/release/build-release.sh
+++ b/dev/release/build-release.sh
@@ -135,26 +135,28 @@ JVM_TARGET_DIR="$PROJECT_HOME/core/target/classes/org/apache/datafusion"
mkdir -p "$JVM_TARGET_DIR/linux/amd64"
docker cp \
- "$CONTAINER_AMD64:/opt/datafusion-java-rm/datafusion-java/native/target/release/libdatafusion_jni.so" \
+ "$CONTAINER_AMD64:/opt/datafusion-java-rm/datafusion-java/rust-target/release/libdatafusion_jni.so" \
"$JVM_TARGET_DIR/linux/amd64/"
mkdir -p "$JVM_TARGET_DIR/linux/aarch64"
docker cp \
- "$CONTAINER_ARM64:/opt/datafusion-java-rm/datafusion-java/native/target/release/libdatafusion_jni.so" \
+ "$CONTAINER_ARM64:/opt/datafusion-java-rm/datafusion-java/rust-target/release/libdatafusion_jni.so" \
"$JVM_TARGET_DIR/linux/aarch64/"
echo "Building macOS native libs on the host (host=$HOST_ARCH)"
rustup target add "$OTHER_DARWIN_TARGET"
-(cd "$PROJECT_HOME/native" && cargo build --release)
-(cd "$PROJECT_HOME/native" && cargo build --release --target "$OTHER_DARWIN_TARGET")
+# Cargo writes to the workspace `rust-target/` dir (set in .cargo/config.toml),
+# not the per-crate `native/target/`, so build from the repo root.
+(cd "$PROJECT_HOME" && cargo build --release -p datafusion-jni)
+(cd "$PROJECT_HOME" && cargo build --release -p datafusion-jni --target "$OTHER_DARWIN_TARGET")
mkdir -p "$JVM_TARGET_DIR/darwin/$HOST_DARWIN_DIR"
-cp "$PROJECT_HOME/native/target/release/libdatafusion_jni.dylib" \
+cp "$PROJECT_HOME/rust-target/release/libdatafusion_jni.dylib" \
"$JVM_TARGET_DIR/darwin/$HOST_DARWIN_DIR/"
mkdir -p "$JVM_TARGET_DIR/darwin/$OTHER_DARWIN_DIR"
-cp "$PROJECT_HOME/native/target/$OTHER_DARWIN_TARGET/release/libdatafusion_jni.dylib" \
+cp "$PROJECT_HOME/rust-target/$OTHER_DARWIN_TARGET/release/libdatafusion_jni.dylib" \
"$JVM_TARGET_DIR/darwin/$OTHER_DARWIN_DIR/"
echo "Installing JAR into local Maven repo"
diff --git a/dev/release/datafusion-java-rm/build-native-libs.sh b/dev/release/datafusion-java-rm/build-native-libs.sh
index 5f273cc..79f8ae0 100755
--- a/dev/release/datafusion-java-rm/build-native-libs.sh
+++ b/dev/release/datafusion-java-rm/build-native-libs.sh
@@ -38,8 +38,9 @@ git clone "$REPO" datafusion-java
cd datafusion-java
git checkout "$BRANCH"
-cd native
-cargo build --release
+# Cargo writes to the workspace `rust-target/` dir (set in .cargo/config.toml),
+# not the per-crate `native/target/`, so build from the repo root.
+cargo build --release -p datafusion-jni
-echo "Built $(pwd)/target/release/libdatafusion_jni.so"
-ls -l target/release/libdatafusion_jni.so
+echo "Built $(pwd)/rust-target/release/libdatafusion_jni.so"
+ls -l rust-target/release/libdatafusion_jni.so
diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt
index 81d83e8..3dbd90f 100644
--- a/dev/release/rat_exclude_files.txt
+++ b/dev/release/rat_exclude_files.txt
@@ -7,7 +7,7 @@
.mvn/wrapper/maven-wrapper.properties
mvnw
mvnw.cmd
-native/Cargo.lock
+Cargo.lock
dev/release/rat_exclude_files.txt
docs/source/_static/**
docs/source/conf.py
diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh
index e486adc..c7767bf 100755
--- a/dev/release/verify-release-candidate.sh
+++ b/dev/release/verify-release-candidate.sh
@@ -150,7 +150,8 @@ test_source_distribution() {
# raises on any formatting errors
rustup component add rustfmt
- (cd native && cargo fmt --all -- --check)
+ # Workspace-wide: covers native, native-common, and any future members.
+ cargo fmt --all -- --check
# build native + JVM and run the full test suite
make test
diff --git a/docs/source/contributor-guide/development.md b/docs/source/contributor-guide/development.md
index 984d77c..61d4fb0 100644
--- a/docs/source/contributor-guide/development.md
+++ b/docs/source/contributor-guide/development.md
@@ -42,7 +42,7 @@ This builds the native Rust crate and runs the JUnit tests. The steps can
be run individually:
```sh
-cd native && cargo build
+cargo build --workspace
./mvnw test
```
@@ -74,6 +74,11 @@ disk space.
The repository is a multi-module Maven build:
+- `Cargo.toml` — Rust workspace root declaring the crate members
+ (`native`, `native-common`) and `[workspace.dependencies]` that pin
+ shared versions in one place. Cargo writes artifacts to `rust-target/`
+ (overridden in `.cargo/config.toml`) so `mvn clean` at the repo root does
+ not nuke the Rust build cache.
- `pom.xml` — parent POM declaring the `core` and `examples` modules and
shared plugin/dependency versions.
- `core/` — `datafusion-java` library module (Java sources, tests, and
@@ -81,7 +86,10 @@ The repository is a multi-module Maven build:
- `examples/` — `datafusion-java-examples` module containing runnable
examples that depend on the library; built alongside the library so they
cannot fall out of sync with the API.
-- `native/` — Rust crate (JNI + Arrow C Data Interface).
+- `native/` — `datafusion-jni` Rust crate (JNI + Arrow C Data Interface).
+- `native-common/` — `datafusion-jni-common` Rust crate: JNI plumbing
+ shared across native crates (error→exception mapping, the per-cdylib
+ Tokio runtime singleton, the async-stream→`FFI_ArrowArrayStream` bridge).
- `proto/` — Protobuf definitions shared between Java and Rust.
- `Makefile` — top-level build orchestration (`make test`, `make format`,
`make tpch-data`).
diff --git a/docs/source/contributor-guide/updating-datafusion-version.md b/docs/source/contributor-guide/updating-datafusion-version.md
index 56d50dc..6e3b90b 100644
--- a/docs/source/contributor-guide/updating-datafusion-version.md
+++ b/docs/source/contributor-guide/updating-datafusion-version.md
@@ -21,7 +21,9 @@ under the License.
Three things must move together when bumping DataFusion:
-1. `native/Cargo.toml` — the `datafusion` crate dependency.
+1. `Cargo.toml` (workspace root) — the `datafusion`, `datafusion-proto`,
+ `datafusion-spark`, and `datafusion-substrait` entries in
+ `[workspace.dependencies]`. Members inherit from there.
2. `pom.xml` — the `` Maven property. **Must equal
the Cargo version**; a mismatch means JVM-built protobuf plans won't
deserialize on the native side.
@@ -32,9 +34,9 @@ Three things must move together when bumping DataFusion:
## Recipe
```sh
-# 1. Bump the Cargo dep
-$EDITOR native/Cargo.toml # set datafusion = ""
-(cd native && cargo update -p datafusion)
+# 1. Bump the workspace dep
+$EDITOR Cargo.toml # set datafusion = "" in [workspace.dependencies]
+cargo update -p datafusion
# 2. Bump the Maven property to match
$EDITOR pom.xml # set
diff --git a/docs/source/user-guide/sessioncontext.md b/docs/source/user-guide/sessioncontext.md
index 818ee0e..f2408c9 100644
--- a/docs/source/user-guide/sessioncontext.md
+++ b/docs/source/user-guide/sessioncontext.md
@@ -60,3 +60,24 @@ try (SessionContext ctx = SessionContext.builder()
// ...
}
```
+
+## Spark-compatible functions
+
+`withSparkFunctions()` registers Apache Spark–compatible functions and
+expression planners (from the
+[`datafusion-spark`](https://crates.io/crates/datafusion-spark) crate) on the
+context:
+
+```java
+try (SessionContext ctx = SessionContext.builder()
+ .withSparkFunctions()
+ .build();
+ DataFrame df = ctx.sql("SELECT crc32('Spark')")) {
+ // ...
+}
+```
+
+When enabled, Spark-compatible functions override any DataFusion built-in of
+the same name. This requires the native library to be built with the `spark`
+Cargo feature, which is enabled in the default build; otherwise `build()`
+throws explaining the feature is missing.
diff --git a/native-common/Cargo.toml b/native-common/Cargo.toml
new file mode 100644
index 0000000..d136dcd
--- /dev/null
+++ b/native-common/Cargo.toml
@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+name = "datafusion-jni-common"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+# Implementation detail of datafusion-java's native crates, not a standalone
+# crates.io library. Matches `publish = false` on the `datafusion-jni` crate.
+publish = false
+readme = "README.md"
+description = "Shared JNI plumbing for DataFusion Java native crates: error-to-exception mapping, the per-cdylib Tokio runtime singleton, and the async-stream-to-FFI_ArrowArrayStream bridge."
+
+[dependencies]
+datafusion = { workspace = true }
+futures = { workspace = true }
+jni = { workspace = true }
+tokio = { workspace = true }
diff --git a/native-common/README.md b/native-common/README.md
new file mode 100644
index 0000000..aadf877
--- /dev/null
+++ b/native-common/README.md
@@ -0,0 +1,37 @@
+
+
+# datafusion-jni-common
+
+Shared JNI plumbing for the [Apache DataFusion Java](https://github.com/apache/datafusion-java)
+native crates. It holds the pieces every DataFusion-backed `cdylib` loaded into a
+JVM needs, factored out so they live in one place.
+
+## Linking model
+
+Each consuming `cdylib` statically links its own copy of this crate, so the
+runtime singleton is per-library, not per-process. Nothing here is exported with
+`#[no_mangle]`, so linking it into several `cdylib`s loaded in one JVM cannot
+collide.
+
+## Status
+
+This crate is an implementation detail of Apache DataFusion Java. Its API may
+change between releases to track the needs of the native crates that depend on
+it.
diff --git a/native/src/errors.rs b/native-common/src/errors.rs
similarity index 93%
rename from native/src/errors.rs
rename to native-common/src/errors.rs
index d926544..b3e438d 100644
--- a/native/src/errors.rs
+++ b/native-common/src/errors.rs
@@ -96,8 +96,7 @@ fn classify(err: &DataFusionError) -> &'static str {
}
DataFusionError::IoError(_)
| DataFusionError::ObjectStore(_)
- | DataFusionError::ParquetError(_)
- | DataFusionError::AvroError(_) => "org/apache/datafusion/IoException",
+ | DataFusionError::ParquetError(_) => "org/apache/datafusion/IoException",
// ArrowError is a 21-variant grab bag -- only some of those variants
// are actually IO-shaped. DivideByZero / ArithmeticOverflow / Compute
// / Cast / InvalidArgument / Memory etc. are execution-time failures
@@ -122,9 +121,15 @@ fn classify(err: &DataFusionError) -> &'static str {
/// Map an [`ArrowError`] variant onto the Java exception class to throw.
/// Only the genuinely IO-shaped variants (`IoError`, `IpcError`) land on
/// `IoException`; everything else is execution-time and routes through
-/// `ExecutionException`. Schema/parse-shaped variants route through
-/// `PlanException` so a malformed IPC schema or a parse error surfaces as a
-/// query problem rather than an execution failure.
+/// `ExecutionException` -- including `AvroError`, which is where Avro decode
+/// failures land now that DataFusion 54 reads Avro through `arrow-avro`
+/// instead of `apache-avro` (the old `DataFusionError::AvroError` variant is
+/// gone). That puts Avro alongside the `CsvError` / `JsonError` decoder
+/// variants, which this arm already classified as execution failures.
+///
+/// Schema/parse-shaped variants route through `PlanException` so a malformed
+/// IPC schema or a parse error surfaces as a query problem rather than an
+/// execution failure.
///
/// Variants without a clean caller-facing category (`CDataInterface`, the
/// various overflow/index-overflow markers) fall through to the parent.
@@ -161,7 +166,10 @@ fn throw(env: &mut JNIEnv, class: &str, message: &str) {
let _ = env.throw_new(class, message);
}
-fn panic_message(panic: &Box) -> String {
+/// Best-effort extraction of a panic payload's message. `catch_unwind` hands
+/// back a `Box`; the payload is a `String` or `&str` for ordinary
+/// `panic!`/`unwrap` sites, anything else is opaque.
+pub fn panic_message(panic: &Box) -> String {
if let Some(s) = panic.downcast_ref::() {
s.clone()
} else if let Some(s) = panic.downcast_ref::<&str>() {
diff --git a/native-common/src/lib.rs b/native-common/src/lib.rs
new file mode 100644
index 0000000..ba47004
--- /dev/null
+++ b/native-common/src/lib.rs
@@ -0,0 +1,98 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! JNI plumbing shared by this workspace's native crates (`datafusion-jni`
+//! and `datafusion-spark-bridge`, and through the latter every bridge
+//! cdylib): the error-to-Java-exception mapping, the per-cdylib Tokio
+//! runtime singleton, and the async-stream-to-`FFI_ArrowArrayStream`
+//! bridge.
+//!
+//! Each cdylib statically links its own copy of this rlib, so [`runtime`] is
+//! a per-cdylib singleton -- exactly the behaviour each crate had when this
+//! code lived inline. Nothing here is exported with `#[no_mangle]`, so
+//! linking this crate into several cdylibs loaded in one JVM cannot collide.
+
+pub mod errors;
+
+use std::panic::{catch_unwind, AssertUnwindSafe};
+use std::sync::OnceLock;
+
+use datafusion::arrow::array::RecordBatch;
+use datafusion::arrow::datatypes::SchemaRef;
+use datafusion::arrow::error::ArrowError;
+use datafusion::arrow::record_batch::RecordBatchReader;
+use datafusion::execution::SendableRecordBatchStream;
+use futures::StreamExt;
+use tokio::runtime::{Handle, Runtime};
+
+static RT: OnceLock = OnceLock::new();
+
+/// The cdylib-wide Tokio runtime.
+pub fn runtime() -> &'static Runtime {
+ runtime_with_init(|_| {})
+}
+
+/// Same singleton as [`runtime`], with a hook that runs exactly once, when
+/// the runtime is created. `datafusion-jni` uses it to install its
+/// runtime-metrics accumulator so the sampling baseline coincides with
+/// runtime start; every later call (either entry point) returns the existing
+/// runtime without invoking the hook.
+pub fn runtime_with_init(init: impl FnOnce(&Handle)) -> &'static Runtime {
+ RT.get_or_init(|| {
+ let rt = Runtime::new().expect("failed to create Tokio runtime");
+ init(rt.handle());
+ rt
+ })
+}
+
+/// Bridges DataFusion's async [`SendableRecordBatchStream`] to the synchronous
+/// [`RecordBatchReader`] interface that `FFI_ArrowArrayStream` (and therefore
+/// the Java `ArrowReader`) consumes. Each call to `next()` drives one
+/// `runtime().block_on(stream.next())`, so memory pressure stays bounded by the
+/// executor pipeline plus a single in-flight batch.
+pub struct StreamingReader {
+ pub schema: SchemaRef,
+ pub stream: SendableRecordBatchStream,
+}
+
+impl Iterator for StreamingReader {
+ type Item = Result;
+
+ fn next(&mut self) -> Option {
+ // Arrow's C ABI invokes this iterator through FFI_ArrowArrayStream's
+ // vtable, outside the JNI handler's try_unwrap_or_throw guard. A panic
+ // here (buggy UDF, arrow cast that panics, runtime poison) would
+ // unwind across C/FFI -- undefined behaviour. Catch it and surface as
+ // an ArrowError so the Java side sees a normal exception instead.
+ let next = catch_unwind(AssertUnwindSafe(|| runtime().block_on(self.stream.next())));
+ match next {
+ Ok(item) => item.map(|r| r.map_err(|e| ArrowError::ExternalError(Box::new(e)))),
+ Err(panic) => {
+ let msg = errors::panic_message(&panic);
+ Some(Err(ArrowError::ExternalError(
+ format!("panic in DataFrame stream: {msg}").into(),
+ )))
+ }
+ }
+ }
+}
+
+impl RecordBatchReader for StreamingReader {
+ fn schema(&self) -> SchemaRef {
+ self.schema.clone()
+ }
+}
diff --git a/native/Cargo.toml b/native/Cargo.toml
index c462408..8f4add5 100644
--- a/native/Cargo.toml
+++ b/native/Cargo.toml
@@ -17,14 +17,17 @@
[package]
name = "datafusion-jni"
-version = "0.1.0"
-edition = "2021"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+# cdylib JNI artifact loaded by the JVM, not a crates.io library.
publish = false
[lib]
# `rlib` alongside `cdylib` so `cargo test` has a Rust-level harness for
-# native-only invariants (e.g. error-classification routing through wrapped
-# DataFusionError chains). The `cdylib` is still the artifact the JVM loads.
+# native-only invariants (the error-classification tests now live in
+# `datafusion-jni-common`). The `cdylib` is still the artifact the JVM loads.
crate-type = ["cdylib", "rlib"]
[features]
@@ -38,6 +41,7 @@ default = [
"object-store-aws",
"object-store-gcp",
"object-store-http",
+ "spark",
]
object-store-aws = ["object_store/aws"]
object-store-gcp = ["object_store/gcp"]
@@ -67,26 +71,33 @@ protoc = ["datafusion-substrait?/protoc"]
# invoked in a build that compiled the feature off, so the Java surface is
# unchanged either way.
runtime-metrics = ["dep:tokio-metrics"]
+# Apache Spark-compatible functions, registered when a caller builds a
+# context with `SessionContextBuilder.withSparkFunctions()`. On by default so
+# Java callers don't have to think about features; builds that strip it get a
+# clear runtime error from the JVM if the flag is requested.
+spark = ["dep:datafusion-spark"]
[dependencies]
-arrow = { version = "58", features = ["ffi"] }
-async-trait = "0.1"
-datafusion = { version = "53.1.0", features = ["avro"] }
-datafusion-proto = "53.1.0"
-datafusion-substrait = { version = "53.1.0", optional = true }
-futures = "0.3"
-jni = "0.21"
-# Pin to the same major as DataFusion 53.1 pulls in transitively (0.13.x)
-# so we share the same `dyn ObjectStore` vtable and don't double-link.
-object_store = { version = "0.13", default-features = false }
-prost = "0.14"
-tokio = { version = "1", features = ["rt-multi-thread"] }
-# Tokio runtime metrics. Optional + cfg-gated: this crate's API surface lives
-# behind `--cfg tokio_unstable`, so enabling the `runtime-metrics` feature also
-# requires the caller to set `RUSTFLAGS="--cfg tokio_unstable"` at build time.
-tokio-metrics = { version = "0.5", optional = true }
-url = "2"
+arrow = { workspace = true }
+async-trait = { workspace = true }
+datafusion = { workspace = true, features = ["avro"] }
+# Shared JNI plumbing (error->exception mapping, runtime singleton,
+# StreamingReader).
+datafusion-jni-common = { path = "../native-common" }
+datafusion-proto = { workspace = true }
+# Apache Spark-compatible functions + expression planners. Optional and
+# gated behind the `spark` feature (in the default set). The `core` feature
+# of the crate is what exposes `SessionStateBuilderSpark`.
+datafusion-spark = { workspace = true, features = ["core"], optional = true }
+datafusion-substrait = { workspace = true, optional = true }
+futures = { workspace = true }
+jni = { workspace = true }
+object_store = { workspace = true }
+prost = { workspace = true }
+tokio = { workspace = true }
+tokio-metrics = { workspace = true, optional = true }
+url = { workspace = true }
[build-dependencies]
-prost-build = "0.14"
-protoc-bin-vendored = "3"
+prost-build = { workspace = true }
+protoc-bin-vendored = { workspace = true }
diff --git a/native/src/arrow.rs b/native/src/arrow.rs
index 2bbe7b0..67e5caf 100644
--- a/native/src/arrow.rs
+++ b/native/src/arrow.rs
@@ -23,10 +23,10 @@ use jni::sys::jlong;
use jni::JNIEnv;
use prost::Message;
-use crate::errors::{try_unwrap_or_throw, JniResult};
use crate::proto_gen::ArrowReadOptionsProto;
use crate::runtime;
use crate::schema::decode_optional_schema;
+use datafusion_jni_common::errors::{try_unwrap_or_throw, JniResult};
fn with_arrow_options(
env: &mut JNIEnv,
diff --git a/native/src/avro.rs b/native/src/avro.rs
index 85d4a07..257ae32 100644
--- a/native/src/avro.rs
+++ b/native/src/avro.rs
@@ -23,10 +23,10 @@ use jni::sys::jlong;
use jni::JNIEnv;
use prost::Message;
-use crate::errors::{try_unwrap_or_throw, JniResult};
use crate::proto_gen::AvroReadOptionsProto;
use crate::runtime;
use crate::schema::decode_optional_schema;
+use datafusion_jni_common::errors::{try_unwrap_or_throw, JniResult};
fn with_avro_options(
env: &mut JNIEnv,
diff --git a/native/src/cache_manager.rs b/native/src/cache_manager.rs
index 3b9e286..4be7070 100644
--- a/native/src/cache_manager.rs
+++ b/native/src/cache_manager.rs
@@ -29,13 +29,11 @@ use std::sync::Arc;
use std::time::Duration;
use datafusion::execution::cache::cache_manager::CacheManagerConfig;
-use datafusion::execution::cache::cache_unit::{
- DefaultFileStatisticsCache, DefaultFilesMetadataCache,
-};
-use datafusion::execution::cache::DefaultListFilesCache;
+use datafusion::execution::cache::file_statistics_cache::DefaultFileStatisticsCache;
+use datafusion::execution::cache::{DefaultFilesMetadataCache, DefaultListFilesCache};
-use crate::errors::JniResult;
use crate::proto_gen::CacheManagerOptionsProto;
+use datafusion_jni_common::errors::JniResult;
/// Build a [`CacheManagerConfig`] from the proto. Returns `Ok(None)` if the
/// caller did not set any cache-manager field, so the JNI layer can skip the
@@ -76,8 +74,20 @@ pub(crate) fn build_config(
config.list_files_cache_ttl = ttl;
}
- if opts.file_statistics_cache_enabled.unwrap_or(false) {
- config.table_files_statistics_cache = Some(Arc::new(DefaultFileStatisticsCache::default()));
+ // DataFusion 54 renamed `table_files_statistics_cache` to
+ // `file_statistics_cache` and made the *limit* the on/off switch:
+ // `CacheManager::try_new` now builds a `DefaultFileStatisticsCache`
+ // whenever `file_statistics_cache_limit > 0`, even when the cache slot is
+ // `None`. The default limit is non-zero, so an explicit `false` from the
+ // Java surface has to zero the limit -- otherwise upstream would install a
+ // stats cache the caller just asked us not to.
+ if let Some(enabled) = opts.file_statistics_cache_enabled {
+ if enabled {
+ config.file_statistics_cache = Some(Arc::new(DefaultFileStatisticsCache::default()));
+ } else {
+ config.file_statistics_cache = None;
+ config.file_statistics_cache_limit = 0;
+ }
}
Ok(Some(config))
diff --git a/native/src/csv.rs b/native/src/csv.rs
index 3ae4627..b79ed59 100644
--- a/native/src/csv.rs
+++ b/native/src/csv.rs
@@ -26,12 +26,12 @@ use jni::sys::jlong;
use jni::JNIEnv;
use prost::Message;
-use crate::errors::{try_unwrap_or_throw, JniResult};
use crate::proto_gen::{
CsvReadOptionsProto, CsvWriteOptionsProto, FileCompressionType as ProtoFileCompressionType,
};
use crate::runtime;
use crate::schema::decode_optional_schema;
+use datafusion_jni_common::errors::{try_unwrap_or_throw, JniResult};
fn with_csv_options(
env: &mut JNIEnv,
diff --git a/native/src/json.rs b/native/src/json.rs
index 8eea32f..b87be78 100644
--- a/native/src/json.rs
+++ b/native/src/json.rs
@@ -27,12 +27,12 @@ use jni::sys::jlong;
use jni::JNIEnv;
use prost::Message;
-use crate::errors::{try_unwrap_or_throw, JniResult};
use crate::proto_gen::{
FileCompressionType as ProtoFileCompressionType, JsonWriteOptionsProto, NdJsonReadOptionsProto,
};
use crate::runtime;
use crate::schema::decode_optional_schema;
+use datafusion_jni_common::errors::{try_unwrap_or_throw, JniResult};
fn with_json_options(
env: &mut JNIEnv,
diff --git a/native/src/lib.rs b/native/src/lib.rs
index 4fd7a8a..56bef5d 100644
--- a/native/src/lib.rs
+++ b/native/src/lib.rs
@@ -19,7 +19,6 @@ mod arrow;
mod avro;
mod cache_manager;
mod csv;
-mod errors;
mod jni_util;
mod json;
mod memory;
@@ -34,28 +33,23 @@ pub(crate) mod proto_gen {
include!(concat!(env!("OUT_DIR"), "/datafusion_java.rs"));
}
-use std::panic::{catch_unwind, AssertUnwindSafe};
use std::path::PathBuf;
use std::sync::{Arc, OnceLock};
-use datafusion::arrow::array::RecordBatch;
use datafusion::arrow::datatypes::SchemaRef;
-use datafusion::arrow::error::ArrowError;
use datafusion::arrow::ffi_stream::FFI_ArrowArrayStream;
use datafusion::arrow::ipc::writer::StreamWriter;
-use datafusion::arrow::record_batch::{RecordBatchIterator, RecordBatchReader};
+use datafusion::arrow::record_batch::RecordBatchIterator;
use datafusion::common::{JoinType, UnnestOptions};
use datafusion::config::TableParquetOptions;
use datafusion::dataframe::DataFrame;
use datafusion::dataframe::DataFrameWriteOptions;
use datafusion::error::DataFusionError;
use datafusion::execution::disk_manager::{DiskManagerBuilder, DiskManagerMode};
-use datafusion::execution::runtime_env::RuntimeEnvBuilder;
-use datafusion::execution::SendableRecordBatchStream;
+use datafusion::execution::runtime_env::{RuntimeEnv, RuntimeEnvBuilder};
use datafusion::logical_expr::Expr;
use datafusion::logical_expr::{col, Partitioning, ScalarUDF, Signature, SortExpr};
use datafusion::prelude::{ParquetReadOptions, SessionConfig, SessionContext};
-use futures::StreamExt;
use jni::objects::{JBooleanArray, JByteArray, JClass, JObject, JObjectArray, JString};
use jni::sys::{jboolean, jbyte, jbyteArray, jint, jlong};
use jni::JNIEnv;
@@ -63,7 +57,10 @@ use jni::JavaVM;
use prost::Message;
use tokio::runtime::Runtime;
-use crate::errors::{try_unwrap_or_throw, JniResult};
+use datafusion_jni_common::errors::{try_unwrap_or_throw, JniResult};
+// Re-exported so sibling modules keep their crate-local `crate::StreamingReader` path.
+pub(crate) use datafusion_jni_common::StreamingReader;
+
use crate::proto_gen::ParquetReadOptionsProto;
use crate::proto_gen::SessionOptions;
use crate::schema::decode_optional_schema;
@@ -84,18 +81,15 @@ pub(crate) fn jvm() -> &'static JavaVM {
}
pub(crate) fn runtime() -> &'static Runtime {
- static RT: OnceLock = OnceLock::new();
- RT.get_or_init(|| {
- let rt = Runtime::new().expect("failed to create Tokio runtime");
- // Eagerly install the runtime-metrics accumulator (no-op when the
- // `runtime-metrics` Cargo feature is off). Initialising here -- not
- // lazily on the first `runtimeStats()` call -- means the
- // RuntimeMonitor's sampling baseline coincides with runtime start, so
- // poll/park/busy totals reflect activity from the first query onward
- // rather than from the first observation.
- crate::runtime_metrics::init(rt.handle());
- rt
- })
+ // The singleton itself lives in datafusion-jni-common (shared with the
+ // datafusion-spark-bridge SDK; each cdylib statically links its own
+ // copy, so the runtime stays per-library). The init hook eagerly installs the
+ // runtime-metrics accumulator (no-op when the `runtime-metrics` Cargo
+ // feature is off). Initialising here -- not lazily on the first
+ // `runtimeStats()` call -- means the RuntimeMonitor's sampling baseline
+ // coincides with runtime start, so poll/park/busy totals reflect activity
+ // from the first query onward rather than from the first observation.
+ datafusion_jni_common::runtime_with_init(crate::runtime_metrics::init)
}
/// Wrap the (already-built) `RuntimeEnvBuilder`'s memory pool with a
@@ -117,6 +111,40 @@ fn install_memory_tracker(
tracker
}
+/// Build a `SessionContext` from a prepared config + runtime env. When
+/// `spark_functions` is set, register Apache Spark-compatible functions and
+/// expression planners via the `datafusion-spark` crate (requires the `spark`
+/// Cargo feature); otherwise build the plain context. Returns an error if the
+/// caller asked for Spark functions in a build that compiled the feature off.
+fn build_session_context(
+ config: SessionConfig,
+ runtime_env: Arc,
+ spark_functions: bool,
+) -> JniResult {
+ if spark_functions {
+ #[cfg(feature = "spark")]
+ {
+ use datafusion::execution::SessionStateBuilder;
+ use datafusion_spark::SessionStateBuilderSpark;
+ // `with_spark_features` runs after the default features so Spark
+ // implementations override the built-ins of the same name.
+ let state = SessionStateBuilder::new_with_default_features()
+ .with_config(config)
+ .with_runtime_env(runtime_env)
+ .with_spark_features()
+ .build();
+ Ok(SessionContext::new_with_state(state))
+ }
+ #[cfg(not(feature = "spark"))]
+ {
+ let _ = (config, runtime_env);
+ Err("spark Cargo feature is not enabled in this build of datafusion-jni".into())
+ }
+ } else {
+ Ok(SessionContext::new_with_config_rt(config, runtime_env))
+ }
+}
+
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_SessionContext_createSessionContext<'local>(
mut env: JNIEnv<'local>,
@@ -225,8 +253,9 @@ pub extern "system" fn Java_org_apache_datafusion_SessionContext_createSessionCo
// update two atomics.
let tracker = install_memory_tracker(&mut runtime_builder);
- let runtime_env = runtime_builder.build()?;
- let ctx = SessionContext::new_with_config_rt(config, Arc::new(runtime_env));
+ let runtime_env = Arc::new(runtime_builder.build()?);
+ let ctx =
+ build_session_context(config, runtime_env, opts.spark_functions.unwrap_or(false))?;
// Object-store registrations come last because they need a built
// RuntimeEnv to register against. A failure here drops `ctx` (and its
@@ -289,50 +318,6 @@ pub extern "system" fn Java_org_apache_datafusion_DataFrame_collectDataFrame<'lo
})
}
-/// Bridges DataFusion's async [`SendableRecordBatchStream`] to the synchronous
-/// [`RecordBatchReader`] interface that `FFI_ArrowArrayStream` (and therefore
-/// the Java `ArrowReader`) consumes. Each call to `next()` drives one
-/// `runtime().block_on(stream.next())`, so memory pressure stays bounded by the
-/// executor pipeline plus a single in-flight batch.
-struct StreamingReader {
- schema: SchemaRef,
- stream: SendableRecordBatchStream,
-}
-
-impl Iterator for StreamingReader {
- type Item = Result;
-
- fn next(&mut self) -> Option {
- // Arrow's C ABI invokes this iterator through FFI_ArrowArrayStream's
- // vtable, outside the JNI handler's try_unwrap_or_throw guard. A panic
- // here (buggy UDF, arrow cast that panics, runtime poison) would
- // unwind across C/FFI -- undefined behaviour. Catch it and surface as
- // an ArrowError so the Java side sees a normal exception instead.
- let next = catch_unwind(AssertUnwindSafe(|| runtime().block_on(self.stream.next())));
- match next {
- Ok(item) => item.map(|r| r.map_err(|e| ArrowError::ExternalError(Box::new(e)))),
- Err(panic) => {
- let msg = if let Some(s) = panic.downcast_ref::() {
- s.clone()
- } else if let Some(s) = panic.downcast_ref::<&str>() {
- (*s).to_string()
- } else {
- "rust panic with non-string payload".to_string()
- };
- Some(Err(ArrowError::ExternalError(
- format!("panic in DataFrame stream: {msg}").into(),
- )))
- }
- }
- }
-}
-
-impl RecordBatchReader for StreamingReader {
- fn schema(&self) -> SchemaRef {
- self.schema.clone()
- }
-}
-
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_DataFrame_executeStreamDataFrame<'local>(
mut env: JNIEnv<'local>,
@@ -1042,6 +1027,49 @@ pub extern "system" fn Java_org_apache_datafusion_SessionContext_getOptionNative
)
}
+/// Converts a raw JNI handle into a shared reference to a [`SessionContext`].
+///
+/// # Safety for the unsafe usage
+/// The caller must ensure `handle` was produced by `Box::into_raw(Box::new(ctx))`
+/// in `createSessionContext` and has not yet been freed. The Java side zeroes
+/// `nativeHandle` on `close()`, so the `handle == 0` guard in every JNI handler
+/// ensures this invariant holds before `unwrap_context` is called.
+fn unwrap_context(handle: jlong) -> JniResult<&'static SessionContext> {
+ if handle == 0 {
+ return Err("SessionContext handle is null".into());
+ }
+ Ok(unsafe { &*(handle as *const SessionContext) })
+}
+
+#[no_mangle]
+pub extern "system" fn Java_org_apache_datafusion_SessionContext_tableExists<'local>(
+ mut env: JNIEnv<'local>,
+ _class: JClass<'local>,
+ handle: jlong,
+ name: JString<'local>,
+) -> jboolean {
+ try_unwrap_or_throw(&mut env, 0, |env| -> JniResult {
+ let ctx = unwrap_context(handle)?;
+ let name: String = env.get_string(&name)?.into();
+ Ok(ctx.table_exist(&name)? as jboolean)
+ })
+}
+
+#[no_mangle]
+pub extern "system" fn Java_org_apache_datafusion_SessionContext_deregisterTable<'local>(
+ mut env: JNIEnv<'local>,
+ _class: JClass<'local>,
+ handle: jlong,
+ name: JString<'local>,
+) {
+ try_unwrap_or_throw(&mut env, (), |env| -> JniResult<()> {
+ let ctx = unwrap_context(handle)?;
+ let name: String = env.get_string(&name)?.into();
+ ctx.deregister_table(&name)?;
+ Ok(())
+ })
+}
+
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_SessionContext_closeSessionContext<'local>(
mut env: JNIEnv<'local>,
@@ -1294,3 +1322,28 @@ pub extern "system" fn Java_org_apache_datafusion_SessionContext_registerTableNa
Ok(())
})
}
+
+#[cfg(all(test, feature = "spark"))]
+mod spark_tests {
+ use super::*;
+
+ #[test]
+ fn spark_functions_register_crc32() {
+ let rt = Arc::new(RuntimeEnvBuilder::new().build().unwrap());
+ let ctx = build_session_context(SessionConfig::new(), rt, true).unwrap();
+ assert!(
+ ctx.state().scalar_functions().contains_key("crc32"),
+ "expected Spark crc32 to be registered when spark_functions = true"
+ );
+ }
+
+ #[test]
+ fn plain_context_has_no_crc32() {
+ let rt = Arc::new(RuntimeEnvBuilder::new().build().unwrap());
+ let ctx = build_session_context(SessionConfig::new(), rt, false).unwrap();
+ assert!(
+ !ctx.state().scalar_functions().contains_key("crc32"),
+ "crc32 must not be registered without spark_functions"
+ );
+ }
+}
diff --git a/native/src/memory.rs b/native/src/memory.rs
index e105299..8068aba 100644
--- a/native/src/memory.rs
+++ b/native/src/memory.rs
@@ -120,7 +120,27 @@ impl TrackingMemoryPool {
}
}
+/// DataFusion 54 added `Display` as a supertrait of [`MemoryPool`] so pools
+/// can render themselves in resource-exhausted messages. Mirror upstream's
+/// wrapper convention: name the wrapper, then defer to the inner pool for the
+/// limit/usage detail, and add the counters this wrapper exists to expose.
+impl std::fmt::Display for TrackingMemoryPool {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ let (current, peak) = self.snapshot();
+ write!(
+ f,
+ "{}(current: {current}, peak: {peak}, inner: {})",
+ self.name(),
+ self.inner
+ )
+ }
+}
+
impl MemoryPool for TrackingMemoryPool {
+ fn name(&self) -> &str {
+ "tracking"
+ }
+
fn register(&self, consumer: &datafusion::execution::memory_pool::MemoryConsumer) {
self.inner.register(consumer);
}
diff --git a/native/src/object_store.rs b/native/src/object_store.rs
index eefccf2..985d721 100644
--- a/native/src/object_store.rs
+++ b/native/src/object_store.rs
@@ -28,9 +28,9 @@ use std::sync::Arc;
use datafusion::prelude::SessionContext;
use url::Url;
-use crate::errors::JniResult;
use crate::proto_gen::object_store_registration::Backend;
use crate::proto_gen::ObjectStoreRegistration;
+use datafusion_jni_common::errors::JniResult;
#[cfg(feature = "object-store-gcp")]
use crate::proto_gen::GcsOptions;
diff --git a/native/src/proto.rs b/native/src/proto.rs
index 4f187bc..c1315f9 100644
--- a/native/src/proto.rs
+++ b/native/src/proto.rs
@@ -28,8 +28,8 @@ use jni::sys::{jbyteArray, jlong};
use jni::JNIEnv;
use prost::Message;
-use crate::errors::{try_unwrap_or_throw, JniResult};
use crate::runtime;
+use datafusion_jni_common::errors::{try_unwrap_or_throw, JniResult};
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_SessionContext_createDataFrameFromProto<
diff --git a/native/src/runtime_metrics.rs b/native/src/runtime_metrics.rs
index e69410e..dd60dcb 100644
--- a/native/src/runtime_metrics.rs
+++ b/native/src/runtime_metrics.rs
@@ -38,7 +38,7 @@
//! 10 totalOverflowCount
#[cfg(not(feature = "runtime-metrics"))]
-use crate::errors::JniResult;
+use datafusion_jni_common::errors::JniResult;
/// Number of i64 values in the snapshot array; kept here so the Java side and
/// the feature-off stub agree on the layout.
@@ -51,7 +51,7 @@ mod imp {
use tokio_metrics::{RuntimeIntervals, RuntimeMonitor};
use super::STATS_FIELD_COUNT;
- use crate::errors::JniResult;
+ use datafusion_jni_common::errors::JniResult;
/// `RuntimeMonitor::intervals().next()` returns *delta* metrics covering
/// the period since the previous call (or, on the very first call, since
@@ -196,7 +196,7 @@ pub fn runtime_stats() -> JniResult<[i64; STATS_FIELD_COUNT]> {
Err(
"datafusion-jni was built without the `runtime-metrics` Cargo feature; \
rebuild the native crate with \
- `RUSTFLAGS=\"--cfg tokio_unstable\" cargo build --features runtime-metrics` \
+ `RUSTFLAGS=\"--cfg tokio_unstable\" cargo build -p datafusion-jni --features runtime-metrics` \
to enable SessionContext.runtimeStats"
.into(),
)
diff --git a/native/src/schema.rs b/native/src/schema.rs
index 968a73a..0c3c7ab 100644
--- a/native/src/schema.rs
+++ b/native/src/schema.rs
@@ -20,7 +20,7 @@ use datafusion::arrow::ipc::reader::StreamReader;
use jni::objects::JByteArray;
use jni::JNIEnv;
-use crate::errors::JniResult;
+use datafusion_jni_common::errors::JniResult;
/// Decode an optional Arrow-IPC schema byte array passed in from Java.
/// Returns `None` if the byte-array reference is null.
diff --git a/native/src/table_provider.rs b/native/src/table_provider.rs
index 70eaac2..51c892b 100644
--- a/native/src/table_provider.rs
+++ b/native/src/table_provider.rs
@@ -22,7 +22,6 @@
//! `TableProvider` trait; it currently only supports a single-partition, no-pushdown scan,
//! with future pushdown and partitioning support tracked as follow-ups.
-use std::any::Any;
use std::fmt;
use std::sync::Arc;
@@ -74,10 +73,6 @@ impl fmt::Debug for JavaTableProvider {
#[async_trait]
impl TableProvider for JavaTableProvider {
- fn as_any(&self) -> &dyn Any {
- self
- }
-
fn schema(&self) -> SchemaRef {
Arc::clone(&self.schema)
}
@@ -152,10 +147,6 @@ impl ExecutionPlan for JavaScanExec {
"JavaScanExec"
}
- fn as_any(&self) -> &dyn Any {
- self
- }
-
fn properties(&self) -> &Arc {
&self.plan_properties
}
diff --git a/native/src/udf.rs b/native/src/udf.rs
index da7b260..3c3c7da 100644
--- a/native/src/udf.rs
+++ b/native/src/udf.rs
@@ -17,7 +17,6 @@
//! Java-backed scalar UDF support.
-use std::any::Any;
use std::fmt;
use datafusion::arrow::array::{make_array, Array, ArrayRef, StructArray};
@@ -80,10 +79,6 @@ impl std::hash::Hash for JavaScalarUdf {
}
impl ScalarUDFImpl for JavaScalarUdf {
- fn as_any(&self) -> &dyn Any {
- self
- }
-
fn name(&self) -> &str {
&self.name
}
diff --git a/pom.xml b/pom.xml
index 6210841..d9f3212 100644
--- a/pom.xml
+++ b/pom.xml
@@ -40,7 +40,7 @@ under the License.
17
UTF-8
5.11.3
- 53.1.0
+ 54.1.0
3.25.5
19.0.0
1.12.0
@@ -95,6 +95,11 @@ under the License.
+
+ org.apache.maven.plugins
+ maven-compiler-plugin
+ 3.13.0
+
org.apache.maven.plugins
maven-surefire-plugin
@@ -173,10 +178,10 @@ under the License.
.mvn/**
**/target/**
- native/target/**
+ rust-target/**
tpch-data/**
-
- native/Cargo.lock
+
+ Cargo.lock
dev/release/rat_exclude_files.txt
diff --git a/proto/session_options.proto b/proto/session_options.proto
index 516e3c4..cfed21e 100644
--- a/proto/session_options.proto
+++ b/proto/session_options.proto
@@ -64,6 +64,12 @@ message SessionOptions {
// Disk-manager configuration (disable spill, size cap). Unset fields
// leave the upstream `RuntimeEnvBuilder` default in place.
optional DiskManagerOptions disk_manager = 10;
+ // Register Apache Spark-compatible functions and expression planners on the
+ // new context (via the datafusion-spark crate). Requires the native library
+ // to be built with the `spark` Cargo feature (on in the default build);
+ // otherwise construction fails with a clear error. Spark functions override
+ // DataFusion built-ins of the same name.
+ optional bool spark_functions = 11;
}
// Disk-manager configuration. The two fields are independent on the wire;