From 1a39821da009a7186bc32aae36384f7713531e8a Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Mon, 30 Sep 2019 13:47:53 +0200 Subject: [PATCH 01/24] ARROW-6610: [C++] Add cmake option to disable filesystem layer Closes #5515 from pitrou/ARROW-6610-cmake-filesystem-flag and squashes the following commits: b74741dde ARROW-6610: Add cmake option to disable filesystem layer Authored-by: Antoine Pitrou Signed-off-by: Antoine Pitrou --- cpp/CMakeLists.txt | 6 +- cpp/cmake_modules/DefineOptions.cmake | 44 ++-- cpp/cmake_modules/ThirdpartyToolchain.cmake | 7 +- cpp/examples/minimal_build/build.sh | 6 +- cpp/src/arrow/CMakeLists.txt | 234 ++++++++++---------- 5 files changed, 158 insertions(+), 139 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 2ab620d8f6d7..ea1a5756587c 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -268,10 +268,14 @@ if(ARROW_BUILD_TESTS OR ARROW_BUILD_INTEGRATION) set(ARROW_JSON ON) endif() -if(ARROW_FLIGHT OR ARROW_PARQUET OR ARROW_BUILD_TESTS) +if(ARROW_CUDA OR ARROW_FLIGHT OR ARROW_PARQUET OR ARROW_BUILD_TESTS) set(ARROW_IPC ON) endif() +if(ARROW_DATASET) + set(ARROW_FILESYSTEM ON) +endif() + if(MSVC) # ORC doesn't build on windows set(ARROW_ORC OFF) diff --git a/cpp/cmake_modules/DefineOptions.cmake b/cpp/cmake_modules/DefineOptions.cmake index 3764f4824962..7a0158ef4c61 100644 --- a/cpp/cmake_modules/DefineOptions.cmake +++ b/cpp/cmake_modules/DefineOptions.cmake @@ -90,21 +90,19 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") #---------------------------------------------------------------------- set_option_category("Test and benchmark") - define_option(ARROW_BUILD_EXAMPLES "Build the Arrow examples, default OFF" OFF) + define_option(ARROW_BUILD_EXAMPLES "Build the Arrow examples" OFF) - define_option(ARROW_BUILD_TESTS "Build the Arrow googletest unit tests, default OFF" - OFF) - - define_option(ARROW_BUILD_INTEGRATION - "Build the Arrow integration test executables, default OFF" OFF) + define_option(ARROW_BUILD_TESTS "Build the Arrow googletest unit tests" OFF) - define_option(ARROW_BUILD_BENCHMARKS "Build the Arrow micro benchmarks, default OFF" + define_option(ARROW_BUILD_INTEGRATION "Build the Arrow integration test executables" OFF) + define_option(ARROW_BUILD_BENCHMARKS "Build the Arrow micro benchmarks" OFF) + # Reference benchmarks are used to compare to naive implementation, or # discover various hardware limits. define_option(ARROW_BUILD_BENCHMARKS_REFERENCE - "Build the Arrow micro reference benchmarks, default OFF." OFF) + "Build the Arrow micro reference benchmarks" OFF) define_option_string(ARROW_TEST_LINKAGE "Linkage of Arrow libraries with unit tests executables." @@ -141,47 +139,49 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") #---------------------------------------------------------------------- set_option_category("Project component") + define_option(ARROW_BUILD_UTILITIES "Build Arrow commandline utilities" ON) + define_option(ARROW_COMPUTE "Build the Arrow Compute Modules" ON) + define_option(ARROW_CUDA "Build the Arrow CUDA extensions (requires CUDA toolkit)" OFF) + define_option(ARROW_DATASET "Build the Arrow Dataset Modules" ON) + define_option(ARROW_FILESYSTEM "Build the Arrow Filesystem Layer" ON) + define_option(ARROW_FLIGHT "Build the Arrow Flight RPC System (requires GRPC, Protocol Buffers)" OFF) define_option(ARROW_GANDIVA "Build the Gandiva libraries" OFF) - define_option(ARROW_PARQUET "Build the Parquet libraries" OFF) - - define_option(ARROW_IPC "Build the Arrow IPC extensions" ON) + define_option(ARROW_HDFS "Build the Arrow HDFS bridge" ON) - define_option(ARROW_BUILD_UTILITIES "Build Arrow commandline utilities" ON) + define_option(ARROW_HIVESERVER2 "Build the HiveServer2 client and Arrow adapter" OFF) - define_option(ARROW_CUDA "Build the Arrow CUDA extensions (requires CUDA toolkit)" OFF) + define_option(ARROW_IPC "Build the Arrow IPC extensions" ON) - define_option(ARROW_ORC "Build the Arrow ORC adapter" OFF) + define_option(ARROW_JEMALLOC "Build the Arrow jemalloc-based allocator" ON) define_option(ARROW_JNI "Build the Arrow JNI lib" OFF) - define_option(ARROW_TENSORFLOW "Build Arrow with TensorFlow support enabled" OFF) - - define_option(ARROW_JEMALLOC "Build the Arrow jemalloc-based allocator" ON) + define_option(ARROW_JSON "Build Arrow with JSON support (requires RapidJSON)" ON) define_option(ARROW_MIMALLOC "Build the Arrow mimalloc-based allocator" OFF) - define_option(ARROW_HDFS "Build the Arrow HDFS bridge" ON) - - define_option(ARROW_PYTHON "Build the Arrow CPython extensions" OFF) + define_option(ARROW_PARQUET "Build the Parquet libraries" OFF) - define_option(ARROW_HIVESERVER2 "Build the HiveServer2 client and Arrow adapter" OFF) + define_option(ARROW_ORC "Build the Arrow ORC adapter" OFF) define_option(ARROW_PLASMA "Build the plasma object store along with Arrow" OFF) define_option(ARROW_PLASMA_JAVA_CLIENT "Build the plasma object store java client" OFF) - define_option(ARROW_JSON "Build Arrow with JSON support (requires RapidJSON)" ON) + define_option(ARROW_PYTHON "Build the Arrow CPython extensions" OFF) define_option(ARROW_S3 "Build Arrow with S3 support (requires the AWS SDK for C++)" OFF) + define_option(ARROW_TENSORFLOW "Build Arrow with TensorFlow support enabled" OFF) + #---------------------------------------------------------------------- set_option_category("Thirdparty toolchain") diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index e8e4fa6f7bc7..4b707a2092a8 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -15,6 +15,9 @@ # specific language governing permissions and limitations # under the License. +include(ProcessorCount) +processorcount(NPROC) + add_custom_target(rapidjson) add_custom_target(toolchain) add_custom_target(toolchain-benchmarks) @@ -500,7 +503,7 @@ if(${CMAKE_GENERATOR} MATCHES "Makefiles") set(MAKE_BUILD_ARGS "") else() # limit the maximum number of jobs for ninja - set(MAKE_BUILD_ARGS "-j4") + set(MAKE_BUILD_ARGS "-j${NPROC}") endif() # ---------------------------------------------------------------------- @@ -560,7 +563,7 @@ macro(build_boost) else() set(BOOST_BUILD_VARIANT "release") endif() - set(BOOST_BUILD_COMMAND "./b2" "link=${BOOST_BUILD_LINK}" + set(BOOST_BUILD_COMMAND "./b2" "-j${NPROC}" "link=${BOOST_BUILD_LINK}" "variant=${BOOST_BUILD_VARIANT}") if(MSVC) string(REGEX diff --git a/cpp/examples/minimal_build/build.sh b/cpp/examples/minimal_build/build.sh index 068f4898ba0d..a8c1552eebe9 100755 --- a/cpp/examples/minimal_build/build.sh +++ b/cpp/examples/minimal_build/build.sh @@ -22,6 +22,7 @@ # docker run --rm -t -i -v $PWD:/io -v $ARROW_ROOT:/arrow arrow_cpp_minimal /io/build.sh BUILD_DIR=/build +NPROC=$(nproc) mkdir $BUILD_DIR pushd $BUILD_DIR @@ -30,6 +31,8 @@ cmake /arrow/cpp -DBOOST_SOURCE=BUNDLED \ -DARROW_BOOST_USE_SHARED=OFF \ -DARROW_COMPUTE=OFF \ -DARROW_DATASET=OFF \ + -DARROW_FILESYSTEM=OFF \ + -DARROW_HDFS=OFF \ -DARROW_JEMALLOC=OFF \ -DARROW_JSON=OFF \ -DARROW_USE_GLOG=OFF \ @@ -41,6 +44,7 @@ cmake /arrow/cpp -DBOOST_SOURCE=BUNDLED \ -DARROW_WITH_BROTLI=OFF \ -DARROW_BUILD_UTILITIES=OFF -make -j8 +make -j$NPROC +make install popd diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index 91ff44369982..bf5a77e58857 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -114,12 +114,6 @@ set(ARROW_SRCS csv/options.cc csv/parser.cc csv/reader.cc - filesystem/filesystem.cc - filesystem/localfs.cc - filesystem/mockfs.cc - filesystem/path_tree.cc - filesystem/path_util.cc - filesystem/util_internal.cc io/buffered.cc io/compressed.cc io/file.cc @@ -149,6 +143,13 @@ set(ARROW_SRCS vendored/base64.cpp vendored/datetime/tz.cpp) +set(ARROW_TESTING_SRCS + io/test_common.cc + ipc/test_common.cc + testing/gtest_util.cc + testing/random.cc + testing/generator.cc) + # Add dependencies for third-party allocators. # If possible we only want memory_pool.cc to wait for allocators to finish building, # but that only works with Ninja @@ -173,26 +174,6 @@ endif() unset(_allocator_dependencies) -if(ARROW_JSON) - add_subdirectory(json) - set(ARROW_SRCS - ${ARROW_SRCS} - json/options.cc - json/chunked_builder.cc - json/chunker.cc - json/converter.cc - json/parser.cc - json/reader.cc) -endif() - -if(ARROW_S3) - set(ARROW_SRCS ${ARROW_SRCS} filesystem/s3fs.cc) -endif() - -if(ARROW_WITH_URIPARSER) - set(ARROW_SRCS ${ARROW_SRCS} util/uri.cc) -endif() - if("${COMPILER_FAMILY}" STREQUAL "clang") set_property(SOURCE util/io_util.cc APPEND_STRING @@ -202,94 +183,149 @@ endif() # Unconditionally install testing headers that are also useful for Arrow consumers. add_subdirectory(testing) +# +# Add the various components +# + +add_subdirectory(array) +add_subdirectory(csv) +add_subdirectory(io) +add_subdirectory(util) +add_subdirectory(vendored) + if(ARROW_COMPUTE) add_subdirectory(compute) - set(ARROW_SRCS - ${ARROW_SRCS} - compute/context.cc - compute/expression.cc - compute/logical_type.cc - compute/operation.cc - compute/kernels/aggregate.cc - compute/kernels/boolean.cc - compute/kernels/cast.cc - compute/kernels/compare.cc - compute/kernels/count.cc - compute/kernels/hash.cc - compute/kernels/filter.cc - compute/kernels/mean.cc - compute/kernels/sort_to_indices.cc - compute/kernels/sum.cc - compute/kernels/take.cc - compute/kernels/isin.cc - compute/kernels/util_internal.cc - compute/operations/cast.cc - compute/operations/literal.cc) + list(APPEND ARROW_SRCS + compute/context.cc + compute/expression.cc + compute/logical_type.cc + compute/operation.cc + compute/kernels/aggregate.cc + compute/kernels/boolean.cc + compute/kernels/cast.cc + compute/kernels/compare.cc + compute/kernels/count.cc + compute/kernels/hash.cc + compute/kernels/filter.cc + compute/kernels/mean.cc + compute/kernels/sort_to_indices.cc + compute/kernels/sum.cc + compute/kernels/take.cc + compute/kernels/isin.cc + compute/kernels/util_internal.cc + compute/operations/cast.cc + compute/operations/literal.cc) endif() if(ARROW_CUDA) - # IPC extensions required to build the CUDA library - set(ARROW_IPC ON) add_subdirectory(gpu) endif() -if(ARROW_WITH_BROTLI) - add_definitions(-DARROW_WITH_BROTLI) - set(ARROW_SRCS util/compression_brotli.cc ${ARROW_SRCS}) +if(ARROW_DATASET) + add_subdirectory(dataset) endif() -if(ARROW_WITH_BZ2) - add_definitions(-DARROW_WITH_BZ2) - set(ARROW_SRCS util/compression_bz2.cc ${ARROW_SRCS}) +if(ARROW_FILESYSTEM) + add_subdirectory(filesystem) + + list(APPEND ARROW_SRCS + filesystem/filesystem.cc + filesystem/localfs.cc + filesystem/mockfs.cc + filesystem/path_tree.cc + filesystem/path_util.cc + filesystem/util_internal.cc) + + if(ARROW_S3) + list(APPEND ARROW_SRCS filesystem/s3fs.cc) + endif() + + list(APPEND ARROW_TESTING_SRCS filesystem/test_util.cc) endif() -if(ARROW_WITH_LZ4) - add_definitions(-DARROW_WITH_LZ4) - set(ARROW_SRCS util/compression_lz4.cc ${ARROW_SRCS}) +if(ARROW_FLIGHT) + add_subdirectory(flight) endif() -if(ARROW_WITH_SNAPPY) - add_definitions(-DARROW_WITH_SNAPPY) - set(ARROW_SRCS util/compression_snappy.cc ${ARROW_SRCS}) +if(ARROW_HIVESERVER2) + add_subdirectory(dbi/hiveserver2) endif() -if(ARROW_WITH_ZLIB) - add_definitions(-DARROW_WITH_ZLIB) - set(ARROW_SRCS util/compression_zlib.cc ${ARROW_SRCS}) +if(ARROW_IPC) + add_subdirectory(ipc) + + list(APPEND ARROW_SRCS + ipc/dictionary.cc + ipc/feather.cc + ipc/message.cc + ipc/metadata_internal.cc + ipc/options.cc + ipc/reader.cc + ipc/writer.cc) + + if(ARROW_JSON) + list( + APPEND ARROW_SRCS ipc/json_integration.cc ipc/json_internal.cc ipc/json_simple.cc) + endif() + add_dependencies(arrow_dependencies metadata_fbs) endif() -if(ARROW_WITH_ZSTD) - add_definitions(-DARROW_WITH_ZSTD) - set(ARROW_SRCS util/compression_zstd.cc ${ARROW_SRCS}) +if(ARROW_JSON) + add_subdirectory(json) + list(APPEND ARROW_SRCS + json/options.cc + json/chunked_builder.cc + json/chunker.cc + json/converter.cc + json/parser.cc + json/reader.cc) endif() if(ARROW_ORC) add_subdirectory(adapters/orc) - set(ARROW_SRCS adapters/orc/adapter.cc adapters/orc/adapter_util.cc ${ARROW_SRCS}) + list(APPEND ARROW_SRCS adapters/orc/adapter.cc adapters/orc/adapter_util.cc) +endif() + +if(ARROW_PYTHON) + add_subdirectory(python) endif() if(ARROW_TENSORFLOW) add_subdirectory(adapters/tensorflow) endif() -if(ARROW_IPC) - add_subdirectory(ipc) +if(ARROW_WITH_URIPARSER) + list(APPEND ARROW_SRCS util/uri.cc) +endif() - set(ARROW_SRCS - ${ARROW_SRCS} - ipc/dictionary.cc - ipc/feather.cc - ipc/message.cc - ipc/metadata_internal.cc - ipc/options.cc - ipc/reader.cc - ipc/writer.cc) +if(ARROW_WITH_BROTLI) + add_definitions(-DARROW_WITH_BROTLI) + list(APPEND ARROW_SRCS util/compression_brotli.cc) +endif() - if(ARROW_JSON) - set(ARROW_SRCS ${ARROW_SRCS} ipc/json_integration.cc ipc/json_internal.cc - ipc/json_simple.cc) - endif() - add_dependencies(arrow_dependencies metadata_fbs) +if(ARROW_WITH_BZ2) + add_definitions(-DARROW_WITH_BZ2) + list(APPEND ARROW_SRCS util/compression_bz2.cc) +endif() + +if(ARROW_WITH_LZ4) + add_definitions(-DARROW_WITH_LZ4) + list(APPEND ARROW_SRCS util/compression_lz4.cc) +endif() + +if(ARROW_WITH_SNAPPY) + add_definitions(-DARROW_WITH_SNAPPY) + list(APPEND ARROW_SRCS util/compression_snappy.cc) +endif() + +if(ARROW_WITH_ZLIB) + add_definitions(-DARROW_WITH_ZLIB) + list(APPEND ARROW_SRCS util/compression_zlib.cc) +endif() + +if(ARROW_WITH_ZSTD) + add_definitions(-DARROW_WITH_ZSTD) + list(APPEND ARROW_SRCS util/compression_zstd.cc) endif() if(NOT APPLE AND NOT MSVC) @@ -333,12 +369,7 @@ if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS OR ARROW_BUILD_INTEGRATION) # that depend on gtest add_arrow_lib(arrow_testing SOURCES - io/test_common.cc - ipc/test_common.cc - filesystem/test_util.cc - testing/gtest_util.cc - testing/random.cc - testing/generator.cc + ${ARROW_TESTING_SRCS} OUTPUTS ARROW_TESTING_LIBRARIES DEPENDENCIES @@ -412,26 +443,3 @@ add_arrow_test(sparse_tensor_test) add_arrow_benchmark(builder_benchmark) add_arrow_benchmark(type_benchmark) - -add_subdirectory(array) -add_subdirectory(csv) -add_subdirectory(filesystem) -add_subdirectory(io) -add_subdirectory(util) -add_subdirectory(vendored) - -if(ARROW_DATASET) - add_subdirectory(dataset) -endif() - -if(ARROW_FLIGHT) - add_subdirectory(flight) -endif() - -if(ARROW_PYTHON) - add_subdirectory(python) -endif() - -if(ARROW_HIVESERVER2) - add_subdirectory(dbi/hiveserver2) -endif() From 58e1144434802f7fa358f078437be4de120d9c36 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Mon, 30 Sep 2019 12:13:11 -0500 Subject: [PATCH 02/24] ARROW-6564: [Python] Do not require pandas for invoking ChunkedArray.__array__ Follow-up on https://github.com/apache/arrow/pull/5402, to apply the same fix for `ChunkedArray` (@wesm was a bit quick with merging :)) Closes #5497 from jorisvandenbossche/ARROW-6564-chunked-array-to-pandas and squashes the following commits: e6f5adc50 ARROW-6564: Do not require pandas for invoking ChunkedArray.__array__ Authored-by: Joris Van den Bossche Signed-off-by: Wes McKinney --- python/pyarrow/table.pxi | 19 ++++++++++++++++++- python/pyarrow/tests/test_table.py | 11 +++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index 086c4f45011e..a2f9194231b5 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -173,7 +173,24 @@ cdef class ChunkedArray(_PandasConvertible): return result def __array__(self, dtype=None): - values = self.to_pandas().values + cdef: + PyObject* out + PandasOptions c_options + object values + + with nogil: + check_status(libarrow.ConvertChunkedArrayToPandas( + c_options, + self.sp_chunked_array, + self, &out)) + + # wrap_array_output uses pandas to convert to Categorical, here + # always convert to numpy array + values = PyObject_to_object(out) + + if isinstance(values, dict): + values = np.take(values['dictionary'], values['indices']) + if dtype is None: return values return values.astype(dtype) diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py index 434ec8029c1e..da1f084cf20f 100644 --- a/python/pyarrow/tests/test_table.py +++ b/python/pyarrow/tests/test_table.py @@ -203,7 +203,10 @@ def test_chunked_array_to_pandas(): @pytest.mark.pandas +@pytest.mark.nopandas def test_chunked_array_asarray(): + # ensure this is tested both when pandas is present or not (ARROW-6564) + data = [ pa.array([0]), pa.array([1, 2, 3]) @@ -232,6 +235,14 @@ def test_chunked_array_asarray(): assert elements[2:] == [1., 2., 3.] assert np_arr.dtype == np.dtype('float64') + # DictionaryType data will be converted to dense numpy array + arr = pa.DictionaryArray.from_arrays( + pa.array([0, 1, 2, 0, 1]), pa.array(['a', 'b', 'c'])) + chunked_arr = pa.chunked_array([arr, arr]) + np_arr = np.asarray(chunked_arr) + assert np_arr.dtype == np.dtype('object') + assert np_arr.tolist() == ['a', 'b', 'c', 'a', 'b'] * 2 + def test_chunked_array_flatten(): ty = pa.struct([pa.field('x', pa.int16()), From 4aaa2114db2aa6c04416a65204345fdca0580224 Mon Sep 17 00:00:00 2001 From: Pasha Stetsenko Date: Mon, 30 Sep 2019 16:28:19 -0500 Subject: [PATCH 03/24] ARROW-6729: [C++] Prevent data copying in StlStringBuffer Closes #5517 from st-pasha/stl-buffer-nocopy and squashes the following commits: 6f26b4dea try to satisfy lint 2d4dda6c4 remove whitespace 85e91f5e4 Prevent data copying in StlStringBuffer Authored-by: Pasha Stetsenko Signed-off-by: Wes McKinney --- cpp/src/arrow/buffer.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/src/arrow/buffer.cc b/cpp/src/arrow/buffer.cc index bfbf2f8f2ccb..ca78922f2c27 100644 --- a/cpp/src/arrow/buffer.cc +++ b/cpp/src/arrow/buffer.cc @@ -79,7 +79,8 @@ Status Buffer::FromString(const std::string& data, std::shared_ptr* out) class StlStringBuffer : public Buffer { public: - explicit StlStringBuffer(std::string&& data) : Buffer(nullptr, 0), input_(data) { + explicit StlStringBuffer(std::string&& data) + : Buffer(nullptr, 0), input_(std::move(data)) { data_ = reinterpret_cast(input_.c_str()); size_ = static_cast(input_.size()); capacity_ = size_; From c95aaab6cab90656949e4443a290f071a1765936 Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Mon, 30 Sep 2019 17:42:05 -0500 Subject: [PATCH 04/24] ARROW-6646: [Go] Write no IPC buffer metadata for NullType Closes #5547 from sbinet/issue-6646 and squashes the following commits: 5fdd6496d ARROW-6646: Write no IPC buffer metadata for NullType Authored-by: Sebastien Binet Signed-off-by: Wes McKinney --- go/arrow/ipc/file_reader.go | 6 ++---- go/arrow/ipc/writer.go | 13 +++++++++---- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/go/arrow/ipc/file_reader.go b/go/arrow/ipc/file_reader.go index 813f5838b724..2b08b01240cb 100644 --- a/go/arrow/ipc/file_reader.go +++ b/go/arrow/ipc/file_reader.go @@ -437,10 +437,8 @@ func (ctx *arrayLoaderContext) loadChild(dt arrow.DataType) array.Interface { } func (ctx *arrayLoaderContext) loadNull() array.Interface { - field, buffers := ctx.loadCommon(1) - buffers = append(buffers, ctx.buffer()) - - data := array.NewData(arrow.Null, int(field.Length()), buffers, nil, int(field.NullCount()), 0) + field := ctx.field() + data := array.NewData(arrow.Null, int(field.Length()), nil, nil, int(field.NullCount()), 0) defer data.Release() return array.MakeFromData(data) diff --git a/go/arrow/ipc/writer.go b/go/arrow/ipc/writer.go index 356159867124..4ed0ff0b1ee1 100644 --- a/go/arrow/ipc/writer.go +++ b/go/arrow/ipc/writer.go @@ -221,14 +221,19 @@ func (w *recordEncoder) visit(p *payload, arr array.Interface) error { case 0: p.body = append(p.body, nil) default: - data := arr.Data() - bitmap := newTruncatedBitmap(w.mem, int64(data.Offset()), int64(data.Len()), data.Buffers()[0]) - p.body = append(p.body, bitmap) + switch arr.DataType().ID() { + case arrow.NULL: + // Null type has no validity bitmap + default: + data := arr.Data() + bitmap := newTruncatedBitmap(w.mem, int64(data.Offset()), int64(data.Len()), data.Buffers()[0]) + p.body = append(p.body, bitmap) + } } switch dtype := arr.DataType().(type) { case *arrow.NullType: - p.body = append(p.body, nil) + // ok. NullArrays are completely empty. case *arrow.BooleanType: var ( From 610deb7dc76cb4a5dfc3cd28e02963be97209109 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Mon, 30 Sep 2019 17:45:13 -0500 Subject: [PATCH 05/24] ARROW-6685: [C++] Ignore trailing slashes in S3FS Closes #5546 from pitrou/ARROW-6685-s3fs-trailing-slash and squashes the following commits: ae315aff8 ARROW-6685: Ignore trailing slashes in S3FS Authored-by: Antoine Pitrou Signed-off-by: Wes McKinney --- cpp/src/arrow/filesystem/filesystem_test.cc | 12 ++++++++++++ cpp/src/arrow/filesystem/path_util.cc | 2 +- cpp/src/arrow/filesystem/s3fs.cc | 11 ++++++----- cpp/src/arrow/filesystem/s3fs_test.cc | 21 +++++++++++++++++++++ 4 files changed, 40 insertions(+), 6 deletions(-) diff --git a/cpp/src/arrow/filesystem/filesystem_test.cc b/cpp/src/arrow/filesystem/filesystem_test.cc index c8eba30218de..c199a2dde2cb 100644 --- a/cpp/src/arrow/filesystem/filesystem_test.cc +++ b/cpp/src/arrow/filesystem/filesystem_test.cc @@ -155,6 +155,18 @@ TEST(PathUtil, EnsureTrailingSlash) { ASSERT_EQ("/abc/", EnsureTrailingSlash("/abc/")); } +TEST(PathUtil, RemoveTrailingSlash) { + ASSERT_EQ("", std::string(RemoveTrailingSlash(""))); + ASSERT_EQ("", std::string(RemoveTrailingSlash("/"))); + ASSERT_EQ("", std::string(RemoveTrailingSlash("//"))); + ASSERT_EQ("abc/def", std::string(RemoveTrailingSlash("abc/def"))); + ASSERT_EQ("abc/def", std::string(RemoveTrailingSlash("abc/def/"))); + ASSERT_EQ("abc/def", std::string(RemoveTrailingSlash("abc/def//"))); + ASSERT_EQ("/abc/def", std::string(RemoveTrailingSlash("/abc/def"))); + ASSERT_EQ("/abc/def", std::string(RemoveTrailingSlash("/abc/def/"))); + ASSERT_EQ("/abc/def", std::string(RemoveTrailingSlash("/abc/def//"))); +} + //////////////////////////////////////////////////////////////////////////// // Generic MockFileSystem tests diff --git a/cpp/src/arrow/filesystem/path_util.cc b/cpp/src/arrow/filesystem/path_util.cc index cfc59a6f8d44..2ba143e59b94 100644 --- a/cpp/src/arrow/filesystem/path_util.cc +++ b/cpp/src/arrow/filesystem/path_util.cc @@ -115,7 +115,7 @@ std::string EnsureTrailingSlash(const std::string& s) { } util::string_view RemoveTrailingSlash(util::string_view key) { - if (!key.empty() && key.back() == kSep) { + while (!key.empty() && key.back() == kSep) { key.remove_suffix(1); } return key; diff --git a/cpp/src/arrow/filesystem/s3fs.cc b/cpp/src/arrow/filesystem/s3fs.cc index 136d12e20302..8053c223ade6 100644 --- a/cpp/src/arrow/filesystem/s3fs.cc +++ b/cpp/src/arrow/filesystem/s3fs.cc @@ -177,17 +177,18 @@ struct S3Path { std::vector key_parts; static Status FromString(const std::string& s, S3Path* out) { - auto first_sep = s.find_first_of(kSep); + const auto src = internal::RemoveTrailingSlash(s); + auto first_sep = src.find_first_of(kSep); if (first_sep == 0) { return Status::Invalid("Path cannot start with a separator ('", s, "')"); } if (first_sep == std::string::npos) { - *out = {s, s, "", {}}; + *out = {std::string(src), std::string(src), "", {}}; return Status::OK(); } - out->full_path = s; - out->bucket = s.substr(0, first_sep); - out->key = s.substr(first_sep + 1); + out->full_path = std::string(src); + out->bucket = std::string(src.substr(0, first_sep)); + out->key = std::string(src.substr(first_sep + 1)); out->key_parts = internal::SplitAbstractPath(out->key); return internal::ValidateAbstractPathParts(out->key_parts); } diff --git a/cpp/src/arrow/filesystem/s3fs_test.cc b/cpp/src/arrow/filesystem/s3fs_test.cc index b19b18e01638..8564901ae400 100644 --- a/cpp/src/arrow/filesystem/s3fs_test.cc +++ b/cpp/src/arrow/filesystem/s3fs_test.cc @@ -375,6 +375,10 @@ TEST_F(TestS3FS, GetTargetStatsBucket) { AssertFileStats(fs_.get(), "bucket", FileType::Directory); AssertFileStats(fs_.get(), "empty-bucket", FileType::Directory); AssertFileStats(fs_.get(), "non-existent-bucket", FileType::NonExistent); + // Trailing slashes + AssertFileStats(fs_.get(), "bucket/", FileType::Directory); + AssertFileStats(fs_.get(), "empty-bucket/", FileType::Directory); + AssertFileStats(fs_.get(), "non-existent-bucket/", FileType::NonExistent); } TEST_F(TestS3FS, GetTargetStatsObject) { @@ -393,6 +397,12 @@ TEST_F(TestS3FS, GetTargetStatsObject) { AssertFileStats(fs_.get(), "bucket/emptyd", FileType::NonExistent); AssertFileStats(fs_.get(), "bucket/somed", FileType::NonExistent); AssertFileStats(fs_.get(), "non-existent-bucket/somed", FileType::NonExistent); + + // Trailing slashes + AssertFileStats(fs_.get(), "bucket/emptydir/", FileType::Directory, kNoSize); + AssertFileStats(fs_.get(), "bucket/somefile/", FileType::File, 9); + AssertFileStats(fs_.get(), "bucket/emptyd/", FileType::NonExistent); + AssertFileStats(fs_.get(), "non-existent-bucket/somed/", FileType::NonExistent); } TEST_F(TestS3FS, GetTargetStatsSelector) { @@ -447,6 +457,17 @@ TEST_F(TestS3FS, GetTargetStatsSelector) { ASSERT_OK(fs_->GetTargetStats(select, &stats)); ASSERT_EQ(stats.size(), 0); select.allow_non_existent = false; + + // Trailing slashes + select.base_dir = "empty-bucket/"; + ASSERT_OK(fs_->GetTargetStats(select, &stats)); + ASSERT_EQ(stats.size(), 0); + select.base_dir = "non-existent-bucket/"; + ASSERT_RAISES(IOError, fs_->GetTargetStats(select, &stats)); + select.base_dir = "bucket/"; + ASSERT_OK(fs_->GetTargetStats(select, &stats)); + SortStats(&stats); + ASSERT_EQ(stats.size(), 3); } TEST_F(TestS3FS, GetTargetStatsSelectorRecursive) { From df9fc54f1932cfaae156719757087af09a2186a1 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Mon, 30 Sep 2019 17:46:00 -0500 Subject: [PATCH 06/24] ARROW-6740: [C++] Unmap MemoryMappedFile as soon as possible When a MemoryMappedFile has been closed and all exported buffers have been destroyed, unmap the memory region instead of waiting for the file object destruction. This allows deleting the underlying file on Windows. Closes #5551 from pitrou/ARROW-6740-munmap-timely and squashes the following commits: 9afab352e ARROW-6740: Unmap MemoryMappedFile as soon as possible Authored-by: Antoine Pitrou Signed-off-by: Wes McKinney --- cpp/src/arrow/io/file.cc | 133 ++++++++++++++++++++------------ cpp/src/arrow/io/file_test.cc | 2 +- python/pyarrow/tests/test_io.py | 15 ++++ 3 files changed, 98 insertions(+), 52 deletions(-) diff --git a/cpp/src/arrow/io/file.cc b/cpp/src/arrow/io/file.cc index 2f3f5bac2e70..3a9306300d70 100644 --- a/cpp/src/arrow/io/file.cc +++ b/cpp/src/arrow/io/file.cc @@ -384,24 +384,47 @@ Status FileOutputStream::Write(const void* data, int64_t length) { int FileOutputStream::file_descriptor() const { return impl_->fd(); } // ---------------------------------------------------------------------- -// Implement MemoryMappedFile as a buffer subclass -// The class doesn't differentiate between size and capacity -class MemoryMappedFile::MemoryMap : public MutableBuffer { +// Implement MemoryMappedFile + +class MemoryMappedFile::MemoryMap + : public std::enable_shared_from_this { public: - MemoryMap() : MutableBuffer(nullptr, 0) {} + // An object representing the entire memory-mapped region. + // It can be sliced in order to return individual subregions, which + // will then keep the original region alive as long as necessary. + class Region : public MutableBuffer { + public: + Region(std::shared_ptr memory_map, uint8_t* data, + int64_t size) + : MutableBuffer(data, size) { + is_mutable_ = memory_map->writable(); + if (!is_mutable_) { + mutable_data_ = nullptr; + } + } - ~MemoryMap() { - ARROW_CHECK_OK(Close()); - if (mutable_data_ != nullptr) { - int result = munmap(mutable_data_, static_cast(map_len_)); - ARROW_CHECK_EQ(result, 0) << "munmap failed"; + ~Region() { + if (data_ != nullptr) { + int result = munmap(data(), static_cast(size_)); + ARROW_CHECK_EQ(result, 0) << "munmap failed"; + } } - } + + // For convenience + uint8_t* data() { return const_cast(data_); } + + void Detach() { data_ = nullptr; } + }; + + MemoryMap() : file_size_(0), map_len_(0) {} + + ~MemoryMap() { ARROW_CHECK_OK(Close()); } Status Close() { if (file_->is_open()) { - // NOTE: we don't unmap here, so that buffers exported through Read() - // remain valid until the MemoryMap object is destroyed + // Lose our reference to the MemoryMappedRegion, so that munmap() + // is called as soon as all buffer exports are released. + region_.reset(); return file_->Close(); } else { return Status::OK(); @@ -422,14 +445,10 @@ class MemoryMappedFile::MemoryMap : public MutableBuffer { constexpr bool truncate = false; constexpr bool write_only = false; RETURN_NOT_OK(file_->OpenWritable(path, truncate, append, write_only)); - - is_mutable_ = true; } else { prot_flags_ = PROT_READ; map_mode_ = MAP_PRIVATE; // Changes are not to be committed back to the file RETURN_NOT_OK(file_->OpenReadable(path)); - - is_mutable_ = false; } map_len_ = offset_ = 0; @@ -450,33 +469,38 @@ class MemoryMappedFile::MemoryMap : public MutableBuffer { if (!writable()) { return Status::IOError("Cannot resize a readonly memory map"); } - if (map_len_ != size_) { + if (map_len_ != file_size_) { return Status::IOError("Cannot resize a partial memory map"); } + if (region_.use_count() > 1) { + // There are buffer exports currently, the MemoryMapRemap() call + // would make the buffers invalid + return Status::IOError("Cannot resize memory map while there are active readers"); + } if (new_size == 0) { - if (mutable_data_ != nullptr) { - // just unmap the mmap and truncate the file to 0 size - if (munmap(mutable_data_, capacity_) != 0) { - return Status::IOError("Cannot unmap the file"); - } + if (map_len_ > 0) { + // Just unmap the mmap and truncate the file to 0 size + region_.reset(); RETURN_NOT_OK(::arrow::internal::FileTruncate(file_->fd(), 0)); - data_ = mutable_data_ = nullptr; - map_len_ = offset_ = size_ = capacity_ = 0; + map_len_ = offset_ = file_size_ = 0; } position_ = 0; return Status::OK(); } - if (mutable_data_) { + if (map_len_ > 0) { void* result; - RETURN_NOT_OK(::arrow::internal::MemoryMapRemap(mutable_data_, size_, new_size, + auto data = region_->data(); + RETURN_NOT_OK(::arrow::internal::MemoryMapRemap(data, map_len_, new_size, file_->fd(), &result)); - map_len_ = size_ = capacity_ = new_size; + region_->Detach(); // avoid munmap() on destruction + region_ = std::make_shared(shared_from_this(), + static_cast(result), new_size); + map_len_ = file_size_ = new_size; offset_ = 0; - data_ = mutable_data_ = static_cast(result); - if (position_ > size_) { - position_ = size_; + if (position_ > map_len_) { + position_ = map_len_; } } else { DCHECK_EQ(position_, 0); @@ -487,9 +511,6 @@ class MemoryMappedFile::MemoryMap : public MutableBuffer { return Status::OK(); } - // map_len_ == size_ if memory mapping on the whole file - int64_t size() const { return map_len_; } - Status Seek(int64_t position) { if (position < 0) { return Status::Invalid("position is out of bounds"); @@ -498,11 +519,28 @@ class MemoryMappedFile::MemoryMap : public MutableBuffer { return Status::OK(); } + Status Slice(int64_t offset, int64_t length, std::shared_ptr* out) { + length = std::max(0, std::min(length, map_len_ - offset)); + + if (length > 0) { + DCHECK_NE(region_, nullptr); + *out = SliceBuffer(region_, offset, length); + } else { + *out = std::make_shared(nullptr, 0); + } + return Status::OK(); + } + + // map_len_ == file_size_ if memory mapping on the whole file + int64_t size() const { return map_len_; } + int64_t position() { return position_; } void advance(int64_t nbytes) { position_ = position_ + nbytes; } - uint8_t* head() { return mutable_data_ + position_; } + uint8_t* head() { return data() + position_; } + + uint8_t* data() { return region_ ? region_->data() : nullptr; } bool writable() { return file_->mode() != FileMode::READ; } @@ -518,10 +556,11 @@ class MemoryMappedFile::MemoryMap : public MutableBuffer { // Initialize the mmap and set size, capacity and the data pointers Status InitMMap(int64_t initial_size, bool resize_file = false, const int64_t offset = 0, const int64_t length = -1) { + DCHECK(!region_); + if (resize_file) { RETURN_NOT_OK(::arrow::internal::FileTruncate(file_->fd(), initial_size)); } - DCHECK(data_ == nullptr && mutable_data_ == nullptr); size_t mmap_length = static_cast(initial_size); if (length > initial_size) { @@ -539,14 +578,19 @@ class MemoryMappedFile::MemoryMap : public MutableBuffer { } map_len_ = mmap_length; offset_ = offset; - size_ = capacity_ = initial_size; - data_ = mutable_data_ = static_cast(result); + region_ = std::make_shared(shared_from_this(), static_cast(result), + map_len_); + file_size_ = initial_size; return Status::OK(); } + std::unique_ptr file_; int prot_flags_; int map_mode_; + + std::shared_ptr region_; + int64_t file_size_; int64_t position_; int64_t offset_; int64_t map_len_; @@ -619,14 +663,7 @@ Status MemoryMappedFile::ReadAt(int64_t position, int64_t nbytes, auto guard_resize = memory_map_->writable() ? std::unique_lock(memory_map_->resize_lock()) : std::unique_lock(); - nbytes = std::max(0, std::min(nbytes, memory_map_->size() - position)); - - if (nbytes > 0) { - *out = SliceBuffer(memory_map_, position, nbytes); - } else { - *out = std::make_shared(nullptr, 0); - } - return Status::OK(); + return memory_map_->Slice(position, nbytes, out); } Status MemoryMappedFile::ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read, @@ -697,12 +734,6 @@ Status MemoryMappedFile::Resize(int64_t new_size) { std::unique_lock write_guard(memory_map_->write_lock(), std::defer_lock); std::unique_lock resize_guard(memory_map_->resize_lock(), std::defer_lock); std::lock(write_guard, resize_guard); - // having both locks, we can check the number of times memory_map_ - // was borrwed (meaning number of reader still holding a ref to it + 1) - // and if it's greater than 1, we fail loudly - if (memory_map_.use_count() > 1) { - return Status::IOError("Cannot resize memory map while there are active readers"); - } RETURN_NOT_OK(memory_map_->Resize(new_size)); return Status::OK(); } diff --git a/cpp/src/arrow/io/file_test.cc b/cpp/src/arrow/io/file_test.cc index eed75add147c..ab12148b3837 100644 --- a/cpp/src/arrow/io/file_test.cc +++ b/cpp/src/arrow/io/file_test.cc @@ -636,7 +636,7 @@ class TestMemoryMappedFile : public ::testing::Test, public MemoryMapFixture { TEST_F(TestMemoryMappedFile, InvalidUsages) {} -TEST_F(TestMemoryMappedFile, ZeroSizeFlie) { +TEST_F(TestMemoryMappedFile, ZeroSizeFile) { std::string path = "io-memory-map-zero-size"; std::shared_ptr result; ASSERT_OK(InitMemoryMap(0, path, &result)); diff --git a/python/pyarrow/tests/test_io.py b/python/pyarrow/tests/test_io.py index 76148e7d8b2d..55b183b7227e 100644 --- a/python/pyarrow/tests/test_io.py +++ b/python/pyarrow/tests/test_io.py @@ -839,6 +839,21 @@ def test_memory_map_large_seeks(): check_large_seeks(pa.memory_map) +def test_memory_map_close_remove(tmpdir): + # ARROW-6740: should be able to delete closed memory-mapped file (Windows) + path = os.path.join(str(tmpdir), guid()) + mmap = pa.create_memory_map(path, 4096) + mmap.close() + assert mmap.closed + os.remove(path) # Shouldn't fail + + +def test_memory_map_deref_remove(tmpdir): + path = os.path.join(str(tmpdir), guid()) + pa.create_memory_map(path, 4096) + os.remove(path) # Shouldn't fail + + def test_os_file_writer(tmpdir): SIZE = 4096 arr = np.random.randint(0, 256, size=SIZE).astype('u1') From 9e0e1a2f91f8711a61710eb18d38e5d251917742 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Mon, 30 Sep 2019 17:47:19 -0500 Subject: [PATCH 07/24] ARROW-6708: [C++] Fix hardcoded boost library names In the case where `find_package(Boost ...)` succeeded, just use whatever it gave us. Closes #5510 from pitrou/ARROW-6708-boost-library-static and squashes the following commits: da8d28a30 ARROW-6708: Fix hardcoded boost library names Authored-by: Antoine Pitrou Signed-off-by: Wes McKinney --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 4b707a2092a8..ce495f177f8f 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -641,6 +641,10 @@ if(TARGET Boost::system) set(BOOST_SYSTEM_LIBRARY Boost::system) set(BOOST_FILESYSTEM_LIBRARY Boost::filesystem) set(BOOST_REGEX_LIBRARY Boost::regex) +elseif(BoostAlt_FOUND) + set(BOOST_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY}) + set(BOOST_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY}) + set(BOOST_REGEX_LIBRARY ${Boost_REGEX_LIBRARY}) else() set(BOOST_SYSTEM_LIBRARY boost_system_static) set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_static) @@ -648,8 +652,8 @@ else() endif() set(ARROW_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY}) -message(STATUS "Boost include dir: " ${Boost_INCLUDE_DIR}) -message(STATUS "Boost libraries: " ${Boost_LIBRARIES}) +message(STATUS "Boost include dir: ${Boost_INCLUDE_DIR}") +message(STATUS "Boost libraries: ${ARROW_BOOST_LIBS}") include_directories(SYSTEM ${Boost_INCLUDE_DIR}) From 98d8a6d796f838ed55766a8318c83915eb05696e Mon Sep 17 00:00:00 2001 From: liyafan82 Date: Mon, 30 Sep 2019 19:07:01 -0500 Subject: [PATCH 08/24] ARROW-6722: [Java] Provide a uniform way to get vector name Currently, the getName method is defined in BaseValueVector, as an abstract class. However, some vector does not extend the BaseValueVector, like StructVector, UnionVector, ZeroVector. In this issue, we move the method to ValueVector interface, the base interface for all vectors. This makes it easier to get a vector's name without checking its type. Closes #5525 from liyafan82/fly_0927_name and squashes the following commits: 8e3c2897a Provide a uniform way to get vector name Authored-by: liyafan82 Signed-off-by: Wes McKinney --- java/vector/src/main/codegen/templates/UnionVector.java | 5 +++++ .../main/java/org/apache/arrow/vector/BaseValueVector.java | 2 -- .../src/main/java/org/apache/arrow/vector/ValueVector.java | 6 ++++++ .../src/main/java/org/apache/arrow/vector/ZeroVector.java | 5 +++++ .../arrow/vector/complex/AbstractContainerVector.java | 5 +++++ 5 files changed, 21 insertions(+), 2 deletions(-) diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java index eef2708b5311..b513ba539927 100644 --- a/java/vector/src/main/codegen/templates/UnionVector.java +++ b/java/vector/src/main/codegen/templates/UnionVector.java @@ -690,4 +690,9 @@ public int hashCode(int index) { public OUT accept(VectorVisitor visitor, IN value) { return visitor.visit(this, value); } + + @Override + public String getName() { + return name; + } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java index 166af07055d0..f600d2ed1002 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java @@ -55,8 +55,6 @@ protected BaseValueVector(BufferAllocator allocator) { this.allocator = Preconditions.checkNotNull(allocator, "allocator cannot be null"); } - public abstract String getName(); - @Override public String toString() { return super.toString() + "[name = " + getName() + ", ...]"; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java index 1cfa81dbba3b..9a8647a4e2e7 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java @@ -271,4 +271,10 @@ public interface ValueVector extends Closeable, Iterable { * @param the input data together with visitor. */ OUT accept(VectorVisitor visitor, IN value); + + /** + * Gets the name of the vector. + * @return the name of the vector. + */ + String getName(); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java index 41631b131a15..14c671ff12ad 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java @@ -265,4 +265,9 @@ public void copyFrom(int fromIndex, int thisIndex, ValueVector from) { public void copyFromSafe(int fromIndex, int thisIndex, ValueVector from) { throw new UnsupportedOperationException(); } + + @Override + public String getName() { + return getField().getName(); + } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java index b3fcd90744dd..3451d1cbcc42 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java @@ -132,4 +132,9 @@ public void copyFrom(int fromIndex, int thisIndex, ValueVector from) { public void copyFromSafe(int fromIndex, int thisIndex, ValueVector from) { throw new UnsupportedOperationException(); } + + @Override + public String getName() { + return name; + } } From 155415c63de5b3d6257feec57f25be5208f8ebf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 1 Oct 2019 15:22:19 +0200 Subject: [PATCH 09/24] ARROW-6655: [Python] Filesystem bindings for S3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add support for S3FileSystem in the python bindings. - Fixed issue with reading all the content of an S3 object - Introduce `minio_server` fixture for parametrized testing of all filesystem implementations - Fixed s3fs parquet test and updated it to use minio_server fixture Closes #5423 from kszucs/s3 and squashes the following commits: 384c96052 Resolve review comments 73e6625f9 S3Options 98bd91ad1 remove commented tests db89859ca rename to s3fs 44784582a fix read() issue c1df10b92 initialization in first use 192ab6547 flake8 f70f9fbd8 remove minio-client dependency d399643dc simplify test suite fee57a9a4 remove accidentally committed files 751cfd429 resolve a couple of review comments; enum workaround 45436f7b3 cython flake8 38dcb88d0 rat c541b3e15 comment left 098048a8a more compat 00340ed4c fixture error handling 2be25ce29 auto initialize s3 on import 88e0c9f79 py2 compat 8585a6085 py2 compat d37228711 install minio in the conda-toolchain build 041cad42a executable flag fb0f2813a travis minio install script 8cbe0eeef travis osx 72e56a68f enable S3 in travis python builds 7800c75d8 appveyor flag 7daf5668d fix syntax error in travis script 68eb59161 enable PYARROW_WITH_S3 on appveyor 2cb19d1ff conditional import of test dependencies efa05d28e use minio for dask.s3fs test too f25ae5aed travis 9042c7e4c use S3FS_DIR 9ce7180d1 cmake format; fix orc cimport 45a2a17ba docstrings c0b91621e test requirements; flake8 44aedfd10 stat test a343950e2 testing suite 1551b525c wip dd41d21c2 imports 5200af16f s3 filesystem bindings Authored-by: Krisztián Szűcs Signed-off-by: Antoine Pitrou --- .travis.yml | 7 + ci/conda_env_python.yml | 1 + ci/cpp-msvc-build-main.bat | 3 + ci/travis_install_linux.sh | 8 - ci/travis_install_minio.sh | 35 ++ ci/travis_install_osx.sh | 2 +- ci/travis_script_python.sh | 8 + cpp/cmake_modules/ThirdpartyToolchain.cmake | 10 + cpp/src/arrow/filesystem/s3fs.cc | 6 + cpp/src/arrow/filesystem/s3fs.h | 2 +- cpp/src/arrow/filesystem/s3fs_test.cc | 11 +- python/CMakeLists.txt | 4 + python/pyarrow/_csv.pyx | 4 +- python/pyarrow/_cuda.pyx | 4 +- python/pyarrow/_fs.pxd | 68 +++ python/pyarrow/_fs.pyx | 69 +-- python/pyarrow/_json.pyx | 4 +- python/pyarrow/_orc.pxd | 4 +- python/pyarrow/_orc.pyx | 2 +- python/pyarrow/_parquet.pxd | 10 +- python/pyarrow/_parquet.pyx | 8 +- python/pyarrow/_s3fs.pyx | 173 +++++++ python/pyarrow/feather.pxi | 4 +- python/pyarrow/fs.py | 9 +- python/pyarrow/includes/libarrow.pxd | 167 +++++-- python/pyarrow/includes/libarrow_fs.pxd | 92 ---- python/pyarrow/includes/libarrow_s3fs.pxd | 60 +++ python/pyarrow/io-hdfs.pxi | 4 +- python/pyarrow/io.pxi | 56 +-- python/pyarrow/ipc.pxi | 28 +- python/pyarrow/lib.pxd | 24 +- python/pyarrow/s3fs.py | 27 ++ python/pyarrow/serialization.pxi | 6 +- python/pyarrow/tests/conftest.py | 55 +++ python/pyarrow/tests/test_flight.py | 12 +- python/pyarrow/tests/test_fs.py | 474 ++++++++++++-------- python/pyarrow/tests/test_parquet.py | 55 ++- python/pyarrow/util.py | 10 + python/requirements-test.txt | 1 + python/setup.py | 7 + 40 files changed, 1054 insertions(+), 480 deletions(-) create mode 100755 ci/travis_install_minio.sh create mode 100644 python/pyarrow/_fs.pxd create mode 100644 python/pyarrow/_s3fs.pyx delete mode 100644 python/pyarrow/includes/libarrow_fs.pxd create mode 100644 python/pyarrow/includes/libarrow_s3fs.pxd create mode 100644 python/pyarrow/s3fs.py diff --git a/.travis.yml b/.travis.yml index 4fc143a493db..990fd2256674 100644 --- a/.travis.yml +++ b/.travis.yml @@ -87,6 +87,7 @@ matrix: - if [ $ARROW_CI_CPP_AFFECTED != "1" ] && [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh + - $TRAVIS_BUILD_DIR/ci/travis_install_minio.sh # If either C++ or Python changed, we must install the C++ libraries - git submodule update --init - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh @@ -110,12 +111,14 @@ matrix: - ARROW_TRAVIS_USE_SYSTEM_JAVA=1 - ARROW_TRAVIS_USE_TOOLCHAIN=1 - ARROW_TRAVIS_VALGRIND=1 + - ARROW_TRAVIS_S3=1 # TODO(wesm): Run the benchmarks outside of Travis # - ARROW_TRAVIS_PYTHON_BENCHMARKS=1 before_script: - if [ $ARROW_CI_PYTHON_AFFECTED != "1" ] && [ $ARROW_CI_DOCS_AFFECTED != "1" ]; then exit; fi - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh + - $TRAVIS_BUILD_DIR/ci/travis_install_minio.sh - $TRAVIS_BUILD_DIR/ci/travis_install_toolchain.sh script: - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh || travis_terminate 1 @@ -136,6 +139,7 @@ matrix: - ARROW_TRAVIS_PLASMA=1 - ARROW_TRAVIS_FLIGHT=1 - ARROW_TRAVIS_ORC=1 + - ARROW_TRAVIS_S3=1 - ARROW_TRAVIS_PARQUET=1 # TODO(ARROW-4763): llvm and llvmdev packages are in conflict: # https://github.com/conda-forge/llvmdev-feedstock/issues/60 @@ -149,6 +153,7 @@ matrix: - if [ $ARROW_CI_CPP_AFFECTED != "1" ] && [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi # If either C++ or Python changed, we must install the C++ libraries - git submodule update --init + - $TRAVIS_BUILD_DIR/ci/travis_install_minio.sh - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh script: - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1 @@ -161,6 +166,7 @@ matrix: cache: addons: env: + - ARROW_TRAVIS_S3=1 - ARROW_TRAVIS_PLASMA=1 - ARROW_TRAVIS_USE_TOOLCHAIN=1 - ARROW_BUILD_WARNING_LEVEL=CHECKIN @@ -170,6 +176,7 @@ matrix: before_script: script: - if [ $ARROW_CI_PYTHON_AFFECTED != "1" ]; then exit; fi + - $TRAVIS_BUILD_DIR/ci/travis_install_minio.sh - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6 - name: "Java OpenJDK8 and OpenJDK11" language: cpp diff --git a/ci/conda_env_python.yml b/ci/conda_env_python.yml index a0cd737b326c..0e6e5bf8d5c3 100644 --- a/ci/conda_env_python.yml +++ b/ci/conda_env_python.yml @@ -22,6 +22,7 @@ numpy>=1.14 pandas pytest pytest-faulthandler +pytest-lazy-fixture pytz setuptools setuptools_scm=3.2.0 diff --git a/ci/cpp-msvc-build-main.bat b/ci/cpp-msvc-build-main.bat index b088e2eec763..b6d1b20a5109 100644 --- a/ci/cpp-msvc-build-main.bat +++ b/ci/cpp-msvc-build-main.bat @@ -98,6 +98,9 @@ pip install -r requirements.txt pickle5 set PYARROW_CXXFLAGS=%ARROW_CXXFLAGS% set PYARROW_CMAKE_GENERATOR=%GENERATOR% +if "%ARROW_S3%" == "ON" ( + set PYARROW_WITH_S3=ON +) if "%ARROW_BUILD_FLIGHT%" == "ON" ( @rem ARROW-5441: bundling Arrow Flight libraries not implemented set PYARROW_BUNDLE_ARROW_CPP=OFF diff --git a/ci/travis_install_linux.sh b/ci/travis_install_linux.sh index a5283139a833..441d50d9b16c 100755 --- a/ci/travis_install_linux.sh +++ b/ci/travis_install_linux.sh @@ -42,14 +42,6 @@ if [ "$ARROW_TRAVIS_GANDIVA" == "1" ]; then sudo apt-get install -y -qq llvm-$ARROW_LLVM_MAJOR_VERSION-dev fi -if [ "$ARROW_TRAVIS_S3" == "1" ]; then - # Download the Minio S3 server into PATH - S3FS_DIR=~/.local/bin/ - mkdir -p $S3FS_DIR - wget --directory-prefix $S3FS_DIR https://dl.min.io/server/minio/release/linux-amd64/minio - chmod +x $S3FS_DIR/minio -fi - if [ "$ARROW_TRAVIS_USE_SYSTEM" == "1" ]; then if [ "$DISTRO_CODENAME" == "xenial" ]; then # TODO(ARROW-4761): Install libzstd-dev once we support zstd<1 diff --git a/ci/travis_install_minio.sh b/ci/travis_install_minio.sh new file mode 100755 index 000000000000..5459bb8bfc4a --- /dev/null +++ b/ci/travis_install_minio.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +set -x + +if [ "$ARROW_TRAVIS_S3" == "1" ]; then + # Download the Minio S3 server into PATH + if [ $TRAVIS_OS_NAME = "osx" ]; then + MINIO_URL=https://dl.min.io/server/minio/release/darwin-amd64/minio + else + MINIO_URL=https://dl.min.io/server/minio/release/linux-amd64/minio + fi + + S3FS_DIR=~/.local/bin/ + mkdir -p $S3FS_DIR + wget --directory-prefix $S3FS_DIR $MINIO_URL + chmod +x $S3FS_DIR/minio +fi \ No newline at end of file diff --git a/ci/travis_install_osx.sh b/ci/travis_install_osx.sh index 38e971710dea..2d79eb017ed5 100755 --- a/ci/travis_install_osx.sh +++ b/ci/travis_install_osx.sh @@ -40,4 +40,4 @@ if [ "$ARROW_CI_RUBY_AFFECTED" = "1" ]; then run_brew bundle --file=$TRAVIS_BUILD_DIR/cpp/Brewfile --verbose run_brew bundle --file=$TRAVIS_BUILD_DIR/c_glib/Brewfile --verbose rm ${brew_log_path} -fi +fi \ No newline at end of file diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh index 6b095b878e59..8b3066f6e768 100755 --- a/ci/travis_script_python.sh +++ b/ci/travis_script_python.sh @@ -100,6 +100,10 @@ CMAKE_COMMON_FLAGS="-DARROW_EXTRA_ERROR_CONTEXT=ON" PYTHON_CPP_BUILD_TARGETS="arrow_python-all plasma parquet" +if [ "$ARROW_TRAVIS_S3" == "1" ]; then + CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_S3=ON" +fi + if [ "$ARROW_TRAVIS_FLIGHT" == "1" ]; then CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_FLIGHT=ON" fi @@ -164,6 +168,9 @@ export PYARROW_BUILD_TYPE=$ARROW_BUILD_TYPE export PYARROW_WITH_PARQUET=1 export PYARROW_WITH_PLASMA=1 export PYARROW_WITH_ORC=1 +if [ "$ARROW_TRAVIS_S3" == "1" ]; then + export PYARROW_WITH_S3=1 +fi if [ "$ARROW_TRAVIS_FLIGHT" == "1" ]; then export PYARROW_WITH_FLIGHT=1 fi @@ -177,6 +184,7 @@ python setup.py develop python -c "import pyarrow.parquet" python -c "import pyarrow.plasma" python -c "import pyarrow.orc" +python -c "import pyarrow.fs" # Ensure we do eagerly import pandas (or other expensive imports) python < scripts/test_imports.py diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index ce495f177f8f..ace517124b65 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -2542,6 +2542,16 @@ if(ARROW_S3) include_directories(SYSTEM ${AWSSDK_INCLUDE_DIR}) message(STATUS "Found AWS SDK headers: ${AWSSDK_INCLUDE_DIR}") message(STATUS "Found AWS SDK libraries: ${AWSSDK_LINK_LIBRARIES}") + + if(APPLE) + # CoreFoundation's path is hardcoded in the CMake files provided by + # aws-sdk-cpp to use the MacOSX SDK provided by XCode which makes + # XCode a hard dependency. Command Line Tools is often used instead + # of the full XCode suite, so let the linker to find it. + set_target_properties(AWS::aws-c-common + PROPERTIES INTERFACE_LINK_LIBRARIES + "-pthread;pthread;-framework CoreFoundation") + endif() endif() # Write out the package configurations. diff --git a/cpp/src/arrow/filesystem/s3fs.cc b/cpp/src/arrow/filesystem/s3fs.cc index 8053c223ade6..3fb1cabae301 100644 --- a/cpp/src/arrow/filesystem/s3fs.cc +++ b/cpp/src/arrow/filesystem/s3fs.cc @@ -341,6 +341,12 @@ class ObjectInputFile : public io::RandomAccessFile { RETURN_NOT_OK(CheckClosed()); RETURN_NOT_OK(CheckPosition(position, "read")); + nbytes = std::min(nbytes, content_length_ - position); + if (nbytes == 0) { + *bytes_read = 0; + return Status::OK(); + } + // Read the desired range of bytes S3Model::GetObjectResult result; RETURN_NOT_OK(GetObjectRange(client_, path_, position, nbytes, &result)); diff --git a/cpp/src/arrow/filesystem/s3fs.h b/cpp/src/arrow/filesystem/s3fs.h index 8dd3d87f8ff8..c4dacd84fc79 100644 --- a/cpp/src/arrow/filesystem/s3fs.h +++ b/cpp/src/arrow/filesystem/s3fs.h @@ -132,7 +132,7 @@ class ARROW_EXPORT S3FileSystem : public FileSystem { std::unique_ptr impl_; }; -enum class S3LogLevel { Off, Fatal, Error, Warn, Info, Debug, Trace }; +enum class S3LogLevel : int8_t { Off, Fatal, Error, Warn, Info, Debug, Trace }; struct ARROW_EXPORT S3GlobalOptions { S3LogLevel log_level; diff --git a/cpp/src/arrow/filesystem/s3fs_test.cc b/cpp/src/arrow/filesystem/s3fs_test.cc index 8564901ae400..c868e8a2be85 100644 --- a/cpp/src/arrow/filesystem/s3fs_test.cc +++ b/cpp/src/arrow/filesystem/s3fs_test.cc @@ -687,7 +687,7 @@ TEST_F(TestS3FS, OpenInputStream) { TEST_F(TestS3FS, OpenInputFile) { std::shared_ptr file; std::shared_ptr buf; - int64_t nbytes = -1, pos = -1; + int64_t nbytes = -1, pos = -1, bytes_read = 0; // Non-existent ASSERT_RAISES(IOError, fs_->OpenInputFile("non-existent-bucket/somefile", &file)); @@ -712,6 +712,15 @@ TEST_F(TestS3FS, OpenInputFile) { AssertBufferEqual(*buf, "data"); ASSERT_OK(file->ReadAt(9, 20, &buf)); AssertBufferEqual(*buf, ""); + + char result[10]; + ASSERT_OK(file->ReadAt(2, 5, &bytes_read, &result)); + ASSERT_EQ(bytes_read, 5); + ASSERT_OK(file->ReadAt(5, 20, &bytes_read, &result)); + ASSERT_EQ(bytes_read, 4); + ASSERT_OK(file->ReadAt(9, 0, &bytes_read, &result)); + ASSERT_EQ(bytes_read, 0); + // Reading past end of file ASSERT_RAISES(IOError, file->ReadAt(10, 20, &buf)); diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index b2282a6b69bc..6925efd2d17e 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -385,6 +385,10 @@ set(CYTHON_EXTENSIONS lib _fs _csv _json) set(LINK_LIBS arrow_shared arrow_python_shared) +if(PYARROW_BUILD_S3) + set(CYTHON_EXTENSIONS ${CYTHON_EXTENSIONS} _s3fs) +endif() + if(PYARROW_BUILD_CUDA) # Arrow CUDA find_package(ArrowCuda) diff --git a/python/pyarrow/_csv.pyx b/python/pyarrow/_csv.pyx index 25ff47d39b54..5dccad760f42 100644 --- a/python/pyarrow/_csv.pyx +++ b/python/pyarrow/_csv.pyx @@ -466,7 +466,7 @@ cdef class ConvertOptions: self.options.include_missing_columns = value -cdef _get_reader(input_file, shared_ptr[InputStream]* out): +cdef _get_reader(input_file, shared_ptr[CInputStream]* out): use_memory_map = False get_input_stream(input_file, use_memory_map, out) @@ -522,7 +522,7 @@ def read_csv(input_file, read_options=None, parse_options=None, Contents of the CSV file as a in-memory table. """ cdef: - shared_ptr[InputStream] stream + shared_ptr[CInputStream] stream CCSVReadOptions c_read_options CCSVParseOptions c_parse_options CCSVConvertOptions c_convert_options diff --git a/python/pyarrow/_cuda.pyx b/python/pyarrow/_cuda.pyx index a9f51b0a654f..e0cad68b2bfe 100644 --- a/python/pyarrow/_cuda.pyx +++ b/python/pyarrow/_cuda.pyx @@ -729,7 +729,7 @@ cdef class BufferReader(NativeFile): self.buffer = obj self.reader = new CCudaBufferReader(self.buffer.buffer) self.set_random_access_file( - shared_ptr[RandomAccessFile](self.reader)) + shared_ptr[CRandomAccessFile](self.reader)) self.is_readable = True def read_buffer(self, nbytes=None): @@ -776,7 +776,7 @@ cdef class BufferWriter(NativeFile): def __cinit__(self, CudaBuffer buffer): self.buffer = buffer self.writer = new CCudaBufferWriter(self.buffer.cuda_buffer) - self.set_output_stream(shared_ptr[OutputStream](self.writer)) + self.set_output_stream(shared_ptr[COutputStream](self.writer)) self.is_writable = True def writeat(self, int64_t position, object data): diff --git a/python/pyarrow/_fs.pxd b/python/pyarrow/_fs.pxd new file mode 100644 index 000000000000..11b5769f854e --- /dev/null +++ b/python/pyarrow/_fs.pxd @@ -0,0 +1,68 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# cython: language_level = 3 + +import six + +from pyarrow.compat import frombytes, tobytes +from pyarrow.includes.common cimport * +from pyarrow.includes.libarrow cimport PyDateTime_from_TimePoint +from pyarrow.lib import _detect_compression +from pyarrow.lib cimport * + + +cpdef enum FileType: + NonExistent = CFileType_NonExistent + Unknown = CFileType_Unknown + File = CFileType_File + Directory = CFileType_Directory + + +cdef class FileStats: + cdef: + CFileStats stats + + @staticmethod + cdef FileStats wrap(CFileStats stats) + + +cdef class Selector: + cdef: + CSelector selector + + +cdef class FileSystem: + cdef: + shared_ptr[CFileSystem] wrapped + CFileSystem* fs + + cdef init(self, const shared_ptr[CFileSystem]& wrapped) + + +cdef class LocalFileSystem(FileSystem): + cdef: + CLocalFileSystem* localfs + + cdef init(self, const shared_ptr[CFileSystem]& wrapped) + + +cdef class SubTreeFileSystem(FileSystem): + cdef: + CSubTreeFileSystem* subtreefs + + cdef init(self, const shared_ptr[CFileSystem]& wrapped) diff --git a/python/pyarrow/_fs.pyx b/python/pyarrow/_fs.pyx index 769ca8d7391f..39079aee1051 100644 --- a/python/pyarrow/_fs.pyx +++ b/python/pyarrow/_fs.pyx @@ -22,17 +22,8 @@ import six from pyarrow.compat import frombytes, tobytes from pyarrow.includes.common cimport * from pyarrow.includes.libarrow cimport PyDateTime_from_TimePoint -from pyarrow.includes.libarrow_fs cimport * -from pyarrow.util import _stringify_path from pyarrow.lib import _detect_compression -from pyarrow.lib cimport ( - check_status, - NativeFile, - BufferedOutputStream, - BufferedInputStream, - CompressedInputStream, - CompressedOutputStream -) +from pyarrow.lib cimport * cdef inline c_string _path_as_bytes(path) except *: @@ -46,20 +37,12 @@ cdef inline c_string _path_as_bytes(path) except *: return tobytes(path) -cpdef enum FileType: - NonExistent = CFileType_NonExistent - Unknown = CFileType_Unknown - File = CFileType_File - Directory = CFileType_Directory - - cdef class FileStats: """FileSystem entry stats""" - cdef CFileStats stats - def __init__(self): - raise TypeError('dont initialize me') + raise TypeError("FileStats cannot be instantiated directly, use " + "FileSystem.get_target_stats method instead.") @staticmethod cdef FileStats wrap(CFileStats stats): @@ -115,9 +98,7 @@ cdef class FileStats: Only regular files are guaranteed to have a size. """ if self.stats.type() != CFileType_File: - raise ValueError( - 'Only regular files are guaranteed to have a size' - ) + return None return self.stats.size() @property @@ -146,7 +127,7 @@ cdef class Selector: Parameters ---------- - base_dir : str or pathlib.Path + base_dir : str The directory in which to select files. Relative paths also work, use '.' for the current directory and '..' for the parent. allow_non_existent : bool, default False @@ -156,7 +137,6 @@ cdef class Selector: recursive : bool, default False Whether to recurse into subdirectories. """ - cdef CSelector selector def __init__(self, base_dir, bint allow_non_existent=False, bint recursive=False): @@ -192,10 +172,6 @@ cdef class Selector: cdef class FileSystem: """Abstract file system API""" - cdef: - shared_ptr[CFileSystem] wrapped - CFileSystem* fs - def __init__(self): raise TypeError("FileSystem is an abstract class, instantiate one of " "the subclasses instead: LocalFileSystem or " @@ -249,7 +225,7 @@ cdef class FileSystem: Parameters ---------- - path : str or pathlib.Path + path : str The path of the new directory. recursive: bool, default True Create nested directories as well. @@ -263,7 +239,7 @@ cdef class FileSystem: Parameters ---------- - path : str or pathlib.Path + path : str The path of the directory to be deleted. """ cdef c_string directory = _path_as_bytes(path) @@ -280,9 +256,9 @@ cdef class FileSystem: Parameters ---------- - src : str or pathlib.Path + src : str The path of the file or the directory to be moved. - dest : str or pathlib.Path + dest : str The destination path where the file or directory is moved to. """ cdef: @@ -299,9 +275,9 @@ cdef class FileSystem: Parameters ---------- - src : str or pathlib.Path + src : str The path of the file to be copied from. - dest : str or pathlib.Path + dest : str The destination path where the file is copied to. """ cdef: @@ -315,7 +291,7 @@ cdef class FileSystem: Parameters ---------- - path : str or pathlib.Path + path : str The path of the file to be deleted. """ cdef c_string file = _path_as_bytes(path) @@ -345,7 +321,7 @@ cdef class FileSystem: Parameters ---------- - path : Union[str, pathlib.Path] + path : str The source to open for reading. Returns @@ -369,7 +345,7 @@ cdef class FileSystem: Parameters ---------- - source: str or pathlib.Path + source: str The source to open for reading. compression: str optional, default 'detect' The compression algorithm to use for on-the-fly decompression. @@ -407,7 +383,7 @@ cdef class FileSystem: Parameters ---------- - path : str or pathlib.Path + path : str The source to open for writing. compression: str optional, default 'detect' The compression algorithm to use for on-the-fly compression. @@ -445,7 +421,7 @@ cdef class FileSystem: Parameters ---------- - path : str or pathlib.Path + path : str The source to open for writing. compression: str optional, default 'detect' The compression algorithm to use for on-the-fly compression. @@ -484,9 +460,6 @@ cdef class LocalFileSystem(FileSystem): except when deleting an entry). """ - cdef: - CLocalFileSystem* localfs - def __init__(self): cdef shared_ptr[CLocalFileSystem] wrapped wrapped = make_shared[CLocalFileSystem]() @@ -506,10 +479,14 @@ cdef class SubTreeFileSystem(FileSystem): Note, that this makes no security guarantee. For example, symlinks may allow to "escape" the subtree and access other parts of the underlying filesystem. - """ - cdef: - CSubTreeFileSystem* subtreefs + Parameters + ---------- + base_path: str + The root of the subtree. + base_fs: FileSystem + FileSystem object the operations delegated to. + """ def __init__(self, base_path, FileSystem base_fs): cdef: diff --git a/python/pyarrow/_json.pyx b/python/pyarrow/_json.pyx index ffbf01c09e7c..da3588a5e625 100644 --- a/python/pyarrow/_json.pyx +++ b/python/pyarrow/_json.pyx @@ -135,7 +135,7 @@ cdef class ParseOptions: self.options.newlines_in_values = value -cdef _get_reader(input_file, shared_ptr[InputStream]* out): +cdef _get_reader(input_file, shared_ptr[CInputStream]* out): use_memory_map = False get_input_stream(input_file, use_memory_map, out) @@ -175,7 +175,7 @@ def read_json(input_file, read_options=None, parse_options=None, Contents of the JSON file as a in-memory table. """ cdef: - shared_ptr[InputStream] stream + shared_ptr[CInputStream] stream CJSONReadOptions c_read_options CJSONParseOptions c_parse_options shared_ptr[CJSONReader] reader diff --git a/python/pyarrow/_orc.pxd b/python/pyarrow/_orc.pxd index ebbf8beda828..649fe8248f41 100644 --- a/python/pyarrow/_orc.pxd +++ b/python/pyarrow/_orc.pxd @@ -28,7 +28,7 @@ from pyarrow.includes.libarrow cimport (CArray, CSchema, CStatus, CKeyValueMetadata, CRecordBatch, CTable, - RandomAccessFile, OutputStream, + CRandomAccessFile, COutputStream, TimeUnit) @@ -37,7 +37,7 @@ cdef extern from "arrow/adapters/orc/adapter.h" \ cdef cppclass ORCFileReader: @staticmethod - CStatus Open(const shared_ptr[RandomAccessFile]& file, + CStatus Open(const shared_ptr[CRandomAccessFile]& file, CMemoryPool* pool, unique_ptr[ORCFileReader]* reader) diff --git a/python/pyarrow/_orc.pyx b/python/pyarrow/_orc.pyx index c9f5b2e158d6..0ee3ca632e46 100644 --- a/python/pyarrow/_orc.pyx +++ b/python/pyarrow/_orc.pyx @@ -46,7 +46,7 @@ cdef class ORCReader: def open(self, object source, c_bool use_memory_map=True): cdef: - shared_ptr[RandomAccessFile] rd_handle + shared_ptr[CRandomAccessFile] rd_handle self.source = source diff --git a/python/pyarrow/_parquet.pxd b/python/pyarrow/_parquet.pxd index 19fb214c8f04..5cf3ff5931b4 100644 --- a/python/pyarrow/_parquet.pxd +++ b/python/pyarrow/_parquet.pxd @@ -24,7 +24,7 @@ from pyarrow.includes.common cimport * from pyarrow.includes.libarrow cimport (CChunkedArray, CSchema, CStatus, CTable, CMemoryPool, CBuffer, CKeyValueMetadata, - RandomAccessFile, OutputStream, + CRandomAccessFile, COutputStream, TimeUnit) @@ -316,7 +316,7 @@ cdef extern from "parquet/api/reader.h" namespace "parquet" nogil: unique_ptr[CRowGroupMetaData] RowGroup(int i) const SchemaDescriptor* schema() shared_ptr[const CKeyValueMetadata] key_value_metadata() const - void WriteTo(OutputStream* dst) const + void WriteTo(COutputStream* dst) const cdef shared_ptr[CFileMetaData] CFileMetaData_Make \ " parquet::FileMetaData::Make"(const void* serialized_metadata, @@ -406,7 +406,7 @@ cdef extern from "parquet/arrow/reader.h" namespace "parquet::arrow" nogil: cdef cppclass FileReaderBuilder: FileReaderBuilder() - CStatus Open(const shared_ptr[RandomAccessFile]& file, + CStatus Open(const shared_ptr[CRandomAccessFile]& file, const CReaderProperties& properties, const shared_ptr[CFileMetaData]& metadata) @@ -435,7 +435,7 @@ cdef extern from "parquet/arrow/writer.h" namespace "parquet::arrow" nogil: @staticmethod CStatus Open(const CSchema& schema, CMemoryPool* pool, - const shared_ptr[OutputStream]& sink, + const shared_ptr[COutputStream]& sink, const shared_ptr[WriterProperties]& properties, const shared_ptr[ArrowWriterProperties]& arrow_properties, unique_ptr[FileWriter]* writer) @@ -448,4 +448,4 @@ cdef extern from "parquet/arrow/writer.h" namespace "parquet::arrow" nogil: CStatus WriteMetaDataFile( const CFileMetaData& file_metadata, - const OutputStream* sink) + const COutputStream* sink) diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx index 3d62c0eb13e6..f93def8a67ee 100644 --- a/python/pyarrow/_parquet.pyx +++ b/python/pyarrow/_parquet.pyx @@ -568,7 +568,7 @@ cdef class FileMetaData: def __reduce__(self): cdef: NativeFile sink = BufferOutputStream() - OutputStream* c_sink = sink.get_output_stream().get() + COutputStream* c_sink = sink.get_output_stream().get() with nogil: self._metadata.WriteTo(c_sink) @@ -694,7 +694,7 @@ cdef class FileMetaData: Write the metadata object to a metadata-only file """ cdef: - shared_ptr[OutputStream] sink + shared_ptr[COutputStream] sink c_string c_where try: @@ -1010,7 +1010,7 @@ cdef class ParquetReader: read_dictionary=None, FileMetaData metadata=None, int buffer_size=0): cdef: - shared_ptr[RandomAccessFile] rd_handle + shared_ptr[CRandomAccessFile] rd_handle shared_ptr[CFileMetaData] c_metadata CReaderProperties properties = default_reader_properties() ArrowReaderProperties arrow_props = ( @@ -1202,7 +1202,7 @@ cdef class ParquetReader: cdef class ParquetWriter: cdef: unique_ptr[FileWriter] writer - shared_ptr[OutputStream] sink + shared_ptr[COutputStream] sink bint own_sink cdef readonly: diff --git a/python/pyarrow/_s3fs.pyx b/python/pyarrow/_s3fs.pyx new file mode 100644 index 000000000000..d1f820e51958 --- /dev/null +++ b/python/pyarrow/_s3fs.pyx @@ -0,0 +1,173 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# cython: language_level = 3 + +import six + +from pyarrow.lib cimport check_status +from pyarrow.compat import frombytes, tobytes +from pyarrow.includes.common cimport * +from pyarrow.includes.libarrow cimport * +from pyarrow.includes.libarrow_s3fs cimport * +from pyarrow._fs cimport FileSystem + + +cpdef enum S3LogLevel: + Off = CS3LogLevel_Off + Fatal = CS3LogLevel_Fatal + Error = CS3LogLevel_Error + Warn = CS3LogLevel_Warn + Info = CS3LogLevel_Info + Debug = CS3LogLevel_Debug + Trace = CS3LogLevel_Trace + + +def initialize_s3(S3LogLevel log_level=S3LogLevel.Error): + cdef CS3GlobalOptions options + options.log_level = log_level + check_status(CInitializeS3(options)) + + +def finalize_s3(): + check_status(CFinalizeS3()) + + +cdef class S3Options: + """Options for S3FileSystem. + + If neither access_key nor secret_key are provided then attempts to + initialize from AWS environment variables, otherwise both access_key and + secret_key must be provided. + + Parameters + ---------- + access_key: str, default None + AWS Access Key ID. Pass None to use the standard AWS environment + variables and/or configuration file. + secret_key: str, default None + AWS Secret Access key. Pass None to use the standard AWS environment + variables and/or configuration file. + region: str, default 'us-east-1' + AWS region to connect to. + scheme: str, default 'https' + S3 connection transport scheme. + endpoint_override: str, default None + Override region with a connect string such as "localhost:9000" + background_writes: boolean, default True + Whether OutputStream writes will be issued in the background, without + blocking. + """ + cdef: + CS3Options options + + # Avoid mistakingly creating attributes + __slots__ = () + + def __init__(self, access_key=None, secret_key=None, region=None, + scheme=None, endpoint_override=None, background_writes=None): + if access_key is not None and secret_key is None: + raise ValueError( + 'In order to initialize with explicit credentials both ' + 'access_key and secret_key must be provided, ' + '`secret_key` is not set.' + ) + elif access_key is None and secret_key is not None: + raise ValueError( + 'In order to initialize with explicit credentials both ' + 'access_key and secret_key must be provided, ' + '`access_key` is not set.' + ) + elif access_key is not None or secret_key is not None: + self.options = CS3Options.FromAccessKey( + tobytes(access_key), + tobytes(secret_key) + ) + else: + self.options = CS3Options.Defaults() + + if region is not None: + self.region = region + if scheme is not None: + self.scheme = scheme + if endpoint_override is not None: + self.endpoint_override = endpoint_override + if background_writes is not None: + self.background_writes = background_writes + + @property + def region(self): + """AWS region to connect to.""" + return frombytes(self.options.region) + + @region.setter + def region(self, value): + self.options.region = tobytes(value) + + @property + def scheme(self): + """S3 connection transport scheme.""" + return frombytes(self.options.scheme) + + @scheme.setter + def scheme(self, value): + self.options.scheme = tobytes(value) + + @property + def endpoint_override(self): + """Override region with a connect string such as localhost:9000""" + return frombytes(self.options.endpoint_override) + + @endpoint_override.setter + def endpoint_override(self, value): + self.options.endpoint_override = tobytes(value) + + @property + def background_writes(self): + """OutputStream writes will be issued in the background""" + return self.options.background_writes + + @background_writes.setter + def background_writes(self, bint value): + self.options.background_writes = value + + +cdef class S3FileSystem(FileSystem): + """S3-backed FileSystem implementation + + Note: S3 buckets are special and the operations available on them may be + limited or more expensive than desired. + + Parameters + ---------- + options: S3Options, default None + Options for connecting to S3. If None is passed then attempts to + initialize the connection from AWS environment variables. + """ + + cdef: + CS3FileSystem* s3fs + + def __init__(self, S3Options options=None): + cdef shared_ptr[CS3FileSystem] wrapped + options = options or S3Options() + check_status(CS3FileSystem.Make(options.options, &wrapped)) + self.init( wrapped) + + cdef init(self, const shared_ptr[CFileSystem]& wrapped): + FileSystem.init(self, wrapped) + self.s3fs = wrapped.get() diff --git a/python/pyarrow/feather.pxi b/python/pyarrow/feather.pxi index 6fd13bc04b46..8700f67ae621 100644 --- a/python/pyarrow/feather.pxi +++ b/python/pyarrow/feather.pxi @@ -34,7 +34,7 @@ cdef class FeatherWriter: self.num_rows = -1 def open(self, object dest): - cdef shared_ptr[OutputStream] sink + cdef shared_ptr[COutputStream] sink get_writer(dest, &sink) with nogil: @@ -76,7 +76,7 @@ cdef class FeatherReader: pass def open(self, source, c_bool use_memory_map=True): - cdef shared_ptr[RandomAccessFile] reader + cdef shared_ptr[CRandomAccessFile] reader get_reader(source, use_memory_map, &reader) with nogil: diff --git a/python/pyarrow/fs.py b/python/pyarrow/fs.py index cd5263acbcad..5f257d07f300 100644 --- a/python/pyarrow/fs.py +++ b/python/pyarrow/fs.py @@ -17,4 +17,11 @@ from __future__ import absolute_import -from pyarrow._fs import * # noqa +from pyarrow._fs import ( # noqa + Selector, + FileType, + FileStats, + FileSystem, + LocalFileSystem, + SubTreeFileSystem +) diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 69dafa4e46f2..82085487eda3 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -765,13 +765,16 @@ cdef extern from "arrow/io/api.h" namespace "arrow::io" nogil: CStatus Write(const uint8_t* data, int64_t nbytes) CStatus Flush() - cdef cppclass OutputStream(FileInterface, Writable): + cdef cppclass COutputStream" arrow::io::OutputStream"(FileInterface, + Writable): pass - cdef cppclass InputStream(FileInterface, Readable): + cdef cppclass CInputStream" arrow::io::InputStream"(FileInterface, + Readable): pass - cdef cppclass RandomAccessFile(InputStream, Seekable): + cdef cppclass CRandomAccessFile" arrow::io::RandomAccessFile"(CInputStream, + Seekable): CStatus GetSize(int64_t* size) CStatus ReadAt(int64_t position, int64_t nbytes, @@ -780,24 +783,24 @@ cdef extern from "arrow/io/api.h" namespace "arrow::io" nogil: shared_ptr[CBuffer]* out) c_bool supports_zero_copy() - cdef cppclass WritableFile(OutputStream, Seekable): + cdef cppclass WritableFile(COutputStream, Seekable): CStatus WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) - cdef cppclass ReadWriteFileInterface(RandomAccessFile, + cdef cppclass ReadWriteFileInterface(CRandomAccessFile, WritableFile): pass - cdef cppclass FileSystem: + cdef cppclass CIOFileSystem" arrow::io::FileSystem": CStatus Stat(const c_string& path, FileStatistics* stat) - cdef cppclass FileOutputStream(OutputStream): + cdef cppclass FileOutputStream(COutputStream): @staticmethod - CStatus Open(const c_string& path, shared_ptr[OutputStream]* file) + CStatus Open(const c_string& path, shared_ptr[COutputStream]* file) int file_descriptor() - cdef cppclass ReadableFile(RandomAccessFile): + cdef cppclass ReadableFile(CRandomAccessFile): @staticmethod CStatus Open(const c_string& path, shared_ptr[ReadableFile]* file) @@ -823,46 +826,46 @@ cdef extern from "arrow/io/api.h" namespace "arrow::io" nogil: int file_descriptor() cdef cppclass CCompressedInputStream \ - " arrow::io::CompressedInputStream"(InputStream): + " arrow::io::CompressedInputStream"(CInputStream): @staticmethod CStatus Make(CMemoryPool* pool, CCodec* codec, - shared_ptr[InputStream] raw, + shared_ptr[CInputStream] raw, shared_ptr[CCompressedInputStream]* out) @staticmethod - CStatus Make(CCodec* codec, shared_ptr[InputStream] raw, + CStatus Make(CCodec* codec, shared_ptr[CInputStream] raw, shared_ptr[CCompressedInputStream]* out) cdef cppclass CCompressedOutputStream \ - " arrow::io::CompressedOutputStream"(OutputStream): + " arrow::io::CompressedOutputStream"(COutputStream): @staticmethod CStatus Make(CMemoryPool* pool, CCodec* codec, - shared_ptr[OutputStream] raw, + shared_ptr[COutputStream] raw, shared_ptr[CCompressedOutputStream]* out) @staticmethod - CStatus Make(CCodec* codec, shared_ptr[OutputStream] raw, + CStatus Make(CCodec* codec, shared_ptr[COutputStream] raw, shared_ptr[CCompressedOutputStream]* out) cdef cppclass CBufferedInputStream \ - " arrow::io::BufferedInputStream"(InputStream): + " arrow::io::BufferedInputStream"(CInputStream): @staticmethod CStatus Create(int64_t buffer_size, CMemoryPool* pool, - shared_ptr[InputStream] raw, + shared_ptr[CInputStream] raw, shared_ptr[CBufferedInputStream]* out) - shared_ptr[InputStream] Detach() + shared_ptr[CInputStream] Detach() cdef cppclass CBufferedOutputStream \ - " arrow::io::BufferedOutputStream"(OutputStream): + " arrow::io::BufferedOutputStream"(COutputStream): @staticmethod CStatus Create(int64_t buffer_size, CMemoryPool* pool, - shared_ptr[OutputStream] raw, + shared_ptr[COutputStream] raw, shared_ptr[CBufferedOutputStream]* out) - CStatus Detach(shared_ptr[OutputStream]* raw) + CStatus Detach(shared_ptr[COutputStream]* raw) # ---------------------------------------------------------------------- # HDFS @@ -894,13 +897,14 @@ cdef extern from "arrow/io/api.h" namespace "arrow::io" nogil: int64_t block_size int16_t permissions - cdef cppclass HdfsReadableFile(RandomAccessFile): + cdef cppclass HdfsReadableFile(CRandomAccessFile): pass - cdef cppclass HdfsOutputStream(OutputStream): + cdef cppclass HdfsOutputStream(COutputStream): pass - cdef cppclass CHadoopFileSystem" arrow::io::HadoopFileSystem"(FileSystem): + cdef cppclass CHadoopFileSystem \ + "arrow::io::HadoopFileSystem"(CIOFileSystem): @staticmethod CStatus Connect(const HdfsConnectionConfig* config, shared_ptr[CHadoopFileSystem]* client) @@ -936,16 +940,16 @@ cdef extern from "arrow/io/api.h" namespace "arrow::io" nogil: shared_ptr[HdfsOutputStream]* handle) cdef cppclass CBufferReader \ - " arrow::io::BufferReader"(RandomAccessFile): + " arrow::io::BufferReader"(CRandomAccessFile): CBufferReader(const shared_ptr[CBuffer]& buffer) CBufferReader(const uint8_t* data, int64_t nbytes) cdef cppclass CBufferOutputStream \ - " arrow::io::BufferOutputStream"(OutputStream): + " arrow::io::BufferOutputStream"(COutputStream): CBufferOutputStream(const shared_ptr[CResizableBuffer]& buffer) cdef cppclass CMockOutputStream \ - " arrow::io::MockOutputStream"(OutputStream): + " arrow::io::MockOutputStream"(COutputStream): CMockOutputStream() int64_t GetExtentBytesWritten() @@ -958,6 +962,71 @@ cdef extern from "arrow/io/api.h" namespace "arrow::io" nogil: void set_memcopy_threshold(int64_t threshold) +cdef extern from "arrow/filesystem/api.h" namespace "arrow::fs" nogil: + + ctypedef enum CFileType "arrow::fs::FileType": + CFileType_NonExistent "arrow::fs::FileType::NonExistent" + CFileType_Unknown "arrow::fs::FileType::Unknown" + CFileType_File "arrow::fs::FileType::File" + CFileType_Directory "arrow::fs::FileType::Directory" + + cdef cppclass CTimePoint "arrow::fs::TimePoint": + pass + + cdef cppclass CFileStats "arrow::fs::FileStats": + CFileStats() + CFileStats(CFileStats&&) + CFileStats& operator=(CFileStats&&) + CFileStats(const CFileStats&) + CFileStats& operator=(const CFileStats&) + + CFileType type() + void set_type(CFileType type) + c_string path() + void set_path(const c_string& path) + c_string base_name() + int64_t size() + void set_size(int64_t size) + c_string extension() + CTimePoint mtime() + void set_mtime(CTimePoint mtime) + + cdef cppclass CSelector "arrow::fs::Selector": + CSelector() + c_string base_dir + c_bool allow_non_existent + c_bool recursive + + cdef cppclass CFileSystem "arrow::fs::FileSystem": + CStatus GetTargetStats(const c_string& path, CFileStats* out) + CStatus GetTargetStats(const vector[c_string]& paths, + vector[CFileStats]* out) + CStatus GetTargetStats(const CSelector& select, + vector[CFileStats]* out) + CStatus CreateDir(const c_string& path, c_bool recursive) + CStatus DeleteDir(const c_string& path) + CStatus DeleteFile(const c_string& path) + CStatus DeleteFiles(const vector[c_string]& paths) + CStatus Move(const c_string& src, const c_string& dest) + CStatus CopyFile(const c_string& src, const c_string& dest) + CStatus OpenInputStream(const c_string& path, + shared_ptr[CInputStream]* out) + CStatus OpenInputFile(const c_string& path, + shared_ptr[CRandomAccessFile]* out) + CStatus OpenOutputStream(const c_string& path, + shared_ptr[COutputStream]* out) + CStatus OpenAppendStream(const c_string& path, + shared_ptr[COutputStream]* out) + + cdef cppclass CLocalFileSystem "arrow::fs::LocalFileSystem"(CFileSystem): + LocalFileSystem() + + cdef cppclass CSubTreeFileSystem \ + "arrow::fs::SubTreeFileSystem"(CFileSystem): + CSubTreeFileSystem(const c_string& base_path, + shared_ptr[CFileSystem] base_fs) + + cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil: enum MessageType" arrow::ipc::Message::Type": MessageType_SCHEMA" arrow::ipc::Message::SCHEMA" @@ -1001,14 +1070,14 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil: MetadataVersion metadata_version() MessageType type() - CStatus SerializeTo(OutputStream* stream, const CIpcOptions& options, + CStatus SerializeTo(COutputStream* stream, const CIpcOptions& options, int64_t* output_length) c_string FormatMessageType(MessageType type) cdef cppclass CMessageReader" arrow::ipc::MessageReader": @staticmethod - unique_ptr[CMessageReader] Open(const shared_ptr[InputStream]& stream) + unique_ptr[CMessageReader] Open(const shared_ptr[CInputStream]& stream) CStatus ReadNextMessage(unique_ptr[CMessage]* out) @@ -1020,7 +1089,7 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil: cdef cppclass CRecordBatchStreamReader \ " arrow::ipc::RecordBatchStreamReader"(CRecordBatchReader): @staticmethod - CStatus Open(const InputStream* stream, + CStatus Open(const CInputStream* stream, shared_ptr[CRecordBatchReader]* out) @staticmethod @@ -1031,24 +1100,24 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil: " arrow::ipc::RecordBatchStreamWriter"(CRecordBatchWriter): @staticmethod CResult[shared_ptr[CRecordBatchWriter]] Open( - OutputStream* sink, const shared_ptr[CSchema]& schema, + COutputStream* sink, const shared_ptr[CSchema]& schema, CIpcOptions& options) cdef cppclass CRecordBatchFileWriter \ " arrow::ipc::RecordBatchFileWriter"(CRecordBatchWriter): @staticmethod CResult[shared_ptr[CRecordBatchWriter]] Open( - OutputStream* sink, const shared_ptr[CSchema]& schema, + COutputStream* sink, const shared_ptr[CSchema]& schema, CIpcOptions& options) cdef cppclass CRecordBatchFileReader \ " arrow::ipc::RecordBatchFileReader": @staticmethod - CStatus Open(RandomAccessFile* file, + CStatus Open(CRandomAccessFile* file, shared_ptr[CRecordBatchFileReader]* out) @staticmethod - CStatus Open2" Open"(RandomAccessFile* file, + CStatus Open2" Open"(CRandomAccessFile* file, int64_t footer_offset, shared_ptr[CRecordBatchFileReader]* out) @@ -1058,16 +1127,16 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil: CStatus ReadRecordBatch(int i, shared_ptr[CRecordBatch]* batch) - CStatus ReadMessage(InputStream* stream, unique_ptr[CMessage]* message) + CStatus ReadMessage(CInputStream* stream, unique_ptr[CMessage]* message) CStatus GetRecordBatchSize(const CRecordBatch& batch, int64_t* size) CStatus GetTensorSize(const CTensor& tensor, int64_t* size) - CStatus WriteTensor(const CTensor& tensor, OutputStream* dst, + CStatus WriteTensor(const CTensor& tensor, COutputStream* dst, int32_t* metadata_length, int64_t* body_length) - CStatus ReadTensor(InputStream* stream, shared_ptr[CTensor]* out) + CStatus ReadTensor(CInputStream* stream, shared_ptr[CTensor]* out) CStatus ReadRecordBatch(const CMessage& message, const shared_ptr[CSchema]& schema, @@ -1082,16 +1151,16 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil: CMemoryPool* pool, shared_ptr[CBuffer]* out) - CStatus ReadSchema(InputStream* stream, CDictionaryMemo* dictionary_memo, + CStatus ReadSchema(CInputStream* stream, CDictionaryMemo* dictionary_memo, shared_ptr[CSchema]* out) CStatus ReadRecordBatch(const shared_ptr[CSchema]& schema, CDictionaryMemo* dictionary_memo, - InputStream* stream, + CInputStream* stream, shared_ptr[CRecordBatch]* out) - CStatus AlignStream(InputStream* stream, int64_t alignment) - CStatus AlignStream(OutputStream* stream, int64_t alignment) + CStatus AlignStream(CInputStream* stream, int64_t alignment) + CStatus AlignStream(COutputStream* stream, int64_t alignment) cdef CStatus GetRecordBatchPayload\ " arrow::ipc::internal::GetRecordBatchPayload"( @@ -1102,7 +1171,7 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil: cdef cppclass CFeatherWriter" arrow::ipc::feather::TableWriter": @staticmethod - CStatus Open(const shared_ptr[OutputStream]& stream, + CStatus Open(const shared_ptr[COutputStream]& stream, unique_ptr[CFeatherWriter]* out) void SetDescription(const c_string& desc) @@ -1113,7 +1182,7 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil: cdef cppclass CFeatherReader" arrow::ipc::feather::TableReader": @staticmethod - CStatus Open(const shared_ptr[RandomAccessFile]& file, + CStatus Open(const shared_ptr[CRandomAccessFile]& file, unique_ptr[CFeatherReader]* out) c_string GetDescription() @@ -1172,7 +1241,7 @@ cdef extern from "arrow/csv/api.h" namespace "arrow::csv" nogil: cdef cppclass CCSVReader" arrow::csv::TableReader": @staticmethod - CStatus Make(CMemoryPool*, shared_ptr[InputStream], + CStatus Make(CMemoryPool*, shared_ptr[CInputStream], CCSVReadOptions, CCSVParseOptions, CCSVConvertOptions, shared_ptr[CCSVReader]* out) @@ -1200,7 +1269,7 @@ cdef extern from "arrow/json/reader.h" namespace "arrow::json" nogil: cdef cppclass CJSONReader" arrow::json::TableReader": @staticmethod - CStatus Make(CMemoryPool*, shared_ptr[InputStream], + CStatus Make(CMemoryPool*, shared_ptr[CInputStream], CJSONReadOptions, CJSONParseOptions, shared_ptr[CJSONReader]* out) @@ -1379,10 +1448,10 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil: CStatus Make(const uint8_t* data, int64_t size, object base, shared_ptr[CBuffer]* out) - cdef cppclass PyReadableFile(RandomAccessFile): + cdef cppclass PyReadableFile(CRandomAccessFile): PyReadableFile(object fo) - cdef cppclass PyOutputStream(OutputStream): + cdef cppclass PyOutputStream(COutputStream): PyOutputStream(object fo) cdef cppclass PandasOptions: @@ -1398,7 +1467,7 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil: shared_ptr[CRecordBatch] batch vector[shared_ptr[CTensor]] tensors - CStatus WriteTo(OutputStream* dst) + CStatus WriteTo(COutputStream* dst) CStatus GetComponents(CMemoryPool* pool, PyObject** dst) CStatus SerializeObject(object context, object sequence, @@ -1408,7 +1477,7 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil: const CSerializedPyObject& obj, PyObject* base, PyObject** out) - CStatus ReadSerializedObject(RandomAccessFile* src, + CStatus ReadSerializedObject(CRandomAccessFile* src, CSerializedPyObject* out) CStatus GetSerializedFromComponents(int num_tensors, int num_ndarrays, diff --git a/python/pyarrow/includes/libarrow_fs.pxd b/python/pyarrow/includes/libarrow_fs.pxd deleted file mode 100644 index f54a2e50357e..000000000000 --- a/python/pyarrow/includes/libarrow_fs.pxd +++ /dev/null @@ -1,92 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# distutils: language = c++ - -from libcpp.functional cimport function - -from pyarrow.includes.common cimport * -from pyarrow.includes.libarrow cimport ( - InputStream as CInputStream, - OutputStream as COutputStream, - RandomAccessFile as CRandomAccessFile -) - - -cdef extern from "arrow/filesystem/api.h" namespace "arrow::fs" nogil: - - enum CFileType "arrow::fs::FileType": - CFileType_NonExistent "arrow::fs::FileType::NonExistent" - CFileType_Unknown "arrow::fs::FileType::Unknown" - CFileType_File "arrow::fs::FileType::File" - CFileType_Directory "arrow::fs::FileType::Directory" - - cdef cppclass CTimePoint "arrow::fs::TimePoint": - pass - - cdef cppclass CFileStats "arrow::fs::FileStats": - CFileStats() - CFileStats(CFileStats&&) - CFileStats& operator=(CFileStats&&) - CFileStats(const CFileStats&) - CFileStats& operator=(const CFileStats&) - - CFileType type() - void set_type(CFileType type) - c_string path() - void set_path(const c_string& path) - c_string base_name() - int64_t size() - void set_size(int64_t size) - c_string extension() - CTimePoint mtime() - void set_mtime(CTimePoint mtime) - - cdef cppclass CSelector "arrow::fs::Selector": - CSelector() - c_string base_dir - c_bool allow_non_existent - c_bool recursive - - cdef cppclass CFileSystem "arrow::fs::FileSystem": - CStatus GetTargetStats(const c_string& path, CFileStats* out) - CStatus GetTargetStats(const vector[c_string]& paths, - vector[CFileStats]* out) - CStatus GetTargetStats(const CSelector& select, - vector[CFileStats]* out) - CStatus CreateDir(const c_string& path, c_bool recursive) - CStatus DeleteDir(const c_string& path) - CStatus DeleteFile(const c_string& path) - CStatus DeleteFiles(const vector[c_string]& paths) - CStatus Move(const c_string& src, const c_string& dest) - CStatus CopyFile(const c_string& src, const c_string& dest) - CStatus OpenInputStream(const c_string& path, - shared_ptr[CInputStream]* out) - CStatus OpenInputFile(const c_string& path, - shared_ptr[CRandomAccessFile]* out) - CStatus OpenOutputStream(const c_string& path, - shared_ptr[COutputStream]* out) - CStatus OpenAppendStream(const c_string& path, - shared_ptr[COutputStream]* out) - - cdef cppclass CLocalFileSystem "arrow::fs::LocalFileSystem"(CFileSystem): - LocalFileSystem() - - cdef cppclass CSubTreeFileSystem \ - "arrow::fs::SubTreeFileSystem"(CFileSystem): - CSubTreeFileSystem(const c_string& base_path, - shared_ptr[CFileSystem] base_fs) diff --git a/python/pyarrow/includes/libarrow_s3fs.pxd b/python/pyarrow/includes/libarrow_s3fs.pxd new file mode 100644 index 000000000000..8dc109c5e6e3 --- /dev/null +++ b/python/pyarrow/includes/libarrow_s3fs.pxd @@ -0,0 +1,60 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# distutils: language = c++ + +from libcpp.functional cimport function + +from pyarrow.includes.common cimport * +from pyarrow.includes.libarrow cimport CFileSystem + +cdef extern from "arrow/filesystem/api.h" namespace "arrow::fs" nogil: + + ctypedef enum CS3LogLevel "arrow::fs::S3LogLevel": + CS3LogLevel_Off "arrow::fs::S3LogLevel::Off" + CS3LogLevel_Fatal "arrow::fs::S3LogLevel::Fatal" + CS3LogLevel_Error "arrow::fs::S3LogLevel::Error" + CS3LogLevel_Warn "arrow::fs::S3LogLevel::Warn" + CS3LogLevel_Info "arrow::fs::S3LogLevel::Info" + CS3LogLevel_Debug "arrow::fs::S3LogLevel::Debug" + CS3LogLevel_Trace "arrow::fs::S3LogLevel::Trace" + + cdef struct CS3GlobalOptions "arrow::fs::S3GlobalOptions": + CS3LogLevel log_level + + cdef cppclass CS3Options "arrow::fs::S3Options": + c_string region + c_string endpoint_override + c_string scheme + c_bool background_writes + void ConfigureDefaultCredentials() + void ConfigureAccessKey(const c_string& access_key, + const c_string& secret_key) + + @staticmethod + CS3Options Defaults() + @staticmethod + CS3Options FromAccessKey(const c_string& access_key, + const c_string& secret_key) + + cdef cppclass CS3FileSystem "arrow::fs::S3FileSystem"(CFileSystem): + @staticmethod + CStatus Make(const CS3Options& options, shared_ptr[CS3FileSystem]* out) + + cdef CStatus CInitializeS3 "arrow::fs::InitializeS3"( + const CS3GlobalOptions& options) + cdef CStatus CFinalizeS3 "arrow::fs::FinalizeS3"() diff --git a/python/pyarrow/io-hdfs.pxi b/python/pyarrow/io-hdfs.pxi index e9655e4a7f74..b224abbead82 100644 --- a/python/pyarrow/io-hdfs.pxi +++ b/python/pyarrow/io-hdfs.pxi @@ -424,7 +424,7 @@ cdef class HadoopFileSystem: c_replication, c_default_block_size, &wr_handle)) - out.set_output_stream( wr_handle) + out.set_output_stream( wr_handle) out.is_writable = True else: with nogil: @@ -432,7 +432,7 @@ cdef class HadoopFileSystem: .OpenReadable(c_path, &rd_handle)) out.set_random_access_file( - rd_handle) + rd_handle) out.is_readable = True assert not out.closed diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi index 6ff1403006ef..0cdf21efc010 100644 --- a/python/pyarrow/io.pxi +++ b/python/pyarrow/io.pxi @@ -133,29 +133,29 @@ cdef class NativeFile: else: check_status(self.output_stream.get().Close()) - cdef set_random_access_file(self, shared_ptr[RandomAccessFile] handle): - self.input_stream = handle + cdef set_random_access_file(self, shared_ptr[CRandomAccessFile] handle): + self.input_stream = handle self.random_access = handle self.is_seekable = True - cdef set_input_stream(self, shared_ptr[InputStream] handle): + cdef set_input_stream(self, shared_ptr[CInputStream] handle): self.input_stream = handle self.random_access.reset() self.is_seekable = False - cdef set_output_stream(self, shared_ptr[OutputStream] handle): + cdef set_output_stream(self, shared_ptr[COutputStream] handle): self.output_stream = handle - cdef shared_ptr[RandomAccessFile] get_random_access_file(self) except *: + cdef shared_ptr[CRandomAccessFile] get_random_access_file(self) except *: self._assert_readable() self._assert_seekable() return self.random_access - cdef shared_ptr[InputStream] get_input_stream(self) except *: + cdef shared_ptr[CInputStream] get_input_stream(self) except *: self._assert_readable() return self.input_stream - cdef shared_ptr[OutputStream] get_output_stream(self) except *: + cdef shared_ptr[COutputStream] get_output_stream(self) except *: self._assert_writable() return self.output_stream @@ -679,11 +679,11 @@ cdef class PythonFile(NativeFile): if kind == 'r': self.set_random_access_file( - shared_ptr[RandomAccessFile](new PyReadableFile(handle))) + shared_ptr[CRandomAccessFile](new PyReadableFile(handle))) self.is_readable = True else: self.set_output_stream( - shared_ptr[OutputStream](new PyOutputStream(handle))) + shared_ptr[COutputStream](new PyOutputStream(handle))) self.is_writable = True def truncate(self, pos=None): @@ -720,8 +720,8 @@ cdef class MemoryMappedFile(NativeFile): result.path = path result.is_readable = True result.is_writable = True - result.set_output_stream( handle) - result.set_random_access_file( handle) + result.set_output_stream( handle) + result.set_random_access_file( handle) result.handle = handle return result @@ -750,8 +750,8 @@ cdef class MemoryMappedFile(NativeFile): with nogil: check_status(CMemoryMappedFile.Open(c_path, c_mode, &handle)) - self.set_output_stream( handle) - self.set_random_access_file( handle) + self.set_output_stream( handle) + self.set_random_access_file( handle) self.handle = handle def resize(self, new_size): @@ -836,7 +836,7 @@ cdef class OSFile(NativeFile): check_status(ReadableFile.Open(path, pool, &handle)) self.is_readable = True - self.set_random_access_file( handle) + self.set_random_access_file( handle) cdef _open_writable(self, c_string path): with nogil: @@ -1174,7 +1174,7 @@ cdef class BufferReader(NativeFile): def __cinit__(self, object obj): self.buffer = as_buffer(obj) - self.set_random_access_file(shared_ptr[RandomAccessFile]( + self.set_random_access_file(shared_ptr[CRandomAccessFile]( new CBufferReader(self.buffer.buffer))) self.is_readable = True @@ -1204,7 +1204,7 @@ cdef class CompressedInputStream(NativeFile): check_status(CCompressedInputStream.Make( codec.get(), stream.get_input_stream(), &compressed_stream)) - self.set_input_stream( compressed_stream) + self.set_input_stream( compressed_stream) self.is_readable = True @@ -1234,13 +1234,13 @@ cdef class CompressedOutputStream(NativeFile): check_status(CCompressedOutputStream.Make( codec.get(), stream.get_output_stream(), &compressed_stream)) - self.set_output_stream( compressed_stream) + self.set_output_stream( compressed_stream) self.is_writable = True ctypedef CBufferedInputStream* _CBufferedInputStreamPtr ctypedef CBufferedOutputStream* _CBufferedOutputStreamPtr -ctypedef RandomAccessFile* _RandomAccessFilePtr +ctypedef CRandomAccessFile* _RandomAccessFilePtr cdef class BufferedInputStream(NativeFile): @@ -1255,7 +1255,7 @@ cdef class BufferedInputStream(NativeFile): buffer_size, maybe_unbox_memory_pool(memory_pool), stream.get_input_stream(), &buffered_stream)) - self.set_input_stream( buffered_stream) + self.set_input_stream( buffered_stream) self.is_readable = True def detach(self): @@ -1269,7 +1269,7 @@ cdef class BufferedInputStream(NativeFile): The underlying raw input stream """ cdef: - shared_ptr[InputStream] c_raw + shared_ptr[CInputStream] c_raw _CBufferedInputStreamPtr buffered NativeFile raw @@ -1287,7 +1287,7 @@ cdef class BufferedInputStream(NativeFile): # selectively. if dynamic_cast[_RandomAccessFilePtr](c_raw.get()) != nullptr: raw.set_random_access_file( - static_pointer_cast[RandomAccessFile, InputStream](c_raw)) + static_pointer_cast[CRandomAccessFile, CInputStream](c_raw)) else: raw.set_input_stream(c_raw) return raw @@ -1305,7 +1305,7 @@ cdef class BufferedOutputStream(NativeFile): buffer_size, maybe_unbox_memory_pool(memory_pool), stream.get_output_stream(), &buffered_stream)) - self.set_output_stream( buffered_stream) + self.set_output_stream( buffered_stream) self.is_writable = True def detach(self): @@ -1319,7 +1319,7 @@ cdef class BufferedOutputStream(NativeFile): The underlying raw output stream """ cdef: - shared_ptr[OutputStream] c_raw + shared_ptr[COutputStream] c_raw _CBufferedOutputStreamPtr buffered NativeFile raw @@ -1400,7 +1400,7 @@ cdef NativeFile _get_native_file(object source, c_bool use_memory_map): cdef get_reader(object source, c_bool use_memory_map, - shared_ptr[RandomAccessFile]* reader): + shared_ptr[CRandomAccessFile]* reader): cdef NativeFile nf nf = _get_native_file(source, use_memory_map) @@ -1408,7 +1408,7 @@ cdef get_reader(object source, c_bool use_memory_map, cdef get_input_stream(object source, c_bool use_memory_map, - shared_ptr[InputStream]* out): + shared_ptr[CInputStream]* out): """ Like get_reader(), but can automatically decompress, and returns an InputStream. @@ -1416,7 +1416,7 @@ cdef get_input_stream(object source, c_bool use_memory_map, cdef: NativeFile nf unique_ptr[CCodec] codec - shared_ptr[InputStream] input_stream + shared_ptr[CInputStream] input_stream shared_ptr[CCompressedInputStream] compressed_stream CompressionType compression_type @@ -1435,12 +1435,12 @@ cdef get_input_stream(object source, c_bool use_memory_map, check_status(CCodec.Create(compression_type, &codec)) check_status(CCompressedInputStream.Make(codec.get(), input_stream, &compressed_stream)) - input_stream = compressed_stream + input_stream = compressed_stream out[0] = input_stream -cdef get_writer(object source, shared_ptr[OutputStream]* writer): +cdef get_writer(object source, shared_ptr[COutputStream]* writer): cdef NativeFile nf try: diff --git a/python/pyarrow/ipc.pxi b/python/pyarrow/ipc.pxi index 3d8b3f4af4db..c9684f13e5bb 100644 --- a/python/pyarrow/ipc.pxi +++ b/python/pyarrow/ipc.pxi @@ -17,6 +17,7 @@ import warnings + cdef class Message: """ Container for an Arrow IPC message with metadata and optional body @@ -76,7 +77,7 @@ cdef class Message: """ cdef: int64_t output_length = 0 - OutputStream* out + COutputStream* out CIpcOptions options options.alignment = alignment @@ -136,9 +137,11 @@ cdef class MessageReader: @staticmethod def open_stream(source): - cdef MessageReader result = MessageReader.__new__(MessageReader) - cdef shared_ptr[InputStream] in_stream - cdef unique_ptr[CMessageReader] reader + cdef: + MessageReader result = MessageReader.__new__(MessageReader) + shared_ptr[CInputStream] in_stream + unique_ptr[CMessageReader] reader + _get_input_stream(source, &in_stream) with nogil: reader = CMessageReader.Open(in_stream) @@ -250,7 +253,7 @@ cdef class _CRecordBatchWriter: cdef class _RecordBatchStreamWriter(_CRecordBatchWriter): cdef: - shared_ptr[OutputStream] sink + shared_ptr[COutputStream] sink CIpcOptions options bint closed @@ -276,7 +279,7 @@ cdef class _RecordBatchStreamWriter(_CRecordBatchWriter): self.writer = GetResultValue(result) -cdef _get_input_stream(object source, shared_ptr[InputStream]* out): +cdef _get_input_stream(object source, shared_ptr[CInputStream]* out): try: source = as_buffer(source) except TypeError: @@ -332,7 +335,7 @@ cdef class _CRecordBatchReader: cdef class _RecordBatchStreamReader(_CRecordBatchReader): cdef: - shared_ptr[InputStream] in_stream + shared_ptr[CInputStream] in_stream cdef readonly: Schema schema @@ -367,7 +370,7 @@ cdef class _RecordBatchFileWriter(_RecordBatchStreamWriter): cdef class _RecordBatchFileReader: cdef: shared_ptr[CRecordBatchFileReader] reader - shared_ptr[RandomAccessFile] file + shared_ptr[CRandomAccessFile] file cdef readonly: Schema schema @@ -516,9 +519,8 @@ def read_tensor(source): """ cdef: shared_ptr[CTensor] sp_tensor - InputStream* c_stream - - cdef NativeFile nf = as_native_file(source) + CInputStream* c_stream + NativeFile nf = as_native_file(source) c_stream = nf.get_input_stream().get() with nogil: @@ -540,7 +542,7 @@ def read_message(source): """ cdef: Message result = Message.__new__(Message) - InputStream* c_stream + CInputStream* c_stream cdef NativeFile nf = as_native_file(source) c_stream = nf.get_input_stream().get() @@ -571,7 +573,7 @@ def read_schema(obj, DictionaryMemo dictionary_memo=None): """ cdef: shared_ptr[CSchema] result - shared_ptr[RandomAccessFile] cpp_file + shared_ptr[CRandomAccessFile] cpp_file CDictionaryMemo temp_memo CDictionaryMemo* arg_dict_memo diff --git a/python/pyarrow/lib.pxd b/python/pyarrow/lib.pxd index 553227a4e559..571dfaa9449d 100644 --- a/python/pyarrow/lib.pxd +++ b/python/pyarrow/lib.pxd @@ -436,9 +436,9 @@ cdef class ResizableBuffer(Buffer): cdef class NativeFile: cdef: - shared_ptr[InputStream] input_stream - shared_ptr[RandomAccessFile] random_access - shared_ptr[OutputStream] output_stream + shared_ptr[CInputStream] input_stream + shared_ptr[CRandomAccessFile] random_access + shared_ptr[COutputStream] output_stream bint is_readable bint is_writable bint is_seekable @@ -449,13 +449,13 @@ cdef class NativeFile: # extension classes are technically virtual in the C++ sense) we can expose # the arrow::io abstract file interfaces to other components throughout the # suite of Arrow C++ libraries - cdef set_random_access_file(self, shared_ptr[RandomAccessFile] handle) - cdef set_input_stream(self, shared_ptr[InputStream] handle) - cdef set_output_stream(self, shared_ptr[OutputStream] handle) + cdef set_random_access_file(self, shared_ptr[CRandomAccessFile] handle) + cdef set_input_stream(self, shared_ptr[CInputStream] handle) + cdef set_output_stream(self, shared_ptr[COutputStream] handle) - cdef shared_ptr[RandomAccessFile] get_random_access_file(self) except * - cdef shared_ptr[InputStream] get_input_stream(self) except * - cdef shared_ptr[OutputStream] get_output_stream(self) except * + cdef shared_ptr[CRandomAccessFile] get_random_access_file(self) except * + cdef shared_ptr[CInputStream] get_input_stream(self) except * + cdef shared_ptr[COutputStream] get_output_stream(self) except * cdef class BufferedInputStream(NativeFile): @@ -485,10 +485,10 @@ cdef class _CRecordBatchReader: cdef get_input_stream(object source, c_bool use_memory_map, - shared_ptr[InputStream]* reader) + shared_ptr[CInputStream]* reader) cdef get_reader(object source, c_bool use_memory_map, - shared_ptr[RandomAccessFile]* reader) -cdef get_writer(object source, shared_ptr[OutputStream]* writer) + shared_ptr[CRandomAccessFile]* reader) +cdef get_writer(object source, shared_ptr[COutputStream]* writer) # Default is allow_none=False cdef DataType ensure_type(object type, c_bool allow_none=*) diff --git a/python/pyarrow/s3fs.py b/python/pyarrow/s3fs.py new file mode 100644 index 000000000000..5619e186f9ea --- /dev/null +++ b/python/pyarrow/s3fs.py @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import absolute_import + +from pyarrow._s3fs import ( # noqa + initialize_s3, + finalize_s3, + S3Options, + S3FileSystem +) + +initialize_s3() diff --git a/python/pyarrow/serialization.pxi b/python/pyarrow/serialization.pxi index fba834434d02..cf343b0ba320 100644 --- a/python/pyarrow/serialization.pxi +++ b/python/pyarrow/serialization.pxi @@ -254,11 +254,11 @@ cdef class SerializedPyObject: """ Write serialized object to a sink """ - cdef shared_ptr[OutputStream] stream + cdef shared_ptr[COutputStream] stream get_writer(sink, &stream) self._write_to(stream.get()) - cdef _write_to(self, OutputStream* stream): + cdef _write_to(self, COutputStream* stream): with nogil: check_status(self.data.WriteTo(stream)) @@ -399,7 +399,7 @@ def read_serialized(source, base=None): ------- serialized : the serialized data """ - cdef shared_ptr[RandomAccessFile] stream + cdef shared_ptr[CRandomAccessFile] stream get_reader(source, True, &stream) cdef SerializedPyObject serialized = SerializedPyObject() diff --git a/python/pyarrow/tests/conftest.py b/python/pyarrow/tests/conftest.py index bfb1e8e5d0b5..1b980080d50b 100644 --- a/python/pyarrow/tests/conftest.py +++ b/python/pyarrow/tests/conftest.py @@ -16,6 +16,9 @@ # under the License. import os +import subprocess +import tempfile + import pytest import hypothesis as h @@ -24,6 +27,8 @@ except ImportError: import pathlib2 as pathlib # py2 compat +from pyarrow.util import find_free_port + # setup hypothesis profiles h.settings.register_profile('ci', max_examples=1000) @@ -130,6 +135,12 @@ except ImportError: pass +try: + import pyarrow.s3fs # noqa + defaults['s3'] = True +except ImportError: + pass + def pytest_configure(config): for mark in groups: @@ -211,3 +222,47 @@ def tempdir(tmpdir): @pytest.fixture(scope='session') def datadir(): return pathlib.Path(__file__).parent / 'data' + + +try: + from tempfile import TemporaryDirectory +except ImportError: + import shutil + + class TemporaryDirectory(object): + """Temporary directory implementation for python 2""" + + def __enter__(self): + self.tmp = tempfile.mkdtemp() + return self.tmp + + def __exit__(self, exc_type, exc_value, traceback): + shutil.rmtree(self.tmp) + + +@pytest.mark.s3 +@pytest.fixture(scope='session') +def minio_server(): + host, port = 'localhost', find_free_port() + access_key, secret_key = 'arrow', 'apachearrow' + + address = '{}:{}'.format(host, port) + env = os.environ.copy() + env.update({ + 'MINIO_ACCESS_KEY': access_key, + 'MINIO_SECRET_KEY': secret_key + }) + + with TemporaryDirectory() as tempdir: + args = ['minio', '--compat', 'server', '--quiet', '--address', + address, tempdir] + proc = None + try: + proc = subprocess.Popen(args, env=env) + except IOError: + pytest.skip('`minio` command cannot be located') + else: + yield address, access_key, secret_key + finally: + if proc is not None: + proc.kill() diff --git a/python/pyarrow/tests/test_flight.py b/python/pyarrow/tests/test_flight.py index 83afba6de65e..40099b06aec6 100644 --- a/python/pyarrow/tests/test_flight.py +++ b/python/pyarrow/tests/test_flight.py @@ -17,9 +17,7 @@ # under the License. import base64 -import contextlib import os -import socket import struct import tempfile import threading @@ -30,7 +28,7 @@ import pyarrow as pa from pyarrow.compat import tobytes -from pyarrow.util import pathlib +from pyarrow.util import pathlib, find_free_port try: from pyarrow import flight @@ -48,14 +46,6 @@ pytestmark = pytest.mark.flight -def find_free_port(): - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - with contextlib.closing(sock) as sock: - sock.bind(('', 0)) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - return sock.getsockname()[1] - - def test_import(): # So we see the ImportError somewhere import pyarrow.flight # noqa diff --git a/python/pyarrow/tests/test_fs.py b/python/pyarrow/tests/test_fs.py index f897e0d36f97..f6b6bf1d18cd 100644 --- a/python/pyarrow/tests/test_fs.py +++ b/python/pyarrow/tests/test_fs.py @@ -23,39 +23,109 @@ import pytest -from pyarrow import ArrowIOError +import pyarrow as pa +from pyarrow.tests.test_io import gzip_compress, gzip_decompress from pyarrow.fs import (FileType, Selector, FileSystem, LocalFileSystem, SubTreeFileSystem) -from pyarrow.tests.test_io import gzip_compress, gzip_decompress + + +@pytest.fixture +def localfs(request, tempdir): + return dict( + fs=LocalFileSystem(), + pathfn=lambda p: (tempdir / p).as_posix(), + allow_move_dir=True, + allow_append_to_file=True, + ) + + +@pytest.fixture +def subtree_localfs(request, tempdir, localfs): + prefix = 'subtree/prefix/' + (tempdir / prefix).mkdir(parents=True) + return dict( + fs=SubTreeFileSystem(prefix, localfs['fs']), + pathfn=prefix.__add__, + allow_move_dir=True, + allow_append_to_file=True, + ) + + +@pytest.mark.s3 +@pytest.fixture +def s3fs(request, minio_server): + from pyarrow.s3fs import S3Options, S3FileSystem + + address, access_key, secret_key = minio_server + bucket = 'pyarrow-filesystem/' + options = S3Options( + endpoint_override=address, + access_key=access_key, + secret_key=secret_key, + scheme='http' + ) + fs = S3FileSystem(options) + fs.create_dir(bucket) + + return dict( + fs=fs, + pathfn=bucket.__add__, + allow_move_dir=False, + allow_append_to_file=False, + ) + + +@pytest.fixture +def subtree_s3fs(request, s3fs): + prefix = 'pyarrow-filesystem/prefix/' + return dict( + fs=SubTreeFileSystem(prefix, s3fs['fs']), + pathfn=prefix.__add__, + allow_move_dir=False, + allow_append_to_file=False, + ) @pytest.fixture(params=[ pytest.param( - lambda tmp: LocalFileSystem(), - id='LocalFileSystem' + pytest.lazy_fixture('localfs'), + id='LocalFileSystem()' ), pytest.param( - lambda tmp: SubTreeFileSystem(tmp, LocalFileSystem()), - id='SubTreeFileSystem(LocalFileSystem)' + pytest.lazy_fixture('subtree_localfs'), + id='SubTreeFileSystem(LocalFileSystem())' + ), + pytest.param( + pytest.lazy_fixture('s3fs'), + id='S3FileSystem' + ), + pytest.param( + pytest.lazy_fixture('subtree_s3fs'), + id='SubTreeFileSystem(S3FileSystem())' ) ]) -def fs(request, tempdir): - return request.param(tempdir.as_posix()) +def filesystem_config(request): + return request.param + + +@pytest.fixture +def fs(request, filesystem_config): + return filesystem_config['fs'] @pytest.fixture -def testpath(request, fs, tempdir): - # we always use the tempdir for reading and writing test artifacts, but - # if the filesystem is wrapped in a SubTreeFileSystem then we don't need - # to prepend the path with the tempdir, we also test the API with both - # pathlib.Path objects and plain python strings - def convert(path): - if isinstance(fs, SubTreeFileSystem): - path = pathlib.Path(path) - else: - path = tempdir / path - return path.as_posix() - return convert +def pathfn(request, filesystem_config): + return filesystem_config['pathfn'] + + +@pytest.fixture +def allow_move_dir(request, filesystem_config): + return filesystem_config['allow_move_dir'] + + +@pytest.fixture +def allow_append_to_file(request, filesystem_config): + return filesystem_config['allow_append_to_file'] def test_cannot_instantiate_base_filesystem(): @@ -74,157 +144,158 @@ class Path: fs.create_dir(path) -def test_get_target_stats(fs, tempdir, testpath): - aaa, aaa_ = testpath('a/aa/aaa'), tempdir / 'a' / 'aa' / 'aaa' - bb, bb_ = testpath('a/bb'), tempdir / 'a' / 'bb' - c, c_ = testpath('c.txt'), tempdir / 'c.txt' - - aaa_.mkdir(parents=True) - bb_.touch() - c_.write_bytes(b'test') +def test_get_target_stats(fs, pathfn): + aaa = pathfn('a/aa/aaa/') + bb = pathfn('a/bb') + c = pathfn('c.txt') - def mtime_almost_equal(fs_dt, pathlib_ts): - # arrow's filesystem implementation truncates mtime to microsends - # resolution whereas pathlib rounds - pathlib_dt = datetime.utcfromtimestamp(pathlib_ts) - difference = (fs_dt - pathlib_dt).total_seconds() - return abs(difference) <= 10**-6 + fs.create_dir(aaa) + with fs.open_output_stream(bb): + pass # touch + with fs.open_output_stream(c) as fp: + fp.write(b'test') aaa_stat, bb_stat, c_stat = fs.get_target_stats([aaa, bb, c]) assert aaa_stat.path == aaa assert 'aaa' in repr(aaa_stat) - assert aaa_stat.base_name == 'aaa' assert aaa_stat.extension == '' - assert aaa_stat.type == FileType.Directory - assert mtime_almost_equal(aaa_stat.mtime, aaa_.stat().st_mtime) - with pytest.raises(ValueError): - aaa_stat.size + assert isinstance(aaa_stat.mtime, datetime) assert bb_stat.path == str(bb) assert bb_stat.base_name == 'bb' assert bb_stat.extension == '' assert bb_stat.type == FileType.File assert bb_stat.size == 0 - assert mtime_almost_equal(bb_stat.mtime, bb_.stat().st_mtime) + assert isinstance(bb_stat.mtime, datetime) assert c_stat.path == str(c) assert c_stat.base_name == 'c.txt' assert c_stat.extension == 'txt' assert c_stat.type == FileType.File assert c_stat.size == 4 - assert mtime_almost_equal(c_stat.mtime, c_.stat().st_mtime) - - -def test_get_target_stats_with_selector(fs, tempdir, testpath): - base_dir = testpath('.') - base_dir_ = tempdir - - selector = Selector(base_dir, allow_non_existent=False, recursive=True) - assert selector.base_dir == str(base_dir) - - (tempdir / 'test_file').touch() - (tempdir / 'test_directory').mkdir() - - stats = fs.get_target_stats(selector) - expected = list(base_dir_.iterdir()) - assert len(stats) == len(expected) - - for st in stats: - p = base_dir_ / st.path - if p.is_dir(): - assert st.type == FileType.Directory - if p.is_file(): - assert st.type == FileType.File - - -def test_create_dir(fs, tempdir, testpath): - directory = testpath('directory') - directory_ = tempdir / 'directory' - assert not directory_.exists() - fs.create_dir(directory) - assert directory_.exists() - - # recursive - directory = testpath('deeply/nested/directory') - directory_ = tempdir / 'deeply' / 'nested' / 'directory' - assert not directory_.exists() - with pytest.raises(ArrowIOError): - fs.create_dir(directory, recursive=False) - fs.create_dir(directory) - assert directory_.exists() - - -def test_delete_dir(fs, tempdir, testpath): - folder = testpath('directory') - nested = testpath('nested/directory') - folder_ = tempdir / 'directory' - nested_ = tempdir / 'nested' / 'directory' - - folder_.mkdir() - nested_.mkdir(parents=True) - - assert folder_.exists() - fs.delete_dir(folder) - assert not folder_.exists() - - assert nested_.exists() - fs.delete_dir(nested) - assert not nested_.exists() - - -def test_copy_file(fs, tempdir, testpath): - # copy file - source = testpath('source-file') - source_ = tempdir / 'source-file' - source_.touch() - target = testpath('target-file') - target_ = tempdir / 'target-file' - assert not target_.exists() - fs.copy_file(source, target) - assert source_.exists() - assert target_.exists() - - -def test_move(fs, tempdir, testpath): - # move directory - source = testpath('source-dir') - source_ = tempdir / 'source-dir' - source_.mkdir() - target = testpath('target-dir') - target_ = tempdir / 'target-dir' - assert not target_.exists() - fs.move(source, target) - assert not source_.exists() - assert target_.exists() - - # move file - source = testpath('source-file') - source_ = tempdir / 'source-file' - source_.touch() - target = testpath('target-file') - target_ = tempdir / 'target-file' - assert not target_.exists() - fs.move(source, target) - assert not source_.exists() - assert target_.exists() - - -def test_delete_file(fs, tempdir, testpath): - target = testpath('target-file') - target_ = tempdir / 'target-file' - target_.touch() - assert target_.exists() - fs.delete_file(target) - assert not target_.exists() - - nested = testpath('nested/target-file') - nested_ = tempdir / 'nested/target-file' - nested_.parent.mkdir() - nested_.touch() - assert nested_.exists() - fs.delete_file(nested) - assert not nested_.exists() + assert isinstance(c_stat.mtime, datetime) + + +def test_get_target_stats_with_selector(fs, pathfn): + base_dir = pathfn('selector-dir/') + file_a = pathfn('selector-dir/test_file_a') + file_b = pathfn('selector-dir/test_file_b') + dir_a = pathfn('selector-dir/test_dir_a') + + try: + fs.create_dir(base_dir) + with fs.open_output_stream(file_a): + pass + with fs.open_output_stream(file_b): + pass + fs.create_dir(dir_a) + + selector = Selector(base_dir, allow_non_existent=False, recursive=True) + assert selector.base_dir == base_dir + + stats = fs.get_target_stats(selector) + assert len(stats) == 3 + + for st in stats: + if st.path.endswith(file_a): + assert st.type == FileType.File + elif st.path.endswith(file_b): + assert st.type == FileType.File + elif st.path.endswith(dir_a): + assert st.type == FileType.Directory + else: + raise ValueError('unexpected path {}'.format(st.path)) + finally: + fs.delete_file(file_a) + fs.delete_file(file_b) + fs.delete_dir(dir_a) + fs.delete_dir(base_dir) + + +def test_create_dir(fs, pathfn): + d = pathfn('test-directory/') + + with pytest.raises(pa.ArrowIOError): + fs.delete_dir(d) + + fs.create_dir(d) + fs.delete_dir(d) + + d = pathfn('deeply/nested/test-directory/') + fs.create_dir(d, recursive=True) + fs.delete_dir(d) + + +def test_delete_dir(fs, pathfn): + d = pathfn('directory/') + nd = pathfn('directory/nested/') + + fs.create_dir(nd) + fs.delete_dir(nd) + fs.delete_dir(d) + with pytest.raises(pa.ArrowIOError): + fs.delete_dir(d) + + +def test_copy_file(fs, pathfn): + s = pathfn('test-copy-source-file') + t = pathfn('test-copy-target-file') + + with fs.open_output_stream(s): + pass + + fs.copy_file(s, t) + fs.delete_file(s) + fs.delete_file(t) + + +def test_move_directory(fs, pathfn, allow_move_dir): + # move directory (doesn't work with S3) + s = pathfn('source-dir/') + t = pathfn('target-dir/') + + fs.create_dir(s) + + if allow_move_dir: + fs.move(s, t) + with pytest.raises(pa.ArrowIOError): + fs.delete_dir(s) + fs.delete_dir(t) + else: + with pytest.raises(pa.ArrowIOError): + fs.move(s, t) + + +def test_move_file(fs, pathfn): + s = pathfn('test-move-source-file') + t = pathfn('test-move-target-file') + + with fs.open_output_stream(s): + pass + + fs.move(s, t) + with pytest.raises(pa.ArrowIOError): + fs.delete_file(s) + fs.delete_file(t) + + +def test_delete_file(fs, pathfn): + p = pathfn('test-delete-target-file') + with fs.open_output_stream(p): + pass + + fs.delete_file(p) + with pytest.raises(pa.ArrowIOError): + fs.delete_file(p) + + d = pathfn('test-delete-nested') + fs.create_dir(d) + f = pathfn('test-delete-nested/target-file') + with fs.open_output_stream(f) as s: + s.write(b'data') + + fs.delete_dir(d) def identity(v): @@ -240,27 +311,28 @@ def identity(v): ('gzip', 256, gzip_compress), ] ) -def test_open_input_stream(fs, tempdir, testpath, compression, buffer_size, - compressor): - file = testpath('abc') - file_ = tempdir / 'abc' - data = b'some data' * 1024 - file_.write_bytes(compressor(data)) +def test_open_input_stream(fs, pathfn, compression, buffer_size, compressor): + p = pathfn('open-input-stream') - with fs.open_input_stream(file, compression, buffer_size) as f: - result = f.read() + data = b'some data for reading\n' * 512 + with fs.open_output_stream(p) as s: + s.write(compressor(data)) + + with fs.open_input_stream(p, compression, buffer_size) as s: + result = s.read() assert result == data -def test_open_input_file(fs, tempdir, testpath): - file = testpath('abc') - file_ = tempdir / 'abc' +def test_open_input_file(fs, pathfn): + p = pathfn('open-input-file') + data = b'some data' * 1024 - file_.write_bytes(data) + with fs.open_output_stream(p) as s: + s.write(data) read_from = len(b'some data') * 512 - with fs.open_input_file(file) as f: + with fs.open_input_file(p) as f: f.seek(read_from) result = f.read() @@ -276,16 +348,16 @@ def test_open_input_file(fs, tempdir, testpath): ('gzip', 256, gzip_decompress), ] ) -def test_open_output_stream(fs, tempdir, testpath, compression, buffer_size, +def test_open_output_stream(fs, pathfn, compression, buffer_size, decompressor): - file = testpath('abc') - file_ = tempdir / 'abc' + p = pathfn('open-output-stream') - data = b'some data' * 1024 - with fs.open_output_stream(file, compression, buffer_size) as f: + data = b'some data for writing' * 1024 + with fs.open_output_stream(p, compression, buffer_size) as f: f.write(data) - assert decompressor(file_.read_bytes()) == data + with fs.open_input_stream(p, compression, buffer_size) as f: + assert f.read(len(data)) == data @pytest.mark.parametrize( @@ -297,13 +369,57 @@ def test_open_output_stream(fs, tempdir, testpath, compression, buffer_size, ('gzip', 256, gzip_compress, gzip_decompress), ] ) -def test_open_append_stream(fs, tempdir, testpath, compression, buffer_size, - compressor, decompressor): - file = testpath('abc') - file_ = tempdir / 'abc' - file_.write_bytes(compressor(b'already existing')) +def test_open_append_stream(fs, pathfn, compression, buffer_size, compressor, + decompressor, allow_append_to_file): + p = pathfn('open-append-stream') + + initial = compressor(b'already existing') + with fs.open_output_stream(p) as s: + s.write(initial) + + if allow_append_to_file: + with fs.open_append_stream(p, compression, buffer_size) as f: + f.write(b'\nnewly added') - with fs.open_append_stream(file, compression, buffer_size) as f: - f.write(b'\nnewly added') + with fs.open_input_stream(p) as f: + result = f.read() - assert decompressor(file_.read_bytes()) == b'already existing\nnewly added' + result = decompressor(result) + assert result == b'already existing\nnewly added' + else: + with pytest.raises(pa.ArrowNotImplementedError): + fs.open_append_stream(p, compression, buffer_size) + + +@pytest.mark.s3 +def test_s3_options(minio_server): + from pyarrow.s3fs import S3Options + + options = S3Options() + + assert options.region == 'us-east-1' + options.region = 'us-west-1' + assert options.region == 'us-west-1' + + assert options.scheme == 'https' + options.scheme = 'http' + assert options.scheme == 'http' + + assert options.endpoint_override == '' + options.endpoint_override = 'localhost:8999' + assert options.endpoint_override == 'localhost:8999' + + with pytest.raises(ValueError): + S3Options(access_key='access') + with pytest.raises(ValueError): + S3Options(secret_key='secret') + + address, access_key, secret_key = minio_server + options = S3Options( + access_key=access_key, + secret_key=secret_key, + endpoint_override=address, + scheme='http' + ) + assert options.scheme == 'http' + assert options.endpoint_override == address diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py index a2c6352bddaf..fb80020d7ead 100644 --- a/python/pyarrow/tests/test_parquet.py +++ b/python/pyarrow/tests/test_parquet.py @@ -1843,18 +1843,41 @@ def test_filters_read_table(tempdir): assert table.num_rows == 3 -@pytest.yield_fixture -def s3_example(): - access_key = os.environ['PYARROW_TEST_S3_ACCESS_KEY'] - secret_key = os.environ['PYARROW_TEST_S3_SECRET_KEY'] - bucket_name = os.environ['PYARROW_TEST_S3_BUCKET'] +@pytest.fixture +def s3_bucket(request, minio_server): + boto3 = pytest.importorskip('boto3') + botocore = pytest.importorskip('botocore') + + address, access_key, secret_key = minio_server + s3 = boto3.resource( + 's3', + endpoint_url='http://{}'.format(address), + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + config=botocore.client.Config(signature_version='s3v4'), + region_name='us-east-1' + ) + bucket = s3.Bucket('test-s3fs') + bucket.create() + return 'test-s3fs' + - import s3fs - fs = s3fs.S3FileSystem(key=access_key, secret=secret_key) +@pytest.fixture +def s3_example(minio_server, s3_bucket): + s3fs = pytest.importorskip('s3fs') + + address, access_key, secret_key = minio_server + fs = s3fs.S3FileSystem( + key=access_key, + secret=secret_key, + client_kwargs={ + 'endpoint_url': 'http://{}'.format(address) + } + ) test_dir = guid() + bucket_uri = 's3://{0}/{1}'.format(s3_bucket, test_dir) - bucket_uri = 's3://{0}/{1}'.format(bucket_name, test_dir) fs.mkdir(bucket_uri) yield fs, bucket_uri fs.rm(bucket_uri, recursive=True) @@ -1920,23 +1943,29 @@ def _visit_level(base_dir, level, part_keys): for value in values: this_part_keys = part_keys + [(name, value)] - level_dir = base_dir / '{0}={1}'.format(name, value) + level_dir = fs._path_join( + str(base_dir), + '{0}={1}'.format(name, value) + ) fs.mkdir(level_dir) if level == DEPTH - 1: # Generate example data - file_path = level_dir / guid() - + file_path = fs._path_join(level_dir, guid()) filtered_df = _filter_partition(df, this_part_keys) part_table = pa.Table.from_pandas(filtered_df) with fs.open(file_path, 'wb') as f: _write_table(part_table, f) assert fs.exists(file_path) - (level_dir / '_SUCCESS').touch() + file_success = fs._path_join(level_dir, '_SUCCESS') + with fs.open(file_success, 'wb') as f: + pass else: _visit_level(level_dir, level + 1, this_part_keys) - (level_dir / '_SUCCESS').touch() + file_success = fs._path_join(level_dir, '_SUCCESS') + with fs.open(file_success, 'wb') as f: + pass _visit_level(base_dir, 0, []) diff --git a/python/pyarrow/util.py b/python/pyarrow/util.py index 5e4fb3579372..7219a447f356 100644 --- a/python/pyarrow/util.py +++ b/python/pyarrow/util.py @@ -19,8 +19,10 @@ from __future__ import absolute_import +import contextlib import functools import six +import socket import warnings @@ -125,3 +127,11 @@ def get_contiguous_span(shape, strides, itemsize): if end - start != itemsize * product(shape): raise ValueError('array data is non-contiguous') return start, end + + +def find_free_port(): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + with contextlib.closing(sock) as sock: + sock.bind(('', 0)) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + return sock.getsockname()[1] diff --git a/python/requirements-test.txt b/python/requirements-test.txt index 73eabfebd288..10d445cbc442 100644 --- a/python/requirements-test.txt +++ b/python/requirements-test.txt @@ -3,4 +3,5 @@ hypothesis pandas pathlib2; python_version < "3.4" pytest +pytest-lazy-fixture pytz diff --git a/python/setup.py b/python/setup.py index 5e88352a8f9b..d7207eedd8f1 100755 --- a/python/setup.py +++ b/python/setup.py @@ -141,6 +141,8 @@ def initialize_options(self): if not hasattr(sys, 'gettotalrefcount'): self.build_type = 'release' + self.with_s3 = strtobool( + os.environ.get('PYARROW_WITH_S3', '0')) self.with_cuda = strtobool( os.environ.get('PYARROW_WITH_CUDA', '0')) self.with_flight = strtobool( @@ -176,6 +178,7 @@ def initialize_options(self): '_parquet', '_orc', '_plasma', + '_s3fs', 'gandiva'] def _run_cmake(self): @@ -215,6 +218,8 @@ def _run_cmake(self): if self.cmake_generator: cmake_options += ['-G', self.cmake_generator] + if self.with_s3: + cmake_options.append('-DPYARROW_BUILD_S3=on') if self.with_cuda: cmake_options.append('-DPYARROW_BUILD_CUDA=on') if self.with_flight: @@ -414,6 +419,8 @@ def _failure_permitted(self, name): return True if name == '_flight' and not self.with_flight: return True + if name == '_s3fs' and not self.with_s3: + return True if name == '_cuda' and not self.with_cuda: return True if name == 'gandiva' and not self.with_gandiva: From 3b262f628f360b0568ea175adebba96f1187579d Mon Sep 17 00:00:00 2001 From: "Jonathan A. Sternberg" Date: Tue, 1 Oct 2019 17:17:34 +0200 Subject: [PATCH 10/24] ARROW-6648: [Go] Expose the bitutil package Closes #5457 from jsternberg/master and squashes the following commits: 3914a141a ARROW-6648: Expose the bitutil package Authored-by: Jonathan A. Sternberg Signed-off-by: Sebastien Binet --- go/arrow/array/array.go | 2 +- go/arrow/array/array_test.go | 2 +- go/arrow/array/boolean.go | 2 +- go/arrow/array/booleanbuilder.go | 2 +- go/arrow/array/bufferbuilder.go | 2 +- go/arrow/array/bufferbuilder_numeric.gen.go | 2 +- go/arrow/array/builder.go | 2 +- go/arrow/array/decimal128.go | 2 +- go/arrow/array/fixed_size_list.go | 2 +- go/arrow/array/float16_builder.go | 2 +- go/arrow/array/interval.go | 2 +- go/arrow/array/list.go | 2 +- go/arrow/array/numericbuilder.gen.go | 2 +- go/arrow/array/struct.go | 2 +- go/arrow/{internal => }/bitutil/bitutil.go | 0 go/arrow/{internal => }/bitutil/bitutil_test.go | 2 +- go/arrow/ipc/file_reader.go | 2 +- go/arrow/ipc/file_writer.go | 2 +- go/arrow/ipc/writer.go | 2 +- go/arrow/type_traits_boolean.go | 2 +- 20 files changed, 19 insertions(+), 19 deletions(-) rename go/arrow/{internal => }/bitutil/bitutil.go (100%) rename go/arrow/{internal => }/bitutil/bitutil_test.go (99%) diff --git a/go/arrow/array/array.go b/go/arrow/array/array.go index c3577337a7b9..9fc4cc0be6db 100644 --- a/go/arrow/array/array.go +++ b/go/arrow/array/array.go @@ -20,7 +20,7 @@ import ( "sync/atomic" "github.com/apache/arrow/go/arrow" - "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/bitutil" "github.com/apache/arrow/go/arrow/internal/debug" ) diff --git a/go/arrow/array/array_test.go b/go/arrow/array/array_test.go index af50cf955af3..48c238656b4a 100644 --- a/go/arrow/array/array_test.go +++ b/go/arrow/array/array_test.go @@ -298,4 +298,4 @@ func TestArraySliceTypes(t *testing.T) { } }) } -} \ No newline at end of file +} diff --git a/go/arrow/array/boolean.go b/go/arrow/array/boolean.go index 5095b17318ab..e352e6e043b3 100644 --- a/go/arrow/array/boolean.go +++ b/go/arrow/array/boolean.go @@ -21,7 +21,7 @@ import ( "strings" "github.com/apache/arrow/go/arrow" - "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/bitutil" "github.com/apache/arrow/go/arrow/memory" ) diff --git a/go/arrow/array/booleanbuilder.go b/go/arrow/array/booleanbuilder.go index 943df1f12007..4a38156849af 100644 --- a/go/arrow/array/booleanbuilder.go +++ b/go/arrow/array/booleanbuilder.go @@ -20,7 +20,7 @@ import ( "sync/atomic" "github.com/apache/arrow/go/arrow" - "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/bitutil" "github.com/apache/arrow/go/arrow/internal/debug" "github.com/apache/arrow/go/arrow/memory" ) diff --git a/go/arrow/array/bufferbuilder.go b/go/arrow/array/bufferbuilder.go index 29d97fc8d41f..bcc7153b5598 100644 --- a/go/arrow/array/bufferbuilder.go +++ b/go/arrow/array/bufferbuilder.go @@ -19,7 +19,7 @@ package array import ( "sync/atomic" - "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/bitutil" "github.com/apache/arrow/go/arrow/internal/debug" "github.com/apache/arrow/go/arrow/memory" ) diff --git a/go/arrow/array/bufferbuilder_numeric.gen.go b/go/arrow/array/bufferbuilder_numeric.gen.go index e39b2528a9ef..4cdf42685f90 100644 --- a/go/arrow/array/bufferbuilder_numeric.gen.go +++ b/go/arrow/array/bufferbuilder_numeric.gen.go @@ -20,7 +20,7 @@ package array import ( "github.com/apache/arrow/go/arrow" - "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/bitutil" "github.com/apache/arrow/go/arrow/memory" ) diff --git a/go/arrow/array/builder.go b/go/arrow/array/builder.go index 6f5beb355d3a..bdd871611afc 100644 --- a/go/arrow/array/builder.go +++ b/go/arrow/array/builder.go @@ -21,7 +21,7 @@ import ( "sync/atomic" "github.com/apache/arrow/go/arrow" - "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/bitutil" "github.com/apache/arrow/go/arrow/memory" ) diff --git a/go/arrow/array/decimal128.go b/go/arrow/array/decimal128.go index 390d87e9a496..3acf6b9c444d 100644 --- a/go/arrow/array/decimal128.go +++ b/go/arrow/array/decimal128.go @@ -22,8 +22,8 @@ import ( "sync/atomic" "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/bitutil" "github.com/apache/arrow/go/arrow/decimal128" - "github.com/apache/arrow/go/arrow/internal/bitutil" "github.com/apache/arrow/go/arrow/internal/debug" "github.com/apache/arrow/go/arrow/memory" ) diff --git a/go/arrow/array/fixed_size_list.go b/go/arrow/array/fixed_size_list.go index 6c61513cf0e1..b0a2ce75b978 100644 --- a/go/arrow/array/fixed_size_list.go +++ b/go/arrow/array/fixed_size_list.go @@ -22,7 +22,7 @@ import ( "sync/atomic" "github.com/apache/arrow/go/arrow" - "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/bitutil" "github.com/apache/arrow/go/arrow/internal/debug" "github.com/apache/arrow/go/arrow/memory" ) diff --git a/go/arrow/array/float16_builder.go b/go/arrow/array/float16_builder.go index 9c36dd172a18..80864279a061 100644 --- a/go/arrow/array/float16_builder.go +++ b/go/arrow/array/float16_builder.go @@ -20,8 +20,8 @@ import ( "sync/atomic" "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/bitutil" "github.com/apache/arrow/go/arrow/float16" - "github.com/apache/arrow/go/arrow/internal/bitutil" "github.com/apache/arrow/go/arrow/internal/debug" "github.com/apache/arrow/go/arrow/memory" ) diff --git a/go/arrow/array/interval.go b/go/arrow/array/interval.go index 21efd6e4078c..cfdd59379af0 100644 --- a/go/arrow/array/interval.go +++ b/go/arrow/array/interval.go @@ -22,7 +22,7 @@ import ( "sync/atomic" "github.com/apache/arrow/go/arrow" - "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/bitutil" "github.com/apache/arrow/go/arrow/internal/debug" "github.com/apache/arrow/go/arrow/memory" "github.com/pkg/errors" diff --git a/go/arrow/array/list.go b/go/arrow/array/list.go index 2fccdeadb945..711e8497f6da 100644 --- a/go/arrow/array/list.go +++ b/go/arrow/array/list.go @@ -22,7 +22,7 @@ import ( "sync/atomic" "github.com/apache/arrow/go/arrow" - "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/bitutil" "github.com/apache/arrow/go/arrow/internal/debug" "github.com/apache/arrow/go/arrow/memory" ) diff --git a/go/arrow/array/numericbuilder.gen.go b/go/arrow/array/numericbuilder.gen.go index b113908af67c..5423d8ce515c 100644 --- a/go/arrow/array/numericbuilder.gen.go +++ b/go/arrow/array/numericbuilder.gen.go @@ -22,7 +22,7 @@ import ( "sync/atomic" "github.com/apache/arrow/go/arrow" - "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/bitutil" "github.com/apache/arrow/go/arrow/internal/debug" "github.com/apache/arrow/go/arrow/memory" ) diff --git a/go/arrow/array/struct.go b/go/arrow/array/struct.go index b70d9539a488..fd2294e0e468 100644 --- a/go/arrow/array/struct.go +++ b/go/arrow/array/struct.go @@ -22,7 +22,7 @@ import ( "sync/atomic" "github.com/apache/arrow/go/arrow" - "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/bitutil" "github.com/apache/arrow/go/arrow/internal/debug" "github.com/apache/arrow/go/arrow/memory" ) diff --git a/go/arrow/internal/bitutil/bitutil.go b/go/arrow/bitutil/bitutil.go similarity index 100% rename from go/arrow/internal/bitutil/bitutil.go rename to go/arrow/bitutil/bitutil.go diff --git a/go/arrow/internal/bitutil/bitutil_test.go b/go/arrow/bitutil/bitutil_test.go similarity index 99% rename from go/arrow/internal/bitutil/bitutil_test.go rename to go/arrow/bitutil/bitutil_test.go index 62134dc64d45..fcb362f9daa2 100644 --- a/go/arrow/internal/bitutil/bitutil_test.go +++ b/go/arrow/bitutil/bitutil_test.go @@ -21,7 +21,7 @@ import ( "math/rand" "testing" - "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/bitutil" "github.com/apache/arrow/go/arrow/internal/testing/tools" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/ipc/file_reader.go b/go/arrow/ipc/file_reader.go index 2b08b01240cb..54213ceced1a 100644 --- a/go/arrow/ipc/file_reader.go +++ b/go/arrow/ipc/file_reader.go @@ -24,7 +24,7 @@ import ( "github.com/apache/arrow/go/arrow" "github.com/apache/arrow/go/arrow/array" - "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/bitutil" "github.com/apache/arrow/go/arrow/internal/flatbuf" "github.com/apache/arrow/go/arrow/memory" "github.com/pkg/errors" diff --git a/go/arrow/ipc/file_writer.go b/go/arrow/ipc/file_writer.go index 1fb6f529302e..8c6dec825c76 100644 --- a/go/arrow/ipc/file_writer.go +++ b/go/arrow/ipc/file_writer.go @@ -22,7 +22,7 @@ import ( "github.com/apache/arrow/go/arrow" "github.com/apache/arrow/go/arrow/array" - "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/bitutil" "github.com/apache/arrow/go/arrow/internal/flatbuf" "github.com/apache/arrow/go/arrow/memory" "github.com/pkg/errors" diff --git a/go/arrow/ipc/writer.go b/go/arrow/ipc/writer.go index 4ed0ff0b1ee1..3262083ce464 100644 --- a/go/arrow/ipc/writer.go +++ b/go/arrow/ipc/writer.go @@ -23,7 +23,7 @@ import ( "github.com/apache/arrow/go/arrow" "github.com/apache/arrow/go/arrow/array" - "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/bitutil" "github.com/apache/arrow/go/arrow/memory" "github.com/pkg/errors" ) diff --git a/go/arrow/type_traits_boolean.go b/go/arrow/type_traits_boolean.go index c30ae0fec4fe..a3a5c59257b1 100644 --- a/go/arrow/type_traits_boolean.go +++ b/go/arrow/type_traits_boolean.go @@ -17,7 +17,7 @@ package arrow import ( - "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/bitutil" ) type booleanTraits struct{} From 8231fcb428b2537ebaf635a56d11ae925f77d225 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Tue, 1 Oct 2019 11:34:50 -0500 Subject: [PATCH 11/24] ARROW-5831: [Release] Add Python program to download binary artifacts in parallel, allow abort/resume This script only uses the Python standard library and curl. It does 8 downloads in parallel by default. Since Bintray returns sha256 checksums we compute these on any local files and do not re-download files, so that interrupted downloads can be resumed. Closes #5550 from wesm/parallel-rc-binary-verification and squashes the following commits: ff207e670 More robust python3 checking 1d78b9f41 Add Python-based parallel bintray artifact download script that can resume Authored-by: Wes McKinney Signed-off-by: Wes McKinney --- dev/release/download_rc_binaries.py | 161 ++++++++++++++++++++++++ dev/release/verify-release-candidate.sh | 42 +------ 2 files changed, 164 insertions(+), 39 deletions(-) create mode 100644 dev/release/download_rc_binaries.py diff --git a/dev/release/download_rc_binaries.py b/dev/release/download_rc_binaries.py new file mode 100644 index 000000000000..01ab2c468eaf --- /dev/null +++ b/dev/release/download_rc_binaries.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import sys + +try: + import argparse + import concurrent.futures as cf + import functools + import hashlib + import json + import os + import subprocess + import urllib.request +except ImportError: + if sys.version_info.major < 3: + raise Exception("Please use Python 3 to run this script") + raise + + +BINTRAY_API_ROOT = "https://bintray.com/api/v1" +BINTRAY_DL_ROOT = "https://dl.bintray.com" +BINTRAY_REPO = 'apache/arrow' +DEFAULT_PARALLEL_DOWNLOADS = 8 + + +class Bintray: + + def __init__(self, repo=BINTRAY_REPO): + self.repo = repo + + def get_file_list(self, package, version): + url = os.path.join(BINTRAY_API_ROOT, 'packages', self.repo, package, + 'versions', version, 'files') + request = urllib.request.urlopen(url).read() + return json.loads(request) + + def download_files(self, files, dest=None, num_parallel=None): + """ + Download files from Bintray in parallel. If file already exists, will + overwrite if the checksum does not match what Bintray says it should be + + Parameters + ---------- + files : List[Dict] + File listing from Bintray + dest : str, default None + Defaults to current working directory + num_parallel : int, default 8 + Number of files to download in parallel. If set to None, uses + default + """ + if dest is None: + dest = os.getcwd() + if num_parallel is None: + num_parallel = DEFAULT_PARALLEL_DOWNLOADS + + if num_parallel == 1: + for path in files: + self._download_file(dest, path) + else: + parallel_map_terminate_early( + functools.partial(self._download_file, dest), + files, + num_parallel + ) + + def _download_file(self, dest, info): + relpath = info['path'] + + base, filename = os.path.split(relpath) + + dest_dir = os.path.join(dest, base) + os.makedirs(dest_dir, exist_ok=True) + + dest_path = os.path.join(dest_dir, filename) + + if os.path.exists(dest_path): + with open(dest_path, 'rb') as f: + sha256sum = hashlib.sha256(f.read()).hexdigest() + if sha256sum == info['sha256']: + print('Local file {} sha256 matches, skipping' + .format(dest_path)) + return + else: + print('Local file sha256 does not match, overwriting') + + print("Downloading {} to {}".format(relpath, dest_path)) + + bintray_abspath = os.path.join(BINTRAY_DL_ROOT, self.repo, relpath) + + cmd = [ + 'curl', '--fail', '--location', + '--output', dest_path, bintray_abspath + ] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, stderr = proc.communicate() + if proc.returncode != 0: + raise Exception("Downloading {} failed\nstdout: {}\nstderr: {}" + .format(relpath, stdout, stderr)) + + +def parallel_map_terminate_early(f, iterable, num_parallel): + tasks = [] + with cf.ProcessPoolExecutor(num_parallel) as pool: + for v in iterable: + tasks.append(pool.submit(functools.partial(f, v))) + + for task in cf.as_completed(tasks): + if task.exception() is not None: + e = task.exception() + for task in tasks: + task.cancel() + raise e + + +ARROW_PACKAGE_TYPES = ['centos', 'debian', 'python', 'ubuntu'] + + +def download_rc_binaries(version, rc_number, dest=None, num_parallel=None): + bintray = Bintray() + + version_string = '{}-rc{}'.format(version, rc_number) + for package_type in ARROW_PACKAGE_TYPES: + files = bintray.get_file_list('{}-rc'.format(package_type), + version_string) + bintray.download_files(files, dest=dest, num_parallel=num_parallel) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Download release candidate binaries' + ) + parser.add_argument('version', type=str, help='The version number') + parser.add_argument('rc_number', type=int, + help='The release candidate number, e.g. 0, 1, etc') + parser.add_argument('--dest', type=str, default=os.getcwd(), + help='The output folder for the downloaded files') + parser.add_argument('--num_parallel', type=int, default=8, + help='The number of concurrent downloads to do') + args = parser.parse_args() + + download_rc_binaries(args.version, args.rc_number, dest=args.dest, + num_parallel=args.num_parallel) diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index b1a5997de348..d09d13659eaf 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -99,49 +99,13 @@ fetch_archive() { shasum -a 512 -c ${dist_name}.tar.gz.sha512 } -bintray() { - local command=$1 - shift - local path=$1 - shift - local url=https://bintray.com/api/v1${path} - echo "${command} ${url}" 1>&2 - curl \ - --fail \ - --request ${command} \ - ${url} \ - "$@" | \ - jq . -} - -download_bintray_files() { - local target=$1 - - local version_name=${VERSION}-rc${RC_NUMBER} - - local file - bintray \ - GET /packages/${BINTRAY_REPOSITORY}/${target}-rc/versions/${version_name}/files | \ - jq -r ".[].path" | \ - while read file; do - mkdir -p "$(dirname ${file})" - curl \ - --fail \ - --location \ - --output ${file} \ - https://dl.bintray.com/${BINTRAY_REPOSITORY}/${file} - done -} - test_binary() { local download_dir=binaries mkdir -p ${download_dir} - pushd ${download_dir} - # takes longer on slow network - for target in centos debian python ubuntu; do - download_bintray_files ${target} - done + python3 $SOURCE_DIR/download_rc_binaries.py $VERSION $RC_NUMBER --dest=${download_dir} + + pushd ${download_dir} # verify the signature and the checksums of each artifact find . -name '*.asc' | while read sigfile; do From 871aedbad26e05911b844d92197f975533c599bf Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 1 Oct 2019 23:59:57 +0200 Subject: [PATCH 12/24] ARROW-6751: [CI] Fix ccache setup on Travis-CI Closes #5554 from pitrou/ARROW-6751-travis-ccache and squashes the following commits: cbc10f7ea Again 08857c57e Again 4ea772d5e ARROW-6751: Fix ccache setup on Travis-CI Authored-by: Antoine Pitrou Signed-off-by: Antoine Pitrou --- .travis.yml | 43 ++++++++++++++++++++++++++++--------- ci/docker_build_cpp.sh | 4 ++++ ci/travis/before-install.sh | 5 +++++ 3 files changed, 42 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 990fd2256674..534004d0d160 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,10 +26,6 @@ cache: - $HOME/.ccache - $HOME/.m2 # Maven -env: - - CCACHE_COMPRESS: 1 - - CCACHE_COMPRESSLEVEL: 5 - git: # ARROW-3083: We use a higher clone depth to also have the latest tag in the cloned history. depth: 250 @@ -40,6 +36,8 @@ before_install: matrix: fast_finish: true include: + + # ------------------------------------------------------------------------- - name: "Lint, Release tests" os: linux language: python @@ -60,6 +58,8 @@ matrix: - docker-compose build --pull cpp-system-deps script: - docker-compose run cpp-system-deps + + # ------------------------------------------------------------------------- # Separating Valgrind and C++ coverage makes individual jobs shorter - name: "C++ unit tests w/ conda-forge toolchain, coverage" compiler: gcc @@ -95,6 +95,8 @@ matrix: - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1 - $TRAVIS_BUILD_DIR/ci/travis_script_gandiva_java.sh || travis_terminate 1 - $TRAVIS_BUILD_DIR/ci/travis_upload_cpp_coverage.sh || travis_terminate 1 + + # ------------------------------------------------------------------------- - name: "Python 3.6 unit tests w/ Valgrind, conda-forge toolchain, coverage" compiler: gcc language: cpp @@ -128,11 +130,12 @@ matrix: - export PLASMA_VALGRIND=1 - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6 - $TRAVIS_BUILD_DIR/ci/travis_upload_cpp_coverage.sh + + # ------------------------------------------------------------------------- - name: "[OS X] C++ w/ XCode 9.3" compiler: clang osx_image: xcode9.3 os: osx - cache: addons: env: - ARROW_TRAVIS_USE_TOOLCHAIN=1 @@ -159,11 +162,12 @@ matrix: - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1 # Disabled because of ARROW-4763 # - $TRAVIS_BUILD_DIR/ci/travis_script_gandiva_java.sh + + # ------------------------------------------------------------------------- - name: "[OS X] Python w/ XCode 9.3" compiler: clang osx_image: xcode9.3 os: osx - cache: addons: env: - ARROW_TRAVIS_S3=1 @@ -178,6 +182,8 @@ matrix: - if [ $ARROW_CI_PYTHON_AFFECTED != "1" ]; then exit; fi - $TRAVIS_BUILD_DIR/ci/travis_install_minio.sh - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6 + + # ------------------------------------------------------------------------- - name: "Java OpenJDK8 and OpenJDK11" language: cpp os: linux @@ -186,6 +192,8 @@ matrix: - docker-compose build --pull java-all-jdks script: - docker-compose run java-all-jdks + + # ------------------------------------------------------------------------- - name: "Integration w/ OpenJDK 8, conda-forge toolchain" language: java os: linux @@ -213,6 +221,8 @@ matrix: script: - $TRAVIS_BUILD_DIR/ci/travis_script_integration.sh # - $TRAVIS_BUILD_DIR/ci/travis_script_plasma_java_client.sh + + # ------------------------------------------------------------------------- - name: "NodeJS" language: node_js os: linux @@ -224,6 +234,8 @@ matrix: - $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh script: - $TRAVIS_BUILD_DIR/ci/travis_script_js.sh + + # ------------------------------------------------------------------------- - name: "C++ & GLib & Ruby w/ gcc 5.4" compiler: gcc os: linux @@ -243,6 +255,8 @@ matrix: script: - if [ $ARROW_CI_C_GLIB_AFFECTED = "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh; fi - $TRAVIS_BUILD_DIR/ci/travis_script_ruby.sh + + # ------------------------------------------------------------------------- - name: "[OS X] C++ & GLib & Ruby w/ XCode 10.2 & Homebrew" compiler: clang osx_image: xcode10.2 @@ -256,7 +270,6 @@ matrix: - ARROW_TRAVIS_PYTHON=1 - ARROW_TRAVIS_VERBOSE=1 - ARROW_TRAVIS_MIMALLOC=1 - cache: addons: before_script: - if [ $ARROW_CI_RUBY_AFFECTED != "1" ]; then exit; fi @@ -267,6 +280,8 @@ matrix: script: - if [ $ARROW_CI_C_GLIB_AFFECTED = "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh; fi - $TRAVIS_BUILD_DIR/ci/travis_script_ruby.sh + + # ------------------------------------------------------------------------- - name: Rust language: rust cache: cargo @@ -293,6 +308,8 @@ matrix: - export PARQUET_TEST_DATA=$TRAVIS_BUILD_DIR/cpp/submodules/parquet-testing/data - cargo tarpaulin --out Xml - bash <(curl -s https://codecov.io/bash) || echo "Codecov did not collect coverage reports" + + # ------------------------------------------------------------------------- - name: Go language: go go_import_path: github.com/apache/arrow @@ -306,9 +323,14 @@ matrix: after_success: - pushd ${TRAVIS_BUILD_DIR}/go/arrow - bash <(curl -s https://codecov.io/bash) || echo "Codecov did not collect coverage reports" + + # ------------------------------------------------------------------------- - name: R (with and without libarrow) language: r - cache: packages + cache: + directories: + - $HOME/.ccache + packages: true dist: xenial env: - ARROW_TRAVIS_PARQUET=1 @@ -318,6 +340,7 @@ matrix: # Have to copy-paste this here because of how R's build steps work - eval `python $TRAVIS_BUILD_DIR/ci/detect-changes.py` - if [ $ARROW_CI_R_AFFECTED != "1" ]; then exit; fi + - source ci/travis/before-install.sh # First check that it builds without libarrow - pushd ${TRAVIS_BUILD_DIR}/r - R -e 'install.packages("remotes"); remotes::install_deps(dep = TRUE)' @@ -348,10 +371,10 @@ matrix: - Rscript ../ci/travis_upload_r_coverage.R after_failure: - - source ci/travis/after-failure.sh + - source ${TRAVIS_BUILD_DIR}/ci/travis/after-failure.sh after_script: - - source ci/travis/after-script.sh + - source ${TRAVIS_BUILD_DIR}/ci/travis/after-script.sh env: global: diff --git a/ci/docker_build_cpp.sh b/ci/docker_build_cpp.sh index 29860bf7199a..8eb96a21bfd5 100755 --- a/ci/docker_build_cpp.sh +++ b/ci/docker_build_cpp.sh @@ -22,7 +22,11 @@ source_dir=${1:-/arrow/cpp} build_dir=${2:-/build/cpp} install_dir=${3:-${ARROW_HOME:-/usr/local}} +export CCACHE_COMPRESS=1 +export CCACHE_COMPRESSLEVEL=5 +export CCACHE_COMPILERCHECK=content export CCACHE_DIR=/build/ccache +export PATH=/usr/lib/ccache/:$PATH rm -rf ${build_dir} mkdir -p ${build_dir} diff --git a/ci/travis/before-install.sh b/ci/travis/before-install.sh index fd97b469341c..1eb45fd436b5 100755 --- a/ci/travis/before-install.sh +++ b/ci/travis/before-install.sh @@ -30,6 +30,11 @@ if [[ "${TRAVIS_OS_NAME}" == "linux" ]]; then echo -e 'Acquire::Retries 10; Acquire::http::Timeout \"20\";' | \ sudo tee /etc/apt/apt.conf.d/99-travis-retry sudo apt-get update -qq + + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=5 + export CCACHE_COMPILERCHECK=content + export PATH=/usr/lib/ccache/:$PATH ccache --show-stats fi From 9694200d2760f320cb662c069360f810eac86416 Mon Sep 17 00:00:00 2001 From: Bruce Mitchener Date: Tue, 1 Oct 2019 19:38:21 -0400 Subject: [PATCH 13/24] ARROW-6745: [Rust] Fix a variety of minor typos. Closes #5548 from waywardmonkeys/fix-typos and squashes the following commits: 0ee704fbb ARROW-6745: Fix a variety of minor typos. Authored-by: Bruce Mitchener Signed-off-by: Paddy Horan --- rust/README.md | 2 +- rust/arrow/README.md | 2 +- rust/arrow/src/compute/kernels/take.rs | 6 +++--- rust/arrow/src/csv/reader.rs | 2 +- rust/arrow/src/datatypes.rs | 2 +- rust/datafusion/README.md | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/rust/README.md b/rust/README.md index 377cf6c6fef7..7b712511c14b 100644 --- a/rust/README.md +++ b/rust/README.md @@ -29,7 +29,7 @@ ## Prerequisites -Before running tests and examples it is necessary to set up the local development enviroment. +Before running tests and examples it is necessary to set up the local development environment. ### Git Submodules diff --git a/rust/arrow/README.md b/rust/arrow/README.md index 4bc6b062648e..7dffc3432024 100644 --- a/rust/arrow/README.md +++ b/rust/arrow/README.md @@ -99,7 +99,7 @@ cargo publish If the Cargo.toml does not have the correct version then it will be necessary to modify it manually. Since there is now a modified file locally that is not -committed to github it will be necessary to use the following command. +committed to GitHub it will be necessary to use the following command. ```bash cargo publish --allow-dirty diff --git a/rust/arrow/src/compute/kernels/take.rs b/rust/arrow/src/compute/kernels/take.rs index 6cce7fb47d95..b591982c56e5 100644 --- a/rust/arrow/src/compute/kernels/take.rs +++ b/rust/arrow/src/compute/kernels/take.rs @@ -359,7 +359,7 @@ mod tests { let a = take(&list_array, &index, None).unwrap(); let a: &ListArray = a.as_any().downcast_ref::().unwrap(); - // construct a value aray with expected results: + // construct a value array with expected results: // [[2,3], null, [-1,-2,-1], [2,3], [0,0,0]] let expected_data = Int32Array::from(vec![ Some(2), @@ -424,7 +424,7 @@ mod tests { let a = take(&list_array, &index, None).unwrap(); let a: &ListArray = a.as_any().downcast_ref::().unwrap(); - // construct a value aray with expected results: + // construct a value array with expected results: // [[null], null, [-1,-2,3], [5,null], [0,null,0]] let expected_data = Int32Array::from(vec![ None, @@ -487,7 +487,7 @@ mod tests { let a = take(&list_array, &index, None).unwrap(); let a: &ListArray = a.as_any().downcast_ref::().unwrap(); - // construct a value aray with expected results: + // construct a value array with expected results: // [null, null, [-1,-2,3], [5,null], [0,null,0]] let expected_data = Int32Array::from(vec![ Some(-1), diff --git a/rust/arrow/src/csv/reader.rs b/rust/arrow/src/csv/reader.rs index a031e199beff..15e0d33ae6d3 100644 --- a/rust/arrow/src/csv/reader.rs +++ b/rust/arrow/src/csv/reader.rs @@ -405,7 +405,7 @@ pub struct ReaderBuilder { /// If schema inference is run on a file with no headers, default column names /// are created. has_headers: bool, - /// An optional column delimiter. Defauits to `b','` + /// An optional column delimiter. Defaults to `b','` delimiter: Option, /// Optional maximum number of records to read during schema inference /// diff --git a/rust/arrow/src/datatypes.rs b/rust/arrow/src/datatypes.rs index 5698ccbc1bd2..40e0153e81d8 100644 --- a/rust/arrow/src/datatypes.rs +++ b/rust/arrow/src/datatypes.rs @@ -785,7 +785,7 @@ impl Field { }) } - /// Converts to a `String` representation of the the `Field` + /// Converts to a `String` representation of the `Field` pub fn to_string(&self) -> String { format!("{}: {:?}", self.name, self.data_type) } diff --git a/rust/datafusion/README.md b/rust/datafusion/README.md index 7058a3b1b433..f3dc512bb032 100644 --- a/rust/datafusion/README.md +++ b/rust/datafusion/README.md @@ -35,7 +35,7 @@ datafusion = "0.15.0-SNAPSHOT" #### Use as a bin ##### Build your own bin(requires rust toolchains) ```sh -git clone https://github/apache/arrow +git clone https://github.com/apache/arrow cd arrow/rust/datafusion cargo run --bin datafusion-cli ``` From d75d18674c9cf6066e30c0f82ba5c37d186d5184 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Saint-Jacques?= Date: Wed, 2 Oct 2019 15:36:16 +0900 Subject: [PATCH 14/24] ARROW-6730: [CI] Use GitHub Actions for "C++ with clang 7" docker image MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #5530 from fsaintjacques/github-action-docker and squashes the following commits: 4bb16ce1a Add GitHub Actions 648f2f093 Use expression c849d7719 Run all tests for push 473c1c2b7 GitHub 99a874846 Try github action Lead-authored-by: François Saint-Jacques Co-authored-by: Sutou Kouhei Signed-off-by: Sutou Kouhei --- .github/workflows/linux-docker-compose.yml | 47 ++++++++++++++++++ ci/detect-changes.py | 55 +++++++++++++++++++++- cpp/Dockerfile.ubuntu-bionic | 10 ++-- docker-compose.yml | 1 - 4 files changed, 106 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/linux-docker-compose.yml diff --git a/.github/workflows/linux-docker-compose.yml b/.github/workflows/linux-docker-compose.yml new file mode 100644 index 000000000000..d1198572b554 --- /dev/null +++ b/.github/workflows/linux-docker-compose.yml @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Linux docker-compose +on: + - push + - pull_request +jobs: + build: + name: Test + strategy: + matrix: + label: + - C++ w/ clang-7 & system packages + include: + - label: C++ w/ clang-7 & system packages + image: cpp-system-deps + skip_expression: | + ${ARROW_CI_CPP_AFFECTED} != "1" + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@master + with: + submodules: true + - name: docker-compose + run: | + eval "$(python ci/detect-changes.py)" + if [[ ${{ matrix.skip_expression }} ]]; then + exit + fi + + docker-compose build --pull ${{ matrix.image }} + docker-compose run ${{ matrix.image }} diff --git a/ci/detect-changes.py b/ci/detect-changes.py index d35d9b570b9a..424cd99d8663 100644 --- a/ci/detect-changes.py +++ b/ci/detect-changes.py @@ -129,6 +129,16 @@ def list_appveyor_affected_files(): return list_affected_files("{0}..HEAD".format(merge_base)) +def list_github_actions_affected_files(): + """ + Return a list of files affected in the current GitHub Actions build. + """ + # GitHub Actions checkout `refs/remotes/pull/$PR/merge` where `HEAD` points + # to the merge commit while `HEAD^` points to the commit before. Hence, + # `..HEAD^` points to all commit between master and the PR. + return list_affected_files("HEAD^..") + + LANGUAGE_TOPICS = ['c_glib', 'cpp', 'docs', 'go', 'java', 'js', 'python', 'r', 'ruby', 'rust', 'csharp'] @@ -143,6 +153,9 @@ def list_appveyor_affected_files(): 'format': LANGUAGE_TOPICS, 'go': ['integration'], '.travis.yml': ALL_TOPICS, + # In theory, it should ignore CONTRIBUTING.md and ISSUE_TEMPLATE.md, but in + # practice it's going to be CI + '.github': ALL_TOPICS, 'c_glib': ['ruby'] } @@ -255,6 +268,23 @@ def run_from_appveyor(): return get_windows_shell_eval(make_env_for_topics(affected)) +def run_from_github(): + perr("Environment variables (excerpt):") + dump_env_vars('GITHUB_', '(REPOSITORY|ACTOR|SHA|REF|HEAD_REF|BASE_REF|EVENT_NAME)') + if os.environ['GITHUB_EVENT_NAME'] != 'pull_request': + # Not a PR build, test everything + affected = dict.fromkeys(ALL_TOPICS, True) + else: + affected_files = list_github_actions_affected_files() + perr("Affected files:", affected_files) + affected = get_affected_topics(affected_files) + assert set(affected) <= set(ALL_TOPICS), affected + + perr("Affected topics:") + perr(pprint.pformat(affected)) + return get_unix_shell_eval(make_env_for_topics(affected)) + + def test_get_affected_topics(): affected_topics = get_affected_topics(['cpp/CMakeLists.txt']) assert affected_topics == { @@ -290,6 +320,23 @@ def test_get_affected_topics(): 'dev': False } + affected_topics = get_affected_topics(['.github/workflows']) + assert affected_topics == { + 'c_glib': True, + 'cpp': True, + 'docs': True, + 'go': True, + 'java': True, + 'js': True, + 'python': True, + 'r': True, + 'ruby': True, + 'rust': True, + 'csharp': True, + 'integration': True, + 'dev': True, + } + if __name__ == "__main__": # This script should have its output evaluated by a shell, @@ -307,5 +354,11 @@ def test_get_affected_topics(): except Exception: print("exit 1") raise + elif os.environ.get('GITHUB_WORKFLOW'): + try: + print(run_from_github()) + except Exception: + print("exit 1") + raise else: - sys.exit("Script must be run under Travis-CI or AppVeyor") + sys.exit("Script must be run under Travis-CI, AppVeyor or GitHub Actions") diff --git a/cpp/Dockerfile.ubuntu-bionic b/cpp/Dockerfile.ubuntu-bionic index 763e13e50c87..325ebb7ee76b 100644 --- a/cpp/Dockerfile.ubuntu-bionic +++ b/cpp/Dockerfile.ubuntu-bionic @@ -71,12 +71,15 @@ RUN apt-get update -y -q && \ libgoogle-glog-dev \ liblz4-dev \ liblzma-dev \ + libprotobuf-dev \ + libprotoc-dev \ libre2-dev \ libsnappy-dev \ libssl-dev \ libzstd-dev \ ninja-build \ pkg-config \ + protobuf-compiler \ rapidjson-dev \ thrift-compiler \ tzdata && \ @@ -92,14 +95,11 @@ RUN apt-get update -y -q && \ ENV CMAKE_ARGS="-DThrift_SOURCE=BUNDLED \ -DFlatbuffers_SOURCE=BUNDLED \ -DGTest_SOURCE=BUNDLED \ --DORC_SOURCE=BUNDLED \ --Dc-ares_SOURCE=BUNDLED \ --DgRPC_SOURCE=BUNDLED \ --DProtobuf_SOURCE=BUNDLED ${CMAKE_ARGS}" +-DORC_SOURCE=BUNDLED" # Prioritize system packages and local installation ENV ARROW_DEPENDENCY_SOURCE=SYSTEM \ - ARROW_FLIGHT=ON \ + ARROW_FLIGHT=OFF \ ARROW_GANDIVA=ON \ ARROW_HDFS=ON \ ARROW_ORC=ON \ diff --git a/docker-compose.yml b/docker-compose.yml index a5cb6061f617..0d0f6f9fc6bc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -131,7 +131,6 @@ services: args: LLVM_VERSION: 7 environment: - ARROW_FLIGHT: "ON" ARROW_USE_ASAN: "ON" ARROW_USE_UBSAN: "ON" volumes: From e72a0da1498fad739c630b51449d76e82887a54e Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Wed, 2 Oct 2019 10:32:42 +0200 Subject: [PATCH 15/24] =?UTF-8?q?ARROW-6752:=20[Go]=20make=20Null=20array?= =?UTF-8?q?=20implement=20Stringer,=20add=20tests=20for=20Nu=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …ll arrays Closes #5555 from sbinet/issue-6752 and squashes the following commits: 18b5387cc ARROW-6752: make Null array implement Stringer, add tests for Null arrays Authored-by: Sebastien Binet Signed-off-by: Sebastien Binet --- go/arrow/array/compare_test.go | 14 ++++---- go/arrow/array/null.go | 14 ++++++++ go/arrow/internal/arrdata/arrdata.go | 52 ++++++++++++++++++++++++++++ go/arrow/internal/arrjson/arrjson.go | 13 +++++++ 4 files changed, 86 insertions(+), 7 deletions(-) diff --git a/go/arrow/array/compare_test.go b/go/arrow/array/compare_test.go index e9927f09f308..3ed326be1758 100644 --- a/go/arrow/array/compare_test.go +++ b/go/arrow/array/compare_test.go @@ -44,7 +44,7 @@ func TestArrayEqual(t *testing.T) { sub2 := array.NewSlice(arr, 0, int64(arr.Len()-1)) defer sub2.Release() - if array.ArrayEqual(sub1, sub2) { + if array.ArrayEqual(sub1, sub2) && name != "nulls" { t.Fatalf("non-identical arrays should not compare equal:\nsub1=%v\nsub2=%v\narrf=%v\n", sub1, sub2, arr) } }) @@ -73,7 +73,7 @@ func TestArraySliceEqual(t *testing.T) { sub2 := array.NewSlice(arr, 0, int64(arr.Len()-1)) defer sub2.Release() - if array.ArraySliceEqual(sub1, 0, int64(sub1.Len()), sub2, 0, int64(sub2.Len())) { + if array.ArraySliceEqual(sub1, 0, int64(sub1.Len()), sub2, 0, int64(sub2.Len())) && name != "nulls" { t.Fatalf("non-identical slices should not compare equal:\nsub1=%v\nsub2=%v\narrf=%v\n", sub1, sub2, arr) } }) @@ -99,7 +99,7 @@ func TestArrayApproxEqual(t *testing.T) { sub2 := array.NewSlice(arr, 0, int64(arr.Len()-1)) defer sub2.Release() - if array.ArrayApproxEqual(sub1, sub2) { + if array.ArrayApproxEqual(sub1, sub2) && name != "nulls" { t.Fatalf("non-identical arrays should not compare equal:\nsub1=%v\nsub2=%v\narrf=%v\n", sub1, sub2, arr) } }) @@ -489,7 +489,7 @@ func TestRecordEqual(t *testing.T) { t.Fatalf("identical records should compare equal:\nrecord:\n%v", rec0) } - if array.RecordEqual(rec0, rec1) { + if array.RecordEqual(rec0, rec1) && name != "nulls" { t.Fatalf("non-identical records should not compare equal:\nrec0:\n%v\nrec1:\n%v", rec0, rec1) } @@ -498,7 +498,7 @@ func TestRecordEqual(t *testing.T) { sub01 := rec0.NewSlice(1, recs[0].NumRows()) defer sub01.Release() - if array.RecordEqual(sub00, sub01) { + if array.RecordEqual(sub00, sub01) && name != "nulls" { t.Fatalf("non-identical records should not compare equal:\nsub0:\n%v\nsub1:\n%v", sub00, sub01) } }) @@ -514,7 +514,7 @@ func TestRecordApproxEqual(t *testing.T) { t.Fatalf("identical records should compare equal:\nrecord:\n%v", rec0) } - if array.RecordApproxEqual(rec0, rec1) { + if array.RecordApproxEqual(rec0, rec1) && name != "nulls" { t.Fatalf("non-identical records should not compare equal:\nrec0:\n%v\nrec1:\n%v", rec0, rec1) } @@ -523,7 +523,7 @@ func TestRecordApproxEqual(t *testing.T) { sub01 := rec0.NewSlice(1, recs[0].NumRows()) defer sub01.Release() - if array.RecordApproxEqual(sub00, sub01) { + if array.RecordApproxEqual(sub00, sub01) && name != "nulls" { t.Fatalf("non-identical records should not compare equal:\nsub0:\n%v\nsub1:\n%v", sub00, sub01) } }) diff --git a/go/arrow/array/null.go b/go/arrow/array/null.go index 346445d5bec9..56d7fa45fa18 100644 --- a/go/arrow/array/null.go +++ b/go/arrow/array/null.go @@ -17,6 +17,7 @@ package array import ( + "strings" "sync/atomic" "github.com/apache/arrow/go/arrow" @@ -53,6 +54,19 @@ func NewNullData(data *Data) *Null { return a } +func (a *Null) String() string { + o := new(strings.Builder) + o.WriteString("[") + for i := 0; i < a.Len(); i++ { + if i > 0 { + o.WriteString(" ") + } + o.WriteString("(null)") + } + o.WriteString("]") + return o.String() +} + func (a *Null) setData(data *Data) { a.array.setData(data) a.array.nullBitmapBytes = nil diff --git a/go/arrow/internal/arrdata/arrdata.go b/go/arrow/internal/arrdata/arrdata.go index c24923631fa3..40dee3b54fd6 100644 --- a/go/arrow/internal/arrdata/arrdata.go +++ b/go/arrow/internal/arrdata/arrdata.go @@ -34,6 +34,7 @@ var ( ) func init() { + Records["nulls"] = makeNullRecords() Records["primitives"] = makePrimitiveRecords() Records["structs"] = makeStructsRecords() Records["lists"] = makeListsRecords() @@ -51,6 +52,49 @@ func init() { sort.Strings(RecordNames) } +func makeNullRecords() []array.Record { + mem := memory.NewGoAllocator() + + meta := arrow.NewMetadata( + []string{"k1", "k2", "k3"}, + []string{"v1", "v2", "v3"}, + ) + + schema := arrow.NewSchema( + []arrow.Field{ + arrow.Field{Name: "nulls", Type: arrow.Null, Nullable: true}, + }, &meta, + ) + + mask := []bool{true, false, false, true, true} + chunks := [][]array.Interface{ + []array.Interface{ + arrayOf(mem, []nullT{null, null, null, null, null}, mask), + }, + []array.Interface{ + arrayOf(mem, []nullT{null, null, null, null, null}, mask), + }, + []array.Interface{ + arrayOf(mem, []nullT{null, null, null, null, null}, mask), + }, + } + + defer func() { + for _, chunk := range chunks { + for _, col := range chunk { + col.Release() + } + } + }() + + recs := make([]array.Record, len(chunks)) + for i, chunk := range chunks { + recs[i] = array.NewRecord(schema, chunk, -1) + } + + return recs +} + func makePrimitiveRecords() []array.Record { mem := memory.NewGoAllocator() @@ -362,6 +406,7 @@ func makeStringsRecords() []array.Record { } type ( + nullT struct{} time32s arrow.Time32 time32ms arrow.Time32 time64ns arrow.Time64 @@ -372,6 +417,10 @@ type ( timestamp_ns arrow.Timestamp ) +var ( + null nullT +) + func makeFixedWidthTypesRecords() []array.Record { mem := memory.NewGoAllocator() schema := arrow.NewSchema( @@ -650,6 +699,9 @@ func arrayOf(mem memory.Allocator, a interface{}, valids []bool) array.Interface } switch a := a.(type) { + case []nullT: + return array.NewNull(len(a)) + case []bool: bldr := array.NewBooleanBuilder(mem) defer bldr.Release() diff --git a/go/arrow/internal/arrjson/arrjson.go b/go/arrow/internal/arrjson/arrjson.go index 11cc3d0f2a1f..2b2bcaf92599 100644 --- a/go/arrow/internal/arrjson/arrjson.go +++ b/go/arrow/internal/arrjson/arrjson.go @@ -65,6 +65,8 @@ type dataType struct { func dtypeToJSON(dt arrow.DataType) dataType { switch dt := dt.(type) { + case *arrow.NullType: + return dataType{Name: "null"} case *arrow.BooleanType: return dataType{Name: "bool"} case *arrow.Int8Type: @@ -155,6 +157,8 @@ func dtypeToJSON(dt arrow.DataType) dataType { func dtypeFromJSON(dt dataType, children []Field) arrow.DataType { switch dt.Name { + case "null": + return arrow.Null case "bool": return arrow.FixedWidthTypes.Boolean case "int": @@ -364,6 +368,9 @@ func arraysToJSON(schema *arrow.Schema, arrs []array.Interface) []Array { func arrayFromJSON(mem memory.Allocator, dt arrow.DataType, arr Array) array.Interface { switch dt := dt.(type) { + case *arrow.NullType: + return array.NewNull(arr.Count) + case *arrow.BooleanType: bldr := array.NewBooleanBuilder(mem) defer bldr.Release() @@ -618,6 +625,12 @@ func arrayFromJSON(mem memory.Allocator, dt arrow.DataType, arr Array) array.Int func arrayToJSON(field arrow.Field, arr array.Interface) Array { switch arr := arr.(type) { + case *array.Null: + return Array{ + Name: field.Name, + Count: arr.Len(), + } + case *array.Boolean: return Array{ Name: field.Name, From 7d18c1cc307558423da144f31e65983b846e0f10 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Wed, 2 Oct 2019 13:37:29 +0200 Subject: [PATCH 16/24] ARROW-6750: [Python] Silence S3 error logs by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #5553 from pitrou/ARROW-6750-python-silence-s3-error-logs and squashes the following commits: 33edb9dd2 ARROW-6750: Silence S3 error logs by default Authored-by: Antoine Pitrou Signed-off-by: Krisztián Szűcs --- python/pyarrow/_s3fs.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyarrow/_s3fs.pyx b/python/pyarrow/_s3fs.pyx index d1f820e51958..96b78f8b0a73 100644 --- a/python/pyarrow/_s3fs.pyx +++ b/python/pyarrow/_s3fs.pyx @@ -37,7 +37,7 @@ cpdef enum S3LogLevel: Trace = CS3LogLevel_Trace -def initialize_s3(S3LogLevel log_level=S3LogLevel.Error): +def initialize_s3(S3LogLevel log_level=S3LogLevel.Fatal): cdef CS3GlobalOptions options options.log_level = log_level check_status(CInitializeS3(options)) From 5f93f85ebd941f8b11e4633fa83b04447b31e2af Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Wed, 2 Oct 2019 16:15:37 +0200 Subject: [PATCH 17/24] ARROW-6755: [Release] Improve Windows release verification script * Improve performance by only building shared libraries * Use `--no-shortcuts` conda option to avoid annoying popups * Use ninja-build for better performance Closes #5559 from wesm/windows-rc-script-improvements and squashes the following commits: b0d3092b5 Do not build static libraries, avoid annoying UAC popup Authored-by: Wes McKinney Signed-off-by: Antoine Pitrou --- dev/release/verify-release-candidate.bat | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/dev/release/verify-release-candidate.bat b/dev/release/verify-release-candidate.bat index 299297e95c54..cfbf56502ab8 100644 --- a/dev/release/verify-release-candidate.bat +++ b/dev/release/verify-release-candidate.bat @@ -41,11 +41,14 @@ set PYTHON=3.6 @rem Using call with conda.bat seems necessary to avoid terminating the batch @rem script execution -call conda create -p %_VERIFICATION_CONDA_ENV% -f -q -y python=%PYTHON% || exit /B +call conda create -p %_VERIFICATION_CONDA_ENV% ^ + --no-shortcuts -f -q -y python=%PYTHON% ^ + || exit /B call activate %_VERIFICATION_CONDA_ENV% || exit /B call conda install -y ^ + --no-shortcuts ^ python=3.7 ^ git ^ --file=ci\conda_env_cpp.yml ^ @@ -68,8 +71,9 @@ pushd %ARROW_SOURCE%\cpp\build @rem This is the path for Visual Studio Community 2017 call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\Tools\VsDevCmd.bat" -arch=amd64 -cmake -G "%GENERATOR%" ^ +cmake -G "Ninja" ^ -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^ + -DARROW_BUILD_STATIC=OFF ^ -DARROW_BOOST_USE_SHARED=ON ^ -DARROW_BUILD_TESTS=ON ^ -DGTest_SOURCE=BUNDLED ^ @@ -79,7 +83,12 @@ cmake -G "%GENERATOR%" ^ -DARROW_PYTHON=ON ^ -DARROW_PARQUET=ON ^ .. || exit /B -cmake --build . --target INSTALL --config %CONFIGURATION% || exit /B + +@rem NOTE(wesm): Building googletest is flaky for me with ninja. Building it +@rem first fixes the problem +ninja googletest_ep || exit /B + +ninja install || exit /B @rem Get testing datasets for Parquet unit tests git clone https://github.com/apache/parquet-testing.git %_VERIFICATION_DIR%\parquet-testing From b70f04a7abf5b7c74724faa9d834ebd0daf3f745 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Saint-Jacques?= Date: Wed, 2 Oct 2019 13:17:22 -0400 Subject: [PATCH 18/24] ARROW-6614: [C++][Dataset] Add DataSourceDiscovery class MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DataSourceDiscovery is a factory-like interface to build DataSource. The class exists for the purpose of unifying/discovery schema of datasource before materializing it (since all DataSources in a Dataset must have a matching schema). - Add MockFileSystem::Make - Refactor FileSystemBaseDataSource to use fs::PathTree and support partition pruning. Closes #5529 from fsaintjacques/ARROW-6614-file-discovery and squashes the following commits: 31736072e Address comments 81d97aa90 Draft DataSourceDiscovery interface 38c2c1cc7 Add Schema inspection to FileFormat 802836d28 Refactor FileSystemBasedDataSource to use PathTree 0d739e915 Add MockFileSystem::Make utility Authored-by: François Saint-Jacques Signed-off-by: Benjamin Kietzman --- cpp/src/arrow/dataset/CMakeLists.txt | 8 +- cpp/src/arrow/dataset/dataset.cc | 22 +-- cpp/src/arrow/dataset/dataset.h | 9 - cpp/src/arrow/dataset/discovery.cc | 91 ++++++++++ cpp/src/arrow/dataset/discovery.h | 69 +++++++- cpp/src/arrow/dataset/discovery_test.cc | 84 ++++++++++ cpp/src/arrow/dataset/file_base.cc | 131 ++++++++------- cpp/src/arrow/dataset/file_base.h | 61 ++++--- cpp/src/arrow/dataset/file_parquet.cc | 28 +++- cpp/src/arrow/dataset/file_parquet.h | 11 ++ cpp/src/arrow/dataset/file_parquet_test.cc | 27 ++- cpp/src/arrow/dataset/file_test.cc | 109 ++++++++---- cpp/src/arrow/dataset/filter.cc | 38 ++--- cpp/src/arrow/dataset/filter.h | 3 + cpp/src/arrow/dataset/scanner.h | 6 +- cpp/src/arrow/dataset/test_util.h | 177 ++++++++------------ cpp/src/arrow/filesystem/filesystem_test.cc | 13 ++ cpp/src/arrow/filesystem/mockfs.cc | 34 ++++ cpp/src/arrow/filesystem/mockfs.h | 9 + cpp/src/arrow/type.h | 1 + 20 files changed, 638 insertions(+), 293 deletions(-) create mode 100644 cpp/src/arrow/dataset/discovery.cc create mode 100644 cpp/src/arrow/dataset/discovery_test.cc diff --git a/cpp/src/arrow/dataset/CMakeLists.txt b/cpp/src/arrow/dataset/CMakeLists.txt index 6f94297e272c..442e9e6587bc 100644 --- a/cpp/src/arrow/dataset/CMakeLists.txt +++ b/cpp/src/arrow/dataset/CMakeLists.txt @@ -23,7 +23,12 @@ arrow_install_all_headers("arrow/dataset") # pkg-config support arrow_add_pkg_config("arrow-dataset") -set(ARROW_DATASET_SRCS dataset.cc file_base.cc filter.cc scanner.cc) +set(ARROW_DATASET_SRCS + dataset.cc + discovery.cc + file_base.cc + filter.cc + scanner.cc) set(ARROW_DATASET_LINK_STATIC arrow_static) set(ARROW_DATASET_LINK_SHARED arrow_shared) @@ -91,6 +96,7 @@ endfunction() if(NOT WIN32) add_arrow_dataset_test(dataset_test) + add_arrow_dataset_test(discovery_test) add_arrow_dataset_test(file_test) add_arrow_dataset_test(filter_test) add_arrow_dataset_test(scanner_test) diff --git a/cpp/src/arrow/dataset/dataset.cc b/cpp/src/arrow/dataset/dataset.cc index e3b81977e2b6..34b01a27c3e1 100644 --- a/cpp/src/arrow/dataset/dataset.cc +++ b/cpp/src/arrow/dataset/dataset.cc @@ -65,26 +65,28 @@ Status Dataset::NewScan(std::unique_ptr* out) { bool DataSource::AssumePartitionExpression( const std::shared_ptr& scan_options, std::shared_ptr* simplified_scan_options) const { - DCHECK_NE(simplified_scan_options, nullptr); - if (scan_options == nullptr) { - // null scan options; no selector to simplify - *simplified_scan_options = scan_options; + auto filter = scan_options->filter; + if (filter == nullptr || partition_expression_ == nullptr) { + if (simplified_scan_options != nullptr) { + *simplified_scan_options = scan_options; + } return true; } - auto c = SelectorAssume(scan_options->selector, partition_expression_); + auto c = filter->Assume(*partition_expression_); DCHECK_OK(c.status()); auto expr = std::move(c).ValueOrDie(); - bool trivial = true; - if (expr->IsNull() || (expr->IsTrivialCondition(&trivial) && !trivial)) { + if (expr->IsNull() || expr->IsTrivialFalseCondition()) { // selector is not satisfiable; yield no fragments return false; } - auto copy = std::make_shared(*scan_options); - copy->selector = ExpressionSelector(std::move(expr)); - *simplified_scan_options = std::move(copy); + if (simplified_scan_options != nullptr) { + auto copy = std::make_shared(*scan_options); + copy->filter = std::move(expr); + *simplified_scan_options = std::move(copy); + } return true; } diff --git a/cpp/src/arrow/dataset/dataset.h b/cpp/src/arrow/dataset/dataset.h index b6f7ca7106a7..7ff580706861 100644 --- a/cpp/src/arrow/dataset/dataset.h +++ b/cpp/src/arrow/dataset/dataset.h @@ -148,14 +148,5 @@ class ARROW_DS_EXPORT Dataset : public std::enable_shared_from_this { std::vector> sources_; }; -/// \brief Conditions to apply to a dataset when reading to include or -/// exclude fragments, filter out rows, etc. -struct DataSelector { - std::vector> filters; - - // TODO(wesm): Select specific partition keys, file path globs, or - // other common desirable selections -}; - } // namespace dataset } // namespace arrow diff --git a/cpp/src/arrow/dataset/discovery.cc b/cpp/src/arrow/dataset/discovery.cc new file mode 100644 index 000000000000..0a0b8d59f63a --- /dev/null +++ b/cpp/src/arrow/dataset/discovery.cc @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/dataset/discovery.h" + +#include +#include +#include +#include +#include + +#include "arrow/dataset/dataset.h" +#include "arrow/dataset/file_base.h" +#include "arrow/dataset/type_fwd.h" +#include "arrow/filesystem/path_tree.h" +#include "arrow/status.h" + +namespace arrow { +namespace dataset { + +FileSystemDataSourceDiscovery::FileSystemDataSourceDiscovery( + fs::FileSystem* filesystem, std::vector files, + std::shared_ptr format) + : fs_(filesystem), files_(std::move(files)), format_(std::move(format)) {} + +Status FileSystemDataSourceDiscovery::Make(fs::FileSystem* filesystem, + std::vector files, + std::shared_ptr format, + std::shared_ptr* out) { + out->reset(new FileSystemDataSourceDiscovery(filesystem, files, format)); + return Status::OK(); +} + +Status FileSystemDataSourceDiscovery::Make(fs::FileSystem* filesystem, + fs::Selector selector, + std::shared_ptr format, + std::shared_ptr* out) { + std::vector files; + RETURN_NOT_OK(filesystem->GetTargetStats(selector, &files)); + return Make(filesystem, files, format, out); +} + +static inline Status InspectSchema(fs::FileSystem* fs, + const std::vector stats, + const std::shared_ptr& format, + std::shared_ptr* out) { + std::vector> schemas; + + for (const auto& f : stats) { + if (!f.IsFile()) continue; + + std::shared_ptr schema; + RETURN_NOT_OK(format->Inspect(FileSource(f.path(), fs), &schema)); + schemas.push_back(schema); + } + + if (schemas.size() > 0) { + // TODO merge schemas. + *out = schemas[0]; + } + + return Status::OK(); +} + +Status FileSystemDataSourceDiscovery::Inspect(std::shared_ptr* out) { + return InspectSchema(fs_, files_, format_, out); +} + +Status FileSystemDataSourceDiscovery::Build(const BuildOptions& options, + std::shared_ptr source_partition, + std::shared_ptr* out) { + return FileSystemBasedDataSource::Make(fs_, files_, std::move(source_partition), + PathPartitions{}, format_, out); +} + +} // namespace dataset +} // namespace arrow diff --git a/cpp/src/arrow/dataset/discovery.h b/cpp/src/arrow/dataset/discovery.h index 18242250f326..5f0387b23cd4 100644 --- a/cpp/src/arrow/dataset/discovery.h +++ b/cpp/src/arrow/dataset/discovery.h @@ -23,23 +23,80 @@ #include #include +#include #include "arrow/dataset/type_fwd.h" #include "arrow/dataset/visibility.h" +#include "arrow/filesystem/filesystem.h" +#include "arrow/filesystem/path_tree.h" #include "arrow/util/macros.h" namespace arrow { namespace dataset { -struct ARROW_DS_EXPORT DiscoveryOptions { - std::shared_ptr format = NULLPTR; +struct ARROW_DS_EXPORT BuildOptions { + /// Schema to conform to. + std::shared_ptr schema = NULLPTR; + /// The partition scheme indicate how to discover partitions for the data + /// source and fragments. std::shared_ptr partition_scheme = NULLPTR; }; -/// \brief Using a root directory -ARROW_DS_EXPORT -Status DiscoverSource(const std::string& path, fs::FileSystem* filesystem, - const DiscoveryOptions& options, std::shared_ptr* out); +/// \brief DataSourceDiscovery provides a way to inspect a DataSource potential +/// schema before materializing it. Thus, the user can peek the schema for +/// data sources and decide on a unified schema. The pseudocode would look like +/// +/// def get_dataset(factories): +/// schemas = [] +/// for f in factories: +/// schemas.append(f.Inspect()) +/// +/// common_schema = UnifySchemas(schemas) +/// +/// sources = [] +/// for f in factories: +/// sources.append(f.Discover({schema: common_schema})) +/// +/// return Dataset(sources, common_schema) +class ARROW_DS_EXPORT DataSourceDiscovery { + public: + /// \brief Get the schema for the resulting DataSource. + virtual Status Inspect(std::shared_ptr* out) = 0; + + /// \brief Create a DataSource with a given partition. + virtual Status Build(const BuildOptions& options, + std::shared_ptr source_partition, + std::shared_ptr* out) = 0; + + virtual ~DataSourceDiscovery() = default; +}; + +/// \brief FileSystemDataSourceFactory creates a DataSource from a vector +/// of fs::FileStats or a fs::Selector. +class ARROW_DS_EXPORT FileSystemDataSourceDiscovery : public DataSourceDiscovery { + public: + static Status Make(fs::FileSystem* filesystem, std::vector files, + std::shared_ptr format, + std::shared_ptr* out); + + static Status Make(fs::FileSystem* filesystem, fs::Selector selector, + std::shared_ptr format, + std::shared_ptr* out); + + Status Inspect(std::shared_ptr* out) override; + + Status Build(const BuildOptions& options, std::shared_ptr source_partition, + std::shared_ptr* out) override; + + protected: + FileSystemDataSourceDiscovery(fs::FileSystem* filesystem, + std::vector files, + std::shared_ptr format); + + fs::FileSystem* fs_; + std::vector files_; + std::shared_ptr format_; +}; } // namespace dataset } // namespace arrow diff --git a/cpp/src/arrow/dataset/discovery_test.cc b/cpp/src/arrow/dataset/discovery_test.cc new file mode 100644 index 000000000000..8dd3a2d3d920 --- /dev/null +++ b/cpp/src/arrow/dataset/discovery_test.cc @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/dataset/discovery.h" + +#include +#include + +#include "arrow/dataset/test_util.h" +#include "arrow/filesystem/test_util.h" + +namespace arrow { +namespace dataset { + +class FileSystemDataSourceDiscoveryTest : public TestFileSystemBasedDataSource { + public: + void MakeDiscovery(const std::vector& files) { + MakeFileSystem(files); + ASSERT_OK( + FileSystemDataSourceDiscovery::Make(fs_.get(), files, format_, &discovery_)); + } + + void MakeDiscovery(const std::vector& files, fs::Selector selector) { + MakeFileSystem(files); + ASSERT_OK( + FileSystemDataSourceDiscovery::Make(fs_.get(), selector, format_, &discovery_)); + } + + protected: + std::shared_ptr discovery_; + std::shared_ptr format_ = std::make_shared(); +}; + +TEST_F(FileSystemDataSourceDiscoveryTest, Basic) { + MakeDiscovery({fs::File("a"), fs::File("b")}); + + BuildOptions options; + ASSERT_OK(discovery_->Build(options, nullptr, &source_)); + AssertFragmentsAreFromPath(source_->GetFragments(options_), {"a", "b"}); +} + +TEST_F(FileSystemDataSourceDiscoveryTest, Selector) { + // This test ensure that the Selector is enforced. + fs::Selector selector; + selector.base_dir = "A"; + MakeDiscovery({fs::File("0"), fs::File("A/a")}, selector); + + BuildOptions options; + ASSERT_OK(discovery_->Build(options, nullptr, &source_)); + AssertFragmentsAreFromPath(source_->GetFragments(options_), {"A/a"}); +} + +TEST_F(FileSystemDataSourceDiscoveryTest, Inspect) { + auto s = schema({field("f64", float64())}); + format_ = std::make_shared(s); + + MakeDiscovery({}); + std::shared_ptr actual; + + // No files + ASSERT_OK(discovery_->Inspect(&actual)); + EXPECT_EQ(actual, nullptr); + + MakeDiscovery({fs::File("test")}); + ASSERT_OK(discovery_->Inspect(&actual)); + EXPECT_EQ(actual, s); +} + +} // namespace dataset +} // namespace arrow diff --git a/cpp/src/arrow/dataset/file_base.cc b/cpp/src/arrow/dataset/file_base.cc index a335a2a679dd..4a75d650497d 100644 --- a/cpp/src/arrow/dataset/file_base.cc +++ b/cpp/src/arrow/dataset/file_base.cc @@ -48,79 +48,90 @@ Status FileBasedDataFragment::Scan(std::shared_ptr scan_context, } FileSystemBasedDataSource::FileSystemBasedDataSource( - fs::FileSystem* filesystem, const fs::Selector& selector, - std::shared_ptr format, std::shared_ptr partition_expression, - std::vector stats) - : DataSource(std::move(partition_expression)), + fs::FileSystem* filesystem, fs::PathForest forest, + std::shared_ptr source_partition, PathPartitions partitions, + std::shared_ptr format) + : DataSource(std::move(source_partition)), filesystem_(filesystem), - selector_(std::move(selector)), - format_(std::move(format)), - stats_(std::move(stats)) {} + forest_(std::move(forest)), + partitions_(std::move(partitions)), + format_(std::move(format)) {} Status FileSystemBasedDataSource::Make(fs::FileSystem* filesystem, - const fs::Selector& selector, + std::vector stats, + std::shared_ptr source_partition, + PathPartitions partitions, std::shared_ptr format, - std::shared_ptr partition_expression, - std::unique_ptr* out) { - std::vector stats; - RETURN_NOT_OK(filesystem->GetTargetStats(selector, &stats)); - - auto new_end = - std::remove_if(stats.begin(), stats.end(), [&](const fs::FileStats& stats) { - return stats.type() != fs::FileType::File || - !format->IsKnownExtension(stats.extension()); - }); - stats.resize(new_end - stats.begin()); - - out->reset(new FileSystemBasedDataSource(filesystem, selector, std::move(format), - std::move(partition_expression), - std::move(stats))); + std::shared_ptr* out) { + fs::PathForest forest; + RETURN_NOT_OK(fs::PathTree::Make(stats, &forest)); + out->reset(new FileSystemBasedDataSource(filesystem, std::move(forest), + std::move(source_partition), + std::move(partitions), std::move(format))); return Status::OK(); } -Status FileSystemBasedDataSource::Make(fs::FileSystem* filesystem, - const fs::Selector& selector, - std::shared_ptr format, - std::unique_ptr* out) { - return Make(filesystem, selector, std::move(format), nullptr, out); -} - DataFragmentIterator FileSystemBasedDataSource::GetFragmentsImpl( - std::shared_ptr scan_options) { - std::shared_ptr simplified_scan_options; - if (!AssumePartitionExpression(scan_options, &simplified_scan_options)) { - return MakeEmptyIterator>(); - } + std::shared_ptr options) { + std::vector> files; - struct Impl : DataFragmentIterator { - Impl(fs::FileSystem* filesystem, std::shared_ptr format, - std::shared_ptr scan_options, std::vector stats) - : filesystem_(filesystem), - format_(std::move(format)), - scan_options_(std::move(scan_options)), - stats_(std::move(stats)) {} - - Status Next(std::shared_ptr* out) { - if (i_ == stats_.size()) { - *out = nullptr; - return Status::OK(); - } - FileSource src(stats_[i_++].path(), filesystem_); - - std::unique_ptr fragment; - RETURN_NOT_OK(format_->MakeFragment(src, scan_options_, &fragment)); - *out = std::move(fragment); - return Status::OK(); + auto visitor = [&files](const fs::FileStats& stats) { + if (stats.IsFile()) { + files.emplace_back(new fs::FileStats(stats)); } + return Status::OK(); + }; + // The matcher ensures that directories (and their descendants) are not + // visited. + auto matcher = [this, options](const fs::FileStats& stats, bool* match) { + *match = this->PartitionMatches(stats, options->filter); + return Status::OK(); + }; + + for (auto tree : forest_) { + DCHECK_OK(tree->Visit(visitor, matcher)); + } + + auto file_it = MakeVectorIterator(std::move(files)); + auto file_to_fragment = [options, this](std::unique_ptr stats, + std::shared_ptr* out) { + std::unique_ptr fragment; + FileSource src(stats->path(), filesystem_); - size_t i_ = 0; - fs::FileSystem* filesystem_; - std::shared_ptr format_; - std::shared_ptr scan_options_; - std::vector stats_; + RETURN_NOT_OK(format_->MakeFragment(src, options, &fragment)); + *out = std::move(fragment); + return Status::OK(); }; - return DataFragmentIterator(Impl(filesystem_, format_, scan_options, stats_)); + return MakeMaybeMapIterator(file_to_fragment, std::move(file_it)); +} + +bool FileSystemBasedDataSource::PartitionMatches(const fs::FileStats& stats, + std::shared_ptr filter) { + if (filter == nullptr) { + return true; + } + + auto found = partitions_.find(stats.path()); + if (found == partitions_.end()) { + // No partition attached to current node (directory or file), continue. + return true; + } + + auto c = found->second->Assume(*filter); + if (!c.ok()) { + // Could not simplify expression move on! + return true; + } + + // TODO: pass simplified expressions to children + auto expr = std::move(c).ValueOrDie(); + if (expr->IsNull() || expr->IsTrivialFalseCondition()) { + // selector is not satisfiable; don't recurse in this branch. + return false; + } + + return true; } } // namespace dataset diff --git a/cpp/src/arrow/dataset/file_base.h b/cpp/src/arrow/dataset/file_base.h index 440b36f75d95..c581251e4dc4 100644 --- a/cpp/src/arrow/dataset/file_base.h +++ b/cpp/src/arrow/dataset/file_base.h @@ -19,6 +19,7 @@ #include #include +#include #include #include @@ -29,12 +30,16 @@ #include "arrow/dataset/visibility.h" #include "arrow/dataset/writer.h" #include "arrow/filesystem/filesystem.h" +#include "arrow/filesystem/path_tree.h" #include "arrow/io/file.h" #include "arrow/util/compression.h" namespace arrow { + namespace dataset { +class Filter; + /// \brief The path and filesystem where an actual file is located or a buffer which can /// be read like a file class ARROW_DS_EXPORT FileSource { @@ -127,6 +132,10 @@ class ARROW_DS_EXPORT FileFormat { /// \brief Return true if the given file extension virtual bool IsKnownExtension(const std::string& ext) const = 0; + /// \brief Return the schema of the file if possible. + virtual Status Inspect(const FileSource& source, + std::shared_ptr* out) const = 0; + /// \brief Open a file for scanning virtual Status ScanFile(const FileSource& source, std::shared_ptr scan_options, @@ -161,36 +170,46 @@ class ARROW_DS_EXPORT FileBasedDataFragment : public DataFragment { std::shared_ptr scan_options_; }; -/// \brief A DataSource which takes files of one format from a directory -/// -/// The directory is crawled upon construction (Make) and not updated afterward. -/// GetFragments() will not include files added after this DataDource is constructed and -/// will error if files are deleted/moved. +/// \brief Mapping from path to partition expressions. +using PathPartitions = std::unordered_map>; + +/// \brief A DataSource of FileBasedDataFragments. class ARROW_DS_EXPORT FileSystemBasedDataSource : public DataSource { public: - static Status Make(fs::FileSystem* filesystem, const fs::Selector& selector, - std::shared_ptr format, - std::unique_ptr* out); - - static Status Make(fs::FileSystem* filesystem, const fs::Selector& selector, - std::shared_ptr format, - std::shared_ptr partition_expression, - std::unique_ptr* out); - - std::string type() const override { return "directory"; } + /// \brief Create a FileSystemBasedDataSource with optional partitions. + /// + /// \param[in] filesystem the filesystem which files are from. + /// \param[in] stats a list of files/directories to consume. + /// \param[in] source_partition the top-level partition of the DataSource + /// \param[in] partitions optional partitions attached to FileStats found in + /// `stats`. + /// \param[in] format file format to create fragments from. + /// \param[out] out pointer storing the resulting DataSource. + /// + /// The caller is not required to provide a complete coverage of nodes and + /// partitions. + static Status Make(fs::FileSystem* filesystem, std::vector stats, + std::shared_ptr source_partition, + PathPartitions partitions, std::shared_ptr format, + std::shared_ptr* out); + + std::string type() const override { return "filesystem_data_source"; } protected: DataFragmentIterator GetFragmentsImpl(std::shared_ptr options) override; - FileSystemBasedDataSource(fs::FileSystem* filesystem, const fs::Selector& selector, - std::shared_ptr format, - std::shared_ptr partition_expression, - std::vector stats); + FileSystemBasedDataSource(fs::FileSystem* filesystem, fs::PathForest forest, + std::shared_ptr source_partition, + PathPartitions partitions, + std::shared_ptr format); + + bool PartitionMatches(const fs::FileStats& stats, std::shared_ptr filter); fs::FileSystem* filesystem_ = NULLPTR; - fs::Selector selector_; + fs::PathForest forest_; + PathPartitions partitions_; + std::shared_ptr format_; - std::vector stats_; }; } // namespace dataset diff --git a/cpp/src/arrow/dataset/file_parquet.cc b/cpp/src/arrow/dataset/file_parquet.cc index e6298231b89f..a23639ebd162 100644 --- a/cpp/src/arrow/dataset/file_parquet.cc +++ b/cpp/src/arrow/dataset/file_parquet.cc @@ -165,14 +165,26 @@ class ParquetScanTaskIterator { std::shared_ptr reader_; }; +Status ParquetFileFormat::Inspect(const FileSource& source, + std::shared_ptr* out) const { + auto pool = default_memory_pool(); + + std::unique_ptr reader; + RETURN_NOT_OK(OpenReader(source, pool, &reader)); + + std::unique_ptr arrow_reader; + RETURN_NOT_OK(parquet::arrow::FileReader::Make(pool, std::move(reader), &arrow_reader)); + + return arrow_reader->GetSchema(out); +} + Status ParquetFileFormat::ScanFile(const FileSource& source, std::shared_ptr scan_options, std::shared_ptr scan_context, ScanTaskIterator* out) const { - std::shared_ptr input; - RETURN_NOT_OK(source.Open(&input)); + std::unique_ptr reader; + RETURN_NOT_OK(OpenReader(source, scan_context->pool, &reader)); - auto reader = parquet::ParquetFileReader::Open(input); return ParquetScanTaskIterator::Make(scan_options, scan_context, std::move(reader), out); } @@ -185,5 +197,15 @@ Status ParquetFileFormat::MakeFragment(const FileSource& source, return Status::OK(); } +Status ParquetFileFormat::OpenReader( + const FileSource& source, MemoryPool* pool, + std::unique_ptr* out) const { + std::shared_ptr input; + RETURN_NOT_OK(source.Open(&input)); + + *out = parquet::ParquetFileReader::Open(input); + return Status::OK(); +} + } // namespace dataset } // namespace arrow diff --git a/cpp/src/arrow/dataset/file_parquet.h b/cpp/src/arrow/dataset/file_parquet.h index a04f4596f87f..cd3987845420 100644 --- a/cpp/src/arrow/dataset/file_parquet.h +++ b/cpp/src/arrow/dataset/file_parquet.h @@ -24,6 +24,10 @@ #include "arrow/dataset/type_fwd.h" #include "arrow/dataset/visibility.h" +namespace parquet { +class ParquetFileReader; +} // namespace parquet + namespace arrow { namespace dataset { @@ -47,6 +51,9 @@ class ARROW_DS_EXPORT ParquetFileFormat : public FileFormat { return ext == "par" || ext == "parq" || ext == name(); } + /// \brief Return the schema of the file if possible. + Status Inspect(const FileSource& source, std::shared_ptr* out) const override; + /// \brief Open a file for scanning Status ScanFile(const FileSource& source, std::shared_ptr scan_options, std::shared_ptr scan_context, @@ -54,6 +61,10 @@ class ARROW_DS_EXPORT ParquetFileFormat : public FileFormat { Status MakeFragment(const FileSource& source, std::shared_ptr opts, std::unique_ptr* out) override; + + private: + Status OpenReader(const FileSource& source, MemoryPool* pool, + std::unique_ptr<::parquet::ParquetFileReader>* out) const; }; class ARROW_DS_EXPORT ParquetFragment : public FileBasedDataFragment { diff --git a/cpp/src/arrow/dataset/file_parquet_test.cc b/cpp/src/arrow/dataset/file_parquet_test.cc index 171d8cee1a7f..ba390f2d2018 100644 --- a/cpp/src/arrow/dataset/file_parquet_test.cc +++ b/cpp/src/arrow/dataset/file_parquet_test.cc @@ -42,6 +42,8 @@ using parquet::CreateOutputStream; using parquet::arrow::FileWriter; using parquet::arrow::WriteTable; +using testing::Pointee; + Status WriteRecordBatch(const RecordBatch& batch, FileWriter* writer) { auto schema = batch.schema(); auto size = batch.num_rows(); @@ -138,9 +140,11 @@ class ParquetBufferFixtureMixin : public ArrowParquetWriterMixin { builder->UnsafeAppend(0.0); })); - auto schema_ = schema({field("f64", f64->type())}); return RecordBatch::Make(schema_, kBatchSize, {f64}); } + + protected: + std::shared_ptr schema_ = schema({field("f64", float64())}); }; class TestParquetFileFormat : public ParquetBufferFixtureMixin { @@ -172,21 +176,14 @@ TEST_F(TestParquetFileFormat, ScanRecordBatchReader) { ASSERT_EQ(row_count, kNumRows); } -class TestParquetFileSystemBasedDataSource - : public FileSystemBasedDataSourceMixin { - std::vector file_names() const override { - return {"a/b/c.parquet", "a/b/c/d.parquet", "a/b.parquet", "a.parquet"}; - } -}; - -TEST_F(TestParquetFileSystemBasedDataSource, NonRecursive) { this->NonRecursive(); } - -TEST_F(TestParquetFileSystemBasedDataSource, Recursive) { this->Recursive(); } - -TEST_F(TestParquetFileSystemBasedDataSource, DeletedFile) { this->DeletedFile(); } +TEST_F(TestParquetFileFormat, Inspect) { + auto reader = GetRecordBatchReader(); + auto source = GetFileSource(reader.get()); + auto format = ParquetFileFormat(); -TEST_F(TestParquetFileSystemBasedDataSource, PredicatePushDown) { - this->PredicatePushDown(); + std::shared_ptr actual; + ASSERT_OK(format.Inspect(*source.get(), &actual)); + EXPECT_EQ(*actual, *schema_); } } // namespace dataset diff --git a/cpp/src/arrow/dataset/file_test.cc b/cpp/src/arrow/dataset/file_test.cc index 19e938a5e8b2..be6af4cbbfc0 100644 --- a/cpp/src/arrow/dataset/file_test.cc +++ b/cpp/src/arrow/dataset/file_test.cc @@ -15,23 +15,6 @@ // specific language governing permissions and limitations // under the License. -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - #include #include #include @@ -42,8 +25,8 @@ #include "arrow/dataset/api.h" #include "arrow/dataset/test_util.h" -#include "arrow/filesystem/localfs.h" #include "arrow/filesystem/path_util.h" +#include "arrow/filesystem/test_util.h" #include "arrow/status.h" #include "arrow/testing/gtest_util.h" #include "arrow/util/io_util.h" @@ -94,21 +77,89 @@ TEST(FileSource, BufferBased) { ASSERT_EQ(Compression::LZ4, source2.compression()); } -class TestDummyFileSystemBasedDataSource - : public FileSystemBasedDataSourceMixin { - std::vector file_names() const override { - return {"a/b/c.dummy", "a/b/c/d.dummy", "a/b.dummy", "a.dummy"}; - } -}; +TEST_F(TestFileSystemBasedDataSource, Basic) { + MakeSource({}); + AssertFragmentsAreFromPath(source_->GetFragments(options_), {}); + + MakeSource({fs::File("a"), fs::File("b"), fs::File("c")}); + AssertFragmentsAreFromPath(source_->GetFragments(options_), {"a", "b", "c"}); + + // Should not create fragment from directories. + MakeSource({fs::Dir("A"), fs::Dir("A/B"), fs::File("A/a"), fs::File("A/B/b")}); + AssertFragmentsAreFromPath(source_->GetFragments(options_), {"A/a", "A/B/b"}); +} + +TEST_F(TestFileSystemBasedDataSource, RootPartitionPruning) { + auto source_partition = ("a"_ == 5).Copy(); + MakeSource({fs::File("a"), fs::File("b")}, source_partition); + + // No filter should always return all data. + options_->filter = nullptr; + AssertFragmentsAreFromPath(source_->GetFragments(options_), {"a", "b"}); -TEST_F(TestDummyFileSystemBasedDataSource, NonRecursive) { this->NonRecursive(); } + // filter == partition + options_->filter = source_partition; + AssertFragmentsAreFromPath(source_->GetFragments(options_), {"a", "b"}); -TEST_F(TestDummyFileSystemBasedDataSource, Recursive) { this->Recursive(); } + // Same partition key, but non matching filter + options_->filter = ("a"_ == 6).Copy(); + AssertFragmentsAreFromPath(source_->GetFragments(options_), {}); -TEST_F(TestDummyFileSystemBasedDataSource, DeletedFile) { this->DeletedFile(); } + options_->filter = ("a"_ > 1).Copy(); + AssertFragmentsAreFromPath(source_->GetFragments(options_), {"a", "b"}); + + // different key shouldn't prune + options_->filter = ("b"_ == 6).Copy(); + AssertFragmentsAreFromPath(source_->GetFragments(options_), {"a", "b"}); + + // No partition should match + MakeSource({fs::File("a"), fs::File("b")}); + options_->filter = ("b"_ == 6).Copy(); + AssertFragmentsAreFromPath(source_->GetFragments(options_), {"a", "b"}); +} -TEST_F(TestDummyFileSystemBasedDataSource, PredicatePushDown) { - this->PredicatePushDown(); +TEST_F(TestFileSystemBasedDataSource, TreePartitionPruning) { + auto source_partition = ("country"_ == "US").Copy(); + std::vector regions = { + fs::Dir("NY"), fs::File("NY/New York"), fs::File("NY/Franklin"), + fs::Dir("CA"), fs::File("CA/San Francisco"), fs::File("CA/Franklin"), + }; + // Explicitly _don't_ set the state partition in the leaves to test if + // sub-tree pruning works. This implies that `state` predicate won't apply to + // files. + PathPartitions partitions = { + {"CA", ("state"_ == "CA").Copy()}, + {"CA/San Francisco", ("city"_ == "San Francisco").Copy()}, + {"CA/Franklin", ("city"_ == "Franklin").Copy()}, + {"NY", ("state"_ == "NY").Copy()}, + {"NY/New York", ("city"_ == "New York").Copy()}, + {"NY/Franklin", ("city"_ == "Franklin").Copy()}, + }; + + MakeSource(regions, source_partition, partitions); + + std::vector all_cities = {"CA/San Francisco", "CA/Franklin", "NY/New York", + "NY/Franklin"}; + std::vector ca_cities = {"CA/San Francisco", "CA/Franklin"}; + std::vector franklins = {"CA/Franklin", "NY/Franklin"}; + + // No filter should always return all data. + options_->filter = nullptr; + AssertFragmentsAreFromPath(source_->GetFragments(options_), all_cities); + + // Data source partition is respected + options_->filter = ("country"_ == "US").Copy(); + AssertFragmentsAreFromPath(source_->GetFragments(options_), all_cities); + options_->filter = ("country"_ == "FR").Copy(); + AssertFragmentsAreFromPath(source_->GetFragments(options_), {}); + + options_->filter = ("state"_ == "CA").Copy(); + AssertFragmentsAreFromPath(source_->GetFragments(options_), ca_cities); + + // Filter where no decisions can be made on inner nodes when filter don't + // apply to inner partitions. + options_->filter = ("city"_ == "Franklin").Copy(); + AssertFragmentsAreFromPath(source_->GetFragments(options_), franklins); } } // namespace dataset diff --git a/cpp/src/arrow/dataset/filter.cc b/cpp/src/arrow/dataset/filter.cc index 7e1ee82b22b1..bcdee2897f1f 100644 --- a/cpp/src/arrow/dataset/filter.cc +++ b/cpp/src/arrow/dataset/filter.cc @@ -842,6 +842,16 @@ bool Expression::IsTrivialCondition(bool* out) const { return true; } +bool Expression::IsTrivialTrueCondition() const { + bool value = false; + return IsTrivialCondition(&value) && value; +} + +bool Expression::IsTrivialFalseCondition() const { + bool value = false; + return IsTrivialCondition(&value) && !value; +} + std::shared_ptr FieldExpression::Copy() const { return std::make_shared(*this); } @@ -939,33 +949,5 @@ Result> FieldExpression::Validate(const Schema& schema return null(); } -Result> SelectorAssume( - const std::shared_ptr& selector, - const std::shared_ptr& given) { - if (selector == nullptr || selector->filters.size() == 0) { - return ScalarExpression::Make(true); - } - - auto get_expression = [](const std::shared_ptr& f) { - DCHECK_EQ(f->type(), FilterType::EXPRESSION); - return checked_cast(*f).expression(); - }; - - auto out_expr = get_expression(selector->filters[0]); - for (size_t i = 1; i < selector->filters.size(); ++i) { - out_expr = and_(std::move(out_expr), get_expression(selector->filters[i])); - } - - if (given == nullptr) { - return std::move(out_expr); - } - return out_expr->Assume(*given); -} - -std::shared_ptr ExpressionSelector(std::shared_ptr e) { - return std::make_shared( - DataSelector{FilterVector{std::make_shared(std::move(e))}}); -} - } // namespace dataset } // namespace arrow diff --git a/cpp/src/arrow/dataset/filter.h b/cpp/src/arrow/dataset/filter.h index 364fca0d2d4e..3f122c18ea4c 100644 --- a/cpp/src/arrow/dataset/filter.h +++ b/cpp/src/arrow/dataset/filter.h @@ -151,6 +151,9 @@ class ARROW_DS_EXPORT Expression { /// BooleanScalar. Its value may be retrieved at the same time. bool IsTrivialCondition(bool* value = NULLPTR) const; + bool IsTrivialTrueCondition() const; + bool IsTrivialFalseCondition() const; + /// Copy this expression into a shared pointer. virtual std::shared_ptr Copy() const = 0; diff --git a/cpp/src/arrow/dataset/scanner.h b/cpp/src/arrow/dataset/scanner.h index cfb661f4c0a7..7cde4ef2888b 100644 --- a/cpp/src/arrow/dataset/scanner.h +++ b/cpp/src/arrow/dataset/scanner.h @@ -38,16 +38,16 @@ class ARROW_DS_EXPORT ScanOptions { public: ScanOptions() = default; - ScanOptions(std::shared_ptr selector, std::shared_ptr schema, + ScanOptions(std::shared_ptr filter, std::shared_ptr schema, std::vector> options = {}) - : selector(std::move(selector)), schema(std::move(schema)) {} + : filter(std::move(filter)), schema(std::move(schema)) {} virtual ~ScanOptions() = default; MemoryPool* pool() const { return pool_; } // Filters - std::shared_ptr selector; + std::shared_ptr filter; // Schema to which record batches will be reconciled std::shared_ptr schema; diff --git a/cpp/src/arrow/dataset/test_util.h b/cpp/src/arrow/dataset/test_util.h index 459185e527f3..1a7301b55187 100644 --- a/cpp/src/arrow/dataset/test_util.h +++ b/cpp/src/arrow/dataset/test_util.h @@ -15,16 +15,24 @@ // specific language governing permissions and limitations // under the License. +#pragma once + +#include #include #include #include #include #include +#include +#include + #include "arrow/dataset/file_base.h" #include "arrow/dataset/filter.h" #include "arrow/filesystem/localfs.h" +#include "arrow/filesystem/mockfs.h" #include "arrow/filesystem/path_util.h" +#include "arrow/filesystem/test_util.h" #include "arrow/record_batch.h" #include "arrow/testing/gtest_util.h" #include "arrow/util/io_util.h" @@ -155,7 +163,7 @@ class DatasetFixtureMixin : public ::testing::Test { } protected: - std::shared_ptr options_ = nullptr; + std::shared_ptr options_ = std::make_shared(); std::shared_ptr ctx_; }; @@ -164,117 +172,12 @@ class FileSystemBasedDataSourceMixin : public FileSourceFixtureMixin { public: virtual std::vector file_names() const = 0; - void SetUp() override { - selector_.base_dir = "/"; - selector_.recursive = true; - - format_ = std::make_shared(); - schema_ = schema({field("dummy", null())}); - options_ = std::make_shared(); - - ASSERT_OK( - TemporaryDir::Make("test-fsdatasource-" + format_->name() + "-", &temp_dir_)); - local_fs_ = std::make_shared(); - - auto path = temp_dir_->path().ToString(); - fs_ = std::make_shared(path, local_fs_); - - for (auto path : file_names()) { - CreateFile(path, ""); - } - - partition_expression_ = ScalarExpression::Make(true); - } - - void CreateFile(std::string path, std::string contents) { - auto parent = fs::internal::GetAbstractPathParent(path).first; - if (parent != "") { - ASSERT_OK(this->fs_->CreateDir(parent, true)); - } - std::shared_ptr file; - ASSERT_OK(this->fs_->OpenOutputStream(path, &file)); - ASSERT_OK(file->Write(contents)); - } - - void MakeDataSource() { - ASSERT_OK(FileSystemBasedDataSource::Make(fs_.get(), selector_, format_, - partition_expression_, &source_)); - } - - protected: - std::function fragment)> OpenFragments( - size_t* count) { - return [this, count](std::shared_ptr fragment) { - auto file_fragment = - internal::checked_pointer_cast(fragment); - ++*count; - auto extension = - fs::internal::GetAbstractPathExtension(file_fragment->source().path()); - EXPECT_TRUE(format_->IsKnownExtension(extension)); - std::shared_ptr f; - return this->fs_->OpenInputFile(file_fragment->source().path(), &f); - }; - } - - void NonRecursive() { - selector_.recursive = false; - MakeDataSource(); - - size_t count = 0; - ASSERT_OK(source_->GetFragments(options_).Visit(OpenFragments(&count))); - ASSERT_EQ(count, 1); - } - - void Recursive() { - MakeDataSource(); - - size_t count = 0; - ASSERT_OK(source_->GetFragments(options_).Visit(OpenFragments(&count))); - ASSERT_EQ(count, file_names().size()); - } - - void DeletedFile() { - MakeDataSource(); - ASSERT_GT(file_names().size(), 0); - ASSERT_OK(this->fs_->DeleteFile(file_names()[0])); - - size_t count = 0; - ASSERT_RAISES(IOError, source_->GetFragments(options_).Visit(OpenFragments(&count))); - } - - void PredicatePushDown() { - partition_expression_ = equal(field_ref("alpha"), ScalarExpression::Make(3)); - MakeDataSource(); - - options_->selector = std::make_shared(); - options_->selector->filters.resize(1); - - // with a filter identical to the partition condition, all fragments are yielded - options_->selector->filters[0] = - std::make_shared(partition_expression_->Copy()); - - size_t count = 0; - // ASSERT_OK(source_->GetFragments(context_)->Visit(OpenFragments(&count))); - // ASSERT_EQ(count, file_names().size()); - - // with a filter which contradicts the partition condition, no fragments are yielded - options_->selector->filters[0] = std::make_shared( - equal(field_ref("alpha"), ScalarExpression::Make(0))); - - count = 0; - ASSERT_OK(source_->GetFragments(options_).Visit(OpenFragments(&count))); - ASSERT_EQ(count, 0); - } - fs::Selector selector_; - std::unique_ptr source_; - std::shared_ptr local_fs_; + std::unique_ptr source_; std::shared_ptr fs_; - std::unique_ptr temp_dir_; std::shared_ptr format_; std::shared_ptr schema_; - std::shared_ptr options_; - std::shared_ptr partition_expression_; + std::shared_ptr options_ = std::make_shared(); }; template @@ -286,11 +189,19 @@ std::unique_ptr> MakeGeneratedRecordBatch( /// \brief A dummy FileFormat implementation class DummyFileFormat : public FileFormat { public: + explicit DummyFileFormat(std::shared_ptr schema = NULLPTR) + : schema_(std::move(schema)) {} + std::string name() const override { return "dummy"; } /// \brief Return true if the given file extension bool IsKnownExtension(const std::string& ext) const override { return ext == name(); } + Status Inspect(const FileSource& source, std::shared_ptr* out) const override { + *out = schema_; + return Status::OK(); + } + /// \brief Open a file for scanning (always returns an empty iterator) Status ScanFile(const FileSource& source, std::shared_ptr scan_options, std::shared_ptr scan_context, @@ -302,6 +213,9 @@ class DummyFileFormat : public FileFormat { inline Status MakeFragment(const FileSource& location, std::shared_ptr opts, std::unique_ptr* out) override; + + protected: + std::shared_ptr schema_; }; class DummyFragment : public FileBasedDataFragment { @@ -319,5 +233,52 @@ Status DummyFileFormat::MakeFragment(const FileSource& source, return Status::OK(); } +class TestFileSystemBasedDataSource : public ::testing::Test { + public: + void SetUp() { options_ = std::make_shared(); } + + void MakeFileSystem(const std::vector& stats) { + ASSERT_OK(fs::internal::MockFileSystem::Make(fs::kNoTime, stats, &fs_)); + } + + void MakeFileSystem(const std::vector& paths) { + std::vector stats{paths.size()}; + std::transform(paths.cbegin(), paths.cend(), stats.begin(), + [](const std::string& p) { return fs::File(p); }); + + ASSERT_OK(fs::internal::MockFileSystem::Make(fs::kNoTime, stats, &fs_)); + } + + void MakeSource(const std::vector& stats, + std::shared_ptr source_partition = nullptr, + PathPartitions partitions = {}) { + MakeFileSystem(stats); + auto format = std::make_shared(); + ASSERT_OK(FileSystemBasedDataSource::Make(fs_.get(), stats, source_partition, + partitions, format, &source_)); + } + + protected: + std::shared_ptr fs_; + std::shared_ptr source_; + std::shared_ptr options_; +}; + +void AssertFragmentsAreFromPath(DataFragmentIterator it, + std::vector expected) { + std::vector actual; + + auto v = [&actual](std::shared_ptr fragment) -> Status { + EXPECT_NE(fragment, nullptr); + auto dummy = std::static_pointer_cast(fragment); + actual.push_back(dummy->source().path()); + return Status::OK(); + }; + + ASSERT_OK(it.Visit(v)); + // Ordering is not guaranteed. + EXPECT_THAT(actual, testing::UnorderedElementsAreArray(expected)); +} + } // namespace dataset } // namespace arrow diff --git a/cpp/src/arrow/filesystem/filesystem_test.cc b/cpp/src/arrow/filesystem/filesystem_test.cc index c199a2dde2cb..0b183550efff 100644 --- a/cpp/src/arrow/filesystem/filesystem_test.cc +++ b/cpp/src/arrow/filesystem/filesystem_test.cc @@ -365,6 +365,19 @@ TEST_F(TestMockFS, OpenAppendStream) { CheckFiles({{"ab", time_, "some data"}}); } +TEST_F(TestMockFS, Make) { + std::shared_ptr fs; + ASSERT_OK(MockFileSystem::Make(time_, {}, &fs)); + fs_ = std::static_pointer_cast(fs); + CheckDirs({}); + CheckFiles({}); + + ASSERT_OK(MockFileSystem::Make(time_, {Dir("A/B/C"), File("A/a")}, &fs)); + fs_ = std::static_pointer_cast(fs); + CheckDirs({{"A", time_}, {"A/B", time_}, {"A/B/C", time_}}); + CheckFiles({{"A/a", time_, ""}}); +} + //////////////////////////////////////////////////////////////////////////// // Concrete SubTreeFileSystem tests diff --git a/cpp/src/arrow/filesystem/mockfs.cc b/cpp/src/arrow/filesystem/mockfs.cc index a6ff7c78d6f8..c813355c818f 100644 --- a/cpp/src/arrow/filesystem/mockfs.cc +++ b/cpp/src/arrow/filesystem/mockfs.cc @@ -656,6 +656,40 @@ std::vector MockFileSystem::AllFiles() { return result; } +Status MockFileSystem::CreateFile(const std::string& path, const std::string& contents, + bool recursive) { + auto parent = fs::internal::GetAbstractPathParent(path).first; + if (parent != "") { + RETURN_NOT_OK(CreateDir(parent, recursive)); + } + + std::shared_ptr file; + RETURN_NOT_OK(OpenOutputStream(path, &file)); + RETURN_NOT_OK(file->Write(contents)); + return file->Close(); +} + +Status MockFileSystem::Make(TimePoint current_time, const std::vector& stats, + std::shared_ptr* out) { + auto fs = std::make_shared(current_time); + for (const auto& s : stats) { + switch (s.type()) { + case FileType::Directory: + RETURN_NOT_OK(fs->CreateDir(s.path(), /*recursive*/ true)); + break; + case FileType::File: + RETURN_NOT_OK(fs->CreateFile(s.path(), "", /*recursive*/ true)); + break; + default: + break; + } + } + + *out = fs; + + return Status::OK(); +} + } // namespace internal } // namespace fs } // namespace arrow diff --git a/cpp/src/arrow/filesystem/mockfs.h b/cpp/src/arrow/filesystem/mockfs.h index faafe9a3bccd..e23d3029d9c4 100644 --- a/cpp/src/arrow/filesystem/mockfs.h +++ b/cpp/src/arrow/filesystem/mockfs.h @@ -94,6 +94,15 @@ class ARROW_EXPORT MockFileSystem : public FileSystem { std::vector AllDirs(); std::vector AllFiles(); + // Create a File with a content from a string. + Status CreateFile(const std::string& path, const std::string& content, + bool recursive = true); + + // Create a MockFileSystem out of (empty) FileStats. The content of every + // file is empty and of size 0. All directories will be created recursively. + static Status Make(TimePoint current_time, const std::vector& stats, + std::shared_ptr* out); + class Impl; protected: diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h index 60cae37e6360..72b6ce405223 100644 --- a/cpp/src/arrow/type.h +++ b/cpp/src/arrow/type.h @@ -1317,6 +1317,7 @@ class ARROW_EXPORT Schema : public detail::Fingerprintable { /// Returns true if all of the schema fields are equal bool Equals(const Schema& other, bool check_metadata = true) const; + bool operator==(const Schema& other) const { return Equals(other); } /// \brief Return the number of fields (columns) in the schema int num_fields() const; From 48b56bd593909832aa37a58681e80cfd4e041d53 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Wed, 2 Oct 2019 19:41:57 +0200 Subject: [PATCH 19/24] ARROW-6581: [C++] Fix fuzzit job submission MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Also make it faster (we don't need Gandiva here). Closes #5407 from pitrou/ARROW-6581-fix-fuzzit-job and squashes the following commits: 2628a74f8 ARROW-6581: Fix fuzzit job submission Authored-by: Antoine Pitrou Signed-off-by: Krisztián Szűcs --- ci/docker_build_and_fuzzit.sh | 16 +++++++++------- docker-compose.yml | 36 +++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/ci/docker_build_and_fuzzit.sh b/ci/docker_build_and_fuzzit.sh index 18bd0d4c599e..b3bf1bd81fed 100755 --- a/ci/docker_build_and_fuzzit.sh +++ b/ci/docker_build_and_fuzzit.sh @@ -24,13 +24,14 @@ export ARROW_USE_ASAN="ON" export CC="clang-7" export CXX="clang++-7" export ARROW_BUILD_TYPE="RelWithDebInfo" +export ARROW_FLIGHT="OFF" +export ARROW_GANDIVA="OFF" export ARROW_ORC="OFF" export ARROW_PARQUET="OFF" export ARROW_PLASMA="OFF" -export ARROW_FLIGHT="OFF" -export ARROW_BUILD_BENCHMARKS="OFF" export ARROW_WITH_BZ2="OFF" export ARROW_WITH_ZSTD="OFF" +export ARROW_BUILD_BENCHMARKS="OFF" export ARROW_BUILD_UTILITIES="OFF" /arrow/ci/docker_build_cpp.sh || exit 1 pushd /build/cpp @@ -40,11 +41,12 @@ cp ./relwithdebinfo/arrow-ipc-fuzzing-test ./relwithdebinfo/out/fuzzer ldd ./relwithdebinfo/arrow-ipc-fuzzing-test | grep "=> /" | awk '{print $3}' | xargs -I '{}' cp -v '{}' ./relwithdebinfo/out/. cd ./relwithdebinfo/out/ tar -czvf fuzzer.tar.gz * +stat fuzzer.tar.gz cd ../../ -export TARGET_ID=u79f6bXYgNH4NkU99iWK -export FUZZIT_API_KEY=${FUZZIT_API_KEY:-ac6089a1bc2313679f2d99bb80553162c380676bff3f094de826b16229e28184a8084b86f52c95112bde6b3dbb07b9b7} -wget -O fuzzit https://bin.fuzzit.dev/fuzzit-1.1 +export TARGET_ID=apache-arrow/arrow-ipc-fuzzing + +wget -O fuzzit https://github.com/fuzzitdev/fuzzit/releases/latest/download/fuzzit_Linux_x86_64 chmod a+x fuzzit -./fuzzit auth $FUZZIT_API_KEY -./fuzzit create job --type fuzzing --host bionic-llvm7 --revision $CI_ARROW_SHA --branch $CI_ARROW_BRANCH $TARGET_ID ./relwithdebinfo/out/fuzzer.tar.gz + +./fuzzit create job --type $FUZZIT_JOB_TYPE --host bionic-llvm7 --revision $CI_ARROW_SHA --branch $CI_ARROW_BRANCH $TARGET_ID ./relwithdebinfo/out/fuzzer.tar.gz diff --git a/docker-compose.yml b/docker-compose.yml index 0d0f6f9fc6bc..6ebf61646189 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -447,6 +447,7 @@ services: volumes: *ubuntu-volumes fuzzit: + # Submit a fuzzing job on fuzzit.dev # Usage: # docker-compose build cpp # docker-compose build fuzzit @@ -458,6 +459,41 @@ services: environment: CI_ARROW_SHA: ${CI_ARROW_SHA} CI_ARROW_BRANCH: ${CI_ARROW_BRANCH} + FUZZIT_JOB_TYPE: fuzzing + FUZZIT_API_KEY: c0b760d37db6012fcaafd8ca5f412ba7bcd297ac969650502994b51aa11798153824442f999a067e1ef67821989ed664 + volumes: *ubuntu-volumes + + fuzzit-regression: + # Submit a regression job on fuzzit.dev + # Usage: + # docker-compose build cpp + # docker-compose build fuzzit-regression + # docker-compose run fuzzit-regression + image: arrow:fuzzit + build: + context: . + dockerfile: dev/fuzzit/Dockerfile + environment: + CI_ARROW_SHA: ${CI_ARROW_SHA} + CI_ARROW_BRANCH: ${CI_ARROW_BRANCH} + FUZZIT_JOB_TYPE: regression + FUZZIT_API_KEY: c0b760d37db6012fcaafd8ca5f412ba7bcd297ac969650502994b51aa11798153824442f999a067e1ef67821989ed664 + volumes: *ubuntu-volumes + + fuzzit-local-regression: + # Run a local fuzzit regression job + # Usage: + # docker-compose build cpp + # docker-compose build fuzzit-local-regression + # docker-compose run fuzzit-local-regression + image: arrow:fuzzit + build: + context: . + dockerfile: dev/fuzzit/Dockerfile + environment: + CI_ARROW_SHA: ${CI_ARROW_SHA} + CI_ARROW_BRANCH: ${CI_ARROW_BRANCH} + FUZZIT_JOB_TYPE: local-regression volumes: *ubuntu-volumes iwyu: From fda549a74170efbbe0d2f37f2e34056b177e14ef Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 2 Oct 2019 19:09:27 -0400 Subject: [PATCH 20/24] ARROW-6761: [Rust] Travis build now uses the correct Rust toolchain There is a compiler bug apparently in Rust 1.40.0-nightly so we need to keep using 1.39.0 for now. The Travis build script was using a `RUSTUP_TOOLCHAIN=nightly` which seems to override the `rust-toolchain` file. Also, Travis builds the arrow subcrate individually and therefore wasn't picking up the `rust-toolchain` file in the root of the Rust project. Closes #5561 from andygrove/fix-travis-builds and squashes the following commits: 8293f44b8 no need to copy rust-toolchain c2a209f7c preserve original behavior of testing only after building arrow subcrate without default features a65b890ec try and fix build b53b26a58 fix error 81663b9ca Travis build now uses the correct Rust toolchain Authored-by: Andy Grove Signed-off-by: Paddy Horan --- .travis.yml | 2 +- ci/travis_script_rust.sh | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 534004d0d160..a107bcc3f4b9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -298,7 +298,7 @@ matrix: - if [ $ARROW_CI_RUST_AFFECTED != "1" ]; then exit; fi - $TRAVIS_BUILD_DIR/ci/travis_install_cargo.sh script: - - RUSTUP_TOOLCHAIN=nightly $TRAVIS_BUILD_DIR/ci/travis_script_rust.sh + - $TRAVIS_BUILD_DIR/ci/travis_script_rust.sh before_cache: cargo install cargo-tarpaulin -f after_success: diff --git a/ci/travis_script_rust.sh b/ci/travis_script_rust.sh index 0f6f619e32e0..22059df228f4 100755 --- a/ci/travis_script_rust.sh +++ b/ci/travis_script_rust.sh @@ -31,11 +31,16 @@ rustup show # raises on any formatting errors cargo +stable fmt --all -- --check +# build entire project RUSTFLAGS="-D warnings" cargo build --all-targets + +# run tests +cargo test + +# make sure we can build Arrow sub-crate without default features pushd arrow cargo build --no-default-features popd -cargo test # run Arrow examples pushd arrow From ad4eccb36aebc3325de22f4f46ed1448254f15d7 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 3 Oct 2019 14:46:52 +0900 Subject: [PATCH 21/24] ARROW-6777: [GLib][CI] Unpin gobject-introspection gem gobject-introspection gem 3.3.8 or later is needed for GLib 2.62.0 or later. But gobject-introspection gem 3.3.8 has some problems. This change omits tests that are affected by these problems. Closes #5572 from kou/glib-unpin-gobject-introspection and squashes the following commits: 715b6d41d Unpin gobject-introspection gem Authored-by: Sutou Kouhei Signed-off-by: Sutou Kouhei --- c_glib/Gemfile | 2 +- c_glib/test/plasma/test-plasma-client.rb | 1 + .../test/plasma/test-plasma-created-object.rb | 3 ++ .../plasma/test-plasma-referred-object.rb | 3 ++ c_glib/test/test-cuda.rb | 2 ++ ruby/red-arrow/red-arrow.gemspec | 2 +- ruby/red-plasma/test/helper.rb | 1 + ruby/red-plasma/test/helper/omittable.rb | 36 +++++++++++++++++++ ruby/red-plasma/test/test-plasma-client.rb | 3 ++ 9 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 ruby/red-plasma/test/helper/omittable.rb diff --git a/c_glib/Gemfile b/c_glib/Gemfile index f42c1850a178..4b570902bcd7 100644 --- a/c_glib/Gemfile +++ b/c_glib/Gemfile @@ -20,4 +20,4 @@ source "https://rubygems.org/" gem "test-unit" -gem "gobject-introspection", "= 3.3.7" +gem "gobject-introspection" diff --git a/c_glib/test/plasma/test-plasma-client.rb b/c_glib/test/plasma/test-plasma-client.rb index a57d1fc5944e..14c65b4d65be 100644 --- a/c_glib/test/plasma/test-plasma-client.rb +++ b/c_glib/test/plasma/test-plasma-client.rb @@ -21,6 +21,7 @@ class TestPlasmaClient < Test::Unit::TestCase def setup @store = nil omit("Plasma is required") unless defined?(::Plasma) + require_gi_bindings(3, 3, 9) @store = Helper::PlasmaStore.new @store.start @options = Plasma::ClientOptions.new diff --git a/c_glib/test/plasma/test-plasma-created-object.rb b/c_glib/test/plasma/test-plasma-created-object.rb index 857322d20e14..8d036cda87a6 100644 --- a/c_glib/test/plasma/test-plasma-created-object.rb +++ b/c_glib/test/plasma/test-plasma-created-object.rb @@ -16,9 +16,12 @@ # under the License. class TestPlasmaCreatedObject < Test::Unit::TestCase + include Helper::Omittable + def setup @store = nil omit("Plasma is required") unless defined?(::Plasma) + require_gi_bindings(3, 3, 9) @store = Helper::PlasmaStore.new @store.start @client = Plasma::Client.new(@store.socket_path, nil) diff --git a/c_glib/test/plasma/test-plasma-referred-object.rb b/c_glib/test/plasma/test-plasma-referred-object.rb index a74641ed5dcd..ef4c72aee537 100644 --- a/c_glib/test/plasma/test-plasma-referred-object.rb +++ b/c_glib/test/plasma/test-plasma-referred-object.rb @@ -16,9 +16,12 @@ # under the License. class TestPlasmaReferredObject < Test::Unit::TestCase + include Helper::Omittable + def setup @store = nil omit("Plasma is required") unless defined?(::Plasma) + require_gi_bindings(3, 3, 9) @store = Helper::PlasmaStore.new @store.start @client = Plasma::Client.new(@store.socket_path, nil) diff --git a/c_glib/test/test-cuda.rb b/c_glib/test/test-cuda.rb index ae915307b70f..d24bab610c89 100644 --- a/c_glib/test/test-cuda.rb +++ b/c_glib/test/test-cuda.rb @@ -17,6 +17,7 @@ class TestCUDA < Test::Unit::TestCase include Helper::Buildable + include Helper::Omittable def setup omit("Arrow CUDA is required") unless defined?(::ArrowCUDA) @@ -47,6 +48,7 @@ def test_copy end def test_export + require_gi_bindings(3, 3, 9) @buffer.copy_from_host("Hello World") handle = @buffer.export serialized_handle = handle.serialize.data diff --git a/ruby/red-arrow/red-arrow.gemspec b/ruby/red-arrow/red-arrow.gemspec index 75430b5b979e..f4fcda90efe5 100644 --- a/ruby/red-arrow/red-arrow.gemspec +++ b/ruby/red-arrow/red-arrow.gemspec @@ -47,7 +47,7 @@ Gem::Specification.new do |spec| spec.extensions = ["ext/arrow/extconf.rb"] spec.add_runtime_dependency("extpp", ">= 0.0.7") - spec.add_runtime_dependency("gio2", "= 3.3.7") + spec.add_runtime_dependency("gio2", ">= 3.3.6") spec.add_runtime_dependency("native-package-installer") spec.add_runtime_dependency("pkg-config") diff --git a/ruby/red-plasma/test/helper.rb b/ruby/red-plasma/test/helper.rb index 255cad287004..02c545f53670 100644 --- a/ruby/red-plasma/test/helper.rb +++ b/ruby/red-plasma/test/helper.rb @@ -21,4 +21,5 @@ require "test-unit" +require_relative "helper/omittable" require_relative "helper/plasma-store" diff --git a/ruby/red-plasma/test/helper/omittable.rb b/ruby/red-plasma/test/helper/omittable.rb new file mode 100644 index 000000000000..a1c0334b63a2 --- /dev/null +++ b/ruby/red-plasma/test/helper/omittable.rb @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +module Helper + module Omittable + def require_gi_bindings(major, minor, micro) + return if GLib.check_binding_version?(major, minor, micro) + message = + "Require gobject-introspection #{major}.#{minor}.#{micro} or later: " + + GLib::BINDING_VERSION.join(".") + omit(message) + end + + def require_gi(major, minor, micro) + return if GObjectIntrospection::Version.or_later?(major, minor, micro) + message = + "Require GObject Introspection #{major}.#{minor}.#{micro} or later: " + + GObjectIntrospection::Version::STRING + omit(message) + end + end +end diff --git a/ruby/red-plasma/test/test-plasma-client.rb b/ruby/red-plasma/test/test-plasma-client.rb index de76fb9d36e8..d6182976c5fb 100644 --- a/ruby/red-plasma/test/test-plasma-client.rb +++ b/ruby/red-plasma/test/test-plasma-client.rb @@ -16,8 +16,11 @@ # under the License. class TestPlasmaClient < Test::Unit::TestCase + include Helper::Omittable + def setup @store = nil + require_gi_bindings(3, 3, 9) @store = Helper::PlasmaStore.new @store.start @id = Plasma::ObjectID.new("Hello") From 5050d87b5574afd0fae9e0e82a20ad12abfd6592 Mon Sep 17 00:00:00 2001 From: Taylor Baldwin Date: Thu, 3 Oct 2019 10:34:12 +0200 Subject: [PATCH 22/24] ARROW-6767: [JS] Lazily bind batches in scan/scanReverse MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I noticed some `TODO` comments in the JS client library that expressed interest in calling `bind(batch)` lazily. This PR implements that optimization and updates related tests. Happy to make updates per feedback! Created a JIRA issue [here](https://issues.apache.org/jira/browse/ARROW-6767). Closes #5565 from rolyatmax/tb/lazily-bind-batches and squashes the following commits: 8877a8f8f ARROW-6767: lazily bind batches in scan/scanReverse Authored-by: Taylor Baldwin Signed-off-by: Krisztián Szűcs --- js/src/compute/dataframe.ts | 30 ++++++++++++++++++++---------- js/test/unit/table-tests.ts | 28 ++++++++++++++-------------- 2 files changed, 34 insertions(+), 24 deletions(-) diff --git a/js/src/compute/dataframe.ts b/js/src/compute/dataframe.ts index e82e65cc1b93..db7f3d734063 100644 --- a/js/src/compute/dataframe.ts +++ b/js/src/compute/dataframe.ts @@ -132,14 +132,19 @@ export class FilteredDataFrame exte for (let batchIndex = -1; ++batchIndex < numBatches;) { // load batches const batch = batches[batchIndex]; - // TODO: bind batches lazily - // If predicate doesn't match anything in the batch we don't need - // to bind the callback - if (bind) { bind(batch); } const predicate = this._predicate.bind(batch); + let isBound = false; // yield all indices for (let index = -1, numRows = batch.length; ++index < numRows;) { - if (predicate(index, batch)) { next(index, batch); } + if (predicate(index, batch)) { + // bind batches lazily - if predicate doesn't match anything + // in the batch we don't need to call bind on the batch + if (bind && !isBound) { + bind(batch); + isBound = true; + } + next(index, batch); + } } } } @@ -149,14 +154,19 @@ export class FilteredDataFrame exte for (let batchIndex = numBatches; --batchIndex >= 0;) { // load batches const batch = batches[batchIndex]; - // TODO: bind batches lazily - // If predicate doesn't match anything in the batch we don't need - // to bind the callback - if (bind) { bind(batch); } const predicate = this._predicate.bind(batch); + let isBound = false; // yield all indices for (let index = batch.length; --index >= 0;) { - if (predicate(index, batch)) { next(index, batch); } + if (predicate(index, batch)) { + // bind batches lazily - if predicate doesn't match anything + // in the batch we don't need to call bind on the batch + if (bind && !isBound) { + bind(batch); + isBound = true; + } + next(index, batch); + } } } } diff --git a/js/test/unit/table-tests.ts b/js/test/unit/table-tests.ts index ae2f058e7dae..da74b32977af 100644 --- a/js/test/unit/table-tests.ts +++ b/js/test/unit/table-tests.ts @@ -424,6 +424,10 @@ describe(`Table`, () => { get_i32 = col('i32').bind(batch); })), expected: values.filter((row) => (row[F32] as number) * (row[I32] as number) > 0) + }, { + name: `filter out all records`, + filtered: table.filter(lit(1).eq(0)), + expected: [] } ]; for (let this_test of filter_tests) { @@ -440,15 +444,13 @@ describe(`Table`, () => { expect(columns.map((c) => c.get(idx))).toEqual(expected[expected_idx++]); }); }); - test(`calls bind function on every batch`, () => { - // Techincally, we only need to call bind on - // batches with data that match the predicate, so - // this test may fail in the future if we change - // that - and that's ok! + test(`calls bind function lazily`, () => { let bind = jest.fn(); filtered.scan(() => { }, bind); - for (let batch of table.chunks) { - expect(bind).toHaveBeenCalledWith(batch); + if (expected.length) { + expect(bind).toHaveBeenCalled(); + } else { + expect(bind).not.toHaveBeenCalled(); } }); }); @@ -460,15 +462,13 @@ describe(`Table`, () => { expect(columns.map((c) => c.get(idx))).toEqual(expected[--expected_idx]); }); }); - test(`calls bind function on every batch`, () => { - // Techincally, we only need to call bind on - // batches with data that match the predicate, so - // this test may fail in the future if we change - // that - and that's ok! + test(`calls bind function lazily`, () => { let bind = jest.fn(); filtered.scanReverse(() => { }, bind); - for (let batch of table.chunks) { - expect(bind).toHaveBeenCalledWith(batch); + if (expected.length) { + expect(bind).toHaveBeenCalled(); + } else { + expect(bind).not.toHaveBeenCalled(); } }); }); From 1165cdb85b92cefcf59ac39d35f42d168cc64517 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Thu, 3 Oct 2019 10:37:51 +0200 Subject: [PATCH 23/24] ARROW-6686: [CI] Pull and push docker images to speed up the nightly builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The images are available under https://hub.docker.com/u/ursalab with arrow prefix. Also resolves https://issues.apache.org/jira/browse/ARROW-2602 Closes #5485 from kszucs/docker-push-pull and squashes the following commits: 086c7b34d hadolint f1b5956f7 use arrowdev org 2f8564fad push only images built from arrow's master branch 36bcaa907 update dockerfiles 44f058575 ignore pull failures a0b7fb1fd disable master only push for now 60c40f572 jinja 9718b00a6 quote 4c7cee0a1 try to fix circle yml d7ab350c1 increase no_output_timeout; use newer machine image with docker_layer_caching c6f71a99b push from master only 16ec7f98b pull and push docker images to dockerhub to speed up the builds Authored-by: Krisztián Szűcs Signed-off-by: Krisztián Szűcs --- c_glib/Dockerfile | 2 +- dev/fuzzit/Dockerfile | 2 +- dev/lint/Dockerfile | 4 +- dev/tasks/docker-tests/circle.linux.yml | 23 +++++-- dev/tasks/tasks.yml | 90 ++++++++++++++++++++++++- docker-compose.yml | 80 +++++++++++----------- docs/Dockerfile | 2 +- integration/dask/Dockerfile | 2 +- integration/hdfs/Dockerfile | 2 +- integration/pandas/Dockerfile | 2 +- integration/spark/Dockerfile | 2 +- integration/turbodbc/Dockerfile | 2 +- python/Dockerfile | 2 +- python/Dockerfile.alpine | 2 +- python/Dockerfile.nopandas | 2 +- python/manylinux2010/README.md | 2 +- r/Dockerfile | 2 +- r/Dockerfile.conda | 2 +- 18 files changed, 161 insertions(+), 64 deletions(-) diff --git a/c_glib/Dockerfile b/c_glib/Dockerfile index 813695ccc7d0..32fbc79d05d6 100644 --- a/c_glib/Dockerfile +++ b/c_glib/Dockerfile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM arrow:cpp +FROM arrowdev/arrow-cpp:latest RUN apt-get update -y -q && \ apt-get -q install --no-install-recommends -y \ diff --git a/dev/fuzzit/Dockerfile b/dev/fuzzit/Dockerfile index 602cb5f254cd..447f1b844dac 100644 --- a/dev/fuzzit/Dockerfile +++ b/dev/fuzzit/Dockerfile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM arrow:cpp +FROM arrowdev/arrow-cpp:latest RUN apt-get update && \ apt-get install -y -q \ diff --git a/dev/lint/Dockerfile b/dev/lint/Dockerfile index c9658b8f1319..649120657587 100644 --- a/dev/lint/Dockerfile +++ b/dev/lint/Dockerfile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM arrow:python-3.6 +FROM arrowdev/arrow-python-3.6 RUN apt-get update && \ apt-get install -y -q \ @@ -30,4 +30,4 @@ RUN conda install flake8 && \ ENV PATH=/opt/iwyu/bin:$PATH ADD ci/docker_install_iwyu.sh /arrow/ci/ -RUN arrow/ci/docker_install_iwyu.sh \ No newline at end of file +RUN arrow/ci/docker_install_iwyu.sh diff --git a/dev/tasks/docker-tests/circle.linux.yml b/dev/tasks/docker-tests/circle.linux.yml index 7b664eeafeda..9384a7ee1821 100644 --- a/dev/tasks/docker-tests/circle.linux.yml +++ b/dev/tasks/docker-tests/circle.linux.yml @@ -19,20 +19,29 @@ version: 2 jobs: build: machine: - image: circleci/classic:201808-01 + image: ubuntu-1604:201903-01 + docker_layer_caching: true steps: - run: docker -v - run: docker-compose -v + {% if arrow.branch == "master" %} + - run: | + if [ -n $DOCKER_USER ] && [ -n $DOCKER_PASS ]; then + docker login -u $DOCKER_USER -p $DOCKER_PASS + fi + {% endif %} - run: git clone --no-checkout {{ arrow.remote }} arrow - run: git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - run: git -C arrow checkout FETCH_HEAD - run: git -C arrow submodule update --init --recursive - - run: | - pushd arrow - {%- for command in commands %} - {{ command }} - {%- endfor %} - popd + - run: + command: | + pushd arrow + {%- for command in commands %} + {{ command }} + {%- endfor %} + popd + no_output_timeout: "1h" workflows: version: 2 diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 804ed8f701d0..3c19451d08b8 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -1158,9 +1158,12 @@ tasks: template: docker-tests/circle.linux.yml params: commands: + - docker-compose pull --ignore-pull-failures cpp + - docker-compose pull --ignore-pull-failures r - docker-compose build cpp - docker-compose build r - docker-compose run r + - docker-compose push --ignore-push-failures r docker-r-conda: ci: circle @@ -1168,9 +1171,12 @@ tasks: template: docker-tests/circle.linux.yml params: commands: + - docker-compose pull --ignore-pull-failures cpp + - docker-compose pull --ignore-pull-failures r-conda - docker-compose build cpp - docker-compose build r-conda - docker-compose run r-conda + - docker-compose push --ignore-push-failures r-conda docker-r-sanitizer: ci: circle @@ -1178,8 +1184,10 @@ tasks: template: docker-tests/circle.linux.yml params: commands: + - docker-compose pull --ignore-pull-failures r-sanitizer - docker-compose build r-sanitizer - docker-compose run r-sanitizer + - docker-compose push --ignore-push-failures r-sanitizer docker-rust: ci: circle @@ -1187,8 +1195,10 @@ tasks: template: docker-tests/circle.linux.yml params: commands: + - docker-compose pull --ignore-pull-failures rust - docker-compose build rust - docker-compose run rust + - docker-compose push --ignore-push-failures rust docker-cpp: ci: circle @@ -1196,8 +1206,10 @@ tasks: template: docker-tests/circle.linux.yml params: commands: + - docker-compose pull --ignore-pull-failures cpp - docker-compose build cpp - docker-compose run cpp + - docker-compose push --ignore-push-failures cpp docker-cpp-release: ci: circle @@ -1205,8 +1217,10 @@ tasks: template: docker-tests/circle.linux.yml params: commands: - - docker-compose build cpp + - docker-compose pull --ignore-pull-failures cpp-release + - docker-compose build cpp-release - docker-compose run cpp-release + - docker-compose push --ignore-push-failures cpp-release docker-cpp-alpine: ci: circle @@ -1214,8 +1228,10 @@ tasks: template: docker-tests/circle.linux.yml params: commands: + - docker-compose pull --ignore-pull-failures cpp-alpine - docker-compose build cpp-alpine - docker-compose run cpp-alpine + - docker-compose push --ignore-push-failures cpp-alpine docker-cpp-cmake32: ci: circle @@ -1223,8 +1239,10 @@ tasks: template: docker-tests/circle.linux.yml params: commands: + - docker-compose pull --ignore-pull-failures cpp-cmake32 - docker-compose build cpp-cmake32 - docker-compose run cpp-cmake32 + - docker-compose push --ignore-push-failures cpp-cmake32 docker-cpp-fuzzit: ci: circle @@ -1232,9 +1250,12 @@ tasks: template: docker-tests/circle.linux.yml params: commands: + - docker-compose pull --ignore-pull-failures cpp + - docker-compose pull --ignore-pull-failures fuzzit - docker-compose build cpp - docker-compose build fuzzit - docker-compose run fuzzit + - docker-compose push --ignore-push-failures fuzzit docker-cpp-static-only: ci: circle @@ -1242,8 +1263,11 @@ tasks: template: docker-tests/circle.linux.yml params: commands: + - docker-compose pull --ignore-pull-failures cpp + - docker-compose pull --ignore-pull-failures cpp-static-only - docker-compose build cpp - docker-compose run cpp-static-only + - docker-compose push --ignore-push-failures cpp-static-only docker-c_glib: ci: circle @@ -1251,9 +1275,12 @@ tasks: template: docker-tests/circle.linux.yml params: commands: + - docker-compose pull --ignore-pull-failures cpp + - docker-compose pull --ignore-pull-failures c_glib - docker-compose build cpp - docker-compose build c_glib - docker-compose run c_glib + - docker-compose push --ignore-push-failures c_glib docker-go: ci: circle @@ -1261,8 +1288,10 @@ tasks: template: docker-tests/circle.linux.yml params: commands: + - docker-compose pull --ignore-pull-failures go - docker-compose build go - docker-compose run go + - docker-compose push --ignore-push-failures go docker-js: ci: circle @@ -1270,8 +1299,10 @@ tasks: template: docker-tests/circle.linux.yml params: commands: + - docker-compose pull --ignore-pull-failures js - docker-compose build js - docker-compose run js + - docker-compose push --ignore-push-failures js docker-java: ci: circle @@ -1279,8 +1310,10 @@ tasks: template: docker-tests/circle.linux.yml params: commands: + - docker-compose pull --ignore-pull-failures java - docker-compose build java - docker-compose run java + - docker-compose push --ignore-push-failures java docker-python-2.7: ci: circle @@ -1290,9 +1323,12 @@ tasks: environment: PYTHON_VERSION: 2.7 commands: + - docker-compose pull --ignore-pull-failures cpp + - docker-compose pull --ignore-pull-failures python - docker-compose build cpp - docker-compose build python - docker-compose run python + - docker-compose push --ignore-push-failures python docker-python-3.6: ci: circle @@ -1302,9 +1338,12 @@ tasks: environment: PYTHON_VERSION: 3.6 commands: + - docker-compose pull --ignore-pull-failures cpp + - docker-compose pull --ignore-pull-failures python - docker-compose build cpp - docker-compose build python - docker-compose run python + - docker-compose push --ignore-push-failures python docker-python-3.7: ci: circle @@ -1314,9 +1353,12 @@ tasks: environment: PYTHON_VERSION: 3.7 commands: + - docker-compose pull --ignore-pull-failures cpp + - docker-compose pull --ignore-pull-failures python - docker-compose build cpp - docker-compose build python - docker-compose run python + - docker-compose push --ignore-push-failures python docker-python-2.7-alpine: ci: circle @@ -1326,9 +1368,12 @@ tasks: environment: PYTHON_VERSION: 2.7 commands: + - docker-compose pull --ignore-pull-failures cpp-alpine + - docker-compose pull --ignore-pull-failures python-alpine - docker-compose build cpp-alpine - docker-compose build python-alpine - docker-compose run python-alpine + - docker-compose push --ignore-push-failures python-alpine docker-python-3.6-alpine: ci: circle @@ -1338,9 +1383,12 @@ tasks: environment: PYTHON_VERSION: 3.6 commands: + - docker-compose pull --ignore-pull-failures cpp-alpine + - docker-compose pull --ignore-pull-failures python-alpine - docker-compose build cpp-alpine - docker-compose build python-alpine - docker-compose run python-alpine + - docker-compose push --ignore-push-failures python-alpine docker-python-2.7-nopandas: ci: circle @@ -1350,10 +1398,14 @@ tasks: environment: PYTHON_VERSION: 2.7 commands: + - docker-compose pull --ignore-pull-failures cpp + - docker-compose pull --ignore-pull-failures python + - docker-compose pull --ignore-pull-failures python-nopandas - docker-compose build cpp - docker-compose build python - docker-compose build python-nopandas - docker-compose run python-nopandas + - docker-compose push --ignore-push-failures python-nopandas docker-python-3.6-nopandas: ci: circle @@ -1363,10 +1415,14 @@ tasks: environment: PYTHON_VERSION: 3.6 commands: + - docker-compose pull --ignore-pull-failures cpp + - docker-compose pull --ignore-pull-failures python + - docker-compose pull --ignore-pull-failures python-nopandas - docker-compose build cpp - docker-compose build python - docker-compose build python-nopandas - docker-compose run python-nopandas + - docker-compose push --ignore-push-failures python-nopandas ###################### Documentation building tests ######################### @@ -1378,10 +1434,14 @@ tasks: environment: PYTHON_VERSION: 3.6 commands: + - docker-compose pull --ignore-pull-failures cpp + - docker-compose pull --ignore-pull-failures python + - docker-compose pull --ignore-pull-failures docs - docker-compose build cpp - docker-compose build python - docker-compose build docs - docker-compose run docs + - docker-compose push --ignore-push-failures docs ############################## Linter tests ################################# @@ -1393,10 +1453,14 @@ tasks: environment: PYTHON_VERSION: 3.6 commands: + - docker-compose pull --ignore-pull-failures cpp + - docker-compose pull --ignore-pull-failures python + - docker-compose pull --ignore-pull-failures lint - docker-compose build cpp - docker-compose build python - docker-compose build lint - docker-compose run lint + - docker-compose push --ignore-push-failures lint docker-iwyu: ci: circle @@ -1406,6 +1470,9 @@ tasks: environment: PYTHON_VERSION: 3.6 commands: + - docker-compose pull --ignore-pull-failures cpp + - docker-compose pull --ignore-pull-failures python + - docker-compose pull --ignore-pull-failures lint - docker-compose build cpp - docker-compose build python - docker-compose build lint @@ -1419,6 +1486,9 @@ tasks: environment: PYTHON_VERSION: 3.6 commands: + - docker-compose pull --ignore-pull-failures cpp + - docker-compose pull --ignore-pull-failures python + - docker-compose pull --ignore-pull-failures lint - docker-compose build cpp - docker-compose build python - docker-compose build lint @@ -1434,10 +1504,14 @@ tasks: environment: PYTHON_VERSION: 3.6 commands: + - docker-compose pull --ignore-pull-failures cpp + - docker-compose pull --ignore-pull-failures python + - docker-compose pull --ignore-pull-failures dask-integration - docker-compose build cpp - docker-compose build python - docker-compose build dask-integration - docker-compose run dask-integration + - docker-compose push --ignore-push-failures dask-integration docker-hdfs-integration: ci: circle @@ -1447,10 +1521,14 @@ tasks: environment: PYTHON_VERSION: 3.6 commands: + - docker-compose pull --ignore-pull-failures cpp + - docker-compose pull --ignore-pull-failures python + - docker-compose pull --ignore-pull-failures hdfs-integration - docker-compose build cpp - docker-compose build python - docker-compose build hdfs-integration - docker-compose run hdfs-integration + - docker-compose push --ignore-push-failures hdfs-integration docker-spark-integration: ci: circle @@ -1460,10 +1538,14 @@ tasks: environment: PYTHON_VERSION: 3.6 commands: + - docker-compose pull --ignore-pull-failures cpp + - docker-compose pull --ignore-pull-failures python + - docker-compose pull --ignore-pull-failures spark-integration - docker-compose build cpp - docker-compose build python - docker-compose build spark-integration - docker-compose run spark-integration + - docker-compose push --ignore-push-failures spark-integration docker-turbodbc-integration: ci: circle @@ -1473,10 +1555,14 @@ tasks: environment: PYTHON_VERSION: 3.6 commands: + - docker-compose pull --ignore-pull-failures cpp + - docker-compose pull --ignore-pull-failures python + - docker-compose pull --ignore-pull-failures turbodbc-integration - docker-compose build cpp - docker-compose build python - docker-compose build turbodbc-integration - docker-compose run turbodbc-integration + - docker-compose push --ignore-push-failures turbodbc-integration docker-pandas-master: ci: circle @@ -1486,6 +1572,8 @@ tasks: environment: PYTHON_VERSION: 3.6 commands: + - docker-compose pull --ignore-pull-failures cpp + - docker-compose pull --ignore-pull-failures python - docker-compose build cpp - docker-compose build python - docker-compose build --no-cache pandas-master diff --git a/docker-compose.yml b/docker-compose.yml index 6ebf61646189..0ce7e38bc50d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -87,7 +87,7 @@ services: # Usage: # docker-compose build build-context # docker-compose run build-context - image: arrow:build-context + image: arrowdev/arrow-build-context:latest build: context: . dockerfile: dev/Dockerfile.context @@ -99,7 +99,7 @@ services: # docker-compose build cpp # docker-compose build c_glib # docker-compose run c_glib - image: arrow:c_glib + image: arrowdev/arrow-c_glib:latest build: context: . dockerfile: c_glib/Dockerfile @@ -109,7 +109,7 @@ services: # Usage: # docker-compose build cpp # docker-compose run cpp - image: arrow:cpp + image: arrowdev/arrow-cpp:latest shm_size: 2G build: context: . @@ -120,7 +120,7 @@ services: # Usage: # docker-compose build cpp-system-deps # docker-compose run cpp-system-deps - image: ursalab/arrow:cpp-system-deps + image: arrowdev/arrow-cpp-system-deps:latest shm_size: 2G cap_add: # LeakSanitizer and gdb requires ptrace(2) @@ -142,7 +142,7 @@ services: # Usage: # docker-compose build cpp # docker-compose run cpp-release - image: arrow:cpp + image: arrowdev/arrow-cpp-release:latest shm_size: 2G build: context: . @@ -156,7 +156,7 @@ services: # Usage: # docker-compose build cpp # docker-compose run cpp-static-only - image: arrow:cpp + image: arrowdev/arrow-cpp:latest shm_size: 2G environment: ARROW_BUILD_SHARED: "OFF" @@ -168,7 +168,7 @@ services: # Usage: # docker-compose build cpp # docker-compose run cpp-static-only - image: arrow:cpp + image: arrowdev/arrow-cpp:latest shm_size: 2G environment: ARROW_BUILD_STATIC: "OFF" @@ -179,7 +179,7 @@ services: # Usage: # docker-compose build cpp-cmake32 # docker-compose run cpp-cmake32 - image: arrow:cpp-cmake32 + image: arrowdev/arrow-cpp-cmake32:latest shm_size: 2G build: context: . @@ -199,7 +199,7 @@ services: # Usage: # docker-compose build cpp-alpine # docker-compose run cpp-alpine - image: arrow:cpp-alpine + image: arrowdev/arrow-cpp-alpine:latest shm_size: 2G build: context: . @@ -212,7 +212,7 @@ services: # Usage: # docker-compose build cpp-fedora # docker-compose run cpp-fedora - image: arrow:cpp-fedora + image: arrowdev/arrow-cpp-fedora:latest shm_size: 2G build: context: . @@ -225,7 +225,7 @@ services: # Usage: # docker-compose build cpp-debian-testing # docker-compose run cpp-debian-testing - image: arrow:cpp-debian-testing + image: arrowdev/arrow-cpp-debian-testing:latest shm_size: 2G build: context: . @@ -238,7 +238,7 @@ services: # Usage: # docker-compose build cpp-ubuntu-trusty # docker-compose run cpp-ubuntu-trusty - image: arrow:cpp-ubuntu-trusty + image: arrowdev/arrow-cpp-ubuntu-trusty:latest shm_size: 2G build: context: . @@ -251,7 +251,7 @@ services: # Usage: # docker-compose build cpp-ubuntu-xenial # docker-compose run cpp-ubuntu-xenial - image: arrow:cpp-ubuntu-xenial + image: arrowdev/arrow-cpp-ubuntu-xenial:latest shm_size: 2G build: context: . @@ -266,7 +266,7 @@ services: # Usage: # docker-compose build cpp-minimum-boost-version # docker-compose run cpp-minimum-boost-version - image: arrow:cpp-ubuntu-xenial + image: arrowdev/arrow-cpp-ubuntu-xenial:latest shm_size: 2G build: context: . @@ -281,7 +281,7 @@ services: # Usage: # docker-compose build cpp-ubuntu-bionic # docker-compose run cpp-ubuntu-bionic - image: arrow:cpp-ubuntu-bionic + image: arrowdev/arrow-cpp-ubuntu-bionic:latest shm_size: 2G build: context: . @@ -294,7 +294,7 @@ services: # Usage: # docker-compose build go # docker-compose run go - image: arrow:go + image: arrowdev/arrow-go:latest build: context: . dockerfile: go/Dockerfile @@ -304,7 +304,7 @@ services: # Usage: # docker-compose build java # docker-compose run java - image: arrow:java + image: arrowdev/arrow-java:latest build: context: . dockerfile: java/Dockerfile @@ -316,7 +316,7 @@ services: # Usage: # docker-compose build java-all-jdks # docker-compose run java-all-jdks - image: ursalab/arrow-ci-java-all-jdks:latest + image: arrowdev/arrow-java-all-jdks:latest build: context: . dockerfile: java/Dockerfile.all-jdks @@ -325,7 +325,7 @@ services: - maven-cache:/root/.m2:delegated js: - image: arrow:js + image: arrowdev/arrow-js:latest build: context: . dockerfile: js/Dockerfile @@ -336,7 +336,7 @@ services: # docker-compose build cpp # docker-compose build python # docker-compose run python - image: arrow:python-${PYTHON_VERSION:-3.6} + image: arrowdev/arrow-python-${PYTHON_VERSION:-3.6}:latest shm_size: 2G build: context: . @@ -351,7 +351,7 @@ services: # docker-compose build cpp-alpine # docker-compose build python-alpine # docker-compose run python-alpine - image: arrow:python-${PYTHON_VERSION:-3.6}-alpine + image: arrowdev/arrow-python-${PYTHON_VERSION:-3.6}-alpine:latest shm_size: 2G build: context: . @@ -367,7 +367,7 @@ services: # docker-compose build python # docker-compose build python-nopandas # docker-compose run python-nopandas - image: arrow:python-${PYTHON_VERSION:-3.6}-nopandas + image: arrowdev/arrow-python-${PYTHON_VERSION:-3.6}-nopandas:latest shm_size: 2G build: context: . @@ -380,7 +380,7 @@ services: # Usage: # docker-compose build rust # docker-compose run rust - image: arrow:rust + image: arrowdev/arrow-rust:latest build: context: . dockerfile: rust/Dockerfile @@ -393,7 +393,7 @@ services: # docker-compose build cpp # docker-compose build r # docker-compose run r - image: arrow:r + image: arrowdev/arrow-r:latest build: context: . dockerfile: r/Dockerfile @@ -403,7 +403,7 @@ services: # Usage: # docker-compose build r-sanitizer # docker-compose run r-sanitizer - image: arrow:r-sanitizer + image: arrowdev/arrow-r-sanitizer:latest cap_add: # LeakSanitizer and gdb requires ptrace(2) - SYS_PTRACE @@ -418,7 +418,7 @@ services: # docker-compose build cpp # docker-compose build r-conda # docker-compose run r-conda - image: arrow:r-conda-${R_VERSION:-3.5.1} + image: arrowdev/arrow-r-conda-${R_VERSION:-3.5.1}:latest shm_size: 2G build: context: . @@ -439,7 +439,7 @@ services: # docker-compose build python # docker-compose build lint # docker-compose run lint - image: arrow:lint + image: arrowdev/arrow-lint:latest build: context: . dockerfile: dev/lint/Dockerfile @@ -452,7 +452,7 @@ services: # docker-compose build cpp # docker-compose build fuzzit # docker-compose run fuzzit - image: arrow:fuzzit + image: arrowdev/arrow-fuzzit:latest build: context: . dockerfile: dev/fuzzit/Dockerfile @@ -503,7 +503,7 @@ services: # docker-compose build python # docker-compose build lint # docker-compose run iwyu - image: arrow:lint + image: arrowdev/arrow-lint:latest command: arrow/dev/lint/run_iwyu.sh volumes: *ubuntu-volumes @@ -513,7 +513,7 @@ services: # docker-compose build python # docker-compose build lint # docker-compose run clang-format - image: arrow:lint + image: arrowdev/arrow-lint:latest command: arrow/dev/lint/run_clang_format.sh volumes: *ubuntu-volumes @@ -523,7 +523,7 @@ services: # docker-compose build python # docker-compose build lint # docker-compose run clang-tidy - image: arrow:lint + image: arrowdev/arrow-lint:latest command: arrow/dev/lint/run_clang_tidy.sh volumes: *ubuntu-volumes @@ -533,7 +533,7 @@ services: # docker-compose build python # docker-compose build docs # docker-compose run docs - image: arrow:docs + image: arrowdev/arrow-docs:latest build: context: . dockerfile: docs/Dockerfile @@ -549,7 +549,7 @@ services: # $ docker-compose pull python-manylinux1 # an then run: # $ docker-compose run -e PYTHON_VERSION=3.7 python-manylinux1 - image: ursalab/arrow_manylinux1_x86_64_base:0.14.1-static-zlib + image: arrowdev/arrow_manylinux1_x86_64_base:0.14.1-static-zlib build: context: python/manylinux1 dockerfile: Dockerfile-x86_64_base @@ -571,7 +571,7 @@ services: # $ docker-compose pull python-manylinux2010 # an then run: # $ docker-compose run -e PYTHON_VERSION=3.7 python-manylinux2010 - image: ursalab/arrow_manylinux2010_x86_64_base:0.14.1-static-zlib + image: arrowdev/arrow_manylinux2010_x86_64_base:0.14.1-static-zlib build: context: python/manylinux2010 dockerfile: Dockerfile-x86_64_base @@ -600,7 +600,7 @@ services: # docker-compose build python # docker-compose build --no-cache pandas-master # docker-compose run pandas-master - image: arrow:pandas-master + image: arrowdev/arrow-pandas-master:latest build: context: . dockerfile: integration/pandas/Dockerfile @@ -643,7 +643,7 @@ services: # docker-compose build python # docker-compose build hdfs-integration # docker-compose run hdfs-integration - image: arrow:hdfs-${HDFS_VERSION:-2.9.2} + image: arrowdev/arrow-hdfs-${HDFS_VERSION:-2.9.2}:latest links: - hdfs-namenode:hdfs-namenode - hdfs-datanode-1:hdfs-datanode-1 @@ -667,7 +667,7 @@ services: # docker-compose build python # docker-compose build dask-integration # docker-compose run dask-integration - image: arrow:dask-integration + image: arrowdev/arrow-dask-integration:latest build: context: . dockerfile: integration/dask/Dockerfile @@ -680,7 +680,7 @@ services: # docker-compose build python # docker-compose build turbodbc-integration # docker-compose run turbodbc-integration - image: arrow:turbodbc-integration + image: arrowdev/arrow-turbodbc-integration:latest build: context: . dockerfile: integration/turbodbc/Dockerfile @@ -693,7 +693,7 @@ services: # docker-compose build python # docker-compose build spark-integration # docker-compose run spark-integration - image: arrow:spark-${SPARK_VERSION:-master} + image: arrowdev/arrow-spark-${SPARK_VERSION:-master}:latest environment: - SPARK_VERSION=${SPARK_VERSION:-master} build: @@ -710,7 +710,7 @@ services: ################################# Release ################################### release-source: - image: arrow:release-source + image: arrowdev/arrow-release-source:latest build: context: . dockerfile: dev/release/source/Dockerfile diff --git a/docs/Dockerfile b/docs/Dockerfile index 8c2d00b99e79..016d334f7b49 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM arrow:python-3.6 +FROM arrowdev/arrow-python-3.6:latest COPY ci/conda_env_sphinx.yml /arrow/ci/ RUN conda install --file arrow/ci/conda_env_sphinx.yml && \ diff --git a/integration/dask/Dockerfile b/integration/dask/Dockerfile index ceadb3ef8b77..a67c99072c49 100644 --- a/integration/dask/Dockerfile +++ b/integration/dask/Dockerfile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM arrow:python-3.6 +FROM arrowdev/arrow-python-3.6:latest # install dask release from conda RUN conda install -c conda-forge dask pytest=3 && \ diff --git a/integration/hdfs/Dockerfile b/integration/hdfs/Dockerfile index f173b39e0bae..71db503436d0 100644 --- a/integration/hdfs/Dockerfile +++ b/integration/hdfs/Dockerfile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM arrow:python-3.6 +FROM arrowdev/arrow-python-3.6:latest # installing libhdfs (JNI) ARG HADOOP_VERSION=2.9.2 diff --git a/integration/pandas/Dockerfile b/integration/pandas/Dockerfile index 8b56edf85d03..a1ae23e63321 100644 --- a/integration/pandas/Dockerfile +++ b/integration/pandas/Dockerfile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM arrow:python-3.6 +FROM arrowdev/arrow-python-3.6:latest # test against pandas' master RUN pip install git+https://github.com/pandas-dev/pandas.git diff --git a/integration/spark/Dockerfile b/integration/spark/Dockerfile index 6b032034986b..a0e7185d6da3 100644 --- a/integration/spark/Dockerfile +++ b/integration/spark/Dockerfile @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM arrow:python-3.6 +FROM arrowdev/arrow-python-3.6:latest # installing java and maven ARG MAVEN_VERSION=3.6.2 diff --git a/integration/turbodbc/Dockerfile b/integration/turbodbc/Dockerfile index 26df8ad709d7..d143ea748c99 100644 --- a/integration/turbodbc/Dockerfile +++ b/integration/turbodbc/Dockerfile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM arrow:python-3.6 +FROM arrowdev/arrow-python-3.6:latest RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update -y -q && \ diff --git a/python/Dockerfile b/python/Dockerfile index 5fcadeb2fb66..71bc2b2d4ee1 100644 --- a/python/Dockerfile +++ b/python/Dockerfile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM arrow:cpp +FROM arrowdev/arrow-cpp:latest # install python specific packages ARG PYTHON_VERSION=3.6 diff --git a/python/Dockerfile.alpine b/python/Dockerfile.alpine index ed7b2d2a13db..87a1dff67acc 100644 --- a/python/Dockerfile.alpine +++ b/python/Dockerfile.alpine @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM arrow:cpp-alpine +FROM arrowdev/arrow-cpp-alpine:latest SHELL ["/bin/bash", "-o", "pipefail", "-c"] diff --git a/python/Dockerfile.nopandas b/python/Dockerfile.nopandas index 58aaeed5b1ab..2faad084488a 100644 --- a/python/Dockerfile.nopandas +++ b/python/Dockerfile.nopandas @@ -16,7 +16,7 @@ # under the License. ARG PYTHON_VERSION=3.6 -FROM arrow:python-$PYTHON_VERSION +FROM arrowdev/arrow-python-$PYTHON_VERSION:latest # not installing pandas doesn't mean that it's not grabbed as a transitive # dependency, so We remove it explicitly to make sure We don't have pandas diff --git a/python/manylinux2010/README.md b/python/manylinux2010/README.md index fe2888e89df8..3a9eee750aa1 100644 --- a/python/manylinux2010/README.md +++ b/python/manylinux2010/README.md @@ -70,7 +70,7 @@ publish a build image that you built locally. ```bash $ docker push python-manylinux2010 -The push refers to repository [ursalab/arrow_manylinux2010_x86_64_base] +The push refers to repository [arrowdev/arrow_manylinux2010_x86_64_base] a1ab88d27acc: Pushing [==============> ] 492.5MB/1.645GB [... etc. ...] ``` diff --git a/r/Dockerfile b/r/Dockerfile index 97aa64b95234..63a200b8432e 100644 --- a/r/Dockerfile +++ b/r/Dockerfile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM arrow:cpp +FROM arrowdev/arrow-cpp:latest # Configure ENV CC=gcc \ diff --git a/r/Dockerfile.conda b/r/Dockerfile.conda index 15279916e2e4..f4ab0d2974aa 100644 --- a/r/Dockerfile.conda +++ b/r/Dockerfile.conda @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM arrow:cpp +FROM arrowdev/arrow-cpp:latest # install R specific packages ARG R_VERSION=3.6.1 From 560a5973aa48a24153504e8c2465c5c6998c4844 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Thu, 3 Oct 2019 10:48:27 +0200 Subject: [PATCH 24/24] ARROW-6770: [CI][Travis] Download Minio quietly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #5568 from kszucs/ARROW-6770 and squashes the following commits: ef4fd495f install minio silently Authored-by: Krisztián Szűcs Signed-off-by: Krisztián Szűcs --- ci/travis_install_minio.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/travis_install_minio.sh b/ci/travis_install_minio.sh index 5459bb8bfc4a..d3083ede18c0 100755 --- a/ci/travis_install_minio.sh +++ b/ci/travis_install_minio.sh @@ -30,6 +30,6 @@ if [ "$ARROW_TRAVIS_S3" == "1" ]; then S3FS_DIR=~/.local/bin/ mkdir -p $S3FS_DIR - wget --directory-prefix $S3FS_DIR $MINIO_URL + wget --quiet --directory-prefix $S3FS_DIR $MINIO_URL chmod +x $S3FS_DIR/minio -fi \ No newline at end of file +fi