diff --git a/.github/workflows/linux-docker-compose.yml b/.github/workflows/linux-docker-compose.yml new file mode 100644 index 000000000000..d1198572b554 --- /dev/null +++ b/.github/workflows/linux-docker-compose.yml @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Linux docker-compose +on: + - push + - pull_request +jobs: + build: + name: Test + strategy: + matrix: + label: + - C++ w/ clang-7 & system packages + include: + - label: C++ w/ clang-7 & system packages + image: cpp-system-deps + skip_expression: | + ${ARROW_CI_CPP_AFFECTED} != "1" + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@master + with: + submodules: true + - name: docker-compose + run: | + eval "$(python ci/detect-changes.py)" + if [[ ${{ matrix.skip_expression }} ]]; then + exit + fi + + docker-compose build --pull ${{ matrix.image }} + docker-compose run ${{ matrix.image }} diff --git a/.travis.yml b/.travis.yml index 4fc143a493db..a107bcc3f4b9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,10 +26,6 @@ cache: - $HOME/.ccache - $HOME/.m2 # Maven -env: - - CCACHE_COMPRESS: 1 - - CCACHE_COMPRESSLEVEL: 5 - git: # ARROW-3083: We use a higher clone depth to also have the latest tag in the cloned history. depth: 250 @@ -40,6 +36,8 @@ before_install: matrix: fast_finish: true include: + + # ------------------------------------------------------------------------- - name: "Lint, Release tests" os: linux language: python @@ -60,6 +58,8 @@ matrix: - docker-compose build --pull cpp-system-deps script: - docker-compose run cpp-system-deps + + # ------------------------------------------------------------------------- # Separating Valgrind and C++ coverage makes individual jobs shorter - name: "C++ unit tests w/ conda-forge toolchain, coverage" compiler: gcc @@ -87,6 +87,7 @@ matrix: - if [ $ARROW_CI_CPP_AFFECTED != "1" ] && [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh + - $TRAVIS_BUILD_DIR/ci/travis_install_minio.sh # If either C++ or Python changed, we must install the C++ libraries - git submodule update --init - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh @@ -94,6 +95,8 @@ matrix: - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1 - $TRAVIS_BUILD_DIR/ci/travis_script_gandiva_java.sh || travis_terminate 1 - $TRAVIS_BUILD_DIR/ci/travis_upload_cpp_coverage.sh || travis_terminate 1 + + # ------------------------------------------------------------------------- - name: "Python 3.6 unit tests w/ Valgrind, conda-forge toolchain, coverage" compiler: gcc language: cpp @@ -110,12 +113,14 @@ matrix: - ARROW_TRAVIS_USE_SYSTEM_JAVA=1 - ARROW_TRAVIS_USE_TOOLCHAIN=1 - ARROW_TRAVIS_VALGRIND=1 + - ARROW_TRAVIS_S3=1 # TODO(wesm): Run the benchmarks outside of Travis # - ARROW_TRAVIS_PYTHON_BENCHMARKS=1 before_script: - if [ $ARROW_CI_PYTHON_AFFECTED != "1" ] && [ $ARROW_CI_DOCS_AFFECTED != "1" ]; then exit; fi - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh + - $TRAVIS_BUILD_DIR/ci/travis_install_minio.sh - $TRAVIS_BUILD_DIR/ci/travis_install_toolchain.sh script: - $TRAVIS_BUILD_DIR/ci/travis_script_java.sh || travis_terminate 1 @@ -125,17 +130,19 @@ matrix: - export PLASMA_VALGRIND=1 - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6 - $TRAVIS_BUILD_DIR/ci/travis_upload_cpp_coverage.sh + + # ------------------------------------------------------------------------- - name: "[OS X] C++ w/ XCode 9.3" compiler: clang osx_image: xcode9.3 os: osx - cache: addons: env: - ARROW_TRAVIS_USE_TOOLCHAIN=1 - ARROW_TRAVIS_PLASMA=1 - ARROW_TRAVIS_FLIGHT=1 - ARROW_TRAVIS_ORC=1 + - ARROW_TRAVIS_S3=1 - ARROW_TRAVIS_PARQUET=1 # TODO(ARROW-4763): llvm and llvmdev packages are in conflict: # https://github.com/conda-forge/llvmdev-feedstock/issues/60 @@ -149,18 +156,21 @@ matrix: - if [ $ARROW_CI_CPP_AFFECTED != "1" ] && [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi # If either C++ or Python changed, we must install the C++ libraries - git submodule update --init + - $TRAVIS_BUILD_DIR/ci/travis_install_minio.sh - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh script: - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1 # Disabled because of ARROW-4763 # - $TRAVIS_BUILD_DIR/ci/travis_script_gandiva_java.sh + + # ------------------------------------------------------------------------- - name: "[OS X] Python w/ XCode 9.3" compiler: clang osx_image: xcode9.3 os: osx - cache: addons: env: + - ARROW_TRAVIS_S3=1 - ARROW_TRAVIS_PLASMA=1 - ARROW_TRAVIS_USE_TOOLCHAIN=1 - ARROW_BUILD_WARNING_LEVEL=CHECKIN @@ -170,7 +180,10 @@ matrix: before_script: script: - if [ $ARROW_CI_PYTHON_AFFECTED != "1" ]; then exit; fi + - $TRAVIS_BUILD_DIR/ci/travis_install_minio.sh - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6 + + # ------------------------------------------------------------------------- - name: "Java OpenJDK8 and OpenJDK11" language: cpp os: linux @@ -179,6 +192,8 @@ matrix: - docker-compose build --pull java-all-jdks script: - docker-compose run java-all-jdks + + # ------------------------------------------------------------------------- - name: "Integration w/ OpenJDK 8, conda-forge toolchain" language: java os: linux @@ -206,6 +221,8 @@ matrix: script: - $TRAVIS_BUILD_DIR/ci/travis_script_integration.sh # - $TRAVIS_BUILD_DIR/ci/travis_script_plasma_java_client.sh + + # ------------------------------------------------------------------------- - name: "NodeJS" language: node_js os: linux @@ -217,6 +234,8 @@ matrix: - $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh script: - $TRAVIS_BUILD_DIR/ci/travis_script_js.sh + + # ------------------------------------------------------------------------- - name: "C++ & GLib & Ruby w/ gcc 5.4" compiler: gcc os: linux @@ -236,6 +255,8 @@ matrix: script: - if [ $ARROW_CI_C_GLIB_AFFECTED = "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh; fi - $TRAVIS_BUILD_DIR/ci/travis_script_ruby.sh + + # ------------------------------------------------------------------------- - name: "[OS X] C++ & GLib & Ruby w/ XCode 10.2 & Homebrew" compiler: clang osx_image: xcode10.2 @@ -249,7 +270,6 @@ matrix: - ARROW_TRAVIS_PYTHON=1 - ARROW_TRAVIS_VERBOSE=1 - ARROW_TRAVIS_MIMALLOC=1 - cache: addons: before_script: - if [ $ARROW_CI_RUBY_AFFECTED != "1" ]; then exit; fi @@ -260,6 +280,8 @@ matrix: script: - if [ $ARROW_CI_C_GLIB_AFFECTED = "1" ]; then $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh; fi - $TRAVIS_BUILD_DIR/ci/travis_script_ruby.sh + + # ------------------------------------------------------------------------- - name: Rust language: rust cache: cargo @@ -276,7 +298,7 @@ matrix: - if [ $ARROW_CI_RUST_AFFECTED != "1" ]; then exit; fi - $TRAVIS_BUILD_DIR/ci/travis_install_cargo.sh script: - - RUSTUP_TOOLCHAIN=nightly $TRAVIS_BUILD_DIR/ci/travis_script_rust.sh + - $TRAVIS_BUILD_DIR/ci/travis_script_rust.sh before_cache: cargo install cargo-tarpaulin -f after_success: @@ -286,6 +308,8 @@ matrix: - export PARQUET_TEST_DATA=$TRAVIS_BUILD_DIR/cpp/submodules/parquet-testing/data - cargo tarpaulin --out Xml - bash <(curl -s https://codecov.io/bash) || echo "Codecov did not collect coverage reports" + + # ------------------------------------------------------------------------- - name: Go language: go go_import_path: github.com/apache/arrow @@ -299,9 +323,14 @@ matrix: after_success: - pushd ${TRAVIS_BUILD_DIR}/go/arrow - bash <(curl -s https://codecov.io/bash) || echo "Codecov did not collect coverage reports" + + # ------------------------------------------------------------------------- - name: R (with and without libarrow) language: r - cache: packages + cache: + directories: + - $HOME/.ccache + packages: true dist: xenial env: - ARROW_TRAVIS_PARQUET=1 @@ -311,6 +340,7 @@ matrix: # Have to copy-paste this here because of how R's build steps work - eval `python $TRAVIS_BUILD_DIR/ci/detect-changes.py` - if [ $ARROW_CI_R_AFFECTED != "1" ]; then exit; fi + - source ci/travis/before-install.sh # First check that it builds without libarrow - pushd ${TRAVIS_BUILD_DIR}/r - R -e 'install.packages("remotes"); remotes::install_deps(dep = TRUE)' @@ -341,10 +371,10 @@ matrix: - Rscript ../ci/travis_upload_r_coverage.R after_failure: - - source ci/travis/after-failure.sh + - source ${TRAVIS_BUILD_DIR}/ci/travis/after-failure.sh after_script: - - source ci/travis/after-script.sh + - source ${TRAVIS_BUILD_DIR}/ci/travis/after-script.sh env: global: diff --git a/c_glib/Dockerfile b/c_glib/Dockerfile index 813695ccc7d0..32fbc79d05d6 100644 --- a/c_glib/Dockerfile +++ b/c_glib/Dockerfile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM arrow:cpp +FROM arrowdev/arrow-cpp:latest RUN apt-get update -y -q && \ apt-get -q install --no-install-recommends -y \ diff --git a/c_glib/Gemfile b/c_glib/Gemfile index f42c1850a178..4b570902bcd7 100644 --- a/c_glib/Gemfile +++ b/c_glib/Gemfile @@ -20,4 +20,4 @@ source "https://rubygems.org/" gem "test-unit" -gem "gobject-introspection", "= 3.3.7" +gem "gobject-introspection" diff --git a/c_glib/test/plasma/test-plasma-client.rb b/c_glib/test/plasma/test-plasma-client.rb index a57d1fc5944e..14c65b4d65be 100644 --- a/c_glib/test/plasma/test-plasma-client.rb +++ b/c_glib/test/plasma/test-plasma-client.rb @@ -21,6 +21,7 @@ class TestPlasmaClient < Test::Unit::TestCase def setup @store = nil omit("Plasma is required") unless defined?(::Plasma) + require_gi_bindings(3, 3, 9) @store = Helper::PlasmaStore.new @store.start @options = Plasma::ClientOptions.new diff --git a/c_glib/test/plasma/test-plasma-created-object.rb b/c_glib/test/plasma/test-plasma-created-object.rb index 857322d20e14..8d036cda87a6 100644 --- a/c_glib/test/plasma/test-plasma-created-object.rb +++ b/c_glib/test/plasma/test-plasma-created-object.rb @@ -16,9 +16,12 @@ # under the License. class TestPlasmaCreatedObject < Test::Unit::TestCase + include Helper::Omittable + def setup @store = nil omit("Plasma is required") unless defined?(::Plasma) + require_gi_bindings(3, 3, 9) @store = Helper::PlasmaStore.new @store.start @client = Plasma::Client.new(@store.socket_path, nil) diff --git a/c_glib/test/plasma/test-plasma-referred-object.rb b/c_glib/test/plasma/test-plasma-referred-object.rb index a74641ed5dcd..ef4c72aee537 100644 --- a/c_glib/test/plasma/test-plasma-referred-object.rb +++ b/c_glib/test/plasma/test-plasma-referred-object.rb @@ -16,9 +16,12 @@ # under the License. class TestPlasmaReferredObject < Test::Unit::TestCase + include Helper::Omittable + def setup @store = nil omit("Plasma is required") unless defined?(::Plasma) + require_gi_bindings(3, 3, 9) @store = Helper::PlasmaStore.new @store.start @client = Plasma::Client.new(@store.socket_path, nil) diff --git a/c_glib/test/test-cuda.rb b/c_glib/test/test-cuda.rb index ae915307b70f..d24bab610c89 100644 --- a/c_glib/test/test-cuda.rb +++ b/c_glib/test/test-cuda.rb @@ -17,6 +17,7 @@ class TestCUDA < Test::Unit::TestCase include Helper::Buildable + include Helper::Omittable def setup omit("Arrow CUDA is required") unless defined?(::ArrowCUDA) @@ -47,6 +48,7 @@ def test_copy end def test_export + require_gi_bindings(3, 3, 9) @buffer.copy_from_host("Hello World") handle = @buffer.export serialized_handle = handle.serialize.data diff --git a/ci/conda_env_python.yml b/ci/conda_env_python.yml index a0cd737b326c..0e6e5bf8d5c3 100644 --- a/ci/conda_env_python.yml +++ b/ci/conda_env_python.yml @@ -22,6 +22,7 @@ numpy>=1.14 pandas pytest pytest-faulthandler +pytest-lazy-fixture pytz setuptools setuptools_scm=3.2.0 diff --git a/ci/cpp-msvc-build-main.bat b/ci/cpp-msvc-build-main.bat index b088e2eec763..b6d1b20a5109 100644 --- a/ci/cpp-msvc-build-main.bat +++ b/ci/cpp-msvc-build-main.bat @@ -98,6 +98,9 @@ pip install -r requirements.txt pickle5 set PYARROW_CXXFLAGS=%ARROW_CXXFLAGS% set PYARROW_CMAKE_GENERATOR=%GENERATOR% +if "%ARROW_S3%" == "ON" ( + set PYARROW_WITH_S3=ON +) if "%ARROW_BUILD_FLIGHT%" == "ON" ( @rem ARROW-5441: bundling Arrow Flight libraries not implemented set PYARROW_BUNDLE_ARROW_CPP=OFF diff --git a/ci/detect-changes.py b/ci/detect-changes.py index d35d9b570b9a..424cd99d8663 100644 --- a/ci/detect-changes.py +++ b/ci/detect-changes.py @@ -129,6 +129,16 @@ def list_appveyor_affected_files(): return list_affected_files("{0}..HEAD".format(merge_base)) +def list_github_actions_affected_files(): + """ + Return a list of files affected in the current GitHub Actions build. + """ + # GitHub Actions checkout `refs/remotes/pull/$PR/merge` where `HEAD` points + # to the merge commit while `HEAD^` points to the commit before. Hence, + # `..HEAD^` points to all commit between master and the PR. + return list_affected_files("HEAD^..") + + LANGUAGE_TOPICS = ['c_glib', 'cpp', 'docs', 'go', 'java', 'js', 'python', 'r', 'ruby', 'rust', 'csharp'] @@ -143,6 +153,9 @@ def list_appveyor_affected_files(): 'format': LANGUAGE_TOPICS, 'go': ['integration'], '.travis.yml': ALL_TOPICS, + # In theory, it should ignore CONTRIBUTING.md and ISSUE_TEMPLATE.md, but in + # practice it's going to be CI + '.github': ALL_TOPICS, 'c_glib': ['ruby'] } @@ -255,6 +268,23 @@ def run_from_appveyor(): return get_windows_shell_eval(make_env_for_topics(affected)) +def run_from_github(): + perr("Environment variables (excerpt):") + dump_env_vars('GITHUB_', '(REPOSITORY|ACTOR|SHA|REF|HEAD_REF|BASE_REF|EVENT_NAME)') + if os.environ['GITHUB_EVENT_NAME'] != 'pull_request': + # Not a PR build, test everything + affected = dict.fromkeys(ALL_TOPICS, True) + else: + affected_files = list_github_actions_affected_files() + perr("Affected files:", affected_files) + affected = get_affected_topics(affected_files) + assert set(affected) <= set(ALL_TOPICS), affected + + perr("Affected topics:") + perr(pprint.pformat(affected)) + return get_unix_shell_eval(make_env_for_topics(affected)) + + def test_get_affected_topics(): affected_topics = get_affected_topics(['cpp/CMakeLists.txt']) assert affected_topics == { @@ -290,6 +320,23 @@ def test_get_affected_topics(): 'dev': False } + affected_topics = get_affected_topics(['.github/workflows']) + assert affected_topics == { + 'c_glib': True, + 'cpp': True, + 'docs': True, + 'go': True, + 'java': True, + 'js': True, + 'python': True, + 'r': True, + 'ruby': True, + 'rust': True, + 'csharp': True, + 'integration': True, + 'dev': True, + } + if __name__ == "__main__": # This script should have its output evaluated by a shell, @@ -307,5 +354,11 @@ def test_get_affected_topics(): except Exception: print("exit 1") raise + elif os.environ.get('GITHUB_WORKFLOW'): + try: + print(run_from_github()) + except Exception: + print("exit 1") + raise else: - sys.exit("Script must be run under Travis-CI or AppVeyor") + sys.exit("Script must be run under Travis-CI, AppVeyor or GitHub Actions") diff --git a/ci/docker_build_and_fuzzit.sh b/ci/docker_build_and_fuzzit.sh index 18bd0d4c599e..b3bf1bd81fed 100755 --- a/ci/docker_build_and_fuzzit.sh +++ b/ci/docker_build_and_fuzzit.sh @@ -24,13 +24,14 @@ export ARROW_USE_ASAN="ON" export CC="clang-7" export CXX="clang++-7" export ARROW_BUILD_TYPE="RelWithDebInfo" +export ARROW_FLIGHT="OFF" +export ARROW_GANDIVA="OFF" export ARROW_ORC="OFF" export ARROW_PARQUET="OFF" export ARROW_PLASMA="OFF" -export ARROW_FLIGHT="OFF" -export ARROW_BUILD_BENCHMARKS="OFF" export ARROW_WITH_BZ2="OFF" export ARROW_WITH_ZSTD="OFF" +export ARROW_BUILD_BENCHMARKS="OFF" export ARROW_BUILD_UTILITIES="OFF" /arrow/ci/docker_build_cpp.sh || exit 1 pushd /build/cpp @@ -40,11 +41,12 @@ cp ./relwithdebinfo/arrow-ipc-fuzzing-test ./relwithdebinfo/out/fuzzer ldd ./relwithdebinfo/arrow-ipc-fuzzing-test | grep "=> /" | awk '{print $3}' | xargs -I '{}' cp -v '{}' ./relwithdebinfo/out/. cd ./relwithdebinfo/out/ tar -czvf fuzzer.tar.gz * +stat fuzzer.tar.gz cd ../../ -export TARGET_ID=u79f6bXYgNH4NkU99iWK -export FUZZIT_API_KEY=${FUZZIT_API_KEY:-ac6089a1bc2313679f2d99bb80553162c380676bff3f094de826b16229e28184a8084b86f52c95112bde6b3dbb07b9b7} -wget -O fuzzit https://bin.fuzzit.dev/fuzzit-1.1 +export TARGET_ID=apache-arrow/arrow-ipc-fuzzing + +wget -O fuzzit https://github.com/fuzzitdev/fuzzit/releases/latest/download/fuzzit_Linux_x86_64 chmod a+x fuzzit -./fuzzit auth $FUZZIT_API_KEY -./fuzzit create job --type fuzzing --host bionic-llvm7 --revision $CI_ARROW_SHA --branch $CI_ARROW_BRANCH $TARGET_ID ./relwithdebinfo/out/fuzzer.tar.gz + +./fuzzit create job --type $FUZZIT_JOB_TYPE --host bionic-llvm7 --revision $CI_ARROW_SHA --branch $CI_ARROW_BRANCH $TARGET_ID ./relwithdebinfo/out/fuzzer.tar.gz diff --git a/ci/docker_build_cpp.sh b/ci/docker_build_cpp.sh index 29860bf7199a..8eb96a21bfd5 100755 --- a/ci/docker_build_cpp.sh +++ b/ci/docker_build_cpp.sh @@ -22,7 +22,11 @@ source_dir=${1:-/arrow/cpp} build_dir=${2:-/build/cpp} install_dir=${3:-${ARROW_HOME:-/usr/local}} +export CCACHE_COMPRESS=1 +export CCACHE_COMPRESSLEVEL=5 +export CCACHE_COMPILERCHECK=content export CCACHE_DIR=/build/ccache +export PATH=/usr/lib/ccache/:$PATH rm -rf ${build_dir} mkdir -p ${build_dir} diff --git a/ci/travis/before-install.sh b/ci/travis/before-install.sh index fd97b469341c..1eb45fd436b5 100755 --- a/ci/travis/before-install.sh +++ b/ci/travis/before-install.sh @@ -30,6 +30,11 @@ if [[ "${TRAVIS_OS_NAME}" == "linux" ]]; then echo -e 'Acquire::Retries 10; Acquire::http::Timeout \"20\";' | \ sudo tee /etc/apt/apt.conf.d/99-travis-retry sudo apt-get update -qq + + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=5 + export CCACHE_COMPILERCHECK=content + export PATH=/usr/lib/ccache/:$PATH ccache --show-stats fi diff --git a/ci/travis_install_linux.sh b/ci/travis_install_linux.sh index a5283139a833..441d50d9b16c 100755 --- a/ci/travis_install_linux.sh +++ b/ci/travis_install_linux.sh @@ -42,14 +42,6 @@ if [ "$ARROW_TRAVIS_GANDIVA" == "1" ]; then sudo apt-get install -y -qq llvm-$ARROW_LLVM_MAJOR_VERSION-dev fi -if [ "$ARROW_TRAVIS_S3" == "1" ]; then - # Download the Minio S3 server into PATH - S3FS_DIR=~/.local/bin/ - mkdir -p $S3FS_DIR - wget --directory-prefix $S3FS_DIR https://dl.min.io/server/minio/release/linux-amd64/minio - chmod +x $S3FS_DIR/minio -fi - if [ "$ARROW_TRAVIS_USE_SYSTEM" == "1" ]; then if [ "$DISTRO_CODENAME" == "xenial" ]; then # TODO(ARROW-4761): Install libzstd-dev once we support zstd<1 diff --git a/ci/travis_install_minio.sh b/ci/travis_install_minio.sh new file mode 100755 index 000000000000..d3083ede18c0 --- /dev/null +++ b/ci/travis_install_minio.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +set -x + +if [ "$ARROW_TRAVIS_S3" == "1" ]; then + # Download the Minio S3 server into PATH + if [ $TRAVIS_OS_NAME = "osx" ]; then + MINIO_URL=https://dl.min.io/server/minio/release/darwin-amd64/minio + else + MINIO_URL=https://dl.min.io/server/minio/release/linux-amd64/minio + fi + + S3FS_DIR=~/.local/bin/ + mkdir -p $S3FS_DIR + wget --quiet --directory-prefix $S3FS_DIR $MINIO_URL + chmod +x $S3FS_DIR/minio +fi diff --git a/ci/travis_install_osx.sh b/ci/travis_install_osx.sh index 38e971710dea..2d79eb017ed5 100755 --- a/ci/travis_install_osx.sh +++ b/ci/travis_install_osx.sh @@ -40,4 +40,4 @@ if [ "$ARROW_CI_RUBY_AFFECTED" = "1" ]; then run_brew bundle --file=$TRAVIS_BUILD_DIR/cpp/Brewfile --verbose run_brew bundle --file=$TRAVIS_BUILD_DIR/c_glib/Brewfile --verbose rm ${brew_log_path} -fi +fi \ No newline at end of file diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh index 6b095b878e59..8b3066f6e768 100755 --- a/ci/travis_script_python.sh +++ b/ci/travis_script_python.sh @@ -100,6 +100,10 @@ CMAKE_COMMON_FLAGS="-DARROW_EXTRA_ERROR_CONTEXT=ON" PYTHON_CPP_BUILD_TARGETS="arrow_python-all plasma parquet" +if [ "$ARROW_TRAVIS_S3" == "1" ]; then + CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_S3=ON" +fi + if [ "$ARROW_TRAVIS_FLIGHT" == "1" ]; then CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -DARROW_FLIGHT=ON" fi @@ -164,6 +168,9 @@ export PYARROW_BUILD_TYPE=$ARROW_BUILD_TYPE export PYARROW_WITH_PARQUET=1 export PYARROW_WITH_PLASMA=1 export PYARROW_WITH_ORC=1 +if [ "$ARROW_TRAVIS_S3" == "1" ]; then + export PYARROW_WITH_S3=1 +fi if [ "$ARROW_TRAVIS_FLIGHT" == "1" ]; then export PYARROW_WITH_FLIGHT=1 fi @@ -177,6 +184,7 @@ python setup.py develop python -c "import pyarrow.parquet" python -c "import pyarrow.plasma" python -c "import pyarrow.orc" +python -c "import pyarrow.fs" # Ensure we do eagerly import pandas (or other expensive imports) python < scripts/test_imports.py diff --git a/ci/travis_script_rust.sh b/ci/travis_script_rust.sh index 0f6f619e32e0..22059df228f4 100755 --- a/ci/travis_script_rust.sh +++ b/ci/travis_script_rust.sh @@ -31,11 +31,16 @@ rustup show # raises on any formatting errors cargo +stable fmt --all -- --check +# build entire project RUSTFLAGS="-D warnings" cargo build --all-targets + +# run tests +cargo test + +# make sure we can build Arrow sub-crate without default features pushd arrow cargo build --no-default-features popd -cargo test # run Arrow examples pushd arrow diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 2ab620d8f6d7..ea1a5756587c 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -268,10 +268,14 @@ if(ARROW_BUILD_TESTS OR ARROW_BUILD_INTEGRATION) set(ARROW_JSON ON) endif() -if(ARROW_FLIGHT OR ARROW_PARQUET OR ARROW_BUILD_TESTS) +if(ARROW_CUDA OR ARROW_FLIGHT OR ARROW_PARQUET OR ARROW_BUILD_TESTS) set(ARROW_IPC ON) endif() +if(ARROW_DATASET) + set(ARROW_FILESYSTEM ON) +endif() + if(MSVC) # ORC doesn't build on windows set(ARROW_ORC OFF) diff --git a/cpp/Dockerfile.ubuntu-bionic b/cpp/Dockerfile.ubuntu-bionic index 763e13e50c87..325ebb7ee76b 100644 --- a/cpp/Dockerfile.ubuntu-bionic +++ b/cpp/Dockerfile.ubuntu-bionic @@ -71,12 +71,15 @@ RUN apt-get update -y -q && \ libgoogle-glog-dev \ liblz4-dev \ liblzma-dev \ + libprotobuf-dev \ + libprotoc-dev \ libre2-dev \ libsnappy-dev \ libssl-dev \ libzstd-dev \ ninja-build \ pkg-config \ + protobuf-compiler \ rapidjson-dev \ thrift-compiler \ tzdata && \ @@ -92,14 +95,11 @@ RUN apt-get update -y -q && \ ENV CMAKE_ARGS="-DThrift_SOURCE=BUNDLED \ -DFlatbuffers_SOURCE=BUNDLED \ -DGTest_SOURCE=BUNDLED \ --DORC_SOURCE=BUNDLED \ --Dc-ares_SOURCE=BUNDLED \ --DgRPC_SOURCE=BUNDLED \ --DProtobuf_SOURCE=BUNDLED ${CMAKE_ARGS}" +-DORC_SOURCE=BUNDLED" # Prioritize system packages and local installation ENV ARROW_DEPENDENCY_SOURCE=SYSTEM \ - ARROW_FLIGHT=ON \ + ARROW_FLIGHT=OFF \ ARROW_GANDIVA=ON \ ARROW_HDFS=ON \ ARROW_ORC=ON \ diff --git a/cpp/cmake_modules/DefineOptions.cmake b/cpp/cmake_modules/DefineOptions.cmake index 3764f4824962..7a0158ef4c61 100644 --- a/cpp/cmake_modules/DefineOptions.cmake +++ b/cpp/cmake_modules/DefineOptions.cmake @@ -90,21 +90,19 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") #---------------------------------------------------------------------- set_option_category("Test and benchmark") - define_option(ARROW_BUILD_EXAMPLES "Build the Arrow examples, default OFF" OFF) + define_option(ARROW_BUILD_EXAMPLES "Build the Arrow examples" OFF) - define_option(ARROW_BUILD_TESTS "Build the Arrow googletest unit tests, default OFF" - OFF) - - define_option(ARROW_BUILD_INTEGRATION - "Build the Arrow integration test executables, default OFF" OFF) + define_option(ARROW_BUILD_TESTS "Build the Arrow googletest unit tests" OFF) - define_option(ARROW_BUILD_BENCHMARKS "Build the Arrow micro benchmarks, default OFF" + define_option(ARROW_BUILD_INTEGRATION "Build the Arrow integration test executables" OFF) + define_option(ARROW_BUILD_BENCHMARKS "Build the Arrow micro benchmarks" OFF) + # Reference benchmarks are used to compare to naive implementation, or # discover various hardware limits. define_option(ARROW_BUILD_BENCHMARKS_REFERENCE - "Build the Arrow micro reference benchmarks, default OFF." OFF) + "Build the Arrow micro reference benchmarks" OFF) define_option_string(ARROW_TEST_LINKAGE "Linkage of Arrow libraries with unit tests executables." @@ -141,47 +139,49 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") #---------------------------------------------------------------------- set_option_category("Project component") + define_option(ARROW_BUILD_UTILITIES "Build Arrow commandline utilities" ON) + define_option(ARROW_COMPUTE "Build the Arrow Compute Modules" ON) + define_option(ARROW_CUDA "Build the Arrow CUDA extensions (requires CUDA toolkit)" OFF) + define_option(ARROW_DATASET "Build the Arrow Dataset Modules" ON) + define_option(ARROW_FILESYSTEM "Build the Arrow Filesystem Layer" ON) + define_option(ARROW_FLIGHT "Build the Arrow Flight RPC System (requires GRPC, Protocol Buffers)" OFF) define_option(ARROW_GANDIVA "Build the Gandiva libraries" OFF) - define_option(ARROW_PARQUET "Build the Parquet libraries" OFF) - - define_option(ARROW_IPC "Build the Arrow IPC extensions" ON) + define_option(ARROW_HDFS "Build the Arrow HDFS bridge" ON) - define_option(ARROW_BUILD_UTILITIES "Build Arrow commandline utilities" ON) + define_option(ARROW_HIVESERVER2 "Build the HiveServer2 client and Arrow adapter" OFF) - define_option(ARROW_CUDA "Build the Arrow CUDA extensions (requires CUDA toolkit)" OFF) + define_option(ARROW_IPC "Build the Arrow IPC extensions" ON) - define_option(ARROW_ORC "Build the Arrow ORC adapter" OFF) + define_option(ARROW_JEMALLOC "Build the Arrow jemalloc-based allocator" ON) define_option(ARROW_JNI "Build the Arrow JNI lib" OFF) - define_option(ARROW_TENSORFLOW "Build Arrow with TensorFlow support enabled" OFF) - - define_option(ARROW_JEMALLOC "Build the Arrow jemalloc-based allocator" ON) + define_option(ARROW_JSON "Build Arrow with JSON support (requires RapidJSON)" ON) define_option(ARROW_MIMALLOC "Build the Arrow mimalloc-based allocator" OFF) - define_option(ARROW_HDFS "Build the Arrow HDFS bridge" ON) - - define_option(ARROW_PYTHON "Build the Arrow CPython extensions" OFF) + define_option(ARROW_PARQUET "Build the Parquet libraries" OFF) - define_option(ARROW_HIVESERVER2 "Build the HiveServer2 client and Arrow adapter" OFF) + define_option(ARROW_ORC "Build the Arrow ORC adapter" OFF) define_option(ARROW_PLASMA "Build the plasma object store along with Arrow" OFF) define_option(ARROW_PLASMA_JAVA_CLIENT "Build the plasma object store java client" OFF) - define_option(ARROW_JSON "Build Arrow with JSON support (requires RapidJSON)" ON) + define_option(ARROW_PYTHON "Build the Arrow CPython extensions" OFF) define_option(ARROW_S3 "Build Arrow with S3 support (requires the AWS SDK for C++)" OFF) + define_option(ARROW_TENSORFLOW "Build Arrow with TensorFlow support enabled" OFF) + #---------------------------------------------------------------------- set_option_category("Thirdparty toolchain") diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index e8e4fa6f7bc7..ace517124b65 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -15,6 +15,9 @@ # specific language governing permissions and limitations # under the License. +include(ProcessorCount) +processorcount(NPROC) + add_custom_target(rapidjson) add_custom_target(toolchain) add_custom_target(toolchain-benchmarks) @@ -500,7 +503,7 @@ if(${CMAKE_GENERATOR} MATCHES "Makefiles") set(MAKE_BUILD_ARGS "") else() # limit the maximum number of jobs for ninja - set(MAKE_BUILD_ARGS "-j4") + set(MAKE_BUILD_ARGS "-j${NPROC}") endif() # ---------------------------------------------------------------------- @@ -560,7 +563,7 @@ macro(build_boost) else() set(BOOST_BUILD_VARIANT "release") endif() - set(BOOST_BUILD_COMMAND "./b2" "link=${BOOST_BUILD_LINK}" + set(BOOST_BUILD_COMMAND "./b2" "-j${NPROC}" "link=${BOOST_BUILD_LINK}" "variant=${BOOST_BUILD_VARIANT}") if(MSVC) string(REGEX @@ -638,6 +641,10 @@ if(TARGET Boost::system) set(BOOST_SYSTEM_LIBRARY Boost::system) set(BOOST_FILESYSTEM_LIBRARY Boost::filesystem) set(BOOST_REGEX_LIBRARY Boost::regex) +elseif(BoostAlt_FOUND) + set(BOOST_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY}) + set(BOOST_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY}) + set(BOOST_REGEX_LIBRARY ${Boost_REGEX_LIBRARY}) else() set(BOOST_SYSTEM_LIBRARY boost_system_static) set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_static) @@ -645,8 +652,8 @@ else() endif() set(ARROW_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY}) -message(STATUS "Boost include dir: " ${Boost_INCLUDE_DIR}) -message(STATUS "Boost libraries: " ${Boost_LIBRARIES}) +message(STATUS "Boost include dir: ${Boost_INCLUDE_DIR}") +message(STATUS "Boost libraries: ${ARROW_BOOST_LIBS}") include_directories(SYSTEM ${Boost_INCLUDE_DIR}) @@ -2535,6 +2542,16 @@ if(ARROW_S3) include_directories(SYSTEM ${AWSSDK_INCLUDE_DIR}) message(STATUS "Found AWS SDK headers: ${AWSSDK_INCLUDE_DIR}") message(STATUS "Found AWS SDK libraries: ${AWSSDK_LINK_LIBRARIES}") + + if(APPLE) + # CoreFoundation's path is hardcoded in the CMake files provided by + # aws-sdk-cpp to use the MacOSX SDK provided by XCode which makes + # XCode a hard dependency. Command Line Tools is often used instead + # of the full XCode suite, so let the linker to find it. + set_target_properties(AWS::aws-c-common + PROPERTIES INTERFACE_LINK_LIBRARIES + "-pthread;pthread;-framework CoreFoundation") + endif() endif() # Write out the package configurations. diff --git a/cpp/examples/minimal_build/build.sh b/cpp/examples/minimal_build/build.sh index 068f4898ba0d..a8c1552eebe9 100755 --- a/cpp/examples/minimal_build/build.sh +++ b/cpp/examples/minimal_build/build.sh @@ -22,6 +22,7 @@ # docker run --rm -t -i -v $PWD:/io -v $ARROW_ROOT:/arrow arrow_cpp_minimal /io/build.sh BUILD_DIR=/build +NPROC=$(nproc) mkdir $BUILD_DIR pushd $BUILD_DIR @@ -30,6 +31,8 @@ cmake /arrow/cpp -DBOOST_SOURCE=BUNDLED \ -DARROW_BOOST_USE_SHARED=OFF \ -DARROW_COMPUTE=OFF \ -DARROW_DATASET=OFF \ + -DARROW_FILESYSTEM=OFF \ + -DARROW_HDFS=OFF \ -DARROW_JEMALLOC=OFF \ -DARROW_JSON=OFF \ -DARROW_USE_GLOG=OFF \ @@ -41,6 +44,7 @@ cmake /arrow/cpp -DBOOST_SOURCE=BUNDLED \ -DARROW_WITH_BROTLI=OFF \ -DARROW_BUILD_UTILITIES=OFF -make -j8 +make -j$NPROC +make install popd diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index 91ff44369982..bf5a77e58857 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -114,12 +114,6 @@ set(ARROW_SRCS csv/options.cc csv/parser.cc csv/reader.cc - filesystem/filesystem.cc - filesystem/localfs.cc - filesystem/mockfs.cc - filesystem/path_tree.cc - filesystem/path_util.cc - filesystem/util_internal.cc io/buffered.cc io/compressed.cc io/file.cc @@ -149,6 +143,13 @@ set(ARROW_SRCS vendored/base64.cpp vendored/datetime/tz.cpp) +set(ARROW_TESTING_SRCS + io/test_common.cc + ipc/test_common.cc + testing/gtest_util.cc + testing/random.cc + testing/generator.cc) + # Add dependencies for third-party allocators. # If possible we only want memory_pool.cc to wait for allocators to finish building, # but that only works with Ninja @@ -173,26 +174,6 @@ endif() unset(_allocator_dependencies) -if(ARROW_JSON) - add_subdirectory(json) - set(ARROW_SRCS - ${ARROW_SRCS} - json/options.cc - json/chunked_builder.cc - json/chunker.cc - json/converter.cc - json/parser.cc - json/reader.cc) -endif() - -if(ARROW_S3) - set(ARROW_SRCS ${ARROW_SRCS} filesystem/s3fs.cc) -endif() - -if(ARROW_WITH_URIPARSER) - set(ARROW_SRCS ${ARROW_SRCS} util/uri.cc) -endif() - if("${COMPILER_FAMILY}" STREQUAL "clang") set_property(SOURCE util/io_util.cc APPEND_STRING @@ -202,94 +183,149 @@ endif() # Unconditionally install testing headers that are also useful for Arrow consumers. add_subdirectory(testing) +# +# Add the various components +# + +add_subdirectory(array) +add_subdirectory(csv) +add_subdirectory(io) +add_subdirectory(util) +add_subdirectory(vendored) + if(ARROW_COMPUTE) add_subdirectory(compute) - set(ARROW_SRCS - ${ARROW_SRCS} - compute/context.cc - compute/expression.cc - compute/logical_type.cc - compute/operation.cc - compute/kernels/aggregate.cc - compute/kernels/boolean.cc - compute/kernels/cast.cc - compute/kernels/compare.cc - compute/kernels/count.cc - compute/kernels/hash.cc - compute/kernels/filter.cc - compute/kernels/mean.cc - compute/kernels/sort_to_indices.cc - compute/kernels/sum.cc - compute/kernels/take.cc - compute/kernels/isin.cc - compute/kernels/util_internal.cc - compute/operations/cast.cc - compute/operations/literal.cc) + list(APPEND ARROW_SRCS + compute/context.cc + compute/expression.cc + compute/logical_type.cc + compute/operation.cc + compute/kernels/aggregate.cc + compute/kernels/boolean.cc + compute/kernels/cast.cc + compute/kernels/compare.cc + compute/kernels/count.cc + compute/kernels/hash.cc + compute/kernels/filter.cc + compute/kernels/mean.cc + compute/kernels/sort_to_indices.cc + compute/kernels/sum.cc + compute/kernels/take.cc + compute/kernels/isin.cc + compute/kernels/util_internal.cc + compute/operations/cast.cc + compute/operations/literal.cc) endif() if(ARROW_CUDA) - # IPC extensions required to build the CUDA library - set(ARROW_IPC ON) add_subdirectory(gpu) endif() -if(ARROW_WITH_BROTLI) - add_definitions(-DARROW_WITH_BROTLI) - set(ARROW_SRCS util/compression_brotli.cc ${ARROW_SRCS}) +if(ARROW_DATASET) + add_subdirectory(dataset) endif() -if(ARROW_WITH_BZ2) - add_definitions(-DARROW_WITH_BZ2) - set(ARROW_SRCS util/compression_bz2.cc ${ARROW_SRCS}) +if(ARROW_FILESYSTEM) + add_subdirectory(filesystem) + + list(APPEND ARROW_SRCS + filesystem/filesystem.cc + filesystem/localfs.cc + filesystem/mockfs.cc + filesystem/path_tree.cc + filesystem/path_util.cc + filesystem/util_internal.cc) + + if(ARROW_S3) + list(APPEND ARROW_SRCS filesystem/s3fs.cc) + endif() + + list(APPEND ARROW_TESTING_SRCS filesystem/test_util.cc) endif() -if(ARROW_WITH_LZ4) - add_definitions(-DARROW_WITH_LZ4) - set(ARROW_SRCS util/compression_lz4.cc ${ARROW_SRCS}) +if(ARROW_FLIGHT) + add_subdirectory(flight) endif() -if(ARROW_WITH_SNAPPY) - add_definitions(-DARROW_WITH_SNAPPY) - set(ARROW_SRCS util/compression_snappy.cc ${ARROW_SRCS}) +if(ARROW_HIVESERVER2) + add_subdirectory(dbi/hiveserver2) endif() -if(ARROW_WITH_ZLIB) - add_definitions(-DARROW_WITH_ZLIB) - set(ARROW_SRCS util/compression_zlib.cc ${ARROW_SRCS}) +if(ARROW_IPC) + add_subdirectory(ipc) + + list(APPEND ARROW_SRCS + ipc/dictionary.cc + ipc/feather.cc + ipc/message.cc + ipc/metadata_internal.cc + ipc/options.cc + ipc/reader.cc + ipc/writer.cc) + + if(ARROW_JSON) + list( + APPEND ARROW_SRCS ipc/json_integration.cc ipc/json_internal.cc ipc/json_simple.cc) + endif() + add_dependencies(arrow_dependencies metadata_fbs) endif() -if(ARROW_WITH_ZSTD) - add_definitions(-DARROW_WITH_ZSTD) - set(ARROW_SRCS util/compression_zstd.cc ${ARROW_SRCS}) +if(ARROW_JSON) + add_subdirectory(json) + list(APPEND ARROW_SRCS + json/options.cc + json/chunked_builder.cc + json/chunker.cc + json/converter.cc + json/parser.cc + json/reader.cc) endif() if(ARROW_ORC) add_subdirectory(adapters/orc) - set(ARROW_SRCS adapters/orc/adapter.cc adapters/orc/adapter_util.cc ${ARROW_SRCS}) + list(APPEND ARROW_SRCS adapters/orc/adapter.cc adapters/orc/adapter_util.cc) +endif() + +if(ARROW_PYTHON) + add_subdirectory(python) endif() if(ARROW_TENSORFLOW) add_subdirectory(adapters/tensorflow) endif() -if(ARROW_IPC) - add_subdirectory(ipc) +if(ARROW_WITH_URIPARSER) + list(APPEND ARROW_SRCS util/uri.cc) +endif() - set(ARROW_SRCS - ${ARROW_SRCS} - ipc/dictionary.cc - ipc/feather.cc - ipc/message.cc - ipc/metadata_internal.cc - ipc/options.cc - ipc/reader.cc - ipc/writer.cc) +if(ARROW_WITH_BROTLI) + add_definitions(-DARROW_WITH_BROTLI) + list(APPEND ARROW_SRCS util/compression_brotli.cc) +endif() - if(ARROW_JSON) - set(ARROW_SRCS ${ARROW_SRCS} ipc/json_integration.cc ipc/json_internal.cc - ipc/json_simple.cc) - endif() - add_dependencies(arrow_dependencies metadata_fbs) +if(ARROW_WITH_BZ2) + add_definitions(-DARROW_WITH_BZ2) + list(APPEND ARROW_SRCS util/compression_bz2.cc) +endif() + +if(ARROW_WITH_LZ4) + add_definitions(-DARROW_WITH_LZ4) + list(APPEND ARROW_SRCS util/compression_lz4.cc) +endif() + +if(ARROW_WITH_SNAPPY) + add_definitions(-DARROW_WITH_SNAPPY) + list(APPEND ARROW_SRCS util/compression_snappy.cc) +endif() + +if(ARROW_WITH_ZLIB) + add_definitions(-DARROW_WITH_ZLIB) + list(APPEND ARROW_SRCS util/compression_zlib.cc) +endif() + +if(ARROW_WITH_ZSTD) + add_definitions(-DARROW_WITH_ZSTD) + list(APPEND ARROW_SRCS util/compression_zstd.cc) endif() if(NOT APPLE AND NOT MSVC) @@ -333,12 +369,7 @@ if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS OR ARROW_BUILD_INTEGRATION) # that depend on gtest add_arrow_lib(arrow_testing SOURCES - io/test_common.cc - ipc/test_common.cc - filesystem/test_util.cc - testing/gtest_util.cc - testing/random.cc - testing/generator.cc + ${ARROW_TESTING_SRCS} OUTPUTS ARROW_TESTING_LIBRARIES DEPENDENCIES @@ -412,26 +443,3 @@ add_arrow_test(sparse_tensor_test) add_arrow_benchmark(builder_benchmark) add_arrow_benchmark(type_benchmark) - -add_subdirectory(array) -add_subdirectory(csv) -add_subdirectory(filesystem) -add_subdirectory(io) -add_subdirectory(util) -add_subdirectory(vendored) - -if(ARROW_DATASET) - add_subdirectory(dataset) -endif() - -if(ARROW_FLIGHT) - add_subdirectory(flight) -endif() - -if(ARROW_PYTHON) - add_subdirectory(python) -endif() - -if(ARROW_HIVESERVER2) - add_subdirectory(dbi/hiveserver2) -endif() diff --git a/cpp/src/arrow/buffer.cc b/cpp/src/arrow/buffer.cc index bfbf2f8f2ccb..ca78922f2c27 100644 --- a/cpp/src/arrow/buffer.cc +++ b/cpp/src/arrow/buffer.cc @@ -79,7 +79,8 @@ Status Buffer::FromString(const std::string& data, std::shared_ptr* out) class StlStringBuffer : public Buffer { public: - explicit StlStringBuffer(std::string&& data) : Buffer(nullptr, 0), input_(data) { + explicit StlStringBuffer(std::string&& data) + : Buffer(nullptr, 0), input_(std::move(data)) { data_ = reinterpret_cast(input_.c_str()); size_ = static_cast(input_.size()); capacity_ = size_; diff --git a/cpp/src/arrow/dataset/CMakeLists.txt b/cpp/src/arrow/dataset/CMakeLists.txt index 6f94297e272c..442e9e6587bc 100644 --- a/cpp/src/arrow/dataset/CMakeLists.txt +++ b/cpp/src/arrow/dataset/CMakeLists.txt @@ -23,7 +23,12 @@ arrow_install_all_headers("arrow/dataset") # pkg-config support arrow_add_pkg_config("arrow-dataset") -set(ARROW_DATASET_SRCS dataset.cc file_base.cc filter.cc scanner.cc) +set(ARROW_DATASET_SRCS + dataset.cc + discovery.cc + file_base.cc + filter.cc + scanner.cc) set(ARROW_DATASET_LINK_STATIC arrow_static) set(ARROW_DATASET_LINK_SHARED arrow_shared) @@ -91,6 +96,7 @@ endfunction() if(NOT WIN32) add_arrow_dataset_test(dataset_test) + add_arrow_dataset_test(discovery_test) add_arrow_dataset_test(file_test) add_arrow_dataset_test(filter_test) add_arrow_dataset_test(scanner_test) diff --git a/cpp/src/arrow/dataset/dataset.cc b/cpp/src/arrow/dataset/dataset.cc index e3b81977e2b6..34b01a27c3e1 100644 --- a/cpp/src/arrow/dataset/dataset.cc +++ b/cpp/src/arrow/dataset/dataset.cc @@ -65,26 +65,28 @@ Status Dataset::NewScan(std::unique_ptr* out) { bool DataSource::AssumePartitionExpression( const std::shared_ptr& scan_options, std::shared_ptr* simplified_scan_options) const { - DCHECK_NE(simplified_scan_options, nullptr); - if (scan_options == nullptr) { - // null scan options; no selector to simplify - *simplified_scan_options = scan_options; + auto filter = scan_options->filter; + if (filter == nullptr || partition_expression_ == nullptr) { + if (simplified_scan_options != nullptr) { + *simplified_scan_options = scan_options; + } return true; } - auto c = SelectorAssume(scan_options->selector, partition_expression_); + auto c = filter->Assume(*partition_expression_); DCHECK_OK(c.status()); auto expr = std::move(c).ValueOrDie(); - bool trivial = true; - if (expr->IsNull() || (expr->IsTrivialCondition(&trivial) && !trivial)) { + if (expr->IsNull() || expr->IsTrivialFalseCondition()) { // selector is not satisfiable; yield no fragments return false; } - auto copy = std::make_shared(*scan_options); - copy->selector = ExpressionSelector(std::move(expr)); - *simplified_scan_options = std::move(copy); + if (simplified_scan_options != nullptr) { + auto copy = std::make_shared(*scan_options); + copy->filter = std::move(expr); + *simplified_scan_options = std::move(copy); + } return true; } diff --git a/cpp/src/arrow/dataset/dataset.h b/cpp/src/arrow/dataset/dataset.h index b6f7ca7106a7..7ff580706861 100644 --- a/cpp/src/arrow/dataset/dataset.h +++ b/cpp/src/arrow/dataset/dataset.h @@ -148,14 +148,5 @@ class ARROW_DS_EXPORT Dataset : public std::enable_shared_from_this { std::vector> sources_; }; -/// \brief Conditions to apply to a dataset when reading to include or -/// exclude fragments, filter out rows, etc. -struct DataSelector { - std::vector> filters; - - // TODO(wesm): Select specific partition keys, file path globs, or - // other common desirable selections -}; - } // namespace dataset } // namespace arrow diff --git a/cpp/src/arrow/dataset/discovery.cc b/cpp/src/arrow/dataset/discovery.cc new file mode 100644 index 000000000000..0a0b8d59f63a --- /dev/null +++ b/cpp/src/arrow/dataset/discovery.cc @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/dataset/discovery.h" + +#include +#include +#include +#include +#include + +#include "arrow/dataset/dataset.h" +#include "arrow/dataset/file_base.h" +#include "arrow/dataset/type_fwd.h" +#include "arrow/filesystem/path_tree.h" +#include "arrow/status.h" + +namespace arrow { +namespace dataset { + +FileSystemDataSourceDiscovery::FileSystemDataSourceDiscovery( + fs::FileSystem* filesystem, std::vector files, + std::shared_ptr format) + : fs_(filesystem), files_(std::move(files)), format_(std::move(format)) {} + +Status FileSystemDataSourceDiscovery::Make(fs::FileSystem* filesystem, + std::vector files, + std::shared_ptr format, + std::shared_ptr* out) { + out->reset(new FileSystemDataSourceDiscovery(filesystem, files, format)); + return Status::OK(); +} + +Status FileSystemDataSourceDiscovery::Make(fs::FileSystem* filesystem, + fs::Selector selector, + std::shared_ptr format, + std::shared_ptr* out) { + std::vector files; + RETURN_NOT_OK(filesystem->GetTargetStats(selector, &files)); + return Make(filesystem, files, format, out); +} + +static inline Status InspectSchema(fs::FileSystem* fs, + const std::vector stats, + const std::shared_ptr& format, + std::shared_ptr* out) { + std::vector> schemas; + + for (const auto& f : stats) { + if (!f.IsFile()) continue; + + std::shared_ptr schema; + RETURN_NOT_OK(format->Inspect(FileSource(f.path(), fs), &schema)); + schemas.push_back(schema); + } + + if (schemas.size() > 0) { + // TODO merge schemas. + *out = schemas[0]; + } + + return Status::OK(); +} + +Status FileSystemDataSourceDiscovery::Inspect(std::shared_ptr* out) { + return InspectSchema(fs_, files_, format_, out); +} + +Status FileSystemDataSourceDiscovery::Build(const BuildOptions& options, + std::shared_ptr source_partition, + std::shared_ptr* out) { + return FileSystemBasedDataSource::Make(fs_, files_, std::move(source_partition), + PathPartitions{}, format_, out); +} + +} // namespace dataset +} // namespace arrow diff --git a/cpp/src/arrow/dataset/discovery.h b/cpp/src/arrow/dataset/discovery.h index 18242250f326..5f0387b23cd4 100644 --- a/cpp/src/arrow/dataset/discovery.h +++ b/cpp/src/arrow/dataset/discovery.h @@ -23,23 +23,80 @@ #include #include +#include #include "arrow/dataset/type_fwd.h" #include "arrow/dataset/visibility.h" +#include "arrow/filesystem/filesystem.h" +#include "arrow/filesystem/path_tree.h" #include "arrow/util/macros.h" namespace arrow { namespace dataset { -struct ARROW_DS_EXPORT DiscoveryOptions { - std::shared_ptr format = NULLPTR; +struct ARROW_DS_EXPORT BuildOptions { + /// Schema to conform to. + std::shared_ptr schema = NULLPTR; + /// The partition scheme indicate how to discover partitions for the data + /// source and fragments. std::shared_ptr partition_scheme = NULLPTR; }; -/// \brief Using a root directory -ARROW_DS_EXPORT -Status DiscoverSource(const std::string& path, fs::FileSystem* filesystem, - const DiscoveryOptions& options, std::shared_ptr* out); +/// \brief DataSourceDiscovery provides a way to inspect a DataSource potential +/// schema before materializing it. Thus, the user can peek the schema for +/// data sources and decide on a unified schema. The pseudocode would look like +/// +/// def get_dataset(factories): +/// schemas = [] +/// for f in factories: +/// schemas.append(f.Inspect()) +/// +/// common_schema = UnifySchemas(schemas) +/// +/// sources = [] +/// for f in factories: +/// sources.append(f.Discover({schema: common_schema})) +/// +/// return Dataset(sources, common_schema) +class ARROW_DS_EXPORT DataSourceDiscovery { + public: + /// \brief Get the schema for the resulting DataSource. + virtual Status Inspect(std::shared_ptr* out) = 0; + + /// \brief Create a DataSource with a given partition. + virtual Status Build(const BuildOptions& options, + std::shared_ptr source_partition, + std::shared_ptr* out) = 0; + + virtual ~DataSourceDiscovery() = default; +}; + +/// \brief FileSystemDataSourceFactory creates a DataSource from a vector +/// of fs::FileStats or a fs::Selector. +class ARROW_DS_EXPORT FileSystemDataSourceDiscovery : public DataSourceDiscovery { + public: + static Status Make(fs::FileSystem* filesystem, std::vector files, + std::shared_ptr format, + std::shared_ptr* out); + + static Status Make(fs::FileSystem* filesystem, fs::Selector selector, + std::shared_ptr format, + std::shared_ptr* out); + + Status Inspect(std::shared_ptr* out) override; + + Status Build(const BuildOptions& options, std::shared_ptr source_partition, + std::shared_ptr* out) override; + + protected: + FileSystemDataSourceDiscovery(fs::FileSystem* filesystem, + std::vector files, + std::shared_ptr format); + + fs::FileSystem* fs_; + std::vector files_; + std::shared_ptr format_; +}; } // namespace dataset } // namespace arrow diff --git a/cpp/src/arrow/dataset/discovery_test.cc b/cpp/src/arrow/dataset/discovery_test.cc new file mode 100644 index 000000000000..8dd3a2d3d920 --- /dev/null +++ b/cpp/src/arrow/dataset/discovery_test.cc @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/dataset/discovery.h" + +#include +#include + +#include "arrow/dataset/test_util.h" +#include "arrow/filesystem/test_util.h" + +namespace arrow { +namespace dataset { + +class FileSystemDataSourceDiscoveryTest : public TestFileSystemBasedDataSource { + public: + void MakeDiscovery(const std::vector& files) { + MakeFileSystem(files); + ASSERT_OK( + FileSystemDataSourceDiscovery::Make(fs_.get(), files, format_, &discovery_)); + } + + void MakeDiscovery(const std::vector& files, fs::Selector selector) { + MakeFileSystem(files); + ASSERT_OK( + FileSystemDataSourceDiscovery::Make(fs_.get(), selector, format_, &discovery_)); + } + + protected: + std::shared_ptr discovery_; + std::shared_ptr format_ = std::make_shared(); +}; + +TEST_F(FileSystemDataSourceDiscoveryTest, Basic) { + MakeDiscovery({fs::File("a"), fs::File("b")}); + + BuildOptions options; + ASSERT_OK(discovery_->Build(options, nullptr, &source_)); + AssertFragmentsAreFromPath(source_->GetFragments(options_), {"a", "b"}); +} + +TEST_F(FileSystemDataSourceDiscoveryTest, Selector) { + // This test ensure that the Selector is enforced. + fs::Selector selector; + selector.base_dir = "A"; + MakeDiscovery({fs::File("0"), fs::File("A/a")}, selector); + + BuildOptions options; + ASSERT_OK(discovery_->Build(options, nullptr, &source_)); + AssertFragmentsAreFromPath(source_->GetFragments(options_), {"A/a"}); +} + +TEST_F(FileSystemDataSourceDiscoveryTest, Inspect) { + auto s = schema({field("f64", float64())}); + format_ = std::make_shared(s); + + MakeDiscovery({}); + std::shared_ptr actual; + + // No files + ASSERT_OK(discovery_->Inspect(&actual)); + EXPECT_EQ(actual, nullptr); + + MakeDiscovery({fs::File("test")}); + ASSERT_OK(discovery_->Inspect(&actual)); + EXPECT_EQ(actual, s); +} + +} // namespace dataset +} // namespace arrow diff --git a/cpp/src/arrow/dataset/file_base.cc b/cpp/src/arrow/dataset/file_base.cc index a335a2a679dd..4a75d650497d 100644 --- a/cpp/src/arrow/dataset/file_base.cc +++ b/cpp/src/arrow/dataset/file_base.cc @@ -48,79 +48,90 @@ Status FileBasedDataFragment::Scan(std::shared_ptr scan_context, } FileSystemBasedDataSource::FileSystemBasedDataSource( - fs::FileSystem* filesystem, const fs::Selector& selector, - std::shared_ptr format, std::shared_ptr partition_expression, - std::vector stats) - : DataSource(std::move(partition_expression)), + fs::FileSystem* filesystem, fs::PathForest forest, + std::shared_ptr source_partition, PathPartitions partitions, + std::shared_ptr format) + : DataSource(std::move(source_partition)), filesystem_(filesystem), - selector_(std::move(selector)), - format_(std::move(format)), - stats_(std::move(stats)) {} + forest_(std::move(forest)), + partitions_(std::move(partitions)), + format_(std::move(format)) {} Status FileSystemBasedDataSource::Make(fs::FileSystem* filesystem, - const fs::Selector& selector, + std::vector stats, + std::shared_ptr source_partition, + PathPartitions partitions, std::shared_ptr format, - std::shared_ptr partition_expression, - std::unique_ptr* out) { - std::vector stats; - RETURN_NOT_OK(filesystem->GetTargetStats(selector, &stats)); - - auto new_end = - std::remove_if(stats.begin(), stats.end(), [&](const fs::FileStats& stats) { - return stats.type() != fs::FileType::File || - !format->IsKnownExtension(stats.extension()); - }); - stats.resize(new_end - stats.begin()); - - out->reset(new FileSystemBasedDataSource(filesystem, selector, std::move(format), - std::move(partition_expression), - std::move(stats))); + std::shared_ptr* out) { + fs::PathForest forest; + RETURN_NOT_OK(fs::PathTree::Make(stats, &forest)); + out->reset(new FileSystemBasedDataSource(filesystem, std::move(forest), + std::move(source_partition), + std::move(partitions), std::move(format))); return Status::OK(); } -Status FileSystemBasedDataSource::Make(fs::FileSystem* filesystem, - const fs::Selector& selector, - std::shared_ptr format, - std::unique_ptr* out) { - return Make(filesystem, selector, std::move(format), nullptr, out); -} - DataFragmentIterator FileSystemBasedDataSource::GetFragmentsImpl( - std::shared_ptr scan_options) { - std::shared_ptr simplified_scan_options; - if (!AssumePartitionExpression(scan_options, &simplified_scan_options)) { - return MakeEmptyIterator>(); - } + std::shared_ptr options) { + std::vector> files; - struct Impl : DataFragmentIterator { - Impl(fs::FileSystem* filesystem, std::shared_ptr format, - std::shared_ptr scan_options, std::vector stats) - : filesystem_(filesystem), - format_(std::move(format)), - scan_options_(std::move(scan_options)), - stats_(std::move(stats)) {} - - Status Next(std::shared_ptr* out) { - if (i_ == stats_.size()) { - *out = nullptr; - return Status::OK(); - } - FileSource src(stats_[i_++].path(), filesystem_); - - std::unique_ptr fragment; - RETURN_NOT_OK(format_->MakeFragment(src, scan_options_, &fragment)); - *out = std::move(fragment); - return Status::OK(); + auto visitor = [&files](const fs::FileStats& stats) { + if (stats.IsFile()) { + files.emplace_back(new fs::FileStats(stats)); } + return Status::OK(); + }; + // The matcher ensures that directories (and their descendants) are not + // visited. + auto matcher = [this, options](const fs::FileStats& stats, bool* match) { + *match = this->PartitionMatches(stats, options->filter); + return Status::OK(); + }; + + for (auto tree : forest_) { + DCHECK_OK(tree->Visit(visitor, matcher)); + } + + auto file_it = MakeVectorIterator(std::move(files)); + auto file_to_fragment = [options, this](std::unique_ptr stats, + std::shared_ptr* out) { + std::unique_ptr fragment; + FileSource src(stats->path(), filesystem_); - size_t i_ = 0; - fs::FileSystem* filesystem_; - std::shared_ptr format_; - std::shared_ptr scan_options_; - std::vector stats_; + RETURN_NOT_OK(format_->MakeFragment(src, options, &fragment)); + *out = std::move(fragment); + return Status::OK(); }; - return DataFragmentIterator(Impl(filesystem_, format_, scan_options, stats_)); + return MakeMaybeMapIterator(file_to_fragment, std::move(file_it)); +} + +bool FileSystemBasedDataSource::PartitionMatches(const fs::FileStats& stats, + std::shared_ptr filter) { + if (filter == nullptr) { + return true; + } + + auto found = partitions_.find(stats.path()); + if (found == partitions_.end()) { + // No partition attached to current node (directory or file), continue. + return true; + } + + auto c = found->second->Assume(*filter); + if (!c.ok()) { + // Could not simplify expression move on! + return true; + } + + // TODO: pass simplified expressions to children + auto expr = std::move(c).ValueOrDie(); + if (expr->IsNull() || expr->IsTrivialFalseCondition()) { + // selector is not satisfiable; don't recurse in this branch. + return false; + } + + return true; } } // namespace dataset diff --git a/cpp/src/arrow/dataset/file_base.h b/cpp/src/arrow/dataset/file_base.h index 440b36f75d95..c581251e4dc4 100644 --- a/cpp/src/arrow/dataset/file_base.h +++ b/cpp/src/arrow/dataset/file_base.h @@ -19,6 +19,7 @@ #include #include +#include #include #include @@ -29,12 +30,16 @@ #include "arrow/dataset/visibility.h" #include "arrow/dataset/writer.h" #include "arrow/filesystem/filesystem.h" +#include "arrow/filesystem/path_tree.h" #include "arrow/io/file.h" #include "arrow/util/compression.h" namespace arrow { + namespace dataset { +class Filter; + /// \brief The path and filesystem where an actual file is located or a buffer which can /// be read like a file class ARROW_DS_EXPORT FileSource { @@ -127,6 +132,10 @@ class ARROW_DS_EXPORT FileFormat { /// \brief Return true if the given file extension virtual bool IsKnownExtension(const std::string& ext) const = 0; + /// \brief Return the schema of the file if possible. + virtual Status Inspect(const FileSource& source, + std::shared_ptr* out) const = 0; + /// \brief Open a file for scanning virtual Status ScanFile(const FileSource& source, std::shared_ptr scan_options, @@ -161,36 +170,46 @@ class ARROW_DS_EXPORT FileBasedDataFragment : public DataFragment { std::shared_ptr scan_options_; }; -/// \brief A DataSource which takes files of one format from a directory -/// -/// The directory is crawled upon construction (Make) and not updated afterward. -/// GetFragments() will not include files added after this DataDource is constructed and -/// will error if files are deleted/moved. +/// \brief Mapping from path to partition expressions. +using PathPartitions = std::unordered_map>; + +/// \brief A DataSource of FileBasedDataFragments. class ARROW_DS_EXPORT FileSystemBasedDataSource : public DataSource { public: - static Status Make(fs::FileSystem* filesystem, const fs::Selector& selector, - std::shared_ptr format, - std::unique_ptr* out); - - static Status Make(fs::FileSystem* filesystem, const fs::Selector& selector, - std::shared_ptr format, - std::shared_ptr partition_expression, - std::unique_ptr* out); - - std::string type() const override { return "directory"; } + /// \brief Create a FileSystemBasedDataSource with optional partitions. + /// + /// \param[in] filesystem the filesystem which files are from. + /// \param[in] stats a list of files/directories to consume. + /// \param[in] source_partition the top-level partition of the DataSource + /// \param[in] partitions optional partitions attached to FileStats found in + /// `stats`. + /// \param[in] format file format to create fragments from. + /// \param[out] out pointer storing the resulting DataSource. + /// + /// The caller is not required to provide a complete coverage of nodes and + /// partitions. + static Status Make(fs::FileSystem* filesystem, std::vector stats, + std::shared_ptr source_partition, + PathPartitions partitions, std::shared_ptr format, + std::shared_ptr* out); + + std::string type() const override { return "filesystem_data_source"; } protected: DataFragmentIterator GetFragmentsImpl(std::shared_ptr options) override; - FileSystemBasedDataSource(fs::FileSystem* filesystem, const fs::Selector& selector, - std::shared_ptr format, - std::shared_ptr partition_expression, - std::vector stats); + FileSystemBasedDataSource(fs::FileSystem* filesystem, fs::PathForest forest, + std::shared_ptr source_partition, + PathPartitions partitions, + std::shared_ptr format); + + bool PartitionMatches(const fs::FileStats& stats, std::shared_ptr filter); fs::FileSystem* filesystem_ = NULLPTR; - fs::Selector selector_; + fs::PathForest forest_; + PathPartitions partitions_; + std::shared_ptr format_; - std::vector stats_; }; } // namespace dataset diff --git a/cpp/src/arrow/dataset/file_parquet.cc b/cpp/src/arrow/dataset/file_parquet.cc index e6298231b89f..a23639ebd162 100644 --- a/cpp/src/arrow/dataset/file_parquet.cc +++ b/cpp/src/arrow/dataset/file_parquet.cc @@ -165,14 +165,26 @@ class ParquetScanTaskIterator { std::shared_ptr reader_; }; +Status ParquetFileFormat::Inspect(const FileSource& source, + std::shared_ptr* out) const { + auto pool = default_memory_pool(); + + std::unique_ptr reader; + RETURN_NOT_OK(OpenReader(source, pool, &reader)); + + std::unique_ptr arrow_reader; + RETURN_NOT_OK(parquet::arrow::FileReader::Make(pool, std::move(reader), &arrow_reader)); + + return arrow_reader->GetSchema(out); +} + Status ParquetFileFormat::ScanFile(const FileSource& source, std::shared_ptr scan_options, std::shared_ptr scan_context, ScanTaskIterator* out) const { - std::shared_ptr input; - RETURN_NOT_OK(source.Open(&input)); + std::unique_ptr reader; + RETURN_NOT_OK(OpenReader(source, scan_context->pool, &reader)); - auto reader = parquet::ParquetFileReader::Open(input); return ParquetScanTaskIterator::Make(scan_options, scan_context, std::move(reader), out); } @@ -185,5 +197,15 @@ Status ParquetFileFormat::MakeFragment(const FileSource& source, return Status::OK(); } +Status ParquetFileFormat::OpenReader( + const FileSource& source, MemoryPool* pool, + std::unique_ptr* out) const { + std::shared_ptr input; + RETURN_NOT_OK(source.Open(&input)); + + *out = parquet::ParquetFileReader::Open(input); + return Status::OK(); +} + } // namespace dataset } // namespace arrow diff --git a/cpp/src/arrow/dataset/file_parquet.h b/cpp/src/arrow/dataset/file_parquet.h index a04f4596f87f..cd3987845420 100644 --- a/cpp/src/arrow/dataset/file_parquet.h +++ b/cpp/src/arrow/dataset/file_parquet.h @@ -24,6 +24,10 @@ #include "arrow/dataset/type_fwd.h" #include "arrow/dataset/visibility.h" +namespace parquet { +class ParquetFileReader; +} // namespace parquet + namespace arrow { namespace dataset { @@ -47,6 +51,9 @@ class ARROW_DS_EXPORT ParquetFileFormat : public FileFormat { return ext == "par" || ext == "parq" || ext == name(); } + /// \brief Return the schema of the file if possible. + Status Inspect(const FileSource& source, std::shared_ptr* out) const override; + /// \brief Open a file for scanning Status ScanFile(const FileSource& source, std::shared_ptr scan_options, std::shared_ptr scan_context, @@ -54,6 +61,10 @@ class ARROW_DS_EXPORT ParquetFileFormat : public FileFormat { Status MakeFragment(const FileSource& source, std::shared_ptr opts, std::unique_ptr* out) override; + + private: + Status OpenReader(const FileSource& source, MemoryPool* pool, + std::unique_ptr<::parquet::ParquetFileReader>* out) const; }; class ARROW_DS_EXPORT ParquetFragment : public FileBasedDataFragment { diff --git a/cpp/src/arrow/dataset/file_parquet_test.cc b/cpp/src/arrow/dataset/file_parquet_test.cc index 171d8cee1a7f..ba390f2d2018 100644 --- a/cpp/src/arrow/dataset/file_parquet_test.cc +++ b/cpp/src/arrow/dataset/file_parquet_test.cc @@ -42,6 +42,8 @@ using parquet::CreateOutputStream; using parquet::arrow::FileWriter; using parquet::arrow::WriteTable; +using testing::Pointee; + Status WriteRecordBatch(const RecordBatch& batch, FileWriter* writer) { auto schema = batch.schema(); auto size = batch.num_rows(); @@ -138,9 +140,11 @@ class ParquetBufferFixtureMixin : public ArrowParquetWriterMixin { builder->UnsafeAppend(0.0); })); - auto schema_ = schema({field("f64", f64->type())}); return RecordBatch::Make(schema_, kBatchSize, {f64}); } + + protected: + std::shared_ptr schema_ = schema({field("f64", float64())}); }; class TestParquetFileFormat : public ParquetBufferFixtureMixin { @@ -172,21 +176,14 @@ TEST_F(TestParquetFileFormat, ScanRecordBatchReader) { ASSERT_EQ(row_count, kNumRows); } -class TestParquetFileSystemBasedDataSource - : public FileSystemBasedDataSourceMixin { - std::vector file_names() const override { - return {"a/b/c.parquet", "a/b/c/d.parquet", "a/b.parquet", "a.parquet"}; - } -}; - -TEST_F(TestParquetFileSystemBasedDataSource, NonRecursive) { this->NonRecursive(); } - -TEST_F(TestParquetFileSystemBasedDataSource, Recursive) { this->Recursive(); } - -TEST_F(TestParquetFileSystemBasedDataSource, DeletedFile) { this->DeletedFile(); } +TEST_F(TestParquetFileFormat, Inspect) { + auto reader = GetRecordBatchReader(); + auto source = GetFileSource(reader.get()); + auto format = ParquetFileFormat(); -TEST_F(TestParquetFileSystemBasedDataSource, PredicatePushDown) { - this->PredicatePushDown(); + std::shared_ptr actual; + ASSERT_OK(format.Inspect(*source.get(), &actual)); + EXPECT_EQ(*actual, *schema_); } } // namespace dataset diff --git a/cpp/src/arrow/dataset/file_test.cc b/cpp/src/arrow/dataset/file_test.cc index 19e938a5e8b2..be6af4cbbfc0 100644 --- a/cpp/src/arrow/dataset/file_test.cc +++ b/cpp/src/arrow/dataset/file_test.cc @@ -15,23 +15,6 @@ // specific language governing permissions and limitations // under the License. -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - #include #include #include @@ -42,8 +25,8 @@ #include "arrow/dataset/api.h" #include "arrow/dataset/test_util.h" -#include "arrow/filesystem/localfs.h" #include "arrow/filesystem/path_util.h" +#include "arrow/filesystem/test_util.h" #include "arrow/status.h" #include "arrow/testing/gtest_util.h" #include "arrow/util/io_util.h" @@ -94,21 +77,89 @@ TEST(FileSource, BufferBased) { ASSERT_EQ(Compression::LZ4, source2.compression()); } -class TestDummyFileSystemBasedDataSource - : public FileSystemBasedDataSourceMixin { - std::vector file_names() const override { - return {"a/b/c.dummy", "a/b/c/d.dummy", "a/b.dummy", "a.dummy"}; - } -}; +TEST_F(TestFileSystemBasedDataSource, Basic) { + MakeSource({}); + AssertFragmentsAreFromPath(source_->GetFragments(options_), {}); + + MakeSource({fs::File("a"), fs::File("b"), fs::File("c")}); + AssertFragmentsAreFromPath(source_->GetFragments(options_), {"a", "b", "c"}); + + // Should not create fragment from directories. + MakeSource({fs::Dir("A"), fs::Dir("A/B"), fs::File("A/a"), fs::File("A/B/b")}); + AssertFragmentsAreFromPath(source_->GetFragments(options_), {"A/a", "A/B/b"}); +} + +TEST_F(TestFileSystemBasedDataSource, RootPartitionPruning) { + auto source_partition = ("a"_ == 5).Copy(); + MakeSource({fs::File("a"), fs::File("b")}, source_partition); + + // No filter should always return all data. + options_->filter = nullptr; + AssertFragmentsAreFromPath(source_->GetFragments(options_), {"a", "b"}); -TEST_F(TestDummyFileSystemBasedDataSource, NonRecursive) { this->NonRecursive(); } + // filter == partition + options_->filter = source_partition; + AssertFragmentsAreFromPath(source_->GetFragments(options_), {"a", "b"}); -TEST_F(TestDummyFileSystemBasedDataSource, Recursive) { this->Recursive(); } + // Same partition key, but non matching filter + options_->filter = ("a"_ == 6).Copy(); + AssertFragmentsAreFromPath(source_->GetFragments(options_), {}); -TEST_F(TestDummyFileSystemBasedDataSource, DeletedFile) { this->DeletedFile(); } + options_->filter = ("a"_ > 1).Copy(); + AssertFragmentsAreFromPath(source_->GetFragments(options_), {"a", "b"}); + + // different key shouldn't prune + options_->filter = ("b"_ == 6).Copy(); + AssertFragmentsAreFromPath(source_->GetFragments(options_), {"a", "b"}); + + // No partition should match + MakeSource({fs::File("a"), fs::File("b")}); + options_->filter = ("b"_ == 6).Copy(); + AssertFragmentsAreFromPath(source_->GetFragments(options_), {"a", "b"}); +} -TEST_F(TestDummyFileSystemBasedDataSource, PredicatePushDown) { - this->PredicatePushDown(); +TEST_F(TestFileSystemBasedDataSource, TreePartitionPruning) { + auto source_partition = ("country"_ == "US").Copy(); + std::vector regions = { + fs::Dir("NY"), fs::File("NY/New York"), fs::File("NY/Franklin"), + fs::Dir("CA"), fs::File("CA/San Francisco"), fs::File("CA/Franklin"), + }; + // Explicitly _don't_ set the state partition in the leaves to test if + // sub-tree pruning works. This implies that `state` predicate won't apply to + // files. + PathPartitions partitions = { + {"CA", ("state"_ == "CA").Copy()}, + {"CA/San Francisco", ("city"_ == "San Francisco").Copy()}, + {"CA/Franklin", ("city"_ == "Franklin").Copy()}, + {"NY", ("state"_ == "NY").Copy()}, + {"NY/New York", ("city"_ == "New York").Copy()}, + {"NY/Franklin", ("city"_ == "Franklin").Copy()}, + }; + + MakeSource(regions, source_partition, partitions); + + std::vector all_cities = {"CA/San Francisco", "CA/Franklin", "NY/New York", + "NY/Franklin"}; + std::vector ca_cities = {"CA/San Francisco", "CA/Franklin"}; + std::vector franklins = {"CA/Franklin", "NY/Franklin"}; + + // No filter should always return all data. + options_->filter = nullptr; + AssertFragmentsAreFromPath(source_->GetFragments(options_), all_cities); + + // Data source partition is respected + options_->filter = ("country"_ == "US").Copy(); + AssertFragmentsAreFromPath(source_->GetFragments(options_), all_cities); + options_->filter = ("country"_ == "FR").Copy(); + AssertFragmentsAreFromPath(source_->GetFragments(options_), {}); + + options_->filter = ("state"_ == "CA").Copy(); + AssertFragmentsAreFromPath(source_->GetFragments(options_), ca_cities); + + // Filter where no decisions can be made on inner nodes when filter don't + // apply to inner partitions. + options_->filter = ("city"_ == "Franklin").Copy(); + AssertFragmentsAreFromPath(source_->GetFragments(options_), franklins); } } // namespace dataset diff --git a/cpp/src/arrow/dataset/filter.cc b/cpp/src/arrow/dataset/filter.cc index 7e1ee82b22b1..bcdee2897f1f 100644 --- a/cpp/src/arrow/dataset/filter.cc +++ b/cpp/src/arrow/dataset/filter.cc @@ -842,6 +842,16 @@ bool Expression::IsTrivialCondition(bool* out) const { return true; } +bool Expression::IsTrivialTrueCondition() const { + bool value = false; + return IsTrivialCondition(&value) && value; +} + +bool Expression::IsTrivialFalseCondition() const { + bool value = false; + return IsTrivialCondition(&value) && !value; +} + std::shared_ptr FieldExpression::Copy() const { return std::make_shared(*this); } @@ -939,33 +949,5 @@ Result> FieldExpression::Validate(const Schema& schema return null(); } -Result> SelectorAssume( - const std::shared_ptr& selector, - const std::shared_ptr& given) { - if (selector == nullptr || selector->filters.size() == 0) { - return ScalarExpression::Make(true); - } - - auto get_expression = [](const std::shared_ptr& f) { - DCHECK_EQ(f->type(), FilterType::EXPRESSION); - return checked_cast(*f).expression(); - }; - - auto out_expr = get_expression(selector->filters[0]); - for (size_t i = 1; i < selector->filters.size(); ++i) { - out_expr = and_(std::move(out_expr), get_expression(selector->filters[i])); - } - - if (given == nullptr) { - return std::move(out_expr); - } - return out_expr->Assume(*given); -} - -std::shared_ptr ExpressionSelector(std::shared_ptr e) { - return std::make_shared( - DataSelector{FilterVector{std::make_shared(std::move(e))}}); -} - } // namespace dataset } // namespace arrow diff --git a/cpp/src/arrow/dataset/filter.h b/cpp/src/arrow/dataset/filter.h index 364fca0d2d4e..3f122c18ea4c 100644 --- a/cpp/src/arrow/dataset/filter.h +++ b/cpp/src/arrow/dataset/filter.h @@ -151,6 +151,9 @@ class ARROW_DS_EXPORT Expression { /// BooleanScalar. Its value may be retrieved at the same time. bool IsTrivialCondition(bool* value = NULLPTR) const; + bool IsTrivialTrueCondition() const; + bool IsTrivialFalseCondition() const; + /// Copy this expression into a shared pointer. virtual std::shared_ptr Copy() const = 0; diff --git a/cpp/src/arrow/dataset/scanner.h b/cpp/src/arrow/dataset/scanner.h index cfb661f4c0a7..7cde4ef2888b 100644 --- a/cpp/src/arrow/dataset/scanner.h +++ b/cpp/src/arrow/dataset/scanner.h @@ -38,16 +38,16 @@ class ARROW_DS_EXPORT ScanOptions { public: ScanOptions() = default; - ScanOptions(std::shared_ptr selector, std::shared_ptr schema, + ScanOptions(std::shared_ptr filter, std::shared_ptr schema, std::vector> options = {}) - : selector(std::move(selector)), schema(std::move(schema)) {} + : filter(std::move(filter)), schema(std::move(schema)) {} virtual ~ScanOptions() = default; MemoryPool* pool() const { return pool_; } // Filters - std::shared_ptr selector; + std::shared_ptr filter; // Schema to which record batches will be reconciled std::shared_ptr schema; diff --git a/cpp/src/arrow/dataset/test_util.h b/cpp/src/arrow/dataset/test_util.h index 459185e527f3..1a7301b55187 100644 --- a/cpp/src/arrow/dataset/test_util.h +++ b/cpp/src/arrow/dataset/test_util.h @@ -15,16 +15,24 @@ // specific language governing permissions and limitations // under the License. +#pragma once + +#include #include #include #include #include #include +#include +#include + #include "arrow/dataset/file_base.h" #include "arrow/dataset/filter.h" #include "arrow/filesystem/localfs.h" +#include "arrow/filesystem/mockfs.h" #include "arrow/filesystem/path_util.h" +#include "arrow/filesystem/test_util.h" #include "arrow/record_batch.h" #include "arrow/testing/gtest_util.h" #include "arrow/util/io_util.h" @@ -155,7 +163,7 @@ class DatasetFixtureMixin : public ::testing::Test { } protected: - std::shared_ptr options_ = nullptr; + std::shared_ptr options_ = std::make_shared(); std::shared_ptr ctx_; }; @@ -164,117 +172,12 @@ class FileSystemBasedDataSourceMixin : public FileSourceFixtureMixin { public: virtual std::vector file_names() const = 0; - void SetUp() override { - selector_.base_dir = "/"; - selector_.recursive = true; - - format_ = std::make_shared(); - schema_ = schema({field("dummy", null())}); - options_ = std::make_shared(); - - ASSERT_OK( - TemporaryDir::Make("test-fsdatasource-" + format_->name() + "-", &temp_dir_)); - local_fs_ = std::make_shared(); - - auto path = temp_dir_->path().ToString(); - fs_ = std::make_shared(path, local_fs_); - - for (auto path : file_names()) { - CreateFile(path, ""); - } - - partition_expression_ = ScalarExpression::Make(true); - } - - void CreateFile(std::string path, std::string contents) { - auto parent = fs::internal::GetAbstractPathParent(path).first; - if (parent != "") { - ASSERT_OK(this->fs_->CreateDir(parent, true)); - } - std::shared_ptr file; - ASSERT_OK(this->fs_->OpenOutputStream(path, &file)); - ASSERT_OK(file->Write(contents)); - } - - void MakeDataSource() { - ASSERT_OK(FileSystemBasedDataSource::Make(fs_.get(), selector_, format_, - partition_expression_, &source_)); - } - - protected: - std::function fragment)> OpenFragments( - size_t* count) { - return [this, count](std::shared_ptr fragment) { - auto file_fragment = - internal::checked_pointer_cast(fragment); - ++*count; - auto extension = - fs::internal::GetAbstractPathExtension(file_fragment->source().path()); - EXPECT_TRUE(format_->IsKnownExtension(extension)); - std::shared_ptr f; - return this->fs_->OpenInputFile(file_fragment->source().path(), &f); - }; - } - - void NonRecursive() { - selector_.recursive = false; - MakeDataSource(); - - size_t count = 0; - ASSERT_OK(source_->GetFragments(options_).Visit(OpenFragments(&count))); - ASSERT_EQ(count, 1); - } - - void Recursive() { - MakeDataSource(); - - size_t count = 0; - ASSERT_OK(source_->GetFragments(options_).Visit(OpenFragments(&count))); - ASSERT_EQ(count, file_names().size()); - } - - void DeletedFile() { - MakeDataSource(); - ASSERT_GT(file_names().size(), 0); - ASSERT_OK(this->fs_->DeleteFile(file_names()[0])); - - size_t count = 0; - ASSERT_RAISES(IOError, source_->GetFragments(options_).Visit(OpenFragments(&count))); - } - - void PredicatePushDown() { - partition_expression_ = equal(field_ref("alpha"), ScalarExpression::Make(3)); - MakeDataSource(); - - options_->selector = std::make_shared(); - options_->selector->filters.resize(1); - - // with a filter identical to the partition condition, all fragments are yielded - options_->selector->filters[0] = - std::make_shared(partition_expression_->Copy()); - - size_t count = 0; - // ASSERT_OK(source_->GetFragments(context_)->Visit(OpenFragments(&count))); - // ASSERT_EQ(count, file_names().size()); - - // with a filter which contradicts the partition condition, no fragments are yielded - options_->selector->filters[0] = std::make_shared( - equal(field_ref("alpha"), ScalarExpression::Make(0))); - - count = 0; - ASSERT_OK(source_->GetFragments(options_).Visit(OpenFragments(&count))); - ASSERT_EQ(count, 0); - } - fs::Selector selector_; - std::unique_ptr source_; - std::shared_ptr local_fs_; + std::unique_ptr source_; std::shared_ptr fs_; - std::unique_ptr temp_dir_; std::shared_ptr format_; std::shared_ptr schema_; - std::shared_ptr options_; - std::shared_ptr partition_expression_; + std::shared_ptr options_ = std::make_shared(); }; template @@ -286,11 +189,19 @@ std::unique_ptr> MakeGeneratedRecordBatch( /// \brief A dummy FileFormat implementation class DummyFileFormat : public FileFormat { public: + explicit DummyFileFormat(std::shared_ptr schema = NULLPTR) + : schema_(std::move(schema)) {} + std::string name() const override { return "dummy"; } /// \brief Return true if the given file extension bool IsKnownExtension(const std::string& ext) const override { return ext == name(); } + Status Inspect(const FileSource& source, std::shared_ptr* out) const override { + *out = schema_; + return Status::OK(); + } + /// \brief Open a file for scanning (always returns an empty iterator) Status ScanFile(const FileSource& source, std::shared_ptr scan_options, std::shared_ptr scan_context, @@ -302,6 +213,9 @@ class DummyFileFormat : public FileFormat { inline Status MakeFragment(const FileSource& location, std::shared_ptr opts, std::unique_ptr* out) override; + + protected: + std::shared_ptr schema_; }; class DummyFragment : public FileBasedDataFragment { @@ -319,5 +233,52 @@ Status DummyFileFormat::MakeFragment(const FileSource& source, return Status::OK(); } +class TestFileSystemBasedDataSource : public ::testing::Test { + public: + void SetUp() { options_ = std::make_shared(); } + + void MakeFileSystem(const std::vector& stats) { + ASSERT_OK(fs::internal::MockFileSystem::Make(fs::kNoTime, stats, &fs_)); + } + + void MakeFileSystem(const std::vector& paths) { + std::vector stats{paths.size()}; + std::transform(paths.cbegin(), paths.cend(), stats.begin(), + [](const std::string& p) { return fs::File(p); }); + + ASSERT_OK(fs::internal::MockFileSystem::Make(fs::kNoTime, stats, &fs_)); + } + + void MakeSource(const std::vector& stats, + std::shared_ptr source_partition = nullptr, + PathPartitions partitions = {}) { + MakeFileSystem(stats); + auto format = std::make_shared(); + ASSERT_OK(FileSystemBasedDataSource::Make(fs_.get(), stats, source_partition, + partitions, format, &source_)); + } + + protected: + std::shared_ptr fs_; + std::shared_ptr source_; + std::shared_ptr options_; +}; + +void AssertFragmentsAreFromPath(DataFragmentIterator it, + std::vector expected) { + std::vector actual; + + auto v = [&actual](std::shared_ptr fragment) -> Status { + EXPECT_NE(fragment, nullptr); + auto dummy = std::static_pointer_cast(fragment); + actual.push_back(dummy->source().path()); + return Status::OK(); + }; + + ASSERT_OK(it.Visit(v)); + // Ordering is not guaranteed. + EXPECT_THAT(actual, testing::UnorderedElementsAreArray(expected)); +} + } // namespace dataset } // namespace arrow diff --git a/cpp/src/arrow/filesystem/filesystem_test.cc b/cpp/src/arrow/filesystem/filesystem_test.cc index c8eba30218de..0b183550efff 100644 --- a/cpp/src/arrow/filesystem/filesystem_test.cc +++ b/cpp/src/arrow/filesystem/filesystem_test.cc @@ -155,6 +155,18 @@ TEST(PathUtil, EnsureTrailingSlash) { ASSERT_EQ("/abc/", EnsureTrailingSlash("/abc/")); } +TEST(PathUtil, RemoveTrailingSlash) { + ASSERT_EQ("", std::string(RemoveTrailingSlash(""))); + ASSERT_EQ("", std::string(RemoveTrailingSlash("/"))); + ASSERT_EQ("", std::string(RemoveTrailingSlash("//"))); + ASSERT_EQ("abc/def", std::string(RemoveTrailingSlash("abc/def"))); + ASSERT_EQ("abc/def", std::string(RemoveTrailingSlash("abc/def/"))); + ASSERT_EQ("abc/def", std::string(RemoveTrailingSlash("abc/def//"))); + ASSERT_EQ("/abc/def", std::string(RemoveTrailingSlash("/abc/def"))); + ASSERT_EQ("/abc/def", std::string(RemoveTrailingSlash("/abc/def/"))); + ASSERT_EQ("/abc/def", std::string(RemoveTrailingSlash("/abc/def//"))); +} + //////////////////////////////////////////////////////////////////////////// // Generic MockFileSystem tests @@ -353,6 +365,19 @@ TEST_F(TestMockFS, OpenAppendStream) { CheckFiles({{"ab", time_, "some data"}}); } +TEST_F(TestMockFS, Make) { + std::shared_ptr fs; + ASSERT_OK(MockFileSystem::Make(time_, {}, &fs)); + fs_ = std::static_pointer_cast(fs); + CheckDirs({}); + CheckFiles({}); + + ASSERT_OK(MockFileSystem::Make(time_, {Dir("A/B/C"), File("A/a")}, &fs)); + fs_ = std::static_pointer_cast(fs); + CheckDirs({{"A", time_}, {"A/B", time_}, {"A/B/C", time_}}); + CheckFiles({{"A/a", time_, ""}}); +} + //////////////////////////////////////////////////////////////////////////// // Concrete SubTreeFileSystem tests diff --git a/cpp/src/arrow/filesystem/mockfs.cc b/cpp/src/arrow/filesystem/mockfs.cc index a6ff7c78d6f8..c813355c818f 100644 --- a/cpp/src/arrow/filesystem/mockfs.cc +++ b/cpp/src/arrow/filesystem/mockfs.cc @@ -656,6 +656,40 @@ std::vector MockFileSystem::AllFiles() { return result; } +Status MockFileSystem::CreateFile(const std::string& path, const std::string& contents, + bool recursive) { + auto parent = fs::internal::GetAbstractPathParent(path).first; + if (parent != "") { + RETURN_NOT_OK(CreateDir(parent, recursive)); + } + + std::shared_ptr file; + RETURN_NOT_OK(OpenOutputStream(path, &file)); + RETURN_NOT_OK(file->Write(contents)); + return file->Close(); +} + +Status MockFileSystem::Make(TimePoint current_time, const std::vector& stats, + std::shared_ptr* out) { + auto fs = std::make_shared(current_time); + for (const auto& s : stats) { + switch (s.type()) { + case FileType::Directory: + RETURN_NOT_OK(fs->CreateDir(s.path(), /*recursive*/ true)); + break; + case FileType::File: + RETURN_NOT_OK(fs->CreateFile(s.path(), "", /*recursive*/ true)); + break; + default: + break; + } + } + + *out = fs; + + return Status::OK(); +} + } // namespace internal } // namespace fs } // namespace arrow diff --git a/cpp/src/arrow/filesystem/mockfs.h b/cpp/src/arrow/filesystem/mockfs.h index faafe9a3bccd..e23d3029d9c4 100644 --- a/cpp/src/arrow/filesystem/mockfs.h +++ b/cpp/src/arrow/filesystem/mockfs.h @@ -94,6 +94,15 @@ class ARROW_EXPORT MockFileSystem : public FileSystem { std::vector AllDirs(); std::vector AllFiles(); + // Create a File with a content from a string. + Status CreateFile(const std::string& path, const std::string& content, + bool recursive = true); + + // Create a MockFileSystem out of (empty) FileStats. The content of every + // file is empty and of size 0. All directories will be created recursively. + static Status Make(TimePoint current_time, const std::vector& stats, + std::shared_ptr* out); + class Impl; protected: diff --git a/cpp/src/arrow/filesystem/path_util.cc b/cpp/src/arrow/filesystem/path_util.cc index cfc59a6f8d44..2ba143e59b94 100644 --- a/cpp/src/arrow/filesystem/path_util.cc +++ b/cpp/src/arrow/filesystem/path_util.cc @@ -115,7 +115,7 @@ std::string EnsureTrailingSlash(const std::string& s) { } util::string_view RemoveTrailingSlash(util::string_view key) { - if (!key.empty() && key.back() == kSep) { + while (!key.empty() && key.back() == kSep) { key.remove_suffix(1); } return key; diff --git a/cpp/src/arrow/filesystem/s3fs.cc b/cpp/src/arrow/filesystem/s3fs.cc index 136d12e20302..3fb1cabae301 100644 --- a/cpp/src/arrow/filesystem/s3fs.cc +++ b/cpp/src/arrow/filesystem/s3fs.cc @@ -177,17 +177,18 @@ struct S3Path { std::vector key_parts; static Status FromString(const std::string& s, S3Path* out) { - auto first_sep = s.find_first_of(kSep); + const auto src = internal::RemoveTrailingSlash(s); + auto first_sep = src.find_first_of(kSep); if (first_sep == 0) { return Status::Invalid("Path cannot start with a separator ('", s, "')"); } if (first_sep == std::string::npos) { - *out = {s, s, "", {}}; + *out = {std::string(src), std::string(src), "", {}}; return Status::OK(); } - out->full_path = s; - out->bucket = s.substr(0, first_sep); - out->key = s.substr(first_sep + 1); + out->full_path = std::string(src); + out->bucket = std::string(src.substr(0, first_sep)); + out->key = std::string(src.substr(first_sep + 1)); out->key_parts = internal::SplitAbstractPath(out->key); return internal::ValidateAbstractPathParts(out->key_parts); } @@ -340,6 +341,12 @@ class ObjectInputFile : public io::RandomAccessFile { RETURN_NOT_OK(CheckClosed()); RETURN_NOT_OK(CheckPosition(position, "read")); + nbytes = std::min(nbytes, content_length_ - position); + if (nbytes == 0) { + *bytes_read = 0; + return Status::OK(); + } + // Read the desired range of bytes S3Model::GetObjectResult result; RETURN_NOT_OK(GetObjectRange(client_, path_, position, nbytes, &result)); diff --git a/cpp/src/arrow/filesystem/s3fs.h b/cpp/src/arrow/filesystem/s3fs.h index 8dd3d87f8ff8..c4dacd84fc79 100644 --- a/cpp/src/arrow/filesystem/s3fs.h +++ b/cpp/src/arrow/filesystem/s3fs.h @@ -132,7 +132,7 @@ class ARROW_EXPORT S3FileSystem : public FileSystem { std::unique_ptr impl_; }; -enum class S3LogLevel { Off, Fatal, Error, Warn, Info, Debug, Trace }; +enum class S3LogLevel : int8_t { Off, Fatal, Error, Warn, Info, Debug, Trace }; struct ARROW_EXPORT S3GlobalOptions { S3LogLevel log_level; diff --git a/cpp/src/arrow/filesystem/s3fs_test.cc b/cpp/src/arrow/filesystem/s3fs_test.cc index b19b18e01638..c868e8a2be85 100644 --- a/cpp/src/arrow/filesystem/s3fs_test.cc +++ b/cpp/src/arrow/filesystem/s3fs_test.cc @@ -375,6 +375,10 @@ TEST_F(TestS3FS, GetTargetStatsBucket) { AssertFileStats(fs_.get(), "bucket", FileType::Directory); AssertFileStats(fs_.get(), "empty-bucket", FileType::Directory); AssertFileStats(fs_.get(), "non-existent-bucket", FileType::NonExistent); + // Trailing slashes + AssertFileStats(fs_.get(), "bucket/", FileType::Directory); + AssertFileStats(fs_.get(), "empty-bucket/", FileType::Directory); + AssertFileStats(fs_.get(), "non-existent-bucket/", FileType::NonExistent); } TEST_F(TestS3FS, GetTargetStatsObject) { @@ -393,6 +397,12 @@ TEST_F(TestS3FS, GetTargetStatsObject) { AssertFileStats(fs_.get(), "bucket/emptyd", FileType::NonExistent); AssertFileStats(fs_.get(), "bucket/somed", FileType::NonExistent); AssertFileStats(fs_.get(), "non-existent-bucket/somed", FileType::NonExistent); + + // Trailing slashes + AssertFileStats(fs_.get(), "bucket/emptydir/", FileType::Directory, kNoSize); + AssertFileStats(fs_.get(), "bucket/somefile/", FileType::File, 9); + AssertFileStats(fs_.get(), "bucket/emptyd/", FileType::NonExistent); + AssertFileStats(fs_.get(), "non-existent-bucket/somed/", FileType::NonExistent); } TEST_F(TestS3FS, GetTargetStatsSelector) { @@ -447,6 +457,17 @@ TEST_F(TestS3FS, GetTargetStatsSelector) { ASSERT_OK(fs_->GetTargetStats(select, &stats)); ASSERT_EQ(stats.size(), 0); select.allow_non_existent = false; + + // Trailing slashes + select.base_dir = "empty-bucket/"; + ASSERT_OK(fs_->GetTargetStats(select, &stats)); + ASSERT_EQ(stats.size(), 0); + select.base_dir = "non-existent-bucket/"; + ASSERT_RAISES(IOError, fs_->GetTargetStats(select, &stats)); + select.base_dir = "bucket/"; + ASSERT_OK(fs_->GetTargetStats(select, &stats)); + SortStats(&stats); + ASSERT_EQ(stats.size(), 3); } TEST_F(TestS3FS, GetTargetStatsSelectorRecursive) { @@ -666,7 +687,7 @@ TEST_F(TestS3FS, OpenInputStream) { TEST_F(TestS3FS, OpenInputFile) { std::shared_ptr file; std::shared_ptr buf; - int64_t nbytes = -1, pos = -1; + int64_t nbytes = -1, pos = -1, bytes_read = 0; // Non-existent ASSERT_RAISES(IOError, fs_->OpenInputFile("non-existent-bucket/somefile", &file)); @@ -691,6 +712,15 @@ TEST_F(TestS3FS, OpenInputFile) { AssertBufferEqual(*buf, "data"); ASSERT_OK(file->ReadAt(9, 20, &buf)); AssertBufferEqual(*buf, ""); + + char result[10]; + ASSERT_OK(file->ReadAt(2, 5, &bytes_read, &result)); + ASSERT_EQ(bytes_read, 5); + ASSERT_OK(file->ReadAt(5, 20, &bytes_read, &result)); + ASSERT_EQ(bytes_read, 4); + ASSERT_OK(file->ReadAt(9, 0, &bytes_read, &result)); + ASSERT_EQ(bytes_read, 0); + // Reading past end of file ASSERT_RAISES(IOError, file->ReadAt(10, 20, &buf)); diff --git a/cpp/src/arrow/io/file.cc b/cpp/src/arrow/io/file.cc index 2f3f5bac2e70..3a9306300d70 100644 --- a/cpp/src/arrow/io/file.cc +++ b/cpp/src/arrow/io/file.cc @@ -384,24 +384,47 @@ Status FileOutputStream::Write(const void* data, int64_t length) { int FileOutputStream::file_descriptor() const { return impl_->fd(); } // ---------------------------------------------------------------------- -// Implement MemoryMappedFile as a buffer subclass -// The class doesn't differentiate between size and capacity -class MemoryMappedFile::MemoryMap : public MutableBuffer { +// Implement MemoryMappedFile + +class MemoryMappedFile::MemoryMap + : public std::enable_shared_from_this { public: - MemoryMap() : MutableBuffer(nullptr, 0) {} + // An object representing the entire memory-mapped region. + // It can be sliced in order to return individual subregions, which + // will then keep the original region alive as long as necessary. + class Region : public MutableBuffer { + public: + Region(std::shared_ptr memory_map, uint8_t* data, + int64_t size) + : MutableBuffer(data, size) { + is_mutable_ = memory_map->writable(); + if (!is_mutable_) { + mutable_data_ = nullptr; + } + } - ~MemoryMap() { - ARROW_CHECK_OK(Close()); - if (mutable_data_ != nullptr) { - int result = munmap(mutable_data_, static_cast(map_len_)); - ARROW_CHECK_EQ(result, 0) << "munmap failed"; + ~Region() { + if (data_ != nullptr) { + int result = munmap(data(), static_cast(size_)); + ARROW_CHECK_EQ(result, 0) << "munmap failed"; + } } - } + + // For convenience + uint8_t* data() { return const_cast(data_); } + + void Detach() { data_ = nullptr; } + }; + + MemoryMap() : file_size_(0), map_len_(0) {} + + ~MemoryMap() { ARROW_CHECK_OK(Close()); } Status Close() { if (file_->is_open()) { - // NOTE: we don't unmap here, so that buffers exported through Read() - // remain valid until the MemoryMap object is destroyed + // Lose our reference to the MemoryMappedRegion, so that munmap() + // is called as soon as all buffer exports are released. + region_.reset(); return file_->Close(); } else { return Status::OK(); @@ -422,14 +445,10 @@ class MemoryMappedFile::MemoryMap : public MutableBuffer { constexpr bool truncate = false; constexpr bool write_only = false; RETURN_NOT_OK(file_->OpenWritable(path, truncate, append, write_only)); - - is_mutable_ = true; } else { prot_flags_ = PROT_READ; map_mode_ = MAP_PRIVATE; // Changes are not to be committed back to the file RETURN_NOT_OK(file_->OpenReadable(path)); - - is_mutable_ = false; } map_len_ = offset_ = 0; @@ -450,33 +469,38 @@ class MemoryMappedFile::MemoryMap : public MutableBuffer { if (!writable()) { return Status::IOError("Cannot resize a readonly memory map"); } - if (map_len_ != size_) { + if (map_len_ != file_size_) { return Status::IOError("Cannot resize a partial memory map"); } + if (region_.use_count() > 1) { + // There are buffer exports currently, the MemoryMapRemap() call + // would make the buffers invalid + return Status::IOError("Cannot resize memory map while there are active readers"); + } if (new_size == 0) { - if (mutable_data_ != nullptr) { - // just unmap the mmap and truncate the file to 0 size - if (munmap(mutable_data_, capacity_) != 0) { - return Status::IOError("Cannot unmap the file"); - } + if (map_len_ > 0) { + // Just unmap the mmap and truncate the file to 0 size + region_.reset(); RETURN_NOT_OK(::arrow::internal::FileTruncate(file_->fd(), 0)); - data_ = mutable_data_ = nullptr; - map_len_ = offset_ = size_ = capacity_ = 0; + map_len_ = offset_ = file_size_ = 0; } position_ = 0; return Status::OK(); } - if (mutable_data_) { + if (map_len_ > 0) { void* result; - RETURN_NOT_OK(::arrow::internal::MemoryMapRemap(mutable_data_, size_, new_size, + auto data = region_->data(); + RETURN_NOT_OK(::arrow::internal::MemoryMapRemap(data, map_len_, new_size, file_->fd(), &result)); - map_len_ = size_ = capacity_ = new_size; + region_->Detach(); // avoid munmap() on destruction + region_ = std::make_shared(shared_from_this(), + static_cast(result), new_size); + map_len_ = file_size_ = new_size; offset_ = 0; - data_ = mutable_data_ = static_cast(result); - if (position_ > size_) { - position_ = size_; + if (position_ > map_len_) { + position_ = map_len_; } } else { DCHECK_EQ(position_, 0); @@ -487,9 +511,6 @@ class MemoryMappedFile::MemoryMap : public MutableBuffer { return Status::OK(); } - // map_len_ == size_ if memory mapping on the whole file - int64_t size() const { return map_len_; } - Status Seek(int64_t position) { if (position < 0) { return Status::Invalid("position is out of bounds"); @@ -498,11 +519,28 @@ class MemoryMappedFile::MemoryMap : public MutableBuffer { return Status::OK(); } + Status Slice(int64_t offset, int64_t length, std::shared_ptr* out) { + length = std::max(0, std::min(length, map_len_ - offset)); + + if (length > 0) { + DCHECK_NE(region_, nullptr); + *out = SliceBuffer(region_, offset, length); + } else { + *out = std::make_shared(nullptr, 0); + } + return Status::OK(); + } + + // map_len_ == file_size_ if memory mapping on the whole file + int64_t size() const { return map_len_; } + int64_t position() { return position_; } void advance(int64_t nbytes) { position_ = position_ + nbytes; } - uint8_t* head() { return mutable_data_ + position_; } + uint8_t* head() { return data() + position_; } + + uint8_t* data() { return region_ ? region_->data() : nullptr; } bool writable() { return file_->mode() != FileMode::READ; } @@ -518,10 +556,11 @@ class MemoryMappedFile::MemoryMap : public MutableBuffer { // Initialize the mmap and set size, capacity and the data pointers Status InitMMap(int64_t initial_size, bool resize_file = false, const int64_t offset = 0, const int64_t length = -1) { + DCHECK(!region_); + if (resize_file) { RETURN_NOT_OK(::arrow::internal::FileTruncate(file_->fd(), initial_size)); } - DCHECK(data_ == nullptr && mutable_data_ == nullptr); size_t mmap_length = static_cast(initial_size); if (length > initial_size) { @@ -539,14 +578,19 @@ class MemoryMappedFile::MemoryMap : public MutableBuffer { } map_len_ = mmap_length; offset_ = offset; - size_ = capacity_ = initial_size; - data_ = mutable_data_ = static_cast(result); + region_ = std::make_shared(shared_from_this(), static_cast(result), + map_len_); + file_size_ = initial_size; return Status::OK(); } + std::unique_ptr file_; int prot_flags_; int map_mode_; + + std::shared_ptr region_; + int64_t file_size_; int64_t position_; int64_t offset_; int64_t map_len_; @@ -619,14 +663,7 @@ Status MemoryMappedFile::ReadAt(int64_t position, int64_t nbytes, auto guard_resize = memory_map_->writable() ? std::unique_lock(memory_map_->resize_lock()) : std::unique_lock(); - nbytes = std::max(0, std::min(nbytes, memory_map_->size() - position)); - - if (nbytes > 0) { - *out = SliceBuffer(memory_map_, position, nbytes); - } else { - *out = std::make_shared(nullptr, 0); - } - return Status::OK(); + return memory_map_->Slice(position, nbytes, out); } Status MemoryMappedFile::ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read, @@ -697,12 +734,6 @@ Status MemoryMappedFile::Resize(int64_t new_size) { std::unique_lock write_guard(memory_map_->write_lock(), std::defer_lock); std::unique_lock resize_guard(memory_map_->resize_lock(), std::defer_lock); std::lock(write_guard, resize_guard); - // having both locks, we can check the number of times memory_map_ - // was borrwed (meaning number of reader still holding a ref to it + 1) - // and if it's greater than 1, we fail loudly - if (memory_map_.use_count() > 1) { - return Status::IOError("Cannot resize memory map while there are active readers"); - } RETURN_NOT_OK(memory_map_->Resize(new_size)); return Status::OK(); } diff --git a/cpp/src/arrow/io/file_test.cc b/cpp/src/arrow/io/file_test.cc index eed75add147c..ab12148b3837 100644 --- a/cpp/src/arrow/io/file_test.cc +++ b/cpp/src/arrow/io/file_test.cc @@ -636,7 +636,7 @@ class TestMemoryMappedFile : public ::testing::Test, public MemoryMapFixture { TEST_F(TestMemoryMappedFile, InvalidUsages) {} -TEST_F(TestMemoryMappedFile, ZeroSizeFlie) { +TEST_F(TestMemoryMappedFile, ZeroSizeFile) { std::string path = "io-memory-map-zero-size"; std::shared_ptr result; ASSERT_OK(InitMemoryMap(0, path, &result)); diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h index 60cae37e6360..72b6ce405223 100644 --- a/cpp/src/arrow/type.h +++ b/cpp/src/arrow/type.h @@ -1317,6 +1317,7 @@ class ARROW_EXPORT Schema : public detail::Fingerprintable { /// Returns true if all of the schema fields are equal bool Equals(const Schema& other, bool check_metadata = true) const; + bool operator==(const Schema& other) const { return Equals(other); } /// \brief Return the number of fields (columns) in the schema int num_fields() const; diff --git a/dev/fuzzit/Dockerfile b/dev/fuzzit/Dockerfile index 602cb5f254cd..447f1b844dac 100644 --- a/dev/fuzzit/Dockerfile +++ b/dev/fuzzit/Dockerfile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM arrow:cpp +FROM arrowdev/arrow-cpp:latest RUN apt-get update && \ apt-get install -y -q \ diff --git a/dev/lint/Dockerfile b/dev/lint/Dockerfile index c9658b8f1319..649120657587 100644 --- a/dev/lint/Dockerfile +++ b/dev/lint/Dockerfile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM arrow:python-3.6 +FROM arrowdev/arrow-python-3.6 RUN apt-get update && \ apt-get install -y -q \ @@ -30,4 +30,4 @@ RUN conda install flake8 && \ ENV PATH=/opt/iwyu/bin:$PATH ADD ci/docker_install_iwyu.sh /arrow/ci/ -RUN arrow/ci/docker_install_iwyu.sh \ No newline at end of file +RUN arrow/ci/docker_install_iwyu.sh diff --git a/dev/release/download_rc_binaries.py b/dev/release/download_rc_binaries.py new file mode 100644 index 000000000000..01ab2c468eaf --- /dev/null +++ b/dev/release/download_rc_binaries.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import sys + +try: + import argparse + import concurrent.futures as cf + import functools + import hashlib + import json + import os + import subprocess + import urllib.request +except ImportError: + if sys.version_info.major < 3: + raise Exception("Please use Python 3 to run this script") + raise + + +BINTRAY_API_ROOT = "https://bintray.com/api/v1" +BINTRAY_DL_ROOT = "https://dl.bintray.com" +BINTRAY_REPO = 'apache/arrow' +DEFAULT_PARALLEL_DOWNLOADS = 8 + + +class Bintray: + + def __init__(self, repo=BINTRAY_REPO): + self.repo = repo + + def get_file_list(self, package, version): + url = os.path.join(BINTRAY_API_ROOT, 'packages', self.repo, package, + 'versions', version, 'files') + request = urllib.request.urlopen(url).read() + return json.loads(request) + + def download_files(self, files, dest=None, num_parallel=None): + """ + Download files from Bintray in parallel. If file already exists, will + overwrite if the checksum does not match what Bintray says it should be + + Parameters + ---------- + files : List[Dict] + File listing from Bintray + dest : str, default None + Defaults to current working directory + num_parallel : int, default 8 + Number of files to download in parallel. If set to None, uses + default + """ + if dest is None: + dest = os.getcwd() + if num_parallel is None: + num_parallel = DEFAULT_PARALLEL_DOWNLOADS + + if num_parallel == 1: + for path in files: + self._download_file(dest, path) + else: + parallel_map_terminate_early( + functools.partial(self._download_file, dest), + files, + num_parallel + ) + + def _download_file(self, dest, info): + relpath = info['path'] + + base, filename = os.path.split(relpath) + + dest_dir = os.path.join(dest, base) + os.makedirs(dest_dir, exist_ok=True) + + dest_path = os.path.join(dest_dir, filename) + + if os.path.exists(dest_path): + with open(dest_path, 'rb') as f: + sha256sum = hashlib.sha256(f.read()).hexdigest() + if sha256sum == info['sha256']: + print('Local file {} sha256 matches, skipping' + .format(dest_path)) + return + else: + print('Local file sha256 does not match, overwriting') + + print("Downloading {} to {}".format(relpath, dest_path)) + + bintray_abspath = os.path.join(BINTRAY_DL_ROOT, self.repo, relpath) + + cmd = [ + 'curl', '--fail', '--location', + '--output', dest_path, bintray_abspath + ] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, stderr = proc.communicate() + if proc.returncode != 0: + raise Exception("Downloading {} failed\nstdout: {}\nstderr: {}" + .format(relpath, stdout, stderr)) + + +def parallel_map_terminate_early(f, iterable, num_parallel): + tasks = [] + with cf.ProcessPoolExecutor(num_parallel) as pool: + for v in iterable: + tasks.append(pool.submit(functools.partial(f, v))) + + for task in cf.as_completed(tasks): + if task.exception() is not None: + e = task.exception() + for task in tasks: + task.cancel() + raise e + + +ARROW_PACKAGE_TYPES = ['centos', 'debian', 'python', 'ubuntu'] + + +def download_rc_binaries(version, rc_number, dest=None, num_parallel=None): + bintray = Bintray() + + version_string = '{}-rc{}'.format(version, rc_number) + for package_type in ARROW_PACKAGE_TYPES: + files = bintray.get_file_list('{}-rc'.format(package_type), + version_string) + bintray.download_files(files, dest=dest, num_parallel=num_parallel) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Download release candidate binaries' + ) + parser.add_argument('version', type=str, help='The version number') + parser.add_argument('rc_number', type=int, + help='The release candidate number, e.g. 0, 1, etc') + parser.add_argument('--dest', type=str, default=os.getcwd(), + help='The output folder for the downloaded files') + parser.add_argument('--num_parallel', type=int, default=8, + help='The number of concurrent downloads to do') + args = parser.parse_args() + + download_rc_binaries(args.version, args.rc_number, dest=args.dest, + num_parallel=args.num_parallel) diff --git a/dev/release/verify-release-candidate.bat b/dev/release/verify-release-candidate.bat index 299297e95c54..cfbf56502ab8 100644 --- a/dev/release/verify-release-candidate.bat +++ b/dev/release/verify-release-candidate.bat @@ -41,11 +41,14 @@ set PYTHON=3.6 @rem Using call with conda.bat seems necessary to avoid terminating the batch @rem script execution -call conda create -p %_VERIFICATION_CONDA_ENV% -f -q -y python=%PYTHON% || exit /B +call conda create -p %_VERIFICATION_CONDA_ENV% ^ + --no-shortcuts -f -q -y python=%PYTHON% ^ + || exit /B call activate %_VERIFICATION_CONDA_ENV% || exit /B call conda install -y ^ + --no-shortcuts ^ python=3.7 ^ git ^ --file=ci\conda_env_cpp.yml ^ @@ -68,8 +71,9 @@ pushd %ARROW_SOURCE%\cpp\build @rem This is the path for Visual Studio Community 2017 call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\Tools\VsDevCmd.bat" -arch=amd64 -cmake -G "%GENERATOR%" ^ +cmake -G "Ninja" ^ -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^ + -DARROW_BUILD_STATIC=OFF ^ -DARROW_BOOST_USE_SHARED=ON ^ -DARROW_BUILD_TESTS=ON ^ -DGTest_SOURCE=BUNDLED ^ @@ -79,7 +83,12 @@ cmake -G "%GENERATOR%" ^ -DARROW_PYTHON=ON ^ -DARROW_PARQUET=ON ^ .. || exit /B -cmake --build . --target INSTALL --config %CONFIGURATION% || exit /B + +@rem NOTE(wesm): Building googletest is flaky for me with ninja. Building it +@rem first fixes the problem +ninja googletest_ep || exit /B + +ninja install || exit /B @rem Get testing datasets for Parquet unit tests git clone https://github.com/apache/parquet-testing.git %_VERIFICATION_DIR%\parquet-testing diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index b1a5997de348..d09d13659eaf 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -99,49 +99,13 @@ fetch_archive() { shasum -a 512 -c ${dist_name}.tar.gz.sha512 } -bintray() { - local command=$1 - shift - local path=$1 - shift - local url=https://bintray.com/api/v1${path} - echo "${command} ${url}" 1>&2 - curl \ - --fail \ - --request ${command} \ - ${url} \ - "$@" | \ - jq . -} - -download_bintray_files() { - local target=$1 - - local version_name=${VERSION}-rc${RC_NUMBER} - - local file - bintray \ - GET /packages/${BINTRAY_REPOSITORY}/${target}-rc/versions/${version_name}/files | \ - jq -r ".[].path" | \ - while read file; do - mkdir -p "$(dirname ${file})" - curl \ - --fail \ - --location \ - --output ${file} \ - https://dl.bintray.com/${BINTRAY_REPOSITORY}/${file} - done -} - test_binary() { local download_dir=binaries mkdir -p ${download_dir} - pushd ${download_dir} - # takes longer on slow network - for target in centos debian python ubuntu; do - download_bintray_files ${target} - done + python3 $SOURCE_DIR/download_rc_binaries.py $VERSION $RC_NUMBER --dest=${download_dir} + + pushd ${download_dir} # verify the signature and the checksums of each artifact find . -name '*.asc' | while read sigfile; do diff --git a/dev/tasks/docker-tests/circle.linux.yml b/dev/tasks/docker-tests/circle.linux.yml index 7b664eeafeda..9384a7ee1821 100644 --- a/dev/tasks/docker-tests/circle.linux.yml +++ b/dev/tasks/docker-tests/circle.linux.yml @@ -19,20 +19,29 @@ version: 2 jobs: build: machine: - image: circleci/classic:201808-01 + image: ubuntu-1604:201903-01 + docker_layer_caching: true steps: - run: docker -v - run: docker-compose -v + {% if arrow.branch == "master" %} + - run: | + if [ -n $DOCKER_USER ] && [ -n $DOCKER_PASS ]; then + docker login -u $DOCKER_USER -p $DOCKER_PASS + fi + {% endif %} - run: git clone --no-checkout {{ arrow.remote }} arrow - run: git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - run: git -C arrow checkout FETCH_HEAD - run: git -C arrow submodule update --init --recursive - - run: | - pushd arrow - {%- for command in commands %} - {{ command }} - {%- endfor %} - popd + - run: + command: | + pushd arrow + {%- for command in commands %} + {{ command }} + {%- endfor %} + popd + no_output_timeout: "1h" workflows: version: 2 diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 804ed8f701d0..3c19451d08b8 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -1158,9 +1158,12 @@ tasks: template: docker-tests/circle.linux.yml params: commands: + - docker-compose pull --ignore-pull-failures cpp + - docker-compose pull --ignore-pull-failures r - docker-compose build cpp - docker-compose build r - docker-compose run r + - docker-compose push --ignore-push-failures r docker-r-conda: ci: circle @@ -1168,9 +1171,12 @@ tasks: template: docker-tests/circle.linux.yml params: commands: + - docker-compose pull --ignore-pull-failures cpp + - docker-compose pull --ignore-pull-failures r-conda - docker-compose build cpp - docker-compose build r-conda - docker-compose run r-conda + - docker-compose push --ignore-push-failures r-conda docker-r-sanitizer: ci: circle @@ -1178,8 +1184,10 @@ tasks: template: docker-tests/circle.linux.yml params: commands: + - docker-compose pull --ignore-pull-failures r-sanitizer - docker-compose build r-sanitizer - docker-compose run r-sanitizer + - docker-compose push --ignore-push-failures r-sanitizer docker-rust: ci: circle @@ -1187,8 +1195,10 @@ tasks: template: docker-tests/circle.linux.yml params: commands: + - docker-compose pull --ignore-pull-failures rust - docker-compose build rust - docker-compose run rust + - docker-compose push --ignore-push-failures rust docker-cpp: ci: circle @@ -1196,8 +1206,10 @@ tasks: template: docker-tests/circle.linux.yml params: commands: + - docker-compose pull --ignore-pull-failures cpp - docker-compose build cpp - docker-compose run cpp + - docker-compose push --ignore-push-failures cpp docker-cpp-release: ci: circle @@ -1205,8 +1217,10 @@ tasks: template: docker-tests/circle.linux.yml params: commands: - - docker-compose build cpp + - docker-compose pull --ignore-pull-failures cpp-release + - docker-compose build cpp-release - docker-compose run cpp-release + - docker-compose push --ignore-push-failures cpp-release docker-cpp-alpine: ci: circle @@ -1214,8 +1228,10 @@ tasks: template: docker-tests/circle.linux.yml params: commands: + - docker-compose pull --ignore-pull-failures cpp-alpine - docker-compose build cpp-alpine - docker-compose run cpp-alpine + - docker-compose push --ignore-push-failures cpp-alpine docker-cpp-cmake32: ci: circle @@ -1223,8 +1239,10 @@ tasks: template: docker-tests/circle.linux.yml params: commands: + - docker-compose pull --ignore-pull-failures cpp-cmake32 - docker-compose build cpp-cmake32 - docker-compose run cpp-cmake32 + - docker-compose push --ignore-push-failures cpp-cmake32 docker-cpp-fuzzit: ci: circle @@ -1232,9 +1250,12 @@ tasks: template: docker-tests/circle.linux.yml params: commands: + - docker-compose pull --ignore-pull-failures cpp + - docker-compose pull --ignore-pull-failures fuzzit - docker-compose build cpp - docker-compose build fuzzit - docker-compose run fuzzit + - docker-compose push --ignore-push-failures fuzzit docker-cpp-static-only: ci: circle @@ -1242,8 +1263,11 @@ tasks: template: docker-tests/circle.linux.yml params: commands: + - docker-compose pull --ignore-pull-failures cpp + - docker-compose pull --ignore-pull-failures cpp-static-only - docker-compose build cpp - docker-compose run cpp-static-only + - docker-compose push --ignore-push-failures cpp-static-only docker-c_glib: ci: circle @@ -1251,9 +1275,12 @@ tasks: template: docker-tests/circle.linux.yml params: commands: + - docker-compose pull --ignore-pull-failures cpp + - docker-compose pull --ignore-pull-failures c_glib - docker-compose build cpp - docker-compose build c_glib - docker-compose run c_glib + - docker-compose push --ignore-push-failures c_glib docker-go: ci: circle @@ -1261,8 +1288,10 @@ tasks: template: docker-tests/circle.linux.yml params: commands: + - docker-compose pull --ignore-pull-failures go - docker-compose build go - docker-compose run go + - docker-compose push --ignore-push-failures go docker-js: ci: circle @@ -1270,8 +1299,10 @@ tasks: template: docker-tests/circle.linux.yml params: commands: + - docker-compose pull --ignore-pull-failures js - docker-compose build js - docker-compose run js + - docker-compose push --ignore-push-failures js docker-java: ci: circle @@ -1279,8 +1310,10 @@ tasks: template: docker-tests/circle.linux.yml params: commands: + - docker-compose pull --ignore-pull-failures java - docker-compose build java - docker-compose run java + - docker-compose push --ignore-push-failures java docker-python-2.7: ci: circle @@ -1290,9 +1323,12 @@ tasks: environment: PYTHON_VERSION: 2.7 commands: + - docker-compose pull --ignore-pull-failures cpp + - docker-compose pull --ignore-pull-failures python - docker-compose build cpp - docker-compose build python - docker-compose run python + - docker-compose push --ignore-push-failures python docker-python-3.6: ci: circle @@ -1302,9 +1338,12 @@ tasks: environment: PYTHON_VERSION: 3.6 commands: + - docker-compose pull --ignore-pull-failures cpp + - docker-compose pull --ignore-pull-failures python - docker-compose build cpp - docker-compose build python - docker-compose run python + - docker-compose push --ignore-push-failures python docker-python-3.7: ci: circle @@ -1314,9 +1353,12 @@ tasks: environment: PYTHON_VERSION: 3.7 commands: + - docker-compose pull --ignore-pull-failures cpp + - docker-compose pull --ignore-pull-failures python - docker-compose build cpp - docker-compose build python - docker-compose run python + - docker-compose push --ignore-push-failures python docker-python-2.7-alpine: ci: circle @@ -1326,9 +1368,12 @@ tasks: environment: PYTHON_VERSION: 2.7 commands: + - docker-compose pull --ignore-pull-failures cpp-alpine + - docker-compose pull --ignore-pull-failures python-alpine - docker-compose build cpp-alpine - docker-compose build python-alpine - docker-compose run python-alpine + - docker-compose push --ignore-push-failures python-alpine docker-python-3.6-alpine: ci: circle @@ -1338,9 +1383,12 @@ tasks: environment: PYTHON_VERSION: 3.6 commands: + - docker-compose pull --ignore-pull-failures cpp-alpine + - docker-compose pull --ignore-pull-failures python-alpine - docker-compose build cpp-alpine - docker-compose build python-alpine - docker-compose run python-alpine + - docker-compose push --ignore-push-failures python-alpine docker-python-2.7-nopandas: ci: circle @@ -1350,10 +1398,14 @@ tasks: environment: PYTHON_VERSION: 2.7 commands: + - docker-compose pull --ignore-pull-failures cpp + - docker-compose pull --ignore-pull-failures python + - docker-compose pull --ignore-pull-failures python-nopandas - docker-compose build cpp - docker-compose build python - docker-compose build python-nopandas - docker-compose run python-nopandas + - docker-compose push --ignore-push-failures python-nopandas docker-python-3.6-nopandas: ci: circle @@ -1363,10 +1415,14 @@ tasks: environment: PYTHON_VERSION: 3.6 commands: + - docker-compose pull --ignore-pull-failures cpp + - docker-compose pull --ignore-pull-failures python + - docker-compose pull --ignore-pull-failures python-nopandas - docker-compose build cpp - docker-compose build python - docker-compose build python-nopandas - docker-compose run python-nopandas + - docker-compose push --ignore-push-failures python-nopandas ###################### Documentation building tests ######################### @@ -1378,10 +1434,14 @@ tasks: environment: PYTHON_VERSION: 3.6 commands: + - docker-compose pull --ignore-pull-failures cpp + - docker-compose pull --ignore-pull-failures python + - docker-compose pull --ignore-pull-failures docs - docker-compose build cpp - docker-compose build python - docker-compose build docs - docker-compose run docs + - docker-compose push --ignore-push-failures docs ############################## Linter tests ################################# @@ -1393,10 +1453,14 @@ tasks: environment: PYTHON_VERSION: 3.6 commands: + - docker-compose pull --ignore-pull-failures cpp + - docker-compose pull --ignore-pull-failures python + - docker-compose pull --ignore-pull-failures lint - docker-compose build cpp - docker-compose build python - docker-compose build lint - docker-compose run lint + - docker-compose push --ignore-push-failures lint docker-iwyu: ci: circle @@ -1406,6 +1470,9 @@ tasks: environment: PYTHON_VERSION: 3.6 commands: + - docker-compose pull --ignore-pull-failures cpp + - docker-compose pull --ignore-pull-failures python + - docker-compose pull --ignore-pull-failures lint - docker-compose build cpp - docker-compose build python - docker-compose build lint @@ -1419,6 +1486,9 @@ tasks: environment: PYTHON_VERSION: 3.6 commands: + - docker-compose pull --ignore-pull-failures cpp + - docker-compose pull --ignore-pull-failures python + - docker-compose pull --ignore-pull-failures lint - docker-compose build cpp - docker-compose build python - docker-compose build lint @@ -1434,10 +1504,14 @@ tasks: environment: PYTHON_VERSION: 3.6 commands: + - docker-compose pull --ignore-pull-failures cpp + - docker-compose pull --ignore-pull-failures python + - docker-compose pull --ignore-pull-failures dask-integration - docker-compose build cpp - docker-compose build python - docker-compose build dask-integration - docker-compose run dask-integration + - docker-compose push --ignore-push-failures dask-integration docker-hdfs-integration: ci: circle @@ -1447,10 +1521,14 @@ tasks: environment: PYTHON_VERSION: 3.6 commands: + - docker-compose pull --ignore-pull-failures cpp + - docker-compose pull --ignore-pull-failures python + - docker-compose pull --ignore-pull-failures hdfs-integration - docker-compose build cpp - docker-compose build python - docker-compose build hdfs-integration - docker-compose run hdfs-integration + - docker-compose push --ignore-push-failures hdfs-integration docker-spark-integration: ci: circle @@ -1460,10 +1538,14 @@ tasks: environment: PYTHON_VERSION: 3.6 commands: + - docker-compose pull --ignore-pull-failures cpp + - docker-compose pull --ignore-pull-failures python + - docker-compose pull --ignore-pull-failures spark-integration - docker-compose build cpp - docker-compose build python - docker-compose build spark-integration - docker-compose run spark-integration + - docker-compose push --ignore-push-failures spark-integration docker-turbodbc-integration: ci: circle @@ -1473,10 +1555,14 @@ tasks: environment: PYTHON_VERSION: 3.6 commands: + - docker-compose pull --ignore-pull-failures cpp + - docker-compose pull --ignore-pull-failures python + - docker-compose pull --ignore-pull-failures turbodbc-integration - docker-compose build cpp - docker-compose build python - docker-compose build turbodbc-integration - docker-compose run turbodbc-integration + - docker-compose push --ignore-push-failures turbodbc-integration docker-pandas-master: ci: circle @@ -1486,6 +1572,8 @@ tasks: environment: PYTHON_VERSION: 3.6 commands: + - docker-compose pull --ignore-pull-failures cpp + - docker-compose pull --ignore-pull-failures python - docker-compose build cpp - docker-compose build python - docker-compose build --no-cache pandas-master diff --git a/docker-compose.yml b/docker-compose.yml index a5cb6061f617..0ce7e38bc50d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -87,7 +87,7 @@ services: # Usage: # docker-compose build build-context # docker-compose run build-context - image: arrow:build-context + image: arrowdev/arrow-build-context:latest build: context: . dockerfile: dev/Dockerfile.context @@ -99,7 +99,7 @@ services: # docker-compose build cpp # docker-compose build c_glib # docker-compose run c_glib - image: arrow:c_glib + image: arrowdev/arrow-c_glib:latest build: context: . dockerfile: c_glib/Dockerfile @@ -109,7 +109,7 @@ services: # Usage: # docker-compose build cpp # docker-compose run cpp - image: arrow:cpp + image: arrowdev/arrow-cpp:latest shm_size: 2G build: context: . @@ -120,7 +120,7 @@ services: # Usage: # docker-compose build cpp-system-deps # docker-compose run cpp-system-deps - image: ursalab/arrow:cpp-system-deps + image: arrowdev/arrow-cpp-system-deps:latest shm_size: 2G cap_add: # LeakSanitizer and gdb requires ptrace(2) @@ -131,7 +131,6 @@ services: args: LLVM_VERSION: 7 environment: - ARROW_FLIGHT: "ON" ARROW_USE_ASAN: "ON" ARROW_USE_UBSAN: "ON" volumes: @@ -143,7 +142,7 @@ services: # Usage: # docker-compose build cpp # docker-compose run cpp-release - image: arrow:cpp + image: arrowdev/arrow-cpp-release:latest shm_size: 2G build: context: . @@ -157,7 +156,7 @@ services: # Usage: # docker-compose build cpp # docker-compose run cpp-static-only - image: arrow:cpp + image: arrowdev/arrow-cpp:latest shm_size: 2G environment: ARROW_BUILD_SHARED: "OFF" @@ -169,7 +168,7 @@ services: # Usage: # docker-compose build cpp # docker-compose run cpp-static-only - image: arrow:cpp + image: arrowdev/arrow-cpp:latest shm_size: 2G environment: ARROW_BUILD_STATIC: "OFF" @@ -180,7 +179,7 @@ services: # Usage: # docker-compose build cpp-cmake32 # docker-compose run cpp-cmake32 - image: arrow:cpp-cmake32 + image: arrowdev/arrow-cpp-cmake32:latest shm_size: 2G build: context: . @@ -200,7 +199,7 @@ services: # Usage: # docker-compose build cpp-alpine # docker-compose run cpp-alpine - image: arrow:cpp-alpine + image: arrowdev/arrow-cpp-alpine:latest shm_size: 2G build: context: . @@ -213,7 +212,7 @@ services: # Usage: # docker-compose build cpp-fedora # docker-compose run cpp-fedora - image: arrow:cpp-fedora + image: arrowdev/arrow-cpp-fedora:latest shm_size: 2G build: context: . @@ -226,7 +225,7 @@ services: # Usage: # docker-compose build cpp-debian-testing # docker-compose run cpp-debian-testing - image: arrow:cpp-debian-testing + image: arrowdev/arrow-cpp-debian-testing:latest shm_size: 2G build: context: . @@ -239,7 +238,7 @@ services: # Usage: # docker-compose build cpp-ubuntu-trusty # docker-compose run cpp-ubuntu-trusty - image: arrow:cpp-ubuntu-trusty + image: arrowdev/arrow-cpp-ubuntu-trusty:latest shm_size: 2G build: context: . @@ -252,7 +251,7 @@ services: # Usage: # docker-compose build cpp-ubuntu-xenial # docker-compose run cpp-ubuntu-xenial - image: arrow:cpp-ubuntu-xenial + image: arrowdev/arrow-cpp-ubuntu-xenial:latest shm_size: 2G build: context: . @@ -267,7 +266,7 @@ services: # Usage: # docker-compose build cpp-minimum-boost-version # docker-compose run cpp-minimum-boost-version - image: arrow:cpp-ubuntu-xenial + image: arrowdev/arrow-cpp-ubuntu-xenial:latest shm_size: 2G build: context: . @@ -282,7 +281,7 @@ services: # Usage: # docker-compose build cpp-ubuntu-bionic # docker-compose run cpp-ubuntu-bionic - image: arrow:cpp-ubuntu-bionic + image: arrowdev/arrow-cpp-ubuntu-bionic:latest shm_size: 2G build: context: . @@ -295,7 +294,7 @@ services: # Usage: # docker-compose build go # docker-compose run go - image: arrow:go + image: arrowdev/arrow-go:latest build: context: . dockerfile: go/Dockerfile @@ -305,7 +304,7 @@ services: # Usage: # docker-compose build java # docker-compose run java - image: arrow:java + image: arrowdev/arrow-java:latest build: context: . dockerfile: java/Dockerfile @@ -317,7 +316,7 @@ services: # Usage: # docker-compose build java-all-jdks # docker-compose run java-all-jdks - image: ursalab/arrow-ci-java-all-jdks:latest + image: arrowdev/arrow-java-all-jdks:latest build: context: . dockerfile: java/Dockerfile.all-jdks @@ -326,7 +325,7 @@ services: - maven-cache:/root/.m2:delegated js: - image: arrow:js + image: arrowdev/arrow-js:latest build: context: . dockerfile: js/Dockerfile @@ -337,7 +336,7 @@ services: # docker-compose build cpp # docker-compose build python # docker-compose run python - image: arrow:python-${PYTHON_VERSION:-3.6} + image: arrowdev/arrow-python-${PYTHON_VERSION:-3.6}:latest shm_size: 2G build: context: . @@ -352,7 +351,7 @@ services: # docker-compose build cpp-alpine # docker-compose build python-alpine # docker-compose run python-alpine - image: arrow:python-${PYTHON_VERSION:-3.6}-alpine + image: arrowdev/arrow-python-${PYTHON_VERSION:-3.6}-alpine:latest shm_size: 2G build: context: . @@ -368,7 +367,7 @@ services: # docker-compose build python # docker-compose build python-nopandas # docker-compose run python-nopandas - image: arrow:python-${PYTHON_VERSION:-3.6}-nopandas + image: arrowdev/arrow-python-${PYTHON_VERSION:-3.6}-nopandas:latest shm_size: 2G build: context: . @@ -381,7 +380,7 @@ services: # Usage: # docker-compose build rust # docker-compose run rust - image: arrow:rust + image: arrowdev/arrow-rust:latest build: context: . dockerfile: rust/Dockerfile @@ -394,7 +393,7 @@ services: # docker-compose build cpp # docker-compose build r # docker-compose run r - image: arrow:r + image: arrowdev/arrow-r:latest build: context: . dockerfile: r/Dockerfile @@ -404,7 +403,7 @@ services: # Usage: # docker-compose build r-sanitizer # docker-compose run r-sanitizer - image: arrow:r-sanitizer + image: arrowdev/arrow-r-sanitizer:latest cap_add: # LeakSanitizer and gdb requires ptrace(2) - SYS_PTRACE @@ -419,7 +418,7 @@ services: # docker-compose build cpp # docker-compose build r-conda # docker-compose run r-conda - image: arrow:r-conda-${R_VERSION:-3.5.1} + image: arrowdev/arrow-r-conda-${R_VERSION:-3.5.1}:latest shm_size: 2G build: context: . @@ -440,7 +439,7 @@ services: # docker-compose build python # docker-compose build lint # docker-compose run lint - image: arrow:lint + image: arrowdev/arrow-lint:latest build: context: . dockerfile: dev/lint/Dockerfile @@ -448,10 +447,45 @@ services: volumes: *ubuntu-volumes fuzzit: + # Submit a fuzzing job on fuzzit.dev # Usage: # docker-compose build cpp # docker-compose build fuzzit # docker-compose run fuzzit + image: arrowdev/arrow-fuzzit:latest + build: + context: . + dockerfile: dev/fuzzit/Dockerfile + environment: + CI_ARROW_SHA: ${CI_ARROW_SHA} + CI_ARROW_BRANCH: ${CI_ARROW_BRANCH} + FUZZIT_JOB_TYPE: fuzzing + FUZZIT_API_KEY: c0b760d37db6012fcaafd8ca5f412ba7bcd297ac969650502994b51aa11798153824442f999a067e1ef67821989ed664 + volumes: *ubuntu-volumes + + fuzzit-regression: + # Submit a regression job on fuzzit.dev + # Usage: + # docker-compose build cpp + # docker-compose build fuzzit-regression + # docker-compose run fuzzit-regression + image: arrow:fuzzit + build: + context: . + dockerfile: dev/fuzzit/Dockerfile + environment: + CI_ARROW_SHA: ${CI_ARROW_SHA} + CI_ARROW_BRANCH: ${CI_ARROW_BRANCH} + FUZZIT_JOB_TYPE: regression + FUZZIT_API_KEY: c0b760d37db6012fcaafd8ca5f412ba7bcd297ac969650502994b51aa11798153824442f999a067e1ef67821989ed664 + volumes: *ubuntu-volumes + + fuzzit-local-regression: + # Run a local fuzzit regression job + # Usage: + # docker-compose build cpp + # docker-compose build fuzzit-local-regression + # docker-compose run fuzzit-local-regression image: arrow:fuzzit build: context: . @@ -459,6 +493,7 @@ services: environment: CI_ARROW_SHA: ${CI_ARROW_SHA} CI_ARROW_BRANCH: ${CI_ARROW_BRANCH} + FUZZIT_JOB_TYPE: local-regression volumes: *ubuntu-volumes iwyu: @@ -468,7 +503,7 @@ services: # docker-compose build python # docker-compose build lint # docker-compose run iwyu - image: arrow:lint + image: arrowdev/arrow-lint:latest command: arrow/dev/lint/run_iwyu.sh volumes: *ubuntu-volumes @@ -478,7 +513,7 @@ services: # docker-compose build python # docker-compose build lint # docker-compose run clang-format - image: arrow:lint + image: arrowdev/arrow-lint:latest command: arrow/dev/lint/run_clang_format.sh volumes: *ubuntu-volumes @@ -488,7 +523,7 @@ services: # docker-compose build python # docker-compose build lint # docker-compose run clang-tidy - image: arrow:lint + image: arrowdev/arrow-lint:latest command: arrow/dev/lint/run_clang_tidy.sh volumes: *ubuntu-volumes @@ -498,7 +533,7 @@ services: # docker-compose build python # docker-compose build docs # docker-compose run docs - image: arrow:docs + image: arrowdev/arrow-docs:latest build: context: . dockerfile: docs/Dockerfile @@ -514,7 +549,7 @@ services: # $ docker-compose pull python-manylinux1 # an then run: # $ docker-compose run -e PYTHON_VERSION=3.7 python-manylinux1 - image: ursalab/arrow_manylinux1_x86_64_base:0.14.1-static-zlib + image: arrowdev/arrow_manylinux1_x86_64_base:0.14.1-static-zlib build: context: python/manylinux1 dockerfile: Dockerfile-x86_64_base @@ -536,7 +571,7 @@ services: # $ docker-compose pull python-manylinux2010 # an then run: # $ docker-compose run -e PYTHON_VERSION=3.7 python-manylinux2010 - image: ursalab/arrow_manylinux2010_x86_64_base:0.14.1-static-zlib + image: arrowdev/arrow_manylinux2010_x86_64_base:0.14.1-static-zlib build: context: python/manylinux2010 dockerfile: Dockerfile-x86_64_base @@ -565,7 +600,7 @@ services: # docker-compose build python # docker-compose build --no-cache pandas-master # docker-compose run pandas-master - image: arrow:pandas-master + image: arrowdev/arrow-pandas-master:latest build: context: . dockerfile: integration/pandas/Dockerfile @@ -608,7 +643,7 @@ services: # docker-compose build python # docker-compose build hdfs-integration # docker-compose run hdfs-integration - image: arrow:hdfs-${HDFS_VERSION:-2.9.2} + image: arrowdev/arrow-hdfs-${HDFS_VERSION:-2.9.2}:latest links: - hdfs-namenode:hdfs-namenode - hdfs-datanode-1:hdfs-datanode-1 @@ -632,7 +667,7 @@ services: # docker-compose build python # docker-compose build dask-integration # docker-compose run dask-integration - image: arrow:dask-integration + image: arrowdev/arrow-dask-integration:latest build: context: . dockerfile: integration/dask/Dockerfile @@ -645,7 +680,7 @@ services: # docker-compose build python # docker-compose build turbodbc-integration # docker-compose run turbodbc-integration - image: arrow:turbodbc-integration + image: arrowdev/arrow-turbodbc-integration:latest build: context: . dockerfile: integration/turbodbc/Dockerfile @@ -658,7 +693,7 @@ services: # docker-compose build python # docker-compose build spark-integration # docker-compose run spark-integration - image: arrow:spark-${SPARK_VERSION:-master} + image: arrowdev/arrow-spark-${SPARK_VERSION:-master}:latest environment: - SPARK_VERSION=${SPARK_VERSION:-master} build: @@ -675,7 +710,7 @@ services: ################################# Release ################################### release-source: - image: arrow:release-source + image: arrowdev/arrow-release-source:latest build: context: . dockerfile: dev/release/source/Dockerfile diff --git a/docs/Dockerfile b/docs/Dockerfile index 8c2d00b99e79..016d334f7b49 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM arrow:python-3.6 +FROM arrowdev/arrow-python-3.6:latest COPY ci/conda_env_sphinx.yml /arrow/ci/ RUN conda install --file arrow/ci/conda_env_sphinx.yml && \ diff --git a/go/arrow/array/array.go b/go/arrow/array/array.go index c3577337a7b9..9fc4cc0be6db 100644 --- a/go/arrow/array/array.go +++ b/go/arrow/array/array.go @@ -20,7 +20,7 @@ import ( "sync/atomic" "github.com/apache/arrow/go/arrow" - "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/bitutil" "github.com/apache/arrow/go/arrow/internal/debug" ) diff --git a/go/arrow/array/array_test.go b/go/arrow/array/array_test.go index af50cf955af3..48c238656b4a 100644 --- a/go/arrow/array/array_test.go +++ b/go/arrow/array/array_test.go @@ -298,4 +298,4 @@ func TestArraySliceTypes(t *testing.T) { } }) } -} \ No newline at end of file +} diff --git a/go/arrow/array/boolean.go b/go/arrow/array/boolean.go index 5095b17318ab..e352e6e043b3 100644 --- a/go/arrow/array/boolean.go +++ b/go/arrow/array/boolean.go @@ -21,7 +21,7 @@ import ( "strings" "github.com/apache/arrow/go/arrow" - "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/bitutil" "github.com/apache/arrow/go/arrow/memory" ) diff --git a/go/arrow/array/booleanbuilder.go b/go/arrow/array/booleanbuilder.go index 943df1f12007..4a38156849af 100644 --- a/go/arrow/array/booleanbuilder.go +++ b/go/arrow/array/booleanbuilder.go @@ -20,7 +20,7 @@ import ( "sync/atomic" "github.com/apache/arrow/go/arrow" - "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/bitutil" "github.com/apache/arrow/go/arrow/internal/debug" "github.com/apache/arrow/go/arrow/memory" ) diff --git a/go/arrow/array/bufferbuilder.go b/go/arrow/array/bufferbuilder.go index 29d97fc8d41f..bcc7153b5598 100644 --- a/go/arrow/array/bufferbuilder.go +++ b/go/arrow/array/bufferbuilder.go @@ -19,7 +19,7 @@ package array import ( "sync/atomic" - "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/bitutil" "github.com/apache/arrow/go/arrow/internal/debug" "github.com/apache/arrow/go/arrow/memory" ) diff --git a/go/arrow/array/bufferbuilder_numeric.gen.go b/go/arrow/array/bufferbuilder_numeric.gen.go index e39b2528a9ef..4cdf42685f90 100644 --- a/go/arrow/array/bufferbuilder_numeric.gen.go +++ b/go/arrow/array/bufferbuilder_numeric.gen.go @@ -20,7 +20,7 @@ package array import ( "github.com/apache/arrow/go/arrow" - "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/bitutil" "github.com/apache/arrow/go/arrow/memory" ) diff --git a/go/arrow/array/builder.go b/go/arrow/array/builder.go index 6f5beb355d3a..bdd871611afc 100644 --- a/go/arrow/array/builder.go +++ b/go/arrow/array/builder.go @@ -21,7 +21,7 @@ import ( "sync/atomic" "github.com/apache/arrow/go/arrow" - "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/bitutil" "github.com/apache/arrow/go/arrow/memory" ) diff --git a/go/arrow/array/compare_test.go b/go/arrow/array/compare_test.go index e9927f09f308..3ed326be1758 100644 --- a/go/arrow/array/compare_test.go +++ b/go/arrow/array/compare_test.go @@ -44,7 +44,7 @@ func TestArrayEqual(t *testing.T) { sub2 := array.NewSlice(arr, 0, int64(arr.Len()-1)) defer sub2.Release() - if array.ArrayEqual(sub1, sub2) { + if array.ArrayEqual(sub1, sub2) && name != "nulls" { t.Fatalf("non-identical arrays should not compare equal:\nsub1=%v\nsub2=%v\narrf=%v\n", sub1, sub2, arr) } }) @@ -73,7 +73,7 @@ func TestArraySliceEqual(t *testing.T) { sub2 := array.NewSlice(arr, 0, int64(arr.Len()-1)) defer sub2.Release() - if array.ArraySliceEqual(sub1, 0, int64(sub1.Len()), sub2, 0, int64(sub2.Len())) { + if array.ArraySliceEqual(sub1, 0, int64(sub1.Len()), sub2, 0, int64(sub2.Len())) && name != "nulls" { t.Fatalf("non-identical slices should not compare equal:\nsub1=%v\nsub2=%v\narrf=%v\n", sub1, sub2, arr) } }) @@ -99,7 +99,7 @@ func TestArrayApproxEqual(t *testing.T) { sub2 := array.NewSlice(arr, 0, int64(arr.Len()-1)) defer sub2.Release() - if array.ArrayApproxEqual(sub1, sub2) { + if array.ArrayApproxEqual(sub1, sub2) && name != "nulls" { t.Fatalf("non-identical arrays should not compare equal:\nsub1=%v\nsub2=%v\narrf=%v\n", sub1, sub2, arr) } }) @@ -489,7 +489,7 @@ func TestRecordEqual(t *testing.T) { t.Fatalf("identical records should compare equal:\nrecord:\n%v", rec0) } - if array.RecordEqual(rec0, rec1) { + if array.RecordEqual(rec0, rec1) && name != "nulls" { t.Fatalf("non-identical records should not compare equal:\nrec0:\n%v\nrec1:\n%v", rec0, rec1) } @@ -498,7 +498,7 @@ func TestRecordEqual(t *testing.T) { sub01 := rec0.NewSlice(1, recs[0].NumRows()) defer sub01.Release() - if array.RecordEqual(sub00, sub01) { + if array.RecordEqual(sub00, sub01) && name != "nulls" { t.Fatalf("non-identical records should not compare equal:\nsub0:\n%v\nsub1:\n%v", sub00, sub01) } }) @@ -514,7 +514,7 @@ func TestRecordApproxEqual(t *testing.T) { t.Fatalf("identical records should compare equal:\nrecord:\n%v", rec0) } - if array.RecordApproxEqual(rec0, rec1) { + if array.RecordApproxEqual(rec0, rec1) && name != "nulls" { t.Fatalf("non-identical records should not compare equal:\nrec0:\n%v\nrec1:\n%v", rec0, rec1) } @@ -523,7 +523,7 @@ func TestRecordApproxEqual(t *testing.T) { sub01 := rec0.NewSlice(1, recs[0].NumRows()) defer sub01.Release() - if array.RecordApproxEqual(sub00, sub01) { + if array.RecordApproxEqual(sub00, sub01) && name != "nulls" { t.Fatalf("non-identical records should not compare equal:\nsub0:\n%v\nsub1:\n%v", sub00, sub01) } }) diff --git a/go/arrow/array/decimal128.go b/go/arrow/array/decimal128.go index 390d87e9a496..3acf6b9c444d 100644 --- a/go/arrow/array/decimal128.go +++ b/go/arrow/array/decimal128.go @@ -22,8 +22,8 @@ import ( "sync/atomic" "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/bitutil" "github.com/apache/arrow/go/arrow/decimal128" - "github.com/apache/arrow/go/arrow/internal/bitutil" "github.com/apache/arrow/go/arrow/internal/debug" "github.com/apache/arrow/go/arrow/memory" ) diff --git a/go/arrow/array/fixed_size_list.go b/go/arrow/array/fixed_size_list.go index 6c61513cf0e1..b0a2ce75b978 100644 --- a/go/arrow/array/fixed_size_list.go +++ b/go/arrow/array/fixed_size_list.go @@ -22,7 +22,7 @@ import ( "sync/atomic" "github.com/apache/arrow/go/arrow" - "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/bitutil" "github.com/apache/arrow/go/arrow/internal/debug" "github.com/apache/arrow/go/arrow/memory" ) diff --git a/go/arrow/array/float16_builder.go b/go/arrow/array/float16_builder.go index 9c36dd172a18..80864279a061 100644 --- a/go/arrow/array/float16_builder.go +++ b/go/arrow/array/float16_builder.go @@ -20,8 +20,8 @@ import ( "sync/atomic" "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/bitutil" "github.com/apache/arrow/go/arrow/float16" - "github.com/apache/arrow/go/arrow/internal/bitutil" "github.com/apache/arrow/go/arrow/internal/debug" "github.com/apache/arrow/go/arrow/memory" ) diff --git a/go/arrow/array/interval.go b/go/arrow/array/interval.go index 21efd6e4078c..cfdd59379af0 100644 --- a/go/arrow/array/interval.go +++ b/go/arrow/array/interval.go @@ -22,7 +22,7 @@ import ( "sync/atomic" "github.com/apache/arrow/go/arrow" - "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/bitutil" "github.com/apache/arrow/go/arrow/internal/debug" "github.com/apache/arrow/go/arrow/memory" "github.com/pkg/errors" diff --git a/go/arrow/array/list.go b/go/arrow/array/list.go index 2fccdeadb945..711e8497f6da 100644 --- a/go/arrow/array/list.go +++ b/go/arrow/array/list.go @@ -22,7 +22,7 @@ import ( "sync/atomic" "github.com/apache/arrow/go/arrow" - "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/bitutil" "github.com/apache/arrow/go/arrow/internal/debug" "github.com/apache/arrow/go/arrow/memory" ) diff --git a/go/arrow/array/null.go b/go/arrow/array/null.go index 346445d5bec9..56d7fa45fa18 100644 --- a/go/arrow/array/null.go +++ b/go/arrow/array/null.go @@ -17,6 +17,7 @@ package array import ( + "strings" "sync/atomic" "github.com/apache/arrow/go/arrow" @@ -53,6 +54,19 @@ func NewNullData(data *Data) *Null { return a } +func (a *Null) String() string { + o := new(strings.Builder) + o.WriteString("[") + for i := 0; i < a.Len(); i++ { + if i > 0 { + o.WriteString(" ") + } + o.WriteString("(null)") + } + o.WriteString("]") + return o.String() +} + func (a *Null) setData(data *Data) { a.array.setData(data) a.array.nullBitmapBytes = nil diff --git a/go/arrow/array/numericbuilder.gen.go b/go/arrow/array/numericbuilder.gen.go index b113908af67c..5423d8ce515c 100644 --- a/go/arrow/array/numericbuilder.gen.go +++ b/go/arrow/array/numericbuilder.gen.go @@ -22,7 +22,7 @@ import ( "sync/atomic" "github.com/apache/arrow/go/arrow" - "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/bitutil" "github.com/apache/arrow/go/arrow/internal/debug" "github.com/apache/arrow/go/arrow/memory" ) diff --git a/go/arrow/array/struct.go b/go/arrow/array/struct.go index b70d9539a488..fd2294e0e468 100644 --- a/go/arrow/array/struct.go +++ b/go/arrow/array/struct.go @@ -22,7 +22,7 @@ import ( "sync/atomic" "github.com/apache/arrow/go/arrow" - "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/bitutil" "github.com/apache/arrow/go/arrow/internal/debug" "github.com/apache/arrow/go/arrow/memory" ) diff --git a/go/arrow/internal/bitutil/bitutil.go b/go/arrow/bitutil/bitutil.go similarity index 100% rename from go/arrow/internal/bitutil/bitutil.go rename to go/arrow/bitutil/bitutil.go diff --git a/go/arrow/internal/bitutil/bitutil_test.go b/go/arrow/bitutil/bitutil_test.go similarity index 99% rename from go/arrow/internal/bitutil/bitutil_test.go rename to go/arrow/bitutil/bitutil_test.go index 62134dc64d45..fcb362f9daa2 100644 --- a/go/arrow/internal/bitutil/bitutil_test.go +++ b/go/arrow/bitutil/bitutil_test.go @@ -21,7 +21,7 @@ import ( "math/rand" "testing" - "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/bitutil" "github.com/apache/arrow/go/arrow/internal/testing/tools" "github.com/stretchr/testify/assert" ) diff --git a/go/arrow/internal/arrdata/arrdata.go b/go/arrow/internal/arrdata/arrdata.go index c24923631fa3..40dee3b54fd6 100644 --- a/go/arrow/internal/arrdata/arrdata.go +++ b/go/arrow/internal/arrdata/arrdata.go @@ -34,6 +34,7 @@ var ( ) func init() { + Records["nulls"] = makeNullRecords() Records["primitives"] = makePrimitiveRecords() Records["structs"] = makeStructsRecords() Records["lists"] = makeListsRecords() @@ -51,6 +52,49 @@ func init() { sort.Strings(RecordNames) } +func makeNullRecords() []array.Record { + mem := memory.NewGoAllocator() + + meta := arrow.NewMetadata( + []string{"k1", "k2", "k3"}, + []string{"v1", "v2", "v3"}, + ) + + schema := arrow.NewSchema( + []arrow.Field{ + arrow.Field{Name: "nulls", Type: arrow.Null, Nullable: true}, + }, &meta, + ) + + mask := []bool{true, false, false, true, true} + chunks := [][]array.Interface{ + []array.Interface{ + arrayOf(mem, []nullT{null, null, null, null, null}, mask), + }, + []array.Interface{ + arrayOf(mem, []nullT{null, null, null, null, null}, mask), + }, + []array.Interface{ + arrayOf(mem, []nullT{null, null, null, null, null}, mask), + }, + } + + defer func() { + for _, chunk := range chunks { + for _, col := range chunk { + col.Release() + } + } + }() + + recs := make([]array.Record, len(chunks)) + for i, chunk := range chunks { + recs[i] = array.NewRecord(schema, chunk, -1) + } + + return recs +} + func makePrimitiveRecords() []array.Record { mem := memory.NewGoAllocator() @@ -362,6 +406,7 @@ func makeStringsRecords() []array.Record { } type ( + nullT struct{} time32s arrow.Time32 time32ms arrow.Time32 time64ns arrow.Time64 @@ -372,6 +417,10 @@ type ( timestamp_ns arrow.Timestamp ) +var ( + null nullT +) + func makeFixedWidthTypesRecords() []array.Record { mem := memory.NewGoAllocator() schema := arrow.NewSchema( @@ -650,6 +699,9 @@ func arrayOf(mem memory.Allocator, a interface{}, valids []bool) array.Interface } switch a := a.(type) { + case []nullT: + return array.NewNull(len(a)) + case []bool: bldr := array.NewBooleanBuilder(mem) defer bldr.Release() diff --git a/go/arrow/internal/arrjson/arrjson.go b/go/arrow/internal/arrjson/arrjson.go index 11cc3d0f2a1f..2b2bcaf92599 100644 --- a/go/arrow/internal/arrjson/arrjson.go +++ b/go/arrow/internal/arrjson/arrjson.go @@ -65,6 +65,8 @@ type dataType struct { func dtypeToJSON(dt arrow.DataType) dataType { switch dt := dt.(type) { + case *arrow.NullType: + return dataType{Name: "null"} case *arrow.BooleanType: return dataType{Name: "bool"} case *arrow.Int8Type: @@ -155,6 +157,8 @@ func dtypeToJSON(dt arrow.DataType) dataType { func dtypeFromJSON(dt dataType, children []Field) arrow.DataType { switch dt.Name { + case "null": + return arrow.Null case "bool": return arrow.FixedWidthTypes.Boolean case "int": @@ -364,6 +368,9 @@ func arraysToJSON(schema *arrow.Schema, arrs []array.Interface) []Array { func arrayFromJSON(mem memory.Allocator, dt arrow.DataType, arr Array) array.Interface { switch dt := dt.(type) { + case *arrow.NullType: + return array.NewNull(arr.Count) + case *arrow.BooleanType: bldr := array.NewBooleanBuilder(mem) defer bldr.Release() @@ -618,6 +625,12 @@ func arrayFromJSON(mem memory.Allocator, dt arrow.DataType, arr Array) array.Int func arrayToJSON(field arrow.Field, arr array.Interface) Array { switch arr := arr.(type) { + case *array.Null: + return Array{ + Name: field.Name, + Count: arr.Len(), + } + case *array.Boolean: return Array{ Name: field.Name, diff --git a/go/arrow/ipc/file_reader.go b/go/arrow/ipc/file_reader.go index 813f5838b724..54213ceced1a 100644 --- a/go/arrow/ipc/file_reader.go +++ b/go/arrow/ipc/file_reader.go @@ -24,7 +24,7 @@ import ( "github.com/apache/arrow/go/arrow" "github.com/apache/arrow/go/arrow/array" - "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/bitutil" "github.com/apache/arrow/go/arrow/internal/flatbuf" "github.com/apache/arrow/go/arrow/memory" "github.com/pkg/errors" @@ -437,10 +437,8 @@ func (ctx *arrayLoaderContext) loadChild(dt arrow.DataType) array.Interface { } func (ctx *arrayLoaderContext) loadNull() array.Interface { - field, buffers := ctx.loadCommon(1) - buffers = append(buffers, ctx.buffer()) - - data := array.NewData(arrow.Null, int(field.Length()), buffers, nil, int(field.NullCount()), 0) + field := ctx.field() + data := array.NewData(arrow.Null, int(field.Length()), nil, nil, int(field.NullCount()), 0) defer data.Release() return array.MakeFromData(data) diff --git a/go/arrow/ipc/file_writer.go b/go/arrow/ipc/file_writer.go index 1fb6f529302e..8c6dec825c76 100644 --- a/go/arrow/ipc/file_writer.go +++ b/go/arrow/ipc/file_writer.go @@ -22,7 +22,7 @@ import ( "github.com/apache/arrow/go/arrow" "github.com/apache/arrow/go/arrow/array" - "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/bitutil" "github.com/apache/arrow/go/arrow/internal/flatbuf" "github.com/apache/arrow/go/arrow/memory" "github.com/pkg/errors" diff --git a/go/arrow/ipc/writer.go b/go/arrow/ipc/writer.go index 356159867124..3262083ce464 100644 --- a/go/arrow/ipc/writer.go +++ b/go/arrow/ipc/writer.go @@ -23,7 +23,7 @@ import ( "github.com/apache/arrow/go/arrow" "github.com/apache/arrow/go/arrow/array" - "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/bitutil" "github.com/apache/arrow/go/arrow/memory" "github.com/pkg/errors" ) @@ -221,14 +221,19 @@ func (w *recordEncoder) visit(p *payload, arr array.Interface) error { case 0: p.body = append(p.body, nil) default: - data := arr.Data() - bitmap := newTruncatedBitmap(w.mem, int64(data.Offset()), int64(data.Len()), data.Buffers()[0]) - p.body = append(p.body, bitmap) + switch arr.DataType().ID() { + case arrow.NULL: + // Null type has no validity bitmap + default: + data := arr.Data() + bitmap := newTruncatedBitmap(w.mem, int64(data.Offset()), int64(data.Len()), data.Buffers()[0]) + p.body = append(p.body, bitmap) + } } switch dtype := arr.DataType().(type) { case *arrow.NullType: - p.body = append(p.body, nil) + // ok. NullArrays are completely empty. case *arrow.BooleanType: var ( diff --git a/go/arrow/type_traits_boolean.go b/go/arrow/type_traits_boolean.go index c30ae0fec4fe..a3a5c59257b1 100644 --- a/go/arrow/type_traits_boolean.go +++ b/go/arrow/type_traits_boolean.go @@ -17,7 +17,7 @@ package arrow import ( - "github.com/apache/arrow/go/arrow/internal/bitutil" + "github.com/apache/arrow/go/arrow/bitutil" ) type booleanTraits struct{} diff --git a/integration/dask/Dockerfile b/integration/dask/Dockerfile index ceadb3ef8b77..a67c99072c49 100644 --- a/integration/dask/Dockerfile +++ b/integration/dask/Dockerfile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM arrow:python-3.6 +FROM arrowdev/arrow-python-3.6:latest # install dask release from conda RUN conda install -c conda-forge dask pytest=3 && \ diff --git a/integration/hdfs/Dockerfile b/integration/hdfs/Dockerfile index f173b39e0bae..71db503436d0 100644 --- a/integration/hdfs/Dockerfile +++ b/integration/hdfs/Dockerfile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM arrow:python-3.6 +FROM arrowdev/arrow-python-3.6:latest # installing libhdfs (JNI) ARG HADOOP_VERSION=2.9.2 diff --git a/integration/pandas/Dockerfile b/integration/pandas/Dockerfile index 8b56edf85d03..a1ae23e63321 100644 --- a/integration/pandas/Dockerfile +++ b/integration/pandas/Dockerfile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM arrow:python-3.6 +FROM arrowdev/arrow-python-3.6:latest # test against pandas' master RUN pip install git+https://github.com/pandas-dev/pandas.git diff --git a/integration/spark/Dockerfile b/integration/spark/Dockerfile index 6b032034986b..a0e7185d6da3 100644 --- a/integration/spark/Dockerfile +++ b/integration/spark/Dockerfile @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM arrow:python-3.6 +FROM arrowdev/arrow-python-3.6:latest # installing java and maven ARG MAVEN_VERSION=3.6.2 diff --git a/integration/turbodbc/Dockerfile b/integration/turbodbc/Dockerfile index 26df8ad709d7..d143ea748c99 100644 --- a/integration/turbodbc/Dockerfile +++ b/integration/turbodbc/Dockerfile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM arrow:python-3.6 +FROM arrowdev/arrow-python-3.6:latest RUN export DEBIAN_FRONTEND=noninteractive && \ apt-get update -y -q && \ diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java index eef2708b5311..b513ba539927 100644 --- a/java/vector/src/main/codegen/templates/UnionVector.java +++ b/java/vector/src/main/codegen/templates/UnionVector.java @@ -690,4 +690,9 @@ public int hashCode(int index) { public OUT accept(VectorVisitor visitor, IN value) { return visitor.visit(this, value); } + + @Override + public String getName() { + return name; + } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java index 166af07055d0..f600d2ed1002 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java @@ -55,8 +55,6 @@ protected BaseValueVector(BufferAllocator allocator) { this.allocator = Preconditions.checkNotNull(allocator, "allocator cannot be null"); } - public abstract String getName(); - @Override public String toString() { return super.toString() + "[name = " + getName() + ", ...]"; diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java index 1cfa81dbba3b..9a8647a4e2e7 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java @@ -271,4 +271,10 @@ public interface ValueVector extends Closeable, Iterable { * @param the input data together with visitor. */ OUT accept(VectorVisitor visitor, IN value); + + /** + * Gets the name of the vector. + * @return the name of the vector. + */ + String getName(); } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java index 41631b131a15..14c671ff12ad 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java @@ -265,4 +265,9 @@ public void copyFrom(int fromIndex, int thisIndex, ValueVector from) { public void copyFromSafe(int fromIndex, int thisIndex, ValueVector from) { throw new UnsupportedOperationException(); } + + @Override + public String getName() { + return getField().getName(); + } } diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java index b3fcd90744dd..3451d1cbcc42 100644 --- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java +++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java @@ -132,4 +132,9 @@ public void copyFrom(int fromIndex, int thisIndex, ValueVector from) { public void copyFromSafe(int fromIndex, int thisIndex, ValueVector from) { throw new UnsupportedOperationException(); } + + @Override + public String getName() { + return name; + } } diff --git a/js/src/compute/dataframe.ts b/js/src/compute/dataframe.ts index e82e65cc1b93..db7f3d734063 100644 --- a/js/src/compute/dataframe.ts +++ b/js/src/compute/dataframe.ts @@ -132,14 +132,19 @@ export class FilteredDataFrame exte for (let batchIndex = -1; ++batchIndex < numBatches;) { // load batches const batch = batches[batchIndex]; - // TODO: bind batches lazily - // If predicate doesn't match anything in the batch we don't need - // to bind the callback - if (bind) { bind(batch); } const predicate = this._predicate.bind(batch); + let isBound = false; // yield all indices for (let index = -1, numRows = batch.length; ++index < numRows;) { - if (predicate(index, batch)) { next(index, batch); } + if (predicate(index, batch)) { + // bind batches lazily - if predicate doesn't match anything + // in the batch we don't need to call bind on the batch + if (bind && !isBound) { + bind(batch); + isBound = true; + } + next(index, batch); + } } } } @@ -149,14 +154,19 @@ export class FilteredDataFrame exte for (let batchIndex = numBatches; --batchIndex >= 0;) { // load batches const batch = batches[batchIndex]; - // TODO: bind batches lazily - // If predicate doesn't match anything in the batch we don't need - // to bind the callback - if (bind) { bind(batch); } const predicate = this._predicate.bind(batch); + let isBound = false; // yield all indices for (let index = batch.length; --index >= 0;) { - if (predicate(index, batch)) { next(index, batch); } + if (predicate(index, batch)) { + // bind batches lazily - if predicate doesn't match anything + // in the batch we don't need to call bind on the batch + if (bind && !isBound) { + bind(batch); + isBound = true; + } + next(index, batch); + } } } } diff --git a/js/test/unit/table-tests.ts b/js/test/unit/table-tests.ts index ae2f058e7dae..da74b32977af 100644 --- a/js/test/unit/table-tests.ts +++ b/js/test/unit/table-tests.ts @@ -424,6 +424,10 @@ describe(`Table`, () => { get_i32 = col('i32').bind(batch); })), expected: values.filter((row) => (row[F32] as number) * (row[I32] as number) > 0) + }, { + name: `filter out all records`, + filtered: table.filter(lit(1).eq(0)), + expected: [] } ]; for (let this_test of filter_tests) { @@ -440,15 +444,13 @@ describe(`Table`, () => { expect(columns.map((c) => c.get(idx))).toEqual(expected[expected_idx++]); }); }); - test(`calls bind function on every batch`, () => { - // Techincally, we only need to call bind on - // batches with data that match the predicate, so - // this test may fail in the future if we change - // that - and that's ok! + test(`calls bind function lazily`, () => { let bind = jest.fn(); filtered.scan(() => { }, bind); - for (let batch of table.chunks) { - expect(bind).toHaveBeenCalledWith(batch); + if (expected.length) { + expect(bind).toHaveBeenCalled(); + } else { + expect(bind).not.toHaveBeenCalled(); } }); }); @@ -460,15 +462,13 @@ describe(`Table`, () => { expect(columns.map((c) => c.get(idx))).toEqual(expected[--expected_idx]); }); }); - test(`calls bind function on every batch`, () => { - // Techincally, we only need to call bind on - // batches with data that match the predicate, so - // this test may fail in the future if we change - // that - and that's ok! + test(`calls bind function lazily`, () => { let bind = jest.fn(); filtered.scanReverse(() => { }, bind); - for (let batch of table.chunks) { - expect(bind).toHaveBeenCalledWith(batch); + if (expected.length) { + expect(bind).toHaveBeenCalled(); + } else { + expect(bind).not.toHaveBeenCalled(); } }); }); diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index b2282a6b69bc..6925efd2d17e 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -385,6 +385,10 @@ set(CYTHON_EXTENSIONS lib _fs _csv _json) set(LINK_LIBS arrow_shared arrow_python_shared) +if(PYARROW_BUILD_S3) + set(CYTHON_EXTENSIONS ${CYTHON_EXTENSIONS} _s3fs) +endif() + if(PYARROW_BUILD_CUDA) # Arrow CUDA find_package(ArrowCuda) diff --git a/python/Dockerfile b/python/Dockerfile index 5fcadeb2fb66..71bc2b2d4ee1 100644 --- a/python/Dockerfile +++ b/python/Dockerfile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM arrow:cpp +FROM arrowdev/arrow-cpp:latest # install python specific packages ARG PYTHON_VERSION=3.6 diff --git a/python/Dockerfile.alpine b/python/Dockerfile.alpine index ed7b2d2a13db..87a1dff67acc 100644 --- a/python/Dockerfile.alpine +++ b/python/Dockerfile.alpine @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM arrow:cpp-alpine +FROM arrowdev/arrow-cpp-alpine:latest SHELL ["/bin/bash", "-o", "pipefail", "-c"] diff --git a/python/Dockerfile.nopandas b/python/Dockerfile.nopandas index 58aaeed5b1ab..2faad084488a 100644 --- a/python/Dockerfile.nopandas +++ b/python/Dockerfile.nopandas @@ -16,7 +16,7 @@ # under the License. ARG PYTHON_VERSION=3.6 -FROM arrow:python-$PYTHON_VERSION +FROM arrowdev/arrow-python-$PYTHON_VERSION:latest # not installing pandas doesn't mean that it's not grabbed as a transitive # dependency, so We remove it explicitly to make sure We don't have pandas diff --git a/python/manylinux2010/README.md b/python/manylinux2010/README.md index fe2888e89df8..3a9eee750aa1 100644 --- a/python/manylinux2010/README.md +++ b/python/manylinux2010/README.md @@ -70,7 +70,7 @@ publish a build image that you built locally. ```bash $ docker push python-manylinux2010 -The push refers to repository [ursalab/arrow_manylinux2010_x86_64_base] +The push refers to repository [arrowdev/arrow_manylinux2010_x86_64_base] a1ab88d27acc: Pushing [==============> ] 492.5MB/1.645GB [... etc. ...] ``` diff --git a/python/pyarrow/_csv.pyx b/python/pyarrow/_csv.pyx index 25ff47d39b54..5dccad760f42 100644 --- a/python/pyarrow/_csv.pyx +++ b/python/pyarrow/_csv.pyx @@ -466,7 +466,7 @@ cdef class ConvertOptions: self.options.include_missing_columns = value -cdef _get_reader(input_file, shared_ptr[InputStream]* out): +cdef _get_reader(input_file, shared_ptr[CInputStream]* out): use_memory_map = False get_input_stream(input_file, use_memory_map, out) @@ -522,7 +522,7 @@ def read_csv(input_file, read_options=None, parse_options=None, Contents of the CSV file as a in-memory table. """ cdef: - shared_ptr[InputStream] stream + shared_ptr[CInputStream] stream CCSVReadOptions c_read_options CCSVParseOptions c_parse_options CCSVConvertOptions c_convert_options diff --git a/python/pyarrow/_cuda.pyx b/python/pyarrow/_cuda.pyx index a9f51b0a654f..e0cad68b2bfe 100644 --- a/python/pyarrow/_cuda.pyx +++ b/python/pyarrow/_cuda.pyx @@ -729,7 +729,7 @@ cdef class BufferReader(NativeFile): self.buffer = obj self.reader = new CCudaBufferReader(self.buffer.buffer) self.set_random_access_file( - shared_ptr[RandomAccessFile](self.reader)) + shared_ptr[CRandomAccessFile](self.reader)) self.is_readable = True def read_buffer(self, nbytes=None): @@ -776,7 +776,7 @@ cdef class BufferWriter(NativeFile): def __cinit__(self, CudaBuffer buffer): self.buffer = buffer self.writer = new CCudaBufferWriter(self.buffer.cuda_buffer) - self.set_output_stream(shared_ptr[OutputStream](self.writer)) + self.set_output_stream(shared_ptr[COutputStream](self.writer)) self.is_writable = True def writeat(self, int64_t position, object data): diff --git a/python/pyarrow/_fs.pxd b/python/pyarrow/_fs.pxd new file mode 100644 index 000000000000..11b5769f854e --- /dev/null +++ b/python/pyarrow/_fs.pxd @@ -0,0 +1,68 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# cython: language_level = 3 + +import six + +from pyarrow.compat import frombytes, tobytes +from pyarrow.includes.common cimport * +from pyarrow.includes.libarrow cimport PyDateTime_from_TimePoint +from pyarrow.lib import _detect_compression +from pyarrow.lib cimport * + + +cpdef enum FileType: + NonExistent = CFileType_NonExistent + Unknown = CFileType_Unknown + File = CFileType_File + Directory = CFileType_Directory + + +cdef class FileStats: + cdef: + CFileStats stats + + @staticmethod + cdef FileStats wrap(CFileStats stats) + + +cdef class Selector: + cdef: + CSelector selector + + +cdef class FileSystem: + cdef: + shared_ptr[CFileSystem] wrapped + CFileSystem* fs + + cdef init(self, const shared_ptr[CFileSystem]& wrapped) + + +cdef class LocalFileSystem(FileSystem): + cdef: + CLocalFileSystem* localfs + + cdef init(self, const shared_ptr[CFileSystem]& wrapped) + + +cdef class SubTreeFileSystem(FileSystem): + cdef: + CSubTreeFileSystem* subtreefs + + cdef init(self, const shared_ptr[CFileSystem]& wrapped) diff --git a/python/pyarrow/_fs.pyx b/python/pyarrow/_fs.pyx index 769ca8d7391f..39079aee1051 100644 --- a/python/pyarrow/_fs.pyx +++ b/python/pyarrow/_fs.pyx @@ -22,17 +22,8 @@ import six from pyarrow.compat import frombytes, tobytes from pyarrow.includes.common cimport * from pyarrow.includes.libarrow cimport PyDateTime_from_TimePoint -from pyarrow.includes.libarrow_fs cimport * -from pyarrow.util import _stringify_path from pyarrow.lib import _detect_compression -from pyarrow.lib cimport ( - check_status, - NativeFile, - BufferedOutputStream, - BufferedInputStream, - CompressedInputStream, - CompressedOutputStream -) +from pyarrow.lib cimport * cdef inline c_string _path_as_bytes(path) except *: @@ -46,20 +37,12 @@ cdef inline c_string _path_as_bytes(path) except *: return tobytes(path) -cpdef enum FileType: - NonExistent = CFileType_NonExistent - Unknown = CFileType_Unknown - File = CFileType_File - Directory = CFileType_Directory - - cdef class FileStats: """FileSystem entry stats""" - cdef CFileStats stats - def __init__(self): - raise TypeError('dont initialize me') + raise TypeError("FileStats cannot be instantiated directly, use " + "FileSystem.get_target_stats method instead.") @staticmethod cdef FileStats wrap(CFileStats stats): @@ -115,9 +98,7 @@ cdef class FileStats: Only regular files are guaranteed to have a size. """ if self.stats.type() != CFileType_File: - raise ValueError( - 'Only regular files are guaranteed to have a size' - ) + return None return self.stats.size() @property @@ -146,7 +127,7 @@ cdef class Selector: Parameters ---------- - base_dir : str or pathlib.Path + base_dir : str The directory in which to select files. Relative paths also work, use '.' for the current directory and '..' for the parent. allow_non_existent : bool, default False @@ -156,7 +137,6 @@ cdef class Selector: recursive : bool, default False Whether to recurse into subdirectories. """ - cdef CSelector selector def __init__(self, base_dir, bint allow_non_existent=False, bint recursive=False): @@ -192,10 +172,6 @@ cdef class Selector: cdef class FileSystem: """Abstract file system API""" - cdef: - shared_ptr[CFileSystem] wrapped - CFileSystem* fs - def __init__(self): raise TypeError("FileSystem is an abstract class, instantiate one of " "the subclasses instead: LocalFileSystem or " @@ -249,7 +225,7 @@ cdef class FileSystem: Parameters ---------- - path : str or pathlib.Path + path : str The path of the new directory. recursive: bool, default True Create nested directories as well. @@ -263,7 +239,7 @@ cdef class FileSystem: Parameters ---------- - path : str or pathlib.Path + path : str The path of the directory to be deleted. """ cdef c_string directory = _path_as_bytes(path) @@ -280,9 +256,9 @@ cdef class FileSystem: Parameters ---------- - src : str or pathlib.Path + src : str The path of the file or the directory to be moved. - dest : str or pathlib.Path + dest : str The destination path where the file or directory is moved to. """ cdef: @@ -299,9 +275,9 @@ cdef class FileSystem: Parameters ---------- - src : str or pathlib.Path + src : str The path of the file to be copied from. - dest : str or pathlib.Path + dest : str The destination path where the file is copied to. """ cdef: @@ -315,7 +291,7 @@ cdef class FileSystem: Parameters ---------- - path : str or pathlib.Path + path : str The path of the file to be deleted. """ cdef c_string file = _path_as_bytes(path) @@ -345,7 +321,7 @@ cdef class FileSystem: Parameters ---------- - path : Union[str, pathlib.Path] + path : str The source to open for reading. Returns @@ -369,7 +345,7 @@ cdef class FileSystem: Parameters ---------- - source: str or pathlib.Path + source: str The source to open for reading. compression: str optional, default 'detect' The compression algorithm to use for on-the-fly decompression. @@ -407,7 +383,7 @@ cdef class FileSystem: Parameters ---------- - path : str or pathlib.Path + path : str The source to open for writing. compression: str optional, default 'detect' The compression algorithm to use for on-the-fly compression. @@ -445,7 +421,7 @@ cdef class FileSystem: Parameters ---------- - path : str or pathlib.Path + path : str The source to open for writing. compression: str optional, default 'detect' The compression algorithm to use for on-the-fly compression. @@ -484,9 +460,6 @@ cdef class LocalFileSystem(FileSystem): except when deleting an entry). """ - cdef: - CLocalFileSystem* localfs - def __init__(self): cdef shared_ptr[CLocalFileSystem] wrapped wrapped = make_shared[CLocalFileSystem]() @@ -506,10 +479,14 @@ cdef class SubTreeFileSystem(FileSystem): Note, that this makes no security guarantee. For example, symlinks may allow to "escape" the subtree and access other parts of the underlying filesystem. - """ - cdef: - CSubTreeFileSystem* subtreefs + Parameters + ---------- + base_path: str + The root of the subtree. + base_fs: FileSystem + FileSystem object the operations delegated to. + """ def __init__(self, base_path, FileSystem base_fs): cdef: diff --git a/python/pyarrow/_json.pyx b/python/pyarrow/_json.pyx index ffbf01c09e7c..da3588a5e625 100644 --- a/python/pyarrow/_json.pyx +++ b/python/pyarrow/_json.pyx @@ -135,7 +135,7 @@ cdef class ParseOptions: self.options.newlines_in_values = value -cdef _get_reader(input_file, shared_ptr[InputStream]* out): +cdef _get_reader(input_file, shared_ptr[CInputStream]* out): use_memory_map = False get_input_stream(input_file, use_memory_map, out) @@ -175,7 +175,7 @@ def read_json(input_file, read_options=None, parse_options=None, Contents of the JSON file as a in-memory table. """ cdef: - shared_ptr[InputStream] stream + shared_ptr[CInputStream] stream CJSONReadOptions c_read_options CJSONParseOptions c_parse_options shared_ptr[CJSONReader] reader diff --git a/python/pyarrow/_orc.pxd b/python/pyarrow/_orc.pxd index ebbf8beda828..649fe8248f41 100644 --- a/python/pyarrow/_orc.pxd +++ b/python/pyarrow/_orc.pxd @@ -28,7 +28,7 @@ from pyarrow.includes.libarrow cimport (CArray, CSchema, CStatus, CKeyValueMetadata, CRecordBatch, CTable, - RandomAccessFile, OutputStream, + CRandomAccessFile, COutputStream, TimeUnit) @@ -37,7 +37,7 @@ cdef extern from "arrow/adapters/orc/adapter.h" \ cdef cppclass ORCFileReader: @staticmethod - CStatus Open(const shared_ptr[RandomAccessFile]& file, + CStatus Open(const shared_ptr[CRandomAccessFile]& file, CMemoryPool* pool, unique_ptr[ORCFileReader]* reader) diff --git a/python/pyarrow/_orc.pyx b/python/pyarrow/_orc.pyx index c9f5b2e158d6..0ee3ca632e46 100644 --- a/python/pyarrow/_orc.pyx +++ b/python/pyarrow/_orc.pyx @@ -46,7 +46,7 @@ cdef class ORCReader: def open(self, object source, c_bool use_memory_map=True): cdef: - shared_ptr[RandomAccessFile] rd_handle + shared_ptr[CRandomAccessFile] rd_handle self.source = source diff --git a/python/pyarrow/_parquet.pxd b/python/pyarrow/_parquet.pxd index 19fb214c8f04..5cf3ff5931b4 100644 --- a/python/pyarrow/_parquet.pxd +++ b/python/pyarrow/_parquet.pxd @@ -24,7 +24,7 @@ from pyarrow.includes.common cimport * from pyarrow.includes.libarrow cimport (CChunkedArray, CSchema, CStatus, CTable, CMemoryPool, CBuffer, CKeyValueMetadata, - RandomAccessFile, OutputStream, + CRandomAccessFile, COutputStream, TimeUnit) @@ -316,7 +316,7 @@ cdef extern from "parquet/api/reader.h" namespace "parquet" nogil: unique_ptr[CRowGroupMetaData] RowGroup(int i) const SchemaDescriptor* schema() shared_ptr[const CKeyValueMetadata] key_value_metadata() const - void WriteTo(OutputStream* dst) const + void WriteTo(COutputStream* dst) const cdef shared_ptr[CFileMetaData] CFileMetaData_Make \ " parquet::FileMetaData::Make"(const void* serialized_metadata, @@ -406,7 +406,7 @@ cdef extern from "parquet/arrow/reader.h" namespace "parquet::arrow" nogil: cdef cppclass FileReaderBuilder: FileReaderBuilder() - CStatus Open(const shared_ptr[RandomAccessFile]& file, + CStatus Open(const shared_ptr[CRandomAccessFile]& file, const CReaderProperties& properties, const shared_ptr[CFileMetaData]& metadata) @@ -435,7 +435,7 @@ cdef extern from "parquet/arrow/writer.h" namespace "parquet::arrow" nogil: @staticmethod CStatus Open(const CSchema& schema, CMemoryPool* pool, - const shared_ptr[OutputStream]& sink, + const shared_ptr[COutputStream]& sink, const shared_ptr[WriterProperties]& properties, const shared_ptr[ArrowWriterProperties]& arrow_properties, unique_ptr[FileWriter]* writer) @@ -448,4 +448,4 @@ cdef extern from "parquet/arrow/writer.h" namespace "parquet::arrow" nogil: CStatus WriteMetaDataFile( const CFileMetaData& file_metadata, - const OutputStream* sink) + const COutputStream* sink) diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx index 3d62c0eb13e6..f93def8a67ee 100644 --- a/python/pyarrow/_parquet.pyx +++ b/python/pyarrow/_parquet.pyx @@ -568,7 +568,7 @@ cdef class FileMetaData: def __reduce__(self): cdef: NativeFile sink = BufferOutputStream() - OutputStream* c_sink = sink.get_output_stream().get() + COutputStream* c_sink = sink.get_output_stream().get() with nogil: self._metadata.WriteTo(c_sink) @@ -694,7 +694,7 @@ cdef class FileMetaData: Write the metadata object to a metadata-only file """ cdef: - shared_ptr[OutputStream] sink + shared_ptr[COutputStream] sink c_string c_where try: @@ -1010,7 +1010,7 @@ cdef class ParquetReader: read_dictionary=None, FileMetaData metadata=None, int buffer_size=0): cdef: - shared_ptr[RandomAccessFile] rd_handle + shared_ptr[CRandomAccessFile] rd_handle shared_ptr[CFileMetaData] c_metadata CReaderProperties properties = default_reader_properties() ArrowReaderProperties arrow_props = ( @@ -1202,7 +1202,7 @@ cdef class ParquetReader: cdef class ParquetWriter: cdef: unique_ptr[FileWriter] writer - shared_ptr[OutputStream] sink + shared_ptr[COutputStream] sink bint own_sink cdef readonly: diff --git a/python/pyarrow/_s3fs.pyx b/python/pyarrow/_s3fs.pyx new file mode 100644 index 000000000000..96b78f8b0a73 --- /dev/null +++ b/python/pyarrow/_s3fs.pyx @@ -0,0 +1,173 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# cython: language_level = 3 + +import six + +from pyarrow.lib cimport check_status +from pyarrow.compat import frombytes, tobytes +from pyarrow.includes.common cimport * +from pyarrow.includes.libarrow cimport * +from pyarrow.includes.libarrow_s3fs cimport * +from pyarrow._fs cimport FileSystem + + +cpdef enum S3LogLevel: + Off = CS3LogLevel_Off + Fatal = CS3LogLevel_Fatal + Error = CS3LogLevel_Error + Warn = CS3LogLevel_Warn + Info = CS3LogLevel_Info + Debug = CS3LogLevel_Debug + Trace = CS3LogLevel_Trace + + +def initialize_s3(S3LogLevel log_level=S3LogLevel.Fatal): + cdef CS3GlobalOptions options + options.log_level = log_level + check_status(CInitializeS3(options)) + + +def finalize_s3(): + check_status(CFinalizeS3()) + + +cdef class S3Options: + """Options for S3FileSystem. + + If neither access_key nor secret_key are provided then attempts to + initialize from AWS environment variables, otherwise both access_key and + secret_key must be provided. + + Parameters + ---------- + access_key: str, default None + AWS Access Key ID. Pass None to use the standard AWS environment + variables and/or configuration file. + secret_key: str, default None + AWS Secret Access key. Pass None to use the standard AWS environment + variables and/or configuration file. + region: str, default 'us-east-1' + AWS region to connect to. + scheme: str, default 'https' + S3 connection transport scheme. + endpoint_override: str, default None + Override region with a connect string such as "localhost:9000" + background_writes: boolean, default True + Whether OutputStream writes will be issued in the background, without + blocking. + """ + cdef: + CS3Options options + + # Avoid mistakingly creating attributes + __slots__ = () + + def __init__(self, access_key=None, secret_key=None, region=None, + scheme=None, endpoint_override=None, background_writes=None): + if access_key is not None and secret_key is None: + raise ValueError( + 'In order to initialize with explicit credentials both ' + 'access_key and secret_key must be provided, ' + '`secret_key` is not set.' + ) + elif access_key is None and secret_key is not None: + raise ValueError( + 'In order to initialize with explicit credentials both ' + 'access_key and secret_key must be provided, ' + '`access_key` is not set.' + ) + elif access_key is not None or secret_key is not None: + self.options = CS3Options.FromAccessKey( + tobytes(access_key), + tobytes(secret_key) + ) + else: + self.options = CS3Options.Defaults() + + if region is not None: + self.region = region + if scheme is not None: + self.scheme = scheme + if endpoint_override is not None: + self.endpoint_override = endpoint_override + if background_writes is not None: + self.background_writes = background_writes + + @property + def region(self): + """AWS region to connect to.""" + return frombytes(self.options.region) + + @region.setter + def region(self, value): + self.options.region = tobytes(value) + + @property + def scheme(self): + """S3 connection transport scheme.""" + return frombytes(self.options.scheme) + + @scheme.setter + def scheme(self, value): + self.options.scheme = tobytes(value) + + @property + def endpoint_override(self): + """Override region with a connect string such as localhost:9000""" + return frombytes(self.options.endpoint_override) + + @endpoint_override.setter + def endpoint_override(self, value): + self.options.endpoint_override = tobytes(value) + + @property + def background_writes(self): + """OutputStream writes will be issued in the background""" + return self.options.background_writes + + @background_writes.setter + def background_writes(self, bint value): + self.options.background_writes = value + + +cdef class S3FileSystem(FileSystem): + """S3-backed FileSystem implementation + + Note: S3 buckets are special and the operations available on them may be + limited or more expensive than desired. + + Parameters + ---------- + options: S3Options, default None + Options for connecting to S3. If None is passed then attempts to + initialize the connection from AWS environment variables. + """ + + cdef: + CS3FileSystem* s3fs + + def __init__(self, S3Options options=None): + cdef shared_ptr[CS3FileSystem] wrapped + options = options or S3Options() + check_status(CS3FileSystem.Make(options.options, &wrapped)) + self.init( wrapped) + + cdef init(self, const shared_ptr[CFileSystem]& wrapped): + FileSystem.init(self, wrapped) + self.s3fs = wrapped.get() diff --git a/python/pyarrow/feather.pxi b/python/pyarrow/feather.pxi index 6fd13bc04b46..8700f67ae621 100644 --- a/python/pyarrow/feather.pxi +++ b/python/pyarrow/feather.pxi @@ -34,7 +34,7 @@ cdef class FeatherWriter: self.num_rows = -1 def open(self, object dest): - cdef shared_ptr[OutputStream] sink + cdef shared_ptr[COutputStream] sink get_writer(dest, &sink) with nogil: @@ -76,7 +76,7 @@ cdef class FeatherReader: pass def open(self, source, c_bool use_memory_map=True): - cdef shared_ptr[RandomAccessFile] reader + cdef shared_ptr[CRandomAccessFile] reader get_reader(source, use_memory_map, &reader) with nogil: diff --git a/python/pyarrow/fs.py b/python/pyarrow/fs.py index cd5263acbcad..5f257d07f300 100644 --- a/python/pyarrow/fs.py +++ b/python/pyarrow/fs.py @@ -17,4 +17,11 @@ from __future__ import absolute_import -from pyarrow._fs import * # noqa +from pyarrow._fs import ( # noqa + Selector, + FileType, + FileStats, + FileSystem, + LocalFileSystem, + SubTreeFileSystem +) diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 69dafa4e46f2..82085487eda3 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -765,13 +765,16 @@ cdef extern from "arrow/io/api.h" namespace "arrow::io" nogil: CStatus Write(const uint8_t* data, int64_t nbytes) CStatus Flush() - cdef cppclass OutputStream(FileInterface, Writable): + cdef cppclass COutputStream" arrow::io::OutputStream"(FileInterface, + Writable): pass - cdef cppclass InputStream(FileInterface, Readable): + cdef cppclass CInputStream" arrow::io::InputStream"(FileInterface, + Readable): pass - cdef cppclass RandomAccessFile(InputStream, Seekable): + cdef cppclass CRandomAccessFile" arrow::io::RandomAccessFile"(CInputStream, + Seekable): CStatus GetSize(int64_t* size) CStatus ReadAt(int64_t position, int64_t nbytes, @@ -780,24 +783,24 @@ cdef extern from "arrow/io/api.h" namespace "arrow::io" nogil: shared_ptr[CBuffer]* out) c_bool supports_zero_copy() - cdef cppclass WritableFile(OutputStream, Seekable): + cdef cppclass WritableFile(COutputStream, Seekable): CStatus WriteAt(int64_t position, const uint8_t* data, int64_t nbytes) - cdef cppclass ReadWriteFileInterface(RandomAccessFile, + cdef cppclass ReadWriteFileInterface(CRandomAccessFile, WritableFile): pass - cdef cppclass FileSystem: + cdef cppclass CIOFileSystem" arrow::io::FileSystem": CStatus Stat(const c_string& path, FileStatistics* stat) - cdef cppclass FileOutputStream(OutputStream): + cdef cppclass FileOutputStream(COutputStream): @staticmethod - CStatus Open(const c_string& path, shared_ptr[OutputStream]* file) + CStatus Open(const c_string& path, shared_ptr[COutputStream]* file) int file_descriptor() - cdef cppclass ReadableFile(RandomAccessFile): + cdef cppclass ReadableFile(CRandomAccessFile): @staticmethod CStatus Open(const c_string& path, shared_ptr[ReadableFile]* file) @@ -823,46 +826,46 @@ cdef extern from "arrow/io/api.h" namespace "arrow::io" nogil: int file_descriptor() cdef cppclass CCompressedInputStream \ - " arrow::io::CompressedInputStream"(InputStream): + " arrow::io::CompressedInputStream"(CInputStream): @staticmethod CStatus Make(CMemoryPool* pool, CCodec* codec, - shared_ptr[InputStream] raw, + shared_ptr[CInputStream] raw, shared_ptr[CCompressedInputStream]* out) @staticmethod - CStatus Make(CCodec* codec, shared_ptr[InputStream] raw, + CStatus Make(CCodec* codec, shared_ptr[CInputStream] raw, shared_ptr[CCompressedInputStream]* out) cdef cppclass CCompressedOutputStream \ - " arrow::io::CompressedOutputStream"(OutputStream): + " arrow::io::CompressedOutputStream"(COutputStream): @staticmethod CStatus Make(CMemoryPool* pool, CCodec* codec, - shared_ptr[OutputStream] raw, + shared_ptr[COutputStream] raw, shared_ptr[CCompressedOutputStream]* out) @staticmethod - CStatus Make(CCodec* codec, shared_ptr[OutputStream] raw, + CStatus Make(CCodec* codec, shared_ptr[COutputStream] raw, shared_ptr[CCompressedOutputStream]* out) cdef cppclass CBufferedInputStream \ - " arrow::io::BufferedInputStream"(InputStream): + " arrow::io::BufferedInputStream"(CInputStream): @staticmethod CStatus Create(int64_t buffer_size, CMemoryPool* pool, - shared_ptr[InputStream] raw, + shared_ptr[CInputStream] raw, shared_ptr[CBufferedInputStream]* out) - shared_ptr[InputStream] Detach() + shared_ptr[CInputStream] Detach() cdef cppclass CBufferedOutputStream \ - " arrow::io::BufferedOutputStream"(OutputStream): + " arrow::io::BufferedOutputStream"(COutputStream): @staticmethod CStatus Create(int64_t buffer_size, CMemoryPool* pool, - shared_ptr[OutputStream] raw, + shared_ptr[COutputStream] raw, shared_ptr[CBufferedOutputStream]* out) - CStatus Detach(shared_ptr[OutputStream]* raw) + CStatus Detach(shared_ptr[COutputStream]* raw) # ---------------------------------------------------------------------- # HDFS @@ -894,13 +897,14 @@ cdef extern from "arrow/io/api.h" namespace "arrow::io" nogil: int64_t block_size int16_t permissions - cdef cppclass HdfsReadableFile(RandomAccessFile): + cdef cppclass HdfsReadableFile(CRandomAccessFile): pass - cdef cppclass HdfsOutputStream(OutputStream): + cdef cppclass HdfsOutputStream(COutputStream): pass - cdef cppclass CHadoopFileSystem" arrow::io::HadoopFileSystem"(FileSystem): + cdef cppclass CHadoopFileSystem \ + "arrow::io::HadoopFileSystem"(CIOFileSystem): @staticmethod CStatus Connect(const HdfsConnectionConfig* config, shared_ptr[CHadoopFileSystem]* client) @@ -936,16 +940,16 @@ cdef extern from "arrow/io/api.h" namespace "arrow::io" nogil: shared_ptr[HdfsOutputStream]* handle) cdef cppclass CBufferReader \ - " arrow::io::BufferReader"(RandomAccessFile): + " arrow::io::BufferReader"(CRandomAccessFile): CBufferReader(const shared_ptr[CBuffer]& buffer) CBufferReader(const uint8_t* data, int64_t nbytes) cdef cppclass CBufferOutputStream \ - " arrow::io::BufferOutputStream"(OutputStream): + " arrow::io::BufferOutputStream"(COutputStream): CBufferOutputStream(const shared_ptr[CResizableBuffer]& buffer) cdef cppclass CMockOutputStream \ - " arrow::io::MockOutputStream"(OutputStream): + " arrow::io::MockOutputStream"(COutputStream): CMockOutputStream() int64_t GetExtentBytesWritten() @@ -958,6 +962,71 @@ cdef extern from "arrow/io/api.h" namespace "arrow::io" nogil: void set_memcopy_threshold(int64_t threshold) +cdef extern from "arrow/filesystem/api.h" namespace "arrow::fs" nogil: + + ctypedef enum CFileType "arrow::fs::FileType": + CFileType_NonExistent "arrow::fs::FileType::NonExistent" + CFileType_Unknown "arrow::fs::FileType::Unknown" + CFileType_File "arrow::fs::FileType::File" + CFileType_Directory "arrow::fs::FileType::Directory" + + cdef cppclass CTimePoint "arrow::fs::TimePoint": + pass + + cdef cppclass CFileStats "arrow::fs::FileStats": + CFileStats() + CFileStats(CFileStats&&) + CFileStats& operator=(CFileStats&&) + CFileStats(const CFileStats&) + CFileStats& operator=(const CFileStats&) + + CFileType type() + void set_type(CFileType type) + c_string path() + void set_path(const c_string& path) + c_string base_name() + int64_t size() + void set_size(int64_t size) + c_string extension() + CTimePoint mtime() + void set_mtime(CTimePoint mtime) + + cdef cppclass CSelector "arrow::fs::Selector": + CSelector() + c_string base_dir + c_bool allow_non_existent + c_bool recursive + + cdef cppclass CFileSystem "arrow::fs::FileSystem": + CStatus GetTargetStats(const c_string& path, CFileStats* out) + CStatus GetTargetStats(const vector[c_string]& paths, + vector[CFileStats]* out) + CStatus GetTargetStats(const CSelector& select, + vector[CFileStats]* out) + CStatus CreateDir(const c_string& path, c_bool recursive) + CStatus DeleteDir(const c_string& path) + CStatus DeleteFile(const c_string& path) + CStatus DeleteFiles(const vector[c_string]& paths) + CStatus Move(const c_string& src, const c_string& dest) + CStatus CopyFile(const c_string& src, const c_string& dest) + CStatus OpenInputStream(const c_string& path, + shared_ptr[CInputStream]* out) + CStatus OpenInputFile(const c_string& path, + shared_ptr[CRandomAccessFile]* out) + CStatus OpenOutputStream(const c_string& path, + shared_ptr[COutputStream]* out) + CStatus OpenAppendStream(const c_string& path, + shared_ptr[COutputStream]* out) + + cdef cppclass CLocalFileSystem "arrow::fs::LocalFileSystem"(CFileSystem): + LocalFileSystem() + + cdef cppclass CSubTreeFileSystem \ + "arrow::fs::SubTreeFileSystem"(CFileSystem): + CSubTreeFileSystem(const c_string& base_path, + shared_ptr[CFileSystem] base_fs) + + cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil: enum MessageType" arrow::ipc::Message::Type": MessageType_SCHEMA" arrow::ipc::Message::SCHEMA" @@ -1001,14 +1070,14 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil: MetadataVersion metadata_version() MessageType type() - CStatus SerializeTo(OutputStream* stream, const CIpcOptions& options, + CStatus SerializeTo(COutputStream* stream, const CIpcOptions& options, int64_t* output_length) c_string FormatMessageType(MessageType type) cdef cppclass CMessageReader" arrow::ipc::MessageReader": @staticmethod - unique_ptr[CMessageReader] Open(const shared_ptr[InputStream]& stream) + unique_ptr[CMessageReader] Open(const shared_ptr[CInputStream]& stream) CStatus ReadNextMessage(unique_ptr[CMessage]* out) @@ -1020,7 +1089,7 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil: cdef cppclass CRecordBatchStreamReader \ " arrow::ipc::RecordBatchStreamReader"(CRecordBatchReader): @staticmethod - CStatus Open(const InputStream* stream, + CStatus Open(const CInputStream* stream, shared_ptr[CRecordBatchReader]* out) @staticmethod @@ -1031,24 +1100,24 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil: " arrow::ipc::RecordBatchStreamWriter"(CRecordBatchWriter): @staticmethod CResult[shared_ptr[CRecordBatchWriter]] Open( - OutputStream* sink, const shared_ptr[CSchema]& schema, + COutputStream* sink, const shared_ptr[CSchema]& schema, CIpcOptions& options) cdef cppclass CRecordBatchFileWriter \ " arrow::ipc::RecordBatchFileWriter"(CRecordBatchWriter): @staticmethod CResult[shared_ptr[CRecordBatchWriter]] Open( - OutputStream* sink, const shared_ptr[CSchema]& schema, + COutputStream* sink, const shared_ptr[CSchema]& schema, CIpcOptions& options) cdef cppclass CRecordBatchFileReader \ " arrow::ipc::RecordBatchFileReader": @staticmethod - CStatus Open(RandomAccessFile* file, + CStatus Open(CRandomAccessFile* file, shared_ptr[CRecordBatchFileReader]* out) @staticmethod - CStatus Open2" Open"(RandomAccessFile* file, + CStatus Open2" Open"(CRandomAccessFile* file, int64_t footer_offset, shared_ptr[CRecordBatchFileReader]* out) @@ -1058,16 +1127,16 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil: CStatus ReadRecordBatch(int i, shared_ptr[CRecordBatch]* batch) - CStatus ReadMessage(InputStream* stream, unique_ptr[CMessage]* message) + CStatus ReadMessage(CInputStream* stream, unique_ptr[CMessage]* message) CStatus GetRecordBatchSize(const CRecordBatch& batch, int64_t* size) CStatus GetTensorSize(const CTensor& tensor, int64_t* size) - CStatus WriteTensor(const CTensor& tensor, OutputStream* dst, + CStatus WriteTensor(const CTensor& tensor, COutputStream* dst, int32_t* metadata_length, int64_t* body_length) - CStatus ReadTensor(InputStream* stream, shared_ptr[CTensor]* out) + CStatus ReadTensor(CInputStream* stream, shared_ptr[CTensor]* out) CStatus ReadRecordBatch(const CMessage& message, const shared_ptr[CSchema]& schema, @@ -1082,16 +1151,16 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil: CMemoryPool* pool, shared_ptr[CBuffer]* out) - CStatus ReadSchema(InputStream* stream, CDictionaryMemo* dictionary_memo, + CStatus ReadSchema(CInputStream* stream, CDictionaryMemo* dictionary_memo, shared_ptr[CSchema]* out) CStatus ReadRecordBatch(const shared_ptr[CSchema]& schema, CDictionaryMemo* dictionary_memo, - InputStream* stream, + CInputStream* stream, shared_ptr[CRecordBatch]* out) - CStatus AlignStream(InputStream* stream, int64_t alignment) - CStatus AlignStream(OutputStream* stream, int64_t alignment) + CStatus AlignStream(CInputStream* stream, int64_t alignment) + CStatus AlignStream(COutputStream* stream, int64_t alignment) cdef CStatus GetRecordBatchPayload\ " arrow::ipc::internal::GetRecordBatchPayload"( @@ -1102,7 +1171,7 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil: cdef cppclass CFeatherWriter" arrow::ipc::feather::TableWriter": @staticmethod - CStatus Open(const shared_ptr[OutputStream]& stream, + CStatus Open(const shared_ptr[COutputStream]& stream, unique_ptr[CFeatherWriter]* out) void SetDescription(const c_string& desc) @@ -1113,7 +1182,7 @@ cdef extern from "arrow/ipc/api.h" namespace "arrow::ipc" nogil: cdef cppclass CFeatherReader" arrow::ipc::feather::TableReader": @staticmethod - CStatus Open(const shared_ptr[RandomAccessFile]& file, + CStatus Open(const shared_ptr[CRandomAccessFile]& file, unique_ptr[CFeatherReader]* out) c_string GetDescription() @@ -1172,7 +1241,7 @@ cdef extern from "arrow/csv/api.h" namespace "arrow::csv" nogil: cdef cppclass CCSVReader" arrow::csv::TableReader": @staticmethod - CStatus Make(CMemoryPool*, shared_ptr[InputStream], + CStatus Make(CMemoryPool*, shared_ptr[CInputStream], CCSVReadOptions, CCSVParseOptions, CCSVConvertOptions, shared_ptr[CCSVReader]* out) @@ -1200,7 +1269,7 @@ cdef extern from "arrow/json/reader.h" namespace "arrow::json" nogil: cdef cppclass CJSONReader" arrow::json::TableReader": @staticmethod - CStatus Make(CMemoryPool*, shared_ptr[InputStream], + CStatus Make(CMemoryPool*, shared_ptr[CInputStream], CJSONReadOptions, CJSONParseOptions, shared_ptr[CJSONReader]* out) @@ -1379,10 +1448,10 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil: CStatus Make(const uint8_t* data, int64_t size, object base, shared_ptr[CBuffer]* out) - cdef cppclass PyReadableFile(RandomAccessFile): + cdef cppclass PyReadableFile(CRandomAccessFile): PyReadableFile(object fo) - cdef cppclass PyOutputStream(OutputStream): + cdef cppclass PyOutputStream(COutputStream): PyOutputStream(object fo) cdef cppclass PandasOptions: @@ -1398,7 +1467,7 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil: shared_ptr[CRecordBatch] batch vector[shared_ptr[CTensor]] tensors - CStatus WriteTo(OutputStream* dst) + CStatus WriteTo(COutputStream* dst) CStatus GetComponents(CMemoryPool* pool, PyObject** dst) CStatus SerializeObject(object context, object sequence, @@ -1408,7 +1477,7 @@ cdef extern from "arrow/python/api.h" namespace "arrow::py" nogil: const CSerializedPyObject& obj, PyObject* base, PyObject** out) - CStatus ReadSerializedObject(RandomAccessFile* src, + CStatus ReadSerializedObject(CRandomAccessFile* src, CSerializedPyObject* out) CStatus GetSerializedFromComponents(int num_tensors, int num_ndarrays, diff --git a/python/pyarrow/includes/libarrow_fs.pxd b/python/pyarrow/includes/libarrow_fs.pxd deleted file mode 100644 index f54a2e50357e..000000000000 --- a/python/pyarrow/includes/libarrow_fs.pxd +++ /dev/null @@ -1,92 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# distutils: language = c++ - -from libcpp.functional cimport function - -from pyarrow.includes.common cimport * -from pyarrow.includes.libarrow cimport ( - InputStream as CInputStream, - OutputStream as COutputStream, - RandomAccessFile as CRandomAccessFile -) - - -cdef extern from "arrow/filesystem/api.h" namespace "arrow::fs" nogil: - - enum CFileType "arrow::fs::FileType": - CFileType_NonExistent "arrow::fs::FileType::NonExistent" - CFileType_Unknown "arrow::fs::FileType::Unknown" - CFileType_File "arrow::fs::FileType::File" - CFileType_Directory "arrow::fs::FileType::Directory" - - cdef cppclass CTimePoint "arrow::fs::TimePoint": - pass - - cdef cppclass CFileStats "arrow::fs::FileStats": - CFileStats() - CFileStats(CFileStats&&) - CFileStats& operator=(CFileStats&&) - CFileStats(const CFileStats&) - CFileStats& operator=(const CFileStats&) - - CFileType type() - void set_type(CFileType type) - c_string path() - void set_path(const c_string& path) - c_string base_name() - int64_t size() - void set_size(int64_t size) - c_string extension() - CTimePoint mtime() - void set_mtime(CTimePoint mtime) - - cdef cppclass CSelector "arrow::fs::Selector": - CSelector() - c_string base_dir - c_bool allow_non_existent - c_bool recursive - - cdef cppclass CFileSystem "arrow::fs::FileSystem": - CStatus GetTargetStats(const c_string& path, CFileStats* out) - CStatus GetTargetStats(const vector[c_string]& paths, - vector[CFileStats]* out) - CStatus GetTargetStats(const CSelector& select, - vector[CFileStats]* out) - CStatus CreateDir(const c_string& path, c_bool recursive) - CStatus DeleteDir(const c_string& path) - CStatus DeleteFile(const c_string& path) - CStatus DeleteFiles(const vector[c_string]& paths) - CStatus Move(const c_string& src, const c_string& dest) - CStatus CopyFile(const c_string& src, const c_string& dest) - CStatus OpenInputStream(const c_string& path, - shared_ptr[CInputStream]* out) - CStatus OpenInputFile(const c_string& path, - shared_ptr[CRandomAccessFile]* out) - CStatus OpenOutputStream(const c_string& path, - shared_ptr[COutputStream]* out) - CStatus OpenAppendStream(const c_string& path, - shared_ptr[COutputStream]* out) - - cdef cppclass CLocalFileSystem "arrow::fs::LocalFileSystem"(CFileSystem): - LocalFileSystem() - - cdef cppclass CSubTreeFileSystem \ - "arrow::fs::SubTreeFileSystem"(CFileSystem): - CSubTreeFileSystem(const c_string& base_path, - shared_ptr[CFileSystem] base_fs) diff --git a/python/pyarrow/includes/libarrow_s3fs.pxd b/python/pyarrow/includes/libarrow_s3fs.pxd new file mode 100644 index 000000000000..8dc109c5e6e3 --- /dev/null +++ b/python/pyarrow/includes/libarrow_s3fs.pxd @@ -0,0 +1,60 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# distutils: language = c++ + +from libcpp.functional cimport function + +from pyarrow.includes.common cimport * +from pyarrow.includes.libarrow cimport CFileSystem + +cdef extern from "arrow/filesystem/api.h" namespace "arrow::fs" nogil: + + ctypedef enum CS3LogLevel "arrow::fs::S3LogLevel": + CS3LogLevel_Off "arrow::fs::S3LogLevel::Off" + CS3LogLevel_Fatal "arrow::fs::S3LogLevel::Fatal" + CS3LogLevel_Error "arrow::fs::S3LogLevel::Error" + CS3LogLevel_Warn "arrow::fs::S3LogLevel::Warn" + CS3LogLevel_Info "arrow::fs::S3LogLevel::Info" + CS3LogLevel_Debug "arrow::fs::S3LogLevel::Debug" + CS3LogLevel_Trace "arrow::fs::S3LogLevel::Trace" + + cdef struct CS3GlobalOptions "arrow::fs::S3GlobalOptions": + CS3LogLevel log_level + + cdef cppclass CS3Options "arrow::fs::S3Options": + c_string region + c_string endpoint_override + c_string scheme + c_bool background_writes + void ConfigureDefaultCredentials() + void ConfigureAccessKey(const c_string& access_key, + const c_string& secret_key) + + @staticmethod + CS3Options Defaults() + @staticmethod + CS3Options FromAccessKey(const c_string& access_key, + const c_string& secret_key) + + cdef cppclass CS3FileSystem "arrow::fs::S3FileSystem"(CFileSystem): + @staticmethod + CStatus Make(const CS3Options& options, shared_ptr[CS3FileSystem]* out) + + cdef CStatus CInitializeS3 "arrow::fs::InitializeS3"( + const CS3GlobalOptions& options) + cdef CStatus CFinalizeS3 "arrow::fs::FinalizeS3"() diff --git a/python/pyarrow/io-hdfs.pxi b/python/pyarrow/io-hdfs.pxi index e9655e4a7f74..b224abbead82 100644 --- a/python/pyarrow/io-hdfs.pxi +++ b/python/pyarrow/io-hdfs.pxi @@ -424,7 +424,7 @@ cdef class HadoopFileSystem: c_replication, c_default_block_size, &wr_handle)) - out.set_output_stream( wr_handle) + out.set_output_stream( wr_handle) out.is_writable = True else: with nogil: @@ -432,7 +432,7 @@ cdef class HadoopFileSystem: .OpenReadable(c_path, &rd_handle)) out.set_random_access_file( - rd_handle) + rd_handle) out.is_readable = True assert not out.closed diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi index 6ff1403006ef..0cdf21efc010 100644 --- a/python/pyarrow/io.pxi +++ b/python/pyarrow/io.pxi @@ -133,29 +133,29 @@ cdef class NativeFile: else: check_status(self.output_stream.get().Close()) - cdef set_random_access_file(self, shared_ptr[RandomAccessFile] handle): - self.input_stream = handle + cdef set_random_access_file(self, shared_ptr[CRandomAccessFile] handle): + self.input_stream = handle self.random_access = handle self.is_seekable = True - cdef set_input_stream(self, shared_ptr[InputStream] handle): + cdef set_input_stream(self, shared_ptr[CInputStream] handle): self.input_stream = handle self.random_access.reset() self.is_seekable = False - cdef set_output_stream(self, shared_ptr[OutputStream] handle): + cdef set_output_stream(self, shared_ptr[COutputStream] handle): self.output_stream = handle - cdef shared_ptr[RandomAccessFile] get_random_access_file(self) except *: + cdef shared_ptr[CRandomAccessFile] get_random_access_file(self) except *: self._assert_readable() self._assert_seekable() return self.random_access - cdef shared_ptr[InputStream] get_input_stream(self) except *: + cdef shared_ptr[CInputStream] get_input_stream(self) except *: self._assert_readable() return self.input_stream - cdef shared_ptr[OutputStream] get_output_stream(self) except *: + cdef shared_ptr[COutputStream] get_output_stream(self) except *: self._assert_writable() return self.output_stream @@ -679,11 +679,11 @@ cdef class PythonFile(NativeFile): if kind == 'r': self.set_random_access_file( - shared_ptr[RandomAccessFile](new PyReadableFile(handle))) + shared_ptr[CRandomAccessFile](new PyReadableFile(handle))) self.is_readable = True else: self.set_output_stream( - shared_ptr[OutputStream](new PyOutputStream(handle))) + shared_ptr[COutputStream](new PyOutputStream(handle))) self.is_writable = True def truncate(self, pos=None): @@ -720,8 +720,8 @@ cdef class MemoryMappedFile(NativeFile): result.path = path result.is_readable = True result.is_writable = True - result.set_output_stream( handle) - result.set_random_access_file( handle) + result.set_output_stream( handle) + result.set_random_access_file( handle) result.handle = handle return result @@ -750,8 +750,8 @@ cdef class MemoryMappedFile(NativeFile): with nogil: check_status(CMemoryMappedFile.Open(c_path, c_mode, &handle)) - self.set_output_stream( handle) - self.set_random_access_file( handle) + self.set_output_stream( handle) + self.set_random_access_file( handle) self.handle = handle def resize(self, new_size): @@ -836,7 +836,7 @@ cdef class OSFile(NativeFile): check_status(ReadableFile.Open(path, pool, &handle)) self.is_readable = True - self.set_random_access_file( handle) + self.set_random_access_file( handle) cdef _open_writable(self, c_string path): with nogil: @@ -1174,7 +1174,7 @@ cdef class BufferReader(NativeFile): def __cinit__(self, object obj): self.buffer = as_buffer(obj) - self.set_random_access_file(shared_ptr[RandomAccessFile]( + self.set_random_access_file(shared_ptr[CRandomAccessFile]( new CBufferReader(self.buffer.buffer))) self.is_readable = True @@ -1204,7 +1204,7 @@ cdef class CompressedInputStream(NativeFile): check_status(CCompressedInputStream.Make( codec.get(), stream.get_input_stream(), &compressed_stream)) - self.set_input_stream( compressed_stream) + self.set_input_stream( compressed_stream) self.is_readable = True @@ -1234,13 +1234,13 @@ cdef class CompressedOutputStream(NativeFile): check_status(CCompressedOutputStream.Make( codec.get(), stream.get_output_stream(), &compressed_stream)) - self.set_output_stream( compressed_stream) + self.set_output_stream( compressed_stream) self.is_writable = True ctypedef CBufferedInputStream* _CBufferedInputStreamPtr ctypedef CBufferedOutputStream* _CBufferedOutputStreamPtr -ctypedef RandomAccessFile* _RandomAccessFilePtr +ctypedef CRandomAccessFile* _RandomAccessFilePtr cdef class BufferedInputStream(NativeFile): @@ -1255,7 +1255,7 @@ cdef class BufferedInputStream(NativeFile): buffer_size, maybe_unbox_memory_pool(memory_pool), stream.get_input_stream(), &buffered_stream)) - self.set_input_stream( buffered_stream) + self.set_input_stream( buffered_stream) self.is_readable = True def detach(self): @@ -1269,7 +1269,7 @@ cdef class BufferedInputStream(NativeFile): The underlying raw input stream """ cdef: - shared_ptr[InputStream] c_raw + shared_ptr[CInputStream] c_raw _CBufferedInputStreamPtr buffered NativeFile raw @@ -1287,7 +1287,7 @@ cdef class BufferedInputStream(NativeFile): # selectively. if dynamic_cast[_RandomAccessFilePtr](c_raw.get()) != nullptr: raw.set_random_access_file( - static_pointer_cast[RandomAccessFile, InputStream](c_raw)) + static_pointer_cast[CRandomAccessFile, CInputStream](c_raw)) else: raw.set_input_stream(c_raw) return raw @@ -1305,7 +1305,7 @@ cdef class BufferedOutputStream(NativeFile): buffer_size, maybe_unbox_memory_pool(memory_pool), stream.get_output_stream(), &buffered_stream)) - self.set_output_stream( buffered_stream) + self.set_output_stream( buffered_stream) self.is_writable = True def detach(self): @@ -1319,7 +1319,7 @@ cdef class BufferedOutputStream(NativeFile): The underlying raw output stream """ cdef: - shared_ptr[OutputStream] c_raw + shared_ptr[COutputStream] c_raw _CBufferedOutputStreamPtr buffered NativeFile raw @@ -1400,7 +1400,7 @@ cdef NativeFile _get_native_file(object source, c_bool use_memory_map): cdef get_reader(object source, c_bool use_memory_map, - shared_ptr[RandomAccessFile]* reader): + shared_ptr[CRandomAccessFile]* reader): cdef NativeFile nf nf = _get_native_file(source, use_memory_map) @@ -1408,7 +1408,7 @@ cdef get_reader(object source, c_bool use_memory_map, cdef get_input_stream(object source, c_bool use_memory_map, - shared_ptr[InputStream]* out): + shared_ptr[CInputStream]* out): """ Like get_reader(), but can automatically decompress, and returns an InputStream. @@ -1416,7 +1416,7 @@ cdef get_input_stream(object source, c_bool use_memory_map, cdef: NativeFile nf unique_ptr[CCodec] codec - shared_ptr[InputStream] input_stream + shared_ptr[CInputStream] input_stream shared_ptr[CCompressedInputStream] compressed_stream CompressionType compression_type @@ -1435,12 +1435,12 @@ cdef get_input_stream(object source, c_bool use_memory_map, check_status(CCodec.Create(compression_type, &codec)) check_status(CCompressedInputStream.Make(codec.get(), input_stream, &compressed_stream)) - input_stream = compressed_stream + input_stream = compressed_stream out[0] = input_stream -cdef get_writer(object source, shared_ptr[OutputStream]* writer): +cdef get_writer(object source, shared_ptr[COutputStream]* writer): cdef NativeFile nf try: diff --git a/python/pyarrow/ipc.pxi b/python/pyarrow/ipc.pxi index 3d8b3f4af4db..c9684f13e5bb 100644 --- a/python/pyarrow/ipc.pxi +++ b/python/pyarrow/ipc.pxi @@ -17,6 +17,7 @@ import warnings + cdef class Message: """ Container for an Arrow IPC message with metadata and optional body @@ -76,7 +77,7 @@ cdef class Message: """ cdef: int64_t output_length = 0 - OutputStream* out + COutputStream* out CIpcOptions options options.alignment = alignment @@ -136,9 +137,11 @@ cdef class MessageReader: @staticmethod def open_stream(source): - cdef MessageReader result = MessageReader.__new__(MessageReader) - cdef shared_ptr[InputStream] in_stream - cdef unique_ptr[CMessageReader] reader + cdef: + MessageReader result = MessageReader.__new__(MessageReader) + shared_ptr[CInputStream] in_stream + unique_ptr[CMessageReader] reader + _get_input_stream(source, &in_stream) with nogil: reader = CMessageReader.Open(in_stream) @@ -250,7 +253,7 @@ cdef class _CRecordBatchWriter: cdef class _RecordBatchStreamWriter(_CRecordBatchWriter): cdef: - shared_ptr[OutputStream] sink + shared_ptr[COutputStream] sink CIpcOptions options bint closed @@ -276,7 +279,7 @@ cdef class _RecordBatchStreamWriter(_CRecordBatchWriter): self.writer = GetResultValue(result) -cdef _get_input_stream(object source, shared_ptr[InputStream]* out): +cdef _get_input_stream(object source, shared_ptr[CInputStream]* out): try: source = as_buffer(source) except TypeError: @@ -332,7 +335,7 @@ cdef class _CRecordBatchReader: cdef class _RecordBatchStreamReader(_CRecordBatchReader): cdef: - shared_ptr[InputStream] in_stream + shared_ptr[CInputStream] in_stream cdef readonly: Schema schema @@ -367,7 +370,7 @@ cdef class _RecordBatchFileWriter(_RecordBatchStreamWriter): cdef class _RecordBatchFileReader: cdef: shared_ptr[CRecordBatchFileReader] reader - shared_ptr[RandomAccessFile] file + shared_ptr[CRandomAccessFile] file cdef readonly: Schema schema @@ -516,9 +519,8 @@ def read_tensor(source): """ cdef: shared_ptr[CTensor] sp_tensor - InputStream* c_stream - - cdef NativeFile nf = as_native_file(source) + CInputStream* c_stream + NativeFile nf = as_native_file(source) c_stream = nf.get_input_stream().get() with nogil: @@ -540,7 +542,7 @@ def read_message(source): """ cdef: Message result = Message.__new__(Message) - InputStream* c_stream + CInputStream* c_stream cdef NativeFile nf = as_native_file(source) c_stream = nf.get_input_stream().get() @@ -571,7 +573,7 @@ def read_schema(obj, DictionaryMemo dictionary_memo=None): """ cdef: shared_ptr[CSchema] result - shared_ptr[RandomAccessFile] cpp_file + shared_ptr[CRandomAccessFile] cpp_file CDictionaryMemo temp_memo CDictionaryMemo* arg_dict_memo diff --git a/python/pyarrow/lib.pxd b/python/pyarrow/lib.pxd index 553227a4e559..571dfaa9449d 100644 --- a/python/pyarrow/lib.pxd +++ b/python/pyarrow/lib.pxd @@ -436,9 +436,9 @@ cdef class ResizableBuffer(Buffer): cdef class NativeFile: cdef: - shared_ptr[InputStream] input_stream - shared_ptr[RandomAccessFile] random_access - shared_ptr[OutputStream] output_stream + shared_ptr[CInputStream] input_stream + shared_ptr[CRandomAccessFile] random_access + shared_ptr[COutputStream] output_stream bint is_readable bint is_writable bint is_seekable @@ -449,13 +449,13 @@ cdef class NativeFile: # extension classes are technically virtual in the C++ sense) we can expose # the arrow::io abstract file interfaces to other components throughout the # suite of Arrow C++ libraries - cdef set_random_access_file(self, shared_ptr[RandomAccessFile] handle) - cdef set_input_stream(self, shared_ptr[InputStream] handle) - cdef set_output_stream(self, shared_ptr[OutputStream] handle) + cdef set_random_access_file(self, shared_ptr[CRandomAccessFile] handle) + cdef set_input_stream(self, shared_ptr[CInputStream] handle) + cdef set_output_stream(self, shared_ptr[COutputStream] handle) - cdef shared_ptr[RandomAccessFile] get_random_access_file(self) except * - cdef shared_ptr[InputStream] get_input_stream(self) except * - cdef shared_ptr[OutputStream] get_output_stream(self) except * + cdef shared_ptr[CRandomAccessFile] get_random_access_file(self) except * + cdef shared_ptr[CInputStream] get_input_stream(self) except * + cdef shared_ptr[COutputStream] get_output_stream(self) except * cdef class BufferedInputStream(NativeFile): @@ -485,10 +485,10 @@ cdef class _CRecordBatchReader: cdef get_input_stream(object source, c_bool use_memory_map, - shared_ptr[InputStream]* reader) + shared_ptr[CInputStream]* reader) cdef get_reader(object source, c_bool use_memory_map, - shared_ptr[RandomAccessFile]* reader) -cdef get_writer(object source, shared_ptr[OutputStream]* writer) + shared_ptr[CRandomAccessFile]* reader) +cdef get_writer(object source, shared_ptr[COutputStream]* writer) # Default is allow_none=False cdef DataType ensure_type(object type, c_bool allow_none=*) diff --git a/python/pyarrow/s3fs.py b/python/pyarrow/s3fs.py new file mode 100644 index 000000000000..5619e186f9ea --- /dev/null +++ b/python/pyarrow/s3fs.py @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import absolute_import + +from pyarrow._s3fs import ( # noqa + initialize_s3, + finalize_s3, + S3Options, + S3FileSystem +) + +initialize_s3() diff --git a/python/pyarrow/serialization.pxi b/python/pyarrow/serialization.pxi index fba834434d02..cf343b0ba320 100644 --- a/python/pyarrow/serialization.pxi +++ b/python/pyarrow/serialization.pxi @@ -254,11 +254,11 @@ cdef class SerializedPyObject: """ Write serialized object to a sink """ - cdef shared_ptr[OutputStream] stream + cdef shared_ptr[COutputStream] stream get_writer(sink, &stream) self._write_to(stream.get()) - cdef _write_to(self, OutputStream* stream): + cdef _write_to(self, COutputStream* stream): with nogil: check_status(self.data.WriteTo(stream)) @@ -399,7 +399,7 @@ def read_serialized(source, base=None): ------- serialized : the serialized data """ - cdef shared_ptr[RandomAccessFile] stream + cdef shared_ptr[CRandomAccessFile] stream get_reader(source, True, &stream) cdef SerializedPyObject serialized = SerializedPyObject() diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index 086c4f45011e..a2f9194231b5 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -173,7 +173,24 @@ cdef class ChunkedArray(_PandasConvertible): return result def __array__(self, dtype=None): - values = self.to_pandas().values + cdef: + PyObject* out + PandasOptions c_options + object values + + with nogil: + check_status(libarrow.ConvertChunkedArrayToPandas( + c_options, + self.sp_chunked_array, + self, &out)) + + # wrap_array_output uses pandas to convert to Categorical, here + # always convert to numpy array + values = PyObject_to_object(out) + + if isinstance(values, dict): + values = np.take(values['dictionary'], values['indices']) + if dtype is None: return values return values.astype(dtype) diff --git a/python/pyarrow/tests/conftest.py b/python/pyarrow/tests/conftest.py index bfb1e8e5d0b5..1b980080d50b 100644 --- a/python/pyarrow/tests/conftest.py +++ b/python/pyarrow/tests/conftest.py @@ -16,6 +16,9 @@ # under the License. import os +import subprocess +import tempfile + import pytest import hypothesis as h @@ -24,6 +27,8 @@ except ImportError: import pathlib2 as pathlib # py2 compat +from pyarrow.util import find_free_port + # setup hypothesis profiles h.settings.register_profile('ci', max_examples=1000) @@ -130,6 +135,12 @@ except ImportError: pass +try: + import pyarrow.s3fs # noqa + defaults['s3'] = True +except ImportError: + pass + def pytest_configure(config): for mark in groups: @@ -211,3 +222,47 @@ def tempdir(tmpdir): @pytest.fixture(scope='session') def datadir(): return pathlib.Path(__file__).parent / 'data' + + +try: + from tempfile import TemporaryDirectory +except ImportError: + import shutil + + class TemporaryDirectory(object): + """Temporary directory implementation for python 2""" + + def __enter__(self): + self.tmp = tempfile.mkdtemp() + return self.tmp + + def __exit__(self, exc_type, exc_value, traceback): + shutil.rmtree(self.tmp) + + +@pytest.mark.s3 +@pytest.fixture(scope='session') +def minio_server(): + host, port = 'localhost', find_free_port() + access_key, secret_key = 'arrow', 'apachearrow' + + address = '{}:{}'.format(host, port) + env = os.environ.copy() + env.update({ + 'MINIO_ACCESS_KEY': access_key, + 'MINIO_SECRET_KEY': secret_key + }) + + with TemporaryDirectory() as tempdir: + args = ['minio', '--compat', 'server', '--quiet', '--address', + address, tempdir] + proc = None + try: + proc = subprocess.Popen(args, env=env) + except IOError: + pytest.skip('`minio` command cannot be located') + else: + yield address, access_key, secret_key + finally: + if proc is not None: + proc.kill() diff --git a/python/pyarrow/tests/test_flight.py b/python/pyarrow/tests/test_flight.py index 83afba6de65e..40099b06aec6 100644 --- a/python/pyarrow/tests/test_flight.py +++ b/python/pyarrow/tests/test_flight.py @@ -17,9 +17,7 @@ # under the License. import base64 -import contextlib import os -import socket import struct import tempfile import threading @@ -30,7 +28,7 @@ import pyarrow as pa from pyarrow.compat import tobytes -from pyarrow.util import pathlib +from pyarrow.util import pathlib, find_free_port try: from pyarrow import flight @@ -48,14 +46,6 @@ pytestmark = pytest.mark.flight -def find_free_port(): - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - with contextlib.closing(sock) as sock: - sock.bind(('', 0)) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - return sock.getsockname()[1] - - def test_import(): # So we see the ImportError somewhere import pyarrow.flight # noqa diff --git a/python/pyarrow/tests/test_fs.py b/python/pyarrow/tests/test_fs.py index f897e0d36f97..f6b6bf1d18cd 100644 --- a/python/pyarrow/tests/test_fs.py +++ b/python/pyarrow/tests/test_fs.py @@ -23,39 +23,109 @@ import pytest -from pyarrow import ArrowIOError +import pyarrow as pa +from pyarrow.tests.test_io import gzip_compress, gzip_decompress from pyarrow.fs import (FileType, Selector, FileSystem, LocalFileSystem, SubTreeFileSystem) -from pyarrow.tests.test_io import gzip_compress, gzip_decompress + + +@pytest.fixture +def localfs(request, tempdir): + return dict( + fs=LocalFileSystem(), + pathfn=lambda p: (tempdir / p).as_posix(), + allow_move_dir=True, + allow_append_to_file=True, + ) + + +@pytest.fixture +def subtree_localfs(request, tempdir, localfs): + prefix = 'subtree/prefix/' + (tempdir / prefix).mkdir(parents=True) + return dict( + fs=SubTreeFileSystem(prefix, localfs['fs']), + pathfn=prefix.__add__, + allow_move_dir=True, + allow_append_to_file=True, + ) + + +@pytest.mark.s3 +@pytest.fixture +def s3fs(request, minio_server): + from pyarrow.s3fs import S3Options, S3FileSystem + + address, access_key, secret_key = minio_server + bucket = 'pyarrow-filesystem/' + options = S3Options( + endpoint_override=address, + access_key=access_key, + secret_key=secret_key, + scheme='http' + ) + fs = S3FileSystem(options) + fs.create_dir(bucket) + + return dict( + fs=fs, + pathfn=bucket.__add__, + allow_move_dir=False, + allow_append_to_file=False, + ) + + +@pytest.fixture +def subtree_s3fs(request, s3fs): + prefix = 'pyarrow-filesystem/prefix/' + return dict( + fs=SubTreeFileSystem(prefix, s3fs['fs']), + pathfn=prefix.__add__, + allow_move_dir=False, + allow_append_to_file=False, + ) @pytest.fixture(params=[ pytest.param( - lambda tmp: LocalFileSystem(), - id='LocalFileSystem' + pytest.lazy_fixture('localfs'), + id='LocalFileSystem()' ), pytest.param( - lambda tmp: SubTreeFileSystem(tmp, LocalFileSystem()), - id='SubTreeFileSystem(LocalFileSystem)' + pytest.lazy_fixture('subtree_localfs'), + id='SubTreeFileSystem(LocalFileSystem())' + ), + pytest.param( + pytest.lazy_fixture('s3fs'), + id='S3FileSystem' + ), + pytest.param( + pytest.lazy_fixture('subtree_s3fs'), + id='SubTreeFileSystem(S3FileSystem())' ) ]) -def fs(request, tempdir): - return request.param(tempdir.as_posix()) +def filesystem_config(request): + return request.param + + +@pytest.fixture +def fs(request, filesystem_config): + return filesystem_config['fs'] @pytest.fixture -def testpath(request, fs, tempdir): - # we always use the tempdir for reading and writing test artifacts, but - # if the filesystem is wrapped in a SubTreeFileSystem then we don't need - # to prepend the path with the tempdir, we also test the API with both - # pathlib.Path objects and plain python strings - def convert(path): - if isinstance(fs, SubTreeFileSystem): - path = pathlib.Path(path) - else: - path = tempdir / path - return path.as_posix() - return convert +def pathfn(request, filesystem_config): + return filesystem_config['pathfn'] + + +@pytest.fixture +def allow_move_dir(request, filesystem_config): + return filesystem_config['allow_move_dir'] + + +@pytest.fixture +def allow_append_to_file(request, filesystem_config): + return filesystem_config['allow_append_to_file'] def test_cannot_instantiate_base_filesystem(): @@ -74,157 +144,158 @@ class Path: fs.create_dir(path) -def test_get_target_stats(fs, tempdir, testpath): - aaa, aaa_ = testpath('a/aa/aaa'), tempdir / 'a' / 'aa' / 'aaa' - bb, bb_ = testpath('a/bb'), tempdir / 'a' / 'bb' - c, c_ = testpath('c.txt'), tempdir / 'c.txt' - - aaa_.mkdir(parents=True) - bb_.touch() - c_.write_bytes(b'test') +def test_get_target_stats(fs, pathfn): + aaa = pathfn('a/aa/aaa/') + bb = pathfn('a/bb') + c = pathfn('c.txt') - def mtime_almost_equal(fs_dt, pathlib_ts): - # arrow's filesystem implementation truncates mtime to microsends - # resolution whereas pathlib rounds - pathlib_dt = datetime.utcfromtimestamp(pathlib_ts) - difference = (fs_dt - pathlib_dt).total_seconds() - return abs(difference) <= 10**-6 + fs.create_dir(aaa) + with fs.open_output_stream(bb): + pass # touch + with fs.open_output_stream(c) as fp: + fp.write(b'test') aaa_stat, bb_stat, c_stat = fs.get_target_stats([aaa, bb, c]) assert aaa_stat.path == aaa assert 'aaa' in repr(aaa_stat) - assert aaa_stat.base_name == 'aaa' assert aaa_stat.extension == '' - assert aaa_stat.type == FileType.Directory - assert mtime_almost_equal(aaa_stat.mtime, aaa_.stat().st_mtime) - with pytest.raises(ValueError): - aaa_stat.size + assert isinstance(aaa_stat.mtime, datetime) assert bb_stat.path == str(bb) assert bb_stat.base_name == 'bb' assert bb_stat.extension == '' assert bb_stat.type == FileType.File assert bb_stat.size == 0 - assert mtime_almost_equal(bb_stat.mtime, bb_.stat().st_mtime) + assert isinstance(bb_stat.mtime, datetime) assert c_stat.path == str(c) assert c_stat.base_name == 'c.txt' assert c_stat.extension == 'txt' assert c_stat.type == FileType.File assert c_stat.size == 4 - assert mtime_almost_equal(c_stat.mtime, c_.stat().st_mtime) - - -def test_get_target_stats_with_selector(fs, tempdir, testpath): - base_dir = testpath('.') - base_dir_ = tempdir - - selector = Selector(base_dir, allow_non_existent=False, recursive=True) - assert selector.base_dir == str(base_dir) - - (tempdir / 'test_file').touch() - (tempdir / 'test_directory').mkdir() - - stats = fs.get_target_stats(selector) - expected = list(base_dir_.iterdir()) - assert len(stats) == len(expected) - - for st in stats: - p = base_dir_ / st.path - if p.is_dir(): - assert st.type == FileType.Directory - if p.is_file(): - assert st.type == FileType.File - - -def test_create_dir(fs, tempdir, testpath): - directory = testpath('directory') - directory_ = tempdir / 'directory' - assert not directory_.exists() - fs.create_dir(directory) - assert directory_.exists() - - # recursive - directory = testpath('deeply/nested/directory') - directory_ = tempdir / 'deeply' / 'nested' / 'directory' - assert not directory_.exists() - with pytest.raises(ArrowIOError): - fs.create_dir(directory, recursive=False) - fs.create_dir(directory) - assert directory_.exists() - - -def test_delete_dir(fs, tempdir, testpath): - folder = testpath('directory') - nested = testpath('nested/directory') - folder_ = tempdir / 'directory' - nested_ = tempdir / 'nested' / 'directory' - - folder_.mkdir() - nested_.mkdir(parents=True) - - assert folder_.exists() - fs.delete_dir(folder) - assert not folder_.exists() - - assert nested_.exists() - fs.delete_dir(nested) - assert not nested_.exists() - - -def test_copy_file(fs, tempdir, testpath): - # copy file - source = testpath('source-file') - source_ = tempdir / 'source-file' - source_.touch() - target = testpath('target-file') - target_ = tempdir / 'target-file' - assert not target_.exists() - fs.copy_file(source, target) - assert source_.exists() - assert target_.exists() - - -def test_move(fs, tempdir, testpath): - # move directory - source = testpath('source-dir') - source_ = tempdir / 'source-dir' - source_.mkdir() - target = testpath('target-dir') - target_ = tempdir / 'target-dir' - assert not target_.exists() - fs.move(source, target) - assert not source_.exists() - assert target_.exists() - - # move file - source = testpath('source-file') - source_ = tempdir / 'source-file' - source_.touch() - target = testpath('target-file') - target_ = tempdir / 'target-file' - assert not target_.exists() - fs.move(source, target) - assert not source_.exists() - assert target_.exists() - - -def test_delete_file(fs, tempdir, testpath): - target = testpath('target-file') - target_ = tempdir / 'target-file' - target_.touch() - assert target_.exists() - fs.delete_file(target) - assert not target_.exists() - - nested = testpath('nested/target-file') - nested_ = tempdir / 'nested/target-file' - nested_.parent.mkdir() - nested_.touch() - assert nested_.exists() - fs.delete_file(nested) - assert not nested_.exists() + assert isinstance(c_stat.mtime, datetime) + + +def test_get_target_stats_with_selector(fs, pathfn): + base_dir = pathfn('selector-dir/') + file_a = pathfn('selector-dir/test_file_a') + file_b = pathfn('selector-dir/test_file_b') + dir_a = pathfn('selector-dir/test_dir_a') + + try: + fs.create_dir(base_dir) + with fs.open_output_stream(file_a): + pass + with fs.open_output_stream(file_b): + pass + fs.create_dir(dir_a) + + selector = Selector(base_dir, allow_non_existent=False, recursive=True) + assert selector.base_dir == base_dir + + stats = fs.get_target_stats(selector) + assert len(stats) == 3 + + for st in stats: + if st.path.endswith(file_a): + assert st.type == FileType.File + elif st.path.endswith(file_b): + assert st.type == FileType.File + elif st.path.endswith(dir_a): + assert st.type == FileType.Directory + else: + raise ValueError('unexpected path {}'.format(st.path)) + finally: + fs.delete_file(file_a) + fs.delete_file(file_b) + fs.delete_dir(dir_a) + fs.delete_dir(base_dir) + + +def test_create_dir(fs, pathfn): + d = pathfn('test-directory/') + + with pytest.raises(pa.ArrowIOError): + fs.delete_dir(d) + + fs.create_dir(d) + fs.delete_dir(d) + + d = pathfn('deeply/nested/test-directory/') + fs.create_dir(d, recursive=True) + fs.delete_dir(d) + + +def test_delete_dir(fs, pathfn): + d = pathfn('directory/') + nd = pathfn('directory/nested/') + + fs.create_dir(nd) + fs.delete_dir(nd) + fs.delete_dir(d) + with pytest.raises(pa.ArrowIOError): + fs.delete_dir(d) + + +def test_copy_file(fs, pathfn): + s = pathfn('test-copy-source-file') + t = pathfn('test-copy-target-file') + + with fs.open_output_stream(s): + pass + + fs.copy_file(s, t) + fs.delete_file(s) + fs.delete_file(t) + + +def test_move_directory(fs, pathfn, allow_move_dir): + # move directory (doesn't work with S3) + s = pathfn('source-dir/') + t = pathfn('target-dir/') + + fs.create_dir(s) + + if allow_move_dir: + fs.move(s, t) + with pytest.raises(pa.ArrowIOError): + fs.delete_dir(s) + fs.delete_dir(t) + else: + with pytest.raises(pa.ArrowIOError): + fs.move(s, t) + + +def test_move_file(fs, pathfn): + s = pathfn('test-move-source-file') + t = pathfn('test-move-target-file') + + with fs.open_output_stream(s): + pass + + fs.move(s, t) + with pytest.raises(pa.ArrowIOError): + fs.delete_file(s) + fs.delete_file(t) + + +def test_delete_file(fs, pathfn): + p = pathfn('test-delete-target-file') + with fs.open_output_stream(p): + pass + + fs.delete_file(p) + with pytest.raises(pa.ArrowIOError): + fs.delete_file(p) + + d = pathfn('test-delete-nested') + fs.create_dir(d) + f = pathfn('test-delete-nested/target-file') + with fs.open_output_stream(f) as s: + s.write(b'data') + + fs.delete_dir(d) def identity(v): @@ -240,27 +311,28 @@ def identity(v): ('gzip', 256, gzip_compress), ] ) -def test_open_input_stream(fs, tempdir, testpath, compression, buffer_size, - compressor): - file = testpath('abc') - file_ = tempdir / 'abc' - data = b'some data' * 1024 - file_.write_bytes(compressor(data)) +def test_open_input_stream(fs, pathfn, compression, buffer_size, compressor): + p = pathfn('open-input-stream') - with fs.open_input_stream(file, compression, buffer_size) as f: - result = f.read() + data = b'some data for reading\n' * 512 + with fs.open_output_stream(p) as s: + s.write(compressor(data)) + + with fs.open_input_stream(p, compression, buffer_size) as s: + result = s.read() assert result == data -def test_open_input_file(fs, tempdir, testpath): - file = testpath('abc') - file_ = tempdir / 'abc' +def test_open_input_file(fs, pathfn): + p = pathfn('open-input-file') + data = b'some data' * 1024 - file_.write_bytes(data) + with fs.open_output_stream(p) as s: + s.write(data) read_from = len(b'some data') * 512 - with fs.open_input_file(file) as f: + with fs.open_input_file(p) as f: f.seek(read_from) result = f.read() @@ -276,16 +348,16 @@ def test_open_input_file(fs, tempdir, testpath): ('gzip', 256, gzip_decompress), ] ) -def test_open_output_stream(fs, tempdir, testpath, compression, buffer_size, +def test_open_output_stream(fs, pathfn, compression, buffer_size, decompressor): - file = testpath('abc') - file_ = tempdir / 'abc' + p = pathfn('open-output-stream') - data = b'some data' * 1024 - with fs.open_output_stream(file, compression, buffer_size) as f: + data = b'some data for writing' * 1024 + with fs.open_output_stream(p, compression, buffer_size) as f: f.write(data) - assert decompressor(file_.read_bytes()) == data + with fs.open_input_stream(p, compression, buffer_size) as f: + assert f.read(len(data)) == data @pytest.mark.parametrize( @@ -297,13 +369,57 @@ def test_open_output_stream(fs, tempdir, testpath, compression, buffer_size, ('gzip', 256, gzip_compress, gzip_decompress), ] ) -def test_open_append_stream(fs, tempdir, testpath, compression, buffer_size, - compressor, decompressor): - file = testpath('abc') - file_ = tempdir / 'abc' - file_.write_bytes(compressor(b'already existing')) +def test_open_append_stream(fs, pathfn, compression, buffer_size, compressor, + decompressor, allow_append_to_file): + p = pathfn('open-append-stream') + + initial = compressor(b'already existing') + with fs.open_output_stream(p) as s: + s.write(initial) + + if allow_append_to_file: + with fs.open_append_stream(p, compression, buffer_size) as f: + f.write(b'\nnewly added') - with fs.open_append_stream(file, compression, buffer_size) as f: - f.write(b'\nnewly added') + with fs.open_input_stream(p) as f: + result = f.read() - assert decompressor(file_.read_bytes()) == b'already existing\nnewly added' + result = decompressor(result) + assert result == b'already existing\nnewly added' + else: + with pytest.raises(pa.ArrowNotImplementedError): + fs.open_append_stream(p, compression, buffer_size) + + +@pytest.mark.s3 +def test_s3_options(minio_server): + from pyarrow.s3fs import S3Options + + options = S3Options() + + assert options.region == 'us-east-1' + options.region = 'us-west-1' + assert options.region == 'us-west-1' + + assert options.scheme == 'https' + options.scheme = 'http' + assert options.scheme == 'http' + + assert options.endpoint_override == '' + options.endpoint_override = 'localhost:8999' + assert options.endpoint_override == 'localhost:8999' + + with pytest.raises(ValueError): + S3Options(access_key='access') + with pytest.raises(ValueError): + S3Options(secret_key='secret') + + address, access_key, secret_key = minio_server + options = S3Options( + access_key=access_key, + secret_key=secret_key, + endpoint_override=address, + scheme='http' + ) + assert options.scheme == 'http' + assert options.endpoint_override == address diff --git a/python/pyarrow/tests/test_io.py b/python/pyarrow/tests/test_io.py index 76148e7d8b2d..55b183b7227e 100644 --- a/python/pyarrow/tests/test_io.py +++ b/python/pyarrow/tests/test_io.py @@ -839,6 +839,21 @@ def test_memory_map_large_seeks(): check_large_seeks(pa.memory_map) +def test_memory_map_close_remove(tmpdir): + # ARROW-6740: should be able to delete closed memory-mapped file (Windows) + path = os.path.join(str(tmpdir), guid()) + mmap = pa.create_memory_map(path, 4096) + mmap.close() + assert mmap.closed + os.remove(path) # Shouldn't fail + + +def test_memory_map_deref_remove(tmpdir): + path = os.path.join(str(tmpdir), guid()) + pa.create_memory_map(path, 4096) + os.remove(path) # Shouldn't fail + + def test_os_file_writer(tmpdir): SIZE = 4096 arr = np.random.randint(0, 256, size=SIZE).astype('u1') diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py index a2c6352bddaf..fb80020d7ead 100644 --- a/python/pyarrow/tests/test_parquet.py +++ b/python/pyarrow/tests/test_parquet.py @@ -1843,18 +1843,41 @@ def test_filters_read_table(tempdir): assert table.num_rows == 3 -@pytest.yield_fixture -def s3_example(): - access_key = os.environ['PYARROW_TEST_S3_ACCESS_KEY'] - secret_key = os.environ['PYARROW_TEST_S3_SECRET_KEY'] - bucket_name = os.environ['PYARROW_TEST_S3_BUCKET'] +@pytest.fixture +def s3_bucket(request, minio_server): + boto3 = pytest.importorskip('boto3') + botocore = pytest.importorskip('botocore') + + address, access_key, secret_key = minio_server + s3 = boto3.resource( + 's3', + endpoint_url='http://{}'.format(address), + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + config=botocore.client.Config(signature_version='s3v4'), + region_name='us-east-1' + ) + bucket = s3.Bucket('test-s3fs') + bucket.create() + return 'test-s3fs' + - import s3fs - fs = s3fs.S3FileSystem(key=access_key, secret=secret_key) +@pytest.fixture +def s3_example(minio_server, s3_bucket): + s3fs = pytest.importorskip('s3fs') + + address, access_key, secret_key = minio_server + fs = s3fs.S3FileSystem( + key=access_key, + secret=secret_key, + client_kwargs={ + 'endpoint_url': 'http://{}'.format(address) + } + ) test_dir = guid() + bucket_uri = 's3://{0}/{1}'.format(s3_bucket, test_dir) - bucket_uri = 's3://{0}/{1}'.format(bucket_name, test_dir) fs.mkdir(bucket_uri) yield fs, bucket_uri fs.rm(bucket_uri, recursive=True) @@ -1920,23 +1943,29 @@ def _visit_level(base_dir, level, part_keys): for value in values: this_part_keys = part_keys + [(name, value)] - level_dir = base_dir / '{0}={1}'.format(name, value) + level_dir = fs._path_join( + str(base_dir), + '{0}={1}'.format(name, value) + ) fs.mkdir(level_dir) if level == DEPTH - 1: # Generate example data - file_path = level_dir / guid() - + file_path = fs._path_join(level_dir, guid()) filtered_df = _filter_partition(df, this_part_keys) part_table = pa.Table.from_pandas(filtered_df) with fs.open(file_path, 'wb') as f: _write_table(part_table, f) assert fs.exists(file_path) - (level_dir / '_SUCCESS').touch() + file_success = fs._path_join(level_dir, '_SUCCESS') + with fs.open(file_success, 'wb') as f: + pass else: _visit_level(level_dir, level + 1, this_part_keys) - (level_dir / '_SUCCESS').touch() + file_success = fs._path_join(level_dir, '_SUCCESS') + with fs.open(file_success, 'wb') as f: + pass _visit_level(base_dir, 0, []) diff --git a/python/pyarrow/tests/test_table.py b/python/pyarrow/tests/test_table.py index 434ec8029c1e..da1f084cf20f 100644 --- a/python/pyarrow/tests/test_table.py +++ b/python/pyarrow/tests/test_table.py @@ -203,7 +203,10 @@ def test_chunked_array_to_pandas(): @pytest.mark.pandas +@pytest.mark.nopandas def test_chunked_array_asarray(): + # ensure this is tested both when pandas is present or not (ARROW-6564) + data = [ pa.array([0]), pa.array([1, 2, 3]) @@ -232,6 +235,14 @@ def test_chunked_array_asarray(): assert elements[2:] == [1., 2., 3.] assert np_arr.dtype == np.dtype('float64') + # DictionaryType data will be converted to dense numpy array + arr = pa.DictionaryArray.from_arrays( + pa.array([0, 1, 2, 0, 1]), pa.array(['a', 'b', 'c'])) + chunked_arr = pa.chunked_array([arr, arr]) + np_arr = np.asarray(chunked_arr) + assert np_arr.dtype == np.dtype('object') + assert np_arr.tolist() == ['a', 'b', 'c', 'a', 'b'] * 2 + def test_chunked_array_flatten(): ty = pa.struct([pa.field('x', pa.int16()), diff --git a/python/pyarrow/util.py b/python/pyarrow/util.py index 5e4fb3579372..7219a447f356 100644 --- a/python/pyarrow/util.py +++ b/python/pyarrow/util.py @@ -19,8 +19,10 @@ from __future__ import absolute_import +import contextlib import functools import six +import socket import warnings @@ -125,3 +127,11 @@ def get_contiguous_span(shape, strides, itemsize): if end - start != itemsize * product(shape): raise ValueError('array data is non-contiguous') return start, end + + +def find_free_port(): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + with contextlib.closing(sock) as sock: + sock.bind(('', 0)) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + return sock.getsockname()[1] diff --git a/python/requirements-test.txt b/python/requirements-test.txt index 73eabfebd288..10d445cbc442 100644 --- a/python/requirements-test.txt +++ b/python/requirements-test.txt @@ -3,4 +3,5 @@ hypothesis pandas pathlib2; python_version < "3.4" pytest +pytest-lazy-fixture pytz diff --git a/python/setup.py b/python/setup.py index 5e88352a8f9b..d7207eedd8f1 100755 --- a/python/setup.py +++ b/python/setup.py @@ -141,6 +141,8 @@ def initialize_options(self): if not hasattr(sys, 'gettotalrefcount'): self.build_type = 'release' + self.with_s3 = strtobool( + os.environ.get('PYARROW_WITH_S3', '0')) self.with_cuda = strtobool( os.environ.get('PYARROW_WITH_CUDA', '0')) self.with_flight = strtobool( @@ -176,6 +178,7 @@ def initialize_options(self): '_parquet', '_orc', '_plasma', + '_s3fs', 'gandiva'] def _run_cmake(self): @@ -215,6 +218,8 @@ def _run_cmake(self): if self.cmake_generator: cmake_options += ['-G', self.cmake_generator] + if self.with_s3: + cmake_options.append('-DPYARROW_BUILD_S3=on') if self.with_cuda: cmake_options.append('-DPYARROW_BUILD_CUDA=on') if self.with_flight: @@ -414,6 +419,8 @@ def _failure_permitted(self, name): return True if name == '_flight' and not self.with_flight: return True + if name == '_s3fs' and not self.with_s3: + return True if name == '_cuda' and not self.with_cuda: return True if name == 'gandiva' and not self.with_gandiva: diff --git a/r/Dockerfile b/r/Dockerfile index 97aa64b95234..63a200b8432e 100644 --- a/r/Dockerfile +++ b/r/Dockerfile @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM arrow:cpp +FROM arrowdev/arrow-cpp:latest # Configure ENV CC=gcc \ diff --git a/r/Dockerfile.conda b/r/Dockerfile.conda index 15279916e2e4..f4ab0d2974aa 100644 --- a/r/Dockerfile.conda +++ b/r/Dockerfile.conda @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM arrow:cpp +FROM arrowdev/arrow-cpp:latest # install R specific packages ARG R_VERSION=3.6.1 diff --git a/ruby/red-arrow/red-arrow.gemspec b/ruby/red-arrow/red-arrow.gemspec index 75430b5b979e..f4fcda90efe5 100644 --- a/ruby/red-arrow/red-arrow.gemspec +++ b/ruby/red-arrow/red-arrow.gemspec @@ -47,7 +47,7 @@ Gem::Specification.new do |spec| spec.extensions = ["ext/arrow/extconf.rb"] spec.add_runtime_dependency("extpp", ">= 0.0.7") - spec.add_runtime_dependency("gio2", "= 3.3.7") + spec.add_runtime_dependency("gio2", ">= 3.3.6") spec.add_runtime_dependency("native-package-installer") spec.add_runtime_dependency("pkg-config") diff --git a/ruby/red-plasma/test/helper.rb b/ruby/red-plasma/test/helper.rb index 255cad287004..02c545f53670 100644 --- a/ruby/red-plasma/test/helper.rb +++ b/ruby/red-plasma/test/helper.rb @@ -21,4 +21,5 @@ require "test-unit" +require_relative "helper/omittable" require_relative "helper/plasma-store" diff --git a/ruby/red-plasma/test/helper/omittable.rb b/ruby/red-plasma/test/helper/omittable.rb new file mode 100644 index 000000000000..a1c0334b63a2 --- /dev/null +++ b/ruby/red-plasma/test/helper/omittable.rb @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +module Helper + module Omittable + def require_gi_bindings(major, minor, micro) + return if GLib.check_binding_version?(major, minor, micro) + message = + "Require gobject-introspection #{major}.#{minor}.#{micro} or later: " + + GLib::BINDING_VERSION.join(".") + omit(message) + end + + def require_gi(major, minor, micro) + return if GObjectIntrospection::Version.or_later?(major, minor, micro) + message = + "Require GObject Introspection #{major}.#{minor}.#{micro} or later: " + + GObjectIntrospection::Version::STRING + omit(message) + end + end +end diff --git a/ruby/red-plasma/test/test-plasma-client.rb b/ruby/red-plasma/test/test-plasma-client.rb index de76fb9d36e8..d6182976c5fb 100644 --- a/ruby/red-plasma/test/test-plasma-client.rb +++ b/ruby/red-plasma/test/test-plasma-client.rb @@ -16,8 +16,11 @@ # under the License. class TestPlasmaClient < Test::Unit::TestCase + include Helper::Omittable + def setup @store = nil + require_gi_bindings(3, 3, 9) @store = Helper::PlasmaStore.new @store.start @id = Plasma::ObjectID.new("Hello") diff --git a/rust/README.md b/rust/README.md index 377cf6c6fef7..7b712511c14b 100644 --- a/rust/README.md +++ b/rust/README.md @@ -29,7 +29,7 @@ ## Prerequisites -Before running tests and examples it is necessary to set up the local development enviroment. +Before running tests and examples it is necessary to set up the local development environment. ### Git Submodules diff --git a/rust/arrow/README.md b/rust/arrow/README.md index 4bc6b062648e..7dffc3432024 100644 --- a/rust/arrow/README.md +++ b/rust/arrow/README.md @@ -99,7 +99,7 @@ cargo publish If the Cargo.toml does not have the correct version then it will be necessary to modify it manually. Since there is now a modified file locally that is not -committed to github it will be necessary to use the following command. +committed to GitHub it will be necessary to use the following command. ```bash cargo publish --allow-dirty diff --git a/rust/arrow/src/compute/kernels/take.rs b/rust/arrow/src/compute/kernels/take.rs index 6cce7fb47d95..b591982c56e5 100644 --- a/rust/arrow/src/compute/kernels/take.rs +++ b/rust/arrow/src/compute/kernels/take.rs @@ -359,7 +359,7 @@ mod tests { let a = take(&list_array, &index, None).unwrap(); let a: &ListArray = a.as_any().downcast_ref::().unwrap(); - // construct a value aray with expected results: + // construct a value array with expected results: // [[2,3], null, [-1,-2,-1], [2,3], [0,0,0]] let expected_data = Int32Array::from(vec![ Some(2), @@ -424,7 +424,7 @@ mod tests { let a = take(&list_array, &index, None).unwrap(); let a: &ListArray = a.as_any().downcast_ref::().unwrap(); - // construct a value aray with expected results: + // construct a value array with expected results: // [[null], null, [-1,-2,3], [5,null], [0,null,0]] let expected_data = Int32Array::from(vec![ None, @@ -487,7 +487,7 @@ mod tests { let a = take(&list_array, &index, None).unwrap(); let a: &ListArray = a.as_any().downcast_ref::().unwrap(); - // construct a value aray with expected results: + // construct a value array with expected results: // [null, null, [-1,-2,3], [5,null], [0,null,0]] let expected_data = Int32Array::from(vec![ Some(-1), diff --git a/rust/arrow/src/csv/reader.rs b/rust/arrow/src/csv/reader.rs index a031e199beff..15e0d33ae6d3 100644 --- a/rust/arrow/src/csv/reader.rs +++ b/rust/arrow/src/csv/reader.rs @@ -405,7 +405,7 @@ pub struct ReaderBuilder { /// If schema inference is run on a file with no headers, default column names /// are created. has_headers: bool, - /// An optional column delimiter. Defauits to `b','` + /// An optional column delimiter. Defaults to `b','` delimiter: Option, /// Optional maximum number of records to read during schema inference /// diff --git a/rust/arrow/src/datatypes.rs b/rust/arrow/src/datatypes.rs index 5698ccbc1bd2..40e0153e81d8 100644 --- a/rust/arrow/src/datatypes.rs +++ b/rust/arrow/src/datatypes.rs @@ -785,7 +785,7 @@ impl Field { }) } - /// Converts to a `String` representation of the the `Field` + /// Converts to a `String` representation of the `Field` pub fn to_string(&self) -> String { format!("{}: {:?}", self.name, self.data_type) } diff --git a/rust/datafusion/README.md b/rust/datafusion/README.md index 7058a3b1b433..f3dc512bb032 100644 --- a/rust/datafusion/README.md +++ b/rust/datafusion/README.md @@ -35,7 +35,7 @@ datafusion = "0.15.0-SNAPSHOT" #### Use as a bin ##### Build your own bin(requires rust toolchains) ```sh -git clone https://github/apache/arrow +git clone https://github.com/apache/arrow cd arrow/rust/datafusion cargo run --bin datafusion-cli ```