diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml new file mode 100644 index 0000000..cf5dc26 --- /dev/null +++ b/.github/workflows/pre-commit.yaml @@ -0,0 +1,39 @@ +name: pre-commit + +on: + pull_request: + branches: + - master + - staging + types: + - opened + - reopened + - ready_for_review + - synchronize + +env: + SKIP: pytest-check + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + fetch-depth: 0 # get full git history + - uses: actions/setup-python@v5 + with: + cache: 'pip' + - name: Install pre-commit + run: | + pip install pre-commit + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v46 + with: + token: ${{ secrets.GITHUB_TOKEN }} + - name: Run pre-commit + uses: pre-commit/action@v3.0.1 + with: + extra_args: --files ${{ steps.changed-files.outputs.all_changed_files }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 87de83e..5186bd8 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -6,32 +6,33 @@ on: - synchronize push: tags: - - 'v[0-9]+.[0-9]+.[0-9]+*' + - "v[0-9]+.[0-9]+.[0-9]+*" jobs: - test-suite: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: [3.6, 3.7, 3.8] + python-version: [3.8, 3.9, "3.10", "3.11"] # TODO 3.12 os: [ubuntu-latest, macos-latest, windows-latest] steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} ${{ matrix.os }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install -r tests/requirements.txt - - name: Test with pytest - run: | - pytest - + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} ${{ matrix.os }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r tests/requirements.txt + - name: Test with pytest + run: | + pytest --cov-report= --cov=btrdb # suppress coverage report here + - name: Coverage Report + run: | + coverage report -m release: needs: @@ -39,7 +40,7 @@ jobs: if: startsWith(github.ref, 'refs/tags/') runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Create Release id: create_release uses: actions/create-release@v1 @@ -51,7 +52,6 @@ jobs: draft: false prerelease: false - deploy: needs: - release @@ -59,20 +59,20 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.8' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel - - name: Build and publish - run: | - python setup.py sdist bdist_wheel - - name: Publish package - uses: pypa/gh-action-pypi-publish@master - with: - user: __token__ - password: ${{ secrets.PYPI_DEPLOYMENT_TOKEN }} \ No newline at end of file + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel build + - name: Build and publish + run: | + python -m build --sdist --wheel --outdir dist/ . + - name: Publish package + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.PYPI_DEPLOYMENT_TOKEN }} diff --git a/.gitignore b/.gitignore index e7c86be..4629a15 100644 --- a/.gitignore +++ b/.gitignore @@ -118,3 +118,13 @@ dmypy.json # Pyre type checker .pyre/ + +# arrow parquet files +*.parquet + +.idea +.idea/misc.xml +.idea/vcs.xml +.idea/inspectionProfiles/profiles_settings.xml +.idea/inspectionProfiles/Project_Default.xml +/.idea/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..2d8e501 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,35 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + exclude: ^(setup.cfg|btrdb/grpcinterface) +- repo: https://github.com/psf/black + rev: 24.3.0 + hooks: + - id: black-jupyter + args: [--line-length=88] + exclude: btrdb/grpcinterface/.*\.py +- repo: https://github.com/pycqa/isort + rev: 5.13.2 + hooks: + - id: isort + name: isort (python) + args: [--profile=black, --line-length=88] + exclude: btrdb/grpcinterface/.*\.py +- repo: https://github.com/PyCQA/flake8 + rev: 7.0.0 + hooks: + - id: flake8 + args: [--config=setup.cfg] + exclude: ^(btrdb/grpcinterface|tests|setup.py|btrdb4|docs|benchmarks) +- repo: local + hooks: + - id: pytest-check + name: pytest-check + entry: pytest + language: system + pass_filenames: false + always_run: true diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..2095729 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,34 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the OS, Python version and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.10" + # You can also specify other tool versions: + # nodejs: "19" + # rust: "1.64" + # golang: "1.19" + +# Build documentation in the "docs/" directory with Sphinx +sphinx: + configuration: docs/source/conf.py + +# Optionally build your docs in additional formats such as PDF and ePub +formats: + - pdf +# - epub + +# Optional but recommended, declare the Python requirements required +# to build your documentation +# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +python: + install: + - requirements: docs/requirements.txt + - method: pip + path: . diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..78b4425 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,76 @@ +# Changelog + +## 5.32.0 + +### What's Changed +- Initial docstring overhaul and a new test for better documentation and test coverage. by @JustinGilmer in [#82](https://github.com/pingthingsio/btrdb-python/pull/82) +- Test new join logic for improved data loading for windowed queries. by @JustinGilmer in [#80](https://github.com/pingthingsio/btrdb-python/pull/80) +- Improve `arrow_to_dataframe` function for handling large amounts of columns, enhancing performance and usability. by @Jefflinf in [#73](https://github.com/pingthingsio/btrdb-python/pull/73) +- Expand testing to include Python 3.11, ensuring compatibility and stability. by @JustinGilmer in [#74](https://github.com/pingthingsio/btrdb-python/pull/74) +- Update exception handling to better support `RpcErrors`, improving error management and debugging. by @JustinGilmer in [#72](https://github.com/pingthingsio/btrdb-python/pull/72) +- Introduce an option for specifying the schema of the returned raw data, allowing for more flexibility in data handling. by @andrewchambers in [#51](https://github.com/pingthingsio/btrdb-python/pull/51) +- Remove non-required dependencies and migrate to 'data' optional dependency for a lighter package and easier installation. by @JustinGilmer in [#71](https://github.com/pingthingsio/btrdb-python/pull/71) +- New method to get first and last timestamps from `aligned_windows`, enhancing data analysis capabilities. by @Jefflinf in [#70](https://github.com/pingthingsio/btrdb-python/pull/70) +- Add `to_timedelta` method for `pointwidth` class, providing more options for time-based data manipulation. by @Jefflinf in [#69](https://github.com/pingthingsio/btrdb-python/pull/69) + +### Fixed +- Fix `NoneType` error for `earliest/latest` for empty streams, ensuring reliability and error handling. by @Jefflinf in [#64](https://github.com/pingthingsio/btrdb-python/pull/64) +- Correct integration tests where the time column is not automatically set as the index, improving test accuracy and reliability. by @JustinGilmer in [#56](https://github.com/pingthingsio/btrdb-python/pull/56) + +### Deprecated +- FutureWarning for `streams_in_collection` to return `StreamSet` in the future, preparing users for upcoming API changes. by @Jefflinf in [#60](https://github.com/pingthingsio/btrdb-python/pull/60) + +**Full Changelog**: [GitHub compare view](https://github.com/PingThingsIO/btrdb-python/compare/v5.31.0...v5.32.0) + + +## 5.31.0 +## What's Changed +* Have release script update pyproject.toml file by @youngale-pingthings in https://github.com/PingThingsIO/btrdb-python/pull/48 +* Provide option to sort the arrow tables by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/47 +* Remove 4MB limit for gRPC message payloads by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/49 +* Update documentation for arrow methods by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/50 +* Update from staging by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/54 +* Sort tables by time by default for any `pyarrow` tables. by @justinGilmer in +* Fix deprecation warnings for pip installations. by @jleifnf in + +**Full Changelog**: [GitHub compare view](https://github.com/PingThingsIO/btrdb-python/compare/v5.30.2...v5.31.0) + +## 5.30.2 +### What's Changed +* Update readthedocs to new yaml for testing. by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/40 +* Converting pandas index takes very long, add in arrow_table. by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/41 + + +**Full Changelog**: https://github.com/PingThingsIO/btrdb-python/compare/v5.30.1...v5.30.2 + +## 5.30.1 +### What's Changed +* Small version bump for pypi release + + +**Full Changelog**: https://github.com/PingThingsIO/btrdb-python/compare/v5.30.0...v5.30.1 + + +## 5.30.0 +### What's Changed +* Merge Arrow support into Main for Release by @youngale-pingthings in https://github.com/PingThingsIO/btrdb-python/pull/37 + * This PR contains many changes that support the commercial Arrow data fetches and inserts + * `arrow_` prefixed methods for `Stream` Objects: + * `insert, aligned_windows, windows, values` + * `arrow_` prefixed methods for StreamSet` objects: + * `insert, values, to_dataframe, to_polars, to_arrow_table, to_numpy, to_dict, to_series` +* Justin gilmer patch 1 by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/39 + + +**Full Changelog**: https://github.com/PingThingsIO/btrdb-python/compare/v5.28.1...v5.30.0 + + +## 5.28.1 +### What's Changed +* Upgrade ray versions by @jleifnf in https://github.com/PingThingsIO/btrdb-python/pull/15 +* Release v5.28.1 and Update Python by @youngale-pingthings in https://github.com/PingThingsIO/btrdb-python/pull/17 + +### New Contributors +* @jleifnf made their first contribution in https://github.com/PingThingsIO/btrdb-python/pull/15 + +**Full Changelog**: https://github.com/PingThingsIO/btrdb-python/compare/v5.15.1...v5.28.1 diff --git a/MANIFEST.in b/MANIFEST.in index 89c5b46..7f67c08 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -20,4 +20,4 @@ global-exclude *.py[co] global-exclude .ipynb_checkpoints global-exclude .DS_Store global-exclude .env -global-exclude .coverage.* \ No newline at end of file +global-exclude .coverage.* diff --git a/README.md b/README.md index 7a808fe..af73c28 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,10 @@ # BTrDB Bindings for Python -These are BTrDB Bindings for Python allowing you painless and productive access to the Berkeley Tree Database (BTrDB). BTrDB is a time series database focusing on blazing speed with respect to univariate time series data at the nanosecond scale. - +These are BTrDB Bindings for Python allowing you painless and productive access to the Berkeley Tree Database (BTrDB). BTrDB is a time series database focusing on blazing speed with respect to univariate time series data at the nanosecond scale. ## Sample Code -Our goal is to make BTrDB as easy to use as possible, focusing on integration with other tools and the productivity of our users. In keeping with this we continue to add new features such as easy transformation to numpy arrays, pandas Series, etc. See the sample code below and then checkout our [documentation](https://btrdb.readthedocs.io/en/latest/) for more in depth instructions. - +Our goal is to make BTrDB as easy to use as possible, focusing on integration with other tools and the productivity of our users. In keeping with this we continue to add new features such as easy transformation to numpy arrays, pandas Series, etc. See the sample code below and then checkout our [documentation](https://btrdb-python.readthedocs.io/en/latest/) for more in depth instructions. import btrdb @@ -40,8 +38,6 @@ Our goal is to make BTrDB as easy to use as possible, focusing on integration wi >> StatPoint(1500000000300000000, 4.0, 5.0, 6.0, 3, 0.816496580927726) >> StatPoint(1500000000600000000, 7.0, 8.0, 9.0, 3, 0.816496580927726) - - You can also easily work with a group of streams for when you need to evaluate data across multiple time series or serialize to disk. from btrdb.utils.timez import to_nanoseconds @@ -69,17 +65,15 @@ You can also easily work with a group of streams for when you need to evaluate d 8 1500000000800000000 NaN 9.0 9 1500000000900000000 10.0 NaN - ## Installation -See our documentation on [installing](https://btrdb.readthedocs.io/en/latest/installing.html) the bindings for more detailed instructions. However, to quickly get started using the latest available versions you can use `pip` to install from pypi with `conda` support coming in the near future. +See our documentation on [installing](https://btrdb-python.readthedocs.io/en/latest/installing.html) the bindings for more detailed instructions. However, to quickly get started using the latest available versions you can use `pip` to install from pypi with `conda` support coming in the near future. $ pip install btrdb - ## Tests -This project includes a suite of automated tests based upon [pytest](https://docs.pytest.org/en/latest/). For your convenience, a `Makefile` has been provided with a target for evaluating the test suite. Use the following command to run the tests. +This project includes a suite of automated tests based upon [pytest](https://docs.pytest.org/en/latest/). For your convenience, a `Makefile` has been provided with a target for evaluating the test suite. Use the following command to run the tests. $ make test @@ -89,13 +83,13 @@ Note that the test suite has additional dependencies that must be installed for ## Releases -This codebase uses github actions to control the release process. To create a new release of the software, run `release.sh` with arguments for the new version as shown below. Make sure you are in the master branch when running this script. +This codebase uses github actions to control the release process. To create a new release of the software, run `release.sh` with arguments for the new version as shown below. Make sure you are in the master branch when running this script. ``` ./release.sh 5 11 4 ``` -This will tag and push the current commit and github actions will run the test suite, build the package, and push it to pypi. If any issues are encountered with the automated tests, the build will fail and you will have a tag with no corresponding release. +This will tag and push the current commit and github actions will run the test suite, build the package, and push it to pypi. If any issues are encountered with the automated tests, the build will fail and you will have a tag with no corresponding release. After a release is created, you can manually edit the release description through github. @@ -105,10 +99,16 @@ The project documentation is written in reStructuredText and is built using Sphi $ make html -This will build the HTML documentation locally in `docs/build`, which can be viewed using `open docs/build/index.html`. Other formats (PDF, epub, etc) can be built using `docs/Makefile`. The documentation is automatically built on every GitHub release and hosted on [Read The Docs](https://btrdb.readthedocs.io/en/latest/). +This will build the HTML documentation locally in `docs/build`, which can be viewed using `open docs/build/index.html`. Other formats (PDF, epub, etc) can be built using `docs/Makefile`. The documentation is automatically built on every GitHub release and hosted on [Read The Docs](https://btrdb-python.readthedocs.io/en/latest/). Note that the documentation also requires Sphix and other dependencies to successfully build: `pip install -r docs/requirements.txt`. ## Versioning -This codebases uses a form of [Semantic Versioning](http://semver.org/) to structure version numbers. In general, the major version number will track with the BTrDB codebase to transparently maintain version compatibility. Planned features between major versions will increment the minor version while any special releases (bug fixes, etc.) will increment the patch number. +This codebases uses a form of [Semantic Versioning](http://semver.org/) to structure version numbers. In general, the major version number will track with the BTrDB codebase to transparently maintain version compatibility. Planned features between major versions will increment the minor version while any special releases (bug fixes, etc.) will increment the patch number. + +## Pre-commit hooks + +`pip install pre-commit` and then run +`pre-commit run --all-files` to comb through changes and make sure they are formatted correctly. +`pre-commit install` and then run `git commit` to automatically run the pre-commit hooks on every commit. diff --git a/benchmarks/benchmark_stream_inserts.py b/benchmarks/benchmark_stream_inserts.py new file mode 100644 index 0000000..003c386 --- /dev/null +++ b/benchmarks/benchmark_stream_inserts.py @@ -0,0 +1,80 @@ +from time import perf_counter +from typing import Dict, List, Tuple, Union + +import pyarrow + +import btrdb + + +def time_stream_insert( + stream: btrdb.stream.Stream, + data: List[Tuple[int, float]], + merge_policy: str = "never", +) -> Dict[str, Union[int, float, str]]: + """Insert raw data into a single stream, where data is a List of tuples of int64 timestamps and float64 values. + + Parameters + ---------- + stream : btrdb.stream.Stream, required + The stream to insert data into. + data : List[Tuple[int, float]], required + The data to insert into stream. + merge_policy : str, optional, default = 'never' + How should the platform handle duplicated data? + Valid policies: + `never`: the default, no points are merged + `equal`: points are deduplicated if the time and value are equal + `retain`: if two points have the same timestamp, the old one is kept + `replace`: if two points have the same timestamp, the new one is kept + """ + prev_ver = stream.version() + tic = perf_counter() + new_ver = stream.insert(data, merge=merge_policy) + toc = perf_counter() + run_time = toc - tic + n_points = len(data) + result = { + "uuid": stream.uuid, + "previous_version": prev_ver, + "new_version": new_ver, + "points_to_insert": n_points, + "total_time_seconds": run_time, + "merge_policy": merge_policy, + } + return result + + +def time_stream_arrow_insert( + stream: btrdb.stream.Stream, data: pyarrow.Table, merge_policy: str = "never" +) -> Dict[str, Union[int, float, str]]: + """Insert raw data into a single stream, where data is a pyarrow Table of timestamps and float values. + + Parameters + ---------- + stream : btrdb.stream.Stream, required + The stream to insert data into. + data : pyarrow.Table, required + The table of data to insert into stream. + merge_policy : str, optional, default = 'never' + How should the platform handle duplicated data? + Valid policies: + `never`: the default, no points are merged + `equal`: points are deduplicated if the time and value are equal + `retain`: if two points have the same timestamp, the old one is kept + `replace`: if two points have the same timestamp, the new one is kept + """ + prev_ver = stream.version() + tic = perf_counter() + new_ver = stream.arrow_insert(data, merge=merge_policy) + toc = perf_counter() + run_time = toc - tic + n_points = data.num_rows + result = { + "uuid": stream.uuid, + "previous_version": prev_ver, + "new_version": new_ver, + "points_to_insert": n_points, + "total_time_seconds": run_time, + "merge_policy": merge_policy, + } + return result diff --git a/benchmarks/benchmark_stream_reads.py b/benchmarks/benchmark_stream_reads.py new file mode 100644 index 0000000..426bdfb --- /dev/null +++ b/benchmarks/benchmark_stream_reads.py @@ -0,0 +1,298 @@ +import pprint +import uuid +from time import perf_counter +from typing import Dict, Union + +import pandas as pd + +import btrdb + + +def time_single_stream_raw_values( + stream: btrdb.stream.Stream, start: int, end: int, version: int = 0 +) -> Dict[str, Union[int, str]]: + """Return the elapsed time for the stream raw values query + + Parameters + ---------- + stream : btrdb.Stream, required + The data stream to return raw values. + start : int, required + The start time (in nanoseconds) to return raw values (inclusive). + end : int, required + The end time (in nanoseconds) to return raw values (exclusive) + version : int, optional, default : 0 + The version of the stream to query for points. + + Notes + ----- + The data points returned will be [start, end) + + Returns + ------- + results : dict + The performance results of the stream method + """ + expected_count = stream.count(start, end, version=version, precise=True) + tic = perf_counter() + vals = stream.values(start, end, version=version) + toc = perf_counter() + # minus 1 to account for the exclusive end time + queried_points = len(vals) + assert queried_points == expected_count + # time in seconds to run + run_time = toc - tic + results = _create_stream_result_dict( + stream.uuid, point_count=queried_points, total_time=run_time, version=version + ) + return results + + +def time_single_stream_arrow_raw_values( + stream: btrdb.stream.Stream, start: int, end: int, version: int = 0 +) -> Dict[str, Union[str, int, float]]: + """Return the elapsed time for the stream arrow raw values query + + Parameters + ---------- + stream : btrdb.Stream, required + The data stream to return the raw data as an arrow table. + start : int, required + The start time (in nanoseconds) to return raw values (inclusive). + end : int, required + The end time (in nanoseconds) to return raw values (exclusive) + version : int, optional, default : 0 + The version of the stream to query for points. + + Notes + ----- + The data points returned will be [start, end) + + Returns + ------- + results : dict + The performance results of the stream method + """ + # minus 1 to account for the exclusive end time for the values query + expected_count = stream.count(start, end, version=version, precise=True) + tic = perf_counter() + vals = stream.arrow_values(start, end, version=version) + toc = perf_counter() + # num of rows + queried_points = vals.num_rows + assert queried_points == expected_count + # time in seconds to run + run_time = toc - tic + results = _create_stream_result_dict( + stream.uuid, point_count=queried_points, total_time=run_time, version=version + ) + return results + + +def time_single_stream_windows_values( + stream: btrdb.stream.Stream, start: int, end: int, width_ns: int, version: int = 0 +) -> Dict[str, Union[str, int, float]]: + """Return the elapsed time for the stream windows values query + + Parameters + ---------- + stream : btrdb.Stream, required + The data stream to return the windowed data as a list of statpoints + start : int, required + The start time (in nanoseconds) to return statpoint values + end : int, required + The end time (in nanoseconds) to return statpoint values + width_ns : int, required + The window width (in nanoseconds) for the statpoints + version : int, optional, default : 0 + The version of the stream to query for points. + + Returns + ------- + results : dict + The performance results of the stream method + """ + tic = perf_counter() + vals = stream.windows(start, end, width=width_ns, version=version) + toc = perf_counter() + # num of statpoints + queried_points = len(vals) + assert queried_points != 0 + # time in seconds to run + run_time = toc - tic + results = _create_stream_result_dict( + stream.uuid, point_count=queried_points, total_time=run_time, version=version + ) + return results + + +def time_single_stream_arrow_windows_values( + stream: btrdb.stream.Stream, start: int, end: int, width_ns: int, version: int = 0 +) -> Dict[str, Union[str, int, float]]: + """Return the elapsed time for the stream arrow window values query + + Parameters + ---------- + stream : btrdb.Stream, required + The data stream to return the windowed data as an arrow table. + start : int, required + The start time (in nanoseconds) to return statpoint values + end : int, required + The end time (in nanoseconds) to return statpoint values + width_ns : int, required + The window width (in nanoseconds) for the statpoints + version : int, optional, default : 0 + The version of the stream to query for points. + + Notes + ----- + The data points returned will be [start, end) + + Returns + ------- + results : dict + The performance results of the stream method + """ + tic = perf_counter() + vals = stream.arrow_windows(start, end, width=width_ns, version=version) + toc = perf_counter() + # num of statpoints + queried_points = vals.num_rows + assert queried_points != 0 + # time in seconds to run + run_time = toc - tic + results = _create_stream_result_dict( + stream.uuid, point_count=queried_points, total_time=run_time, version=version + ) + return results + + +def time_single_stream_aligned_windows_values( + stream: btrdb.stream.Stream, start: int, end: int, pointwidth: int, version: int = 0 +) -> Dict[str, Union[str, int, float]]: + """Return the elapsed time for the stream window values query + + Parameters + ---------- + stream : btrdb.Stream, required + The data stream to return the windowed data as a list of statpoints + start : int, required + The start time (in nanoseconds) to return statpoint values + end : int, required + The end time (in nanoseconds) to return statpoint values + pointwidth : int, required + The level of the tree to return statpoints (the exponent k in 2**k) + version : int, optional, default : 0 + The version of the stream to query for points. + + Returns + ------- + results : dict + The performance results of the stream method + """ + tic = perf_counter() + vals = stream.aligned_windows(start, end, pointwidth=pointwidth, version=version) + toc = perf_counter() + # num of statpoints + queried_points = len(vals) + assert queried_points != 0 + # time in seconds to run + run_time = toc - tic + results = _create_stream_result_dict( + stream.uuid, point_count=queried_points, total_time=run_time, version=version + ) + return results + + +def time_single_stream_arrow_aligned_windows_values( + stream: btrdb.stream.Stream, start: int, end: int, pointwidth: int, version: int = 0 +) -> Dict[str, Union[str, int, float]]: + """Return the elapsed time for the stream arrow aligned window values query + + Parameters + ---------- + stream : btrdb.Stream, required + The data stream to return the windowed data as an arrow table. + start : int, required + The start time (in nanoseconds) to return statpoint values + end : int, required + The end time (in nanoseconds) to return statpoint values + pointwidth : int, required + The level of the tree to return statpoints (the exponent k in 2**k) + version : int, optional, default : 0 + The version of the stream to query for points. + + Returns + ------- + results : dict + The performance results of the stream method + """ + tic = perf_counter() + vals = stream.arrow_aligned_windows( + start, end, pointwidth=pointwidth, version=version + ) + toc = perf_counter() + # num of statpoints + queried_points = vals.num_rows + assert queried_points != 0 + # time in seconds to run + run_time = toc - tic + results = _create_stream_result_dict( + stream.uuid, point_count=queried_points, total_time=run_time, version=version + ) + return results + + +def _create_stream_result_dict( + uu: uuid.UUID, + point_count: int, + total_time: float, + version: int, +) -> Dict[str, Union[str, int, float]]: + return { + "uuid": str(uu), + "total_points": point_count, + "total_time_seconds": total_time, + "stream_version": version, + } + + +def main(): + """Run a single run of the benchmarks""" + conn = btrdb.connect(profile="andy") + stream1 = conn.stream_from_uuid( + list(conn.streams_in_collection("andy/7064-6684-5e6e-9e14-ff9ca7bae46e"))[ + 0 + ].uuid + ) + start = stream1.earliest()[0].time + end = stream1.latest()[0].time + width_ns = btrdb.utils.timez.ns_delta(minutes=1) + pointwidth = btrdb.utils.general.pointwidth(38) + print(f"pointwidth of: {pointwidth}") + res_list = [] + for f in [time_single_stream_arrow_raw_values, time_single_stream_raw_values]: + res = f(stream1, start, end, 0) + res["func"] = f.__name__ + res_list.append(res) + for f in [ + time_single_stream_arrow_windows_values, + time_single_stream_windows_values, + ]: + res = f(stream1, start, end, width_ns=width_ns, version=0) + res["func"] = f.__name__ + res_list.append(res) + for f in [ + time_single_stream_arrow_aligned_windows_values, + time_single_stream_aligned_windows_values, + ]: + res = f(stream1, start, end, pointwidth=pointwidth, version=0) + res["func"] = f.__name__ + res_list.append(res) + return res_list + + +if __name__ == "__main__": + result = main() + pprint.pprint(result) + print(pd.DataFrame.from_dict(result, orient="columns")) diff --git a/benchmarks/benchmark_streamset_inserts.py b/benchmarks/benchmark_streamset_inserts.py new file mode 100644 index 0000000..4a03e1f --- /dev/null +++ b/benchmarks/benchmark_streamset_inserts.py @@ -0,0 +1,103 @@ +import uuid +from time import perf_counter +from typing import Dict, List, Tuple, Union + +import pyarrow + +import btrdb + + +def time_streamset_inserts( + streamset: btrdb.stream.StreamSet, + data_map: Dict[uuid.UUID, List[Tuple[int, float]]], + merge_policy: str = "never", +) -> Dict[str, Union[str, int, float, List, Dict]]: + """Insert data into a streamset with a provided data map. + + Parameters + ---------- + streamset : btrdb.stream.StreamSet, required + The streams to insert data into. + data_map : Dict[uuid.UUID, List[Tuple[int, float]], required + The mappings of streamset uuids to data to insert. + merge_policy : str, optional, default = 'never' + How should the platform handle duplicated data? + Valid policies: + `never`: the default, no points are merged + `equal`: points are deduplicated if the time and value are equal + `retain`: if two points have the same timestamp, the old one is kept + `replace`: if two points have the same timestamp, the new one is kept + """ + _ensure_all_streams_in_data_map(streamset, data_map) + prev_vers = streamset.versions() + tic = perf_counter() + new_vers = streamset.insert(data_map=data_map, merge=merge_policy) + toc = perf_counter() + n_streams = len(streamset) + total_points = sum([len(v) for v in data_map.values()]) + points_insert_per_stream = {s.uuid: len(data_map[s.uuid]) for s in streamset} + total_time = toc - tic + result = { + "n_streams": n_streams, + "total_points": total_points, + "previous_versions": prev_vers, + "new_versions": new_vers, + "total_time_seconds": total_time, + "points_insert_per_stream": points_insert_per_stream, + "merge_policy": merge_policy, + } + return result + + +def time_streamset_arrow_inserts( + streamset: btrdb.stream.StreamSet, + data_map: Dict[uuid.UUID, pyarrow.Table], + merge_policy: str = "never", +) -> Dict[str, Union[str, int, float, List, Dict]]: + """Insert data into streamset using pyarrow Tables. + + Parameters + ---------- + streamset : btrdb.stream.StreamSet, required + The streams to insert data into. + data_map : Dict[uuid.UUID, pyarrow.Table], required + The mappings of streamset uuids to pyarrow tables to insert. + merge_policy : str, optional, default = 'never' + How should the platform handle duplicated data? + Valid policies: + `never`: the default, no points are merged + `equal`: points are deduplicated if the time and value are equal + `retain`: if two points have the same timestamp, the old one is kept + `replace`: if two points have the same timestamp, the new one is kept + """ + _ensure_all_streams_in_data_map(streamset, data_map) + prev_vers = streamset.versions() + tic = perf_counter() + new_vers = streamset.arrow_insert(data_map=data_map, merge=merge_policy) + toc = perf_counter() + n_streams = len(streamset) + total_points = sum([v.num_rows for v in data_map.values()]) + points_insert_per_stream = {s.uuid: data_map[s.uuid].num_rows for s in streamset} + total_time = toc - tic + result = { + "n_streams": n_streams, + "total_points": total_points, + "previous_versions": prev_vers, + "new_versions": new_vers, + "total_time_seconds": total_time, + "points_insert_per_stream": points_insert_per_stream, + "merge_policy": merge_policy, + } + return result + + +def _ensure_all_streams_in_data_map( + streamset: btrdb.stream.StreamSet, + data_map: Dict[uuid.UUID, Union[pyarrow.Table, List[Tuple[int, float]]]], +) -> None: + truthy = [s.uuid in data_map for s in streamset] + uus = [s.uuid for s in streamset] + if not all(truthy): + raise ValueError( + f"Streams in streamset are not present in the data map. Data_map keys: {data_map.keys()}, streamset uuids: {uus}" + ) diff --git a/benchmarks/benchmark_streamset_reads.py b/benchmarks/benchmark_streamset_reads.py new file mode 100644 index 0000000..7c2c836 --- /dev/null +++ b/benchmarks/benchmark_streamset_reads.py @@ -0,0 +1,440 @@ +import pprint +import uuid +from time import perf_counter +from typing import Dict, Union + +import pandas + +import btrdb + + +def time_streamset_raw_values( + streamset: btrdb.stream.StreamSet, start: int, end: int, version: int = 0 +) -> Dict[str, Union[int, str]]: + """Return the elapsed time for the streamset raw values query + + Parameters + ---------- + streamset : btrdb.StreamSet, required + The data streamset to return raw values. + start : int, required + The start time (in nanoseconds) to return raw values (inclusive). + end : int, required + The end time (in nanoseconds) to return raw values (exclusive) + version : int, optional, default : 0 + The version of the streamset to query for points. + + Notes + ----- + The data points returned will be [start, end) + + Returns + ------- + results : dict + The performance results of the streamset method + """ + streamset = streamset.filter(start=start, end=end) + versions = {s.uuid: 0 for s in streamset} + streamset.pin_versions(versions) + # minus 1 * n_streams to account for the exclusive end time + expected_count = streamset.count(precise=True) - len(streamset) + tic = perf_counter() + vals = streamset.values() + toc = perf_counter() + # print(vals) + queried_points = 0 + for points in vals: + print(len(points)) + queried_points += len(points) + expected_count = streamset.count(precise=True) + print(queried_points) + print(expected_count) + assert queried_points == expected_count + # time in seconds to run + run_time = toc - tic + res = _create_streamset_result_dict( + streamset, point_count=queried_points, total_time=run_time, version=version + ) + return res + + +def time_streamset_arrow_raw_values( + streamset: btrdb.stream.StreamSet, start: int, end: int, version: int = 0 +) -> Dict[str, Union[str, int, float]]: + """Return the elapsed time for the streamset arrow raw values query + + Parameters + ---------- + streamset : btrdb.StreamSet, required + The data streamset to return the raw data as an arrow table. + start : int, required + The start time (in nanoseconds) to return raw values (inclusive). + end : int, required + The end time (in nanoseconds) to return raw values (exclusive) + version : int, optional, default : 0 + The version of the streamset to query for points. + + Notes + ----- + The data points returned will be [start, end) + + Returns + ------- + results : dict + The performance results of the streamset method + """ + streamset = streamset.filter(start=start, end=end) + versions = {s.uuid: 0 for s in streamset} + streamset.pin_versions(versions) + # minus 1 * n_streams to account for the exclusive end time + expected_count = streamset.count(precise=True) - len(streamset) + tic = perf_counter() + vals = streamset.arrow_values() + toc = perf_counter() + # TODO: These point counts and assertion needs to be column specific + queried_points = len(streamset) * (vals.num_rows - 1) + print(queried_points) + print(expected_count) + assert queried_points == expected_count + # time in seconds to run + run_time = toc - tic + res = _create_streamset_result_dict( + streamset, point_count=queried_points, total_time=run_time, version=version + ) + return res + + +def time_streamset_windows_values( + streamset: btrdb.stream.StreamSet, + start: int, + end: int, + width_ns: int, + version: int = 0, +) -> Dict[str, Union[str, int, float]]: + """Return the elapsed time for the streamset windows values query + + Parameters + ---------- + streamset : btrdb.StreamSet, required + The data streamset to return the windowed data as a list of statpoints + start : int, required + The start time (in nanoseconds) to return statpoint values + end : int, required + The end time (in nanoseconds) to return statpoint values + width_ns : int, required + The window width (in nanoseconds) for the statpoints + version : int, optional, default : 0 + The version of the streamset to query for points. + + Returns + ------- + results : dict + The performance results of the streamset method + """ + streamset = streamset.filter(start=start, end=end) + versions = {s.uuid: 0 for s in streamset} + streamset.pin_versions(versions) + streamset = streamset.windows(width=width_ns) + tic = perf_counter() + vals = streamset.values() + toc = perf_counter() + # num of statpoints + queried_points = len(vals[0]) * len(streamset) + assert queried_points != 0 + # time in seconds to run + run_time = toc - tic + results = _create_streamset_result_dict( + streamset, point_count=queried_points, total_time=run_time, version=version + ) + return results + + +def time_streamset_arrow_windows_values( + streamset: btrdb.stream.StreamSet, + start: int, + end: int, + width_ns: int, + version: int = 0, +) -> Dict[str, Union[str, int, float]]: + """Return the elapsed time for the streamset arrow window values query + + Parameters + ---------- + streamset : btrdb.StreamSet, required + The data streamset to return the windowed data as an arrow table. + start : int, required + The start time (in nanoseconds) to return statpoint values + end : int, required + The end time (in nanoseconds) to return statpoint values + width_ns : int, required + The window width (in nanoseconds) for the statpoints + version : int, optional, default : 0 + The version of the streamset to query for points. + + Notes + ----- + The data points returned will be [start, end) + + Returns + ------- + results : dict + The performance results of the streamset method + """ + streamset = streamset.filter(start=start, end=end) + versions = {s.uuid: 0 for s in streamset} + streamset.pin_versions(versions) + streamset = streamset.windows(width=width_ns) + tic = perf_counter() + vals = streamset.arrow_values() + toc = perf_counter() + # num of statpoints + queried_points = vals.num_rows * len(streamset) + assert queried_points != 0 + # time in seconds to run + run_time = toc - tic + results = _create_streamset_result_dict( + streamset, point_count=queried_points, total_time=run_time, version=version + ) + return results + + +def time_streamset_aligned_windows_values( + streamset: btrdb.stream.StreamSet, + start: int, + end: int, + pointwidth: int, + version: int = 0, +) -> Dict[str, Union[str, int, float]]: + """Return the elapsed time for the streamset window values query + + Parameters + ---------- + streamset : btrdb.StreamSet, required + The data streamset to return the windowed data as a list of statpoints + start : int, required + The start time (in nanoseconds) to return statpoint values + end : int, required + The end time (in nanoseconds) to return statpoint values + pointwidth : int, required + The level of the tree to return statpoints (the exponent k in 2**k) + version : int, optional, default : 0 + The version of the streamset to query for points. + + Returns + ------- + results : dict + The performance results of the streamset method + """ + streamset = streamset.filter(start=start, end=end) + versions = {s.uuid: 0 for s in streamset} + streamset.pin_versions(versions) + streamset = streamset.aligned_windows(pointwidth=pointwidth) + tic = perf_counter() + vals = streamset.values() + toc = perf_counter() + # num of statpoints + queried_points = len(vals[0]) * len(streamset) + assert queried_points != 0 + # time in seconds to run + run_time = toc - tic + results = _create_streamset_result_dict( + streamset, point_count=queried_points, total_time=run_time, version=version + ) + return results + + +def time_streamset_arrow_aligned_windows_values( + streamset: btrdb.stream.StreamSet, + start: int, + end: int, + pointwidth: int, + version: int = 0, +) -> Dict[str, Union[str, int, float]]: + """Return the elapsed time for the streamset arrow aligned window values query + + Parameters + ---------- + streamset : btrdb.StreamSet, required + The data streamset to return the windowed data as an arrow table. + start : int, required + The start time (in nanoseconds) to return statpoint values + end : int, required + The end time (in nanoseconds) to return statpoint values + pointwidth : int, required + The level of the tree to return statpoints (the exponent k in 2**k) + version : int, optional, default : 0 + The version of the streamset to query for points. + + Returns + ------- + results : dict + The performance results of the streamset method + """ + streamset = streamset.filter(start=start, end=end) + versions = {s.uuid: 0 for s in streamset} + streamset.pin_versions(versions) + streamset = streamset.aligned_windows(pointwidth=pointwidth) + tic = perf_counter() + vals = streamset.arrow_values() + toc = perf_counter() + # num of statpoints + queried_points = vals.num_rows * len(streamset) + assert queried_points != 0 + # time in seconds to run + run_time = toc - tic + results = _create_streamset_result_dict( + streamset, point_count=queried_points, total_time=run_time, version=version + ) + return results + + +def time_streamset_arrow_multistream_raw_values_non_timesnapped( + streamset: btrdb.stream.StreamSet, + start: int, + end: int, + version: int = 0, + sampling_frequency: int = None, +) -> Dict[str, Union[str, int, float]]: + """Use the arrow multistream endpoint that joins the stream data on-server before sending to the client. + + We make sure to set a sampling rate of 0 to ensure that we do not time snap and just perform a full-outer join on + the streams. + + Parameters + ---------- + streamset : btrdb.stream.StreamSet, required + The streamset to perform the multistream query on. + start : int, required + The start time (in nanoseconds) to query raw data from. + end : int, required + The end time (in nanoseconds) non-exclusive, to query raw data from. + version : int, optional, default=0 + The version of the stream to pin against, currently this is unused. + sampling_frequency : int, optional, ignored + The sampling frequency of the data stream in Hz + + Notes + ----- + Sampling_frequency is not used here, it will be manually set. + """ + streamset = streamset.filter(start=start, end=end, sampling_frequency=0) + versions = {s.uuid: 0 for s in streamset} + streamset = streamset.pin_versions(versions) + tic = perf_counter() + vals = streamset.arrow_values() + toc = perf_counter() + queried_points = vals.num_rows * len(streamset) + # print(vals) + # print(vals.to_pandas().describe()) + run_time = toc - tic + results = _create_streamset_result_dict( + streamset=streamset, total_time=run_time, point_count=queried_points, version=0 + ) + return results + + +def time_streamset_arrow_multistream_raw_values_timesnapped( + streamset: btrdb.stream.StreamSet, + start: int, + end: int, + sampling_frequency: int, + version: int = 0, +) -> Dict[str, Union[str, int, float]]: + """Use the arrow multistream endpoint that joins the stream data on-server before sending to the client. + + We make sure to set a sampling rate to ensure that we time snap the returned data. + + Parameters + ---------- + streamset : btrdb.stream.StreamSet, required + The streamset to perform the multistream query on. + start : int, required + The start time (in nanoseconds) to query raw data from. + end : int, required + The end time (in nanoseconds) non-exclusive, to query raw data from. + sampling_frequency : int, required + The common sampling frequency (in Hz) of the data to snap the data points to. + version : int, optional, default=0 + The version of the stream to pin against, currently this is unused. + """ + streamset = streamset.filter( + start=start, end=end, sampling_frequency=sampling_frequency + ) + versions = {s.uuid: 0 for s in streamset} + streamset = streamset.pin_versions(versions) + tic = perf_counter() + vals = streamset.arrow_values() + toc = perf_counter() + queried_points = vals.num_rows * len(streamset) + # print(vals) + run_time = toc - tic + results = _create_streamset_result_dict( + streamset=streamset, total_time=run_time, point_count=queried_points, version=0 + ) + return results + + +def _create_streamset_result_dict( + streamset: btrdb.stream.StreamSet, + point_count: int, + total_time: float, + version: int, +) -> Dict[str, Union[str, int, float]]: + return { + "n_streams": len(streamset), + "total_points": point_count, + "total_time_seconds": total_time, + "streamset_version": version, + } + + +def main(): + """Run a single run of the benchmarks""" + conn = btrdb.connect(profile="andy") + stream1 = conn.stream_from_uuid( + list(conn.streams_in_collection("andy/7064-6684-5e6e-9e14-ff9ca7bae46e"))[ + 0 + ].uuid + ) + stream2 = conn.stream_from_uuid( + list(conn.streams_in_collection("andy/30e6-d72f-5cc7-9966-bc1579dc4a72"))[ + 0 + ].uuid + ) + streamset = btrdb.stream.StreamSet([stream1, stream2]) + start = max(stream1.earliest()[0].time, stream2.earliest()[0].time) + end = min(stream1.latest()[0].time, stream2.latest()[0].time) + width_ns = btrdb.utils.timez.ns_delta(minutes=1) + pointwidth = btrdb.utils.general.pointwidth(38) + print(f"pointwidth of: {pointwidth}") + res_list = [] + for f in [time_streamset_raw_values, time_streamset_arrow_raw_values]: + res = f(streamset, start, end, 0) + res["func"] = f.__name__ + res_list.append(res) + for f in [time_streamset_windows_values, time_streamset_arrow_windows_values]: + res = f(streamset, start, end, width_ns=width_ns, version=0) + res["func"] = f.__name__ + res_list.append(res) + for f in [ + time_streamset_aligned_windows_values, + time_streamset_arrow_aligned_windows_values, + ]: + res = f(streamset, start, end, pointwidth=pointwidth, version=0) + res["func"] = f.__name__ + res_list.append(res) + for f in [ + time_streamset_arrow_multistream_raw_values_non_timesnapped, + time_streamset_arrow_multistream_raw_values_timesnapped, + ]: + res = f(streamset, start, end, sampling_frequency=2, version=0) + res["func"] = f.__name__ + res_list.append(res) + + return res_list + + +if __name__ == "__main__": + results = main() + pprint.pprint(results) + print(pandas.DataFrame.from_dict(results, orient="columns")) diff --git a/btrdb/__init__.py b/btrdb/__init__.py index a7019c7..1584b3b 100644 --- a/btrdb/__init__.py +++ b/btrdb/__init__.py @@ -15,12 +15,15 @@ ## Imports ########################################################################## -from btrdb.conn import Connection, BTrDB +from warnings import warn + +from btrdb.conn import BTrDB, Connection from btrdb.endpoint import Endpoint from btrdb.exceptions import ConnectionError +from btrdb.stream import MAXIMUM_TIME, MINIMUM_TIME +from btrdb.utils.credentials import credentials, credentials_by_profile +from btrdb.utils.ray import register_serializer from btrdb.version import get_version -from btrdb.utils.credentials import credentials_by_profile, credentials -from btrdb.stream import MINIMUM_TIME, MAXIMUM_TIME ########################################################################## ## Module Variables @@ -37,37 +40,71 @@ ## Functions ########################################################################## + def _connect(endpoints=None, apikey=None): return BTrDB(Endpoint(Connection(endpoints, apikey=apikey).channel)) -def connect(conn_str=None, apikey=None, profile=None): + +def connect(conn_str=None, apikey=None, profile=None, shareable=False): """ Connect to a BTrDB server. Parameters ---------- conn_str: str, default=None - The address and port of the cluster to connect to, e.g. `192.168.1.1:4411`. - If set to None, will look in the environment variable `$BTRDB_ENDPOINTS` + The address and port of the cluster to connect to, e.g. ``192.168.1.1:4411``. + If set to None, will look in the environment variable ``$BTRDB_ENDPOINTS`` (recommended). apikey: str, default=None The API key used to authenticate requests (optional). If None, the key - is looked up from the environment variable `$BTRDB_API_KEY`. + is looked up from the environment variable ``$BTRDB_API_KEY``. profile: str, default=None The name of a profile containing the required connection information as found in the user's predictive grid credentials file - `~/.predictivegrid/credentials.yaml`. + ``~/.predictivegrid/credentials.yaml``. + shareable: bool, default=False + Whether the connection can be "shared" in a distributed setting such + as Ray workers. If set to True, the connection can be serialized and sent + to other workers so that data can be retrieved in parallel; **however**, this + is less secure because it is possible for other users of the Ray cluster to + use your API key to fetch data. Returns ------- db : BTrDB An instance of the BTrDB context to directly interact with the database. + Examples + -------- + This example looks for the env variables: ``BTRDB_ENDPOINTS`` and ``BTRDB_API_KEY``. + + >>> conn = btrdb.connect() + + + + Connect to the platform by looking for the relevant platform profile + in ``${HOME}/.predictivegrid/credentials.yaml`` if the file is present. + + >>> conn = btrdb.connect(profile='test') + + + If you provide incorrect credentials, you will get an error. + + >>> conn = btrdb.connect(conn_str="192.168.1.1:4411", apikey="NONSENSICAL_API_KEY") + + """ # do not allow user to provide both address and profile if conn_str and profile: raise ValueError("Received both conn_str and profile arguments.") + # check shareable flag and register custom serializer if necessary + if shareable: + warn( + "a shareable connection is potentially insecure; other users of the same cluster may be able to access your API key" + ) + register_serializer(conn_str=conn_str, apikey=apikey, profile=profile) + # use specific profile if requested if profile: return _connect(**credentials_by_profile(profile)) @@ -78,4 +115,3 @@ def connect(conn_str=None, apikey=None, profile=None): return _connect(**creds) raise ConnectionError("Could not determine credentials to use.") - diff --git a/btrdb/conn.py b/btrdb/conn.py index 55058a4..4d26400 100644 --- a/btrdb/conn.py +++ b/btrdb/conn.py @@ -15,18 +15,24 @@ ## Imports ########################################################################## +import importlib.metadata +import json +import logging import os import re -import json import uuid as uuidlib +from concurrent.futures import ThreadPoolExecutor +from typing import List, Tuple, Union +from warnings import warn +import certifi import grpc from grpc._cython.cygrpc import CompressionAlgorithm +from btrdb.exceptions import BTrDBError, InvalidOperation, StreamNotFoundError, retry from btrdb.stream import Stream, StreamSet -from btrdb.utils.general import unpack_stream_descriptor from btrdb.utils.conversion import to_uuid -from btrdb.exceptions import StreamNotFoundError, InvalidOperation +from btrdb.utils.general import unpack_stream_descriptor ########################################################################## ## Module Variables @@ -40,23 +46,37 @@ ########################################################################## ## Classes ########################################################################## +logger = logging.getLogger(__name__) -class Connection(object): +class Connection(object): def __init__(self, addrportstr, apikey=None): """ Connects to a BTrDB server Parameters ---------- - addrportstr: str + addrportstr : str, required The address of the cluster to connect to, e.g 123.123.123:4411 - apikey: str - The option API key to authenticate requests + apikey : str, optional + The optional API key to authenticate requests + Notes + ----- + The ``btrdb.connect`` method is a helper function to make connecting to the platform easier + usually that will be sufficient for most users. """ + # 4 Is a magic number to make sure the error propagates where btrdb.connect is called. + warn( + "This API is deprecated in favor of the pingthings_api, refer to your hub landing page for further documentation.", + FutureWarning, + stacklevel=4, + ) addrport = addrportstr.split(":", 2) - chan_ops = [('grpc.default_compression_algorithm', CompressionAlgorithm.gzip)] + # 100MB size limit ~ 2500 streams for 5000 points with each point being 64bit + # 500MB size limit ~ 13K streams for 5000 points + # -1 size limit = no limit of size to send + chan_ops = [("grpc.max_receive_message_length", -1)] if len(addrport) != 2: raise ValueError("expecting address:port") @@ -65,34 +85,131 @@ def __init__(self, addrportstr, apikey=None): # grpc bundles its own CA certs which will work for all normal SSL # certificates but will fail for custom CA certs. Allow the user # to specify a CA bundle via env var to overcome this - ca_bundle = os.getenv("BTRDB_CA_BUNDLE","") - if ca_bundle != "": + env_bundle = os.getenv("BTRDB_CA_BUNDLE", "") + + # certifi certs are provided as part of this package install + # https://github.com/certifi/python-certifi + lib_certs = certifi.where() + + ca_bundle = env_bundle + + if ca_bundle == "": + ca_bundle = lib_certs + try: with open(ca_bundle, "rb") as f: contents = f.read() - else: - contents = None + except Exception: + if env_bundle != "": + # The user has given us something but we can't use it, we need to make noise + raise Exception( + "BTRDB_CA_BUNDLE(%s) env is defined but could not read file" + % ca_bundle + ) + else: + contents = None if apikey is None: self.channel = grpc.secure_channel( addrportstr, grpc.ssl_channel_credentials(contents), - options=chan_ops + options=chan_ops, ) else: self.channel = grpc.secure_channel( addrportstr, grpc.composite_channel_credentials( grpc.ssl_channel_credentials(contents), - grpc.access_token_call_credentials(apikey) + grpc.access_token_call_credentials(apikey), ), - options=chan_ops + options=chan_ops, ) else: - if apikey is not None: - raise ValueError("cannot use an API key with an insecure (port 4410) BTrDB API. Try port 4411") self.channel = grpc.insecure_channel(addrportstr, chan_ops) + if apikey is not None: + + class AuthCallDetails(grpc.ClientCallDetails): + def __init__(self, apikey, client_call_details): + metadata = [] + if client_call_details.metadata is not None: + metadata = list(client_call_details.metadata) + metadata.append(("authorization", "Bearer " + apikey)) + version = "unknown" + try: + version = importlib.metadata.version("btrdb") + except Exception: + pass + metadata.append( + ( + "x-api-client", + "btrdbpy-" + version, + ) + ) + self.method = client_call_details.method + self.timeout = client_call_details.timeout + self.credentials = client_call_details.credentials + self.wait_for_ready = client_call_details.wait_for_ready + self.compression = client_call_details.compression + self.metadata = metadata + + class AuthorizationInterceptor( + grpc.UnaryUnaryClientInterceptor, + grpc.UnaryStreamClientInterceptor, + grpc.StreamUnaryClientInterceptor, + grpc.StreamStreamClientInterceptor, + ): + def __init__(self, apikey): + self.apikey = apikey + + def intercept_unary_unary( + self, continuation, client_call_details, request + ): + return continuation( + AuthCallDetails(self.apikey, client_call_details), request + ) + + def intercept_unary_stream( + self, continuation, client_call_details, request + ): + return continuation( + AuthCallDetails(self.apikey, client_call_details), request + ) + + def intercept_stream_unary( + self, continuation, client_call_details, request_iterator + ): + return continuation( + AuthCallDetails(self.apikey, client_call_details), + request_iterator, + ) + + def intercept_stream_stream( + self, continuation, client_call_details, request_iterator + ): + return continuation( + AuthCallDetails(self.apikey, client_call_details), + request_iterator, + ) + + self.channel = grpc.intercept_channel( + self.channel, + AuthorizationInterceptor(apikey), + ) +def _is_arrow_enabled(info): + info = { + "majorVersion": info.majorVersion, + "minorVersion": info.minorVersion, + } + major = info.get("majorVersion", -1) + minor = info.get("minorVersion", -1) + if logger.isEnabledFor(logging.DEBUG): + logger.debug(f"major version: {major}") + logger.debug(f"minor version: {minor}") + if major >= 5 and minor >= 30: + return True + else: + return False class BTrDB(object): @@ -102,22 +219,53 @@ class BTrDB(object): def __init__(self, endpoint): self.ep = endpoint - - def query(self, stmt, params=[]): + try: + _ = self.ep.info() + except Exception as err: + raise BTrDBError(f"Could not connect to the database, error message: {err}") + self._executor = ThreadPoolExecutor() + try: + self._ARROW_ENABLED = _is_arrow_enabled(self.ep.info()) + except Exception: + self._ARROW_ENABLED = False + if logger.isEnabledFor(logging.DEBUG): + logger.debug(f"ARROW ENABLED: {self._ARROW_ENABLED}") + + @retry + def query( + self, + stmt: str, + params: Union[Tuple[str], List[str]] = None, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Performs a SQL query on the database metadata and returns a list of dictionaries from the resulting cursor. Parameters ---------- - stmt: str + stmt : str a SQL statement to be executed on the BTrDB metadata. Available tables are noted below. To sanitize inputs use a `$1` style parameter such as `select * from streams where name = $1 or name = $2`. - params: list or tuple + params : list or tuple a list of parameter values to be sanitized and interpolated into the SQL statement. Using parameters forces value/type checking and is considered a best practice at the very least. + auto_retry : bool, default: False + Whether to retry this request in the event of an error + retries : int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay : int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff : int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- @@ -128,42 +276,118 @@ def query(self, stmt, params=[]): Notes ------- Parameters will be inserted into the SQL statement as noted by the - paramter number such as `$1`, `$2`, or `$3`. The `streams` table is + parameter number such as `$1`, `$2`, or `$3`. The `streams` table is available for `SELECT` statements only. See https://btrdb.readthedocs.io/en/latest/ for more info. + + The following are the queryable columns in the postgres ``streams`` table. + + +------------------+------------------------+-----------+ + | Column | Type | Nullable | + +==================+========================+===========+ + | uuid | uuid | not null | + +------------------+------------------------+-----------+ + | collection | character varying(256) | not null | + +------------------+------------------------+-----------+ + | name | character varying(256) | not null | + +------------------+------------------------+-----------+ + | unit | character varying(256) | not null | + +------------------+------------------------+-----------+ + | ingress | character varying(256) | not null | + +------------------+------------------------+-----------+ + | property_version | bigint | not null | + +------------------+------------------------+-----------+ + | annotations | hstore | | + +------------------+------------------------+-----------+ + + Examples + -------- + Count all streams in the platform. + + >>> conn = btrdb.connect() + >>> conn.query("SELECT COUNT(uuid) FROM streams") + [{'count': ...}] + + Count all streams in the collection ``foo/bar`` by passing in the variable as a parameter. + + >>> conn.query("SELECT COUNT(uuid) FROM streams WHERE collection=$1::text", params=["foo/bar"]) + [{'count': ...}] + + Count all streams in the platform that has a non-null entry for the metadata annotation ``foo``. + + >>> conn.query("SELECT COUNT(uuid) FROM streams WHERE annotations->$1::text IS NOT NULL", params=["foo"]) + [{'count': ...}] """ + if params is None: + params = list() return [ json.loads(row.decode("utf-8")) for page in self.ep.sql_query(stmt, params) for row in page ] - def streams(self, *identifiers, versions=None, is_collection_prefix=False): """ Returns a StreamSet object with BTrDB streams from the supplied identifiers. If any streams cannot be found matching the identifier - than StreamNotFoundError will be returned. + then a ``StreamNotFoundError`` will be returned. Parameters ---------- - identifiers: str or UUID + identifiers : str or UUID a single item or iterable of items which can be used to query for - streams. identiers are expected to be UUID as string, UUID as UUID, + streams. Identifiers are expected to be UUID as string, UUID as UUID, or collection/name string. - versions: list[int] + versions : list[int] a single or iterable of version numbers to match the identifiers + is_collection_prefix : bool, default=False + If providing a collection string, is that string just a prefix, or the entire collection name? + This will impact how many streams are returned. + + + Returns + ------- + :class:`StreamSet` + Collection of streams. + + Examples + -------- + With a sequence of uuids. + + >>> conn = btrdb.connect() + >>> conn.streams(identifiers=list_of_uuids) + + + With a sequence of uuids and version numbers. + Here we are using version 0 to use the latest data points. + + >>> conn.streams(identifiers=list_of_uuids, versions=[0 for _ in list_of_uuids]) + + + Filtering by ``collection`` prefix ``"foo"`` where multiple collections exist like the following: + ``foo/bar``, ``foo/baz``, ``foo/bar/new``, and ``foo``. + If we set `is_collection_prefix`` to ``True``, this will return all streams that exist in the collections defined above. + It is similar to a regex pattern ``^foo.*`` for matching purposes. + + >>> conn.streams(identifiers="foo", is_collection_prefix=True) + + + If you set ``is_collection_prefix`` to ``False``, this will assume that the string identifier you provide is the full collection name. + Matching like the regex here: ``^foo`` + + >>> conn.streams(identifiers="foo", is_collection_prefix=False) + + """ if versions is not None and not isinstance(versions, list): raise TypeError("versions argument must be of type list") if versions and len(versions) != len(identifiers): raise ValueError("number of versions does not match identifiers") - - streams = [] + streams: List[Stream] = [] for ident in identifiers: if isinstance(ident, uuidlib.UUID): streams.append(self.stream_from_uuid(ident)) @@ -182,20 +406,26 @@ def streams(self, *identifiers, versions=None, is_collection_prefix=False): found = self.streams_in_collection( "/".join(parts[:-1]), is_collection_prefix=is_collection_prefix, - tags={"name": parts[-1]} + tags={"name": parts[-1]}, ) - if len(found) == 1: + if isinstance(found, Stream): + streams.append(found) + continue + if isinstance(found, list) and len(found) == 1: streams.append(found[0]) continue raise StreamNotFoundError(f"Could not identify stream `{ident}`") - raise ValueError(f"Could not identify stream based on `{ident}`. Identifier must be UUID or collection/name.") - + raise ValueError( + f"Could not identify stream based on `{ident}`. Identifier must be UUID or collection/name." + ) obj = StreamSet(streams) if versions: - version_dict = {streams[idx].uuid: versions[idx] for idx in range(len(versions))} + version_dict = { + streams[idx].uuid: versions[idx] for idx in range(len(versions)) + } obj.pin_versions(version_dict) return obj @@ -210,7 +440,7 @@ def stream_from_uuid(self, uuid): Parameters ---------- - uuid: UUID + uuid : UUID The uuid of the requested stream. Returns @@ -218,22 +448,72 @@ def stream_from_uuid(self, uuid): Stream instance of Stream class or None + Examples + -------- + + >>> import btrdb + >>> conn = btrdb.connect() + >>> uuid = "f98f4b4e-9fab-46b5-8a80-f282059d69b1" + >>> stream = conn.stream_from_uuid(uuid) + >>> stream + + """ return Stream(self, to_uuid(uuid)) - def create(self, uuid, collection, tags=None, annotations=None): + @retry + def create( + self, + uuid, + collection, + tags=None, + annotations=None, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Tells BTrDB to create a new stream with UUID `uuid` in `collection` with specified `tags` and `annotations`. Parameters ---------- - uuid: UUID + uuid : UUID, required The uuid of the requested stream. + collection : str, required + The collection string prefix that the stream will belong to. + tags : dict, required + The tags-level metadata key:value pairs. + annotations : dict, optional + The mutable metadata of the stream, key:value pairs + auto_retry : bool, default: False + Whether to retry this request in the event of an error + retries : int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay : int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff : int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- Stream instance of Stream class + + + Examples + -------- + >>> import btrdb + >>> from uuid import uuid4 # this generates a random uuid + >>> conn = btrdb.connect() + >>> collection = "new/stream/collection" + >>> tags = {"name":"foo", "unit":"V"} + >>> annotations = {"bar": "baz"} + >>> s = conn.create(uuid=uuid4(), tags=tags, annotations=annotations, collection=collection) + """ if tags is None: @@ -243,29 +523,43 @@ def create(self, uuid, collection, tags=None, annotations=None): annotations = {} self.ep.create(uuid, collection, tags, annotations) - return Stream(self, uuid, + return Stream( + self, + uuid, known_to_exist=True, collection=collection, tags=tags.copy(), annotations=annotations.copy(), - property_version=0 + property_version=1, ) def info(self): """ - Returns information about the connected BTrDB srerver. + Returns information about the platform proxy server. Returns ------- dict - server connection and status information + Proxy server connection and status information + + Examples + -------- + >>> conn = btrdb.connect() + >>> conn.info() + { + .. 'majorVersion': 5, + .. 'minorVersion': 8, + .. 'build': ..., + .. 'proxy': ..., + } """ info = self.ep.info() return { "majorVersion": info.majorVersion, + "minorVersion": info.minorVersion, "build": info.build, - "proxy": { "proxyEndpoints": [ep for ep in info.proxy.proxyEndpoints] }, + "proxy": {"proxyEndpoints": [ep for ep in info.proxy.proxyEndpoints]}, } def list_collections(self, starts_with=""): @@ -273,14 +567,140 @@ def list_collections(self, starts_with=""): Returns a list of collection paths using the `starts_with` argument for filtering. + Parameters + ---------- + starts_with : str, optional, default: '' + Filter collections that start with the string provided, if none passed, will list all collections. + Returns ------- - collection paths: list[str] + collections: List[str] + + Examples + -------- + + Assuming we have the following collections in the platform: + ``foo``, ``bar``, ``foo/baz``, ``bar/baz`` + + >>> conn = btrdb.connect() + >>> conn.list_collections().sort() + ["bar", "bar/baz", "foo", "foo/bar"] + + >>> conn.list_collections(starts_with="foo") + ["foo", "foo/bar"] + """ return [c for some in self.ep.listCollections(starts_with) for c in some] - def streams_in_collection(self, *collection, is_collection_prefix=True, tags=None, annotations=None): + def _list_unique_tags_annotations(self, key, collection): + """ + Returns a SQL statement and parameters to get list of tags or annotations. + """ + if key == "annotations": + query = "select distinct({}) as {} from streams".format( + "skeys(annotations)", "annotations" + ) + else: + query = "select distinct({}) as {} from streams".format(key, key) + params = [] + if isinstance(collection, str): + params.append("{}%".format(collection)) + query = " where ".join([query, """collection like $1"""]) + return [metadata[key] for metadata in self.query(query, params)] + + def list_unique_annotations(self, collection=None): + """ + Returns a list of annotation keys used in a given collection prefix. + + Parameters + ---------- + collection : str + Prefix of the collection to filter. + + Returns + ------- + annotations : list[str] + + Notes + ----- + This query treats the ``collection`` string as a prefix, so ``collection="foo"`` will match with the following wildcard syntax ``foo%``. + If you only want to filter for a single collection, you will need to provide the full collection, if there are other collections + that match the ``foo%`` pattern, you might need to use a custom SQL query using ``conn.query``. + + Examples + -------- + >>> conn.list_unique_annotations(collection="sunshine/PMU1") + ['foo', 'location', 'impedance'] + + """ + return self._list_unique_tags_annotations("annotations", collection) + + def list_unique_names(self, collection=None): + """ + Returns a list of names used in a given collection prefix. + + Parameters + ---------- + collection : str + Prefix of the collection to filter. + + Returns + ------- + names : list[str] + + Examples + -------- + Can specify a full ``collection`` name. + + >>> conn.list_unique_names(collection="sunshine/PMU1") + ['C1ANG', 'C1MAG', 'C2ANG', 'C2MAG', 'C3ANG', 'C3MAG', 'L1ANG', 'L1MAG', 'L2ANG', 'L2MAG', 'L3ANG', 'L3MAG', 'LSTATE'] + + And also provide a ``collection`` prefix. + + >>> conn.list_unique_names(collection="sunshine/") + ['C1ANG', 'C1MAG', 'C2ANG', 'C2MAG', 'C3ANG', 'C3MAG', 'L1ANG', 'L1MAG', 'L2ANG', 'L2MAG', 'L3ANG', 'L3MAG', 'LSTATE'] + + + + """ + return self._list_unique_tags_annotations("name", collection) + + def list_unique_units(self, collection=None): + """ + Returns a list of units used in a given collection prefix. + + Parameters + ---------- + collection : str + Prefix of the collection to filter. + + Returns + ------- + units : list[str] + + + Examples + -------- + + >>> conn.list_unique_units(collection="sunshine/PMU1") + ['amps', 'deg', 'mask', 'volts'] + + """ + return self._list_unique_tags_annotations("unit", collection) + + @retry + def streams_in_collection( + self, + *collection, + is_collection_prefix=True, + tags=None, + annotations=None, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Search for streams matching given parameters @@ -288,20 +708,54 @@ def streams_in_collection(self, *collection, is_collection_prefix=True, tags=Non Parameters ---------- - collection: str + collection : str collections to use when searching for streams, case sensitive. - is_collection_prefix: bool + is_collection_prefix : bool Whether the collection is a prefix. - tags: Dict[str, str] + tags : Dict[str, str] The tags to identify the stream. - annotations: Dict[str, str] + annotations : Dict[str, str] The annotations to identify the stream. + auto_retry : bool, default: False + Whether to retry this request in the event of an error + retries : int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay : int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff : int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------ - list - A list of stream objects found with the provided search arguments. + list[Stream] + A list of ``Stream`` objects found with the provided search arguments. + + + .. note:: + + In a future release, the default return value of this function will be a ``StreamSet`` + + Examples + -------- + + >>> conn = btrdb.connect() + >>> conn.streams_in_collection(collection="foo", is_collection_prefix=True) + [, , ] + + >>> conn.streams_in_collection(collection="foo", is_collection_prefix=False) + [, ] + >>> conn.streams_in_collection(collection="foo", + ... is_collection_prefix=False, tags={"unit":"Volts"}) + [] + + >>> conn.streams_in_collection(collection="foo", + ... is_collection_prefix=False, tags={"unit":"UNKNOWN"}) + [] """ result = [] @@ -315,28 +769,59 @@ def streams_in_collection(self, *collection, is_collection_prefix=True, tags=Non collection = [None] for item in collection: - streams = self.ep.lookupStreams(item, is_collection_prefix, tags, annotations) + streams = self.ep.lookupStreams( + item, is_collection_prefix, tags, annotations + ) for desclist in streams: for desc in desclist: tagsanns = unpack_stream_descriptor(desc) - result.append(Stream( - self, uuidlib.UUID(bytes = desc.uuid), - known_to_exist=True, collection=desc.collection, - tags=tagsanns[0], annotations=tagsanns[1], - property_version=desc.propertyVersion - )) - + result.append( + Stream( + self, + uuidlib.UUID(bytes=desc.uuid), + known_to_exist=True, + collection=desc.collection, + tags=tagsanns[0], + annotations=tagsanns[1], + property_version=desc.propertyVersion, + ) + ) + # TODO: In future release update this method to return a streamset object. + warn( + "StreamSet will be the default return object for ``streams_in_collection`` in a future release.", + FutureWarning, + stacklevel=2, + ) return result - def collection_metadata(self, prefix): + @retry + def collection_metadata( + self, + prefix, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Gives statistics about metadata for collections that match a - prefix. + ``prefix``. Parameters ---------- - prefix: str + prefix : str, required A prefix of the collection names to look at + auto_retry : bool, default: False + Whether to retry this request in the event of an error + retries : int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay : int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff : int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- @@ -344,6 +829,16 @@ def collection_metadata(self, prefix): A tuple of dictionaries containing metadata on the streams in the provided collection. + Examples + -------- + >>> conn.collection_metadata("sunshine/PMU1") + ({'name': 0, 'unit': 0, 'ingress': 0, 'distiller': 0}, + .. {'foo': 1, 'impedance': 12, 'location': 12}) + + >>> conn.collection_metadata("sunshine/") + ({'name': 0, 'unit': 0, 'ingress': 0, 'distiller': 0}, + .. {'foo': 1, 'impedance': 72, 'location': 72}) + """ ep = self.ep tags, annotations = ep.getMetadataUsage(prefix) diff --git a/btrdb/endpoint.py b/btrdb/endpoint.py index de5378e..c632bbb 100644 --- a/btrdb/endpoint.py +++ b/btrdb/endpoint.py @@ -24,15 +24,26 @@ # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import io +import typing +import uuid -from btrdb.grpcinterface import btrdb_pb2 -from btrdb.grpcinterface import btrdb_pb2_grpc +from btrdb.exceptions import check_proto_stat, error_handler +from btrdb.grpcinterface import btrdb_pb2, btrdb_pb2_grpc from btrdb.point import RawPoint -from btrdb.exceptions import BTrDBError, error_handler, check_proto_stat from btrdb.utils.general import unpack_stream_descriptor +try: + import pyarrow as pa +except ImportError: + pa = None + +_ARROW_IMPORT_MSG = """Package `pyarrow` required, please pip install.""" + class Endpoint(object): + """Server endpoint where we make specific requests.""" + def __init__(self, channel): self.stub = btrdb_pb2_grpc.BTrDBStub(channel) @@ -45,6 +56,77 @@ def rawValues(self, uu, start, end, version=0): check_proto_stat(result.stat) yield result.values, result.versionMajor + @error_handler + def arrowRawValues(self, uu, start, end, version=0, schema=None): + if pa is None: + raise ImportError(_ARROW_IMPORT_MSG) + templateBytes = b"" + if schema is not None: + byte_io = io.BytesIO() + with pa.ipc.new_stream(sink=byte_io, schema=schema) as _: + pass + templateBytes = byte_io.getvalue() + params = btrdb_pb2.ArrowRawValuesParams( + uuid=uu.bytes, + start=start, + end=end, + versionMajor=version, + templateBytes=templateBytes, + ) + for result in self.stub.ArrowRawValues(params): + check_proto_stat(result.stat) + with pa.ipc.open_stream(result.arrowBytes) as reader: + yield reader.read_all(), result.versionMajor + + @error_handler + def arrowMultiValues( + self, uu_list, start, end, version_list, snap_periodNS=None, schema=None + ): + if pa is None: + raise ImportError(_ARROW_IMPORT_MSG) + templateBytes = b"" + if schema is not None: + byte_io = io.BytesIO() + with pa.ipc.new_stream(sink=byte_io, schema=schema) as _: + pass + templateBytes = byte_io.getvalue() + params = btrdb_pb2.ArrowMultiValuesParams( + uuid=[uu.bytes for uu in uu_list], + start=start, + end=end, + versionMajor=[ver for ver in version_list], + snapPeriodNs=int(snap_periodNS), + templateBytes=templateBytes, + ) + for result in self.stub.ArrowMultiValues(params): + check_proto_stat(result.stat) + with pa.ipc.open_stream(result.arrowBytes) as reader: + yield reader.read_all() + + @error_handler + def arrowInsertValues(self, uu: uuid.UUID, values: pa.Table, policy: str): + if pa is None: + raise ImportError(_ARROW_IMPORT_MSG) + policy_map = { + "never": btrdb_pb2.MergePolicy.NEVER, + "equal": btrdb_pb2.MergePolicy.EQUAL, + "retain": btrdb_pb2.MergePolicy.RETAIN, + "replace": btrdb_pb2.MergePolicy.REPLACE, + } + byte_io = io.BytesIO() + with pa.ipc.new_stream(sink=byte_io, schema=values.schema) as writer: + writer.write(values) + arrow_bytes = byte_io.getvalue() + params = btrdb_pb2.ArrowInsertParams( + uuid=uu.bytes, + sync=False, + arrowBytes=arrow_bytes, + merge_policy=policy_map[policy], + ) + result = self.stub.ArrowInsert(params) + check_proto_stat(result.stat) + return result.versionMajor + @error_handler def alignedWindows(self, uu, start, end, pointwidth, version=0): params = btrdb_pb2.AlignedWindowsParams( @@ -58,6 +140,22 @@ def alignedWindows(self, uu, start, end, pointwidth, version=0): check_proto_stat(result.stat) yield result.values, result.versionMajor + @error_handler + def arrowAlignedWindows(self, uu, start, end, pointwidth, version=0): + if pa is None: + raise ImportError(_ARROW_IMPORT_MSG) + params = btrdb_pb2.AlignedWindowsParams( + uuid=uu.bytes, + start=start, + end=end, + versionMajor=version, + pointWidth=int(pointwidth), + ) + for result in self.stub.ArrowAlignedWindows(params): + check_proto_stat(result.stat) + with pa.ipc.open_stream(result.arrowBytes) as reader: + yield reader.read_all(), result.versionMajor + @error_handler def windows(self, uu, start, end, width, depth, version=0): params = btrdb_pb2.WindowsParams( @@ -72,6 +170,23 @@ def windows(self, uu, start, end, width, depth, version=0): check_proto_stat(result.stat) yield result.values, result.versionMajor + @error_handler + def arrowWindows(self, uu, start, end, width, depth, version=0): + if pa is None: + raise ImportError(_ARROW_IMPORT_MSG) + params = btrdb_pb2.WindowsParams( + uuid=uu.bytes, + start=start, + end=end, + versionMajor=version, + width=width, + depth=depth, + ) + for result in self.stub.ArrowWindows(params): + check_proto_stat(result.stat) + with pa.ipc.open_stream(result.arrowBytes) as reader: + yield reader.read_all(), result.versionMajor + @error_handler def streamInfo(self, uu, omitDescriptor, omitVersion): params = btrdb_pb2.StreamInfoParams( @@ -81,7 +196,13 @@ def streamInfo(self, uu, omitDescriptor, omitVersion): desc = result.descriptor check_proto_stat(result.stat) tagsanns = unpack_stream_descriptor(desc) - return desc.collection, desc.propertyVersion, tagsanns[0], tagsanns[1], result.versionMajor + return ( + desc.collection, + desc.propertyVersion, + tagsanns[0], + tagsanns[1], + result.versionMajor, + ) @error_handler def obliterate(self, uu): @@ -135,11 +256,11 @@ def setStreamTags(self, uu, expected, tags, collection): def create(self, uu, collection, tags, annotations): tagkvlist = [] for k, v in tags.items(): - kv = btrdb_pb2.KeyOptValue(key = k, val = btrdb_pb2.OptValue(value=v)) + kv = btrdb_pb2.KeyOptValue(key=k, val=btrdb_pb2.OptValue(value=v)) tagkvlist.append(kv) annkvlist = [] for k, v in annotations.items(): - kv = btrdb_pb2.KeyOptValue(key = k, val = btrdb_pb2.OptValue(value=v)) + kv = btrdb_pb2.KeyOptValue(key=k, val=btrdb_pb2.OptValue(value=v)) annkvlist.append(kv) params = btrdb_pb2.CreateParams( uuid=uu.bytes, collection=collection, tags=tagkvlist, annotations=annkvlist @@ -201,7 +322,7 @@ def nearest(self, uu, time, version, backward): result = self.stub.Nearest(params) check_proto_stat(result.stat) return result.value, result.versionMajor - + @error_handler def changes(self, uu, fromVersion, toVersion, resolution): params = btrdb_pb2.ChangesParams( @@ -268,25 +389,53 @@ def getMetadataUsage(self, prefix): return result.tags, result.annotations @error_handler - def generateCSV(self, queryType, start, end, width, depth, includeVersions, *streams): - protoStreams = [btrdb_pb2.StreamCSVConfig(version = stream[0], - label = stream[1], - uuid = stream[2].bytes) - for stream in streams] - params = btrdb_pb2.GenerateCSVParams(queryType = queryType.to_proto(), - startTime = start, - endTime = end, - windowSize = width, - depth = depth, - includeVersions = includeVersions, - streams = protoStreams) + def generateCSV( + self, queryType, start, end, width, depth, includeVersions, *streams + ): + protoStreams = [ + btrdb_pb2.StreamCSVConfig( + version=stream[0], label=stream[1], uuid=stream[2].bytes + ) + for stream in streams + ] + params = btrdb_pb2.GenerateCSVParams( + queryType=queryType.to_proto(), + startTime=start, + endTime=end, + windowSize=width, + depth=depth, + includeVersions=includeVersions, + streams=protoStreams, + ) for result in self.stub.GenerateCSV(params): check_proto_stat(result.stat) yield result.row @error_handler - def sql_query(self, stmt, params=[]): + def sql_query(self, stmt, params: typing.List): request = btrdb_pb2.SQLQueryParams(query=stmt, params=params) for page in self.stub.SQLQuery(request): check_proto_stat(page.stat) yield page.SQLQueryRow + + @error_handler + def subscribe(self, update_queue): + def updates(): + while True: + update = update_queue.get() + if update is None: + return + (to_add, to_remove) = update + if len(to_add) != 0: + yield btrdb_pb2.SubscriptionUpdate( + op=0, uuid=[uu.bytes for uu in to_add] + ) + if len(to_remove) != 0: + yield btrdb_pb2.SubscriptionUpdate( + op=1, uuid=[uu.bytes for uu in to_remove] + ) + + for response in self.stub.Subscribe(updates()): + check_proto_stat(response.stat) + with pa.ipc.open_stream(response.arrowBytes) as reader: + yield uuid.UUID(bytes=response.uuid), reader.read_all() diff --git a/btrdb/exceptions.py b/btrdb/exceptions.py index 2007503..8507f7c 100644 --- a/btrdb/exceptions.py +++ b/btrdb/exceptions.py @@ -12,13 +12,64 @@ ########################################################################## import inspect -from grpc import RpcError +import logging +import time from functools import wraps +from grpc import RpcError + +########################################################################## +## Module Variables and Constants +########################################################################## +MAX_RETRIES = 5 +INITIAL_RETRY_DELAY = 3 # time to wait before first retry in seconds +RETRY_BACKOFF_FACTOR = ( + 4 # exponential factor by which retry backoff increases between tries +) + + ########################################################################## ## Decorators ########################################################################## +# config logging for auto-retry +logging.basicConfig( + format="%(asctime)s:%(levelname)s:%(message)s", + level=logging.INFO, + datefmt="%m/%d/%Y %I:%M:%S", +) + + +def retry(fn, *args, **kwargs): + """ + decorates functions and retries it in the event of BTrDBErrors or RpcErrors + """ + + @wraps(fn) + def retry_func(*args, **kwargs): + # don't retry if arg is set to False + if not kwargs.get("auto_retry"): + return fn(*args, **kwargs) + + total_retries = kwargs.get("retries") or MAX_RETRIES + retries = total_retries + delay = kwargs.get("retry_delay") or INITIAL_RETRY_DELAY + backoff = kwargs.get("retry_backoff") or RETRY_BACKOFF_FACTOR + while retries > 0: + try: + return fn(*args, **kwargs) + except (BTrDBError, RpcError) as e: + msg = f"ERROR: {e}, attempting retry {total_retries-retries+1}/{total_retries} in {delay} seconds..." + logging.info(msg) + time.sleep(delay) + retries -= 1 + delay *= backoff + err = e + handle_grpc_error(err) + + return retry_func + + def consume_generator(fn, *args, **kwargs): # when a generator is passed back to the calling function, it may encounter an error # when trying to call next(), in that case we want to yield an Exception @@ -27,6 +78,7 @@ def consume_generator(fn, *args, **kwargs): except RpcError as e: handle_grpc_error(e) + def error_handler(fn): """ decorates endpoint functions and checks for grpc.RpcErrors @@ -35,6 +87,7 @@ def error_handler(fn): ---------- fn: function """ + # allows input func to keep its name and metadata @wraps(fn) def wrap(*args, **kwargs): @@ -44,12 +97,15 @@ def wrap(*args, **kwargs): return fn(*args, **kwargs) except RpcError as e: handle_grpc_error(e) + return wrap + ########################################################################## ## gRPC error handling ########################################################################## + # NOTE: this function relies on matching strings and isn't really ideal. # this is more of a band-aid solution while we figure out how to send # better errors from btrdb-server @@ -61,18 +117,21 @@ def handle_grpc_error(err): Parameters ---------- - err: grpc.RpcError + err: Union[grpc.RpcError, btrdb.BTrDBError] """ details = err.details() - if details == "[404] stream does not exist": - raise StreamNotFoundError("Stream not found with provided uuid") from None - elif details == "failed to connect to all addresses": - raise ConnectionError("Failed to connect to BTrDB") from None - elif any(str(e) in err.details() for e in BTRDB_SERVER_ERRORS): - raise BTRDBServerError("An error has occured with btrdb-server") from None - elif str(err.code()) == "StatusCode.PERMISSION_DENIED": - raise PermissionDenied(details) from None - raise BTrDBError(details) from None + if isinstance(err, BTrDBError): + if details == "[404] stream does not exist": + raise StreamNotFoundError("Stream not found with provided uuid") from None + elif details == "failed to connect to all addresses": + raise ConnectionError("Failed to connect to BTrDB") from None + elif any(str(e) in err.details() for e in BTRDB_SERVER_ERRORS): + raise BTRDBServerError("An error has occured with btrdb-server") from None + elif str(err.code()) == "StatusCode.PERMISSION_DENIED": + raise PermissionDenied(details) from None + raise BTrDBError(details) from None + else: + raise err def check_proto_stat(stat): @@ -89,144 +148,189 @@ def check_proto_stat(stat): if code in BTRDB_ERRORS: raise BTRDB_ERRORS[code](stat.msg) elif code in BTRDB_SERVER_ERRORS: - raise BTRDBServerError(stat.msg) + raise BTRDBServerError(str(code) + ": " + stat.msg) raise BTrDBError(stat.msg) + ########################################################################## ## BTrDB Exceptions ########################################################################## + class BTrDBError(Exception): """ The primary exception for BTrDB errors. """ + pass + class ConnectionError(BTrDBError): """ Raised when an error occurrs while trying to establish a connection with BTrDB. """ + pass + class StreamNotFoundError(BTrDBError): """ Raised when attempting to perform an operation on a stream that does not exist in the specified BTrDB allocation. """ + pass + class CredentialsFileNotFound(FileNotFoundError, BTrDBError): """ Raised when a credentials file could not be found. """ + pass + class ProfileNotFound(BTrDBError): """ Raised when a requested profile could not be found in the credentials file. """ + pass + class BTRDBServerError(BTrDBError): """ Raised when an error occurs with btrdb-server. """ + pass + class BTRDBTypeError(TypeError, BTrDBError): """ Raised when attempting to perform an operation with an invalid type. """ + pass + class InvalidOperation(BTrDBError): """ Raised when an invalid BTrDB operation has been requested. """ + pass + class StreamExists(InvalidOperation): """ Raised when create() has been attempted and the uuid already exists. """ + pass + class AmbiguousStream(InvalidOperation): """ Raised when create() has been attempted and uuid is different, but collection and tags already exist """ + pass + class PermissionDenied(InvalidOperation): """ Raised when user does not have permission to perform an operation. """ + pass + class BTRDBValueError(ValueError, BTrDBError): """ Raised when an invalid value has been passed to a BTrDB operation. """ + pass + class InvalidCollection(BTRDBValueError): """ Raised when a collection name is invalid. It is either too long or not a valid string. """ + pass + class InvalidTagKey(BTRDBValueError): """ Raised when a tag key is invalid. Must be one of ("name", "unit", "ingress", "distiller"). """ + pass + class InvalidTagValue(BTRDBValueError): """ Raised when a tag value is invalid. It is either too long or not a valid string. """ + pass + class InvalidTimeRange(BTRDBValueError): """ Raised when insert data contains a timestamp outside the range of (btrdb.MINIMUM_TIME, btrdb.MAXIMUM_TIME) """ + pass + class InvalidPointWidth(BTRDBValueError): """ Raised when attempting to use a pointwidth that is not a whole number between 0 and 64 (exclusive). """ + pass + class BadValue(BTRDBValueError): """ Raised when attempting to insert data that contains non-float values such as None. """ + pass + class RecycledUUID(BTRDBValueError): """ Raised when attempting to create a stream with a uuid that matches a previously deleted stream. """ + pass + class BadSQLValue(BTRDBValueError): """ Raised when invalid parameters have been passed to metadata db. """ + pass + class VersionNotAvailable(BTRDBValueError): """ Raised when querying a stream at a pruned or otherwise invalid version number. """ + pass + class NoSuchPoint(BTRDBValueError): """ Raised when asking for next/previous point and there isn't one. """ + pass @@ -248,35 +352,35 @@ class NoSuchPoint(BTRDBValueError): 425: BadValue, 429: RecycledUUID, 441: BadSQLValue, - 450: VersionNotAvailable + 450: VersionNotAvailable, } # All of these raise BTRDBServerError BTRDB_SERVER_ERRORS = [ - 402, # ContextError - 403, # InsertFailure - 405, # WrongEndpoint - 414, # InsertTooBig - 421, # WrongArgs - 423, # AnnotationVersionMismatch - 424, # FaultInjectionDisabled - 426, # ResourceDepleted - 427, # InvalidVersions - 431, # ObliterateDisabled - 432, # CephError - 433, # NodeExisted - 434, # JournalError - 438, # InvalidParameter - 440, # MetadataConnectionError - 452, # OverlappingTrimRange - 453, # LockFailed - 454, # NoLeafNode - 455, # TierCacheError - 456, # StreamEmpty - 457, # TieredStorageBackendError - 458, # TieredStorageOutOfBounds - 459, # TieredStorageTemporaryError - 500, # InvariantFailure - 501, # NotImplemented - 502, # UnsupportedRollback -] \ No newline at end of file + 402, # ContextError + 403, # InsertFailure + 405, # WrongEndpoint + 414, # InsertTooBig + 421, # WrongArgs + 423, # AnnotationVersionMismatch + 424, # FaultInjectionDisabled + 426, # ResourceDepleted + 427, # InvalidVersions + 431, # ObliterateDisabled + 432, # CephError + 433, # NodeExisted + 434, # JournalError + 438, # InvalidParameter + 440, # MetadataConnectionError + 452, # OverlappingTrimRange + 453, # LockFailed + 454, # NoLeafNode + 455, # TierCacheError + 456, # StreamEmpty + 457, # TieredStorageBackendError + 458, # TieredStorageOutOfBounds + 459, # TieredStorageTemporaryError + 500, # InvariantFailure + 501, # NotImplemented + 502, # UnsupportedRollback +] diff --git a/btrdb/grpcinterface/README.md b/btrdb/grpcinterface/README.md new file mode 100644 index 0000000..24987ee --- /dev/null +++ b/btrdb/grpcinterface/README.md @@ -0,0 +1,6 @@ +To create new protobuf files in this folder: +```commandline +python -m grpc_tools.protoc --proto_path=./ --python_out=./ --pyi_out=. --grpc_python_out=./ btrdb.proto +``` + +Make sure the proto file is newest. And that you run the above command from within this folder. diff --git a/btrdb/grpcinterface/btrdb.proto b/btrdb/grpcinterface/btrdb.proto index 93fdff2..1e147d6 100644 --- a/btrdb/grpcinterface/btrdb.proto +++ b/btrdb/grpcinterface/btrdb.proto @@ -4,8 +4,12 @@ package v5api; service BTrDB { rpc RawValues(RawValuesParams) returns (stream RawValuesResponse); + rpc ArrowRawValues(ArrowRawValuesParams) returns (stream ArrowRawValuesResponse); + rpc ArrowMultiValues(ArrowMultiValuesParams) returns (stream ArrowMultiValuesResponse); rpc AlignedWindows(AlignedWindowsParams) returns (stream AlignedWindowsResponse); + rpc ArrowAlignedWindows(AlignedWindowsParams) returns (stream ArrowAlignedWindowsResponse); rpc Windows(WindowsParams) returns (stream WindowsResponse); + rpc ArrowWindows(WindowsParams) returns (stream ArrowWindowsResponse); rpc StreamInfo(StreamInfoParams) returns (StreamInfoResponse); rpc SetStreamAnnotations(SetStreamAnnotationsParams) returns (SetStreamAnnotationsResponse); rpc SetStreamTags(SetStreamTagsParams) returns (SetStreamTagsResponse); @@ -15,6 +19,7 @@ service BTrDB { rpc Nearest(NearestParams) returns (NearestResponse); rpc Changes(ChangesParams) returns (stream ChangesResponse); rpc Insert(InsertParams) returns (InsertResponse); + rpc ArrowInsert(ArrowInsertParams) returns (InsertResponse); rpc Delete(DeleteParams) returns (DeleteResponse); rpc Info(InfoParams) returns (InfoResponse); rpc FaultInject(FaultInjectParams) returns (FaultInjectResponse); @@ -23,6 +28,7 @@ service BTrDB { rpc GetMetadataUsage(MetadataUsageParams) returns (MetadataUsageResponse); rpc GenerateCSV(GenerateCSVParams) returns (stream GenerateCSVResponse); rpc SQLQuery(SQLQueryParams) returns (stream SQLQueryResponse); + rpc Subscribe(stream SubscriptionUpdate) returns (stream SubscriptionResp); //rpc SetCompactionConfig(SetCompactionConfigParams) returns (SetCompactionConfigResponse); //rpc GetCompactionConfig(GetCompactionConfigParams) returns (GetCompactionConfigResponse); } @@ -39,6 +45,35 @@ message RawValuesResponse { uint64 versionMinor = 3; repeated RawPoint values = 4; } +message ArrowRawValuesParams { + bytes uuid = 1; + sfixed64 start = 2; + sfixed64 end = 3; + uint64 versionMajor = 4; + bytes templateBytes = 5; +} +message ArrowRawValuesResponse { + Status stat = 1; + uint64 versionMajor = 2; + uint64 versionMinor = 3; + bytes arrowBytes = 4; +} +message ArrowMultiValuesParams { + repeated bytes uuid = 1; + repeated uint64 versionMajor = 2; + sfixed64 start = 3; + sfixed64 end = 4; + int64 snapPeriodNs = 5; + bytes templateBytes = 6; +} +message ArrowMultiValuesResponse { + Status stat = 1; + bytes arrowBytes = 2; +} +message RawPointVec { + sfixed64 time = 1; + repeated double value = 2; +}; message AlignedWindowsParams { bytes uuid = 1; sfixed64 start = 2; @@ -52,6 +87,12 @@ message AlignedWindowsResponse { uint64 versionMinor = 3; repeated StatPoint values = 4; } +message ArrowAlignedWindowsResponse { + Status stat = 1; + uint64 versionMajor = 2; + uint64 versionMinor = 3; + bytes arrowBytes = 4; +} message WindowsParams { bytes uuid = 1; sfixed64 start = 2; @@ -66,6 +107,12 @@ message WindowsResponse { uint64 versionMinor = 3; repeated StatPoint values = 4; } +message ArrowWindowsResponse { + Status stat = 1; + uint64 versionMajor = 2; + uint64 versionMinor = 3; + bytes arrowBytes = 4; +} message StreamInfoParams { bytes uuid = 1; bool omitVersion = 2; @@ -175,12 +222,25 @@ enum MergePolicy { RETAIN = 2; // When timestamps are equal, keep old value REPLACE = 3; // When timestamps are equal, keep new value } +message RoundSpec { + oneof spec { + int32 bits = 2; + } +} message InsertParams { bytes uuid = 1; bool sync = 2; MergePolicy merge_policy = 4; + RoundSpec rounding=5; repeated RawPoint values = 3; } +message ArrowInsertParams { + bytes uuid = 1; + bool sync = 2; + MergePolicy merge_policy = 3; + RoundSpec rounding = 4; + bytes arrowBytes = 5; +} message InsertResponse { Status stat = 1; uint64 versionMajor = 2; @@ -375,3 +435,19 @@ message ReducedResolutionRange { int64 End = 2; uint32 Resolution = 3; } + +enum SubscriptionUpdateOp { + ADD_UUIDS = 0; + REMOVE_UUIDS = 1; +} + +message SubscriptionUpdate { + SubscriptionUpdateOp op = 1; + repeated bytes uuid = 2; +} + +message SubscriptionResp { + Status stat = 1; + bytes uuid = 2; + bytes arrowBytes = 3; +} diff --git a/btrdb/grpcinterface/btrdb_pb2.py b/btrdb/grpcinterface/btrdb_pb2.py index 8104a34..9dcef1a 100644 --- a/btrdb/grpcinterface/btrdb_pb2.py +++ b/btrdb/grpcinterface/btrdb_pb2.py @@ -1,13 +1,12 @@ +# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: btrdb.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf.internal import enum_type_wrapper +# Protobuf Python Version: 4.25.0 +"""Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection +from google.protobuf import descriptor_pool as _descriptor_pool from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -15,3542 +14,159 @@ -DESCRIPTOR = _descriptor.FileDescriptor( - name='btrdb.proto', - package='v5api', - syntax='proto3', - serialized_options=None, - serialized_pb=_b('\n\x0b\x62trdb.proto\x12\x05v5api\"Q\n\x0fRawValuesParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\"}\n\x11RawValuesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x1f\n\x06values\x18\x04 \x03(\x0b\x32\x0f.v5api.RawPoint\"j\n\x14\x41lignedWindowsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\x12\x12\n\npointWidth\x18\x05 \x01(\r\"\x83\x01\n\x16\x41lignedWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12 \n\x06values\x18\x04 \x03(\x0b\x32\x10.v5api.StatPoint\"m\n\rWindowsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\x12\r\n\x05width\x18\x05 \x01(\x04\x12\r\n\x05\x64\x65pth\x18\x06 \x01(\r\"|\n\x0fWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12 \n\x06values\x18\x04 \x03(\x0b\x32\x10.v5api.StatPoint\"h\n\x10StreamInfoParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x13\n\x0bomitVersion\x18\x02 \x01(\x08\x12\x16\n\x0eomitDescriptor\x18\x03 \x01(\x08\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"\x8a\x01\n\x12StreamInfoResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12+\n\ndescriptor\x18\x04 \x01(\x0b\x32\x17.v5api.StreamDescriptor\"\x98\x01\n\x10StreamDescriptor\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x12\n\ncollection\x18\x02 \x01(\t\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x17\n\x0fpropertyVersion\x18\x05 \x01(\x04\"\x82\x01\n\x1aSetStreamAnnotationsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x1f\n\x17\x65xpectedPropertyVersion\x18\x02 \x01(\x04\x12#\n\x07\x63hanges\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x10\n\x08removals\x18\x04 \x03(\t\";\n\x1cSetStreamAnnotationsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"\x8a\x01\n\x13SetStreamTagsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x1f\n\x17\x65xpectedPropertyVersion\x18\x02 \x01(\x04\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x12\n\ncollection\x18\x04 \x01(\t\x12\x0e\n\x06remove\x18\x05 \x03(\t\"4\n\x15SetStreamTagsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"{\n\x0c\x43reateParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x12\n\ncollection\x18\x02 \x01(\t\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\"-\n\x0e\x43reateResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"@\n\x13MetadataUsageParams\x12\x0e\n\x06prefix\x18\x01 \x01(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"y\n\x15MetadataUsageResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x1d\n\x04tags\x18\x02 \x03(\x0b\x32\x0f.v5api.KeyCount\x12$\n\x0b\x61nnotations\x18\x03 \x03(\x0b\x32\x0f.v5api.KeyCount\"&\n\x08KeyCount\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05\x63ount\x18\x02 \x01(\x04\"B\n\x15ListCollectionsParams\x12\x0e\n\x06prefix\x18\x01 \x01(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"K\n\x17ListCollectionsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x13\n\x0b\x63ollections\x18\x02 \x03(\t\"\xab\x01\n\x13LookupStreamsParams\x12\x12\n\ncollection\x18\x01 \x01(\t\x12\x1a\n\x12isCollectionPrefix\x18\x02 \x01(\x08\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"^\n\x15LookupStreamsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12(\n\x07results\x18\x02 \x03(\x0b\x32\x17.v5api.StreamDescriptor\"S\n\rNearestParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x0c\n\x04time\x18\x02 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x03 \x01(\x04\x12\x10\n\x08\x62\x61\x63kward\x18\x04 \x01(\x08\"z\n\x0fNearestResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x1e\n\x05value\x18\x04 \x01(\x0b\x32\x0f.v5api.RawPoint\"U\n\rChangesParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x11\n\tfromMajor\x18\x02 \x01(\x04\x12\x0f\n\x07toMajor\x18\x03 \x01(\x04\x12\x12\n\nresolution\x18\x04 \x01(\r\"\x7f\n\x0f\x43hangesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12#\n\x06ranges\x18\x04 \x03(\x0b\x32\x13.v5api.ChangedRange\"u\n\x0cInsertParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x0c\n\x04sync\x18\x02 \x01(\x08\x12(\n\x0cmerge_policy\x18\x04 \x01(\x0e\x32\x12.v5api.MergePolicy\x12\x1f\n\x06values\x18\x03 \x03(\x0b\x32\x0f.v5api.RawPoint\"Y\n\x0eInsertResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\"8\n\x0c\x44\x65leteParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\"Y\n\x0e\x44\x65leteResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\"\x0c\n\nInfoParams\"\xa2\x01\n\x0cInfoResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x19\n\x04mash\x18\x02 \x01(\x0b\x32\x0b.v5api.Mash\x12\x14\n\x0cmajorVersion\x18\x03 \x01(\r\x12\x14\n\x0cminorVersion\x18\x04 \x01(\r\x12\r\n\x05\x62uild\x18\x05 \x01(\t\x12\x1f\n\x05proxy\x18\x06 \x01(\x0b\x32\x10.v5api.ProxyInfo\"#\n\tProxyInfo\x12\x16\n\x0eproxyEndpoints\x18\x01 \x03(\t\"1\n\x11\x46\x61ultInjectParams\x12\x0c\n\x04type\x18\x01 \x01(\x04\x12\x0e\n\x06params\x18\x02 \x01(\x0c\">\n\x13\x46\x61ultInjectResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\n\n\x02rv\x18\x02 \x01(\x0c\"\x1b\n\x0b\x46lushParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"X\n\rFlushResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\" \n\x10ObliterateParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"1\n\x12ObliterateResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"\'\n\x08RawPoint\x12\x0c\n\x04time\x18\x01 \x01(\x10\x12\r\n\x05value\x18\x02 \x01(\x01\"`\n\tStatPoint\x12\x0c\n\x04time\x18\x01 \x01(\x10\x12\x0b\n\x03min\x18\x02 \x01(\x01\x12\x0c\n\x04mean\x18\x03 \x01(\x01\x12\x0b\n\x03max\x18\x04 \x01(\x01\x12\r\n\x05\x63ount\x18\x05 \x01(\x06\x12\x0e\n\x06stddev\x18\x06 \x01(\x01\"*\n\x0c\x43hangedRange\x12\r\n\x05start\x18\x01 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x02 \x01(\x10\">\n\x06Status\x12\x0c\n\x04\x63ode\x18\x01 \x01(\r\x12\x0b\n\x03msg\x18\x02 \x01(\t\x12\x19\n\x04mash\x18\x03 \x01(\x0b\x32\x0b.v5api.Mash\"\x98\x01\n\x04Mash\x12\x10\n\x08revision\x18\x01 \x01(\x03\x12\x0e\n\x06leader\x18\x02 \x01(\t\x12\x16\n\x0eleaderRevision\x18\x03 \x01(\x03\x12\x13\n\x0btotalWeight\x18\x04 \x01(\x03\x12\x0f\n\x07healthy\x18\x05 \x01(\x08\x12\x10\n\x08unmapped\x18\x06 \x01(\x01\x12\x1e\n\x07members\x18\x07 \x03(\x0b\x32\r.v5api.Member\"\xc3\x01\n\x06Member\x12\x0c\n\x04hash\x18\x01 \x01(\r\x12\x10\n\x08nodename\x18\x02 \x01(\t\x12\n\n\x02up\x18\x03 \x01(\x08\x12\n\n\x02in\x18\x04 \x01(\x08\x12\x0f\n\x07\x65nabled\x18\x05 \x01(\x08\x12\r\n\x05start\x18\x06 \x01(\x03\x12\x0b\n\x03\x65nd\x18\x07 \x01(\x03\x12\x0e\n\x06weight\x18\x08 \x01(\x03\x12\x16\n\x0ereadPreference\x18\t \x01(\x01\x12\x15\n\rhttpEndpoints\x18\n \x01(\t\x12\x15\n\rgrpcEndpoints\x18\x0b \x01(\t\"8\n\x0bKeyOptValue\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x1c\n\x03val\x18\x02 \x01(\x0b\x32\x0f.v5api.OptValue\"\x19\n\x08OptValue\x12\r\n\x05value\x18\x01 \x01(\t\"&\n\x08KeyValue\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"?\n\x0fStreamCSVConfig\x12\x0f\n\x07version\x18\x01 \x01(\x04\x12\r\n\x05label\x18\x02 \x01(\t\x12\x0c\n\x04uuid\x18\x03 \x01(\x0c\"\x9d\x02\n\x11GenerateCSVParams\x12\x35\n\tqueryType\x18\x01 \x01(\x0e\x32\".v5api.GenerateCSVParams.QueryType\x12\x11\n\tstartTime\x18\x02 \x01(\x03\x12\x0f\n\x07\x65ndTime\x18\x03 \x01(\x03\x12\x12\n\nwindowSize\x18\x04 \x01(\x04\x12\r\n\x05\x64\x65pth\x18\x05 \x01(\r\x12\x17\n\x0fincludeVersions\x18\x06 \x01(\x08\x12\'\n\x07streams\x18\x07 \x03(\x0b\x32\x16.v5api.StreamCSVConfig\"H\n\tQueryType\x12\x19\n\x15\x41LIGNED_WINDOWS_QUERY\x10\x00\x12\x11\n\rWINDOWS_QUERY\x10\x01\x12\r\n\tRAW_QUERY\x10\x02\"Q\n\x13GenerateCSVResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x10\n\x08isHeader\x18\x02 \x01(\x08\x12\x0b\n\x03row\x18\x03 \x03(\t\"J\n\x0eSQLQueryParams\x12\r\n\x05query\x18\x01 \x01(\t\x12\x0e\n\x06params\x18\x02 \x03(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"D\n\x10SQLQueryResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x13\n\x0bSQLQueryRow\x18\x02 \x03(\x0c\"\x14\n\x04Role\x12\x0c\n\x04name\x18\x01 \x01(\t\"\x94\x01\n\x19SetCompactionConfigParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x18\n\x10\x43ompactedVersion\x18\x02 \x01(\x04\x12>\n\x17reducedResolutionRanges\x18\x03 \x03(\x0b\x32\x1d.v5api.ReducedResolutionRange\x12\x0f\n\x07unused0\x18\x04 \x01(\x04\":\n\x1bSetCompactionConfigResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\")\n\x19GetCompactionConfigParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"\xc1\x01\n\x1bGetCompactionConfigResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x1a\n\x12LatestMajorVersion\x18\x02 \x01(\x04\x12\x18\n\x10\x43ompactedVersion\x18\x03 \x01(\x04\x12>\n\x17reducedResolutionRanges\x18\x04 \x03(\x0b\x32\x1d.v5api.ReducedResolutionRange\x12\x0f\n\x07unused0\x18\x05 \x01(\x04\"H\n\x16ReducedResolutionRange\x12\r\n\x05Start\x18\x01 \x01(\x03\x12\x0b\n\x03\x45nd\x18\x02 \x01(\x03\x12\x12\n\nResolution\x18\x03 \x01(\r*<\n\x0bMergePolicy\x12\t\n\x05NEVER\x10\x00\x12\t\n\x05\x45QUAL\x10\x01\x12\n\n\x06RETAIN\x10\x02\x12\x0b\n\x07REPLACE\x10\x03\x32\xb3\n\n\x05\x42TrDB\x12?\n\tRawValues\x12\x16.v5api.RawValuesParams\x1a\x18.v5api.RawValuesResponse0\x01\x12N\n\x0e\x41lignedWindows\x12\x1b.v5api.AlignedWindowsParams\x1a\x1d.v5api.AlignedWindowsResponse0\x01\x12\x39\n\x07Windows\x12\x14.v5api.WindowsParams\x1a\x16.v5api.WindowsResponse0\x01\x12@\n\nStreamInfo\x12\x17.v5api.StreamInfoParams\x1a\x19.v5api.StreamInfoResponse\x12^\n\x14SetStreamAnnotations\x12!.v5api.SetStreamAnnotationsParams\x1a#.v5api.SetStreamAnnotationsResponse\x12I\n\rSetStreamTags\x12\x1a.v5api.SetStreamTagsParams\x1a\x1c.v5api.SetStreamTagsResponse\x12\x34\n\x06\x43reate\x12\x13.v5api.CreateParams\x1a\x15.v5api.CreateResponse\x12Q\n\x0fListCollections\x12\x1c.v5api.ListCollectionsParams\x1a\x1e.v5api.ListCollectionsResponse0\x01\x12K\n\rLookupStreams\x12\x1a.v5api.LookupStreamsParams\x1a\x1c.v5api.LookupStreamsResponse0\x01\x12\x37\n\x07Nearest\x12\x14.v5api.NearestParams\x1a\x16.v5api.NearestResponse\x12\x39\n\x07\x43hanges\x12\x14.v5api.ChangesParams\x1a\x16.v5api.ChangesResponse0\x01\x12\x34\n\x06Insert\x12\x13.v5api.InsertParams\x1a\x15.v5api.InsertResponse\x12\x34\n\x06\x44\x65lete\x12\x13.v5api.DeleteParams\x1a\x15.v5api.DeleteResponse\x12.\n\x04Info\x12\x11.v5api.InfoParams\x1a\x13.v5api.InfoResponse\x12\x43\n\x0b\x46\x61ultInject\x12\x18.v5api.FaultInjectParams\x1a\x1a.v5api.FaultInjectResponse\x12\x31\n\x05\x46lush\x12\x12.v5api.FlushParams\x1a\x14.v5api.FlushResponse\x12@\n\nObliterate\x12\x17.v5api.ObliterateParams\x1a\x19.v5api.ObliterateResponse\x12L\n\x10GetMetadataUsage\x12\x1a.v5api.MetadataUsageParams\x1a\x1c.v5api.MetadataUsageResponse\x12\x45\n\x0bGenerateCSV\x12\x18.v5api.GenerateCSVParams\x1a\x1a.v5api.GenerateCSVResponse0\x01\x12<\n\x08SQLQuery\x12\x15.v5api.SQLQueryParams\x1a\x17.v5api.SQLQueryResponse0\x01\x62\x06proto3') -) - -_MERGEPOLICY = _descriptor.EnumDescriptor( - name='MergePolicy', - full_name='v5api.MergePolicy', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='NEVER', index=0, number=0, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='EQUAL', index=1, number=1, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='RETAIN', index=2, number=2, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='REPLACE', index=3, number=3, - serialized_options=None, - type=None), - ], - containing_type=None, - serialized_options=None, - serialized_start=5490, - serialized_end=5550, -) -_sym_db.RegisterEnumDescriptor(_MERGEPOLICY) - -MergePolicy = enum_type_wrapper.EnumTypeWrapper(_MERGEPOLICY) -NEVER = 0 -EQUAL = 1 -RETAIN = 2 -REPLACE = 3 - - -_GENERATECSVPARAMS_QUERYTYPE = _descriptor.EnumDescriptor( - name='QueryType', - full_name='v5api.GenerateCSVParams.QueryType', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='ALIGNED_WINDOWS_QUERY', index=0, number=0, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='WINDOWS_QUERY', index=1, number=1, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='RAW_QUERY', index=2, number=2, - serialized_options=None, - type=None), - ], - containing_type=None, - serialized_options=None, - serialized_start=4641, - serialized_end=4713, -) -_sym_db.RegisterEnumDescriptor(_GENERATECSVPARAMS_QUERYTYPE) - - -_RAWVALUESPARAMS = _descriptor.Descriptor( - name='RawValuesParams', - full_name='v5api.RawValuesParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.RawValuesParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='start', full_name='v5api.RawValuesParams.start', index=1, - number=2, type=16, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='end', full_name='v5api.RawValuesParams.end', index=2, - number=3, type=16, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMajor', full_name='v5api.RawValuesParams.versionMajor', index=3, - number=4, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=22, - serialized_end=103, -) - - -_RAWVALUESRESPONSE = _descriptor.Descriptor( - name='RawValuesResponse', - full_name='v5api.RawValuesResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.RawValuesResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMajor', full_name='v5api.RawValuesResponse.versionMajor', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMinor', full_name='v5api.RawValuesResponse.versionMinor', index=2, - number=3, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='values', full_name='v5api.RawValuesResponse.values', index=3, - number=4, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=105, - serialized_end=230, -) - - -_ALIGNEDWINDOWSPARAMS = _descriptor.Descriptor( - name='AlignedWindowsParams', - full_name='v5api.AlignedWindowsParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.AlignedWindowsParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='start', full_name='v5api.AlignedWindowsParams.start', index=1, - number=2, type=16, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='end', full_name='v5api.AlignedWindowsParams.end', index=2, - number=3, type=16, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMajor', full_name='v5api.AlignedWindowsParams.versionMajor', index=3, - number=4, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='pointWidth', full_name='v5api.AlignedWindowsParams.pointWidth', index=4, - number=5, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=232, - serialized_end=338, -) - - -_ALIGNEDWINDOWSRESPONSE = _descriptor.Descriptor( - name='AlignedWindowsResponse', - full_name='v5api.AlignedWindowsResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.AlignedWindowsResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMajor', full_name='v5api.AlignedWindowsResponse.versionMajor', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMinor', full_name='v5api.AlignedWindowsResponse.versionMinor', index=2, - number=3, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='values', full_name='v5api.AlignedWindowsResponse.values', index=3, - number=4, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=341, - serialized_end=472, -) - - -_WINDOWSPARAMS = _descriptor.Descriptor( - name='WindowsParams', - full_name='v5api.WindowsParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.WindowsParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='start', full_name='v5api.WindowsParams.start', index=1, - number=2, type=16, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='end', full_name='v5api.WindowsParams.end', index=2, - number=3, type=16, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMajor', full_name='v5api.WindowsParams.versionMajor', index=3, - number=4, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='width', full_name='v5api.WindowsParams.width', index=4, - number=5, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='depth', full_name='v5api.WindowsParams.depth', index=5, - number=6, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=474, - serialized_end=583, -) - - -_WINDOWSRESPONSE = _descriptor.Descriptor( - name='WindowsResponse', - full_name='v5api.WindowsResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.WindowsResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMajor', full_name='v5api.WindowsResponse.versionMajor', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMinor', full_name='v5api.WindowsResponse.versionMinor', index=2, - number=3, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='values', full_name='v5api.WindowsResponse.values', index=3, - number=4, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=585, - serialized_end=709, -) - - -_STREAMINFOPARAMS = _descriptor.Descriptor( - name='StreamInfoParams', - full_name='v5api.StreamInfoParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.StreamInfoParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='omitVersion', full_name='v5api.StreamInfoParams.omitVersion', index=1, - number=2, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='omitDescriptor', full_name='v5api.StreamInfoParams.omitDescriptor', index=2, - number=3, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='role', full_name='v5api.StreamInfoParams.role', index=3, - number=100, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=711, - serialized_end=815, -) - - -_STREAMINFORESPONSE = _descriptor.Descriptor( - name='StreamInfoResponse', - full_name='v5api.StreamInfoResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.StreamInfoResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMajor', full_name='v5api.StreamInfoResponse.versionMajor', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMinor', full_name='v5api.StreamInfoResponse.versionMinor', index=2, - number=3, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='descriptor', full_name='v5api.StreamInfoResponse.descriptor', index=3, - number=4, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=818, - serialized_end=956, -) - - -_STREAMDESCRIPTOR = _descriptor.Descriptor( - name='StreamDescriptor', - full_name='v5api.StreamDescriptor', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.StreamDescriptor.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='collection', full_name='v5api.StreamDescriptor.collection', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='tags', full_name='v5api.StreamDescriptor.tags', index=2, - number=3, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='annotations', full_name='v5api.StreamDescriptor.annotations', index=3, - number=4, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='propertyVersion', full_name='v5api.StreamDescriptor.propertyVersion', index=4, - number=5, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=959, - serialized_end=1111, -) - - -_SETSTREAMANNOTATIONSPARAMS = _descriptor.Descriptor( - name='SetStreamAnnotationsParams', - full_name='v5api.SetStreamAnnotationsParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.SetStreamAnnotationsParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='expectedPropertyVersion', full_name='v5api.SetStreamAnnotationsParams.expectedPropertyVersion', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='changes', full_name='v5api.SetStreamAnnotationsParams.changes', index=2, - number=3, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='removals', full_name='v5api.SetStreamAnnotationsParams.removals', index=3, - number=4, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1114, - serialized_end=1244, -) - - -_SETSTREAMANNOTATIONSRESPONSE = _descriptor.Descriptor( - name='SetStreamAnnotationsResponse', - full_name='v5api.SetStreamAnnotationsResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.SetStreamAnnotationsResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1246, - serialized_end=1305, -) - - -_SETSTREAMTAGSPARAMS = _descriptor.Descriptor( - name='SetStreamTagsParams', - full_name='v5api.SetStreamTagsParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.SetStreamTagsParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='expectedPropertyVersion', full_name='v5api.SetStreamTagsParams.expectedPropertyVersion', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='tags', full_name='v5api.SetStreamTagsParams.tags', index=2, - number=3, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='collection', full_name='v5api.SetStreamTagsParams.collection', index=3, - number=4, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='remove', full_name='v5api.SetStreamTagsParams.remove', index=4, - number=5, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1308, - serialized_end=1446, -) - - -_SETSTREAMTAGSRESPONSE = _descriptor.Descriptor( - name='SetStreamTagsResponse', - full_name='v5api.SetStreamTagsResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.SetStreamTagsResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1448, - serialized_end=1500, -) - - -_CREATEPARAMS = _descriptor.Descriptor( - name='CreateParams', - full_name='v5api.CreateParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.CreateParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='collection', full_name='v5api.CreateParams.collection', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='tags', full_name='v5api.CreateParams.tags', index=2, - number=3, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='annotations', full_name='v5api.CreateParams.annotations', index=3, - number=4, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1502, - serialized_end=1625, -) - - -_CREATERESPONSE = _descriptor.Descriptor( - name='CreateResponse', - full_name='v5api.CreateResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.CreateResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1627, - serialized_end=1672, -) - - -_METADATAUSAGEPARAMS = _descriptor.Descriptor( - name='MetadataUsageParams', - full_name='v5api.MetadataUsageParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='prefix', full_name='v5api.MetadataUsageParams.prefix', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='role', full_name='v5api.MetadataUsageParams.role', index=1, - number=100, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1674, - serialized_end=1738, -) - - -_METADATAUSAGERESPONSE = _descriptor.Descriptor( - name='MetadataUsageResponse', - full_name='v5api.MetadataUsageResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.MetadataUsageResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='tags', full_name='v5api.MetadataUsageResponse.tags', index=1, - number=2, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='annotations', full_name='v5api.MetadataUsageResponse.annotations', index=2, - number=3, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1740, - serialized_end=1861, -) - - -_KEYCOUNT = _descriptor.Descriptor( - name='KeyCount', - full_name='v5api.KeyCount', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='key', full_name='v5api.KeyCount.key', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='count', full_name='v5api.KeyCount.count', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1863, - serialized_end=1901, -) - - -_LISTCOLLECTIONSPARAMS = _descriptor.Descriptor( - name='ListCollectionsParams', - full_name='v5api.ListCollectionsParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='prefix', full_name='v5api.ListCollectionsParams.prefix', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='role', full_name='v5api.ListCollectionsParams.role', index=1, - number=100, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1903, - serialized_end=1969, -) - - -_LISTCOLLECTIONSRESPONSE = _descriptor.Descriptor( - name='ListCollectionsResponse', - full_name='v5api.ListCollectionsResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.ListCollectionsResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='collections', full_name='v5api.ListCollectionsResponse.collections', index=1, - number=2, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1971, - serialized_end=2046, -) - - -_LOOKUPSTREAMSPARAMS = _descriptor.Descriptor( - name='LookupStreamsParams', - full_name='v5api.LookupStreamsParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='collection', full_name='v5api.LookupStreamsParams.collection', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='isCollectionPrefix', full_name='v5api.LookupStreamsParams.isCollectionPrefix', index=1, - number=2, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='tags', full_name='v5api.LookupStreamsParams.tags', index=2, - number=3, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='annotations', full_name='v5api.LookupStreamsParams.annotations', index=3, - number=4, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='role', full_name='v5api.LookupStreamsParams.role', index=4, - number=100, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=2049, - serialized_end=2220, -) - - -_LOOKUPSTREAMSRESPONSE = _descriptor.Descriptor( - name='LookupStreamsResponse', - full_name='v5api.LookupStreamsResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.LookupStreamsResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='results', full_name='v5api.LookupStreamsResponse.results', index=1, - number=2, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=2222, - serialized_end=2316, -) - - -_NEARESTPARAMS = _descriptor.Descriptor( - name='NearestParams', - full_name='v5api.NearestParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.NearestParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='time', full_name='v5api.NearestParams.time', index=1, - number=2, type=16, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMajor', full_name='v5api.NearestParams.versionMajor', index=2, - number=3, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='backward', full_name='v5api.NearestParams.backward', index=3, - number=4, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=2318, - serialized_end=2401, -) - - -_NEARESTRESPONSE = _descriptor.Descriptor( - name='NearestResponse', - full_name='v5api.NearestResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.NearestResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMajor', full_name='v5api.NearestResponse.versionMajor', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMinor', full_name='v5api.NearestResponse.versionMinor', index=2, - number=3, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='value', full_name='v5api.NearestResponse.value', index=3, - number=4, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=2403, - serialized_end=2525, -) - - -_CHANGESPARAMS = _descriptor.Descriptor( - name='ChangesParams', - full_name='v5api.ChangesParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.ChangesParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='fromMajor', full_name='v5api.ChangesParams.fromMajor', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='toMajor', full_name='v5api.ChangesParams.toMajor', index=2, - number=3, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='resolution', full_name='v5api.ChangesParams.resolution', index=3, - number=4, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=2527, - serialized_end=2612, -) - - -_CHANGESRESPONSE = _descriptor.Descriptor( - name='ChangesResponse', - full_name='v5api.ChangesResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.ChangesResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMajor', full_name='v5api.ChangesResponse.versionMajor', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMinor', full_name='v5api.ChangesResponse.versionMinor', index=2, - number=3, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='ranges', full_name='v5api.ChangesResponse.ranges', index=3, - number=4, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=2614, - serialized_end=2741, -) - - -_INSERTPARAMS = _descriptor.Descriptor( - name='InsertParams', - full_name='v5api.InsertParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.InsertParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='sync', full_name='v5api.InsertParams.sync', index=1, - number=2, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='merge_policy', full_name='v5api.InsertParams.merge_policy', index=2, - number=4, type=14, cpp_type=8, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='values', full_name='v5api.InsertParams.values', index=3, - number=3, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=2743, - serialized_end=2860, -) - - -_INSERTRESPONSE = _descriptor.Descriptor( - name='InsertResponse', - full_name='v5api.InsertResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.InsertResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMajor', full_name='v5api.InsertResponse.versionMajor', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMinor', full_name='v5api.InsertResponse.versionMinor', index=2, - number=3, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=2862, - serialized_end=2951, -) - - -_DELETEPARAMS = _descriptor.Descriptor( - name='DeleteParams', - full_name='v5api.DeleteParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.DeleteParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='start', full_name='v5api.DeleteParams.start', index=1, - number=2, type=16, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='end', full_name='v5api.DeleteParams.end', index=2, - number=3, type=16, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=2953, - serialized_end=3009, -) - - -_DELETERESPONSE = _descriptor.Descriptor( - name='DeleteResponse', - full_name='v5api.DeleteResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.DeleteResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMajor', full_name='v5api.DeleteResponse.versionMajor', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMinor', full_name='v5api.DeleteResponse.versionMinor', index=2, - number=3, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3011, - serialized_end=3100, -) - - -_INFOPARAMS = _descriptor.Descriptor( - name='InfoParams', - full_name='v5api.InfoParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3102, - serialized_end=3114, -) - - -_INFORESPONSE = _descriptor.Descriptor( - name='InfoResponse', - full_name='v5api.InfoResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.InfoResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='mash', full_name='v5api.InfoResponse.mash', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='majorVersion', full_name='v5api.InfoResponse.majorVersion', index=2, - number=3, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='minorVersion', full_name='v5api.InfoResponse.minorVersion', index=3, - number=4, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='build', full_name='v5api.InfoResponse.build', index=4, - number=5, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='proxy', full_name='v5api.InfoResponse.proxy', index=5, - number=6, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3117, - serialized_end=3279, -) - - -_PROXYINFO = _descriptor.Descriptor( - name='ProxyInfo', - full_name='v5api.ProxyInfo', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='proxyEndpoints', full_name='v5api.ProxyInfo.proxyEndpoints', index=0, - number=1, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3281, - serialized_end=3316, -) - - -_FAULTINJECTPARAMS = _descriptor.Descriptor( - name='FaultInjectParams', - full_name='v5api.FaultInjectParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='type', full_name='v5api.FaultInjectParams.type', index=0, - number=1, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='params', full_name='v5api.FaultInjectParams.params', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3318, - serialized_end=3367, -) - - -_FAULTINJECTRESPONSE = _descriptor.Descriptor( - name='FaultInjectResponse', - full_name='v5api.FaultInjectResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.FaultInjectResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='rv', full_name='v5api.FaultInjectResponse.rv', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3369, - serialized_end=3431, -) - - -_FLUSHPARAMS = _descriptor.Descriptor( - name='FlushParams', - full_name='v5api.FlushParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.FlushParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3433, - serialized_end=3460, -) - - -_FLUSHRESPONSE = _descriptor.Descriptor( - name='FlushResponse', - full_name='v5api.FlushResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.FlushResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMajor', full_name='v5api.FlushResponse.versionMajor', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMinor', full_name='v5api.FlushResponse.versionMinor', index=2, - number=3, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3462, - serialized_end=3550, -) - - -_OBLITERATEPARAMS = _descriptor.Descriptor( - name='ObliterateParams', - full_name='v5api.ObliterateParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.ObliterateParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3552, - serialized_end=3584, -) - - -_OBLITERATERESPONSE = _descriptor.Descriptor( - name='ObliterateResponse', - full_name='v5api.ObliterateResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.ObliterateResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3586, - serialized_end=3635, -) - - -_RAWPOINT = _descriptor.Descriptor( - name='RawPoint', - full_name='v5api.RawPoint', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='time', full_name='v5api.RawPoint.time', index=0, - number=1, type=16, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='value', full_name='v5api.RawPoint.value', index=1, - number=2, type=1, cpp_type=5, label=1, - has_default_value=False, default_value=float(0), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3637, - serialized_end=3676, -) - - -_STATPOINT = _descriptor.Descriptor( - name='StatPoint', - full_name='v5api.StatPoint', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='time', full_name='v5api.StatPoint.time', index=0, - number=1, type=16, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='min', full_name='v5api.StatPoint.min', index=1, - number=2, type=1, cpp_type=5, label=1, - has_default_value=False, default_value=float(0), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='mean', full_name='v5api.StatPoint.mean', index=2, - number=3, type=1, cpp_type=5, label=1, - has_default_value=False, default_value=float(0), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='max', full_name='v5api.StatPoint.max', index=3, - number=4, type=1, cpp_type=5, label=1, - has_default_value=False, default_value=float(0), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='count', full_name='v5api.StatPoint.count', index=4, - number=5, type=6, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='stddev', full_name='v5api.StatPoint.stddev', index=5, - number=6, type=1, cpp_type=5, label=1, - has_default_value=False, default_value=float(0), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3678, - serialized_end=3774, -) - - -_CHANGEDRANGE = _descriptor.Descriptor( - name='ChangedRange', - full_name='v5api.ChangedRange', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='start', full_name='v5api.ChangedRange.start', index=0, - number=1, type=16, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='end', full_name='v5api.ChangedRange.end', index=1, - number=2, type=16, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3776, - serialized_end=3818, -) - - -_STATUS = _descriptor.Descriptor( - name='Status', - full_name='v5api.Status', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='code', full_name='v5api.Status.code', index=0, - number=1, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='msg', full_name='v5api.Status.msg', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='mash', full_name='v5api.Status.mash', index=2, - number=3, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3820, - serialized_end=3882, -) - - -_MASH = _descriptor.Descriptor( - name='Mash', - full_name='v5api.Mash', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='revision', full_name='v5api.Mash.revision', index=0, - number=1, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='leader', full_name='v5api.Mash.leader', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='leaderRevision', full_name='v5api.Mash.leaderRevision', index=2, - number=3, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='totalWeight', full_name='v5api.Mash.totalWeight', index=3, - number=4, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='healthy', full_name='v5api.Mash.healthy', index=4, - number=5, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='unmapped', full_name='v5api.Mash.unmapped', index=5, - number=6, type=1, cpp_type=5, label=1, - has_default_value=False, default_value=float(0), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='members', full_name='v5api.Mash.members', index=6, - number=7, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3885, - serialized_end=4037, -) - - -_MEMBER = _descriptor.Descriptor( - name='Member', - full_name='v5api.Member', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='hash', full_name='v5api.Member.hash', index=0, - number=1, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='nodename', full_name='v5api.Member.nodename', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='up', full_name='v5api.Member.up', index=2, - number=3, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='in', full_name='v5api.Member.in', index=3, - number=4, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='enabled', full_name='v5api.Member.enabled', index=4, - number=5, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='start', full_name='v5api.Member.start', index=5, - number=6, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='end', full_name='v5api.Member.end', index=6, - number=7, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='weight', full_name='v5api.Member.weight', index=7, - number=8, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='readPreference', full_name='v5api.Member.readPreference', index=8, - number=9, type=1, cpp_type=5, label=1, - has_default_value=False, default_value=float(0), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='httpEndpoints', full_name='v5api.Member.httpEndpoints', index=9, - number=10, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='grpcEndpoints', full_name='v5api.Member.grpcEndpoints', index=10, - number=11, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=4040, - serialized_end=4235, -) - - -_KEYOPTVALUE = _descriptor.Descriptor( - name='KeyOptValue', - full_name='v5api.KeyOptValue', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='key', full_name='v5api.KeyOptValue.key', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='val', full_name='v5api.KeyOptValue.val', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=4237, - serialized_end=4293, -) - - -_OPTVALUE = _descriptor.Descriptor( - name='OptValue', - full_name='v5api.OptValue', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='value', full_name='v5api.OptValue.value', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=4295, - serialized_end=4320, -) - - -_KEYVALUE = _descriptor.Descriptor( - name='KeyValue', - full_name='v5api.KeyValue', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='key', full_name='v5api.KeyValue.key', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='value', full_name='v5api.KeyValue.value', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=4322, - serialized_end=4360, -) - - -_STREAMCSVCONFIG = _descriptor.Descriptor( - name='StreamCSVConfig', - full_name='v5api.StreamCSVConfig', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='version', full_name='v5api.StreamCSVConfig.version', index=0, - number=1, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='label', full_name='v5api.StreamCSVConfig.label', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.StreamCSVConfig.uuid', index=2, - number=3, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=4362, - serialized_end=4425, -) - - -_GENERATECSVPARAMS = _descriptor.Descriptor( - name='GenerateCSVParams', - full_name='v5api.GenerateCSVParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='queryType', full_name='v5api.GenerateCSVParams.queryType', index=0, - number=1, type=14, cpp_type=8, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='startTime', full_name='v5api.GenerateCSVParams.startTime', index=1, - number=2, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='endTime', full_name='v5api.GenerateCSVParams.endTime', index=2, - number=3, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='windowSize', full_name='v5api.GenerateCSVParams.windowSize', index=3, - number=4, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='depth', full_name='v5api.GenerateCSVParams.depth', index=4, - number=5, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='includeVersions', full_name='v5api.GenerateCSVParams.includeVersions', index=5, - number=6, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='streams', full_name='v5api.GenerateCSVParams.streams', index=6, - number=7, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - _GENERATECSVPARAMS_QUERYTYPE, - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=4428, - serialized_end=4713, -) - - -_GENERATECSVRESPONSE = _descriptor.Descriptor( - name='GenerateCSVResponse', - full_name='v5api.GenerateCSVResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.GenerateCSVResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='isHeader', full_name='v5api.GenerateCSVResponse.isHeader', index=1, - number=2, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='row', full_name='v5api.GenerateCSVResponse.row', index=2, - number=3, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=4715, - serialized_end=4796, -) - - -_SQLQUERYPARAMS = _descriptor.Descriptor( - name='SQLQueryParams', - full_name='v5api.SQLQueryParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='query', full_name='v5api.SQLQueryParams.query', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='params', full_name='v5api.SQLQueryParams.params', index=1, - number=2, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='role', full_name='v5api.SQLQueryParams.role', index=2, - number=100, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=4798, - serialized_end=4872, -) - - -_SQLQUERYRESPONSE = _descriptor.Descriptor( - name='SQLQueryResponse', - full_name='v5api.SQLQueryResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.SQLQueryResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='SQLQueryRow', full_name='v5api.SQLQueryResponse.SQLQueryRow', index=1, - number=2, type=12, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=4874, - serialized_end=4942, -) - - -_ROLE = _descriptor.Descriptor( - name='Role', - full_name='v5api.Role', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='v5api.Role.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=4944, - serialized_end=4964, -) - - -_SETCOMPACTIONCONFIGPARAMS = _descriptor.Descriptor( - name='SetCompactionConfigParams', - full_name='v5api.SetCompactionConfigParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.SetCompactionConfigParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='CompactedVersion', full_name='v5api.SetCompactionConfigParams.CompactedVersion', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='reducedResolutionRanges', full_name='v5api.SetCompactionConfigParams.reducedResolutionRanges', index=2, - number=3, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='unused0', full_name='v5api.SetCompactionConfigParams.unused0', index=3, - number=4, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=4967, - serialized_end=5115, -) - - -_SETCOMPACTIONCONFIGRESPONSE = _descriptor.Descriptor( - name='SetCompactionConfigResponse', - full_name='v5api.SetCompactionConfigResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.SetCompactionConfigResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=5117, - serialized_end=5175, -) - - -_GETCOMPACTIONCONFIGPARAMS = _descriptor.Descriptor( - name='GetCompactionConfigParams', - full_name='v5api.GetCompactionConfigParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.GetCompactionConfigParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=5177, - serialized_end=5218, -) - - -_GETCOMPACTIONCONFIGRESPONSE = _descriptor.Descriptor( - name='GetCompactionConfigResponse', - full_name='v5api.GetCompactionConfigResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.GetCompactionConfigResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='LatestMajorVersion', full_name='v5api.GetCompactionConfigResponse.LatestMajorVersion', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='CompactedVersion', full_name='v5api.GetCompactionConfigResponse.CompactedVersion', index=2, - number=3, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='reducedResolutionRanges', full_name='v5api.GetCompactionConfigResponse.reducedResolutionRanges', index=3, - number=4, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='unused0', full_name='v5api.GetCompactionConfigResponse.unused0', index=4, - number=5, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=5221, - serialized_end=5414, -) - - -_REDUCEDRESOLUTIONRANGE = _descriptor.Descriptor( - name='ReducedResolutionRange', - full_name='v5api.ReducedResolutionRange', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='Start', full_name='v5api.ReducedResolutionRange.Start', index=0, - number=1, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='End', full_name='v5api.ReducedResolutionRange.End', index=1, - number=2, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='Resolution', full_name='v5api.ReducedResolutionRange.Resolution', index=2, - number=3, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=5416, - serialized_end=5488, -) - -_RAWVALUESRESPONSE.fields_by_name['stat'].message_type = _STATUS -_RAWVALUESRESPONSE.fields_by_name['values'].message_type = _RAWPOINT -_ALIGNEDWINDOWSRESPONSE.fields_by_name['stat'].message_type = _STATUS -_ALIGNEDWINDOWSRESPONSE.fields_by_name['values'].message_type = _STATPOINT -_WINDOWSRESPONSE.fields_by_name['stat'].message_type = _STATUS -_WINDOWSRESPONSE.fields_by_name['values'].message_type = _STATPOINT -_STREAMINFOPARAMS.fields_by_name['role'].message_type = _ROLE -_STREAMINFORESPONSE.fields_by_name['stat'].message_type = _STATUS -_STREAMINFORESPONSE.fields_by_name['descriptor'].message_type = _STREAMDESCRIPTOR -_STREAMDESCRIPTOR.fields_by_name['tags'].message_type = _KEYOPTVALUE -_STREAMDESCRIPTOR.fields_by_name['annotations'].message_type = _KEYOPTVALUE -_SETSTREAMANNOTATIONSPARAMS.fields_by_name['changes'].message_type = _KEYOPTVALUE -_SETSTREAMANNOTATIONSRESPONSE.fields_by_name['stat'].message_type = _STATUS -_SETSTREAMTAGSPARAMS.fields_by_name['tags'].message_type = _KEYOPTVALUE -_SETSTREAMTAGSRESPONSE.fields_by_name['stat'].message_type = _STATUS -_CREATEPARAMS.fields_by_name['tags'].message_type = _KEYOPTVALUE -_CREATEPARAMS.fields_by_name['annotations'].message_type = _KEYOPTVALUE -_CREATERESPONSE.fields_by_name['stat'].message_type = _STATUS -_METADATAUSAGEPARAMS.fields_by_name['role'].message_type = _ROLE -_METADATAUSAGERESPONSE.fields_by_name['stat'].message_type = _STATUS -_METADATAUSAGERESPONSE.fields_by_name['tags'].message_type = _KEYCOUNT -_METADATAUSAGERESPONSE.fields_by_name['annotations'].message_type = _KEYCOUNT -_LISTCOLLECTIONSPARAMS.fields_by_name['role'].message_type = _ROLE -_LISTCOLLECTIONSRESPONSE.fields_by_name['stat'].message_type = _STATUS -_LOOKUPSTREAMSPARAMS.fields_by_name['tags'].message_type = _KEYOPTVALUE -_LOOKUPSTREAMSPARAMS.fields_by_name['annotations'].message_type = _KEYOPTVALUE -_LOOKUPSTREAMSPARAMS.fields_by_name['role'].message_type = _ROLE -_LOOKUPSTREAMSRESPONSE.fields_by_name['stat'].message_type = _STATUS -_LOOKUPSTREAMSRESPONSE.fields_by_name['results'].message_type = _STREAMDESCRIPTOR -_NEARESTRESPONSE.fields_by_name['stat'].message_type = _STATUS -_NEARESTRESPONSE.fields_by_name['value'].message_type = _RAWPOINT -_CHANGESRESPONSE.fields_by_name['stat'].message_type = _STATUS -_CHANGESRESPONSE.fields_by_name['ranges'].message_type = _CHANGEDRANGE -_INSERTPARAMS.fields_by_name['merge_policy'].enum_type = _MERGEPOLICY -_INSERTPARAMS.fields_by_name['values'].message_type = _RAWPOINT -_INSERTRESPONSE.fields_by_name['stat'].message_type = _STATUS -_DELETERESPONSE.fields_by_name['stat'].message_type = _STATUS -_INFORESPONSE.fields_by_name['stat'].message_type = _STATUS -_INFORESPONSE.fields_by_name['mash'].message_type = _MASH -_INFORESPONSE.fields_by_name['proxy'].message_type = _PROXYINFO -_FAULTINJECTRESPONSE.fields_by_name['stat'].message_type = _STATUS -_FLUSHRESPONSE.fields_by_name['stat'].message_type = _STATUS -_OBLITERATERESPONSE.fields_by_name['stat'].message_type = _STATUS -_STATUS.fields_by_name['mash'].message_type = _MASH -_MASH.fields_by_name['members'].message_type = _MEMBER -_KEYOPTVALUE.fields_by_name['val'].message_type = _OPTVALUE -_GENERATECSVPARAMS.fields_by_name['queryType'].enum_type = _GENERATECSVPARAMS_QUERYTYPE -_GENERATECSVPARAMS.fields_by_name['streams'].message_type = _STREAMCSVCONFIG -_GENERATECSVPARAMS_QUERYTYPE.containing_type = _GENERATECSVPARAMS -_GENERATECSVRESPONSE.fields_by_name['stat'].message_type = _STATUS -_SQLQUERYPARAMS.fields_by_name['role'].message_type = _ROLE -_SQLQUERYRESPONSE.fields_by_name['stat'].message_type = _STATUS -_SETCOMPACTIONCONFIGPARAMS.fields_by_name['reducedResolutionRanges'].message_type = _REDUCEDRESOLUTIONRANGE -_SETCOMPACTIONCONFIGRESPONSE.fields_by_name['stat'].message_type = _STATUS -_GETCOMPACTIONCONFIGRESPONSE.fields_by_name['stat'].message_type = _STATUS -_GETCOMPACTIONCONFIGRESPONSE.fields_by_name['reducedResolutionRanges'].message_type = _REDUCEDRESOLUTIONRANGE -DESCRIPTOR.message_types_by_name['RawValuesParams'] = _RAWVALUESPARAMS -DESCRIPTOR.message_types_by_name['RawValuesResponse'] = _RAWVALUESRESPONSE -DESCRIPTOR.message_types_by_name['AlignedWindowsParams'] = _ALIGNEDWINDOWSPARAMS -DESCRIPTOR.message_types_by_name['AlignedWindowsResponse'] = _ALIGNEDWINDOWSRESPONSE -DESCRIPTOR.message_types_by_name['WindowsParams'] = _WINDOWSPARAMS -DESCRIPTOR.message_types_by_name['WindowsResponse'] = _WINDOWSRESPONSE -DESCRIPTOR.message_types_by_name['StreamInfoParams'] = _STREAMINFOPARAMS -DESCRIPTOR.message_types_by_name['StreamInfoResponse'] = _STREAMINFORESPONSE -DESCRIPTOR.message_types_by_name['StreamDescriptor'] = _STREAMDESCRIPTOR -DESCRIPTOR.message_types_by_name['SetStreamAnnotationsParams'] = _SETSTREAMANNOTATIONSPARAMS -DESCRIPTOR.message_types_by_name['SetStreamAnnotationsResponse'] = _SETSTREAMANNOTATIONSRESPONSE -DESCRIPTOR.message_types_by_name['SetStreamTagsParams'] = _SETSTREAMTAGSPARAMS -DESCRIPTOR.message_types_by_name['SetStreamTagsResponse'] = _SETSTREAMTAGSRESPONSE -DESCRIPTOR.message_types_by_name['CreateParams'] = _CREATEPARAMS -DESCRIPTOR.message_types_by_name['CreateResponse'] = _CREATERESPONSE -DESCRIPTOR.message_types_by_name['MetadataUsageParams'] = _METADATAUSAGEPARAMS -DESCRIPTOR.message_types_by_name['MetadataUsageResponse'] = _METADATAUSAGERESPONSE -DESCRIPTOR.message_types_by_name['KeyCount'] = _KEYCOUNT -DESCRIPTOR.message_types_by_name['ListCollectionsParams'] = _LISTCOLLECTIONSPARAMS -DESCRIPTOR.message_types_by_name['ListCollectionsResponse'] = _LISTCOLLECTIONSRESPONSE -DESCRIPTOR.message_types_by_name['LookupStreamsParams'] = _LOOKUPSTREAMSPARAMS -DESCRIPTOR.message_types_by_name['LookupStreamsResponse'] = _LOOKUPSTREAMSRESPONSE -DESCRIPTOR.message_types_by_name['NearestParams'] = _NEARESTPARAMS -DESCRIPTOR.message_types_by_name['NearestResponse'] = _NEARESTRESPONSE -DESCRIPTOR.message_types_by_name['ChangesParams'] = _CHANGESPARAMS -DESCRIPTOR.message_types_by_name['ChangesResponse'] = _CHANGESRESPONSE -DESCRIPTOR.message_types_by_name['InsertParams'] = _INSERTPARAMS -DESCRIPTOR.message_types_by_name['InsertResponse'] = _INSERTRESPONSE -DESCRIPTOR.message_types_by_name['DeleteParams'] = _DELETEPARAMS -DESCRIPTOR.message_types_by_name['DeleteResponse'] = _DELETERESPONSE -DESCRIPTOR.message_types_by_name['InfoParams'] = _INFOPARAMS -DESCRIPTOR.message_types_by_name['InfoResponse'] = _INFORESPONSE -DESCRIPTOR.message_types_by_name['ProxyInfo'] = _PROXYINFO -DESCRIPTOR.message_types_by_name['FaultInjectParams'] = _FAULTINJECTPARAMS -DESCRIPTOR.message_types_by_name['FaultInjectResponse'] = _FAULTINJECTRESPONSE -DESCRIPTOR.message_types_by_name['FlushParams'] = _FLUSHPARAMS -DESCRIPTOR.message_types_by_name['FlushResponse'] = _FLUSHRESPONSE -DESCRIPTOR.message_types_by_name['ObliterateParams'] = _OBLITERATEPARAMS -DESCRIPTOR.message_types_by_name['ObliterateResponse'] = _OBLITERATERESPONSE -DESCRIPTOR.message_types_by_name['RawPoint'] = _RAWPOINT -DESCRIPTOR.message_types_by_name['StatPoint'] = _STATPOINT -DESCRIPTOR.message_types_by_name['ChangedRange'] = _CHANGEDRANGE -DESCRIPTOR.message_types_by_name['Status'] = _STATUS -DESCRIPTOR.message_types_by_name['Mash'] = _MASH -DESCRIPTOR.message_types_by_name['Member'] = _MEMBER -DESCRIPTOR.message_types_by_name['KeyOptValue'] = _KEYOPTVALUE -DESCRIPTOR.message_types_by_name['OptValue'] = _OPTVALUE -DESCRIPTOR.message_types_by_name['KeyValue'] = _KEYVALUE -DESCRIPTOR.message_types_by_name['StreamCSVConfig'] = _STREAMCSVCONFIG -DESCRIPTOR.message_types_by_name['GenerateCSVParams'] = _GENERATECSVPARAMS -DESCRIPTOR.message_types_by_name['GenerateCSVResponse'] = _GENERATECSVRESPONSE -DESCRIPTOR.message_types_by_name['SQLQueryParams'] = _SQLQUERYPARAMS -DESCRIPTOR.message_types_by_name['SQLQueryResponse'] = _SQLQUERYRESPONSE -DESCRIPTOR.message_types_by_name['Role'] = _ROLE -DESCRIPTOR.message_types_by_name['SetCompactionConfigParams'] = _SETCOMPACTIONCONFIGPARAMS -DESCRIPTOR.message_types_by_name['SetCompactionConfigResponse'] = _SETCOMPACTIONCONFIGRESPONSE -DESCRIPTOR.message_types_by_name['GetCompactionConfigParams'] = _GETCOMPACTIONCONFIGPARAMS -DESCRIPTOR.message_types_by_name['GetCompactionConfigResponse'] = _GETCOMPACTIONCONFIGRESPONSE -DESCRIPTOR.message_types_by_name['ReducedResolutionRange'] = _REDUCEDRESOLUTIONRANGE -DESCRIPTOR.enum_types_by_name['MergePolicy'] = _MERGEPOLICY -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -RawValuesParams = _reflection.GeneratedProtocolMessageType('RawValuesParams', (_message.Message,), dict( - DESCRIPTOR = _RAWVALUESPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.RawValuesParams) - )) -_sym_db.RegisterMessage(RawValuesParams) - -RawValuesResponse = _reflection.GeneratedProtocolMessageType('RawValuesResponse', (_message.Message,), dict( - DESCRIPTOR = _RAWVALUESRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.RawValuesResponse) - )) -_sym_db.RegisterMessage(RawValuesResponse) - -AlignedWindowsParams = _reflection.GeneratedProtocolMessageType('AlignedWindowsParams', (_message.Message,), dict( - DESCRIPTOR = _ALIGNEDWINDOWSPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.AlignedWindowsParams) - )) -_sym_db.RegisterMessage(AlignedWindowsParams) - -AlignedWindowsResponse = _reflection.GeneratedProtocolMessageType('AlignedWindowsResponse', (_message.Message,), dict( - DESCRIPTOR = _ALIGNEDWINDOWSRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.AlignedWindowsResponse) - )) -_sym_db.RegisterMessage(AlignedWindowsResponse) - -WindowsParams = _reflection.GeneratedProtocolMessageType('WindowsParams', (_message.Message,), dict( - DESCRIPTOR = _WINDOWSPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.WindowsParams) - )) -_sym_db.RegisterMessage(WindowsParams) - -WindowsResponse = _reflection.GeneratedProtocolMessageType('WindowsResponse', (_message.Message,), dict( - DESCRIPTOR = _WINDOWSRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.WindowsResponse) - )) -_sym_db.RegisterMessage(WindowsResponse) - -StreamInfoParams = _reflection.GeneratedProtocolMessageType('StreamInfoParams', (_message.Message,), dict( - DESCRIPTOR = _STREAMINFOPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.StreamInfoParams) - )) -_sym_db.RegisterMessage(StreamInfoParams) - -StreamInfoResponse = _reflection.GeneratedProtocolMessageType('StreamInfoResponse', (_message.Message,), dict( - DESCRIPTOR = _STREAMINFORESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.StreamInfoResponse) - )) -_sym_db.RegisterMessage(StreamInfoResponse) - -StreamDescriptor = _reflection.GeneratedProtocolMessageType('StreamDescriptor', (_message.Message,), dict( - DESCRIPTOR = _STREAMDESCRIPTOR, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.StreamDescriptor) - )) -_sym_db.RegisterMessage(StreamDescriptor) - -SetStreamAnnotationsParams = _reflection.GeneratedProtocolMessageType('SetStreamAnnotationsParams', (_message.Message,), dict( - DESCRIPTOR = _SETSTREAMANNOTATIONSPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.SetStreamAnnotationsParams) - )) -_sym_db.RegisterMessage(SetStreamAnnotationsParams) - -SetStreamAnnotationsResponse = _reflection.GeneratedProtocolMessageType('SetStreamAnnotationsResponse', (_message.Message,), dict( - DESCRIPTOR = _SETSTREAMANNOTATIONSRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.SetStreamAnnotationsResponse) - )) -_sym_db.RegisterMessage(SetStreamAnnotationsResponse) - -SetStreamTagsParams = _reflection.GeneratedProtocolMessageType('SetStreamTagsParams', (_message.Message,), dict( - DESCRIPTOR = _SETSTREAMTAGSPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.SetStreamTagsParams) - )) -_sym_db.RegisterMessage(SetStreamTagsParams) - -SetStreamTagsResponse = _reflection.GeneratedProtocolMessageType('SetStreamTagsResponse', (_message.Message,), dict( - DESCRIPTOR = _SETSTREAMTAGSRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.SetStreamTagsResponse) - )) -_sym_db.RegisterMessage(SetStreamTagsResponse) - -CreateParams = _reflection.GeneratedProtocolMessageType('CreateParams', (_message.Message,), dict( - DESCRIPTOR = _CREATEPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.CreateParams) - )) -_sym_db.RegisterMessage(CreateParams) - -CreateResponse = _reflection.GeneratedProtocolMessageType('CreateResponse', (_message.Message,), dict( - DESCRIPTOR = _CREATERESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.CreateResponse) - )) -_sym_db.RegisterMessage(CreateResponse) - -MetadataUsageParams = _reflection.GeneratedProtocolMessageType('MetadataUsageParams', (_message.Message,), dict( - DESCRIPTOR = _METADATAUSAGEPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.MetadataUsageParams) - )) -_sym_db.RegisterMessage(MetadataUsageParams) - -MetadataUsageResponse = _reflection.GeneratedProtocolMessageType('MetadataUsageResponse', (_message.Message,), dict( - DESCRIPTOR = _METADATAUSAGERESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.MetadataUsageResponse) - )) -_sym_db.RegisterMessage(MetadataUsageResponse) - -KeyCount = _reflection.GeneratedProtocolMessageType('KeyCount', (_message.Message,), dict( - DESCRIPTOR = _KEYCOUNT, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.KeyCount) - )) -_sym_db.RegisterMessage(KeyCount) - -ListCollectionsParams = _reflection.GeneratedProtocolMessageType('ListCollectionsParams', (_message.Message,), dict( - DESCRIPTOR = _LISTCOLLECTIONSPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.ListCollectionsParams) - )) -_sym_db.RegisterMessage(ListCollectionsParams) - -ListCollectionsResponse = _reflection.GeneratedProtocolMessageType('ListCollectionsResponse', (_message.Message,), dict( - DESCRIPTOR = _LISTCOLLECTIONSRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.ListCollectionsResponse) - )) -_sym_db.RegisterMessage(ListCollectionsResponse) - -LookupStreamsParams = _reflection.GeneratedProtocolMessageType('LookupStreamsParams', (_message.Message,), dict( - DESCRIPTOR = _LOOKUPSTREAMSPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.LookupStreamsParams) - )) -_sym_db.RegisterMessage(LookupStreamsParams) - -LookupStreamsResponse = _reflection.GeneratedProtocolMessageType('LookupStreamsResponse', (_message.Message,), dict( - DESCRIPTOR = _LOOKUPSTREAMSRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.LookupStreamsResponse) - )) -_sym_db.RegisterMessage(LookupStreamsResponse) - -NearestParams = _reflection.GeneratedProtocolMessageType('NearestParams', (_message.Message,), dict( - DESCRIPTOR = _NEARESTPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.NearestParams) - )) -_sym_db.RegisterMessage(NearestParams) - -NearestResponse = _reflection.GeneratedProtocolMessageType('NearestResponse', (_message.Message,), dict( - DESCRIPTOR = _NEARESTRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.NearestResponse) - )) -_sym_db.RegisterMessage(NearestResponse) - -ChangesParams = _reflection.GeneratedProtocolMessageType('ChangesParams', (_message.Message,), dict( - DESCRIPTOR = _CHANGESPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.ChangesParams) - )) -_sym_db.RegisterMessage(ChangesParams) - -ChangesResponse = _reflection.GeneratedProtocolMessageType('ChangesResponse', (_message.Message,), dict( - DESCRIPTOR = _CHANGESRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.ChangesResponse) - )) -_sym_db.RegisterMessage(ChangesResponse) - -InsertParams = _reflection.GeneratedProtocolMessageType('InsertParams', (_message.Message,), dict( - DESCRIPTOR = _INSERTPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.InsertParams) - )) -_sym_db.RegisterMessage(InsertParams) - -InsertResponse = _reflection.GeneratedProtocolMessageType('InsertResponse', (_message.Message,), dict( - DESCRIPTOR = _INSERTRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.InsertResponse) - )) -_sym_db.RegisterMessage(InsertResponse) - -DeleteParams = _reflection.GeneratedProtocolMessageType('DeleteParams', (_message.Message,), dict( - DESCRIPTOR = _DELETEPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.DeleteParams) - )) -_sym_db.RegisterMessage(DeleteParams) - -DeleteResponse = _reflection.GeneratedProtocolMessageType('DeleteResponse', (_message.Message,), dict( - DESCRIPTOR = _DELETERESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.DeleteResponse) - )) -_sym_db.RegisterMessage(DeleteResponse) - -InfoParams = _reflection.GeneratedProtocolMessageType('InfoParams', (_message.Message,), dict( - DESCRIPTOR = _INFOPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.InfoParams) - )) -_sym_db.RegisterMessage(InfoParams) - -InfoResponse = _reflection.GeneratedProtocolMessageType('InfoResponse', (_message.Message,), dict( - DESCRIPTOR = _INFORESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.InfoResponse) - )) -_sym_db.RegisterMessage(InfoResponse) - -ProxyInfo = _reflection.GeneratedProtocolMessageType('ProxyInfo', (_message.Message,), dict( - DESCRIPTOR = _PROXYINFO, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.ProxyInfo) - )) -_sym_db.RegisterMessage(ProxyInfo) - -FaultInjectParams = _reflection.GeneratedProtocolMessageType('FaultInjectParams', (_message.Message,), dict( - DESCRIPTOR = _FAULTINJECTPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.FaultInjectParams) - )) -_sym_db.RegisterMessage(FaultInjectParams) - -FaultInjectResponse = _reflection.GeneratedProtocolMessageType('FaultInjectResponse', (_message.Message,), dict( - DESCRIPTOR = _FAULTINJECTRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.FaultInjectResponse) - )) -_sym_db.RegisterMessage(FaultInjectResponse) - -FlushParams = _reflection.GeneratedProtocolMessageType('FlushParams', (_message.Message,), dict( - DESCRIPTOR = _FLUSHPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.FlushParams) - )) -_sym_db.RegisterMessage(FlushParams) - -FlushResponse = _reflection.GeneratedProtocolMessageType('FlushResponse', (_message.Message,), dict( - DESCRIPTOR = _FLUSHRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.FlushResponse) - )) -_sym_db.RegisterMessage(FlushResponse) - -ObliterateParams = _reflection.GeneratedProtocolMessageType('ObliterateParams', (_message.Message,), dict( - DESCRIPTOR = _OBLITERATEPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.ObliterateParams) - )) -_sym_db.RegisterMessage(ObliterateParams) - -ObliterateResponse = _reflection.GeneratedProtocolMessageType('ObliterateResponse', (_message.Message,), dict( - DESCRIPTOR = _OBLITERATERESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.ObliterateResponse) - )) -_sym_db.RegisterMessage(ObliterateResponse) - -RawPoint = _reflection.GeneratedProtocolMessageType('RawPoint', (_message.Message,), dict( - DESCRIPTOR = _RAWPOINT, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.RawPoint) - )) -_sym_db.RegisterMessage(RawPoint) - -StatPoint = _reflection.GeneratedProtocolMessageType('StatPoint', (_message.Message,), dict( - DESCRIPTOR = _STATPOINT, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.StatPoint) - )) -_sym_db.RegisterMessage(StatPoint) - -ChangedRange = _reflection.GeneratedProtocolMessageType('ChangedRange', (_message.Message,), dict( - DESCRIPTOR = _CHANGEDRANGE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.ChangedRange) - )) -_sym_db.RegisterMessage(ChangedRange) - -Status = _reflection.GeneratedProtocolMessageType('Status', (_message.Message,), dict( - DESCRIPTOR = _STATUS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.Status) - )) -_sym_db.RegisterMessage(Status) - -Mash = _reflection.GeneratedProtocolMessageType('Mash', (_message.Message,), dict( - DESCRIPTOR = _MASH, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.Mash) - )) -_sym_db.RegisterMessage(Mash) - -Member = _reflection.GeneratedProtocolMessageType('Member', (_message.Message,), dict( - DESCRIPTOR = _MEMBER, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.Member) - )) -_sym_db.RegisterMessage(Member) - -KeyOptValue = _reflection.GeneratedProtocolMessageType('KeyOptValue', (_message.Message,), dict( - DESCRIPTOR = _KEYOPTVALUE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.KeyOptValue) - )) -_sym_db.RegisterMessage(KeyOptValue) - -OptValue = _reflection.GeneratedProtocolMessageType('OptValue', (_message.Message,), dict( - DESCRIPTOR = _OPTVALUE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.OptValue) - )) -_sym_db.RegisterMessage(OptValue) - -KeyValue = _reflection.GeneratedProtocolMessageType('KeyValue', (_message.Message,), dict( - DESCRIPTOR = _KEYVALUE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.KeyValue) - )) -_sym_db.RegisterMessage(KeyValue) - -StreamCSVConfig = _reflection.GeneratedProtocolMessageType('StreamCSVConfig', (_message.Message,), dict( - DESCRIPTOR = _STREAMCSVCONFIG, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.StreamCSVConfig) - )) -_sym_db.RegisterMessage(StreamCSVConfig) - -GenerateCSVParams = _reflection.GeneratedProtocolMessageType('GenerateCSVParams', (_message.Message,), dict( - DESCRIPTOR = _GENERATECSVPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.GenerateCSVParams) - )) -_sym_db.RegisterMessage(GenerateCSVParams) - -GenerateCSVResponse = _reflection.GeneratedProtocolMessageType('GenerateCSVResponse', (_message.Message,), dict( - DESCRIPTOR = _GENERATECSVRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.GenerateCSVResponse) - )) -_sym_db.RegisterMessage(GenerateCSVResponse) - -SQLQueryParams = _reflection.GeneratedProtocolMessageType('SQLQueryParams', (_message.Message,), dict( - DESCRIPTOR = _SQLQUERYPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.SQLQueryParams) - )) -_sym_db.RegisterMessage(SQLQueryParams) - -SQLQueryResponse = _reflection.GeneratedProtocolMessageType('SQLQueryResponse', (_message.Message,), dict( - DESCRIPTOR = _SQLQUERYRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.SQLQueryResponse) - )) -_sym_db.RegisterMessage(SQLQueryResponse) - -Role = _reflection.GeneratedProtocolMessageType('Role', (_message.Message,), dict( - DESCRIPTOR = _ROLE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.Role) - )) -_sym_db.RegisterMessage(Role) - -SetCompactionConfigParams = _reflection.GeneratedProtocolMessageType('SetCompactionConfigParams', (_message.Message,), dict( - DESCRIPTOR = _SETCOMPACTIONCONFIGPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.SetCompactionConfigParams) - )) -_sym_db.RegisterMessage(SetCompactionConfigParams) - -SetCompactionConfigResponse = _reflection.GeneratedProtocolMessageType('SetCompactionConfigResponse', (_message.Message,), dict( - DESCRIPTOR = _SETCOMPACTIONCONFIGRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.SetCompactionConfigResponse) - )) -_sym_db.RegisterMessage(SetCompactionConfigResponse) - -GetCompactionConfigParams = _reflection.GeneratedProtocolMessageType('GetCompactionConfigParams', (_message.Message,), dict( - DESCRIPTOR = _GETCOMPACTIONCONFIGPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.GetCompactionConfigParams) - )) -_sym_db.RegisterMessage(GetCompactionConfigParams) - -GetCompactionConfigResponse = _reflection.GeneratedProtocolMessageType('GetCompactionConfigResponse', (_message.Message,), dict( - DESCRIPTOR = _GETCOMPACTIONCONFIGRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.GetCompactionConfigResponse) - )) -_sym_db.RegisterMessage(GetCompactionConfigResponse) - -ReducedResolutionRange = _reflection.GeneratedProtocolMessageType('ReducedResolutionRange', (_message.Message,), dict( - DESCRIPTOR = _REDUCEDRESOLUTIONRANGE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.ReducedResolutionRange) - )) -_sym_db.RegisterMessage(ReducedResolutionRange) - - - -_BTRDB = _descriptor.ServiceDescriptor( - name='BTrDB', - full_name='v5api.BTrDB', - file=DESCRIPTOR, - index=0, - serialized_options=None, - serialized_start=5553, - serialized_end=6884, - methods=[ - _descriptor.MethodDescriptor( - name='RawValues', - full_name='v5api.BTrDB.RawValues', - index=0, - containing_service=None, - input_type=_RAWVALUESPARAMS, - output_type=_RAWVALUESRESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='AlignedWindows', - full_name='v5api.BTrDB.AlignedWindows', - index=1, - containing_service=None, - input_type=_ALIGNEDWINDOWSPARAMS, - output_type=_ALIGNEDWINDOWSRESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='Windows', - full_name='v5api.BTrDB.Windows', - index=2, - containing_service=None, - input_type=_WINDOWSPARAMS, - output_type=_WINDOWSRESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='StreamInfo', - full_name='v5api.BTrDB.StreamInfo', - index=3, - containing_service=None, - input_type=_STREAMINFOPARAMS, - output_type=_STREAMINFORESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='SetStreamAnnotations', - full_name='v5api.BTrDB.SetStreamAnnotations', - index=4, - containing_service=None, - input_type=_SETSTREAMANNOTATIONSPARAMS, - output_type=_SETSTREAMANNOTATIONSRESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='SetStreamTags', - full_name='v5api.BTrDB.SetStreamTags', - index=5, - containing_service=None, - input_type=_SETSTREAMTAGSPARAMS, - output_type=_SETSTREAMTAGSRESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='Create', - full_name='v5api.BTrDB.Create', - index=6, - containing_service=None, - input_type=_CREATEPARAMS, - output_type=_CREATERESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='ListCollections', - full_name='v5api.BTrDB.ListCollections', - index=7, - containing_service=None, - input_type=_LISTCOLLECTIONSPARAMS, - output_type=_LISTCOLLECTIONSRESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='LookupStreams', - full_name='v5api.BTrDB.LookupStreams', - index=8, - containing_service=None, - input_type=_LOOKUPSTREAMSPARAMS, - output_type=_LOOKUPSTREAMSRESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='Nearest', - full_name='v5api.BTrDB.Nearest', - index=9, - containing_service=None, - input_type=_NEARESTPARAMS, - output_type=_NEARESTRESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='Changes', - full_name='v5api.BTrDB.Changes', - index=10, - containing_service=None, - input_type=_CHANGESPARAMS, - output_type=_CHANGESRESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='Insert', - full_name='v5api.BTrDB.Insert', - index=11, - containing_service=None, - input_type=_INSERTPARAMS, - output_type=_INSERTRESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='Delete', - full_name='v5api.BTrDB.Delete', - index=12, - containing_service=None, - input_type=_DELETEPARAMS, - output_type=_DELETERESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='Info', - full_name='v5api.BTrDB.Info', - index=13, - containing_service=None, - input_type=_INFOPARAMS, - output_type=_INFORESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='FaultInject', - full_name='v5api.BTrDB.FaultInject', - index=14, - containing_service=None, - input_type=_FAULTINJECTPARAMS, - output_type=_FAULTINJECTRESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='Flush', - full_name='v5api.BTrDB.Flush', - index=15, - containing_service=None, - input_type=_FLUSHPARAMS, - output_type=_FLUSHRESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='Obliterate', - full_name='v5api.BTrDB.Obliterate', - index=16, - containing_service=None, - input_type=_OBLITERATEPARAMS, - output_type=_OBLITERATERESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='GetMetadataUsage', - full_name='v5api.BTrDB.GetMetadataUsage', - index=17, - containing_service=None, - input_type=_METADATAUSAGEPARAMS, - output_type=_METADATAUSAGERESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='GenerateCSV', - full_name='v5api.BTrDB.GenerateCSV', - index=18, - containing_service=None, - input_type=_GENERATECSVPARAMS, - output_type=_GENERATECSVRESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='SQLQuery', - full_name='v5api.BTrDB.SQLQuery', - index=19, - containing_service=None, - input_type=_SQLQUERYPARAMS, - output_type=_SQLQUERYRESPONSE, - serialized_options=None, - ), -]) -_sym_db.RegisterServiceDescriptor(_BTRDB) - -DESCRIPTOR.services_by_name['BTrDB'] = _BTRDB - +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0b\x62trdb.proto\x12\x05v5api\"Q\n\x0fRawValuesParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\"}\n\x11RawValuesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x1f\n\x06values\x18\x04 \x03(\x0b\x32\x0f.v5api.RawPoint\"m\n\x14\x41rrowRawValuesParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\x12\x15\n\rtemplateBytes\x18\x05 \x01(\x0c\"u\n\x16\x41rrowRawValuesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x12\n\narrowBytes\x18\x04 \x01(\x0c\"\x85\x01\n\x16\x41rrowMultiValuesParams\x12\x0c\n\x04uuid\x18\x01 \x03(\x0c\x12\x14\n\x0cversionMajor\x18\x02 \x03(\x04\x12\r\n\x05start\x18\x03 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x04 \x01(\x10\x12\x14\n\x0csnapPeriodNs\x18\x05 \x01(\x03\x12\x15\n\rtemplateBytes\x18\x06 \x01(\x0c\"K\n\x18\x41rrowMultiValuesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x12\n\narrowBytes\x18\x02 \x01(\x0c\"*\n\x0bRawPointVec\x12\x0c\n\x04time\x18\x01 \x01(\x10\x12\r\n\x05value\x18\x02 \x03(\x01\"j\n\x14\x41lignedWindowsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\x12\x12\n\npointWidth\x18\x05 \x01(\r\"\x83\x01\n\x16\x41lignedWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12 \n\x06values\x18\x04 \x03(\x0b\x32\x10.v5api.StatPoint\"z\n\x1b\x41rrowAlignedWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x12\n\narrowBytes\x18\x04 \x01(\x0c\"m\n\rWindowsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\x12\r\n\x05width\x18\x05 \x01(\x04\x12\r\n\x05\x64\x65pth\x18\x06 \x01(\r\"|\n\x0fWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12 \n\x06values\x18\x04 \x03(\x0b\x32\x10.v5api.StatPoint\"s\n\x14\x41rrowWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x12\n\narrowBytes\x18\x04 \x01(\x0c\"h\n\x10StreamInfoParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x13\n\x0bomitVersion\x18\x02 \x01(\x08\x12\x16\n\x0eomitDescriptor\x18\x03 \x01(\x08\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"\x8a\x01\n\x12StreamInfoResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12+\n\ndescriptor\x18\x04 \x01(\x0b\x32\x17.v5api.StreamDescriptor\"\x98\x01\n\x10StreamDescriptor\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x12\n\ncollection\x18\x02 \x01(\t\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x17\n\x0fpropertyVersion\x18\x05 \x01(\x04\"\x82\x01\n\x1aSetStreamAnnotationsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x1f\n\x17\x65xpectedPropertyVersion\x18\x02 \x01(\x04\x12#\n\x07\x63hanges\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x10\n\x08removals\x18\x04 \x03(\t\";\n\x1cSetStreamAnnotationsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"\x8a\x01\n\x13SetStreamTagsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x1f\n\x17\x65xpectedPropertyVersion\x18\x02 \x01(\x04\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x12\n\ncollection\x18\x04 \x01(\t\x12\x0e\n\x06remove\x18\x05 \x03(\t\"4\n\x15SetStreamTagsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"{\n\x0c\x43reateParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x12\n\ncollection\x18\x02 \x01(\t\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\"-\n\x0e\x43reateResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"@\n\x13MetadataUsageParams\x12\x0e\n\x06prefix\x18\x01 \x01(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"y\n\x15MetadataUsageResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x1d\n\x04tags\x18\x02 \x03(\x0b\x32\x0f.v5api.KeyCount\x12$\n\x0b\x61nnotations\x18\x03 \x03(\x0b\x32\x0f.v5api.KeyCount\"&\n\x08KeyCount\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05\x63ount\x18\x02 \x01(\x04\"B\n\x15ListCollectionsParams\x12\x0e\n\x06prefix\x18\x01 \x01(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"K\n\x17ListCollectionsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x13\n\x0b\x63ollections\x18\x02 \x03(\t\"\xab\x01\n\x13LookupStreamsParams\x12\x12\n\ncollection\x18\x01 \x01(\t\x12\x1a\n\x12isCollectionPrefix\x18\x02 \x01(\x08\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"^\n\x15LookupStreamsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12(\n\x07results\x18\x02 \x03(\x0b\x32\x17.v5api.StreamDescriptor\"S\n\rNearestParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x0c\n\x04time\x18\x02 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x03 \x01(\x04\x12\x10\n\x08\x62\x61\x63kward\x18\x04 \x01(\x08\"z\n\x0fNearestResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x1e\n\x05value\x18\x04 \x01(\x0b\x32\x0f.v5api.RawPoint\"U\n\rChangesParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x11\n\tfromMajor\x18\x02 \x01(\x04\x12\x0f\n\x07toMajor\x18\x03 \x01(\x04\x12\x12\n\nresolution\x18\x04 \x01(\r\"\x7f\n\x0f\x43hangesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12#\n\x06ranges\x18\x04 \x03(\x0b\x32\x13.v5api.ChangedRange\"#\n\tRoundSpec\x12\x0e\n\x04\x62its\x18\x02 \x01(\x05H\x00\x42\x06\n\x04spec\"\x99\x01\n\x0cInsertParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x0c\n\x04sync\x18\x02 \x01(\x08\x12(\n\x0cmerge_policy\x18\x04 \x01(\x0e\x32\x12.v5api.MergePolicy\x12\"\n\x08rounding\x18\x05 \x01(\x0b\x32\x10.v5api.RoundSpec\x12\x1f\n\x06values\x18\x03 \x03(\x0b\x32\x0f.v5api.RawPoint\"\x91\x01\n\x11\x41rrowInsertParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x0c\n\x04sync\x18\x02 \x01(\x08\x12(\n\x0cmerge_policy\x18\x03 \x01(\x0e\x32\x12.v5api.MergePolicy\x12\"\n\x08rounding\x18\x04 \x01(\x0b\x32\x10.v5api.RoundSpec\x12\x12\n\narrowBytes\x18\x05 \x01(\x0c\"Y\n\x0eInsertResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\"8\n\x0c\x44\x65leteParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\"Y\n\x0e\x44\x65leteResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\"\x0c\n\nInfoParams\"\xa2\x01\n\x0cInfoResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x19\n\x04mash\x18\x02 \x01(\x0b\x32\x0b.v5api.Mash\x12\x14\n\x0cmajorVersion\x18\x03 \x01(\r\x12\x14\n\x0cminorVersion\x18\x04 \x01(\r\x12\r\n\x05\x62uild\x18\x05 \x01(\t\x12\x1f\n\x05proxy\x18\x06 \x01(\x0b\x32\x10.v5api.ProxyInfo\"#\n\tProxyInfo\x12\x16\n\x0eproxyEndpoints\x18\x01 \x03(\t\"1\n\x11\x46\x61ultInjectParams\x12\x0c\n\x04type\x18\x01 \x01(\x04\x12\x0e\n\x06params\x18\x02 \x01(\x0c\">\n\x13\x46\x61ultInjectResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\n\n\x02rv\x18\x02 \x01(\x0c\"\x1b\n\x0b\x46lushParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"X\n\rFlushResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\" \n\x10ObliterateParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"1\n\x12ObliterateResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"\'\n\x08RawPoint\x12\x0c\n\x04time\x18\x01 \x01(\x10\x12\r\n\x05value\x18\x02 \x01(\x01\"`\n\tStatPoint\x12\x0c\n\x04time\x18\x01 \x01(\x10\x12\x0b\n\x03min\x18\x02 \x01(\x01\x12\x0c\n\x04mean\x18\x03 \x01(\x01\x12\x0b\n\x03max\x18\x04 \x01(\x01\x12\r\n\x05\x63ount\x18\x05 \x01(\x06\x12\x0e\n\x06stddev\x18\x06 \x01(\x01\"*\n\x0c\x43hangedRange\x12\r\n\x05start\x18\x01 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x02 \x01(\x10\">\n\x06Status\x12\x0c\n\x04\x63ode\x18\x01 \x01(\r\x12\x0b\n\x03msg\x18\x02 \x01(\t\x12\x19\n\x04mash\x18\x03 \x01(\x0b\x32\x0b.v5api.Mash\"\x98\x01\n\x04Mash\x12\x10\n\x08revision\x18\x01 \x01(\x03\x12\x0e\n\x06leader\x18\x02 \x01(\t\x12\x16\n\x0eleaderRevision\x18\x03 \x01(\x03\x12\x13\n\x0btotalWeight\x18\x04 \x01(\x03\x12\x0f\n\x07healthy\x18\x05 \x01(\x08\x12\x10\n\x08unmapped\x18\x06 \x01(\x01\x12\x1e\n\x07members\x18\x07 \x03(\x0b\x32\r.v5api.Member\"\xc3\x01\n\x06Member\x12\x0c\n\x04hash\x18\x01 \x01(\r\x12\x10\n\x08nodename\x18\x02 \x01(\t\x12\n\n\x02up\x18\x03 \x01(\x08\x12\n\n\x02in\x18\x04 \x01(\x08\x12\x0f\n\x07\x65nabled\x18\x05 \x01(\x08\x12\r\n\x05start\x18\x06 \x01(\x03\x12\x0b\n\x03\x65nd\x18\x07 \x01(\x03\x12\x0e\n\x06weight\x18\x08 \x01(\x03\x12\x16\n\x0ereadPreference\x18\t \x01(\x01\x12\x15\n\rhttpEndpoints\x18\n \x01(\t\x12\x15\n\rgrpcEndpoints\x18\x0b \x01(\t\"8\n\x0bKeyOptValue\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x1c\n\x03val\x18\x02 \x01(\x0b\x32\x0f.v5api.OptValue\"\x19\n\x08OptValue\x12\r\n\x05value\x18\x01 \x01(\t\"&\n\x08KeyValue\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"?\n\x0fStreamCSVConfig\x12\x0f\n\x07version\x18\x01 \x01(\x04\x12\r\n\x05label\x18\x02 \x01(\t\x12\x0c\n\x04uuid\x18\x03 \x01(\x0c\"\x9d\x02\n\x11GenerateCSVParams\x12\x35\n\tqueryType\x18\x01 \x01(\x0e\x32\".v5api.GenerateCSVParams.QueryType\x12\x11\n\tstartTime\x18\x02 \x01(\x03\x12\x0f\n\x07\x65ndTime\x18\x03 \x01(\x03\x12\x12\n\nwindowSize\x18\x04 \x01(\x04\x12\r\n\x05\x64\x65pth\x18\x05 \x01(\r\x12\x17\n\x0fincludeVersions\x18\x06 \x01(\x08\x12\'\n\x07streams\x18\x07 \x03(\x0b\x32\x16.v5api.StreamCSVConfig\"H\n\tQueryType\x12\x19\n\x15\x41LIGNED_WINDOWS_QUERY\x10\x00\x12\x11\n\rWINDOWS_QUERY\x10\x01\x12\r\n\tRAW_QUERY\x10\x02\"Q\n\x13GenerateCSVResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x10\n\x08isHeader\x18\x02 \x01(\x08\x12\x0b\n\x03row\x18\x03 \x03(\t\"J\n\x0eSQLQueryParams\x12\r\n\x05query\x18\x01 \x01(\t\x12\x0e\n\x06params\x18\x02 \x03(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"D\n\x10SQLQueryResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x13\n\x0bSQLQueryRow\x18\x02 \x03(\x0c\"\x14\n\x04Role\x12\x0c\n\x04name\x18\x01 \x01(\t\"\x94\x01\n\x19SetCompactionConfigParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x18\n\x10\x43ompactedVersion\x18\x02 \x01(\x04\x12>\n\x17reducedResolutionRanges\x18\x03 \x03(\x0b\x32\x1d.v5api.ReducedResolutionRange\x12\x0f\n\x07unused0\x18\x04 \x01(\x04\":\n\x1bSetCompactionConfigResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\")\n\x19GetCompactionConfigParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"\xc1\x01\n\x1bGetCompactionConfigResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x1a\n\x12LatestMajorVersion\x18\x02 \x01(\x04\x12\x18\n\x10\x43ompactedVersion\x18\x03 \x01(\x04\x12>\n\x17reducedResolutionRanges\x18\x04 \x03(\x0b\x32\x1d.v5api.ReducedResolutionRange\x12\x0f\n\x07unused0\x18\x05 \x01(\x04\"H\n\x16ReducedResolutionRange\x12\r\n\x05Start\x18\x01 \x01(\x03\x12\x0b\n\x03\x45nd\x18\x02 \x01(\x03\x12\x12\n\nResolution\x18\x03 \x01(\r\"K\n\x12SubscriptionUpdate\x12\'\n\x02op\x18\x01 \x01(\x0e\x32\x1b.v5api.SubscriptionUpdateOp\x12\x0c\n\x04uuid\x18\x02 \x03(\x0c\"Q\n\x10SubscriptionResp\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x0c\n\x04uuid\x18\x02 \x01(\x0c\x12\x12\n\narrowBytes\x18\x03 \x01(\x0c*<\n\x0bMergePolicy\x12\t\n\x05NEVER\x10\x00\x12\t\n\x05\x45QUAL\x10\x01\x12\n\n\x06RETAIN\x10\x02\x12\x0b\n\x07REPLACE\x10\x03*7\n\x14SubscriptionUpdateOp\x12\r\n\tADD_UUIDS\x10\x00\x12\x10\n\x0cREMOVE_UUIDS\x10\x01\x32\xfd\r\n\x05\x42TrDB\x12?\n\tRawValues\x12\x16.v5api.RawValuesParams\x1a\x18.v5api.RawValuesResponse0\x01\x12N\n\x0e\x41rrowRawValues\x12\x1b.v5api.ArrowRawValuesParams\x1a\x1d.v5api.ArrowRawValuesResponse0\x01\x12T\n\x10\x41rrowMultiValues\x12\x1d.v5api.ArrowMultiValuesParams\x1a\x1f.v5api.ArrowMultiValuesResponse0\x01\x12N\n\x0e\x41lignedWindows\x12\x1b.v5api.AlignedWindowsParams\x1a\x1d.v5api.AlignedWindowsResponse0\x01\x12X\n\x13\x41rrowAlignedWindows\x12\x1b.v5api.AlignedWindowsParams\x1a\".v5api.ArrowAlignedWindowsResponse0\x01\x12\x39\n\x07Windows\x12\x14.v5api.WindowsParams\x1a\x16.v5api.WindowsResponse0\x01\x12\x43\n\x0c\x41rrowWindows\x12\x14.v5api.WindowsParams\x1a\x1b.v5api.ArrowWindowsResponse0\x01\x12@\n\nStreamInfo\x12\x17.v5api.StreamInfoParams\x1a\x19.v5api.StreamInfoResponse\x12^\n\x14SetStreamAnnotations\x12!.v5api.SetStreamAnnotationsParams\x1a#.v5api.SetStreamAnnotationsResponse\x12I\n\rSetStreamTags\x12\x1a.v5api.SetStreamTagsParams\x1a\x1c.v5api.SetStreamTagsResponse\x12\x34\n\x06\x43reate\x12\x13.v5api.CreateParams\x1a\x15.v5api.CreateResponse\x12Q\n\x0fListCollections\x12\x1c.v5api.ListCollectionsParams\x1a\x1e.v5api.ListCollectionsResponse0\x01\x12K\n\rLookupStreams\x12\x1a.v5api.LookupStreamsParams\x1a\x1c.v5api.LookupStreamsResponse0\x01\x12\x37\n\x07Nearest\x12\x14.v5api.NearestParams\x1a\x16.v5api.NearestResponse\x12\x39\n\x07\x43hanges\x12\x14.v5api.ChangesParams\x1a\x16.v5api.ChangesResponse0\x01\x12\x34\n\x06Insert\x12\x13.v5api.InsertParams\x1a\x15.v5api.InsertResponse\x12>\n\x0b\x41rrowInsert\x12\x18.v5api.ArrowInsertParams\x1a\x15.v5api.InsertResponse\x12\x34\n\x06\x44\x65lete\x12\x13.v5api.DeleteParams\x1a\x15.v5api.DeleteResponse\x12.\n\x04Info\x12\x11.v5api.InfoParams\x1a\x13.v5api.InfoResponse\x12\x43\n\x0b\x46\x61ultInject\x12\x18.v5api.FaultInjectParams\x1a\x1a.v5api.FaultInjectResponse\x12\x31\n\x05\x46lush\x12\x12.v5api.FlushParams\x1a\x14.v5api.FlushResponse\x12@\n\nObliterate\x12\x17.v5api.ObliterateParams\x1a\x19.v5api.ObliterateResponse\x12L\n\x10GetMetadataUsage\x12\x1a.v5api.MetadataUsageParams\x1a\x1c.v5api.MetadataUsageResponse\x12\x45\n\x0bGenerateCSV\x12\x18.v5api.GenerateCSVParams\x1a\x1a.v5api.GenerateCSVResponse0\x01\x12<\n\x08SQLQuery\x12\x15.v5api.SQLQueryParams\x1a\x17.v5api.SQLQueryResponse0\x01\x12\x43\n\tSubscribe\x12\x19.v5api.SubscriptionUpdate\x1a\x17.v5api.SubscriptionResp(\x01\x30\x01\x62\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'btrdb_pb2', _globals) +if _descriptor._USE_C_DESCRIPTORS == False: + DESCRIPTOR._options = None + _globals['_MERGEPOLICY']._serialized_start=6600 + _globals['_MERGEPOLICY']._serialized_end=6660 + _globals['_SUBSCRIPTIONUPDATEOP']._serialized_start=6662 + _globals['_SUBSCRIPTIONUPDATEOP']._serialized_end=6717 + _globals['_RAWVALUESPARAMS']._serialized_start=22 + _globals['_RAWVALUESPARAMS']._serialized_end=103 + _globals['_RAWVALUESRESPONSE']._serialized_start=105 + _globals['_RAWVALUESRESPONSE']._serialized_end=230 + _globals['_ARROWRAWVALUESPARAMS']._serialized_start=232 + _globals['_ARROWRAWVALUESPARAMS']._serialized_end=341 + _globals['_ARROWRAWVALUESRESPONSE']._serialized_start=343 + _globals['_ARROWRAWVALUESRESPONSE']._serialized_end=460 + _globals['_ARROWMULTIVALUESPARAMS']._serialized_start=463 + _globals['_ARROWMULTIVALUESPARAMS']._serialized_end=596 + _globals['_ARROWMULTIVALUESRESPONSE']._serialized_start=598 + _globals['_ARROWMULTIVALUESRESPONSE']._serialized_end=673 + _globals['_RAWPOINTVEC']._serialized_start=675 + _globals['_RAWPOINTVEC']._serialized_end=717 + _globals['_ALIGNEDWINDOWSPARAMS']._serialized_start=719 + _globals['_ALIGNEDWINDOWSPARAMS']._serialized_end=825 + _globals['_ALIGNEDWINDOWSRESPONSE']._serialized_start=828 + _globals['_ALIGNEDWINDOWSRESPONSE']._serialized_end=959 + _globals['_ARROWALIGNEDWINDOWSRESPONSE']._serialized_start=961 + _globals['_ARROWALIGNEDWINDOWSRESPONSE']._serialized_end=1083 + _globals['_WINDOWSPARAMS']._serialized_start=1085 + _globals['_WINDOWSPARAMS']._serialized_end=1194 + _globals['_WINDOWSRESPONSE']._serialized_start=1196 + _globals['_WINDOWSRESPONSE']._serialized_end=1320 + _globals['_ARROWWINDOWSRESPONSE']._serialized_start=1322 + _globals['_ARROWWINDOWSRESPONSE']._serialized_end=1437 + _globals['_STREAMINFOPARAMS']._serialized_start=1439 + _globals['_STREAMINFOPARAMS']._serialized_end=1543 + _globals['_STREAMINFORESPONSE']._serialized_start=1546 + _globals['_STREAMINFORESPONSE']._serialized_end=1684 + _globals['_STREAMDESCRIPTOR']._serialized_start=1687 + _globals['_STREAMDESCRIPTOR']._serialized_end=1839 + _globals['_SETSTREAMANNOTATIONSPARAMS']._serialized_start=1842 + _globals['_SETSTREAMANNOTATIONSPARAMS']._serialized_end=1972 + _globals['_SETSTREAMANNOTATIONSRESPONSE']._serialized_start=1974 + _globals['_SETSTREAMANNOTATIONSRESPONSE']._serialized_end=2033 + _globals['_SETSTREAMTAGSPARAMS']._serialized_start=2036 + _globals['_SETSTREAMTAGSPARAMS']._serialized_end=2174 + _globals['_SETSTREAMTAGSRESPONSE']._serialized_start=2176 + _globals['_SETSTREAMTAGSRESPONSE']._serialized_end=2228 + _globals['_CREATEPARAMS']._serialized_start=2230 + _globals['_CREATEPARAMS']._serialized_end=2353 + _globals['_CREATERESPONSE']._serialized_start=2355 + _globals['_CREATERESPONSE']._serialized_end=2400 + _globals['_METADATAUSAGEPARAMS']._serialized_start=2402 + _globals['_METADATAUSAGEPARAMS']._serialized_end=2466 + _globals['_METADATAUSAGERESPONSE']._serialized_start=2468 + _globals['_METADATAUSAGERESPONSE']._serialized_end=2589 + _globals['_KEYCOUNT']._serialized_start=2591 + _globals['_KEYCOUNT']._serialized_end=2629 + _globals['_LISTCOLLECTIONSPARAMS']._serialized_start=2631 + _globals['_LISTCOLLECTIONSPARAMS']._serialized_end=2697 + _globals['_LISTCOLLECTIONSRESPONSE']._serialized_start=2699 + _globals['_LISTCOLLECTIONSRESPONSE']._serialized_end=2774 + _globals['_LOOKUPSTREAMSPARAMS']._serialized_start=2777 + _globals['_LOOKUPSTREAMSPARAMS']._serialized_end=2948 + _globals['_LOOKUPSTREAMSRESPONSE']._serialized_start=2950 + _globals['_LOOKUPSTREAMSRESPONSE']._serialized_end=3044 + _globals['_NEARESTPARAMS']._serialized_start=3046 + _globals['_NEARESTPARAMS']._serialized_end=3129 + _globals['_NEARESTRESPONSE']._serialized_start=3131 + _globals['_NEARESTRESPONSE']._serialized_end=3253 + _globals['_CHANGESPARAMS']._serialized_start=3255 + _globals['_CHANGESPARAMS']._serialized_end=3340 + _globals['_CHANGESRESPONSE']._serialized_start=3342 + _globals['_CHANGESRESPONSE']._serialized_end=3469 + _globals['_ROUNDSPEC']._serialized_start=3471 + _globals['_ROUNDSPEC']._serialized_end=3506 + _globals['_INSERTPARAMS']._serialized_start=3509 + _globals['_INSERTPARAMS']._serialized_end=3662 + _globals['_ARROWINSERTPARAMS']._serialized_start=3665 + _globals['_ARROWINSERTPARAMS']._serialized_end=3810 + _globals['_INSERTRESPONSE']._serialized_start=3812 + _globals['_INSERTRESPONSE']._serialized_end=3901 + _globals['_DELETEPARAMS']._serialized_start=3903 + _globals['_DELETEPARAMS']._serialized_end=3959 + _globals['_DELETERESPONSE']._serialized_start=3961 + _globals['_DELETERESPONSE']._serialized_end=4050 + _globals['_INFOPARAMS']._serialized_start=4052 + _globals['_INFOPARAMS']._serialized_end=4064 + _globals['_INFORESPONSE']._serialized_start=4067 + _globals['_INFORESPONSE']._serialized_end=4229 + _globals['_PROXYINFO']._serialized_start=4231 + _globals['_PROXYINFO']._serialized_end=4266 + _globals['_FAULTINJECTPARAMS']._serialized_start=4268 + _globals['_FAULTINJECTPARAMS']._serialized_end=4317 + _globals['_FAULTINJECTRESPONSE']._serialized_start=4319 + _globals['_FAULTINJECTRESPONSE']._serialized_end=4381 + _globals['_FLUSHPARAMS']._serialized_start=4383 + _globals['_FLUSHPARAMS']._serialized_end=4410 + _globals['_FLUSHRESPONSE']._serialized_start=4412 + _globals['_FLUSHRESPONSE']._serialized_end=4500 + _globals['_OBLITERATEPARAMS']._serialized_start=4502 + _globals['_OBLITERATEPARAMS']._serialized_end=4534 + _globals['_OBLITERATERESPONSE']._serialized_start=4536 + _globals['_OBLITERATERESPONSE']._serialized_end=4585 + _globals['_RAWPOINT']._serialized_start=4587 + _globals['_RAWPOINT']._serialized_end=4626 + _globals['_STATPOINT']._serialized_start=4628 + _globals['_STATPOINT']._serialized_end=4724 + _globals['_CHANGEDRANGE']._serialized_start=4726 + _globals['_CHANGEDRANGE']._serialized_end=4768 + _globals['_STATUS']._serialized_start=4770 + _globals['_STATUS']._serialized_end=4832 + _globals['_MASH']._serialized_start=4835 + _globals['_MASH']._serialized_end=4987 + _globals['_MEMBER']._serialized_start=4990 + _globals['_MEMBER']._serialized_end=5185 + _globals['_KEYOPTVALUE']._serialized_start=5187 + _globals['_KEYOPTVALUE']._serialized_end=5243 + _globals['_OPTVALUE']._serialized_start=5245 + _globals['_OPTVALUE']._serialized_end=5270 + _globals['_KEYVALUE']._serialized_start=5272 + _globals['_KEYVALUE']._serialized_end=5310 + _globals['_STREAMCSVCONFIG']._serialized_start=5312 + _globals['_STREAMCSVCONFIG']._serialized_end=5375 + _globals['_GENERATECSVPARAMS']._serialized_start=5378 + _globals['_GENERATECSVPARAMS']._serialized_end=5663 + _globals['_GENERATECSVPARAMS_QUERYTYPE']._serialized_start=5591 + _globals['_GENERATECSVPARAMS_QUERYTYPE']._serialized_end=5663 + _globals['_GENERATECSVRESPONSE']._serialized_start=5665 + _globals['_GENERATECSVRESPONSE']._serialized_end=5746 + _globals['_SQLQUERYPARAMS']._serialized_start=5748 + _globals['_SQLQUERYPARAMS']._serialized_end=5822 + _globals['_SQLQUERYRESPONSE']._serialized_start=5824 + _globals['_SQLQUERYRESPONSE']._serialized_end=5892 + _globals['_ROLE']._serialized_start=5894 + _globals['_ROLE']._serialized_end=5914 + _globals['_SETCOMPACTIONCONFIGPARAMS']._serialized_start=5917 + _globals['_SETCOMPACTIONCONFIGPARAMS']._serialized_end=6065 + _globals['_SETCOMPACTIONCONFIGRESPONSE']._serialized_start=6067 + _globals['_SETCOMPACTIONCONFIGRESPONSE']._serialized_end=6125 + _globals['_GETCOMPACTIONCONFIGPARAMS']._serialized_start=6127 + _globals['_GETCOMPACTIONCONFIGPARAMS']._serialized_end=6168 + _globals['_GETCOMPACTIONCONFIGRESPONSE']._serialized_start=6171 + _globals['_GETCOMPACTIONCONFIGRESPONSE']._serialized_end=6364 + _globals['_REDUCEDRESOLUTIONRANGE']._serialized_start=6366 + _globals['_REDUCEDRESOLUTIONRANGE']._serialized_end=6438 + _globals['_SUBSCRIPTIONUPDATE']._serialized_start=6440 + _globals['_SUBSCRIPTIONUPDATE']._serialized_end=6515 + _globals['_SUBSCRIPTIONRESP']._serialized_start=6517 + _globals['_SUBSCRIPTIONRESP']._serialized_end=6598 + _globals['_BTRDB']._serialized_start=6720 + _globals['_BTRDB']._serialized_end=8509 # @@protoc_insertion_point(module_scope) diff --git a/btrdb/grpcinterface/btrdb_pb2.pyi b/btrdb/grpcinterface/btrdb_pb2.pyi new file mode 100644 index 0000000..cc6320c --- /dev/null +++ b/btrdb/grpcinterface/btrdb_pb2.pyi @@ -0,0 +1,768 @@ +from google.protobuf.internal import containers as _containers +from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from typing import ClassVar as _ClassVar, Iterable as _Iterable, Mapping as _Mapping, Optional as _Optional, Union as _Union + +DESCRIPTOR: _descriptor.FileDescriptor + +class MergePolicy(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = () + NEVER: _ClassVar[MergePolicy] + EQUAL: _ClassVar[MergePolicy] + RETAIN: _ClassVar[MergePolicy] + REPLACE: _ClassVar[MergePolicy] + +class SubscriptionUpdateOp(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = () + ADD_UUIDS: _ClassVar[SubscriptionUpdateOp] + REMOVE_UUIDS: _ClassVar[SubscriptionUpdateOp] +NEVER: MergePolicy +EQUAL: MergePolicy +RETAIN: MergePolicy +REPLACE: MergePolicy +ADD_UUIDS: SubscriptionUpdateOp +REMOVE_UUIDS: SubscriptionUpdateOp + +class RawValuesParams(_message.Message): + __slots__ = ("uuid", "start", "end", "versionMajor") + UUID_FIELD_NUMBER: _ClassVar[int] + START_FIELD_NUMBER: _ClassVar[int] + END_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + uuid: bytes + start: int + end: int + versionMajor: int + def __init__(self, uuid: _Optional[bytes] = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., versionMajor: _Optional[int] = ...) -> None: ... + +class RawValuesResponse(_message.Message): + __slots__ = ("stat", "versionMajor", "versionMinor", "values") + STAT_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + VALUES_FIELD_NUMBER: _ClassVar[int] + stat: Status + versionMajor: int + versionMinor: int + values: _containers.RepeatedCompositeFieldContainer[RawPoint] + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., values: _Optional[_Iterable[_Union[RawPoint, _Mapping]]] = ...) -> None: ... + +class ArrowRawValuesParams(_message.Message): + __slots__ = ("uuid", "start", "end", "versionMajor", "templateBytes") + UUID_FIELD_NUMBER: _ClassVar[int] + START_FIELD_NUMBER: _ClassVar[int] + END_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + TEMPLATEBYTES_FIELD_NUMBER: _ClassVar[int] + uuid: bytes + start: int + end: int + versionMajor: int + templateBytes: bytes + def __init__(self, uuid: _Optional[bytes] = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., versionMajor: _Optional[int] = ..., templateBytes: _Optional[bytes] = ...) -> None: ... + +class ArrowRawValuesResponse(_message.Message): + __slots__ = ("stat", "versionMajor", "versionMinor", "arrowBytes") + STAT_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + ARROWBYTES_FIELD_NUMBER: _ClassVar[int] + stat: Status + versionMajor: int + versionMinor: int + arrowBytes: bytes + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., arrowBytes: _Optional[bytes] = ...) -> None: ... + +class ArrowMultiValuesParams(_message.Message): + __slots__ = ("uuid", "versionMajor", "start", "end", "snapPeriodNs", "templateBytes") + UUID_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + START_FIELD_NUMBER: _ClassVar[int] + END_FIELD_NUMBER: _ClassVar[int] + SNAPPERIODNS_FIELD_NUMBER: _ClassVar[int] + TEMPLATEBYTES_FIELD_NUMBER: _ClassVar[int] + uuid: _containers.RepeatedScalarFieldContainer[bytes] + versionMajor: _containers.RepeatedScalarFieldContainer[int] + start: int + end: int + snapPeriodNs: int + templateBytes: bytes + def __init__(self, uuid: _Optional[_Iterable[bytes]] = ..., versionMajor: _Optional[_Iterable[int]] = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., snapPeriodNs: _Optional[int] = ..., templateBytes: _Optional[bytes] = ...) -> None: ... + +class ArrowMultiValuesResponse(_message.Message): + __slots__ = ("stat", "arrowBytes") + STAT_FIELD_NUMBER: _ClassVar[int] + ARROWBYTES_FIELD_NUMBER: _ClassVar[int] + stat: Status + arrowBytes: bytes + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., arrowBytes: _Optional[bytes] = ...) -> None: ... + +class RawPointVec(_message.Message): + __slots__ = ("time", "value") + TIME_FIELD_NUMBER: _ClassVar[int] + VALUE_FIELD_NUMBER: _ClassVar[int] + time: int + value: _containers.RepeatedScalarFieldContainer[float] + def __init__(self, time: _Optional[int] = ..., value: _Optional[_Iterable[float]] = ...) -> None: ... + +class AlignedWindowsParams(_message.Message): + __slots__ = ("uuid", "start", "end", "versionMajor", "pointWidth") + UUID_FIELD_NUMBER: _ClassVar[int] + START_FIELD_NUMBER: _ClassVar[int] + END_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + POINTWIDTH_FIELD_NUMBER: _ClassVar[int] + uuid: bytes + start: int + end: int + versionMajor: int + pointWidth: int + def __init__(self, uuid: _Optional[bytes] = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., versionMajor: _Optional[int] = ..., pointWidth: _Optional[int] = ...) -> None: ... + +class AlignedWindowsResponse(_message.Message): + __slots__ = ("stat", "versionMajor", "versionMinor", "values") + STAT_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + VALUES_FIELD_NUMBER: _ClassVar[int] + stat: Status + versionMajor: int + versionMinor: int + values: _containers.RepeatedCompositeFieldContainer[StatPoint] + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., values: _Optional[_Iterable[_Union[StatPoint, _Mapping]]] = ...) -> None: ... + +class ArrowAlignedWindowsResponse(_message.Message): + __slots__ = ("stat", "versionMajor", "versionMinor", "arrowBytes") + STAT_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + ARROWBYTES_FIELD_NUMBER: _ClassVar[int] + stat: Status + versionMajor: int + versionMinor: int + arrowBytes: bytes + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., arrowBytes: _Optional[bytes] = ...) -> None: ... + +class WindowsParams(_message.Message): + __slots__ = ("uuid", "start", "end", "versionMajor", "width", "depth") + UUID_FIELD_NUMBER: _ClassVar[int] + START_FIELD_NUMBER: _ClassVar[int] + END_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + WIDTH_FIELD_NUMBER: _ClassVar[int] + DEPTH_FIELD_NUMBER: _ClassVar[int] + uuid: bytes + start: int + end: int + versionMajor: int + width: int + depth: int + def __init__(self, uuid: _Optional[bytes] = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., versionMajor: _Optional[int] = ..., width: _Optional[int] = ..., depth: _Optional[int] = ...) -> None: ... + +class WindowsResponse(_message.Message): + __slots__ = ("stat", "versionMajor", "versionMinor", "values") + STAT_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + VALUES_FIELD_NUMBER: _ClassVar[int] + stat: Status + versionMajor: int + versionMinor: int + values: _containers.RepeatedCompositeFieldContainer[StatPoint] + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., values: _Optional[_Iterable[_Union[StatPoint, _Mapping]]] = ...) -> None: ... + +class ArrowWindowsResponse(_message.Message): + __slots__ = ("stat", "versionMajor", "versionMinor", "arrowBytes") + STAT_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + ARROWBYTES_FIELD_NUMBER: _ClassVar[int] + stat: Status + versionMajor: int + versionMinor: int + arrowBytes: bytes + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., arrowBytes: _Optional[bytes] = ...) -> None: ... + +class StreamInfoParams(_message.Message): + __slots__ = ("uuid", "omitVersion", "omitDescriptor", "role") + UUID_FIELD_NUMBER: _ClassVar[int] + OMITVERSION_FIELD_NUMBER: _ClassVar[int] + OMITDESCRIPTOR_FIELD_NUMBER: _ClassVar[int] + ROLE_FIELD_NUMBER: _ClassVar[int] + uuid: bytes + omitVersion: bool + omitDescriptor: bool + role: Role + def __init__(self, uuid: _Optional[bytes] = ..., omitVersion: bool = ..., omitDescriptor: bool = ..., role: _Optional[_Union[Role, _Mapping]] = ...) -> None: ... + +class StreamInfoResponse(_message.Message): + __slots__ = ("stat", "versionMajor", "versionMinor", "descriptor") + STAT_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + DESCRIPTOR_FIELD_NUMBER: _ClassVar[int] + stat: Status + versionMajor: int + versionMinor: int + descriptor: StreamDescriptor + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., descriptor: _Optional[_Union[StreamDescriptor, _Mapping]] = ...) -> None: ... + +class StreamDescriptor(_message.Message): + __slots__ = ("uuid", "collection", "tags", "annotations", "propertyVersion") + UUID_FIELD_NUMBER: _ClassVar[int] + COLLECTION_FIELD_NUMBER: _ClassVar[int] + TAGS_FIELD_NUMBER: _ClassVar[int] + ANNOTATIONS_FIELD_NUMBER: _ClassVar[int] + PROPERTYVERSION_FIELD_NUMBER: _ClassVar[int] + uuid: bytes + collection: str + tags: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + annotations: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + propertyVersion: int + def __init__(self, uuid: _Optional[bytes] = ..., collection: _Optional[str] = ..., tags: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., annotations: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., propertyVersion: _Optional[int] = ...) -> None: ... + +class SetStreamAnnotationsParams(_message.Message): + __slots__ = ("uuid", "expectedPropertyVersion", "changes", "removals") + UUID_FIELD_NUMBER: _ClassVar[int] + EXPECTEDPROPERTYVERSION_FIELD_NUMBER: _ClassVar[int] + CHANGES_FIELD_NUMBER: _ClassVar[int] + REMOVALS_FIELD_NUMBER: _ClassVar[int] + uuid: bytes + expectedPropertyVersion: int + changes: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + removals: _containers.RepeatedScalarFieldContainer[str] + def __init__(self, uuid: _Optional[bytes] = ..., expectedPropertyVersion: _Optional[int] = ..., changes: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., removals: _Optional[_Iterable[str]] = ...) -> None: ... + +class SetStreamAnnotationsResponse(_message.Message): + __slots__ = ("stat",) + STAT_FIELD_NUMBER: _ClassVar[int] + stat: Status + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ...) -> None: ... + +class SetStreamTagsParams(_message.Message): + __slots__ = ("uuid", "expectedPropertyVersion", "tags", "collection", "remove") + UUID_FIELD_NUMBER: _ClassVar[int] + EXPECTEDPROPERTYVERSION_FIELD_NUMBER: _ClassVar[int] + TAGS_FIELD_NUMBER: _ClassVar[int] + COLLECTION_FIELD_NUMBER: _ClassVar[int] + REMOVE_FIELD_NUMBER: _ClassVar[int] + uuid: bytes + expectedPropertyVersion: int + tags: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + collection: str + remove: _containers.RepeatedScalarFieldContainer[str] + def __init__(self, uuid: _Optional[bytes] = ..., expectedPropertyVersion: _Optional[int] = ..., tags: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., collection: _Optional[str] = ..., remove: _Optional[_Iterable[str]] = ...) -> None: ... + +class SetStreamTagsResponse(_message.Message): + __slots__ = ("stat",) + STAT_FIELD_NUMBER: _ClassVar[int] + stat: Status + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ...) -> None: ... + +class CreateParams(_message.Message): + __slots__ = ("uuid", "collection", "tags", "annotations") + UUID_FIELD_NUMBER: _ClassVar[int] + COLLECTION_FIELD_NUMBER: _ClassVar[int] + TAGS_FIELD_NUMBER: _ClassVar[int] + ANNOTATIONS_FIELD_NUMBER: _ClassVar[int] + uuid: bytes + collection: str + tags: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + annotations: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + def __init__(self, uuid: _Optional[bytes] = ..., collection: _Optional[str] = ..., tags: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., annotations: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ...) -> None: ... + +class CreateResponse(_message.Message): + __slots__ = ("stat",) + STAT_FIELD_NUMBER: _ClassVar[int] + stat: Status + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ...) -> None: ... + +class MetadataUsageParams(_message.Message): + __slots__ = ("prefix", "role") + PREFIX_FIELD_NUMBER: _ClassVar[int] + ROLE_FIELD_NUMBER: _ClassVar[int] + prefix: str + role: Role + def __init__(self, prefix: _Optional[str] = ..., role: _Optional[_Union[Role, _Mapping]] = ...) -> None: ... + +class MetadataUsageResponse(_message.Message): + __slots__ = ("stat", "tags", "annotations") + STAT_FIELD_NUMBER: _ClassVar[int] + TAGS_FIELD_NUMBER: _ClassVar[int] + ANNOTATIONS_FIELD_NUMBER: _ClassVar[int] + stat: Status + tags: _containers.RepeatedCompositeFieldContainer[KeyCount] + annotations: _containers.RepeatedCompositeFieldContainer[KeyCount] + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., tags: _Optional[_Iterable[_Union[KeyCount, _Mapping]]] = ..., annotations: _Optional[_Iterable[_Union[KeyCount, _Mapping]]] = ...) -> None: ... + +class KeyCount(_message.Message): + __slots__ = ("key", "count") + KEY_FIELD_NUMBER: _ClassVar[int] + COUNT_FIELD_NUMBER: _ClassVar[int] + key: str + count: int + def __init__(self, key: _Optional[str] = ..., count: _Optional[int] = ...) -> None: ... + +class ListCollectionsParams(_message.Message): + __slots__ = ("prefix", "role") + PREFIX_FIELD_NUMBER: _ClassVar[int] + ROLE_FIELD_NUMBER: _ClassVar[int] + prefix: str + role: Role + def __init__(self, prefix: _Optional[str] = ..., role: _Optional[_Union[Role, _Mapping]] = ...) -> None: ... + +class ListCollectionsResponse(_message.Message): + __slots__ = ("stat", "collections") + STAT_FIELD_NUMBER: _ClassVar[int] + COLLECTIONS_FIELD_NUMBER: _ClassVar[int] + stat: Status + collections: _containers.RepeatedScalarFieldContainer[str] + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., collections: _Optional[_Iterable[str]] = ...) -> None: ... + +class LookupStreamsParams(_message.Message): + __slots__ = ("collection", "isCollectionPrefix", "tags", "annotations", "role") + COLLECTION_FIELD_NUMBER: _ClassVar[int] + ISCOLLECTIONPREFIX_FIELD_NUMBER: _ClassVar[int] + TAGS_FIELD_NUMBER: _ClassVar[int] + ANNOTATIONS_FIELD_NUMBER: _ClassVar[int] + ROLE_FIELD_NUMBER: _ClassVar[int] + collection: str + isCollectionPrefix: bool + tags: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + annotations: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + role: Role + def __init__(self, collection: _Optional[str] = ..., isCollectionPrefix: bool = ..., tags: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., annotations: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., role: _Optional[_Union[Role, _Mapping]] = ...) -> None: ... + +class LookupStreamsResponse(_message.Message): + __slots__ = ("stat", "results") + STAT_FIELD_NUMBER: _ClassVar[int] + RESULTS_FIELD_NUMBER: _ClassVar[int] + stat: Status + results: _containers.RepeatedCompositeFieldContainer[StreamDescriptor] + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., results: _Optional[_Iterable[_Union[StreamDescriptor, _Mapping]]] = ...) -> None: ... + +class NearestParams(_message.Message): + __slots__ = ("uuid", "time", "versionMajor", "backward") + UUID_FIELD_NUMBER: _ClassVar[int] + TIME_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + BACKWARD_FIELD_NUMBER: _ClassVar[int] + uuid: bytes + time: int + versionMajor: int + backward: bool + def __init__(self, uuid: _Optional[bytes] = ..., time: _Optional[int] = ..., versionMajor: _Optional[int] = ..., backward: bool = ...) -> None: ... + +class NearestResponse(_message.Message): + __slots__ = ("stat", "versionMajor", "versionMinor", "value") + STAT_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + VALUE_FIELD_NUMBER: _ClassVar[int] + stat: Status + versionMajor: int + versionMinor: int + value: RawPoint + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., value: _Optional[_Union[RawPoint, _Mapping]] = ...) -> None: ... + +class ChangesParams(_message.Message): + __slots__ = ("uuid", "fromMajor", "toMajor", "resolution") + UUID_FIELD_NUMBER: _ClassVar[int] + FROMMAJOR_FIELD_NUMBER: _ClassVar[int] + TOMAJOR_FIELD_NUMBER: _ClassVar[int] + RESOLUTION_FIELD_NUMBER: _ClassVar[int] + uuid: bytes + fromMajor: int + toMajor: int + resolution: int + def __init__(self, uuid: _Optional[bytes] = ..., fromMajor: _Optional[int] = ..., toMajor: _Optional[int] = ..., resolution: _Optional[int] = ...) -> None: ... + +class ChangesResponse(_message.Message): + __slots__ = ("stat", "versionMajor", "versionMinor", "ranges") + STAT_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + RANGES_FIELD_NUMBER: _ClassVar[int] + stat: Status + versionMajor: int + versionMinor: int + ranges: _containers.RepeatedCompositeFieldContainer[ChangedRange] + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., ranges: _Optional[_Iterable[_Union[ChangedRange, _Mapping]]] = ...) -> None: ... + +class RoundSpec(_message.Message): + __slots__ = ("bits",) + BITS_FIELD_NUMBER: _ClassVar[int] + bits: int + def __init__(self, bits: _Optional[int] = ...) -> None: ... + +class InsertParams(_message.Message): + __slots__ = ("uuid", "sync", "merge_policy", "rounding", "values") + UUID_FIELD_NUMBER: _ClassVar[int] + SYNC_FIELD_NUMBER: _ClassVar[int] + MERGE_POLICY_FIELD_NUMBER: _ClassVar[int] + ROUNDING_FIELD_NUMBER: _ClassVar[int] + VALUES_FIELD_NUMBER: _ClassVar[int] + uuid: bytes + sync: bool + merge_policy: MergePolicy + rounding: RoundSpec + values: _containers.RepeatedCompositeFieldContainer[RawPoint] + def __init__(self, uuid: _Optional[bytes] = ..., sync: bool = ..., merge_policy: _Optional[_Union[MergePolicy, str]] = ..., rounding: _Optional[_Union[RoundSpec, _Mapping]] = ..., values: _Optional[_Iterable[_Union[RawPoint, _Mapping]]] = ...) -> None: ... + +class ArrowInsertParams(_message.Message): + __slots__ = ("uuid", "sync", "merge_policy", "rounding", "arrowBytes") + UUID_FIELD_NUMBER: _ClassVar[int] + SYNC_FIELD_NUMBER: _ClassVar[int] + MERGE_POLICY_FIELD_NUMBER: _ClassVar[int] + ROUNDING_FIELD_NUMBER: _ClassVar[int] + ARROWBYTES_FIELD_NUMBER: _ClassVar[int] + uuid: bytes + sync: bool + merge_policy: MergePolicy + rounding: RoundSpec + arrowBytes: bytes + def __init__(self, uuid: _Optional[bytes] = ..., sync: bool = ..., merge_policy: _Optional[_Union[MergePolicy, str]] = ..., rounding: _Optional[_Union[RoundSpec, _Mapping]] = ..., arrowBytes: _Optional[bytes] = ...) -> None: ... + +class InsertResponse(_message.Message): + __slots__ = ("stat", "versionMajor", "versionMinor") + STAT_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + stat: Status + versionMajor: int + versionMinor: int + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ...) -> None: ... + +class DeleteParams(_message.Message): + __slots__ = ("uuid", "start", "end") + UUID_FIELD_NUMBER: _ClassVar[int] + START_FIELD_NUMBER: _ClassVar[int] + END_FIELD_NUMBER: _ClassVar[int] + uuid: bytes + start: int + end: int + def __init__(self, uuid: _Optional[bytes] = ..., start: _Optional[int] = ..., end: _Optional[int] = ...) -> None: ... + +class DeleteResponse(_message.Message): + __slots__ = ("stat", "versionMajor", "versionMinor") + STAT_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + stat: Status + versionMajor: int + versionMinor: int + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ...) -> None: ... + +class InfoParams(_message.Message): + __slots__ = () + def __init__(self) -> None: ... + +class InfoResponse(_message.Message): + __slots__ = ("stat", "mash", "majorVersion", "minorVersion", "build", "proxy") + STAT_FIELD_NUMBER: _ClassVar[int] + MASH_FIELD_NUMBER: _ClassVar[int] + MAJORVERSION_FIELD_NUMBER: _ClassVar[int] + MINORVERSION_FIELD_NUMBER: _ClassVar[int] + BUILD_FIELD_NUMBER: _ClassVar[int] + PROXY_FIELD_NUMBER: _ClassVar[int] + stat: Status + mash: Mash + majorVersion: int + minorVersion: int + build: str + proxy: ProxyInfo + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., mash: _Optional[_Union[Mash, _Mapping]] = ..., majorVersion: _Optional[int] = ..., minorVersion: _Optional[int] = ..., build: _Optional[str] = ..., proxy: _Optional[_Union[ProxyInfo, _Mapping]] = ...) -> None: ... + +class ProxyInfo(_message.Message): + __slots__ = ("proxyEndpoints",) + PROXYENDPOINTS_FIELD_NUMBER: _ClassVar[int] + proxyEndpoints: _containers.RepeatedScalarFieldContainer[str] + def __init__(self, proxyEndpoints: _Optional[_Iterable[str]] = ...) -> None: ... + +class FaultInjectParams(_message.Message): + __slots__ = ("type", "params") + TYPE_FIELD_NUMBER: _ClassVar[int] + PARAMS_FIELD_NUMBER: _ClassVar[int] + type: int + params: bytes + def __init__(self, type: _Optional[int] = ..., params: _Optional[bytes] = ...) -> None: ... + +class FaultInjectResponse(_message.Message): + __slots__ = ("stat", "rv") + STAT_FIELD_NUMBER: _ClassVar[int] + RV_FIELD_NUMBER: _ClassVar[int] + stat: Status + rv: bytes + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., rv: _Optional[bytes] = ...) -> None: ... + +class FlushParams(_message.Message): + __slots__ = ("uuid",) + UUID_FIELD_NUMBER: _ClassVar[int] + uuid: bytes + def __init__(self, uuid: _Optional[bytes] = ...) -> None: ... + +class FlushResponse(_message.Message): + __slots__ = ("stat", "versionMajor", "versionMinor") + STAT_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + stat: Status + versionMajor: int + versionMinor: int + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ...) -> None: ... + +class ObliterateParams(_message.Message): + __slots__ = ("uuid",) + UUID_FIELD_NUMBER: _ClassVar[int] + uuid: bytes + def __init__(self, uuid: _Optional[bytes] = ...) -> None: ... + +class ObliterateResponse(_message.Message): + __slots__ = ("stat",) + STAT_FIELD_NUMBER: _ClassVar[int] + stat: Status + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ...) -> None: ... + +class RawPoint(_message.Message): + __slots__ = ("time", "value") + TIME_FIELD_NUMBER: _ClassVar[int] + VALUE_FIELD_NUMBER: _ClassVar[int] + time: int + value: float + def __init__(self, time: _Optional[int] = ..., value: _Optional[float] = ...) -> None: ... + +class StatPoint(_message.Message): + __slots__ = ("time", "min", "mean", "max", "count", "stddev") + TIME_FIELD_NUMBER: _ClassVar[int] + MIN_FIELD_NUMBER: _ClassVar[int] + MEAN_FIELD_NUMBER: _ClassVar[int] + MAX_FIELD_NUMBER: _ClassVar[int] + COUNT_FIELD_NUMBER: _ClassVar[int] + STDDEV_FIELD_NUMBER: _ClassVar[int] + time: int + min: float + mean: float + max: float + count: int + stddev: float + def __init__(self, time: _Optional[int] = ..., min: _Optional[float] = ..., mean: _Optional[float] = ..., max: _Optional[float] = ..., count: _Optional[int] = ..., stddev: _Optional[float] = ...) -> None: ... + +class ChangedRange(_message.Message): + __slots__ = ("start", "end") + START_FIELD_NUMBER: _ClassVar[int] + END_FIELD_NUMBER: _ClassVar[int] + start: int + end: int + def __init__(self, start: _Optional[int] = ..., end: _Optional[int] = ...) -> None: ... + +class Status(_message.Message): + __slots__ = ("code", "msg", "mash") + CODE_FIELD_NUMBER: _ClassVar[int] + MSG_FIELD_NUMBER: _ClassVar[int] + MASH_FIELD_NUMBER: _ClassVar[int] + code: int + msg: str + mash: Mash + def __init__(self, code: _Optional[int] = ..., msg: _Optional[str] = ..., mash: _Optional[_Union[Mash, _Mapping]] = ...) -> None: ... + +class Mash(_message.Message): + __slots__ = ("revision", "leader", "leaderRevision", "totalWeight", "healthy", "unmapped", "members") + REVISION_FIELD_NUMBER: _ClassVar[int] + LEADER_FIELD_NUMBER: _ClassVar[int] + LEADERREVISION_FIELD_NUMBER: _ClassVar[int] + TOTALWEIGHT_FIELD_NUMBER: _ClassVar[int] + HEALTHY_FIELD_NUMBER: _ClassVar[int] + UNMAPPED_FIELD_NUMBER: _ClassVar[int] + MEMBERS_FIELD_NUMBER: _ClassVar[int] + revision: int + leader: str + leaderRevision: int + totalWeight: int + healthy: bool + unmapped: float + members: _containers.RepeatedCompositeFieldContainer[Member] + def __init__(self, revision: _Optional[int] = ..., leader: _Optional[str] = ..., leaderRevision: _Optional[int] = ..., totalWeight: _Optional[int] = ..., healthy: bool = ..., unmapped: _Optional[float] = ..., members: _Optional[_Iterable[_Union[Member, _Mapping]]] = ...) -> None: ... + +class Member(_message.Message): + __slots__ = ("hash", "nodename", "up", "enabled", "start", "end", "weight", "readPreference", "httpEndpoints", "grpcEndpoints") + HASH_FIELD_NUMBER: _ClassVar[int] + NODENAME_FIELD_NUMBER: _ClassVar[int] + UP_FIELD_NUMBER: _ClassVar[int] + IN_FIELD_NUMBER: _ClassVar[int] + ENABLED_FIELD_NUMBER: _ClassVar[int] + START_FIELD_NUMBER: _ClassVar[int] + END_FIELD_NUMBER: _ClassVar[int] + WEIGHT_FIELD_NUMBER: _ClassVar[int] + READPREFERENCE_FIELD_NUMBER: _ClassVar[int] + HTTPENDPOINTS_FIELD_NUMBER: _ClassVar[int] + GRPCENDPOINTS_FIELD_NUMBER: _ClassVar[int] + hash: int + nodename: str + up: bool + enabled: bool + start: int + end: int + weight: int + readPreference: float + httpEndpoints: str + grpcEndpoints: str + def __init__(self, hash: _Optional[int] = ..., nodename: _Optional[str] = ..., up: bool = ..., enabled: bool = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., weight: _Optional[int] = ..., readPreference: _Optional[float] = ..., httpEndpoints: _Optional[str] = ..., grpcEndpoints: _Optional[str] = ..., **kwargs) -> None: ... + +class KeyOptValue(_message.Message): + __slots__ = ("key", "val") + KEY_FIELD_NUMBER: _ClassVar[int] + VAL_FIELD_NUMBER: _ClassVar[int] + key: str + val: OptValue + def __init__(self, key: _Optional[str] = ..., val: _Optional[_Union[OptValue, _Mapping]] = ...) -> None: ... + +class OptValue(_message.Message): + __slots__ = ("value",) + VALUE_FIELD_NUMBER: _ClassVar[int] + value: str + def __init__(self, value: _Optional[str] = ...) -> None: ... + +class KeyValue(_message.Message): + __slots__ = ("key", "value") + KEY_FIELD_NUMBER: _ClassVar[int] + VALUE_FIELD_NUMBER: _ClassVar[int] + key: str + value: str + def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ... + +class StreamCSVConfig(_message.Message): + __slots__ = ("version", "label", "uuid") + VERSION_FIELD_NUMBER: _ClassVar[int] + LABEL_FIELD_NUMBER: _ClassVar[int] + UUID_FIELD_NUMBER: _ClassVar[int] + version: int + label: str + uuid: bytes + def __init__(self, version: _Optional[int] = ..., label: _Optional[str] = ..., uuid: _Optional[bytes] = ...) -> None: ... + +class GenerateCSVParams(_message.Message): + __slots__ = ("queryType", "startTime", "endTime", "windowSize", "depth", "includeVersions", "streams") + class QueryType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = () + ALIGNED_WINDOWS_QUERY: _ClassVar[GenerateCSVParams.QueryType] + WINDOWS_QUERY: _ClassVar[GenerateCSVParams.QueryType] + RAW_QUERY: _ClassVar[GenerateCSVParams.QueryType] + ALIGNED_WINDOWS_QUERY: GenerateCSVParams.QueryType + WINDOWS_QUERY: GenerateCSVParams.QueryType + RAW_QUERY: GenerateCSVParams.QueryType + QUERYTYPE_FIELD_NUMBER: _ClassVar[int] + STARTTIME_FIELD_NUMBER: _ClassVar[int] + ENDTIME_FIELD_NUMBER: _ClassVar[int] + WINDOWSIZE_FIELD_NUMBER: _ClassVar[int] + DEPTH_FIELD_NUMBER: _ClassVar[int] + INCLUDEVERSIONS_FIELD_NUMBER: _ClassVar[int] + STREAMS_FIELD_NUMBER: _ClassVar[int] + queryType: GenerateCSVParams.QueryType + startTime: int + endTime: int + windowSize: int + depth: int + includeVersions: bool + streams: _containers.RepeatedCompositeFieldContainer[StreamCSVConfig] + def __init__(self, queryType: _Optional[_Union[GenerateCSVParams.QueryType, str]] = ..., startTime: _Optional[int] = ..., endTime: _Optional[int] = ..., windowSize: _Optional[int] = ..., depth: _Optional[int] = ..., includeVersions: bool = ..., streams: _Optional[_Iterable[_Union[StreamCSVConfig, _Mapping]]] = ...) -> None: ... + +class GenerateCSVResponse(_message.Message): + __slots__ = ("stat", "isHeader", "row") + STAT_FIELD_NUMBER: _ClassVar[int] + ISHEADER_FIELD_NUMBER: _ClassVar[int] + ROW_FIELD_NUMBER: _ClassVar[int] + stat: Status + isHeader: bool + row: _containers.RepeatedScalarFieldContainer[str] + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., isHeader: bool = ..., row: _Optional[_Iterable[str]] = ...) -> None: ... + +class SQLQueryParams(_message.Message): + __slots__ = ("query", "params", "role") + QUERY_FIELD_NUMBER: _ClassVar[int] + PARAMS_FIELD_NUMBER: _ClassVar[int] + ROLE_FIELD_NUMBER: _ClassVar[int] + query: str + params: _containers.RepeatedScalarFieldContainer[str] + role: Role + def __init__(self, query: _Optional[str] = ..., params: _Optional[_Iterable[str]] = ..., role: _Optional[_Union[Role, _Mapping]] = ...) -> None: ... + +class SQLQueryResponse(_message.Message): + __slots__ = ("stat", "SQLQueryRow") + STAT_FIELD_NUMBER: _ClassVar[int] + SQLQUERYROW_FIELD_NUMBER: _ClassVar[int] + stat: Status + SQLQueryRow: _containers.RepeatedScalarFieldContainer[bytes] + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., SQLQueryRow: _Optional[_Iterable[bytes]] = ...) -> None: ... + +class Role(_message.Message): + __slots__ = ("name",) + NAME_FIELD_NUMBER: _ClassVar[int] + name: str + def __init__(self, name: _Optional[str] = ...) -> None: ... + +class SetCompactionConfigParams(_message.Message): + __slots__ = ("uuid", "CompactedVersion", "reducedResolutionRanges", "unused0") + UUID_FIELD_NUMBER: _ClassVar[int] + COMPACTEDVERSION_FIELD_NUMBER: _ClassVar[int] + REDUCEDRESOLUTIONRANGES_FIELD_NUMBER: _ClassVar[int] + UNUSED0_FIELD_NUMBER: _ClassVar[int] + uuid: bytes + CompactedVersion: int + reducedResolutionRanges: _containers.RepeatedCompositeFieldContainer[ReducedResolutionRange] + unused0: int + def __init__(self, uuid: _Optional[bytes] = ..., CompactedVersion: _Optional[int] = ..., reducedResolutionRanges: _Optional[_Iterable[_Union[ReducedResolutionRange, _Mapping]]] = ..., unused0: _Optional[int] = ...) -> None: ... + +class SetCompactionConfigResponse(_message.Message): + __slots__ = ("stat",) + STAT_FIELD_NUMBER: _ClassVar[int] + stat: Status + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ...) -> None: ... + +class GetCompactionConfigParams(_message.Message): + __slots__ = ("uuid",) + UUID_FIELD_NUMBER: _ClassVar[int] + uuid: bytes + def __init__(self, uuid: _Optional[bytes] = ...) -> None: ... + +class GetCompactionConfigResponse(_message.Message): + __slots__ = ("stat", "LatestMajorVersion", "CompactedVersion", "reducedResolutionRanges", "unused0") + STAT_FIELD_NUMBER: _ClassVar[int] + LATESTMAJORVERSION_FIELD_NUMBER: _ClassVar[int] + COMPACTEDVERSION_FIELD_NUMBER: _ClassVar[int] + REDUCEDRESOLUTIONRANGES_FIELD_NUMBER: _ClassVar[int] + UNUSED0_FIELD_NUMBER: _ClassVar[int] + stat: Status + LatestMajorVersion: int + CompactedVersion: int + reducedResolutionRanges: _containers.RepeatedCompositeFieldContainer[ReducedResolutionRange] + unused0: int + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., LatestMajorVersion: _Optional[int] = ..., CompactedVersion: _Optional[int] = ..., reducedResolutionRanges: _Optional[_Iterable[_Union[ReducedResolutionRange, _Mapping]]] = ..., unused0: _Optional[int] = ...) -> None: ... + +class ReducedResolutionRange(_message.Message): + __slots__ = ("Start", "End", "Resolution") + START_FIELD_NUMBER: _ClassVar[int] + END_FIELD_NUMBER: _ClassVar[int] + RESOLUTION_FIELD_NUMBER: _ClassVar[int] + Start: int + End: int + Resolution: int + def __init__(self, Start: _Optional[int] = ..., End: _Optional[int] = ..., Resolution: _Optional[int] = ...) -> None: ... + +class SubscriptionUpdate(_message.Message): + __slots__ = ("op", "uuid") + OP_FIELD_NUMBER: _ClassVar[int] + UUID_FIELD_NUMBER: _ClassVar[int] + op: SubscriptionUpdateOp + uuid: _containers.RepeatedScalarFieldContainer[bytes] + def __init__(self, op: _Optional[_Union[SubscriptionUpdateOp, str]] = ..., uuid: _Optional[_Iterable[bytes]] = ...) -> None: ... + +class SubscriptionResp(_message.Message): + __slots__ = ("stat", "uuid", "arrowBytes") + STAT_FIELD_NUMBER: _ClassVar[int] + UUID_FIELD_NUMBER: _ClassVar[int] + ARROWBYTES_FIELD_NUMBER: _ClassVar[int] + stat: Status + uuid: bytes + arrowBytes: bytes + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., uuid: _Optional[bytes] = ..., arrowBytes: _Optional[bytes] = ...) -> None: ... diff --git a/btrdb/grpcinterface/btrdb_pb2_grpc.py b/btrdb/grpcinterface/btrdb_pb2_grpc.py index 89b9ce0..d0db693 100644 --- a/btrdb/grpcinterface/btrdb_pb2_grpc.py +++ b/btrdb/grpcinterface/btrdb_pb2_grpc.py @@ -1,370 +1,893 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" import grpc import btrdb.grpcinterface.btrdb_pb2 as btrdb__pb2 class BTrDBStub(object): - # missing associated documentation comment in .proto file - pass - - def __init__(self, channel): - """Constructor. - - Args: - channel: A grpc.Channel. - """ - self.RawValues = channel.unary_stream( - '/v5api.BTrDB/RawValues', - request_serializer=btrdb__pb2.RawValuesParams.SerializeToString, - response_deserializer=btrdb__pb2.RawValuesResponse.FromString, - ) - self.AlignedWindows = channel.unary_stream( - '/v5api.BTrDB/AlignedWindows', - request_serializer=btrdb__pb2.AlignedWindowsParams.SerializeToString, - response_deserializer=btrdb__pb2.AlignedWindowsResponse.FromString, - ) - self.Windows = channel.unary_stream( - '/v5api.BTrDB/Windows', - request_serializer=btrdb__pb2.WindowsParams.SerializeToString, - response_deserializer=btrdb__pb2.WindowsResponse.FromString, - ) - self.StreamInfo = channel.unary_unary( - '/v5api.BTrDB/StreamInfo', - request_serializer=btrdb__pb2.StreamInfoParams.SerializeToString, - response_deserializer=btrdb__pb2.StreamInfoResponse.FromString, - ) - self.SetStreamAnnotations = channel.unary_unary( - '/v5api.BTrDB/SetStreamAnnotations', - request_serializer=btrdb__pb2.SetStreamAnnotationsParams.SerializeToString, - response_deserializer=btrdb__pb2.SetStreamAnnotationsResponse.FromString, - ) - self.SetStreamTags = channel.unary_unary( - '/v5api.BTrDB/SetStreamTags', - request_serializer=btrdb__pb2.SetStreamTagsParams.SerializeToString, - response_deserializer=btrdb__pb2.SetStreamTagsResponse.FromString, - ) - self.Create = channel.unary_unary( - '/v5api.BTrDB/Create', - request_serializer=btrdb__pb2.CreateParams.SerializeToString, - response_deserializer=btrdb__pb2.CreateResponse.FromString, - ) - self.ListCollections = channel.unary_stream( - '/v5api.BTrDB/ListCollections', - request_serializer=btrdb__pb2.ListCollectionsParams.SerializeToString, - response_deserializer=btrdb__pb2.ListCollectionsResponse.FromString, - ) - self.LookupStreams = channel.unary_stream( - '/v5api.BTrDB/LookupStreams', - request_serializer=btrdb__pb2.LookupStreamsParams.SerializeToString, - response_deserializer=btrdb__pb2.LookupStreamsResponse.FromString, - ) - self.Nearest = channel.unary_unary( - '/v5api.BTrDB/Nearest', - request_serializer=btrdb__pb2.NearestParams.SerializeToString, - response_deserializer=btrdb__pb2.NearestResponse.FromString, - ) - self.Changes = channel.unary_stream( - '/v5api.BTrDB/Changes', - request_serializer=btrdb__pb2.ChangesParams.SerializeToString, - response_deserializer=btrdb__pb2.ChangesResponse.FromString, - ) - self.Insert = channel.unary_unary( - '/v5api.BTrDB/Insert', - request_serializer=btrdb__pb2.InsertParams.SerializeToString, - response_deserializer=btrdb__pb2.InsertResponse.FromString, - ) - self.Delete = channel.unary_unary( - '/v5api.BTrDB/Delete', - request_serializer=btrdb__pb2.DeleteParams.SerializeToString, - response_deserializer=btrdb__pb2.DeleteResponse.FromString, - ) - self.Info = channel.unary_unary( - '/v5api.BTrDB/Info', - request_serializer=btrdb__pb2.InfoParams.SerializeToString, - response_deserializer=btrdb__pb2.InfoResponse.FromString, - ) - self.FaultInject = channel.unary_unary( - '/v5api.BTrDB/FaultInject', - request_serializer=btrdb__pb2.FaultInjectParams.SerializeToString, - response_deserializer=btrdb__pb2.FaultInjectResponse.FromString, - ) - self.Flush = channel.unary_unary( - '/v5api.BTrDB/Flush', - request_serializer=btrdb__pb2.FlushParams.SerializeToString, - response_deserializer=btrdb__pb2.FlushResponse.FromString, - ) - self.Obliterate = channel.unary_unary( - '/v5api.BTrDB/Obliterate', - request_serializer=btrdb__pb2.ObliterateParams.SerializeToString, - response_deserializer=btrdb__pb2.ObliterateResponse.FromString, - ) - self.GetMetadataUsage = channel.unary_unary( - '/v5api.BTrDB/GetMetadataUsage', - request_serializer=btrdb__pb2.MetadataUsageParams.SerializeToString, - response_deserializer=btrdb__pb2.MetadataUsageResponse.FromString, - ) - self.GenerateCSV = channel.unary_stream( - '/v5api.BTrDB/GenerateCSV', - request_serializer=btrdb__pb2.GenerateCSVParams.SerializeToString, - response_deserializer=btrdb__pb2.GenerateCSVResponse.FromString, - ) - self.SQLQuery = channel.unary_stream( - '/v5api.BTrDB/SQLQuery', - request_serializer=btrdb__pb2.SQLQueryParams.SerializeToString, - response_deserializer=btrdb__pb2.SQLQueryResponse.FromString, - ) + """Missing associated documentation comment in .proto file.""" + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.RawValues = channel.unary_stream( + '/v5api.BTrDB/RawValues', + request_serializer=btrdb__pb2.RawValuesParams.SerializeToString, + response_deserializer=btrdb__pb2.RawValuesResponse.FromString, + ) + self.ArrowRawValues = channel.unary_stream( + '/v5api.BTrDB/ArrowRawValues', + request_serializer=btrdb__pb2.ArrowRawValuesParams.SerializeToString, + response_deserializer=btrdb__pb2.ArrowRawValuesResponse.FromString, + ) + self.ArrowMultiValues = channel.unary_stream( + '/v5api.BTrDB/ArrowMultiValues', + request_serializer=btrdb__pb2.ArrowMultiValuesParams.SerializeToString, + response_deserializer=btrdb__pb2.ArrowMultiValuesResponse.FromString, + ) + self.AlignedWindows = channel.unary_stream( + '/v5api.BTrDB/AlignedWindows', + request_serializer=btrdb__pb2.AlignedWindowsParams.SerializeToString, + response_deserializer=btrdb__pb2.AlignedWindowsResponse.FromString, + ) + self.ArrowAlignedWindows = channel.unary_stream( + '/v5api.BTrDB/ArrowAlignedWindows', + request_serializer=btrdb__pb2.AlignedWindowsParams.SerializeToString, + response_deserializer=btrdb__pb2.ArrowAlignedWindowsResponse.FromString, + ) + self.Windows = channel.unary_stream( + '/v5api.BTrDB/Windows', + request_serializer=btrdb__pb2.WindowsParams.SerializeToString, + response_deserializer=btrdb__pb2.WindowsResponse.FromString, + ) + self.ArrowWindows = channel.unary_stream( + '/v5api.BTrDB/ArrowWindows', + request_serializer=btrdb__pb2.WindowsParams.SerializeToString, + response_deserializer=btrdb__pb2.ArrowWindowsResponse.FromString, + ) + self.StreamInfo = channel.unary_unary( + '/v5api.BTrDB/StreamInfo', + request_serializer=btrdb__pb2.StreamInfoParams.SerializeToString, + response_deserializer=btrdb__pb2.StreamInfoResponse.FromString, + ) + self.SetStreamAnnotations = channel.unary_unary( + '/v5api.BTrDB/SetStreamAnnotations', + request_serializer=btrdb__pb2.SetStreamAnnotationsParams.SerializeToString, + response_deserializer=btrdb__pb2.SetStreamAnnotationsResponse.FromString, + ) + self.SetStreamTags = channel.unary_unary( + '/v5api.BTrDB/SetStreamTags', + request_serializer=btrdb__pb2.SetStreamTagsParams.SerializeToString, + response_deserializer=btrdb__pb2.SetStreamTagsResponse.FromString, + ) + self.Create = channel.unary_unary( + '/v5api.BTrDB/Create', + request_serializer=btrdb__pb2.CreateParams.SerializeToString, + response_deserializer=btrdb__pb2.CreateResponse.FromString, + ) + self.ListCollections = channel.unary_stream( + '/v5api.BTrDB/ListCollections', + request_serializer=btrdb__pb2.ListCollectionsParams.SerializeToString, + response_deserializer=btrdb__pb2.ListCollectionsResponse.FromString, + ) + self.LookupStreams = channel.unary_stream( + '/v5api.BTrDB/LookupStreams', + request_serializer=btrdb__pb2.LookupStreamsParams.SerializeToString, + response_deserializer=btrdb__pb2.LookupStreamsResponse.FromString, + ) + self.Nearest = channel.unary_unary( + '/v5api.BTrDB/Nearest', + request_serializer=btrdb__pb2.NearestParams.SerializeToString, + response_deserializer=btrdb__pb2.NearestResponse.FromString, + ) + self.Changes = channel.unary_stream( + '/v5api.BTrDB/Changes', + request_serializer=btrdb__pb2.ChangesParams.SerializeToString, + response_deserializer=btrdb__pb2.ChangesResponse.FromString, + ) + self.Insert = channel.unary_unary( + '/v5api.BTrDB/Insert', + request_serializer=btrdb__pb2.InsertParams.SerializeToString, + response_deserializer=btrdb__pb2.InsertResponse.FromString, + ) + self.ArrowInsert = channel.unary_unary( + '/v5api.BTrDB/ArrowInsert', + request_serializer=btrdb__pb2.ArrowInsertParams.SerializeToString, + response_deserializer=btrdb__pb2.InsertResponse.FromString, + ) + self.Delete = channel.unary_unary( + '/v5api.BTrDB/Delete', + request_serializer=btrdb__pb2.DeleteParams.SerializeToString, + response_deserializer=btrdb__pb2.DeleteResponse.FromString, + ) + self.Info = channel.unary_unary( + '/v5api.BTrDB/Info', + request_serializer=btrdb__pb2.InfoParams.SerializeToString, + response_deserializer=btrdb__pb2.InfoResponse.FromString, + ) + self.FaultInject = channel.unary_unary( + '/v5api.BTrDB/FaultInject', + request_serializer=btrdb__pb2.FaultInjectParams.SerializeToString, + response_deserializer=btrdb__pb2.FaultInjectResponse.FromString, + ) + self.Flush = channel.unary_unary( + '/v5api.BTrDB/Flush', + request_serializer=btrdb__pb2.FlushParams.SerializeToString, + response_deserializer=btrdb__pb2.FlushResponse.FromString, + ) + self.Obliterate = channel.unary_unary( + '/v5api.BTrDB/Obliterate', + request_serializer=btrdb__pb2.ObliterateParams.SerializeToString, + response_deserializer=btrdb__pb2.ObliterateResponse.FromString, + ) + self.GetMetadataUsage = channel.unary_unary( + '/v5api.BTrDB/GetMetadataUsage', + request_serializer=btrdb__pb2.MetadataUsageParams.SerializeToString, + response_deserializer=btrdb__pb2.MetadataUsageResponse.FromString, + ) + self.GenerateCSV = channel.unary_stream( + '/v5api.BTrDB/GenerateCSV', + request_serializer=btrdb__pb2.GenerateCSVParams.SerializeToString, + response_deserializer=btrdb__pb2.GenerateCSVResponse.FromString, + ) + self.SQLQuery = channel.unary_stream( + '/v5api.BTrDB/SQLQuery', + request_serializer=btrdb__pb2.SQLQueryParams.SerializeToString, + response_deserializer=btrdb__pb2.SQLQueryResponse.FromString, + ) + self.Subscribe = channel.stream_stream( + '/v5api.BTrDB/Subscribe', + request_serializer=btrdb__pb2.SubscriptionUpdate.SerializeToString, + response_deserializer=btrdb__pb2.SubscriptionResp.FromString, + ) class BTrDBServicer(object): - # missing associated documentation comment in .proto file - pass - - def RawValues(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def AlignedWindows(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Windows(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def StreamInfo(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def SetStreamAnnotations(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def SetStreamTags(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Create(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def ListCollections(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def LookupStreams(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Nearest(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Changes(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Insert(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Delete(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Info(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def FaultInject(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Flush(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Obliterate(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def GetMetadataUsage(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def GenerateCSV(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def SQLQuery(self, request, context): - """rpc SetCompactionConfig(SetCompactionConfigParams) returns (SetCompactionConfigResponse); - rpc GetCompactionConfig(GetCompactionConfigParams) returns (GetCompactionConfigResponse); - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') + """Missing associated documentation comment in .proto file.""" + + def RawValues(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def ArrowRawValues(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def ArrowMultiValues(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def AlignedWindows(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def ArrowAlignedWindows(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Windows(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def ArrowWindows(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def StreamInfo(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def SetStreamAnnotations(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def SetStreamTags(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Create(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def ListCollections(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def LookupStreams(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Nearest(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Changes(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Insert(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def ArrowInsert(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Delete(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Info(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def FaultInject(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Flush(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Obliterate(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetMetadataUsage(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GenerateCSV(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def SQLQuery(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Subscribe(self, request_iterator, context): + """rpc SetCompactionConfig(SetCompactionConfigParams) returns (SetCompactionConfigResponse); + rpc GetCompactionConfig(GetCompactionConfigParams) returns (GetCompactionConfigResponse); + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') def add_BTrDBServicer_to_server(servicer, server): - rpc_method_handlers = { - 'RawValues': grpc.unary_stream_rpc_method_handler( - servicer.RawValues, - request_deserializer=btrdb__pb2.RawValuesParams.FromString, - response_serializer=btrdb__pb2.RawValuesResponse.SerializeToString, - ), - 'AlignedWindows': grpc.unary_stream_rpc_method_handler( - servicer.AlignedWindows, - request_deserializer=btrdb__pb2.AlignedWindowsParams.FromString, - response_serializer=btrdb__pb2.AlignedWindowsResponse.SerializeToString, - ), - 'Windows': grpc.unary_stream_rpc_method_handler( - servicer.Windows, - request_deserializer=btrdb__pb2.WindowsParams.FromString, - response_serializer=btrdb__pb2.WindowsResponse.SerializeToString, - ), - 'StreamInfo': grpc.unary_unary_rpc_method_handler( - servicer.StreamInfo, - request_deserializer=btrdb__pb2.StreamInfoParams.FromString, - response_serializer=btrdb__pb2.StreamInfoResponse.SerializeToString, - ), - 'SetStreamAnnotations': grpc.unary_unary_rpc_method_handler( - servicer.SetStreamAnnotations, - request_deserializer=btrdb__pb2.SetStreamAnnotationsParams.FromString, - response_serializer=btrdb__pb2.SetStreamAnnotationsResponse.SerializeToString, - ), - 'SetStreamTags': grpc.unary_unary_rpc_method_handler( - servicer.SetStreamTags, - request_deserializer=btrdb__pb2.SetStreamTagsParams.FromString, - response_serializer=btrdb__pb2.SetStreamTagsResponse.SerializeToString, - ), - 'Create': grpc.unary_unary_rpc_method_handler( - servicer.Create, - request_deserializer=btrdb__pb2.CreateParams.FromString, - response_serializer=btrdb__pb2.CreateResponse.SerializeToString, - ), - 'ListCollections': grpc.unary_stream_rpc_method_handler( - servicer.ListCollections, - request_deserializer=btrdb__pb2.ListCollectionsParams.FromString, - response_serializer=btrdb__pb2.ListCollectionsResponse.SerializeToString, - ), - 'LookupStreams': grpc.unary_stream_rpc_method_handler( - servicer.LookupStreams, - request_deserializer=btrdb__pb2.LookupStreamsParams.FromString, - response_serializer=btrdb__pb2.LookupStreamsResponse.SerializeToString, - ), - 'Nearest': grpc.unary_unary_rpc_method_handler( - servicer.Nearest, - request_deserializer=btrdb__pb2.NearestParams.FromString, - response_serializer=btrdb__pb2.NearestResponse.SerializeToString, - ), - 'Changes': grpc.unary_stream_rpc_method_handler( - servicer.Changes, - request_deserializer=btrdb__pb2.ChangesParams.FromString, - response_serializer=btrdb__pb2.ChangesResponse.SerializeToString, - ), - 'Insert': grpc.unary_unary_rpc_method_handler( - servicer.Insert, - request_deserializer=btrdb__pb2.InsertParams.FromString, - response_serializer=btrdb__pb2.InsertResponse.SerializeToString, - ), - 'Delete': grpc.unary_unary_rpc_method_handler( - servicer.Delete, - request_deserializer=btrdb__pb2.DeleteParams.FromString, - response_serializer=btrdb__pb2.DeleteResponse.SerializeToString, - ), - 'Info': grpc.unary_unary_rpc_method_handler( - servicer.Info, - request_deserializer=btrdb__pb2.InfoParams.FromString, - response_serializer=btrdb__pb2.InfoResponse.SerializeToString, - ), - 'FaultInject': grpc.unary_unary_rpc_method_handler( - servicer.FaultInject, - request_deserializer=btrdb__pb2.FaultInjectParams.FromString, - response_serializer=btrdb__pb2.FaultInjectResponse.SerializeToString, - ), - 'Flush': grpc.unary_unary_rpc_method_handler( - servicer.Flush, - request_deserializer=btrdb__pb2.FlushParams.FromString, - response_serializer=btrdb__pb2.FlushResponse.SerializeToString, - ), - 'Obliterate': grpc.unary_unary_rpc_method_handler( - servicer.Obliterate, - request_deserializer=btrdb__pb2.ObliterateParams.FromString, - response_serializer=btrdb__pb2.ObliterateResponse.SerializeToString, - ), - 'GetMetadataUsage': grpc.unary_unary_rpc_method_handler( - servicer.GetMetadataUsage, - request_deserializer=btrdb__pb2.MetadataUsageParams.FromString, - response_serializer=btrdb__pb2.MetadataUsageResponse.SerializeToString, - ), - 'GenerateCSV': grpc.unary_stream_rpc_method_handler( - servicer.GenerateCSV, - request_deserializer=btrdb__pb2.GenerateCSVParams.FromString, - response_serializer=btrdb__pb2.GenerateCSVResponse.SerializeToString, - ), - 'SQLQuery': grpc.unary_stream_rpc_method_handler( - servicer.SQLQuery, - request_deserializer=btrdb__pb2.SQLQueryParams.FromString, - response_serializer=btrdb__pb2.SQLQueryResponse.SerializeToString, - ), - } - generic_handler = grpc.method_handlers_generic_handler( - 'v5api.BTrDB', rpc_method_handlers) - server.add_generic_rpc_handlers((generic_handler,)) + rpc_method_handlers = { + 'RawValues': grpc.unary_stream_rpc_method_handler( + servicer.RawValues, + request_deserializer=btrdb__pb2.RawValuesParams.FromString, + response_serializer=btrdb__pb2.RawValuesResponse.SerializeToString, + ), + 'ArrowRawValues': grpc.unary_stream_rpc_method_handler( + servicer.ArrowRawValues, + request_deserializer=btrdb__pb2.ArrowRawValuesParams.FromString, + response_serializer=btrdb__pb2.ArrowRawValuesResponse.SerializeToString, + ), + 'ArrowMultiValues': grpc.unary_stream_rpc_method_handler( + servicer.ArrowMultiValues, + request_deserializer=btrdb__pb2.ArrowMultiValuesParams.FromString, + response_serializer=btrdb__pb2.ArrowMultiValuesResponse.SerializeToString, + ), + 'AlignedWindows': grpc.unary_stream_rpc_method_handler( + servicer.AlignedWindows, + request_deserializer=btrdb__pb2.AlignedWindowsParams.FromString, + response_serializer=btrdb__pb2.AlignedWindowsResponse.SerializeToString, + ), + 'ArrowAlignedWindows': grpc.unary_stream_rpc_method_handler( + servicer.ArrowAlignedWindows, + request_deserializer=btrdb__pb2.AlignedWindowsParams.FromString, + response_serializer=btrdb__pb2.ArrowAlignedWindowsResponse.SerializeToString, + ), + 'Windows': grpc.unary_stream_rpc_method_handler( + servicer.Windows, + request_deserializer=btrdb__pb2.WindowsParams.FromString, + response_serializer=btrdb__pb2.WindowsResponse.SerializeToString, + ), + 'ArrowWindows': grpc.unary_stream_rpc_method_handler( + servicer.ArrowWindows, + request_deserializer=btrdb__pb2.WindowsParams.FromString, + response_serializer=btrdb__pb2.ArrowWindowsResponse.SerializeToString, + ), + 'StreamInfo': grpc.unary_unary_rpc_method_handler( + servicer.StreamInfo, + request_deserializer=btrdb__pb2.StreamInfoParams.FromString, + response_serializer=btrdb__pb2.StreamInfoResponse.SerializeToString, + ), + 'SetStreamAnnotations': grpc.unary_unary_rpc_method_handler( + servicer.SetStreamAnnotations, + request_deserializer=btrdb__pb2.SetStreamAnnotationsParams.FromString, + response_serializer=btrdb__pb2.SetStreamAnnotationsResponse.SerializeToString, + ), + 'SetStreamTags': grpc.unary_unary_rpc_method_handler( + servicer.SetStreamTags, + request_deserializer=btrdb__pb2.SetStreamTagsParams.FromString, + response_serializer=btrdb__pb2.SetStreamTagsResponse.SerializeToString, + ), + 'Create': grpc.unary_unary_rpc_method_handler( + servicer.Create, + request_deserializer=btrdb__pb2.CreateParams.FromString, + response_serializer=btrdb__pb2.CreateResponse.SerializeToString, + ), + 'ListCollections': grpc.unary_stream_rpc_method_handler( + servicer.ListCollections, + request_deserializer=btrdb__pb2.ListCollectionsParams.FromString, + response_serializer=btrdb__pb2.ListCollectionsResponse.SerializeToString, + ), + 'LookupStreams': grpc.unary_stream_rpc_method_handler( + servicer.LookupStreams, + request_deserializer=btrdb__pb2.LookupStreamsParams.FromString, + response_serializer=btrdb__pb2.LookupStreamsResponse.SerializeToString, + ), + 'Nearest': grpc.unary_unary_rpc_method_handler( + servicer.Nearest, + request_deserializer=btrdb__pb2.NearestParams.FromString, + response_serializer=btrdb__pb2.NearestResponse.SerializeToString, + ), + 'Changes': grpc.unary_stream_rpc_method_handler( + servicer.Changes, + request_deserializer=btrdb__pb2.ChangesParams.FromString, + response_serializer=btrdb__pb2.ChangesResponse.SerializeToString, + ), + 'Insert': grpc.unary_unary_rpc_method_handler( + servicer.Insert, + request_deserializer=btrdb__pb2.InsertParams.FromString, + response_serializer=btrdb__pb2.InsertResponse.SerializeToString, + ), + 'ArrowInsert': grpc.unary_unary_rpc_method_handler( + servicer.ArrowInsert, + request_deserializer=btrdb__pb2.ArrowInsertParams.FromString, + response_serializer=btrdb__pb2.InsertResponse.SerializeToString, + ), + 'Delete': grpc.unary_unary_rpc_method_handler( + servicer.Delete, + request_deserializer=btrdb__pb2.DeleteParams.FromString, + response_serializer=btrdb__pb2.DeleteResponse.SerializeToString, + ), + 'Info': grpc.unary_unary_rpc_method_handler( + servicer.Info, + request_deserializer=btrdb__pb2.InfoParams.FromString, + response_serializer=btrdb__pb2.InfoResponse.SerializeToString, + ), + 'FaultInject': grpc.unary_unary_rpc_method_handler( + servicer.FaultInject, + request_deserializer=btrdb__pb2.FaultInjectParams.FromString, + response_serializer=btrdb__pb2.FaultInjectResponse.SerializeToString, + ), + 'Flush': grpc.unary_unary_rpc_method_handler( + servicer.Flush, + request_deserializer=btrdb__pb2.FlushParams.FromString, + response_serializer=btrdb__pb2.FlushResponse.SerializeToString, + ), + 'Obliterate': grpc.unary_unary_rpc_method_handler( + servicer.Obliterate, + request_deserializer=btrdb__pb2.ObliterateParams.FromString, + response_serializer=btrdb__pb2.ObliterateResponse.SerializeToString, + ), + 'GetMetadataUsage': grpc.unary_unary_rpc_method_handler( + servicer.GetMetadataUsage, + request_deserializer=btrdb__pb2.MetadataUsageParams.FromString, + response_serializer=btrdb__pb2.MetadataUsageResponse.SerializeToString, + ), + 'GenerateCSV': grpc.unary_stream_rpc_method_handler( + servicer.GenerateCSV, + request_deserializer=btrdb__pb2.GenerateCSVParams.FromString, + response_serializer=btrdb__pb2.GenerateCSVResponse.SerializeToString, + ), + 'SQLQuery': grpc.unary_stream_rpc_method_handler( + servicer.SQLQuery, + request_deserializer=btrdb__pb2.SQLQueryParams.FromString, + response_serializer=btrdb__pb2.SQLQueryResponse.SerializeToString, + ), + 'Subscribe': grpc.stream_stream_rpc_method_handler( + servicer.Subscribe, + request_deserializer=btrdb__pb2.SubscriptionUpdate.FromString, + response_serializer=btrdb__pb2.SubscriptionResp.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'v5api.BTrDB', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + + + # This class is part of an EXPERIMENTAL API. +class BTrDB(object): + """Missing associated documentation comment in .proto file.""" + + @staticmethod + def RawValues(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream(request, target, '/v5api.BTrDB/RawValues', + btrdb__pb2.RawValuesParams.SerializeToString, + btrdb__pb2.RawValuesResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def ArrowRawValues(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream(request, target, '/v5api.BTrDB/ArrowRawValues', + btrdb__pb2.ArrowRawValuesParams.SerializeToString, + btrdb__pb2.ArrowRawValuesResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def ArrowMultiValues(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream(request, target, '/v5api.BTrDB/ArrowMultiValues', + btrdb__pb2.ArrowMultiValuesParams.SerializeToString, + btrdb__pb2.ArrowMultiValuesResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def AlignedWindows(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream(request, target, '/v5api.BTrDB/AlignedWindows', + btrdb__pb2.AlignedWindowsParams.SerializeToString, + btrdb__pb2.AlignedWindowsResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def ArrowAlignedWindows(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream(request, target, '/v5api.BTrDB/ArrowAlignedWindows', + btrdb__pb2.AlignedWindowsParams.SerializeToString, + btrdb__pb2.ArrowAlignedWindowsResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Windows(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream(request, target, '/v5api.BTrDB/Windows', + btrdb__pb2.WindowsParams.SerializeToString, + btrdb__pb2.WindowsResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def ArrowWindows(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream(request, target, '/v5api.BTrDB/ArrowWindows', + btrdb__pb2.WindowsParams.SerializeToString, + btrdb__pb2.ArrowWindowsResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def StreamInfo(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/v5api.BTrDB/StreamInfo', + btrdb__pb2.StreamInfoParams.SerializeToString, + btrdb__pb2.StreamInfoResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def SetStreamAnnotations(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/v5api.BTrDB/SetStreamAnnotations', + btrdb__pb2.SetStreamAnnotationsParams.SerializeToString, + btrdb__pb2.SetStreamAnnotationsResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def SetStreamTags(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/v5api.BTrDB/SetStreamTags', + btrdb__pb2.SetStreamTagsParams.SerializeToString, + btrdb__pb2.SetStreamTagsResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Create(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/v5api.BTrDB/Create', + btrdb__pb2.CreateParams.SerializeToString, + btrdb__pb2.CreateResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def ListCollections(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream(request, target, '/v5api.BTrDB/ListCollections', + btrdb__pb2.ListCollectionsParams.SerializeToString, + btrdb__pb2.ListCollectionsResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def LookupStreams(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream(request, target, '/v5api.BTrDB/LookupStreams', + btrdb__pb2.LookupStreamsParams.SerializeToString, + btrdb__pb2.LookupStreamsResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Nearest(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/v5api.BTrDB/Nearest', + btrdb__pb2.NearestParams.SerializeToString, + btrdb__pb2.NearestResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Changes(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream(request, target, '/v5api.BTrDB/Changes', + btrdb__pb2.ChangesParams.SerializeToString, + btrdb__pb2.ChangesResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Insert(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/v5api.BTrDB/Insert', + btrdb__pb2.InsertParams.SerializeToString, + btrdb__pb2.InsertResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def ArrowInsert(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/v5api.BTrDB/ArrowInsert', + btrdb__pb2.ArrowInsertParams.SerializeToString, + btrdb__pb2.InsertResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Delete(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/v5api.BTrDB/Delete', + btrdb__pb2.DeleteParams.SerializeToString, + btrdb__pb2.DeleteResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Info(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/v5api.BTrDB/Info', + btrdb__pb2.InfoParams.SerializeToString, + btrdb__pb2.InfoResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def FaultInject(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/v5api.BTrDB/FaultInject', + btrdb__pb2.FaultInjectParams.SerializeToString, + btrdb__pb2.FaultInjectResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Flush(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/v5api.BTrDB/Flush', + btrdb__pb2.FlushParams.SerializeToString, + btrdb__pb2.FlushResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Obliterate(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/v5api.BTrDB/Obliterate', + btrdb__pb2.ObliterateParams.SerializeToString, + btrdb__pb2.ObliterateResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def GetMetadataUsage(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/v5api.BTrDB/GetMetadataUsage', + btrdb__pb2.MetadataUsageParams.SerializeToString, + btrdb__pb2.MetadataUsageResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def GenerateCSV(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream(request, target, '/v5api.BTrDB/GenerateCSV', + btrdb__pb2.GenerateCSVParams.SerializeToString, + btrdb__pb2.GenerateCSVResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def SQLQuery(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream(request, target, '/v5api.BTrDB/SQLQuery', + btrdb__pb2.SQLQueryParams.SerializeToString, + btrdb__pb2.SQLQueryResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Subscribe(request_iterator, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.stream_stream(request_iterator, target, '/v5api.BTrDB/Subscribe', + btrdb__pb2.SubscriptionUpdate.SerializeToString, + btrdb__pb2.SubscriptionResp.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/btrdb/point.py b/btrdb/point.py index fa767f9..00cb7b0 100644 --- a/btrdb/point.py +++ b/btrdb/point.py @@ -17,11 +17,11 @@ from btrdb.grpcinterface import btrdb_pb2 - ########################################################################## ## Point Classes ########################################################################## + class RawPoint(object): """ A point of data representing a single position within a time series. Each @@ -75,7 +75,7 @@ def __getitem__(self, index): @staticmethod def to_proto(point): - return btrdb_pb2.RawPoint(time = point[0], value = point[1]) + return btrdb_pb2.RawPoint(time=point[0], value=point[1]) @staticmethod def to_proto_list(points): @@ -148,7 +148,9 @@ def __init__(self, time, minv, meanv, maxv, count, stddev): @classmethod def from_proto(cls, proto): - return cls(proto.time, proto.min, proto.mean, proto.max, proto.count, proto.stddev) + return cls( + proto.time, proto.min, proto.mean, proto.max, proto.count, proto.stddev + ) @classmethod def from_proto_list(cls, proto_list): @@ -214,12 +216,7 @@ def __getitem__(self, index): def __repr__(self): return "StatPoint({}, {}, {}, {}, {}, {})".format( - self.time, - self.min, - self.mean, - self.max, - self.count, - self.stddev + self.time, self.min, self.mean, self.max, self.count, self.stddev ) def __str__(self): @@ -230,9 +227,11 @@ def __eq__(self, other): if not hasattr(other, attr): return False - return self.time == other.time and \ - self.min == other.min and \ - self.mean == other.mean and \ - self.max == other.max and \ - self.count == other.count and \ - self.stddev == other.stddev + return ( + self.time == other.time + and self.min == other.min + and self.mean == other.mean + and self.max == other.max + and self.count == other.count + and self.stddev == other.stddev + ) diff --git a/btrdb/stream.py b/btrdb/stream.py index 109346c..e1cd2d7 100644 --- a/btrdb/stream.py +++ b/btrdb/stream.py @@ -14,48 +14,69 @@ ########################################################################## ## Imports ########################################################################## - -import re import json +import logging +import re +import uuid import uuid as uuidlib -from copy import deepcopy +import warnings +from collections import deque from collections.abc import Sequence +from copy import deepcopy +from typing import TYPE_CHECKING, Dict, List + +import pyarrow +import pyarrow.compute as pc + +try: + import pyarrow as pa +except ImportError: + pa = None + +if TYPE_CHECKING: + import pyarrow as pa -from btrdb.utils.buffer import PointBuffer -from btrdb.point import RawPoint, StatPoint -from btrdb.transformers import StreamSetTransformer -from btrdb.utils.timez import currently_as_ns, to_nanoseconds -from btrdb.utils.conversion import AnnotationEncoder, AnnotationDecoder -from btrdb.utils.general import pointwidth as pw from btrdb.exceptions import ( BTrDBError, BTRDBTypeError, BTRDBValueError, - InvalidOperation, InvalidCollection, + InvalidOperation, + NoSuchPoint, StreamNotFoundError, - NoSuchPoint + retry, ) - +from btrdb.point import RawPoint, StatPoint +from btrdb.transformers import _STAT_PROPERTIES, StreamSetTransformer +from btrdb.utils.buffer import PointBuffer +from btrdb.utils.conversion import AnnotationDecoder, AnnotationEncoder +from btrdb.utils.general import pointwidth as pw +from btrdb.utils.timez import currently_as_ns, to_nanoseconds ########################################################################## ## Module Variables ########################################################################## - +logger = logging.getLogger(__name__) +IS_DEBUG = logger.isEnabledFor(logging.DEBUG) INSERT_BATCH_SIZE = 50000 MINIMUM_TIME = -(16 << 56) MAXIMUM_TIME = (48 << 56) - 1 + +_ARROW_IMPORT_MSG = """Package pyarrow required, please pip install.""" + try: RE_PATTERN = re._pattern_type except Exception: RE_PATTERN = re.Pattern +_arrow_not_impl_str = "The BTrDB server you are using does not support {}." ########################################################################## ## Stream Classes ########################################################################## + class Stream(object): """ An object that represents a specific time series stream in the BTrDB database. @@ -72,30 +93,44 @@ class Stream(object): """ def __init__(self, btrdb, uuid, **db_values): - db_args = ('known_to_exist', 'collection', 'tags', 'annotations', 'property_version') + db_args = ( + "known_to_exist", + "collection", + "tags", + "annotations", + "property_version", + ) for key in db_args: value = db_values.pop(key, None) setattr(self, "_{}".format(key), value) if db_values: bad_keys = ", ".join(db_values.keys()) - raise BTRDBTypeError("got unexpected db_values argument(s) '{}'".format(bad_keys)) + raise BTRDBTypeError( + "got unexpected db_values argument(s) '{}'".format(bad_keys) + ) self._btrdb = btrdb self._uuid = uuid def refresh_metadata(self): """ - Refreshes the locally cached meta data for a stream + Refreshes the locally cached metadata for a stream from the server. Queries the BTrDB server for all stream metadata including collection, annotation, and tags. This method requires a round trip to the server. """ ep = self._btrdb.ep - self._collection, self._property_version, self._tags, self._annotations, _ = ep.streamInfo(self._uuid, False, True) + ( + self._collection, + self._property_version, + self._tags, + self._annotations, + _, + ) = ep.streamInfo(self._uuid, False, True) self._known_to_exist = True - # deserialize annoation values + # deserialize annotation values self._annotations = { key: json.loads(val, cls=AnnotationDecoder) for key, val in self._annotations.items() @@ -118,6 +153,18 @@ def exists(self): ------- bool Indicates whether stream is extant in the BTrDB server. + + Examples + -------- + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream = conn.stream_from_uuid("...") + >>> stream.uuid + UUID('...') + >>> stream.exists() + True + + """ if self._known_to_exist: @@ -131,20 +178,19 @@ def exists(self): return False raise bte - def count(self, start=MINIMUM_TIME, end=MAXIMUM_TIME, pointwidth=62, precise=False, version=0): + def count( + self, + start=MINIMUM_TIME, + end=MAXIMUM_TIME, + pointwidth=62, + precise=False, + version=0, + ): """ Compute the total number of points in the stream Counts the number of points in the specified window and version. By - default returns the latest total count of points in the stream. This - helper method sums the counts of all StatPoints returned by - ``aligned_windows``. Because of this, note that the start and end - timestamps may be adjusted if they are not powers of 2. For smaller - windows of time, you may also need to adjust the pointwidth to ensure - that the count granularity is captured appropriately. - - Alternatively you can set the precise argument to True which will - give an exact count to the nanosecond but may be slower to execute. + default, returns the latest total count of points in the stream. Parameters ---------- @@ -173,32 +219,70 @@ def count(self, start=MINIMUM_TIME, end=MAXIMUM_TIME, pointwidth=62, precise=Fal ------- int The total number of points in the stream for the specified window. + + + .. note:: + + This helper method sums the counts of all StatPoints returned by + ``aligned_windows``. Because of this, note that the start and end + timestamps may be adjusted if they are not powers of 2. For smaller + windows of time, you may also need to adjust the ``pointwidth`` to ensure + that the count granularity is captured appropriately. + + Alternatively you can set the ``precise`` argument to ``True`` which will + give an exact count to the nanosecond but may be slower to execute. + + + + Examples + -------- + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream = conn.stream_from_uuid("...") + >>> stream.count() + 1234 + >>> stream.count(start=1500000000000000000, end=1603680000000000000, pointwidth=55) + 567 + >>> stream.count(start=1500000000000000000, end=1603680000000000000, precise=True) + 789 """ if not precise: - pointwidth = min(pointwidth, pw.from_nanoseconds(to_nanoseconds(end) - to_nanoseconds(start))-1) + pointwidth = min( + pointwidth, + pw.from_nanoseconds(to_nanoseconds(end) - to_nanoseconds(start)) - 1, + ) points = self.aligned_windows(start, end, pointwidth, version) - return sum([point.count for point, _ in points]) + return sum([point.count for point, _ in points]) depth = 0 width = to_nanoseconds(end) - to_nanoseconds(start) points = self.windows(start, end, width, depth, version) - return sum([point.count for point, _ in points]) - + return sum([point.count for point, _ in points]) @property def btrdb(self): """ - Returns the stream's BTrDB object. + Returns the stream's BTrDB object. - Parameters - ---------- - None + Parameters + ---------- + None - Returns - ------- - BTrDB - The BTrDB database object. + Returns + ------- + BTrDB + The BTrDB database object. + + Examples + -------- + + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream = conn.stream_from_uuid("...") + >>> btrdb_obj = stream.btrdb + >>> btrdb_obj + """ return self._btrdb @@ -206,7 +290,7 @@ def btrdb(self): @property def uuid(self): """ - Returns the stream's UUID. The stream may nor may not exist + Returns the stream's UUID. The stream may or may not exist yet, depending on how the stream object was obtained. Returns @@ -234,6 +318,14 @@ def name(self): str The name of the stream. + Examples + -------- + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream = conn.stream_from_uuid("...") + >>> stream.name + 'foo' + """ return self.tags()["name"] @@ -249,6 +341,14 @@ def unit(self): str The unit for values of the stream. + Examples + -------- + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream = conn.stream_from_uuid("...") + >>> stream.unit + 'volts' + """ return self.tags()["unit"] @@ -267,6 +367,14 @@ def collection(self): str the collection of the stream + Examples + -------- + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream = conn.stream_from_uuid("...") + >>> stream.collection + 'foo/bar' + """ if self._collection is not None: return self._collection @@ -274,7 +382,15 @@ def collection(self): self.refresh_metadata() return self._collection - def earliest(self, version=0): + @retry + def earliest( + self, + version=0, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Returns the first point of data in the stream. Returns None if error encountered during lookup or no values in stream. @@ -284,6 +400,17 @@ def earliest(self, version=0): version : int, default: 0 Specify the version of the stream to query; if zero, queries the latest stream state rather than pinning to a version. + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- @@ -291,11 +418,33 @@ def earliest(self, version=0): The first data point in the stream and the version of the stream the value was retrieved at (tuple(RawPoint, int)). + + Examples + -------- + Get the earliest point for a stream using ``version`` ``0``. + + >>> stream.earliest(version=0) + (, 1234567) + + Extract just the ``RawPoint`` data. + + >>> pt, _ = stream.earliest(version=0) + >>> print(pt.time, pt.value) + 1547241923338098176 123.7 + """ start = MINIMUM_TIME return self.nearest(start, version=version, backward=False) - def latest(self, version=0): + @retry + def latest( + self, + version=0, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Returns last point of data in the stream. Returns None if error encountered during lookup or no values in stream. @@ -305,6 +454,17 @@ def latest(self, version=0): version : int, default: 0 Specify the version of the stream to query; if zero, queries the latest stream state rather than pinning to a version. + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- @@ -312,21 +472,52 @@ def latest(self, version=0): The last data point in the stream and the version of the stream the value was retrieved at (tuple(RawPoint, int)). + Examples + -------- + Get the latest point for a stream using ``version`` ``0``. + + >>> stream.latest(version=0) + (, 1234567) + + Extract just the ``RawPoint`` data. + + >>> pt, _ = stream.latest(version=0) + >>> print(pt.time, pt.value) + 1547241923338098176 123.7 """ start = MAXIMUM_TIME return self.nearest(start, version=version, backward=True) - def current(self, version=0): + @retry + def current( + self, + version=0, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Returns the point that is closest to the current timestamp, e.g. the latest point in the stream up until now. Note that no future values will be returned. - Returns None if errors during lookup or there are no values before now. + Returns None if errors occur during lookup or there are no values before now. Parameters ---------- version : int, default: 0 Specify the version of the stream to query; if zero, queries the latest stream state rather than pinning to a version. + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- @@ -337,7 +528,15 @@ def current(self, version=0): start = currently_as_ns() return self.nearest(start, version, backward=True) - def tags(self, refresh=False): + @retry + def tags( + self, + refresh=False, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Returns the stream's tags. @@ -346,9 +545,20 @@ def tags(self, refresh=False): Parameters ---------- - refresh: bool + refresh: bool, default: False Indicates whether a round trip to the server should be implemented regardless of whether there is a local copy. + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- @@ -361,22 +571,39 @@ def tags(self, refresh=False): return deepcopy(self._tags) - def annotations(self, refresh=False): + @retry + def annotations( + self, + refresh=False, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Returns a stream's annotations Annotations returns the annotations of the stream (and the annotation - version). It will always require a round trip to the server. If you are - ok with stale data and want a higher performance version, use - Stream.CachedAnnotations(). + version). Do not modify the resulting map. Parameters ---------- - refresh: bool + refresh: bool, default: False Indicates whether a round trip to the server should be implemented regardless of whether there is a local copy. + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- @@ -384,23 +611,61 @@ def annotations(self, refresh=False): A tuple containing a dictionary of annotations and an integer representing the version of the metadata (tuple(dict, int)). + + .. note:: + + This ``version`` is not the same as the ``stream.version``. + + Examples + -------- + Accessing a streams annotations. + + >>> stream.annotations() + ({"foo":"bar", "baz":"bazaar"}, 231) + + Extract the version and metadata separately. + + >>> annotations, metadata_version = stream.annotations() + >>> annotations + {"foo":"bar", "baz":"bazaar"} + >>> metadata_version + 231 """ if refresh or self._annotations is None: self.refresh_metadata() return deepcopy(self._annotations), deepcopy(self._property_version) - def version(self): + @retry + def version( + self, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Returns the current data version of the stream. - Version returns the current data version of the stream. This is not - cached, it queries each time. Take care that you do not intorduce races - in your code by assuming this function will always return the same vaue + .. warning:: + + Version returns the current data version of the stream. This is not + cached, it queries each time. Take care that you do not introduce races + in your code by assuming this function will always return the same value. Parameters ---------- - None + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- @@ -410,7 +675,7 @@ def version(self): """ return self._btrdb.ep.streamInfo(self._uuid, True, False)[4] - def insert(self, data, merge='never'): + def insert(self, data, merge="never"): """ Insert new data in the form (time, value) into the series. @@ -437,29 +702,145 @@ def insert(self, data, merge='never'): int The version of the stream after inserting new points. + + Examples + -------- + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream = conn.stream_from_uuid("...") + >>> data = [(1500000000000000000, 1.0), (1500000000100000000, 2.0)] + >>> stream.insert(data) + 1234 + >>> stream.insert(data, merge="replace") + 1235 """ - i = 0 version = 0 + i = 0 while i < len(data): - thisBatch = data[i:i + INSERT_BATCH_SIZE] + thisBatch = data[i : i + INSERT_BATCH_SIZE] version = self._btrdb.ep.insert(self._uuid, thisBatch, merge) i += INSERT_BATCH_SIZE return version + def arrow_insert(self, data: pa.Table, merge: str = "never") -> int: + """ + Insert new data in the form of a pyarrow Table with (time, value) columns. + + Inserts a table of new (time, value) columns into the stream. The values + in the table need not be sorted by time. If the arrays are larger than + appropriate, this function will automatically chunk the inserts. As a + consequence, the insert is not necessarily atomic, but can be used with + a very large array. + + Parameters + ---------- + data: pyarrow.Table, required + A pyarrow table with a schema of time:Timestamp[ns, tz=UTC], value:float64 + This schema will be validated and converted if necessary. + merge: str + A string describing the merge policy. Valid policies are: + - 'never': the default, no points are merged + - 'equal': points are deduplicated if the time and value are equal + - 'retain': if two points have the same timestamp, the old one is kept + - 'replace': if two points have the same timestamp, the new one is kept + + Returns + ------- + int + The version of the stream after inserting new points. + + + .. note:: + + This method is available for commercial customers with arrow-enabled servers. + + Examples + -------- + Assuming we have a sequence of ``times`` and ``values`` where ``times`` are in nanoseconds. + Insert the data as a pyarrow table, and if there are duplicate timestamps already in the database, + replace with the new ones in ``payload``. + + >>> conn = btrdb.connect() + >>> import pyarrow as pa + >>> for t, v in zip(times, vals): + ... print(t,v) + 1500000000000000000 1.0 + 1500000000100000000 2.0 + 1500000000200000000 3.0 + 1500000000300000000 4.0 + 1500000000400000000 5.0 + 1500000000500000000 6.0 + 1500000000600000000 7.0 + 1500000000700000000 8.0 + 1500000000800000000 9.0 + 1500000000900000000 10.0 + >>> schema = pa.schema( + ... [ + ... pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False), + ... pa.field("value", pa.float64(), nullable=False), + ... ] + ... ) + >>> payload = pa.Table.from_arrays([times, vals], schema=schema) + >>> version = stream.arrow_insert(payload, merge="replace") + """ + if not self._btrdb._ARROW_ENABLED: + raise NotImplementedError(_arrow_not_impl_str.format("arrow_insert")) + if pa is None: + raise ImportError(_ARROW_IMPORT_MSG) + chunksize = INSERT_BATCH_SIZE + assert isinstance(data, pa.Table) + tmp_table = data.rename_columns(["time", "value"]) + new_schema = pa.schema( + [ + (pa.field("time", pa.timestamp(unit="ns", tz="UTC"), nullable=False)), + (pa.field("value", pa.float64(), nullable=False)), + ] + ) + tmp_table = tmp_table.cast(new_schema) + num_rows = tmp_table.num_rows + + # Calculate the number of batches based on the chunk size + num_batches = num_rows // chunksize + if num_rows % chunksize != 0 or num_batches == 0: + num_batches = num_batches + 1 + + table_slices = [] + + for i in range(num_batches): + start_idx = i * chunksize + t = tmp_table.slice(offset=start_idx, length=chunksize) + table_slices.append(t) + + # Process the batches as needed + version = [] + for tab in table_slices: + version.append( + self._btrdb.ep.arrowInsertValues(uu=self.uuid, values=tab, policy=merge) + ) + return max(version) + def _update_tags_collection(self, tags, collection): - tags = self.tags() if tags is None else tags + """ + :meta private: + """ + tags = {} if tags is None else tags collection = self.collection if collection is None else collection if collection is None: - raise BTRDBValueError("collection must be provided to update tags or collection") + raise BTRDBValueError( + "collection must be provided to update tags or collection" + ) self._btrdb.ep.setStreamTags( uu=self.uuid, expected=self._property_version, tags=tags, - collection=collection + collection=collection, ) def _update_annotations(self, annotations, encoder, replace): + """ + :meta private: + """ # make a copy of the annotations to prevent accidental mutable object mutation serialized = deepcopy(annotations) if encoder is not None: @@ -470,16 +851,30 @@ def _update_annotations(self, annotations, encoder, replace): removals = [] if replace: - removals = [i for i in self._annotations.keys() if i not in annotations.keys()] + removals = [ + i for i in self._annotations.keys() if i not in annotations.keys() + ] self._btrdb.ep.setStreamAnnotations( uu=self.uuid, expected=self._property_version, changes=serialized, - removals=removals + removals=removals, ) - def update(self, tags=None, annotations=None, collection=None, encoder=AnnotationEncoder, replace=False): + @retry + def update( + self, + tags=None, + annotations=None, + collection=None, + encoder=AnnotationEncoder, + replace=False, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Updates metadata including tags, annotations, and collection as an UPSERT operation. @@ -491,14 +886,10 @@ def update(self, tags=None, annotations=None, collection=None, encoder=Annotatio None, they will remain unchanged in the database. To delete either tags or annotations, you must specify exactly which - keys and values you want set for the field and set `replace=True`. For - example: + keys and values you want set for the field and set ``replace=True``. - >>> annotations, _ = stream.anotations() - >>> del annotations["key_to_delete"] - >>> stream.update(annotations=annotations, replace=True) - This ensures that all of the keys and values for the annotations are + This ensures that all the keys and values for the annotations are preserved except for the key to be deleted. Parameters @@ -514,12 +905,23 @@ def update(self, tags=None, annotations=None, collection=None, encoder=Annotatio Specify a new collection for the stream. If None, the collection will remain unchanged. encoder : json.JSONEncoder or None - JSON encoder class to use for annotation serialization. Set to None + JSON encoder class to use for annotation serialization. Set to ``None`` to prevent JSON encoding of the annotations. replace : bool, default: False Replace all annotations or tags with the specified dictionaries - instead of performing the normal upsert operation. Specifying True + instead of performing the normal upsert operation. Specifying ``True`` is the only way to remove annotation keys. + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- @@ -527,9 +929,22 @@ def update(self, tags=None, annotations=None, collection=None, encoder=Annotatio The version of the metadata (separate from the version of the data) also known as the "property version". + + Examples + -------- + >>> annotations, _ = stream.anotations() + >>> del annotations["key_to_delete"] + >>> stream.update(annotations=annotations, replace=True) + 12345 + >>> annotations, _ = stream.annotations() + >>> "key_to_delete" in annotations + False + """ if tags is None and annotations is None and collection is None: - raise BTRDBValueError("you must supply a tags, annotations, or collection argument") + raise BTRDBValueError( + "you must supply a tags, annotations, or collection argument" + ) if tags is not None and isinstance(tags, dict) is False: raise BTRDBTypeError("tags must be of type dict") @@ -550,34 +965,81 @@ def update(self, tags=None, annotations=None, collection=None, encoder=Annotatio return self._property_version - def delete(self, start, end): + @retry + def delete( + self, + start, + end, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ - "Delete" all points between [`start`, `end`) + "Delete" all points between [``start``, ``end``) - "Delete" all points between `start` (inclusive) and `end` (exclusive), - both in nanoseconds. As BTrDB has persistent multiversioning, the - deleted points will still exist as part of an older version of the - stream. + "Delete" all points between ``start`` (inclusive) and ``end`` (exclusive), + both in nanoseconds. + + .. note:: + + As ``BTrDB`` has persistent multiversioning, the + deleted points will still exist as part of an older version of the + stream. Parameters ---------- start : int or datetime like object The start time in nanoseconds for the range to be deleted. (see - :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) + :func:``btrdb.utils.timez.to_nanoseconds`` for valid input types) end : int or datetime like object The end time in nanoseconds for the range to be deleted. (see - :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) + :func:``btrdb.utils.timez.to_nanoseconds`` for valid input types) + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- int The version of the new stream created - """ - return self._btrdb.ep.deleteRange(self._uuid, to_nanoseconds(start), - to_nanoseconds(end)) - def values(self, start, end, version=0): + Examples + -------- + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream = conn.stream_from_uuid("...") + >>> start = 1500000000000000000 + >>> end = 1500000001000000000 + >>> stream.delete(start, end) + 1234 + >>> stream.count(start=start, end=end) + 0 + """ + return self._btrdb.ep.deleteRange( + self._uuid, to_nanoseconds(start), to_nanoseconds(end) + ) + + @retry + def values( + self, + start, + end, + version=0, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Read raw values from BTrDB between time [a, b) in nanoseconds. @@ -595,6 +1057,17 @@ def values(self, start, end, version=0): :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) version: int The version of the stream to be queried + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------ @@ -603,40 +1076,138 @@ def values(self, start, end, version=0): version (list(tuple(RawPoint,int))). - Notes - ----- - Note that the raw data points are the original values at the sensor's - native sampling rate (assuming the time series represents measurements - from a sensor). This is the lowest level of data with the finest time - granularity. In the tree data structure of BTrDB, this data is stored in - the vector nodes. + .. note:: + + Note that the raw data points are the original values at the sensor's + native sampling rate (assuming the time series represents measurements + from a sensor). This is the lowest level of data with the finest time + granularity. In the tree data structure of BTrDB, this data is stored in + the vector nodes. """ materialized = [] start = to_nanoseconds(start) end = to_nanoseconds(end) - point_windows = self._btrdb.ep.rawValues(self._uuid, start, end, version) for point_list, version in point_windows: for point in point_list: materialized.append((RawPoint.from_proto(point), version)) return materialized - def aligned_windows(self, start, end, pointwidth, version=0): + @retry + def arrow_values( + self, + start, + end, + version: int = 0, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + schema=None, + ) -> pa.Table: + """Read raw values from BTrDB between time [a, b) in nanoseconds. + + RawValues queries BTrDB for the raw time series data points between + `start` and `end` time, both in nanoseconds since the Epoch for the + specified stream `version`. + + start : int or datetime like object + The start time in nanoseconds for the range to be queried. (see + :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) + end : int or datetime like object + The end time in nanoseconds for the range to be queried. (see + :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) + version: int, default: 0 + The version of the stream to be queried + schema: pyarrow.Schema + Optional arrow schema the server will cast the returned data to before sending it over + the network. You can use this to change the timestamp format, column names or data sizes. + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False + + + Returns + ------ + pyarrow.Table + A pyarrow table of the raw values with time and value columns. + + + .. note:: + + Note that the raw data points are the original values at the sensor's + native sampling rate (assuming the time series represents measurements + from a sensor). This is the lowest level of data with the finest time + granularity. In the tree data structure of BTrDB, this data is stored in + the vector nodes. + + + .. note:: + + This method is available for commercial customers with arrow-enabled servers. + + + """ + if not self._btrdb._ARROW_ENABLED: + raise NotImplementedError(_arrow_not_impl_str.format("arrow_values")) + if pa is None: + raise ImportError(_ARROW_IMPORT_MSG) + start = to_nanoseconds(start) + end = to_nanoseconds(end) + arrow_and_versions = self._btrdb.ep.arrowRawValues( + uu=self.uuid, start=start, end=end, version=version, schema=schema + ) + tables = list(arrow_and_versions) + if len(tables) > 0: + tabs, ver = zip(*tables) + return pa.concat_tables(tabs) + else: + if schema is None: + schema = pa.schema( + [ + pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False), + pa.field("value", pa.float64(), nullable=False), + ] + ) + return pa.Table.from_arrays([pa.array([]), pa.array([])], schema=schema) + + @retry + def aligned_windows( + self, + start, + end, + pointwidth, + version=0, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Read statistical aggregates of windows of data from BTrDB. Query BTrDB for aggregates (or roll ups or windows) of the time series - with `version` between time `start` (inclusive) and `end` (exclusive) in + with `version` between time ``start`` (inclusive) and ``end`` (exclusive) in nanoseconds. Each point returned is a statistical aggregate of all the - raw data within a window of width 2**`pointwidth` nanoseconds. These + raw data within a window of width ``2**pointwidth`` nanoseconds. These statistical aggregates currently include the mean, minimum, and maximum of the data and the count of data points composing the window. - Note that `start` is inclusive, but `end` is exclusive. That is, results + .. note:: + + ``start`` is inclusive, but ``end`` is exclusive. That is, results will be returned for all windows that start in the interval [start, end). If end < start+2^pointwidth you will not get any results. If start and - end are not powers of two, the bottom pointwidth bits will be cleared. + end are not powers of two, the bottom ``pointwidth`` bits will be cleared. Each window will contain statistical summaries of the window. Statistical points with count == 0 will be omitted. @@ -652,6 +1223,17 @@ def aligned_windows(self, start, end, pointwidth, version=0): Specify the number of ns between data points (2**pointwidth) version : int Version of the stream to query + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- @@ -660,23 +1242,141 @@ def aligned_windows(self, start, end, pointwidth, version=0): containing data tuples. Each data tuple contains a StatPoint and the stream version. - Notes - ----- - As the window-width is a power-of-two, it aligns with BTrDB internal - tree data structure and is faster to execute than `windows()`. + + .. note:: + + As the window-width is a power-of-two, it aligns with ``BTrDB`` internal + tree data structure and is faster to execute than ``windows()``. + """ materialized = [] start = to_nanoseconds(start) end = to_nanoseconds(end) - - windows = self._btrdb.ep.alignedWindows(self._uuid, start, end, pointwidth, version) + windows = self._btrdb.ep.alignedWindows( + self._uuid, start, end, pointwidth, version + ) for stat_points, version in windows: for point in stat_points: materialized.append((StatPoint.from_proto(point), version)) - return tuple(materialized) - def windows(self, start, end, width, depth=0, version=0): + @retry + def arrow_aligned_windows( + self, + start: int, + end: int, + pointwidth: int, + version: int = 0, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ) -> pa.Table: + """Read statistical aggregates of windows of data from BTrDB. + + Query BTrDB for aggregates (or roll ups or windows) of the time series + with `version` between time `start` (inclusive) and `end` (exclusive) in + nanoseconds [start, end). Each point returned is a statistical aggregate of all the + raw data within a window of width 2**`pointwidth` nanoseconds. These + statistical aggregates currently include the mean, minimum, and maximum + of the data and the count of data points composing the window. + + + .. note:: + + ``start`` is inclusive, but ``end`` is exclusive. That is, results + will be returned for all windows that start in the interval [start, end). + If end < start+2^pointwidth you will not get any results. If start and + end are not powers of two, the bottom ``pointwidth`` bits will be cleared. + Each window will contain statistical summaries of the window. + Statistical points with ``count == 0`` will be omitted. + + Parameters + ---------- + start : int or datetime like object, required + The start time in nanoseconds for the range to be queried. (see + :func:``btrdb.utils.timez.to_nanoseconds`` for valid input types) + end : int or datetime like object, required + The end time in nanoseconds for the range to be queried. (see + :func:``btrdb.utils.timez.to_nanoseconds`` for valid input types) + pointwidth : int, required + Specify the number of ns between data points (2**pointwidth) + version : int, default: 0 + Version of the stream to query + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False + + Returns + ------- + pyarrow.Table + Returns a pyarrow table containing the windows of data. + + + .. note:: + + As the window-width is a power-of-two, it aligns with BTrDB internal + tree data structure and is faster to execute than `windows()`. + + .. note:: + + This method is available for commercial customers with arrow-enabled servers. + """ + if not self._btrdb._ARROW_ENABLED: + raise NotImplementedError( + _arrow_not_impl_str.format("arrow_aligned_windows") + ) + if pa is None: + raise ImportError(_ARROW_IMPORT_MSG) + + if IS_DEBUG: + logger.debug(f"For stream - {self.uuid} - {self.name}") + start = to_nanoseconds(start) + end = to_nanoseconds(end) + tables = list( + self._btrdb.ep.arrowAlignedWindows( + self.uuid, start=start, end=end, pointwidth=pointwidth, version=version + ) + ) + if len(tables) > 0: + tabs, ver = zip(*tables) + # assume that time column is the first column returned + time_col = tabs[0].column_names[0] + return pa.concat_tables(tabs).sort_by(time_col) + else: + schema = pa.schema( + [ + pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False), + pa.field("min", pa.float64(), nullable=False), + pa.field("mean", pa.float64(), nullable=False), + pa.field("max", pa.float64(), nullable=False), + pa.field("count", pa.uint64(), nullable=False), + pa.field("stddev", pa.float64(), nullable=False), + ] + ) + return pa.Table.from_arrays([pa.array([]) for _ in range(5)], schema=schema) + + @retry + def windows( + self, + start, + end, + width, + depth=0, + version=0, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Read arbitrarily-sized windows of data from BTrDB. StatPoint objects will be returned representing the data for each window. @@ -690,13 +1390,20 @@ def windows(self, start, end, width, depth=0, version=0): The end time in nanoseconds for the range to be queried. (see :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) width : int - The number of nanoseconds in each window, subject to the depth - parameter. - depth : int - The precision of the window duration as a power of 2 in nanoseconds. - E.g 30 would make the window duration accurate to roughly 1 second + The number of nanoseconds in each window. version : int The version of the stream to query. + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- @@ -705,30 +1412,25 @@ def windows(self, start, end, width, depth=0, version=0): containing data tuples. Each data tuple contains a StatPoint and the stream version (tuple(tuple(StatPoint, int), ...)). - Notes - ----- - Windows returns arbitrary precision windows from BTrDB. It is slower - than AlignedWindows, but still significantly faster than RawValues. Each - returned window will be `width` nanoseconds long. `start` is inclusive, - but `end` is exclusive (e.g if end < start+width you will get no - results). That is, results will be returned for all windows that start - at a time less than the end timestamp. If (`end` - `start`) is not a - multiple of width, then end will be decreased to the greatest value less - than end such that (end - start) is a multiple of `width` (i.e., we set - end = start + width * floordiv(end - start, width). The `depth` - parameter is an optimization that can be used to speed up queries on - fast queries. Each window will be accurate to 2^depth nanoseconds. If - depth is zero, the results are accurate to the nanosecond. On a dense - stream for large windows, this accuracy may not be required. For example - for a window of a day, +- one second may be appropriate, so a depth of - 30 can be specified. This is much faster to execute on the database - side. + + .. note:: + + ``windows`` returns arbitrary precision windows from BTrDB. It is slower + than ``aligned_windows``, but can be significantly faster than raw value queries (``values``). Each + returned window will be ``width`` nanoseconds long. ``start`` is inclusive, + but ``end`` is exclusive (e.g if ``end < start+width`` you will get no + results). That is, results will be returned for all windows that start + at a time less than the end timestamp. If (``end`` - ``start``) is not a + multiple of ``width``, then ``end`` will be decreased to the greatest value less + than ``end`` such that (``end`` - ``start``) is a multiple of ``width`` (i.e., we set + ``end = start + width * floordiv(end - start, width)``. The ``depth`` + parameter previously available has been deprecated. The only valid value + for ``depth`` is now ``0``. """ materialized = [] start = to_nanoseconds(start) end = to_nanoseconds(end) - windows = self._btrdb.ep.windows(self._uuid, start, end, width, depth, version) for stat_points, version in windows: for point in stat_points: @@ -736,25 +1438,141 @@ def windows(self, start, end, width, depth=0, version=0): return tuple(materialized) - def nearest(self, time, version, backward=False): + @retry + def arrow_windows( + self, + start: int, + end: int, + width: int, + version: int = 0, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ) -> pa.Table: + """Read arbitrarily-sized windows of data from BTrDB. + + Parameters + ---------- + start : int or datetime like object, required + The start time in nanoseconds for the range to be queried. (see + :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) + end : int or datetime like object, required + The end time in nanoseconds for the range to be queried. (see + :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) + width : int, required + The number of nanoseconds in each window. + version : int, default=0, optional + The version of the stream to query. + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False + + Returns + ------- + pyarrow.Table + Returns a pyarrow Table containing windows of data. + + + .. note:: + + ``windows`` returns arbitrary precision windows from ``BTrDB``. It is slower + than ``aligned_windows``, but still significantly faster than RawValues. Each + returned window will be ``width`` nanoseconds long. ``start`` is inclusive, + but ``end`` is exclusive (e.g if end < start+width you will get no + results). That is, results will be returned for all windows that start + at a time less than the ``end`` timestamp. If (``end`` - ``start``) is not a + multiple of ``width``, then ``end`` will be decreased to the greatest value less + than ``end`` such that (``end`` - ``start``) is a multiple of `width` (i.e., we set + ``end = start + width * floordiv(end - start, width)``. The ``depth`` + parameter previously available has been deprecated. The only valid value + for ``depth`` is now 0. + + .. note:: + + This method is available for commercial customers with arrow-enabled servers. + """ + if not self._btrdb._ARROW_ENABLED: + raise NotImplementedError(_arrow_not_impl_str.format("arrow_windows")) + if pa is None: + raise ImportError(_ARROW_IMPORT_MSG) + start = to_nanoseconds(start) + end = to_nanoseconds(end) + tables = list( + self._btrdb.ep.arrowWindows( + self.uuid, + start=start, + end=end, + width=width, + depth=0, + version=version, + ) + ) + if len(tables) > 0: + tabs, ver = zip(*tables) + time_col = tabs[0].column_names[0] + return pa.concat_tables(tabs).sort_by(time_col) + else: + schema = pa.schema( + [ + pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False), + pa.field("min", pa.float64(), nullable=False), + pa.field("mean", pa.float64(), nullable=False), + pa.field("max", pa.float64(), nullable=False), + pa.field("count", pa.uint64(), nullable=False), + pa.field("stddev", pa.float64(), nullable=False), + ] + ) + return pa.Table.from_arrays([pa.array([]) for _ in range(5)], schema=schema) + + @retry + def nearest( + self, + time, + version, + backward=False, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Finds the closest point in the stream to a specified time. - Return the point nearest to the specified `time` in nanoseconds since - Epoch in the stream with `version` while specifying whether to search - forward or backward in time. If `backward` is false, the returned point - will be >= `time`. If backward is true, the returned point will be < - `time`. The version of the stream used to satisfy the query is returned. + Return the point nearest to the specified ``time`` in nanoseconds since + Epoch in the stream with ``version`` while specifying whether to search + forward or backward in time. If ``backward`` is ``false``, the returned point + will be >= ``time``. If ``backward`` is ``true``, the returned point will be < + ``time``. The ``version`` of the ``stream`` used to satisfy the query is returned. Parameters ---------- time : int or datetime like object The time (in nanoseconds since Epoch) to search near (see - :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) + :func:``btrdb.utils.timez.to_nanoseconds`` for valid input types) version : int Version of the stream to use in search backward : boolean True to search backwards from time, else false for forward + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- @@ -764,8 +1582,15 @@ def nearest(self, time, version, backward=False): """ try: - rp, version = self._btrdb.ep.nearest(self._uuid, - to_nanoseconds(time), version, backward) + if IS_DEBUG: + logger.debug( + f"checking nearest for: {self.uuid}\t\t{time}\t\t{version}" + ) + rp, version = self._btrdb.ep.nearest( + self._uuid, to_nanoseconds(time), version, backward + ) + if IS_DEBUG: + logger.debug(f"Nearest for stream: {self.uuid} - {rp}") except BTrDBError as exc: if not isinstance(exc, NoSuchPoint): raise @@ -773,62 +1598,121 @@ def nearest(self, time, version, backward=False): return RawPoint.from_proto(rp), version - - def obliterate(self): + @retry + def obliterate( + self, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Obliterate deletes a stream from the BTrDB server. An exception will be raised if the stream could not be found. + Parameters + ---------- + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False + Raises ------ BTrDBError [404] stream does not exist The stream could not be found in BTrDB - """ self._btrdb.ep.obliterate(self._uuid) - def flush(self): + @retry + def flush( + self, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Flush writes the stream buffers out to persistent storage. + Parameters + ---------- + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False """ self._btrdb.ep.flush(self._uuid) def __repr__(self): - return "".format(self.collection, - self.name) + return "".format(self.collection, self.name) + ########################################################################## ## StreamSet Classes ########################################################################## + class StreamSetBase(Sequence): """ A lighweight wrapper around a list of stream objects """ - def __init__(self, streams): - self._streams = streams + def __init__(self, streams: List[Stream]): + self._streams: List[Stream] = [] + for stream in streams: + if not isinstance(stream, Stream): + raise BTRDBTypeError( + f"streams must be of type Stream {stream}, {type(stream)}" + ) + self._streams.append(stream) + if len(self._streams) < 1: + raise ValueError( + f"Trying to create streamset with an empty list of streams {self._streams}." + ) + try: + self._btrdb = self._streams[0]._btrdb + except Exception as e: + warnings.warn(f"Issue setting btrdb object from stream: {e}") self._pinned_versions = None self.filters = [] self.pointwidth = None self.width = None - self.depth = None + self.depth = 0 @property def allow_window(self): - return not bool(self.pointwidth or (self.width and self.depth)) + return not bool(self.pointwidth or (self.width and self.depth == 0)) def _latest_versions(self): - return {s.uuid: s.version() for s in self._streams} - + """ + :meta private: + """ + uuid_ver_tups = self._btrdb._executor.map( + lambda s: (s.uuid, s.version()), self._streams + ) + return {uu: v for uu, v in uuid_ver_tups} def pin_versions(self, versions=None): """ Saves the stream versions that future materializations should use. If no pin is requested then the first materialization will automatically - pin the return versions. Versions can also be supplied through a dict + pin the return versions. Versions can also be supplied through a ``dict`` object with key:UUID, value:stream.version(). Parameters @@ -841,6 +1725,10 @@ def pin_versions(self, versions=None): StreamSet Returns self + Examples + -------- + >>> version_map = {s.uuid: 0 for s in streamset} + >>> pinned_streamset = streamset.pin_versions(versions=version_map) """ if versions is not None: if not isinstance(versions, dict): @@ -850,7 +1738,6 @@ def pin_versions(self, versions=None): if not isinstance(key, uuidlib.UUID): raise BTRDBTypeError("version keys must be type UUID") - self._pinned_versions = self._latest_versions() if not versions else versions return self @@ -870,10 +1757,24 @@ def versions(self): dict A dict containing the stream UUID and version ints as key/values - """ - return self._pinned_versions if self._pinned_versions else self._latest_versions() - def count(self): + Examples + -------- + A pinned vs non-pinned streamset + + >>> streamset = btrdb.stream.StreamSet([stream1, stream2]) + >>> version_map = {s.uuid: 0 for s streamset} + >>> pinned_streamset = streamset.pin_versions(versions=version_map) + >>> pinned_streamset.versions() + {UUID('fa42f64a-a851-408f-aa7e-88a85b3d295c'): 0, UUID('18e5527a-ed13-424d-bb97-3e06a763609e'): 0} + >>> streamset.versions() + {UUID('fa42f64a-a851-408f-aa7e-88a85b3d295c'): 34532, UUID('18e5527a-ed13-424d-bb97-3e06a763609e'): 12345} + """ + return ( + self._pinned_versions if self._pinned_versions else self._latest_versions() + ) + + def count(self, precise: bool = False): """ Compute the total number of points in the streams using filters. @@ -882,32 +1783,53 @@ def count(self): all points in the streams. The count is modified by start and end filters or by pinning versions. - Note that this helper method sums the counts of all StatPoints returned - by ``aligned_windows``. Because of this the start and end timestamps - may be adjusted if they are not powers of 2. You can also set the - pointwidth property for smaller windows of time to ensure that the - count granularity is captured appropriately. - Parameters ---------- - None + precise : bool, default = False + Use statpoint counts using aligned_windows which trades accuracy for speed. Returns ------- int The total number of points in all streams for the specified filters. + + .. note:: + + Note that this helper method sums the counts of all StatPoints returned + by ``aligned_windows``. Because of this the start and end timestamps + may be adjusted if they are not powers of 2. + + Examples + -------- + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream1 = conn.stream_from_uuid("...") + >>> stream2 = conn.stream_from_uuid("...") + >>> streamset = btrdb.stream.StreamSet([stream1, stream2]) + >>> streamset.count() + 2345 + >>> filtered_streamset = streamset.filter(start=1500000000000000000, end=1500000001000000000) + >>> filtered_streamset.count(precise=True) + 734 + >>> streamset.filter(start=1500000000000000000, end=1500000001000000000).count(precise=True) + 734 """ params = self._params_from_filters() start = params.get("start", MINIMUM_TIME) end = params.get("end", MAXIMUM_TIME) - versions = self._pinned_versions if self._pinned_versions else {} - count = 0 - for s in self._streams: - version = versions.get(s.uuid, 0) - count += s.count(start, end, version=version) + versions = ( + self._pinned_versions if self._pinned_versions else self._latest_versions() + ) + + my_counts_gen = self._btrdb._executor.map( + lambda s: s.count( + start, end, version=versions.get(s.uuid, 0), precise=precise + ), + self._streams, + ) - return count + return sum(list(my_counts_gen)) def earliest(self): """ @@ -922,16 +1844,31 @@ def earliest(self): tuple The earliest points of data found among all streams + Examples + -------- + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream1 = conn.stream_from_uuid("...") + >>> stream2 = conn.stream_from_uuid("...") + >>> streamset = btrdb.stream.StreamSet([stream1, stream2]) + >>> streamset.earliest() + (, ) """ earliest = [] params = self._params_from_filters() start = params.get("start", MINIMUM_TIME) versions = self.versions() - - for s in self._streams: - version = versions.get(s.uuid, 0) - point, _ = s.nearest(start, version=version, backward=False) - earliest.append(point) + if IS_DEBUG: + logger.debug(f"versions: {versions}") + earliest_points_gen = self._btrdb._executor.map( + lambda s: s.nearest(start, version=versions.get(s.uuid, 0), backward=False), + self._streams, + ) + for point in earliest_points_gen: + if point is not None: + earliest.append(point[0]) + else: + earliest.append(None) return tuple(earliest) @@ -948,33 +1885,52 @@ def latest(self): tuple The latest points of data found among all streams + Examples + -------- + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream1 = conn.stream_from_uuid("...") + >>> stream2 = conn.stream_from_uuid("...") + >>> streamset = btrdb.stream.StreamSet([stream1, stream2]) + >>> streamset.earliest() + (, ) """ latest = [] params = self._params_from_filters() start = params.get("end", MAXIMUM_TIME) versions = self.versions() - - for s in self._streams: - version = versions.get(s.uuid, 0) - point, _ = s.nearest(start, version=version, backward=True) - latest.append(point) + latest_points_gen = self._btrdb._executor.map( + lambda s: s.nearest(start, version=versions.get(s.uuid, 0), backward=True), + self._streams, + ) + for point in latest_points_gen: + if point is not None: + latest.append(point[0]) + else: + latest.append(None) return tuple(latest) def current(self): """ Returns the points of data in the streams closest to the current timestamp. If - the current timestamp is outside of the filtered range of data, a ValueError is + the current timestamp is outside the filtered range of data, a ValueError is raised. - Parameters - ---------- - None - Returns ------- tuple The latest points of data found among all streams + + Examples + -------- + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream1 = conn.stream_from_uuid("...") + >>> stream2 = conn.stream_from_uuid("...") + >>> streamset = btrdb.stream.StreamSet([stream1, stream2]) + >>> streamset.current() + (, ) """ latest = [] params = self._params_from_filters() @@ -983,17 +1939,35 @@ def current(self): start = params.get("start", None) if (end is not None and end <= now) or (start is not None and start > now): - raise BTRDBValueError("current time is not included in filtered stream range") + raise BTRDBValueError( + "current time is not included in filtered stream range" + ) - for s in self._streams: - version = self.versions()[s.uuid] - point, _ = s.nearest(now, version=version, backward=True) - latest.append(point) + versions = self.versions() + current_points_gen = self._btrdb._executor.map( + lambda s: (s.nearest(now, version=versions.get(s.uuid, 0), backward=True)), + self._streams, + ) + for point in current_points_gen: + if point is not None: + latest.append(point[0]) + else: + latest.append(None) return tuple(latest) - def filter(self, start=None, end=None, collection=None, name=None, unit=None, - tags=None, annotations=None): + def filter( + self, + start=None, + end=None, + collection=None, + name=None, + unit=None, + tags=None, + annotations=None, + sampling_frequency=None, + schema=None, + ): """ Provides a new StreamSet instance containing stored query parameters and stream objects that match filtering criteria. @@ -1028,58 +2002,126 @@ def filter(self, start=None, end=None, collection=None, name=None, unit=None, key/value pairs for filtering streams based on tags annotations : dict key/value pairs for filtering streams based on annotations + sampling_frequency : float + The sampling frequency of the data streams in Hz, set this if you want timesnapped values. + schema: pyarrow.Schema + Optional arrow schema the server will cast the returned data to before sending it over + the network. You can use this to change the timestamp format, column names or data sizes. Returns ------- StreamSet a new instance cloned from the original with filters applied + + .. note:: + + If you set ``sampling_frequency`` to a non-zero value, the stream data returned will be aligned to a + grid of timestamps based on the period of the sampling frequency. For example, a sampling rate of 30hz will + have a sampling period of 1/30hz -> ~33_333_333 ns per sample. Leave ``sampling_frequency`` as ``None``, or set to ``0`` to + prevent time alignment. You should **not** use aligned data for frequency-based analysis. + + + Examples + -------- + create a streamset and apply a few filters + + >>> streamset = btrdb.stream.StreamSet(list_of_streams) + >>> print(f"Total streams: {len(streamset)}") + Total streams: 89 + + >>> streamset.filter(units="Volts") + >>> print(f"Total streams: {len(streamset)}") + Total streams: 89 + + >>> filtered_streamset = streamset.filter(units="Volts") + >>> print(f"Total streams: {len(filtered_streamset)}") + Total streams: 23 + + >>> multiple_filters_streamset = (streamset.filter(unit="Volts") + >>> .filter(name="Sensor 1") + >>> .filter(annotations={"phase":"A"}) + >>> ) + >>> print(f"Total streams: {len(multiple_filters_streamset)}") + Total streams: 1 """ obj = self.clone() - if start is not None or end is not None: - obj.filters.append(StreamFilter(start, end)) + if ( + start is not None + or end is not None + or sampling_frequency is not None + or schema is not None + ): + obj.filters.append( + StreamFilter( + start=start, + end=end, + sampling_frequency=sampling_frequency, + schema=schema, + ) + ) # filter by collection if collection is not None: if isinstance(collection, RE_PATTERN): - obj._streams = [s for s in obj._streams for m in [collection.search(s.collection)] if m] + obj._streams = [ + s + for s in obj._streams + for m in [collection.search(s.collection)] + if m + ] elif isinstance(collection, str): - obj._streams = [s for s in obj._streams if s.collection.lower() == collection.lower()] + obj._streams = [ + s + for s in obj._streams + if s.collection.lower() == collection.lower() + ] else: raise BTRDBTypeError("collection must be string or compiled regex") # filter by name if name is not None: if isinstance(name, RE_PATTERN): - obj._streams = [s for s in obj._streams for m in [name.search(s.name)] if m] + obj._streams = [ + s for s in obj._streams for m in [name.search(s.name)] if m + ] elif isinstance(name, str): - obj._streams = [s for s in obj._streams if s.name.lower() == name.lower()] + obj._streams = [ + s for s in obj._streams if s.name.lower() == name.lower() + ] else: raise BTRDBTypeError("name must be string or compiled regex") # filter by unit if unit is not None: if isinstance(unit, RE_PATTERN): - obj._streams = [s for s in obj._streams for m in [unit.search(s.tags()["unit"])] if m] + obj._streams = [ + s + for s in obj._streams + for m in [unit.search(s.tags()["unit"])] + if m + ] elif isinstance(unit, str): - obj._streams = [s for s in obj._streams if s.tags().get("unit", "").lower() == unit.lower()] + obj._streams = [ + s + for s in obj._streams + if s.tags().get("unit", "").lower() == unit.lower() + ] else: raise BTRDBTypeError("unit must be string or compiled regex") # filter by tags if tags: # filters if the subset of the tags matches the given tags - obj._streams = [ - s for s in obj._streams - if tags.items() <= s.tags().items() - ] + obj._streams = [s for s in obj._streams if tags.items() <= s.tags().items()] # filter by annotations if annotations: # filters if the subset of the annotations matches the given annotations obj._streams = [ - s for s in obj._streams + s + for s in obj._streams if annotations.items() <= s.annotations()[0].items() ] @@ -1100,14 +2142,14 @@ def clone(self): Returns a new copy of the instance """ - protected = ('_streams', ) + protected = ("_streams", "_btrdb") clone = self.__class__(self._streams) for attr, val in self.__dict__.items(): if attr not in protected: setattr(clone, attr, deepcopy(val)) return clone - def windows(self, width, depth): + def windows(self, width, depth=0): """ Stores the request for a windowing operation when the query is eventually materialized. @@ -1118,7 +2160,8 @@ def windows(self, width, depth): The number of nanoseconds to use for each window size. depth : int The requested accuracy of the data up to 2^depth nanoseconds. A - depth of 0 is accurate to the nanosecond. + depth of 0 is accurate to the nanosecond. This is now the only + valid value for depth. Returns ------- @@ -1126,31 +2169,46 @@ def windows(self, width, depth): Returns self - Notes - ----- - Windows returns arbitrary precision windows from BTrDB. It is slower - than aligned_windows, but still significantly faster than values. Each - returned window will be width nanoseconds long. start is inclusive, but - end is exclusive (e.g if end < start+width you will get no results). - That is, results will be returned for all windows that start at a time - less than the end timestamp. If (end - start) is not a multiple of - width, then end will be decreased to the greatest value less than end - such that (end - start) is a multiple of width (i.e., we set end = start - + width * floordiv(end - start, width). The depth parameter is an - optimization that can be used to speed up queries on fast queries. Each - window will be accurate to 2^depth nanoseconds. If depth is zero, the - results are accurate to the nanosecond. On a dense stream for large - windows, this accuracy may not be required. For example for a window of - a day, +- one second may be appropriate, so a depth of 30 can be - specified. This is much faster to execute on the database side. + .. note:: + ``windows`` returns arbitrary precision windows from BTrDB. It is slower + than ``aligned_windows``, but can be significantly faster than values. + Each returned window will be ``width`` nanoseconds long. + ``start`` is inclusive, but ``end`` is exclusive ( ``[start, end) ) + (e.g. if end < start+width you will get no results). + That is, results will be returned for all windows that start at a time + less than the end timestamp. If (``end`` - ``start``) is not a multiple of + ``width``, then ``end`` will be decreased to the greatest value less than ``end`` + such that (end - start) is a multiple of width (i.e., we set end = start + + width * floordiv(end - start, width)). The ``depth`` parameter previously + available has been deprecated. The only valid value for ``depth`` is now ``0``. + + Examples + -------- + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream1 = conn.stream_from_uuid("...") + >>> stream2 = conn.stream_from_uuid("...") + >>> streamset = btrdb.stream.StreamSet([stream1, stream2]) + >>> streamset.windows(width=1000000000) + + >>> streamset.windows(width=1000000000, depth=0) + + >>> streamset.aligned_windows(pointwidth=30) + Traceback (most recent call last): + ... + btrdb.exceptions.InvalidOperation: A window operation is already requested """ if not self.allow_window: raise InvalidOperation("A window operation is already requested") # TODO: refactor keeping in mind how exception is raised self.width = int(width) - self.depth = int(depth) + if depth != 0: + warnings.warn( + f"The only valid value for `depth` is now 0. You provided: {depth}. Overriding this value to 0." + ) + self.depth = 0 return self def aligned_windows(self, pointwidth): @@ -1168,18 +2226,32 @@ def aligned_windows(self, pointwidth): StreamSet Returns self - Notes - ----- - `aligned_windows` reads power-of-two aligned windows from BTrDB. It is - faster than Windows(). Each returned window will be 2^pointwidth - nanoseconds long, starting at start. Note that start is inclusive, but - end is exclusive. That is, results will be returned for all windows that - start in the interval [start, end). If end < start+2^pointwidth you will - not get any results. If start and end are not powers of two, the bottom - pointwidth bits will be cleared. Each window will contain statistical - summaries of the window. Statistical points with count == 0 will be - omitted. + .. note:: + + ``aligned_windows`` reads power-of-two aligned windows from BTrDB. It is + faster than ``windows()``. Each returned window will be 2^``pointwidth`` + nanoseconds long, beginning at ``start``. Note that start is inclusive, but + end is exclusive. That is, results will be returned for all windows that + start in the interval ``[start, end)``. If ``end`` < ``start``+2^``pointwidth`` you will + not get any results. If ``start`` and ``end`` are not powers of two, the bottom + ``pointwidth`` bits will be cleared. Each window will contain statistical + summaries of the window. Statistical points with ``count`` == 0 will be + omitted. + + Examples + -------- + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream1 = conn.stream_from_uuid("...") + >>> stream2 = conn.stream_from_uuid("...") + >>> streamset = btrdb.stream.StreamSet([stream1, stream2]) + >>> streamset.aligned_windows(pointwidth=30) + + >>> streamset.windows(width=1000000000) + Traceback (most recent call last): + ... + btrdb.exceptions.InvalidOperation: A window operation is already requested """ if not self.allow_window: raise InvalidOperation("A window operation is already requested") @@ -1196,37 +2268,50 @@ def _streamset_data(self, as_iterators=False): ---------- as_iterators : bool Returns each single stream's data as an iterator. Defaults to False. + + :meta private: """ params = self._params_from_filters() - versions = self.versions() - data = [] + # sampling freq not supported for non-arrow streamset ops + _ = params.pop("sampling_frequency", None) + versions = self._pinned_versions + if versions is None: + versions = {s.uuid: 0 for s in self} if self.pointwidth is not None: # create list of stream.aligned_windows data params.update({"pointwidth": self.pointwidth}) - for s in self._streams: - params.update({"version": versions[s.uuid]}) - data.append(s.aligned_windows(**params)) - + # need to update params based on version of stream, use dict merge + aligned_windows_gen = self._btrdb._executor.map( + lambda s: s.aligned_windows( + **{**params, **{"version": versions[s.uuid]}} + ), + self._streams, + ) + data = list(aligned_windows_gen) elif self.width is not None and self.depth is not None: # create list of stream.windows data (the windows method should # prevent the possibility that only one of these is None) params.update({"width": self.width, "depth": self.depth}) - for s in self._streams: - params.update({"version": versions[s.uuid]}) - data.append(s.windows(**params)) + windows_gen = self._btrdb._executor.map( + lambda s: s.windows(**{**params, **{"version": versions[s.uuid]}}), + self._streams, + ) + data = list(windows_gen) else: # create list of stream.values - data = [s.values(**params) for s in self._streams] + values_gen = self._btrdb._executor.map( + lambda s: s.values(**params), self._streams + ) + data = list(values_gen) if as_iterators: return [iter(ii) for ii in data] return data - def rows(self): """ Returns a materialized list of tuples where each tuple contains the @@ -1243,6 +2328,20 @@ def rows(self): A list of tuples containing a RawPoint (or StatPoint) and the stream version (list(tuple(RawPoint, int))). + Examples + -------- + >>> for row in streams.rows(): + >>> print(row) + (None, RawPoint(1500000000000000000, 1.0), RawPoint(1500000000000000000, 1.0), RawPoint(1500000000000000000, 1.0)) + (RawPoint(1500000000100000000, 2.0), None, RawPoint(1500000000100000000, 2.0), RawPoint(1500000000100000000, 2.0)) + (None, RawPoint(1500000000200000000, 3.0), None, RawPoint(1500000000200000000, 3.0)) + (RawPoint(1500000000300000000, 4.0), None, RawPoint(1500000000300000000, 4.0), RawPoint(1500000000300000000, 4.0)) + (None, RawPoint(1500000000400000000, 5.0), RawPoint(1500000000400000000, 5.0), RawPoint(1500000000400000000, 5.0)) + (RawPoint(1500000000500000000, 6.0), None, None, RawPoint(1500000000500000000, 6.0)) + (None, RawPoint(1500000000600000000, 7.0), RawPoint(1500000000600000000, 7.0), RawPoint(1500000000600000000, 7.0)) + (RawPoint(1500000000700000000, 8.0), None, RawPoint(1500000000700000000, 8.0), RawPoint(1500000000700000000, 8.0)) + (None, RawPoint(1500000000800000000, 9.0), RawPoint(1500000000800000000, 9.0), RawPoint(1500000000800000000, 9.0)) + (RawPoint(1500000000900000000, 10.0), None, RawPoint(1500000000900000000, 10.0), RawPoint(1500000000900000000, 10.0)) """ result = [] streamset_data = self._streamset_data(as_iterators=True) @@ -1253,7 +2352,6 @@ def rows(self): # add next values from streams into buffer for stream_idx, data in enumerate(streamset_data): - if buffer.active[stream_idx]: try: point, _ = next(data) @@ -1272,18 +2370,135 @@ def rows(self): return result + def insert(self, data_map: dict, merge: str = "never") -> dict: + """Insert new data in the form (time, value) into their mapped streams. + + The times in the dataframe need not be sorted by time. If the point counts are larger than + appropriate, this function will automatically chunk the inserts. As a + consequence, the insert is not necessarily atomic, but can be used with + a very large array. + + Parameters + ---------- + data_map: dict[uuid, pandas.DataFrame] + A dictionary mapping stream uuids to insert data into and their value as a + pandas dataframe containing two columns, one named "time" which contains int64 utc+0 timestamps + and a "value" column containing float64 measurements. These columns will be typecast into these types. + merge: str + A string describing the merge policy. Valid policies are: + - 'never': the default, no points are merged + - 'equal': points are deduplicated if the time and value are equal + - 'retain': if two points have the same timestamp, the old one is kept + - 'replace': if two points have the same timestamp, the new one is kept + + + Returns + ------- + dict[uuid, int] + The versions of the stream after inserting new points. + + + .. note:: + + You MUST convert your datetimes into UTC+0 **yourself**. BTrDB expects UTC+0 datetimes. + + + Examples + -------- + >>> import pandas as pd + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream1 = conn.stream_from_uuid("...") + >>> stream2 = conn.stream_from_uuid("...") + >>> streamset = btrdb.stream.StreamSet([stream1, stream2]) + >>> data_map = { + ... stream1.uuid: pd.DataFrame({'time': [1500000000000000000, 1500000000100000000], 'value': [1.0, 2.0]}), + ... stream2.uuid: pd.DataFrame({'time': [1500000000000000000, 1500000000100000000], 'value': [3.0, 4.0]}) + ... } + >>> streamset.insert(data_map) + {UUID('...'): 1234, UUID('...'): 5678} + >>> streamset.insert(data_map, merge='replace') + {UUID('...'): 1235, UUID('...'): 5679} + """ + filtered_data_map = {s.uuid: data_map[s.uuid] for s in self._streams} + for key, dat in filtered_data_map.items(): + data_list = [ + (times, vals) + for times, vals in zip( + dat["time"].astype(int).values, dat["value"].astype(float).values + ) + ] + filtered_data_map[key] = data_list + versions_gen = self._btrdb._executor.map( + lambda s: (s.uuid, s.insert(data=filtered_data_map[s.uuid], merge=merge)), + self._streams, + ) + versions = {uu: ver for uu, ver in versions_gen} + return versions + + def arrow_insert(self, data_map: dict, merge: str = "never") -> dict: + """Insert new data in the form (time, value) into their mapped streams using pyarrow tables. + + The times in the arrow table need not be sorted by time. If the point counts are larger than + appropriate, this function will automatically chunk the inserts. As a + consequence, the insert is not necessarily atomic, but can be used with + a very large array. + + Parameters + ---------- + data_map: dict[uuid, pyarrow.Table] + A dictionary keyed on stream uuids and mapped to pyarrow tables with a schema + of time:Timestamp[ns, tz=UTC], value:float64. This schema will be validated and converted if necessary. + merge: str + A string describing the merge policy. Valid policies are: + - 'never': the default, no points are merged + - 'equal': points are deduplicated if the time and value are equal + - 'retain': if two points have the same timestamp, the old one is kept + - 'replace': if two points have the same timestamp, the new one is kept + + Notes + ----- + BTrDB expects datetimes to be in UTC+0. + + This method is available for commercial customers with arrow-enabled servers. + + Returns + ------- + dict[uuid, int] + The versions of the stream after inserting new points. + """ + filtered_data_map = {s.uuid: data_map[s.uuid] for s in self._streams} + versions_gen = self._btrdb._executor.map( + lambda s: ( + s.uuid, + s.arrow_insert(data=filtered_data_map[s.uuid], merge=merge), + ), + self._streams, + ) + versions = {uu: ver for uu, ver in versions_gen} + return versions + def _params_from_filters(self): + """ + :meta private: + """ params = {} for filter in self.filters: if filter.start is not None: params["start"] = filter.start if filter.end is not None: params["end"] = filter.end + if filter.sampling_frequency is not None: + params["sampling_frequency"] = filter.sampling_frequency + if filter.schema is not None: + params["schema"] = filter.schema return params def values_iter(self): """ Must return context object which would then close server cursor on __exit__ + + :meta private: """ raise NotImplementedError() @@ -1295,14 +2510,105 @@ def values(self): streamset_data = self._streamset_data() for stream_data in streamset_data: result.append([point[0] for point in stream_data]) - return result + def arrow_values( + self, + ): + """Return a pyarrow table of stream values based on the streamset parameters. + + This data will be sorted by the 'time' column. + + .. note:: + + This method is available for commercial customers with arrow-enabled servers. + """ + if pa is None: + raise ImportError(_ARROW_IMPORT_MSG) + params = self._params_from_filters() + versions = self._pinned_versions + if versions is None: + versions = {s.uuid: 0 for s in self} + + if params.get("sampling_frequency", None) is None: + _ = params.pop("sampling_frequency", None) + + if self.pointwidth is not None: + if params.pop("schema", None) is not None: + raise NotImplementedError( + "aligned windows queries do not yet support an arrow schema" + ) + # create list of stream.aligned_windows data + params.update({"pointwidth": self.pointwidth}) + _ = params.pop("sampling_frequency", None) + # need to update params based on version of stream, use dict merge + # {**dict1, **dict2} creates a new dict which updates the key/values if there are any + # same keys in dict1 and dict2, then the second dict in the expansion (dict2 in this case) + # will update the key (similar to dict1.update(dict2)) + aligned_windows_gen = self._btrdb._executor.map( + lambda s: s.arrow_aligned_windows( + **{**params, **{"version": versions[s.uuid]}} + ), + self._streams, + ) + stream_uus = [str(s.uuid) for s in self._streams] + data = list(aligned_windows_gen) + table_joined = _merge_pyarrow_tables( + {uu: tab for uu, tab in zip(stream_uus, data)} + ) + data = table_joined + + elif self.width is not None and self.depth is not None: + # create list of stream.windows data (the windows method should + # prevent the possibility that only one of these is None) + if params.pop("schema", None) is not None: + raise NotImplementedError( + "windows queries do not yet support an arrow schema" + ) + _ = params.pop("sampling_frequency", None) + params.update({"width": self.width}) + windows_gen = self._btrdb._executor.map( + lambda s: s.arrow_windows( + **{**params, **{"version": versions[s.uuid]}} + ), + self._streams, + ) + stream_uus = [str(s.uuid) for s in self._streams] + data = list(windows_gen) + table_joined = _merge_pyarrow_tables( + {uu: tab for uu, tab in zip(stream_uus, data)} + ) + data = table_joined + else: + sampling_freq = params.pop("sampling_frequency", 0) + period_ns = 0 + if sampling_freq > 0: + period_ns = _to_period_ns(sampling_freq) + params["uu_list"] = [s.uuid for s in self._streams] + params["version_list"] = [versions[s.uuid] for s in self._streams] + params["snap_periodNS"] = period_ns + table = list(self._btrdb.ep.arrowMultiValues(**params)) + if len(table) > 0: + data = pa.concat_tables(table) + else: + schema = params.pop("schema", None) + if schema is None: + schema = pa.schema( + [pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False)] + + [ + pa.field(str(s.uuid), pa.float64(), nullable=False) + for s in self._streams + ], + ) + data = pa.Table.from_arrays( + [pa.array([]) for i in range(1 + len(self._streams))], schema=schema + ) + time_col = data.column_names[0] + return data.sort_by(time_col) + def __repr__(self): token = "stream" if len(self) == 1 else "streams" - return "<{}({} {})>".format( - self.__class__.__name__, len(self._streams), token - ) + return "<{}({} {})>".format(self.__class__.__name__, len(self._streams), token) def __str__(self): token = "stream" if len(self) == 1 else "streams" @@ -1322,6 +2628,15 @@ def __getitem__(self, item): return self._streams[item] + def __contains__(self, item): + if isinstance(item, str): + for stream in self._streams: + if str(stream.uuid()) == item: + return True + return False + + return item in self._streams + def __len__(self): return len(self._streams) @@ -1330,6 +2645,7 @@ class StreamSet(StreamSetBase, StreamSetTransformer): """ Public class for a collection of streams """ + pass @@ -1337,16 +2653,119 @@ class StreamSet(StreamSetBase, StreamSetTransformer): ## Utility Classes ########################################################################## + class StreamFilter(object): """ Object for storing requested filtering options """ - def __init__(self, start=None, end=None): + + def __init__( + self, + start: int = None, + end: int = None, + sampling_frequency: int = None, + schema=None, + ): self.start = to_nanoseconds(start) if start else None self.end = to_nanoseconds(end) if end else None + self.schema = schema + self.sampling_frequency = sampling_frequency if sampling_frequency else None if self.start is None and self.end is None: raise BTRDBValueError("A valid `start` or `end` must be supplied") if self.start is not None and self.end is not None and self.start >= self.end: raise BTRDBValueError("`start` must be strictly less than `end` argument") + + +def _to_period_ns(fs: int): + """Convert sampling rate to sampling period in ns.""" + period = 1 / fs + period_ns = period * 1e9 + return int(period_ns) + + +def _coalesce_table_deque(tables: deque): + main_table = tables.popleft() + idx = 0 + while len(tables) != 0: + idx = idx + 1 + t2 = tables.popleft() + main_table = main_table.join( + t2, "time", join_type="full outer", right_suffix=f"_{idx}" + ) + return main_table + + +def _extract_unique_times(stream_map: Dict[uuid.UUID, pa.Table]) -> pa.Array: + """Extracts and returns unique 'time' values from all tables.""" + time_arrays = [ + table.column("time").combine_chunks() + for table in stream_map.values() + if table.num_rows > 0 + ] + if not time_arrays: # Check if the list is empty + return pa.array([], type=pa.timestamp("ns")) + + all_times = pa.concat_arrays(time_arrays) + return pc.unique(all_times).sort() + + +def _build_combined_schema( + stream_map: Dict[uuid.UUID, pa.Table], unique_times: pa.Array +) -> pa.Schema: + """Constructs a combined schema for the merged table, ensuring unique column names.""" + combined_schema = [("time", unique_times.type)] + # Use a list comprehension to flatten the loop into a single iterable over all columns + combined_schema += [ + pa.field( + f"{uu}/{col_name}", + table.column(col_name).type if table.num_rows > 0 else pa.null(), + ) + for uu, table in stream_map.items() + for col_name in table.column_names + if col_name != "time" + ] + + return pa.schema(combined_schema) + + +def _merge_pyarrow_tables(stream_map: Dict[uuid.UUID, pa.Table]) -> pa.Table: + """Merges PyArrow tables based on 'time' values into a single table.""" + unique_times = _extract_unique_times(stream_map) + combined_schema = _build_combined_schema(stream_map, unique_times) + + none_data = [None] * len(unique_times) + preallocated_data = { + field.name: pa.array(none_data, type=field.type) for field in combined_schema + } + preallocated_data["time"] = unique_times + + for uu, table in stream_map.items(): + if table.num_rows > 0: + time_indices = pc.index_in( + preallocated_data["time"], + value_set=table.column("time"), + skip_nulls=True, + ) + for col_name in table.column_names: + if col_name == "time": + continue + combined_col_name = f"{str(uu)}/{col_name}" + preallocated_data[combined_col_name] = pc.take( + table.column(col_name), indices=time_indices + ) + else: + # For empty tables, ensure their columns are represented with all nulls + for col_name in combined_schema.names: + if ( + col_name.startswith(f"{str(uu)}/") + and col_name not in preallocated_data + ): + field_type = combined_schema.field(col_name).type + preallocated_data[col_name] = pa.array( + [None] * preallocated_data["time"].length(), type=field_type + ) + + arrays = [preallocated_data[col] for col in combined_schema.names] + return pa.Table.from_arrays(arrays=arrays, schema=combined_schema) diff --git a/btrdb/transformers.py b/btrdb/transformers.py index 12f7979..dcdee4f 100644 --- a/btrdb/transformers.py +++ b/btrdb/transformers.py @@ -15,20 +15,49 @@ ## Imports ########################################################################## -import csv import contextlib +import csv from collections import OrderedDict -from warnings import warn +from typing import TYPE_CHECKING + +try: + import pyarrow as pa +except ImportError: + pa = None +try: + import polars as pl +except ImportError: + pl = None +try: + import pandas as pd +except ImportError: + pd = None +try: + import numpy as np +except ImportError: + np = None + +if TYPE_CHECKING: + import numpy as np + import pandas as pd + import polars as pl + import pyarrow as pa + +_IMPORT_ERR_MSG = ( + """Package(s) expected, but not found. Please pip install the following: {}""" +) ########################################################################## ## Helper Functions ########################################################################## -_STAT_PROPERTIES = ('min', 'mean', 'max', 'count', 'stddev') +_STAT_PROPERTIES = ("min", "mean", "max", "count", "stddev") + def _get_time_from_row(row): for item in row: - if item: return item.time + if item: + return item.time raise Exception("Row contains no data") @@ -38,15 +67,14 @@ def _stream_names(streamset, func): before sending a collection of streams (dataframe, etc.) back to the user. """ - return tuple( - func(s) for s in streamset._streams - ) + return tuple(func(s) for s in streamset._streams) ########################################################################## ## Transform Functions ########################################################################## + def to_series(streamset, datetime64_index=True, agg="mean", name_callable=None): """ Returns a list of Pandas Series objects indexed by time @@ -63,22 +91,24 @@ def to_series(streamset, datetime64_index=True, agg="mean", name_callable=None): argument is ignored if RawPoint values are passed into the function. name_callable : lambda, default: lambda s: s.collection + "/" + s.name - Sprecify a callable that can be used to determine the series name given a + Specify a callable that can be used to determine the series name given a Stream object. + + .. note:: + + This method does **not** use the ``arrow`` -accelerated endpoints for faster and more efficient data retrieval. + """ - try: - import pandas as pd - except ImportError: - raise ImportError("Please install Pandas to use this transformation function.") + if pd is None: + raise ImportError(_IMPORT_ERR_MSG.format("pandas")) # TODO: allow this at some future point if agg == "all": raise AttributeError("cannot use 'all' as aggregate at this time") if not callable(name_callable): - name_callable = lambda s: s.collection + "/" + s.name - + name_callable = lambda s: s.collection + "/" + s.name result = [] stream_names = _stream_names(streamset, name_callable) @@ -93,67 +123,452 @@ def to_series(streamset, datetime64_index=True, agg="mean", name_callable=None): values.append(getattr(point, agg)) if datetime64_index: - times = pd.Index(times, dtype='datetime64[ns]') + times = pd.Index(times, dtype="datetime64[ns]", name="time") - result.append(pd.Series( - data=values, index=times, name=stream_names[idx] - )) + result.append(pd.Series(data=values, index=times, name=stream_names[idx])) return result -def to_dataframe(streamset, columns=None, agg="mean", name_callable=None): +def arrow_to_series(streamset, agg=None, name_callable=None): + """ + Returns a list of Pandas Series objects indexed by time + + Parameters + ---------- + agg : List[str], default: ["mean"] + Specify the StatPoint field or fields (e.g. aggregating function) to create the Series + from. Must be one or more of "min", "mean", "max", "count", or "stddev". This + argument is ignored if RawPoint values are passed into the function. + + name_callable : lambda, default: lambda s: s.collection + "/" + s.name + Specify a callable that can be used to determine the series name given a + Stream object. + + + Returns + ------- + List[pandas.Series] + + + .. note:: + + This method is available for commercial customers with arrow-enabled servers. + + + .. note:: + + If you are not performing a ``window`` or ``aligned_window`` query, the ``agg`` parameter will be ignored. + + Examples + -------- + Return a list of series of raw data per stream. + + >>> conn = btrdb.connect() + >>> s1 = conn.stream_from_uuid('c9fd8735-5ec5-4141-9a51-d23e1b2dfa42') + >>> s2 = conn.stream_from_uuid('9173fa70-87ab-4ac8-ac08-4fd63b910cae' + >>> streamset = btrdb.stream.StreamSet([s1,s2]) + >>> streamset.filter(start=1500000000000000000, end=1500000000900000001).arrow_to_series(agg=None) + [time + 2017-07-14 02:40:00+00:00 1.0 + 2017-07-14 02:40:00.100000+00:00 2.0 + 2017-07-14 02:40:00.200000+00:00 3.0 + 2017-07-14 02:40:00.300000+00:00 4.0 + 2017-07-14 02:40:00.400000+00:00 5.0 + 2017-07-14 02:40:00.500000+00:00 6.0 + 2017-07-14 02:40:00.600000+00:00 7.0 + 2017-07-14 02:40:00.700000+00:00 8.0 + 2017-07-14 02:40:00.800000+00:00 9.0 + 2017-07-14 02:40:00.900000+00:00 10.0 + Name: new/stream/collection/foo, dtype: double[pyarrow], + time + 2017-07-14 02:40:00+00:00 1.0 + 2017-07-14 02:40:00.100000+00:00 2.0 + 2017-07-14 02:40:00.200000+00:00 3.0 + 2017-07-14 02:40:00.300000+00:00 4.0 + 2017-07-14 02:40:00.400000+00:00 5.0 + 2017-07-14 02:40:00.500000+00:00 6.0 + 2017-07-14 02:40:00.600000+00:00 7.0 + 2017-07-14 02:40:00.700000+00:00 8.0 + 2017-07-14 02:40:00.800000+00:00 9.0 + 2017-07-14 02:40:00.900000+00:00 10.0 + Name: new/stream/bar, dtype: double[pyarrow]] + + + A window query of 0.5seconds long. + + >>> streamset.filter(start=1500000000000000000, end=1500000000900000001) + ... .windows(width=int(0.5 * 10**9)) + ... .arrow_to_series(agg=["mean", "count"]) + [time + 2017-07-14 02:40:00+00:00 2.5 + 2017-07-14 02:40:00.400000+00:00 6.5 + Name: new/stream/collection/foo/mean, dtype: double[pyarrow], + ... + time + 2017-07-14 02:40:00+00:00 4 + 2017-07-14 02:40:00.400000+00:00 4 + Name: new/stream/collection/foo/count, dtype: uint64[pyarrow], + ... + time + 2017-07-14 02:40:00+00:00 2.5 + 2017-07-14 02:40:00.400000+00:00 6.5 + Name: new/stream/bar/mean, dtype: double[pyarrow], + ... + time + 2017-07-14 02:40:00+00:00 4 + 2017-07-14 02:40:00.400000+00:00 4 + Name: new/stream/bar/count, dtype: uint64[pyarrow]] + + + """ + if not streamset._btrdb._ARROW_ENABLED: + raise NotImplementedError( + "arrow_to_series requires an arrow-enabled BTrDB server." + ) + if pa is None or pd is None: + raise ImportError(_IMPORT_ERR_MSG.format(",".join(["pyarrow", "pandas"]))) + if agg is None: + agg = ["mean"] + if not isinstance(agg, list): + raise ValueError( + f"Expected argument 'agg' to be a list of strings, was provided type - {type(agg)} with values - {agg}" + ) + arrow_df = arrow_to_dataframe( + streamset=streamset, agg=agg, name_callable=name_callable + ) + return [arrow_df[col] for col in arrow_df] + + +def arrow_to_dataframe(streamset, agg=None, name_callable=None) -> pd.DataFrame: """ Returns a Pandas DataFrame object indexed by time and using the values of a stream for each column. Parameters ---------- - columns: sequence - column names to use for DataFrame. Deprecated and not compatible with name_callable. + agg : List[str], default: ["mean"] + Specify the StatPoint fields (e.g. aggregating function) to create the dataframe + from. Must be one or more of "min", "mean", "max", "count", "stddev", or "all". This + argument is ignored if not using StatPoints. + + name_callable : lambda, default: lambda s: s.collection + "/" + s.name + Specify a callable that can be used to determine the series name given a + Stream object. + + + .. note:: + + This method is available for commercial customers with arrow-enabled servers. + + + Examples + -------- + + >>> conn = btrdb.connect() + >>> s1 = conn.stream_from_uuid('c9fd8735-5ec5-4141-9a51-d23e1b2dfa42') + >>> s2 = conn.stream_from_uuid('9173fa70-87ab-4ac8-ac08-4fd63b910cae' + >>> streamset = btrdb.stream.StreamSet([s1,s2]) + >>> streamset.filter(start=1500000000000000000, end=1500000000900000001).arrow_to_dataframe() + new/stream/collection/foo new/stream/bar + time + 2017-07-14 02:40:00+00:00 1.0 1.0 + 2017-07-14 02:40:00.100000+00:00 2.0 2.0 + 2017-07-14 02:40:00.200000+00:00 3.0 3.0 + 2017-07-14 02:40:00.300000+00:00 4.0 4.0 + 2017-07-14 02:40:00.400000+00:00 5.0 5.0 + 2017-07-14 02:40:00.500000+00:00 6.0 6.0 + 2017-07-14 02:40:00.600000+00:00 7.0 7.0 + 2017-07-14 02:40:00.700000+00:00 8.0 8.0 + 2017-07-14 02:40:00.800000+00:00 9.0 9.0 + 2017-07-14 02:40:00.900000+00:00 10.0 10.0 + + + Use the stream uuids as their column names instead, using a lambda function. + + >>> streamset.filter(start=1500000000000000000, end=1500000000900000001) + ... .arrow_to_dataframe( + ... name_callable=lambda s: str(s.uuid) + ... ) + c9fd8735-5ec5-4141-9a51-d23e1b2dfa42 9173fa70-87ab-4ac8-ac08-4fd63b910cae + time + 2017-07-14 02:40:00+00:00 1.0 1.0 + 2017-07-14 02:40:00.100000+00:00 2.0 2.0 + 2017-07-14 02:40:00.200000+00:00 3.0 3.0 + 2017-07-14 02:40:00.300000+00:00 4.0 4.0 + 2017-07-14 02:40:00.400000+00:00 5.0 5.0 + 2017-07-14 02:40:00.500000+00:00 6.0 6.0 + 2017-07-14 02:40:00.600000+00:00 7.0 7.0 + 2017-07-14 02:40:00.700000+00:00 8.0 8.0 + 2017-07-14 02:40:00.800000+00:00 9.0 9.0 + 2017-07-14 02:40:00.900000+00:00 10.0 10.0 + + + A window query, with a window width of 0.4 seconds, and only showing the ``mean`` statpoint. + + >>> streamset.filter(start=1500000000000000000, end=1500000000900000001) + ... .windows(width=int(0.4*10**9)) + ... .arrow_to_dataframe(agg=["mean"]) + new/stream/collection/foo/mean new/stream/bar/mean + time + 2017-07-14 02:40:00+00:00 2.5 2.5 + 2017-07-14 02:40:00.400000+00:00 6.5 6.5 + + + A window query, with a window width of 0.4 seconds, and only showing the ``mean`` and ``count`` statpoints. + + >>> streamset.filter(start=1500000000000000000, end=1500000000900000001) + ... .windows(width=int(0.4*10**9)) + ... .arrow_to_dataframe(agg=["mean", "count"]) + new/stream/collection/foo/mean new/stream/collection/foo/count new/stream/bar/mean new/stream/bar/count + time + 2017-07-14 02:40:00+00:00 2.5 4 2.5 4 + 2017-07-14 02:40:00.400000+00:00 6.5 4 6.5 4 + """ + + def _rename(col_name, col_names_map): + if col_name == "time": + return col_name + col_name_parts = col_name.split("/") + if len(col_name_parts) == 1: + uuid = col_name_parts[0] + new_col_name = col_names_map[uuid] + elif len(col_name_parts) == 2: + uuid, _ = col_name_parts + new_col_name = col_name.replace(uuid, col_names_map[uuid]) + return new_col_name + + if not streamset._btrdb._ARROW_ENABLED: + raise NotImplementedError( + "arrow_to_dataframe requires an arrow-enabled BTrDB server." + ) + if pa is None or pd is None: + raise ImportError(_IMPORT_ERR_MSG.format(",".join(["pyarrow", "pandas"]))) + + if agg is None: + agg = ["mean"] + if not isinstance(agg, list): + raise ValueError( + f"Argument 'agg' not provided as a list of strings was provided type - {type(agg)} with values - {agg}" + ) + # do not allow agg="all" with RawPoints + if "all" in agg and streamset.allow_window: + agg = "" + elif any(["all" in val for val in agg]) and not streamset.allow_window: + agg = ["min", "mean", "max", "count", "stddev"] + else: + agg = agg + + # default arg values + if not callable(name_callable): + name_callable = lambda s: s.collection + "/" + s.name + # format is: uuid/stat_type + tmp = streamset.arrow_values() + # assume time col is the first column + time_col = tmp.column_names[0] + + tmp_df = tmp.to_pandas( + date_as_object=False, + types_mapper=pd.ArrowDtype, + split_blocks=True, + self_destruct=True, + ).set_index(time_col) + tmp_df.index.name = time_col + col_names = _stream_names(streamset, name_callable) + col_names_map = {str(s.uuid): c for s, c in zip(streamset, col_names)} + tmp_df.rename( + columns=lambda col_name: _rename(col_name, col_names_map), inplace=True + ) + if not streamset.allow_window: + usable_cols = [] + for column_str in tmp_df.columns: + for agg_name in agg: + if agg_name in column_str: + usable_cols.append(column_str) + tmp_df = tmp_df.loc[:, usable_cols] + return tmp_df + + +def to_dataframe(streamset, agg="mean", name_callable=None): + """ + Returns a Pandas DataFrame object indexed by time and using the values of a + stream for each column. + + Parameters + ---------- agg : str, default: "mean" Specify the StatPoint field (e.g. aggregating function) to create the Series from. Must be one of "min", "mean", "max", "count", "stddev", or "all". This argument is ignored if not using StatPoints. name_callable : lambda, default: lambda s: s.collection + "/" + s.name - Sprecify a callable that can be used to determine the series name given a + Specify a callable that can be used to determine the series name given a Stream object. This is not compatible with agg == "all" at this time - """ - try: - import pandas as pd - except ImportError: - raise ImportError("Please install Pandas to use this transformation function.") + .. note:: - # deprecation warning added in v5.8 - if columns: - warn("the columns argument is deprecated and will be removed in a future release", DeprecationWarning, stacklevel=2) + This method does **not** use the ``arrow`` -accelerated endpoints for faster and more efficient data retrieval. + + """ + if pd is None: + raise ImportError(_IMPORT_ERR_MSG.format("pandas")) # TODO: allow this at some future point if agg == "all" and name_callable is not None: - raise AttributeError("cannot provide name_callable when using 'all' as aggregate at this time") + raise AttributeError( + "cannot provide name_callable when using 'all' as aggregate at this time" + ) # do not allow agg="all" with RawPoints if agg == "all" and streamset.allow_window: - agg="" + agg = "" # default arg values if not callable(name_callable): - name_callable = lambda s: s.collection + "/" + s.name + name_callable = lambda s: s.collection + "/" + s.name + + df = pd.DataFrame(to_dict(streamset, agg=agg, name_callable=name_callable)) + + if not df.empty: + df = df.set_index("time") + df.index.name = "time" + + if agg == "all" and not streamset.allow_window: + stream_names = [ + [s.collection, s.name, prop] + for s in streamset._streams + for prop in _STAT_PROPERTIES + ] + df.columns = pd.MultiIndex.from_tuples(stream_names) + else: + df.columns = _stream_names(streamset, name_callable) + + return df + + +def arrow_to_polars(streamset, agg=None, name_callable=None): + """ + Returns a Polars DataFrame object with time as a column and the values of a + stream for each additional column from an arrow table. + Parameters + ---------- + agg : List[str], default: ["mean"] + Specify the StatPoint field or fields (e.g. aggregating function) to create the dataframe + from. Must be one or multiple of "min", "mean", "max", "count", "stddev", or "all". This + argument is ignored if not using StatPoints. - df = pd.DataFrame(to_dict(streamset,agg=agg)) - df = df.set_index("time") + name_callable : lambda, default: lambda s: s.collection + "/" + s.name + Specify a callable that can be used to determine the series name given a + Stream object. + + .. note:: + + This method is available for commercial customers with arrow-enabled servers. + + """ + if not streamset._btrdb._ARROW_ENABLED: + raise NotImplementedError( + "arrow_to_polars requires an arrow-enabled BTrDB server." + ) + if pa is None or pd is None or pl is None: + raise ImportError( + _IMPORT_ERR_MSG.format(",".join(["pyarrow", "pandas", "polars"])) + ) + if agg is None: + agg = ["mean"] + if not isinstance(agg, list): + raise ValueError( + f"Expected argument 'agg' to be a list of strings, was provided type - {type(agg)} with values - {agg}" + ) + arrow_df = arrow_to_dataframe( + streamset=streamset, agg=agg, name_callable=name_callable + ) + return pl.from_pandas(arrow_df, include_index=True, nan_to_null=False) + + +def arrow_to_arrow_table(streamset): + """Return a pyarrow table of data. + + + .. note:: + + This method is available for commercial customers with arrow-enabled servers. + + """ + if not streamset._btrdb._ARROW_ENABLED: + raise NotImplementedError( + "arrow_to_arrow_table requires an arrow-enabled BTrDB server." + ) + if pa is None: + raise ImportError(_IMPORT_ERR_MSG.format("pyarrow")) + return streamset.arrow_values() + + +def to_polars(streamset, agg="mean", name_callable=None): + """ + Returns a Polars DataFrame object with time as a column and the values of a + stream for each additional column. + + Parameters + ---------- + agg : str, default: "mean" + Specify the StatPoint field (e.g. aggregating function) to create the Series + from. Must be one of "min", "mean", "max", "count", "stddev", or "all". This + argument is ignored if not using StatPoints. + + name_callable : lambda, default: lambda s: s.collection + "/" + s.name + Specify a callable that can be used to determine the series name given a + Stream object. This is not compatible with agg == "all" at this time + + + + .. note:: + + This method does **not** use the ``arrow`` -accelerated endpoints for faster and more efficient data retrieval. + + """ + if pl is None or pd is None: + raise ImportError(_IMPORT_ERR_MSG.format(",".join(["polars", "pandas"]))) + + # TODO: allow this at some future point + if agg == "all" and name_callable is not None: + raise AttributeError( + "cannot provide name_callable when using 'all' as aggregate at this time" + ) + + # do not allow agg="all" with RawPoints if agg == "all" and not streamset.allow_window: - stream_names = [[s.collection, s.name, prop] for s in streamset._streams for prop in _STAT_PROPERTIES] - df.columns=pd.MultiIndex.from_tuples(stream_names) + agg = "" + + # default arg values + if not callable(name_callable): + name_callable = lambda s: s.collection + "/" + s.name + + df = streamset.to_dataframe(agg=agg, name_callable=name_callable) else: - df.columns = columns if columns else _stream_names(streamset, name_callable) + df = pd.DataFrame(to_dict(streamset, agg=agg, name_callable=name_callable)) - return df + if not df.empty: + if df.index.name == "time": + pass + else: + df = df.set_index("time") + + df.index = pd.DatetimeIndex(df.index, tz="UTC", name="time") + if agg == "all" and streamset.allow_window: + stream_names = [ + [s.collection, s.name, prop] + for s in streamset._streams + for prop in _STAT_PROPERTIES + ] + df.columns = pd.MultiIndex.from_tuples(stream_names) + else: + df.columns = _stream_names(streamset, name_callable) + + return pl.from_pandas(df, nan_to_null=False, include_index=True) def to_array(streamset, agg="mean"): @@ -168,11 +583,14 @@ def to_array(streamset, agg="mean"): arrays. Must be one of "min", "mean", "max", "count", or "stddev". This argument is ignored if RawPoint values are passed into the function. + + .. note:: + + This method does **not** use the ``arrow`` -accelerated endpoints for faster and more efficient data retrieval. + """ - try: - import numpy as np - except ImportError: - raise ImportError("Please install Numpy to use this transformation function.") + if np is None: + raise ImportError(_IMPORT_ERR_MSG.format("numpy")) # TODO: allow this at some future point if agg == "all": @@ -186,8 +604,41 @@ def to_array(streamset, agg="mean"): segment.append(point.value) else: segment.append(getattr(point, agg)) - results.append(segment) - return np.array(results) + results.append(np.array(segment)) + return np.array(results, dtype=object) + + +def arrow_to_numpy(streamset, agg=None): + """Return a multidimensional array in the numpy format. + + Parameters + ---------- + agg : List[str], default: ["mean"] + Specify the StatPoint field or fields (e.g. aggregating function) to return for the + arrays. Must be one or more of "min", "mean", "max", "count", or "stddev". This + argument is ignored if RawPoint values are passed into the function. + + + .. note:: + + This method first converts to a pandas data frame then to a numpy array. + + + .. note:: + + This method is available for commercial customers with arrow-enabled servers. + + """ + if not streamset._btrdb._ARROW_ENABLED: + raise NotImplementedError( + "arrow_to_numpy requires an arrow-enabled BTrDB server." + ) + if np is None or pa is None or pd is None: + raise ImportError( + _IMPORT_ERR_MSG.format(",".join(["numpy", "pyarrow", "pandas"])) + ) + arrow_df = arrow_to_dataframe(streamset=streamset, agg=agg, name_callable=None) + return arrow_df.values def to_dict(streamset, agg="mean", name_callable=None): @@ -203,34 +654,85 @@ def to_dict(streamset, agg="mean", name_callable=None): argument is ignored if RawPoint values are passed into the function. name_callable : lambda, default: lambda s: s.collection + "/" + s.name - Sprecify a callable that can be used to determine the series name given a + Specify a callable that can be used to determine the series name given a Stream object. + + .. note:: + + This method does **not** use the ``arrow`` -accelerated endpoints for faster and more efficient data retrieval. + """ if not callable(name_callable): - name_callable = lambda s: s.collection + "/" + s.name + name_callable = lambda s: s.collection + "/" + s.name data = [] stream_names = _stream_names(streamset, name_callable) for row in streamset.rows(): - item = OrderedDict({ - "time": _get_time_from_row(row), - }) + item = OrderedDict( + { + "time": _get_time_from_row(row), + } + ) for idx, col in enumerate(stream_names): if row[idx].__class__.__name__ == "RawPoint": item[col] = row[idx].value if row[idx] else None else: if agg == "all": for stat in _STAT_PROPERTIES: - item["{}-{}".format(col, stat)] = getattr(row[idx], stat) if row[idx] else None + item["{}-{}".format(col, stat)] = ( + getattr(row[idx], stat) if row[idx] else None + ) else: item[col] = getattr(row[idx], agg) if row[idx] else None data.append(item) return data -def to_csv(streamset, fobj, dialect=None, fieldnames=None, agg="mean", name_callable=None): +def arrow_to_dict(streamset, agg=None, name_callable=None): + """ + Returns a list of dicts for each time code with the appropriate + stream data attached. + + Parameters + ---------- + agg : List[str], default: ["mean"] + Specify the StatPoint field or fields (e.g. aggregating function) to constrain dict + keys. Must be one or more of "min", "mean", "max", "count", or "stddev". This + argument is ignored if RawPoint values are passed into the function. + + name_callable : lambda, default: lambda s: s.collection + "/" + s.name + Specify a callable that can be used to determine the series name given a + Stream object. + + + .. note:: + + This method is available for commercial customers with arrow-enabled servers. + + """ + if not streamset._btrdb._ARROW_ENABLED: + raise NotImplementedError( + "arrow_to_dict requires an arrow-enabled BTrDB server." + ) + if pa is None or pd is None: + raise ImportError(_IMPORT_ERR_MSG.format(",".join(["pyarrow", "pandas"]))) + if agg is None: + agg = ["mean"] + if not isinstance(agg, list): + raise ValueError( + f"Expected a list of strings for 'agg', was provided type - {type(agg)} with values {agg}." + ) + arrow_df = arrow_to_dataframe( + streamset=streamset, agg=agg, name_callable=name_callable + ) + return arrow_df.to_dict(orient="index") + + +def to_csv( + streamset, fobj, dialect=None, fieldnames=None, agg="mean", name_callable=None +): """ Saves stream data as a CSV file. @@ -253,8 +755,14 @@ def to_csv(streamset, fobj, dialect=None, fieldnames=None, agg="mean", name_call This argument is ignored if RawPoint values are passed into the function. name_callable : lambda, default: lambda s: s.collection + "/" + s.name - Sprecify a callable that can be used to determine the series name given a + Specify a callable that can be used to determine the series name given a Stream object. + + + .. note:: + + This method does **not** use the ``arrow`` -accelerated endpoints for faster and more efficient data retrieval. + """ # TODO: allow this at some future point @@ -262,12 +770,12 @@ def to_csv(streamset, fobj, dialect=None, fieldnames=None, agg="mean", name_call raise AttributeError("cannot use 'all' as aggregate at this time") if not callable(name_callable): - name_callable = lambda s: s.collection + "/" + s.name + name_callable = lambda s: s.collection + "/" + s.name @contextlib.contextmanager def open_path_or_file(path_or_file): if isinstance(path_or_file, str): - f = file_to_close = open(path_or_file, 'w', newline='') + f = file_to_close = open(path_or_file, "w", newline="") else: f = path_or_file file_to_close = None @@ -301,37 +809,60 @@ def to_table(streamset, agg="mean", name_callable=None): argument is ignored if RawPoint values are passed into the function. name_callable : lambda, default: lambda s: s.collection + "/" + s.name - Sprecify a callable that can be used to determine the column name given a + Specify a callable that can be used to determine the column name given a Stream object. + + .. note:: + + This method does **not** use the ``arrow`` -accelerated endpoints for faster and more efficient data retrieval. + """ try: from tabulate import tabulate except ImportError: - raise ImportError("Please install tabulate to use this transformation function.") + raise ImportError( + "Please install tabulate to use this transformation function." + ) # TODO: allow this at some future point if agg == "all": raise AttributeError("cannot use 'all' as aggregate at this time") if not callable(name_callable): - name_callable = lambda s: s.collection + "/" + s.name + name_callable = lambda s: s.collection + "/" + s.name - return tabulate(streamset.to_dict(agg=agg, name_callable=name_callable), headers="keys") + return tabulate( + streamset.to_dict(agg=agg, name_callable=name_callable), headers="keys" + ) ########################################################################## ## Transform Classes ########################################################################## + class StreamSetTransformer(object): """ Base class for StreamSet or Stream transformations """ + to_dict = to_dict + arrow_to_dict = arrow_to_dict + to_array = to_array + arrow_to_numpy = arrow_to_numpy + to_series = to_series + arrow_to_series = arrow_to_series + to_dataframe = to_dataframe + arrow_to_dataframe = arrow_to_dataframe + + to_polars = to_polars + arrow_to_polars = arrow_to_polars + + arrow_to_arrow_table = arrow_to_arrow_table to_csv = to_csv to_table = to_table diff --git a/btrdb/utils/buffer.py b/btrdb/utils/buffer.py index 02b2fc5..d06f7b5 100644 --- a/btrdb/utils/buffer.py +++ b/btrdb/utils/buffer.py @@ -17,20 +17,18 @@ from collections import defaultdict - ########################################################################## ## Classes ########################################################################## -class PointBuffer(defaultdict): +class PointBuffer(defaultdict): def __init__(self, length, *args, **kwargs): super().__init__(lambda: [None] * length, *args, **kwargs) self.num_streams = length self.active = [True] * length self.last_known_time = [None] * length - def add_point(self, stream_index, point): self.last_known_time[stream_index] = max( (self.last_known_time[stream_index] or -1), point.time @@ -53,7 +51,6 @@ def is_ready(self, key): """ # check each stream value to see if its valid/ready for idx, latest_time in enumerate(self.last_known_time): - # return False if stream is active and we are waiting on val if self.active[idx] and (latest_time is None or key > latest_time): return False @@ -61,9 +58,7 @@ def is_ready(self, key): return True def next_key_ready(self): - """ - - """ + """ """ keys = list(self.keys()) keys.sort() for key in keys: diff --git a/btrdb/utils/conversion.py b/btrdb/utils/conversion.py index a1b979f..687be95 100644 --- a/btrdb/utils/conversion.py +++ b/btrdb/utils/conversion.py @@ -17,12 +17,12 @@ ## Imports ########################################################################## -import uuid import json -import pytz - +import uuid from datetime import datetime +import pytz + try: import numpy as np except ImportError: @@ -36,6 +36,7 @@ ## Classes ########################################################################## + class AnnotationEncoder(json.JSONEncoder): """Default JSON encoder class for saving stream annotations""" @@ -87,6 +88,7 @@ def decode(self, s): ## Functions ########################################################################## + def to_uuid(obj): """ Converts argument to UUID @@ -104,6 +106,4 @@ def to_uuid(obj): if isinstance(obj, str): return uuid.UUID(obj) - raise TypeError("Cannot convert object to UUID ({})".format( - obj.__class__.__name__) - ) + raise TypeError("Cannot convert object to UUID ({})".format(obj.__class__.__name__)) diff --git a/btrdb/utils/credentials.py b/btrdb/utils/credentials.py index e7d2510..d0c8263 100644 --- a/btrdb/utils/credentials.py +++ b/btrdb/utils/credentials.py @@ -16,10 +16,11 @@ ########################################################################## import os -import yaml from functools import wraps -from btrdb.exceptions import ProfileNotFound, CredentialsFileNotFound +import yaml + +from btrdb.exceptions import CredentialsFileNotFound, ProfileNotFound ########################################################################## ## Module Variables @@ -27,21 +28,27 @@ CONFIG_DIR = ".predictivegrid" CREDENTIALS_FILENAME = "credentials.yaml" -CREDENTIALS_PATH = os.path.join(os.path.expanduser("~"), CONFIG_DIR, CREDENTIALS_FILENAME) +CREDENTIALS_PATH = os.path.join( + os.path.expanduser("~"), CONFIG_DIR, CREDENTIALS_FILENAME +) ########################################################################## ## Functions ########################################################################## + def filter_none(f): """ decorator for removing dict items with None as value """ + @wraps(f) def inner(*args, **kwargs): - return { k: v for k, v in f(*args, **kwargs).items() if v is not None } + return {k: v for k, v in f(*args, **kwargs).items() if v is not None} + return inner + def load_credentials_from_file(): """ Returns a dict of the credentials file contents @@ -52,10 +59,13 @@ def load_credentials_from_file(): A dictionary of profile connection information """ try: - with open(CREDENTIALS_PATH, 'r') as stream: + with open(CREDENTIALS_PATH, "r") as stream: return yaml.safe_load(stream) except FileNotFoundError as exc: - raise CredentialsFileNotFound("Cound not find `{}`".format(CREDENTIALS_PATH)) from exc + raise CredentialsFileNotFound( + "Cound not find `{}`".format(CREDENTIALS_PATH) + ) from exc + @filter_none def credentials_by_profile(name=None): @@ -82,14 +92,16 @@ def credentials_by_profile(name=None): The requested profile could not be found in the credentials file """ if not name: - name = os.environ.get("BTRDB_PROFILE", 'default') + name = os.environ.get("BTRDB_PROFILE", "default") # load from credentials yaml file creds = load_credentials_from_file() if name not in creds.keys(): - if name == 'default': + if name == "default": return {} - raise ProfileNotFound("Profile `{}` not found in credentials file.".format(name)) + raise ProfileNotFound( + "Profile `{}` not found in credentials file.".format(name) + ) # rename api_key if needed and return fragment = creds[name].get("btrdb", {}) @@ -97,6 +109,7 @@ def credentials_by_profile(name=None): fragment["apikey"] = fragment.pop("api_key") return fragment + @filter_none def credentials_by_env(): """ @@ -110,7 +123,7 @@ def credentials_by_env(): """ return { "endpoints": os.environ.get("BTRDB_ENDPOINTS", None), - "apikey": os.environ.get("BTRDB_API_KEY", None), + "apikey": os.environ.get("BTRDB_API_KEY", None), } @@ -131,7 +144,12 @@ def credentials(endpoints=None, apikey=None): """ creds = {} - credentials_by_arg = filter_none(lambda: { "endpoints": endpoints, "apikey": apikey, }) + credentials_by_arg = filter_none( + lambda: { + "endpoints": endpoints, + "apikey": apikey, + } + ) pipeline = [credentials_by_env, credentials_by_arg] if os.path.exists(CREDENTIALS_PATH): pipeline.insert(0, credentials_by_profile) diff --git a/btrdb/utils/general.py b/btrdb/utils/general.py index 15ce1b8..d438fce 100644 --- a/btrdb/utils/general.py +++ b/btrdb/utils/general.py @@ -12,12 +12,14 @@ """ General utilities for btrdb bindings """ +from datetime import timedelta + +from btrdb.utils.timez import to_nanoseconds + ########################################################################## ## Functions ########################################################################## - - def unpack_stream_descriptor(desc): """ Returns dicts for tags and annotations found in supplied stream @@ -36,6 +38,7 @@ def unpack_stream_descriptor(desc): ## Pointwidth Helpers ########################################################################## + class pointwidth(object): """ A representation of a period of time described by the BTrDB tree (and used in @@ -58,7 +61,7 @@ def from_timedelta(cls, delta): specified duration. Because pointwidths are in powers of 2, be sure to check that the returned real duration is sufficient. """ - return cls.from_nanoseconds(int(delta.total_seconds()*1e9)) + return cls.from_nanoseconds(int(delta.total_seconds() * 1e9)) @classmethod def from_nanoseconds(cls, nsec): @@ -73,6 +76,50 @@ def from_nanoseconds(cls, nsec): break return cls(pos) + def for_aligned_windows(self, start, end): + """ + Returns the aligned_windows's first and last timestamps, as well as the number of windows, + based on a given pointwidth set. + + Parameters + ---------- + start : int or datetime like object + The start time in nanoseconds for the range to be queried. (see + :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) + + end : int or datetime like object + The end time in nanoseconds for the range to be queried. (see + :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) + + + Returns + ------- + aligned_start: int + First timestamp would be returned by `aligned_windows` that is inclusive of specified start + timestamp. + aligned_end: int + Last timestamp would be returned by `aligned_windows` that is inclusive of specified end + timestamp. + n_windows: int + The number of windows would be returned by `aligned_windows`. + + Examples + -------- + Querying `aligned_windows` of pointwidth of 30, spaning 1 day (~24 hours). + + >>> start, end = "2016-03-01", "2016-03-02" + >>> pointwidth(30).for_aligned_windows(start, end) + (1456790399996657664, 1456876798632525824, 80466) + # output timestamp's `strftime` is: + # ['2016-02-29 23:59:59.996657664', '2016-03-01 23:59:58.632525824'] + + """ + start, end = to_nanoseconds(start), to_nanoseconds(end) + aligned_start = start - (start % self.nanoseconds) + n_windows = (end - aligned_start) // self.nanoseconds + aligned_end = aligned_start + (self.nanoseconds * (n_windows - 1)) + return aligned_start, aligned_end, n_windows + def __init__(self, p): self._pointwidth = int(p) @@ -117,10 +164,16 @@ def years(self): return self.nanoseconds / 3.154e16 def decr(self): - return pointwidth(self-1) + return pointwidth(self - 1) def incr(self): - return pointwidth(self+1) + return pointwidth(self + 1) + + def to_timedelta(self): + """ + Returns the timedelta of the pointwidth. + """ + return timedelta(microseconds=self.microseconds) def __int__(self): return self._pointwidth @@ -141,10 +194,10 @@ def __ge__(self, other): return int(self) >= int(other) def __add__(self, other): - return pointwidth(int(self)+int(other)) + return pointwidth(int(self) + int(other)) def __sub__(self, other): - return pointwidth(int(self)-int(other)) + return pointwidth(int(self) - int(other)) def __str__(self): """ @@ -180,4 +233,4 @@ def __str__(self): return "{:0.0f} nanoseconds".format(self.nanoseconds) def __repr__(self): - return "".format(int(self)) \ No newline at end of file + return "".format(int(self)) diff --git a/btrdb/utils/ray.py b/btrdb/utils/ray.py index 8df9527..b1bbb66 100644 --- a/btrdb/utils/ray.py +++ b/btrdb/utils/ray.py @@ -1,10 +1,9 @@ from functools import partial -import ray - import btrdb from btrdb.conn import BTrDB + def register_serializer(conn_str=None, apikey=None, profile=None): """ Register serializer for BTrDB Object @@ -22,8 +21,44 @@ def register_serializer(conn_str=None, apikey=None, profile=None): found in the user's predictive grid credentials file `~/.predictivegrid/credentials.yaml`. """ - ray.register_custom_serializer( - BTrDB, serializer=btrdb_serializer, deserializer=partial(btrdb_deserializer, conn_str=conn_str, apikey=apikey, profile=profile)) + try: + import ray + except ImportError: + raise ImportError("must pip install ray to register custom serializer") + try: + import semver + except ImportError: + raise ImportError("must pip install semver to register custom serializer") + + assert ( + ray.is_initialized() + ), "Need to call ray.init() before registering custom serializer" + ver = semver.VersionInfo.parse(ray.__version__) + if ver.major == 0: + ray.register_custom_serializer( + BTrDB, + serializer=btrdb_serializer, + deserializer=partial( + btrdb_deserializer, conn_str=conn_str, apikey=apikey, profile=profile + ), + ) + elif ver.major == 1 and ver.minor in range( + 2, 13 + ): # as of May 2022, latest Ray version is at 1.12 + # TODO: check different versions of ray? + ray.util.register_serializer( + BTrDB, + serializer=btrdb_serializer, + deserializer=partial( + btrdb_deserializer, conn_str=conn_str, apikey=apikey, profile=profile + ), + ) + else: + raise Exception( + "Ray version %s does not have custom serialization. Please upgrade to >= 1.4.0" + % ray.__version__ + ) + def btrdb_serializer(_): """ @@ -31,10 +66,11 @@ def btrdb_serializer(_): """ return None + def btrdb_deserializer(_, conn_str=None, apikey=None, profile=None): """ deserialize function - + Parameters ---------- conn_str: str, default=None diff --git a/btrdb/utils/timez.py b/btrdb/utils/timez.py index cc311dc..3c0428a 100644 --- a/btrdb/utils/timez.py +++ b/btrdb/utils/timez.py @@ -18,27 +18,25 @@ ########################################################################## from datetime import datetime - -from operator import mul from decimal import Decimal +from operator import mul import pytz - ########################################################################## ## Module Variables ########################################################################## DATETIME_FORMATS = ( "%Y-%m-%d %H:%M:%S.%f%z", # most common RFC3339 nanoseconds - "%Y-%m-%d %H:%M:%S.%f", # expects UTC default timezone - "%Y-%m-%dT%H:%M:%S.%fZ", # JSON encoding, UTC timezone - "%Y-%m-%dT%H:%M:%SZ", # JSON encoding, UTC timezone + "%Y-%m-%d %H:%M:%S.%f", # expects UTC default timezone + "%Y-%m-%dT%H:%M:%S.%fZ", # JSON encoding, UTC timezone + "%Y-%m-%dT%H:%M:%SZ", # JSON encoding, UTC timezone "%Y-%m-%dT%H:%M:%S.%f%z", # less common JSON-ish encoding - "%Y-%m-%dT%H:%M:%S.%f", # for completeness, UTC default timezone - "%Y-%m-%d %H:%M:%S%z", # human readable date time with TZ - "%Y-%m-%d %H:%M:%S", # human readable date time UTC default - "%Y-%m-%d", # helper to get midnight on a particular date + "%Y-%m-%dT%H:%M:%S.%f", # for completeness, UTC default timezone + "%Y-%m-%d %H:%M:%S%z", # human readable date time with TZ + "%Y-%m-%d %H:%M:%S", # human readable date time UTC default + "%Y-%m-%d", # helper to get midnight on a particular date ) @@ -46,6 +44,7 @@ ## Functions ########################################################################## + def currently_as_ns(): """ Returns the current UTC time as nanoseconds since epoch @@ -56,7 +55,7 @@ def currently_as_ns(): def ns_to_datetime(ns): """ - Converts nanoseconds to a datetime object (UTC) + Converts nanoseconds to a naive datetime object (UTC+0) Parameters ---------- @@ -141,6 +140,7 @@ def to_nanoseconds(val): try: import numpy as np + if isinstance(val, np.datetime64): val = val.astype(datetime) except ImportError: @@ -174,10 +174,11 @@ def to_nanoseconds(val): raise TypeError("only int, float, str, datetime, and datetime64 are allowed") -def ns_delta(days=0, hours=0, minutes=0, seconds=0, milliseconds=0, \ - microseconds=0, nanoseconds=0): +def ns_delta( + days=0, hours=0, minutes=0, seconds=0, milliseconds=0, microseconds=0, nanoseconds=0 +): """ - Similar to `timedelta`, ns_delta represents a span of time but as + Similar to ``timedelta``, ``ns_delta`` represents a span of time but as the total number of nanoseconds. Parameters @@ -201,6 +202,14 @@ def ns_delta(days=0, hours=0, minutes=0, seconds=0, milliseconds=0, \ ------- amount of time in nanoseconds : int + + Examples + -------- + 1 minute time delta should be 60 billion nanoseconds + + >>> deltaT = ns_delta(minutes=1) + >>> deltaT == 1 * 60 * 10**9 # 1 minute * 60 seconds * 1billion nanoseconds / second + True """ MICROSECOND = 1000 MILLISECOND = MICROSECOND * 1000 diff --git a/btrdb/version.py b/btrdb/version.py index 6df466a..6727e46 100644 --- a/btrdb/version.py +++ b/btrdb/version.py @@ -15,20 +15,25 @@ ## Module Info ########################################################################## -__version_info__ = { 'major': 5, 'minor': 11, 'micro': 0, 'releaselevel': 'final'} + +__version_info__ = {"major": 5, "minor": 34, "micro": 2, "releaselevel": "final"} + ########################################################################## ## Helper Functions ########################################################################## + def get_version(short=False): """ Prints the version. """ - assert __version_info__['releaselevel'] in ('alpha', 'beta', 'final') - vers = ["%(major)i.%(minor)i" % __version_info__, ] - if __version_info__['micro']: + assert __version_info__["releaselevel"] in ("alpha", "beta", "final") + vers = [ + "%(major)i.%(minor)i" % __version_info__, + ] + if __version_info__["micro"]: vers.append(".%(micro)i" % __version_info__) - if __version_info__['releaselevel'] != 'final' and not short: - vers.append('%s%i' % (__version_info__['releaselevel'][0])) - return ''.join(vers) + if __version_info__["releaselevel"] != "final" and not short: + vers.append("%s%i" % (__version_info__["releaselevel"][0])) + return "".join(vers) diff --git a/btrdb4/__init__.py b/btrdb4/__init__.py index 85af01b..ee79a41 100644 --- a/btrdb4/__init__.py +++ b/btrdb4/__init__.py @@ -21,11 +21,13 @@ ## Warning ########################################################################## -MESSAGE = ("The btrdb4 package is not available in this version. To install " - "with pip use `pip install btrdb==4.*`") +MESSAGE = ( + "The btrdb4 package is not available in this version. To install " + "with pip use `pip install btrdb==4.*`" +) # raise exception on ImportWarning -warnings.simplefilter('error', ImportWarning) +warnings.simplefilter("error", ImportWarning) # warn user of version issue warnings.warn(MESSAGE, ImportWarning) diff --git a/docs/requirements.txt b/docs/requirements.txt index 074ee76..6d1cc57 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,14 @@ alabaster>=0.7.12 -Sphinx>=1.7 \ No newline at end of file +Sphinx>=1.7 +ipython +sphinx-rtd-theme +sphinx-copybutton +sphinx-design +numpydoc +pydata-sphinx-theme +pyarrow +pandas +polars +grpcio +grpcio-tools +pyyaml diff --git a/docs/source/api-index.rst b/docs/source/api-index.rst new file mode 100644 index 0000000..72b478f --- /dev/null +++ b/docs/source/api-index.rst @@ -0,0 +1,16 @@ + +.. _API REF: + +API Reference +------------- + +.. toctree:: + :maxdepth: 2 + + api/package + api/conn + api/streams + api/points + api/exceptions + api/transformers + api/utils-timez diff --git a/docs/source/api/conn.rst b/docs/source/api/conn.rst index c577eaa..62ba065 100644 --- a/docs/source/api/conn.rst +++ b/docs/source/api/conn.rst @@ -1,6 +1,7 @@ btrdb.conn ============= +.. _Conn info: .. automodule:: btrdb.conn .. autoclass:: BTrDB diff --git a/docs/source/api/streams.rst b/docs/source/api/streams.rst index 6c55473..7707bbc 100644 --- a/docs/source/api/streams.rst +++ b/docs/source/api/streams.rst @@ -1,10 +1,15 @@ btrdb.stream ============== + +.. _StreamGeneralDocs: .. automodule:: btrdb.stream .. autoclass:: Stream :members: -.. autoclass:: StreamSetBase +.. _StreamSet API: +.. autoclass:: StreamSet + :show-inheritance: + :inherited-members: :members: diff --git a/docs/source/api/transformers.rst b/docs/source/api/transformers.rst index 289a1ea..830e0b5 100644 --- a/docs/source/api/transformers.rst +++ b/docs/source/api/transformers.rst @@ -1,17 +1,31 @@ +.. _TransformersDocs: btrdb.transformers ========================= -A number of tranformation and serialization functions have been developed so +A number of transformation and serialization functions have been developed so you can use the data in the format of your choice. These functions are provided -in the :code:`btrdb.utils.transformers` module but are also available directly -off the the :code:`StreamSet` class. +in the :code:`StreamSet` class. .. automodule:: btrdb.transformers +.. autofunction:: arrow_to_dict + +.. autofunction:: arrow_to_numpy + +.. autofunction:: arrow_to_series + +.. autofunction:: arrow_to_dataframe + +.. autofunction:: arrow_to_polars + +.. autofunction:: arrow_to_arrow_table + .. autofunction:: to_dict .. autofunction:: to_array +.. autofunction:: to_polars + .. autofunction:: to_series .. autofunction:: to_dataframe diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst new file mode 100644 index 0000000..59ff417 --- /dev/null +++ b/docs/source/changelog.rst @@ -0,0 +1,121 @@ +.. _changelog: +Changelog +========= + +5.32.0 +------ + +What's Changed +^^^^^^^^^^^^^^ +- Initial docstring overhaul and a new test for better documentation and test coverage. by @JustinGilmer in `#82 `_ +- Test new join logic for improved data loading for windowed queries. by @JustinGilmer in `#80 `_ +- Improve :code:`arrow_to_dataframe` function for handling large amounts of columns, enhancing performance and usability. by @Jefflinf in `#73 `_ +- Expand testing to include Python 3.11, ensuring compatibility and stability. by @JustinGilmer in `#74 `_ +- Update exception handling to better support :code:`RpcErrors`, improving error management and debugging. by @JustinGilmer in `#72 `_ +- Introduce an option for specifying the schema of the returned raw data, allowing for more flexibility in data handling. by @andrewchambers in `#51 `_ +- Remove non-required dependencies and migrate to 'data' optional dependency for a lighter package and easier installation. by @JustinGilmer in `#71 `_ +- New method to get first and last timestamps from ``aligned_windows``, enhancing data analysis capabilities. by @Jefflinf in `#70 `_ +- Add ``to_timedelta`` method for ``pointwidth`` class, providing more options for time-based data manipulation. by @Jefflinf in `#69 `_ + +Fixed +^^^^^ +- Fix ``NoneType`` error for ``earliest/latest`` for empty streams, ensuring reliability and error handling. by @Jefflinf in `#64 `_ +- Correct integration tests where the time column is not automatically set as the index, improving test accuracy and reliability. by @JustinGilmer in `#56 `_ + +Deprecated +^^^^^^^^^^ +- FutureWarning for ``streams_in_collection`` to return ``StreamSet`` in the future, preparing users for upcoming API changes. by @Jefflinf in `#60 `_ + +**Full Changelog**: `GitHub compare view `_ + + +5.31.0 +------ +What's Changed +^^^^^^^^^^^^^^ +* Have release script update pyproject.toml file by @youngale-pingthings in https://github.com/PingThingsIO/btrdb-python/pull/48 +* Provide option to sort the arrow tables by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/47 +* Remove 4MB limit for gRPC message payloads by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/49 +* Update documentation for arrow methods by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/50 +* Update from staging by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/54 +* Sort tables by time by default for any ``pyarrow`` tables. by @justinGilmer in +* Fix deprecation warnings for pip installations. by @jleifnf in + +**Full Changelog**: `GitHub compare view `_ + +5.30.2 +------ +What’s Changed +^^^^^^^^^^^^^^ +- Update readthedocs to new yaml for testing. by @justinGilmer in + https://github.com/PingThingsIO/btrdb-python/pull/40 +- Converting pandas index takes very long, fix this in arrow_table. by + @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/41 + +**Full Changelog**: +`5.30.2 `_ + +.. _section-1: + +5.30.1 +------ +.. _whats-changed-1: + +What’s Changed +^^^^^^^^^^^^^^ + +- Small version bump for pypi release + +**Full Changelog**: +`5.30.1 `_ + +.. _section-2: + +5.30.0 +------ +.. _whats-changed-2: + +What’s Changed +^^^^^^^^^^^^^^ + +- Merge Arrow support into Main for Release by @youngale-pingthings in + https://github.com/PingThingsIO/btrdb-python/pull/37 + + - This PR contains many changes that support the commercial Arrow + data fetches and inserts + - ``arrow_`` prefixed methods for ``Stream`` Objects: + + - ``insert, aligned_windows, windows, values`` + + - ``arrow_`` prefixed methods for StreamSet\` objects: + + - ``insert, values, to_dataframe, to_polars, to_arrow_table, to_numpy, to_dict, to_series`` + +- Justin gilmer patch 1 by @justinGilmer in + https://github.com/PingThingsIO/btrdb-python/pull/39 + +**Full Changelog**: +`5.30.0 `_ + +.. _section-3: + +5.28.1 +------ +.. _whats-changed-3: + +What’s Changed +^^^^^^^^^^^^^^ + +- Upgrade ray versions by @jleifnf in + https://github.com/PingThingsIO/btrdb-python/pull/15 +- Release v5.28.1 and Update Python by @youngale-pingthings in + https://github.com/PingThingsIO/btrdb-python/pull/17 + +New Contributors +^^^^^^^^^^^^^^^^ + +- @jleifnf made their first contribution in + https://github.com/PingThingsIO/btrdb-python/pull/15 + +**Full Changelog**: +`5.28.1 `_ diff --git a/docs/source/concepts.rst b/docs/source/concepts.rst index 09c6675..2392034 100644 --- a/docs/source/concepts.rst +++ b/docs/source/concepts.rst @@ -11,7 +11,7 @@ of their behavior which will allow you to use them effectively. .. note:: - Data requests are fully materialized at this time. A future release will include the option to process data using generators to save on memory usage. + Data requests are fully materialized at this time. A future release will include the option to process data using generators to save on memory usage. BTrDB Server @@ -22,6 +22,7 @@ BTrDB focuses on univariate data which opens a host of benefits and is one of the reasons BTrDB is able to process incredibly large amounts of data quickly and easily. +.. _Points Described: Points ------------ Points of data within a time series make up the smallest objects you will be @@ -37,98 +38,106 @@ value within the stream. .. code-block:: python - # view time and value of a single point in the stream + >>> # view time and value of a single point in the stream + >>> point.time + 1547241923338098176 - point.time - >> 1547241923338098176 - - point.value - >> 120.5 + >>> point.value + 120.5 StatPoint ^^^^^^^^^^^^ The StatPoint provides statistics about multiple points and gives -aggregation values such as `min`, `max`, `mean`, and `stddev` (standard deviation). +aggregation values such as :code:`min`, :code:`max`, :code:`mean`, :code:`count` and :code:`stddev` (standard deviation). This is most useful when you don't need to touch every individual value such as when you only need the count of the values over a range of time. These statistical queries execute in time proportional to the number of results, not the number of underlying points (i.e logarithmic time) and so you can attain valuable data in a fraction of the time when compared with retrieving -all of the individual values. Due to the internal data structures, BTrDB does +all of the individual values. Due to the internal data structures, :code:`BTrDB` does not need to read the underlying points to return these statistics! .. code-block:: python - # view aggregate values for points in a stream + >>> # view aggregate values for points in a stream + >>> point.time + 1547241923338098176 + + >>>point.min + 42.1 - point.time - >> 1547241923338098176 + >>> point.mean + 78.477 - point.min - >> 42.1 + >>> point.max + 122.4 - point.mean - >> 78.477 + >>> point.count + 18600 - point.max - >> 122.4 + >>> point.stddev + 3.4 - point.count - >> 18600 - point.stddev - >> 3.4 +Tabular Data +------------ +In addition to working with the :code:`RawPoint` or :code:`StatPoint` objects, newer versions of the platform now natively support some tabular data formats as well. +This is enabled for commercial customers and are available using the :code:`stream.arrow_` or :code:`streamset.arrow_` methods. +Refer to the :ref:`arrow enabled queries page ` and the :ref:`API docs ` Streams ------------ -Streams represent a single series of time/value pairs. As such, the database +:code:`Stream` s represent a single series of time/value pairs. As such, the database can hold an almost unlimited amount of individual streams. Each stream has a -`collection` which is similar to a "path" or grouping for multiple streams. Each -steam will also have a `name` as well as a `uuid` which is guaranteed to be unique +:code:`collection` which is similar to a "path" or grouping for multiple streams. Each +steam will also have a :code:`name` as well as a :code:`uuid` which is guaranteed to be unique across streams. -BTrDB data is versioned such that changes to a given stream (time series) will +:code:`BTrDB` data is versioned such that changes to a given stream (time series) will result in a new version for the stream. In this manner, you can pin your interactions to a specific version ensuring the values do not change over the course of your -interactions. If you want to work with the most recent version/data then -specify a version of zero (the default). +interactions. + +.. note:: + + If you want to work with the most recent version/data then specify a version of :code:`0` (the default). Each stream has a number of attributes and methods available and these are documented -within the API section of this publication. But the most common interactions -by users are to access the UUID, tags, annotations, version, and underlying data. +within the :ref:`API Reference ` section of this publication. But the most common interactions +by users are to access the :code:`UUID`, :code:`tags`, :code:`annotations`, :code:`version`, and underlying data. -Each stream uses a UUID as its unique identifier which can also be used when querying -for streams. Metadata is provided by tags and annotations which are both provided -as dictionaries of data. Tags are used internally and have very specific keys -while annotations are more free-form and can be used by you to store your own +Each stream uses a :code:`UUID` as its unique identifier which can also be used when querying +for streams. Metadata is provided by :code:`tags` and :code:`annotations` which are both provided +as dictionaries of data. :code:`tags` are used internally and have very specific keys +while :code:`annotations` are more free-form and can be used by you to store your own metadata. .. code-block:: python - # retrieve stream's UUID - stream.uuid - >> UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + >>> # retrieve stream's UUID + >>> stream.uuid + UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") - # retrieve stream's current version - stream.version() - >> 244 + >>> # retrieve stream's current version + >>> stream.version() + 244 - # retrieve stream tags - stream.tags() - >> {'name': 'L1MAG', 'unit': 'volts', 'ingress': ''} + >>> # retrieve stream tags + >>> stream.tags() + {'name': 'L1MAG', 'unit': 'volts', 'ingress': ''} - # retrieve stream annotations - stream.annotations() - >> {'poc': 'Salvatore McFesterson', 'region': 'northwest', 'state': 'WA'} + >>> # retrieve stream annotations + >>> stream.annotations() + ({'poc': 'Salvatore McFesterson', 'region': 'northwest', 'state': 'WA'}, 23) - # loop through points in the stream - for point, _ in stream.values(end=1547241923338098176, version=133): - print(point) - >> RawPoint(1500000000100000000, 2.4) - >> RawPoint(1500000000200000000, 2.8) - >> RawPoint(1500000000300000000, 3.6) + >>> # loop through points in the stream + >>> for point, _ in stream.values(end=1547241923338098176, version=133): + >>> print(point) + RawPoint(1500000000100000000, 2.4) + RawPoint(1500000000200000000, 2.8) + RawPoint(1500000000300000000, 3.6) ... @@ -136,8 +145,8 @@ StreamSets ------------ Often you will want to query and work with multiple streams instead of just an -individual stream - StreamSets allow you to do this effectively. It is a light -wrapper around a list of Stream objects with convenience methods provided to +individual stream - :code:`StreamSets` allow you to do this effectively. It is a light +wrapper around a list of :code:`Stream` objects with convenience methods provided to help you work with multiple streams of data. As an example, you can filter the stream data with a single method call and then @@ -145,17 +154,45 @@ easily transform the data into other data types such as a pandas DataFrame or to disk as a CSV file. See the examples below for a quick sample and then visit our API docs to see the full list of features provided to you. +.. note:: + + :code:`StreamSet` methods that filter and operate on the :code:`StreamSet` object (like :code:`StreamSet.filter` ) return new copies of the :code:`StreamSet` itself rather than modifying in place. Similar to how most :code:`pandas.DataFrame` methods return a new :code:`DataFrame` object. This lets you compose multiple functions in a single call, which can improve readability, but can be tricky if you are not expecting this behavior. + +Lets explore a common use-case, filtering a streamset. + .. code-block:: python - # establish database connection and query for streams by UUID - db = connect() - uuid_list = ["0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a", ...] - streams = db.streams(*uuid_list) + >>> # create a streamset and apply a few filters + >>> streamset = btrdb.stream.StreamSet(list_of_streams) + >>> print(f"Total streams: {len(streamset)}") + Total streams: 89 - streams.filter(start=1500000000000000000).to_csv("data.csv") + >>> streamset.filter(units="Volts") + >>> print(f"Total streams: {len(streamset)}") + Total streams: 89 - streams.filter(start=1500000000000000000).to_dataframe() - >> time NW/stream0 NW/stream1 + >>> filtered_streamset = streamset.filter(units="Volts") + >>> print(f"Total streams: {len(filtered_streamset)}") + Total streams: 23 + + >>> multiple_filters_streamset = (streamset.filter(unit="Volts") + >>> .filter(name="Sensor 1") + >>> .filter(annotations={"phase":"A"}) + >>> ) + >>> print(f"Total streams: {len(multiple_filters_streamset)}") + Total streams: 1 + +.. code-block:: python + + >>> # establish database connection and query for streams by UUID + >>> db = connect() + >>> uuid_list = ["0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a", ...] + >>> streams = db.streams(*uuid_list) + + >>> streams.filter(start=1500000000000000000).to_csv("data.csv") + + >>> streams.filter(start=1500000000000000000).to_dataframe() + time NW/stream0 NW/stream1 0 1500000000000000000 NaN 1.0 1 1500000000100000000 2.0 NaN 2 1500000000200000000 NaN 3.0 @@ -166,3 +203,9 @@ our API docs to see the full list of features provided to you. 7 1500000000700000000 8.0 NaN 8 1500000000800000000 NaN 9.0 9 1500000000900000000 10.0 NaN + + +Apache-Arrow Accelerated Methods +-------------------------------- + +* Refer to :ref:`Arrow Page` diff --git a/docs/source/conf.py b/docs/source/conf.py index e221fcb..5db17d4 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -18,21 +18,22 @@ import os import sys -sys.path.insert(0, os.path.abspath('_themes')) -sys.path.insert(0, os.path.abspath('../..')) + +sys.path.insert(0, os.path.abspath("_themes")) +sys.path.insert(0, os.path.abspath("../..")) from btrdb.version import get_version # -- Project information ----------------------------------------------------- -project = 'btrdb' -copyright = '2019, Michael P. Andersen' -author = 'Michael P. Andersen' +project = "btrdb" +copyright = "2024, Ping Things, Inc." +author = "PingThingsIO" # The short X.Y version version = get_version() # The full version, including alpha/beta/rc tags -release = 'v' + get_version() +release = "v" + get_version() # -- General configuration --------------------------------------------------- @@ -45,36 +46,45 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.napoleon', - 'sphinx.ext.todo', - 'sphinx.ext.githubpages', - 'sphinx.ext.intersphinx', + "sphinx.ext.autodoc", + "sphinx.ext.napoleon", + "sphinx.ext.autosummary", + # 'sphinx.ext.inheritance_diagram', + # 'sphinx.ext.intersphinx', + "sphinx.ext.ifconfig", + # 'IPython.sphinxext.ipython_console_highlighting', + # 'IPython.sphinxext.ipython_directive', + "sphinx.ext.todo", + "sphinx.ext.githubpages", + # "sphinx.ext.intersphinx", + "numpydoc", + "sphinx_copybutton", + "sphinx_design", ] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] -source_suffix = '.rst' +source_suffix = ".rst" # The master toctree document. -master_doc = 'index' +master_doc = "index" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] # The name of the Pygments (syntax highlighting) style to use. pygments_style = None @@ -85,31 +95,34 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'alabaster' +# html_theme = "alabaster" +# html_theme = "sphinx_rtd_theme" +html_theme = "pydata_sphinx_theme" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. # html_theme_options = { - 'show_powered_by': False, - 'github_user': 'BTrDB', - 'github_repo': 'btrdb-python', - 'travis_button': False, - 'github_banner': False, - 'show_related': False, - 'note_bg': '#FFF59C', - 'description': 'A midweight library to converse with the BTrDB database.', - 'extra_nav_links': { - "btrdb": "http://btrdb.io" - }, - 'show_relbars': True, + # "show_powered_by": False, + # "github_user": "PingThingsIO", + # "github_repo": "btrdb-python", + # "travis_button": False, + # "github_banner": False, + # "show_related": False, + # "note_bg": "#FFF59C", + # "description": "A midweight library to converse with the BTrDB database.", + # "extra_nav_links": {"btrdb": "http://btrdb-python.readthedocs.io"}, + # "show_relbars": True, + "show_toc_level": 2, + "navigation_depth": 4, # Adjust the depth of the sidebar TOC + "show_nav_level": 2, # Initially shown levels of the TOC } # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # Custom sidebar templates, must be a dictionary that maps document names # to template names. @@ -131,7 +144,7 @@ # -- Options for HTMLHelp output --------------------------------------------- # Output file base name for HTML help builder. -htmlhelp_basename = 'btrdb-pythondoc' +htmlhelp_basename = "btrdb-pythondoc" # -- Options for LaTeX output ------------------------------------------------ @@ -140,15 +153,12 @@ # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. # # 'preamble': '', - # Latex figure (float) alignment # # 'figure_align': 'htbp', @@ -158,8 +168,13 @@ # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'btrdb-python.tex', 'btrdb-python Documentation', - 'Michael Andersen', 'manual'), + ( + master_doc, + "btrdb-python.tex", + "btrdb-python Documentation", + "PingThingsIO", + "manual", + ), ] @@ -167,10 +182,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'btrdb-python', 'btrdb-python Documentation', - [author], 1) -] +man_pages = [(master_doc, "btrdb-python", "btrdb-python Documentation", [author], 1)] # -- Options for Texinfo output ---------------------------------------------- @@ -179,9 +191,15 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'btrdb-python', 'btrdb-python Documentation', - author, 'btrdb-python', 'One line description of project.', - 'Miscellaneous'), + ( + master_doc, + "btrdb-python", + "btrdb-python Documentation", + author, + "btrdb-python", + "One line description of project.", + "Miscellaneous", + ), ] @@ -200,7 +218,7 @@ # epub_uid = '' # A list of files that should not be packed into the epub file. -epub_exclude_files = ['search.html'] +epub_exclude_files = ["search.html"] # -- Extension configuration ------------------------------------------------- @@ -210,4 +228,4 @@ # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = True -numfig = True \ No newline at end of file +numfig = True diff --git a/docs/source/explained.rst b/docs/source/explained.rst index 6ace50e..d40eea7 100644 --- a/docs/source/explained.rst +++ b/docs/source/explained.rst @@ -5,7 +5,7 @@ BTrDB Explained **A next-gen timeseries database for dense, streaming telemetry.** -**Problem**: Existing timeseries databases are poorly equipped for a new generation of ultra-fafst sensor telemetry. Specifically, millions of high-precision power meters are to be deployed through the power grid to help analyze and prevent blackouts. Thus, new software must be built to facilitate the storage and analysis of its data. +**Problem**: Existing timeseries databases are poorly equipped for a new generation of ultra-fast sensor telemetry. Specifically, millions of high-precision power meters are to be deployed through the power grid to help analyze and prevent blackouts. Thus, new software must be built to facilitate the storage and analysis of its data. **Baseline**: We need 1.4M inserts/second and 5x that in reads if we are to support 1000 `micro-synchrophasors`_ per server node. No timeseries database can do this. @@ -126,6 +126,5 @@ The original version of this page can be found at: This page is written based on the following sources: -- `Homepage `_ - `Whitepaper `_ - `Code `_ diff --git a/docs/source/index.rst b/docs/source/index.rst index 66d2de0..5320d6f 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,8 +1,8 @@ Welcome to btrdb docs! ====================== -.. image:: https://img.shields.io/travis/BTrDB/btrdb-python/master.svg - :target: https://travis-ci.org/BTrDB/btrdb-python +.. image:: https://github.com/PingThingsIO/btrdb-python/actions/workflows/release.yaml/badge.svg + :target: https://github.com/PingThingsIO/btrdb-python/actions .. image:: https://readthedocs.org/projects/btrdb/badge/?version=latest :target: https://btrdb.readthedocs.io/en/latest/ @@ -16,15 +16,6 @@ Welcome to btrdb docs! .. image:: https://img.shields.io/pypi/v/btrdb.svg :target: https://pypi.python.org/project/btrdb/ -.. note:: - - Starting with the 5.0 release, btrdb-python will be Python 3 only! This decision - was not made lightly but is necessary to keep compatibility with underlying - packages. - - In addition, this software is only compatible with version 5.x of the BTrDB - server. To communicate with a 4.x server, please install an earlier version - of this software. Welcome to btrdb-python's documentation. We provide Python access to the Berkeley Tree Database (BTrBD) along with some select convenience methods. If you are @@ -42,7 +33,7 @@ your appetite. import btrdb from btrdb.utils.timez import to_nanoseconds - # establish connection to server + # establish connection to a server conn = btrdb.connect("192.168.1.101:4410") # search for streams and view metadata @@ -59,6 +50,9 @@ your appetite. for point, _ in stream.values(start, end): print(point.time, point.value) + # return the data as an arrow table instead + data = stream.arrow_values(start, end) + User Guide ---------- @@ -75,27 +69,8 @@ in Github! concepts working explained - - -API Reference -------------- - -.. toctree:: - :maxdepth: 2 - - api/package - api/conn - api/streams - api/points - api/exceptions - api/transformers - api/utils-timez - - -Maintainers ------------ - -* :doc:`maintainers/anaconda` + api-index + changelog diff --git a/docs/source/installing.rst b/docs/source/installing.rst index 394c068..99a873b 100644 --- a/docs/source/installing.rst +++ b/docs/source/installing.rst @@ -1,25 +1,36 @@ Installing ======================== -The btrdb package has only a few requirements and is relatively easy to install. +The :code:`btrdb` package has only a few requirements and is relatively easy to install. A number of installation options are available as detailed below. Installing with pip ------------------- -We recommend using pip to install btrdb-python on all platforms: +We recommend using :code:`pip` to install :code:`btrdb-python` on all platforms: .. code-block:: bash $ pip install btrdb -To get a specific version of btrdb-python supply the version number. The major -version of this library is pegged to the major version of the BTrDB database as -in the 4.x bindings are best used to speak to a 4.x BTrDB database. +With :code:`btrdb>=5.30.2`, there are now extra dependencies that can be installed with ``pip``. +We recommend installing the :code:`data` extra dependencies (the second option in the code block below). + + +.. code-block:: shell-session + + $ pip install "btrdb>=5.30.2" # standard btrdb + $ pip install "btrdb[data]>=5.30.2" # btrdb with data science packages included (recommended) + $ pip install "btrdb[all]>=5.30.2" # btrdb with testing, data science and all other optional packages + + +To get a specific version of :code:`btrdb-python` supply the version number. The major +version of this library is tied to the major version of the BTrDB database as +in the 4.X bindings are best used to speak to a 4.X BTrDB database, the 5.X bindings for 5.X platform.. .. code-block:: bash - $ pip install btrdb==4.11.2 + $ pip install "btrdb[data]==5.30.2" To upgrade using pip: @@ -32,14 +43,4 @@ To upgrade using pip: Installing with Anaconda ------------------------ -If you'd like to use Anaconda, you'll need to download the library from the pingthings -channel as shown below. - -Note however that only the version 5 bindings are available in Anaconda Cloud. If you'd -like to install the version 4 bindings you will need to use `pip` as shown above. - -.. code-block:: bash - - $ conda install -c pingthings btrdb - - +We recommend installing using :code:`pip`. diff --git a/docs/source/quick-start.rst b/docs/source/quick-start.rst index bc4af5f..7e70ce8 100644 --- a/docs/source/quick-start.rst +++ b/docs/source/quick-start.rst @@ -9,23 +9,31 @@ Connecting to a server is easy with the supplied :code:`connect` function from t .. code-block:: python - import btrdb + >>> import btrdb + >>> # connect with API key + >>> conn = btrdb.connect("192.168.1.101:4411", apikey="123456789123456789") + >>> conn + + +Get Platform Information +^^^^^^^^^^^^^^^^^^^^^^^^ +.. code-block:: python - # connect without credentials - conn = btrdb.connect("192.168.1.101:4410") + >>> conn.info() + {'majorVersion': ..., + 'minorVersion': ..., + 'build': ..., + 'proxy': {...}} - # connect using TLS - conn = btrdb.connect("192.168.1.101:4411") - # connect with API key - conn = btrdb.connect("192.168.1.101:4411", apikey="123456789123456789") +Refer to :ref:`the connection API documentation page. ` Retrieving a Stream ---------------------- -In order to interact with data, you'll need to obtain or create a :code:`Stream` object. A -number of options are available to get existing streams. +In order to interact with data, you'll need to obtain or create a :code:`Stream` object. +A number of options are available to get existing streams. Find streams by collection ^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -35,9 +43,9 @@ collection you can use the :code:`streams_in_collection` method. .. code-block:: python - streams = conn.streams_in_collection("USEAST_NOC1/90807") - for stream in streams: - print(stream.uuid, stream.name) + >>> streams = conn.streams_in_collection("USEAST_NOC1/90807") + >>> for stream in streams: + >>> print(stream.uuid, stream.name) Find stream by UUID ^^^^^^^^^^^^^^^^^^^^^ @@ -47,7 +55,7 @@ would like to retrieve. For convenience, this method accepts instances of either .. code-block:: python - stream = conn.stream_from_uuid("07d28a44-4991-492d-b9c5-2d8cec5aa6d4") + >>> stream = conn.stream_from_uuid("07d28a44-4991-492d-b9c5-2d8cec5aa6d4") @@ -61,42 +69,65 @@ if needed. .. code-block:: python - start = datetime(2018,1,1,12,30, tzinfo=timezone.utc) - start = start.timestamp() * 1e9 - end = start + (3600 * 1e9) + >>> start = datetime(2018,1,1,12,30, tzinfo=timezone.utc) + >>> start = start.timestamp() * 1e9 + >>> end = start + (3600 * 1e9) - for point, _ in stream.values(start, end): - print(point.time, point.value) + >>> for point, _ in stream.values(start, end): + >>> print(point.time, point.value) Some convenience functions are available to make it easier to deal with converting to nanoseconds. .. code-block:: python - from btrdb.utils.timez import to_nanoseconds, currently_as_ns + >>> from btrdb.utils.timez import to_nanoseconds, currently_as_ns - start = to_nanoseconds(datetime(2018,1,1, tzinfo=timezone.utc)) - end = currently_as_ns() + >>> start = to_nanoseconds(datetime(2018,1,1, tzinfo=timezone.utc)) + >>> end = currently_as_ns() - for point, _ in stream.values(start, end): - print(point.time, point.value) + >>> for point, _ in stream.values(start, end): + >>> print(point.time, point.value) You can also view windows of data at arbitrary levels of detail. One such windowing feature is shown below. .. code-block:: python - # query for windows of data 10,000 nanoseconds wide using a depth of zero - # which is accurate to the nanosecond. - params = { - "start": 1500000000000000000, - "end": 1500000000010000000, - "width": 2000000, - "depth": 0, - } - for window in stream.windows(**params): - for point, version in window: - print(point, version) + >>> # query for windows of data 10,000 nanoseconds wide using a depth of zero + >>> # which is accurate to the nanosecond. + >>> params = { + ... "start": 1500000000000000000, + ... "end": 1500000000010000000, + ... "width": 2000000, + ... "depth": 0, + ... } + >>> for window in stream.windows(**params): + >>> for point, version in window: + >>> print(point, version) + + +Return data as :code:`arrow` tables +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Instead of returning data a :code:`RawPoint` at a time, which can be more computationally intensive, there is now the ability to return the data in a tabular format from the start, which can drastically save on run time as well as facilitate interoperability with many more data-science driven tools. +`Apache Arrow is a language agnostic columnar data schema `_ that has become a defacto standard for in-memory data analytics. +All data retrieval methods in :code:`BTrDB` now have corresponding :code:`arrow-` prepended methods that natively return :code:`pyarrow` data tables. + +.. code-block:: python + + >>> s.arrow_values(start=1500000000000000000, end=1500000002000000001).to_pandas() + time value + 0 2017-07-14 02:40:00+00:00 1.0 + 1 2017-07-14 02:40:00.100000+00:00 2.0 + 2 2017-07-14 02:40:00.200000+00:00 3.0 + 3 2017-07-14 02:40:00.300000+00:00 4.0 + 4 2017-07-14 02:40:00.400000+00:00 5.0 + 5 2017-07-14 02:40:00.500000+00:00 6.0 + 6 2017-07-14 02:40:00.600000+00:00 7.0 + 7 2017-07-14 02:40:00.700000+00:00 8.0 + 8 2017-07-14 02:40:00.800000+00:00 9.0 + 9 2017-07-14 02:40:00.900000+00:00 10.0 + Using StreamSets -------------------- @@ -110,33 +141,45 @@ level of the individual :code:`Stream` object. Aside from being useful to see concurrent data across streams, you can also easily transform the data to other data structures or even serialize the data to disk in one operation. -Some quick examples are shown below but please review the API docs for the full +Some quick examples are shown below but please review the :ref:`API docs ` for the full list of features. + +.. note:: + + In the following examples, notice that the end time is **not** inclusive of the data that is present at :code:`end` . :code:`start` is **inclusive** while :code:`end` is **exclusive**. This is the case for **all** :code:`BTrDB` data query operations. + + .. math:: + [start, end) + + .. code-block:: python - streams = db.streams(*uuid_list) - - # serialize data to disk as CSV - streams.filter(start=1500000000000000000).to_csv("data.csv") - - # convert data to a pandas DataFrame - streams.filter(start=1500000000000000000).to_dataframe() - >> time NW/stream0 NW/stream1 - 0 1500000000000000000 NaN 1.0 - 1 1500000000100000000 2.0 NaN - 2 1500000000200000000 NaN 3.0 - 3 1500000000300000000 4.0 NaN - 4 1500000000400000000 NaN 5.0 - 5 1500000000500000000 6.0 NaN - 6 1500000000600000000 NaN 7.0 - 7 1500000000700000000 8.0 NaN - 8 1500000000800000000 NaN 9.0 - 9 1500000000900000000 10.0 NaN - - # materialize the streams' data - streams.filter(start=1500000000000000000).values() - >> [[RawPoint(1500000000100000000, 2.0), + >>> streams = db.streams(*uuid_list) + + >>> # serialize data to disk as CSV + >>> streams.filter(start=1500000000000000000, end=1500000000900000000).to_csv("data.csv") + + >>> # convert data to a pandas DataFrame + >>> streams.filter(start=1500000000000000000, end=1500000000900000000).to_dataframe() + nw/stream0 nw/stream1 + time + 1500000000000000000 nan 1.0 + 1500000000100000000 2.0 nan + 1500000000200000000 nan 3.0 + 1500000000300000000 4.0 nan + 1500000000400000000 nan 5.0 + 1500000000500000000 6.0 nan + 1500000000600000000 nan 7.0 + 1500000000700000000 8.0 nan + 1500000000800000000 nan 9.0 + + + + + >>> # materialize the streams' data + >>> streams.filter(start=1500000000000000000, end=1500000000900000000).values() + [[RawPoint(1500000000100000000, 2.0), RawPoint(1500000000300000000, 4.0), RawPoint(1500000000500000000, 6.0), RawPoint(1500000000700000000, 8.0), @@ -145,3 +188,48 @@ list of features. RawPoint(1500000000200000000, 3.0), RawPoint(1500000000400000000, 5.0), ... + + + + + + +Return data as :code:`arrow` tables +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:code:`StreamSets` are also able to return :code:`arrow` tables for the group of streams they represent. +This is especially convenient and is usually **much** faster than using the traditional :code:`RawPoint` -based data representation. +We recommend using the :code:`arrow` functions whenever possible. + +.. code-block:: python + + >>> # convert data to a pandas DataFrame, using pyarrow + >>> streams.filter(start=1500000000000000000, end=1500000000900000000) + ... .arrow_to_dataframe() + NW/stream0 NW/stream1 + time + 2017-07-14 02:40:00+00:00 NaN 1.0 + 2017-07-14 02:40:00.100000+00:00 2.0 NaN + 2017-07-14 02:40:00.200000+00:00 NaN 3.0 + 2017-07-14 02:40:00.300000+00:00 4.0 NaN + 2017-07-14 02:40:00.400000+00:00 NaN 5.0 + 2017-07-14 02:40:00.500000+00:00 6.0 NaN + 2017-07-14 02:40:00.600000+00:00 NaN 7.0 + 2017-07-14 02:40:00.700000+00:00 8.0 NaN + 2017-07-14 02:40:00.800000+00:00 NaN 9.0 + + + >>> # materialize the streams' data as an arrow table + >>> streams.filter(start=1500000000000000000, end=1500000000900000000).arrow_values() + pyarrow.Table + time: timestamp[ns, tz=UTC] not null + b29204f4-6c13-4ec7-a149-88e2ff950a72: double not null + 99a0d0b0-e24f-4875-b7d8-eae0036f2149: double not null + ---- + time: [ + ... [2017-07-14 02:40:00.000000000Z,2017-07-14 02:40:00.100000000Z, + ... 2017-07-14 02:40:00.200000000Z,2017-07-14 02:40:00.300000000Z, + ... 2017-07-14 02:40:00.400000000Z,2017-07-14 02:40:00.500000000Z, + ... 2017-07-14 02:40:00.600000000Z,2017-07-14 02:40:00.700000000Z, + ... 2017-07-14 02:40:00.800000000Z]] + b29204f4-6c13-4ec7-a149-88e2ff950a72: [[nan,2,nan,4,nan,6,nan,8,nan]] + 99a0d0b0-e24f-4875-b7d8-eae0036f2149: [[1,nan,3,nan,5,nan,7,nan,9]] diff --git a/docs/source/working.rst b/docs/source/working.rst index 8773907..cc33d2c 100644 --- a/docs/source/working.rst +++ b/docs/source/working.rst @@ -15,4 +15,7 @@ to interact with the BTrDB database. working/stream-view-data working/streamsets working/multiprocessing + working/multistream + working/arrow-enabled-queries + working/dash working/ray diff --git a/docs/source/working/arrow-enabled-queries.rst b/docs/source/working/arrow-enabled-queries.rst new file mode 100644 index 0000000..fa531c9 --- /dev/null +++ b/docs/source/working/arrow-enabled-queries.rst @@ -0,0 +1,32 @@ +.. -*- mode: rst -*- + +.. _Arrow Page: + +Arrow-enabled Queries +===================== + +In more recent deployments of the BTrDB platform (>=5.30.0), commercial customers also have access to additional accelerated functionality for data fetching and inserting. + +Also, most :code:`StreamSet` based value queries (:code:`AlignedWindows`, :code:`Windows`, :code:`Values`) are multithreaded by default. +This leads to decent performance improvements for fetching and inserting data using the standard :code:`StreamSet` api without any edits by the user. +Refer to :ref:`The StreamSet API ` + +In addition to these improvements to the standard API, commercial customers also have access to additional accelerated data fetching and inserting methods that can dramatically speed up their workflows. + +Apache Arrow Data Format +^^^^^^^^^^^^^^^^^^^^^^^^ + +While keeping our standard API consistent with our :code:`Point` and :code:`StatPoint` :ref:`python object model `, we have also created additional methods that will provide this same type of data, but in a tabular format by default. +Leveraging the `language-agnostic columnar data format Arrow `_, we can transmit our timeseries data in a format that is already optimized for data analytics with well-defined schemas that take the guesswork out of the data types, timezones, etc. +To learn more about these methods, please refer to the :ref:`arrow_ prefixed methods for both Stream and StreamSet objects ` and the :ref:`StreamSet transformer methods `. + +.. _ArrowMultistreamDocs: +True Multistream Support +^^^^^^^^^^^^^^^^^^^^^^^^ + +Until now, there has not been a true multistream query support, our previous api and with the new edits, emulates multistream support with :code:`StreamSet` s and using multithreading. +However, this will still only scale to an amount of streams based on the amount of threads that the python threadpool logic can support. + +Due to this, raw data queries for :code:`StreamSet` s using our arrow api :code:`StreamSet.filter(start=X, end=Y,).arrow_values()` will now perform true multistream queries. +The platform, instead of the python client, will now quickly grab all the stream data for all streams in your streamset, and then package that back to the python client in an :code:`arrow` table! +This leads to data fetch speedups on the order of 10-50x based on the amount and kind of streams. diff --git a/docs/source/working/dash.rst b/docs/source/working/dash.rst new file mode 100644 index 0000000..b6bf10d --- /dev/null +++ b/docs/source/working/dash.rst @@ -0,0 +1,89 @@ +.. -*- mode: rst -*- + +Working with Dash and Plotly +============================ +From `Plotly's getting start guide: `_ "The plotly Python library is an interactive, open-source plotting library that supports over 40 unique chart types covering a wide range of statistical, financial, geographic, scientific, and 3-dimensional use-cases." + +These tools are usable in jupyter notebooks and can also be ran as their own standalone apps using plotly-dash. + + +Below are two examples using the standard API and the Arrow enabled API to retrieve data as a pandas.DataFrame, and then plotting the results. +These examples are based off of the `minimal dash app `_. + +Non-Multistream API +------------------- + +.. code-block:: python + + from dash import Dash, html, dcc, callback, Output, Input + import plotly.express as px + import pandas as pd + import btrdb + + conn = btrdb.connect() + streams = conn.streams_in_collection("YOUR_COLLECTION_HERE") + streamset = conn.streams(*[s.uuid for s in streams]) + latest = streamset.latest() + end = min([pt.time for pt in latest]) + start = end - btrdb.utils.timez.ns_delta(minutes=5) + + df = streamset.filter(start=start, end=end).to_dataframe() + + app = Dash(__name__) + + app.layout = html.Div([ + html.H1(children='Title of Dash App', style={'textAlign':'center'}), + dcc.Dropdown(df.columns, id='dropdown-selection'), + dcc.Graph(id='graph-content') + ]) + + @callback( + Output('graph-content', 'figure'), + Input('dropdown-selection', 'value') + ) + def update_graph(value): + dff = df[value] + return px.line(dff, x=dff.index, y=value) + + if __name__ == '__main__': + app.run(debug=True) + + + +Multistream API +--------------- +.. code-block:: python + + from dash import Dash, html, dcc, callback, Output, Input + import plotly.express as px + import pandas as pd + import btrdb + + conn = btrdb.connect() + streams = conn.streams_in_collection("YOUR_COLLECTION_HERE") + streamset = conn.streams(*[s.uuid for s in streams]) + latest = streamset.latest() + end = min([pt.time for pt in latest]) + start = end - btrdb.utils.timez.ns_delta(minutes=5) + + df = streamset.filter(start=start, end=end).arrow_to_dataframe() + df = df.set_index('time') + + app = Dash(__name__) + + app.layout = html.Div([ + html.H1(children='Title of Dash App', style={'textAlign':'center'}), + dcc.Dropdown(df.columns, id='dropdown-selection'), + dcc.Graph(id='graph-content') + ]) + + @callback( + Output('graph-content', 'figure'), + Input('dropdown-selection', 'value') + ) + def update_graph(value): + dff = df[value] + return px.line(dff, x=dff.index, y=value) + + if __name__ == '__main__': + app.run(debug=True) diff --git a/docs/source/working/multiprocessing.rst b/docs/source/working/multiprocessing.rst index 0aa6a67..c474154 100644 --- a/docs/source/working/multiprocessing.rst +++ b/docs/source/working/multiprocessing.rst @@ -78,4 +78,4 @@ Let's break this down quickly since this is a very common design pattern. First The ``stream_quality`` function is our parallelizable function (e.g. computing the data quality for multiple streams at a time). Depending on how long the ``data_quality`` function takes to compute, we may also want to parallelize ``(stream, start, end)`` tuples. -If you would like features like a connection pool object (as other databases have) or multiprocessing helpers, please leave us a note in our GitHub issues! \ No newline at end of file +If you would like features like a connection pool object (as other databases have) or multiprocessing helpers, please leave us a note in our GitHub issues! diff --git a/docs/source/working/multistream.rst b/docs/source/working/multistream.rst new file mode 100644 index 0000000..77548ff --- /dev/null +++ b/docs/source/working/multistream.rst @@ -0,0 +1,6 @@ +.. -*- mode: rst -*- + +Multistream Queries +=================== + +Refer to :ref:`ArrowMultistreamDocs`. diff --git a/docs/source/working/ray.rst b/docs/source/working/ray.rst index 98a7fbd..f13fc80 100644 --- a/docs/source/working/ray.rst +++ b/docs/source/working/ray.rst @@ -18,7 +18,7 @@ Setting up the ray serializer ray.init() conn_params = {"profile": "profile_name"} - + # register serializer with the connection parameters register_serializer(**conn_params) @@ -55,4 +55,4 @@ Setting up the ray serializer >>(pid=28482) (RawPoint(1533210100000000000, 0.0), 0) # output of test_streamset >>(pid=28481) (RawPoint(1533210100000000000, 0.0), RawPoint(1533210100000000000, 0.0)) - >>(pid=28481) StreamSet with 2 streams \ No newline at end of file + >>(pid=28481) StreamSet with 2 streams diff --git a/docs/source/working/server.rst b/docs/source/working/server.rst index cd479c0..4eaf1b4 100644 --- a/docs/source/working/server.rst +++ b/docs/source/working/server.rst @@ -14,7 +14,7 @@ an API key will raise an exception. Connecting to servers --------------------------- -The btrdb library comes with a high level :code:`connnect` function to interface +The btrdb library comes with a high level :code:`connect` function to interface with a BTrDB server. Upon successfully connecting, you will be returned a :code:`BTrDB` object which is the starting point for all of your server interactions. diff --git a/docs/source/working/stream-manage-metadata.rst b/docs/source/working/stream-manage-metadata.rst index 9452253..7d90655 100644 --- a/docs/source/working/stream-manage-metadata.rst +++ b/docs/source/working/stream-manage-metadata.rst @@ -102,12 +102,10 @@ that the data has already been changed by another user or process. annotations=annotations ) -If you would like to remove any keys from your annotations you must use the `replace=True` keyword argument. This will ensure that the annotations dictionary you provide completely replaces the existing values rather than perform an UPSERT operation. The example below shows how you could remove an existing key from the annotations dictionary. +If you would like to remove any keys from your annotations you must use the :code:`replace=True` keyword argument. This will ensure that the annotations dictionary you provide completely replaces the existing values rather than perform an UPSERT operation. The example below shows how you could remove an existing key from the annotations dictionary. .. code-block:: python annotations, _ = stream.anotations() del annotations["key_to_delete"] stream.update(annotations=annotations, replace=True) - - diff --git a/docs/source/working/stream-query-manage.rst b/docs/source/working/stream-query-manage.rst index d0f83c6..9476381 100644 --- a/docs/source/working/stream-query-manage.rst +++ b/docs/source/working/stream-query-manage.rst @@ -75,15 +75,15 @@ detail. Querying Metadata ----------------- Finally, you can query for metadata using standard SQL although at the moment, only the -`streams` table is available. SQL queries can be submitted using the `query` -method which accepts both a `stmt` and `params` argument. The `stmt` should -contain the SQL you'd like executed with parameter placeholders such as `$1` or -`$2` as shown below. +:code:`streams` table is available. SQL queries can be submitted using the :code:`query` +method which accepts both a :code:`stmt` and :code:`params` argument. The :code:`stmt` should +contain the SQL you'd like executed with parameter placeholders such as :code:`$1` or +:code:`$2` as shown below. .. code-block:: python conn = btrdb.connect() - stmt = "select uuid from streams where name = $1 or name = $2" + stmt = "SELECT uuid FROM streams WHERE name = $1 OR name = $2" params = ["Boston_1", "Boston_2"] for row in conn.query(stmt, params): diff --git a/docs/source/working/stream-view-data.rst b/docs/source/working/stream-view-data.rst index 915e75e..b9c41c7 100644 --- a/docs/source/working/stream-view-data.rst +++ b/docs/source/working/stream-view-data.rst @@ -45,7 +45,7 @@ you query for all the data using the :code:`Stream.values` method due to the mem consumption implied). Each of these three methods returns a tuple containing a RawPoint and the data version number. The exact timestamp can be obtained from the RawPoint. Keep in mind that all of these methods accept a :code:`version` argument so that -you can ask for the earliest, latest, or nearest point from a previous version of the stream. +you can ask for the :code:`earliest`, :code:`latest`, or :code:`nearest` point from a previous version of the stream. .. code-block:: python @@ -86,7 +86,7 @@ aggregate of all the raw data within a window of width 2^pointwidth nanoseconds. Note that :code:`start` is inclusive, but :code:`end` is exclusive. That is, results -will be returned for all windows that start in the interval [start, end). +will be returned for all windows that start in the interval :math:`[start, end)`. If end < start+2^pointwidth you will not get any results. If start and end are not powers of two, the bottom pointwidth bits will be cleared. Each window will contain statistical summaries of the window. Statistical points @@ -150,37 +150,37 @@ of sample depths are provided below. +-------+-------------+---------------------------+-----------------+ As usual when querying data from BTrDB, the :code:`start` time is inclusive -while the :code:`end` time is exclusive. Note that if your last window spans +while the :code:`end` time is exclusive. **Note**: that if your last window spans across the end time then it will not be included in the results. .. code-block:: python - start = 1500000000000000000 - end = 1500000001000000000 - - # view underlying data for comparison - for point, _ in stream.values(start=start, end=end): - print(point) - >> RawPoint(1500000000000000000, 1.0) - >> RawPoint(1500000000100000000, 2.0) - >> RawPoint(1500000000200000000, 3.0) - >> RawPoint(1500000000300000000, 4.0) - >> RawPoint(1500000000400000000, 5.0) - >> RawPoint(1500000000500000000, 6.0) - >> RawPoint(1500000000600000000, 7.0) - >> RawPoint(1500000000700000000, 8.0) - >> RawPoint(1500000000800000000, 9.0) - >> RawPoint(1500000000900000000, 10.0) - - # each window spans 300 milleseconds - width = 300000000 - - # request a precision of roughly 1 millesecond - depth = 20 - - # view windowed data - for point, _ in stream.windows(start=start, end=end, - width=width, depth=depth): - >> StatPoint(1500000000000000000, 1.0, 2.0, 3.0, 3, 0.816496580927726) - >> StatPoint(1500000000300000000, 4.0, 5.0, 6.0, 3, 0.816496580927726) - >> StatPoint(1500000000600000000, 7.0, 8.0, 9.0, 3, 0.816496580927726) + >>> start = 1500000000000000000 + >>> end = 1500000001000000000 + + >>> # view underlying data for comparison + >>> for point, _ in stream.values(start=start, end=end): + >>> print(point) + RawPoint(1500000000000000000, 1.0) + RawPoint(1500000000100000000, 2.0) + RawPoint(1500000000200000000, 3.0) + RawPoint(1500000000300000000, 4.0) + RawPoint(1500000000400000000, 5.0) + RawPoint(1500000000500000000, 6.0) + RawPoint(1500000000600000000, 7.0) + RawPoint(1500000000700000000, 8.0) + RawPoint(1500000000800000000, 9.0) + RawPoint(1500000000900000000, 10.0) + + >>> # each window spans 300 milliseconds + >>> width = 300000000 + + >>> # request a precision of roughly 1 millisecond + >>> depth = 20 + + >>> # view windowed data + >>> for point, _ in stream.windows(start=start, end=end, + ... width=width, depth=depth): + StatPoint(1500000000000000000, 1.0, 2.0, 3.0, 3, 0.816496580927726) + StatPoint(1500000000300000000, 4.0, 5.0, 6.0, 3, 0.816496580927726) + StatPoint(1500000000600000000, 7.0, 8.0, 9.0, 3, 0.816496580927726) diff --git a/docs/source/working/streamsets.rst b/docs/source/working/streamsets.rst index 8a9d807..da6c1d0 100644 --- a/docs/source/working/streamsets.rst +++ b/docs/source/working/streamsets.rst @@ -38,8 +38,8 @@ Filtering To apply query parameters to your request, you should use the :code:`filter` method to supply a :code:`start` or :code:`end` argument. -Keep in mind that :code:`filter` will return a new object so you can keep -multiple filtered StreamSets in memory while you explore your data. The +Keep in mind that :code:`filter` will **return a new object so you can keep +multiple filtered StreamSets in memory while you explore your data**. The :code:`filter` method may be called multiple times but only the final values will be used when it is time to fulfill the request by the server. @@ -83,8 +83,8 @@ re.search to choose the streams to include. Retrieving Data ---------------- -There are two options available when you are ready to process the data from the -server. Both options are fully materialized but are organized in different ways +There are three options available when you are ready to process the data from the +server. All options are fully materialized but are organized in different ways according to what is more convenient for you. StreamSet.values() @@ -131,6 +131,10 @@ consists of 4 streams. >> RawPoint(1500000000900000000, 10.0)]] +.. attention:: + + + StreamSet.rows() ^^^^^^^^^^^^^^^^^^ By contrast, the :code:`rows` method aligns data by time rather than by stream. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..238dae6 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,80 @@ +[project] +name = "btrdb" +version = "5.34.2" +authors = [ + {name="PingThingsIO", email="support@pingthings.io"}, +] +maintainers = [ + {name="PingThingsIO", email="support@pingthings.io"}, +] +description = "Bindings to interact with the Berkeley Tree Database using gRPC." +readme = "README.md" +license = {file="LICENSE.txt"} +requires-python = ">=3.7,<=3.12" +classifiers = [ + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'Intended Audience :: Information Technology', + 'Intended Audience :: Science/Research', + 'License :: OSI Approved :: BSD License', + 'Natural Language :: English', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Topic :: Database', + 'Topic :: Software Development :: Libraries :: Python Modules', +] +keywords = [ + 'btrdb', 'berkeley', 'timeseries', 'database', 'bindings', 'gRPC', +] +dependencies = [ + "grpcio", + "grpcio-tools", + "pytz", + "pyyaml", + "certifi", +] + +[project.optional-dependencies] +data = [ + "pyarrow", + "pandas>=2.0", + "polars", + "numpy", + "tabulate", +] +ray = [ + "btrdb[data]", + "ray" +] +testing = [ + "btrdb[data]", + "pytest", + "freezegun", + "pytest-cov", + "pytest-flake8" +] +all = [ + "btrdb[data]", + "btrdb[ray]", + "btrdb[testing]", +] + +[project.urls] +"Docs" = "https://btrdb-python.readthedocs.io/" +"Repository" = "https://github.com/pingthingsio/btrdb-python.git" + +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.exclude-package-data] +drop = [ + "tests", + "docs" +] diff --git a/release.sh b/release.sh index 6f6595a..b63f28c 100755 --- a/release.sh +++ b/release.sh @@ -35,13 +35,12 @@ fi echo "Setting version to v$1.$2.$3" -VERION_CODE="__version_info__ = { 'major': $1, 'minor': $2, 'micro': $3, 'releaselevel': 'final'}" -sed -i "s/^__version_info__.*$/${VERION_CODE}/g" btrdb/version.py +VERSION_CODE="__version_info__ = {\"major\": $1, \"minor\": $2, \"micro\": $3, \"releaselevel\": \"final\"}" +sed -i.bak "s/^__version_info__.*$/${VERSION_CODE}/g" btrdb/version.py +sed -i.bak "s/^version.*$/version\ = \"$1.$2.$3\"/g" pyproject.toml git add btrdb/version.py +git add pyproject.toml git commit -m "Release v$1.$2.$3" -git tag v$1.$2.$3 -git push origin v$1.$2.$3 -sleep 10 -git push \ No newline at end of file +echo "Now make a PR and merge it into main, then make tag :)" diff --git a/requirements-all.txt b/requirements-all.txt new file mode 100644 index 0000000..4a71eb1 --- /dev/null +++ b/requirements-all.txt @@ -0,0 +1,57 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --extra=all --output-file=requirements-all.txt --resolver=backtracking pyproject.toml +# +# Pinned versions and dependencies of dependencies were removed after the autogeneration. +certifi + # via + # btrdb (pyproject.toml) +freezegun + # via btrdb +grpcio + # via + # btrdb (pyproject.toml) + # grpcio-tools + # ray +grpcio-tools + # via + # btrdb (pyproject.toml) +numpy + # via + # btrdb + # pandas + # pyarrow + # ray +pandas + # via btrdb +polars + # via btrdb +protobuf + # via + # grpcio-tools + # ray +pyarrow + # via btrdb +pytest + # via + # btrdb + # pytest-cov + # pytest-flake8 +pytest-cov + # via btrdb +pytest-flake8 + # via btrdb +pytz + # via + # btrdb (pyproject.toml) + # pandas +pyyaml + # via + # btrdb (pyproject.toml) + # ray +ray + # via btrdb +tabulate + # via btrdb diff --git a/requirements.txt b/requirements.txt index b12c1a6..a921486 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,24 @@ -# GRPC / Protobuff related -grpcio>=1.19.0 -grpcio-tools>=1.19.0 - -# Time related utils +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --output-file=requirements.txt --resolver=backtracking pyproject.toml +# +# This file was modified to remove version pins. +certifi + # via btrdb (pyproject.toml) +grpcio + # via + # btrdb (pyproject.toml) + # grpcio-tools +grpcio-tools + # via btrdb (pyproject.toml) +protobuf + # via grpcio-tools pytz + # via btrdb (pyproject.toml) +pyyaml + # via btrdb (pyproject.toml) -# Misc libraries -pyyaml \ No newline at end of file +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/setup.cfg b/setup.cfg index cded2bd..8d1166f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,43 +1,12 @@ [metadata] -description-file = DESCRIPTION.md -license_file = LICENSE.txt +description_file = DESCRIPTION.md +license_files = LICENSE.txt [aliases] test=pytest [tool:pytest] -addopts = --cov=btrdb --flake8 python_files = tests/* -flake8-ignore = - E111 - E114 - E121 - E122 - E123 - E124 - E128 - E201 - E202 - E221 - E222 - E225 - E226 - E231 - E241 - E251 - E266 - E271 - E302 - E303 - E402 - E501 - E502 - E701 - E731 - F401 - W292 - W293 - W391 - tests/* ALL - btrdb/grpcinterface/btrdb_pb2.py UnusedImport - btrdb/__init__.py UnusedImport + +[flake8] +extend-ignore = E111, E114, E121, E122, E123, E124, E128, E201, E202, E203, E221, E222, E225, E226, E231, E241, E251, E266, E271, E302, E303, E402, E501, E502,E701,E731,F401,W292,W293,W391 diff --git a/setup.py b/setup.py index 0e515f2..df6ad76 100644 --- a/setup.py +++ b/setup.py @@ -16,69 +16,70 @@ ## Imports ########################################################################## -import os import codecs +import os -from setuptools import setup -from setuptools import find_packages +from setuptools import find_packages, setup ########################################################################## ## Package Information ########################################################################## ## Basic information -NAME = "btrdb" -DESCRIPTION = "Bindings to interact with the Berkeley Tree Database using gRPC." -AUTHOR = "Michael Andersen, Allen Leis" -EMAIL = "michael@steelcode.com" -MAINTAINER = "Michael Andersen" -LICENSE = "BSD-3-Clause" -REPOSITORY = "https://github.com/BTrDB/btrdb-python" -PACKAGE = "btrdb" -URL = "http://btrdb.io/" -DOCS_URL = "https://btrdb.readthedocs.io/en/latest/" +NAME = "btrdb" +DESCRIPTION = "Bindings to interact with the Berkeley Tree Database using gRPC." +AUTHOR = "PingThingsIO" +EMAIL = "support@pingthings.io" +MAINTAINER = "PingThingsIO" +LICENSE = "BSD-3-Clause" +REPOSITORY = "https://github.com/PingThingsIO/btrdb-python" +PACKAGE = "btrdb" +URL = "http://btrdb.io/" +DOCS_URL = "https://btrdb-python.readthedocs.io/" ## Define the keywords -KEYWORDS = ('btrdb', 'berkeley', 'timeseries', 'database', 'bindings' 'gRPC') +KEYWORDS = ("btrdb", "berkeley", "timeseries", "database", "bindings" "gRPC") ## Define the classifiers ## See https://pypi.python.org/pypi?%3Aaction=list_classifiers -CLASSIFIERS = ( - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'Intended Audience :: Information Technology', - 'Intended Audience :: Science/Research', - 'License :: OSI Approved :: BSD License', - 'Natural Language :: English', - 'Operating System :: OS Independent', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Topic :: Database', - 'Topic :: Software Development :: Libraries :: Python Modules', +CLASSIFIERS = ( + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: BSD License", + "Natural Language :: English", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Topic :: Database", + "Topic :: Software Development :: Libraries :: Python Modules", ) ## Important Paths -PROJECT = os.path.abspath(os.path.dirname(__file__)) +PROJECT = os.path.abspath(os.path.dirname(__file__)) REQUIRE_PATH = "requirements.txt" VERSION_PATH = os.path.join(PACKAGE, "version.py") PKG_DESCRIBE = "DESCRIPTION.md" ## Directories to ignore in find_packages -EXCLUDES = ( - "tests", "docs", +EXCLUDES = ( + "tests", + "docs", ) ########################################################################## ## Helper Functions ########################################################################## + def read(*parts): """ Assume UTF-8 encoding and return the contents of the file located at the absolute path from the REPOSITORY joined with *parts. """ - with codecs.open(os.path.join(PROJECT, *parts), 'rb', 'utf-8') as f: + with codecs.open(os.path.join(PROJECT, *parts), "rb", "utf-8") as f: return f.read() @@ -90,7 +91,7 @@ def get_version(path=VERSION_PATH): """ namespace = {} exec(read(path), namespace) - return namespace['get_version'](short=True) + return namespace["get_version"](short=True) def get_requires(path=REQUIRE_PATH): @@ -100,7 +101,7 @@ def get_requires(path=REQUIRE_PATH): """ for line in read(path).splitlines(): line = line.strip() - if line and not line.startswith('#'): + if line and not line.startswith("#"): yield line @@ -132,7 +133,6 @@ def get_description_type(path=PKG_DESCRIBE): "license": LICENSE, "author": AUTHOR, "author_email": EMAIL, - "url": URL, "maintainer": MAINTAINER, "maintainer_email": EMAIL, "project_urls": { @@ -152,8 +152,8 @@ def get_description_type(path=PKG_DESCRIBE): }, "install_requires": list(get_requires()), "python_requires": ">=3.6, <4", - "setup_requires":["pytest-runner"], - "tests_require":["pytest"], + "setup_requires": ["pytest-runner"], + "tests_require": ["pytest"], } @@ -161,5 +161,5 @@ def get_description_type(path=PKG_DESCRIBE): ## Run setup script ########################################################################## -if __name__ == '__main__': +if __name__ == "__main__": setup(**config) diff --git a/tests/btrdb/test_arrow_streams.py b/tests/btrdb/test_arrow_streams.py new file mode 100644 index 0000000..301fa78 --- /dev/null +++ b/tests/btrdb/test_arrow_streams.py @@ -0,0 +1,47 @@ +import concurrent +import uuid +from unittest.mock import Mock, PropertyMock, patch + +import pytest + +import btrdb +from btrdb.point import RawPoint +from btrdb.stream import Stream + + +@pytest.fixture +def stream1(): + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + stream = Mock(Stream) + stream.version = Mock(return_value=11) + stream.uuid = Mock(return_value=uu) + stream.nearest = Mock(return_value=(RawPoint(time=10, value=1), 11)) + type(stream).collection = PropertyMock(return_value="fruits/apple") + type(stream).name = PropertyMock(return_value="gala") + stream.tags = Mock(return_value={"name": "gala", "unit": "volts"}) + stream.annotations = Mock(return_value=({"owner": "ABC", "color": "red"}, 11)) + stream._btrdb = Mock() + stream._btrdb._executor = concurrent.futures.ThreadPoolExecutor() + stream._btrdb._ARROW_ENABLED = Mock(return_value=True) + return stream + + +@pytest.fixture +def stream2(): + uu = uuid.UUID("17dbe387-89ea-42b6-864b-f505cdb483f5") + stream = Mock(Stream) + stream.version = Mock(return_value=22) + stream.uuid = Mock(return_value=uu) + stream.nearest = Mock(return_value=(RawPoint(time=20, value=1), 22)) + type(stream).collection = PropertyMock(return_value="fruits/orange") + type(stream).name = PropertyMock(return_value="blood") + stream.tags = Mock(return_value={"name": "blood", "unit": "amps"}) + stream.annotations = Mock(return_value=({"owner": "ABC", "color": "orange"}, 22)) + stream._btrdb = Mock() + stream._btrdb._executor = Mock() + stream._btrdb._ARROW_ENABLED = Mock(return_value=True) + return stream + + +class TestArrowStreams(object): + pass diff --git a/tests/btrdb/test_base.py b/tests/btrdb/test_base.py index c44acf8..0fddc90 100644 --- a/tests/btrdb/test_base.py +++ b/tests/btrdb/test_base.py @@ -16,27 +16,27 @@ ########################################################################## import os +from unittest.mock import MagicMock, Mock, patch + import pytest -from unittest.mock import patch -from btrdb import connect, __version__, BTRDB_ENDPOINTS, BTRDB_API_KEY +from btrdb import BTRDB_API_KEY, BTRDB_ENDPOINTS, Endpoint, __version__, connect from btrdb.exceptions import ConnectionError - ########################################################################## ## Initialization Tests ########################################################################## -class TestConnect(object): - def setup(self): +class TestConnect(object): + def setup_method(self): for env in ["BTRDB_ENDPOINTS", "BTRDB_PROFILE", "BTRDB_API_KEY"]: try: del os.environ[env] except KeyError: pass - @patch('btrdb.credentials') + @patch("btrdb.credentials") def test_raises_err_with_no_inputs(self, mock_credentials): """ Assert ConnectionError raised if no connection arg, ENV, profile @@ -51,53 +51,64 @@ def test_raises_err_if_both_profile_and_credentials(self): Assert error is raised if both profile and credentials are sent """ with pytest.raises(ValueError): - connect("192.168.1.100:4410",None,"default") + connect("192.168.1.100:4410", None, "default") - @patch('btrdb.utils.credentials.credentials_by_profile') - @patch('btrdb.utils.credentials.credentials_by_env') - @patch('btrdb._connect') - def test_uses_args_over_env(self, mock_connect, mock_credentials_by_env, mock_credentials_by_profile): + @patch("btrdb.utils.credentials.credentials_by_profile") + @patch("btrdb.utils.credentials.credentials_by_env") + @patch("btrdb._connect") + def test_uses_args_over_env( + self, mock_connect, mock_credentials_by_env, mock_credentials_by_profile + ): """ Assert uses connect args over env """ mock_credentials_by_profile.return_value = {} - mock_credentials_by_env.return_value = { - "endpoints": "a", "apikey": "b" - } + mock_credentials_by_env.return_value = {"endpoints": "a", "apikey": "b"} connect("cat", "dog") mock_connect.assert_called_once_with(endpoints="cat", apikey="dog") - @patch('btrdb.utils.credentials.credentials_by_profile') - @patch('btrdb.utils.credentials.credentials_by_env') - @patch('btrdb._connect') - def test_uses_env_over_profile(self, mock_connect, mock_credentials_by_env, mock_credentials_by_profile): + @patch("btrdb.utils.credentials.credentials_by_profile") + @patch("btrdb.utils.credentials.credentials_by_env") + @patch("btrdb._connect") + def test_uses_env_over_profile( + self, mock_connect, mock_credentials_by_env, mock_credentials_by_profile + ): """ Assert connect uses env over profile info """ - mock_credentials_by_profile.return_value = { - "endpoints": "a", "apikey": "b" - } - mock_credentials_by_env.return_value = { - "endpoints": "c", "apikey": "d" - } + mock_credentials_by_profile.return_value = {"endpoints": "a", "apikey": "b"} + mock_credentials_by_env.return_value = {"endpoints": "c", "apikey": "d"} connect() mock_connect.assert_called_once_with(endpoints="c", apikey="d") - @patch('btrdb.utils.credentials.credentials_by_profile') - @patch('btrdb.Connection') - def test_connect_with_env(self, mock_conn, mock_credentials_by_profile): + # @patch("btrdb.utils.credentials.credentials_by_env") + @patch("btrdb.utils.credentials.credentials_by_env") + @patch("btrdb.utils.credentials.credentials_by_profile") + @patch("btrdb.Connection") + def test_connect_with_env( + self, mock_conn, mock_credentials_by_profile, mock_credentials_by_env + ): """ Assert connect uses ENV variables """ - mock_credentials_by_profile.return_value = {} - address = "127.0.0.1:4410" - os.environ[BTRDB_ENDPOINTS] = address + with patch( + "btrdb.endpoint.Endpoint.info", return_value=Mock(Endpoint) + ) as mock_ep_info: + mock_ep_info.return_value = MagicMock() + mock_credentials_by_profile.return_value = {} + address = "127.0.0.1:4410" + mock_credentials_by_env.return_value = {"endpoints": address} + os.environ[BTRDB_ENDPOINTS] = address - btrdb = connect() - mock_conn.assert_called_once_with(address, apikey=None) - mock_conn.reset_mock() + btrdb = connect() + mock_conn.assert_called_once_with(address, apikey=None) + mock_conn.reset_mock() - apikey = "abcd" - os.environ[BTRDB_API_KEY] = apikey - btrdb = connect() - mock_conn.assert_called_once_with(address, apikey=apikey) + apikey = "abcd" + os.environ[BTRDB_API_KEY] = apikey + mock_credentials_by_env.return_value = { + "endpoints": address, + "apikey": apikey, + } + btrdb = connect() + mock_conn.assert_called_once_with(address, apikey=apikey) diff --git a/tests/btrdb/test_conn.py b/tests/btrdb/test_conn.py index f863187..3720ef2 100644 --- a/tests/btrdb/test_conn.py +++ b/tests/btrdb/test_conn.py @@ -16,20 +16,69 @@ ########################################################################## import uuid as uuidlib +from unittest.mock import MagicMock, Mock, PropertyMock, call, patch + import pytest -from unittest.mock import Mock, PropertyMock, patch, call -from btrdb.conn import Connection, BTrDB +from btrdb.conn import BTrDB, Connection from btrdb.endpoint import Endpoint -from btrdb.grpcinterface import btrdb_pb2 from btrdb.exceptions import * +from btrdb.grpcinterface import btrdb_pb2 +from btrdb.stream import Stream + +########################################################################## +## Fixtures +########################################################################## + + +@pytest.fixture +def stream1(): + uu = uuidlib.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + stream = Mock(Stream) + stream.version = Mock(return_value=11) + stream.uuid = Mock(return_value=uu) + type(stream).collection = PropertyMock(return_value="fruits/apple") + type(stream).name = PropertyMock(return_value="gala") + stream.tags = Mock(return_value={"name": "gala", "unit": "volts"}) + stream.annotations = Mock(return_value=({"owner": "ABC", "color": "red"}, 11)) + stream._btrdb = MagicMock() + return stream + + +@pytest.fixture +def stream2(): + uu = uuidlib.UUID("17dbe387-89ea-42b6-864b-f505cdb483f5") + stream = Mock(Stream) + stream.version = Mock(return_value=22) + stream.uuid = Mock(return_value=uu) + type(stream).collection = PropertyMock(return_value="fruits/orange") + type(stream).name = PropertyMock(return_value="blood") + stream.tags = Mock(return_value={"name": "blood", "unit": "amps"}) + stream.annotations = Mock(return_value=({"owner": "ABC", "color": "orange"}, 22)) + stream._btrdb = MagicMock() + return stream + + +@pytest.fixture +def stream3(): + uu = uuidlib.UUID("17dbe387-89ea-42b6-864b-e2ef0d22a53b") + stream = Mock(Stream) + stream.version = Mock(return_value=33) + stream.uuid = Mock(return_value=uu) + type(stream).collection = PropertyMock(return_value="fruits/banana") + type(stream).name = PropertyMock(return_value="yellow") + stream.tags = Mock(return_value={"name": "yellow", "unit": "watts"}) + stream.annotations = Mock(return_value=({"owner": "ABC", "color": "yellow"}, 33)) + stream._btrdb = MagicMock() + return stream + ########################################################################## ## Connection Tests ########################################################################## -class TestConnection(object): +class TestConnection(object): def test_raises_err_invalid_address(self): """ Assert ValueError is raised if address:port is invalidly formatted @@ -40,137 +89,129 @@ def test_raises_err_invalid_address(self): assert "expecting address:port" in str(exc) - def test_raises_err_for_apikey_insecure_port(self): - """ - Assert error is raised if apikey used on insecure port - """ - address = "127.0.0.1:4410" - with pytest.raises(ValueError) as exc: - conn = Connection(address, apikey="abcd") - assert "cannot use an API key with an insecure" in str(exc) - - ########################################################################## ## BTrDB Tests ########################################################################## -class TestBTrDB(object): +class TestBTrDB(object): ########################################################################## ## .streams tests ########################################################################## + # TODO: remove this when removing future warnings from `streams_in_collection` + pytestmark = pytest.mark.filterwarnings("ignore::FutureWarning") def test_streams_raises_err_if_version_not_list(self): """ Assert streams raises TypeError if versions is not list """ - db = BTrDB(None) - with pytest.raises(TypeError) as exc: - db.streams('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a', versions="2,2") - - assert "versions argument must be of type list" in str(exc) + with patch("btrdb.endpoint.Endpoint", return_value=MagicMock()) as ep: + db = BTrDB(ep) + with pytest.raises(TypeError) as exc: + db.streams("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a", versions="2,2") + assert "versions argument must be of type list" in str(exc) def test_streams_raises_err_if_version_argument_mismatch(self): """ Assert streams raises ValueError if len(identifiers) doesnt match length of versions """ - db = BTrDB(None) - with pytest.raises(ValueError) as exc: - db.streams('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a', versions=[2,2]) - - assert "versions does not match identifiers" in str(exc) + with patch("btrdb.endpoint.Endpoint", return_value=MagicMock()) as ep: + db = BTrDB(ep) + with pytest.raises(ValueError) as exc: + db.streams("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a", versions=[2, 2]) + assert "versions does not match identifiers" in str(exc) def test_streams_stores_versions(self): """ Assert streams correctly stores supplied version info """ - db = BTrDB(None) - uuid1 = uuidlib.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') - uuid2 = uuidlib.UUID('17dbe387-89ea-42b6-864b-f505cdb483f5') - versions = [22,44] - expected = dict(zip([uuid1, uuid2], versions)) + with patch("btrdb.endpoint.Endpoint", return_value=MagicMock()) as ep: + db = BTrDB(ep) + uuid1 = uuidlib.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + uuid2 = uuidlib.UUID("17dbe387-89ea-42b6-864b-f505cdb483f5") + versions = [22, 44] + expected = dict(zip([uuid1, uuid2], versions)) - streams = db.streams(uuid1, uuid2, versions=versions) - assert streams._pinned_versions == expected + streams = db.streams(uuid1, uuid2, versions=versions) + assert streams._pinned_versions == expected - - @patch('btrdb.conn.BTrDB.stream_from_uuid') + @patch("btrdb.conn.BTrDB.stream_from_uuid") def test_streams_recognizes_uuid(self, mock_func): """ Assert streams recognizes uuid strings """ - db = BTrDB(None) - uuid1 = uuidlib.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') - mock_func.return_value = [1] - db.streams(uuid1) - - mock_func.assert_called_once() - assert mock_func.call_args[0][0] == uuid1 + with patch("btrdb.endpoint.Endpoint", return_value=MagicMock()) as ep: + db = BTrDB(ep) + uuid1 = uuidlib.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + mock_func.return_value = Stream(db, uuid1) + db.streams(uuid1) + mock_func.assert_called_once() + assert mock_func.call_args[0][0] == uuid1 - @patch('btrdb.conn.BTrDB.stream_from_uuid') + @patch("btrdb.conn.BTrDB.stream_from_uuid") def test_streams_recognizes_uuid_string(self, mock_func): """ Assert streams recognizes uuid strings """ - db = BTrDB(None) - uuid1 = '0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a' - mock_func.return_value = [1] - db.streams(uuid1) + with patch("btrdb.endpoint.Endpoint", return_value=MagicMock()) as ep: + db = BTrDB(ep) + uuid1 = "0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a" + mock_func.return_value = Stream(db, uuid1) + db.streams(uuid1) - mock_func.assert_called_once() - assert mock_func.call_args[0][0] == uuid1 + mock_func.assert_called_once() + assert mock_func.call_args[0][0] == uuid1 - - @patch('btrdb.conn.BTrDB.streams_in_collection') + @patch("btrdb.conn.BTrDB.streams_in_collection") def test_streams_handles_path(self, mock_func): """ Assert streams calls streams_in_collection for collection/name paths """ - db = BTrDB(None) - ident = "zoo/animal/dog" - mock_func.return_value = [1] - db.streams(ident, '0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') - - mock_func.assert_called_once() - assert mock_func.call_args[0][0] == 'zoo/animal' - assert mock_func.call_args[1] == { - 'is_collection_prefix': False, - 'tags': {'name': 'dog'} - } - - - @patch('btrdb.conn.BTrDB.streams_in_collection') + with patch("btrdb.endpoint.Endpoint", return_value=MagicMock()) as ep: + db = BTrDB(ep) + ident = "zoo/animal/dog" + mock_func.return_value = [ + Stream(db, "0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a"), + ] + db.streams(ident, "0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + + mock_func.assert_called_once() + assert mock_func.call_args[0][0] == "zoo/animal" + assert mock_func.call_args[1] == { + "is_collection_prefix": False, + "tags": {"name": "dog"}, + } + + @patch("btrdb.conn.BTrDB.streams_in_collection") def test_streams_raises_err(self, mock_func): """ Assert streams raises StreamNotFoundError """ - db = BTrDB(None) - ident = "zoo/animal/dog" - - mock_func.return_value = [] - with pytest.raises(StreamNotFoundError) as exc: - db.streams(ident) - - mock_func.return_value = [1,2] - with pytest.raises(StreamNotFoundError) as exc: + with patch("btrdb.endpoint.Endpoint", return_value=MagicMock()) as ep: + db = BTrDB(ep) + ident = "zoo/animal/dog" + + mock_func.return_value = [] + with pytest.raises(StreamNotFoundError) as exc: + db.streams(ident) + + # check that does not raise if one returned + mock_func.return_value = [ + Stream(db, ident), + ] db.streams(ident) - # check that does not raise if one returned - mock_func.return_value = [1] - db.streams(ident) - - def test_streams_raises_valueerror(self): """ Assert streams raises ValueError if not uuid, uuid str, or path """ - db = BTrDB(None) - with pytest.raises(ValueError) as exc: - db.streams(11) - + with patch("btrdb.endpoint.Endpoint", return_value=MagicMock()) as ep: + db = BTrDB(ep) + with pytest.raises(ValueError) as exc: + db.streams(11) ########################################################################## ## other tests @@ -180,7 +221,7 @@ def test_info(self): """ Assert info method returns a dict """ - serialized = b'\x18\x05*\x055.0.02\x10\n\x0elocalhost:4410' + serialized = b"\x18\x05*\x055.0.02\x10\n\x0elocalhost:4410" info = btrdb_pb2.InfoResponse.FromString(serialized) endpoint = Mock(Endpoint) @@ -189,8 +230,11 @@ def test_info(self): truth = { "majorVersion": 5, + "minorVersion": 0, "build": "5.0.0", - "proxy": { "proxyEndpoints": ["localhost:4410"], }, + "proxy": { + "proxyEndpoints": ["localhost:4410"], + }, } info = conn.info() assert info == truth @@ -203,16 +247,15 @@ def test_list_collections(self): Assert list_collections method works """ endpoint = Mock(Endpoint) - endpoint.listCollections = Mock(side_effect=[iter([ - ['allen/automated'], - ['allen/bindings'] - ])]) + endpoint.listCollections = Mock( + side_effect=[iter([["allen/automated"], ["allen/bindings"]])] + ) conn = BTrDB(endpoint) - truth = ['allen/automated', 'allen/bindings'] + truth = ["allen/automated", "allen/bindings"] assert conn.list_collections() == truth - @patch('btrdb.conn.unpack_stream_descriptor') + @patch("btrdb.conn.unpack_stream_descriptor") def test_streams_in_collections_args(self, mock_util): """ Assert streams_in_collections correctly sends *collection, tags, annotations @@ -230,13 +273,17 @@ def test_streams_in_collections_args(self, mock_util): conn = BTrDB(endpoint) tags = {"unit": "volts"} annotations = {"size": "large"} - streams = conn.streams_in_collection("a", "b", is_collection_prefix=False, - tags=tags, annotations=annotations) + streams = conn.streams_in_collection( + "a", "b", is_collection_prefix=False, tags=tags, annotations=annotations + ) assert streams[0].name == "gala" assert streams[1].name == "fuji" - expected = [call('a', False, tags, annotations), call('b', False, tags, annotations)] + expected = [ + call("a", False, tags, annotations), + call("b", False, tags, annotations), + ] assert endpoint.lookupStreams.call_args_list == expected def test_streams_in_collections_no_arg(self): diff --git a/tests/btrdb/test_point.py b/tests/btrdb/test_point.py index 5085b00..b596523 100644 --- a/tests/btrdb/test_point.py +++ b/tests/btrdb/test_point.py @@ -17,23 +17,22 @@ import pytest -from btrdb.point import RawPoint, StatPoint from btrdb.grpcinterface import btrdb_pb2 - +from btrdb.point import RawPoint, StatPoint ########################################################################## ## ProtoBuff Classes ########################################################################## -RawPointProto = btrdb_pb2.RawPoint -StatPointProto = btrdb_pb2.StatPoint +RawPointProto = btrdb_pb2.RawPoint +StatPointProto = btrdb_pb2.StatPoint ########################################################################## ## RawPoint Tests ########################################################################## -class TestRawPoint(object): +class TestRawPoint(object): def test_create(self): """ Ensure we can create the object @@ -150,19 +149,19 @@ def test_repr_str(self): ## StatPoint Tests ########################################################################## -class TestStatPoint(object): +class TestStatPoint(object): def test_create(self): """ Ensure we can create the object """ - StatPoint(1,1,1,1,1,1) + StatPoint(1, 1, 1, 1, 1, 1) def test_from_proto(self): """ Assert from_proto creates a correct object """ - proto = StatPointProto(time=1,min=2,mean=3,max=4,count=5,stddev=6) + proto = StatPointProto(time=1, min=2, mean=3, max=4, count=5, stddev=6) point = StatPoint.from_proto(proto) assert point.time == proto.time assert point.min == proto.min @@ -175,7 +174,7 @@ def test_from_proto_creates_rawpoint(self): """ Assert from_proto creates a new instance of StatPoint """ - proto = StatPointProto(time=1,min=2,mean=3,max=4,count=5,stddev=6) + proto = StatPointProto(time=1, min=2, mean=3, max=4, count=5, stddev=6) point = StatPoint.from_proto(proto) assert isinstance(point, StatPoint) @@ -183,8 +182,10 @@ def test_from_proto_list(self): """ Assert from_proto_list creates valid objects """ - protos = [StatPointProto(time=1,min=2,mean=3,max=4,count=5,stddev=6), - StatPointProto(time=11,min=12,mean=13,max=14,count=15,stddev=16)] + protos = [ + StatPointProto(time=1, min=2, mean=3, max=4, count=5, stddev=6), + StatPointProto(time=11, min=12, mean=13, max=14, count=15, stddev=16), + ] points = StatPoint.from_proto_list(protos) assert points[0].time == protos[0].time assert points[0].min == protos[0].min @@ -203,8 +204,10 @@ def test_from_proto_list_returns_list_of_statpoint(self): """ Assert from_proto_list returns list of StatPoint """ - protos = [StatPointProto(time=1,min=2,mean=3,max=4,count=5,stddev=6), - StatPointProto(time=11,min=12,mean=13,max=14,count=15,stddev=16)] + protos = [ + StatPointProto(time=1, min=2, mean=3, max=4, count=5, stddev=6), + StatPointProto(time=11, min=12, mean=13, max=14, count=15, stddev=16), + ] points = StatPoint.from_proto_list(protos) assert len(points) == 2 diff --git a/tests/btrdb/test_stream.py b/tests/btrdb/test_stream.py index 3ba9461..5887afa 100644 --- a/tests/btrdb/test_stream.py +++ b/tests/btrdb/test_stream.py @@ -10,46 +10,49 @@ """ Testing package for the btrdb stream module """ - -########################################################################## -## Imports -########################################################################## - +import concurrent.futures +import datetime +import json import re import sys -import json import uuid -import pytz +from unittest.mock import Mock, PropertyMock, call, patch + import pytest -import datetime -from unittest.mock import Mock, PropertyMock, patch, call +import pytz +from btrdb import MAXIMUM_TIME, MINIMUM_TIME from btrdb.conn import BTrDB from btrdb.endpoint import Endpoint -from btrdb import MINIMUM_TIME, MAXIMUM_TIME -from btrdb.stream import Stream, StreamSet, StreamFilter, INSERT_BATCH_SIZE -from btrdb.point import RawPoint, StatPoint from btrdb.exceptions import ( BTrDBError, + InvalidCollection, InvalidOperation, + NoSuchPoint, StreamNotFoundError, - InvalidCollection, - NoSuchPoint ) from btrdb.grpcinterface import btrdb_pb2 +from btrdb.point import RawPoint, StatPoint +from btrdb.stream import INSERT_BATCH_SIZE, Stream, StreamFilter, StreamSet + +########################################################################## +## Imports +########################################################################## + -RawPointProto = btrdb_pb2.RawPoint -StatPointProto = btrdb_pb2.StatPoint -EST = pytz.timezone('America/New_York') +RawPointProto = btrdb_pb2.RawPoint +StatPointProto = btrdb_pb2.StatPoint +EST = pytz.timezone("America/New_York") ########################################################################## ## Fixtures ########################################################################## + @pytest.fixture def stream1(): - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") stream = Mock(Stream) stream.version = Mock(return_value=11) stream.uuid = Mock(return_value=uu) @@ -58,12 +61,15 @@ def stream1(): type(stream).name = PropertyMock(return_value="gala") stream.tags = Mock(return_value={"name": "gala", "unit": "volts"}) stream.annotations = Mock(return_value=({"owner": "ABC", "color": "red"}, 11)) + stream._btrdb = Mock() + stream._btrdb._executor = concurrent.futures.ThreadPoolExecutor() + stream._btrdb._ARROW_ENABLED = Mock(return_value=False) return stream @pytest.fixture def stream2(): - uu = uuid.UUID('17dbe387-89ea-42b6-864b-f505cdb483f5') + uu = uuid.UUID("17dbe387-89ea-42b6-864b-f505cdb483f5") stream = Mock(Stream) stream.version = Mock(return_value=22) stream.uuid = Mock(return_value=uu) @@ -72,6 +78,43 @@ def stream2(): type(stream).name = PropertyMock(return_value="blood") stream.tags = Mock(return_value={"name": "blood", "unit": "amps"}) stream.annotations = Mock(return_value=({"owner": "ABC", "color": "orange"}, 22)) + stream._btrdb = Mock() + stream._btrdb._executor = Mock() + stream._btrdb._ARROW_ENABLED = Mock(return_value=False) + return stream + + +@pytest.fixture +def stream3(): + uu = uuid.UUID("17dbe387-89ea-42b6-864b-e2ef0d22a53b") + stream = Mock(Stream) + stream.version = Mock(return_value=33) + stream.uuid = Mock(return_value=uu) + stream.nearest = Mock(return_value=(RawPoint(time=30, value=1), 33)) + type(stream).collection = PropertyMock(return_value="fruits/banana") + type(stream).name = PropertyMock(return_value="yellow") + stream.tags = Mock(return_value={"name": "yellow", "unit": "watts"}) + stream.annotations = Mock(return_value=({"owner": "ABC", "color": "yellow"}, 3)) + stream._btrdb = Mock() + stream._btrdb._executor = Mock() + stream._btrdb._ARROW_ENABLED = Mock(return_value=False) + return stream + + +@pytest.fixture +def arrow_stream3(): + uu = uuid.UUID("17dbe387-89ea-42b6-864b-f505cdb483f5") + stream = Mock(Stream) + stream.version = Mock(return_value=33) + stream.uuid = Mock(return_value=uu) + stream.nearest = Mock(return_value=(RawPoint(time=20, value=1), 22)) + type(stream).collection = PropertyMock(return_value="fruits/orange") + type(stream).name = PropertyMock(return_value="blood") + stream.tags = Mock(return_value={"name": "blood", "unit": "amps"}) + stream.annotations = Mock(return_value=({"owner": "ABC", "color": "orange"}, 22)) + stream._btrdb = Mock() + stream._btrdb._executor = Mock() + stream._btrdb._ARROW_ENABLED = Mock(return_value=True) return stream @@ -79,14 +122,13 @@ def stream2(): ## Stream Tests ########################################################################## -class TestStream(object): +class TestStream(object): def test_create(self): """ Assert we can create the object """ - Stream(None, "FAKE") - + Stream(None, "FAKE_UUID") def test_repr_str(self): """ @@ -98,64 +140,61 @@ def test_repr_str(self): stream._collection = COLLECTION stream._tags = {"name": NAME} - expected = "".format( - COLLECTION, NAME - ) + expected = "".format(COLLECTION, NAME) assert stream.__repr__() == expected assert stream.__str__() == expected - def test_refresh_metadata(self): """ Assert refresh_metadata calls Endpoint.streamInfo """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) endpoint.streamInfo = Mock(return_value=("koala", 42, {}, {}, None)) + endpoint.info = Mock(return_value={"majorVersion": 5, "minorVersion": 30}) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) stream.refresh_metadata() stream._btrdb.ep.streamInfo.assert_called_once_with(uu, False, True) - def test_refresh_metadata_deserializes_annotations(self): """ Assert refresh_metadata deserializes annotation values """ uu = uuid.uuid4() serialized = { - 'acronym': 'VPHM', - 'description': 'El Segundo PMU 42 Ean', - 'devacronym': 'PMU!EL_SEG_PMU_42', - 'enabled': 'true', - 'id': '76932ae4-09bc-472c-8dc6-64fea68d2797', - 'phase': 'A', - 'label': 'null', - 'frequency': '30', - 'control': '2019-11-07 13:21:23.000000-0500', + "acronym": "VPHM", + "description": "El Segundo PMU 42 Ean", + "devacronym": "PMU!EL_SEG_PMU_42", + "enabled": "true", + "id": "76932ae4-09bc-472c-8dc6-64fea68d2797", + "phase": "A", + "label": "null", + "frequency": "30", + "control": "2019-11-07 13:21:23.000000-0500", "calibrate": '{"racf": 1.8, "pacf": 0.005}', } expected = { - 'acronym': 'VPHM', - 'description': 'El Segundo PMU 42 Ean', - 'devacronym': 'PMU!EL_SEG_PMU_42', - 'enabled': True, - 'id': '76932ae4-09bc-472c-8dc6-64fea68d2797', - 'phase': 'A', - 'label': None, - 'frequency': 30, - 'control': '2019-11-07 13:21:23.000000-0500', + "acronym": "VPHM", + "description": "El Segundo PMU 42 Ean", + "devacronym": "PMU!EL_SEG_PMU_42", + "enabled": True, + "id": "76932ae4-09bc-472c-8dc6-64fea68d2797", + "phase": "A", + "label": None, + "frequency": 30, + "control": "2019-11-07 13:21:23.000000-0500", "calibrate": {"racf": 1.8, "pacf": 0.005}, } endpoint = Mock(Endpoint) endpoint.streamInfo = Mock(return_value=("koala", 42, {}, serialized, None)) + endpoint.info = Mock(return_value={"majorVersion": 5, "minorVersion": 30}) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) stream.refresh_metadata() assert stream.annotations()[0] == expected - def test_stream_name_property(self): """ Assert name property comes from tags @@ -165,7 +204,6 @@ def test_stream_name_property(self): stream._tags = {"name": name} assert stream.name == name - def test_stream_unit_property(self): """ Assert unit property comes from tags @@ -175,7 +213,6 @@ def test_stream_unit_property(self): stream._tags = {"unit": unit} assert stream.unit == unit - ########################################################################## ## update tests ########################################################################## @@ -184,8 +221,9 @@ def test_update_arguments(self): """ Assert update raises errors on invalid arguments """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) + endpoint.info = Mock(return_value={"majorVersion": 5, "minorVersion": 30}) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) # no arguments @@ -208,46 +246,46 @@ def test_update_arguments(self): stream.update(collection=42) assert "collection must be of type string" in str(exc) - def test_update_tags(self): """ Assert update calls correct Endpoint methods for tags update """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) + endpoint.info = Mock(return_value={"majorVersion": 5, "minorVersion": 30}) endpoint.streamInfo = Mock(return_value=("koala", 42, {}, {}, None)) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) tags = {"cat": "dog"} stream.update(tags=tags) - stream._btrdb.ep.setStreamTags.assert_called_once_with(uu=uu, expected=42, - tags=tags, collection="koala") + stream._btrdb.ep.setStreamTags.assert_called_once_with( + uu=uu, expected=42, tags=tags, collection="koala" + ) stream._btrdb.ep.setStreamAnnotations.assert_not_called() - def test_update_collection(self): """ Assert update calls correct Endpoint methods for collection update """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) endpoint.streamInfo = Mock(return_value=("koala", 42, {}, {}, None)) - stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) + endpoint.info = Mock(return_value={"majorVersion": 5, "minorVersion": 30}) + stream = Stream(btrdb=BTrDB(endpoint), uuid=uu, property_version=42) collection = "giraffe" stream.update(collection=collection) - stream._btrdb.ep.setStreamTags.assert_called_once_with(uu=uu, expected=42, - tags=stream.tags(), collection=collection) - stream._btrdb.ep.setStreamAnnotations.assert_not_called() - + stream._btrdb.ep.setStreamTags.assert_called_once_with( + uu=uu, expected=42, tags={}, collection=collection + ) def test_update_annotations(self): """ Assert update calls correct Endpoint methods for annotations update """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) endpoint.streamInfo = Mock(return_value=("koala", 42, {}, {}, None)) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) @@ -258,7 +296,7 @@ def test_update_annotations(self): "description": "El Segundo PMU 42 Ean", "devacronym": "PMU!EL_SEG_PMU_42", "enabled": True, - "id": uuid.UUID('76932ae4-09bc-472c-8dc6-64fea68d2797'), + "id": uuid.UUID("76932ae4-09bc-472c-8dc6-64fea68d2797"), "phase": "A", "label": None, "frequency": 30, @@ -272,22 +310,21 @@ def test_update_annotations(self): uu=uu, expected=42, changes={ - 'acronym': 'VPHM', - 'description': 'El Segundo PMU 42 Ean', - 'devacronym': 'PMU!EL_SEG_PMU_42', - 'enabled': 'true', - 'id': '76932ae4-09bc-472c-8dc6-64fea68d2797', - 'phase': 'A', - 'label': 'null', - 'frequency': '30', - 'control': '2019-11-07 13:21:23.000000-0500', + "acronym": "VPHM", + "description": "El Segundo PMU 42 Ean", + "devacronym": "PMU!EL_SEG_PMU_42", + "enabled": "true", + "id": "76932ae4-09bc-472c-8dc6-64fea68d2797", + "phase": "A", + "label": "null", + "frequency": "30", + "control": "2019-11-07 13:21:23.000000-0500", "calibrate": '{"racf": 1.8, "pacf": 0.005}', }, removals=[], ) stream._btrdb.ep.setStreamTags.assert_not_called() - def test_update_annotations_nested_conversions(self): """ Assert update correctly encodes nested annotation data @@ -304,21 +341,23 @@ def test_update_annotations_nested_conversions(self): "num": 12, "float": 1.3, "string": "the quick brown fox is 12", - } - } + }, + }, } - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) - endpoint.streamInfo = Mock(return_value=("koala", 42, {}, {"foo": "42 Cherry Hill"}, None)) + endpoint.streamInfo = Mock( + return_value=("koala", 42, {}, {"foo": "42 Cherry Hill"}, None) + ) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) stream.refresh_metadata() stream.update(annotations=annotations) # spot check a nested value for Python 3.4 and 3.5 compatability - changes = stream._btrdb.ep.setStreamAnnotations.call_args[1]['changes'] - assert changes['nested'].__class__ == str - assert json.loads(changes['nested']) == annotations['nested'] + changes = stream._btrdb.ep.setStreamAnnotations.call_args[1]["changes"] + assert changes["nested"].__class__ == str + assert json.loads(changes["nested"]) == annotations["nested"] # check all args if Python > 3.5 if sys.version_info[0] > 3.5: @@ -326,18 +365,18 @@ def test_update_annotations_nested_conversions(self): uu=uu, expected=42, changes={ - 'num': '10', - 'float': '1.3', - 'string': '"the quick brown fox is 10"', - 'nested': '{"num": 11, "float": 1.3, "string": "the quick brown fox is 11", "nested": {"num": 12, "float": 1.3, "string": "the quick brown fox is 12"}}' - } + "num": "10", + "float": "1.3", + "string": '"the quick brown fox is 10"', + "nested": '{"num": 11, "float": 1.3, "string": "the quick brown fox is 11", "nested": {"num": 12, "float": 1.3, "string": "the quick brown fox is 12"}}', + }, ) def test_update_annotations_no_encoder(self): """ Assert update annotations works with None as encoder argument """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) endpoint.streamInfo = Mock(return_value=("koala", 42, {}, {}, None)) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) @@ -361,12 +400,17 @@ def test_update_annotations_replace(self): Assert that replace argument will add proper keys to removals array in endpoint call. """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) - endpoint.streamInfo = Mock(return_value=("koala", 42, {}, {"phase": "A", "source": "PJM"}, None)) + endpoint.streamInfo = Mock( + return_value=("koala", 42, {}, {"phase": "A", "source": "PJM"}, None) + ) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) - annotations = {"foo": "this is a string", "phase": "A", } + annotations = { + "foo": "this is a string", + "phase": "A", + } stream.refresh_metadata() @@ -396,45 +440,42 @@ def test_exists_cached_value(self): """ Assert exists first uses cached value """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") stream = Stream(btrdb=BTrDB(Mock(Endpoint)), uuid=uu, known_to_exist=True) stream.refresh_metadata = Mock() assert stream.exists() stream.refresh_metadata.assert_not_called() - def test_exists(self): """ Assert exists refreshes data if value is unknown """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") stream = Stream(btrdb=BTrDB(Mock(Endpoint)), uuid=uu) stream.refresh_metadata = Mock(return_value=True) assert stream.exists() assert stream.refresh_metadata.call_count == 1 - def test_exists_returns_false_on_404(self): """ Assert exists returns False on 404 error """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") stream = Stream(btrdb=BTrDB(Mock(Endpoint)), uuid=uu) - stream.refresh_metadata = Mock(side_effect=StreamNotFoundError( - "stream not found with provided uuid" - )) + stream.refresh_metadata = Mock( + side_effect=StreamNotFoundError("stream not found with provided uuid") + ) assert stream.exists() == False assert stream.refresh_metadata.call_count == 1 - def test_exists_passes_other_errors(self): """ Assert exists does not keep non 404 errors trapped """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") stream = Stream(btrdb=BTrDB(Mock(Endpoint)), uuid=uu) stream.refresh_metadata = Mock(side_effect=ValueError()) @@ -442,7 +483,6 @@ def test_exists_passes_other_errors(self): stream.exists() assert stream.refresh_metadata.call_count == 1 - ########################################################################## ## tag/annotation tests ########################################################################## @@ -451,34 +491,33 @@ def test_tags_returns_copy(self): """ Assert tags returns a copy of the tags dict """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") token = {"cat": "dog"} stream = Stream(btrdb=BTrDB(Mock(Endpoint)), uuid=uu, tags=token) assert stream.tags() is not token assert stream.tags() == token - def test_tags_returns_cached_values(self): """ Assert tags returns a copy of the tags dict """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") token = {"cat": "dog"} - stream = Stream(btrdb=BTrDB(Mock(Endpoint)), uuid=uu, tags=token, - property_version=42) + stream = Stream( + btrdb=BTrDB(Mock(Endpoint)), uuid=uu, tags=token, property_version=42 + ) stream.refresh_metadata = Mock() assert stream.tags(refresh=False) == token stream.refresh_metadata.assert_not_called() - def test_tags_forces_refresh_if_requested(self): """ Assert tags calls refresh_metadata if requested even though a cached copy is available """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") token = {"cat": "dog"} stream = Stream(btrdb=BTrDB(Mock(Endpoint)), uuid=uu, tags=token) stream.refresh_metadata = Mock() @@ -486,42 +525,41 @@ def test_tags_forces_refresh_if_requested(self): stream.tags(refresh=True) assert stream.refresh_metadata.call_count == 1 - def test_annotations_returns_copy_of_value(self): """ Assert annotations returns a copy of the annotations dict """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") token = {"cat": "dog"} - stream = Stream(btrdb=BTrDB(Mock(Endpoint)), uuid=uu, annotations=token, - property_version=42) + stream = Stream( + btrdb=BTrDB(Mock(Endpoint)), uuid=uu, annotations=token, property_version=42 + ) stream.refresh_metadata = Mock() assert stream.annotations(refresh=False)[0] == token assert stream.annotations(refresh=False)[0] is not token assert stream.annotations(refresh=False)[1] == 42 - def test_annotations_returns_cached_values(self): """ Assert annotations returns a copy of the annotations dict """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") token = {"cat": "dog"} - stream = Stream(btrdb=BTrDB(Mock(Endpoint)), uuid=uu, annotations=token, - property_version=42) + stream = Stream( + btrdb=BTrDB(Mock(Endpoint)), uuid=uu, annotations=token, property_version=42 + ) stream.refresh_metadata = Mock() assert stream.annotations(refresh=False)[0] == token stream.refresh_metadata.assert_not_called() - def test_annotations_forces_refresh_if_requested(self): """ Assert annotations calls refresh_metadata if requested even though a cached copy is available """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") token = {"cat": "dog"} stream = Stream(btrdb=BTrDB(Mock(Endpoint)), uuid=uu, annotations=token) stream.refresh_metadata = Mock() @@ -529,7 +567,6 @@ def test_annotations_forces_refresh_if_requested(self): stream.annotations(refresh=True) assert stream.refresh_metadata.call_count == 1 - ########################################################################## ## windowing tests ########################################################################## @@ -538,15 +575,29 @@ def test_windows(self): """ Assert windows returns tuples of data from Endpoint.windows """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) windows = [ - [(StatPointProto(time=1,min=2,mean=3,max=4,count=5,stddev=6), StatPointProto(time=2,min=3,mean=4,max=5,count=6,stddev=7)), 42], - [(StatPointProto(time=3,min=4,mean=5,max=6,count=7,stddev=8), StatPointProto(time=4,min=5,mean=6,max=7,count=8,stddev=9)), 42], + [ + ( + StatPointProto(time=1, min=2, mean=3, max=4, count=5, stddev=6), + StatPointProto(time=2, min=3, mean=4, max=5, count=6, stddev=7), + ), + 42, + ], + [ + ( + StatPointProto(time=3, min=4, mean=5, max=6, count=7, stddev=8), + StatPointProto(time=4, min=5, mean=6, max=7, count=8, stddev=9), + ), + 42, + ], ] expected = ( - (StatPoint(time=1,minv=2.0,meanv=3.0,maxv=4.0,count=5,stddev=6.0), 42), (StatPoint(time=2,minv=3.0,meanv=4.0,maxv=5.0,count=6,stddev=7.0), 42), - (StatPoint(time=3,minv=4.0,meanv=5.0,maxv=6.0,count=7,stddev=8.0), 42), (StatPoint(time=4,minv=5.0,meanv=6.0,maxv=7.0,count=8,stddev=9.0), 42), + (StatPoint(time=1, minv=2.0, meanv=3.0, maxv=4.0, count=5, stddev=6.0), 42), + (StatPoint(time=2, minv=3.0, meanv=4.0, maxv=5.0, count=6, stddev=7.0), 42), + (StatPoint(time=3, minv=4.0, meanv=5.0, maxv=6.0, count=7, stddev=8.0), 42), + (StatPoint(time=4, minv=5.0, meanv=6.0, maxv=7.0, count=8, stddev=9.0), 42), ) endpoint.windows = Mock(return_value=windows) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) @@ -556,24 +607,35 @@ def test_windows(self): assert result == expected assert isinstance(result, tuple) assert isinstance(result[0], tuple) - stream._btrdb.ep.windows.assert_called_once_with( - uu, 100, 500, 2, 0, 0 - ) - + stream._btrdb.ep.windows.assert_called_once_with(uu, 100, 500, 2, 0, 0) def test_aligned_windows(self): """ Assert windows returns tuples of data from Endpoint.alignedWindows """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) windows = [ - [(StatPointProto(time=1,min=2,mean=3,max=4,count=5,stddev=6), StatPointProto(time=2,min=3,mean=4,max=5,count=6,stddev=7)), 42], - [(StatPointProto(time=3,min=4,mean=5,max=6,count=7,stddev=8), StatPointProto(time=4,min=5,mean=6,max=7,count=8,stddev=9)), 42], + [ + ( + StatPointProto(time=1, min=2, mean=3, max=4, count=5, stddev=6), + StatPointProto(time=2, min=3, mean=4, max=5, count=6, stddev=7), + ), + 42, + ], + [ + ( + StatPointProto(time=3, min=4, mean=5, max=6, count=7, stddev=8), + StatPointProto(time=4, min=5, mean=6, max=7, count=8, stddev=9), + ), + 42, + ], ] expected = ( - (StatPoint(time=1,minv=2.0,meanv=3.0,maxv=4.0,count=5,stddev=6.0), 42), (StatPoint(time=2,minv=3.0,meanv=4.0,maxv=5.0,count=6,stddev=7.0), 42), - (StatPoint(time=3,minv=4.0,meanv=5.0,maxv=6.0,count=7,stddev=8.0), 42), (StatPoint(time=4,minv=5.0,meanv=6.0,maxv=7.0,count=8,stddev=9.0), 42), + (StatPoint(time=1, minv=2.0, meanv=3.0, maxv=4.0, count=5, stddev=6.0), 42), + (StatPoint(time=2, minv=3.0, meanv=4.0, maxv=5.0, count=6, stddev=7.0), 42), + (StatPoint(time=3, minv=4.0, meanv=5.0, maxv=6.0, count=7, stddev=8.0), 42), + (StatPoint(time=4, minv=5.0, meanv=6.0, maxv=7.0, count=8, stddev=9.0), 42), ) endpoint.alignedWindows = Mock(return_value=windows) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) @@ -583,20 +645,29 @@ def test_aligned_windows(self): assert result == expected assert isinstance(result, tuple) assert isinstance(result[0], tuple) - stream._btrdb.ep.alignedWindows.assert_called_once_with( - uu, 100, 500, 1, 0 - ) - + stream._btrdb.ep.alignedWindows.assert_called_once_with(uu, 100, 500, 1, 0) def test_count(self): """ Test that stream count method uses aligned windows """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) windows = [ - [(StatPointProto(time=1,min=2,mean=3,max=4,count=5,stddev=6), StatPointProto(time=2,min=3,mean=4,max=5,count=6,stddev=7)), 42], - [(StatPointProto(time=3,min=4,mean=5,max=6,count=7,stddev=8), StatPointProto(time=4,min=5,mean=6,max=7,count=8,stddev=9)), 42], + [ + ( + StatPointProto(time=1, min=2, mean=3, max=4, count=5, stddev=6), + StatPointProto(time=2, min=3, mean=4, max=5, count=6, stddev=7), + ), + 42, + ], + [ + ( + StatPointProto(time=3, min=4, mean=5, max=6, count=7, stddev=8), + StatPointProto(time=4, min=5, mean=6, max=7, count=8, stddev=9), + ), + 42, + ], ] endpoint.alignedWindows = Mock(return_value=windows) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) @@ -609,7 +680,6 @@ def test_count(self): stream.count(10, 1000, 8, version=1200) stream._btrdb.ep.alignedWindows.assert_called_with(uu, 10, 1000, 8, 1200) - ########################################################################## ## earliest/latest tests ########################################################################## @@ -618,7 +688,7 @@ def test_earliest(self): """ Assert earliest calls Endpoint.nearest """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) endpoint.nearest = Mock(return_value=(RawPointProto(time=100, value=1.0), 42)) @@ -627,12 +697,11 @@ def test_earliest(self): assert (point, ver) == (RawPoint(100, 1.0), 42) endpoint.nearest.assert_called_once_with(uu, MINIMUM_TIME, 0, False) - def test_earliest_swallows_exception(self): """ Assert earliest returns None when endpoint throws exception """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) endpoint.nearest = Mock(side_effect=NoSuchPoint("next point does not exist")) @@ -640,12 +709,11 @@ def test_earliest_swallows_exception(self): assert stream.earliest() is None endpoint.nearest.assert_called_once_with(uu, MINIMUM_TIME, 0, False) - def test_earliest_passes_exception(self): """ Assert earliest reraises non 401 exception """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) endpoint.nearest = Mock(side_effect=BTrDBError("empty")) @@ -655,12 +723,11 @@ def test_earliest_passes_exception(self): assert exc.value.args[0] == "empty" endpoint.nearest.assert_called_once_with(uu, MINIMUM_TIME, 0, False) - def test_latest(self): """ Assert latest calls Endpoint.nearest """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) endpoint.nearest = Mock(return_value=(RawPointProto(time=100, value=1.0), 42)) @@ -669,12 +736,11 @@ def test_latest(self): assert (point, ver) == (RawPoint(100, 1.0), 42) endpoint.nearest.assert_called_once_with(uu, MAXIMUM_TIME, 0, True) - def test_latest_swallows_exception(self): """ Assert latest returns None when endpoint throws exception """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) endpoint.nearest = Mock(side_effect=NoSuchPoint("empty")) @@ -682,12 +748,11 @@ def test_latest_swallows_exception(self): assert stream.latest() is None endpoint.nearest.assert_called_once_with(uu, MAXIMUM_TIME, 0, True) - def test_latest_passes_exception(self): """ Assert latest reraises non 401 exception """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) endpoint.nearest = Mock(side_effect=BTrDBError("empty")) @@ -697,13 +762,12 @@ def test_latest_passes_exception(self): assert exc.value.args[0] == "empty" endpoint.nearest.assert_called_once_with(uu, MAXIMUM_TIME, 0, True) - @patch("btrdb.stream.currently_as_ns") def test_currently(self, mocked): """ Assert currently calls Endpoint.nearest """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) endpoint.nearest = Mock(return_value=(RawPointProto(time=100, value=1.0), 42)) @@ -714,13 +778,12 @@ def test_currently(self, mocked): assert (point, ver) == (RawPoint(100, 1.0), 42) endpoint.nearest.assert_called_once_with(uu, ns_fake_time, 0, True) - @patch("btrdb.stream.currently_as_ns") def test_currently_swallows_exception(self, mocked): """ Assert currently returns None when endpoint throws exception """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) endpoint.nearest = Mock(side_effect=NoSuchPoint("empty")) @@ -730,13 +793,12 @@ def test_currently_swallows_exception(self, mocked): assert stream.current() is None endpoint.nearest.assert_called_once_with(uu, ns_fake_time, 0, True) - @patch("btrdb.stream.currently_as_ns") def test_currently_passes_exception(self, mocked): """ Assert currently reraises non 401 exception """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) endpoint.nearest = Mock(side_effect=BTrDBError("empty")) @@ -748,7 +810,6 @@ def test_currently_passes_exception(self, mocked): assert exc.value.args[0] == "empty" endpoint.nearest.assert_called_once_with(uu, ns_fake_time, 0, True) - ########################################################################## ## misc tests ########################################################################## @@ -757,7 +818,7 @@ def test_version(self): """ Assert version calls and returns correct value from streamInfo """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) endpoint.streamInfo = Mock(return_value=("", 0, {}, {}, 42)) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) @@ -765,30 +826,36 @@ def test_version(self): assert stream.version() == 42 stream._btrdb.ep.streamInfo.assert_called_once_with(uu, True, False) - def test_insert(self): """ Assert insert batches data to endpoint insert and returns version """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) - endpoint.insert = Mock(side_effect=[1,2,3]) + endpoint.insert = Mock(side_effect=[1, 2, 3]) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) - data = list(zip(range(10000,120000), map(float, range(110000)))) + data = list(zip(range(10000, 120000), map(float, range(110000)))) version = stream.insert(data) - assert stream._btrdb.ep.insert.call_args_list[0][0][1] == data[:INSERT_BATCH_SIZE] - assert stream._btrdb.ep.insert.call_args_list[1][0][1] == data[INSERT_BATCH_SIZE:2*INSERT_BATCH_SIZE] - assert stream._btrdb.ep.insert.call_args_list[2][0][1] == data[2*INSERT_BATCH_SIZE:] + assert ( + stream._btrdb.ep.insert.call_args_list[0][0][1] == data[:INSERT_BATCH_SIZE] + ) + assert ( + stream._btrdb.ep.insert.call_args_list[1][0][1] + == data[INSERT_BATCH_SIZE : 2 * INSERT_BATCH_SIZE] + ) + assert ( + stream._btrdb.ep.insert.call_args_list[2][0][1] + == data[2 * INSERT_BATCH_SIZE :] + ) assert version == 3 - def test_nearest(self): """ Assert nearest calls Endpoint.nearest with correct arguments """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) endpoint.nearest = Mock(return_value=(RawPointProto(time=100, value=1.0), 42)) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) @@ -798,12 +865,11 @@ def test_nearest(self): assert point == RawPoint(100, 1.0) assert version == 42 - def test_nearest_swallows_exception(self): """ Assert nearest returns None when endpoint throws 401 exception """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) endpoint.nearest = Mock(side_effect=NoSuchPoint("next point does not exist")) @@ -811,12 +877,11 @@ def test_nearest_swallows_exception(self): assert stream.nearest(0, 0, False) is None endpoint.nearest.assert_called_once_with(uu, 0, 0, False) - def test_nearest_passes_exception(self): """ Assert nearest reraises non 401 exception """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) endpoint.nearest = Mock(side_effect=BTrDBError("foo")) @@ -826,45 +891,41 @@ def test_nearest_passes_exception(self): assert exc.value.args[0] == "foo" endpoint.nearest.assert_called_once_with(uu, 0, 0, False) - def test_delete_range(self): """ Assert delete_range calls Endpoint.deleteRange with correct arguments """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") stream = Stream(btrdb=BTrDB(Mock(Endpoint)), uuid=uu) stream.delete(5, 10) stream._btrdb.ep.deleteRange.assert_called_once_with(uu, 5, 10) - def test_flush(self): """ Assert flush calls Endpoint.flush with UUID """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") stream = Stream(btrdb=BTrDB(Mock(Endpoint)), uuid=uu) stream.flush() stream._btrdb.ep.flush.assert_called_once_with(uu) - def test_obliterate(self): """ Assert obliterate calls Endpoint.obliterate with UUID """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") stream = Stream(btrdb=BTrDB(Mock(Endpoint)), uuid=uu) stream.obliterate() stream._btrdb.ep.obliterate.assert_called_once_with(uu) - def test_obliterate_allows_error(self): """ Assert obliterate raises error if stream not found. (does not swallow error) """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) endpoint.obliterate = Mock(side_effect=StreamNotFoundError()) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) @@ -874,20 +935,26 @@ def test_obliterate_allows_error(self): stream._btrdb.ep.obliterate.assert_called_once_with(uu) - - ########################################################################## ## StreamSet Tests ########################################################################## -class TestStreamSet(object): +class TestStreamSet(object): def test_create(self): """ Assert we can create the object """ - StreamSet([]) + StreamSet([Stream(None, "1"), Stream(None, "2"), Stream(None, "3")]) + @pytest.mark.parametrize( + "empty_container", [(tuple()), (dict()), (list()), (set())] + ) + def test_create_error_with_no_objects(self, empty_container): + with pytest.raises( + ValueError, match="Trying to create streamset with an empty list of streams" + ): + StreamSet(empty_container) ########################################################################## ## builtin / magic methods tests @@ -897,66 +964,91 @@ def test_repr(self): """ Assert StreamSet instance repr output """ - data = [11,22,"dog","cat"] + data = [ + Stream(None, "11"), + Stream(None, "22"), + Stream(None, "dog"), + Stream(None, "cat"), + ] streams = StreamSet(data) expected = "" assert streams.__repr__() == expected - data = [1] + data = [Stream(None, "1")] streams = StreamSet(data) expected = "" assert streams.__repr__() == expected - def test_str(self): """ Assert StreamSet instance str output """ - data = [11,22,"dog","cat"] + data = [ + Stream(None, "11"), + Stream(None, "22"), + Stream(None, "dog"), + Stream(None, "cat"), + ] streams = StreamSet(data) expected = "StreamSet with 4 streams" assert str(streams) == expected - data = [1] + data = [Stream(None, "1")] streams = StreamSet(data) expected = "StreamSet with 1 stream" assert str(streams) == expected - def test_subscriptable(self): """ Assert StreamSet instance is subscriptable """ - data = [11,22,"dog","cat"] + data = [ + Stream(None, "11"), + Stream(None, "22"), + Stream(None, "dog"), + Stream(None, "cat"), + ] streams = StreamSet(data) for index, val in enumerate(data): assert streams[index] == val - def test_len(self): """ Assert StreamSet instance support len """ - data = [11,22,"dog","cat"] + data = [ + Stream(None, "11"), + Stream(None, "22"), + Stream(None, "dog"), + Stream(None, "cat"), + ] streams = StreamSet(data) assert len(streams) == len(data) - def test_iter(self): """ Assert StreamSet instance support iteration """ - data = [11,22,"dog","cat"] + data = [ + Stream(None, "11"), + Stream(None, "22"), + Stream(None, "dog"), + Stream(None, "cat"), + ] streams = StreamSet(data) for index, stream in enumerate(streams): assert data[index] == stream - def test_indexing(self): """ Assert StreamSet instance supports indexing """ - data = [11,22,"dog","cat"] + data = [ + Stream(None, "11"), + Stream(None, "22"), + Stream(None, "dog"), + Stream(None, "cat"), + ] streams = StreamSet(data) # verify index lookup @@ -967,7 +1059,6 @@ def test_indexing(self): # verify slicing works assert streams[:2] == data[:2] - def test_mapping(self): """ Assert StreamSet instance support key mapping @@ -990,54 +1081,58 @@ def test_mapping(self): streams[missing] assert str(missing) in str(e) - - def test_contains(self): + def test_contains(self, stream1, stream2, stream3): """ Assert StreamSet instance supports contains """ - data = [11,22,"dog","cat"] + data = [stream1, stream2, stream3] streams = StreamSet(data) - assert "dog" in streams - + assert "0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a" in streams def test_reverse(self): """ Assert StreamSet instance supports reversal """ - data = [11,22,"dog","cat"] + data = [ + Stream(None, "11"), + Stream(None, "22"), + Stream(None, "dog"), + Stream(None, "cat"), + ] streams = StreamSet(data) assert list(reversed(streams)) == list(reversed(data)) - def test_to_list(self): """ Assert StreamSet instance cast to list """ - data = [11,22,"dog","cat"] + data = [ + Stream(None, "11"), + Stream(None, "22"), + Stream(None, "dog"), + Stream(None, "cat"), + ] streams = StreamSet(data) assert list(streams) == data - ########################################################################## ## allow_window tests ########################################################################## - def test_allow_window(self): + def test_allow_window(self, stream1, stream2, stream3): """ Assert allow_window returns False if window already requested """ - streams = StreamSet([1,2,3]) + streams = StreamSet([stream1, stream2, stream3]) assert streams.allow_window == True - streams.windows(30, 4) + streams.windows(30) assert streams.allow_window == False - - streams = StreamSet([1,2,3]) + streams = StreamSet([stream1, stream2, stream3]) streams.aligned_windows(30) assert streams.allow_window == False - ########################################################################## ## _latest_versions tests ########################################################################## @@ -1047,13 +1142,9 @@ def test_latest_versions(self, stream1, stream2): Assert _latest_versions returns correct values """ streams = StreamSet([stream1, stream2]) - expected = { - stream1.uuid: stream1.version(), - stream2.uuid: stream2.version() - } + expected = {stream1.uuid: stream1.version(), stream2.uuid: stream2.version()} assert streams._latest_versions() == expected - ########################################################################## ## pin_versions tests ########################################################################## @@ -1063,75 +1154,63 @@ def test_pin_versions_returns_self(self, stream1): Assert pin_versions returns self """ streams = StreamSet([stream1]) - expected = { - stream1.uuid(): stream1.version() - } - result = streams.pin_versions(expected) + expected = {stream1.uuid(): stream1.version()} + result = streams.pin_versions(expected) assert streams is result - - def test_pin_versions_with_argument(self): + def test_pin_versions_with_argument(self, stream1, stream2): """ Assert pin_versions uses supplied version numbers """ - streams = StreamSet([1,2]) - expected = [3,4] + streams = StreamSet([stream1, stream2]) + expected = [3, 4] assert streams.pin_versions(expected) == streams assert streams._pinned_versions == expected - - def test_pin_versions_with_argument(self): + def test_pin_versions_with_argument(self, stream1, stream2): """ Assert pin_versions uses supplied version numbers """ - streams = StreamSet([1,2]) - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') - expected = { - uu: 42 - } + streams = StreamSet([stream1, stream2]) + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + expected = {uu: 42} assert streams.pin_versions(expected) == streams assert streams._pinned_versions == expected - def test_pin_versions_no_argument(self, stream1, stream2): """ Assert pin_versions uses latest version numbers """ streams = StreamSet([stream1, stream2]) - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') - expected = { - uu: 42 - } + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + expected = {uu: 42} assert streams.pin_versions(expected) == streams assert streams._pinned_versions == expected - - def test_pin_versions_raise_on_non_dict(self): + def test_pin_versions_raise_on_non_dict(self, stream1): """ Assert pin_versions raises if versions argument is not dict """ - streams = StreamSet([1]) + streams = StreamSet([stream1]) expected = "INVALID DATA" with pytest.raises(TypeError) as e: streams.pin_versions(expected) == streams assert "dict" in str(e).lower() - def test_pin_versions_raise_on_non_uuid_key(self): """ Assert pin_versions raises if versions argument dict does not have UUID for keys """ - streams = StreamSet([1]) + streams = StreamSet([Stream(None, "1")]) expected = {"uuid": 42} with pytest.raises(TypeError) as e: streams.pin_versions(expected) == streams assert "uuid" in str(e).lower() - ########################################################################## ## versions tests ########################################################################## @@ -1141,37 +1220,29 @@ def test_versions_no_pin(self, stream1, stream2): Assert versions returns correctly if pin_versions not called """ streams = StreamSet([stream1, stream2]) - expected = { - stream1.uuid: stream1.version(), - stream2.uuid: stream2.version() - } + expected = {stream1.uuid: stream1.version(), stream2.uuid: stream2.version()} assert streams.versions() == expected - def test_versions_with_pin(self, stream1, stream2): """ Assert versions returns correctly if pin_versions called """ - uu1 = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu1 = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") s1 = Mock(Stream) s1.version = Mock(return_value=11) s1.uuid = Mock(return_value=uu1) - uu2 = uuid.UUID('17dbe387-89ea-42b6-864b-f505cdb483f5') + uu2 = uuid.UUID("17dbe387-89ea-42b6-864b-f505cdb483f5") s2 = Mock(Stream) s2.version = Mock(return_value=22) s2.uuid = Mock(return_value=uu2) streams = StreamSet([stream1, stream2]) - expected = { - stream1.uuid(): 88, - stream2.uuid(): 99 - } + expected = {stream1.uuid(): 88, stream2.uuid(): 99} streams.pin_versions(expected) assert streams.versions() == expected - ########################################################################## ## earliest/latest tests ########################################################################## @@ -1181,78 +1252,115 @@ def test_earliest(self, stream1, stream2): Assert earliest returns correct time code """ streams = StreamSet([stream1, stream2]) - assert streams.latest() == (RawPoint(time=10, value=1), RawPoint(time=20, value=1)) - + assert streams.earliest() == ( + RawPoint(time=10, value=1), + RawPoint(time=20, value=1), + ) def test_latest(self, stream1, stream2): """ Assert latest returns correct time code """ streams = StreamSet([stream1, stream2]) - assert streams.latest() == (RawPoint(time=10, value=1), RawPoint(time=20, value=1)) + assert streams.latest() == ( + RawPoint(time=10, value=1), + RawPoint(time=20, value=1), + ) @patch("btrdb.stream.currently_as_ns") def test_current(self, mocked, stream1, stream2): """ Assert current calls nearest with the current time """ - mocked.return_value=15 + mocked.return_value = 15 streams = StreamSet([stream1, stream2]) streams.current() stream1.nearest.assert_called_once_with(15, version=11, backward=True) stream2.nearest.assert_called_once_with(15, version=22, backward=True) @patch("btrdb.stream.currently_as_ns") - def test_currently_out_of_range(self, mocked): + def test_currently_out_of_range(self, mocked, stream1, stream2): """ Assert currently raises an exception if it is not filtered """ - mocked.return_value=15 + mocked.return_value = 15 streams = StreamSet([stream1, stream2]) - with pytest.raises(ValueError, match="current time is not included in filtered stream range"): + with pytest.raises( + ValueError, match="current time is not included in filtered stream range" + ): streams.filter(start=20, end=30).current() - with pytest.raises(ValueError, match="current time is not included in filtered stream range"): + with pytest.raises( + ValueError, match="current time is not included in filtered stream range" + ): streams.filter(start=0, end=10).current() def test_count(self): """ Test the stream set count method """ - uu1 = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') - uu2 = uuid.UUID('4dadf38d-52a5-4b7a-ada9-a5d563f9538c') + uu1 = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + uu2 = uuid.UUID("4dadf38d-52a5-4b7a-ada9-a5d563f9538c") endpoint = Mock(Endpoint) + endpoint.streamInfo = Mock(return_value=[-1, -1, -1, -1, 0]) windows = [ - [(StatPointProto(time=1,min=2,mean=3,max=4,count=5,stddev=6), StatPointProto(time=2,min=3,mean=4,max=5,count=6,stddev=7)), 42], - [(StatPointProto(time=3,min=4,mean=5,max=6,count=7,stddev=8), StatPointProto(time=4,min=5,mean=6,max=7,count=8,stddev=9)), 42], + [ + ( + StatPointProto(time=1, min=2, mean=3, max=4, count=5, stddev=6), + StatPointProto(time=2, min=3, mean=4, max=5, count=6, stddev=7), + ), + 42, + ], + [ + ( + StatPointProto(time=3, min=4, mean=5, max=6, count=7, stddev=8), + StatPointProto(time=4, min=5, mean=6, max=7, count=8, stddev=9), + ), + 42, + ], ] endpoint.alignedWindows = Mock(return_value=windows) - streams = StreamSet([ - Stream(btrdb=BTrDB(endpoint), uuid=uu1), - Stream(btrdb=BTrDB(endpoint), uuid=uu2), - ]) + streams = StreamSet( + [ + Stream(btrdb=BTrDB(endpoint), uuid=uu1), + Stream(btrdb=BTrDB(endpoint), uuid=uu2), + ] + ) assert streams.count() == 52 endpoint.alignedWindows.assert_any_call(uu1, MINIMUM_TIME, MAXIMUM_TIME, 60, 0) endpoint.alignedWindows.assert_any_call(uu2, MINIMUM_TIME, MAXIMUM_TIME, 60, 0) - def test_count_filtered(self): """ Test the stream set count method with filters """ - uu1 = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') - uu2 = uuid.UUID('4dadf38d-52a5-4b7a-ada9-a5d563f9538c') + uu1 = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + uu2 = uuid.UUID("4dadf38d-52a5-4b7a-ada9-a5d563f9538c") endpoint = Mock(Endpoint) endpoint.alignedWindows = Mock(return_value=[]) - streams = StreamSet([ - Stream(btrdb=BTrDB(endpoint), uuid=uu1), - Stream(btrdb=BTrDB(endpoint), uuid=uu2), - ]) + streams = StreamSet( + [ + Stream(btrdb=BTrDB(endpoint), uuid=uu1), + Stream(btrdb=BTrDB(endpoint), uuid=uu2), + ] + ) windows = [ - [(StatPointProto(time=1,min=2,mean=3,max=4,count=5,stddev=6), StatPointProto(time=2,min=3,mean=4,max=5,count=6,stddev=7)), 42], - [(StatPointProto(time=3,min=4,mean=5,max=6,count=7,stddev=8), StatPointProto(time=4,min=5,mean=6,max=7,count=8,stddev=9)), 42], + [ + ( + StatPointProto(time=1, min=2, mean=3, max=4, count=5, stddev=6), + StatPointProto(time=2, min=3, mean=4, max=5, count=6, stddev=7), + ), + 42, + ], + [ + ( + StatPointProto(time=3, min=4, mean=5, max=6, count=7, stddev=8), + StatPointProto(time=4, min=5, mean=6, max=7, count=8, stddev=9), + ), + 42, + ], ] endpoint.alignedWindows = Mock(return_value=windows) @@ -1263,8 +1371,6 @@ def test_count_filtered(self): endpoint.alignedWindows.assert_any_call(uu1, 10, 1000, 8, 42) endpoint.alignedWindows.assert_any_call(uu2, 10, 1000, 8, 99) - - ########################################################################## ## filter tests ########################################################################## @@ -1282,7 +1388,6 @@ def test_filter(self, stream1): assert streams.filters[0].end == end assert isinstance(streams.filters[0], StreamFilter) - def test_filter_returns_new_instance(self, stream1): """ Assert filter returns new instance @@ -1294,7 +1399,6 @@ def test_filter_returns_new_instance(self, stream1): assert other is not streams assert isinstance(other, streams.__class__) - def test_filter_collection(self, stream1, stream2): """ Assert filter collection works as intended @@ -1331,7 +1435,6 @@ def test_filter_collection(self, stream1, stream2): other = streams.filter(collection=re.compile(r"region\.")) assert other._streams == [stream1] - def test_filter_name(self, stream1, stream2): """ Assert filter name works as intended @@ -1366,7 +1469,6 @@ def test_filter_name(self, stream1, stream2): other = streams.filter(name=re.compile(r"region\.")) assert other._streams == [stream1] - def test_filter_unit(self, stream1, stream2): """ Assert filter unit works as intended @@ -1391,7 +1493,6 @@ def test_filter_unit(self, stream1, stream2): other = streams.filter(unit=re.compile("meters")) assert other._streams == [] - def test_filter_tags(self, stream1, stream2): """ Assert filter annotations works as intended @@ -1410,7 +1511,6 @@ def test_filter_tags(self, stream1, stream2): other = streams.filter(tags={"unit": "volts"}) assert other._streams == [stream1, stream2] - def test_filter_annotations(self, stream1, stream2): """ Assert filter annotations works as intended @@ -1429,7 +1529,6 @@ def test_filter_annotations(self, stream1, stream2): other = streams.filter(annotations={"owner": "ABC", "color": "red"}) assert other._streams == [stream1] - ########################################################################## ## clone tests ########################################################################## @@ -1442,14 +1541,14 @@ def test_clone_returns_new_object(self, stream1, stream2): clone = streams.clone() assert id(clone) != id(streams) - assert clone._streams is streams._streams - + assert clone._streams is not streams._streams + assert clone._streams == streams._streams ########################################################################## ## windows tests ########################################################################## - def test_windows_raises_valueerror(self, stream1): + def test_windows_raises_valueerror_and_warning(self, stream1): """ Assert that raises ValueError if arguments not castable to int """ @@ -1458,68 +1557,69 @@ def test_windows_raises_valueerror(self, stream1): streams.windows("invalid", 42) assert "literal" in str(exc).lower() - with pytest.raises(ValueError) as exc: + with pytest.warns(Warning, match="Overriding") as exc: streams.windows(42, "invalid") - assert "literal" in str(exc).lower() - def test_windows_raises_if_not_allowed(self, stream1): """ Assert windows raises Exception if not allowed """ streams = StreamSet([stream1]) - streams.windows(8, 22) + streams.windows(8) with pytest.raises(Exception) as exc: - streams.windows(10, 20) + streams.windows(10) assert "window operation is already requested" in str(exc).lower() - def test_windows_returns_self(self, stream1): """ Assert windows returns self """ streams = StreamSet([stream1]) - result = streams.windows(10, 20) + result = streams.windows(10) assert result is streams - def test_windows_stores_values(self, stream1): """ Assert windows stores values """ streams = StreamSet([stream1]) - result = streams.windows(10, 20) + result = streams.windows(10) # assert stores values - assert streams.width == 10 - assert streams.depth == 20 - + assert result.width == 10 + assert result.depth == 0 def test_windows_values_and_calls_to_endpoint(self): """ assert windows result and endpoint calls are correct """ endpoint = Mock(Endpoint) - window1 = [[(StatPointProto(time=1,min=2,mean=3,max=4,count=5,stddev=6),), 11]] - window2 = [[(StatPointProto(time=2,min=3,mean=4,max=5,count=6,stddev=7),), 12]] - endpoint.windows = Mock(side_effect=[ window1, window2 ]) + window1 = [ + [(StatPointProto(time=1, min=2, mean=3, max=4, count=5, stddev=6),), 11] + ] + window2 = [ + [(StatPointProto(time=2, min=3, mean=4, max=5, count=6, stddev=7),), 12] + ] + endpoint.windows = Mock(side_effect=[window1, window2]) - uu1 = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') - uu2 = uuid.UUID('5d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu1 = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + uu2 = uuid.UUID("5d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") s1 = Stream(btrdb=BTrDB(endpoint), uuid=uu1) s2 = Stream(btrdb=BTrDB(endpoint), uuid=uu2) versions = {uu1: 11, uu2: 12} - start, end, width, depth = 1, 100, 1000, 25 + start, end, width = 1, 100, 1000 streams = StreamSet([s1, s2]) streams.pin_versions(versions) - values = streams.filter(start=start, end=end).windows(width=width, depth=depth).values() + values = ( + streams.filter(start=start, end=end).windows(width=width, depth=0).values() + ) # assert endpoint calls have correct arguments, version expected = [ - call(uu1, start, end, width, depth, versions[uu1]), - call(uu2, start, end, width, depth, versions[uu2]) + call(uu1, start, end, width, 0, versions[uu1]), + call(uu2, start, end, width, 0, versions[uu2]), ] assert endpoint.windows.call_args_list == expected @@ -1530,31 +1630,34 @@ def test_windows_values_and_calls_to_endpoint(self): ] assert values == expected - def test_windows_rows_and_calls_to_endpoint(self): """ assert windows rows result and endpoint calls are correct """ endpoint = Mock(Endpoint) - window1 = [[(StatPointProto(time=1,min=2,mean=3,max=4,count=5,stddev=6),), 11]] - window2 = [[(StatPointProto(time=2,min=3,mean=4,max=5,count=6,stddev=7),), 12]] - endpoint.windows = Mock(side_effect=[ window1, window2 ]) + window1 = [ + [(StatPointProto(time=1, min=2, mean=3, max=4, count=5, stddev=6),), 11] + ] + window2 = [ + [(StatPointProto(time=2, min=3, mean=4, max=5, count=6, stddev=7),), 12] + ] + endpoint.windows = Mock(side_effect=[window1, window2]) - uu1 = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') - uu2 = uuid.UUID('5d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu1 = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + uu2 = uuid.UUID("5d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") s1 = Stream(btrdb=BTrDB(endpoint), uuid=uu1) s2 = Stream(btrdb=BTrDB(endpoint), uuid=uu2) versions = {uu1: 11, uu2: 12} - start, end, width, depth = 1, 100, 1000, 25 + start, end, width = 1, 100, 1000 streams = StreamSet([s1, s2]) streams.pin_versions(versions) - rows = streams.filter(start=start, end=end).windows(width=width, depth=depth).rows() + rows = streams.filter(start=start, end=end).windows(width=width, depth=0).rows() # assert endpoint calls have correct arguments, version expected = [ - call(uu1, start, end, width, depth, versions[uu1]), - call(uu2, start, end, width, depth, versions[uu2]) + call(uu1, start, end, width, 0, versions[uu1]), + call(uu2, start, end, width, 0, versions[uu2]), ] assert endpoint.windows.call_args_list == expected @@ -1565,7 +1668,6 @@ def test_windows_rows_and_calls_to_endpoint(self): ] assert rows == expected - ########################################################################## ## aligned_windows tests ########################################################################## @@ -1579,19 +1681,17 @@ def test_aligned_windows_raises_valueerror(self, stream1): streams.aligned_windows("invalid") assert "literal" in str(exc).lower() - def test_aligned_windows_raises_if_not_allowed(self, stream1): """ Assert that aligned_windows raises Exception if not allowed """ streams = StreamSet([stream1]) - streams.windows(8, 22) + streams.windows(8) with pytest.raises(Exception) as exc: streams.aligned_windows(20) assert "window operation is already requested" in str(exc).lower() - def test_aligned_windows(self, stream1): """ Assert aligned_windows stores objects @@ -1600,7 +1700,6 @@ def test_aligned_windows(self, stream1): result = streams.aligned_windows(20) assert streams.pointwidth == 20 - def test_aligned_windows_returns_self(self, stream1): """ Assert aligned_windows returns self @@ -1609,18 +1708,21 @@ def test_aligned_windows_returns_self(self, stream1): result = streams.aligned_windows(20) assert result is streams - def test_aligned_windows_values_and_calls_to_endpoint(self): """ assert aligned_windows result and endpoint calls are correct """ endpoint = Mock(Endpoint) - window1 = [[(StatPointProto(time=1,min=2,mean=3,max=4,count=5,stddev=6),), 11]] - window2 = [[(StatPointProto(time=2,min=3,mean=4,max=5,count=6,stddev=7),), 12]] - endpoint.alignedWindows = Mock(side_effect=[ window1, window2 ]) + window1 = [ + [(StatPointProto(time=1, min=2, mean=3, max=4, count=5, stddev=6),), 11] + ] + window2 = [ + [(StatPointProto(time=2, min=3, mean=4, max=5, count=6, stddev=7),), 12] + ] + endpoint.alignedWindows = Mock(side_effect=[window1, window2]) - uu1 = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') - uu2 = uuid.UUID('5d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu1 = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + uu2 = uuid.UUID("5d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") s1 = Stream(btrdb=BTrDB(endpoint), uuid=uu1) s2 = Stream(btrdb=BTrDB(endpoint), uuid=uu2) versions = {uu1: 11, uu2: 12} @@ -1628,12 +1730,16 @@ def test_aligned_windows_values_and_calls_to_endpoint(self): start, end, pointwidth = 1, 100, 25 streams = StreamSet([s1, s2]) streams.pin_versions(versions) - values = streams.filter(start=start, end=end).aligned_windows(pointwidth=pointwidth).values() + values = ( + streams.filter(start=start, end=end) + .aligned_windows(pointwidth=pointwidth) + .values() + ) # assert endpoint calls have correct arguments, version expected = [ call(uu1, start, end, pointwidth, versions[uu1]), - call(uu2, start, end, pointwidth, versions[uu2]) + call(uu2, start, end, pointwidth, versions[uu2]), ] assert endpoint.alignedWindows.call_args_list == expected @@ -1644,18 +1750,21 @@ def test_aligned_windows_values_and_calls_to_endpoint(self): ] assert values == expected - def test_aligned_windows_rows_and_calls_to_endpoint(self): """ assert aligned_windows rows result and endpoint calls are correct """ endpoint = Mock(Endpoint) - window1 = [[(StatPointProto(time=1,min=2,mean=3,max=4,count=5,stddev=6),), 11]] - window2 = [[(StatPointProto(time=2,min=3,mean=4,max=5,count=6,stddev=7),), 12]] - endpoint.alignedWindows = Mock(side_effect=[ window1, window2 ]) + window1 = [ + [(StatPointProto(time=1, min=2, mean=3, max=4, count=5, stddev=6),), 11] + ] + window2 = [ + [(StatPointProto(time=2, min=3, mean=4, max=5, count=6, stddev=7),), 12] + ] + endpoint.alignedWindows = Mock(side_effect=[window1, window2]) - uu1 = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') - uu2 = uuid.UUID('5d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu1 = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + uu2 = uuid.UUID("5d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") s1 = Stream(btrdb=BTrDB(endpoint), uuid=uu1) s2 = Stream(btrdb=BTrDB(endpoint), uuid=uu2) versions = {uu1: 11, uu2: 12} @@ -1663,12 +1772,16 @@ def test_aligned_windows_rows_and_calls_to_endpoint(self): start, end, pointwidth = 1, 100, 25 streams = StreamSet([s1, s2]) streams.pin_versions(versions) - rows = streams.filter(start=start, end=end).aligned_windows(pointwidth=pointwidth).rows() + rows = ( + streams.filter(start=start, end=end) + .aligned_windows(pointwidth=pointwidth) + .rows() + ) # assert endpoint calls have correct arguments, version expected = [ call(uu1, start, end, pointwidth, versions[uu1]), - call(uu2, start, end, pointwidth, versions[uu2]) + call(uu2, start, end, pointwidth, versions[uu2]), ] assert endpoint.alignedWindows.call_args_list == expected @@ -1679,8 +1792,6 @@ def test_aligned_windows_rows_and_calls_to_endpoint(self): ] assert rows == expected - - ########################################################################## ## rows tests ########################################################################## @@ -1689,13 +1800,21 @@ def test_rows(self, stream1, stream2): """ Assert rows returns correct values """ - stream1.values = Mock(return_value=iter([ - (RawPoint(time=1, value=1), 1), (RawPoint(time=2, value=2), 1), - (RawPoint(time=3, value=3), 1), (RawPoint(time=4, value=4), 1), - ])) - stream2.values = Mock(return_value=iter([ - (RawPoint(time=1, value=1), 2), (RawPoint(time=3, value=3), 2) - ])) + stream1.values = Mock( + return_value=iter( + [ + (RawPoint(time=1, value=1), 1), + (RawPoint(time=2, value=2), 1), + (RawPoint(time=3, value=3), 1), + (RawPoint(time=4, value=4), 1), + ] + ) + ) + stream2.values = Mock( + return_value=iter( + [(RawPoint(time=1, value=1), 2), (RawPoint(time=3, value=3), 2)] + ) + ) streams = StreamSet([stream1, stream2]) rows = iter(streams.rows()) @@ -1705,7 +1824,6 @@ def test_rows(self, stream1, stream2): assert next(rows) == (RawPoint(time=3, value=3), RawPoint(time=3, value=3)) assert next(rows) == (RawPoint(time=4, value=4), None) - ########################################################################## ## _params_from_filters tests ########################################################################## @@ -1723,7 +1841,6 @@ def test_params_from_filters_latest_value_wins(self, stream1): streams = streams.filter(start=2) assert streams._params_from_filters() == {"start": 2} - def test_params_from_filters_works(self, stream1): """ Assert _params_from_filters returns correct values @@ -1737,7 +1854,6 @@ def test_params_from_filters_works(self, stream1): streams = streams.filter(start=9, end=10) assert streams._params_from_filters() == {"start": 9, "end": 10} - ########################################################################## ## values tests ########################################################################## @@ -1747,13 +1863,16 @@ def test_values(self, stream1, stream2): Assert values returns correct data """ stream1_values = [ - (RawPoint(time=1, value=1), 1), (RawPoint(time=2, value=2), 1), - (RawPoint(time=3, value=3), 1), (RawPoint(time=4, value=4), 1), + (RawPoint(time=1, value=1), 1), + (RawPoint(time=2, value=2), 1), + (RawPoint(time=3, value=3), 1), + (RawPoint(time=4, value=4), 1), ] stream1.values = Mock(return_value=iter(stream1_values)) stream2_values = [ - (RawPoint(time=1, value=1), 2), (RawPoint(time=3, value=3), 2) + (RawPoint(time=1, value=1), 2), + (RawPoint(time=3, value=3), 2), ] stream2.values = Mock(return_value=iter(stream2_values)) @@ -1768,20 +1887,20 @@ def test_values(self, stream1, stream2): ## StreamFilter Tests ########################################################################## -class TestStreamFilter(object): +class TestStreamFilter(object): def test_create(self): """ Assert we can create the object """ - StreamFilter(0,1) + StreamFilter(0, 1) def test_start_larger_or_equal(self): """ Assert we raise ValueError if start is greater than/equal to end """ with pytest.raises(ValueError): - StreamFilter(1,1) + StreamFilter(1, 1) def test_start_valid(self): """ diff --git a/tests/btrdb/test_transformers.py b/tests/btrdb/test_transformers.py index 5cdc0a5..c7ce719 100644 --- a/tests/btrdb/test_transformers.py +++ b/tests/btrdb/test_transformers.py @@ -16,21 +16,23 @@ ########################################################################## import os -from io import StringIO, BytesIO - -import pytest +from io import BytesIO, StringIO from unittest.mock import Mock, PropertyMock + import numpy as np -from pandas import Series, DataFrame, Index +import pytest +from pandas import DataFrame, Index, Series -from btrdb.stream import Stream, StreamSet +from btrdb.conn import BTrDB from btrdb.point import RawPoint, StatPoint +from btrdb.stream import Stream, StreamSet from btrdb.transformers import * ########################################################################## ## Transformer Tests ########################################################################## + @pytest.fixture def streamset(): # list of mock streams @@ -39,26 +41,110 @@ def streamset(): stream = Mock(Stream) type(stream).collection = PropertyMock(return_value="test") type(stream).name = PropertyMock(return_value="stream{}".format(idx)) + stream._btrdb = Mock() + stream._btrdb._ARROW_ENABLED = Mock(return_value=False) streams.append(stream) rows = [ - (None, RawPoint(1500000000000000000, 1.0), RawPoint(1500000000000000000, 1.0), RawPoint(1500000000000000000, 1.0)), - (RawPoint(1500000000100000000, 2.0), None, RawPoint(1500000000100000000, 2.0), RawPoint(1500000000100000000, 2.0)), - (None, RawPoint(1500000000200000000, 3.0), None, RawPoint(1500000000200000000, 3.0)), - (RawPoint(1500000000300000000, 4.0), None, RawPoint(1500000000300000000, 4.0), RawPoint(1500000000300000000, 4.0)), - (None, RawPoint(1500000000400000000, 5.0), RawPoint(1500000000400000000, 5.0), RawPoint(1500000000400000000, 5.0)), - (RawPoint(1500000000500000000, 6.0), None, None, RawPoint(1500000000500000000, 6.0)), - (None, RawPoint(1500000000600000000, 7.0), RawPoint(1500000000600000000, 7.0), RawPoint(1500000000600000000, 7.0)), - (RawPoint(1500000000700000000, 8.0), None, RawPoint(1500000000700000000, 8.0), RawPoint(1500000000700000000, 8.0)), - (None, RawPoint(1500000000800000000, 9.0), RawPoint(1500000000800000000, 9.0), RawPoint(1500000000800000000, 9.0)), - (RawPoint(1500000000900000000, 10.0), None, RawPoint(1500000000900000000, 10.0), RawPoint(1500000000900000000, 10.0) ), + ( + None, + RawPoint(1500000000000000000, 1.0), + RawPoint(1500000000000000000, 1.0), + RawPoint(1500000000000000000, 1.0), + ), + ( + RawPoint(1500000000100000000, 2.0), + None, + RawPoint(1500000000100000000, 2.0), + RawPoint(1500000000100000000, 2.0), + ), + ( + None, + RawPoint(1500000000200000000, 3.0), + None, + RawPoint(1500000000200000000, 3.0), + ), + ( + RawPoint(1500000000300000000, 4.0), + None, + RawPoint(1500000000300000000, 4.0), + RawPoint(1500000000300000000, 4.0), + ), + ( + None, + RawPoint(1500000000400000000, 5.0), + RawPoint(1500000000400000000, 5.0), + RawPoint(1500000000400000000, 5.0), + ), + ( + RawPoint(1500000000500000000, 6.0), + None, + None, + RawPoint(1500000000500000000, 6.0), + ), + ( + None, + RawPoint(1500000000600000000, 7.0), + RawPoint(1500000000600000000, 7.0), + RawPoint(1500000000600000000, 7.0), + ), + ( + RawPoint(1500000000700000000, 8.0), + None, + RawPoint(1500000000700000000, 8.0), + RawPoint(1500000000700000000, 8.0), + ), + ( + None, + RawPoint(1500000000800000000, 9.0), + RawPoint(1500000000800000000, 9.0), + RawPoint(1500000000800000000, 9.0), + ), + ( + RawPoint(1500000000900000000, 10.0), + None, + RawPoint(1500000000900000000, 10.0), + RawPoint(1500000000900000000, 10.0), + ), ] values = [ - [RawPoint(1500000000100000000, 2.0),RawPoint(1500000000300000000, 4.0),RawPoint(1500000000500000000, 6.0),RawPoint(1500000000700000000, 8.0),RawPoint(1500000000900000000, 10.0)], - [RawPoint(1500000000000000000, 1.0),RawPoint(1500000000200000000, 3.0),RawPoint(1500000000400000000, 5.0),RawPoint(1500000000600000000, 7.0),RawPoint(1500000000800000000, 9.0)], - [RawPoint(1500000000000000000, 1.0),RawPoint(1500000000100000000, 2.0),RawPoint(1500000000300000000, 4.0),RawPoint(1500000000400000000, 5.0),RawPoint(1500000000600000000, 7.0),RawPoint(1500000000700000000, 8.0),RawPoint(1500000000800000000, 9.0),RawPoint(1500000000900000000, 10.0)], - [RawPoint(1500000000000000000, 1.0),RawPoint(1500000000100000000, 2.0),RawPoint(1500000000200000000, 3.0),RawPoint(1500000000300000000, 4.0),RawPoint(1500000000400000000, 5.0),RawPoint(1500000000500000000, 6.0),RawPoint(1500000000600000000, 7.0),RawPoint(1500000000700000000, 8.0),RawPoint(1500000000800000000, 9.0),RawPoint(1500000000900000000, 10.0)] + [ + RawPoint(1500000000100000000, 2.0), + RawPoint(1500000000300000000, 4.0), + RawPoint(1500000000500000000, 6.0), + RawPoint(1500000000700000000, 8.0), + RawPoint(1500000000900000000, 10.0), + ], + [ + RawPoint(1500000000000000000, 1.0), + RawPoint(1500000000200000000, 3.0), + RawPoint(1500000000400000000, 5.0), + RawPoint(1500000000600000000, 7.0), + RawPoint(1500000000800000000, 9.0), + ], + [ + RawPoint(1500000000000000000, 1.0), + RawPoint(1500000000100000000, 2.0), + RawPoint(1500000000300000000, 4.0), + RawPoint(1500000000400000000, 5.0), + RawPoint(1500000000600000000, 7.0), + RawPoint(1500000000700000000, 8.0), + RawPoint(1500000000800000000, 9.0), + RawPoint(1500000000900000000, 10.0), + ], + [ + RawPoint(1500000000000000000, 1.0), + RawPoint(1500000000100000000, 2.0), + RawPoint(1500000000200000000, 3.0), + RawPoint(1500000000300000000, 4.0), + RawPoint(1500000000400000000, 5.0), + RawPoint(1500000000500000000, 6.0), + RawPoint(1500000000600000000, 7.0), + RawPoint(1500000000700000000, 8.0), + RawPoint(1500000000800000000, 9.0), + RawPoint(1500000000900000000, 10.0), + ], ] obj = StreamSet(streams) @@ -70,22 +156,63 @@ def streamset(): @pytest.fixture def statpoint_streamset(): rows = [ - [StatPoint(1500000000100000000, 0, 2.0, 2.5, 10, 1),StatPoint(1500000000100000000, 0, 4.0, 4.5, 11, 1),StatPoint(1500000000100000000, 0, 6.0, 6.5, 10, 1),StatPoint(1500000000100000000, 0, 8.0, 8.5, 11, 2)], - [StatPoint(1500000000300000000, 0, 3.0, 3.5, 11, 1),StatPoint(1500000000300000000, 0, 5.0, 5.5, 10, 1),StatPoint(1500000000300000000, 0, 7.0, 7.5, 10, 1),StatPoint(1500000000300000000, 0, 9.0, 9.5, 11, 2)], - [StatPoint(1500000000500000000, 0, 4.0, 4.5, 10, 1),StatPoint(1500000000500000000, 0, 6.0, 6.5, 11, 1),StatPoint(1500000000500000000, 0, 8.0, 8.5, 10, 1),StatPoint(1500000000500000000, 0, 10.0, 10.5, 11, 2)], - [StatPoint(1500000000700000000, 0, 5.0, 5.5, 11, 1),StatPoint(1500000000700000000, 0, 7.0, 7.5, 10, 1),StatPoint(1500000000700000000, 0, 9.0, 9.5, 10, 1),StatPoint(1500000000700000000, 0, 11.0, 11.5, 11, 2)], + [ + StatPoint(1500000000100000000, 0, 2.0, 2.5, 10, 1), + StatPoint(1500000000100000000, 0, 4.0, 4.5, 11, 1), + StatPoint(1500000000100000000, 0, 6.0, 6.5, 10, 1), + StatPoint(1500000000100000000, 0, 8.0, 8.5, 11, 2), + ], + [ + StatPoint(1500000000300000000, 0, 3.0, 3.5, 11, 1), + StatPoint(1500000000300000000, 0, 5.0, 5.5, 10, 1), + StatPoint(1500000000300000000, 0, 7.0, 7.5, 10, 1), + StatPoint(1500000000300000000, 0, 9.0, 9.5, 11, 2), + ], + [ + StatPoint(1500000000500000000, 0, 4.0, 4.5, 10, 1), + StatPoint(1500000000500000000, 0, 6.0, 6.5, 11, 1), + StatPoint(1500000000500000000, 0, 8.0, 8.5, 10, 1), + StatPoint(1500000000500000000, 0, 10.0, 10.5, 11, 2), + ], + [ + StatPoint(1500000000700000000, 0, 5.0, 5.5, 11, 1), + StatPoint(1500000000700000000, 0, 7.0, 7.5, 10, 1), + StatPoint(1500000000700000000, 0, 9.0, 9.5, 10, 1), + StatPoint(1500000000700000000, 0, 11.0, 11.5, 11, 2), + ], ] values = [ - [StatPoint(1500000000100000000, 0, 2.0, 2.5, 10, 1),StatPoint(1500000000300000000, 0, 4.0, 4.5, 11, 1),StatPoint(1500000000500000000, 0, 6.0, 6.5, 10, 1),StatPoint(1500000000700000000, 0, 8.0, 8.5, 11, 2)], - [StatPoint(1500000000100000000, 1, 3.0, 3.5, 11, 1),StatPoint(1500000000300000000, 1, 5.0, 5.5, 10, 1),StatPoint(1500000000500000000, 1, 7.0, 7.5, 10, 1),StatPoint(1500000000700000000, 1, 9.0, 9.5, 11, 2)], - [StatPoint(1500000000100000000, 2, 4.0, 4.5, 10, 1),StatPoint(1500000000300000000, 2, 6.0, 6.5, 11, 1),StatPoint(1500000000500000000, 2, 8.0, 8.5, 10, 1),StatPoint(1500000000700000000, 2, 10.0, 10.5, 11, 2)], - [StatPoint(1500000000100000000, 3, 5.0, 5.5, 11, 1),StatPoint(1500000000300000000, 3, 7.0, 7.5, 10, 1),StatPoint(1500000000500000000, 3, 9.0, 9.5, 10, 1),StatPoint(1500000000700000000, 3, 11.0, 11.5, 11, 2)], + [ + StatPoint(1500000000100000000, 0, 2.0, 2.5, 10, 1), + StatPoint(1500000000300000000, 0, 4.0, 4.5, 11, 1), + StatPoint(1500000000500000000, 0, 6.0, 6.5, 10, 1), + StatPoint(1500000000700000000, 0, 8.0, 8.5, 11, 2), + ], + [ + StatPoint(1500000000100000000, 1, 3.0, 3.5, 11, 1), + StatPoint(1500000000300000000, 1, 5.0, 5.5, 10, 1), + StatPoint(1500000000500000000, 1, 7.0, 7.5, 10, 1), + StatPoint(1500000000700000000, 1, 9.0, 9.5, 11, 2), + ], + [ + StatPoint(1500000000100000000, 2, 4.0, 4.5, 10, 1), + StatPoint(1500000000300000000, 2, 6.0, 6.5, 11, 1), + StatPoint(1500000000500000000, 2, 8.0, 8.5, 10, 1), + StatPoint(1500000000700000000, 2, 10.0, 10.5, 11, 2), + ], + [ + StatPoint(1500000000100000000, 3, 5.0, 5.5, 11, 1), + StatPoint(1500000000300000000, 3, 7.0, 7.5, 10, 1), + StatPoint(1500000000500000000, 3, 9.0, 9.5, 10, 1), + StatPoint(1500000000700000000, 3, 11.0, 11.5, 11, 2), + ], ] streams = [] for idx in range(4): stream = Mock(Stream) type(stream).collection = PropertyMock(return_value="test") type(stream).name = PropertyMock(return_value="stream{}".format(idx)) + stream._btrdb = Mock() streams.append(stream) obj = StreamSet(streams) @@ -97,16 +224,76 @@ def statpoint_streamset(): expected = { "to_dict": [ - {'time': 1500000000000000000, 'test/stream0': None, 'test/stream1': 1.0, 'test/stream2': 1.0, 'test/stream3': 1.0}, - {'time': 1500000000100000000, 'test/stream0': 2.0, 'test/stream1': None, 'test/stream2': 2.0, 'test/stream3': 2.0}, - {'time': 1500000000200000000, 'test/stream0': None, 'test/stream1': 3.0, 'test/stream2': None, 'test/stream3': 3.0}, - {'time': 1500000000300000000, 'test/stream0': 4.0, 'test/stream1': None, 'test/stream2': 4.0, 'test/stream3': 4.0}, - {'time': 1500000000400000000, 'test/stream0': None, 'test/stream1': 5.0, 'test/stream2': 5.0, 'test/stream3': 5.0}, - {'time': 1500000000500000000, 'test/stream0': 6.0, 'test/stream1': None, 'test/stream2': None, 'test/stream3': 6.0}, - {'time': 1500000000600000000, 'test/stream0': None, 'test/stream1': 7.0, 'test/stream2': 7.0, 'test/stream3': 7.0}, - {'time': 1500000000700000000, 'test/stream0': 8.0, 'test/stream1': None, 'test/stream2': 8.0, 'test/stream3': 8.0}, - {'time': 1500000000800000000, 'test/stream0': None, 'test/stream1': 9.0, 'test/stream2': 9.0, 'test/stream3': 9.0}, - {'time': 1500000000900000000, 'test/stream0': 10.0, 'test/stream1': None, 'test/stream2': 10.0, 'test/stream3': 10.0} + { + "time": 1500000000000000000, + "test/stream0": None, + "test/stream1": 1.0, + "test/stream2": 1.0, + "test/stream3": 1.0, + }, + { + "time": 1500000000100000000, + "test/stream0": 2.0, + "test/stream1": None, + "test/stream2": 2.0, + "test/stream3": 2.0, + }, + { + "time": 1500000000200000000, + "test/stream0": None, + "test/stream1": 3.0, + "test/stream2": None, + "test/stream3": 3.0, + }, + { + "time": 1500000000300000000, + "test/stream0": 4.0, + "test/stream1": None, + "test/stream2": 4.0, + "test/stream3": 4.0, + }, + { + "time": 1500000000400000000, + "test/stream0": None, + "test/stream1": 5.0, + "test/stream2": 5.0, + "test/stream3": 5.0, + }, + { + "time": 1500000000500000000, + "test/stream0": 6.0, + "test/stream1": None, + "test/stream2": None, + "test/stream3": 6.0, + }, + { + "time": 1500000000600000000, + "test/stream0": None, + "test/stream1": 7.0, + "test/stream2": 7.0, + "test/stream3": 7.0, + }, + { + "time": 1500000000700000000, + "test/stream0": 8.0, + "test/stream1": None, + "test/stream2": 8.0, + "test/stream3": 8.0, + }, + { + "time": 1500000000800000000, + "test/stream0": None, + "test/stream1": 9.0, + "test/stream2": 9.0, + "test/stream3": 9.0, + }, + { + "time": 1500000000900000000, + "test/stream0": 10.0, + "test/stream1": None, + "test/stream2": 10.0, + "test/stream3": 10.0, + }, ], "to_array": [ np.array([2.0, 4.0, 6.0, 8.0, 10.0]), @@ -140,12 +327,10 @@ def statpoint_streamset(): 1500000000800000000,,9.0,9.0,9.0 1500000000900000000,10.0,,10.0,10.0 """, - } class TestTransformers(object): - ########################################################################## ## to_dict Tests ########################################################################## @@ -155,30 +340,174 @@ def test_to_dict(self, streamset): def test_to_dict_statpoint(self, statpoint_streamset): expected = [ - {'time': 1500000000100000000, 'test/stream0': 2.0, 'test/stream1': 4.0, 'test/stream2': 6.0, 'test/stream3': 8.0}, - {'time': 1500000000300000000, 'test/stream0': 3.0, 'test/stream1': 5.0, 'test/stream2': 7.0, 'test/stream3': 9.0}, - {'time': 1500000000500000000, 'test/stream0': 4.0, 'test/stream1': 6.0, 'test/stream2': 8.0, 'test/stream3': 10.0}, - {'time': 1500000000700000000, 'test/stream0': 5.0, 'test/stream1': 7.0, 'test/stream2': 9.0, 'test/stream3': 11.0}, + { + "time": 1500000000100000000, + "test/stream0": 2.0, + "test/stream1": 4.0, + "test/stream2": 6.0, + "test/stream3": 8.0, + }, + { + "time": 1500000000300000000, + "test/stream0": 3.0, + "test/stream1": 5.0, + "test/stream2": 7.0, + "test/stream3": 9.0, + }, + { + "time": 1500000000500000000, + "test/stream0": 4.0, + "test/stream1": 6.0, + "test/stream2": 8.0, + "test/stream3": 10.0, + }, + { + "time": 1500000000700000000, + "test/stream0": 5.0, + "test/stream1": 7.0, + "test/stream2": 9.0, + "test/stream3": 11.0, + }, ] assert to_dict(statpoint_streamset) == expected def test_to_dict_statpoint_agg_count(self, statpoint_streamset): expected = [ - {'time': 1500000000100000000, 'test/stream0': 10, 'test/stream1': 11, 'test/stream2': 10, 'test/stream3': 11}, - {'time': 1500000000300000000, 'test/stream0': 11, 'test/stream1': 10, 'test/stream2': 10, 'test/stream3': 11}, - {'time': 1500000000500000000, 'test/stream0': 10, 'test/stream1': 11, 'test/stream2': 10, 'test/stream3': 11}, - {'time': 1500000000700000000, 'test/stream0': 11, 'test/stream1': 10, 'test/stream2': 10, 'test/stream3': 11}, + { + "time": 1500000000100000000, + "test/stream0": 10, + "test/stream1": 11, + "test/stream2": 10, + "test/stream3": 11, + }, + { + "time": 1500000000300000000, + "test/stream0": 11, + "test/stream1": 10, + "test/stream2": 10, + "test/stream3": 11, + }, + { + "time": 1500000000500000000, + "test/stream0": 10, + "test/stream1": 11, + "test/stream2": 10, + "test/stream3": 11, + }, + { + "time": 1500000000700000000, + "test/stream0": 11, + "test/stream1": 10, + "test/stream2": 10, + "test/stream3": 11, + }, ] - assert to_dict(statpoint_streamset, agg='count') == expected + assert to_dict(statpoint_streamset, agg="count") == expected def test_to_dict_statpoint_agg_all(self, statpoint_streamset): expected = [ - OrderedDict([('time', 1500000000100000000), ('test/stream0-min', 0), ('test/stream0-mean', 2.0), ('test/stream0-max', 2.5), ('test/stream0-count', 10), ('test/stream0-stddev', 1), ('test/stream1-min', 0), ('test/stream1-mean', 4.0), ('test/stream1-max', 4.5), ('test/stream1-count', 11), ('test/stream1-stddev', 1), ('test/stream2-min', 0), ('test/stream2-mean', 6.0), ('test/stream2-max', 6.5), ('test/stream2-count', 10), ('test/stream2-stddev', 1), ('test/stream3-min', 0), ('test/stream3-mean', 8.0), ('test/stream3-max', 8.5), ('test/stream3-count', 11), ('test/stream3-stddev', 2)]), - OrderedDict([('time', 1500000000300000000), ('test/stream0-min', 0), ('test/stream0-mean', 3.0), ('test/stream0-max', 3.5), ('test/stream0-count', 11), ('test/stream0-stddev', 1), ('test/stream1-min', 0), ('test/stream1-mean', 5.0), ('test/stream1-max', 5.5), ('test/stream1-count', 10), ('test/stream1-stddev', 1), ('test/stream2-min', 0), ('test/stream2-mean', 7.0), ('test/stream2-max', 7.5), ('test/stream2-count', 10), ('test/stream2-stddev', 1), ('test/stream3-min', 0), ('test/stream3-mean', 9.0), ('test/stream3-max', 9.5), ('test/stream3-count', 11), ('test/stream3-stddev', 2)]), - OrderedDict([('time', 1500000000500000000), ('test/stream0-min', 0), ('test/stream0-mean', 4.0), ('test/stream0-max', 4.5), ('test/stream0-count', 10), ('test/stream0-stddev', 1), ('test/stream1-min', 0), ('test/stream1-mean', 6.0), ('test/stream1-max', 6.5), ('test/stream1-count', 11), ('test/stream1-stddev', 1), ('test/stream2-min', 0), ('test/stream2-mean', 8.0), ('test/stream2-max', 8.5), ('test/stream2-count', 10), ('test/stream2-stddev', 1), ('test/stream3-min', 0), ('test/stream3-mean', 10.0), ('test/stream3-max', 10.5), ('test/stream3-count', 11), ('test/stream3-stddev', 2)]), - OrderedDict([('time', 1500000000700000000), ('test/stream0-min', 0), ('test/stream0-mean', 5.0), ('test/stream0-max', 5.5), ('test/stream0-count', 11), ('test/stream0-stddev', 1), ('test/stream1-min', 0), ('test/stream1-mean', 7.0), ('test/stream1-max', 7.5), ('test/stream1-count', 10), ('test/stream1-stddev', 1), ('test/stream2-min', 0), ('test/stream2-mean', 9.0), ('test/stream2-max', 9.5), ('test/stream2-count', 10), ('test/stream2-stddev', 1), ('test/stream3-min', 0), ('test/stream3-mean', 11.0), ('test/stream3-max', 11.5), ('test/stream3-count', 11), ('test/stream3-stddev', 2)]) + OrderedDict( + [ + ("time", 1500000000100000000), + ("test/stream0-min", 0), + ("test/stream0-mean", 2.0), + ("test/stream0-max", 2.5), + ("test/stream0-count", 10), + ("test/stream0-stddev", 1), + ("test/stream1-min", 0), + ("test/stream1-mean", 4.0), + ("test/stream1-max", 4.5), + ("test/stream1-count", 11), + ("test/stream1-stddev", 1), + ("test/stream2-min", 0), + ("test/stream2-mean", 6.0), + ("test/stream2-max", 6.5), + ("test/stream2-count", 10), + ("test/stream2-stddev", 1), + ("test/stream3-min", 0), + ("test/stream3-mean", 8.0), + ("test/stream3-max", 8.5), + ("test/stream3-count", 11), + ("test/stream3-stddev", 2), + ] + ), + OrderedDict( + [ + ("time", 1500000000300000000), + ("test/stream0-min", 0), + ("test/stream0-mean", 3.0), + ("test/stream0-max", 3.5), + ("test/stream0-count", 11), + ("test/stream0-stddev", 1), + ("test/stream1-min", 0), + ("test/stream1-mean", 5.0), + ("test/stream1-max", 5.5), + ("test/stream1-count", 10), + ("test/stream1-stddev", 1), + ("test/stream2-min", 0), + ("test/stream2-mean", 7.0), + ("test/stream2-max", 7.5), + ("test/stream2-count", 10), + ("test/stream2-stddev", 1), + ("test/stream3-min", 0), + ("test/stream3-mean", 9.0), + ("test/stream3-max", 9.5), + ("test/stream3-count", 11), + ("test/stream3-stddev", 2), + ] + ), + OrderedDict( + [ + ("time", 1500000000500000000), + ("test/stream0-min", 0), + ("test/stream0-mean", 4.0), + ("test/stream0-max", 4.5), + ("test/stream0-count", 10), + ("test/stream0-stddev", 1), + ("test/stream1-min", 0), + ("test/stream1-mean", 6.0), + ("test/stream1-max", 6.5), + ("test/stream1-count", 11), + ("test/stream1-stddev", 1), + ("test/stream2-min", 0), + ("test/stream2-mean", 8.0), + ("test/stream2-max", 8.5), + ("test/stream2-count", 10), + ("test/stream2-stddev", 1), + ("test/stream3-min", 0), + ("test/stream3-mean", 10.0), + ("test/stream3-max", 10.5), + ("test/stream3-count", 11), + ("test/stream3-stddev", 2), + ] + ), + OrderedDict( + [ + ("time", 1500000000700000000), + ("test/stream0-min", 0), + ("test/stream0-mean", 5.0), + ("test/stream0-max", 5.5), + ("test/stream0-count", 11), + ("test/stream0-stddev", 1), + ("test/stream1-min", 0), + ("test/stream1-mean", 7.0), + ("test/stream1-max", 7.5), + ("test/stream1-count", 10), + ("test/stream1-stddev", 1), + ("test/stream2-min", 0), + ("test/stream2-mean", 9.0), + ("test/stream2-max", 9.5), + ("test/stream2-count", 10), + ("test/stream2-stddev", 1), + ("test/stream3-min", 0), + ("test/stream3-mean", 11.0), + ("test/stream3-max", 11.5), + ("test/stream3-count", 11), + ("test/stream3-stddev", 2), + ] + ), ] - assert to_dict(statpoint_streamset, agg='all') == expected + assert to_dict(statpoint_streamset, agg="all") == expected ########################################################################## ## to_array Tests @@ -205,10 +534,50 @@ def test_to_array_statpoint(self, statpoint_streamset): result = to_array(statpoint_streamset, agg="min") assert result.__class__ == np.ndarray - assert (result[0] == np.array([0.0, 0.0, 0.0, 0.0,])).all() - assert (result[1] == np.array([1.0, 1.0, 1.0, 1.0, ])).all() - assert (result[2] == np.array([2.0, 2.0, 2.0, 2.0, ])).all() - assert (result[3] == np.array([3.0, 3.0, 3.0, 3.0, ])).all() + assert ( + result[0] + == np.array( + [ + 0.0, + 0.0, + 0.0, + 0.0, + ] + ) + ).all() + assert ( + result[1] + == np.array( + [ + 1.0, + 1.0, + 1.0, + 1.0, + ] + ) + ).all() + assert ( + result[2] + == np.array( + [ + 2.0, + 2.0, + 2.0, + 2.0, + ] + ) + ).all() + assert ( + result[3] + == np.array( + [ + 3.0, + 3.0, + 3.0, + 3.0, + ] + ) + ).all() ########################################################################## ## to_series Tests @@ -222,20 +591,20 @@ def test_to_series(self, streamset): expected_names = [s.collection + "/" + s.name for s in streamset] for idx, points in enumerate(streamset.values()): values, times = list(zip(*[(p.value, p.time) for p in points])) - times = Index(times, dtype='datetime64[ns]') + times = Index(times, dtype="datetime64[ns]") expected.append(Series(values, times, name=expected_names[idx])) result = to_series(streamset) for idx in range(4): - assert (result[idx] == expected[idx]).all() # verify data + assert (result[idx] == expected[idx]).all() # verify data assert result[idx].name == expected_names[idx] # verify name def test_to_series_name_lambda(self, streamset): """ - assert to_dateframe uses name lambda + assert to_series uses name lambda """ result = streamset.to_series(name_callable=lambda s: s.name) - assert [s.name for s in result] == ['stream0', 'stream1', 'stream2', 'stream3'] + assert [s.name for s in result] == ["stream0", "stream1", "stream2", "stream3"] def test_to_series_ignores_rawpoint_with_agg(self, streamset): """ @@ -250,19 +619,34 @@ def test_to_series_statpoint(self, statpoint_streamset): """ result = to_series(statpoint_streamset) values = [2.0, 4.0, 6.0, 8.0] - index = [1500000000100000000, 1500000000300000000, 1500000000500000000, 1500000000700000000] - times = Index(index, dtype='datetime64[ns]') + index = [ + 1500000000100000000, + 1500000000300000000, + 1500000000500000000, + 1500000000700000000, + ] + times = Index(index, dtype="datetime64[ns]") assert (result[0] == Series(values, index=times)).all() values = [3.0, 5.0, 7.0, 9.0] - index = [1500000000100000000, 1500000000300000000, 1500000000500000000, 1500000000700000000] - times = Index(index, dtype='datetime64[ns]') + index = [ + 1500000000100000000, + 1500000000300000000, + 1500000000500000000, + 1500000000700000000, + ] + times = Index(index, dtype="datetime64[ns]") assert (result[1] == Series(values, index=times)).all() result = to_series(statpoint_streamset, agg="max") values = [2.5, 4.5, 6.5, 8.5] - index = [1500000000100000000, 1500000000300000000, 1500000000500000000, 1500000000700000000] - times = Index(index, dtype='datetime64[ns]') + index = [ + 1500000000100000000, + 1500000000300000000, + 1500000000500000000, + 1500000000700000000, + ] + times = Index(index, dtype="datetime64[ns]") assert (result[0] == Series(values, index=times)).all() def test_to_series_index_type(self, streamset): @@ -271,7 +655,7 @@ def test_to_series_index_type(self, streamset): """ result = to_series(streamset) for series in result: - assert series.index.dtype.name == 'datetime64[ns]' + assert series.index.dtype.name == "datetime64[ns]" def test_to_series_index_as_int(self, streamset): """ @@ -279,7 +663,7 @@ def test_to_series_index_as_int(self, streamset): """ result = to_series(streamset, False) for series in result: - assert series.index.dtype.name == 'int64' + assert series.index.dtype.name == "int64" def test_to_series_raises_on_agg_all(self, statpoint_streamset): """ @@ -296,28 +680,27 @@ def test_to_dataframe(self, streamset): """ assert to_dateframe works on RawPoints """ - columns = ["time", "test/stream0", "test/stream1", "test/stream2", "test/stream3"] + columns = [ + "time", + "test/stream0", + "test/stream1", + "test/stream2", + "test/stream3", + ] df = DataFrame(expected["to_dict"], columns=columns) df.set_index("time", inplace=True) assert to_dataframe(streamset).equals(df) - def test_to_dataframe_column_issues_warning(self, statpoint_streamset): + def test_to_dataframe_column_issues_error(self, statpoint_streamset): """ - assert to_dateframe with column argument issues warning + assert to_dateframe with column argument issues error """ columns = ["test/cats", "test/dogs", "test/horses", "test/fishes"] - with pytest.deprecated_call(): + with pytest.raises(TypeError) as unexpected_key_err: statpoint_streamset.to_dataframe(columns=columns) - - def test_to_dataframe_column(self, statpoint_streamset): - """ - assert to_dateframe with column argument actually renames columns - """ - columns = ["test/cats", "test/dogs", "test/horses", "test/fishes"] - with pytest.deprecated_call(): - df = statpoint_streamset.to_dataframe(columns=columns) - - assert df.columns.tolist() == columns + assert "got an unexpected keyword argument 'columns'" in str( + unexpected_key_err.value + ) def test_to_dataframe_multindex(self, statpoint_streamset): """ @@ -325,9 +708,14 @@ def test_to_dataframe_multindex(self, statpoint_streamset): """ df = statpoint_streamset.to_dataframe(agg="all") assert len(df.columns.levels) == 3 - assert list(df.columns.levels[0]) == ['test'] - assert list(df.columns.levels[1]) == ['stream0', 'stream1', 'stream2', 'stream3'] - assert list(df.columns.levels[2]) == ['count', 'max', 'mean', 'min', 'stddev'] + assert list(df.columns.levels[0]) == ["test"] + assert list(df.columns.levels[1]) == [ + "stream0", + "stream1", + "stream2", + "stream3", + ] + assert list(df.columns.levels[2]) == ["count", "max", "mean", "min", "stddev"] def test_to_dataframe_name_lambda(self, streamset): """ @@ -345,8 +733,7 @@ def test_to_dataframe_rawpoint_with_agg(self, streamset): assert to_dateframe ignores agg if RawPoint """ df = streamset.to_dataframe(agg="all") - assert df.shape == (10,4) - + assert df.shape == (10, 4) def test_to_dataframe_statpoints(self, statpoint_streamset): """ @@ -403,8 +790,8 @@ def test_to_csv_as_stringio(self, streamset): # convert back to dicts for assert due to newline issues with CSV result = [dict(row) for row in reader] for item in result: - for k,v in item.items(): - if v is '': + for k, v in item.items(): + if v == "": item[k] = None elif "." in v: item[k] = float(v) @@ -466,4 +853,4 @@ def test_to_table_raises_on_agg_all(self, statpoint_streamset): asserts to_table raises error if using "all" as agg. """ with pytest.raises(AttributeError): - to_table(statpoint_streamset, agg="all") \ No newline at end of file + to_table(statpoint_streamset, agg="all") diff --git a/tests/btrdb/utils/test_buffer.py b/tests/btrdb/utils/test_buffer.py index bd5f626..2786d5b 100644 --- a/tests/btrdb/utils/test_buffer.py +++ b/tests/btrdb/utils/test_buffer.py @@ -16,21 +16,21 @@ ########################################################################## import pytest -from btrdb.utils.buffer import PointBuffer + from btrdb.point import RawPoint +from btrdb.utils.buffer import PointBuffer ########################################################################## ## Test Constants ########################################################################## - ########################################################################## ## Initialization Tests ########################################################################## -class TestPointBuffer(object): +class TestPointBuffer(object): def test_earliest(self): """ Assert earliest returns correct key @@ -106,7 +106,6 @@ def test_is_ready(self): buffer.deactivate(1) assert buffer.is_ready(2000) == True - def test_next_key_ready_with_inactive(self): """ Assert next_key_ready returns correct key with inactive stream @@ -118,7 +117,6 @@ def test_next_key_ready_with_inactive(self): buffer.add_point(0, RawPoint(time=2000, value="leopard")) assert buffer.next_key_ready() == 1000 - # assert first key is 500 even though it was exhausted buffer = PointBuffer(3) buffer.add_point(1, RawPoint(time=500, value="leopard")) diff --git a/tests/btrdb/utils/test_conversions.py b/tests/btrdb/utils/test_conversions.py index 8cc52a1..0f3a963 100644 --- a/tests/btrdb/utils/test_conversions.py +++ b/tests/btrdb/utils/test_conversions.py @@ -15,15 +15,14 @@ ## Imports ########################################################################## -import uuid import json -import pytest -import numpy as np - +import uuid from datetime import datetime -from btrdb.utils.conversion import to_uuid -from btrdb.utils.conversion import AnnotationDecoder, AnnotationEncoder +import numpy as np +import pytest + +from btrdb.utils.conversion import AnnotationDecoder, AnnotationEncoder, to_uuid ########################################################################## ## Test Constants @@ -38,48 +37,56 @@ ## Conversion Tests ########################################################################## -class TestAnnotationJSON(object): - @pytest.mark.parametrize("obj, expected", [ - (True, "true"), - (False, "false"), - (None, "null"), - (3.14, "3.14"), - (42, "42"), - ("foo", "foo"), - ("a long walk on the beach", "a long walk on the beach"), - (['a', 'b', 'c'], '["a", "b", "c"]'), - ({'color': 'red', 'foo': 24}, '{"color": "red", "foo": 24}'), - (datetime(2018, 9, 10, 16, 30), "2018-09-10 16:30:00.000000"), - (np.datetime64(datetime(2018, 9, 10, 16, 30)), "2018-09-10 16:30:00.000000+0000"), - (EXAMPLE_UUID, EXAMPLE_UUID_STR), - ]) +class TestAnnotationJSON(object): + @pytest.mark.parametrize( + "obj, expected", + [ + (True, "true"), + (False, "false"), + (None, "null"), + (3.14, "3.14"), + (42, "42"), + ("foo", "foo"), + ("a long walk on the beach", "a long walk on the beach"), + (["a", "b", "c"], '["a", "b", "c"]'), + ({"color": "red", "foo": 24}, '{"color": "red", "foo": 24}'), + (datetime(2018, 9, 10, 16, 30), "2018-09-10 16:30:00.000000"), + ( + np.datetime64(datetime(2018, 9, 10, 16, 30)), + "2018-09-10 16:30:00.000000+0000", + ), + (EXAMPLE_UUID, EXAMPLE_UUID_STR), + ], + ) def test_annotations_encoder(self, obj, expected): msg = f"did not correctly encode type {type(obj)}" assert json.dumps(obj, cls=AnnotationEncoder, indent=None) == expected, msg - @pytest.mark.parametrize("obj, s", [ - (True, "true"), - (False, "false"), - (None, "null"), - (3.14, "3.14"), - (42, "42"), - ("foo", "foo"), - ("foo", '"foo"'), - ("a long walk on the beach", "a long walk on the beach"), - ("a long walk on the beach", '"a long walk on the beach"'), - (['a', 'b', 'c'], '["a", "b", "c"]'), - ({'color': 'red', 'foo': 24}, '{"color": "red", "foo": 24}'), - ("2018-09-10 16:30:00.000000", "2018-09-10 16:30:00.000000"), - ({'uuid': EXAMPLE_UUID_STR}, f'{{"uuid": "{EXAMPLE_UUID_STR}"}}'), - ]) + @pytest.mark.parametrize( + "obj, s", + [ + (True, "true"), + (False, "false"), + (None, "null"), + (3.14, "3.14"), + (42, "42"), + ("foo", "foo"), + ("foo", '"foo"'), + ("a long walk on the beach", "a long walk on the beach"), + ("a long walk on the beach", '"a long walk on the beach"'), + (["a", "b", "c"], '["a", "b", "c"]'), + ({"color": "red", "foo": 24}, '{"color": "red", "foo": 24}'), + ("2018-09-10 16:30:00.000000", "2018-09-10 16:30:00.000000"), + ({"uuid": EXAMPLE_UUID_STR}, f'{{"uuid": "{EXAMPLE_UUID_STR}"}}'), + ], + ) def test_annotations_decoder(self, obj, s): msg = f"did not correctly decode type {type(obj)}" assert json.loads(s, cls=AnnotationDecoder) == obj, msg class TestToUUID(object): - def test_from_bytes(self): """ Assert that `to_uuid` converts from bytes diff --git a/tests/btrdb/utils/test_credentials.py b/tests/btrdb/utils/test_credentials.py index 6bd057f..7a49018 100644 --- a/tests/btrdb/utils/test_credentials.py +++ b/tests/btrdb/utils/test_credentials.py @@ -15,21 +15,20 @@ ## Imports ########################################################################## -import pytest from unittest.mock import patch +import pytest + from btrdb.exceptions import * from btrdb.utils.credentials import * - ########################################################################## ## Tests ########################################################################## class TestLoadCredentials(object): - - @patch('builtins.open') + @patch("builtins.open") def test_raises_err_if_credentials_not_found(self, mock_open): """ Assert CredentialsFileNotFound is raised if credentials.yaml is not found @@ -40,22 +39,21 @@ def test_raises_err_if_credentials_not_found(self, mock_open): class TestLoadProfile(object): - - def setup(self): + def setup_method(self): for env in ["BTRDB_ENDPOINTS", "BTRDB_PROFILE", "BTRDB_API_KEY"]: try: del os.environ[env] except KeyError: pass - @patch('btrdb.utils.credentials.load_credentials_from_file') + @patch("btrdb.utils.credentials.load_credentials_from_file") def test_raises_err_if_profile_not_found(self, mock_credentials): """ Assert ProfileNotFound is raised if profile is asked for but not found """ mock_credentials.return_value = { "duck": { - "btrdb" : { + "btrdb": { "endpoints": "192.168.1.100:4410", "api_key": "111222333", } @@ -64,7 +62,7 @@ def test_raises_err_if_profile_not_found(self, mock_credentials): with pytest.raises(ProfileNotFound): credentials_by_profile("horse") - @patch('btrdb.utils.credentials.load_credentials_from_file') + @patch("btrdb.utils.credentials.load_credentials_from_file") def test_returns_requested_profile(self, mock_credentials): """ Assert returns correct profile @@ -75,56 +73,56 @@ def test_returns_requested_profile(self, mock_credentials): } assert credentials_by_profile("duck")["endpoints"] == "duck" - @patch('btrdb.utils.credentials.load_credentials_from_file') + @patch("btrdb.utils.credentials.load_credentials_from_file") def test_returns_default_profile(self, mock_credentials): """ Assert default profile is returned """ mock_credentials.return_value = { "duck": { - "btrdb" : { + "btrdb": { "endpoints": "192.168.1.100:4411", "api_key": "111222333", }, - "name": "duck" + "name": "duck", }, "default": { - "btrdb" : { + "btrdb": { "endpoints": "192.168.1.222:4411", "api_key": "333222111", }, - "name": "default" + "name": "default", }, } assert credentials_by_profile()["apikey"] == "333222111" - @patch('btrdb.utils.credentials.load_credentials_from_file') + @patch("btrdb.utils.credentials.load_credentials_from_file") def test_returns_no_default_profile(self, mock_credentials): """ Assert empty credentials are returned if no default profile """ mock_credentials.return_value = { "duck": { - "btrdb" : { + "btrdb": { "endpoints": "192.168.1.100:4411", "api_key": "111222333", }, - "name": "duck" + "name": "duck", }, } assert credentials_by_profile() == {} -class TestCredentials(object): - def setup(self): +class TestCredentials(object): + def setup_method(self): for env in ["BTRDB_ENDPOINTS", "BTRDB_PROFILE", "BTRDB_API_KEY"]: try: del os.environ[env] except KeyError: pass - @patch('btrdb.utils.credentials.load_credentials_from_file') - @patch('os.path.exists') + @patch("btrdb.utils.credentials.load_credentials_from_file") + @patch("os.path.exists") def test_checks_file_existence(self, mock_exists, mock_load): mock_exists.return_value = False credentials() diff --git a/tests/btrdb/utils/test_general.py b/tests/btrdb/utils/test_general.py index 0f09cf5..0edb86e 100644 --- a/tests/btrdb/utils/test_general.py +++ b/tests/btrdb/utils/test_general.py @@ -11,47 +11,81 @@ Testing for the btrdb.utils.general module """ +from datetime import timedelta + import pytest -from datetime import timedelta -from btrdb.utils.timez import ns_delta from btrdb.utils.general import pointwidth +from btrdb.utils.timez import ns_delta class TestPointwidth(object): - - @pytest.mark.parametrize("delta, expected", [ - (timedelta(days=365), 54), - (timedelta(days=30), 51), - (timedelta(days=7), 49), - (timedelta(days=1), 46), - (timedelta(hours=4), 43), - (timedelta(minutes=15), 39), - (timedelta(seconds=30), 34), - ]) - + @pytest.mark.parametrize( + "delta, expected", + [ + (timedelta(days=365), 54), + (timedelta(days=30), 51), + (timedelta(days=7), 49), + (timedelta(days=1), 46), + (timedelta(hours=4), 43), + (timedelta(minutes=15), 39), + (timedelta(seconds=30), 34), + ], + ) def test_from_timedelta(self, delta, expected): """ Test getting the closest point width from a timedelta """ assert pointwidth.from_timedelta(delta) == expected - @pytest.mark.parametrize("nsec, expected", [ - (ns_delta(days=365), 54), - (ns_delta(days=30), 51), - (ns_delta(days=7), 49), - (ns_delta(days=1), 46), - (ns_delta(hours=12), 45), - (ns_delta(minutes=30), 40), - (ns_delta(seconds=1), 29), - ]) - + @pytest.mark.parametrize( + "nsec, expected", + [ + (ns_delta(days=365), 54), + (ns_delta(days=30), 51), + (ns_delta(days=7), 49), + (ns_delta(days=1), 46), + (ns_delta(hours=12), 45), + (ns_delta(minutes=30), 40), + (ns_delta(seconds=1), 29), + ], + ) def test_from_nanoseconds(self, nsec, expected): """ Test getting the closest point width from nanoseconds """ assert pointwidth.from_nanoseconds(nsec) == expected + @pytest.mark.parametrize( + "pw_time_range, expected", + [ + ( + (52, 1560127027000000000, 1702702800000000000), + (1558245471070191616, 1697857059518676992, 32), + ), + ( + (48, 1560127027000000000, 1702702800000000000), + (1559934320930455552, 1702360659146047488, 507), + ), + ( + (44, 1456876800000000000, 1464652800000000000), + (1456861702896222208, 1464619856941809664, 442), + ), + ( + (38, 1456876800000000000, 1464652800000000000), + (1456876546303197184, 1464652292534829056, 28289), + ), + ( + (32, 1456876800000000000, 1464652800000000000), + (1456876799706267648, 1464652795046002688, 1810491), + ), + ], + ) + def test_for_aligned_windows(self, pw_time_range, expected): + pw, start, end = pw_time_range + result = pointwidth(pw).for_aligned_windows(start, end) + assert result == expected + def test_time_conversions(self): """ Test standard pointwidth time conversions @@ -110,3 +144,19 @@ def test_incr(self): Test incrementing a pointwidth """ assert pointwidth(23).incr() == 24 + + @pytest.mark.parametrize( + "pw, expected", + [ + (54, timedelta(days=208, seconds=43198, microseconds=509482)), + (51, timedelta(days=26, seconds=5399, microseconds=813685)), + (49, timedelta(days=6, seconds=44549, microseconds=953421)), + (46, timedelta(seconds=70368, microseconds=744178)), + (43, timedelta(seconds=8796, microseconds=93022)), + (39, timedelta(seconds=549, microseconds=755814)), + (34, timedelta(seconds=17, microseconds=179869)), + (30, timedelta(seconds=1, microseconds=73742)), + ], + ) + def test_to_timedelta(self, pw, expected): + assert pointwidth(pw).to_timedelta() == expected diff --git a/tests/btrdb/utils/test_timez.py b/tests/btrdb/utils/test_timez.py index 45fd3fa..7150fff 100644 --- a/tests/btrdb/utils/test_timez.py +++ b/tests/btrdb/utils/test_timez.py @@ -15,22 +15,27 @@ ## Imports ########################################################################## -import pytz -import pytest import datetime + import numpy as np +import pytest +import pytz from freezegun import freeze_time -from btrdb.utils.timez import (currently_as_ns, datetime_to_ns, ns_to_datetime, - to_nanoseconds, ns_delta) - +from btrdb.utils.timez import ( + currently_as_ns, + datetime_to_ns, + ns_delta, + ns_to_datetime, + to_nanoseconds, +) ########################################################################## ## Initialization Tests ########################################################################## -class TestCurrentlyAsNs(object): +class TestCurrentlyAsNs(object): def test_currently_as_ns(self): """ Assert currently_as_ns returns correct value @@ -41,25 +46,23 @@ def test_currently_as_ns(self): class TestDatetimeToNs(object): - def test_datetime_to_ns_naive(self): """ Assert datetime_to_ns handles naive datetime """ - dt = datetime.datetime(2018,1,1,12) + dt = datetime.datetime(2018, 1, 1, 12) localized = pytz.utc.localize(dt) expected = int(localized.timestamp() * 1e9) assert dt.tzinfo is None assert datetime_to_ns(dt) == expected - def test_datetime_to_ns_aware(self): """ Assert datetime_to_ns handles tz aware datetime """ eastern = pytz.timezone("US/Eastern") - dt = datetime.datetime(2018,1,1,17, tzinfo=eastern) + dt = datetime.datetime(2018, 1, 1, 17, tzinfo=eastern) expected = int(dt.astimezone(pytz.utc).timestamp() * 1e9) assert dt.tzinfo is not None @@ -67,7 +70,6 @@ def test_datetime_to_ns_aware(self): class TestNsToDatetime(object): - def test_ns_to_datetime_is_utc(self): """ Assert ns_to_datetime returns UTC aware datetime @@ -83,18 +85,17 @@ def test_ns_to_datetime_is_correct(self): Assert ns_to_datetime returns correct datetime """ val = 1514808000000000000 - expected = datetime.datetime(2018,1,1,12, tzinfo=pytz.UTC) + expected = datetime.datetime(2018, 1, 1, 12, tzinfo=pytz.UTC) assert ns_to_datetime(val) == expected class TestToNanoseconds(object): - def test_datetime_to_ns_naive(self): """ Assert to_nanoseconds handles naive datetime """ - dt = datetime.datetime(2018,1,1,12) + dt = datetime.datetime(2018, 1, 1, 12) localized = pytz.utc.localize(dt) expected = int(localized.timestamp() * 1e9) @@ -106,7 +107,7 @@ def test_datetime_to_ns_aware(self): Assert to_nanoseconds handles tz aware datetime """ eastern = pytz.timezone("US/Eastern") - dt = datetime.datetime(2018,1,1,17, tzinfo=eastern) + dt = datetime.datetime(2018, 1, 1, 17, tzinfo=eastern) expected = int(dt.astimezone(pytz.utc).timestamp() * 1e9) assert dt.tzinfo is not None @@ -116,7 +117,7 @@ def test_str(self): """ Assert to_nanoseconds handles RFC3339 format """ - dt = datetime.datetime(2018,1,1,12, tzinfo=pytz.utc) + dt = datetime.datetime(2018, 1, 1, 12, tzinfo=pytz.utc) expected = int(dt.timestamp() * 1e9) dt_str = "2018-1-1 12:00:00.0-0000" @@ -124,12 +125,12 @@ def test_str(self): assert to_nanoseconds(dt_str) == expected dt_str = "2018-1-1 7:00:00.0-0500" - dt = datetime.datetime(2018,1,1,12, tzinfo=pytz.timezone("US/Eastern")) + dt = datetime.datetime(2018, 1, 1, 12, tzinfo=pytz.timezone("US/Eastern")) assert dt.tzinfo is not None assert to_nanoseconds(dt_str) == expected dt_str = "2018-01-15 07:32:49" - dt = datetime.datetime(2018,1,15,7,32,49, tzinfo=pytz.utc) + dt = datetime.datetime(2018, 1, 15, 7, 32, 49, tzinfo=pytz.utc) expected = int(dt.timestamp() * 1e9) assert to_nanoseconds(dt_str) == expected @@ -173,20 +174,19 @@ def test_float(self): """ Assert to_nanoseconds handles float """ - dt = datetime.datetime(2018,1,1,12, tzinfo=pytz.utc) + dt = datetime.datetime(2018, 1, 1, 12, tzinfo=pytz.utc) expected = int(dt.timestamp() * 1e9) - dt64 = np.datetime64('2018-01-01T12:00') + dt64 = np.datetime64("2018-01-01T12:00") assert expected == to_nanoseconds(dt64) class TestToNsDelta(object): - def test_ns_delta(self): """ Assert ns_delta converts inputs properly """ - val = ns_delta(1,2,1,3,1,23,1) + val = ns_delta(1, 2, 1, 3, 1, 23, 1) assert val == 93663001023001 def test_ns_delta_precision(self): @@ -200,7 +200,8 @@ def test_returns_int(self): """ Assert ns_delta returns int if floats used for arguments """ - val = ns_delta(1.0,1.0,1.0,1.0,1.0,1.0,1) - assert val == int(1 + 1e3 + 1e6 + 1e9 + (1e9 * 60) + (1e9 * 60 * 60) + \ - (1e9 * 60 * 60 * 24) ) + val = ns_delta(1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1) + assert val == int( + 1 + 1e3 + 1e6 + 1e9 + (1e9 * 60) + (1e9 * 60 * 60) + (1e9 * 60 * 60 * 24) + ) assert isinstance(val, int) diff --git a/tests/btrdb4/test_import.py b/tests/btrdb4/test_import.py index 21c0d6c..4fd170e 100644 --- a/tests/btrdb4/test_import.py +++ b/tests/btrdb4/test_import.py @@ -21,11 +21,11 @@ ## Import Tests ########################################################################## -class TestPackage(object): +class TestPackage(object): def test_import(self): """ Assert that `import brdb4` issues warning exception """ - with pytest.raises(ImportWarning, match='not available'): + with pytest.raises(ImportWarning, match="not available"): import btrdb4 diff --git a/tests/btrdb_integration/__init__.py b/tests/btrdb_integration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/btrdb_integration/conftest.py b/tests/btrdb_integration/conftest.py new file mode 100644 index 0000000..5e3a7f4 --- /dev/null +++ b/tests/btrdb_integration/conftest.py @@ -0,0 +1,68 @@ +import os + +import pyarrow as pa +import pytest + +import btrdb + + +@pytest.fixture +def single_stream_values_arrow_schema(): + schema = pa.schema( + [ + pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False), + pa.field("value", pa.float64(), nullable=False), + ] + ) + return schema + + +@pytest.fixture +def single_stream_windows_all_stats_arrow_schema(): + schema = pa.schema( + [ + pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False), + pa.field("min", pa.float64(), nullable=False), + pa.field("mean", pa.float64(), nullable=False), + pa.field("max", pa.float64(), nullable=False), + pa.field("count", pa.uint64(), nullable=False), + pa.field("stddev", pa.float64(), nullable=False), + ] + ) + return schema + + +def single_stream_windows_mean_stddev_count_stats_arrow_schema(): + schema = ( + pa.schema( + [ + pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False), + pa.field("mean", pa.float64(), nullable=False), + pa.field("count", pa.uint64(), nullable=False), + pa.field("stddev", pa.float64(), nullable=False), + ] + ), + ) + return schema + + +@pytest.fixture +def conn(): + if os.getenv("BTRDB_INTEGRATION_TEST_PROFILE") == None: + pytest.skip("skipping because BTRDB_INTEGRATION_TEST_PROFILE is not set") + conn = btrdb.connect(profile=os.getenv("BTRDB_INTEGRATION_TEST_PROFILE")) + return conn + + +def _delete_collection(conn, col): + streams = conn.streams_in_collection(col) + for stream in streams: + stream.obliterate() + + +@pytest.fixture +def tmp_collection(request, conn): + col = "btrdb_python_integration_tests/" + request.node.name + "/tmp" + _delete_collection(conn, col) + yield col + _delete_collection(conn, col) diff --git a/tests/btrdb_integration/test_conn.py b/tests/btrdb_integration/test_conn.py new file mode 100644 index 0000000..dc6f47a --- /dev/null +++ b/tests/btrdb_integration/test_conn.py @@ -0,0 +1,50 @@ +import logging +import os +from uuid import uuid4 as new_uuid + +import pytest + +import btrdb +from btrdb.exceptions import BTrDBError +from btrdb.utils.credentials import credentials_by_profile + + +@pytest.mark.skipif( + os.getenv("BTRDB_INTEGRATION_TEST_PROFILE") is None, + reason="Need an integration test server to run integration tests.", +) +class TestConnection: + def test_connection_info(self, conn): + info = conn.info() + logging.info(f"connection info: {info}") + + def test_incorrect_connect( + self, + ): + env_name = os.getenv("BTRDB_INTEGRATION_TEST_PROFILE") + creds = credentials_by_profile(env_name) + conn_str = creds["endpoints"] + err_msg = r"""Could not connect to the database, error message: <_InactiveRpcError of RPC that terminated with:\n\tstatus = StatusCode.UNAUTHENTICATED\n\tdetails = "could not validate api key:""" + with pytest.raises(BTrDBError, match=err_msg): + conn = btrdb.connect(conn_str=conn_str, apikey="BOGUS_KEY") + + @pytest.mark.xfail( + reason="Should return BTrDBError, but returns GRPCError instead, FIXME" + ) + def test_from_uuid(self, conn): + # todo, investigate this, the stat code returned is 0, which is why its not being returned as a btrdb error + uu = new_uuid() + stream = conn.stream_from_uuid(uu) + with pytest.raises(btrdb.exceptions.BTrDBError): + print(stream) + + def test_create_stream(self, conn, tmp_collection): + uuid = new_uuid() + stream = conn.create(uuid, tmp_collection, tags={"name": "s"}) + assert stream.uuid == uuid + assert stream.name == "s" + + def test_list_collections(self, conn, tmp_collection): + assert tmp_collection not in conn.list_collections() + stream = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) + assert tmp_collection in conn.list_collections() diff --git a/tests/btrdb_integration/test_regressions.py b/tests/btrdb_integration/test_regressions.py new file mode 100644 index 0000000..db6fd2d --- /dev/null +++ b/tests/btrdb_integration/test_regressions.py @@ -0,0 +1,26 @@ +from uuid import uuid4 as new_uuid + +import pytest + + +@pytest.mark.xfail +def test_create_count_obliterate_concurrency_bug(conn, tmp_collection): + from concurrent.futures import ThreadPoolExecutor + + n_streams = 10 + + def create_stream(i): + return conn.create(new_uuid(), tmp_collection, tags={"name": f"s{i}"}) + + def points_in_stream(s): + return s.count() + + def obliterate_stream(s): + s.obliterate() + + with ThreadPoolExecutor() as executor: + for i in range(2): + pmap = lambda f, it: list(executor.map(f, it, timeout=30)) + streams = pmap(create_stream, range(n_streams)) + assert sum(pmap(points_in_stream, streams)) == 0 + pmap(obliterate_stream, streams) diff --git a/tests/btrdb_integration/test_stream.py b/tests/btrdb_integration/test_stream.py new file mode 100644 index 0000000..5fba2a1 --- /dev/null +++ b/tests/btrdb_integration/test_stream.py @@ -0,0 +1,368 @@ +import logging +from uuid import uuid4 as new_uuid + +import numpy as np +import pandas as pd +import pytest + +import btrdb.utils.timez +from btrdb.utils.timez import currently_as_ns, ns_delta + +try: + import pyarrow as pa +except ImportError: + pa = None + + +def test_stream_methods(conn, tmp_collection): + s1 = conn.create( + new_uuid(), + tmp_collection, + tags={"name": "s1", "unit": "foo"}, + annotations={"test": "bar"}, + ) + + +def test_grpc_insert_and_values(conn, tmp_collection): + s = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) + t = currently_as_ns() + data = [] + duration = 100 + for i in range(duration): + data.append([t + i, i]) + s.insert(data) + fetched_data = s.values(start=t, end=t + duration) + assert len(fetched_data) == len(data) + for i, (p, _) in enumerate(fetched_data): + assert p.time == data[i][0] + assert p.value == data[i][1] + + +def test_arrow_insert_and_values( + conn, tmp_collection, single_stream_values_arrow_schema +): + s = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) + t = currently_as_ns() + times = [] + values = [] + duration = 100 + for i in range(duration): + times.append(t + i) + values.append(i) + data = pa.Table.from_arrays( + [ + pa.array(times), + pa.array(values), + ], + schema=single_stream_values_arrow_schema, + ) + s.arrow_insert(data) + fetched_data = s.arrow_values(start=t, end=t + duration) + assert data == fetched_data + + +def test_arrow_values_template_schema(conn, tmp_collection): + s = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) + t = currently_as_ns() + times = [100, 200, 300, 400] + values = [1.0, 2.0, 3.0, 4.0] + s.insert(list(zip(times, values))) + schema = pa.schema( + [ + pa.field("t", pa.int64(), nullable=False), + pa.field("v", pa.float32(), nullable=False), + ] + ) + expected = pa.Table.from_arrays([pa.array(times), pa.array(values)], schema=schema) + fetched_data = s.arrow_values(start=times[0], end=times[-1] + 1, schema=schema) + assert expected == fetched_data + assert expected.schema.equals(fetched_data.schema) + + +@pytest.mark.parametrize( + "merge_policy,duplicates_expected", + [("never", True), ("equal", True), ("retain", False), ("replace", False)], +) +def test_arrow_insert_duplicate_timestamps( + conn, + tmp_collection, + single_stream_values_arrow_schema, + merge_policy, + duplicates_expected, +): + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + t1 = [100, 105, 110, 110, 120] + d1 = [0.0, 1.0, 2.0, 3.0, 4.0] + s1.insert(list(zip(t1, d1)), merge=merge_policy) + df = s1.arrow_values(start=100, end=121).to_pandas().set_index("time") + assert any(df.index.duplicated().tolist()) == duplicates_expected + + +def test_arrow_values_table_schema( + conn, tmp_collection, single_stream_values_arrow_schema +): + s = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) + t = currently_as_ns() + times = [] + values = [] + duration = 100 + for i in range(duration): + times.append(t + i) + values.append(i) + data = pa.Table.from_arrays( + [ + pa.array(times), + pa.array(values), + ], + schema=single_stream_values_arrow_schema, + ) + s.arrow_insert(data) + fetched_data = s.arrow_values(start=t, end=t + duration) + assert single_stream_values_arrow_schema.equals(fetched_data.schema) + + +def test_arrow_values_table_schema_2( + conn, tmp_collection, single_stream_values_arrow_schema +): + s = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) + t = currently_as_ns() + times = [] + values = [] + duration = 100 + for i in range(duration): + times.append(t + i) + values.append(i) + data = pa.Table.from_arrays( + [ + pa.array(times), + pa.array(values), + ], + schema=single_stream_values_arrow_schema, + ) + s.arrow_insert(data) + fetched_data = s.arrow_values(start=t, end=t + duration) + assert single_stream_values_arrow_schema.equals(fetched_data.schema) + + +def test_arrow_windows( + conn, + tmp_collection, + single_stream_windows_all_stats_arrow_schema, + single_stream_values_arrow_schema, +): + rng = np.random.default_rng(seed=7) + s = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) + t = currently_as_ns() + times = [] + values = [] + duration = btrdb.utils.timez.ns_delta(minutes=1) + nvals = 100 + for i in range(nvals): + times.append(t + i * duration) + values.append(i + rng.random()) + data = pa.Table.from_arrays( + [ + pa.array(times), + pa.array(values), + ], + schema=single_stream_values_arrow_schema, + ) + # group these 5 at a time to go with the 5 min windows from below + mean_dat = np.mean(np.asarray(values).reshape(-1, 5), axis=1) + min_dat = np.min(np.asarray(values).reshape(-1, 5), axis=1) + max_dat = np.max(np.asarray(values).reshape(-1, 5), axis=1) + count_dat = [np.asarray(values).reshape(-1, 5).shape[1]] * int( + np.asarray(values).shape[0] / 5 + ) + stddev_dat = np.std(np.asarray(values).reshape(-1, 5), axis=1) + time_dat = [t for i, t in enumerate(times) if i % 5 == 0] + window_table = pa.Table.from_arrays( + [time_dat, min_dat, mean_dat, max_dat, count_dat, stddev_dat], + schema=single_stream_windows_all_stats_arrow_schema, + ) + s.arrow_insert(data) + # 5 min windows + fetched_data = s.arrow_windows( + start=t, end=t + duration * nvals + 1, width=duration * 5 + ) + fetched_df = fetched_data.to_pandas() + fetched_df["time"] = fetched_df["time"].astype(int) + window_df = window_table.to_pandas() + window_df["time"] = window_df["time"].astype(int) + assert np.allclose(fetched_df.values, window_df.values, rtol=0, atol=1e-9) + + +@pytest.mark.xfail +def test_aligned_windows_no_flush( + conn, + tmp_collection, + single_stream_windows_all_stats_arrow_schema, + single_stream_values_arrow_schema, +): + rng = np.random.default_rng(seed=7) + s = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) + t = currently_as_ns() + times = [] + values = [] + duration = btrdb.utils.timez.ns_delta(minutes=1) + nvals = 100 + for i in range(nvals): + times.append(t + i * duration) + values.append(i + rng.random()) + data = pa.Table.from_arrays( + [ + pa.array(times), + pa.array(values), + ], + schema=single_stream_values_arrow_schema, + ) + s.arrow_insert(data) + fetched_data = s.arrow_aligned_windows( + start=t, + end=t + duration * nvals, + pointwidth=btrdb.utils.general.pointwidth.from_nanoseconds(duration * 6), + ) + fetched_df = fetched_data.to_pandas() + assert fetched_df["stddev"].isna().sum() == 0 + + +def test_arrow_aligned_windows_vs_aligned_windows( + conn, + tmp_collection, + single_stream_windows_all_stats_arrow_schema, + single_stream_values_arrow_schema, +): + rng = np.random.default_rng(seed=7) + s = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) + t = currently_as_ns() + times = [] + values = [] + duration = btrdb.utils.timez.ns_delta(minutes=1) + nvals = 100 + for i in range(nvals): + times.append(t + i * duration) + values.append(i + rng.random()) + data = pa.Table.from_arrays( + [ + pa.array(times), + pa.array(values), + ], + schema=single_stream_values_arrow_schema, + ) + s.arrow_insert(data) + s.flush() + fetched_data = s.arrow_aligned_windows( + start=t, + end=t + duration * nvals, + pointwidth=btrdb.utils.general.pointwidth.from_nanoseconds(duration * 6), + ) + fetched_df = fetched_data.to_pandas() + fetched_df["time"] = fetched_df["time"].astype(int) + fetched_df["count"] = fetched_df["count"].astype(int) + + other_method = s.aligned_windows( + start=t, + end=t + duration * nvals, + pointwidth=btrdb.utils.general.pointwidth.from_nanoseconds(duration * 6), + ) + other_method_data = [] + for statpt, ver in other_method: + tmp = dict() + tmp["time"] = statpt.time + tmp["min"] = statpt.min + tmp["mean"] = statpt.mean + tmp["max"] = statpt.max + tmp["count"] = statpt.count + tmp["stddev"] = statpt.stddev + other_method_data.append(tmp) + other_method_df = pd.DataFrame(other_method_data) + assert fetched_data.schema.equals(single_stream_windows_all_stats_arrow_schema) + assert fetched_df.equals(other_method_df) + + +def compare_arrow_windows_vs_windows( + conn, + tmp_collection, + single_stream_windows_all_stats_arrow_schema, + single_stream_values_arrow_schema, +): + rng = np.random.default_rng(seed=7) + s = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) + t = currently_as_ns() + times = [] + values = [] + duration = btrdb.utils.timez.ns_delta(minutes=1) + nvals = 100 + for i in range(nvals): + times.append(t + i * duration) + values.append(i + rng.random()) + data = pa.Table.from_arrays( + [ + pa.array(times), + pa.array(values), + ], + schema=single_stream_values_arrow_schema, + ) + s.arrow_insert(data) + fetched_data = s.arrow_windows( + start=t, + end=t + duration * nvals, + width=duration * 5, + ) + fetched_df = fetched_data.to_pandas() + fetched_df["time"] = fetched_df["time"].astype(int) + fetched_df["count"] = fetched_df["count"].astype(int) + + other_method = s.windows(start=t, end=t + duration * nvals, width=duration * 5) + other_method_data = [] + for statpt, ver in other_method: + tmp = dict() + tmp["time"] = statpt.time + tmp["min"] = statpt.min + tmp["mean"] = statpt.mean + tmp["max"] = statpt.max + tmp["count"] = statpt.count + tmp["stddev"] = statpt.stddev + other_method_data.append(tmp) + other_method_df = pd.DataFrame(other_method_data) + assert fetched_data.schema.equals(single_stream_windows_all_stats_arrow_schema) + assert fetched_df.equals(other_method_df) + + +def test_arrow_empty_values(conn, tmp_collection): + s = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) + data = s.arrow_values(start=100, end=1000) + assert len(data["time"]) == 0 + assert len(data["value"]) == 0 + + +def test_arrow_empty_values_schema(conn, tmp_collection): + s = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) + data = s.arrow_values(start=100, end=1000) + schema = pa.schema( + [ + pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False), + pa.field("value", pa.float64(), nullable=False), + ] + ) + assert schema.equals(data.schema) + + +def test_stream_annotation_update(conn, tmp_collection): + s = conn.create( + new_uuid(), + tmp_collection + "foo/", + tags={"name": "s"}, + annotations={"foo": "bar"}, + ) + annotations1, version1 = s.annotations() + assert version1 == 1 + assert annotations1["foo"] == "bar" + s.update(annotations={"foo": "baz"}) + annotations2, version2 = s.annotations() + assert version2 > version1 + assert annotations2["foo"] == "baz" + s.update(annotations={}, replace=True) + annotations3, version3 = s.annotations() + assert len(annotations3) == 0 + assert version3 > version2 diff --git a/tests/btrdb_integration/test_streamset.py b/tests/btrdb_integration/test_streamset.py new file mode 100644 index 0000000..0029700 --- /dev/null +++ b/tests/btrdb_integration/test_streamset.py @@ -0,0 +1,558 @@ +import random +from math import isnan, nan +from uuid import uuid4 as new_uuid + +import numpy as np +import pytest +from numpy import testing as np_test + +import btrdb +import btrdb.stream +from btrdb.utils.timez import currently_as_ns + +try: + import pyarrow as pa +except ImportError: + pa = None +try: + import pandas as pd +except ImportError: + pd = None +try: + import polars as pl + from polars import testing as pl_test +except ImportError: + pl = None + + +def test_streamset_values(conn, tmp_collection): + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + t1 = [100, 105, 110, 115, 120] + t2 = [101, 106, 110, 114, 119] + d1 = [0.0, 1.0, 2.0, 3.0, 4.0] + d2 = [5.0, 6.0, 7.0, 8.0, 9.0] + s1.insert(list(zip(t1, d1))) + s2.insert(list(zip(t2, d2))) + ss = btrdb.stream.StreamSet([s1, s2]).filter(start=100, end=121) + values = ss.values() + assert len(values) == 2 + assert len(values[0]) == len(t1) + assert len(values[1]) == len(t2) + assert [(p.time, p.value) for p in values[0]] == list(zip(t1, d1)) + assert [(p.time, p.value) for p in values[1]] == list(zip(t2, d2)) + + +def test_streamset_arrow_values(conn, tmp_collection): + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + t1 = [100, 105, 110, 115, 120] + t2 = [101, 106, 110, 114, 119] + d1 = [0.0, 1.0, 2.0, 3.0, 4.0] + d2 = [5.0, 6.0, 7.0, 8.0, 9.0] + s1.insert(list(zip(t1, d1))) + s2.insert(list(zip(t2, d2))) + ss = btrdb.stream.StreamSet([s1, s2]).filter(start=100, end=121) + expected_times = [100, 101, 105, 106, 110, 114, 115, 119, 120] + expected_col1 = [0.0, np.nan, 1.0, np.nan, 2.0, np.nan, 3.0, np.nan, 4.0] + expected_col2 = [np.nan, 5.0, np.nan, 6.0, 7.0, 8.0, np.nan, 9.0, np.nan] + expected_schema = pa.schema( + [ + pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False), + pa.field(str(s1.uuid), pa.float64(), nullable=False), + pa.field(str(s2.uuid), pa.float64(), nullable=False), + ] + ) + values = ss.arrow_values() + times = [t.value for t in values["time"]] + col1 = [np.nan if isnan(v.as_py()) else v.as_py() for v in values[str(s1.uuid)]] + col2 = [np.nan if isnan(v.as_py()) else v.as_py() for v in values[str(s2.uuid)]] + assert times == expected_times + assert col1 == expected_col1 + assert col2 == expected_col2 + assert expected_schema.equals(values.schema) + + +def test_streamset_template_schema(conn, tmp_collection): + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + t1 = [100, 105, 110, 115, 120] + t2 = [101, 106, 110, 114, 119] + d1 = [0.0, 1.0, 2.0, 3.0, 4.0] + d2 = [5.0, 6.0, 7.0, 8.0, 9.0] + s1.insert(list(zip(t1, d1))) + s2.insert(list(zip(t2, d2))) + schema = pa.schema( + [ + pa.field("t", pa.timestamp("ns", tz="UTC"), nullable=False), + pa.field("a", pa.float64(), nullable=False), + pa.field("b", pa.float32(), nullable=False), + ] + ) + ss = btrdb.stream.StreamSet([s1, s2]).filter(start=100, end=121, schema=schema) + expected_times = [100, 101, 105, 106, 110, 114, 115, 119, 120] + expected_col1 = [0.0, np.nan, 1.0, np.nan, 2.0, np.nan, 3.0, np.nan, 4.0] + expected_col2 = [np.nan, 5.0, np.nan, 6.0, 7.0, 8.0, np.nan, 9.0, np.nan] + values = ss.arrow_values() + times = [t.value for t in values["t"]] + col1 = [np.nan if isnan(v.as_py()) else v.as_py() for v in values["a"]] + col2 = [np.nan if isnan(v.as_py()) else v.as_py() for v in values["b"]] + assert times == expected_times + assert col1 == expected_col1 + assert col2 == expected_col2 + assert schema.equals(values.schema) + + +@pytest.mark.parametrize( + "name_callable", + [(None), (lambda s: str(s.uuid)), (lambda s: s.name + "/" + s.collection)], + ids=["empty", "uu_as_str", "name_collection"], +) +def test_streamset_arrow_windows_vs_windows(conn, tmp_collection, name_callable): + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + s3 = conn.create(new_uuid(), tmp_collection, tags={"name": "s3"}) + t1 = [100, 105, 110, 115, 120] + t2 = [101, 106, 110, 114, 119] + d1 = [0.0, 1.0, 2.0, 3.0, 4.0] + d2 = [5.0, 6.0, 7.0, 8.0, 9.0] + d3 = [1.0, 9.0, 44.0, 8.0, 9.0] + s1.insert(list(zip(t1, d1))) + s2.insert(list(zip(t2, d2))) + s3.insert(list(zip(t2, d3))) + ss = ( + btrdb.stream.StreamSet([s1, s2, s3]) + .filter(start=100, end=121) + .windows(width=btrdb.utils.timez.ns_delta(nanoseconds=10)) + ) + values_arrow = ss.arrow_to_dataframe(name_callable=name_callable) + values_arrow.index = pd.DatetimeIndex(values_arrow.index) + values_prev = ss.to_dataframe(name_callable=name_callable).convert_dtypes( + dtype_backend="pyarrow" + ) + values_prev.index = pd.to_datetime(values_prev.index, utc=True) + values_prev = values_prev.convert_dtypes( + dtype_backend="pyarrow", + ) + col_map = {old_col: old_col + "/mean" for old_col in values_prev.columns} + values_prev = values_prev.rename(columns=col_map) + assert values_arrow.equals(values_prev) + + +def test_streamset_arrow_windows_vs_windows_agg_all(conn, tmp_collection): + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + s3 = conn.create(new_uuid(), tmp_collection, tags={"name": "s3"}) + t1 = [100, 105, 110, 115, 120] + t2 = [101, 106, 110, 114, 119] + d1 = [0.0, 1.0, 2.0, 3.0, 4.0] + d2 = [5.0, 6.0, 7.0, 8.0, 9.0] + d3 = [1.0, 9.0, 44.0, 8.0, 9.0] + s1.insert(list(zip(t1, d1))) + s2.insert(list(zip(t2, d2))) + s3.insert(list(zip(t2, d3))) + ss = ( + btrdb.stream.StreamSet([s1, s2, s3]) + .filter(start=100, end=121) + .windows(width=btrdb.utils.timez.ns_delta(nanoseconds=10)) + ) + values_arrow = ss.arrow_to_dataframe(name_callable=None, agg=["all"]) + values_arrow.index = pd.DatetimeIndex(values_arrow.index) + values_prev = ss.to_dataframe(name_callable=None, agg="all") + values_prev = values_prev.apply(lambda x: x.astype(str(x.dtype) + "[pyarrow]")) + values_prev = values_prev.apply( + lambda x: x.astype("uint64[pyarrow]") if "count" in x.name else x + ) + values_prev.index = pd.DatetimeIndex(values_prev.index, tz="UTC") + values_prev = values_prev.convert_dtypes(dtype_backend="pyarrow") + new_cols = ["/".join(old_col) for old_col in values_prev.columns] + values_prev.columns = new_cols + assert values_arrow.equals(values_prev) + + with pytest.raises( + AttributeError, + match="cannot provide name_callable when using 'all' as aggregate at this time", + ): + values_prev = ss.to_dataframe(name_callable=lambda x: str(x.uuid), agg="all") + other_arrow_df = ss.arrow_to_dataframe( + agg=["all", "mean", "trash"], name_callable=lambda x: str(x.uuid) + ) + assert ( + len( + other_arrow_df.filter( + regex=r".*\/[min,mean,max,count,stddev]", axis=1 + ).columns + ) + == 3 * 5 + ) + + +@pytest.mark.parametrize( + "name_callable", + [(None), (lambda s: str(s.uuid)), (lambda s: s.name + "/" + s.collection)], + ids=["empty", "uu_as_str", "name_collection"], +) +def test_streamset_arrow_aligned_windows_vs_aligned_windows( + conn, tmp_collection, name_callable +): + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + s3 = conn.create(new_uuid(), tmp_collection, tags={"name": "s3"}) + t1 = [100, 105, 110, 115, 120] + t2 = [101, 106, 110, 114, 119] + d1 = [0.0, 1.0, 2.0, 3.0, 4.0] + d2 = [5.0, 6.0, 7.0, 8.0, 9.0] + d3 = [1.0, 9.0, 44.0, 8.0, 9.0] + s1.insert(list(zip(t1, d1))) + s2.insert(list(zip(t2, d2))) + s3.insert(list(zip(t2, d3))) + ss = ( + btrdb.stream.StreamSet([s1, s2, s3]) + .filter(start=100, end=121) + .aligned_windows(pointwidth=btrdb.utils.general.pointwidth.from_nanoseconds(10)) + ) + values_arrow = ss.arrow_to_dataframe(name_callable=name_callable) + values_arrow.index = pd.DatetimeIndex(values_arrow.index) + values_prev = ss.to_dataframe( + name_callable=name_callable + ) # .convert_dtypes(dtype_backend='pyarrow') + values_prev = values_prev.apply(lambda x: x.astype(str(x.dtype) + "[pyarrow]")) + values_prev = values_prev.apply( + lambda x: x.astype("uint64[pyarrow]") if "count" in x.name else x + ) + values_prev.index = pd.DatetimeIndex(values_prev.index, tz="UTC") + col_map = {old_col: old_col + "/mean" for old_col in values_prev.columns} + values_prev = values_prev.rename(columns=col_map) + assert values_arrow.equals(values_prev) + + +@pytest.mark.parametrize( + "name_callable", + [(None), (lambda s: str(s.uuid)), (lambda s: s.name + "/" + s.collection)], + ids=["empty", "uu_as_str", "name_collection"], +) +def test_streamset_arrow_aligned_windows_join_logic( + conn, tmp_collection, name_callable +): + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + s3 = conn.create(new_uuid(), tmp_collection, tags={"name": "s3"}) + t1 = [100, 105, 110, 115, 120] + t2 = [101, 106, 110, 132, 140] + d1 = [0.0, 1.0, 2.0, 3.0, 4.0] + d2 = [5.0, 6.0, 7.0, 8.0, 9.0] + d3 = [1.0, 9.0, 44.0, 8.0, 9.0] + s1.insert(list(zip(t1, d1))) + s2.insert(list(zip(t2, d2))) + s3.insert(list(zip(t2, d3))) + ss = ( + btrdb.stream.StreamSet([s1, s2, s3]) + .filter(start=100, end=141) + .aligned_windows(pointwidth=btrdb.utils.general.pointwidth.from_nanoseconds(8)) + ) + values_arrow = ss.arrow_to_dataframe(name_callable=name_callable) + values_arrow.index = pd.DatetimeIndex(values_arrow.index) + values_prev = ss.to_dataframe( + name_callable=name_callable + ) # .convert_dtypes(dtype_backend='pyarrow') + values_prev = values_prev.apply(lambda x: x.astype(str(x.dtype) + "[pyarrow]")) + values_prev = values_prev.apply( + lambda x: x.astype("uint64[pyarrow]") if "count" in x.name else x + ) + values_prev.index = pd.DatetimeIndex(values_prev.index, tz="UTC") + col_map = {old_col: old_col + "/mean" for old_col in values_prev.columns} + values_prev = values_prev.rename(columns=col_map) + assert values_arrow.equals(values_prev) + + +def test_streamset_empty_arrow_values(conn, tmp_collection): + s = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) + ss = btrdb.stream.StreamSet([s]).filter(start=100, end=200) + values = ss.arrow_values() + expected_schema = pa.schema( + [ + pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False), + pa.field(str(s.uuid), pa.float64(), nullable=False), + ] + ) + assert [t.value for t in values["time"]] == [] + assert [v for v in values[str(s.uuid)]] == [] + assert expected_schema.equals(values.schema) + + +def test_streamset_to_dataframe(conn, tmp_collection): + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + t1 = [100, 105, 110, 115, 120] + t2 = [101, 106, 110, 114, 119] + d1 = [0.0, 1.0, 2.0, 3.0, 4.0] + d2 = [5.0, 6.0, 7.0, 8.0, 9.0] + s1.insert(list(zip(t1, d1))) + s2.insert(list(zip(t2, d2))) + ss = btrdb.stream.StreamSet([s1, s2]).filter(start=100, end=121) + values = ss.to_dataframe() + expected_times = [100, 101, 105, 106, 110, 114, 115, 119, 120] + expected_col1 = [0.0, np.nan, 1.0, np.nan, 2.0, np.nan, 3.0, np.nan, 4.0] + expected_col2 = [np.nan, 5.0, np.nan, 6.0, 7.0, 8.0, np.nan, 9.0, np.nan] + expected_dat = { + tmp_collection + "/s1": expected_col1, + tmp_collection + "/s2": expected_col2, + } + expected_df = pd.DataFrame(expected_dat, index=expected_times) + assert values.equals(expected_df) + + +def test_arrow_streamset_to_dataframe(conn, tmp_collection): + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + t1 = [100, 105, 110, 115, 120] + t2 = [101, 106, 110, 114, 119] + d1 = [0.0, 1.0, 2.0, 3.0, 4.0] + d2 = [5.0, 6.0, 7.0, 8.0, 9.0] + s1.insert(list(zip(t1, d1))) + s2.insert(list(zip(t2, d2))) + ss = btrdb.stream.StreamSet([s1, s2]).filter(start=100, end=121) + values = ss.arrow_to_dataframe() + expected_times = [100, 101, 105, 106, 110, 114, 115, 119, 120] + expected_times = [ + pa.scalar(v, type=pa.timestamp("ns", tz="UTC")).as_py() for v in expected_times + ] + expected_col1 = pa.array( + [0.0, np.nan, 1.0, np.nan, 2.0, np.nan, 3.0, np.nan, 4.0], mask=[False] * 9 + ) + expected_col2 = pa.array( + [np.nan, 5.0, np.nan, 6.0, 7.0, 8.0, np.nan, 9.0, np.nan], mask=[False] * 9 + ) + expected_dat = { + "time": expected_times, + tmp_collection + "/s1": expected_col1, + tmp_collection + "/s2": expected_col2, + } + schema = pa.schema( + fields=[ + pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False), + pa.field(tmp_collection + "/s1", type=pa.float64(), nullable=False), + pa.field(tmp_collection + "/s2", type=pa.float64(), nullable=False), + ] + ) + expected_table = pa.Table.from_pydict(mapping=expected_dat, schema=schema) + expected_df = expected_table.to_pandas( + timestamp_as_object=False, + types_mapper=pd.ArrowDtype, + ).set_index("time") + expected_df.index = pd.DatetimeIndex(expected_df.index, tz="UTC") + np_test.assert_array_equal( + values.values.astype(float), expected_df.values.astype(float) + ) + + +def test_arrow_streamset_to_polars(conn, tmp_collection): + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + t1 = [100, 105, 110, 115, 120] + t2 = [101, 106, 110, 114, 119] + d1 = [0.0, 1.0, 2.0, 3.0, 4.0] + d2 = [5.0, 6.0, 7.0, 8.0, 9.0] + s1.insert(list(zip(t1, d1))) + s2.insert(list(zip(t2, d2))) + ss = btrdb.stream.StreamSet([s1, s2]).filter(start=100, end=121) + values = ss.arrow_to_polars() + expected_times = [100, 101, 105, 106, 110, 114, 115, 119, 120] + expected_times = [ + pa.scalar(v, type=pa.timestamp("ns", tz="UTC")).as_py() for v in expected_times + ] + expected_col1 = [0.0, np.nan, 1.0, np.nan, 2.0, np.nan, 3.0, np.nan, 4.0] + expected_col2 = [np.nan, 5.0, np.nan, 6.0, 7.0, 8.0, np.nan, 9.0, np.nan] + expected_dat = { + tmp_collection + "/s1": expected_col1, + tmp_collection + "/s2": expected_col2, + } + expected_df = pd.DataFrame( + expected_dat, index=pd.DatetimeIndex(expected_times, name="time") + ) + expected_df_pl = pl.from_pandas(expected_df, nan_to_null=False, include_index=True) + pl_test.assert_frame_equal(values, expected_df_pl) + + +@pytest.mark.parametrize( + "name_callable", + [(None), (lambda s: str(s.uuid)), (lambda s: s.name + "/" + s.collection)], + ids=["empty", "uu_as_str", "name_collection"], +) +def test_streamset_arrow_polars_vs_old_to_polars(conn, tmp_collection, name_callable): + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + t1 = [100, 105, 110, 115, 120] + t2 = [101, 106, 110, 114, 119] + d1 = [0.0, 1.0, 2.0, 3.0, 4.0] + d2 = [5.0, 6.0, 7.0, 8.0, 9.0] + s1.insert(list(zip(t1, d1))) + s2.insert(list(zip(t2, d2))) + ss = btrdb.stream.StreamSet([s1, s2]).filter(start=100, end=121) + values_arrow = ss.arrow_to_polars() + values_non_arrow = ss.to_polars() + expected_times = [100, 101, 105, 106, 110, 114, 115, 119, 120] + expected_times = [ + pa.scalar(v, type=pa.timestamp("ns", tz="UTC")).as_py() for v in expected_times + ] + expected_col1 = [0.0, np.nan, 1.0, np.nan, 2.0, np.nan, 3.0, np.nan, 4.0] + expected_col2 = [np.nan, 5.0, np.nan, 6.0, 7.0, 8.0, np.nan, 9.0, np.nan] + expected_dat = { + tmp_collection + "/s1": expected_col1, + tmp_collection + "/s2": expected_col2, + } + expected_df = pd.DataFrame( + expected_dat, index=pd.DatetimeIndex(expected_times, tz="UTC", name="time") + ) + expected_df_pl = pl.from_pandas(expected_df, nan_to_null=False, include_index=True) + pl_test.assert_frame_equal(values_arrow, expected_df_pl) + pl_test.assert_frame_equal(values_non_arrow, expected_df_pl) + pl_test.assert_frame_equal(values_non_arrow, values_arrow) + + +@pytest.mark.parametrize( + "name_callable", + [(None), (lambda s: str(s.uuid)), (lambda s: s.name + "/" + s.collection)], + ids=["empty", "uu_as_str", "name_collection"], +) +def test_streamset_windows_arrow_polars_vs_old_to_polars( + conn, tmp_collection, name_callable +): + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + s3 = conn.create(new_uuid(), tmp_collection, tags={"name": "s3"}) + t1 = [100, 105, 110, 115, 120] + t2 = [101, 106, 110, 114, 119] + d1 = [0.0, 1.0, 2.0, 3.0, 4.0] + d2 = [5.0, 6.0, 7.0, 8.0, 9.0] + d3 = [1.0, 9.0, 44.0, 8.0, 9.0] + s1.insert(list(zip(t1, d1))) + s2.insert(list(zip(t2, d2))) + s3.insert(list(zip(t2, d3))) + ss = ( + btrdb.stream.StreamSet([s1, s2, s3]) + .filter(start=100, end=121) + .windows(width=btrdb.utils.timez.ns_delta(nanoseconds=10)) + ) + values_arrow_pl = ss.arrow_to_polars(name_callable=name_callable) + values_non_arrow_pl = ss.to_polars(name_callable=name_callable) + new_names = { + old_col: str(old_col) + "/" + "mean" + for old_col in values_non_arrow_pl.select(pl.col(pl.Float64)).columns + } + new_names["time"] = "time" + values_non_arrow_pl = values_non_arrow_pl.rename(mapping=new_names) + assert values_arrow_pl.equals(values_non_arrow_pl) + + +def test_streamset_windows_aggregates_filter(conn, tmp_collection): + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + s3 = conn.create(new_uuid(), tmp_collection, tags={"name": "s3"}) + t1 = [100, 105, 110, 115, 120] + t2 = [101, 106, 110, 114, 119] + d1 = [0.0, 1.0, 2.0, 3.0, 4.0] + d2 = [5.0, 6.0, 7.0, 8.0, 9.0] + d3 = [1.0, 9.0, 44.0, 8.0, 9.0] + s1.insert(list(zip(t1, d1))) + s2.insert(list(zip(t2, d2))) + s3.insert(list(zip(t2, d3))) + ss = ( + btrdb.stream.StreamSet([s1, s2, s3]) + .filter(start=100, end=121) + .windows(width=btrdb.utils.timez.ns_delta(nanoseconds=10)) + ) + values_arrow_df = ss.arrow_to_dataframe(agg=["mean", "stddev"]) + values_arrow_df.index = pd.DatetimeIndex(values_arrow_df.index) + values_non_arrow_df = ss.to_dataframe(agg="all") + values_non_arrow_df.index = pd.DatetimeIndex(values_non_arrow_df.index, tz="UTC") + values_non_arrow_df = values_non_arrow_df.apply( + lambda x: x.astype(str(x.dtype) + "[pyarrow]") + ) + values_non_arrow_df = values_non_arrow_df.apply( + lambda x: x.astype("uint64[pyarrow]") if "count" in x.name else x + ) + new_cols = ["/".join(old_col) for old_col in values_non_arrow_df.columns] + values_non_arrow_df.columns = new_cols + cols_to_drop = [ + col + for col in values_non_arrow_df.columns + if ("count" in col) or ("min" in col) or ("max" in col) + ] + values_non_arrow_df.drop(columns=cols_to_drop, inplace=True) + assert values_arrow_df.equals(values_non_arrow_df) + + +def test_timesnap_backward_extends_range(conn, tmp_collection): + sec = 10**9 + tv1 = [ + [int(0.5 * sec), 0.5], + [2 * sec, 2.0], + ] + tv2 = [ + [int(0.5 * sec) - 1, 0.5], + [2 * sec, 2.0], + ] + tv3 = [ + [1 * sec, 1.0], + [2 * sec, 2.0], + ] + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + s3 = conn.create(new_uuid(), tmp_collection, tags={"name": "s3"}) + s1.insert(tv1) + s2.insert(tv2) + s3.insert(tv3) + ss = btrdb.stream.StreamSet([s1, s2, s3]).filter( + start=1 * sec, end=3 * sec, sampling_frequency=1 + ) + values = ss.arrow_values() + assert [1 * sec, 2 * sec] == [t.value for t in values["time"]] + assert [0.5, 2.0] == [v.as_py() for v in values[str(s1.uuid)]] + assert [np.nan, 2.0] == [ + np.nan if isnan(v.as_py()) else v.as_py() for v in values[str(s2.uuid)] + ] + assert [1.0, 2.0] == [v.as_py() for v in values[str(s3.uuid)]] + + +def test_timesnap_forward_restricts_range(conn, tmp_collection): + sec = 10**9 + tv = [ + [1 * sec, 1.0], + [2 * sec, 2.0], + [int(2.75 * sec), 2.75], + ] + s = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) + s.insert(tv) + ss = btrdb.stream.StreamSet([s]).filter(start=1 * sec, sampling_frequency=1) + values = ss.filter(end=int(3.0 * sec)).arrow_values() + assert [1 * sec, 2 * sec] == [t.value for t in values["time"]] + assert [1.0, 2.0] == [v.as_py() for v in values[str(s.uuid)]] + # Same result if skipping past end instead of to end. + assert values == ss.filter(end=int(2.9 * sec)).arrow_values() + + +@pytest.mark.parametrize("freq", [(None), (30), (15), (1), (0.1)]) +def test_timesnap_with_different_sampling_frequencies(freq, conn, tmp_collection): + # 30hz data + data_insert_freq = 30 + period_for_data = int(1 / data_insert_freq * 1e9) + stop = btrdb.utils.timez.currently_as_ns() + start = stop - btrdb.utils.timez.ns_delta(minutes=10) + t1 = [i for i in range(start, stop - period_for_data, period_for_data)] + v1 = [ + random.random() for _ in range(start, stop - period_for_data, period_for_data) + ] + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + stset = btrdb.stream.StreamSet([s1, s2]) + data_map = {s.uuid: pa.table([t1, v1], names=["time", "value"]) for s in stset} + stset.arrow_insert(data_map=data_map) + df = stset.filter( + start=start, end=stop, sampling_frequency=freq + ).arrow_to_dataframe() + total_points = df.shape[0] * df.shape[1] + total_raw_pts = len(v1) * len(stset) + expected_frac_of_pts = 1 if freq is None else freq / data_insert_freq + actual_frac_of_pts = total_points / total_raw_pts + assert np.isclose(expected_frac_of_pts, actual_frac_of_pts, rtol=0.001) diff --git a/tests/requirements.txt b/tests/requirements.txt index 2d03e46..a58a885 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,9 +1,57 @@ -pytest==5.3.2 -pytest-cov==2.8.1 -pytest-flake8==1.0.7 -freezegun==0.3.12 - -# dependencies for conversions -numpy>=1.15 +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --extra=all --output-file=requirements-all.txt --resolver=backtracking pyproject.toml +# +# Pinned versions and dependencies of dependencies were removed after the autogeneration. +certifi + # via + # btrdb (pyproject.toml) +freezegun + # via btrdb +grpcio + # via + # btrdb (pyproject.toml) + # grpcio-tools + # ray +grpcio-tools + # via + # btrdb (pyproject.toml) +numpy + # via + # btrdb + # pandas + # pyarrow + # ray pandas -tabulate==0.8.6 + # via btrdb +polars + # via btrdb +protobuf + # via + # grpcio-tools + # ray +pyarrow + # via btrdb +pytest <9 + # via + # btrdb + # pytest-cov + # pytest-flake8 +pytest-cov + # via btrdb +pytest-flake8 + # via btrdb +pytz + # via + # btrdb (pyproject.toml) + # pandas +pyyaml + # via + # btrdb (pyproject.toml) + # ray +ray + # via btrdb +tabulate + # via btrdb