From e59653e76be7149e08be3af1e2e0a854bd90062f Mon Sep 17 00:00:00 2001 From: Gabe Fierro Date: Wed, 5 May 2021 11:30:01 -0700 Subject: [PATCH 001/131] add assertion to check for initialization, use ray 1.2.0 register_serializer --- btrdb/utils/ray.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/btrdb/utils/ray.py b/btrdb/utils/ray.py index 8df9527..d6e00e0 100644 --- a/btrdb/utils/ray.py +++ b/btrdb/utils/ray.py @@ -22,7 +22,8 @@ def register_serializer(conn_str=None, apikey=None, profile=None): found in the user's predictive grid credentials file `~/.predictivegrid/credentials.yaml`. """ - ray.register_custom_serializer( + assert ray.is_initialized(), "Need to call ray.init() before registering custom serializer" + ray.util.register_serializer( BTrDB, serializer=btrdb_serializer, deserializer=partial(btrdb_deserializer, conn_str=conn_str, apikey=apikey, profile=profile)) def btrdb_serializer(_): From 2fbff728ad45daa4d539f1ec3797543d5bdbeaec Mon Sep 17 00:00:00 2001 From: Gabe Fierro Date: Wed, 5 May 2021 11:34:19 -0700 Subject: [PATCH 002/131] is this check being run? --- btrdb/utils/ray.py | 1 + 1 file changed, 1 insertion(+) diff --git a/btrdb/utils/ray.py b/btrdb/utils/ray.py index d6e00e0..9b1f611 100644 --- a/btrdb/utils/ray.py +++ b/btrdb/utils/ray.py @@ -22,6 +22,7 @@ def register_serializer(conn_str=None, apikey=None, profile=None): found in the user's predictive grid credentials file `~/.predictivegrid/credentials.yaml`. """ + print("check init", ray.is_initialized()) assert ray.is_initialized(), "Need to call ray.init() before registering custom serializer" ray.util.register_serializer( BTrDB, serializer=btrdb_serializer, deserializer=partial(btrdb_deserializer, conn_str=conn_str, apikey=apikey, profile=profile)) From bbaa1696a6ef1566855db70327032f5540df5154 Mon Sep 17 00:00:00 2001 From: Gabe Fierro Date: Wed, 5 May 2021 11:41:17 -0700 Subject: [PATCH 003/131] remove print --- btrdb/utils/ray.py | 1 - 1 file changed, 1 deletion(-) diff --git a/btrdb/utils/ray.py b/btrdb/utils/ray.py index 9b1f611..d6e00e0 100644 --- a/btrdb/utils/ray.py +++ b/btrdb/utils/ray.py @@ -22,7 +22,6 @@ def register_serializer(conn_str=None, apikey=None, profile=None): found in the user's predictive grid credentials file `~/.predictivegrid/credentials.yaml`. """ - print("check init", ray.is_initialized()) assert ray.is_initialized(), "Need to call ray.init() before registering custom serializer" ray.util.register_serializer( BTrDB, serializer=btrdb_serializer, deserializer=partial(btrdb_deserializer, conn_str=conn_str, apikey=apikey, profile=profile)) From e2bb4f91b5581c299c90dde2a52b5e4853d760d0 Mon Sep 17 00:00:00 2001 From: Gabe Fierro Date: Thu, 20 May 2021 12:20:49 -0700 Subject: [PATCH 004/131] trying out custom serialize with a flag on connect --- btrdb/__init__.py | 15 ++++++++++++++- btrdb/utils/ray.py | 10 ++++++++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/btrdb/__init__.py b/btrdb/__init__.py index a7019c7..b5485a5 100644 --- a/btrdb/__init__.py +++ b/btrdb/__init__.py @@ -21,6 +21,8 @@ from btrdb.version import get_version from btrdb.utils.credentials import credentials_by_profile, credentials from btrdb.stream import MINIMUM_TIME, MAXIMUM_TIME +from btrdb.util import register_serializer +from warnings import warn ########################################################################## ## Module Variables @@ -40,7 +42,7 @@ def _connect(endpoints=None, apikey=None): return BTrDB(Endpoint(Connection(endpoints, apikey=apikey).channel)) -def connect(conn_str=None, apikey=None, profile=None): +def connect(conn_str=None, apikey=None, profile=None, shareable=False): """ Connect to a BTrDB server. @@ -57,6 +59,12 @@ def connect(conn_str=None, apikey=None, profile=None): The name of a profile containing the required connection information as found in the user's predictive grid credentials file `~/.predictivegrid/credentials.yaml`. + shareable: bool, default=False + Whether or not the connection can be "shared" in a distributed setting such + as Ray workers. If set to True, the connection can be serialized and sent + to other workers so that data can be retrieved in parallel; **however**, this + is less secure because it is possible for other users of the Ray cluster to + use your API key to fetch data. Returns ------- @@ -68,6 +76,11 @@ def connect(conn_str=None, apikey=None, profile=None): if conn_str and profile: raise ValueError("Received both conn_str and profile arguments.") + # check shareable flag and register custom serializer if necessary + if shareable: + warn("a shareable connection is potentially insecure; other users of the same cluster may be able to access your API key") + register_serializer(conn_str=conn_str, apikey=apikey, profile=profile) + # use specific profile if requested if profile: return _connect(**credentials_by_profile(profile)) diff --git a/btrdb/utils/ray.py b/btrdb/utils/ray.py index d6e00e0..0ab5f65 100644 --- a/btrdb/utils/ray.py +++ b/btrdb/utils/ray.py @@ -23,8 +23,14 @@ def register_serializer(conn_str=None, apikey=None, profile=None): `~/.predictivegrid/credentials.yaml`. """ assert ray.is_initialized(), "Need to call ray.init() before registering custom serializer" - ray.util.register_serializer( - BTrDB, serializer=btrdb_serializer, deserializer=partial(btrdb_deserializer, conn_str=conn_str, apikey=apikey, profile=profile)) + # TODO: check the version using the 'semver' package? + if ray.__version__ == "0.8.4": + ray.register_custom_serializer( + BTrDB, serializer=btrdb_serializer, deserializer=partial(btrdb_deserializer, conn_str=conn_str, apikey=apikey, profile=profile)) + elif ray.__version__ in ["1.2.0", "1.3.0"]: + # TODO: check different versions of ray? + ray.util.register_serializer( + BTrDB, serializer=btrdb_serializer, deserializer=partial(btrdb_deserializer, conn_str=conn_str, apikey=apikey, profile=profile)) def btrdb_serializer(_): """ From 74f5623aaef6b4ba0eff8c661febec51a62f400f Mon Sep 17 00:00:00 2001 From: Gabe Fierro Date: Thu, 20 May 2021 12:23:17 -0700 Subject: [PATCH 005/131] fix import --- btrdb/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/btrdb/__init__.py b/btrdb/__init__.py index b5485a5..95847e7 100644 --- a/btrdb/__init__.py +++ b/btrdb/__init__.py @@ -21,7 +21,7 @@ from btrdb.version import get_version from btrdb.utils.credentials import credentials_by_profile, credentials from btrdb.stream import MINIMUM_TIME, MAXIMUM_TIME -from btrdb.util import register_serializer +from btrdb.utils import register_serializer from warnings import warn ########################################################################## From c122166a775d8670c63884435c3be267735c283b Mon Sep 17 00:00:00 2001 From: Gabe Fierro Date: Thu, 20 May 2021 12:24:49 -0700 Subject: [PATCH 006/131] fix import --- btrdb/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/btrdb/__init__.py b/btrdb/__init__.py index 95847e7..cd14662 100644 --- a/btrdb/__init__.py +++ b/btrdb/__init__.py @@ -20,8 +20,8 @@ from btrdb.exceptions import ConnectionError from btrdb.version import get_version from btrdb.utils.credentials import credentials_by_profile, credentials +from btrdb.utils.ray import register_serializer from btrdb.stream import MINIMUM_TIME, MAXIMUM_TIME -from btrdb.utils import register_serializer from warnings import warn ########################################################################## From cd27a0b5b217fdc2fa0a3496de3271823599da37 Mon Sep 17 00:00:00 2001 From: Gabe Fierro Date: Thu, 20 May 2021 12:29:01 -0700 Subject: [PATCH 007/131] try semver --- btrdb/utils/ray.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/btrdb/utils/ray.py b/btrdb/utils/ray.py index 0ab5f65..64b2990 100644 --- a/btrdb/utils/ray.py +++ b/btrdb/utils/ray.py @@ -2,6 +2,8 @@ import ray +import semver + import btrdb from btrdb.conn import BTrDB @@ -24,13 +26,16 @@ def register_serializer(conn_str=None, apikey=None, profile=None): """ assert ray.is_initialized(), "Need to call ray.init() before registering custom serializer" # TODO: check the version using the 'semver' package? - if ray.__version__ == "0.8.4": + ver = semver.VersionInfo.parse(ray.__version__) + if ver.major == 0: ray.register_custom_serializer( BTrDB, serializer=btrdb_serializer, deserializer=partial(btrdb_deserializer, conn_str=conn_str, apikey=apikey, profile=profile)) - elif ray.__version__ in ["1.2.0", "1.3.0"]: + elif ver.major == 1 and ver.minor in range(2, 4): # TODO: check different versions of ray? ray.util.register_serializer( BTrDB, serializer=btrdb_serializer, deserializer=partial(btrdb_deserializer, conn_str=conn_str, apikey=apikey, profile=profile)) + else: + raise Exception("Ray version %s does not have custom serialization. Please upgrade to >= 1.2.0" % ray.__version__) def btrdb_serializer(_): """ From 80517caf7652db0988b206c840feeec175b922ed Mon Sep 17 00:00:00 2001 From: Michael Chestnut Date: Mon, 24 May 2021 12:41:01 -0500 Subject: [PATCH 008/131] modify setup and docs to use new repo --- docs/source/conf.py | 8 ++++---- setup.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index e221fcb..cebafa7 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -26,8 +26,8 @@ # -- Project information ----------------------------------------------------- project = 'btrdb' -copyright = '2019, Michael P. Andersen' -author = 'Michael P. Andersen' +copyright = '2021, PingThingsIO' +author = 'PingThingsIO' # The short X.Y version version = get_version() @@ -93,7 +93,7 @@ # html_theme_options = { 'show_powered_by': False, - 'github_user': 'BTrDB', + 'github_user': 'PingThingsIO', 'github_repo': 'btrdb-python', 'travis_button': False, 'github_banner': False, @@ -159,7 +159,7 @@ # author, documentclass [howto, manual, or own class]). latex_documents = [ (master_doc, 'btrdb-python.tex', 'btrdb-python Documentation', - 'Michael Andersen', 'manual'), + 'PingThingsIO', 'manual'), ] diff --git a/setup.py b/setup.py index 0e515f2..2997b26 100644 --- a/setup.py +++ b/setup.py @@ -29,11 +29,11 @@ ## Basic information NAME = "btrdb" DESCRIPTION = "Bindings to interact with the Berkeley Tree Database using gRPC." -AUTHOR = "Michael Andersen, Allen Leis" -EMAIL = "michael@steelcode.com" -MAINTAINER = "Michael Andersen" +AUTHOR = "PingThingsIO" +EMAIL = "support@pingthings.io" +MAINTAINER = "PingThingsIO" LICENSE = "BSD-3-Clause" -REPOSITORY = "https://github.com/BTrDB/btrdb-python" +REPOSITORY = "https://github.com/PingThingsIO/btrdb-python" PACKAGE = "btrdb" URL = "http://btrdb.io/" DOCS_URL = "https://btrdb.readthedocs.io/en/latest/" From 8529c5cd25da1b3013cc5188731d8920b194f2f7 Mon Sep 17 00:00:00 2001 From: Michael Chestnut Date: Wed, 26 May 2021 16:29:35 -0500 Subject: [PATCH 009/131] updating copyright --- docs/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index cebafa7..79f76de 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -26,7 +26,7 @@ # -- Project information ----------------------------------------------------- project = 'btrdb' -copyright = '2021, PingThingsIO' +copyright = '2021, Ping Things, Inc.' author = 'PingThingsIO' # The short X.Y version From fc75ca4c3ba2005bb3aaf4c294c3c108b854c3a4 Mon Sep 17 00:00:00 2001 From: Michael Chestnut Date: Thu, 10 Jun 2021 11:43:57 -0500 Subject: [PATCH 010/131] Release v5.11.1 --- btrdb/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/btrdb/version.py b/btrdb/version.py index 6df466a..c337dbe 100644 --- a/btrdb/version.py +++ b/btrdb/version.py @@ -15,7 +15,7 @@ ## Module Info ########################################################################## -__version_info__ = { 'major': 5, 'minor': 11, 'micro': 0, 'releaselevel': 'final'} + ########################################################################## ## Helper Functions From d9e943da3620ec75c8e43fb5bd8dc5c185974a1d Mon Sep 17 00:00:00 2001 From: Michael Chestnut Date: Thu, 10 Jun 2021 14:49:10 -0500 Subject: [PATCH 011/131] Release v5.11.1 --- btrdb/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/btrdb/version.py b/btrdb/version.py index c337dbe..cb44b1d 100644 --- a/btrdb/version.py +++ b/btrdb/version.py @@ -15,7 +15,7 @@ ## Module Info ########################################################################## - +__version_info__ = { 'major': 5, 'minor': 11, 'micro': 1, 'releaselevel': 'final'} ########################################################################## ## Helper Functions From e881d2e6f36b8ee3a3d649f7ab0e4c2a052c02ea Mon Sep 17 00:00:00 2001 From: Michael Chestnut Date: Thu, 10 Jun 2021 14:49:22 -0500 Subject: [PATCH 012/131] Release v5.11.2 --- btrdb/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/btrdb/version.py b/btrdb/version.py index cb44b1d..eff1b61 100644 --- a/btrdb/version.py +++ b/btrdb/version.py @@ -15,7 +15,7 @@ ## Module Info ########################################################################## -__version_info__ = { 'major': 5, 'minor': 11, 'micro': 1, 'releaselevel': 'final'} +__version_info__ = { 'major': 5, 'minor': 11, 'micro': 2, 'releaselevel': 'final'} ########################################################################## ## Helper Functions From f9d0c3d5953a3b225b56984cdcf088f92306f08e Mon Sep 17 00:00:00 2001 From: Michael Chestnut Date: Thu, 10 Jun 2021 15:35:49 -0500 Subject: [PATCH 013/131] make ray and semver dependencies optional --- btrdb/utils/ray.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/btrdb/utils/ray.py b/btrdb/utils/ray.py index 64b2990..674ab64 100644 --- a/btrdb/utils/ray.py +++ b/btrdb/utils/ray.py @@ -1,11 +1,6 @@ -from functools import partial - -import ray - -import semver - import btrdb from btrdb.conn import BTrDB +from functools import partial def register_serializer(conn_str=None, apikey=None, profile=None): """ @@ -24,6 +19,15 @@ def register_serializer(conn_str=None, apikey=None, profile=None): found in the user's predictive grid credentials file `~/.predictivegrid/credentials.yaml`. """ + try: + import ray + except ImportError: + raise ImportError("must pip install ray to register custom serializer") + try: + import semver + except ImportError: + raise ImportError("must pip install semver to register custom serializer") + assert ray.is_initialized(), "Need to call ray.init() before registering custom serializer" # TODO: check the version using the 'semver' package? ver = semver.VersionInfo.parse(ray.__version__) @@ -31,7 +35,7 @@ def register_serializer(conn_str=None, apikey=None, profile=None): ray.register_custom_serializer( BTrDB, serializer=btrdb_serializer, deserializer=partial(btrdb_deserializer, conn_str=conn_str, apikey=apikey, profile=profile)) elif ver.major == 1 and ver.minor in range(2, 4): - # TODO: check different versions of ray? + # TODO: check different versions of ray? ray.util.register_serializer( BTrDB, serializer=btrdb_serializer, deserializer=partial(btrdb_deserializer, conn_str=conn_str, apikey=apikey, profile=profile)) else: From 47ec4f43bddba2e8c7a7d14c75c31406792c5d2d Mon Sep 17 00:00:00 2001 From: Michael Chestnut Date: Thu, 10 Jun 2021 15:53:41 -0500 Subject: [PATCH 014/131] Release v5.11.3 --- btrdb/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/btrdb/version.py b/btrdb/version.py index eff1b61..d6cd184 100644 --- a/btrdb/version.py +++ b/btrdb/version.py @@ -15,7 +15,7 @@ ## Module Info ########################################################################## -__version_info__ = { 'major': 5, 'minor': 11, 'micro': 2, 'releaselevel': 'final'} +__version_info__ = { 'major': 5, 'minor': 11, 'micro': 3, 'releaselevel': 'final'} ########################################################################## ## Helper Functions From 2fd10035f492800004c872cca225235b4ec05461 Mon Sep 17 00:00:00 2001 From: Michael Chestnut Date: Thu, 10 Jun 2021 16:07:44 -0500 Subject: [PATCH 015/131] Release v5.11.4 --- btrdb/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/btrdb/version.py b/btrdb/version.py index d6cd184..69a362a 100644 --- a/btrdb/version.py +++ b/btrdb/version.py @@ -15,7 +15,7 @@ ## Module Info ########################################################################## -__version_info__ = { 'major': 5, 'minor': 11, 'micro': 3, 'releaselevel': 'final'} +__version_info__ = { 'major': 5, 'minor': 11, 'micro': 4, 'releaselevel': 'final'} ########################################################################## ## Helper Functions From 3ca1d530c9bf4f0f76bf80ccab0413e03b722e6c Mon Sep 17 00:00:00 2001 From: Michael Chestnut Date: Thu, 10 Jun 2021 17:17:28 -0500 Subject: [PATCH 016/131] make sed cmd in release.sh compatatible with osx --- release.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release.sh b/release.sh index 6f6595a..9104416 100755 --- a/release.sh +++ b/release.sh @@ -35,8 +35,8 @@ fi echo "Setting version to v$1.$2.$3" -VERION_CODE="__version_info__ = { 'major': $1, 'minor': $2, 'micro': $3, 'releaselevel': 'final'}" -sed -i "s/^__version_info__.*$/${VERION_CODE}/g" btrdb/version.py +VERSION_CODE="__version_info__ = { 'major': $1, 'minor': $2, 'micro': $3, 'releaselevel': 'final'}" +sed -i.bak "s/^__version_info__.*$/${VERSION_CODE}/g" btrdb/version.py git add btrdb/version.py git commit -m "Release v$1.$2.$3" From 0f47f85a6c839dd34d597cc98bf71efe86118f62 Mon Sep 17 00:00:00 2001 From: Michael Chestnut Date: Fri, 25 Jun 2021 09:14:34 -0500 Subject: [PATCH 017/131] update build status in docs to show github actions --- docs/source/index.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index 66d2de0..da1b71e 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,8 +1,8 @@ Welcome to btrdb docs! ====================== -.. image:: https://img.shields.io/travis/BTrDB/btrdb-python/master.svg - :target: https://travis-ci.org/BTrDB/btrdb-python +.. image:: https://github.com/PingThingsIO/btrdb-python/actions/workflows/release.yaml/badge.svg + :target: https://github.com/PingThingsIO/btrdb-python/actions .. image:: https://readthedocs.org/projects/btrdb/badge/?version=latest :target: https://btrdb.readthedocs.io/en/latest/ From d7eef59c1f8c9b05dd58e270256c6e575790c6f3 Mon Sep 17 00:00:00 2001 From: Michael Chestnut Date: Fri, 25 Jun 2021 16:57:12 -0500 Subject: [PATCH 018/131] Release v5.11.5 --- btrdb/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/btrdb/version.py b/btrdb/version.py index 69a362a..b602f1a 100644 --- a/btrdb/version.py +++ b/btrdb/version.py @@ -15,7 +15,7 @@ ## Module Info ########################################################################## -__version_info__ = { 'major': 5, 'minor': 11, 'micro': 4, 'releaselevel': 'final'} +__version_info__ = { 'major': 5, 'minor': 11, 'micro': 5, 'releaselevel': 'final'} ########################################################################## ## Helper Functions From bb16eb1d4e49b0156134436df2c2a12eb2daccc2 Mon Sep 17 00:00:00 2001 From: Michael Chestnut Date: Fri, 25 Jun 2021 17:16:20 -0500 Subject: [PATCH 019/131] Release v5.11.6 --- btrdb/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/btrdb/version.py b/btrdb/version.py index b602f1a..809ca7e 100644 --- a/btrdb/version.py +++ b/btrdb/version.py @@ -15,7 +15,7 @@ ## Module Info ########################################################################## -__version_info__ = { 'major': 5, 'minor': 11, 'micro': 5, 'releaselevel': 'final'} +__version_info__ = { 'major': 5, 'minor': 11, 'micro': 6, 'releaselevel': 'final'} ########################################################################## ## Helper Functions From 6bd543af2c397908263a624e07a36eea987f69e0 Mon Sep 17 00:00:00 2001 From: mchestnut91 <40185521+mchestnut91@users.noreply.github.com> Date: Mon, 28 Jun 2021 15:43:14 -0500 Subject: [PATCH 020/131] Update docs link in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7a808fe..f347a6d 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ These are BTrDB Bindings for Python allowing you painless and productive access ## Sample Code -Our goal is to make BTrDB as easy to use as possible, focusing on integration with other tools and the productivity of our users. In keeping with this we continue to add new features such as easy transformation to numpy arrays, pandas Series, etc. See the sample code below and then checkout our [documentation](https://btrdb.readthedocs.io/en/latest/) for more in depth instructions. +Our goal is to make BTrDB as easy to use as possible, focusing on integration with other tools and the productivity of our users. In keeping with this we continue to add new features such as easy transformation to numpy arrays, pandas Series, etc. See the sample code below and then checkout our [documentation](https://btrdb-python.readthedocs.io/en/latest/) for more in depth instructions. import btrdb From 14a851e5ba5bf3555566ce33fbd3831e02343301 Mon Sep 17 00:00:00 2001 From: Michael Chestnut Date: Mon, 28 Jun 2021 15:45:01 -0500 Subject: [PATCH 021/131] change link to docs in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2997b26..a047ce5 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ REPOSITORY = "https://github.com/PingThingsIO/btrdb-python" PACKAGE = "btrdb" URL = "http://btrdb.io/" -DOCS_URL = "https://btrdb.readthedocs.io/en/latest/" +DOCS_URL = "https://btrdb-python.readthedocs.io/en/latest/" ## Define the keywords KEYWORDS = ('btrdb', 'berkeley', 'timeseries', 'database', 'bindings' 'gRPC') From f07a5d16e744f030889fbe3925aa8915e1a47ab3 Mon Sep 17 00:00:00 2001 From: mchestnut91 <40185521+mchestnut91@users.noreply.github.com> Date: Tue, 29 Jun 2021 11:47:58 -0500 Subject: [PATCH 022/131] update second docs link in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f347a6d..778c1a5 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,7 @@ The project documentation is written in reStructuredText and is built using Sphi $ make html -This will build the HTML documentation locally in `docs/build`, which can be viewed using `open docs/build/index.html`. Other formats (PDF, epub, etc) can be built using `docs/Makefile`. The documentation is automatically built on every GitHub release and hosted on [Read The Docs](https://btrdb.readthedocs.io/en/latest/). +This will build the HTML documentation locally in `docs/build`, which can be viewed using `open docs/build/index.html`. Other formats (PDF, epub, etc) can be built using `docs/Makefile`. The documentation is automatically built on every GitHub release and hosted on [Read The Docs](https://btrdb-python.readthedocs.io/en/latest/). Note that the documentation also requires Sphix and other dependencies to successfully build: `pip install -r docs/requirements.txt`. From e9801fde84a763669b1e9a7cd3f19502d114aa83 Mon Sep 17 00:00:00 2001 From: mchestnut91 <40185521+mchestnut91@users.noreply.github.com> Date: Tue, 29 Jun 2021 15:17:44 -0500 Subject: [PATCH 023/131] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 778c1a5..7a808fe 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ These are BTrDB Bindings for Python allowing you painless and productive access ## Sample Code -Our goal is to make BTrDB as easy to use as possible, focusing on integration with other tools and the productivity of our users. In keeping with this we continue to add new features such as easy transformation to numpy arrays, pandas Series, etc. See the sample code below and then checkout our [documentation](https://btrdb-python.readthedocs.io/en/latest/) for more in depth instructions. +Our goal is to make BTrDB as easy to use as possible, focusing on integration with other tools and the productivity of our users. In keeping with this we continue to add new features such as easy transformation to numpy arrays, pandas Series, etc. See the sample code below and then checkout our [documentation](https://btrdb.readthedocs.io/en/latest/) for more in depth instructions. import btrdb @@ -105,7 +105,7 @@ The project documentation is written in reStructuredText and is built using Sphi $ make html -This will build the HTML documentation locally in `docs/build`, which can be viewed using `open docs/build/index.html`. Other formats (PDF, epub, etc) can be built using `docs/Makefile`. The documentation is automatically built on every GitHub release and hosted on [Read The Docs](https://btrdb-python.readthedocs.io/en/latest/). +This will build the HTML documentation locally in `docs/build`, which can be viewed using `open docs/build/index.html`. Other formats (PDF, epub, etc) can be built using `docs/Makefile`. The documentation is automatically built on every GitHub release and hosted on [Read The Docs](https://btrdb.readthedocs.io/en/latest/). Note that the documentation also requires Sphix and other dependencies to successfully build: `pip install -r docs/requirements.txt`. From a23e628a14f62c36635d6571a3c3607b23e74b78 Mon Sep 17 00:00:00 2001 From: Michael Chestnut Date: Tue, 29 Jun 2021 15:59:03 -0500 Subject: [PATCH 024/131] change path in setup back to old docs --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a047ce5..2997b26 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ REPOSITORY = "https://github.com/PingThingsIO/btrdb-python" PACKAGE = "btrdb" URL = "http://btrdb.io/" -DOCS_URL = "https://btrdb-python.readthedocs.io/en/latest/" +DOCS_URL = "https://btrdb.readthedocs.io/en/latest/" ## Define the keywords KEYWORDS = ('btrdb', 'berkeley', 'timeseries', 'database', 'bindings' 'gRPC') From 31b3cb191fc404df0349b51f749cfd67a5fde5d0 Mon Sep 17 00:00:00 2001 From: David Konigsberg <72822263+davidkonigsberg@users.noreply.github.com> Date: Thu, 30 Sep 2021 15:17:34 -0400 Subject: [PATCH 025/131] fixes Let's Encrypt cert issue (#9) --- btrdb/conn.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/btrdb/conn.py b/btrdb/conn.py index 55058a4..a07c162 100644 --- a/btrdb/conn.py +++ b/btrdb/conn.py @@ -65,12 +65,20 @@ def __init__(self, addrportstr, apikey=None): # grpc bundles its own CA certs which will work for all normal SSL # certificates but will fail for custom CA certs. Allow the user # to specify a CA bundle via env var to overcome this - ca_bundle = os.getenv("BTRDB_CA_BUNDLE","") - if ca_bundle != "": + env_bundle = os.getenv("BTRDB_CA_BUNDLE", "") + os_certs = "/etc/ssl/certs/ca-certificates.crt" + ca_bundle = env_bundle + if ca_bundle == "": + ca_bundle = os_certs + try: with open(ca_bundle, "rb") as f: contents = f.read() - else: - contents = None + except Exception: + if env_bundle != "": + # The user has given us something but we can't use it, we need to make noise + raise Exception("BTRDB_CA_BUNDLE(%s) env is defined but could not read file" % ca_bundle) + else: + contents = None if apikey is None: self.channel = grpc.secure_channel( From 39ec946414d4aa862f510b1f513ab1dc57146cb1 Mon Sep 17 00:00:00 2001 From: Allen Leis Date: Mon, 4 Oct 2021 12:16:27 -0400 Subject: [PATCH 026/131] bundles certifi certs as dependency (#10) --- btrdb/conn.py | 10 ++++++++-- requirements.txt | 3 ++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/btrdb/conn.py b/btrdb/conn.py index a07c162..bb79f2f 100644 --- a/btrdb/conn.py +++ b/btrdb/conn.py @@ -18,6 +18,7 @@ import os import re import json +import certifi import uuid as uuidlib import grpc @@ -66,10 +67,15 @@ def __init__(self, addrportstr, apikey=None): # certificates but will fail for custom CA certs. Allow the user # to specify a CA bundle via env var to overcome this env_bundle = os.getenv("BTRDB_CA_BUNDLE", "") - os_certs = "/etc/ssl/certs/ca-certificates.crt" + + # certifi certs are provided as part of this package install + # https://github.com/certifi/python-certifi + lib_certs = certifi.where() + ca_bundle = env_bundle + if ca_bundle == "": - ca_bundle = os_certs + ca_bundle = lib_certs try: with open(ca_bundle, "rb") as f: contents = f.read() diff --git a/requirements.txt b/requirements.txt index b12c1a6..0d89f56 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,5 @@ grpcio-tools>=1.19.0 pytz # Misc libraries -pyyaml \ No newline at end of file +pyyaml +certifi \ No newline at end of file From ff0824c04b6dc36496956ecc89a8b5963f236e6c Mon Sep 17 00:00:00 2001 From: looselycoupled Date: Mon, 4 Oct 2021 12:24:56 -0400 Subject: [PATCH 027/131] Release v5.11.7 --- btrdb/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/btrdb/version.py b/btrdb/version.py index 809ca7e..4bff3e4 100644 --- a/btrdb/version.py +++ b/btrdb/version.py @@ -15,7 +15,7 @@ ## Module Info ########################################################################## -__version_info__ = { 'major': 5, 'minor': 11, 'micro': 6, 'releaselevel': 'final'} +__version_info__ = { 'major': 5, 'minor': 11, 'micro': 7, 'releaselevel': 'final'} ########################################################################## ## Helper Functions From 621823396f603b76b4e6d1f60ceea42cab9b323f Mon Sep 17 00:00:00 2001 From: Jacob Alperin-Sheriff Date: Wed, 27 Oct 2021 17:21:38 -0400 Subject: [PATCH 028/131] Added code to handle case where dataframe created by to_dataframe is empty due to empty StreamSet(s)? --- btrdb/transformers.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/btrdb/transformers.py b/btrdb/transformers.py index 12f7979..1cf346c 100644 --- a/btrdb/transformers.py +++ b/btrdb/transformers.py @@ -145,13 +145,15 @@ def to_dataframe(streamset, columns=None, agg="mean", name_callable=None): df = pd.DataFrame(to_dict(streamset,agg=agg)) - df = df.set_index("time") - if agg == "all" and not streamset.allow_window: - stream_names = [[s.collection, s.name, prop] for s in streamset._streams for prop in _STAT_PROPERTIES] - df.columns=pd.MultiIndex.from_tuples(stream_names) - else: - df.columns = columns if columns else _stream_names(streamset, name_callable) + if not df.empty: + df = df.set_index("time") + + if agg == "all" and not streamset.allow_window: + stream_names = [[s.collection, s.name, prop] for s in streamset._streams for prop in _STAT_PROPERTIES] + df.columns=pd.MultiIndex.from_tuples(stream_names) + else: + df.columns = columns if columns else _stream_names(streamset, name_callable) return df From e1a7ef51f1ea82761f8a29019dbe5c192cab0756 Mon Sep 17 00:00:00 2001 From: looselycoupled Date: Fri, 29 Oct 2021 11:25:02 -0400 Subject: [PATCH 029/131] Release v5.12.0 --- btrdb/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/btrdb/version.py b/btrdb/version.py index 4bff3e4..2c4489d 100644 --- a/btrdb/version.py +++ b/btrdb/version.py @@ -15,7 +15,7 @@ ## Module Info ########################################################################## -__version_info__ = { 'major': 5, 'minor': 11, 'micro': 7, 'releaselevel': 'final'} +__version_info__ = { 'major': 5, 'minor': 12, 'micro': 0, 'releaselevel': 'final'} ########################################################################## ## Helper Functions From d0aec0db3babd8b2d4bac92a3e626e0f466ac65f Mon Sep 17 00:00:00 2001 From: David <72822263+davidkonigsberg@users.noreply.github.com> Date: Tue, 25 Jan 2022 16:35:00 -0500 Subject: [PATCH 030/131] Release v5.15.1 --- btrdb/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/btrdb/version.py b/btrdb/version.py index 2c4489d..ba57564 100644 --- a/btrdb/version.py +++ b/btrdb/version.py @@ -15,7 +15,7 @@ ## Module Info ########################################################################## -__version_info__ = { 'major': 5, 'minor': 12, 'micro': 0, 'releaselevel': 'final'} +__version_info__ = { 'major': 5, 'minor': 15, 'micro': 1, 'releaselevel': 'final'} ########################################################################## ## Helper Functions From 479ed3365758ddc10ef12434d4c2efd7fd3929c5 Mon Sep 17 00:00:00 2001 From: Jeff Lin <42981468+jleifnf@users.noreply.github.com> Date: Wed, 25 May 2022 14:46:26 -0700 Subject: [PATCH 031/131] Upgrade ray versions (#15) * Release v5.15.0 * update register_serializer for ray to allow ray>=1.2 * fix duplicated import for functools in ray.py * update pytest-flake8 to 1.1.0 to fix StringIO wrapper error with FLAKE8 and fix indentations Co-authored-by: David Konigsberg <72822263+davidkonigsberg@users.noreply.github.com> --- btrdb/utils/ray.py | 15 +++++++++------ tests/requirements.txt | 2 +- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/btrdb/utils/ray.py b/btrdb/utils/ray.py index 674ab64..e83ec2d 100644 --- a/btrdb/utils/ray.py +++ b/btrdb/utils/ray.py @@ -2,6 +2,7 @@ from btrdb.conn import BTrDB from functools import partial + def register_serializer(conn_str=None, apikey=None, profile=None): """ Register serializer for BTrDB Object @@ -27,19 +28,21 @@ def register_serializer(conn_str=None, apikey=None, profile=None): import semver except ImportError: raise ImportError("must pip install semver to register custom serializer") - + assert ray.is_initialized(), "Need to call ray.init() before registering custom serializer" - # TODO: check the version using the 'semver' package? ver = semver.VersionInfo.parse(ray.__version__) if ver.major == 0: ray.register_custom_serializer( - BTrDB, serializer=btrdb_serializer, deserializer=partial(btrdb_deserializer, conn_str=conn_str, apikey=apikey, profile=profile)) - elif ver.major == 1 and ver.minor in range(2, 4): + BTrDB, serializer=btrdb_serializer, + deserializer=partial(btrdb_deserializer, conn_str=conn_str, apikey=apikey, profile=profile)) + elif ver.major == 1 and ver.minor in range(2, 13): # as of May 2022, latest Ray version is at 1.12 # TODO: check different versions of ray? ray.util.register_serializer( - BTrDB, serializer=btrdb_serializer, deserializer=partial(btrdb_deserializer, conn_str=conn_str, apikey=apikey, profile=profile)) + BTrDB, serializer=btrdb_serializer, + deserializer=partial(btrdb_deserializer, conn_str=conn_str, apikey=apikey, profile=profile)) else: - raise Exception("Ray version %s does not have custom serialization. Please upgrade to >= 1.2.0" % ray.__version__) + raise Exception("Ray version %s does not have custom serialization. Please upgrade to >= 1.4.0" % ray.__version__) + def btrdb_serializer(_): """ diff --git a/tests/requirements.txt b/tests/requirements.txt index 2d03e46..2fe0fc4 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,6 +1,6 @@ pytest==5.3.2 pytest-cov==2.8.1 -pytest-flake8==1.0.7 +pytest-flake8==1.1.0 freezegun==0.3.12 # dependencies for conversions From 74e756727aba905e1e7e272bc787fafbffd59465 Mon Sep 17 00:00:00 2001 From: Alexander Young Date: Thu, 9 Mar 2023 17:18:49 -0600 Subject: [PATCH 032/131] Release v5.28.1 and Update Python (#17) * Release v5.28.1 * add py3.9 and remove py3.6 (which is no longer supported * remove extraenous version bak * Pin grpc and grpc-tools to compatible versions * Pin flake8 so the tests hopefully work * Pin numpy so the tests hopefully work --- .github/workflows/release.yaml | 4 ++-- btrdb/version.py | 2 +- requirements.txt | 6 +++--- tests/requirements.txt | 3 ++- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 87de83e..61de84e 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -14,7 +14,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: [3.6, 3.7, 3.8] + python-version: [3.7, 3.8, 3.9] os: [ubuntu-latest, macos-latest, windows-latest] steps: @@ -75,4 +75,4 @@ jobs: uses: pypa/gh-action-pypi-publish@master with: user: __token__ - password: ${{ secrets.PYPI_DEPLOYMENT_TOKEN }} \ No newline at end of file + password: ${{ secrets.PYPI_DEPLOYMENT_TOKEN }} diff --git a/btrdb/version.py b/btrdb/version.py index ba57564..2592e90 100644 --- a/btrdb/version.py +++ b/btrdb/version.py @@ -15,7 +15,7 @@ ## Module Info ########################################################################## -__version_info__ = { 'major': 5, 'minor': 15, 'micro': 1, 'releaselevel': 'final'} +__version_info__ = { 'major': 5, 'minor': 28, 'micro': 1, 'releaselevel': 'final'} ########################################################################## ## Helper Functions diff --git a/requirements.txt b/requirements.txt index 0d89f56..8b35bcf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,10 @@ # GRPC / Protobuff related -grpcio>=1.19.0 -grpcio-tools>=1.19.0 +grpcio>=1.19.0,<=1.48.2 +grpcio-tools>=1.19.0,<=1.48.2 # Time related utils pytz # Misc libraries pyyaml -certifi \ No newline at end of file +certifi diff --git a/tests/requirements.txt b/tests/requirements.txt index 2fe0fc4..534ff57 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -2,8 +2,9 @@ pytest==5.3.2 pytest-cov==2.8.1 pytest-flake8==1.1.0 freezegun==0.3.12 +flake8<5.0.0 # dependencies for conversions -numpy>=1.15 +numpy==1.20.3 pandas tabulate==0.8.6 From f38157d96093f84e2ba2c83894ed8093ff366408 Mon Sep 17 00:00:00 2001 From: Justin Gilmer Date: Tue, 6 Jun 2023 16:57:48 -0500 Subject: [PATCH 033/131] Threadpool executor (#22) * Release v5.15.0 * update protobuf to v4.22.3 * Add threaded streamset calls Using concurrent.futures.ThreadPoolExecutor * Blacken code * Update for failing tests * Ignore flake8 as part of testing pytest-flake8 seems to have issues with the later versions of flake8 https://github.com/tholo/pytest-flake8/issues/92 * Update .gitignore * Update ignore and remove extra print. * Remove idea folder (pycharm) --------- Co-authored-by: David Konigsberg <72822263+davidkonigsberg@users.noreply.github.com> Co-authored-by: Jeff Lin <42981468+jleifnf@users.noreply.github.com> --- .gitignore | 5 + btrdb/conn.py | 68 +- btrdb/grpcinterface/README.md | 6 + btrdb/grpcinterface/btrdb_pb2.py | 3679 +------------------------ btrdb/grpcinterface/btrdb_pb2.pyi | 632 +++++ btrdb/grpcinterface/btrdb_pb2_grpc.py | 1039 ++++--- btrdb/stream.py | 241 +- btrdb/transformers.py | 4 +- setup.cfg | 10 +- tests/btrdb/test_stream.py | 8 +- 10 files changed, 1684 insertions(+), 4008 deletions(-) create mode 100644 btrdb/grpcinterface/README.md create mode 100644 btrdb/grpcinterface/btrdb_pb2.pyi diff --git a/.gitignore b/.gitignore index e7c86be..e703d03 100644 --- a/.gitignore +++ b/.gitignore @@ -118,3 +118,8 @@ dmypy.json # Pyre type checker .pyre/ +.idea/misc.xml +.idea/vcs.xml +.idea/inspectionProfiles/profiles_settings.xml +.idea/inspectionProfiles/Project_Default.xml +/.idea/ diff --git a/btrdb/conn.py b/btrdb/conn.py index bb79f2f..5499fa7 100644 --- a/btrdb/conn.py +++ b/btrdb/conn.py @@ -20,6 +20,7 @@ import json import certifi import uuid as uuidlib +from concurrent.futures import ThreadPoolExecutor import grpc from grpc._cython.cygrpc import CompressionAlgorithm @@ -42,8 +43,8 @@ ## Classes ########################################################################## -class Connection(object): +class Connection(object): def __init__(self, addrportstr, apikey=None): """ Connects to a BTrDB server @@ -57,7 +58,7 @@ def __init__(self, addrportstr, apikey=None): """ addrport = addrportstr.split(":", 2) - chan_ops = [('grpc.default_compression_algorithm', CompressionAlgorithm.gzip)] + chan_ops = [("grpc.default_compression_algorithm", CompressionAlgorithm.gzip)] if len(addrport) != 2: raise ValueError("expecting address:port") @@ -82,7 +83,10 @@ def __init__(self, addrportstr, apikey=None): except Exception: if env_bundle != "": # The user has given us something but we can't use it, we need to make noise - raise Exception("BTRDB_CA_BUNDLE(%s) env is defined but could not read file" % ca_bundle) + raise Exception( + "BTRDB_CA_BUNDLE(%s) env is defined but could not read file" + % ca_bundle + ) else: contents = None @@ -90,25 +94,25 @@ def __init__(self, addrportstr, apikey=None): self.channel = grpc.secure_channel( addrportstr, grpc.ssl_channel_credentials(contents), - options=chan_ops + options=chan_ops, ) else: self.channel = grpc.secure_channel( addrportstr, grpc.composite_channel_credentials( grpc.ssl_channel_credentials(contents), - grpc.access_token_call_credentials(apikey) + grpc.access_token_call_credentials(apikey), ), - options=chan_ops + options=chan_ops, ) else: if apikey is not None: - raise ValueError("cannot use an API key with an insecure (port 4410) BTrDB API. Try port 4411") + raise ValueError( + "cannot use an API key with an insecure (port 4410) BTrDB API. Try port 4411" + ) self.channel = grpc.insecure_channel(addrportstr, chan_ops) - - class BTrDB(object): """ The primary server connection object for communicating with a BTrDB server. @@ -116,6 +120,7 @@ class BTrDB(object): def __init__(self, endpoint): self.ep = endpoint + self._executor = ThreadPoolExecutor() def query(self, stmt, params=[]): """ @@ -153,7 +158,6 @@ def query(self, stmt, params=[]): for row in page ] - def streams(self, *identifiers, versions=None, is_collection_prefix=False): """ Returns a StreamSet object with BTrDB streams from the supplied @@ -196,20 +200,23 @@ def streams(self, *identifiers, versions=None, is_collection_prefix=False): found = self.streams_in_collection( "/".join(parts[:-1]), is_collection_prefix=is_collection_prefix, - tags={"name": parts[-1]} + tags={"name": parts[-1]}, ) if len(found) == 1: streams.append(found[0]) continue raise StreamNotFoundError(f"Could not identify stream `{ident}`") - raise ValueError(f"Could not identify stream based on `{ident}`. Identifier must be UUID or collection/name.") - + raise ValueError( + f"Could not identify stream based on `{ident}`. Identifier must be UUID or collection/name." + ) obj = StreamSet(streams) if versions: - version_dict = {streams[idx].uuid: versions[idx] for idx in range(len(versions))} + version_dict = { + streams[idx].uuid: versions[idx] for idx in range(len(versions)) + } obj.pin_versions(version_dict) return obj @@ -257,12 +264,14 @@ def create(self, uuid, collection, tags=None, annotations=None): annotations = {} self.ep.create(uuid, collection, tags, annotations) - return Stream(self, uuid, + return Stream( + self, + uuid, known_to_exist=True, collection=collection, tags=tags.copy(), annotations=annotations.copy(), - property_version=0 + property_version=0, ) def info(self): @@ -279,7 +288,7 @@ def info(self): return { "majorVersion": info.majorVersion, "build": info.build, - "proxy": { "proxyEndpoints": [ep for ep in info.proxy.proxyEndpoints] }, + "proxy": {"proxyEndpoints": [ep for ep in info.proxy.proxyEndpoints]}, } def list_collections(self, starts_with=""): @@ -294,7 +303,9 @@ def list_collections(self, starts_with=""): """ return [c for some in self.ep.listCollections(starts_with) for c in some] - def streams_in_collection(self, *collection, is_collection_prefix=True, tags=None, annotations=None): + def streams_in_collection( + self, *collection, is_collection_prefix=True, tags=None, annotations=None + ): """ Search for streams matching given parameters @@ -329,16 +340,23 @@ def streams_in_collection(self, *collection, is_collection_prefix=True, tags=Non collection = [None] for item in collection: - streams = self.ep.lookupStreams(item, is_collection_prefix, tags, annotations) + streams = self.ep.lookupStreams( + item, is_collection_prefix, tags, annotations + ) for desclist in streams: for desc in desclist: tagsanns = unpack_stream_descriptor(desc) - result.append(Stream( - self, uuidlib.UUID(bytes = desc.uuid), - known_to_exist=True, collection=desc.collection, - tags=tagsanns[0], annotations=tagsanns[1], - property_version=desc.propertyVersion - )) + result.append( + Stream( + self, + uuidlib.UUID(bytes=desc.uuid), + known_to_exist=True, + collection=desc.collection, + tags=tagsanns[0], + annotations=tagsanns[1], + property_version=desc.propertyVersion, + ) + ) return result diff --git a/btrdb/grpcinterface/README.md b/btrdb/grpcinterface/README.md new file mode 100644 index 0000000..735571a --- /dev/null +++ b/btrdb/grpcinterface/README.md @@ -0,0 +1,6 @@ +To create new protobuf files in this folder: +```commandline +python -m grpc_tools.protoc -I. --python_out=. --pyi_out=. --grpc_python_out=. ./btrdb.proto +``` + +Make sure the proto file is newest. \ No newline at end of file diff --git a/btrdb/grpcinterface/btrdb_pb2.py b/btrdb/grpcinterface/btrdb_pb2.py index 8104a34..a98905b 100644 --- a/btrdb/grpcinterface/btrdb_pb2.py +++ b/btrdb/grpcinterface/btrdb_pb2.py @@ -1,12 +1,10 @@ +# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: btrdb.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf.internal import enum_type_wrapper +"""Generated protocol buffer code.""" +from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection +from google.protobuf import descriptor_pool as _descriptor_pool from google.protobuf import symbol_database as _symbol_database # @@protoc_insertion_point(imports) @@ -15,3542 +13,135 @@ -DESCRIPTOR = _descriptor.FileDescriptor( - name='btrdb.proto', - package='v5api', - syntax='proto3', - serialized_options=None, - serialized_pb=_b('\n\x0b\x62trdb.proto\x12\x05v5api\"Q\n\x0fRawValuesParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\"}\n\x11RawValuesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x1f\n\x06values\x18\x04 \x03(\x0b\x32\x0f.v5api.RawPoint\"j\n\x14\x41lignedWindowsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\x12\x12\n\npointWidth\x18\x05 \x01(\r\"\x83\x01\n\x16\x41lignedWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12 \n\x06values\x18\x04 \x03(\x0b\x32\x10.v5api.StatPoint\"m\n\rWindowsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\x12\r\n\x05width\x18\x05 \x01(\x04\x12\r\n\x05\x64\x65pth\x18\x06 \x01(\r\"|\n\x0fWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12 \n\x06values\x18\x04 \x03(\x0b\x32\x10.v5api.StatPoint\"h\n\x10StreamInfoParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x13\n\x0bomitVersion\x18\x02 \x01(\x08\x12\x16\n\x0eomitDescriptor\x18\x03 \x01(\x08\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"\x8a\x01\n\x12StreamInfoResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12+\n\ndescriptor\x18\x04 \x01(\x0b\x32\x17.v5api.StreamDescriptor\"\x98\x01\n\x10StreamDescriptor\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x12\n\ncollection\x18\x02 \x01(\t\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x17\n\x0fpropertyVersion\x18\x05 \x01(\x04\"\x82\x01\n\x1aSetStreamAnnotationsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x1f\n\x17\x65xpectedPropertyVersion\x18\x02 \x01(\x04\x12#\n\x07\x63hanges\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x10\n\x08removals\x18\x04 \x03(\t\";\n\x1cSetStreamAnnotationsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"\x8a\x01\n\x13SetStreamTagsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x1f\n\x17\x65xpectedPropertyVersion\x18\x02 \x01(\x04\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x12\n\ncollection\x18\x04 \x01(\t\x12\x0e\n\x06remove\x18\x05 \x03(\t\"4\n\x15SetStreamTagsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"{\n\x0c\x43reateParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x12\n\ncollection\x18\x02 \x01(\t\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\"-\n\x0e\x43reateResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"@\n\x13MetadataUsageParams\x12\x0e\n\x06prefix\x18\x01 \x01(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"y\n\x15MetadataUsageResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x1d\n\x04tags\x18\x02 \x03(\x0b\x32\x0f.v5api.KeyCount\x12$\n\x0b\x61nnotations\x18\x03 \x03(\x0b\x32\x0f.v5api.KeyCount\"&\n\x08KeyCount\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05\x63ount\x18\x02 \x01(\x04\"B\n\x15ListCollectionsParams\x12\x0e\n\x06prefix\x18\x01 \x01(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"K\n\x17ListCollectionsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x13\n\x0b\x63ollections\x18\x02 \x03(\t\"\xab\x01\n\x13LookupStreamsParams\x12\x12\n\ncollection\x18\x01 \x01(\t\x12\x1a\n\x12isCollectionPrefix\x18\x02 \x01(\x08\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"^\n\x15LookupStreamsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12(\n\x07results\x18\x02 \x03(\x0b\x32\x17.v5api.StreamDescriptor\"S\n\rNearestParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x0c\n\x04time\x18\x02 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x03 \x01(\x04\x12\x10\n\x08\x62\x61\x63kward\x18\x04 \x01(\x08\"z\n\x0fNearestResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x1e\n\x05value\x18\x04 \x01(\x0b\x32\x0f.v5api.RawPoint\"U\n\rChangesParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x11\n\tfromMajor\x18\x02 \x01(\x04\x12\x0f\n\x07toMajor\x18\x03 \x01(\x04\x12\x12\n\nresolution\x18\x04 \x01(\r\"\x7f\n\x0f\x43hangesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12#\n\x06ranges\x18\x04 \x03(\x0b\x32\x13.v5api.ChangedRange\"u\n\x0cInsertParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x0c\n\x04sync\x18\x02 \x01(\x08\x12(\n\x0cmerge_policy\x18\x04 \x01(\x0e\x32\x12.v5api.MergePolicy\x12\x1f\n\x06values\x18\x03 \x03(\x0b\x32\x0f.v5api.RawPoint\"Y\n\x0eInsertResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\"8\n\x0c\x44\x65leteParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\"Y\n\x0e\x44\x65leteResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\"\x0c\n\nInfoParams\"\xa2\x01\n\x0cInfoResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x19\n\x04mash\x18\x02 \x01(\x0b\x32\x0b.v5api.Mash\x12\x14\n\x0cmajorVersion\x18\x03 \x01(\r\x12\x14\n\x0cminorVersion\x18\x04 \x01(\r\x12\r\n\x05\x62uild\x18\x05 \x01(\t\x12\x1f\n\x05proxy\x18\x06 \x01(\x0b\x32\x10.v5api.ProxyInfo\"#\n\tProxyInfo\x12\x16\n\x0eproxyEndpoints\x18\x01 \x03(\t\"1\n\x11\x46\x61ultInjectParams\x12\x0c\n\x04type\x18\x01 \x01(\x04\x12\x0e\n\x06params\x18\x02 \x01(\x0c\">\n\x13\x46\x61ultInjectResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\n\n\x02rv\x18\x02 \x01(\x0c\"\x1b\n\x0b\x46lushParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"X\n\rFlushResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\" \n\x10ObliterateParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"1\n\x12ObliterateResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"\'\n\x08RawPoint\x12\x0c\n\x04time\x18\x01 \x01(\x10\x12\r\n\x05value\x18\x02 \x01(\x01\"`\n\tStatPoint\x12\x0c\n\x04time\x18\x01 \x01(\x10\x12\x0b\n\x03min\x18\x02 \x01(\x01\x12\x0c\n\x04mean\x18\x03 \x01(\x01\x12\x0b\n\x03max\x18\x04 \x01(\x01\x12\r\n\x05\x63ount\x18\x05 \x01(\x06\x12\x0e\n\x06stddev\x18\x06 \x01(\x01\"*\n\x0c\x43hangedRange\x12\r\n\x05start\x18\x01 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x02 \x01(\x10\">\n\x06Status\x12\x0c\n\x04\x63ode\x18\x01 \x01(\r\x12\x0b\n\x03msg\x18\x02 \x01(\t\x12\x19\n\x04mash\x18\x03 \x01(\x0b\x32\x0b.v5api.Mash\"\x98\x01\n\x04Mash\x12\x10\n\x08revision\x18\x01 \x01(\x03\x12\x0e\n\x06leader\x18\x02 \x01(\t\x12\x16\n\x0eleaderRevision\x18\x03 \x01(\x03\x12\x13\n\x0btotalWeight\x18\x04 \x01(\x03\x12\x0f\n\x07healthy\x18\x05 \x01(\x08\x12\x10\n\x08unmapped\x18\x06 \x01(\x01\x12\x1e\n\x07members\x18\x07 \x03(\x0b\x32\r.v5api.Member\"\xc3\x01\n\x06Member\x12\x0c\n\x04hash\x18\x01 \x01(\r\x12\x10\n\x08nodename\x18\x02 \x01(\t\x12\n\n\x02up\x18\x03 \x01(\x08\x12\n\n\x02in\x18\x04 \x01(\x08\x12\x0f\n\x07\x65nabled\x18\x05 \x01(\x08\x12\r\n\x05start\x18\x06 \x01(\x03\x12\x0b\n\x03\x65nd\x18\x07 \x01(\x03\x12\x0e\n\x06weight\x18\x08 \x01(\x03\x12\x16\n\x0ereadPreference\x18\t \x01(\x01\x12\x15\n\rhttpEndpoints\x18\n \x01(\t\x12\x15\n\rgrpcEndpoints\x18\x0b \x01(\t\"8\n\x0bKeyOptValue\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x1c\n\x03val\x18\x02 \x01(\x0b\x32\x0f.v5api.OptValue\"\x19\n\x08OptValue\x12\r\n\x05value\x18\x01 \x01(\t\"&\n\x08KeyValue\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"?\n\x0fStreamCSVConfig\x12\x0f\n\x07version\x18\x01 \x01(\x04\x12\r\n\x05label\x18\x02 \x01(\t\x12\x0c\n\x04uuid\x18\x03 \x01(\x0c\"\x9d\x02\n\x11GenerateCSVParams\x12\x35\n\tqueryType\x18\x01 \x01(\x0e\x32\".v5api.GenerateCSVParams.QueryType\x12\x11\n\tstartTime\x18\x02 \x01(\x03\x12\x0f\n\x07\x65ndTime\x18\x03 \x01(\x03\x12\x12\n\nwindowSize\x18\x04 \x01(\x04\x12\r\n\x05\x64\x65pth\x18\x05 \x01(\r\x12\x17\n\x0fincludeVersions\x18\x06 \x01(\x08\x12\'\n\x07streams\x18\x07 \x03(\x0b\x32\x16.v5api.StreamCSVConfig\"H\n\tQueryType\x12\x19\n\x15\x41LIGNED_WINDOWS_QUERY\x10\x00\x12\x11\n\rWINDOWS_QUERY\x10\x01\x12\r\n\tRAW_QUERY\x10\x02\"Q\n\x13GenerateCSVResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x10\n\x08isHeader\x18\x02 \x01(\x08\x12\x0b\n\x03row\x18\x03 \x03(\t\"J\n\x0eSQLQueryParams\x12\r\n\x05query\x18\x01 \x01(\t\x12\x0e\n\x06params\x18\x02 \x03(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"D\n\x10SQLQueryResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x13\n\x0bSQLQueryRow\x18\x02 \x03(\x0c\"\x14\n\x04Role\x12\x0c\n\x04name\x18\x01 \x01(\t\"\x94\x01\n\x19SetCompactionConfigParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x18\n\x10\x43ompactedVersion\x18\x02 \x01(\x04\x12>\n\x17reducedResolutionRanges\x18\x03 \x03(\x0b\x32\x1d.v5api.ReducedResolutionRange\x12\x0f\n\x07unused0\x18\x04 \x01(\x04\":\n\x1bSetCompactionConfigResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\")\n\x19GetCompactionConfigParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"\xc1\x01\n\x1bGetCompactionConfigResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x1a\n\x12LatestMajorVersion\x18\x02 \x01(\x04\x12\x18\n\x10\x43ompactedVersion\x18\x03 \x01(\x04\x12>\n\x17reducedResolutionRanges\x18\x04 \x03(\x0b\x32\x1d.v5api.ReducedResolutionRange\x12\x0f\n\x07unused0\x18\x05 \x01(\x04\"H\n\x16ReducedResolutionRange\x12\r\n\x05Start\x18\x01 \x01(\x03\x12\x0b\n\x03\x45nd\x18\x02 \x01(\x03\x12\x12\n\nResolution\x18\x03 \x01(\r*<\n\x0bMergePolicy\x12\t\n\x05NEVER\x10\x00\x12\t\n\x05\x45QUAL\x10\x01\x12\n\n\x06RETAIN\x10\x02\x12\x0b\n\x07REPLACE\x10\x03\x32\xb3\n\n\x05\x42TrDB\x12?\n\tRawValues\x12\x16.v5api.RawValuesParams\x1a\x18.v5api.RawValuesResponse0\x01\x12N\n\x0e\x41lignedWindows\x12\x1b.v5api.AlignedWindowsParams\x1a\x1d.v5api.AlignedWindowsResponse0\x01\x12\x39\n\x07Windows\x12\x14.v5api.WindowsParams\x1a\x16.v5api.WindowsResponse0\x01\x12@\n\nStreamInfo\x12\x17.v5api.StreamInfoParams\x1a\x19.v5api.StreamInfoResponse\x12^\n\x14SetStreamAnnotations\x12!.v5api.SetStreamAnnotationsParams\x1a#.v5api.SetStreamAnnotationsResponse\x12I\n\rSetStreamTags\x12\x1a.v5api.SetStreamTagsParams\x1a\x1c.v5api.SetStreamTagsResponse\x12\x34\n\x06\x43reate\x12\x13.v5api.CreateParams\x1a\x15.v5api.CreateResponse\x12Q\n\x0fListCollections\x12\x1c.v5api.ListCollectionsParams\x1a\x1e.v5api.ListCollectionsResponse0\x01\x12K\n\rLookupStreams\x12\x1a.v5api.LookupStreamsParams\x1a\x1c.v5api.LookupStreamsResponse0\x01\x12\x37\n\x07Nearest\x12\x14.v5api.NearestParams\x1a\x16.v5api.NearestResponse\x12\x39\n\x07\x43hanges\x12\x14.v5api.ChangesParams\x1a\x16.v5api.ChangesResponse0\x01\x12\x34\n\x06Insert\x12\x13.v5api.InsertParams\x1a\x15.v5api.InsertResponse\x12\x34\n\x06\x44\x65lete\x12\x13.v5api.DeleteParams\x1a\x15.v5api.DeleteResponse\x12.\n\x04Info\x12\x11.v5api.InfoParams\x1a\x13.v5api.InfoResponse\x12\x43\n\x0b\x46\x61ultInject\x12\x18.v5api.FaultInjectParams\x1a\x1a.v5api.FaultInjectResponse\x12\x31\n\x05\x46lush\x12\x12.v5api.FlushParams\x1a\x14.v5api.FlushResponse\x12@\n\nObliterate\x12\x17.v5api.ObliterateParams\x1a\x19.v5api.ObliterateResponse\x12L\n\x10GetMetadataUsage\x12\x1a.v5api.MetadataUsageParams\x1a\x1c.v5api.MetadataUsageResponse\x12\x45\n\x0bGenerateCSV\x12\x18.v5api.GenerateCSVParams\x1a\x1a.v5api.GenerateCSVResponse0\x01\x12<\n\x08SQLQuery\x12\x15.v5api.SQLQueryParams\x1a\x17.v5api.SQLQueryResponse0\x01\x62\x06proto3') -) - -_MERGEPOLICY = _descriptor.EnumDescriptor( - name='MergePolicy', - full_name='v5api.MergePolicy', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='NEVER', index=0, number=0, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='EQUAL', index=1, number=1, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='RETAIN', index=2, number=2, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='REPLACE', index=3, number=3, - serialized_options=None, - type=None), - ], - containing_type=None, - serialized_options=None, - serialized_start=5490, - serialized_end=5550, -) -_sym_db.RegisterEnumDescriptor(_MERGEPOLICY) - -MergePolicy = enum_type_wrapper.EnumTypeWrapper(_MERGEPOLICY) -NEVER = 0 -EQUAL = 1 -RETAIN = 2 -REPLACE = 3 - - -_GENERATECSVPARAMS_QUERYTYPE = _descriptor.EnumDescriptor( - name='QueryType', - full_name='v5api.GenerateCSVParams.QueryType', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='ALIGNED_WINDOWS_QUERY', index=0, number=0, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='WINDOWS_QUERY', index=1, number=1, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='RAW_QUERY', index=2, number=2, - serialized_options=None, - type=None), - ], - containing_type=None, - serialized_options=None, - serialized_start=4641, - serialized_end=4713, -) -_sym_db.RegisterEnumDescriptor(_GENERATECSVPARAMS_QUERYTYPE) - - -_RAWVALUESPARAMS = _descriptor.Descriptor( - name='RawValuesParams', - full_name='v5api.RawValuesParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.RawValuesParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='start', full_name='v5api.RawValuesParams.start', index=1, - number=2, type=16, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='end', full_name='v5api.RawValuesParams.end', index=2, - number=3, type=16, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMajor', full_name='v5api.RawValuesParams.versionMajor', index=3, - number=4, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=22, - serialized_end=103, -) - - -_RAWVALUESRESPONSE = _descriptor.Descriptor( - name='RawValuesResponse', - full_name='v5api.RawValuesResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.RawValuesResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMajor', full_name='v5api.RawValuesResponse.versionMajor', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMinor', full_name='v5api.RawValuesResponse.versionMinor', index=2, - number=3, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='values', full_name='v5api.RawValuesResponse.values', index=3, - number=4, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=105, - serialized_end=230, -) - - -_ALIGNEDWINDOWSPARAMS = _descriptor.Descriptor( - name='AlignedWindowsParams', - full_name='v5api.AlignedWindowsParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.AlignedWindowsParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='start', full_name='v5api.AlignedWindowsParams.start', index=1, - number=2, type=16, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='end', full_name='v5api.AlignedWindowsParams.end', index=2, - number=3, type=16, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMajor', full_name='v5api.AlignedWindowsParams.versionMajor', index=3, - number=4, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='pointWidth', full_name='v5api.AlignedWindowsParams.pointWidth', index=4, - number=5, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=232, - serialized_end=338, -) - - -_ALIGNEDWINDOWSRESPONSE = _descriptor.Descriptor( - name='AlignedWindowsResponse', - full_name='v5api.AlignedWindowsResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.AlignedWindowsResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMajor', full_name='v5api.AlignedWindowsResponse.versionMajor', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMinor', full_name='v5api.AlignedWindowsResponse.versionMinor', index=2, - number=3, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='values', full_name='v5api.AlignedWindowsResponse.values', index=3, - number=4, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=341, - serialized_end=472, -) - - -_WINDOWSPARAMS = _descriptor.Descriptor( - name='WindowsParams', - full_name='v5api.WindowsParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.WindowsParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='start', full_name='v5api.WindowsParams.start', index=1, - number=2, type=16, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='end', full_name='v5api.WindowsParams.end', index=2, - number=3, type=16, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMajor', full_name='v5api.WindowsParams.versionMajor', index=3, - number=4, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='width', full_name='v5api.WindowsParams.width', index=4, - number=5, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='depth', full_name='v5api.WindowsParams.depth', index=5, - number=6, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=474, - serialized_end=583, -) - - -_WINDOWSRESPONSE = _descriptor.Descriptor( - name='WindowsResponse', - full_name='v5api.WindowsResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.WindowsResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMajor', full_name='v5api.WindowsResponse.versionMajor', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMinor', full_name='v5api.WindowsResponse.versionMinor', index=2, - number=3, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='values', full_name='v5api.WindowsResponse.values', index=3, - number=4, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=585, - serialized_end=709, -) - - -_STREAMINFOPARAMS = _descriptor.Descriptor( - name='StreamInfoParams', - full_name='v5api.StreamInfoParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.StreamInfoParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='omitVersion', full_name='v5api.StreamInfoParams.omitVersion', index=1, - number=2, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='omitDescriptor', full_name='v5api.StreamInfoParams.omitDescriptor', index=2, - number=3, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='role', full_name='v5api.StreamInfoParams.role', index=3, - number=100, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=711, - serialized_end=815, -) - - -_STREAMINFORESPONSE = _descriptor.Descriptor( - name='StreamInfoResponse', - full_name='v5api.StreamInfoResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.StreamInfoResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMajor', full_name='v5api.StreamInfoResponse.versionMajor', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMinor', full_name='v5api.StreamInfoResponse.versionMinor', index=2, - number=3, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='descriptor', full_name='v5api.StreamInfoResponse.descriptor', index=3, - number=4, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=818, - serialized_end=956, -) - - -_STREAMDESCRIPTOR = _descriptor.Descriptor( - name='StreamDescriptor', - full_name='v5api.StreamDescriptor', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.StreamDescriptor.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='collection', full_name='v5api.StreamDescriptor.collection', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='tags', full_name='v5api.StreamDescriptor.tags', index=2, - number=3, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='annotations', full_name='v5api.StreamDescriptor.annotations', index=3, - number=4, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='propertyVersion', full_name='v5api.StreamDescriptor.propertyVersion', index=4, - number=5, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=959, - serialized_end=1111, -) - - -_SETSTREAMANNOTATIONSPARAMS = _descriptor.Descriptor( - name='SetStreamAnnotationsParams', - full_name='v5api.SetStreamAnnotationsParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.SetStreamAnnotationsParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='expectedPropertyVersion', full_name='v5api.SetStreamAnnotationsParams.expectedPropertyVersion', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='changes', full_name='v5api.SetStreamAnnotationsParams.changes', index=2, - number=3, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='removals', full_name='v5api.SetStreamAnnotationsParams.removals', index=3, - number=4, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1114, - serialized_end=1244, -) - - -_SETSTREAMANNOTATIONSRESPONSE = _descriptor.Descriptor( - name='SetStreamAnnotationsResponse', - full_name='v5api.SetStreamAnnotationsResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.SetStreamAnnotationsResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1246, - serialized_end=1305, -) - - -_SETSTREAMTAGSPARAMS = _descriptor.Descriptor( - name='SetStreamTagsParams', - full_name='v5api.SetStreamTagsParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.SetStreamTagsParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='expectedPropertyVersion', full_name='v5api.SetStreamTagsParams.expectedPropertyVersion', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='tags', full_name='v5api.SetStreamTagsParams.tags', index=2, - number=3, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='collection', full_name='v5api.SetStreamTagsParams.collection', index=3, - number=4, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='remove', full_name='v5api.SetStreamTagsParams.remove', index=4, - number=5, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1308, - serialized_end=1446, -) - - -_SETSTREAMTAGSRESPONSE = _descriptor.Descriptor( - name='SetStreamTagsResponse', - full_name='v5api.SetStreamTagsResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.SetStreamTagsResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1448, - serialized_end=1500, -) - - -_CREATEPARAMS = _descriptor.Descriptor( - name='CreateParams', - full_name='v5api.CreateParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.CreateParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='collection', full_name='v5api.CreateParams.collection', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='tags', full_name='v5api.CreateParams.tags', index=2, - number=3, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='annotations', full_name='v5api.CreateParams.annotations', index=3, - number=4, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1502, - serialized_end=1625, -) - - -_CREATERESPONSE = _descriptor.Descriptor( - name='CreateResponse', - full_name='v5api.CreateResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.CreateResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1627, - serialized_end=1672, -) - - -_METADATAUSAGEPARAMS = _descriptor.Descriptor( - name='MetadataUsageParams', - full_name='v5api.MetadataUsageParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='prefix', full_name='v5api.MetadataUsageParams.prefix', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='role', full_name='v5api.MetadataUsageParams.role', index=1, - number=100, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1674, - serialized_end=1738, -) - - -_METADATAUSAGERESPONSE = _descriptor.Descriptor( - name='MetadataUsageResponse', - full_name='v5api.MetadataUsageResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.MetadataUsageResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='tags', full_name='v5api.MetadataUsageResponse.tags', index=1, - number=2, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='annotations', full_name='v5api.MetadataUsageResponse.annotations', index=2, - number=3, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1740, - serialized_end=1861, -) - - -_KEYCOUNT = _descriptor.Descriptor( - name='KeyCount', - full_name='v5api.KeyCount', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='key', full_name='v5api.KeyCount.key', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='count', full_name='v5api.KeyCount.count', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1863, - serialized_end=1901, -) - - -_LISTCOLLECTIONSPARAMS = _descriptor.Descriptor( - name='ListCollectionsParams', - full_name='v5api.ListCollectionsParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='prefix', full_name='v5api.ListCollectionsParams.prefix', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='role', full_name='v5api.ListCollectionsParams.role', index=1, - number=100, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1903, - serialized_end=1969, -) - - -_LISTCOLLECTIONSRESPONSE = _descriptor.Descriptor( - name='ListCollectionsResponse', - full_name='v5api.ListCollectionsResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.ListCollectionsResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='collections', full_name='v5api.ListCollectionsResponse.collections', index=1, - number=2, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1971, - serialized_end=2046, -) - - -_LOOKUPSTREAMSPARAMS = _descriptor.Descriptor( - name='LookupStreamsParams', - full_name='v5api.LookupStreamsParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='collection', full_name='v5api.LookupStreamsParams.collection', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='isCollectionPrefix', full_name='v5api.LookupStreamsParams.isCollectionPrefix', index=1, - number=2, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='tags', full_name='v5api.LookupStreamsParams.tags', index=2, - number=3, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='annotations', full_name='v5api.LookupStreamsParams.annotations', index=3, - number=4, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='role', full_name='v5api.LookupStreamsParams.role', index=4, - number=100, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=2049, - serialized_end=2220, -) - - -_LOOKUPSTREAMSRESPONSE = _descriptor.Descriptor( - name='LookupStreamsResponse', - full_name='v5api.LookupStreamsResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.LookupStreamsResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='results', full_name='v5api.LookupStreamsResponse.results', index=1, - number=2, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=2222, - serialized_end=2316, -) - - -_NEARESTPARAMS = _descriptor.Descriptor( - name='NearestParams', - full_name='v5api.NearestParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.NearestParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='time', full_name='v5api.NearestParams.time', index=1, - number=2, type=16, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMajor', full_name='v5api.NearestParams.versionMajor', index=2, - number=3, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='backward', full_name='v5api.NearestParams.backward', index=3, - number=4, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=2318, - serialized_end=2401, -) - - -_NEARESTRESPONSE = _descriptor.Descriptor( - name='NearestResponse', - full_name='v5api.NearestResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.NearestResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMajor', full_name='v5api.NearestResponse.versionMajor', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMinor', full_name='v5api.NearestResponse.versionMinor', index=2, - number=3, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='value', full_name='v5api.NearestResponse.value', index=3, - number=4, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=2403, - serialized_end=2525, -) - - -_CHANGESPARAMS = _descriptor.Descriptor( - name='ChangesParams', - full_name='v5api.ChangesParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.ChangesParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='fromMajor', full_name='v5api.ChangesParams.fromMajor', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='toMajor', full_name='v5api.ChangesParams.toMajor', index=2, - number=3, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='resolution', full_name='v5api.ChangesParams.resolution', index=3, - number=4, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=2527, - serialized_end=2612, -) - - -_CHANGESRESPONSE = _descriptor.Descriptor( - name='ChangesResponse', - full_name='v5api.ChangesResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.ChangesResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMajor', full_name='v5api.ChangesResponse.versionMajor', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMinor', full_name='v5api.ChangesResponse.versionMinor', index=2, - number=3, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='ranges', full_name='v5api.ChangesResponse.ranges', index=3, - number=4, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=2614, - serialized_end=2741, -) - - -_INSERTPARAMS = _descriptor.Descriptor( - name='InsertParams', - full_name='v5api.InsertParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.InsertParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='sync', full_name='v5api.InsertParams.sync', index=1, - number=2, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='merge_policy', full_name='v5api.InsertParams.merge_policy', index=2, - number=4, type=14, cpp_type=8, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='values', full_name='v5api.InsertParams.values', index=3, - number=3, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=2743, - serialized_end=2860, -) - - -_INSERTRESPONSE = _descriptor.Descriptor( - name='InsertResponse', - full_name='v5api.InsertResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.InsertResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMajor', full_name='v5api.InsertResponse.versionMajor', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMinor', full_name='v5api.InsertResponse.versionMinor', index=2, - number=3, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=2862, - serialized_end=2951, -) - - -_DELETEPARAMS = _descriptor.Descriptor( - name='DeleteParams', - full_name='v5api.DeleteParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.DeleteParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='start', full_name='v5api.DeleteParams.start', index=1, - number=2, type=16, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='end', full_name='v5api.DeleteParams.end', index=2, - number=3, type=16, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=2953, - serialized_end=3009, -) - - -_DELETERESPONSE = _descriptor.Descriptor( - name='DeleteResponse', - full_name='v5api.DeleteResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.DeleteResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMajor', full_name='v5api.DeleteResponse.versionMajor', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMinor', full_name='v5api.DeleteResponse.versionMinor', index=2, - number=3, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3011, - serialized_end=3100, -) - - -_INFOPARAMS = _descriptor.Descriptor( - name='InfoParams', - full_name='v5api.InfoParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3102, - serialized_end=3114, -) - - -_INFORESPONSE = _descriptor.Descriptor( - name='InfoResponse', - full_name='v5api.InfoResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.InfoResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='mash', full_name='v5api.InfoResponse.mash', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='majorVersion', full_name='v5api.InfoResponse.majorVersion', index=2, - number=3, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='minorVersion', full_name='v5api.InfoResponse.minorVersion', index=3, - number=4, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='build', full_name='v5api.InfoResponse.build', index=4, - number=5, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='proxy', full_name='v5api.InfoResponse.proxy', index=5, - number=6, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3117, - serialized_end=3279, -) - - -_PROXYINFO = _descriptor.Descriptor( - name='ProxyInfo', - full_name='v5api.ProxyInfo', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='proxyEndpoints', full_name='v5api.ProxyInfo.proxyEndpoints', index=0, - number=1, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3281, - serialized_end=3316, -) - - -_FAULTINJECTPARAMS = _descriptor.Descriptor( - name='FaultInjectParams', - full_name='v5api.FaultInjectParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='type', full_name='v5api.FaultInjectParams.type', index=0, - number=1, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='params', full_name='v5api.FaultInjectParams.params', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3318, - serialized_end=3367, -) - - -_FAULTINJECTRESPONSE = _descriptor.Descriptor( - name='FaultInjectResponse', - full_name='v5api.FaultInjectResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.FaultInjectResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='rv', full_name='v5api.FaultInjectResponse.rv', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3369, - serialized_end=3431, -) - - -_FLUSHPARAMS = _descriptor.Descriptor( - name='FlushParams', - full_name='v5api.FlushParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.FlushParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3433, - serialized_end=3460, -) - - -_FLUSHRESPONSE = _descriptor.Descriptor( - name='FlushResponse', - full_name='v5api.FlushResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.FlushResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMajor', full_name='v5api.FlushResponse.versionMajor', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMinor', full_name='v5api.FlushResponse.versionMinor', index=2, - number=3, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3462, - serialized_end=3550, -) - - -_OBLITERATEPARAMS = _descriptor.Descriptor( - name='ObliterateParams', - full_name='v5api.ObliterateParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.ObliterateParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3552, - serialized_end=3584, -) - - -_OBLITERATERESPONSE = _descriptor.Descriptor( - name='ObliterateResponse', - full_name='v5api.ObliterateResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.ObliterateResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3586, - serialized_end=3635, -) - - -_RAWPOINT = _descriptor.Descriptor( - name='RawPoint', - full_name='v5api.RawPoint', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='time', full_name='v5api.RawPoint.time', index=0, - number=1, type=16, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='value', full_name='v5api.RawPoint.value', index=1, - number=2, type=1, cpp_type=5, label=1, - has_default_value=False, default_value=float(0), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3637, - serialized_end=3676, -) - - -_STATPOINT = _descriptor.Descriptor( - name='StatPoint', - full_name='v5api.StatPoint', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='time', full_name='v5api.StatPoint.time', index=0, - number=1, type=16, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='min', full_name='v5api.StatPoint.min', index=1, - number=2, type=1, cpp_type=5, label=1, - has_default_value=False, default_value=float(0), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='mean', full_name='v5api.StatPoint.mean', index=2, - number=3, type=1, cpp_type=5, label=1, - has_default_value=False, default_value=float(0), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='max', full_name='v5api.StatPoint.max', index=3, - number=4, type=1, cpp_type=5, label=1, - has_default_value=False, default_value=float(0), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='count', full_name='v5api.StatPoint.count', index=4, - number=5, type=6, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='stddev', full_name='v5api.StatPoint.stddev', index=5, - number=6, type=1, cpp_type=5, label=1, - has_default_value=False, default_value=float(0), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3678, - serialized_end=3774, -) - - -_CHANGEDRANGE = _descriptor.Descriptor( - name='ChangedRange', - full_name='v5api.ChangedRange', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='start', full_name='v5api.ChangedRange.start', index=0, - number=1, type=16, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='end', full_name='v5api.ChangedRange.end', index=1, - number=2, type=16, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3776, - serialized_end=3818, -) - - -_STATUS = _descriptor.Descriptor( - name='Status', - full_name='v5api.Status', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='code', full_name='v5api.Status.code', index=0, - number=1, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='msg', full_name='v5api.Status.msg', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='mash', full_name='v5api.Status.mash', index=2, - number=3, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3820, - serialized_end=3882, -) - - -_MASH = _descriptor.Descriptor( - name='Mash', - full_name='v5api.Mash', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='revision', full_name='v5api.Mash.revision', index=0, - number=1, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='leader', full_name='v5api.Mash.leader', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='leaderRevision', full_name='v5api.Mash.leaderRevision', index=2, - number=3, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='totalWeight', full_name='v5api.Mash.totalWeight', index=3, - number=4, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='healthy', full_name='v5api.Mash.healthy', index=4, - number=5, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='unmapped', full_name='v5api.Mash.unmapped', index=5, - number=6, type=1, cpp_type=5, label=1, - has_default_value=False, default_value=float(0), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='members', full_name='v5api.Mash.members', index=6, - number=7, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3885, - serialized_end=4037, -) - - -_MEMBER = _descriptor.Descriptor( - name='Member', - full_name='v5api.Member', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='hash', full_name='v5api.Member.hash', index=0, - number=1, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='nodename', full_name='v5api.Member.nodename', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='up', full_name='v5api.Member.up', index=2, - number=3, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='in', full_name='v5api.Member.in', index=3, - number=4, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='enabled', full_name='v5api.Member.enabled', index=4, - number=5, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='start', full_name='v5api.Member.start', index=5, - number=6, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='end', full_name='v5api.Member.end', index=6, - number=7, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='weight', full_name='v5api.Member.weight', index=7, - number=8, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='readPreference', full_name='v5api.Member.readPreference', index=8, - number=9, type=1, cpp_type=5, label=1, - has_default_value=False, default_value=float(0), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='httpEndpoints', full_name='v5api.Member.httpEndpoints', index=9, - number=10, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='grpcEndpoints', full_name='v5api.Member.grpcEndpoints', index=10, - number=11, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=4040, - serialized_end=4235, -) - - -_KEYOPTVALUE = _descriptor.Descriptor( - name='KeyOptValue', - full_name='v5api.KeyOptValue', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='key', full_name='v5api.KeyOptValue.key', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='val', full_name='v5api.KeyOptValue.val', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=4237, - serialized_end=4293, -) - - -_OPTVALUE = _descriptor.Descriptor( - name='OptValue', - full_name='v5api.OptValue', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='value', full_name='v5api.OptValue.value', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=4295, - serialized_end=4320, -) - - -_KEYVALUE = _descriptor.Descriptor( - name='KeyValue', - full_name='v5api.KeyValue', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='key', full_name='v5api.KeyValue.key', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='value', full_name='v5api.KeyValue.value', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=4322, - serialized_end=4360, -) - - -_STREAMCSVCONFIG = _descriptor.Descriptor( - name='StreamCSVConfig', - full_name='v5api.StreamCSVConfig', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='version', full_name='v5api.StreamCSVConfig.version', index=0, - number=1, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='label', full_name='v5api.StreamCSVConfig.label', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.StreamCSVConfig.uuid', index=2, - number=3, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=4362, - serialized_end=4425, -) - - -_GENERATECSVPARAMS = _descriptor.Descriptor( - name='GenerateCSVParams', - full_name='v5api.GenerateCSVParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='queryType', full_name='v5api.GenerateCSVParams.queryType', index=0, - number=1, type=14, cpp_type=8, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='startTime', full_name='v5api.GenerateCSVParams.startTime', index=1, - number=2, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='endTime', full_name='v5api.GenerateCSVParams.endTime', index=2, - number=3, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='windowSize', full_name='v5api.GenerateCSVParams.windowSize', index=3, - number=4, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='depth', full_name='v5api.GenerateCSVParams.depth', index=4, - number=5, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='includeVersions', full_name='v5api.GenerateCSVParams.includeVersions', index=5, - number=6, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='streams', full_name='v5api.GenerateCSVParams.streams', index=6, - number=7, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - _GENERATECSVPARAMS_QUERYTYPE, - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=4428, - serialized_end=4713, -) - - -_GENERATECSVRESPONSE = _descriptor.Descriptor( - name='GenerateCSVResponse', - full_name='v5api.GenerateCSVResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.GenerateCSVResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='isHeader', full_name='v5api.GenerateCSVResponse.isHeader', index=1, - number=2, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='row', full_name='v5api.GenerateCSVResponse.row', index=2, - number=3, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=4715, - serialized_end=4796, -) - - -_SQLQUERYPARAMS = _descriptor.Descriptor( - name='SQLQueryParams', - full_name='v5api.SQLQueryParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='query', full_name='v5api.SQLQueryParams.query', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='params', full_name='v5api.SQLQueryParams.params', index=1, - number=2, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='role', full_name='v5api.SQLQueryParams.role', index=2, - number=100, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=4798, - serialized_end=4872, -) - - -_SQLQUERYRESPONSE = _descriptor.Descriptor( - name='SQLQueryResponse', - full_name='v5api.SQLQueryResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.SQLQueryResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='SQLQueryRow', full_name='v5api.SQLQueryResponse.SQLQueryRow', index=1, - number=2, type=12, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=4874, - serialized_end=4942, -) - - -_ROLE = _descriptor.Descriptor( - name='Role', - full_name='v5api.Role', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='v5api.Role.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=4944, - serialized_end=4964, -) - - -_SETCOMPACTIONCONFIGPARAMS = _descriptor.Descriptor( - name='SetCompactionConfigParams', - full_name='v5api.SetCompactionConfigParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.SetCompactionConfigParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='CompactedVersion', full_name='v5api.SetCompactionConfigParams.CompactedVersion', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='reducedResolutionRanges', full_name='v5api.SetCompactionConfigParams.reducedResolutionRanges', index=2, - number=3, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='unused0', full_name='v5api.SetCompactionConfigParams.unused0', index=3, - number=4, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=4967, - serialized_end=5115, -) - - -_SETCOMPACTIONCONFIGRESPONSE = _descriptor.Descriptor( - name='SetCompactionConfigResponse', - full_name='v5api.SetCompactionConfigResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.SetCompactionConfigResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=5117, - serialized_end=5175, -) - - -_GETCOMPACTIONCONFIGPARAMS = _descriptor.Descriptor( - name='GetCompactionConfigParams', - full_name='v5api.GetCompactionConfigParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.GetCompactionConfigParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=5177, - serialized_end=5218, -) - - -_GETCOMPACTIONCONFIGRESPONSE = _descriptor.Descriptor( - name='GetCompactionConfigResponse', - full_name='v5api.GetCompactionConfigResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.GetCompactionConfigResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='LatestMajorVersion', full_name='v5api.GetCompactionConfigResponse.LatestMajorVersion', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='CompactedVersion', full_name='v5api.GetCompactionConfigResponse.CompactedVersion', index=2, - number=3, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='reducedResolutionRanges', full_name='v5api.GetCompactionConfigResponse.reducedResolutionRanges', index=3, - number=4, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='unused0', full_name='v5api.GetCompactionConfigResponse.unused0', index=4, - number=5, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=5221, - serialized_end=5414, -) - - -_REDUCEDRESOLUTIONRANGE = _descriptor.Descriptor( - name='ReducedResolutionRange', - full_name='v5api.ReducedResolutionRange', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='Start', full_name='v5api.ReducedResolutionRange.Start', index=0, - number=1, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='End', full_name='v5api.ReducedResolutionRange.End', index=1, - number=2, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='Resolution', full_name='v5api.ReducedResolutionRange.Resolution', index=2, - number=3, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=5416, - serialized_end=5488, -) - -_RAWVALUESRESPONSE.fields_by_name['stat'].message_type = _STATUS -_RAWVALUESRESPONSE.fields_by_name['values'].message_type = _RAWPOINT -_ALIGNEDWINDOWSRESPONSE.fields_by_name['stat'].message_type = _STATUS -_ALIGNEDWINDOWSRESPONSE.fields_by_name['values'].message_type = _STATPOINT -_WINDOWSRESPONSE.fields_by_name['stat'].message_type = _STATUS -_WINDOWSRESPONSE.fields_by_name['values'].message_type = _STATPOINT -_STREAMINFOPARAMS.fields_by_name['role'].message_type = _ROLE -_STREAMINFORESPONSE.fields_by_name['stat'].message_type = _STATUS -_STREAMINFORESPONSE.fields_by_name['descriptor'].message_type = _STREAMDESCRIPTOR -_STREAMDESCRIPTOR.fields_by_name['tags'].message_type = _KEYOPTVALUE -_STREAMDESCRIPTOR.fields_by_name['annotations'].message_type = _KEYOPTVALUE -_SETSTREAMANNOTATIONSPARAMS.fields_by_name['changes'].message_type = _KEYOPTVALUE -_SETSTREAMANNOTATIONSRESPONSE.fields_by_name['stat'].message_type = _STATUS -_SETSTREAMTAGSPARAMS.fields_by_name['tags'].message_type = _KEYOPTVALUE -_SETSTREAMTAGSRESPONSE.fields_by_name['stat'].message_type = _STATUS -_CREATEPARAMS.fields_by_name['tags'].message_type = _KEYOPTVALUE -_CREATEPARAMS.fields_by_name['annotations'].message_type = _KEYOPTVALUE -_CREATERESPONSE.fields_by_name['stat'].message_type = _STATUS -_METADATAUSAGEPARAMS.fields_by_name['role'].message_type = _ROLE -_METADATAUSAGERESPONSE.fields_by_name['stat'].message_type = _STATUS -_METADATAUSAGERESPONSE.fields_by_name['tags'].message_type = _KEYCOUNT -_METADATAUSAGERESPONSE.fields_by_name['annotations'].message_type = _KEYCOUNT -_LISTCOLLECTIONSPARAMS.fields_by_name['role'].message_type = _ROLE -_LISTCOLLECTIONSRESPONSE.fields_by_name['stat'].message_type = _STATUS -_LOOKUPSTREAMSPARAMS.fields_by_name['tags'].message_type = _KEYOPTVALUE -_LOOKUPSTREAMSPARAMS.fields_by_name['annotations'].message_type = _KEYOPTVALUE -_LOOKUPSTREAMSPARAMS.fields_by_name['role'].message_type = _ROLE -_LOOKUPSTREAMSRESPONSE.fields_by_name['stat'].message_type = _STATUS -_LOOKUPSTREAMSRESPONSE.fields_by_name['results'].message_type = _STREAMDESCRIPTOR -_NEARESTRESPONSE.fields_by_name['stat'].message_type = _STATUS -_NEARESTRESPONSE.fields_by_name['value'].message_type = _RAWPOINT -_CHANGESRESPONSE.fields_by_name['stat'].message_type = _STATUS -_CHANGESRESPONSE.fields_by_name['ranges'].message_type = _CHANGEDRANGE -_INSERTPARAMS.fields_by_name['merge_policy'].enum_type = _MERGEPOLICY -_INSERTPARAMS.fields_by_name['values'].message_type = _RAWPOINT -_INSERTRESPONSE.fields_by_name['stat'].message_type = _STATUS -_DELETERESPONSE.fields_by_name['stat'].message_type = _STATUS -_INFORESPONSE.fields_by_name['stat'].message_type = _STATUS -_INFORESPONSE.fields_by_name['mash'].message_type = _MASH -_INFORESPONSE.fields_by_name['proxy'].message_type = _PROXYINFO -_FAULTINJECTRESPONSE.fields_by_name['stat'].message_type = _STATUS -_FLUSHRESPONSE.fields_by_name['stat'].message_type = _STATUS -_OBLITERATERESPONSE.fields_by_name['stat'].message_type = _STATUS -_STATUS.fields_by_name['mash'].message_type = _MASH -_MASH.fields_by_name['members'].message_type = _MEMBER -_KEYOPTVALUE.fields_by_name['val'].message_type = _OPTVALUE -_GENERATECSVPARAMS.fields_by_name['queryType'].enum_type = _GENERATECSVPARAMS_QUERYTYPE -_GENERATECSVPARAMS.fields_by_name['streams'].message_type = _STREAMCSVCONFIG -_GENERATECSVPARAMS_QUERYTYPE.containing_type = _GENERATECSVPARAMS -_GENERATECSVRESPONSE.fields_by_name['stat'].message_type = _STATUS -_SQLQUERYPARAMS.fields_by_name['role'].message_type = _ROLE -_SQLQUERYRESPONSE.fields_by_name['stat'].message_type = _STATUS -_SETCOMPACTIONCONFIGPARAMS.fields_by_name['reducedResolutionRanges'].message_type = _REDUCEDRESOLUTIONRANGE -_SETCOMPACTIONCONFIGRESPONSE.fields_by_name['stat'].message_type = _STATUS -_GETCOMPACTIONCONFIGRESPONSE.fields_by_name['stat'].message_type = _STATUS -_GETCOMPACTIONCONFIGRESPONSE.fields_by_name['reducedResolutionRanges'].message_type = _REDUCEDRESOLUTIONRANGE -DESCRIPTOR.message_types_by_name['RawValuesParams'] = _RAWVALUESPARAMS -DESCRIPTOR.message_types_by_name['RawValuesResponse'] = _RAWVALUESRESPONSE -DESCRIPTOR.message_types_by_name['AlignedWindowsParams'] = _ALIGNEDWINDOWSPARAMS -DESCRIPTOR.message_types_by_name['AlignedWindowsResponse'] = _ALIGNEDWINDOWSRESPONSE -DESCRIPTOR.message_types_by_name['WindowsParams'] = _WINDOWSPARAMS -DESCRIPTOR.message_types_by_name['WindowsResponse'] = _WINDOWSRESPONSE -DESCRIPTOR.message_types_by_name['StreamInfoParams'] = _STREAMINFOPARAMS -DESCRIPTOR.message_types_by_name['StreamInfoResponse'] = _STREAMINFORESPONSE -DESCRIPTOR.message_types_by_name['StreamDescriptor'] = _STREAMDESCRIPTOR -DESCRIPTOR.message_types_by_name['SetStreamAnnotationsParams'] = _SETSTREAMANNOTATIONSPARAMS -DESCRIPTOR.message_types_by_name['SetStreamAnnotationsResponse'] = _SETSTREAMANNOTATIONSRESPONSE -DESCRIPTOR.message_types_by_name['SetStreamTagsParams'] = _SETSTREAMTAGSPARAMS -DESCRIPTOR.message_types_by_name['SetStreamTagsResponse'] = _SETSTREAMTAGSRESPONSE -DESCRIPTOR.message_types_by_name['CreateParams'] = _CREATEPARAMS -DESCRIPTOR.message_types_by_name['CreateResponse'] = _CREATERESPONSE -DESCRIPTOR.message_types_by_name['MetadataUsageParams'] = _METADATAUSAGEPARAMS -DESCRIPTOR.message_types_by_name['MetadataUsageResponse'] = _METADATAUSAGERESPONSE -DESCRIPTOR.message_types_by_name['KeyCount'] = _KEYCOUNT -DESCRIPTOR.message_types_by_name['ListCollectionsParams'] = _LISTCOLLECTIONSPARAMS -DESCRIPTOR.message_types_by_name['ListCollectionsResponse'] = _LISTCOLLECTIONSRESPONSE -DESCRIPTOR.message_types_by_name['LookupStreamsParams'] = _LOOKUPSTREAMSPARAMS -DESCRIPTOR.message_types_by_name['LookupStreamsResponse'] = _LOOKUPSTREAMSRESPONSE -DESCRIPTOR.message_types_by_name['NearestParams'] = _NEARESTPARAMS -DESCRIPTOR.message_types_by_name['NearestResponse'] = _NEARESTRESPONSE -DESCRIPTOR.message_types_by_name['ChangesParams'] = _CHANGESPARAMS -DESCRIPTOR.message_types_by_name['ChangesResponse'] = _CHANGESRESPONSE -DESCRIPTOR.message_types_by_name['InsertParams'] = _INSERTPARAMS -DESCRIPTOR.message_types_by_name['InsertResponse'] = _INSERTRESPONSE -DESCRIPTOR.message_types_by_name['DeleteParams'] = _DELETEPARAMS -DESCRIPTOR.message_types_by_name['DeleteResponse'] = _DELETERESPONSE -DESCRIPTOR.message_types_by_name['InfoParams'] = _INFOPARAMS -DESCRIPTOR.message_types_by_name['InfoResponse'] = _INFORESPONSE -DESCRIPTOR.message_types_by_name['ProxyInfo'] = _PROXYINFO -DESCRIPTOR.message_types_by_name['FaultInjectParams'] = _FAULTINJECTPARAMS -DESCRIPTOR.message_types_by_name['FaultInjectResponse'] = _FAULTINJECTRESPONSE -DESCRIPTOR.message_types_by_name['FlushParams'] = _FLUSHPARAMS -DESCRIPTOR.message_types_by_name['FlushResponse'] = _FLUSHRESPONSE -DESCRIPTOR.message_types_by_name['ObliterateParams'] = _OBLITERATEPARAMS -DESCRIPTOR.message_types_by_name['ObliterateResponse'] = _OBLITERATERESPONSE -DESCRIPTOR.message_types_by_name['RawPoint'] = _RAWPOINT -DESCRIPTOR.message_types_by_name['StatPoint'] = _STATPOINT -DESCRIPTOR.message_types_by_name['ChangedRange'] = _CHANGEDRANGE -DESCRIPTOR.message_types_by_name['Status'] = _STATUS -DESCRIPTOR.message_types_by_name['Mash'] = _MASH -DESCRIPTOR.message_types_by_name['Member'] = _MEMBER -DESCRIPTOR.message_types_by_name['KeyOptValue'] = _KEYOPTVALUE -DESCRIPTOR.message_types_by_name['OptValue'] = _OPTVALUE -DESCRIPTOR.message_types_by_name['KeyValue'] = _KEYVALUE -DESCRIPTOR.message_types_by_name['StreamCSVConfig'] = _STREAMCSVCONFIG -DESCRIPTOR.message_types_by_name['GenerateCSVParams'] = _GENERATECSVPARAMS -DESCRIPTOR.message_types_by_name['GenerateCSVResponse'] = _GENERATECSVRESPONSE -DESCRIPTOR.message_types_by_name['SQLQueryParams'] = _SQLQUERYPARAMS -DESCRIPTOR.message_types_by_name['SQLQueryResponse'] = _SQLQUERYRESPONSE -DESCRIPTOR.message_types_by_name['Role'] = _ROLE -DESCRIPTOR.message_types_by_name['SetCompactionConfigParams'] = _SETCOMPACTIONCONFIGPARAMS -DESCRIPTOR.message_types_by_name['SetCompactionConfigResponse'] = _SETCOMPACTIONCONFIGRESPONSE -DESCRIPTOR.message_types_by_name['GetCompactionConfigParams'] = _GETCOMPACTIONCONFIGPARAMS -DESCRIPTOR.message_types_by_name['GetCompactionConfigResponse'] = _GETCOMPACTIONCONFIGRESPONSE -DESCRIPTOR.message_types_by_name['ReducedResolutionRange'] = _REDUCEDRESOLUTIONRANGE -DESCRIPTOR.enum_types_by_name['MergePolicy'] = _MERGEPOLICY -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -RawValuesParams = _reflection.GeneratedProtocolMessageType('RawValuesParams', (_message.Message,), dict( - DESCRIPTOR = _RAWVALUESPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.RawValuesParams) - )) -_sym_db.RegisterMessage(RawValuesParams) - -RawValuesResponse = _reflection.GeneratedProtocolMessageType('RawValuesResponse', (_message.Message,), dict( - DESCRIPTOR = _RAWVALUESRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.RawValuesResponse) - )) -_sym_db.RegisterMessage(RawValuesResponse) - -AlignedWindowsParams = _reflection.GeneratedProtocolMessageType('AlignedWindowsParams', (_message.Message,), dict( - DESCRIPTOR = _ALIGNEDWINDOWSPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.AlignedWindowsParams) - )) -_sym_db.RegisterMessage(AlignedWindowsParams) - -AlignedWindowsResponse = _reflection.GeneratedProtocolMessageType('AlignedWindowsResponse', (_message.Message,), dict( - DESCRIPTOR = _ALIGNEDWINDOWSRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.AlignedWindowsResponse) - )) -_sym_db.RegisterMessage(AlignedWindowsResponse) - -WindowsParams = _reflection.GeneratedProtocolMessageType('WindowsParams', (_message.Message,), dict( - DESCRIPTOR = _WINDOWSPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.WindowsParams) - )) -_sym_db.RegisterMessage(WindowsParams) - -WindowsResponse = _reflection.GeneratedProtocolMessageType('WindowsResponse', (_message.Message,), dict( - DESCRIPTOR = _WINDOWSRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.WindowsResponse) - )) -_sym_db.RegisterMessage(WindowsResponse) - -StreamInfoParams = _reflection.GeneratedProtocolMessageType('StreamInfoParams', (_message.Message,), dict( - DESCRIPTOR = _STREAMINFOPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.StreamInfoParams) - )) -_sym_db.RegisterMessage(StreamInfoParams) - -StreamInfoResponse = _reflection.GeneratedProtocolMessageType('StreamInfoResponse', (_message.Message,), dict( - DESCRIPTOR = _STREAMINFORESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.StreamInfoResponse) - )) -_sym_db.RegisterMessage(StreamInfoResponse) - -StreamDescriptor = _reflection.GeneratedProtocolMessageType('StreamDescriptor', (_message.Message,), dict( - DESCRIPTOR = _STREAMDESCRIPTOR, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.StreamDescriptor) - )) -_sym_db.RegisterMessage(StreamDescriptor) - -SetStreamAnnotationsParams = _reflection.GeneratedProtocolMessageType('SetStreamAnnotationsParams', (_message.Message,), dict( - DESCRIPTOR = _SETSTREAMANNOTATIONSPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.SetStreamAnnotationsParams) - )) -_sym_db.RegisterMessage(SetStreamAnnotationsParams) - -SetStreamAnnotationsResponse = _reflection.GeneratedProtocolMessageType('SetStreamAnnotationsResponse', (_message.Message,), dict( - DESCRIPTOR = _SETSTREAMANNOTATIONSRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.SetStreamAnnotationsResponse) - )) -_sym_db.RegisterMessage(SetStreamAnnotationsResponse) - -SetStreamTagsParams = _reflection.GeneratedProtocolMessageType('SetStreamTagsParams', (_message.Message,), dict( - DESCRIPTOR = _SETSTREAMTAGSPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.SetStreamTagsParams) - )) -_sym_db.RegisterMessage(SetStreamTagsParams) - -SetStreamTagsResponse = _reflection.GeneratedProtocolMessageType('SetStreamTagsResponse', (_message.Message,), dict( - DESCRIPTOR = _SETSTREAMTAGSRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.SetStreamTagsResponse) - )) -_sym_db.RegisterMessage(SetStreamTagsResponse) - -CreateParams = _reflection.GeneratedProtocolMessageType('CreateParams', (_message.Message,), dict( - DESCRIPTOR = _CREATEPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.CreateParams) - )) -_sym_db.RegisterMessage(CreateParams) - -CreateResponse = _reflection.GeneratedProtocolMessageType('CreateResponse', (_message.Message,), dict( - DESCRIPTOR = _CREATERESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.CreateResponse) - )) -_sym_db.RegisterMessage(CreateResponse) - -MetadataUsageParams = _reflection.GeneratedProtocolMessageType('MetadataUsageParams', (_message.Message,), dict( - DESCRIPTOR = _METADATAUSAGEPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.MetadataUsageParams) - )) -_sym_db.RegisterMessage(MetadataUsageParams) - -MetadataUsageResponse = _reflection.GeneratedProtocolMessageType('MetadataUsageResponse', (_message.Message,), dict( - DESCRIPTOR = _METADATAUSAGERESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.MetadataUsageResponse) - )) -_sym_db.RegisterMessage(MetadataUsageResponse) - -KeyCount = _reflection.GeneratedProtocolMessageType('KeyCount', (_message.Message,), dict( - DESCRIPTOR = _KEYCOUNT, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.KeyCount) - )) -_sym_db.RegisterMessage(KeyCount) - -ListCollectionsParams = _reflection.GeneratedProtocolMessageType('ListCollectionsParams', (_message.Message,), dict( - DESCRIPTOR = _LISTCOLLECTIONSPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.ListCollectionsParams) - )) -_sym_db.RegisterMessage(ListCollectionsParams) - -ListCollectionsResponse = _reflection.GeneratedProtocolMessageType('ListCollectionsResponse', (_message.Message,), dict( - DESCRIPTOR = _LISTCOLLECTIONSRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.ListCollectionsResponse) - )) -_sym_db.RegisterMessage(ListCollectionsResponse) - -LookupStreamsParams = _reflection.GeneratedProtocolMessageType('LookupStreamsParams', (_message.Message,), dict( - DESCRIPTOR = _LOOKUPSTREAMSPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.LookupStreamsParams) - )) -_sym_db.RegisterMessage(LookupStreamsParams) - -LookupStreamsResponse = _reflection.GeneratedProtocolMessageType('LookupStreamsResponse', (_message.Message,), dict( - DESCRIPTOR = _LOOKUPSTREAMSRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.LookupStreamsResponse) - )) -_sym_db.RegisterMessage(LookupStreamsResponse) - -NearestParams = _reflection.GeneratedProtocolMessageType('NearestParams', (_message.Message,), dict( - DESCRIPTOR = _NEARESTPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.NearestParams) - )) -_sym_db.RegisterMessage(NearestParams) - -NearestResponse = _reflection.GeneratedProtocolMessageType('NearestResponse', (_message.Message,), dict( - DESCRIPTOR = _NEARESTRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.NearestResponse) - )) -_sym_db.RegisterMessage(NearestResponse) - -ChangesParams = _reflection.GeneratedProtocolMessageType('ChangesParams', (_message.Message,), dict( - DESCRIPTOR = _CHANGESPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.ChangesParams) - )) -_sym_db.RegisterMessage(ChangesParams) - -ChangesResponse = _reflection.GeneratedProtocolMessageType('ChangesResponse', (_message.Message,), dict( - DESCRIPTOR = _CHANGESRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.ChangesResponse) - )) -_sym_db.RegisterMessage(ChangesResponse) - -InsertParams = _reflection.GeneratedProtocolMessageType('InsertParams', (_message.Message,), dict( - DESCRIPTOR = _INSERTPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.InsertParams) - )) -_sym_db.RegisterMessage(InsertParams) - -InsertResponse = _reflection.GeneratedProtocolMessageType('InsertResponse', (_message.Message,), dict( - DESCRIPTOR = _INSERTRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.InsertResponse) - )) -_sym_db.RegisterMessage(InsertResponse) - -DeleteParams = _reflection.GeneratedProtocolMessageType('DeleteParams', (_message.Message,), dict( - DESCRIPTOR = _DELETEPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.DeleteParams) - )) -_sym_db.RegisterMessage(DeleteParams) - -DeleteResponse = _reflection.GeneratedProtocolMessageType('DeleteResponse', (_message.Message,), dict( - DESCRIPTOR = _DELETERESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.DeleteResponse) - )) -_sym_db.RegisterMessage(DeleteResponse) - -InfoParams = _reflection.GeneratedProtocolMessageType('InfoParams', (_message.Message,), dict( - DESCRIPTOR = _INFOPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.InfoParams) - )) -_sym_db.RegisterMessage(InfoParams) - -InfoResponse = _reflection.GeneratedProtocolMessageType('InfoResponse', (_message.Message,), dict( - DESCRIPTOR = _INFORESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.InfoResponse) - )) -_sym_db.RegisterMessage(InfoResponse) - -ProxyInfo = _reflection.GeneratedProtocolMessageType('ProxyInfo', (_message.Message,), dict( - DESCRIPTOR = _PROXYINFO, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.ProxyInfo) - )) -_sym_db.RegisterMessage(ProxyInfo) - -FaultInjectParams = _reflection.GeneratedProtocolMessageType('FaultInjectParams', (_message.Message,), dict( - DESCRIPTOR = _FAULTINJECTPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.FaultInjectParams) - )) -_sym_db.RegisterMessage(FaultInjectParams) - -FaultInjectResponse = _reflection.GeneratedProtocolMessageType('FaultInjectResponse', (_message.Message,), dict( - DESCRIPTOR = _FAULTINJECTRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.FaultInjectResponse) - )) -_sym_db.RegisterMessage(FaultInjectResponse) - -FlushParams = _reflection.GeneratedProtocolMessageType('FlushParams', (_message.Message,), dict( - DESCRIPTOR = _FLUSHPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.FlushParams) - )) -_sym_db.RegisterMessage(FlushParams) - -FlushResponse = _reflection.GeneratedProtocolMessageType('FlushResponse', (_message.Message,), dict( - DESCRIPTOR = _FLUSHRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.FlushResponse) - )) -_sym_db.RegisterMessage(FlushResponse) - -ObliterateParams = _reflection.GeneratedProtocolMessageType('ObliterateParams', (_message.Message,), dict( - DESCRIPTOR = _OBLITERATEPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.ObliterateParams) - )) -_sym_db.RegisterMessage(ObliterateParams) - -ObliterateResponse = _reflection.GeneratedProtocolMessageType('ObliterateResponse', (_message.Message,), dict( - DESCRIPTOR = _OBLITERATERESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.ObliterateResponse) - )) -_sym_db.RegisterMessage(ObliterateResponse) - -RawPoint = _reflection.GeneratedProtocolMessageType('RawPoint', (_message.Message,), dict( - DESCRIPTOR = _RAWPOINT, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.RawPoint) - )) -_sym_db.RegisterMessage(RawPoint) - -StatPoint = _reflection.GeneratedProtocolMessageType('StatPoint', (_message.Message,), dict( - DESCRIPTOR = _STATPOINT, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.StatPoint) - )) -_sym_db.RegisterMessage(StatPoint) - -ChangedRange = _reflection.GeneratedProtocolMessageType('ChangedRange', (_message.Message,), dict( - DESCRIPTOR = _CHANGEDRANGE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.ChangedRange) - )) -_sym_db.RegisterMessage(ChangedRange) - -Status = _reflection.GeneratedProtocolMessageType('Status', (_message.Message,), dict( - DESCRIPTOR = _STATUS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.Status) - )) -_sym_db.RegisterMessage(Status) - -Mash = _reflection.GeneratedProtocolMessageType('Mash', (_message.Message,), dict( - DESCRIPTOR = _MASH, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.Mash) - )) -_sym_db.RegisterMessage(Mash) - -Member = _reflection.GeneratedProtocolMessageType('Member', (_message.Message,), dict( - DESCRIPTOR = _MEMBER, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.Member) - )) -_sym_db.RegisterMessage(Member) - -KeyOptValue = _reflection.GeneratedProtocolMessageType('KeyOptValue', (_message.Message,), dict( - DESCRIPTOR = _KEYOPTVALUE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.KeyOptValue) - )) -_sym_db.RegisterMessage(KeyOptValue) - -OptValue = _reflection.GeneratedProtocolMessageType('OptValue', (_message.Message,), dict( - DESCRIPTOR = _OPTVALUE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.OptValue) - )) -_sym_db.RegisterMessage(OptValue) - -KeyValue = _reflection.GeneratedProtocolMessageType('KeyValue', (_message.Message,), dict( - DESCRIPTOR = _KEYVALUE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.KeyValue) - )) -_sym_db.RegisterMessage(KeyValue) - -StreamCSVConfig = _reflection.GeneratedProtocolMessageType('StreamCSVConfig', (_message.Message,), dict( - DESCRIPTOR = _STREAMCSVCONFIG, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.StreamCSVConfig) - )) -_sym_db.RegisterMessage(StreamCSVConfig) - -GenerateCSVParams = _reflection.GeneratedProtocolMessageType('GenerateCSVParams', (_message.Message,), dict( - DESCRIPTOR = _GENERATECSVPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.GenerateCSVParams) - )) -_sym_db.RegisterMessage(GenerateCSVParams) - -GenerateCSVResponse = _reflection.GeneratedProtocolMessageType('GenerateCSVResponse', (_message.Message,), dict( - DESCRIPTOR = _GENERATECSVRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.GenerateCSVResponse) - )) -_sym_db.RegisterMessage(GenerateCSVResponse) - -SQLQueryParams = _reflection.GeneratedProtocolMessageType('SQLQueryParams', (_message.Message,), dict( - DESCRIPTOR = _SQLQUERYPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.SQLQueryParams) - )) -_sym_db.RegisterMessage(SQLQueryParams) - -SQLQueryResponse = _reflection.GeneratedProtocolMessageType('SQLQueryResponse', (_message.Message,), dict( - DESCRIPTOR = _SQLQUERYRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.SQLQueryResponse) - )) -_sym_db.RegisterMessage(SQLQueryResponse) - -Role = _reflection.GeneratedProtocolMessageType('Role', (_message.Message,), dict( - DESCRIPTOR = _ROLE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.Role) - )) -_sym_db.RegisterMessage(Role) - -SetCompactionConfigParams = _reflection.GeneratedProtocolMessageType('SetCompactionConfigParams', (_message.Message,), dict( - DESCRIPTOR = _SETCOMPACTIONCONFIGPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.SetCompactionConfigParams) - )) -_sym_db.RegisterMessage(SetCompactionConfigParams) - -SetCompactionConfigResponse = _reflection.GeneratedProtocolMessageType('SetCompactionConfigResponse', (_message.Message,), dict( - DESCRIPTOR = _SETCOMPACTIONCONFIGRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.SetCompactionConfigResponse) - )) -_sym_db.RegisterMessage(SetCompactionConfigResponse) - -GetCompactionConfigParams = _reflection.GeneratedProtocolMessageType('GetCompactionConfigParams', (_message.Message,), dict( - DESCRIPTOR = _GETCOMPACTIONCONFIGPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.GetCompactionConfigParams) - )) -_sym_db.RegisterMessage(GetCompactionConfigParams) - -GetCompactionConfigResponse = _reflection.GeneratedProtocolMessageType('GetCompactionConfigResponse', (_message.Message,), dict( - DESCRIPTOR = _GETCOMPACTIONCONFIGRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.GetCompactionConfigResponse) - )) -_sym_db.RegisterMessage(GetCompactionConfigResponse) - -ReducedResolutionRange = _reflection.GeneratedProtocolMessageType('ReducedResolutionRange', (_message.Message,), dict( - DESCRIPTOR = _REDUCEDRESOLUTIONRANGE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.ReducedResolutionRange) - )) -_sym_db.RegisterMessage(ReducedResolutionRange) - - - -_BTRDB = _descriptor.ServiceDescriptor( - name='BTrDB', - full_name='v5api.BTrDB', - file=DESCRIPTOR, - index=0, - serialized_options=None, - serialized_start=5553, - serialized_end=6884, - methods=[ - _descriptor.MethodDescriptor( - name='RawValues', - full_name='v5api.BTrDB.RawValues', - index=0, - containing_service=None, - input_type=_RAWVALUESPARAMS, - output_type=_RAWVALUESRESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='AlignedWindows', - full_name='v5api.BTrDB.AlignedWindows', - index=1, - containing_service=None, - input_type=_ALIGNEDWINDOWSPARAMS, - output_type=_ALIGNEDWINDOWSRESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='Windows', - full_name='v5api.BTrDB.Windows', - index=2, - containing_service=None, - input_type=_WINDOWSPARAMS, - output_type=_WINDOWSRESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='StreamInfo', - full_name='v5api.BTrDB.StreamInfo', - index=3, - containing_service=None, - input_type=_STREAMINFOPARAMS, - output_type=_STREAMINFORESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='SetStreamAnnotations', - full_name='v5api.BTrDB.SetStreamAnnotations', - index=4, - containing_service=None, - input_type=_SETSTREAMANNOTATIONSPARAMS, - output_type=_SETSTREAMANNOTATIONSRESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='SetStreamTags', - full_name='v5api.BTrDB.SetStreamTags', - index=5, - containing_service=None, - input_type=_SETSTREAMTAGSPARAMS, - output_type=_SETSTREAMTAGSRESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='Create', - full_name='v5api.BTrDB.Create', - index=6, - containing_service=None, - input_type=_CREATEPARAMS, - output_type=_CREATERESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='ListCollections', - full_name='v5api.BTrDB.ListCollections', - index=7, - containing_service=None, - input_type=_LISTCOLLECTIONSPARAMS, - output_type=_LISTCOLLECTIONSRESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='LookupStreams', - full_name='v5api.BTrDB.LookupStreams', - index=8, - containing_service=None, - input_type=_LOOKUPSTREAMSPARAMS, - output_type=_LOOKUPSTREAMSRESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='Nearest', - full_name='v5api.BTrDB.Nearest', - index=9, - containing_service=None, - input_type=_NEARESTPARAMS, - output_type=_NEARESTRESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='Changes', - full_name='v5api.BTrDB.Changes', - index=10, - containing_service=None, - input_type=_CHANGESPARAMS, - output_type=_CHANGESRESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='Insert', - full_name='v5api.BTrDB.Insert', - index=11, - containing_service=None, - input_type=_INSERTPARAMS, - output_type=_INSERTRESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='Delete', - full_name='v5api.BTrDB.Delete', - index=12, - containing_service=None, - input_type=_DELETEPARAMS, - output_type=_DELETERESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='Info', - full_name='v5api.BTrDB.Info', - index=13, - containing_service=None, - input_type=_INFOPARAMS, - output_type=_INFORESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='FaultInject', - full_name='v5api.BTrDB.FaultInject', - index=14, - containing_service=None, - input_type=_FAULTINJECTPARAMS, - output_type=_FAULTINJECTRESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='Flush', - full_name='v5api.BTrDB.Flush', - index=15, - containing_service=None, - input_type=_FLUSHPARAMS, - output_type=_FLUSHRESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='Obliterate', - full_name='v5api.BTrDB.Obliterate', - index=16, - containing_service=None, - input_type=_OBLITERATEPARAMS, - output_type=_OBLITERATERESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='GetMetadataUsage', - full_name='v5api.BTrDB.GetMetadataUsage', - index=17, - containing_service=None, - input_type=_METADATAUSAGEPARAMS, - output_type=_METADATAUSAGERESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='GenerateCSV', - full_name='v5api.BTrDB.GenerateCSV', - index=18, - containing_service=None, - input_type=_GENERATECSVPARAMS, - output_type=_GENERATECSVRESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='SQLQuery', - full_name='v5api.BTrDB.SQLQuery', - index=19, - containing_service=None, - input_type=_SQLQUERYPARAMS, - output_type=_SQLQUERYRESPONSE, - serialized_options=None, - ), -]) -_sym_db.RegisterServiceDescriptor(_BTRDB) - -DESCRIPTOR.services_by_name['BTrDB'] = _BTRDB - +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0b\x62trdb.proto\x12\x05v5api\"Q\n\x0fRawValuesParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\"}\n\x11RawValuesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x1f\n\x06values\x18\x04 \x03(\x0b\x32\x0f.v5api.RawPoint\"j\n\x14\x41lignedWindowsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\x12\x12\n\npointWidth\x18\x05 \x01(\r\"\x83\x01\n\x16\x41lignedWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12 \n\x06values\x18\x04 \x03(\x0b\x32\x10.v5api.StatPoint\"m\n\rWindowsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\x12\r\n\x05width\x18\x05 \x01(\x04\x12\r\n\x05\x64\x65pth\x18\x06 \x01(\r\"|\n\x0fWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12 \n\x06values\x18\x04 \x03(\x0b\x32\x10.v5api.StatPoint\"h\n\x10StreamInfoParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x13\n\x0bomitVersion\x18\x02 \x01(\x08\x12\x16\n\x0eomitDescriptor\x18\x03 \x01(\x08\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"\x8a\x01\n\x12StreamInfoResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12+\n\ndescriptor\x18\x04 \x01(\x0b\x32\x17.v5api.StreamDescriptor\"\x98\x01\n\x10StreamDescriptor\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x12\n\ncollection\x18\x02 \x01(\t\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x17\n\x0fpropertyVersion\x18\x05 \x01(\x04\"\x82\x01\n\x1aSetStreamAnnotationsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x1f\n\x17\x65xpectedPropertyVersion\x18\x02 \x01(\x04\x12#\n\x07\x63hanges\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x10\n\x08removals\x18\x04 \x03(\t\";\n\x1cSetStreamAnnotationsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"\x8a\x01\n\x13SetStreamTagsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x1f\n\x17\x65xpectedPropertyVersion\x18\x02 \x01(\x04\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x12\n\ncollection\x18\x04 \x01(\t\x12\x0e\n\x06remove\x18\x05 \x03(\t\"4\n\x15SetStreamTagsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"{\n\x0c\x43reateParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x12\n\ncollection\x18\x02 \x01(\t\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\"-\n\x0e\x43reateResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"@\n\x13MetadataUsageParams\x12\x0e\n\x06prefix\x18\x01 \x01(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"y\n\x15MetadataUsageResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x1d\n\x04tags\x18\x02 \x03(\x0b\x32\x0f.v5api.KeyCount\x12$\n\x0b\x61nnotations\x18\x03 \x03(\x0b\x32\x0f.v5api.KeyCount\"&\n\x08KeyCount\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05\x63ount\x18\x02 \x01(\x04\"B\n\x15ListCollectionsParams\x12\x0e\n\x06prefix\x18\x01 \x01(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"K\n\x17ListCollectionsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x13\n\x0b\x63ollections\x18\x02 \x03(\t\"\xab\x01\n\x13LookupStreamsParams\x12\x12\n\ncollection\x18\x01 \x01(\t\x12\x1a\n\x12isCollectionPrefix\x18\x02 \x01(\x08\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"^\n\x15LookupStreamsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12(\n\x07results\x18\x02 \x03(\x0b\x32\x17.v5api.StreamDescriptor\"S\n\rNearestParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x0c\n\x04time\x18\x02 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x03 \x01(\x04\x12\x10\n\x08\x62\x61\x63kward\x18\x04 \x01(\x08\"z\n\x0fNearestResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x1e\n\x05value\x18\x04 \x01(\x0b\x32\x0f.v5api.RawPoint\"U\n\rChangesParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x11\n\tfromMajor\x18\x02 \x01(\x04\x12\x0f\n\x07toMajor\x18\x03 \x01(\x04\x12\x12\n\nresolution\x18\x04 \x01(\r\"\x7f\n\x0f\x43hangesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12#\n\x06ranges\x18\x04 \x03(\x0b\x32\x13.v5api.ChangedRange\"u\n\x0cInsertParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x0c\n\x04sync\x18\x02 \x01(\x08\x12(\n\x0cmerge_policy\x18\x04 \x01(\x0e\x32\x12.v5api.MergePolicy\x12\x1f\n\x06values\x18\x03 \x03(\x0b\x32\x0f.v5api.RawPoint\"Y\n\x0eInsertResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\"8\n\x0c\x44\x65leteParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\"Y\n\x0e\x44\x65leteResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\"\x0c\n\nInfoParams\"\xa2\x01\n\x0cInfoResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x19\n\x04mash\x18\x02 \x01(\x0b\x32\x0b.v5api.Mash\x12\x14\n\x0cmajorVersion\x18\x03 \x01(\r\x12\x14\n\x0cminorVersion\x18\x04 \x01(\r\x12\r\n\x05\x62uild\x18\x05 \x01(\t\x12\x1f\n\x05proxy\x18\x06 \x01(\x0b\x32\x10.v5api.ProxyInfo\"#\n\tProxyInfo\x12\x16\n\x0eproxyEndpoints\x18\x01 \x03(\t\"1\n\x11\x46\x61ultInjectParams\x12\x0c\n\x04type\x18\x01 \x01(\x04\x12\x0e\n\x06params\x18\x02 \x01(\x0c\">\n\x13\x46\x61ultInjectResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\n\n\x02rv\x18\x02 \x01(\x0c\"\x1b\n\x0b\x46lushParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"X\n\rFlushResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\" \n\x10ObliterateParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"1\n\x12ObliterateResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"\'\n\x08RawPoint\x12\x0c\n\x04time\x18\x01 \x01(\x10\x12\r\n\x05value\x18\x02 \x01(\x01\"`\n\tStatPoint\x12\x0c\n\x04time\x18\x01 \x01(\x10\x12\x0b\n\x03min\x18\x02 \x01(\x01\x12\x0c\n\x04mean\x18\x03 \x01(\x01\x12\x0b\n\x03max\x18\x04 \x01(\x01\x12\r\n\x05\x63ount\x18\x05 \x01(\x06\x12\x0e\n\x06stddev\x18\x06 \x01(\x01\"*\n\x0c\x43hangedRange\x12\r\n\x05start\x18\x01 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x02 \x01(\x10\">\n\x06Status\x12\x0c\n\x04\x63ode\x18\x01 \x01(\r\x12\x0b\n\x03msg\x18\x02 \x01(\t\x12\x19\n\x04mash\x18\x03 \x01(\x0b\x32\x0b.v5api.Mash\"\x98\x01\n\x04Mash\x12\x10\n\x08revision\x18\x01 \x01(\x03\x12\x0e\n\x06leader\x18\x02 \x01(\t\x12\x16\n\x0eleaderRevision\x18\x03 \x01(\x03\x12\x13\n\x0btotalWeight\x18\x04 \x01(\x03\x12\x0f\n\x07healthy\x18\x05 \x01(\x08\x12\x10\n\x08unmapped\x18\x06 \x01(\x01\x12\x1e\n\x07members\x18\x07 \x03(\x0b\x32\r.v5api.Member\"\xc3\x01\n\x06Member\x12\x0c\n\x04hash\x18\x01 \x01(\r\x12\x10\n\x08nodename\x18\x02 \x01(\t\x12\n\n\x02up\x18\x03 \x01(\x08\x12\n\n\x02in\x18\x04 \x01(\x08\x12\x0f\n\x07\x65nabled\x18\x05 \x01(\x08\x12\r\n\x05start\x18\x06 \x01(\x03\x12\x0b\n\x03\x65nd\x18\x07 \x01(\x03\x12\x0e\n\x06weight\x18\x08 \x01(\x03\x12\x16\n\x0ereadPreference\x18\t \x01(\x01\x12\x15\n\rhttpEndpoints\x18\n \x01(\t\x12\x15\n\rgrpcEndpoints\x18\x0b \x01(\t\"8\n\x0bKeyOptValue\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x1c\n\x03val\x18\x02 \x01(\x0b\x32\x0f.v5api.OptValue\"\x19\n\x08OptValue\x12\r\n\x05value\x18\x01 \x01(\t\"&\n\x08KeyValue\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"?\n\x0fStreamCSVConfig\x12\x0f\n\x07version\x18\x01 \x01(\x04\x12\r\n\x05label\x18\x02 \x01(\t\x12\x0c\n\x04uuid\x18\x03 \x01(\x0c\"\x9d\x02\n\x11GenerateCSVParams\x12\x35\n\tqueryType\x18\x01 \x01(\x0e\x32\".v5api.GenerateCSVParams.QueryType\x12\x11\n\tstartTime\x18\x02 \x01(\x03\x12\x0f\n\x07\x65ndTime\x18\x03 \x01(\x03\x12\x12\n\nwindowSize\x18\x04 \x01(\x04\x12\r\n\x05\x64\x65pth\x18\x05 \x01(\r\x12\x17\n\x0fincludeVersions\x18\x06 \x01(\x08\x12\'\n\x07streams\x18\x07 \x03(\x0b\x32\x16.v5api.StreamCSVConfig\"H\n\tQueryType\x12\x19\n\x15\x41LIGNED_WINDOWS_QUERY\x10\x00\x12\x11\n\rWINDOWS_QUERY\x10\x01\x12\r\n\tRAW_QUERY\x10\x02\"Q\n\x13GenerateCSVResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x10\n\x08isHeader\x18\x02 \x01(\x08\x12\x0b\n\x03row\x18\x03 \x03(\t\"J\n\x0eSQLQueryParams\x12\r\n\x05query\x18\x01 \x01(\t\x12\x0e\n\x06params\x18\x02 \x03(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"D\n\x10SQLQueryResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x13\n\x0bSQLQueryRow\x18\x02 \x03(\x0c\"\x14\n\x04Role\x12\x0c\n\x04name\x18\x01 \x01(\t\"\x94\x01\n\x19SetCompactionConfigParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x18\n\x10\x43ompactedVersion\x18\x02 \x01(\x04\x12>\n\x17reducedResolutionRanges\x18\x03 \x03(\x0b\x32\x1d.v5api.ReducedResolutionRange\x12\x0f\n\x07unused0\x18\x04 \x01(\x04\":\n\x1bSetCompactionConfigResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\")\n\x19GetCompactionConfigParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"\xc1\x01\n\x1bGetCompactionConfigResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x1a\n\x12LatestMajorVersion\x18\x02 \x01(\x04\x12\x18\n\x10\x43ompactedVersion\x18\x03 \x01(\x04\x12>\n\x17reducedResolutionRanges\x18\x04 \x03(\x0b\x32\x1d.v5api.ReducedResolutionRange\x12\x0f\n\x07unused0\x18\x05 \x01(\x04\"H\n\x16ReducedResolutionRange\x12\r\n\x05Start\x18\x01 \x01(\x03\x12\x0b\n\x03\x45nd\x18\x02 \x01(\x03\x12\x12\n\nResolution\x18\x03 \x01(\r*<\n\x0bMergePolicy\x12\t\n\x05NEVER\x10\x00\x12\t\n\x05\x45QUAL\x10\x01\x12\n\n\x06RETAIN\x10\x02\x12\x0b\n\x07REPLACE\x10\x03\x32\xb3\n\n\x05\x42TrDB\x12?\n\tRawValues\x12\x16.v5api.RawValuesParams\x1a\x18.v5api.RawValuesResponse0\x01\x12N\n\x0e\x41lignedWindows\x12\x1b.v5api.AlignedWindowsParams\x1a\x1d.v5api.AlignedWindowsResponse0\x01\x12\x39\n\x07Windows\x12\x14.v5api.WindowsParams\x1a\x16.v5api.WindowsResponse0\x01\x12@\n\nStreamInfo\x12\x17.v5api.StreamInfoParams\x1a\x19.v5api.StreamInfoResponse\x12^\n\x14SetStreamAnnotations\x12!.v5api.SetStreamAnnotationsParams\x1a#.v5api.SetStreamAnnotationsResponse\x12I\n\rSetStreamTags\x12\x1a.v5api.SetStreamTagsParams\x1a\x1c.v5api.SetStreamTagsResponse\x12\x34\n\x06\x43reate\x12\x13.v5api.CreateParams\x1a\x15.v5api.CreateResponse\x12Q\n\x0fListCollections\x12\x1c.v5api.ListCollectionsParams\x1a\x1e.v5api.ListCollectionsResponse0\x01\x12K\n\rLookupStreams\x12\x1a.v5api.LookupStreamsParams\x1a\x1c.v5api.LookupStreamsResponse0\x01\x12\x37\n\x07Nearest\x12\x14.v5api.NearestParams\x1a\x16.v5api.NearestResponse\x12\x39\n\x07\x43hanges\x12\x14.v5api.ChangesParams\x1a\x16.v5api.ChangesResponse0\x01\x12\x34\n\x06Insert\x12\x13.v5api.InsertParams\x1a\x15.v5api.InsertResponse\x12\x34\n\x06\x44\x65lete\x12\x13.v5api.DeleteParams\x1a\x15.v5api.DeleteResponse\x12.\n\x04Info\x12\x11.v5api.InfoParams\x1a\x13.v5api.InfoResponse\x12\x43\n\x0b\x46\x61ultInject\x12\x18.v5api.FaultInjectParams\x1a\x1a.v5api.FaultInjectResponse\x12\x31\n\x05\x46lush\x12\x12.v5api.FlushParams\x1a\x14.v5api.FlushResponse\x12@\n\nObliterate\x12\x17.v5api.ObliterateParams\x1a\x19.v5api.ObliterateResponse\x12L\n\x10GetMetadataUsage\x12\x1a.v5api.MetadataUsageParams\x1a\x1c.v5api.MetadataUsageResponse\x12\x45\n\x0bGenerateCSV\x12\x18.v5api.GenerateCSVParams\x1a\x1a.v5api.GenerateCSVResponse0\x01\x12<\n\x08SQLQuery\x12\x15.v5api.SQLQueryParams\x1a\x17.v5api.SQLQueryResponse0\x01\x62\x06proto3') + +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'btrdb_pb2', globals()) +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + _MERGEPOLICY._serialized_start=5490 + _MERGEPOLICY._serialized_end=5550 + _RAWVALUESPARAMS._serialized_start=22 + _RAWVALUESPARAMS._serialized_end=103 + _RAWVALUESRESPONSE._serialized_start=105 + _RAWVALUESRESPONSE._serialized_end=230 + _ALIGNEDWINDOWSPARAMS._serialized_start=232 + _ALIGNEDWINDOWSPARAMS._serialized_end=338 + _ALIGNEDWINDOWSRESPONSE._serialized_start=341 + _ALIGNEDWINDOWSRESPONSE._serialized_end=472 + _WINDOWSPARAMS._serialized_start=474 + _WINDOWSPARAMS._serialized_end=583 + _WINDOWSRESPONSE._serialized_start=585 + _WINDOWSRESPONSE._serialized_end=709 + _STREAMINFOPARAMS._serialized_start=711 + _STREAMINFOPARAMS._serialized_end=815 + _STREAMINFORESPONSE._serialized_start=818 + _STREAMINFORESPONSE._serialized_end=956 + _STREAMDESCRIPTOR._serialized_start=959 + _STREAMDESCRIPTOR._serialized_end=1111 + _SETSTREAMANNOTATIONSPARAMS._serialized_start=1114 + _SETSTREAMANNOTATIONSPARAMS._serialized_end=1244 + _SETSTREAMANNOTATIONSRESPONSE._serialized_start=1246 + _SETSTREAMANNOTATIONSRESPONSE._serialized_end=1305 + _SETSTREAMTAGSPARAMS._serialized_start=1308 + _SETSTREAMTAGSPARAMS._serialized_end=1446 + _SETSTREAMTAGSRESPONSE._serialized_start=1448 + _SETSTREAMTAGSRESPONSE._serialized_end=1500 + _CREATEPARAMS._serialized_start=1502 + _CREATEPARAMS._serialized_end=1625 + _CREATERESPONSE._serialized_start=1627 + _CREATERESPONSE._serialized_end=1672 + _METADATAUSAGEPARAMS._serialized_start=1674 + _METADATAUSAGEPARAMS._serialized_end=1738 + _METADATAUSAGERESPONSE._serialized_start=1740 + _METADATAUSAGERESPONSE._serialized_end=1861 + _KEYCOUNT._serialized_start=1863 + _KEYCOUNT._serialized_end=1901 + _LISTCOLLECTIONSPARAMS._serialized_start=1903 + _LISTCOLLECTIONSPARAMS._serialized_end=1969 + _LISTCOLLECTIONSRESPONSE._serialized_start=1971 + _LISTCOLLECTIONSRESPONSE._serialized_end=2046 + _LOOKUPSTREAMSPARAMS._serialized_start=2049 + _LOOKUPSTREAMSPARAMS._serialized_end=2220 + _LOOKUPSTREAMSRESPONSE._serialized_start=2222 + _LOOKUPSTREAMSRESPONSE._serialized_end=2316 + _NEARESTPARAMS._serialized_start=2318 + _NEARESTPARAMS._serialized_end=2401 + _NEARESTRESPONSE._serialized_start=2403 + _NEARESTRESPONSE._serialized_end=2525 + _CHANGESPARAMS._serialized_start=2527 + _CHANGESPARAMS._serialized_end=2612 + _CHANGESRESPONSE._serialized_start=2614 + _CHANGESRESPONSE._serialized_end=2741 + _INSERTPARAMS._serialized_start=2743 + _INSERTPARAMS._serialized_end=2860 + _INSERTRESPONSE._serialized_start=2862 + _INSERTRESPONSE._serialized_end=2951 + _DELETEPARAMS._serialized_start=2953 + _DELETEPARAMS._serialized_end=3009 + _DELETERESPONSE._serialized_start=3011 + _DELETERESPONSE._serialized_end=3100 + _INFOPARAMS._serialized_start=3102 + _INFOPARAMS._serialized_end=3114 + _INFORESPONSE._serialized_start=3117 + _INFORESPONSE._serialized_end=3279 + _PROXYINFO._serialized_start=3281 + _PROXYINFO._serialized_end=3316 + _FAULTINJECTPARAMS._serialized_start=3318 + _FAULTINJECTPARAMS._serialized_end=3367 + _FAULTINJECTRESPONSE._serialized_start=3369 + _FAULTINJECTRESPONSE._serialized_end=3431 + _FLUSHPARAMS._serialized_start=3433 + _FLUSHPARAMS._serialized_end=3460 + _FLUSHRESPONSE._serialized_start=3462 + _FLUSHRESPONSE._serialized_end=3550 + _OBLITERATEPARAMS._serialized_start=3552 + _OBLITERATEPARAMS._serialized_end=3584 + _OBLITERATERESPONSE._serialized_start=3586 + _OBLITERATERESPONSE._serialized_end=3635 + _RAWPOINT._serialized_start=3637 + _RAWPOINT._serialized_end=3676 + _STATPOINT._serialized_start=3678 + _STATPOINT._serialized_end=3774 + _CHANGEDRANGE._serialized_start=3776 + _CHANGEDRANGE._serialized_end=3818 + _STATUS._serialized_start=3820 + _STATUS._serialized_end=3882 + _MASH._serialized_start=3885 + _MASH._serialized_end=4037 + _MEMBER._serialized_start=4040 + _MEMBER._serialized_end=4235 + _KEYOPTVALUE._serialized_start=4237 + _KEYOPTVALUE._serialized_end=4293 + _OPTVALUE._serialized_start=4295 + _OPTVALUE._serialized_end=4320 + _KEYVALUE._serialized_start=4322 + _KEYVALUE._serialized_end=4360 + _STREAMCSVCONFIG._serialized_start=4362 + _STREAMCSVCONFIG._serialized_end=4425 + _GENERATECSVPARAMS._serialized_start=4428 + _GENERATECSVPARAMS._serialized_end=4713 + _GENERATECSVPARAMS_QUERYTYPE._serialized_start=4641 + _GENERATECSVPARAMS_QUERYTYPE._serialized_end=4713 + _GENERATECSVRESPONSE._serialized_start=4715 + _GENERATECSVRESPONSE._serialized_end=4796 + _SQLQUERYPARAMS._serialized_start=4798 + _SQLQUERYPARAMS._serialized_end=4872 + _SQLQUERYRESPONSE._serialized_start=4874 + _SQLQUERYRESPONSE._serialized_end=4942 + _ROLE._serialized_start=4944 + _ROLE._serialized_end=4964 + _SETCOMPACTIONCONFIGPARAMS._serialized_start=4967 + _SETCOMPACTIONCONFIGPARAMS._serialized_end=5115 + _SETCOMPACTIONCONFIGRESPONSE._serialized_start=5117 + _SETCOMPACTIONCONFIGRESPONSE._serialized_end=5175 + _GETCOMPACTIONCONFIGPARAMS._serialized_start=5177 + _GETCOMPACTIONCONFIGPARAMS._serialized_end=5218 + _GETCOMPACTIONCONFIGRESPONSE._serialized_start=5221 + _GETCOMPACTIONCONFIGRESPONSE._serialized_end=5414 + _REDUCEDRESOLUTIONRANGE._serialized_start=5416 + _REDUCEDRESOLUTIONRANGE._serialized_end=5488 + _BTRDB._serialized_start=5553 + _BTRDB._serialized_end=6884 # @@protoc_insertion_point(module_scope) diff --git a/btrdb/grpcinterface/btrdb_pb2.pyi b/btrdb/grpcinterface/btrdb_pb2.pyi new file mode 100644 index 0000000..e9abf5b --- /dev/null +++ b/btrdb/grpcinterface/btrdb_pb2.pyi @@ -0,0 +1,632 @@ +from google.protobuf.internal import containers as _containers +from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from typing import ClassVar as _ClassVar, Iterable as _Iterable, Mapping as _Mapping, Optional as _Optional, Union as _Union + +DESCRIPTOR: _descriptor.FileDescriptor +EQUAL: MergePolicy +NEVER: MergePolicy +REPLACE: MergePolicy +RETAIN: MergePolicy + +class AlignedWindowsParams(_message.Message): + __slots__ = ["end", "pointWidth", "start", "uuid", "versionMajor"] + END_FIELD_NUMBER: _ClassVar[int] + POINTWIDTH_FIELD_NUMBER: _ClassVar[int] + START_FIELD_NUMBER: _ClassVar[int] + UUID_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + end: int + pointWidth: int + start: int + uuid: bytes + versionMajor: int + def __init__(self, uuid: _Optional[bytes] = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., versionMajor: _Optional[int] = ..., pointWidth: _Optional[int] = ...) -> None: ... + +class AlignedWindowsResponse(_message.Message): + __slots__ = ["stat", "values", "versionMajor", "versionMinor"] + STAT_FIELD_NUMBER: _ClassVar[int] + VALUES_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + stat: Status + values: _containers.RepeatedCompositeFieldContainer[StatPoint] + versionMajor: int + versionMinor: int + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., values: _Optional[_Iterable[_Union[StatPoint, _Mapping]]] = ...) -> None: ... + +class ChangedRange(_message.Message): + __slots__ = ["end", "start"] + END_FIELD_NUMBER: _ClassVar[int] + START_FIELD_NUMBER: _ClassVar[int] + end: int + start: int + def __init__(self, start: _Optional[int] = ..., end: _Optional[int] = ...) -> None: ... + +class ChangesParams(_message.Message): + __slots__ = ["fromMajor", "resolution", "toMajor", "uuid"] + FROMMAJOR_FIELD_NUMBER: _ClassVar[int] + RESOLUTION_FIELD_NUMBER: _ClassVar[int] + TOMAJOR_FIELD_NUMBER: _ClassVar[int] + UUID_FIELD_NUMBER: _ClassVar[int] + fromMajor: int + resolution: int + toMajor: int + uuid: bytes + def __init__(self, uuid: _Optional[bytes] = ..., fromMajor: _Optional[int] = ..., toMajor: _Optional[int] = ..., resolution: _Optional[int] = ...) -> None: ... + +class ChangesResponse(_message.Message): + __slots__ = ["ranges", "stat", "versionMajor", "versionMinor"] + RANGES_FIELD_NUMBER: _ClassVar[int] + STAT_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + ranges: _containers.RepeatedCompositeFieldContainer[ChangedRange] + stat: Status + versionMajor: int + versionMinor: int + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., ranges: _Optional[_Iterable[_Union[ChangedRange, _Mapping]]] = ...) -> None: ... + +class CreateParams(_message.Message): + __slots__ = ["annotations", "collection", "tags", "uuid"] + ANNOTATIONS_FIELD_NUMBER: _ClassVar[int] + COLLECTION_FIELD_NUMBER: _ClassVar[int] + TAGS_FIELD_NUMBER: _ClassVar[int] + UUID_FIELD_NUMBER: _ClassVar[int] + annotations: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + collection: str + tags: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + uuid: bytes + def __init__(self, uuid: _Optional[bytes] = ..., collection: _Optional[str] = ..., tags: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., annotations: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ...) -> None: ... + +class CreateResponse(_message.Message): + __slots__ = ["stat"] + STAT_FIELD_NUMBER: _ClassVar[int] + stat: Status + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ...) -> None: ... + +class DeleteParams(_message.Message): + __slots__ = ["end", "start", "uuid"] + END_FIELD_NUMBER: _ClassVar[int] + START_FIELD_NUMBER: _ClassVar[int] + UUID_FIELD_NUMBER: _ClassVar[int] + end: int + start: int + uuid: bytes + def __init__(self, uuid: _Optional[bytes] = ..., start: _Optional[int] = ..., end: _Optional[int] = ...) -> None: ... + +class DeleteResponse(_message.Message): + __slots__ = ["stat", "versionMajor", "versionMinor"] + STAT_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + stat: Status + versionMajor: int + versionMinor: int + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ...) -> None: ... + +class FaultInjectParams(_message.Message): + __slots__ = ["params", "type"] + PARAMS_FIELD_NUMBER: _ClassVar[int] + TYPE_FIELD_NUMBER: _ClassVar[int] + params: bytes + type: int + def __init__(self, type: _Optional[int] = ..., params: _Optional[bytes] = ...) -> None: ... + +class FaultInjectResponse(_message.Message): + __slots__ = ["rv", "stat"] + RV_FIELD_NUMBER: _ClassVar[int] + STAT_FIELD_NUMBER: _ClassVar[int] + rv: bytes + stat: Status + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., rv: _Optional[bytes] = ...) -> None: ... + +class FlushParams(_message.Message): + __slots__ = ["uuid"] + UUID_FIELD_NUMBER: _ClassVar[int] + uuid: bytes + def __init__(self, uuid: _Optional[bytes] = ...) -> None: ... + +class FlushResponse(_message.Message): + __slots__ = ["stat", "versionMajor", "versionMinor"] + STAT_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + stat: Status + versionMajor: int + versionMinor: int + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ...) -> None: ... + +class GenerateCSVParams(_message.Message): + __slots__ = ["depth", "endTime", "includeVersions", "queryType", "startTime", "streams", "windowSize"] + class QueryType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = [] + ALIGNED_WINDOWS_QUERY: GenerateCSVParams.QueryType + DEPTH_FIELD_NUMBER: _ClassVar[int] + ENDTIME_FIELD_NUMBER: _ClassVar[int] + INCLUDEVERSIONS_FIELD_NUMBER: _ClassVar[int] + QUERYTYPE_FIELD_NUMBER: _ClassVar[int] + RAW_QUERY: GenerateCSVParams.QueryType + STARTTIME_FIELD_NUMBER: _ClassVar[int] + STREAMS_FIELD_NUMBER: _ClassVar[int] + WINDOWSIZE_FIELD_NUMBER: _ClassVar[int] + WINDOWS_QUERY: GenerateCSVParams.QueryType + depth: int + endTime: int + includeVersions: bool + queryType: GenerateCSVParams.QueryType + startTime: int + streams: _containers.RepeatedCompositeFieldContainer[StreamCSVConfig] + windowSize: int + def __init__(self, queryType: _Optional[_Union[GenerateCSVParams.QueryType, str]] = ..., startTime: _Optional[int] = ..., endTime: _Optional[int] = ..., windowSize: _Optional[int] = ..., depth: _Optional[int] = ..., includeVersions: bool = ..., streams: _Optional[_Iterable[_Union[StreamCSVConfig, _Mapping]]] = ...) -> None: ... + +class GenerateCSVResponse(_message.Message): + __slots__ = ["isHeader", "row", "stat"] + ISHEADER_FIELD_NUMBER: _ClassVar[int] + ROW_FIELD_NUMBER: _ClassVar[int] + STAT_FIELD_NUMBER: _ClassVar[int] + isHeader: bool + row: _containers.RepeatedScalarFieldContainer[str] + stat: Status + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., isHeader: bool = ..., row: _Optional[_Iterable[str]] = ...) -> None: ... + +class GetCompactionConfigParams(_message.Message): + __slots__ = ["uuid"] + UUID_FIELD_NUMBER: _ClassVar[int] + uuid: bytes + def __init__(self, uuid: _Optional[bytes] = ...) -> None: ... + +class GetCompactionConfigResponse(_message.Message): + __slots__ = ["CompactedVersion", "LatestMajorVersion", "reducedResolutionRanges", "stat", "unused0"] + COMPACTEDVERSION_FIELD_NUMBER: _ClassVar[int] + CompactedVersion: int + LATESTMAJORVERSION_FIELD_NUMBER: _ClassVar[int] + LatestMajorVersion: int + REDUCEDRESOLUTIONRANGES_FIELD_NUMBER: _ClassVar[int] + STAT_FIELD_NUMBER: _ClassVar[int] + UNUSED0_FIELD_NUMBER: _ClassVar[int] + reducedResolutionRanges: _containers.RepeatedCompositeFieldContainer[ReducedResolutionRange] + stat: Status + unused0: int + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., LatestMajorVersion: _Optional[int] = ..., CompactedVersion: _Optional[int] = ..., reducedResolutionRanges: _Optional[_Iterable[_Union[ReducedResolutionRange, _Mapping]]] = ..., unused0: _Optional[int] = ...) -> None: ... + +class InfoParams(_message.Message): + __slots__ = [] + def __init__(self) -> None: ... + +class InfoResponse(_message.Message): + __slots__ = ["build", "majorVersion", "mash", "minorVersion", "proxy", "stat"] + BUILD_FIELD_NUMBER: _ClassVar[int] + MAJORVERSION_FIELD_NUMBER: _ClassVar[int] + MASH_FIELD_NUMBER: _ClassVar[int] + MINORVERSION_FIELD_NUMBER: _ClassVar[int] + PROXY_FIELD_NUMBER: _ClassVar[int] + STAT_FIELD_NUMBER: _ClassVar[int] + build: str + majorVersion: int + mash: Mash + minorVersion: int + proxy: ProxyInfo + stat: Status + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., mash: _Optional[_Union[Mash, _Mapping]] = ..., majorVersion: _Optional[int] = ..., minorVersion: _Optional[int] = ..., build: _Optional[str] = ..., proxy: _Optional[_Union[ProxyInfo, _Mapping]] = ...) -> None: ... + +class InsertParams(_message.Message): + __slots__ = ["merge_policy", "sync", "uuid", "values"] + MERGE_POLICY_FIELD_NUMBER: _ClassVar[int] + SYNC_FIELD_NUMBER: _ClassVar[int] + UUID_FIELD_NUMBER: _ClassVar[int] + VALUES_FIELD_NUMBER: _ClassVar[int] + merge_policy: MergePolicy + sync: bool + uuid: bytes + values: _containers.RepeatedCompositeFieldContainer[RawPoint] + def __init__(self, uuid: _Optional[bytes] = ..., sync: bool = ..., merge_policy: _Optional[_Union[MergePolicy, str]] = ..., values: _Optional[_Iterable[_Union[RawPoint, _Mapping]]] = ...) -> None: ... + +class InsertResponse(_message.Message): + __slots__ = ["stat", "versionMajor", "versionMinor"] + STAT_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + stat: Status + versionMajor: int + versionMinor: int + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ...) -> None: ... + +class KeyCount(_message.Message): + __slots__ = ["count", "key"] + COUNT_FIELD_NUMBER: _ClassVar[int] + KEY_FIELD_NUMBER: _ClassVar[int] + count: int + key: str + def __init__(self, key: _Optional[str] = ..., count: _Optional[int] = ...) -> None: ... + +class KeyOptValue(_message.Message): + __slots__ = ["key", "val"] + KEY_FIELD_NUMBER: _ClassVar[int] + VAL_FIELD_NUMBER: _ClassVar[int] + key: str + val: OptValue + def __init__(self, key: _Optional[str] = ..., val: _Optional[_Union[OptValue, _Mapping]] = ...) -> None: ... + +class KeyValue(_message.Message): + __slots__ = ["key", "value"] + KEY_FIELD_NUMBER: _ClassVar[int] + VALUE_FIELD_NUMBER: _ClassVar[int] + key: str + value: str + def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ... + +class ListCollectionsParams(_message.Message): + __slots__ = ["prefix", "role"] + PREFIX_FIELD_NUMBER: _ClassVar[int] + ROLE_FIELD_NUMBER: _ClassVar[int] + prefix: str + role: Role + def __init__(self, prefix: _Optional[str] = ..., role: _Optional[_Union[Role, _Mapping]] = ...) -> None: ... + +class ListCollectionsResponse(_message.Message): + __slots__ = ["collections", "stat"] + COLLECTIONS_FIELD_NUMBER: _ClassVar[int] + STAT_FIELD_NUMBER: _ClassVar[int] + collections: _containers.RepeatedScalarFieldContainer[str] + stat: Status + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., collections: _Optional[_Iterable[str]] = ...) -> None: ... + +class LookupStreamsParams(_message.Message): + __slots__ = ["annotations", "collection", "isCollectionPrefix", "role", "tags"] + ANNOTATIONS_FIELD_NUMBER: _ClassVar[int] + COLLECTION_FIELD_NUMBER: _ClassVar[int] + ISCOLLECTIONPREFIX_FIELD_NUMBER: _ClassVar[int] + ROLE_FIELD_NUMBER: _ClassVar[int] + TAGS_FIELD_NUMBER: _ClassVar[int] + annotations: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + collection: str + isCollectionPrefix: bool + role: Role + tags: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + def __init__(self, collection: _Optional[str] = ..., isCollectionPrefix: bool = ..., tags: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., annotations: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., role: _Optional[_Union[Role, _Mapping]] = ...) -> None: ... + +class LookupStreamsResponse(_message.Message): + __slots__ = ["results", "stat"] + RESULTS_FIELD_NUMBER: _ClassVar[int] + STAT_FIELD_NUMBER: _ClassVar[int] + results: _containers.RepeatedCompositeFieldContainer[StreamDescriptor] + stat: Status + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., results: _Optional[_Iterable[_Union[StreamDescriptor, _Mapping]]] = ...) -> None: ... + +class Mash(_message.Message): + __slots__ = ["healthy", "leader", "leaderRevision", "members", "revision", "totalWeight", "unmapped"] + HEALTHY_FIELD_NUMBER: _ClassVar[int] + LEADERREVISION_FIELD_NUMBER: _ClassVar[int] + LEADER_FIELD_NUMBER: _ClassVar[int] + MEMBERS_FIELD_NUMBER: _ClassVar[int] + REVISION_FIELD_NUMBER: _ClassVar[int] + TOTALWEIGHT_FIELD_NUMBER: _ClassVar[int] + UNMAPPED_FIELD_NUMBER: _ClassVar[int] + healthy: bool + leader: str + leaderRevision: int + members: _containers.RepeatedCompositeFieldContainer[Member] + revision: int + totalWeight: int + unmapped: float + def __init__(self, revision: _Optional[int] = ..., leader: _Optional[str] = ..., leaderRevision: _Optional[int] = ..., totalWeight: _Optional[int] = ..., healthy: bool = ..., unmapped: _Optional[float] = ..., members: _Optional[_Iterable[_Union[Member, _Mapping]]] = ...) -> None: ... + +class Member(_message.Message): + __slots__ = ["enabled", "end", "grpcEndpoints", "hash", "httpEndpoints", "nodename", "readPreference", "start", "up", "weight"] + ENABLED_FIELD_NUMBER: _ClassVar[int] + END_FIELD_NUMBER: _ClassVar[int] + GRPCENDPOINTS_FIELD_NUMBER: _ClassVar[int] + HASH_FIELD_NUMBER: _ClassVar[int] + HTTPENDPOINTS_FIELD_NUMBER: _ClassVar[int] + IN_FIELD_NUMBER: _ClassVar[int] + NODENAME_FIELD_NUMBER: _ClassVar[int] + READPREFERENCE_FIELD_NUMBER: _ClassVar[int] + START_FIELD_NUMBER: _ClassVar[int] + UP_FIELD_NUMBER: _ClassVar[int] + WEIGHT_FIELD_NUMBER: _ClassVar[int] + enabled: bool + end: int + grpcEndpoints: str + hash: int + httpEndpoints: str + nodename: str + readPreference: float + start: int + up: bool + weight: int + def __init__(self, hash: _Optional[int] = ..., nodename: _Optional[str] = ..., up: bool = ..., enabled: bool = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., weight: _Optional[int] = ..., readPreference: _Optional[float] = ..., httpEndpoints: _Optional[str] = ..., grpcEndpoints: _Optional[str] = ..., **kwargs) -> None: ... + +class MetadataUsageParams(_message.Message): + __slots__ = ["prefix", "role"] + PREFIX_FIELD_NUMBER: _ClassVar[int] + ROLE_FIELD_NUMBER: _ClassVar[int] + prefix: str + role: Role + def __init__(self, prefix: _Optional[str] = ..., role: _Optional[_Union[Role, _Mapping]] = ...) -> None: ... + +class MetadataUsageResponse(_message.Message): + __slots__ = ["annotations", "stat", "tags"] + ANNOTATIONS_FIELD_NUMBER: _ClassVar[int] + STAT_FIELD_NUMBER: _ClassVar[int] + TAGS_FIELD_NUMBER: _ClassVar[int] + annotations: _containers.RepeatedCompositeFieldContainer[KeyCount] + stat: Status + tags: _containers.RepeatedCompositeFieldContainer[KeyCount] + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., tags: _Optional[_Iterable[_Union[KeyCount, _Mapping]]] = ..., annotations: _Optional[_Iterable[_Union[KeyCount, _Mapping]]] = ...) -> None: ... + +class NearestParams(_message.Message): + __slots__ = ["backward", "time", "uuid", "versionMajor"] + BACKWARD_FIELD_NUMBER: _ClassVar[int] + TIME_FIELD_NUMBER: _ClassVar[int] + UUID_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + backward: bool + time: int + uuid: bytes + versionMajor: int + def __init__(self, uuid: _Optional[bytes] = ..., time: _Optional[int] = ..., versionMajor: _Optional[int] = ..., backward: bool = ...) -> None: ... + +class NearestResponse(_message.Message): + __slots__ = ["stat", "value", "versionMajor", "versionMinor"] + STAT_FIELD_NUMBER: _ClassVar[int] + VALUE_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + stat: Status + value: RawPoint + versionMajor: int + versionMinor: int + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., value: _Optional[_Union[RawPoint, _Mapping]] = ...) -> None: ... + +class ObliterateParams(_message.Message): + __slots__ = ["uuid"] + UUID_FIELD_NUMBER: _ClassVar[int] + uuid: bytes + def __init__(self, uuid: _Optional[bytes] = ...) -> None: ... + +class ObliterateResponse(_message.Message): + __slots__ = ["stat"] + STAT_FIELD_NUMBER: _ClassVar[int] + stat: Status + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ...) -> None: ... + +class OptValue(_message.Message): + __slots__ = ["value"] + VALUE_FIELD_NUMBER: _ClassVar[int] + value: str + def __init__(self, value: _Optional[str] = ...) -> None: ... + +class ProxyInfo(_message.Message): + __slots__ = ["proxyEndpoints"] + PROXYENDPOINTS_FIELD_NUMBER: _ClassVar[int] + proxyEndpoints: _containers.RepeatedScalarFieldContainer[str] + def __init__(self, proxyEndpoints: _Optional[_Iterable[str]] = ...) -> None: ... + +class RawPoint(_message.Message): + __slots__ = ["time", "value"] + TIME_FIELD_NUMBER: _ClassVar[int] + VALUE_FIELD_NUMBER: _ClassVar[int] + time: int + value: float + def __init__(self, time: _Optional[int] = ..., value: _Optional[float] = ...) -> None: ... + +class RawValuesParams(_message.Message): + __slots__ = ["end", "start", "uuid", "versionMajor"] + END_FIELD_NUMBER: _ClassVar[int] + START_FIELD_NUMBER: _ClassVar[int] + UUID_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + end: int + start: int + uuid: bytes + versionMajor: int + def __init__(self, uuid: _Optional[bytes] = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., versionMajor: _Optional[int] = ...) -> None: ... + +class RawValuesResponse(_message.Message): + __slots__ = ["stat", "values", "versionMajor", "versionMinor"] + STAT_FIELD_NUMBER: _ClassVar[int] + VALUES_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + stat: Status + values: _containers.RepeatedCompositeFieldContainer[RawPoint] + versionMajor: int + versionMinor: int + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., values: _Optional[_Iterable[_Union[RawPoint, _Mapping]]] = ...) -> None: ... + +class ReducedResolutionRange(_message.Message): + __slots__ = ["End", "Resolution", "Start"] + END_FIELD_NUMBER: _ClassVar[int] + End: int + RESOLUTION_FIELD_NUMBER: _ClassVar[int] + Resolution: int + START_FIELD_NUMBER: _ClassVar[int] + Start: int + def __init__(self, Start: _Optional[int] = ..., End: _Optional[int] = ..., Resolution: _Optional[int] = ...) -> None: ... + +class Role(_message.Message): + __slots__ = ["name"] + NAME_FIELD_NUMBER: _ClassVar[int] + name: str + def __init__(self, name: _Optional[str] = ...) -> None: ... + +class SQLQueryParams(_message.Message): + __slots__ = ["params", "query", "role"] + PARAMS_FIELD_NUMBER: _ClassVar[int] + QUERY_FIELD_NUMBER: _ClassVar[int] + ROLE_FIELD_NUMBER: _ClassVar[int] + params: _containers.RepeatedScalarFieldContainer[str] + query: str + role: Role + def __init__(self, query: _Optional[str] = ..., params: _Optional[_Iterable[str]] = ..., role: _Optional[_Union[Role, _Mapping]] = ...) -> None: ... + +class SQLQueryResponse(_message.Message): + __slots__ = ["SQLQueryRow", "stat"] + SQLQUERYROW_FIELD_NUMBER: _ClassVar[int] + SQLQueryRow: _containers.RepeatedScalarFieldContainer[bytes] + STAT_FIELD_NUMBER: _ClassVar[int] + stat: Status + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., SQLQueryRow: _Optional[_Iterable[bytes]] = ...) -> None: ... + +class SetCompactionConfigParams(_message.Message): + __slots__ = ["CompactedVersion", "reducedResolutionRanges", "unused0", "uuid"] + COMPACTEDVERSION_FIELD_NUMBER: _ClassVar[int] + CompactedVersion: int + REDUCEDRESOLUTIONRANGES_FIELD_NUMBER: _ClassVar[int] + UNUSED0_FIELD_NUMBER: _ClassVar[int] + UUID_FIELD_NUMBER: _ClassVar[int] + reducedResolutionRanges: _containers.RepeatedCompositeFieldContainer[ReducedResolutionRange] + unused0: int + uuid: bytes + def __init__(self, uuid: _Optional[bytes] = ..., CompactedVersion: _Optional[int] = ..., reducedResolutionRanges: _Optional[_Iterable[_Union[ReducedResolutionRange, _Mapping]]] = ..., unused0: _Optional[int] = ...) -> None: ... + +class SetCompactionConfigResponse(_message.Message): + __slots__ = ["stat"] + STAT_FIELD_NUMBER: _ClassVar[int] + stat: Status + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ...) -> None: ... + +class SetStreamAnnotationsParams(_message.Message): + __slots__ = ["changes", "expectedPropertyVersion", "removals", "uuid"] + CHANGES_FIELD_NUMBER: _ClassVar[int] + EXPECTEDPROPERTYVERSION_FIELD_NUMBER: _ClassVar[int] + REMOVALS_FIELD_NUMBER: _ClassVar[int] + UUID_FIELD_NUMBER: _ClassVar[int] + changes: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + expectedPropertyVersion: int + removals: _containers.RepeatedScalarFieldContainer[str] + uuid: bytes + def __init__(self, uuid: _Optional[bytes] = ..., expectedPropertyVersion: _Optional[int] = ..., changes: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., removals: _Optional[_Iterable[str]] = ...) -> None: ... + +class SetStreamAnnotationsResponse(_message.Message): + __slots__ = ["stat"] + STAT_FIELD_NUMBER: _ClassVar[int] + stat: Status + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ...) -> None: ... + +class SetStreamTagsParams(_message.Message): + __slots__ = ["collection", "expectedPropertyVersion", "remove", "tags", "uuid"] + COLLECTION_FIELD_NUMBER: _ClassVar[int] + EXPECTEDPROPERTYVERSION_FIELD_NUMBER: _ClassVar[int] + REMOVE_FIELD_NUMBER: _ClassVar[int] + TAGS_FIELD_NUMBER: _ClassVar[int] + UUID_FIELD_NUMBER: _ClassVar[int] + collection: str + expectedPropertyVersion: int + remove: _containers.RepeatedScalarFieldContainer[str] + tags: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + uuid: bytes + def __init__(self, uuid: _Optional[bytes] = ..., expectedPropertyVersion: _Optional[int] = ..., tags: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., collection: _Optional[str] = ..., remove: _Optional[_Iterable[str]] = ...) -> None: ... + +class SetStreamTagsResponse(_message.Message): + __slots__ = ["stat"] + STAT_FIELD_NUMBER: _ClassVar[int] + stat: Status + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ...) -> None: ... + +class StatPoint(_message.Message): + __slots__ = ["count", "max", "mean", "min", "stddev", "time"] + COUNT_FIELD_NUMBER: _ClassVar[int] + MAX_FIELD_NUMBER: _ClassVar[int] + MEAN_FIELD_NUMBER: _ClassVar[int] + MIN_FIELD_NUMBER: _ClassVar[int] + STDDEV_FIELD_NUMBER: _ClassVar[int] + TIME_FIELD_NUMBER: _ClassVar[int] + count: int + max: float + mean: float + min: float + stddev: float + time: int + def __init__(self, time: _Optional[int] = ..., min: _Optional[float] = ..., mean: _Optional[float] = ..., max: _Optional[float] = ..., count: _Optional[int] = ..., stddev: _Optional[float] = ...) -> None: ... + +class Status(_message.Message): + __slots__ = ["code", "mash", "msg"] + CODE_FIELD_NUMBER: _ClassVar[int] + MASH_FIELD_NUMBER: _ClassVar[int] + MSG_FIELD_NUMBER: _ClassVar[int] + code: int + mash: Mash + msg: str + def __init__(self, code: _Optional[int] = ..., msg: _Optional[str] = ..., mash: _Optional[_Union[Mash, _Mapping]] = ...) -> None: ... + +class StreamCSVConfig(_message.Message): + __slots__ = ["label", "uuid", "version"] + LABEL_FIELD_NUMBER: _ClassVar[int] + UUID_FIELD_NUMBER: _ClassVar[int] + VERSION_FIELD_NUMBER: _ClassVar[int] + label: str + uuid: bytes + version: int + def __init__(self, version: _Optional[int] = ..., label: _Optional[str] = ..., uuid: _Optional[bytes] = ...) -> None: ... + +class StreamDescriptor(_message.Message): + __slots__ = ["annotations", "collection", "propertyVersion", "tags", "uuid"] + ANNOTATIONS_FIELD_NUMBER: _ClassVar[int] + COLLECTION_FIELD_NUMBER: _ClassVar[int] + PROPERTYVERSION_FIELD_NUMBER: _ClassVar[int] + TAGS_FIELD_NUMBER: _ClassVar[int] + UUID_FIELD_NUMBER: _ClassVar[int] + annotations: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + collection: str + propertyVersion: int + tags: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + uuid: bytes + def __init__(self, uuid: _Optional[bytes] = ..., collection: _Optional[str] = ..., tags: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., annotations: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., propertyVersion: _Optional[int] = ...) -> None: ... + +class StreamInfoParams(_message.Message): + __slots__ = ["omitDescriptor", "omitVersion", "role", "uuid"] + OMITDESCRIPTOR_FIELD_NUMBER: _ClassVar[int] + OMITVERSION_FIELD_NUMBER: _ClassVar[int] + ROLE_FIELD_NUMBER: _ClassVar[int] + UUID_FIELD_NUMBER: _ClassVar[int] + omitDescriptor: bool + omitVersion: bool + role: Role + uuid: bytes + def __init__(self, uuid: _Optional[bytes] = ..., omitVersion: bool = ..., omitDescriptor: bool = ..., role: _Optional[_Union[Role, _Mapping]] = ...) -> None: ... + +class StreamInfoResponse(_message.Message): + __slots__ = ["descriptor", "stat", "versionMajor", "versionMinor"] + DESCRIPTOR_FIELD_NUMBER: _ClassVar[int] + STAT_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + descriptor: StreamDescriptor + stat: Status + versionMajor: int + versionMinor: int + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., descriptor: _Optional[_Union[StreamDescriptor, _Mapping]] = ...) -> None: ... + +class WindowsParams(_message.Message): + __slots__ = ["depth", "end", "start", "uuid", "versionMajor", "width"] + DEPTH_FIELD_NUMBER: _ClassVar[int] + END_FIELD_NUMBER: _ClassVar[int] + START_FIELD_NUMBER: _ClassVar[int] + UUID_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + WIDTH_FIELD_NUMBER: _ClassVar[int] + depth: int + end: int + start: int + uuid: bytes + versionMajor: int + width: int + def __init__(self, uuid: _Optional[bytes] = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., versionMajor: _Optional[int] = ..., width: _Optional[int] = ..., depth: _Optional[int] = ...) -> None: ... + +class WindowsResponse(_message.Message): + __slots__ = ["stat", "values", "versionMajor", "versionMinor"] + STAT_FIELD_NUMBER: _ClassVar[int] + VALUES_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + stat: Status + values: _containers.RepeatedCompositeFieldContainer[StatPoint] + versionMajor: int + versionMinor: int + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., values: _Optional[_Iterable[_Union[StatPoint, _Mapping]]] = ...) -> None: ... + +class MergePolicy(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = [] diff --git a/btrdb/grpcinterface/btrdb_pb2_grpc.py b/btrdb/grpcinterface/btrdb_pb2_grpc.py index 89b9ce0..0bac853 100644 --- a/btrdb/grpcinterface/btrdb_pb2_grpc.py +++ b/btrdb/grpcinterface/btrdb_pb2_grpc.py @@ -1,370 +1,695 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" import grpc import btrdb.grpcinterface.btrdb_pb2 as btrdb__pb2 class BTrDBStub(object): - # missing associated documentation comment in .proto file - pass - - def __init__(self, channel): - """Constructor. - - Args: - channel: A grpc.Channel. - """ - self.RawValues = channel.unary_stream( - '/v5api.BTrDB/RawValues', - request_serializer=btrdb__pb2.RawValuesParams.SerializeToString, - response_deserializer=btrdb__pb2.RawValuesResponse.FromString, - ) - self.AlignedWindows = channel.unary_stream( - '/v5api.BTrDB/AlignedWindows', - request_serializer=btrdb__pb2.AlignedWindowsParams.SerializeToString, - response_deserializer=btrdb__pb2.AlignedWindowsResponse.FromString, - ) - self.Windows = channel.unary_stream( - '/v5api.BTrDB/Windows', - request_serializer=btrdb__pb2.WindowsParams.SerializeToString, - response_deserializer=btrdb__pb2.WindowsResponse.FromString, - ) - self.StreamInfo = channel.unary_unary( - '/v5api.BTrDB/StreamInfo', - request_serializer=btrdb__pb2.StreamInfoParams.SerializeToString, - response_deserializer=btrdb__pb2.StreamInfoResponse.FromString, - ) - self.SetStreamAnnotations = channel.unary_unary( - '/v5api.BTrDB/SetStreamAnnotations', - request_serializer=btrdb__pb2.SetStreamAnnotationsParams.SerializeToString, - response_deserializer=btrdb__pb2.SetStreamAnnotationsResponse.FromString, - ) - self.SetStreamTags = channel.unary_unary( - '/v5api.BTrDB/SetStreamTags', - request_serializer=btrdb__pb2.SetStreamTagsParams.SerializeToString, - response_deserializer=btrdb__pb2.SetStreamTagsResponse.FromString, - ) - self.Create = channel.unary_unary( - '/v5api.BTrDB/Create', - request_serializer=btrdb__pb2.CreateParams.SerializeToString, - response_deserializer=btrdb__pb2.CreateResponse.FromString, - ) - self.ListCollections = channel.unary_stream( - '/v5api.BTrDB/ListCollections', - request_serializer=btrdb__pb2.ListCollectionsParams.SerializeToString, - response_deserializer=btrdb__pb2.ListCollectionsResponse.FromString, - ) - self.LookupStreams = channel.unary_stream( - '/v5api.BTrDB/LookupStreams', - request_serializer=btrdb__pb2.LookupStreamsParams.SerializeToString, - response_deserializer=btrdb__pb2.LookupStreamsResponse.FromString, - ) - self.Nearest = channel.unary_unary( - '/v5api.BTrDB/Nearest', - request_serializer=btrdb__pb2.NearestParams.SerializeToString, - response_deserializer=btrdb__pb2.NearestResponse.FromString, - ) - self.Changes = channel.unary_stream( - '/v5api.BTrDB/Changes', - request_serializer=btrdb__pb2.ChangesParams.SerializeToString, - response_deserializer=btrdb__pb2.ChangesResponse.FromString, - ) - self.Insert = channel.unary_unary( - '/v5api.BTrDB/Insert', - request_serializer=btrdb__pb2.InsertParams.SerializeToString, - response_deserializer=btrdb__pb2.InsertResponse.FromString, - ) - self.Delete = channel.unary_unary( - '/v5api.BTrDB/Delete', - request_serializer=btrdb__pb2.DeleteParams.SerializeToString, - response_deserializer=btrdb__pb2.DeleteResponse.FromString, - ) - self.Info = channel.unary_unary( - '/v5api.BTrDB/Info', - request_serializer=btrdb__pb2.InfoParams.SerializeToString, - response_deserializer=btrdb__pb2.InfoResponse.FromString, - ) - self.FaultInject = channel.unary_unary( - '/v5api.BTrDB/FaultInject', - request_serializer=btrdb__pb2.FaultInjectParams.SerializeToString, - response_deserializer=btrdb__pb2.FaultInjectResponse.FromString, - ) - self.Flush = channel.unary_unary( - '/v5api.BTrDB/Flush', - request_serializer=btrdb__pb2.FlushParams.SerializeToString, - response_deserializer=btrdb__pb2.FlushResponse.FromString, - ) - self.Obliterate = channel.unary_unary( - '/v5api.BTrDB/Obliterate', - request_serializer=btrdb__pb2.ObliterateParams.SerializeToString, - response_deserializer=btrdb__pb2.ObliterateResponse.FromString, - ) - self.GetMetadataUsage = channel.unary_unary( - '/v5api.BTrDB/GetMetadataUsage', - request_serializer=btrdb__pb2.MetadataUsageParams.SerializeToString, - response_deserializer=btrdb__pb2.MetadataUsageResponse.FromString, - ) - self.GenerateCSV = channel.unary_stream( - '/v5api.BTrDB/GenerateCSV', - request_serializer=btrdb__pb2.GenerateCSVParams.SerializeToString, - response_deserializer=btrdb__pb2.GenerateCSVResponse.FromString, - ) - self.SQLQuery = channel.unary_stream( - '/v5api.BTrDB/SQLQuery', - request_serializer=btrdb__pb2.SQLQueryParams.SerializeToString, - response_deserializer=btrdb__pb2.SQLQueryResponse.FromString, - ) + """Missing associated documentation comment in .proto file.""" + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.RawValues = channel.unary_stream( + '/v5api.BTrDB/RawValues', + request_serializer=btrdb__pb2.RawValuesParams.SerializeToString, + response_deserializer=btrdb__pb2.RawValuesResponse.FromString, + ) + self.AlignedWindows = channel.unary_stream( + '/v5api.BTrDB/AlignedWindows', + request_serializer=btrdb__pb2.AlignedWindowsParams.SerializeToString, + response_deserializer=btrdb__pb2.AlignedWindowsResponse.FromString, + ) + self.Windows = channel.unary_stream( + '/v5api.BTrDB/Windows', + request_serializer=btrdb__pb2.WindowsParams.SerializeToString, + response_deserializer=btrdb__pb2.WindowsResponse.FromString, + ) + self.StreamInfo = channel.unary_unary( + '/v5api.BTrDB/StreamInfo', + request_serializer=btrdb__pb2.StreamInfoParams.SerializeToString, + response_deserializer=btrdb__pb2.StreamInfoResponse.FromString, + ) + self.SetStreamAnnotations = channel.unary_unary( + '/v5api.BTrDB/SetStreamAnnotations', + request_serializer=btrdb__pb2.SetStreamAnnotationsParams.SerializeToString, + response_deserializer=btrdb__pb2.SetStreamAnnotationsResponse.FromString, + ) + self.SetStreamTags = channel.unary_unary( + '/v5api.BTrDB/SetStreamTags', + request_serializer=btrdb__pb2.SetStreamTagsParams.SerializeToString, + response_deserializer=btrdb__pb2.SetStreamTagsResponse.FromString, + ) + self.Create = channel.unary_unary( + '/v5api.BTrDB/Create', + request_serializer=btrdb__pb2.CreateParams.SerializeToString, + response_deserializer=btrdb__pb2.CreateResponse.FromString, + ) + self.ListCollections = channel.unary_stream( + '/v5api.BTrDB/ListCollections', + request_serializer=btrdb__pb2.ListCollectionsParams.SerializeToString, + response_deserializer=btrdb__pb2.ListCollectionsResponse.FromString, + ) + self.LookupStreams = channel.unary_stream( + '/v5api.BTrDB/LookupStreams', + request_serializer=btrdb__pb2.LookupStreamsParams.SerializeToString, + response_deserializer=btrdb__pb2.LookupStreamsResponse.FromString, + ) + self.Nearest = channel.unary_unary( + '/v5api.BTrDB/Nearest', + request_serializer=btrdb__pb2.NearestParams.SerializeToString, + response_deserializer=btrdb__pb2.NearestResponse.FromString, + ) + self.Changes = channel.unary_stream( + '/v5api.BTrDB/Changes', + request_serializer=btrdb__pb2.ChangesParams.SerializeToString, + response_deserializer=btrdb__pb2.ChangesResponse.FromString, + ) + self.Insert = channel.unary_unary( + '/v5api.BTrDB/Insert', + request_serializer=btrdb__pb2.InsertParams.SerializeToString, + response_deserializer=btrdb__pb2.InsertResponse.FromString, + ) + self.Delete = channel.unary_unary( + '/v5api.BTrDB/Delete', + request_serializer=btrdb__pb2.DeleteParams.SerializeToString, + response_deserializer=btrdb__pb2.DeleteResponse.FromString, + ) + self.Info = channel.unary_unary( + '/v5api.BTrDB/Info', + request_serializer=btrdb__pb2.InfoParams.SerializeToString, + response_deserializer=btrdb__pb2.InfoResponse.FromString, + ) + self.FaultInject = channel.unary_unary( + '/v5api.BTrDB/FaultInject', + request_serializer=btrdb__pb2.FaultInjectParams.SerializeToString, + response_deserializer=btrdb__pb2.FaultInjectResponse.FromString, + ) + self.Flush = channel.unary_unary( + '/v5api.BTrDB/Flush', + request_serializer=btrdb__pb2.FlushParams.SerializeToString, + response_deserializer=btrdb__pb2.FlushResponse.FromString, + ) + self.Obliterate = channel.unary_unary( + '/v5api.BTrDB/Obliterate', + request_serializer=btrdb__pb2.ObliterateParams.SerializeToString, + response_deserializer=btrdb__pb2.ObliterateResponse.FromString, + ) + self.GetMetadataUsage = channel.unary_unary( + '/v5api.BTrDB/GetMetadataUsage', + request_serializer=btrdb__pb2.MetadataUsageParams.SerializeToString, + response_deserializer=btrdb__pb2.MetadataUsageResponse.FromString, + ) + self.GenerateCSV = channel.unary_stream( + '/v5api.BTrDB/GenerateCSV', + request_serializer=btrdb__pb2.GenerateCSVParams.SerializeToString, + response_deserializer=btrdb__pb2.GenerateCSVResponse.FromString, + ) + self.SQLQuery = channel.unary_stream( + '/v5api.BTrDB/SQLQuery', + request_serializer=btrdb__pb2.SQLQueryParams.SerializeToString, + response_deserializer=btrdb__pb2.SQLQueryResponse.FromString, + ) class BTrDBServicer(object): - # missing associated documentation comment in .proto file - pass - - def RawValues(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def AlignedWindows(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Windows(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def StreamInfo(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def SetStreamAnnotations(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def SetStreamTags(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Create(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def ListCollections(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def LookupStreams(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Nearest(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Changes(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Insert(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Delete(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Info(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def FaultInject(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Flush(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Obliterate(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def GetMetadataUsage(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def GenerateCSV(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def SQLQuery(self, request, context): - """rpc SetCompactionConfig(SetCompactionConfigParams) returns (SetCompactionConfigResponse); - rpc GetCompactionConfig(GetCompactionConfigParams) returns (GetCompactionConfigResponse); - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') + """Missing associated documentation comment in .proto file.""" + + def RawValues(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def AlignedWindows(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Windows(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def StreamInfo(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def SetStreamAnnotations(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def SetStreamTags(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Create(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def ListCollections(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def LookupStreams(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Nearest(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Changes(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Insert(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Delete(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Info(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def FaultInject(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Flush(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Obliterate(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetMetadataUsage(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GenerateCSV(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def SQLQuery(self, request, context): + """rpc SetCompactionConfig(SetCompactionConfigParams) returns (SetCompactionConfigResponse); + rpc GetCompactionConfig(GetCompactionConfigParams) returns (GetCompactionConfigResponse); + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') def add_BTrDBServicer_to_server(servicer, server): - rpc_method_handlers = { - 'RawValues': grpc.unary_stream_rpc_method_handler( - servicer.RawValues, - request_deserializer=btrdb__pb2.RawValuesParams.FromString, - response_serializer=btrdb__pb2.RawValuesResponse.SerializeToString, - ), - 'AlignedWindows': grpc.unary_stream_rpc_method_handler( - servicer.AlignedWindows, - request_deserializer=btrdb__pb2.AlignedWindowsParams.FromString, - response_serializer=btrdb__pb2.AlignedWindowsResponse.SerializeToString, - ), - 'Windows': grpc.unary_stream_rpc_method_handler( - servicer.Windows, - request_deserializer=btrdb__pb2.WindowsParams.FromString, - response_serializer=btrdb__pb2.WindowsResponse.SerializeToString, - ), - 'StreamInfo': grpc.unary_unary_rpc_method_handler( - servicer.StreamInfo, - request_deserializer=btrdb__pb2.StreamInfoParams.FromString, - response_serializer=btrdb__pb2.StreamInfoResponse.SerializeToString, - ), - 'SetStreamAnnotations': grpc.unary_unary_rpc_method_handler( - servicer.SetStreamAnnotations, - request_deserializer=btrdb__pb2.SetStreamAnnotationsParams.FromString, - response_serializer=btrdb__pb2.SetStreamAnnotationsResponse.SerializeToString, - ), - 'SetStreamTags': grpc.unary_unary_rpc_method_handler( - servicer.SetStreamTags, - request_deserializer=btrdb__pb2.SetStreamTagsParams.FromString, - response_serializer=btrdb__pb2.SetStreamTagsResponse.SerializeToString, - ), - 'Create': grpc.unary_unary_rpc_method_handler( - servicer.Create, - request_deserializer=btrdb__pb2.CreateParams.FromString, - response_serializer=btrdb__pb2.CreateResponse.SerializeToString, - ), - 'ListCollections': grpc.unary_stream_rpc_method_handler( - servicer.ListCollections, - request_deserializer=btrdb__pb2.ListCollectionsParams.FromString, - response_serializer=btrdb__pb2.ListCollectionsResponse.SerializeToString, - ), - 'LookupStreams': grpc.unary_stream_rpc_method_handler( - servicer.LookupStreams, - request_deserializer=btrdb__pb2.LookupStreamsParams.FromString, - response_serializer=btrdb__pb2.LookupStreamsResponse.SerializeToString, - ), - 'Nearest': grpc.unary_unary_rpc_method_handler( - servicer.Nearest, - request_deserializer=btrdb__pb2.NearestParams.FromString, - response_serializer=btrdb__pb2.NearestResponse.SerializeToString, - ), - 'Changes': grpc.unary_stream_rpc_method_handler( - servicer.Changes, - request_deserializer=btrdb__pb2.ChangesParams.FromString, - response_serializer=btrdb__pb2.ChangesResponse.SerializeToString, - ), - 'Insert': grpc.unary_unary_rpc_method_handler( - servicer.Insert, - request_deserializer=btrdb__pb2.InsertParams.FromString, - response_serializer=btrdb__pb2.InsertResponse.SerializeToString, - ), - 'Delete': grpc.unary_unary_rpc_method_handler( - servicer.Delete, - request_deserializer=btrdb__pb2.DeleteParams.FromString, - response_serializer=btrdb__pb2.DeleteResponse.SerializeToString, - ), - 'Info': grpc.unary_unary_rpc_method_handler( - servicer.Info, - request_deserializer=btrdb__pb2.InfoParams.FromString, - response_serializer=btrdb__pb2.InfoResponse.SerializeToString, - ), - 'FaultInject': grpc.unary_unary_rpc_method_handler( - servicer.FaultInject, - request_deserializer=btrdb__pb2.FaultInjectParams.FromString, - response_serializer=btrdb__pb2.FaultInjectResponse.SerializeToString, - ), - 'Flush': grpc.unary_unary_rpc_method_handler( - servicer.Flush, - request_deserializer=btrdb__pb2.FlushParams.FromString, - response_serializer=btrdb__pb2.FlushResponse.SerializeToString, - ), - 'Obliterate': grpc.unary_unary_rpc_method_handler( - servicer.Obliterate, - request_deserializer=btrdb__pb2.ObliterateParams.FromString, - response_serializer=btrdb__pb2.ObliterateResponse.SerializeToString, - ), - 'GetMetadataUsage': grpc.unary_unary_rpc_method_handler( - servicer.GetMetadataUsage, - request_deserializer=btrdb__pb2.MetadataUsageParams.FromString, - response_serializer=btrdb__pb2.MetadataUsageResponse.SerializeToString, - ), - 'GenerateCSV': grpc.unary_stream_rpc_method_handler( - servicer.GenerateCSV, - request_deserializer=btrdb__pb2.GenerateCSVParams.FromString, - response_serializer=btrdb__pb2.GenerateCSVResponse.SerializeToString, - ), - 'SQLQuery': grpc.unary_stream_rpc_method_handler( - servicer.SQLQuery, - request_deserializer=btrdb__pb2.SQLQueryParams.FromString, - response_serializer=btrdb__pb2.SQLQueryResponse.SerializeToString, - ), - } - generic_handler = grpc.method_handlers_generic_handler( - 'v5api.BTrDB', rpc_method_handlers) - server.add_generic_rpc_handlers((generic_handler,)) + rpc_method_handlers = { + 'RawValues': grpc.unary_stream_rpc_method_handler( + servicer.RawValues, + request_deserializer=btrdb__pb2.RawValuesParams.FromString, + response_serializer=btrdb__pb2.RawValuesResponse.SerializeToString, + ), + 'AlignedWindows': grpc.unary_stream_rpc_method_handler( + servicer.AlignedWindows, + request_deserializer=btrdb__pb2.AlignedWindowsParams.FromString, + response_serializer=btrdb__pb2.AlignedWindowsResponse.SerializeToString, + ), + 'Windows': grpc.unary_stream_rpc_method_handler( + servicer.Windows, + request_deserializer=btrdb__pb2.WindowsParams.FromString, + response_serializer=btrdb__pb2.WindowsResponse.SerializeToString, + ), + 'StreamInfo': grpc.unary_unary_rpc_method_handler( + servicer.StreamInfo, + request_deserializer=btrdb__pb2.StreamInfoParams.FromString, + response_serializer=btrdb__pb2.StreamInfoResponse.SerializeToString, + ), + 'SetStreamAnnotations': grpc.unary_unary_rpc_method_handler( + servicer.SetStreamAnnotations, + request_deserializer=btrdb__pb2.SetStreamAnnotationsParams.FromString, + response_serializer=btrdb__pb2.SetStreamAnnotationsResponse.SerializeToString, + ), + 'SetStreamTags': grpc.unary_unary_rpc_method_handler( + servicer.SetStreamTags, + request_deserializer=btrdb__pb2.SetStreamTagsParams.FromString, + response_serializer=btrdb__pb2.SetStreamTagsResponse.SerializeToString, + ), + 'Create': grpc.unary_unary_rpc_method_handler( + servicer.Create, + request_deserializer=btrdb__pb2.CreateParams.FromString, + response_serializer=btrdb__pb2.CreateResponse.SerializeToString, + ), + 'ListCollections': grpc.unary_stream_rpc_method_handler( + servicer.ListCollections, + request_deserializer=btrdb__pb2.ListCollectionsParams.FromString, + response_serializer=btrdb__pb2.ListCollectionsResponse.SerializeToString, + ), + 'LookupStreams': grpc.unary_stream_rpc_method_handler( + servicer.LookupStreams, + request_deserializer=btrdb__pb2.LookupStreamsParams.FromString, + response_serializer=btrdb__pb2.LookupStreamsResponse.SerializeToString, + ), + 'Nearest': grpc.unary_unary_rpc_method_handler( + servicer.Nearest, + request_deserializer=btrdb__pb2.NearestParams.FromString, + response_serializer=btrdb__pb2.NearestResponse.SerializeToString, + ), + 'Changes': grpc.unary_stream_rpc_method_handler( + servicer.Changes, + request_deserializer=btrdb__pb2.ChangesParams.FromString, + response_serializer=btrdb__pb2.ChangesResponse.SerializeToString, + ), + 'Insert': grpc.unary_unary_rpc_method_handler( + servicer.Insert, + request_deserializer=btrdb__pb2.InsertParams.FromString, + response_serializer=btrdb__pb2.InsertResponse.SerializeToString, + ), + 'Delete': grpc.unary_unary_rpc_method_handler( + servicer.Delete, + request_deserializer=btrdb__pb2.DeleteParams.FromString, + response_serializer=btrdb__pb2.DeleteResponse.SerializeToString, + ), + 'Info': grpc.unary_unary_rpc_method_handler( + servicer.Info, + request_deserializer=btrdb__pb2.InfoParams.FromString, + response_serializer=btrdb__pb2.InfoResponse.SerializeToString, + ), + 'FaultInject': grpc.unary_unary_rpc_method_handler( + servicer.FaultInject, + request_deserializer=btrdb__pb2.FaultInjectParams.FromString, + response_serializer=btrdb__pb2.FaultInjectResponse.SerializeToString, + ), + 'Flush': grpc.unary_unary_rpc_method_handler( + servicer.Flush, + request_deserializer=btrdb__pb2.FlushParams.FromString, + response_serializer=btrdb__pb2.FlushResponse.SerializeToString, + ), + 'Obliterate': grpc.unary_unary_rpc_method_handler( + servicer.Obliterate, + request_deserializer=btrdb__pb2.ObliterateParams.FromString, + response_serializer=btrdb__pb2.ObliterateResponse.SerializeToString, + ), + 'GetMetadataUsage': grpc.unary_unary_rpc_method_handler( + servicer.GetMetadataUsage, + request_deserializer=btrdb__pb2.MetadataUsageParams.FromString, + response_serializer=btrdb__pb2.MetadataUsageResponse.SerializeToString, + ), + 'GenerateCSV': grpc.unary_stream_rpc_method_handler( + servicer.GenerateCSV, + request_deserializer=btrdb__pb2.GenerateCSVParams.FromString, + response_serializer=btrdb__pb2.GenerateCSVResponse.SerializeToString, + ), + 'SQLQuery': grpc.unary_stream_rpc_method_handler( + servicer.SQLQuery, + request_deserializer=btrdb__pb2.SQLQueryParams.FromString, + response_serializer=btrdb__pb2.SQLQueryResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'v5api.BTrDB', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + + + # This class is part of an EXPERIMENTAL API. +class BTrDB(object): + """Missing associated documentation comment in .proto file.""" + + @staticmethod + def RawValues(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream(request, target, '/v5api.BTrDB/RawValues', + btrdb__pb2.RawValuesParams.SerializeToString, + btrdb__pb2.RawValuesResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def AlignedWindows(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream(request, target, '/v5api.BTrDB/AlignedWindows', + btrdb__pb2.AlignedWindowsParams.SerializeToString, + btrdb__pb2.AlignedWindowsResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Windows(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream(request, target, '/v5api.BTrDB/Windows', + btrdb__pb2.WindowsParams.SerializeToString, + btrdb__pb2.WindowsResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def StreamInfo(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/v5api.BTrDB/StreamInfo', + btrdb__pb2.StreamInfoParams.SerializeToString, + btrdb__pb2.StreamInfoResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def SetStreamAnnotations(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/v5api.BTrDB/SetStreamAnnotations', + btrdb__pb2.SetStreamAnnotationsParams.SerializeToString, + btrdb__pb2.SetStreamAnnotationsResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def SetStreamTags(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/v5api.BTrDB/SetStreamTags', + btrdb__pb2.SetStreamTagsParams.SerializeToString, + btrdb__pb2.SetStreamTagsResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Create(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/v5api.BTrDB/Create', + btrdb__pb2.CreateParams.SerializeToString, + btrdb__pb2.CreateResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def ListCollections(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream(request, target, '/v5api.BTrDB/ListCollections', + btrdb__pb2.ListCollectionsParams.SerializeToString, + btrdb__pb2.ListCollectionsResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def LookupStreams(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream(request, target, '/v5api.BTrDB/LookupStreams', + btrdb__pb2.LookupStreamsParams.SerializeToString, + btrdb__pb2.LookupStreamsResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Nearest(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/v5api.BTrDB/Nearest', + btrdb__pb2.NearestParams.SerializeToString, + btrdb__pb2.NearestResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Changes(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream(request, target, '/v5api.BTrDB/Changes', + btrdb__pb2.ChangesParams.SerializeToString, + btrdb__pb2.ChangesResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Insert(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/v5api.BTrDB/Insert', + btrdb__pb2.InsertParams.SerializeToString, + btrdb__pb2.InsertResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Delete(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/v5api.BTrDB/Delete', + btrdb__pb2.DeleteParams.SerializeToString, + btrdb__pb2.DeleteResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Info(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/v5api.BTrDB/Info', + btrdb__pb2.InfoParams.SerializeToString, + btrdb__pb2.InfoResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def FaultInject(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/v5api.BTrDB/FaultInject', + btrdb__pb2.FaultInjectParams.SerializeToString, + btrdb__pb2.FaultInjectResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Flush(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/v5api.BTrDB/Flush', + btrdb__pb2.FlushParams.SerializeToString, + btrdb__pb2.FlushResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Obliterate(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/v5api.BTrDB/Obliterate', + btrdb__pb2.ObliterateParams.SerializeToString, + btrdb__pb2.ObliterateResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def GetMetadataUsage(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/v5api.BTrDB/GetMetadataUsage', + btrdb__pb2.MetadataUsageParams.SerializeToString, + btrdb__pb2.MetadataUsageResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def GenerateCSV(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream(request, target, '/v5api.BTrDB/GenerateCSV', + btrdb__pb2.GenerateCSVParams.SerializeToString, + btrdb__pb2.GenerateCSVResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def SQLQuery(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream(request, target, '/v5api.BTrDB/SQLQuery', + btrdb__pb2.SQLQueryParams.SerializeToString, + btrdb__pb2.SQLQueryResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/btrdb/stream.py b/btrdb/stream.py index 39d86fe..16db559 100644 --- a/btrdb/stream.py +++ b/btrdb/stream.py @@ -18,6 +18,7 @@ import re import json import uuid as uuidlib +import warnings from copy import deepcopy from collections.abc import Sequence @@ -34,7 +35,7 @@ InvalidOperation, InvalidCollection, StreamNotFoundError, - NoSuchPoint + NoSuchPoint, ) @@ -56,6 +57,7 @@ ## Stream Classes ########################################################################## + class Stream(object): """ An object that represents a specific time series stream in the BTrDB database. @@ -72,13 +74,21 @@ class Stream(object): """ def __init__(self, btrdb, uuid, **db_values): - db_args = ('known_to_exist', 'collection', 'tags', 'annotations', 'property_version') + db_args = ( + "known_to_exist", + "collection", + "tags", + "annotations", + "property_version", + ) for key in db_args: value = db_values.pop(key, None) setattr(self, "_{}".format(key), value) if db_values: bad_keys = ", ".join(db_values.keys()) - raise BTRDBTypeError("got unexpected db_values argument(s) '{}'".format(bad_keys)) + raise BTRDBTypeError( + "got unexpected db_values argument(s) '{}'".format(bad_keys) + ) self._btrdb = btrdb self._uuid = uuid @@ -92,7 +102,13 @@ def refresh_metadata(self): """ ep = self._btrdb.ep - self._collection, self._property_version, self._tags, self._annotations, _ = ep.streamInfo(self._uuid, False, True) + ( + self._collection, + self._property_version, + self._tags, + self._annotations, + _, + ) = ep.streamInfo(self._uuid, False, True) self._known_to_exist = True # deserialize annoation values @@ -131,7 +147,14 @@ def exists(self): return False raise bte - def count(self, start=MINIMUM_TIME, end=MAXIMUM_TIME, pointwidth=62, precise=False, version=0): + def count( + self, + start=MINIMUM_TIME, + end=MAXIMUM_TIME, + pointwidth=62, + precise=False, + version=0, + ): """ Compute the total number of points in the stream @@ -176,15 +199,17 @@ def count(self, start=MINIMUM_TIME, end=MAXIMUM_TIME, pointwidth=62, precise=Fal """ if not precise: - pointwidth = min(pointwidth, pw.from_nanoseconds(to_nanoseconds(end) - to_nanoseconds(start))-1) + pointwidth = min( + pointwidth, + pw.from_nanoseconds(to_nanoseconds(end) - to_nanoseconds(start)) - 1, + ) points = self.aligned_windows(start, end, pointwidth, version) - return sum([point.count for point, _ in points]) + return sum([point.count for point, _ in points]) depth = 0 width = to_nanoseconds(end) - to_nanoseconds(start) points = self.windows(start, end, width, depth, version) - return sum([point.count for point, _ in points]) - + return sum([point.count for point, _ in points]) @property def btrdb(self): @@ -206,7 +231,7 @@ def btrdb(self): @property def uuid(self): """ - Returns the stream's UUID. The stream may nor may not exist + Returns the stream's UUID. The stream may or may not exist yet, depending on how the stream object was obtained. Returns @@ -410,7 +435,7 @@ def version(self): """ return self._btrdb.ep.streamInfo(self._uuid, True, False)[4] - def insert(self, data, merge='never'): + def insert(self, data, merge="never"): """ Insert new data in the form (time, value) into the series. @@ -441,7 +466,7 @@ def insert(self, data, merge='never'): i = 0 version = 0 while i < len(data): - thisBatch = data[i:i + INSERT_BATCH_SIZE] + thisBatch = data[i : i + INSERT_BATCH_SIZE] version = self._btrdb.ep.insert(self._uuid, thisBatch, merge) i += INSERT_BATCH_SIZE return version @@ -450,13 +475,15 @@ def _update_tags_collection(self, tags, collection): tags = self.tags() if tags is None else tags collection = self.collection if collection is None else collection if collection is None: - raise BTRDBValueError("collection must be provided to update tags or collection") + raise BTRDBValueError( + "collection must be provided to update tags or collection" + ) self._btrdb.ep.setStreamTags( uu=self.uuid, expected=self._property_version, tags=tags, - collection=collection + collection=collection, ) def _update_annotations(self, annotations, encoder, replace): @@ -470,16 +497,25 @@ def _update_annotations(self, annotations, encoder, replace): removals = [] if replace: - removals = [i for i in self._annotations.keys() if i not in annotations.keys()] + removals = [ + i for i in self._annotations.keys() if i not in annotations.keys() + ] self._btrdb.ep.setStreamAnnotations( uu=self.uuid, expected=self._property_version, changes=serialized, - removals=removals + removals=removals, ) - def update(self, tags=None, annotations=None, collection=None, encoder=AnnotationEncoder, replace=False): + def update( + self, + tags=None, + annotations=None, + collection=None, + encoder=AnnotationEncoder, + replace=False, + ): """ Updates metadata including tags, annotations, and collection as an UPSERT operation. @@ -529,7 +565,9 @@ def update(self, tags=None, annotations=None, collection=None, encoder=Annotatio """ if tags is None and annotations is None and collection is None: - raise BTRDBValueError("you must supply a tags, annotations, or collection argument") + raise BTRDBValueError( + "you must supply a tags, annotations, or collection argument" + ) if tags is not None and isinstance(tags, dict) is False: raise BTRDBTypeError("tags must be of type dict") @@ -574,8 +612,9 @@ def delete(self, start, end): The version of the new stream created """ - return self._btrdb.ep.deleteRange(self._uuid, to_nanoseconds(start), - to_nanoseconds(end)) + return self._btrdb.ep.deleteRange( + self._uuid, to_nanoseconds(start), to_nanoseconds(end) + ) def values(self, start, end, version=0): """ @@ -669,7 +708,9 @@ def aligned_windows(self, start, end, pointwidth, version=0): start = to_nanoseconds(start) end = to_nanoseconds(end) - windows = self._btrdb.ep.alignedWindows(self._uuid, start, end, pointwidth, version) + windows = self._btrdb.ep.alignedWindows( + self._uuid, start, end, pointwidth, version + ) for stat_points, version in windows: for point in stat_points: materialized.append((StatPoint.from_proto(point), version)) @@ -755,8 +796,9 @@ def nearest(self, time, version, backward=False): """ try: - rp, version = self._btrdb.ep.nearest(self._uuid, - to_nanoseconds(time), version, backward) + rp, version = self._btrdb.ep.nearest( + self._uuid, to_nanoseconds(time), version, backward + ) except BTrDBError as exc: if not isinstance(exc, NoSuchPoint): raise @@ -764,7 +806,6 @@ def nearest(self, time, version, backward=False): return RawPoint.from_proto(rp), version - def obliterate(self): """ Obliterate deletes a stream from the BTrDB server. An exception will be @@ -786,13 +827,14 @@ def flush(self): self._btrdb.ep.flush(self._uuid) def __repr__(self): - return "".format(self.collection, - self.name) + return "".format(self.collection, self.name) + ########################################################################## ## StreamSet Classes ########################################################################## + class StreamSetBase(Sequence): """ A lighweight wrapper around a list of stream objects @@ -800,6 +842,10 @@ class StreamSetBase(Sequence): def __init__(self, streams): self._streams = streams + try: + self._btrdb = self._streams[0]._btrdb + except Exception as e: + warnings.warn(f"Issue setting btrdb object from stream: {e}") self._pinned_versions = None self.filters = [] @@ -812,8 +858,10 @@ def allow_window(self): return not bool(self.pointwidth or (self.width and self.depth)) def _latest_versions(self): - return {s.uuid: s.version() for s in self._streams} - + uuid_ver_tups = self._btrdb._executor.map( + lambda s: (s.uuid, s.version()), self._streams + ) + return {uu: v for uu, v in uuid_ver_tups} def pin_versions(self, versions=None): """ @@ -841,7 +889,6 @@ def pin_versions(self, versions=None): if not isinstance(key, uuidlib.UUID): raise BTRDBTypeError("version keys must be type UUID") - self._pinned_versions = self._latest_versions() if not versions else versions return self @@ -862,7 +909,9 @@ def versions(self): A dict containing the stream UUID and version ints as key/values """ - return self._pinned_versions if self._pinned_versions else self._latest_versions() + return ( + self._pinned_versions if self._pinned_versions else self._latest_versions() + ) def count(self): """ @@ -892,13 +941,13 @@ def count(self): start = params.get("start", MINIMUM_TIME) end = params.get("end", MAXIMUM_TIME) versions = self._pinned_versions if self._pinned_versions else {} - count = 0 - for s in self._streams: - version = versions.get(s.uuid, 0) - count += s.count(start, end, version=version) + my_counts_gen = self._btrdb._executor.map( + lambda s: s.count(start, end, version=versions.get(s.uuid, 0)), + self._streams, + ) - return count + return sum(my_counts_gen) def earliest(self): """ @@ -918,10 +967,11 @@ def earliest(self): params = self._params_from_filters() start = params.get("start", MINIMUM_TIME) versions = self.versions() - - for s in self._streams: - version = versions.get(s.uuid, 0) - point, _ = s.nearest(start, version=version, backward=False) + earliest_points_gen = self._btrdb._executor.map( + lambda s: s.nearest(start, version=versions.get(s.uuid, 0), backward=False), + self._streams, + ) + for point, _ in earliest_points_gen: earliest.append(point) return tuple(earliest) @@ -944,10 +994,11 @@ def latest(self): params = self._params_from_filters() start = params.get("end", MAXIMUM_TIME) versions = self.versions() - - for s in self._streams: - version = versions.get(s.uuid, 0) - point, _ = s.nearest(start, version=version, backward=True) + latest_points_gen = self._btrdb._executor.map( + lambda s: s.nearest(start, version=versions.get(s.uuid, 0), backward=True), + self._streams, + ) + for point, _ in latest_points_gen: latest.append(point) return tuple(latest) @@ -974,17 +1025,30 @@ def current(self): start = params.get("start", None) if (end is not None and end <= now) or (start is not None and start > now): - raise BTRDBValueError("current time is not included in filtered stream range") + raise BTRDBValueError( + "current time is not included in filtered stream range" + ) - for s in self._streams: - version = self.versions()[s.uuid] - point, _ = s.nearest(now, version=version, backward=True) + versions = self.versions() + latest_points_gen = self._btrdb._executor.map( + lambda s: (s.nearest(now, version=versions.get(s.uuid, 0), backward=True)), + self._streams, + ) + for point in latest_points_gen: latest.append(point) return tuple(latest) - def filter(self, start=None, end=None, collection=None, name=None, unit=None, - tags=None, annotations=None): + def filter( + self, + start=None, + end=None, + collection=None, + name=None, + unit=None, + tags=None, + annotations=None, + ): """ Provides a new StreamSet instance containing stored query parameters and stream objects that match filtering criteria. @@ -1034,43 +1098,63 @@ def filter(self, start=None, end=None, collection=None, name=None, unit=None, # filter by collection if collection is not None: if isinstance(collection, RE_PATTERN): - obj._streams = [s for s in obj._streams for m in [collection.search(s.collection)] if m] + obj._streams = [ + s + for s in obj._streams + for m in [collection.search(s.collection)] + if m + ] elif isinstance(collection, str): - obj._streams = [s for s in obj._streams if s.collection.lower() == collection.lower()] + obj._streams = [ + s + for s in obj._streams + if s.collection.lower() == collection.lower() + ] else: raise BTRDBTypeError("collection must be string or compiled regex") # filter by name if name is not None: if isinstance(name, RE_PATTERN): - obj._streams = [s for s in obj._streams for m in [name.search(s.name)] if m] + obj._streams = [ + s for s in obj._streams for m in [name.search(s.name)] if m + ] elif isinstance(name, str): - obj._streams = [s for s in obj._streams if s.name.lower() == name.lower()] + obj._streams = [ + s for s in obj._streams if s.name.lower() == name.lower() + ] else: raise BTRDBTypeError("name must be string or compiled regex") # filter by unit if unit is not None: if isinstance(unit, RE_PATTERN): - obj._streams = [s for s in obj._streams for m in [unit.search(s.tags()["unit"])] if m] + obj._streams = [ + s + for s in obj._streams + for m in [unit.search(s.tags()["unit"])] + if m + ] elif isinstance(unit, str): - obj._streams = [s for s in obj._streams if s.tags().get("unit", "").lower() == unit.lower()] + obj._streams = [ + s + for s in obj._streams + if s.tags().get("unit", "").lower() == unit.lower() + ] else: raise BTRDBTypeError("unit must be string or compiled regex") # filter by tags if tags: # filters if the subset of the tags matches the given tags - obj._streams = [ - s for s in obj._streams - if tags.items() <= s.tags().items() - ] + obj._streams = [s for s in obj._streams if tags.items() <= s.tags().items()] # filter by annotations if annotations: # filters if the subset of the annotations matches the given annotations obj._streams = [ - s for s in obj._streams + s + for s in obj._streams if annotations.items() <= s.annotations()[0].items() ] @@ -1091,7 +1175,7 @@ def clone(self): Returns a new copy of the instance """ - protected = ('_streams', ) + protected = ("_streams", "_btrdb") clone = self.__class__(self._streams) for attr, val in self.__dict__.items(): if attr not in protected: @@ -1186,34 +1270,41 @@ def _streamset_data(self, as_iterators=False): """ params = self._params_from_filters() versions = self.versions() - data = [] if self.pointwidth is not None: # create list of stream.aligned_windows data params.update({"pointwidth": self.pointwidth}) - for s in self._streams: - params.update({"version": versions[s.uuid]}) - data.append(s.aligned_windows(**params)) - + # need to update params based on version of stream, use dict merge + aligned_windows_gen = self._btrdb._executor.map( + lambda s: s.aligned_windows( + **{**params, **{"version": versions[s.uuid]}} + ), + self._streams, + ) + data = list(aligned_windows_gen) elif self.width is not None and self.depth is not None: # create list of stream.windows data (the windows method should # prevent the possibility that only one of these is None) params.update({"width": self.width, "depth": self.depth}) - for s in self._streams: - params.update({"version": versions[s.uuid]}) - data.append(s.windows(**params)) + windows_gen = self._btrdb._executor.map( + lambda s: s.windows(**{**params, **{"version": versions[s.uuid]}}), + self._streams, + ) + data = list(windows_gen) else: # create list of stream.values - data = [s.values(**params) for s in self._streams] + values_gen = self._btrdb._executor.map( + lambda s: s.values(**params), self._streams + ) + data = list(values_gen) if as_iterators: return [iter(ii) for ii in data] return data - def rows(self): """ Returns a materialized list of tuples where each tuple contains the @@ -1240,7 +1331,6 @@ def rows(self): # add next values from streams into buffer for stream_idx, data in enumerate(streamset_data): - if buffer.active[stream_idx]: try: point, _ = next(data) @@ -1287,9 +1377,7 @@ def values(self): def __repr__(self): token = "stream" if len(self) == 1 else "streams" - return "<{}({} {})>".format( - self.__class__.__name__, len(self._streams), token - ) + return "<{}({} {})>".format(self.__class__.__name__, len(self._streams), token) def __str__(self): token = "stream" if len(self) == 1 else "streams" @@ -1317,6 +1405,7 @@ class StreamSet(StreamSetBase, StreamSetTransformer): """ Public class for a collection of streams """ + pass @@ -1324,10 +1413,12 @@ class StreamSet(StreamSetBase, StreamSetTransformer): ## Utility Classes ########################################################################## + class StreamFilter(object): """ Object for storing requested filtering options """ + def __init__(self, start=None, end=None): self.start = to_nanoseconds(start) if start else None self.end = to_nanoseconds(end) if end else None diff --git a/btrdb/transformers.py b/btrdb/transformers.py index 1cf346c..a3a3f94 100644 --- a/btrdb/transformers.py +++ b/btrdb/transformers.py @@ -188,8 +188,8 @@ def to_array(streamset, agg="mean"): segment.append(point.value) else: segment.append(getattr(point, agg)) - results.append(segment) - return np.array(results) + results.append(np.array(segment)) + return np.array(results, dtype=object) def to_dict(streamset, agg="mean", name_callable=None): diff --git a/setup.cfg b/setup.cfg index cded2bd..76684f0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,8 +6,10 @@ license_file = LICENSE.txt test=pytest [tool:pytest] -addopts = --cov=btrdb --flake8 +addopts = --cov=btrdb python_files = tests/* + +[flake8] flake8-ignore = E111 E114 @@ -39,5 +41,7 @@ flake8-ignore = W293 W391 tests/* ALL - btrdb/grpcinterface/btrdb_pb2.py UnusedImport - btrdb/__init__.py UnusedImport + btrdb/grpcinterface/btrdb_pb2.py ALL + btrdb/__init__.py ALL + setup.py ALL + diff --git a/tests/btrdb/test_stream.py b/tests/btrdb/test_stream.py index 3ba9461..6af865e 100644 --- a/tests/btrdb/test_stream.py +++ b/tests/btrdb/test_stream.py @@ -10,7 +10,7 @@ """ Testing package for the btrdb stream module """ - +import concurrent.futures ########################################################################## ## Imports ########################################################################## @@ -58,6 +58,8 @@ def stream1(): type(stream).name = PropertyMock(return_value="gala") stream.tags = Mock(return_value={"name": "gala", "unit": "volts"}) stream.annotations = Mock(return_value=({"owner": "ABC", "color": "red"}, 11)) + stream._btrdb = Mock() + stream._btrdb._executor = concurrent.futures.ThreadPoolExecutor() return stream @@ -72,6 +74,8 @@ def stream2(): type(stream).name = PropertyMock(return_value="blood") stream.tags = Mock(return_value={"name": "blood", "unit": "amps"}) stream.annotations = Mock(return_value=({"owner": "ABC", "color": "orange"}, 22)) + stream._btrdb = Mock() + stream._btrdb._executor = Mock() return stream @@ -1181,7 +1185,7 @@ def test_earliest(self, stream1, stream2): Assert earliest returns correct time code """ streams = StreamSet([stream1, stream2]) - assert streams.latest() == (RawPoint(time=10, value=1), RawPoint(time=20, value=1)) + assert streams.earliest() == (RawPoint(time=10, value=1), RawPoint(time=20, value=1)) def test_latest(self, stream1, stream2): From 354a13ab42ea4547e02c4a57dfcc4facde123349 Mon Sep 17 00:00:00 2001 From: Justin Gilmer Date: Tue, 6 Jun 2023 17:10:53 -0500 Subject: [PATCH 034/131] Threaded arrow (#23) * Release v5.15.0 * update protobuf to v4.22.3 * Add threaded streamset calls Using concurrent.futures.ThreadPoolExecutor * Blacken code * Update for failing tests * Ignore flake8 as part of testing pytest-flake8 seems to have issues with the later versions of flake8 https://github.com/tholo/pytest-flake8/issues/92 * Update .gitignore * Update proto definitions. * Update endpoint to support arrow methods * Support arrow endpoints * Additional arrow updates * Update transformers, add polars conversion * Update .gitignore * Update ignore and remove extra print. * Remove idea folder (pycharm) * Update requirements.txt * Update btrdb/transformers.py * Update the way to check for arrow-enabled btrdb This has not been "turned on" yet though, since we dont know the version number this will be enabled for. The method is currently commented out, but can be re-enabled pretty easily. * Use IPC streams to send the arrow bytes for insert Instead of writing out feather files to an `io.BytesIO` stream and then sending the feather files over the wire, this creates a buffered outputstream and then sends that data back as bytes to btrdb. * Create arrow specific stream methods. * Update test conn object to support minor version * Update tests and migrate arrow code. * Arrow and standard streamset insert * Create basic arrow to dataframe transformer * Support multirawvalues, arrow transformers * Multivalue arrow queries, in progress * Update stream filter to properly filter for sampling frequency * Update arrow values queries for multivalues * Update param passing for sampling frequency * Update index passing, and ignore depth * benchmark raw values queries for arrow and current api * Add aligned windows and run func * Streamset read benchmarks (WIP) In addition: * update streamset.count to support the `precise` boolean flag. * Update mock return value for versionMajor * In progress validation of stream benchs --------- Co-authored-by: David Konigsberg <72822263+davidkonigsberg@users.noreply.github.com> Co-authored-by: Jeff Lin <42981468+jleifnf@users.noreply.github.com> --- .gitignore | 5 + benchmarks/benchmark_stream_reads.py | 276 +++++++++ benchmarks/benchmark_streamset_reads.py | 300 ++++++++++ btrdb/conn.py | 19 + btrdb/endpoint.py | 120 +++- btrdb/grpcinterface/README.md | 4 +- btrdb/grpcinterface/btrdb.proto | 51 ++ btrdb/grpcinterface/btrdb_pb2.py | 259 +++++---- btrdb/grpcinterface/btrdb_pb2.pyi | 92 ++- btrdb/grpcinterface/btrdb_pb2_grpc.py | 165 ++++++ btrdb/stream.py | 593 ++++++++++++++++++-- btrdb/transformers.py | 362 ++++++++++-- requirements.txt | 9 +- tests/btrdb/test_arrow_streams.py | 46 ++ tests/btrdb/test_conn.py | 1 + tests/btrdb/test_stream.py | 711 +++++++++++++----------- tests/btrdb/test_transformers.py | 2 + 17 files changed, 2468 insertions(+), 547 deletions(-) create mode 100644 benchmarks/benchmark_stream_reads.py create mode 100644 benchmarks/benchmark_streamset_reads.py create mode 100644 tests/btrdb/test_arrow_streams.py diff --git a/.gitignore b/.gitignore index e703d03..4629a15 100644 --- a/.gitignore +++ b/.gitignore @@ -118,6 +118,11 @@ dmypy.json # Pyre type checker .pyre/ + +# arrow parquet files +*.parquet + +.idea .idea/misc.xml .idea/vcs.xml .idea/inspectionProfiles/profiles_settings.xml diff --git a/benchmarks/benchmark_stream_reads.py b/benchmarks/benchmark_stream_reads.py new file mode 100644 index 0000000..623695a --- /dev/null +++ b/benchmarks/benchmark_stream_reads.py @@ -0,0 +1,276 @@ +import uuid +from time import perf_counter +from typing import Dict, Union + +import btrdb + + +def time_single_stream_raw_values( + stream: btrdb.stream.Stream, start: int, end: int, version: int = 0 +) -> Dict[str, Union[int, str]]: + """Return the elapsed time for the stream raw values query + + Parameters + ---------- + stream : btrdb.Stream, required + The data stream to return raw values. + start : int, required + The start time (in nanoseconds) to return raw values (inclusive). + end : int, required + The end time (in nanoseconds) to return raw values (exclusive) + version : int, optional, default : 0 + The version of the stream to query for points. + + Notes + ----- + The data points returned will be [start, end) + + Returns + ------- + results : dict + The performance results of the stream method + """ + expected_count = stream.count(start, end, version=version, precise=True) + tic = perf_counter() + vals = stream.values(start, end, version=version) + toc = perf_counter() + # minus 1 to account for the exclusive end time + queried_points = len(vals) + assert queried_points == expected_count + # time in seconds to run + run_time = toc - tic + results = _create_stream_result_dict( + stream.uuid, point_count=queried_points, total_time=run_time, version=version + ) + return results + + +def time_single_stream_arrow_raw_values( + stream: btrdb.stream.Stream, start: int, end: int, version: int = 0 +) -> Dict[str, Union[str, int, float]]: + """Return the elapsed time for the stream arrow raw values query + + Parameters + ---------- + stream : btrdb.Stream, required + The data stream to return the raw data as an arrow table. + start : int, required + The start time (in nanoseconds) to return raw values (inclusive). + end : int, required + The end time (in nanoseconds) to return raw values (exclusive) + version : int, optional, default : 0 + The version of the stream to query for points. + + Notes + ----- + The data points returned will be [start, end) + + Returns + ------- + results : dict + The performance results of the stream method + """ + # minus 1 to account for the exclusive end time for the values query + expected_count = stream.count(start, end, version=version, precise=True) + tic = perf_counter() + vals = stream.arrow_values(start, end, version=version) + toc = perf_counter() + # num of rows + queried_points = vals.num_rows + assert queried_points == expected_count + # time in seconds to run + run_time = toc - tic + results = _create_stream_result_dict( + stream.uuid, point_count=queried_points, total_time=run_time, version=version + ) + return results + + +def time_single_stream_windows_values( + stream: btrdb.stream.Stream, start: int, end: int, width_ns: int, version: int = 0 +) -> Dict[str, Union[str, int, float]]: + """Return the elapsed time for the stream windows values query + + Parameters + ---------- + stream : btrdb.Stream, required + The data stream to return the windowed data as a list of statpoints + start : int, required + The start time (in nanoseconds) to return statpoint values + end : int, required + The end time (in nanoseconds) to return statpoint values + width_ns : int, required + The window width (in nanoseconds) for the statpoints + version : int, optional, default : 0 + The version of the stream to query for points. + + Returns + ------- + results : dict + The performance results of the stream method + """ + tic = perf_counter() + vals = stream.windows(start, end, width=width_ns, version=version) + toc = perf_counter() + # num of statpoints + queried_points = len(vals) + assert queried_points != 0 + # time in seconds to run + run_time = toc - tic + results = _create_stream_result_dict( + stream.uuid, point_count=queried_points, total_time=run_time, version=version + ) + return results + + +def time_single_stream_arrow_windows_values( + stream: btrdb.stream.Stream, start: int, end: int, width_ns: int, version: int = 0 +) -> Dict[str, Union[str, int, float]]: + """Return the elapsed time for the stream arrow window values query + + Parameters + ---------- + stream : btrdb.Stream, required + The data stream to return the windowed data as an arrow table. + start : int, required + The start time (in nanoseconds) to return statpoint values + end : int, required + The end time (in nanoseconds) to return statpoint values + width_ns : int, required + The window width (in nanoseconds) for the statpoints + version : int, optional, default : 0 + The version of the stream to query for points. + + Notes + ----- + The data points returned will be [start, end) + + Returns + ------- + results : dict + The performance results of the stream method + """ + tic = perf_counter() + vals = stream.arrow_windows(start, end, width=width_ns, version=version) + toc = perf_counter() + # num of statpoints + queried_points = vals.num_rows + assert queried_points != 0 + # time in seconds to run + run_time = toc - tic + results = _create_stream_result_dict( + stream.uuid, point_count=queried_points, total_time=run_time, version=version + ) + return results + + +def time_single_stream_aligned_windows_values( + stream: btrdb.stream.Stream, start: int, end: int, pointwidth: int, version: int = 0 +) -> Dict[str, Union[str, int, float]]: + """Return the elapsed time for the stream window values query + + Parameters + ---------- + stream : btrdb.Stream, required + The data stream to return the windowed data as a list of statpoints + start : int, required + The start time (in nanoseconds) to return statpoint values + end : int, required + The end time (in nanoseconds) to return statpoint values + pointwidth : int, required + The level of the tree to return statpoints (the exponent k in 2**k) + version : int, optional, default : 0 + The version of the stream to query for points. + + Returns + ------- + results : dict + The performance results of the stream method + """ + tic = perf_counter() + vals = stream.aligned_windows(start, end, pointwidth=pointwidth, version=version) + toc = perf_counter() + # num of statpoints + queried_points = len(vals) + assert queried_points != 0 + # time in seconds to run + run_time = toc - tic + results = _create_stream_result_dict( + stream.uuid, point_count=queried_points, total_time=run_time, version=version + ) + return results + + +def time_single_stream_arrow_aligned_windows_values( + stream: btrdb.stream.Stream, start: int, end: int, pointwidth: int, version: int = 0 +) -> Dict[str, Union[str, int, float]]: + """Return the elapsed time for the stream arrow aligned window values query + + Parameters + ---------- + stream : btrdb.Stream, required + The data stream to return the windowed data as an arrow table. + start : int, required + The start time (in nanoseconds) to return statpoint values + end : int, required + The end time (in nanoseconds) to return statpoint values + pointwidth : int, required + The level of the tree to return statpoints (the exponent k in 2**k) + version : int, optional, default : 0 + The version of the stream to query for points. + + Returns + ------- + results : dict + The performance results of the stream method + """ + tic = perf_counter() + vals = stream.arrow_aligned_windows( + start, end, pointwidth=pointwidth, version=version + ) + toc = perf_counter() + # num of statpoints + queried_points = vals.num_rows + assert queried_points != 0 + # time in seconds to run + run_time = toc - tic + results = _create_stream_result_dict( + stream.uuid, point_count=queried_points, total_time=run_time, version=version + ) + return results + + +def _create_stream_result_dict( + uu: uuid.UUID, + point_count: int, + total_time: float, + version: int, +) -> Dict[str, Union[str, int, float]]: + return { + "uuid": str(uu), + "total_points": point_count, + "total_time_seconds": total_time, + "stream_version": version, + } + + +def main(): + """Run a single run of the benchmarks""" + conn = btrdb.connect(profile="andy") + stream1 = conn.stream_from_uuid( + list(conn.streams_in_collection("andy/7064-6684-5e6e-9e14-ff9ca7bae46e"))[0].uuid + ) + start = stream1.earliest()[0].time + end = stream1.latest()[0].time + width_ns = btrdb.utils.timez.ns_delta(minutes=1) + pointwidth = btrdb.utils.general.pointwidth(38) + print(f"pointwidth of: {pointwidth}") + for f in [time_single_stream_arrow_raw_values, time_single_stream_raw_values]: + print(f(stream1, start, end, 0)) + for f in [time_single_stream_arrow_windows_values, time_single_stream_windows_values]: + print(f(stream1, start, end, width_ns=width_ns, version=0)) + for f in [time_single_stream_arrow_aligned_windows_values, time_single_stream_aligned_windows_values]: + print(f(stream1, start, end, pointwidth=pointwidth, version=0)) + +if __name__=="__main__": + main() \ No newline at end of file diff --git a/benchmarks/benchmark_streamset_reads.py b/benchmarks/benchmark_streamset_reads.py new file mode 100644 index 0000000..3330fd7 --- /dev/null +++ b/benchmarks/benchmark_streamset_reads.py @@ -0,0 +1,300 @@ +import uuid +from time import perf_counter +from typing import Dict, Union + +import pandas + +import btrdb + + +def time_streamset_raw_values( + streamset: btrdb.stream.StreamSet, start: int, end: int, version: int = 0 +) -> Dict[str, Union[int, str]]: + """Return the elapsed time for the streamset raw values query + + Parameters + ---------- + streamset : btrdb.StreamSet, required + The data streamset to return raw values. + start : int, required + The start time (in nanoseconds) to return raw values (inclusive). + end : int, required + The end time (in nanoseconds) to return raw values (exclusive) + version : int, optional, default : 0 + The version of the streamset to query for points. + + Notes + ----- + The data points returned will be [start, end) + + Returns + ------- + results : dict + The performance results of the streamset method + """ + streamset = streamset.filter(start=start, end=end) + versions = {s.uuid: 0 for s in streamset._streams} + streamset.pin_versions(versions) + expected_count = streamset.count(precise=True) + tic = perf_counter() + vals = streamset.values() + toc = perf_counter() + # print(vals) + # minus 1 to account for the exclusive end time + queried_points = 0 + for points in vals: + print(len(points)) + queried_points += len(points) + expected_count = streamset.count(precise=True) + print(queried_points) + print(expected_count) + assert queried_points == expected_count + # time in seconds to run + run_time = toc - tic + res = _create_streamset_result_dict( + streamset, point_count=queried_points, total_time=run_time, version=version + ) + return res + + +def time_streamset_arrow_raw_values( + streamset: btrdb.stream.StreamSet, start: int, end: int, version: int = 0 +) -> Dict[str, Union[str, int, float]]: + """Return the elapsed time for the streamset arrow raw values query + + Parameters + ---------- + streamset : btrdb.StreamSet, required + The data streamset to return the raw data as an arrow table. + start : int, required + The start time (in nanoseconds) to return raw values (inclusive). + end : int, required + The end time (in nanoseconds) to return raw values (exclusive) + version : int, optional, default : 0 + The version of the streamset to query for points. + + Notes + ----- + The data points returned will be [start, end) + + Returns + ------- + results : dict + The performance results of the streamset method + """ + streamset = streamset.filter(start=start, end=end) + versions = {s.uuid: 0 for s in streamset._streams} + streamset.pin_versions(versions) + expected_count = streamset.count(precise=True) + tic = perf_counter() + vals = streamset.arrow_values() + toc = perf_counter() + queried_points = len(streamset) * (vals.num_rows - 1) + print(queried_points) + print(expected_count) + assert queried_points == expected_count + # time in seconds to run + run_time = toc - tic + res = _create_streamset_result_dict( + streamset, point_count=queried_points, total_time=run_time, version=version + ) + return res + + +def time_streamset_windows_values( + streamset: btrdb.stream.StreamSet, start: int, end: int, width_ns: int, version: int = 0 +) -> Dict[str, Union[str, int, float]]: + """Return the elapsed time for the streamset windows values query + + Parameters + ---------- + streamset : btrdb.StreamSet, required + The data streamset to return the windowed data as a list of statpoints + start : int, required + The start time (in nanoseconds) to return statpoint values + end : int, required + The end time (in nanoseconds) to return statpoint values + width_ns : int, required + The window width (in nanoseconds) for the statpoints + version : int, optional, default : 0 + The version of the streamset to query for points. + + Returns + ------- + results : dict + The performance results of the streamset method + """ + tic = perf_counter() + vals = streamset.windows(start, end, width=width_ns, version=version) + toc = perf_counter() + # num of statpoints + queried_points = len(vals) + assert queried_points != 0 + # time in seconds to run + run_time = toc - tic + results = _create_streamset_result_dict( + streamset.uuid, point_count=queried_points, total_time=run_time, version=version + ) + return results + + +def time_streamset_arrow_windows_values( + streamset: btrdb.stream.StreamSet, start: int, end: int, width_ns: int, version: int = 0 +) -> Dict[str, Union[str, int, float]]: + """Return the elapsed time for the streamset arrow window values query + + Parameters + ---------- + streamset : btrdb.StreamSet, required + The data streamset to return the windowed data as an arrow table. + start : int, required + The start time (in nanoseconds) to return statpoint values + end : int, required + The end time (in nanoseconds) to return statpoint values + width_ns : int, required + The window width (in nanoseconds) for the statpoints + version : int, optional, default : 0 + The version of the streamset to query for points. + + Notes + ----- + The data points returned will be [start, end) + + Returns + ------- + results : dict + The performance results of the streamset method + """ + tic = perf_counter() + vals = streamset.arrow_windows(start, end, width=width_ns, version=version) + toc = perf_counter() + # num of statpoints + queried_points = vals.num_rows + assert queried_points != 0 + # time in seconds to run + run_time = toc - tic + results = _create_streamset_result_dict( + streamset.uuid, point_count=queried_points, total_time=run_time, version=version + ) + return results + + +def time_streamset_aligned_windows_values( + streamset: btrdb.stream.StreamSet, start: int, end: int, pointwidth: int, version: int = 0 +) -> Dict[str, Union[str, int, float]]: + """Return the elapsed time for the streamset window values query + + Parameters + ---------- + streamset : btrdb.StreamSet, required + The data streamset to return the windowed data as a list of statpoints + start : int, required + The start time (in nanoseconds) to return statpoint values + end : int, required + The end time (in nanoseconds) to return statpoint values + pointwidth : int, required + The level of the tree to return statpoints (the exponent k in 2**k) + version : int, optional, default : 0 + The version of the streamset to query for points. + + Returns + ------- + results : dict + The performance results of the streamset method + """ + tic = perf_counter() + vals = streamset.aligned_windows(start, end, pointwidth=pointwidth, version=version) + toc = perf_counter() + # num of statpoints + queried_points = len(vals) + assert queried_points != 0 + # time in seconds to run + run_time = toc - tic + results = _create_streamset_result_dict( + streamset.uuid, point_count=queried_points, total_time=run_time, version=version + ) + return results + + +def time_streamset_arrow_aligned_windows_values( + streamset: btrdb.stream.StreamSet, start: int, end: int, pointwidth: int, version: int = 0 +) -> Dict[str, Union[str, int, float]]: + """Return the elapsed time for the streamset arrow aligned window values query + + Parameters + ---------- + streamset : btrdb.StreamSet, required + The data streamset to return the windowed data as an arrow table. + start : int, required + The start time (in nanoseconds) to return statpoint values + end : int, required + The end time (in nanoseconds) to return statpoint values + pointwidth : int, required + The level of the tree to return statpoints (the exponent k in 2**k) + version : int, optional, default : 0 + The version of the streamset to query for points. + + Returns + ------- + results : dict + The performance results of the streamset method + """ + tic = perf_counter() + vals = streamset.arrow_aligned_windows( + start, end, pointwidth=pointwidth, version=version + ) + toc = perf_counter() + # num of statpoints + queried_points = vals.num_rows + assert queried_points != 0 + # time in seconds to run + run_time = toc - tic + results = _create_streamset_result_dict( + streamset.uuid, point_count=queried_points, total_time=run_time, version=version + ) + return results + + +def _create_streamset_result_dict( + streamset: btrdb.stream.StreamSet, + point_count: int, + total_time: float, + version: int, +) -> Dict[str, Union[str, int, float]]: + return { + "n_streams": len(streamset), + "total_points": point_count, + + "total_time_seconds": total_time, + "streamset_version": version, + } + + +def main(): + """Run a single run of the benchmarks""" + conn = btrdb.connect(profile="andy") + stream1 = conn.stream_from_uuid( + list(conn.streams_in_collection("andy/7064-6684-5e6e-9e14-ff9ca7bae46e"))[0].uuid + ) + stream2 = conn.stream_from_uuid(list(conn.streams_in_collection("andy/30e6-d72f-5cc7-9966-bc1579dc4a72"))[0].uuid) + streamset = btrdb.stream.StreamSet([stream1, stream2]) + start = max(stream1.earliest()[0].time, stream2.earliest()[0].time) + end = min(stream1.latest()[0].time, stream2.latest()[0].time) + width_ns = btrdb.utils.timez.ns_delta(minutes=1) + pointwidth = btrdb.utils.general.pointwidth(38) + print(f"pointwidth of: {pointwidth}") + res_list = [] + for f in [time_streamset_raw_values, time_streamset_arrow_raw_values]: + res = f(streamset, start, end, 0) + res["func"] = f.__name__ + # for f in [time_streamset_windows_values, time_streamset_arrow_windows_values]: + # res = f(streamset, start, end, width_ns=width_ns, version=0) + # res["func"] = f.__name__ + # for f in [time_streamset_aligned_windows_values, time_streamset_arrow_aligned_windows_values]: + # res = f(streamset, start, end, pointwidth=pointwidth, version=0) + # res["func"] = res + + return res +if __name__=="__main__": + results = main() + print(pandas.DataFrame(results)) \ No newline at end of file diff --git a/btrdb/conn.py b/btrdb/conn.py index 5499fa7..617fb1b 100644 --- a/btrdb/conn.py +++ b/btrdb/conn.py @@ -19,6 +19,7 @@ import re import json import certifi +import logging import uuid as uuidlib from concurrent.futures import ThreadPoolExecutor @@ -42,6 +43,7 @@ ########################################################################## ## Classes ########################################################################## +logger = logging.getLogger(__name__) class Connection(object): @@ -112,6 +114,20 @@ def __init__(self, addrportstr, apikey=None): ) self.channel = grpc.insecure_channel(addrportstr, chan_ops) +def _is_arrow_enabled(info): + info = { + "majorVersion": info.majorVersion, + "minorVersion": info.minorVersion, + } + major = info.get("majorVersion", -1) + minor = info.get("minorVersion", -1) + logger.debug(f"major version: {major}") + logger.debug(f"minor version: {minor}") + if major >= 5 and minor >= 30: + return True + else: + return False + class BTrDB(object): """ @@ -121,6 +137,8 @@ class BTrDB(object): def __init__(self, endpoint): self.ep = endpoint self._executor = ThreadPoolExecutor() + self._ARROW_ENABLED = True #_is_arrow_enabled(self.ep.info()) + logger.debug(f"ARROW ENABLED: {self._ARROW_ENABLED}") def query(self, stmt, params=[]): """ @@ -287,6 +305,7 @@ def info(self): info = self.ep.info() return { "majorVersion": info.majorVersion, + "minorVersion": info.minorVersion, "build": info.build, "proxy": {"proxyEndpoints": [ep for ep in info.proxy.proxyEndpoints]}, } diff --git a/btrdb/endpoint.py b/btrdb/endpoint.py index de5378e..aa1b065 100644 --- a/btrdb/endpoint.py +++ b/btrdb/endpoint.py @@ -24,6 +24,10 @@ # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import logging +import uuid + +import grpc from btrdb.grpcinterface import btrdb_pb2 from btrdb.grpcinterface import btrdb_pb2_grpc @@ -31,6 +35,8 @@ from btrdb.exceptions import BTrDBError, error_handler, check_proto_stat from btrdb.utils.general import unpack_stream_descriptor +logger = logging.getLogger(__name__) + class Endpoint(object): def __init__(self, channel): @@ -45,6 +51,45 @@ def rawValues(self, uu, start, end, version=0): check_proto_stat(result.stat) yield result.values, result.versionMajor + @error_handler + def arrowRawValues(self, uu, start, end, version=0): + params = btrdb_pb2.RawValuesParams( + uuid=uu.bytes, start=start, end=end, versionMajor=version + ) + for result in self.stub.ArrowRawValues(params): + check_proto_stat(result.stat) + yield result.arrowBytes, result.versionMajor + + def arrowMultiValues(self, uu_list, start, end, version_list, snap_periodNS): + params = btrdb_pb2.ArrowMultiValuesParams( + uuid=[uu.bytes for uu in uu_list], + start=start, + end=end, + versionMajor=[ver for ver in version_list], + snapPeriodNs=int(snap_periodNS) + ) + for result in self.stub.ArrowMultiValues(params): + check_proto_stat(result.stat) + yield result.arrowBytes, result.versionMajor + + @error_handler + def arrowInsertValues(self, uu: uuid.UUID, values: bytearray, policy: str): + policy_map = { + "never": btrdb_pb2.MergePolicy.NEVER, + "equal": btrdb_pb2.MergePolicy.EQUAL, + "retain": btrdb_pb2.MergePolicy.RETAIN, + "replace": btrdb_pb2.MergePolicy.REPLACE, + } + params = btrdb_pb2.ArrowInsertParams( + uuid=uu.bytes, + sync=False, + arrowBytes=values, + merge_policy=policy_map[policy], + ) + result = self.stub.ArrowInsert(params) + check_proto_stat(result.stat) + return result.versionMajor + @error_handler def alignedWindows(self, uu, start, end, pointwidth, version=0): params = btrdb_pb2.AlignedWindowsParams( @@ -58,6 +103,19 @@ def alignedWindows(self, uu, start, end, pointwidth, version=0): check_proto_stat(result.stat) yield result.values, result.versionMajor + @error_handler + def arrowAlignedWindows(self, uu, start, end, pointwidth, version=0): + params = btrdb_pb2.AlignedWindowsParams( + uuid=uu.bytes, + start=start, + end=end, + versionMajor=version, + pointWidth=int(pointwidth), + ) + for result in self.stub.ArrowAlignedWindows(params): + check_proto_stat(result.stat) + yield result.arrowBytes, result.versionMajor + @error_handler def windows(self, uu, start, end, width, depth, version=0): params = btrdb_pb2.WindowsParams( @@ -72,6 +130,20 @@ def windows(self, uu, start, end, width, depth, version=0): check_proto_stat(result.stat) yield result.values, result.versionMajor + @error_handler + def arrowWindows(self, uu, start, end, width, depth, version=0): + params = btrdb_pb2.WindowsParams( + uuid=uu.bytes, + start=start, + end=end, + versionMajor=version, + width=width, + depth=depth, + ) + for result in self.stub.ArrowWindows(params): + check_proto_stat(result.stat) + yield result.arrowBytes, result.versionMajor + @error_handler def streamInfo(self, uu, omitDescriptor, omitVersion): params = btrdb_pb2.StreamInfoParams( @@ -81,7 +153,13 @@ def streamInfo(self, uu, omitDescriptor, omitVersion): desc = result.descriptor check_proto_stat(result.stat) tagsanns = unpack_stream_descriptor(desc) - return desc.collection, desc.propertyVersion, tagsanns[0], tagsanns[1], result.versionMajor + return ( + desc.collection, + desc.propertyVersion, + tagsanns[0], + tagsanns[1], + result.versionMajor, + ) @error_handler def obliterate(self, uu): @@ -135,11 +213,11 @@ def setStreamTags(self, uu, expected, tags, collection): def create(self, uu, collection, tags, annotations): tagkvlist = [] for k, v in tags.items(): - kv = btrdb_pb2.KeyOptValue(key = k, val = btrdb_pb2.OptValue(value=v)) + kv = btrdb_pb2.KeyOptValue(key=k, val=btrdb_pb2.OptValue(value=v)) tagkvlist.append(kv) annkvlist = [] for k, v in annotations.items(): - kv = btrdb_pb2.KeyOptValue(key = k, val = btrdb_pb2.OptValue(value=v)) + kv = btrdb_pb2.KeyOptValue(key=k, val=btrdb_pb2.OptValue(value=v)) annkvlist.append(kv) params = btrdb_pb2.CreateParams( uuid=uu.bytes, collection=collection, tags=tagkvlist, annotations=annkvlist @@ -195,13 +273,17 @@ def lookupStreams(self, collection, isCollectionPrefix, tags, annotations): @error_handler def nearest(self, uu, time, version, backward): + logger.debug(f"nearest function params: {uu}\t{time}\t{version}\t{backward}") params = btrdb_pb2.NearestParams( uuid=uu.bytes, time=time, versionMajor=version, backward=backward ) + logger.debug(f"params from nearest: {params}") + logger.debug(f"uuid: {uu}") result = self.stub.Nearest(params) + logger.debug(f"nearest, results: {result}") check_proto_stat(result.stat) return result.value, result.versionMajor - + @error_handler def changes(self, uu, fromVersion, toVersion, resolution): params = btrdb_pb2.ChangesParams( @@ -268,18 +350,24 @@ def getMetadataUsage(self, prefix): return result.tags, result.annotations @error_handler - def generateCSV(self, queryType, start, end, width, depth, includeVersions, *streams): - protoStreams = [btrdb_pb2.StreamCSVConfig(version = stream[0], - label = stream[1], - uuid = stream[2].bytes) - for stream in streams] - params = btrdb_pb2.GenerateCSVParams(queryType = queryType.to_proto(), - startTime = start, - endTime = end, - windowSize = width, - depth = depth, - includeVersions = includeVersions, - streams = protoStreams) + def generateCSV( + self, queryType, start, end, width, depth, includeVersions, *streams + ): + protoStreams = [ + btrdb_pb2.StreamCSVConfig( + version=stream[0], label=stream[1], uuid=stream[2].bytes + ) + for stream in streams + ] + params = btrdb_pb2.GenerateCSVParams( + queryType=queryType.to_proto(), + startTime=start, + endTime=end, + windowSize=width, + depth=depth, + includeVersions=includeVersions, + streams=protoStreams, + ) for result in self.stub.GenerateCSV(params): check_proto_stat(result.stat) yield result.row diff --git a/btrdb/grpcinterface/README.md b/btrdb/grpcinterface/README.md index 735571a..24987ee 100644 --- a/btrdb/grpcinterface/README.md +++ b/btrdb/grpcinterface/README.md @@ -1,6 +1,6 @@ To create new protobuf files in this folder: ```commandline -python -m grpc_tools.protoc -I. --python_out=. --pyi_out=. --grpc_python_out=. ./btrdb.proto +python -m grpc_tools.protoc --proto_path=./ --python_out=./ --pyi_out=. --grpc_python_out=./ btrdb.proto ``` -Make sure the proto file is newest. \ No newline at end of file +Make sure the proto file is newest. And that you run the above command from within this folder. diff --git a/btrdb/grpcinterface/btrdb.proto b/btrdb/grpcinterface/btrdb.proto index 93fdff2..884a71c 100644 --- a/btrdb/grpcinterface/btrdb.proto +++ b/btrdb/grpcinterface/btrdb.proto @@ -4,8 +4,12 @@ package v5api; service BTrDB { rpc RawValues(RawValuesParams) returns (stream RawValuesResponse); + rpc ArrowRawValues(RawValuesParams) returns (stream ArrowRawValuesResponse); + rpc ArrowMultiValues(ArrowMultiValuesParams) returns (stream ArrowMultiValuesResponse); rpc AlignedWindows(AlignedWindowsParams) returns (stream AlignedWindowsResponse); + rpc ArrowAlignedWindows(AlignedWindowsParams) returns (stream ArrowAlignedWindowsResponse); rpc Windows(WindowsParams) returns (stream WindowsResponse); + rpc ArrowWindows(WindowsParams) returns (stream ArrowWindowsResponse); rpc StreamInfo(StreamInfoParams) returns (StreamInfoResponse); rpc SetStreamAnnotations(SetStreamAnnotationsParams) returns (SetStreamAnnotationsResponse); rpc SetStreamTags(SetStreamTagsParams) returns (SetStreamTagsResponse); @@ -15,6 +19,7 @@ service BTrDB { rpc Nearest(NearestParams) returns (NearestResponse); rpc Changes(ChangesParams) returns (stream ChangesResponse); rpc Insert(InsertParams) returns (InsertResponse); + rpc ArrowInsert(ArrowInsertParams) returns (InsertResponse); rpc Delete(DeleteParams) returns (DeleteResponse); rpc Info(InfoParams) returns (InfoResponse); rpc FaultInject(FaultInjectParams) returns (FaultInjectResponse); @@ -39,6 +44,27 @@ message RawValuesResponse { uint64 versionMinor = 3; repeated RawPoint values = 4; } +message ArrowRawValuesResponse { + Status stat = 1; + uint64 versionMajor = 2; + uint64 versionMinor = 3; + bytes arrowBytes = 4; +} +message ArrowMultiValuesParams { + repeated bytes uuid = 1; + repeated uint64 versionMajor = 2; + sfixed64 start = 3; + sfixed64 end = 4; + int64 snapPeriodNs = 5; +} +message ArrowMultiValuesResponse { + Status stat = 1; + bytes arrowBytes = 2; +} +message RawPointVec { + sfixed64 time = 1; + repeated double value = 2; +}; message AlignedWindowsParams { bytes uuid = 1; sfixed64 start = 2; @@ -52,6 +78,12 @@ message AlignedWindowsResponse { uint64 versionMinor = 3; repeated StatPoint values = 4; } +message ArrowAlignedWindowsResponse { + Status stat = 1; + uint64 versionMajor = 2; + uint64 versionMinor = 3; + bytes arrowBytes = 4; +} message WindowsParams { bytes uuid = 1; sfixed64 start = 2; @@ -66,6 +98,12 @@ message WindowsResponse { uint64 versionMinor = 3; repeated StatPoint values = 4; } +message ArrowWindowsResponse { + Status stat = 1; + uint64 versionMajor = 2; + uint64 versionMinor = 3; + bytes arrowBytes = 4; +} message StreamInfoParams { bytes uuid = 1; bool omitVersion = 2; @@ -175,12 +213,25 @@ enum MergePolicy { RETAIN = 2; // When timestamps are equal, keep old value REPLACE = 3; // When timestamps are equal, keep new value } +message RoundSpec { + oneof spec { + int32 bits = 2; + } +} message InsertParams { bytes uuid = 1; bool sync = 2; MergePolicy merge_policy = 4; + RoundSpec rounding=5; repeated RawPoint values = 3; } +message ArrowInsertParams { + bytes uuid = 1; + bool sync = 2; + MergePolicy merge_policy = 3; + RoundSpec rounding = 4; + bytes arrowBytes = 5; +} message InsertResponse { Status stat = 1; uint64 versionMajor = 2; diff --git a/btrdb/grpcinterface/btrdb_pb2.py b/btrdb/grpcinterface/btrdb_pb2.py index a98905b..ace467e 100644 --- a/btrdb/grpcinterface/btrdb_pb2.py +++ b/btrdb/grpcinterface/btrdb_pb2.py @@ -13,135 +13,152 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0b\x62trdb.proto\x12\x05v5api\"Q\n\x0fRawValuesParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\"}\n\x11RawValuesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x1f\n\x06values\x18\x04 \x03(\x0b\x32\x0f.v5api.RawPoint\"j\n\x14\x41lignedWindowsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\x12\x12\n\npointWidth\x18\x05 \x01(\r\"\x83\x01\n\x16\x41lignedWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12 \n\x06values\x18\x04 \x03(\x0b\x32\x10.v5api.StatPoint\"m\n\rWindowsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\x12\r\n\x05width\x18\x05 \x01(\x04\x12\r\n\x05\x64\x65pth\x18\x06 \x01(\r\"|\n\x0fWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12 \n\x06values\x18\x04 \x03(\x0b\x32\x10.v5api.StatPoint\"h\n\x10StreamInfoParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x13\n\x0bomitVersion\x18\x02 \x01(\x08\x12\x16\n\x0eomitDescriptor\x18\x03 \x01(\x08\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"\x8a\x01\n\x12StreamInfoResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12+\n\ndescriptor\x18\x04 \x01(\x0b\x32\x17.v5api.StreamDescriptor\"\x98\x01\n\x10StreamDescriptor\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x12\n\ncollection\x18\x02 \x01(\t\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x17\n\x0fpropertyVersion\x18\x05 \x01(\x04\"\x82\x01\n\x1aSetStreamAnnotationsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x1f\n\x17\x65xpectedPropertyVersion\x18\x02 \x01(\x04\x12#\n\x07\x63hanges\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x10\n\x08removals\x18\x04 \x03(\t\";\n\x1cSetStreamAnnotationsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"\x8a\x01\n\x13SetStreamTagsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x1f\n\x17\x65xpectedPropertyVersion\x18\x02 \x01(\x04\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x12\n\ncollection\x18\x04 \x01(\t\x12\x0e\n\x06remove\x18\x05 \x03(\t\"4\n\x15SetStreamTagsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"{\n\x0c\x43reateParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x12\n\ncollection\x18\x02 \x01(\t\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\"-\n\x0e\x43reateResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"@\n\x13MetadataUsageParams\x12\x0e\n\x06prefix\x18\x01 \x01(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"y\n\x15MetadataUsageResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x1d\n\x04tags\x18\x02 \x03(\x0b\x32\x0f.v5api.KeyCount\x12$\n\x0b\x61nnotations\x18\x03 \x03(\x0b\x32\x0f.v5api.KeyCount\"&\n\x08KeyCount\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05\x63ount\x18\x02 \x01(\x04\"B\n\x15ListCollectionsParams\x12\x0e\n\x06prefix\x18\x01 \x01(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"K\n\x17ListCollectionsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x13\n\x0b\x63ollections\x18\x02 \x03(\t\"\xab\x01\n\x13LookupStreamsParams\x12\x12\n\ncollection\x18\x01 \x01(\t\x12\x1a\n\x12isCollectionPrefix\x18\x02 \x01(\x08\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"^\n\x15LookupStreamsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12(\n\x07results\x18\x02 \x03(\x0b\x32\x17.v5api.StreamDescriptor\"S\n\rNearestParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x0c\n\x04time\x18\x02 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x03 \x01(\x04\x12\x10\n\x08\x62\x61\x63kward\x18\x04 \x01(\x08\"z\n\x0fNearestResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x1e\n\x05value\x18\x04 \x01(\x0b\x32\x0f.v5api.RawPoint\"U\n\rChangesParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x11\n\tfromMajor\x18\x02 \x01(\x04\x12\x0f\n\x07toMajor\x18\x03 \x01(\x04\x12\x12\n\nresolution\x18\x04 \x01(\r\"\x7f\n\x0f\x43hangesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12#\n\x06ranges\x18\x04 \x03(\x0b\x32\x13.v5api.ChangedRange\"u\n\x0cInsertParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x0c\n\x04sync\x18\x02 \x01(\x08\x12(\n\x0cmerge_policy\x18\x04 \x01(\x0e\x32\x12.v5api.MergePolicy\x12\x1f\n\x06values\x18\x03 \x03(\x0b\x32\x0f.v5api.RawPoint\"Y\n\x0eInsertResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\"8\n\x0c\x44\x65leteParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\"Y\n\x0e\x44\x65leteResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\"\x0c\n\nInfoParams\"\xa2\x01\n\x0cInfoResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x19\n\x04mash\x18\x02 \x01(\x0b\x32\x0b.v5api.Mash\x12\x14\n\x0cmajorVersion\x18\x03 \x01(\r\x12\x14\n\x0cminorVersion\x18\x04 \x01(\r\x12\r\n\x05\x62uild\x18\x05 \x01(\t\x12\x1f\n\x05proxy\x18\x06 \x01(\x0b\x32\x10.v5api.ProxyInfo\"#\n\tProxyInfo\x12\x16\n\x0eproxyEndpoints\x18\x01 \x03(\t\"1\n\x11\x46\x61ultInjectParams\x12\x0c\n\x04type\x18\x01 \x01(\x04\x12\x0e\n\x06params\x18\x02 \x01(\x0c\">\n\x13\x46\x61ultInjectResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\n\n\x02rv\x18\x02 \x01(\x0c\"\x1b\n\x0b\x46lushParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"X\n\rFlushResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\" \n\x10ObliterateParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"1\n\x12ObliterateResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"\'\n\x08RawPoint\x12\x0c\n\x04time\x18\x01 \x01(\x10\x12\r\n\x05value\x18\x02 \x01(\x01\"`\n\tStatPoint\x12\x0c\n\x04time\x18\x01 \x01(\x10\x12\x0b\n\x03min\x18\x02 \x01(\x01\x12\x0c\n\x04mean\x18\x03 \x01(\x01\x12\x0b\n\x03max\x18\x04 \x01(\x01\x12\r\n\x05\x63ount\x18\x05 \x01(\x06\x12\x0e\n\x06stddev\x18\x06 \x01(\x01\"*\n\x0c\x43hangedRange\x12\r\n\x05start\x18\x01 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x02 \x01(\x10\">\n\x06Status\x12\x0c\n\x04\x63ode\x18\x01 \x01(\r\x12\x0b\n\x03msg\x18\x02 \x01(\t\x12\x19\n\x04mash\x18\x03 \x01(\x0b\x32\x0b.v5api.Mash\"\x98\x01\n\x04Mash\x12\x10\n\x08revision\x18\x01 \x01(\x03\x12\x0e\n\x06leader\x18\x02 \x01(\t\x12\x16\n\x0eleaderRevision\x18\x03 \x01(\x03\x12\x13\n\x0btotalWeight\x18\x04 \x01(\x03\x12\x0f\n\x07healthy\x18\x05 \x01(\x08\x12\x10\n\x08unmapped\x18\x06 \x01(\x01\x12\x1e\n\x07members\x18\x07 \x03(\x0b\x32\r.v5api.Member\"\xc3\x01\n\x06Member\x12\x0c\n\x04hash\x18\x01 \x01(\r\x12\x10\n\x08nodename\x18\x02 \x01(\t\x12\n\n\x02up\x18\x03 \x01(\x08\x12\n\n\x02in\x18\x04 \x01(\x08\x12\x0f\n\x07\x65nabled\x18\x05 \x01(\x08\x12\r\n\x05start\x18\x06 \x01(\x03\x12\x0b\n\x03\x65nd\x18\x07 \x01(\x03\x12\x0e\n\x06weight\x18\x08 \x01(\x03\x12\x16\n\x0ereadPreference\x18\t \x01(\x01\x12\x15\n\rhttpEndpoints\x18\n \x01(\t\x12\x15\n\rgrpcEndpoints\x18\x0b \x01(\t\"8\n\x0bKeyOptValue\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x1c\n\x03val\x18\x02 \x01(\x0b\x32\x0f.v5api.OptValue\"\x19\n\x08OptValue\x12\r\n\x05value\x18\x01 \x01(\t\"&\n\x08KeyValue\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"?\n\x0fStreamCSVConfig\x12\x0f\n\x07version\x18\x01 \x01(\x04\x12\r\n\x05label\x18\x02 \x01(\t\x12\x0c\n\x04uuid\x18\x03 \x01(\x0c\"\x9d\x02\n\x11GenerateCSVParams\x12\x35\n\tqueryType\x18\x01 \x01(\x0e\x32\".v5api.GenerateCSVParams.QueryType\x12\x11\n\tstartTime\x18\x02 \x01(\x03\x12\x0f\n\x07\x65ndTime\x18\x03 \x01(\x03\x12\x12\n\nwindowSize\x18\x04 \x01(\x04\x12\r\n\x05\x64\x65pth\x18\x05 \x01(\r\x12\x17\n\x0fincludeVersions\x18\x06 \x01(\x08\x12\'\n\x07streams\x18\x07 \x03(\x0b\x32\x16.v5api.StreamCSVConfig\"H\n\tQueryType\x12\x19\n\x15\x41LIGNED_WINDOWS_QUERY\x10\x00\x12\x11\n\rWINDOWS_QUERY\x10\x01\x12\r\n\tRAW_QUERY\x10\x02\"Q\n\x13GenerateCSVResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x10\n\x08isHeader\x18\x02 \x01(\x08\x12\x0b\n\x03row\x18\x03 \x03(\t\"J\n\x0eSQLQueryParams\x12\r\n\x05query\x18\x01 \x01(\t\x12\x0e\n\x06params\x18\x02 \x03(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"D\n\x10SQLQueryResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x13\n\x0bSQLQueryRow\x18\x02 \x03(\x0c\"\x14\n\x04Role\x12\x0c\n\x04name\x18\x01 \x01(\t\"\x94\x01\n\x19SetCompactionConfigParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x18\n\x10\x43ompactedVersion\x18\x02 \x01(\x04\x12>\n\x17reducedResolutionRanges\x18\x03 \x03(\x0b\x32\x1d.v5api.ReducedResolutionRange\x12\x0f\n\x07unused0\x18\x04 \x01(\x04\":\n\x1bSetCompactionConfigResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\")\n\x19GetCompactionConfigParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"\xc1\x01\n\x1bGetCompactionConfigResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x1a\n\x12LatestMajorVersion\x18\x02 \x01(\x04\x12\x18\n\x10\x43ompactedVersion\x18\x03 \x01(\x04\x12>\n\x17reducedResolutionRanges\x18\x04 \x03(\x0b\x32\x1d.v5api.ReducedResolutionRange\x12\x0f\n\x07unused0\x18\x05 \x01(\x04\"H\n\x16ReducedResolutionRange\x12\r\n\x05Start\x18\x01 \x01(\x03\x12\x0b\n\x03\x45nd\x18\x02 \x01(\x03\x12\x12\n\nResolution\x18\x03 \x01(\r*<\n\x0bMergePolicy\x12\t\n\x05NEVER\x10\x00\x12\t\n\x05\x45QUAL\x10\x01\x12\n\n\x06RETAIN\x10\x02\x12\x0b\n\x07REPLACE\x10\x03\x32\xb3\n\n\x05\x42TrDB\x12?\n\tRawValues\x12\x16.v5api.RawValuesParams\x1a\x18.v5api.RawValuesResponse0\x01\x12N\n\x0e\x41lignedWindows\x12\x1b.v5api.AlignedWindowsParams\x1a\x1d.v5api.AlignedWindowsResponse0\x01\x12\x39\n\x07Windows\x12\x14.v5api.WindowsParams\x1a\x16.v5api.WindowsResponse0\x01\x12@\n\nStreamInfo\x12\x17.v5api.StreamInfoParams\x1a\x19.v5api.StreamInfoResponse\x12^\n\x14SetStreamAnnotations\x12!.v5api.SetStreamAnnotationsParams\x1a#.v5api.SetStreamAnnotationsResponse\x12I\n\rSetStreamTags\x12\x1a.v5api.SetStreamTagsParams\x1a\x1c.v5api.SetStreamTagsResponse\x12\x34\n\x06\x43reate\x12\x13.v5api.CreateParams\x1a\x15.v5api.CreateResponse\x12Q\n\x0fListCollections\x12\x1c.v5api.ListCollectionsParams\x1a\x1e.v5api.ListCollectionsResponse0\x01\x12K\n\rLookupStreams\x12\x1a.v5api.LookupStreamsParams\x1a\x1c.v5api.LookupStreamsResponse0\x01\x12\x37\n\x07Nearest\x12\x14.v5api.NearestParams\x1a\x16.v5api.NearestResponse\x12\x39\n\x07\x43hanges\x12\x14.v5api.ChangesParams\x1a\x16.v5api.ChangesResponse0\x01\x12\x34\n\x06Insert\x12\x13.v5api.InsertParams\x1a\x15.v5api.InsertResponse\x12\x34\n\x06\x44\x65lete\x12\x13.v5api.DeleteParams\x1a\x15.v5api.DeleteResponse\x12.\n\x04Info\x12\x11.v5api.InfoParams\x1a\x13.v5api.InfoResponse\x12\x43\n\x0b\x46\x61ultInject\x12\x18.v5api.FaultInjectParams\x1a\x1a.v5api.FaultInjectResponse\x12\x31\n\x05\x46lush\x12\x12.v5api.FlushParams\x1a\x14.v5api.FlushResponse\x12@\n\nObliterate\x12\x17.v5api.ObliterateParams\x1a\x19.v5api.ObliterateResponse\x12L\n\x10GetMetadataUsage\x12\x1a.v5api.MetadataUsageParams\x1a\x1c.v5api.MetadataUsageResponse\x12\x45\n\x0bGenerateCSV\x12\x18.v5api.GenerateCSVParams\x1a\x1a.v5api.GenerateCSVResponse0\x01\x12<\n\x08SQLQuery\x12\x15.v5api.SQLQueryParams\x1a\x17.v5api.SQLQueryResponse0\x01\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0b\x62trdb.proto\x12\x05v5api\"Q\n\x0fRawValuesParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\"}\n\x11RawValuesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x1f\n\x06values\x18\x04 \x03(\x0b\x32\x0f.v5api.RawPoint\"u\n\x16\x41rrowRawValuesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x12\n\narrowBytes\x18\x04 \x01(\x0c\"n\n\x16\x41rrowMultiValuesParams\x12\x0c\n\x04uuid\x18\x01 \x03(\x0c\x12\x14\n\x0cversionMajor\x18\x02 \x03(\x04\x12\r\n\x05start\x18\x03 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x04 \x01(\x10\x12\x14\n\x0csnapPeriodNs\x18\x05 \x01(\x03\"K\n\x18\x41rrowMultiValuesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x12\n\narrowBytes\x18\x02 \x01(\x0c\"*\n\x0bRawPointVec\x12\x0c\n\x04time\x18\x01 \x01(\x10\x12\r\n\x05value\x18\x02 \x03(\x01\"j\n\x14\x41lignedWindowsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\x12\x12\n\npointWidth\x18\x05 \x01(\r\"\x83\x01\n\x16\x41lignedWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12 \n\x06values\x18\x04 \x03(\x0b\x32\x10.v5api.StatPoint\"z\n\x1b\x41rrowAlignedWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x12\n\narrowBytes\x18\x04 \x01(\x0c\"m\n\rWindowsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\x12\r\n\x05width\x18\x05 \x01(\x04\x12\r\n\x05\x64\x65pth\x18\x06 \x01(\r\"|\n\x0fWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12 \n\x06values\x18\x04 \x03(\x0b\x32\x10.v5api.StatPoint\"s\n\x14\x41rrowWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x12\n\narrowBytes\x18\x04 \x01(\x0c\"h\n\x10StreamInfoParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x13\n\x0bomitVersion\x18\x02 \x01(\x08\x12\x16\n\x0eomitDescriptor\x18\x03 \x01(\x08\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"\x8a\x01\n\x12StreamInfoResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12+\n\ndescriptor\x18\x04 \x01(\x0b\x32\x17.v5api.StreamDescriptor\"\x98\x01\n\x10StreamDescriptor\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x12\n\ncollection\x18\x02 \x01(\t\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x17\n\x0fpropertyVersion\x18\x05 \x01(\x04\"\x82\x01\n\x1aSetStreamAnnotationsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x1f\n\x17\x65xpectedPropertyVersion\x18\x02 \x01(\x04\x12#\n\x07\x63hanges\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x10\n\x08removals\x18\x04 \x03(\t\";\n\x1cSetStreamAnnotationsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"\x8a\x01\n\x13SetStreamTagsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x1f\n\x17\x65xpectedPropertyVersion\x18\x02 \x01(\x04\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x12\n\ncollection\x18\x04 \x01(\t\x12\x0e\n\x06remove\x18\x05 \x03(\t\"4\n\x15SetStreamTagsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"{\n\x0c\x43reateParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x12\n\ncollection\x18\x02 \x01(\t\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\"-\n\x0e\x43reateResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"@\n\x13MetadataUsageParams\x12\x0e\n\x06prefix\x18\x01 \x01(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"y\n\x15MetadataUsageResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x1d\n\x04tags\x18\x02 \x03(\x0b\x32\x0f.v5api.KeyCount\x12$\n\x0b\x61nnotations\x18\x03 \x03(\x0b\x32\x0f.v5api.KeyCount\"&\n\x08KeyCount\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05\x63ount\x18\x02 \x01(\x04\"B\n\x15ListCollectionsParams\x12\x0e\n\x06prefix\x18\x01 \x01(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"K\n\x17ListCollectionsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x13\n\x0b\x63ollections\x18\x02 \x03(\t\"\xab\x01\n\x13LookupStreamsParams\x12\x12\n\ncollection\x18\x01 \x01(\t\x12\x1a\n\x12isCollectionPrefix\x18\x02 \x01(\x08\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"^\n\x15LookupStreamsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12(\n\x07results\x18\x02 \x03(\x0b\x32\x17.v5api.StreamDescriptor\"S\n\rNearestParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x0c\n\x04time\x18\x02 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x03 \x01(\x04\x12\x10\n\x08\x62\x61\x63kward\x18\x04 \x01(\x08\"z\n\x0fNearestResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x1e\n\x05value\x18\x04 \x01(\x0b\x32\x0f.v5api.RawPoint\"U\n\rChangesParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x11\n\tfromMajor\x18\x02 \x01(\x04\x12\x0f\n\x07toMajor\x18\x03 \x01(\x04\x12\x12\n\nresolution\x18\x04 \x01(\r\"\x7f\n\x0f\x43hangesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12#\n\x06ranges\x18\x04 \x03(\x0b\x32\x13.v5api.ChangedRange\"#\n\tRoundSpec\x12\x0e\n\x04\x62its\x18\x02 \x01(\x05H\x00\x42\x06\n\x04spec\"\x99\x01\n\x0cInsertParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x0c\n\x04sync\x18\x02 \x01(\x08\x12(\n\x0cmerge_policy\x18\x04 \x01(\x0e\x32\x12.v5api.MergePolicy\x12\"\n\x08rounding\x18\x05 \x01(\x0b\x32\x10.v5api.RoundSpec\x12\x1f\n\x06values\x18\x03 \x03(\x0b\x32\x0f.v5api.RawPoint\"\x91\x01\n\x11\x41rrowInsertParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x0c\n\x04sync\x18\x02 \x01(\x08\x12(\n\x0cmerge_policy\x18\x03 \x01(\x0e\x32\x12.v5api.MergePolicy\x12\"\n\x08rounding\x18\x04 \x01(\x0b\x32\x10.v5api.RoundSpec\x12\x12\n\narrowBytes\x18\x05 \x01(\x0c\"Y\n\x0eInsertResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\"8\n\x0c\x44\x65leteParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\"Y\n\x0e\x44\x65leteResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\"\x0c\n\nInfoParams\"\xa2\x01\n\x0cInfoResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x19\n\x04mash\x18\x02 \x01(\x0b\x32\x0b.v5api.Mash\x12\x14\n\x0cmajorVersion\x18\x03 \x01(\r\x12\x14\n\x0cminorVersion\x18\x04 \x01(\r\x12\r\n\x05\x62uild\x18\x05 \x01(\t\x12\x1f\n\x05proxy\x18\x06 \x01(\x0b\x32\x10.v5api.ProxyInfo\"#\n\tProxyInfo\x12\x16\n\x0eproxyEndpoints\x18\x01 \x03(\t\"1\n\x11\x46\x61ultInjectParams\x12\x0c\n\x04type\x18\x01 \x01(\x04\x12\x0e\n\x06params\x18\x02 \x01(\x0c\">\n\x13\x46\x61ultInjectResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\n\n\x02rv\x18\x02 \x01(\x0c\"\x1b\n\x0b\x46lushParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"X\n\rFlushResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\" \n\x10ObliterateParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"1\n\x12ObliterateResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"\'\n\x08RawPoint\x12\x0c\n\x04time\x18\x01 \x01(\x10\x12\r\n\x05value\x18\x02 \x01(\x01\"`\n\tStatPoint\x12\x0c\n\x04time\x18\x01 \x01(\x10\x12\x0b\n\x03min\x18\x02 \x01(\x01\x12\x0c\n\x04mean\x18\x03 \x01(\x01\x12\x0b\n\x03max\x18\x04 \x01(\x01\x12\r\n\x05\x63ount\x18\x05 \x01(\x06\x12\x0e\n\x06stddev\x18\x06 \x01(\x01\"*\n\x0c\x43hangedRange\x12\r\n\x05start\x18\x01 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x02 \x01(\x10\">\n\x06Status\x12\x0c\n\x04\x63ode\x18\x01 \x01(\r\x12\x0b\n\x03msg\x18\x02 \x01(\t\x12\x19\n\x04mash\x18\x03 \x01(\x0b\x32\x0b.v5api.Mash\"\x98\x01\n\x04Mash\x12\x10\n\x08revision\x18\x01 \x01(\x03\x12\x0e\n\x06leader\x18\x02 \x01(\t\x12\x16\n\x0eleaderRevision\x18\x03 \x01(\x03\x12\x13\n\x0btotalWeight\x18\x04 \x01(\x03\x12\x0f\n\x07healthy\x18\x05 \x01(\x08\x12\x10\n\x08unmapped\x18\x06 \x01(\x01\x12\x1e\n\x07members\x18\x07 \x03(\x0b\x32\r.v5api.Member\"\xc3\x01\n\x06Member\x12\x0c\n\x04hash\x18\x01 \x01(\r\x12\x10\n\x08nodename\x18\x02 \x01(\t\x12\n\n\x02up\x18\x03 \x01(\x08\x12\n\n\x02in\x18\x04 \x01(\x08\x12\x0f\n\x07\x65nabled\x18\x05 \x01(\x08\x12\r\n\x05start\x18\x06 \x01(\x03\x12\x0b\n\x03\x65nd\x18\x07 \x01(\x03\x12\x0e\n\x06weight\x18\x08 \x01(\x03\x12\x16\n\x0ereadPreference\x18\t \x01(\x01\x12\x15\n\rhttpEndpoints\x18\n \x01(\t\x12\x15\n\rgrpcEndpoints\x18\x0b \x01(\t\"8\n\x0bKeyOptValue\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x1c\n\x03val\x18\x02 \x01(\x0b\x32\x0f.v5api.OptValue\"\x19\n\x08OptValue\x12\r\n\x05value\x18\x01 \x01(\t\"&\n\x08KeyValue\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"?\n\x0fStreamCSVConfig\x12\x0f\n\x07version\x18\x01 \x01(\x04\x12\r\n\x05label\x18\x02 \x01(\t\x12\x0c\n\x04uuid\x18\x03 \x01(\x0c\"\x9d\x02\n\x11GenerateCSVParams\x12\x35\n\tqueryType\x18\x01 \x01(\x0e\x32\".v5api.GenerateCSVParams.QueryType\x12\x11\n\tstartTime\x18\x02 \x01(\x03\x12\x0f\n\x07\x65ndTime\x18\x03 \x01(\x03\x12\x12\n\nwindowSize\x18\x04 \x01(\x04\x12\r\n\x05\x64\x65pth\x18\x05 \x01(\r\x12\x17\n\x0fincludeVersions\x18\x06 \x01(\x08\x12\'\n\x07streams\x18\x07 \x03(\x0b\x32\x16.v5api.StreamCSVConfig\"H\n\tQueryType\x12\x19\n\x15\x41LIGNED_WINDOWS_QUERY\x10\x00\x12\x11\n\rWINDOWS_QUERY\x10\x01\x12\r\n\tRAW_QUERY\x10\x02\"Q\n\x13GenerateCSVResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x10\n\x08isHeader\x18\x02 \x01(\x08\x12\x0b\n\x03row\x18\x03 \x03(\t\"J\n\x0eSQLQueryParams\x12\r\n\x05query\x18\x01 \x01(\t\x12\x0e\n\x06params\x18\x02 \x03(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"D\n\x10SQLQueryResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x13\n\x0bSQLQueryRow\x18\x02 \x03(\x0c\"\x14\n\x04Role\x12\x0c\n\x04name\x18\x01 \x01(\t\"\x94\x01\n\x19SetCompactionConfigParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x18\n\x10\x43ompactedVersion\x18\x02 \x01(\x04\x12>\n\x17reducedResolutionRanges\x18\x03 \x03(\x0b\x32\x1d.v5api.ReducedResolutionRange\x12\x0f\n\x07unused0\x18\x04 \x01(\x04\":\n\x1bSetCompactionConfigResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\")\n\x19GetCompactionConfigParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"\xc1\x01\n\x1bGetCompactionConfigResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x1a\n\x12LatestMajorVersion\x18\x02 \x01(\x04\x12\x18\n\x10\x43ompactedVersion\x18\x03 \x01(\x04\x12>\n\x17reducedResolutionRanges\x18\x04 \x03(\x0b\x32\x1d.v5api.ReducedResolutionRange\x12\x0f\n\x07unused0\x18\x05 \x01(\x04\"H\n\x16ReducedResolutionRange\x12\r\n\x05Start\x18\x01 \x01(\x03\x12\x0b\n\x03\x45nd\x18\x02 \x01(\x03\x12\x12\n\nResolution\x18\x03 \x01(\r*<\n\x0bMergePolicy\x12\t\n\x05NEVER\x10\x00\x12\t\n\x05\x45QUAL\x10\x01\x12\n\n\x06RETAIN\x10\x02\x12\x0b\n\x07REPLACE\x10\x03\x32\xb3\r\n\x05\x42TrDB\x12?\n\tRawValues\x12\x16.v5api.RawValuesParams\x1a\x18.v5api.RawValuesResponse0\x01\x12I\n\x0e\x41rrowRawValues\x12\x16.v5api.RawValuesParams\x1a\x1d.v5api.ArrowRawValuesResponse0\x01\x12T\n\x10\x41rrowMultiValues\x12\x1d.v5api.ArrowMultiValuesParams\x1a\x1f.v5api.ArrowMultiValuesResponse0\x01\x12N\n\x0e\x41lignedWindows\x12\x1b.v5api.AlignedWindowsParams\x1a\x1d.v5api.AlignedWindowsResponse0\x01\x12X\n\x13\x41rrowAlignedWindows\x12\x1b.v5api.AlignedWindowsParams\x1a\".v5api.ArrowAlignedWindowsResponse0\x01\x12\x39\n\x07Windows\x12\x14.v5api.WindowsParams\x1a\x16.v5api.WindowsResponse0\x01\x12\x43\n\x0c\x41rrowWindows\x12\x14.v5api.WindowsParams\x1a\x1b.v5api.ArrowWindowsResponse0\x01\x12@\n\nStreamInfo\x12\x17.v5api.StreamInfoParams\x1a\x19.v5api.StreamInfoResponse\x12^\n\x14SetStreamAnnotations\x12!.v5api.SetStreamAnnotationsParams\x1a#.v5api.SetStreamAnnotationsResponse\x12I\n\rSetStreamTags\x12\x1a.v5api.SetStreamTagsParams\x1a\x1c.v5api.SetStreamTagsResponse\x12\x34\n\x06\x43reate\x12\x13.v5api.CreateParams\x1a\x15.v5api.CreateResponse\x12Q\n\x0fListCollections\x12\x1c.v5api.ListCollectionsParams\x1a\x1e.v5api.ListCollectionsResponse0\x01\x12K\n\rLookupStreams\x12\x1a.v5api.LookupStreamsParams\x1a\x1c.v5api.LookupStreamsResponse0\x01\x12\x37\n\x07Nearest\x12\x14.v5api.NearestParams\x1a\x16.v5api.NearestResponse\x12\x39\n\x07\x43hanges\x12\x14.v5api.ChangesParams\x1a\x16.v5api.ChangesResponse0\x01\x12\x34\n\x06Insert\x12\x13.v5api.InsertParams\x1a\x15.v5api.InsertResponse\x12>\n\x0b\x41rrowInsert\x12\x18.v5api.ArrowInsertParams\x1a\x15.v5api.InsertResponse\x12\x34\n\x06\x44\x65lete\x12\x13.v5api.DeleteParams\x1a\x15.v5api.DeleteResponse\x12.\n\x04Info\x12\x11.v5api.InfoParams\x1a\x13.v5api.InfoResponse\x12\x43\n\x0b\x46\x61ultInject\x12\x18.v5api.FaultInjectParams\x1a\x1a.v5api.FaultInjectResponse\x12\x31\n\x05\x46lush\x12\x12.v5api.FlushParams\x1a\x14.v5api.FlushResponse\x12@\n\nObliterate\x12\x17.v5api.ObliterateParams\x1a\x19.v5api.ObliterateResponse\x12L\n\x10GetMetadataUsage\x12\x1a.v5api.MetadataUsageParams\x1a\x1c.v5api.MetadataUsageResponse\x12\x45\n\x0bGenerateCSV\x12\x18.v5api.GenerateCSVParams\x1a\x1a.v5api.GenerateCSVResponse0\x01\x12<\n\x08SQLQuery\x12\x15.v5api.SQLQueryParams\x1a\x17.v5api.SQLQueryResponse0\x01\x62\x06proto3') + _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'btrdb_pb2', globals()) if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None - _MERGEPOLICY._serialized_start=5490 - _MERGEPOLICY._serialized_end=5550 + _MERGEPOLICY._serialized_start=6305 + _MERGEPOLICY._serialized_end=6365 _RAWVALUESPARAMS._serialized_start=22 _RAWVALUESPARAMS._serialized_end=103 _RAWVALUESRESPONSE._serialized_start=105 _RAWVALUESRESPONSE._serialized_end=230 - _ALIGNEDWINDOWSPARAMS._serialized_start=232 - _ALIGNEDWINDOWSPARAMS._serialized_end=338 - _ALIGNEDWINDOWSRESPONSE._serialized_start=341 - _ALIGNEDWINDOWSRESPONSE._serialized_end=472 - _WINDOWSPARAMS._serialized_start=474 - _WINDOWSPARAMS._serialized_end=583 - _WINDOWSRESPONSE._serialized_start=585 - _WINDOWSRESPONSE._serialized_end=709 - _STREAMINFOPARAMS._serialized_start=711 - _STREAMINFOPARAMS._serialized_end=815 - _STREAMINFORESPONSE._serialized_start=818 - _STREAMINFORESPONSE._serialized_end=956 - _STREAMDESCRIPTOR._serialized_start=959 - _STREAMDESCRIPTOR._serialized_end=1111 - _SETSTREAMANNOTATIONSPARAMS._serialized_start=1114 - _SETSTREAMANNOTATIONSPARAMS._serialized_end=1244 - _SETSTREAMANNOTATIONSRESPONSE._serialized_start=1246 - _SETSTREAMANNOTATIONSRESPONSE._serialized_end=1305 - _SETSTREAMTAGSPARAMS._serialized_start=1308 - _SETSTREAMTAGSPARAMS._serialized_end=1446 - _SETSTREAMTAGSRESPONSE._serialized_start=1448 - _SETSTREAMTAGSRESPONSE._serialized_end=1500 - _CREATEPARAMS._serialized_start=1502 - _CREATEPARAMS._serialized_end=1625 - _CREATERESPONSE._serialized_start=1627 - _CREATERESPONSE._serialized_end=1672 - _METADATAUSAGEPARAMS._serialized_start=1674 - _METADATAUSAGEPARAMS._serialized_end=1738 - _METADATAUSAGERESPONSE._serialized_start=1740 - _METADATAUSAGERESPONSE._serialized_end=1861 - _KEYCOUNT._serialized_start=1863 - _KEYCOUNT._serialized_end=1901 - _LISTCOLLECTIONSPARAMS._serialized_start=1903 - _LISTCOLLECTIONSPARAMS._serialized_end=1969 - _LISTCOLLECTIONSRESPONSE._serialized_start=1971 - _LISTCOLLECTIONSRESPONSE._serialized_end=2046 - _LOOKUPSTREAMSPARAMS._serialized_start=2049 - _LOOKUPSTREAMSPARAMS._serialized_end=2220 - _LOOKUPSTREAMSRESPONSE._serialized_start=2222 - _LOOKUPSTREAMSRESPONSE._serialized_end=2316 - _NEARESTPARAMS._serialized_start=2318 - _NEARESTPARAMS._serialized_end=2401 - _NEARESTRESPONSE._serialized_start=2403 - _NEARESTRESPONSE._serialized_end=2525 - _CHANGESPARAMS._serialized_start=2527 - _CHANGESPARAMS._serialized_end=2612 - _CHANGESRESPONSE._serialized_start=2614 - _CHANGESRESPONSE._serialized_end=2741 - _INSERTPARAMS._serialized_start=2743 - _INSERTPARAMS._serialized_end=2860 - _INSERTRESPONSE._serialized_start=2862 - _INSERTRESPONSE._serialized_end=2951 - _DELETEPARAMS._serialized_start=2953 - _DELETEPARAMS._serialized_end=3009 - _DELETERESPONSE._serialized_start=3011 - _DELETERESPONSE._serialized_end=3100 - _INFOPARAMS._serialized_start=3102 - _INFOPARAMS._serialized_end=3114 - _INFORESPONSE._serialized_start=3117 - _INFORESPONSE._serialized_end=3279 - _PROXYINFO._serialized_start=3281 - _PROXYINFO._serialized_end=3316 - _FAULTINJECTPARAMS._serialized_start=3318 - _FAULTINJECTPARAMS._serialized_end=3367 - _FAULTINJECTRESPONSE._serialized_start=3369 - _FAULTINJECTRESPONSE._serialized_end=3431 - _FLUSHPARAMS._serialized_start=3433 - _FLUSHPARAMS._serialized_end=3460 - _FLUSHRESPONSE._serialized_start=3462 - _FLUSHRESPONSE._serialized_end=3550 - _OBLITERATEPARAMS._serialized_start=3552 - _OBLITERATEPARAMS._serialized_end=3584 - _OBLITERATERESPONSE._serialized_start=3586 - _OBLITERATERESPONSE._serialized_end=3635 - _RAWPOINT._serialized_start=3637 - _RAWPOINT._serialized_end=3676 - _STATPOINT._serialized_start=3678 - _STATPOINT._serialized_end=3774 - _CHANGEDRANGE._serialized_start=3776 - _CHANGEDRANGE._serialized_end=3818 - _STATUS._serialized_start=3820 - _STATUS._serialized_end=3882 - _MASH._serialized_start=3885 - _MASH._serialized_end=4037 - _MEMBER._serialized_start=4040 - _MEMBER._serialized_end=4235 - _KEYOPTVALUE._serialized_start=4237 - _KEYOPTVALUE._serialized_end=4293 - _OPTVALUE._serialized_start=4295 - _OPTVALUE._serialized_end=4320 - _KEYVALUE._serialized_start=4322 - _KEYVALUE._serialized_end=4360 - _STREAMCSVCONFIG._serialized_start=4362 - _STREAMCSVCONFIG._serialized_end=4425 - _GENERATECSVPARAMS._serialized_start=4428 - _GENERATECSVPARAMS._serialized_end=4713 - _GENERATECSVPARAMS_QUERYTYPE._serialized_start=4641 - _GENERATECSVPARAMS_QUERYTYPE._serialized_end=4713 - _GENERATECSVRESPONSE._serialized_start=4715 - _GENERATECSVRESPONSE._serialized_end=4796 - _SQLQUERYPARAMS._serialized_start=4798 - _SQLQUERYPARAMS._serialized_end=4872 - _SQLQUERYRESPONSE._serialized_start=4874 - _SQLQUERYRESPONSE._serialized_end=4942 - _ROLE._serialized_start=4944 - _ROLE._serialized_end=4964 - _SETCOMPACTIONCONFIGPARAMS._serialized_start=4967 - _SETCOMPACTIONCONFIGPARAMS._serialized_end=5115 - _SETCOMPACTIONCONFIGRESPONSE._serialized_start=5117 - _SETCOMPACTIONCONFIGRESPONSE._serialized_end=5175 - _GETCOMPACTIONCONFIGPARAMS._serialized_start=5177 - _GETCOMPACTIONCONFIGPARAMS._serialized_end=5218 - _GETCOMPACTIONCONFIGRESPONSE._serialized_start=5221 - _GETCOMPACTIONCONFIGRESPONSE._serialized_end=5414 - _REDUCEDRESOLUTIONRANGE._serialized_start=5416 - _REDUCEDRESOLUTIONRANGE._serialized_end=5488 - _BTRDB._serialized_start=5553 - _BTRDB._serialized_end=6884 + _ARROWRAWVALUESRESPONSE._serialized_start=232 + _ARROWRAWVALUESRESPONSE._serialized_end=349 + _ARROWMULTIVALUESPARAMS._serialized_start=351 + _ARROWMULTIVALUESPARAMS._serialized_end=461 + _ARROWMULTIVALUESRESPONSE._serialized_start=463 + _ARROWMULTIVALUESRESPONSE._serialized_end=538 + _RAWPOINTVEC._serialized_start=540 + _RAWPOINTVEC._serialized_end=582 + _ALIGNEDWINDOWSPARAMS._serialized_start=584 + _ALIGNEDWINDOWSPARAMS._serialized_end=690 + _ALIGNEDWINDOWSRESPONSE._serialized_start=693 + _ALIGNEDWINDOWSRESPONSE._serialized_end=824 + _ARROWALIGNEDWINDOWSRESPONSE._serialized_start=826 + _ARROWALIGNEDWINDOWSRESPONSE._serialized_end=948 + _WINDOWSPARAMS._serialized_start=950 + _WINDOWSPARAMS._serialized_end=1059 + _WINDOWSRESPONSE._serialized_start=1061 + _WINDOWSRESPONSE._serialized_end=1185 + _ARROWWINDOWSRESPONSE._serialized_start=1187 + _ARROWWINDOWSRESPONSE._serialized_end=1302 + _STREAMINFOPARAMS._serialized_start=1304 + _STREAMINFOPARAMS._serialized_end=1408 + _STREAMINFORESPONSE._serialized_start=1411 + _STREAMINFORESPONSE._serialized_end=1549 + _STREAMDESCRIPTOR._serialized_start=1552 + _STREAMDESCRIPTOR._serialized_end=1704 + _SETSTREAMANNOTATIONSPARAMS._serialized_start=1707 + _SETSTREAMANNOTATIONSPARAMS._serialized_end=1837 + _SETSTREAMANNOTATIONSRESPONSE._serialized_start=1839 + _SETSTREAMANNOTATIONSRESPONSE._serialized_end=1898 + _SETSTREAMTAGSPARAMS._serialized_start=1901 + _SETSTREAMTAGSPARAMS._serialized_end=2039 + _SETSTREAMTAGSRESPONSE._serialized_start=2041 + _SETSTREAMTAGSRESPONSE._serialized_end=2093 + _CREATEPARAMS._serialized_start=2095 + _CREATEPARAMS._serialized_end=2218 + _CREATERESPONSE._serialized_start=2220 + _CREATERESPONSE._serialized_end=2265 + _METADATAUSAGEPARAMS._serialized_start=2267 + _METADATAUSAGEPARAMS._serialized_end=2331 + _METADATAUSAGERESPONSE._serialized_start=2333 + _METADATAUSAGERESPONSE._serialized_end=2454 + _KEYCOUNT._serialized_start=2456 + _KEYCOUNT._serialized_end=2494 + _LISTCOLLECTIONSPARAMS._serialized_start=2496 + _LISTCOLLECTIONSPARAMS._serialized_end=2562 + _LISTCOLLECTIONSRESPONSE._serialized_start=2564 + _LISTCOLLECTIONSRESPONSE._serialized_end=2639 + _LOOKUPSTREAMSPARAMS._serialized_start=2642 + _LOOKUPSTREAMSPARAMS._serialized_end=2813 + _LOOKUPSTREAMSRESPONSE._serialized_start=2815 + _LOOKUPSTREAMSRESPONSE._serialized_end=2909 + _NEARESTPARAMS._serialized_start=2911 + _NEARESTPARAMS._serialized_end=2994 + _NEARESTRESPONSE._serialized_start=2996 + _NEARESTRESPONSE._serialized_end=3118 + _CHANGESPARAMS._serialized_start=3120 + _CHANGESPARAMS._serialized_end=3205 + _CHANGESRESPONSE._serialized_start=3207 + _CHANGESRESPONSE._serialized_end=3334 + _ROUNDSPEC._serialized_start=3336 + _ROUNDSPEC._serialized_end=3371 + _INSERTPARAMS._serialized_start=3374 + _INSERTPARAMS._serialized_end=3527 + _ARROWINSERTPARAMS._serialized_start=3530 + _ARROWINSERTPARAMS._serialized_end=3675 + _INSERTRESPONSE._serialized_start=3677 + _INSERTRESPONSE._serialized_end=3766 + _DELETEPARAMS._serialized_start=3768 + _DELETEPARAMS._serialized_end=3824 + _DELETERESPONSE._serialized_start=3826 + _DELETERESPONSE._serialized_end=3915 + _INFOPARAMS._serialized_start=3917 + _INFOPARAMS._serialized_end=3929 + _INFORESPONSE._serialized_start=3932 + _INFORESPONSE._serialized_end=4094 + _PROXYINFO._serialized_start=4096 + _PROXYINFO._serialized_end=4131 + _FAULTINJECTPARAMS._serialized_start=4133 + _FAULTINJECTPARAMS._serialized_end=4182 + _FAULTINJECTRESPONSE._serialized_start=4184 + _FAULTINJECTRESPONSE._serialized_end=4246 + _FLUSHPARAMS._serialized_start=4248 + _FLUSHPARAMS._serialized_end=4275 + _FLUSHRESPONSE._serialized_start=4277 + _FLUSHRESPONSE._serialized_end=4365 + _OBLITERATEPARAMS._serialized_start=4367 + _OBLITERATEPARAMS._serialized_end=4399 + _OBLITERATERESPONSE._serialized_start=4401 + _OBLITERATERESPONSE._serialized_end=4450 + _RAWPOINT._serialized_start=4452 + _RAWPOINT._serialized_end=4491 + _STATPOINT._serialized_start=4493 + _STATPOINT._serialized_end=4589 + _CHANGEDRANGE._serialized_start=4591 + _CHANGEDRANGE._serialized_end=4633 + _STATUS._serialized_start=4635 + _STATUS._serialized_end=4697 + _MASH._serialized_start=4700 + _MASH._serialized_end=4852 + _MEMBER._serialized_start=4855 + _MEMBER._serialized_end=5050 + _KEYOPTVALUE._serialized_start=5052 + _KEYOPTVALUE._serialized_end=5108 + _OPTVALUE._serialized_start=5110 + _OPTVALUE._serialized_end=5135 + _KEYVALUE._serialized_start=5137 + _KEYVALUE._serialized_end=5175 + _STREAMCSVCONFIG._serialized_start=5177 + _STREAMCSVCONFIG._serialized_end=5240 + _GENERATECSVPARAMS._serialized_start=5243 + _GENERATECSVPARAMS._serialized_end=5528 + _GENERATECSVPARAMS_QUERYTYPE._serialized_start=5456 + _GENERATECSVPARAMS_QUERYTYPE._serialized_end=5528 + _GENERATECSVRESPONSE._serialized_start=5530 + _GENERATECSVRESPONSE._serialized_end=5611 + _SQLQUERYPARAMS._serialized_start=5613 + _SQLQUERYPARAMS._serialized_end=5687 + _SQLQUERYRESPONSE._serialized_start=5689 + _SQLQUERYRESPONSE._serialized_end=5757 + _ROLE._serialized_start=5759 + _ROLE._serialized_end=5779 + _SETCOMPACTIONCONFIGPARAMS._serialized_start=5782 + _SETCOMPACTIONCONFIGPARAMS._serialized_end=5930 + _SETCOMPACTIONCONFIGRESPONSE._serialized_start=5932 + _SETCOMPACTIONCONFIGRESPONSE._serialized_end=5990 + _GETCOMPACTIONCONFIGPARAMS._serialized_start=5992 + _GETCOMPACTIONCONFIGPARAMS._serialized_end=6033 + _GETCOMPACTIONCONFIGRESPONSE._serialized_start=6036 + _GETCOMPACTIONCONFIGRESPONSE._serialized_end=6229 + _REDUCEDRESOLUTIONRANGE._serialized_start=6231 + _REDUCEDRESOLUTIONRANGE._serialized_end=6303 + _BTRDB._serialized_start=6368 + _BTRDB._serialized_end=8083 # @@protoc_insertion_point(module_scope) diff --git a/btrdb/grpcinterface/btrdb_pb2.pyi b/btrdb/grpcinterface/btrdb_pb2.pyi index e9abf5b..ff9b6fc 100644 --- a/btrdb/grpcinterface/btrdb_pb2.pyi +++ b/btrdb/grpcinterface/btrdb_pb2.pyi @@ -36,6 +36,78 @@ class AlignedWindowsResponse(_message.Message): versionMinor: int def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., values: _Optional[_Iterable[_Union[StatPoint, _Mapping]]] = ...) -> None: ... +class ArrowAlignedWindowsResponse(_message.Message): + __slots__ = ["arrowBytes", "stat", "versionMajor", "versionMinor"] + ARROWBYTES_FIELD_NUMBER: _ClassVar[int] + STAT_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + arrowBytes: bytes + stat: Status + versionMajor: int + versionMinor: int + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., arrowBytes: _Optional[bytes] = ...) -> None: ... + +class ArrowInsertParams(_message.Message): + __slots__ = ["arrowBytes", "merge_policy", "rounding", "sync", "uuid"] + ARROWBYTES_FIELD_NUMBER: _ClassVar[int] + MERGE_POLICY_FIELD_NUMBER: _ClassVar[int] + ROUNDING_FIELD_NUMBER: _ClassVar[int] + SYNC_FIELD_NUMBER: _ClassVar[int] + UUID_FIELD_NUMBER: _ClassVar[int] + arrowBytes: bytes + merge_policy: MergePolicy + rounding: RoundSpec + sync: bool + uuid: bytes + def __init__(self, uuid: _Optional[bytes] = ..., sync: bool = ..., merge_policy: _Optional[_Union[MergePolicy, str]] = ..., rounding: _Optional[_Union[RoundSpec, _Mapping]] = ..., arrowBytes: _Optional[bytes] = ...) -> None: ... + +class ArrowMultiValuesParams(_message.Message): + __slots__ = ["end", "snapPeriodNs", "start", "uuid", "versionMajor"] + END_FIELD_NUMBER: _ClassVar[int] + SNAPPERIODNS_FIELD_NUMBER: _ClassVar[int] + START_FIELD_NUMBER: _ClassVar[int] + UUID_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + end: int + snapPeriodNs: int + start: int + uuid: _containers.RepeatedScalarFieldContainer[bytes] + versionMajor: _containers.RepeatedScalarFieldContainer[int] + def __init__(self, uuid: _Optional[_Iterable[bytes]] = ..., versionMajor: _Optional[_Iterable[int]] = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., snapPeriodNs: _Optional[int] = ...) -> None: ... + +class ArrowMultiValuesResponse(_message.Message): + __slots__ = ["arrowBytes", "stat"] + ARROWBYTES_FIELD_NUMBER: _ClassVar[int] + STAT_FIELD_NUMBER: _ClassVar[int] + arrowBytes: bytes + stat: Status + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., arrowBytes: _Optional[bytes] = ...) -> None: ... + +class ArrowRawValuesResponse(_message.Message): + __slots__ = ["arrowBytes", "stat", "versionMajor", "versionMinor"] + ARROWBYTES_FIELD_NUMBER: _ClassVar[int] + STAT_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + arrowBytes: bytes + stat: Status + versionMajor: int + versionMinor: int + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., arrowBytes: _Optional[bytes] = ...) -> None: ... + +class ArrowWindowsResponse(_message.Message): + __slots__ = ["arrowBytes", "stat", "versionMajor", "versionMinor"] + ARROWBYTES_FIELD_NUMBER: _ClassVar[int] + STAT_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + arrowBytes: bytes + stat: Status + versionMajor: int + versionMinor: int + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., arrowBytes: _Optional[bytes] = ...) -> None: ... + class ChangedRange(_message.Message): __slots__ = ["end", "start"] END_FIELD_NUMBER: _ClassVar[int] @@ -212,16 +284,18 @@ class InfoResponse(_message.Message): def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., mash: _Optional[_Union[Mash, _Mapping]] = ..., majorVersion: _Optional[int] = ..., minorVersion: _Optional[int] = ..., build: _Optional[str] = ..., proxy: _Optional[_Union[ProxyInfo, _Mapping]] = ...) -> None: ... class InsertParams(_message.Message): - __slots__ = ["merge_policy", "sync", "uuid", "values"] + __slots__ = ["merge_policy", "rounding", "sync", "uuid", "values"] MERGE_POLICY_FIELD_NUMBER: _ClassVar[int] + ROUNDING_FIELD_NUMBER: _ClassVar[int] SYNC_FIELD_NUMBER: _ClassVar[int] UUID_FIELD_NUMBER: _ClassVar[int] VALUES_FIELD_NUMBER: _ClassVar[int] merge_policy: MergePolicy + rounding: RoundSpec sync: bool uuid: bytes values: _containers.RepeatedCompositeFieldContainer[RawPoint] - def __init__(self, uuid: _Optional[bytes] = ..., sync: bool = ..., merge_policy: _Optional[_Union[MergePolicy, str]] = ..., values: _Optional[_Iterable[_Union[RawPoint, _Mapping]]] = ...) -> None: ... + def __init__(self, uuid: _Optional[bytes] = ..., sync: bool = ..., merge_policy: _Optional[_Union[MergePolicy, str]] = ..., rounding: _Optional[_Union[RoundSpec, _Mapping]] = ..., values: _Optional[_Iterable[_Union[RawPoint, _Mapping]]] = ...) -> None: ... class InsertResponse(_message.Message): __slots__ = ["stat", "versionMajor", "versionMinor"] @@ -412,6 +486,14 @@ class RawPoint(_message.Message): value: float def __init__(self, time: _Optional[int] = ..., value: _Optional[float] = ...) -> None: ... +class RawPointVec(_message.Message): + __slots__ = ["time", "value"] + TIME_FIELD_NUMBER: _ClassVar[int] + VALUE_FIELD_NUMBER: _ClassVar[int] + time: int + value: _containers.RepeatedScalarFieldContainer[float] + def __init__(self, time: _Optional[int] = ..., value: _Optional[_Iterable[float]] = ...) -> None: ... + class RawValuesParams(_message.Message): __slots__ = ["end", "start", "uuid", "versionMajor"] END_FIELD_NUMBER: _ClassVar[int] @@ -452,6 +534,12 @@ class Role(_message.Message): name: str def __init__(self, name: _Optional[str] = ...) -> None: ... +class RoundSpec(_message.Message): + __slots__ = ["bits"] + BITS_FIELD_NUMBER: _ClassVar[int] + bits: int + def __init__(self, bits: _Optional[int] = ...) -> None: ... + class SQLQueryParams(_message.Message): __slots__ = ["params", "query", "role"] PARAMS_FIELD_NUMBER: _ClassVar[int] diff --git a/btrdb/grpcinterface/btrdb_pb2_grpc.py b/btrdb/grpcinterface/btrdb_pb2_grpc.py index 0bac853..52a72bd 100644 --- a/btrdb/grpcinterface/btrdb_pb2_grpc.py +++ b/btrdb/grpcinterface/btrdb_pb2_grpc.py @@ -19,16 +19,36 @@ def __init__(self, channel): request_serializer=btrdb__pb2.RawValuesParams.SerializeToString, response_deserializer=btrdb__pb2.RawValuesResponse.FromString, ) + self.ArrowRawValues = channel.unary_stream( + '/v5api.BTrDB/ArrowRawValues', + request_serializer=btrdb__pb2.RawValuesParams.SerializeToString, + response_deserializer=btrdb__pb2.ArrowRawValuesResponse.FromString, + ) + self.ArrowMultiValues = channel.unary_stream( + '/v5api.BTrDB/ArrowMultiValues', + request_serializer=btrdb__pb2.ArrowMultiValuesParams.SerializeToString, + response_deserializer=btrdb__pb2.ArrowMultiValuesResponse.FromString, + ) self.AlignedWindows = channel.unary_stream( '/v5api.BTrDB/AlignedWindows', request_serializer=btrdb__pb2.AlignedWindowsParams.SerializeToString, response_deserializer=btrdb__pb2.AlignedWindowsResponse.FromString, ) + self.ArrowAlignedWindows = channel.unary_stream( + '/v5api.BTrDB/ArrowAlignedWindows', + request_serializer=btrdb__pb2.AlignedWindowsParams.SerializeToString, + response_deserializer=btrdb__pb2.ArrowAlignedWindowsResponse.FromString, + ) self.Windows = channel.unary_stream( '/v5api.BTrDB/Windows', request_serializer=btrdb__pb2.WindowsParams.SerializeToString, response_deserializer=btrdb__pb2.WindowsResponse.FromString, ) + self.ArrowWindows = channel.unary_stream( + '/v5api.BTrDB/ArrowWindows', + request_serializer=btrdb__pb2.WindowsParams.SerializeToString, + response_deserializer=btrdb__pb2.ArrowWindowsResponse.FromString, + ) self.StreamInfo = channel.unary_unary( '/v5api.BTrDB/StreamInfo', request_serializer=btrdb__pb2.StreamInfoParams.SerializeToString, @@ -74,6 +94,11 @@ def __init__(self, channel): request_serializer=btrdb__pb2.InsertParams.SerializeToString, response_deserializer=btrdb__pb2.InsertResponse.FromString, ) + self.ArrowInsert = channel.unary_unary( + '/v5api.BTrDB/ArrowInsert', + request_serializer=btrdb__pb2.ArrowInsertParams.SerializeToString, + response_deserializer=btrdb__pb2.InsertResponse.FromString, + ) self.Delete = channel.unary_unary( '/v5api.BTrDB/Delete', request_serializer=btrdb__pb2.DeleteParams.SerializeToString, @@ -125,18 +150,42 @@ def RawValues(self, request, context): context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') + def ArrowRawValues(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def ArrowMultiValues(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + def AlignedWindows(self, request, context): """Missing associated documentation comment in .proto file.""" context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') + def ArrowAlignedWindows(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + def Windows(self, request, context): """Missing associated documentation comment in .proto file.""" context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') + def ArrowWindows(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + def StreamInfo(self, request, context): """Missing associated documentation comment in .proto file.""" context.set_code(grpc.StatusCode.UNIMPLEMENTED) @@ -191,6 +240,12 @@ def Insert(self, request, context): context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') + def ArrowInsert(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + def Delete(self, request, context): """Missing associated documentation comment in .proto file.""" context.set_code(grpc.StatusCode.UNIMPLEMENTED) @@ -249,16 +304,36 @@ def add_BTrDBServicer_to_server(servicer, server): request_deserializer=btrdb__pb2.RawValuesParams.FromString, response_serializer=btrdb__pb2.RawValuesResponse.SerializeToString, ), + 'ArrowRawValues': grpc.unary_stream_rpc_method_handler( + servicer.ArrowRawValues, + request_deserializer=btrdb__pb2.RawValuesParams.FromString, + response_serializer=btrdb__pb2.ArrowRawValuesResponse.SerializeToString, + ), + 'ArrowMultiValues': grpc.unary_stream_rpc_method_handler( + servicer.ArrowMultiValues, + request_deserializer=btrdb__pb2.ArrowMultiValuesParams.FromString, + response_serializer=btrdb__pb2.ArrowMultiValuesResponse.SerializeToString, + ), 'AlignedWindows': grpc.unary_stream_rpc_method_handler( servicer.AlignedWindows, request_deserializer=btrdb__pb2.AlignedWindowsParams.FromString, response_serializer=btrdb__pb2.AlignedWindowsResponse.SerializeToString, ), + 'ArrowAlignedWindows': grpc.unary_stream_rpc_method_handler( + servicer.ArrowAlignedWindows, + request_deserializer=btrdb__pb2.AlignedWindowsParams.FromString, + response_serializer=btrdb__pb2.ArrowAlignedWindowsResponse.SerializeToString, + ), 'Windows': grpc.unary_stream_rpc_method_handler( servicer.Windows, request_deserializer=btrdb__pb2.WindowsParams.FromString, response_serializer=btrdb__pb2.WindowsResponse.SerializeToString, ), + 'ArrowWindows': grpc.unary_stream_rpc_method_handler( + servicer.ArrowWindows, + request_deserializer=btrdb__pb2.WindowsParams.FromString, + response_serializer=btrdb__pb2.ArrowWindowsResponse.SerializeToString, + ), 'StreamInfo': grpc.unary_unary_rpc_method_handler( servicer.StreamInfo, request_deserializer=btrdb__pb2.StreamInfoParams.FromString, @@ -304,6 +379,11 @@ def add_BTrDBServicer_to_server(servicer, server): request_deserializer=btrdb__pb2.InsertParams.FromString, response_serializer=btrdb__pb2.InsertResponse.SerializeToString, ), + 'ArrowInsert': grpc.unary_unary_rpc_method_handler( + servicer.ArrowInsert, + request_deserializer=btrdb__pb2.ArrowInsertParams.FromString, + response_serializer=btrdb__pb2.InsertResponse.SerializeToString, + ), 'Delete': grpc.unary_unary_rpc_method_handler( servicer.Delete, request_deserializer=btrdb__pb2.DeleteParams.FromString, @@ -371,6 +451,40 @@ def RawValues(request, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + @staticmethod + def ArrowRawValues(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream(request, target, '/v5api.BTrDB/ArrowRawValues', + btrdb__pb2.RawValuesParams.SerializeToString, + btrdb__pb2.ArrowRawValuesResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def ArrowMultiValues(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream(request, target, '/v5api.BTrDB/ArrowMultiValues', + btrdb__pb2.ArrowMultiValuesParams.SerializeToString, + btrdb__pb2.ArrowMultiValuesResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + @staticmethod def AlignedWindows(request, target, @@ -388,6 +502,23 @@ def AlignedWindows(request, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + @staticmethod + def ArrowAlignedWindows(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream(request, target, '/v5api.BTrDB/ArrowAlignedWindows', + btrdb__pb2.AlignedWindowsParams.SerializeToString, + btrdb__pb2.ArrowAlignedWindowsResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + @staticmethod def Windows(request, target, @@ -405,6 +536,23 @@ def Windows(request, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + @staticmethod + def ArrowWindows(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream(request, target, '/v5api.BTrDB/ArrowWindows', + btrdb__pb2.WindowsParams.SerializeToString, + btrdb__pb2.ArrowWindowsResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + @staticmethod def StreamInfo(request, target, @@ -558,6 +706,23 @@ def Insert(request, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + @staticmethod + def ArrowInsert(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/v5api.BTrDB/ArrowInsert', + btrdb__pb2.ArrowInsertParams.SerializeToString, + btrdb__pb2.InsertResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + @staticmethod def Delete(request, target, diff --git a/btrdb/stream.py b/btrdb/stream.py index 16db559..4f5bb79 100644 --- a/btrdb/stream.py +++ b/btrdb/stream.py @@ -10,39 +10,41 @@ """ Module for Stream and related classes """ - ########################################################################## ## Imports ########################################################################## - -import re import json +import logging +import re import uuid as uuidlib import warnings -from copy import deepcopy +from collections import deque from collections.abc import Sequence +from copy import deepcopy -from btrdb.utils.buffer import PointBuffer -from btrdb.point import RawPoint, StatPoint -from btrdb.transformers import StreamSetTransformer -from btrdb.utils.timez import currently_as_ns, to_nanoseconds -from btrdb.utils.conversion import AnnotationEncoder, AnnotationDecoder -from btrdb.utils.general import pointwidth as pw +import pyarrow as pa + +import btrdb.grpcinterface.btrdb_pb2_grpc from btrdb.exceptions import ( BTrDBError, BTRDBTypeError, BTRDBValueError, - InvalidOperation, InvalidCollection, - StreamNotFoundError, + InvalidOperation, NoSuchPoint, + StreamNotFoundError, ) - +from btrdb.point import RawPoint, StatPoint +from btrdb.transformers import _STAT_PROPERTIES, StreamSetTransformer +from btrdb.utils.buffer import PointBuffer +from btrdb.utils.conversion import AnnotationDecoder, AnnotationEncoder +from btrdb.utils.general import pointwidth as pw +from btrdb.utils.timez import currently_as_ns, to_nanoseconds ########################################################################## ## Module Variables ########################################################################## - +logger = logging.getLogger(__name__) INSERT_BATCH_SIZE = 50000 MINIMUM_TIME = -(16 << 56) MAXIMUM_TIME = (48 << 56) - 1 @@ -52,6 +54,8 @@ except Exception: RE_PATTERN = re.Pattern +_arrow_not_impl_str = "The BTrDB server you are using does not support {}." +_arrow_available_str = "Pyarrow and arrow table support is available for {}, you can access this method using {}." ########################################################################## ## Stream Classes @@ -463,14 +467,83 @@ def insert(self, data, merge="never"): The version of the stream after inserting new points. """ - i = 0 + if self._btrdb._ARROW_ENABLED: + warnings.warn(_arrow_available_str.format("insert", "arrow_insert")) version = 0 + i = 0 while i < len(data): thisBatch = data[i : i + INSERT_BATCH_SIZE] version = self._btrdb.ep.insert(self._uuid, thisBatch, merge) i += INSERT_BATCH_SIZE return version + def arrow_insert(self, data: pa.Table, merge: str = "never") -> int: + """ + Insert new data in the form of a pyarrow Table with (time, value) columns. + + Inserts a table of new (time, value) columns into the stream. The values + in the table need not be sorted by time. If the arrays are larger than + appropriate, this function will automatically chunk the inserts. As a + consequence, the insert is not necessarily atomic, but can be used with + a very large array. + + Parameters + ---------- + data: pyarrow.Table, required + A pyarrow table with a schema of time:Timestamp[ns, tz=UTC], value:float64 + This schema will be validated and converted if necessary. + merge: str + A string describing the merge policy. Valid policies are: + - 'never': the default, no points are merged + - 'equal': points are deduplicated if the time and value are equal + - 'retain': if two points have the same timestamp, the old one is kept + - 'replace': if two points have the same timestamp, the new one is kept + + Returns + ------- + int + The version of the stream after inserting new points. + + """ + if not self._btrdb._ARROW_ENABLED: + raise NotImplementedError(_arrow_not_impl_str.format("arrow_insert")) + chunksize = INSERT_BATCH_SIZE + assert isinstance(data, pa.Table) + tmp_table = data.rename_columns(["time", "value"]) + logger.debug(f"tmp_table schema: {tmp_table.schema}") + new_schema = pa.schema( + [ + (pa.field("time", pa.timestamp(unit="ns", tz="UTC"))), + (pa.field("value", pa.float64())), + ] + ) + tmp_table = tmp_table.cast(new_schema) + num_rows = tmp_table.num_rows + + # Calculate the number of batches based on the chunk size + num_batches = num_rows // chunksize + if num_rows % chunksize != 0 or num_batches == 0: + num_batches = num_batches + 1 + + table_slices = [] + + for i in range(num_batches): + start_idx = i * chunksize + t = tmp_table.slice(offset=start_idx, length=chunksize) + table_slices.append(t) + + # Process the batches as needed + version = [] + for tab in table_slices: + logger.debug(f"Table Slice: {tab}") + feather_bytes = _table_slice_to_feather_bytes(table_slice=tab) + version.append( + self._btrdb.ep.arrowInsertValues( + uu=self.uuid, values=feather_bytes, policy=merge + ) + ) + return max(version) + def _update_tags_collection(self, tags, collection): tags = self.tags() if tags is None else tags collection = self.collection if collection is None else collection @@ -651,16 +724,64 @@ def values(self, start, end, version=0): the vector nodes. """ + if self._btrdb._ARROW_ENABLED: + warnings.warn(_arrow_available_str.format("values", "arrow_values")) materialized = [] start = to_nanoseconds(start) end = to_nanoseconds(end) - + logger.debug(f"For stream - {self.uuid} - {self.name}") point_windows = self._btrdb.ep.rawValues(self._uuid, start, end, version) for point_list, version in point_windows: for point in point_list: materialized.append((RawPoint.from_proto(point), version)) return materialized + def arrow_values(self, start: int, end: int, version: int = 0) -> pa.Table: + """Read raw values from BTrDB between time [a, b) in nanoseconds. + + RawValues queries BTrDB for the raw time series data points between + `start` and `end` time, both in nanoseconds since the Epoch for the + specified stream `version`. + + start : int or datetime like object + The start time in nanoseconds for the range to be queried. (see + :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) + end : int or datetime like object + The end time in nanoseconds for the range to be queried. (see + :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) + version: int + The version of the stream to be queried + + Returns + ------ + pyarrow.Table + A pyarrow table of the raw values with time and value columns. + + + Notes + ----- + Note that the raw data points are the original values at the sensor's + native sampling rate (assuming the time series represents measurements + from a sensor). This is the lowest level of data with the finest time + granularity. In the tree data structure of BTrDB, this data is stored in + the vector nodes. + """ + if not self._btrdb._ARROW_ENABLED: + raise NotImplementedError(_arrow_not_impl_str.format("arrow_values")) + start = to_nanoseconds(start) + end = to_nanoseconds(end) + logger.debug(f"For stream - {self.uuid} - {self.name}") + arr_bytes = self._btrdb.ep.arrowRawValues( + uu=self.uuid, start=start, end=end, version=version + ) + # exhausting the generator from above + bytes_materialized = list(arr_bytes) + + logger.debug(f"Length of materialized list: {len(bytes_materialized)}") + logger.debug(f"materialized bytes[0:1]: {bytes_materialized[0:1]}") + materialized_tables = _materialize_stream_as_table(bytes_materialized) + return materialized_tables.rename_columns(["time", str(self.uuid)]) + def aligned_windows(self, start, end, pointwidth, version=0): """ Read statistical aggregates of windows of data from BTrDB. @@ -704,19 +825,86 @@ def aligned_windows(self, start, end, pointwidth, version=0): As the window-width is a power-of-two, it aligns with BTrDB internal tree data structure and is faster to execute than `windows()`. """ + if self._btrdb._ARROW_ENABLED: + warnings.warn( + _arrow_available_str.format("aligned_windows", "arrow_aligned_windows") + ) materialized = [] start = to_nanoseconds(start) end = to_nanoseconds(end) - windows = self._btrdb.ep.alignedWindows( self._uuid, start, end, pointwidth, version ) for stat_points, version in windows: for point in stat_points: materialized.append((StatPoint.from_proto(point), version)) - return tuple(materialized) + def arrow_aligned_windows( + self, start: int, end: int, pointwidth: int, version: int = 0 + ) -> pa.Table: + """Read statistical aggregates of windows of data from BTrDB. + + Query BTrDB for aggregates (or roll ups or windows) of the time series + with `version` between time `start` (inclusive) and `end` (exclusive) in + nanoseconds [start, end). Each point returned is a statistical aggregate of all the + raw data within a window of width 2**`pointwidth` nanoseconds. These + statistical aggregates currently include the mean, minimum, and maximum + of the data and the count of data points composing the window. + + Note that `start` is inclusive, but `end` is exclusive. That is, results + will be returned for all windows that start in the interval [start, end). + If end < start+2^pointwidth you will not get any results. If start and + end are not powers of two, the bottom pointwidth bits will be cleared. + Each window will contain statistical summaries of the window. + Statistical points with count == 0 will be omitted. + + Parameters + ---------- + start : int or datetime like object + The start time in nanoseconds for the range to be queried. (see + :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) + end : int or datetime like object + The end time in nanoseconds for the range to be queried. (see + :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) + pointwidth : int + Specify the number of ns between data points (2**pointwidth) + version : int + Version of the stream to query + + Returns + ------- + pyarrow.Table + Returns a pyarrow table containing the windows of data. + + Notes + ----- + As the window-width is a power-of-two, it aligns with BTrDB internal + tree data structure and is faster to execute than `windows()`. + """ + if not self._btrdb._ARROW_ENABLED: + raise NotImplementedError( + _arrow_not_impl_str.format("arrow_aligned_windows") + ) + + logger.debug(f"For stream - {self.uuid} - {self.name}") + start = to_nanoseconds(start) + end = to_nanoseconds(end) + arr_bytes = self._btrdb.ep.arrowAlignedWindows( + self.uuid, start=start, end=end, pointwidth=pointwidth, version=version + ) + # exhausting the generator from above + bytes_materialized = list(arr_bytes) + + logger.debug(f"Length of materialized list: {len(bytes_materialized)}") + logger.debug(f"materialized bytes[0:1]: {bytes_materialized[0:1]}") + # ignore versions for now + materialized_table = _materialize_stream_as_table(bytes_materialized) + stream_names = [ + "/".join([self.collection, self.name, prop]) for prop in _STAT_PROPERTIES + ] + return materialized_table.rename_columns(["time", *stream_names]) + def windows(self, start, end, width, depth=0, version=0): """ Read arbitrarily-sized windows of data from BTrDB. StatPoint objects @@ -757,10 +945,11 @@ def windows(self, start, end, width, depth=0, version=0): for depth is now 0. """ + if self._btrdb._ARROW_ENABLED: + warnings.warn(_arrow_available_str.format("windows", "arrow_windows")) materialized = [] start = to_nanoseconds(start) end = to_nanoseconds(end) - windows = self._btrdb.ep.windows(self._uuid, start, end, width, depth, version) for stat_points, version in windows: for point in stat_points: @@ -768,6 +957,67 @@ def windows(self, start, end, width, depth=0, version=0): return tuple(materialized) + def arrow_windows( + self, start: int, end: int, width: int, version: int = 0 + ) -> pa.Table: + """Read arbitrarily-sized windows of data from BTrDB. + + Parameters + ---------- + start : int or datetime like object, required + The start time in nanoseconds for the range to be queried. (see + :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) + end : int or datetime like object, required + The end time in nanoseconds for the range to be queried. (see + :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) + width : int, required + The number of nanoseconds in each window. + version : int, default=0, optional + The version of the stream to query. + + Returns + ------- + pyarrow.Table + Returns a pyarrow Table containing windows of data. + + Notes + ----- + Windows returns arbitrary precision windows from BTrDB. It is slower + than AlignedWindows, but still significantly faster than RawValues. Each + returned window will be `width` nanoseconds long. `start` is inclusive, + but `end` is exclusive (e.g if end < start+width you will get no + results). That is, results will be returned for all windows that start + at a time less than the end timestamp. If (`end` - `start`) is not a + multiple of width, then end will be decreased to the greatest value less + than end such that (end - start) is a multiple of `width` (i.e., we set + end = start + width * floordiv(end - start, width). The `depth` + parameter previously available has been deprecated. The only valid value + for depth is now 0. + """ + if not self._btrdb._ARROW_ENABLED: + raise NotImplementedError(_arrow_not_impl_str.format("arrow_windows")) + start = to_nanoseconds(start) + end = to_nanoseconds(end) + arr_bytes = self._btrdb.ep.arrowWindows( + self.uuid, + start=start, + end=end, + width=width, + depth=0, + version=version, + ) + # exhausting the generator from above + bytes_materialized = list(arr_bytes) + + logger.debug(f"Length of materialized list: {len(bytes_materialized)}") + logger.debug(f"materialized bytes[0:1]: {bytes_materialized[0:1]}") + # ignore versions for now + materialized = _materialize_stream_as_table(bytes_materialized) + stream_names = [ + "/".join([self.collection, self.name, prop]) for prop in _STAT_PROPERTIES + ] + return materialized.rename_columns(["time", *stream_names]) + def nearest(self, time, version, backward=False): """ Finds the closest point in the stream to a specified time. @@ -796,9 +1046,11 @@ def nearest(self, time, version, backward=False): """ try: + logger.debug(f"checking nearest for: {self.uuid}\t\t{time}\t\t{version}") rp, version = self._btrdb.ep.nearest( self._uuid, to_nanoseconds(time), version, backward ) + logger.debug(f"Nearest for stream: {self.uuid} - {rp}") except BTrDBError as exc: if not isinstance(exc, NoSuchPoint): raise @@ -842,6 +1094,10 @@ class StreamSetBase(Sequence): def __init__(self, streams): self._streams = streams + if len(self._streams) < 1: + raise ValueError( + f"Trying to create streamset with an empty list of streams {self._streams}." + ) try: self._btrdb = self._streams[0]._btrdb except Exception as e: @@ -851,11 +1107,11 @@ def __init__(self, streams): self.filters = [] self.pointwidth = None self.width = None - self.depth = None + self.depth = 0 @property def allow_window(self): - return not bool(self.pointwidth or (self.width and self.depth)) + return not bool(self.pointwidth or (self.width and self.depth == 0)) def _latest_versions(self): uuid_ver_tups = self._btrdb._executor.map( @@ -913,7 +1169,7 @@ def versions(self): self._pinned_versions if self._pinned_versions else self._latest_versions() ) - def count(self): + def count(self, precise:bool=False): """ Compute the total number of points in the streams using filters. @@ -924,13 +1180,12 @@ def count(self): Note that this helper method sums the counts of all StatPoints returned by ``aligned_windows``. Because of this the start and end timestamps - may be adjusted if they are not powers of 2. You can also set the - pointwidth property for smaller windows of time to ensure that the - count granularity is captured appropriately. + may be adjusted if they are not powers of 2. Parameters ---------- - None + precise : bool, default = False + Use statpoint counts using aligned_windows which trades accuracy for speed. Returns ------- @@ -940,14 +1195,15 @@ def count(self): params = self._params_from_filters() start = params.get("start", MINIMUM_TIME) end = params.get("end", MAXIMUM_TIME) - versions = self._pinned_versions if self._pinned_versions else {} + + versions = self._pinned_versions if self._pinned_versions else self._latest_versions() my_counts_gen = self._btrdb._executor.map( - lambda s: s.count(start, end, version=versions.get(s.uuid, 0)), + lambda s: s.count(start, end, version=versions.get(s.uuid, 0), precise=precise), self._streams, ) - return sum(my_counts_gen) + return sum(list(my_counts_gen)) def earliest(self): """ @@ -967,6 +1223,7 @@ def earliest(self): params = self._params_from_filters() start = params.get("start", MINIMUM_TIME) versions = self.versions() + logger.debug(f"versions: {versions}") earliest_points_gen = self._btrdb._executor.map( lambda s: s.nearest(start, version=versions.get(s.uuid, 0), backward=False), self._streams, @@ -1006,13 +1263,9 @@ def latest(self): def current(self): """ Returns the points of data in the streams closest to the current timestamp. If - the current timestamp is outside of the filtered range of data, a ValueError is + the current timestamp is outside the filtered range of data, a ValueError is raised. - Parameters - ---------- - None - Returns ------- tuple @@ -1048,6 +1301,7 @@ def filter( unit=None, tags=None, annotations=None, + sampling_frequency=None, ): """ Provides a new StreamSet instance containing stored query parameters and @@ -1083,17 +1337,29 @@ def filter( key/value pairs for filtering streams based on tags annotations : dict key/value pairs for filtering streams based on annotations + sampling_frequency : int + The sampling frequency of the data streams in Hz, set this if you want timesnapped values. Returns ------- StreamSet a new instance cloned from the original with filters applied + Notes + ----- + If you set `sampling_frequency` to a non-zero value, the stream data returned will be aligned to a + grid of timestamps based on the period of the sampling frequency. For example, a sampling rate of 30hz will + have a sampling period of 1/30hz -> ~33_333_333 ns per sample. Leave sampling_frequency as None, or set to 0 to + prevent time alignment. You should **not** use aligned data for frequency-based analysis. """ obj = self.clone() - if start is not None or end is not None: - obj.filters.append(StreamFilter(start, end)) + if start is not None or end is not None or sampling_frequency is not None: + obj.filters.append( + StreamFilter( + start=start, end=end, sampling_frequency=sampling_frequency + ) + ) # filter by collection if collection is not None: @@ -1182,7 +1448,7 @@ def clone(self): setattr(clone, attr, deepcopy(val)) return clone - def windows(self, width, depth): + def windows(self, width, depth=0): """ Stores the request for a windowing operation when the query is eventually materialized. @@ -1221,7 +1487,11 @@ def windows(self, width, depth): # TODO: refactor keeping in mind how exception is raised self.width = int(width) - self.depth = int(depth) + if depth != 0: + warnings.warn( + f"The only valid value for `depth` is now 0. You provided: {depth}. Overriding this value to 0." + ) + self.depth = 0 return self def aligned_windows(self, pointwidth): @@ -1269,6 +1539,8 @@ def _streamset_data(self, as_iterators=False): Returns each single stream's data as an iterator. Defaults to False. """ params = self._params_from_filters() + # sampling freq not supported for non-arrow streamset ops + _ = params.pop("sampling_frequency", None) versions = self.versions() if self.pointwidth is not None: @@ -1305,6 +1577,85 @@ def _streamset_data(self, as_iterators=False): return data + def _arrow_streamset_data(self): + params = self._params_from_filters() + versions = self.versions() + if params.get("sampling_frequency", None) is None: + _ = params.pop("sampling_frequency", None) + + if self.pointwidth is not None: + # create list of stream.aligned_windows data + params.update({"pointwidth": self.pointwidth}) + _ = params.pop("sampling_frequency", None) + # need to update params based on version of stream, use dict merge + # {**dict1, **dict2} creates a new dict which updates the key/values if there are any + # same keys in dict1 and dict2, then the second dict in the expansion (dict2 in this case) + # will update the key (similar to dict1.update(dict2)) + aligned_windows_gen = self._btrdb._executor.map( + lambda s: s.arrow_aligned_windows( + **{**params, **{"version": versions[s.uuid]}} + ), + self._streams, + ) + data = list(aligned_windows_gen) + tablex = data.pop() + if data: + for tab in data: + tablex = tablex.join(tab, "time", join_type="full outer") + data = tablex + else: + data = tablex + + elif self.width is not None and self.depth is not None: + # create list of stream.windows data (the windows method should + # prevent the possibility that only one of these is None) + _ = params.pop("sampling_frequency", None) + params.update({"width": self.width}) + windows_gen = self._btrdb._executor.map( + lambda s: s.arrow_windows( + **{**params, **{"version": versions[s.uuid]}} + ), + self._streams, + ) + data = list(windows_gen) + tablex = data.pop() + if data: + for tab in data: + tablex = tablex.join(tab, "time", join_type="full outer") + data = tablex + else: + data = tablex + + else: + # determine if we are aligning data or not + sampling_freq = params.get("sampling_frequency", None) + if sampling_freq is None: + sampling_freq = -1 + if sampling_freq > 0: + # We are getting timesnapped data + data = self._arrow_multivalues(period_ns=_to_period_ns(sampling_freq)) + elif sampling_freq == 0: + # outer-joined on time data + data = self._arrow_multivalues(period_ns=0) + else: + # getting raw value data from each stream multithreaded + # create list of stream.values + values_gen = self._btrdb._executor.map( + lambda s: s.arrow_values(**params), self._streams + ) + data = list(values_gen) + tables = deque(data) + main_table = tables.popleft() + idx = 0 + while len(tables) != 0: + idx = idx + 1 + t2 = tables.popleft() + main_table = main_table.join( + t2, "time", join_type="full outer", right_suffix=f"_{idx}" + ) + data = main_table + return data + def rows(self): """ Returns a materialized list of tuples where each tuple contains the @@ -1349,6 +1700,100 @@ def rows(self): return result + def arrow_rows(self): + """Return tuples of rows from arrow table""" + raise NotImplementedError( + f"arrow_rows has not been implemented yet, please use `rows` if you need this functionality." + ) + + def insert(self, data_map: dict, merge: str = "never") -> dict: + """Insert new data in the form (time, value) into their mapped streams. + + The times in the dataframe need not be sorted by time. If the point counts are larger than + appropriate, this function will automatically chunk the inserts. As a + consequence, the insert is not necessarily atomic, but can be used with + a very large array. + + Parameters + ---------- + data_map: dict[uuid, pandas.DataFrame] + A dictionary mapping stream uuids to insert data into and their value as a + pandas dataframe containing two columns, one named "time" which contains int64 utc+0 timestamps + and a "value" column containing float64 measurements. These columns will be typecast into these types. + merge: str + A string describing the merge policy. Valid policies are: + - 'never': the default, no points are merged + - 'equal': points are deduplicated if the time and value are equal + - 'retain': if two points have the same timestamp, the old one is kept + - 'replace': if two points have the same timestamp, the new one is kept + + Notes + ----- + You MUST convert your datetimes into utc+0 yourself. BTrDB expects utc+0 datetimes. + + Returns + ------- + dict[uuid, int] + The versions of the stream after inserting new points. + """ + if self._btrdb._ARROW_ENABLED: + warnings.warn(_arrow_available_str.format("insert", "arrow_insert")) + filtered_data_map = {s.uuid: data_map[s.uuid] for s in self._streams} + for key, dat in filtered_data_map.items(): + data_list = [ + (times, vals) + for times, vals in zip( + dat["time"].astype(int).values, dat["value"].astype(float).values + ) + ] + filtered_data_map[key] = data_list + versions_gen = self._btrdb._executor.map( + lambda s: (s.uuid, s.insert(data=filtered_data_map[s.uuid], merge=merge)), + self._streams, + ) + versions = {uu: ver for uu, ver in versions_gen} + return versions + + def arrow_insert(self, data_map: dict, merge: str = "never") -> dict: + """Insert new data in the form (time, value) into their mapped streams using pyarrow tables. + + The times in the arrow table need not be sorted by time. If the point counts are larger than + appropriate, this function will automatically chunk the inserts. As a + consequence, the insert is not necessarily atomic, but can be used with + a very large array. + + Parameters + ---------- + data_map: dict[uuid, pyarrow.Table] + A dictionary keyed on stream uuids and mapped to pyarrow tables with a schema + of time:Timestamp[ns, tz=UTC], value:float64. This schema will be validated and converted if necessary. + merge: str + A string describing the merge policy. Valid policies are: + - 'never': the default, no points are merged + - 'equal': points are deduplicated if the time and value are equal + - 'retain': if two points have the same timestamp, the old one is kept + - 'replace': if two points have the same timestamp, the new one is kept + + Notes + ----- + BTrDB expects datetimes to be in UTC+0. + + Returns + ------- + dict[uuid, int] + The versions of the stream after inserting new points. + """ + filtered_data_map = {s.uuid: data_map[s.uuid] for s in self._streams} + versions_gen = self._btrdb._executor.map( + lambda s: ( + s.uuid, + s.arrow_insert(data=filtered_data_map[s.uuid], merge=merge), + ), + self._streams, + ) + versions = {uu: ver for uu, ver in versions_gen} + return versions + def _params_from_filters(self): params = {} for filter in self.filters: @@ -1356,6 +1801,8 @@ def _params_from_filters(self): params["start"] = filter.start if filter.end is not None: params["end"] = filter.end + if filter.sampling_frequency is not None: + params["sampling_frequency"] = filter.sampling_frequency return params def values_iter(self): @@ -1372,9 +1819,32 @@ def values(self): streamset_data = self._streamset_data() for stream_data in streamset_data: result.append([point[0] for point in stream_data]) - return result + def arrow_values(self): + """Return a pyarrow table of stream values based on the streamset parameters.""" + streamset_data = self._arrow_streamset_data() + return streamset_data + + def _arrow_multivalues(self, period_ns: int): + if not self._btrdb._ARROW_ENABLED: + raise NotImplementedError( + _arrow_not_impl_str.format("arrow_multirawvalues") + ) + params = self._params_from_filters() + versions = self.versions() + params["uu_list"] = [s.uuid for s in self._streams] + params["versions"] = [versions[s.uuid] for s in self._streams] + params["snap_periodNS"] = period_ns + arr_bytes = self._btrdb.ep.arrowMultiValues(**params) + # exhausting the generator from above + bytes_materialized = list(arr_bytes) + + logger.debug(f"Length of materialized list: {len(bytes_materialized)}") + logger.debug(f"materialized bytes[0:1]: {bytes_materialized[0:1]}") + data = _materialize_stream_as_table(bytes_materialized) + return data + def __repr__(self): token = "stream" if len(self) == 1 else "streams" return "<{}({} {})>".format(self.__class__.__name__, len(self._streams), token) @@ -1419,12 +1889,55 @@ class StreamFilter(object): Object for storing requested filtering options """ - def __init__(self, start=None, end=None): + def __init__( + self, start: int = None, end: int = None, sampling_frequency: int = None + ): self.start = to_nanoseconds(start) if start else None self.end = to_nanoseconds(end) if end else None + self.sampling_frequency = ( + int(sampling_frequency) if sampling_frequency else None + ) if self.start is None and self.end is None: raise BTRDBValueError("A valid `start` or `end` must be supplied") if self.start is not None and self.end is not None and self.start >= self.end: raise BTRDBValueError("`start` must be strictly less than `end` argument") + + +def _to_period_ns(fs: int): + """Convert sampling rate to sampling period in ns.""" + period = 1 / fs + period_ns = period * 1e9 + return int(period_ns) + + +def _materialize_stream_as_table(arrow_bytes): + table_list = [] + for b, _ in arrow_bytes: + with pa.ipc.open_stream(b) as reader: + schema = reader.schema + logger.debug(f"schema: {schema}") + table_list.append(reader.read_all()) + logger.debug(f"table list: {table_list}") + table = pa.concat_tables(table_list) + return table + + +def _table_slice_to_feather_bytes(table_slice: pa.Table) -> bytes: + sink = pa.BufferOutputStream() + with pa.ipc.new_stream(sink=sink, schema=table_slice.schema) as writer: + writer.write_table(table_slice) + return sink.readall() + + +def _coalesce_table_deque(tables: deque): + main_table = tables.popleft() + idx = 0 + while len(tables) != 0: + idx = idx + 1 + t2 = tables.popleft() + main_table = main_table.join( + t2, "time", join_type="full outer", right_suffix=f"_{idx}" + ) + return main_table diff --git a/btrdb/transformers.py b/btrdb/transformers.py index a3a3f94..6f61d49 100644 --- a/btrdb/transformers.py +++ b/btrdb/transformers.py @@ -20,15 +20,20 @@ from collections import OrderedDict from warnings import warn +import pandas as pd + + ########################################################################## ## Helper Functions ########################################################################## -_STAT_PROPERTIES = ('min', 'mean', 'max', 'count', 'stddev') +_STAT_PROPERTIES = ("min", "mean", "max", "count", "stddev") + def _get_time_from_row(row): for item in row: - if item: return item.time + if item: + return item.time raise Exception("Row contains no data") @@ -38,15 +43,14 @@ def _stream_names(streamset, func): before sending a collection of streams (dataframe, etc.) back to the user. """ - return tuple( - func(s) for s in streamset._streams - ) + return tuple(func(s) for s in streamset._streams) ########################################################################## ## Transform Functions ########################################################################## + def to_series(streamset, datetime64_index=True, agg="mean", name_callable=None): """ Returns a list of Pandas Series objects indexed by time @@ -63,7 +67,7 @@ def to_series(streamset, datetime64_index=True, agg="mean", name_callable=None): argument is ignored if RawPoint values are passed into the function. name_callable : lambda, default: lambda s: s.collection + "/" + s.name - Sprecify a callable that can be used to determine the series name given a + Specify a callable that can be used to determine the series name given a Stream object. """ @@ -77,8 +81,7 @@ def to_series(streamset, datetime64_index=True, agg="mean", name_callable=None): raise AttributeError("cannot use 'all' as aggregate at this time") if not callable(name_callable): - name_callable = lambda s: s.collection + "/" + s.name - + name_callable = lambda s: s.collection + "/" + s.name result = [] stream_names = _stream_names(streamset, name_callable) @@ -93,14 +96,121 @@ def to_series(streamset, datetime64_index=True, agg="mean", name_callable=None): values.append(getattr(point, agg)) if datetime64_index: - times = pd.Index(times, dtype='datetime64[ns]') + times = pd.Index(times, dtype="datetime64[ns]") - result.append(pd.Series( - data=values, index=times, name=stream_names[idx] - )) + result.append(pd.Series(data=values, index=times, name=stream_names[idx])) return result +def arrow_to_series(streamset, agg="mean", name_callable=None): + """ + Returns a list of Pandas Series objects indexed by time + + Parameters + ---------- + agg : str, default: "mean" + Specify the StatPoint field (e.g. aggregating function) to create the Series + from. Must be one of "min", "mean", "max", "count", or "stddev". This + argument is ignored if RawPoint values are passed into the function. + + name_callable : lambda, default: lambda s: s.collection + "/" + s.name + Specify a callable that can be used to determine the series name given a + Stream object. + """ + if not streamset._btrdb._ARROW_ENABLED: + raise NotImplementedError( + f"arrow_to_series requires an arrow-enabled BTrDB server." + ) + arrow_df = arrow_to_dataframe( + streamset=streamset, agg=agg, name_callable=name_callable + ) + return [arrow_df[col] for col in arrow_df] + + +def arrow_to_dataframe( + streamset, columns=None, agg="mean", name_callable=None +) -> pd.DataFrame: + """ + Returns a Pandas DataFrame object indexed by time and using the values of a + stream for each column. + + Parameters + ---------- + columns: sequence + column names to use for DataFrame. Deprecated and not compatible with name_callable. + + agg : str, default: "mean" + Specify the StatPoint field (e.g. aggregating function) to create the Series + from. Must be one of "min", "mean", "max", "count", "stddev", or "all". This + argument is ignored if not using StatPoints. + + name_callable : lambda, default: lambda s: s.collection + "/" + s.name + Specify a callable that can be used to determine the series name given a + Stream object. This is not compatible with agg == "all" at this time + """ + if not streamset._btrdb._ARROW_ENABLED: + raise NotImplementedError( + f"arrow_to_dataframe requires an arrow-enabled BTrDB server." + ) + + try: + import pandas as pd + import pyarrow as pa + except ImportError as err: + raise ImportError( + f"Please install Pandas and pyarrow to use this transformation function. ErrorMessage: {err}" + ) + # deprecation warning added in v5.8 + if columns: + warn( + "the columns argument is deprecated and will be removed in a future release", + DeprecationWarning, + stacklevel=2, + ) + + # TODO: allow this at some future point + if agg == "all" and name_callable is not None: + raise AttributeError( + "cannot provide name_callable when using 'all' as aggregate at this time" + ) + + # do not allow agg="all" with RawPoints + if agg == "all" and streamset.allow_window: + agg = "" + + # default arg values + if not callable(name_callable): + name_callable = lambda s: s.collection + "/" + s.name + tmp_table = streamset.arrow_values() + my_cols = [c for c in tmp_table.column_names] + col_names = _stream_names(streamset, name_callable) + cols = [] + for name in col_names: + for prop in _STAT_PROPERTIES: + cols.append(name + "/" + prop) + if agg == "all": + tmp = tmp_table.select(["time", *cols]) + elif not streamset.allow_window: + usable_cols = [val for val in cols if agg in val] + tmp = tmp_table.select(["time", *usable_cols]) + else: + tmp = tmp_table + df = tmp.to_pandas() + if not df.empty: + df = df.set_index("time") + + if agg == "all" and not streamset.allow_window: + stream_names = [ + [s.collection, s.name, prop] + for s in streamset._streams + for prop in _STAT_PROPERTIES + ] + df.columns = pd.MultiIndex.from_tuples(stream_names) + else: + df.columns = columns if columns else _stream_names(streamset, name_callable) + return df + + def to_dataframe(streamset, columns=None, agg="mean", name_callable=None): """ Returns a Pandas DataFrame object indexed by time and using the values of a @@ -117,7 +227,7 @@ def to_dataframe(streamset, columns=None, agg="mean", name_callable=None): argument is ignored if not using StatPoints. name_callable : lambda, default: lambda s: s.collection + "/" + s.name - Sprecify a callable that can be used to determine the series name given a + Specify a callable that can be used to determine the series name given a Stream object. This is not compatible with agg == "all" at this time @@ -129,35 +239,144 @@ def to_dataframe(streamset, columns=None, agg="mean", name_callable=None): # deprecation warning added in v5.8 if columns: - warn("the columns argument is deprecated and will be removed in a future release", DeprecationWarning, stacklevel=2) + warn( + "the columns argument is deprecated and will be removed in a future release", + DeprecationWarning, + stacklevel=2, + ) # TODO: allow this at some future point if agg == "all" and name_callable is not None: - raise AttributeError("cannot provide name_callable when using 'all' as aggregate at this time") + raise AttributeError( + "cannot provide name_callable when using 'all' as aggregate at this time" + ) # do not allow agg="all" with RawPoints if agg == "all" and streamset.allow_window: - agg="" + agg = "" # default arg values if not callable(name_callable): - name_callable = lambda s: s.collection + "/" + s.name - + name_callable = lambda s: s.collection + "/" + s.name - df = pd.DataFrame(to_dict(streamset,agg=agg)) + df = pd.DataFrame(to_dict(streamset, agg=agg)) if not df.empty: df = df.set_index("time") if agg == "all" and not streamset.allow_window: - stream_names = [[s.collection, s.name, prop] for s in streamset._streams for prop in _STAT_PROPERTIES] - df.columns=pd.MultiIndex.from_tuples(stream_names) + stream_names = [ + [s.collection, s.name, prop] + for s in streamset._streams + for prop in _STAT_PROPERTIES + ] + df.columns = pd.MultiIndex.from_tuples(stream_names) else: - df.columns = columns if columns else _stream_names(streamset, name_callable) + df.columns = columns if columns else _stream_names(streamset, name_callable) return df +def arrow_to_polars(streamset, agg="mean", name_callable=None): + """ + Returns a Polars DataFrame object with time as a column and the values of a + stream for each additional column from an arrow table. + + Parameters + ---------- + agg : str, default: "mean" + Specify the StatPoint field (e.g. aggregating function) to create the Series + from. Must be one of "min", "mean", "max", "count", "stddev", or "all". This + argument is ignored if not using StatPoints. + + name_callable : lambda, default: lambda s: s.collection + "/" + s.name + Specify a callable that can be used to determine the series name given a + Stream object. This is not compatible with agg == "all" at this time + + Notes + ----- + This requires a BTrDB server that has arrow support enabled. + """ + if not streamset._btrdb._ARROW_ENABLED: + raise NotImplementedError( + f"arrow_to_polars requires an arrow-enabled BTrDB server." + ) + try: + import polars as pl + except ImportError: + raise ImportError("Please install polars to use this transformation function.") + arrow_df = arrow_to_dataframe( + streamset=streamset, agg=agg, name_callable=name_callable + ) + return pl.from_pandas(arrow_df, include_index=True) + + +def arrow_to_arrow_table(streamset): + if not streamset._btrdb._ARROW_ENABLED: + raise NotImplementedError( + f"arrow_to_arrow_table requires an arrow-enabled BTrDB server." + ) + return streamset.arrow_values() + + +def to_polars(streamset, agg="mean", name_callable=None): + """ + Returns a Polars DataFrame object with time as a column and the values of a + stream for each additional column. + + Parameters + ---------- + agg : str, default: "mean" + Specify the StatPoint field (e.g. aggregating function) to create the Series + from. Must be one of "min", "mean", "max", "count", "stddev", or "all". This + argument is ignored if not using StatPoints. + + name_callable : lambda, default: lambda s: s.collection + "/" + s.name + Specify a callable that can be used to determine the series name given a + Stream object. This is not compatible with agg == "all" at this time + """ + try: + import polars as pl + except ImportError: + raise ImportError("Please install polars to use this transformation function.") + + # TODO: allow this at some future point + if agg == "all" and name_callable is not None: + raise AttributeError( + "cannot provide name_callable when using 'all' as aggregate at this time" + ) + + # do not allow agg="all" with RawPoints + if agg == "all" and not streamset.allow_window: + agg = "" + + # default arg values + if not callable(name_callable): + name_callable = lambda s: s.collection + "/" + s.name + + df = streamset.to_dataframe(agg=agg) + else: + df = pd.DataFrame(to_dict(streamset, agg=agg, name_callable=name_callable)) + + if not df.empty: + if df.index.name == "time": + pass + else: + df = df.set_index("time") + + if agg == "all" and streamset.allow_window: + stream_names = [ + [s.collection, s.name, prop] + for s in streamset._streams + for prop in _STAT_PROPERTIES + ] + df.columns = pd.MultiIndex.from_tuples(stream_names) + else: + df.columns = _stream_names(streamset, name_callable) + + return pl.from_pandas(df.reset_index()) + + def to_array(streamset, agg="mean"): """ Returns a multidimensional numpy array (similar to a list of lists) containing point @@ -192,6 +411,28 @@ def to_array(streamset, agg="mean"): return np.array(results, dtype=object) +def arrow_to_numpy(streamset, agg="mean"): + """Return a multidimensional array in the numpy format. + + Parameters + ---------- + agg : str, default: "mean" + Specify the StatPoint field (e.g. aggregating function) to return for the + arrays. Must be one of "min", "mean", "max", "count", or "stddev". This + argument is ignored if RawPoint values are passed into the function. + + Notes + ----- + This method first converts to a pandas data frame then to a numpy array. + """ + if not streamset._btrdb._ARROW_ENABLED: + raise NotImplementedError( + f"arrow_to_numpy requires an arrow-enabled BTrDB server." + ) + arrow_df = arrow_to_dataframe(streamset=streamset, agg=agg, name_callable=None) + return arrow_df.values + + def to_dict(streamset, agg="mean", name_callable=None): """ Returns a list of OrderedDict for each time code with the appropriate @@ -205,34 +446,67 @@ def to_dict(streamset, agg="mean", name_callable=None): argument is ignored if RawPoint values are passed into the function. name_callable : lambda, default: lambda s: s.collection + "/" + s.name - Sprecify a callable that can be used to determine the series name given a + Specify a callable that can be used to determine the series name given a Stream object. """ if not callable(name_callable): - name_callable = lambda s: s.collection + "/" + s.name + name_callable = lambda s: s.collection + "/" + s.name data = [] stream_names = _stream_names(streamset, name_callable) for row in streamset.rows(): - item = OrderedDict({ - "time": _get_time_from_row(row), - }) + item = OrderedDict( + { + "time": _get_time_from_row(row), + } + ) for idx, col in enumerate(stream_names): if row[idx].__class__.__name__ == "RawPoint": item[col] = row[idx].value if row[idx] else None else: if agg == "all": for stat in _STAT_PROPERTIES: - item["{}-{}".format(col, stat)] = getattr(row[idx], stat) if row[idx] else None + item["{}-{}".format(col, stat)] = ( + getattr(row[idx], stat) if row[idx] else None + ) else: item[col] = getattr(row[idx], agg) if row[idx] else None data.append(item) return data -def to_csv(streamset, fobj, dialect=None, fieldnames=None, agg="mean", name_callable=None): +def arrow_to_dict(streamset, agg="mean", name_callable=None): + """ + Returns a list of dicts for each time code with the appropriate + stream data attached. + + Parameters + ---------- + agg : str, default: "mean" + Specify the StatPoint field (e.g. aggregating function) to constrain dict + keys. Must be one of "min", "mean", "max", "count", or "stddev". This + argument is ignored if RawPoint values are passed into the function. + + name_callable : lambda, default: lambda s: s.collection + "/" + s.name + Specify a callable that can be used to determine the series name given a + Stream object. + + """ + if not streamset._btrdb._ARROW_ENABLED: + raise NotImplementedError( + f"arrow_to_dict requires an arrow-enabled BTrDB server." + ) + arrow_df = arrow_to_dataframe( + streamset=streamset, agg=agg, name_callable=name_callable + ) + return arrow_df.to_dict(orient="index") + + +def to_csv( + streamset, fobj, dialect=None, fieldnames=None, agg="mean", name_callable=None +): """ Saves stream data as a CSV file. @@ -255,7 +529,7 @@ def to_csv(streamset, fobj, dialect=None, fieldnames=None, agg="mean", name_call This argument is ignored if RawPoint values are passed into the function. name_callable : lambda, default: lambda s: s.collection + "/" + s.name - Sprecify a callable that can be used to determine the series name given a + Specify a callable that can be used to determine the series name given a Stream object. """ @@ -264,12 +538,12 @@ def to_csv(streamset, fobj, dialect=None, fieldnames=None, agg="mean", name_call raise AttributeError("cannot use 'all' as aggregate at this time") if not callable(name_callable): - name_callable = lambda s: s.collection + "/" + s.name + name_callable = lambda s: s.collection + "/" + s.name @contextlib.contextmanager def open_path_or_file(path_or_file): if isinstance(path_or_file, str): - f = file_to_close = open(path_or_file, 'w', newline='') + f = file_to_close = open(path_or_file, "w", newline="") else: f = path_or_file file_to_close = None @@ -303,37 +577,53 @@ def to_table(streamset, agg="mean", name_callable=None): argument is ignored if RawPoint values are passed into the function. name_callable : lambda, default: lambda s: s.collection + "/" + s.name - Sprecify a callable that can be used to determine the column name given a + Specify a callable that can be used to determine the column name given a Stream object. """ try: from tabulate import tabulate except ImportError: - raise ImportError("Please install tabulate to use this transformation function.") + raise ImportError( + "Please install tabulate to use this transformation function." + ) # TODO: allow this at some future point if agg == "all": raise AttributeError("cannot use 'all' as aggregate at this time") if not callable(name_callable): - name_callable = lambda s: s.collection + "/" + s.name + name_callable = lambda s: s.collection + "/" + s.name - return tabulate(streamset.to_dict(agg=agg, name_callable=name_callable), headers="keys") + return tabulate( + streamset.to_dict(agg=agg, name_callable=name_callable), headers="keys" + ) ########################################################################## ## Transform Classes ########################################################################## + class StreamSetTransformer(object): """ Base class for StreamSet or Stream transformations """ + to_dict = to_dict + arrow_to_dict = arrow_to_dict + to_array = to_array + arrow_to_numpy = arrow_to_numpy + to_series = to_series + arrow_to_series = arrow_to_series + to_dataframe = to_dataframe + arrow_to_dataframe = arrow_to_dataframe + + to_polars = to_polars + arrow_to_polars = arrow_to_polars to_csv = to_csv to_table = to_table diff --git a/requirements.txt b/requirements.txt index 8b35bcf..254b418 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,11 @@ # GRPC / Protobuff related -grpcio>=1.19.0,<=1.48.2 -grpcio-tools>=1.19.0,<=1.48.2 +grpcio +grpcio-tools + +# pyarrow and other tools +pyarrow +pandas +polars # Time related utils pytz diff --git a/tests/btrdb/test_arrow_streams.py b/tests/btrdb/test_arrow_streams.py new file mode 100644 index 0000000..b284238 --- /dev/null +++ b/tests/btrdb/test_arrow_streams.py @@ -0,0 +1,46 @@ +import concurrent +import uuid +from unittest.mock import Mock, PropertyMock, patch + +import pytest + +import btrdb +from btrdb.point import RawPoint +from btrdb.stream import Stream + + +@pytest.fixture +def stream1(): + uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + stream = Mock(Stream) + stream.version = Mock(return_value=11) + stream.uuid = Mock(return_value=uu) + stream.nearest = Mock(return_value=(RawPoint(time=10, value=1), 11)) + type(stream).collection = PropertyMock(return_value="fruits/apple") + type(stream).name = PropertyMock(return_value="gala") + stream.tags = Mock(return_value={"name": "gala", "unit": "volts"}) + stream.annotations = Mock(return_value=({"owner": "ABC", "color": "red"}, 11)) + stream._btrdb = Mock() + stream._btrdb._executor = concurrent.futures.ThreadPoolExecutor() + stream._btrdb._ARROW_ENABLED = Mock(return_value=True) + return stream + + +@pytest.fixture +def stream2(): + uu = uuid.UUID('17dbe387-89ea-42b6-864b-f505cdb483f5') + stream = Mock(Stream) + stream.version = Mock(return_value=22) + stream.uuid = Mock(return_value=uu) + stream.nearest = Mock(return_value=(RawPoint(time=20, value=1), 22)) + type(stream).collection = PropertyMock(return_value="fruits/orange") + type(stream).name = PropertyMock(return_value="blood") + stream.tags = Mock(return_value={"name": "blood", "unit": "amps"}) + stream.annotations = Mock(return_value=({"owner": "ABC", "color": "orange"}, 22)) + stream._btrdb = Mock() + stream._btrdb._executor = Mock() + stream._btrdb._ARROW_ENABLED = Mock(return_value=True) + return stream + +class TestArrowStreams(object): + pass \ No newline at end of file diff --git a/tests/btrdb/test_conn.py b/tests/btrdb/test_conn.py index f863187..5907458 100644 --- a/tests/btrdb/test_conn.py +++ b/tests/btrdb/test_conn.py @@ -189,6 +189,7 @@ def test_info(self): truth = { "majorVersion": 5, + "minorVersion": 0, "build": "5.0.0", "proxy": { "proxyEndpoints": ["localhost:4410"], }, } diff --git a/tests/btrdb/test_stream.py b/tests/btrdb/test_stream.py index 6af865e..321f336 100644 --- a/tests/btrdb/test_stream.py +++ b/tests/btrdb/test_stream.py @@ -11,6 +11,7 @@ Testing package for the btrdb stream module """ import concurrent.futures + ########################################################################## ## Imports ########################################################################## @@ -34,22 +35,23 @@ InvalidOperation, StreamNotFoundError, InvalidCollection, - NoSuchPoint + NoSuchPoint, ) from btrdb.grpcinterface import btrdb_pb2 -RawPointProto = btrdb_pb2.RawPoint -StatPointProto = btrdb_pb2.StatPoint -EST = pytz.timezone('America/New_York') +RawPointProto = btrdb_pb2.RawPoint +StatPointProto = btrdb_pb2.StatPoint +EST = pytz.timezone("America/New_York") ########################################################################## ## Fixtures ########################################################################## + @pytest.fixture def stream1(): - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") stream = Mock(Stream) stream.version = Mock(return_value=11) stream.uuid = Mock(return_value=uu) @@ -60,12 +62,13 @@ def stream1(): stream.annotations = Mock(return_value=({"owner": "ABC", "color": "red"}, 11)) stream._btrdb = Mock() stream._btrdb._executor = concurrent.futures.ThreadPoolExecutor() + stream._btrdb._ARROW_ENABLED = Mock(return_value=False) return stream @pytest.fixture def stream2(): - uu = uuid.UUID('17dbe387-89ea-42b6-864b-f505cdb483f5') + uu = uuid.UUID("17dbe387-89ea-42b6-864b-f505cdb483f5") stream = Mock(Stream) stream.version = Mock(return_value=22) stream.uuid = Mock(return_value=uu) @@ -76,22 +79,37 @@ def stream2(): stream.annotations = Mock(return_value=({"owner": "ABC", "color": "orange"}, 22)) stream._btrdb = Mock() stream._btrdb._executor = Mock() + stream._btrdb._ARROW_ENABLED = Mock(return_value=False) return stream +@pytest.fixture +def arrow_stream3(): + uu = uuid.UUID("17dbe387-89ea-42b6-864b-f505cdb483f5") + stream = Mock(Stream) + stream.version = Mock(return_value=22) + stream.uuid = Mock(return_value=uu) + stream.nearest = Mock(return_value=(RawPoint(time=20, value=1), 22)) + type(stream).collection = PropertyMock(return_value="fruits/orange") + type(stream).name = PropertyMock(return_value="blood") + stream.tags = Mock(return_value={"name": "blood", "unit": "amps"}) + stream.annotations = Mock(return_value=({"owner": "ABC", "color": "orange"}, 22)) + stream._btrdb = Mock() + stream._btrdb._executor = Mock() + stream._btrdb._ARROW_ENABLED = Mock(return_value=True) + return stream ########################################################################## ## Stream Tests ########################################################################## -class TestStream(object): +class TestStream(object): def test_create(self): """ Assert we can create the object """ Stream(None, "FAKE") - def test_repr_str(self): """ Assert the repr and str output are correct @@ -102,18 +120,15 @@ def test_repr_str(self): stream._collection = COLLECTION stream._tags = {"name": NAME} - expected = "".format( - COLLECTION, NAME - ) + expected = "".format(COLLECTION, NAME) assert stream.__repr__() == expected assert stream.__str__() == expected - def test_refresh_metadata(self): """ Assert refresh_metadata calls Endpoint.streamInfo """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) endpoint.streamInfo = Mock(return_value=("koala", 42, {}, {}, None)) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) @@ -121,34 +136,33 @@ def test_refresh_metadata(self): stream.refresh_metadata() stream._btrdb.ep.streamInfo.assert_called_once_with(uu, False, True) - def test_refresh_metadata_deserializes_annotations(self): """ Assert refresh_metadata deserializes annotation values """ uu = uuid.uuid4() serialized = { - 'acronym': 'VPHM', - 'description': 'El Segundo PMU 42 Ean', - 'devacronym': 'PMU!EL_SEG_PMU_42', - 'enabled': 'true', - 'id': '76932ae4-09bc-472c-8dc6-64fea68d2797', - 'phase': 'A', - 'label': 'null', - 'frequency': '30', - 'control': '2019-11-07 13:21:23.000000-0500', + "acronym": "VPHM", + "description": "El Segundo PMU 42 Ean", + "devacronym": "PMU!EL_SEG_PMU_42", + "enabled": "true", + "id": "76932ae4-09bc-472c-8dc6-64fea68d2797", + "phase": "A", + "label": "null", + "frequency": "30", + "control": "2019-11-07 13:21:23.000000-0500", "calibrate": '{"racf": 1.8, "pacf": 0.005}', } expected = { - 'acronym': 'VPHM', - 'description': 'El Segundo PMU 42 Ean', - 'devacronym': 'PMU!EL_SEG_PMU_42', - 'enabled': True, - 'id': '76932ae4-09bc-472c-8dc6-64fea68d2797', - 'phase': 'A', - 'label': None, - 'frequency': 30, - 'control': '2019-11-07 13:21:23.000000-0500', + "acronym": "VPHM", + "description": "El Segundo PMU 42 Ean", + "devacronym": "PMU!EL_SEG_PMU_42", + "enabled": True, + "id": "76932ae4-09bc-472c-8dc6-64fea68d2797", + "phase": "A", + "label": None, + "frequency": 30, + "control": "2019-11-07 13:21:23.000000-0500", "calibrate": {"racf": 1.8, "pacf": 0.005}, } @@ -159,7 +173,6 @@ def test_refresh_metadata_deserializes_annotations(self): stream.refresh_metadata() assert stream.annotations()[0] == expected - def test_stream_name_property(self): """ Assert name property comes from tags @@ -169,7 +182,6 @@ def test_stream_name_property(self): stream._tags = {"name": name} assert stream.name == name - def test_stream_unit_property(self): """ Assert unit property comes from tags @@ -179,7 +191,6 @@ def test_stream_unit_property(self): stream._tags = {"unit": unit} assert stream.unit == unit - ########################################################################## ## update tests ########################################################################## @@ -188,7 +199,7 @@ def test_update_arguments(self): """ Assert update raises errors on invalid arguments """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) @@ -212,12 +223,11 @@ def test_update_arguments(self): stream.update(collection=42) assert "collection must be of type string" in str(exc) - def test_update_tags(self): """ Assert update calls correct Endpoint methods for tags update """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) endpoint.streamInfo = Mock(return_value=("koala", 42, {}, {}, None)) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) @@ -225,16 +235,16 @@ def test_update_tags(self): tags = {"cat": "dog"} stream.update(tags=tags) - stream._btrdb.ep.setStreamTags.assert_called_once_with(uu=uu, expected=42, - tags=tags, collection="koala") + stream._btrdb.ep.setStreamTags.assert_called_once_with( + uu=uu, expected=42, tags=tags, collection="koala" + ) stream._btrdb.ep.setStreamAnnotations.assert_not_called() - def test_update_collection(self): """ Assert update calls correct Endpoint methods for collection update """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) endpoint.streamInfo = Mock(return_value=("koala", 42, {}, {}, None)) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) @@ -242,16 +252,16 @@ def test_update_collection(self): collection = "giraffe" stream.update(collection=collection) - stream._btrdb.ep.setStreamTags.assert_called_once_with(uu=uu, expected=42, - tags=stream.tags(), collection=collection) + stream._btrdb.ep.setStreamTags.assert_called_once_with( + uu=uu, expected=42, tags=stream.tags(), collection=collection + ) stream._btrdb.ep.setStreamAnnotations.assert_not_called() - def test_update_annotations(self): """ Assert update calls correct Endpoint methods for annotations update """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) endpoint.streamInfo = Mock(return_value=("koala", 42, {}, {}, None)) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) @@ -262,7 +272,7 @@ def test_update_annotations(self): "description": "El Segundo PMU 42 Ean", "devacronym": "PMU!EL_SEG_PMU_42", "enabled": True, - "id": uuid.UUID('76932ae4-09bc-472c-8dc6-64fea68d2797'), + "id": uuid.UUID("76932ae4-09bc-472c-8dc6-64fea68d2797"), "phase": "A", "label": None, "frequency": 30, @@ -276,22 +286,21 @@ def test_update_annotations(self): uu=uu, expected=42, changes={ - 'acronym': 'VPHM', - 'description': 'El Segundo PMU 42 Ean', - 'devacronym': 'PMU!EL_SEG_PMU_42', - 'enabled': 'true', - 'id': '76932ae4-09bc-472c-8dc6-64fea68d2797', - 'phase': 'A', - 'label': 'null', - 'frequency': '30', - 'control': '2019-11-07 13:21:23.000000-0500', + "acronym": "VPHM", + "description": "El Segundo PMU 42 Ean", + "devacronym": "PMU!EL_SEG_PMU_42", + "enabled": "true", + "id": "76932ae4-09bc-472c-8dc6-64fea68d2797", + "phase": "A", + "label": "null", + "frequency": "30", + "control": "2019-11-07 13:21:23.000000-0500", "calibrate": '{"racf": 1.8, "pacf": 0.005}', }, removals=[], ) stream._btrdb.ep.setStreamTags.assert_not_called() - def test_update_annotations_nested_conversions(self): """ Assert update correctly encodes nested annotation data @@ -308,21 +317,23 @@ def test_update_annotations_nested_conversions(self): "num": 12, "float": 1.3, "string": "the quick brown fox is 12", - } - } + }, + }, } - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) - endpoint.streamInfo = Mock(return_value=("koala", 42, {}, {"foo": "42 Cherry Hill"}, None)) + endpoint.streamInfo = Mock( + return_value=("koala", 42, {}, {"foo": "42 Cherry Hill"}, None) + ) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) stream.refresh_metadata() stream.update(annotations=annotations) # spot check a nested value for Python 3.4 and 3.5 compatability - changes = stream._btrdb.ep.setStreamAnnotations.call_args[1]['changes'] - assert changes['nested'].__class__ == str - assert json.loads(changes['nested']) == annotations['nested'] + changes = stream._btrdb.ep.setStreamAnnotations.call_args[1]["changes"] + assert changes["nested"].__class__ == str + assert json.loads(changes["nested"]) == annotations["nested"] # check all args if Python > 3.5 if sys.version_info[0] > 3.5: @@ -330,18 +341,18 @@ def test_update_annotations_nested_conversions(self): uu=uu, expected=42, changes={ - 'num': '10', - 'float': '1.3', - 'string': '"the quick brown fox is 10"', - 'nested': '{"num": 11, "float": 1.3, "string": "the quick brown fox is 11", "nested": {"num": 12, "float": 1.3, "string": "the quick brown fox is 12"}}' - } + "num": "10", + "float": "1.3", + "string": '"the quick brown fox is 10"', + "nested": '{"num": 11, "float": 1.3, "string": "the quick brown fox is 11", "nested": {"num": 12, "float": 1.3, "string": "the quick brown fox is 12"}}', + }, ) def test_update_annotations_no_encoder(self): """ Assert update annotations works with None as encoder argument """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) endpoint.streamInfo = Mock(return_value=("koala", 42, {}, {}, None)) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) @@ -365,12 +376,17 @@ def test_update_annotations_replace(self): Assert that replace argument will add proper keys to removals array in endpoint call. """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) - endpoint.streamInfo = Mock(return_value=("koala", 42, {}, {"phase": "A", "source": "PJM"}, None)) + endpoint.streamInfo = Mock( + return_value=("koala", 42, {}, {"phase": "A", "source": "PJM"}, None) + ) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) - annotations = {"foo": "this is a string", "phase": "A", } + annotations = { + "foo": "this is a string", + "phase": "A", + } stream.refresh_metadata() @@ -400,45 +416,42 @@ def test_exists_cached_value(self): """ Assert exists first uses cached value """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") stream = Stream(btrdb=BTrDB(Mock(Endpoint)), uuid=uu, known_to_exist=True) stream.refresh_metadata = Mock() assert stream.exists() stream.refresh_metadata.assert_not_called() - def test_exists(self): """ Assert exists refreshes data if value is unknown """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") stream = Stream(btrdb=BTrDB(Mock(Endpoint)), uuid=uu) stream.refresh_metadata = Mock(return_value=True) assert stream.exists() assert stream.refresh_metadata.call_count == 1 - def test_exists_returns_false_on_404(self): """ Assert exists returns False on 404 error """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") stream = Stream(btrdb=BTrDB(Mock(Endpoint)), uuid=uu) - stream.refresh_metadata = Mock(side_effect=StreamNotFoundError( - "stream not found with provided uuid" - )) + stream.refresh_metadata = Mock( + side_effect=StreamNotFoundError("stream not found with provided uuid") + ) assert stream.exists() == False assert stream.refresh_metadata.call_count == 1 - def test_exists_passes_other_errors(self): """ Assert exists does not keep non 404 errors trapped """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") stream = Stream(btrdb=BTrDB(Mock(Endpoint)), uuid=uu) stream.refresh_metadata = Mock(side_effect=ValueError()) @@ -446,7 +459,6 @@ def test_exists_passes_other_errors(self): stream.exists() assert stream.refresh_metadata.call_count == 1 - ########################################################################## ## tag/annotation tests ########################################################################## @@ -455,34 +467,33 @@ def test_tags_returns_copy(self): """ Assert tags returns a copy of the tags dict """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") token = {"cat": "dog"} stream = Stream(btrdb=BTrDB(Mock(Endpoint)), uuid=uu, tags=token) assert stream.tags() is not token assert stream.tags() == token - def test_tags_returns_cached_values(self): """ Assert tags returns a copy of the tags dict """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") token = {"cat": "dog"} - stream = Stream(btrdb=BTrDB(Mock(Endpoint)), uuid=uu, tags=token, - property_version=42) + stream = Stream( + btrdb=BTrDB(Mock(Endpoint)), uuid=uu, tags=token, property_version=42 + ) stream.refresh_metadata = Mock() assert stream.tags(refresh=False) == token stream.refresh_metadata.assert_not_called() - def test_tags_forces_refresh_if_requested(self): """ Assert tags calls refresh_metadata if requested even though a cached copy is available """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") token = {"cat": "dog"} stream = Stream(btrdb=BTrDB(Mock(Endpoint)), uuid=uu, tags=token) stream.refresh_metadata = Mock() @@ -490,42 +501,41 @@ def test_tags_forces_refresh_if_requested(self): stream.tags(refresh=True) assert stream.refresh_metadata.call_count == 1 - def test_annotations_returns_copy_of_value(self): """ Assert annotations returns a copy of the annotations dict """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") token = {"cat": "dog"} - stream = Stream(btrdb=BTrDB(Mock(Endpoint)), uuid=uu, annotations=token, - property_version=42) + stream = Stream( + btrdb=BTrDB(Mock(Endpoint)), uuid=uu, annotations=token, property_version=42 + ) stream.refresh_metadata = Mock() assert stream.annotations(refresh=False)[0] == token assert stream.annotations(refresh=False)[0] is not token assert stream.annotations(refresh=False)[1] == 42 - def test_annotations_returns_cached_values(self): """ Assert annotations returns a copy of the annotations dict """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") token = {"cat": "dog"} - stream = Stream(btrdb=BTrDB(Mock(Endpoint)), uuid=uu, annotations=token, - property_version=42) + stream = Stream( + btrdb=BTrDB(Mock(Endpoint)), uuid=uu, annotations=token, property_version=42 + ) stream.refresh_metadata = Mock() assert stream.annotations(refresh=False)[0] == token stream.refresh_metadata.assert_not_called() - def test_annotations_forces_refresh_if_requested(self): """ Assert annotations calls refresh_metadata if requested even though a cached copy is available """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") token = {"cat": "dog"} stream = Stream(btrdb=BTrDB(Mock(Endpoint)), uuid=uu, annotations=token) stream.refresh_metadata = Mock() @@ -533,7 +543,6 @@ def test_annotations_forces_refresh_if_requested(self): stream.annotations(refresh=True) assert stream.refresh_metadata.call_count == 1 - ########################################################################## ## windowing tests ########################################################################## @@ -542,15 +551,29 @@ def test_windows(self): """ Assert windows returns tuples of data from Endpoint.windows """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) windows = [ - [(StatPointProto(time=1,min=2,mean=3,max=4,count=5,stddev=6), StatPointProto(time=2,min=3,mean=4,max=5,count=6,stddev=7)), 42], - [(StatPointProto(time=3,min=4,mean=5,max=6,count=7,stddev=8), StatPointProto(time=4,min=5,mean=6,max=7,count=8,stddev=9)), 42], + [ + ( + StatPointProto(time=1, min=2, mean=3, max=4, count=5, stddev=6), + StatPointProto(time=2, min=3, mean=4, max=5, count=6, stddev=7), + ), + 42, + ], + [ + ( + StatPointProto(time=3, min=4, mean=5, max=6, count=7, stddev=8), + StatPointProto(time=4, min=5, mean=6, max=7, count=8, stddev=9), + ), + 42, + ], ] expected = ( - (StatPoint(time=1,minv=2.0,meanv=3.0,maxv=4.0,count=5,stddev=6.0), 42), (StatPoint(time=2,minv=3.0,meanv=4.0,maxv=5.0,count=6,stddev=7.0), 42), - (StatPoint(time=3,minv=4.0,meanv=5.0,maxv=6.0,count=7,stddev=8.0), 42), (StatPoint(time=4,minv=5.0,meanv=6.0,maxv=7.0,count=8,stddev=9.0), 42), + (StatPoint(time=1, minv=2.0, meanv=3.0, maxv=4.0, count=5, stddev=6.0), 42), + (StatPoint(time=2, minv=3.0, meanv=4.0, maxv=5.0, count=6, stddev=7.0), 42), + (StatPoint(time=3, minv=4.0, meanv=5.0, maxv=6.0, count=7, stddev=8.0), 42), + (StatPoint(time=4, minv=5.0, meanv=6.0, maxv=7.0, count=8, stddev=9.0), 42), ) endpoint.windows = Mock(return_value=windows) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) @@ -560,24 +583,35 @@ def test_windows(self): assert result == expected assert isinstance(result, tuple) assert isinstance(result[0], tuple) - stream._btrdb.ep.windows.assert_called_once_with( - uu, 100, 500, 2, 0, 0 - ) - + stream._btrdb.ep.windows.assert_called_once_with(uu, 100, 500, 2, 0, 0) def test_aligned_windows(self): """ Assert windows returns tuples of data from Endpoint.alignedWindows """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) windows = [ - [(StatPointProto(time=1,min=2,mean=3,max=4,count=5,stddev=6), StatPointProto(time=2,min=3,mean=4,max=5,count=6,stddev=7)), 42], - [(StatPointProto(time=3,min=4,mean=5,max=6,count=7,stddev=8), StatPointProto(time=4,min=5,mean=6,max=7,count=8,stddev=9)), 42], + [ + ( + StatPointProto(time=1, min=2, mean=3, max=4, count=5, stddev=6), + StatPointProto(time=2, min=3, mean=4, max=5, count=6, stddev=7), + ), + 42, + ], + [ + ( + StatPointProto(time=3, min=4, mean=5, max=6, count=7, stddev=8), + StatPointProto(time=4, min=5, mean=6, max=7, count=8, stddev=9), + ), + 42, + ], ] expected = ( - (StatPoint(time=1,minv=2.0,meanv=3.0,maxv=4.0,count=5,stddev=6.0), 42), (StatPoint(time=2,minv=3.0,meanv=4.0,maxv=5.0,count=6,stddev=7.0), 42), - (StatPoint(time=3,minv=4.0,meanv=5.0,maxv=6.0,count=7,stddev=8.0), 42), (StatPoint(time=4,minv=5.0,meanv=6.0,maxv=7.0,count=8,stddev=9.0), 42), + (StatPoint(time=1, minv=2.0, meanv=3.0, maxv=4.0, count=5, stddev=6.0), 42), + (StatPoint(time=2, minv=3.0, meanv=4.0, maxv=5.0, count=6, stddev=7.0), 42), + (StatPoint(time=3, minv=4.0, meanv=5.0, maxv=6.0, count=7, stddev=8.0), 42), + (StatPoint(time=4, minv=5.0, meanv=6.0, maxv=7.0, count=8, stddev=9.0), 42), ) endpoint.alignedWindows = Mock(return_value=windows) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) @@ -587,20 +621,29 @@ def test_aligned_windows(self): assert result == expected assert isinstance(result, tuple) assert isinstance(result[0], tuple) - stream._btrdb.ep.alignedWindows.assert_called_once_with( - uu, 100, 500, 1, 0 - ) - + stream._btrdb.ep.alignedWindows.assert_called_once_with(uu, 100, 500, 1, 0) def test_count(self): """ Test that stream count method uses aligned windows """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) windows = [ - [(StatPointProto(time=1,min=2,mean=3,max=4,count=5,stddev=6), StatPointProto(time=2,min=3,mean=4,max=5,count=6,stddev=7)), 42], - [(StatPointProto(time=3,min=4,mean=5,max=6,count=7,stddev=8), StatPointProto(time=4,min=5,mean=6,max=7,count=8,stddev=9)), 42], + [ + ( + StatPointProto(time=1, min=2, mean=3, max=4, count=5, stddev=6), + StatPointProto(time=2, min=3, mean=4, max=5, count=6, stddev=7), + ), + 42, + ], + [ + ( + StatPointProto(time=3, min=4, mean=5, max=6, count=7, stddev=8), + StatPointProto(time=4, min=5, mean=6, max=7, count=8, stddev=9), + ), + 42, + ], ] endpoint.alignedWindows = Mock(return_value=windows) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) @@ -613,7 +656,6 @@ def test_count(self): stream.count(10, 1000, 8, version=1200) stream._btrdb.ep.alignedWindows.assert_called_with(uu, 10, 1000, 8, 1200) - ########################################################################## ## earliest/latest tests ########################################################################## @@ -622,7 +664,7 @@ def test_earliest(self): """ Assert earliest calls Endpoint.nearest """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) endpoint.nearest = Mock(return_value=(RawPointProto(time=100, value=1.0), 42)) @@ -631,12 +673,11 @@ def test_earliest(self): assert (point, ver) == (RawPoint(100, 1.0), 42) endpoint.nearest.assert_called_once_with(uu, MINIMUM_TIME, 0, False) - def test_earliest_swallows_exception(self): """ Assert earliest returns None when endpoint throws exception """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) endpoint.nearest = Mock(side_effect=NoSuchPoint("next point does not exist")) @@ -644,12 +685,11 @@ def test_earliest_swallows_exception(self): assert stream.earliest() is None endpoint.nearest.assert_called_once_with(uu, MINIMUM_TIME, 0, False) - def test_earliest_passes_exception(self): """ Assert earliest reraises non 401 exception """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) endpoint.nearest = Mock(side_effect=BTrDBError("empty")) @@ -659,12 +699,11 @@ def test_earliest_passes_exception(self): assert exc.value.args[0] == "empty" endpoint.nearest.assert_called_once_with(uu, MINIMUM_TIME, 0, False) - def test_latest(self): """ Assert latest calls Endpoint.nearest """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) endpoint.nearest = Mock(return_value=(RawPointProto(time=100, value=1.0), 42)) @@ -673,12 +712,11 @@ def test_latest(self): assert (point, ver) == (RawPoint(100, 1.0), 42) endpoint.nearest.assert_called_once_with(uu, MAXIMUM_TIME, 0, True) - def test_latest_swallows_exception(self): """ Assert latest returns None when endpoint throws exception """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) endpoint.nearest = Mock(side_effect=NoSuchPoint("empty")) @@ -686,12 +724,11 @@ def test_latest_swallows_exception(self): assert stream.latest() is None endpoint.nearest.assert_called_once_with(uu, MAXIMUM_TIME, 0, True) - def test_latest_passes_exception(self): """ Assert latest reraises non 401 exception """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) endpoint.nearest = Mock(side_effect=BTrDBError("empty")) @@ -701,13 +738,12 @@ def test_latest_passes_exception(self): assert exc.value.args[0] == "empty" endpoint.nearest.assert_called_once_with(uu, MAXIMUM_TIME, 0, True) - @patch("btrdb.stream.currently_as_ns") def test_currently(self, mocked): """ Assert currently calls Endpoint.nearest """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) endpoint.nearest = Mock(return_value=(RawPointProto(time=100, value=1.0), 42)) @@ -718,13 +754,12 @@ def test_currently(self, mocked): assert (point, ver) == (RawPoint(100, 1.0), 42) endpoint.nearest.assert_called_once_with(uu, ns_fake_time, 0, True) - @patch("btrdb.stream.currently_as_ns") def test_currently_swallows_exception(self, mocked): """ Assert currently returns None when endpoint throws exception """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) endpoint.nearest = Mock(side_effect=NoSuchPoint("empty")) @@ -734,13 +769,12 @@ def test_currently_swallows_exception(self, mocked): assert stream.current() is None endpoint.nearest.assert_called_once_with(uu, ns_fake_time, 0, True) - @patch("btrdb.stream.currently_as_ns") def test_currently_passes_exception(self, mocked): """ Assert currently reraises non 401 exception """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) endpoint.nearest = Mock(side_effect=BTrDBError("empty")) @@ -752,7 +786,6 @@ def test_currently_passes_exception(self, mocked): assert exc.value.args[0] == "empty" endpoint.nearest.assert_called_once_with(uu, ns_fake_time, 0, True) - ########################################################################## ## misc tests ########################################################################## @@ -761,7 +794,7 @@ def test_version(self): """ Assert version calls and returns correct value from streamInfo """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) endpoint.streamInfo = Mock(return_value=("", 0, {}, {}, 42)) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) @@ -769,30 +802,36 @@ def test_version(self): assert stream.version() == 42 stream._btrdb.ep.streamInfo.assert_called_once_with(uu, True, False) - def test_insert(self): """ Assert insert batches data to endpoint insert and returns version """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) - endpoint.insert = Mock(side_effect=[1,2,3]) + endpoint.insert = Mock(side_effect=[1, 2, 3]) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) - data = list(zip(range(10000,120000), map(float, range(110000)))) + data = list(zip(range(10000, 120000), map(float, range(110000)))) version = stream.insert(data) - assert stream._btrdb.ep.insert.call_args_list[0][0][1] == data[:INSERT_BATCH_SIZE] - assert stream._btrdb.ep.insert.call_args_list[1][0][1] == data[INSERT_BATCH_SIZE:2*INSERT_BATCH_SIZE] - assert stream._btrdb.ep.insert.call_args_list[2][0][1] == data[2*INSERT_BATCH_SIZE:] + assert ( + stream._btrdb.ep.insert.call_args_list[0][0][1] == data[:INSERT_BATCH_SIZE] + ) + assert ( + stream._btrdb.ep.insert.call_args_list[1][0][1] + == data[INSERT_BATCH_SIZE : 2 * INSERT_BATCH_SIZE] + ) + assert ( + stream._btrdb.ep.insert.call_args_list[2][0][1] + == data[2 * INSERT_BATCH_SIZE :] + ) assert version == 3 - def test_nearest(self): """ Assert nearest calls Endpoint.nearest with correct arguments """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) endpoint.nearest = Mock(return_value=(RawPointProto(time=100, value=1.0), 42)) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) @@ -802,12 +841,11 @@ def test_nearest(self): assert point == RawPoint(100, 1.0) assert version == 42 - def test_nearest_swallows_exception(self): """ Assert nearest returns None when endpoint throws 401 exception """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) endpoint.nearest = Mock(side_effect=NoSuchPoint("next point does not exist")) @@ -815,12 +853,11 @@ def test_nearest_swallows_exception(self): assert stream.nearest(0, 0, False) is None endpoint.nearest.assert_called_once_with(uu, 0, 0, False) - def test_nearest_passes_exception(self): """ Assert nearest reraises non 401 exception """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) endpoint.nearest = Mock(side_effect=BTrDBError("foo")) @@ -830,45 +867,41 @@ def test_nearest_passes_exception(self): assert exc.value.args[0] == "foo" endpoint.nearest.assert_called_once_with(uu, 0, 0, False) - def test_delete_range(self): """ Assert delete_range calls Endpoint.deleteRange with correct arguments """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") stream = Stream(btrdb=BTrDB(Mock(Endpoint)), uuid=uu) stream.delete(5, 10) stream._btrdb.ep.deleteRange.assert_called_once_with(uu, 5, 10) - def test_flush(self): """ Assert flush calls Endpoint.flush with UUID """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") stream = Stream(btrdb=BTrDB(Mock(Endpoint)), uuid=uu) stream.flush() stream._btrdb.ep.flush.assert_called_once_with(uu) - def test_obliterate(self): """ Assert obliterate calls Endpoint.obliterate with UUID """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") stream = Stream(btrdb=BTrDB(Mock(Endpoint)), uuid=uu) stream.obliterate() stream._btrdb.ep.obliterate.assert_called_once_with(uu) - def test_obliterate_allows_error(self): """ Assert obliterate raises error if stream not found. (does not swallow error) """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) endpoint.obliterate = Mock(side_effect=StreamNotFoundError()) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) @@ -878,20 +911,26 @@ def test_obliterate_allows_error(self): stream._btrdb.ep.obliterate.assert_called_once_with(uu) - - ########################################################################## ## StreamSet Tests ########################################################################## -class TestStreamSet(object): +class TestStreamSet(object): def test_create(self): """ Assert we can create the object """ - StreamSet([]) + StreamSet([1]) + @pytest.mark.parametrize( + "empty_container", [(tuple()), (dict()), (list()), (set())] + ) + def test_create_error_with_no_objects(self, empty_container): + with pytest.raises( + ValueError, match="Trying to create streamset with an empty list of streams" + ): + StreamSet(empty_container) ########################################################################## ## builtin / magic methods tests @@ -901,7 +940,7 @@ def test_repr(self): """ Assert StreamSet instance repr output """ - data = [11,22,"dog","cat"] + data = [11, 22, "dog", "cat"] streams = StreamSet(data) expected = "" assert streams.__repr__() == expected @@ -911,12 +950,11 @@ def test_repr(self): expected = "" assert streams.__repr__() == expected - def test_str(self): """ Assert StreamSet instance str output """ - data = [11,22,"dog","cat"] + data = [11, 22, "dog", "cat"] streams = StreamSet(data) expected = "StreamSet with 4 streams" assert str(streams) == expected @@ -926,41 +964,37 @@ def test_str(self): expected = "StreamSet with 1 stream" assert str(streams) == expected - def test_subscriptable(self): """ Assert StreamSet instance is subscriptable """ - data = [11,22,"dog","cat"] + data = [11, 22, "dog", "cat"] streams = StreamSet(data) for index, val in enumerate(data): assert streams[index] == val - def test_len(self): """ Assert StreamSet instance support len """ - data = [11,22,"dog","cat"] + data = [11, 22, "dog", "cat"] streams = StreamSet(data) assert len(streams) == len(data) - def test_iter(self): """ Assert StreamSet instance support iteration """ - data = [11,22,"dog","cat"] + data = [11, 22, "dog", "cat"] streams = StreamSet(data) for index, stream in enumerate(streams): assert data[index] == stream - def test_indexing(self): """ Assert StreamSet instance supports indexing """ - data = [11,22,"dog","cat"] + data = [11, 22, "dog", "cat"] streams = StreamSet(data) # verify index lookup @@ -971,7 +1005,6 @@ def test_indexing(self): # verify slicing works assert streams[:2] == data[:2] - def test_mapping(self): """ Assert StreamSet instance support key mapping @@ -994,34 +1027,30 @@ def test_mapping(self): streams[missing] assert str(missing) in str(e) - def test_contains(self): """ Assert StreamSet instance supports contains """ - data = [11,22,"dog","cat"] + data = [11, 22, "dog", "cat"] streams = StreamSet(data) assert "dog" in streams - def test_reverse(self): """ Assert StreamSet instance supports reversal """ - data = [11,22,"dog","cat"] + data = [11, 22, "dog", "cat"] streams = StreamSet(data) assert list(reversed(streams)) == list(reversed(data)) - def test_to_list(self): """ Assert StreamSet instance cast to list """ - data = [11,22,"dog","cat"] + data = [11, 22, "dog", "cat"] streams = StreamSet(data) assert list(streams) == data - ########################################################################## ## allow_window tests ########################################################################## @@ -1030,18 +1059,16 @@ def test_allow_window(self): """ Assert allow_window returns False if window already requested """ - streams = StreamSet([1,2,3]) + streams = StreamSet([1, 2, 3]) assert streams.allow_window == True streams.windows(30, 4) assert streams.allow_window == False - - streams = StreamSet([1,2,3]) + streams = StreamSet([1, 2, 3]) streams.aligned_windows(30) assert streams.allow_window == False - ########################################################################## ## _latest_versions tests ########################################################################## @@ -1051,13 +1078,9 @@ def test_latest_versions(self, stream1, stream2): Assert _latest_versions returns correct values """ streams = StreamSet([stream1, stream2]) - expected = { - stream1.uuid: stream1.version(), - stream2.uuid: stream2.version() - } + expected = {stream1.uuid: stream1.version(), stream2.uuid: stream2.version()} assert streams._latest_versions() == expected - ########################################################################## ## pin_versions tests ########################################################################## @@ -1067,50 +1090,40 @@ def test_pin_versions_returns_self(self, stream1): Assert pin_versions returns self """ streams = StreamSet([stream1]) - expected = { - stream1.uuid(): stream1.version() - } - result = streams.pin_versions(expected) + expected = {stream1.uuid(): stream1.version()} + result = streams.pin_versions(expected) assert streams is result - def test_pin_versions_with_argument(self): """ Assert pin_versions uses supplied version numbers """ - streams = StreamSet([1,2]) - expected = [3,4] + streams = StreamSet([1, 2]) + expected = [3, 4] assert streams.pin_versions(expected) == streams assert streams._pinned_versions == expected - def test_pin_versions_with_argument(self): """ Assert pin_versions uses supplied version numbers """ - streams = StreamSet([1,2]) - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') - expected = { - uu: 42 - } + streams = StreamSet([1, 2]) + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + expected = {uu: 42} assert streams.pin_versions(expected) == streams assert streams._pinned_versions == expected - def test_pin_versions_no_argument(self, stream1, stream2): """ Assert pin_versions uses latest version numbers """ streams = StreamSet([stream1, stream2]) - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') - expected = { - uu: 42 - } + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + expected = {uu: 42} assert streams.pin_versions(expected) == streams assert streams._pinned_versions == expected - def test_pin_versions_raise_on_non_dict(self): """ Assert pin_versions raises if versions argument is not dict @@ -1122,7 +1135,6 @@ def test_pin_versions_raise_on_non_dict(self): streams.pin_versions(expected) == streams assert "dict" in str(e).lower() - def test_pin_versions_raise_on_non_uuid_key(self): """ Assert pin_versions raises if versions argument dict does not have UUID @@ -1135,7 +1147,6 @@ def test_pin_versions_raise_on_non_uuid_key(self): streams.pin_versions(expected) == streams assert "uuid" in str(e).lower() - ########################################################################## ## versions tests ########################################################################## @@ -1145,37 +1156,29 @@ def test_versions_no_pin(self, stream1, stream2): Assert versions returns correctly if pin_versions not called """ streams = StreamSet([stream1, stream2]) - expected = { - stream1.uuid: stream1.version(), - stream2.uuid: stream2.version() - } + expected = {stream1.uuid: stream1.version(), stream2.uuid: stream2.version()} assert streams.versions() == expected - def test_versions_with_pin(self, stream1, stream2): """ Assert versions returns correctly if pin_versions called """ - uu1 = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu1 = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") s1 = Mock(Stream) s1.version = Mock(return_value=11) s1.uuid = Mock(return_value=uu1) - uu2 = uuid.UUID('17dbe387-89ea-42b6-864b-f505cdb483f5') + uu2 = uuid.UUID("17dbe387-89ea-42b6-864b-f505cdb483f5") s2 = Mock(Stream) s2.version = Mock(return_value=22) s2.uuid = Mock(return_value=uu2) streams = StreamSet([stream1, stream2]) - expected = { - stream1.uuid(): 88, - stream2.uuid(): 99 - } + expected = {stream1.uuid(): 88, stream2.uuid(): 99} streams.pin_versions(expected) assert streams.versions() == expected - ########################################################################## ## earliest/latest tests ########################################################################## @@ -1185,22 +1188,26 @@ def test_earliest(self, stream1, stream2): Assert earliest returns correct time code """ streams = StreamSet([stream1, stream2]) - assert streams.earliest() == (RawPoint(time=10, value=1), RawPoint(time=20, value=1)) - - + assert streams.earliest() == ( + RawPoint(time=10, value=1), + RawPoint(time=20, value=1), + ) def test_latest(self, stream1, stream2): """ Assert latest returns correct time code """ streams = StreamSet([stream1, stream2]) - assert streams.latest() == (RawPoint(time=10, value=1), RawPoint(time=20, value=1)) + assert streams.latest() == ( + RawPoint(time=10, value=1), + RawPoint(time=20, value=1), + ) @patch("btrdb.stream.currently_as_ns") def test_current(self, mocked, stream1, stream2): """ Assert current calls nearest with the current time """ - mocked.return_value=15 + mocked.return_value = 15 streams = StreamSet([stream1, stream2]) streams.current() stream1.nearest.assert_called_once_with(15, version=11, backward=True) @@ -1211,52 +1218,84 @@ def test_currently_out_of_range(self, mocked): """ Assert currently raises an exception if it is not filtered """ - mocked.return_value=15 + mocked.return_value = 15 streams = StreamSet([stream1, stream2]) - with pytest.raises(ValueError, match="current time is not included in filtered stream range"): + with pytest.raises( + ValueError, match="current time is not included in filtered stream range" + ): streams.filter(start=20, end=30).current() - with pytest.raises(ValueError, match="current time is not included in filtered stream range"): + with pytest.raises( + ValueError, match="current time is not included in filtered stream range" + ): streams.filter(start=0, end=10).current() def test_count(self): """ Test the stream set count method """ - uu1 = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') - uu2 = uuid.UUID('4dadf38d-52a5-4b7a-ada9-a5d563f9538c') + uu1 = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + uu2 = uuid.UUID("4dadf38d-52a5-4b7a-ada9-a5d563f9538c") endpoint = Mock(Endpoint) + endpoint.streamInfo = Mock(return_value=[-1, -1, -1, -1, 0]) windows = [ - [(StatPointProto(time=1,min=2,mean=3,max=4,count=5,stddev=6), StatPointProto(time=2,min=3,mean=4,max=5,count=6,stddev=7)), 42], - [(StatPointProto(time=3,min=4,mean=5,max=6,count=7,stddev=8), StatPointProto(time=4,min=5,mean=6,max=7,count=8,stddev=9)), 42], + [ + ( + StatPointProto(time=1, min=2, mean=3, max=4, count=5, stddev=6), + StatPointProto(time=2, min=3, mean=4, max=5, count=6, stddev=7), + ), + 42, + ], + [ + ( + StatPointProto(time=3, min=4, mean=5, max=6, count=7, stddev=8), + StatPointProto(time=4, min=5, mean=6, max=7, count=8, stddev=9), + ), + 42, + ], ] endpoint.alignedWindows = Mock(return_value=windows) - streams = StreamSet([ - Stream(btrdb=BTrDB(endpoint), uuid=uu1), - Stream(btrdb=BTrDB(endpoint), uuid=uu2), - ]) + streams = StreamSet( + [ + Stream(btrdb=BTrDB(endpoint), uuid=uu1), + Stream(btrdb=BTrDB(endpoint), uuid=uu2), + ] + ) assert streams.count() == 52 endpoint.alignedWindows.assert_any_call(uu1, MINIMUM_TIME, MAXIMUM_TIME, 60, 0) endpoint.alignedWindows.assert_any_call(uu2, MINIMUM_TIME, MAXIMUM_TIME, 60, 0) - def test_count_filtered(self): """ Test the stream set count method with filters """ - uu1 = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') - uu2 = uuid.UUID('4dadf38d-52a5-4b7a-ada9-a5d563f9538c') + uu1 = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + uu2 = uuid.UUID("4dadf38d-52a5-4b7a-ada9-a5d563f9538c") endpoint = Mock(Endpoint) endpoint.alignedWindows = Mock(return_value=[]) - streams = StreamSet([ - Stream(btrdb=BTrDB(endpoint), uuid=uu1), - Stream(btrdb=BTrDB(endpoint), uuid=uu2), - ]) + streams = StreamSet( + [ + Stream(btrdb=BTrDB(endpoint), uuid=uu1), + Stream(btrdb=BTrDB(endpoint), uuid=uu2), + ] + ) windows = [ - [(StatPointProto(time=1,min=2,mean=3,max=4,count=5,stddev=6), StatPointProto(time=2,min=3,mean=4,max=5,count=6,stddev=7)), 42], - [(StatPointProto(time=3,min=4,mean=5,max=6,count=7,stddev=8), StatPointProto(time=4,min=5,mean=6,max=7,count=8,stddev=9)), 42], + [ + ( + StatPointProto(time=1, min=2, mean=3, max=4, count=5, stddev=6), + StatPointProto(time=2, min=3, mean=4, max=5, count=6, stddev=7), + ), + 42, + ], + [ + ( + StatPointProto(time=3, min=4, mean=5, max=6, count=7, stddev=8), + StatPointProto(time=4, min=5, mean=6, max=7, count=8, stddev=9), + ), + 42, + ], ] endpoint.alignedWindows = Mock(return_value=windows) @@ -1267,8 +1306,6 @@ def test_count_filtered(self): endpoint.alignedWindows.assert_any_call(uu1, 10, 1000, 8, 42) endpoint.alignedWindows.assert_any_call(uu2, 10, 1000, 8, 99) - - ########################################################################## ## filter tests ########################################################################## @@ -1286,7 +1323,6 @@ def test_filter(self, stream1): assert streams.filters[0].end == end assert isinstance(streams.filters[0], StreamFilter) - def test_filter_returns_new_instance(self, stream1): """ Assert filter returns new instance @@ -1298,7 +1334,6 @@ def test_filter_returns_new_instance(self, stream1): assert other is not streams assert isinstance(other, streams.__class__) - def test_filter_collection(self, stream1, stream2): """ Assert filter collection works as intended @@ -1335,7 +1370,6 @@ def test_filter_collection(self, stream1, stream2): other = streams.filter(collection=re.compile(r"region\.")) assert other._streams == [stream1] - def test_filter_name(self, stream1, stream2): """ Assert filter name works as intended @@ -1370,7 +1404,6 @@ def test_filter_name(self, stream1, stream2): other = streams.filter(name=re.compile(r"region\.")) assert other._streams == [stream1] - def test_filter_unit(self, stream1, stream2): """ Assert filter unit works as intended @@ -1395,7 +1428,6 @@ def test_filter_unit(self, stream1, stream2): other = streams.filter(unit=re.compile("meters")) assert other._streams == [] - def test_filter_tags(self, stream1, stream2): """ Assert filter annotations works as intended @@ -1414,7 +1446,6 @@ def test_filter_tags(self, stream1, stream2): other = streams.filter(tags={"unit": "volts"}) assert other._streams == [stream1, stream2] - def test_filter_annotations(self, stream1, stream2): """ Assert filter annotations works as intended @@ -1433,7 +1464,6 @@ def test_filter_annotations(self, stream1, stream2): other = streams.filter(annotations={"owner": "ABC", "color": "red"}) assert other._streams == [stream1] - ########################################################################## ## clone tests ########################################################################## @@ -1448,12 +1478,11 @@ def test_clone_returns_new_object(self, stream1, stream2): assert id(clone) != id(streams) assert clone._streams is streams._streams - ########################################################################## ## windows tests ########################################################################## - def test_windows_raises_valueerror(self, stream1): + def test_windows_raises_valueerror_and_warning(self, stream1): """ Assert that raises ValueError if arguments not castable to int """ @@ -1462,10 +1491,8 @@ def test_windows_raises_valueerror(self, stream1): streams.windows("invalid", 42) assert "literal" in str(exc).lower() - with pytest.raises(ValueError) as exc: + with pytest.warns(Warning, match="Overriding") as exc: streams.windows(42, "invalid") - assert "literal" in str(exc).lower() - def test_windows_raises_if_not_allowed(self, stream1): """ @@ -1478,7 +1505,6 @@ def test_windows_raises_if_not_allowed(self, stream1): streams.windows(10, 20) assert "window operation is already requested" in str(exc).lower() - def test_windows_returns_self(self, stream1): """ Assert windows returns self @@ -1487,7 +1513,6 @@ def test_windows_returns_self(self, stream1): result = streams.windows(10, 20) assert result is streams - def test_windows_stores_values(self, stream1): """ Assert windows stores values @@ -1496,21 +1521,24 @@ def test_windows_stores_values(self, stream1): result = streams.windows(10, 20) # assert stores values - assert streams.width == 10 - assert streams.depth == 20 - + assert result.width == 10 + assert result.depth == 0 def test_windows_values_and_calls_to_endpoint(self): """ assert windows result and endpoint calls are correct """ endpoint = Mock(Endpoint) - window1 = [[(StatPointProto(time=1,min=2,mean=3,max=4,count=5,stddev=6),), 11]] - window2 = [[(StatPointProto(time=2,min=3,mean=4,max=5,count=6,stddev=7),), 12]] - endpoint.windows = Mock(side_effect=[ window1, window2 ]) + window1 = [ + [(StatPointProto(time=1, min=2, mean=3, max=4, count=5, stddev=6),), 11] + ] + window2 = [ + [(StatPointProto(time=2, min=3, mean=4, max=5, count=6, stddev=7),), 12] + ] + endpoint.windows = Mock(side_effect=[window1, window2]) - uu1 = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') - uu2 = uuid.UUID('5d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu1 = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + uu2 = uuid.UUID("5d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") s1 = Stream(btrdb=BTrDB(endpoint), uuid=uu1) s2 = Stream(btrdb=BTrDB(endpoint), uuid=uu2) versions = {uu1: 11, uu2: 12} @@ -1518,12 +1546,16 @@ def test_windows_values_and_calls_to_endpoint(self): start, end, width, depth = 1, 100, 1000, 25 streams = StreamSet([s1, s2]) streams.pin_versions(versions) - values = streams.filter(start=start, end=end).windows(width=width, depth=depth).values() + values = ( + streams.filter(start=start, end=end) + .windows(width=width, depth=depth) + .values() + ) # assert endpoint calls have correct arguments, version expected = [ - call(uu1, start, end, width, depth, versions[uu1]), - call(uu2, start, end, width, depth, versions[uu2]) + call(uu1, start, end, width, 0, versions[uu1]), + call(uu2, start, end, width, 0, versions[uu2]), ] assert endpoint.windows.call_args_list == expected @@ -1534,18 +1566,21 @@ def test_windows_values_and_calls_to_endpoint(self): ] assert values == expected - def test_windows_rows_and_calls_to_endpoint(self): """ assert windows rows result and endpoint calls are correct """ endpoint = Mock(Endpoint) - window1 = [[(StatPointProto(time=1,min=2,mean=3,max=4,count=5,stddev=6),), 11]] - window2 = [[(StatPointProto(time=2,min=3,mean=4,max=5,count=6,stddev=7),), 12]] - endpoint.windows = Mock(side_effect=[ window1, window2 ]) + window1 = [ + [(StatPointProto(time=1, min=2, mean=3, max=4, count=5, stddev=6),), 11] + ] + window2 = [ + [(StatPointProto(time=2, min=3, mean=4, max=5, count=6, stddev=7),), 12] + ] + endpoint.windows = Mock(side_effect=[window1, window2]) - uu1 = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') - uu2 = uuid.UUID('5d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu1 = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + uu2 = uuid.UUID("5d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") s1 = Stream(btrdb=BTrDB(endpoint), uuid=uu1) s2 = Stream(btrdb=BTrDB(endpoint), uuid=uu2) versions = {uu1: 11, uu2: 12} @@ -1553,12 +1588,16 @@ def test_windows_rows_and_calls_to_endpoint(self): start, end, width, depth = 1, 100, 1000, 25 streams = StreamSet([s1, s2]) streams.pin_versions(versions) - rows = streams.filter(start=start, end=end).windows(width=width, depth=depth).rows() + rows = ( + streams.filter(start=start, end=end) + .windows(width=width, depth=depth) + .rows() + ) # assert endpoint calls have correct arguments, version expected = [ - call(uu1, start, end, width, depth, versions[uu1]), - call(uu2, start, end, width, depth, versions[uu2]) + call(uu1, start, end, width, 0, versions[uu1]), + call(uu2, start, end, width, 0, versions[uu2]), ] assert endpoint.windows.call_args_list == expected @@ -1569,7 +1608,6 @@ def test_windows_rows_and_calls_to_endpoint(self): ] assert rows == expected - ########################################################################## ## aligned_windows tests ########################################################################## @@ -1583,7 +1621,6 @@ def test_aligned_windows_raises_valueerror(self, stream1): streams.aligned_windows("invalid") assert "literal" in str(exc).lower() - def test_aligned_windows_raises_if_not_allowed(self, stream1): """ Assert that aligned_windows raises Exception if not allowed @@ -1595,7 +1632,6 @@ def test_aligned_windows_raises_if_not_allowed(self, stream1): streams.aligned_windows(20) assert "window operation is already requested" in str(exc).lower() - def test_aligned_windows(self, stream1): """ Assert aligned_windows stores objects @@ -1604,7 +1640,6 @@ def test_aligned_windows(self, stream1): result = streams.aligned_windows(20) assert streams.pointwidth == 20 - def test_aligned_windows_returns_self(self, stream1): """ Assert aligned_windows returns self @@ -1613,18 +1648,21 @@ def test_aligned_windows_returns_self(self, stream1): result = streams.aligned_windows(20) assert result is streams - def test_aligned_windows_values_and_calls_to_endpoint(self): """ assert aligned_windows result and endpoint calls are correct """ endpoint = Mock(Endpoint) - window1 = [[(StatPointProto(time=1,min=2,mean=3,max=4,count=5,stddev=6),), 11]] - window2 = [[(StatPointProto(time=2,min=3,mean=4,max=5,count=6,stddev=7),), 12]] - endpoint.alignedWindows = Mock(side_effect=[ window1, window2 ]) + window1 = [ + [(StatPointProto(time=1, min=2, mean=3, max=4, count=5, stddev=6),), 11] + ] + window2 = [ + [(StatPointProto(time=2, min=3, mean=4, max=5, count=6, stddev=7),), 12] + ] + endpoint.alignedWindows = Mock(side_effect=[window1, window2]) - uu1 = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') - uu2 = uuid.UUID('5d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu1 = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + uu2 = uuid.UUID("5d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") s1 = Stream(btrdb=BTrDB(endpoint), uuid=uu1) s2 = Stream(btrdb=BTrDB(endpoint), uuid=uu2) versions = {uu1: 11, uu2: 12} @@ -1632,12 +1670,16 @@ def test_aligned_windows_values_and_calls_to_endpoint(self): start, end, pointwidth = 1, 100, 25 streams = StreamSet([s1, s2]) streams.pin_versions(versions) - values = streams.filter(start=start, end=end).aligned_windows(pointwidth=pointwidth).values() + values = ( + streams.filter(start=start, end=end) + .aligned_windows(pointwidth=pointwidth) + .values() + ) # assert endpoint calls have correct arguments, version expected = [ call(uu1, start, end, pointwidth, versions[uu1]), - call(uu2, start, end, pointwidth, versions[uu2]) + call(uu2, start, end, pointwidth, versions[uu2]), ] assert endpoint.alignedWindows.call_args_list == expected @@ -1648,18 +1690,21 @@ def test_aligned_windows_values_and_calls_to_endpoint(self): ] assert values == expected - def test_aligned_windows_rows_and_calls_to_endpoint(self): """ assert aligned_windows rows result and endpoint calls are correct """ endpoint = Mock(Endpoint) - window1 = [[(StatPointProto(time=1,min=2,mean=3,max=4,count=5,stddev=6),), 11]] - window2 = [[(StatPointProto(time=2,min=3,mean=4,max=5,count=6,stddev=7),), 12]] - endpoint.alignedWindows = Mock(side_effect=[ window1, window2 ]) + window1 = [ + [(StatPointProto(time=1, min=2, mean=3, max=4, count=5, stddev=6),), 11] + ] + window2 = [ + [(StatPointProto(time=2, min=3, mean=4, max=5, count=6, stddev=7),), 12] + ] + endpoint.alignedWindows = Mock(side_effect=[window1, window2]) - uu1 = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') - uu2 = uuid.UUID('5d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu1 = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + uu2 = uuid.UUID("5d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") s1 = Stream(btrdb=BTrDB(endpoint), uuid=uu1) s2 = Stream(btrdb=BTrDB(endpoint), uuid=uu2) versions = {uu1: 11, uu2: 12} @@ -1667,12 +1712,16 @@ def test_aligned_windows_rows_and_calls_to_endpoint(self): start, end, pointwidth = 1, 100, 25 streams = StreamSet([s1, s2]) streams.pin_versions(versions) - rows = streams.filter(start=start, end=end).aligned_windows(pointwidth=pointwidth).rows() + rows = ( + streams.filter(start=start, end=end) + .aligned_windows(pointwidth=pointwidth) + .rows() + ) # assert endpoint calls have correct arguments, version expected = [ call(uu1, start, end, pointwidth, versions[uu1]), - call(uu2, start, end, pointwidth, versions[uu2]) + call(uu2, start, end, pointwidth, versions[uu2]), ] assert endpoint.alignedWindows.call_args_list == expected @@ -1683,8 +1732,6 @@ def test_aligned_windows_rows_and_calls_to_endpoint(self): ] assert rows == expected - - ########################################################################## ## rows tests ########################################################################## @@ -1693,13 +1740,21 @@ def test_rows(self, stream1, stream2): """ Assert rows returns correct values """ - stream1.values = Mock(return_value=iter([ - (RawPoint(time=1, value=1), 1), (RawPoint(time=2, value=2), 1), - (RawPoint(time=3, value=3), 1), (RawPoint(time=4, value=4), 1), - ])) - stream2.values = Mock(return_value=iter([ - (RawPoint(time=1, value=1), 2), (RawPoint(time=3, value=3), 2) - ])) + stream1.values = Mock( + return_value=iter( + [ + (RawPoint(time=1, value=1), 1), + (RawPoint(time=2, value=2), 1), + (RawPoint(time=3, value=3), 1), + (RawPoint(time=4, value=4), 1), + ] + ) + ) + stream2.values = Mock( + return_value=iter( + [(RawPoint(time=1, value=1), 2), (RawPoint(time=3, value=3), 2)] + ) + ) streams = StreamSet([stream1, stream2]) rows = iter(streams.rows()) @@ -1709,7 +1764,6 @@ def test_rows(self, stream1, stream2): assert next(rows) == (RawPoint(time=3, value=3), RawPoint(time=3, value=3)) assert next(rows) == (RawPoint(time=4, value=4), None) - ########################################################################## ## _params_from_filters tests ########################################################################## @@ -1727,7 +1781,6 @@ def test_params_from_filters_latest_value_wins(self, stream1): streams = streams.filter(start=2) assert streams._params_from_filters() == {"start": 2} - def test_params_from_filters_works(self, stream1): """ Assert _params_from_filters returns correct values @@ -1741,7 +1794,6 @@ def test_params_from_filters_works(self, stream1): streams = streams.filter(start=9, end=10) assert streams._params_from_filters() == {"start": 9, "end": 10} - ########################################################################## ## values tests ########################################################################## @@ -1751,13 +1803,16 @@ def test_values(self, stream1, stream2): Assert values returns correct data """ stream1_values = [ - (RawPoint(time=1, value=1), 1), (RawPoint(time=2, value=2), 1), - (RawPoint(time=3, value=3), 1), (RawPoint(time=4, value=4), 1), + (RawPoint(time=1, value=1), 1), + (RawPoint(time=2, value=2), 1), + (RawPoint(time=3, value=3), 1), + (RawPoint(time=4, value=4), 1), ] stream1.values = Mock(return_value=iter(stream1_values)) stream2_values = [ - (RawPoint(time=1, value=1), 2), (RawPoint(time=3, value=3), 2) + (RawPoint(time=1, value=1), 2), + (RawPoint(time=3, value=3), 2), ] stream2.values = Mock(return_value=iter(stream2_values)) @@ -1772,20 +1827,20 @@ def test_values(self, stream1, stream2): ## StreamFilter Tests ########################################################################## -class TestStreamFilter(object): +class TestStreamFilter(object): def test_create(self): """ Assert we can create the object """ - StreamFilter(0,1) + StreamFilter(0, 1) def test_start_larger_or_equal(self): """ Assert we raise ValueError if start is greater than/equal to end """ with pytest.raises(ValueError): - StreamFilter(1,1) + StreamFilter(1, 1) def test_start_valid(self): """ diff --git a/tests/btrdb/test_transformers.py b/tests/btrdb/test_transformers.py index 5cdc0a5..56aad5a 100644 --- a/tests/btrdb/test_transformers.py +++ b/tests/btrdb/test_transformers.py @@ -39,6 +39,8 @@ def streamset(): stream = Mock(Stream) type(stream).collection = PropertyMock(return_value="test") type(stream).name = PropertyMock(return_value="stream{}".format(idx)) + stream._btrdb = Mock() + stream._btrdb._ARROW_ENABLED = Mock(return_value=False) streams.append(stream) rows = [ From 5217656c056ad1fef052d4fbaafb1e6a0279987d Mon Sep 17 00:00:00 2001 From: Justin Gilmer Date: Tue, 6 Jun 2023 17:21:56 -0500 Subject: [PATCH 035/131] Add 3.10 python to the testing matrix (#21) * Add 3.10 python to the testing matrix * Fix yaml parsing * Update requirements to support 3.10 * Use pip-tools `pip-compile` cli tool to generate requirements.txt files from the updated pyproject.toml file * Include pyproject.toml with basic features to support proper extra deps * Support different ways to install btrdb from pip * `btrdb, btrdb[data], btrdb[all], btrdb[testing], btrdb[ray]` * Update transformers.py to build up a numpy array when the subarrays are not the same size (number of entries) * This converts the main array's dtype to `object` * tests still pass with this change * recompile the btrdb proto files with latest protobuf and grpc plugins * Create multiple requirements.txt files for easier updating in the future as well as a locked version with pinned dependencies * Ignore protoc generated flake errors * Update test requirements * Include pre-commit and setup. * Pre-commit lints. --- .github/workflows/pre-commit.yaml | 38 ++ .github/workflows/release.yaml | 2 +- .pre-commit-config.yaml | 35 ++ MANIFEST.in | 2 +- btrdb/__init__.py | 18 +- btrdb/conn.py | 10 +- btrdb/endpoint.py | 5 +- btrdb/exceptions.py | 110 +++- btrdb/grpcinterface/btrdb_pb2.py | 2 +- btrdb/point.py | 29 +- btrdb/stream.py | 2 +- btrdb/transformers.py | 2 +- btrdb/utils/buffer.py | 9 +- btrdb/utils/conversion.py | 12 +- btrdb/utils/credentials.py | 40 +- btrdb/utils/general.py | 13 +- btrdb/utils/ray.py | 37 +- btrdb/utils/timez.py | 25 +- btrdb/version.py | 17 +- btrdb4/__init__.py | 8 +- docs/requirements.txt | 2 +- docs/source/conf.py | 92 +-- docs/source/installing.rst | 2 - docs/source/working/multiprocessing.rst | 2 +- docs/source/working/ray.rst | 4 +- .../source/working/stream-manage-metadata.rst | 2 - pyproject.toml | 76 +++ release.sh | 2 +- requirements-all-locked.txt | 128 +++++ requirements-all.txt | 57 ++ requirements.txt | 31 +- setup.cfg | 36 +- setup.py | 73 +-- tests/btrdb/test_base.py | 48 +- tests/btrdb/test_conn.py | 79 ++- tests/btrdb/test_point.py | 29 +- tests/btrdb/test_stream.py | 17 +- tests/btrdb/test_transformers.py | 544 +++++++++++++++--- tests/btrdb/utils/test_buffer.py | 8 +- tests/btrdb/utils/test_conversions.py | 81 +-- tests/btrdb/utils/test_credentials.py | 36 +- tests/btrdb/utils/test_general.py | 50 +- tests/btrdb/utils/test_timez.py | 51 +- tests/btrdb4/test_import.py | 4 +- tests/requirements.txt | 65 ++- 45 files changed, 1390 insertions(+), 545 deletions(-) create mode 100644 .github/workflows/pre-commit.yaml create mode 100644 .pre-commit-config.yaml create mode 100644 pyproject.toml create mode 100644 requirements-all-locked.txt create mode 100644 requirements-all.txt diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml new file mode 100644 index 0000000..858e465 --- /dev/null +++ b/.github/workflows/pre-commit.yaml @@ -0,0 +1,38 @@ +name: pre-commit + +on: + pull_request: + branches: + - master + types: + - opened + - reopened + - ready_for_review + - synchronize + +env: + SKIP: pytest-check + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + fetch-depth: 0 # get full git history + - uses: actions/setup-python@v3 + with: + cache: 'pip' + - name: Install pre-commit + run: | + pip install pre-commit + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v21 + with: + token: ${{ secrets.GITHUB_TOKEN }} + - name: Run pre-commit + uses: pre-commit/action@v2.0.3 + with: + extra_args: --files ${{ steps.changed-files.outputs.all_changed_files }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 61de84e..8946559 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -14,7 +14,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: [3.7, 3.8, 3.9] + python-version: [3.7, 3.8, 3.9, '3.10'] os: [ubuntu-latest, macos-latest, windows-latest] steps: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..c2da34a --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,35 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + exclude: ^(setup.cfg|btrdb/grpcinterface) +- repo: https://github.com/psf/black + rev: 23.3.0 + hooks: + - id: black-jupyter + args: [--line-length=88] + exclude: btrdb/grpcinterface/.*\.py +- repo: https://github.com/pycqa/isort + rev: 5.11.5 + hooks: + - id: isort + name: isort (python) + args: [--profile=black, --line-length=88] + exclude: btrdb/grpcinterface/.*\.py +- repo: https://github.com/PyCQA/flake8 + rev: 6.0.0 + hooks: + - id: flake8 + args: [--config=setup.cfg] + exclude: ^(btrdb/grpcinterface|tests|setup.py|btrdb4|docs) +- repo: local + hooks: + - id: pytest-check + name: pytest-check + entry: pytest + language: system + pass_filenames: false + always_run: true diff --git a/MANIFEST.in b/MANIFEST.in index 89c5b46..7f67c08 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -20,4 +20,4 @@ global-exclude *.py[co] global-exclude .ipynb_checkpoints global-exclude .DS_Store global-exclude .env -global-exclude .coverage.* \ No newline at end of file +global-exclude .coverage.* diff --git a/btrdb/__init__.py b/btrdb/__init__.py index cd14662..efcb686 100644 --- a/btrdb/__init__.py +++ b/btrdb/__init__.py @@ -15,14 +15,15 @@ ## Imports ########################################################################## -from btrdb.conn import Connection, BTrDB +from warnings import warn + +from btrdb.conn import BTrDB, Connection from btrdb.endpoint import Endpoint from btrdb.exceptions import ConnectionError -from btrdb.version import get_version -from btrdb.utils.credentials import credentials_by_profile, credentials +from btrdb.stream import MAXIMUM_TIME, MINIMUM_TIME +from btrdb.utils.credentials import credentials, credentials_by_profile from btrdb.utils.ray import register_serializer -from btrdb.stream import MINIMUM_TIME, MAXIMUM_TIME -from warnings import warn +from btrdb.version import get_version ########################################################################## ## Module Variables @@ -39,9 +40,11 @@ ## Functions ########################################################################## + def _connect(endpoints=None, apikey=None): return BTrDB(Endpoint(Connection(endpoints, apikey=apikey).channel)) + def connect(conn_str=None, apikey=None, profile=None, shareable=False): """ Connect to a BTrDB server. @@ -78,7 +81,9 @@ def connect(conn_str=None, apikey=None, profile=None, shareable=False): # check shareable flag and register custom serializer if necessary if shareable: - warn("a shareable connection is potentially insecure; other users of the same cluster may be able to access your API key") + warn( + "a shareable connection is potentially insecure; other users of the same cluster may be able to access your API key" + ) register_serializer(conn_str=conn_str, apikey=apikey, profile=profile) # use specific profile if requested @@ -91,4 +96,3 @@ def connect(conn_str=None, apikey=None, profile=None, shareable=False): return _connect(**creds) raise ConnectionError("Could not determine credentials to use.") - diff --git a/btrdb/conn.py b/btrdb/conn.py index 617fb1b..4605c11 100644 --- a/btrdb/conn.py +++ b/btrdb/conn.py @@ -15,21 +15,20 @@ ## Imports ########################################################################## +import json import os import re -import json -import certifi -import logging import uuid as uuidlib from concurrent.futures import ThreadPoolExecutor +import certifi import grpc from grpc._cython.cygrpc import CompressionAlgorithm +from btrdb.exceptions import InvalidOperation, StreamNotFoundError from btrdb.stream import Stream, StreamSet -from btrdb.utils.general import unpack_stream_descriptor from btrdb.utils.conversion import to_uuid -from btrdb.exceptions import StreamNotFoundError, InvalidOperation +from btrdb.utils.general import unpack_stream_descriptor ########################################################################## ## Module Variables @@ -128,7 +127,6 @@ def _is_arrow_enabled(info): else: return False - class BTrDB(object): """ The primary server connection object for communicating with a BTrDB server. diff --git a/btrdb/endpoint.py b/btrdb/endpoint.py index aa1b065..3715d3f 100644 --- a/btrdb/endpoint.py +++ b/btrdb/endpoint.py @@ -29,10 +29,9 @@ import grpc -from btrdb.grpcinterface import btrdb_pb2 -from btrdb.grpcinterface import btrdb_pb2_grpc +from btrdb.exceptions import BTrDBError, check_proto_stat, error_handler +from btrdb.grpcinterface import btrdb_pb2, btrdb_pb2_grpc from btrdb.point import RawPoint -from btrdb.exceptions import BTrDBError, error_handler, check_proto_stat from btrdb.utils.general import unpack_stream_descriptor logger = logging.getLogger(__name__) diff --git a/btrdb/exceptions.py b/btrdb/exceptions.py index 2007503..fe13716 100644 --- a/btrdb/exceptions.py +++ b/btrdb/exceptions.py @@ -12,13 +12,15 @@ ########################################################################## import inspect -from grpc import RpcError from functools import wraps +from grpc import RpcError + ########################################################################## ## Decorators ########################################################################## + def consume_generator(fn, *args, **kwargs): # when a generator is passed back to the calling function, it may encounter an error # when trying to call next(), in that case we want to yield an Exception @@ -27,6 +29,7 @@ def consume_generator(fn, *args, **kwargs): except RpcError as e: handle_grpc_error(e) + def error_handler(fn): """ decorates endpoint functions and checks for grpc.RpcErrors @@ -35,6 +38,7 @@ def error_handler(fn): ---------- fn: function """ + # allows input func to keep its name and metadata @wraps(fn) def wrap(*args, **kwargs): @@ -44,12 +48,15 @@ def wrap(*args, **kwargs): return fn(*args, **kwargs) except RpcError as e: handle_grpc_error(e) + return wrap + ########################################################################## ## gRPC error handling ########################################################################## + # NOTE: this function relies on matching strings and isn't really ideal. # this is more of a band-aid solution while we figure out how to send # better errors from btrdb-server @@ -92,141 +99,186 @@ def check_proto_stat(stat): raise BTRDBServerError(stat.msg) raise BTrDBError(stat.msg) + ########################################################################## ## BTrDB Exceptions ########################################################################## + class BTrDBError(Exception): """ The primary exception for BTrDB errors. """ + pass + class ConnectionError(BTrDBError): """ Raised when an error occurrs while trying to establish a connection with BTrDB. """ + pass + class StreamNotFoundError(BTrDBError): """ Raised when attempting to perform an operation on a stream that does not exist in the specified BTrDB allocation. """ + pass + class CredentialsFileNotFound(FileNotFoundError, BTrDBError): """ Raised when a credentials file could not be found. """ + pass + class ProfileNotFound(BTrDBError): """ Raised when a requested profile could not be found in the credentials file. """ + pass + class BTRDBServerError(BTrDBError): """ Raised when an error occurs with btrdb-server. """ + pass + class BTRDBTypeError(TypeError, BTrDBError): """ Raised when attempting to perform an operation with an invalid type. """ + pass + class InvalidOperation(BTrDBError): """ Raised when an invalid BTrDB operation has been requested. """ + pass + class StreamExists(InvalidOperation): """ Raised when create() has been attempted and the uuid already exists. """ + pass + class AmbiguousStream(InvalidOperation): """ Raised when create() has been attempted and uuid is different, but collection and tags already exist """ + pass + class PermissionDenied(InvalidOperation): """ Raised when user does not have permission to perform an operation. """ + pass + class BTRDBValueError(ValueError, BTrDBError): """ Raised when an invalid value has been passed to a BTrDB operation. """ + pass + class InvalidCollection(BTRDBValueError): """ Raised when a collection name is invalid. It is either too long or not a valid string. """ + pass + class InvalidTagKey(BTRDBValueError): """ Raised when a tag key is invalid. Must be one of ("name", "unit", "ingress", "distiller"). """ + pass + class InvalidTagValue(BTRDBValueError): """ Raised when a tag value is invalid. It is either too long or not a valid string. """ + pass + class InvalidTimeRange(BTRDBValueError): """ Raised when insert data contains a timestamp outside the range of (btrdb.MINIMUM_TIME, btrdb.MAXIMUM_TIME) """ + pass + class InvalidPointWidth(BTRDBValueError): """ Raised when attempting to use a pointwidth that is not a whole number between 0 and 64 (exclusive). """ + pass + class BadValue(BTRDBValueError): """ Raised when attempting to insert data that contains non-float values such as None. """ + pass + class RecycledUUID(BTRDBValueError): """ Raised when attempting to create a stream with a uuid that matches a previously deleted stream. """ + pass + class BadSQLValue(BTRDBValueError): """ Raised when invalid parameters have been passed to metadata db. """ + pass + class VersionNotAvailable(BTRDBValueError): """ Raised when querying a stream at a pruned or otherwise invalid version number. """ + pass + class NoSuchPoint(BTRDBValueError): """ Raised when asking for next/previous point and there isn't one. """ + pass @@ -248,35 +300,35 @@ class NoSuchPoint(BTRDBValueError): 425: BadValue, 429: RecycledUUID, 441: BadSQLValue, - 450: VersionNotAvailable + 450: VersionNotAvailable, } # All of these raise BTRDBServerError BTRDB_SERVER_ERRORS = [ - 402, # ContextError - 403, # InsertFailure - 405, # WrongEndpoint - 414, # InsertTooBig - 421, # WrongArgs - 423, # AnnotationVersionMismatch - 424, # FaultInjectionDisabled - 426, # ResourceDepleted - 427, # InvalidVersions - 431, # ObliterateDisabled - 432, # CephError - 433, # NodeExisted - 434, # JournalError - 438, # InvalidParameter - 440, # MetadataConnectionError - 452, # OverlappingTrimRange - 453, # LockFailed - 454, # NoLeafNode - 455, # TierCacheError - 456, # StreamEmpty - 457, # TieredStorageBackendError - 458, # TieredStorageOutOfBounds - 459, # TieredStorageTemporaryError - 500, # InvariantFailure - 501, # NotImplemented - 502, # UnsupportedRollback -] \ No newline at end of file + 402, # ContextError + 403, # InsertFailure + 405, # WrongEndpoint + 414, # InsertTooBig + 421, # WrongArgs + 423, # AnnotationVersionMismatch + 424, # FaultInjectionDisabled + 426, # ResourceDepleted + 427, # InvalidVersions + 431, # ObliterateDisabled + 432, # CephError + 433, # NodeExisted + 434, # JournalError + 438, # InvalidParameter + 440, # MetadataConnectionError + 452, # OverlappingTrimRange + 453, # LockFailed + 454, # NoLeafNode + 455, # TierCacheError + 456, # StreamEmpty + 457, # TieredStorageBackendError + 458, # TieredStorageOutOfBounds + 459, # TieredStorageTemporaryError + 500, # InvariantFailure + 501, # NotImplemented + 502, # UnsupportedRollback +] diff --git a/btrdb/grpcinterface/btrdb_pb2.py b/btrdb/grpcinterface/btrdb_pb2.py index ace467e..bed2b0d 100644 --- a/btrdb/grpcinterface/btrdb_pb2.py +++ b/btrdb/grpcinterface/btrdb_pb2.py @@ -13,8 +13,8 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0b\x62trdb.proto\x12\x05v5api\"Q\n\x0fRawValuesParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\"}\n\x11RawValuesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x1f\n\x06values\x18\x04 \x03(\x0b\x32\x0f.v5api.RawPoint\"u\n\x16\x41rrowRawValuesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x12\n\narrowBytes\x18\x04 \x01(\x0c\"n\n\x16\x41rrowMultiValuesParams\x12\x0c\n\x04uuid\x18\x01 \x03(\x0c\x12\x14\n\x0cversionMajor\x18\x02 \x03(\x04\x12\r\n\x05start\x18\x03 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x04 \x01(\x10\x12\x14\n\x0csnapPeriodNs\x18\x05 \x01(\x03\"K\n\x18\x41rrowMultiValuesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x12\n\narrowBytes\x18\x02 \x01(\x0c\"*\n\x0bRawPointVec\x12\x0c\n\x04time\x18\x01 \x01(\x10\x12\r\n\x05value\x18\x02 \x03(\x01\"j\n\x14\x41lignedWindowsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\x12\x12\n\npointWidth\x18\x05 \x01(\r\"\x83\x01\n\x16\x41lignedWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12 \n\x06values\x18\x04 \x03(\x0b\x32\x10.v5api.StatPoint\"z\n\x1b\x41rrowAlignedWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x12\n\narrowBytes\x18\x04 \x01(\x0c\"m\n\rWindowsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\x12\r\n\x05width\x18\x05 \x01(\x04\x12\r\n\x05\x64\x65pth\x18\x06 \x01(\r\"|\n\x0fWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12 \n\x06values\x18\x04 \x03(\x0b\x32\x10.v5api.StatPoint\"s\n\x14\x41rrowWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x12\n\narrowBytes\x18\x04 \x01(\x0c\"h\n\x10StreamInfoParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x13\n\x0bomitVersion\x18\x02 \x01(\x08\x12\x16\n\x0eomitDescriptor\x18\x03 \x01(\x08\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"\x8a\x01\n\x12StreamInfoResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12+\n\ndescriptor\x18\x04 \x01(\x0b\x32\x17.v5api.StreamDescriptor\"\x98\x01\n\x10StreamDescriptor\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x12\n\ncollection\x18\x02 \x01(\t\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x17\n\x0fpropertyVersion\x18\x05 \x01(\x04\"\x82\x01\n\x1aSetStreamAnnotationsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x1f\n\x17\x65xpectedPropertyVersion\x18\x02 \x01(\x04\x12#\n\x07\x63hanges\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x10\n\x08removals\x18\x04 \x03(\t\";\n\x1cSetStreamAnnotationsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"\x8a\x01\n\x13SetStreamTagsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x1f\n\x17\x65xpectedPropertyVersion\x18\x02 \x01(\x04\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x12\n\ncollection\x18\x04 \x01(\t\x12\x0e\n\x06remove\x18\x05 \x03(\t\"4\n\x15SetStreamTagsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"{\n\x0c\x43reateParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x12\n\ncollection\x18\x02 \x01(\t\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\"-\n\x0e\x43reateResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"@\n\x13MetadataUsageParams\x12\x0e\n\x06prefix\x18\x01 \x01(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"y\n\x15MetadataUsageResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x1d\n\x04tags\x18\x02 \x03(\x0b\x32\x0f.v5api.KeyCount\x12$\n\x0b\x61nnotations\x18\x03 \x03(\x0b\x32\x0f.v5api.KeyCount\"&\n\x08KeyCount\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05\x63ount\x18\x02 \x01(\x04\"B\n\x15ListCollectionsParams\x12\x0e\n\x06prefix\x18\x01 \x01(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"K\n\x17ListCollectionsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x13\n\x0b\x63ollections\x18\x02 \x03(\t\"\xab\x01\n\x13LookupStreamsParams\x12\x12\n\ncollection\x18\x01 \x01(\t\x12\x1a\n\x12isCollectionPrefix\x18\x02 \x01(\x08\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"^\n\x15LookupStreamsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12(\n\x07results\x18\x02 \x03(\x0b\x32\x17.v5api.StreamDescriptor\"S\n\rNearestParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x0c\n\x04time\x18\x02 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x03 \x01(\x04\x12\x10\n\x08\x62\x61\x63kward\x18\x04 \x01(\x08\"z\n\x0fNearestResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x1e\n\x05value\x18\x04 \x01(\x0b\x32\x0f.v5api.RawPoint\"U\n\rChangesParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x11\n\tfromMajor\x18\x02 \x01(\x04\x12\x0f\n\x07toMajor\x18\x03 \x01(\x04\x12\x12\n\nresolution\x18\x04 \x01(\r\"\x7f\n\x0f\x43hangesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12#\n\x06ranges\x18\x04 \x03(\x0b\x32\x13.v5api.ChangedRange\"#\n\tRoundSpec\x12\x0e\n\x04\x62its\x18\x02 \x01(\x05H\x00\x42\x06\n\x04spec\"\x99\x01\n\x0cInsertParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x0c\n\x04sync\x18\x02 \x01(\x08\x12(\n\x0cmerge_policy\x18\x04 \x01(\x0e\x32\x12.v5api.MergePolicy\x12\"\n\x08rounding\x18\x05 \x01(\x0b\x32\x10.v5api.RoundSpec\x12\x1f\n\x06values\x18\x03 \x03(\x0b\x32\x0f.v5api.RawPoint\"\x91\x01\n\x11\x41rrowInsertParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x0c\n\x04sync\x18\x02 \x01(\x08\x12(\n\x0cmerge_policy\x18\x03 \x01(\x0e\x32\x12.v5api.MergePolicy\x12\"\n\x08rounding\x18\x04 \x01(\x0b\x32\x10.v5api.RoundSpec\x12\x12\n\narrowBytes\x18\x05 \x01(\x0c\"Y\n\x0eInsertResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\"8\n\x0c\x44\x65leteParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\"Y\n\x0e\x44\x65leteResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\"\x0c\n\nInfoParams\"\xa2\x01\n\x0cInfoResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x19\n\x04mash\x18\x02 \x01(\x0b\x32\x0b.v5api.Mash\x12\x14\n\x0cmajorVersion\x18\x03 \x01(\r\x12\x14\n\x0cminorVersion\x18\x04 \x01(\r\x12\r\n\x05\x62uild\x18\x05 \x01(\t\x12\x1f\n\x05proxy\x18\x06 \x01(\x0b\x32\x10.v5api.ProxyInfo\"#\n\tProxyInfo\x12\x16\n\x0eproxyEndpoints\x18\x01 \x03(\t\"1\n\x11\x46\x61ultInjectParams\x12\x0c\n\x04type\x18\x01 \x01(\x04\x12\x0e\n\x06params\x18\x02 \x01(\x0c\">\n\x13\x46\x61ultInjectResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\n\n\x02rv\x18\x02 \x01(\x0c\"\x1b\n\x0b\x46lushParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"X\n\rFlushResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\" \n\x10ObliterateParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"1\n\x12ObliterateResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"\'\n\x08RawPoint\x12\x0c\n\x04time\x18\x01 \x01(\x10\x12\r\n\x05value\x18\x02 \x01(\x01\"`\n\tStatPoint\x12\x0c\n\x04time\x18\x01 \x01(\x10\x12\x0b\n\x03min\x18\x02 \x01(\x01\x12\x0c\n\x04mean\x18\x03 \x01(\x01\x12\x0b\n\x03max\x18\x04 \x01(\x01\x12\r\n\x05\x63ount\x18\x05 \x01(\x06\x12\x0e\n\x06stddev\x18\x06 \x01(\x01\"*\n\x0c\x43hangedRange\x12\r\n\x05start\x18\x01 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x02 \x01(\x10\">\n\x06Status\x12\x0c\n\x04\x63ode\x18\x01 \x01(\r\x12\x0b\n\x03msg\x18\x02 \x01(\t\x12\x19\n\x04mash\x18\x03 \x01(\x0b\x32\x0b.v5api.Mash\"\x98\x01\n\x04Mash\x12\x10\n\x08revision\x18\x01 \x01(\x03\x12\x0e\n\x06leader\x18\x02 \x01(\t\x12\x16\n\x0eleaderRevision\x18\x03 \x01(\x03\x12\x13\n\x0btotalWeight\x18\x04 \x01(\x03\x12\x0f\n\x07healthy\x18\x05 \x01(\x08\x12\x10\n\x08unmapped\x18\x06 \x01(\x01\x12\x1e\n\x07members\x18\x07 \x03(\x0b\x32\r.v5api.Member\"\xc3\x01\n\x06Member\x12\x0c\n\x04hash\x18\x01 \x01(\r\x12\x10\n\x08nodename\x18\x02 \x01(\t\x12\n\n\x02up\x18\x03 \x01(\x08\x12\n\n\x02in\x18\x04 \x01(\x08\x12\x0f\n\x07\x65nabled\x18\x05 \x01(\x08\x12\r\n\x05start\x18\x06 \x01(\x03\x12\x0b\n\x03\x65nd\x18\x07 \x01(\x03\x12\x0e\n\x06weight\x18\x08 \x01(\x03\x12\x16\n\x0ereadPreference\x18\t \x01(\x01\x12\x15\n\rhttpEndpoints\x18\n \x01(\t\x12\x15\n\rgrpcEndpoints\x18\x0b \x01(\t\"8\n\x0bKeyOptValue\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x1c\n\x03val\x18\x02 \x01(\x0b\x32\x0f.v5api.OptValue\"\x19\n\x08OptValue\x12\r\n\x05value\x18\x01 \x01(\t\"&\n\x08KeyValue\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"?\n\x0fStreamCSVConfig\x12\x0f\n\x07version\x18\x01 \x01(\x04\x12\r\n\x05label\x18\x02 \x01(\t\x12\x0c\n\x04uuid\x18\x03 \x01(\x0c\"\x9d\x02\n\x11GenerateCSVParams\x12\x35\n\tqueryType\x18\x01 \x01(\x0e\x32\".v5api.GenerateCSVParams.QueryType\x12\x11\n\tstartTime\x18\x02 \x01(\x03\x12\x0f\n\x07\x65ndTime\x18\x03 \x01(\x03\x12\x12\n\nwindowSize\x18\x04 \x01(\x04\x12\r\n\x05\x64\x65pth\x18\x05 \x01(\r\x12\x17\n\x0fincludeVersions\x18\x06 \x01(\x08\x12\'\n\x07streams\x18\x07 \x03(\x0b\x32\x16.v5api.StreamCSVConfig\"H\n\tQueryType\x12\x19\n\x15\x41LIGNED_WINDOWS_QUERY\x10\x00\x12\x11\n\rWINDOWS_QUERY\x10\x01\x12\r\n\tRAW_QUERY\x10\x02\"Q\n\x13GenerateCSVResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x10\n\x08isHeader\x18\x02 \x01(\x08\x12\x0b\n\x03row\x18\x03 \x03(\t\"J\n\x0eSQLQueryParams\x12\r\n\x05query\x18\x01 \x01(\t\x12\x0e\n\x06params\x18\x02 \x03(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"D\n\x10SQLQueryResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x13\n\x0bSQLQueryRow\x18\x02 \x03(\x0c\"\x14\n\x04Role\x12\x0c\n\x04name\x18\x01 \x01(\t\"\x94\x01\n\x19SetCompactionConfigParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x18\n\x10\x43ompactedVersion\x18\x02 \x01(\x04\x12>\n\x17reducedResolutionRanges\x18\x03 \x03(\x0b\x32\x1d.v5api.ReducedResolutionRange\x12\x0f\n\x07unused0\x18\x04 \x01(\x04\":\n\x1bSetCompactionConfigResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\")\n\x19GetCompactionConfigParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"\xc1\x01\n\x1bGetCompactionConfigResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x1a\n\x12LatestMajorVersion\x18\x02 \x01(\x04\x12\x18\n\x10\x43ompactedVersion\x18\x03 \x01(\x04\x12>\n\x17reducedResolutionRanges\x18\x04 \x03(\x0b\x32\x1d.v5api.ReducedResolutionRange\x12\x0f\n\x07unused0\x18\x05 \x01(\x04\"H\n\x16ReducedResolutionRange\x12\r\n\x05Start\x18\x01 \x01(\x03\x12\x0b\n\x03\x45nd\x18\x02 \x01(\x03\x12\x12\n\nResolution\x18\x03 \x01(\r*<\n\x0bMergePolicy\x12\t\n\x05NEVER\x10\x00\x12\t\n\x05\x45QUAL\x10\x01\x12\n\n\x06RETAIN\x10\x02\x12\x0b\n\x07REPLACE\x10\x03\x32\xb3\r\n\x05\x42TrDB\x12?\n\tRawValues\x12\x16.v5api.RawValuesParams\x1a\x18.v5api.RawValuesResponse0\x01\x12I\n\x0e\x41rrowRawValues\x12\x16.v5api.RawValuesParams\x1a\x1d.v5api.ArrowRawValuesResponse0\x01\x12T\n\x10\x41rrowMultiValues\x12\x1d.v5api.ArrowMultiValuesParams\x1a\x1f.v5api.ArrowMultiValuesResponse0\x01\x12N\n\x0e\x41lignedWindows\x12\x1b.v5api.AlignedWindowsParams\x1a\x1d.v5api.AlignedWindowsResponse0\x01\x12X\n\x13\x41rrowAlignedWindows\x12\x1b.v5api.AlignedWindowsParams\x1a\".v5api.ArrowAlignedWindowsResponse0\x01\x12\x39\n\x07Windows\x12\x14.v5api.WindowsParams\x1a\x16.v5api.WindowsResponse0\x01\x12\x43\n\x0c\x41rrowWindows\x12\x14.v5api.WindowsParams\x1a\x1b.v5api.ArrowWindowsResponse0\x01\x12@\n\nStreamInfo\x12\x17.v5api.StreamInfoParams\x1a\x19.v5api.StreamInfoResponse\x12^\n\x14SetStreamAnnotations\x12!.v5api.SetStreamAnnotationsParams\x1a#.v5api.SetStreamAnnotationsResponse\x12I\n\rSetStreamTags\x12\x1a.v5api.SetStreamTagsParams\x1a\x1c.v5api.SetStreamTagsResponse\x12\x34\n\x06\x43reate\x12\x13.v5api.CreateParams\x1a\x15.v5api.CreateResponse\x12Q\n\x0fListCollections\x12\x1c.v5api.ListCollectionsParams\x1a\x1e.v5api.ListCollectionsResponse0\x01\x12K\n\rLookupStreams\x12\x1a.v5api.LookupStreamsParams\x1a\x1c.v5api.LookupStreamsResponse0\x01\x12\x37\n\x07Nearest\x12\x14.v5api.NearestParams\x1a\x16.v5api.NearestResponse\x12\x39\n\x07\x43hanges\x12\x14.v5api.ChangesParams\x1a\x16.v5api.ChangesResponse0\x01\x12\x34\n\x06Insert\x12\x13.v5api.InsertParams\x1a\x15.v5api.InsertResponse\x12>\n\x0b\x41rrowInsert\x12\x18.v5api.ArrowInsertParams\x1a\x15.v5api.InsertResponse\x12\x34\n\x06\x44\x65lete\x12\x13.v5api.DeleteParams\x1a\x15.v5api.DeleteResponse\x12.\n\x04Info\x12\x11.v5api.InfoParams\x1a\x13.v5api.InfoResponse\x12\x43\n\x0b\x46\x61ultInject\x12\x18.v5api.FaultInjectParams\x1a\x1a.v5api.FaultInjectResponse\x12\x31\n\x05\x46lush\x12\x12.v5api.FlushParams\x1a\x14.v5api.FlushResponse\x12@\n\nObliterate\x12\x17.v5api.ObliterateParams\x1a\x19.v5api.ObliterateResponse\x12L\n\x10GetMetadataUsage\x12\x1a.v5api.MetadataUsageParams\x1a\x1c.v5api.MetadataUsageResponse\x12\x45\n\x0bGenerateCSV\x12\x18.v5api.GenerateCSVParams\x1a\x1a.v5api.GenerateCSVResponse0\x01\x12<\n\x08SQLQuery\x12\x15.v5api.SQLQueryParams\x1a\x17.v5api.SQLQueryResponse0\x01\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0b\x62trdb.proto\x12\x05v5api\"Q\n\x0fRawValuesParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\"}\n\x11RawValuesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x1f\n\x06values\x18\x04 \x03(\x0b\x32\x0f.v5api.RawPoint\"u\n\x16\x41rrowRawValuesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x12\n\narrowBytes\x18\x04 \x01(\x0c\"n\n\x16\x41rrowMultiValuesParams\x12\x0c\n\x04uuid\x18\x01 \x03(\x0c\x12\x14\n\x0cversionMajor\x18\x02 \x03(\x04\x12\r\n\x05start\x18\x03 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x04 \x01(\x10\x12\x14\n\x0csnapPeriodNs\x18\x05 \x01(\x03\"K\n\x18\x41rrowMultiValuesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x12\n\narrowBytes\x18\x02 \x01(\x0c\"*\n\x0bRawPointVec\x12\x0c\n\x04time\x18\x01 \x01(\x10\x12\r\n\x05value\x18\x02 \x03(\x01\"j\n\x14\x41lignedWindowsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\x12\x12\n\npointWidth\x18\x05 \x01(\r\"\x83\x01\n\x16\x41lignedWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12 \n\x06values\x18\x04 \x03(\x0b\x32\x10.v5api.StatPoint\"z\n\x1b\x41rrowAlignedWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x12\n\narrowBytes\x18\x04 \x01(\x0c\"m\n\rWindowsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\x12\r\n\x05width\x18\x05 \x01(\x04\x12\r\n\x05\x64\x65pth\x18\x06 \x01(\r\"|\n\x0fWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12 \n\x06values\x18\x04 \x03(\x0b\x32\x10.v5api.StatPoint\"s\n\x14\x41rrowWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x12\n\narrowBytes\x18\x04 \x01(\x0c\"h\n\x10StreamInfoParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x13\n\x0bomitVersion\x18\x02 \x01(\x08\x12\x16\n\x0eomitDescriptor\x18\x03 \x01(\x08\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"\x8a\x01\n\x12StreamInfoResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12+\n\ndescriptor\x18\x04 \x01(\x0b\x32\x17.v5api.StreamDescriptor\"\x98\x01\n\x10StreamDescriptor\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x12\n\ncollection\x18\x02 \x01(\t\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x17\n\x0fpropertyVersion\x18\x05 \x01(\x04\"\x82\x01\n\x1aSetStreamAnnotationsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x1f\n\x17\x65xpectedPropertyVersion\x18\x02 \x01(\x04\x12#\n\x07\x63hanges\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x10\n\x08removals\x18\x04 \x03(\t\";\n\x1cSetStreamAnnotationsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"\x8a\x01\n\x13SetStreamTagsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x1f\n\x17\x65xpectedPropertyVersion\x18\x02 \x01(\x04\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x12\n\ncollection\x18\x04 \x01(\t\x12\x0e\n\x06remove\x18\x05 \x03(\t\"4\n\x15SetStreamTagsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"{\n\x0c\x43reateParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x12\n\ncollection\x18\x02 \x01(\t\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\"-\n\x0e\x43reateResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"@\n\x13MetadataUsageParams\x12\x0e\n\x06prefix\x18\x01 \x01(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"y\n\x15MetadataUsageResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x1d\n\x04tags\x18\x02 \x03(\x0b\x32\x0f.v5api.KeyCount\x12$\n\x0b\x61nnotations\x18\x03 \x03(\x0b\x32\x0f.v5api.KeyCount\"&\n\x08KeyCount\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05\x63ount\x18\x02 \x01(\x04\"B\n\x15ListCollectionsParams\x12\x0e\n\x06prefix\x18\x01 \x01(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"K\n\x17ListCollectionsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x13\n\x0b\x63ollections\x18\x02 \x03(\t\"\xab\x01\n\x13LookupStreamsParams\x12\x12\n\ncollection\x18\x01 \x01(\t\x12\x1a\n\x12isCollectionPrefix\x18\x02 \x01(\x08\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"^\n\x15LookupStreamsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12(\n\x07results\x18\x02 \x03(\x0b\x32\x17.v5api.StreamDescriptor\"S\n\rNearestParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x0c\n\x04time\x18\x02 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x03 \x01(\x04\x12\x10\n\x08\x62\x61\x63kward\x18\x04 \x01(\x08\"z\n\x0fNearestResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x1e\n\x05value\x18\x04 \x01(\x0b\x32\x0f.v5api.RawPoint\"U\n\rChangesParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x11\n\tfromMajor\x18\x02 \x01(\x04\x12\x0f\n\x07toMajor\x18\x03 \x01(\x04\x12\x12\n\nresolution\x18\x04 \x01(\r\"\x7f\n\x0f\x43hangesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12#\n\x06ranges\x18\x04 \x03(\x0b\x32\x13.v5api.ChangedRange\"#\n\tRoundSpec\x12\x0e\n\x04\x62its\x18\x02 \x01(\x05H\x00\x42\x06\n\x04spec\"\x99\x01\n\x0cInsertParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x0c\n\x04sync\x18\x02 \x01(\x08\x12(\n\x0cmerge_policy\x18\x04 \x01(\x0e\x32\x12.v5api.MergePolicy\x12\"\n\x08rounding\x18\x05 \x01(\x0b\x32\x10.v5api.RoundSpec\x12\x1f\n\x06values\x18\x03 \x03(\x0b\x32\x0f.v5api.RawPoint\"\x91\x01\n\x11\x41rrowInsertParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x0c\n\x04sync\x18\x02 \x01(\x08\x12(\n\x0cmerge_policy\x18\x03 \x01(\x0e\x32\x12.v5api.MergePolicy\x12\"\n\x08rounding\x18\x04 \x01(\x0b\x32\x10.v5api.RoundSpec\x12\x12\n\narrowBytes\x18\x05 \x01(\x0c\"Y\n\x0eInsertResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\"8\n\x0c\x44\x65leteParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\"Y\n\x0e\x44\x65leteResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\"\x0c\n\nInfoParams\"\xa2\x01\n\x0cInfoResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x19\n\x04mash\x18\x02 \x01(\x0b\x32\x0b.v5api.Mash\x12\x14\n\x0cmajorVersion\x18\x03 \x01(\r\x12\x14\n\x0cminorVersion\x18\x04 \x01(\r\x12\r\n\x05\x62uild\x18\x05 \x01(\t\x12\x1f\n\x05proxy\x18\x06 \x01(\x0b\x32\x10.v5api.ProxyInfo\"#\n\tProxyInfo\x12\x16\n\x0eproxyEndpoints\x18\x01 \x03(\t\"1\n\x11\x46\x61ultInjectParams\x12\x0c\n\x04type\x18\x01 \x01(\x04\x12\x0e\n\x06params\x18\x02 \x01(\x0c\">\n\x13\x46\x61ultInjectResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\n\n\x02rv\x18\x02 \x01(\x0c\"\x1b\n\x0b\x46lushParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"X\n\rFlushResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\" \n\x10ObliterateParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"1\n\x12ObliterateResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"\'\n\x08RawPoint\x12\x0c\n\x04time\x18\x01 \x01(\x10\x12\r\n\x05value\x18\x02 \x01(\x01\"`\n\tStatPoint\x12\x0c\n\x04time\x18\x01 \x01(\x10\x12\x0b\n\x03min\x18\x02 \x01(\x01\x12\x0c\n\x04mean\x18\x03 \x01(\x01\x12\x0b\n\x03max\x18\x04 \x01(\x01\x12\r\n\x05\x63ount\x18\x05 \x01(\x06\x12\x0e\n\x06stddev\x18\x06 \x01(\x01\"*\n\x0c\x43hangedRange\x12\r\n\x05start\x18\x01 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x02 \x01(\x10\">\n\x06Status\x12\x0c\n\x04\x63ode\x18\x01 \x01(\r\x12\x0b\n\x03msg\x18\x02 \x01(\t\x12\x19\n\x04mash\x18\x03 \x01(\x0b\x32\x0b.v5api.Mash\"\x98\x01\n\x04Mash\x12\x10\n\x08revision\x18\x01 \x01(\x03\x12\x0e\n\x06leader\x18\x02 \x01(\t\x12\x16\n\x0eleaderRevision\x18\x03 \x01(\x03\x12\x13\n\x0btotalWeight\x18\x04 \x01(\x03\x12\x0f\n\x07healthy\x18\x05 \x01(\x08\x12\x10\n\x08unmapped\x18\x06 \x01(\x01\x12\x1e\n\x07members\x18\x07 \x03(\x0b\x32\r.v5api.Member\"\xc3\x01\n\x06Member\x12\x0c\n\x04hash\x18\x01 \x01(\r\x12\x10\n\x08nodename\x18\x02 \x01(\t\x12\n\n\x02up\x18\x03 \x01(\x08\x12\n\n\x02in\x18\x04 \x01(\x08\x12\x0f\n\x07\x65nabled\x18\x05 \x01(\x08\x12\r\n\x05start\x18\x06 \x01(\x03\x12\x0b\n\x03\x65nd\x18\x07 \x01(\x03\x12\x0e\n\x06weight\x18\x08 \x01(\x03\x12\x16\n\x0ereadPreference\x18\t \x01(\x01\x12\x15\n\rhttpEndpoints\x18\n \x01(\t\x12\x15\n\rgrpcEndpoints\x18\x0b \x01(\t\"8\n\x0bKeyOptValue\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x1c\n\x03val\x18\x02 \x01(\x0b\x32\x0f.v5api.OptValue\"\x19\n\x08OptValue\x12\r\n\x05value\x18\x01 \x01(\t\"&\n\x08KeyValue\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"?\n\x0fStreamCSVConfig\x12\x0f\n\x07version\x18\x01 \x01(\x04\x12\r\n\x05label\x18\x02 \x01(\t\x12\x0c\n\x04uuid\x18\x03 \x01(\x0c\"\x9d\x02\n\x11GenerateCSVParams\x12\x35\n\tqueryType\x18\x01 \x01(\x0e\x32\".v5api.GenerateCSVParams.QueryType\x12\x11\n\tstartTime\x18\x02 \x01(\x03\x12\x0f\n\x07\x65ndTime\x18\x03 \x01(\x03\x12\x12\n\nwindowSize\x18\x04 \x01(\x04\x12\r\n\x05\x64\x65pth\x18\x05 \x01(\r\x12\x17\n\x0fincludeVersions\x18\x06 \x01(\x08\x12\'\n\x07streams\x18\x07 \x03(\x0b\x32\x16.v5api.StreamCSVConfig\"H\n\tQueryType\x12\x19\n\x15\x41LIGNED_WINDOWS_QUERY\x10\x00\x12\x11\n\rWINDOWS_QUERY\x10\x01\x12\r\n\tRAW_QUERY\x10\x02\"Q\n\x13GenerateCSVResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x10\n\x08isHeader\x18\x02 \x01(\x08\x12\x0b\n\x03row\x18\x03 \x03(\t\"J\n\x0eSQLQueryParams\x12\r\n\x05query\x18\x01 \x01(\t\x12\x0e\n\x06params\x18\x02 \x03(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"D\n\x10SQLQueryResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x13\n\x0bSQLQueryRow\x18\x02 \x03(\x0c\"\x14\n\x04Role\x12\x0c\n\x04name\x18\x01 \x01(\t\"\x94\x01\n\x19SetCompactionConfigParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x18\n\x10\x43ompactedVersion\x18\x02 \x01(\x04\x12>\n\x17reducedResolutionRanges\x18\x03 \x03(\x0b\x32\x1d.v5api.ReducedResolutionRange\x12\x0f\n\x07unused0\x18\x04 \x01(\x04\":\n\x1bSetCompactionConfigResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\")\n\x19GetCompactionConfigParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"\xc1\x01\n\x1bGetCompactionConfigResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x1a\n\x12LatestMajorVersion\x18\x02 \x01(\x04\x12\x18\n\x10\x43ompactedVersion\x18\x03 \x01(\x04\x12>\n\x17reducedResolutionRanges\x18\x04 \x03(\x0b\x32\x1d.v5api.ReducedResolutionRange\x12\x0f\n\x07unused0\x18\x05 \x01(\x04\"H\n\x16ReducedResolutionRange\x12\r\n\x05Start\x18\x01 \x01(\x03\x12\x0b\n\x03\x45nd\x18\x02 \x01(\x03\x12\x12\n\nResolution\x18\x03 \x01(\r*<\n\x0bMergePolicy\x12\t\n\x05NEVER\x10\x00\x12\t\n\x05\x45QUAL\x10\x01\x12\n\n\x06RETAIN\x10\x02\x12\x0b\n\x07REPLACE\x10\x03\x32\xb3\r\n\x05\x42TrDB\x12?\n\tRawValues\x12\x16.v5api.RawValuesParams\x1a\x18.v5api.RawValuesResponse0\x01\x12I\n\x0e\x41rrowRawValues\x12\x16.v5api.RawValuesParams\x1a\x1d.v5api.ArrowRawValuesResponse0\x01\x12T\n\x10\x41rrowMultiValues\x12\x1d.v5api.ArrowMultiValuesParams\x1a\x1f.v5api.ArrowMultiValuesResponse0\x01\x12N\n\x0e\x41lignedWindows\x12\x1b.v5api.AlignedWindowsParams\x1a\x1d.v5api.AlignedWindowsResponse0\x01\x12X\n\x13\x41rrowAlignedWindows\x12\x1b.v5api.AlignedWindowsParams\x1a\".v5api.ArrowAlignedWindowsResponse0\x01\x12\x39\n\x07Windows\x12\x14.v5api.WindowsParams\x1a\x16.v5api.WindowsResponse0\x01\x12\x43\n\x0c\x41rrowWindows\x12\x14.v5api.WindowsParams\x1a\x1b.v5api.ArrowWindowsResponse0\x01\x12@\n\nStreamInfo\x12\x17.v5api.StreamInfoParams\x1a\x19.v5api.StreamInfoResponse\x12^\n\x14SetStreamAnnotations\x12!.v5api.SetStreamAnnotationsParams\x1a#.v5api.SetStreamAnnotationsResponse\x12I\n\rSetStreamTags\x12\x1a.v5api.SetStreamTagsParams\x1a\x1c.v5api.SetStreamTagsResponse\x12\x34\n\x06\x43reate\x12\x13.v5api.CreateParams\x1a\x15.v5api.CreateResponse\x12Q\n\x0fListCollections\x12\x1c.v5api.ListCollectionsParams\x1a\x1e.v5api.ListCollectionsResponse0\x01\x12K\n\rLookupStreams\x12\x1a.v5api.LookupStreamsParams\x1a\x1c.v5api.LookupStreamsResponse0\x01\x12\x37\n\x07Nearest\x12\x14.v5api.NearestParams\x1a\x16.v5api.NearestResponse\x12\x39\n\x07\x43hanges\x12\x14.v5api.ChangesParams\x1a\x16.v5api.ChangesResponse0\x01\x12\x34\n\x06Insert\x12\x13.v5api.InsertParams\x1a\x15.v5api.InsertResponse\x12>\n\x0b\x41rrowInsert\x12\x18.v5api.ArrowInsertParams\x1a\x15.v5api.InsertResponse\x12\x34\n\x06\x44\x65lete\x12\x13.v5api.DeleteParams\x1a\x15.v5api.DeleteResponse\x12.\n\x04Info\x12\x11.v5api.InfoParams\x1a\x13.v5api.InfoResponse\x12\x43\n\x0b\x46\x61ultInject\x12\x18.v5api.FaultInjectParams\x1a\x1a.v5api.FaultInjectResponse\x12\x31\n\x05\x46lush\x12\x12.v5api.FlushParams\x1a\x14.v5api.FlushResponse\x12@\n\nObliterate\x12\x17.v5api.ObliterateParams\x1a\x19.v5api.ObliterateResponse\x12L\n\x10GetMetadataUsage\x12\x1a.v5api.MetadataUsageParams\x1a\x1c.v5api.MetadataUsageResponse\x12\x45\n\x0bGenerateCSV\x12\x18.v5api.GenerateCSVParams\x1a\x1a.v5api.GenerateCSVResponse0\x01\x12<\n\x08SQLQuery\x12\x15.v5api.SQLQueryParams\x1a\x17.v5api.SQLQueryResponse0\x01\x62\x06proto3') _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'btrdb_pb2', globals()) diff --git a/btrdb/point.py b/btrdb/point.py index fa767f9..00cb7b0 100644 --- a/btrdb/point.py +++ b/btrdb/point.py @@ -17,11 +17,11 @@ from btrdb.grpcinterface import btrdb_pb2 - ########################################################################## ## Point Classes ########################################################################## + class RawPoint(object): """ A point of data representing a single position within a time series. Each @@ -75,7 +75,7 @@ def __getitem__(self, index): @staticmethod def to_proto(point): - return btrdb_pb2.RawPoint(time = point[0], value = point[1]) + return btrdb_pb2.RawPoint(time=point[0], value=point[1]) @staticmethod def to_proto_list(points): @@ -148,7 +148,9 @@ def __init__(self, time, minv, meanv, maxv, count, stddev): @classmethod def from_proto(cls, proto): - return cls(proto.time, proto.min, proto.mean, proto.max, proto.count, proto.stddev) + return cls( + proto.time, proto.min, proto.mean, proto.max, proto.count, proto.stddev + ) @classmethod def from_proto_list(cls, proto_list): @@ -214,12 +216,7 @@ def __getitem__(self, index): def __repr__(self): return "StatPoint({}, {}, {}, {}, {}, {})".format( - self.time, - self.min, - self.mean, - self.max, - self.count, - self.stddev + self.time, self.min, self.mean, self.max, self.count, self.stddev ) def __str__(self): @@ -230,9 +227,11 @@ def __eq__(self, other): if not hasattr(other, attr): return False - return self.time == other.time and \ - self.min == other.min and \ - self.mean == other.mean and \ - self.max == other.max and \ - self.count == other.count and \ - self.stddev == other.stddev + return ( + self.time == other.time + and self.min == other.min + and self.mean == other.mean + and self.max == other.max + and self.count == other.count + and self.stddev == other.stddev + ) diff --git a/btrdb/stream.py b/btrdb/stream.py index 4f5bb79..e58c426 100644 --- a/btrdb/stream.py +++ b/btrdb/stream.py @@ -1655,7 +1655,7 @@ def _arrow_streamset_data(self): ) data = main_table return data - + def rows(self): """ Returns a materialized list of tuples where each tuple contains the diff --git a/btrdb/transformers.py b/btrdb/transformers.py index 6f61d49..e5ac34b 100644 --- a/btrdb/transformers.py +++ b/btrdb/transformers.py @@ -15,8 +15,8 @@ ## Imports ########################################################################## -import csv import contextlib +import csv from collections import OrderedDict from warnings import warn diff --git a/btrdb/utils/buffer.py b/btrdb/utils/buffer.py index 02b2fc5..d06f7b5 100644 --- a/btrdb/utils/buffer.py +++ b/btrdb/utils/buffer.py @@ -17,20 +17,18 @@ from collections import defaultdict - ########################################################################## ## Classes ########################################################################## -class PointBuffer(defaultdict): +class PointBuffer(defaultdict): def __init__(self, length, *args, **kwargs): super().__init__(lambda: [None] * length, *args, **kwargs) self.num_streams = length self.active = [True] * length self.last_known_time = [None] * length - def add_point(self, stream_index, point): self.last_known_time[stream_index] = max( (self.last_known_time[stream_index] or -1), point.time @@ -53,7 +51,6 @@ def is_ready(self, key): """ # check each stream value to see if its valid/ready for idx, latest_time in enumerate(self.last_known_time): - # return False if stream is active and we are waiting on val if self.active[idx] and (latest_time is None or key > latest_time): return False @@ -61,9 +58,7 @@ def is_ready(self, key): return True def next_key_ready(self): - """ - - """ + """ """ keys = list(self.keys()) keys.sort() for key in keys: diff --git a/btrdb/utils/conversion.py b/btrdb/utils/conversion.py index a1b979f..687be95 100644 --- a/btrdb/utils/conversion.py +++ b/btrdb/utils/conversion.py @@ -17,12 +17,12 @@ ## Imports ########################################################################## -import uuid import json -import pytz - +import uuid from datetime import datetime +import pytz + try: import numpy as np except ImportError: @@ -36,6 +36,7 @@ ## Classes ########################################################################## + class AnnotationEncoder(json.JSONEncoder): """Default JSON encoder class for saving stream annotations""" @@ -87,6 +88,7 @@ def decode(self, s): ## Functions ########################################################################## + def to_uuid(obj): """ Converts argument to UUID @@ -104,6 +106,4 @@ def to_uuid(obj): if isinstance(obj, str): return uuid.UUID(obj) - raise TypeError("Cannot convert object to UUID ({})".format( - obj.__class__.__name__) - ) + raise TypeError("Cannot convert object to UUID ({})".format(obj.__class__.__name__)) diff --git a/btrdb/utils/credentials.py b/btrdb/utils/credentials.py index e7d2510..d0c8263 100644 --- a/btrdb/utils/credentials.py +++ b/btrdb/utils/credentials.py @@ -16,10 +16,11 @@ ########################################################################## import os -import yaml from functools import wraps -from btrdb.exceptions import ProfileNotFound, CredentialsFileNotFound +import yaml + +from btrdb.exceptions import CredentialsFileNotFound, ProfileNotFound ########################################################################## ## Module Variables @@ -27,21 +28,27 @@ CONFIG_DIR = ".predictivegrid" CREDENTIALS_FILENAME = "credentials.yaml" -CREDENTIALS_PATH = os.path.join(os.path.expanduser("~"), CONFIG_DIR, CREDENTIALS_FILENAME) +CREDENTIALS_PATH = os.path.join( + os.path.expanduser("~"), CONFIG_DIR, CREDENTIALS_FILENAME +) ########################################################################## ## Functions ########################################################################## + def filter_none(f): """ decorator for removing dict items with None as value """ + @wraps(f) def inner(*args, **kwargs): - return { k: v for k, v in f(*args, **kwargs).items() if v is not None } + return {k: v for k, v in f(*args, **kwargs).items() if v is not None} + return inner + def load_credentials_from_file(): """ Returns a dict of the credentials file contents @@ -52,10 +59,13 @@ def load_credentials_from_file(): A dictionary of profile connection information """ try: - with open(CREDENTIALS_PATH, 'r') as stream: + with open(CREDENTIALS_PATH, "r") as stream: return yaml.safe_load(stream) except FileNotFoundError as exc: - raise CredentialsFileNotFound("Cound not find `{}`".format(CREDENTIALS_PATH)) from exc + raise CredentialsFileNotFound( + "Cound not find `{}`".format(CREDENTIALS_PATH) + ) from exc + @filter_none def credentials_by_profile(name=None): @@ -82,14 +92,16 @@ def credentials_by_profile(name=None): The requested profile could not be found in the credentials file """ if not name: - name = os.environ.get("BTRDB_PROFILE", 'default') + name = os.environ.get("BTRDB_PROFILE", "default") # load from credentials yaml file creds = load_credentials_from_file() if name not in creds.keys(): - if name == 'default': + if name == "default": return {} - raise ProfileNotFound("Profile `{}` not found in credentials file.".format(name)) + raise ProfileNotFound( + "Profile `{}` not found in credentials file.".format(name) + ) # rename api_key if needed and return fragment = creds[name].get("btrdb", {}) @@ -97,6 +109,7 @@ def credentials_by_profile(name=None): fragment["apikey"] = fragment.pop("api_key") return fragment + @filter_none def credentials_by_env(): """ @@ -110,7 +123,7 @@ def credentials_by_env(): """ return { "endpoints": os.environ.get("BTRDB_ENDPOINTS", None), - "apikey": os.environ.get("BTRDB_API_KEY", None), + "apikey": os.environ.get("BTRDB_API_KEY", None), } @@ -131,7 +144,12 @@ def credentials(endpoints=None, apikey=None): """ creds = {} - credentials_by_arg = filter_none(lambda: { "endpoints": endpoints, "apikey": apikey, }) + credentials_by_arg = filter_none( + lambda: { + "endpoints": endpoints, + "apikey": apikey, + } + ) pipeline = [credentials_by_env, credentials_by_arg] if os.path.exists(CREDENTIALS_PATH): pipeline.insert(0, credentials_by_profile) diff --git a/btrdb/utils/general.py b/btrdb/utils/general.py index 15ce1b8..6344fe6 100644 --- a/btrdb/utils/general.py +++ b/btrdb/utils/general.py @@ -36,6 +36,7 @@ def unpack_stream_descriptor(desc): ## Pointwidth Helpers ########################################################################## + class pointwidth(object): """ A representation of a period of time described by the BTrDB tree (and used in @@ -58,7 +59,7 @@ def from_timedelta(cls, delta): specified duration. Because pointwidths are in powers of 2, be sure to check that the returned real duration is sufficient. """ - return cls.from_nanoseconds(int(delta.total_seconds()*1e9)) + return cls.from_nanoseconds(int(delta.total_seconds() * 1e9)) @classmethod def from_nanoseconds(cls, nsec): @@ -117,10 +118,10 @@ def years(self): return self.nanoseconds / 3.154e16 def decr(self): - return pointwidth(self-1) + return pointwidth(self - 1) def incr(self): - return pointwidth(self+1) + return pointwidth(self + 1) def __int__(self): return self._pointwidth @@ -141,10 +142,10 @@ def __ge__(self, other): return int(self) >= int(other) def __add__(self, other): - return pointwidth(int(self)+int(other)) + return pointwidth(int(self) + int(other)) def __sub__(self, other): - return pointwidth(int(self)-int(other)) + return pointwidth(int(self) - int(other)) def __str__(self): """ @@ -180,4 +181,4 @@ def __str__(self): return "{:0.0f} nanoseconds".format(self.nanoseconds) def __repr__(self): - return "".format(int(self)) \ No newline at end of file + return "".format(int(self)) diff --git a/btrdb/utils/ray.py b/btrdb/utils/ray.py index e83ec2d..b1bbb66 100644 --- a/btrdb/utils/ray.py +++ b/btrdb/utils/ray.py @@ -1,6 +1,7 @@ +from functools import partial + import btrdb from btrdb.conn import BTrDB -from functools import partial def register_serializer(conn_str=None, apikey=None, profile=None): @@ -28,20 +29,35 @@ def register_serializer(conn_str=None, apikey=None, profile=None): import semver except ImportError: raise ImportError("must pip install semver to register custom serializer") - - assert ray.is_initialized(), "Need to call ray.init() before registering custom serializer" + + assert ( + ray.is_initialized() + ), "Need to call ray.init() before registering custom serializer" ver = semver.VersionInfo.parse(ray.__version__) if ver.major == 0: ray.register_custom_serializer( - BTrDB, serializer=btrdb_serializer, - deserializer=partial(btrdb_deserializer, conn_str=conn_str, apikey=apikey, profile=profile)) - elif ver.major == 1 and ver.minor in range(2, 13): # as of May 2022, latest Ray version is at 1.12 + BTrDB, + serializer=btrdb_serializer, + deserializer=partial( + btrdb_deserializer, conn_str=conn_str, apikey=apikey, profile=profile + ), + ) + elif ver.major == 1 and ver.minor in range( + 2, 13 + ): # as of May 2022, latest Ray version is at 1.12 # TODO: check different versions of ray? ray.util.register_serializer( - BTrDB, serializer=btrdb_serializer, - deserializer=partial(btrdb_deserializer, conn_str=conn_str, apikey=apikey, profile=profile)) + BTrDB, + serializer=btrdb_serializer, + deserializer=partial( + btrdb_deserializer, conn_str=conn_str, apikey=apikey, profile=profile + ), + ) else: - raise Exception("Ray version %s does not have custom serialization. Please upgrade to >= 1.4.0" % ray.__version__) + raise Exception( + "Ray version %s does not have custom serialization. Please upgrade to >= 1.4.0" + % ray.__version__ + ) def btrdb_serializer(_): @@ -50,10 +66,11 @@ def btrdb_serializer(_): """ return None + def btrdb_deserializer(_, conn_str=None, apikey=None, profile=None): """ deserialize function - + Parameters ---------- conn_str: str, default=None diff --git a/btrdb/utils/timez.py b/btrdb/utils/timez.py index cc311dc..32cec9e 100644 --- a/btrdb/utils/timez.py +++ b/btrdb/utils/timez.py @@ -18,27 +18,25 @@ ########################################################################## from datetime import datetime - -from operator import mul from decimal import Decimal +from operator import mul import pytz - ########################################################################## ## Module Variables ########################################################################## DATETIME_FORMATS = ( "%Y-%m-%d %H:%M:%S.%f%z", # most common RFC3339 nanoseconds - "%Y-%m-%d %H:%M:%S.%f", # expects UTC default timezone - "%Y-%m-%dT%H:%M:%S.%fZ", # JSON encoding, UTC timezone - "%Y-%m-%dT%H:%M:%SZ", # JSON encoding, UTC timezone + "%Y-%m-%d %H:%M:%S.%f", # expects UTC default timezone + "%Y-%m-%dT%H:%M:%S.%fZ", # JSON encoding, UTC timezone + "%Y-%m-%dT%H:%M:%SZ", # JSON encoding, UTC timezone "%Y-%m-%dT%H:%M:%S.%f%z", # less common JSON-ish encoding - "%Y-%m-%dT%H:%M:%S.%f", # for completeness, UTC default timezone - "%Y-%m-%d %H:%M:%S%z", # human readable date time with TZ - "%Y-%m-%d %H:%M:%S", # human readable date time UTC default - "%Y-%m-%d", # helper to get midnight on a particular date + "%Y-%m-%dT%H:%M:%S.%f", # for completeness, UTC default timezone + "%Y-%m-%d %H:%M:%S%z", # human readable date time with TZ + "%Y-%m-%d %H:%M:%S", # human readable date time UTC default + "%Y-%m-%d", # helper to get midnight on a particular date ) @@ -46,6 +44,7 @@ ## Functions ########################################################################## + def currently_as_ns(): """ Returns the current UTC time as nanoseconds since epoch @@ -141,6 +140,7 @@ def to_nanoseconds(val): try: import numpy as np + if isinstance(val, np.datetime64): val = val.astype(datetime) except ImportError: @@ -174,8 +174,9 @@ def to_nanoseconds(val): raise TypeError("only int, float, str, datetime, and datetime64 are allowed") -def ns_delta(days=0, hours=0, minutes=0, seconds=0, milliseconds=0, \ - microseconds=0, nanoseconds=0): +def ns_delta( + days=0, hours=0, minutes=0, seconds=0, milliseconds=0, microseconds=0, nanoseconds=0 +): """ Similar to `timedelta`, ns_delta represents a span of time but as the total number of nanoseconds. diff --git a/btrdb/version.py b/btrdb/version.py index 2592e90..492de74 100644 --- a/btrdb/version.py +++ b/btrdb/version.py @@ -15,20 +15,23 @@ ## Module Info ########################################################################## -__version_info__ = { 'major': 5, 'minor': 28, 'micro': 1, 'releaselevel': 'final'} +__version_info__ = {"major": 5, "minor": 28, "micro": 1, "releaselevel": "final"} ########################################################################## ## Helper Functions ########################################################################## + def get_version(short=False): """ Prints the version. """ - assert __version_info__['releaselevel'] in ('alpha', 'beta', 'final') - vers = ["%(major)i.%(minor)i" % __version_info__, ] - if __version_info__['micro']: + assert __version_info__["releaselevel"] in ("alpha", "beta", "final") + vers = [ + "%(major)i.%(minor)i" % __version_info__, + ] + if __version_info__["micro"]: vers.append(".%(micro)i" % __version_info__) - if __version_info__['releaselevel'] != 'final' and not short: - vers.append('%s%i' % (__version_info__['releaselevel'][0])) - return ''.join(vers) + if __version_info__["releaselevel"] != "final" and not short: + vers.append("%s%i" % (__version_info__["releaselevel"][0])) + return "".join(vers) diff --git a/btrdb4/__init__.py b/btrdb4/__init__.py index 85af01b..ee79a41 100644 --- a/btrdb4/__init__.py +++ b/btrdb4/__init__.py @@ -21,11 +21,13 @@ ## Warning ########################################################################## -MESSAGE = ("The btrdb4 package is not available in this version. To install " - "with pip use `pip install btrdb==4.*`") +MESSAGE = ( + "The btrdb4 package is not available in this version. To install " + "with pip use `pip install btrdb==4.*`" +) # raise exception on ImportWarning -warnings.simplefilter('error', ImportWarning) +warnings.simplefilter("error", ImportWarning) # warn user of version issue warnings.warn(MESSAGE, ImportWarning) diff --git a/docs/requirements.txt b/docs/requirements.txt index 074ee76..4da8205 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,2 @@ alabaster>=0.7.12 -Sphinx>=1.7 \ No newline at end of file +Sphinx>=1.7 diff --git a/docs/source/conf.py b/docs/source/conf.py index 79f76de..9801730 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -18,21 +18,22 @@ import os import sys -sys.path.insert(0, os.path.abspath('_themes')) -sys.path.insert(0, os.path.abspath('../..')) + +sys.path.insert(0, os.path.abspath("_themes")) +sys.path.insert(0, os.path.abspath("../..")) from btrdb.version import get_version # -- Project information ----------------------------------------------------- -project = 'btrdb' -copyright = '2021, Ping Things, Inc.' -author = 'PingThingsIO' +project = "btrdb" +copyright = "2021, Ping Things, Inc." +author = "PingThingsIO" # The short X.Y version version = get_version() # The full version, including alpha/beta/rc tags -release = 'v' + get_version() +release = "v" + get_version() # -- General configuration --------------------------------------------------- @@ -45,24 +46,24 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.napoleon', - 'sphinx.ext.todo', - 'sphinx.ext.githubpages', - 'sphinx.ext.intersphinx', + "sphinx.ext.autodoc", + "sphinx.ext.napoleon", + "sphinx.ext.todo", + "sphinx.ext.githubpages", + "sphinx.ext.intersphinx", ] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] -source_suffix = '.rst' +source_suffix = ".rst" # The master toctree document. -master_doc = 'index' +master_doc = "index" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -74,7 +75,7 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] # The name of the Pygments (syntax highlighting) style to use. pygments_style = None @@ -85,31 +86,29 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'alabaster' +html_theme = "alabaster" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. # html_theme_options = { - 'show_powered_by': False, - 'github_user': 'PingThingsIO', - 'github_repo': 'btrdb-python', - 'travis_button': False, - 'github_banner': False, - 'show_related': False, - 'note_bg': '#FFF59C', - 'description': 'A midweight library to converse with the BTrDB database.', - 'extra_nav_links': { - "btrdb": "http://btrdb.io" - }, - 'show_relbars': True, + "show_powered_by": False, + "github_user": "PingThingsIO", + "github_repo": "btrdb-python", + "travis_button": False, + "github_banner": False, + "show_related": False, + "note_bg": "#FFF59C", + "description": "A midweight library to converse with the BTrDB database.", + "extra_nav_links": {"btrdb": "http://btrdb.io"}, + "show_relbars": True, } # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # Custom sidebar templates, must be a dictionary that maps document names # to template names. @@ -131,7 +130,7 @@ # -- Options for HTMLHelp output --------------------------------------------- # Output file base name for HTML help builder. -htmlhelp_basename = 'btrdb-pythondoc' +htmlhelp_basename = "btrdb-pythondoc" # -- Options for LaTeX output ------------------------------------------------ @@ -140,15 +139,12 @@ # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. # # 'preamble': '', - # Latex figure (float) alignment # # 'figure_align': 'htbp', @@ -158,8 +154,13 @@ # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'btrdb-python.tex', 'btrdb-python Documentation', - 'PingThingsIO', 'manual'), + ( + master_doc, + "btrdb-python.tex", + "btrdb-python Documentation", + "PingThingsIO", + "manual", + ), ] @@ -167,10 +168,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'btrdb-python', 'btrdb-python Documentation', - [author], 1) -] +man_pages = [(master_doc, "btrdb-python", "btrdb-python Documentation", [author], 1)] # -- Options for Texinfo output ---------------------------------------------- @@ -179,9 +177,15 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'btrdb-python', 'btrdb-python Documentation', - author, 'btrdb-python', 'One line description of project.', - 'Miscellaneous'), + ( + master_doc, + "btrdb-python", + "btrdb-python Documentation", + author, + "btrdb-python", + "One line description of project.", + "Miscellaneous", + ), ] @@ -200,7 +204,7 @@ # epub_uid = '' # A list of files that should not be packed into the epub file. -epub_exclude_files = ['search.html'] +epub_exclude_files = ["search.html"] # -- Extension configuration ------------------------------------------------- @@ -210,4 +214,4 @@ # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = True -numfig = True \ No newline at end of file +numfig = True diff --git a/docs/source/installing.rst b/docs/source/installing.rst index 394c068..5b82210 100644 --- a/docs/source/installing.rst +++ b/docs/source/installing.rst @@ -41,5 +41,3 @@ like to install the version 4 bindings you will need to use `pip` as shown above .. code-block:: bash $ conda install -c pingthings btrdb - - diff --git a/docs/source/working/multiprocessing.rst b/docs/source/working/multiprocessing.rst index 0aa6a67..c474154 100644 --- a/docs/source/working/multiprocessing.rst +++ b/docs/source/working/multiprocessing.rst @@ -78,4 +78,4 @@ Let's break this down quickly since this is a very common design pattern. First The ``stream_quality`` function is our parallelizable function (e.g. computing the data quality for multiple streams at a time). Depending on how long the ``data_quality`` function takes to compute, we may also want to parallelize ``(stream, start, end)`` tuples. -If you would like features like a connection pool object (as other databases have) or multiprocessing helpers, please leave us a note in our GitHub issues! \ No newline at end of file +If you would like features like a connection pool object (as other databases have) or multiprocessing helpers, please leave us a note in our GitHub issues! diff --git a/docs/source/working/ray.rst b/docs/source/working/ray.rst index 98a7fbd..f13fc80 100644 --- a/docs/source/working/ray.rst +++ b/docs/source/working/ray.rst @@ -18,7 +18,7 @@ Setting up the ray serializer ray.init() conn_params = {"profile": "profile_name"} - + # register serializer with the connection parameters register_serializer(**conn_params) @@ -55,4 +55,4 @@ Setting up the ray serializer >>(pid=28482) (RawPoint(1533210100000000000, 0.0), 0) # output of test_streamset >>(pid=28481) (RawPoint(1533210100000000000, 0.0), RawPoint(1533210100000000000, 0.0)) - >>(pid=28481) StreamSet with 2 streams \ No newline at end of file + >>(pid=28481) StreamSet with 2 streams diff --git a/docs/source/working/stream-manage-metadata.rst b/docs/source/working/stream-manage-metadata.rst index 9452253..75b7fd6 100644 --- a/docs/source/working/stream-manage-metadata.rst +++ b/docs/source/working/stream-manage-metadata.rst @@ -109,5 +109,3 @@ If you would like to remove any keys from your annotations you must use the `rep annotations, _ = stream.anotations() del annotations["key_to_delete"] stream.update(annotations=annotations, replace=True) - - diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..066ed3b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,76 @@ +[project] +name = "btrdb" +version = "5.28.1" +authors = [ + {name="PingThingsIO", email="support@pingthings.io"}, +] +description = "Bindings to interact with the Berkeley Tree Database using gRPC." +readme = "README.md" +license = {file="LICENSE.txt"} +requires-python = ">=3.7,<=3.10" +classifiers = [ + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'Intended Audience :: Information Technology', + 'Intended Audience :: Science/Research', + 'License :: OSI Approved :: BSD License', + 'Natural Language :: English', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Topic :: Database', + 'Topic :: Software Development :: Libraries :: Python Modules', +] +keywords = [ + 'btrdb', 'berkeley', 'timeseries', 'database', 'bindings', 'gRPC', +] +dependencies = [ + "grpcio", + "grpcio-tools", + "pytz", + "pyyaml", + "certifi", +] + +[project.optional-dependencies] +data = [ + "pyarrow", + "pandas>=2.0", + "polars", + "numpy", + "tabulate", +] +ray = [ + "btrdb[data]", + "ray" +] +testing = [ + "btrdb[data]", + "pytest", + "freezegun", + "pytest-cov", + "pytest-flake8" +] +all = [ + "btrdb[data]", + "btrdb[ray]", + "btrdb[testing]", +] + +[project.urls] +"Homepage" = "https://btrdb.io" +"Docs" = "https://btrdb.readthedocs.io" +"Repository" = "https://github.com/pingthingsio/btrdb-python.git" + +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.exclude-package-data] +drop = [ + "tests", + "docs" +] diff --git a/release.sh b/release.sh index 9104416..d630e0e 100755 --- a/release.sh +++ b/release.sh @@ -44,4 +44,4 @@ git tag v$1.$2.$3 git push origin v$1.$2.$3 sleep 10 -git push \ No newline at end of file +git push diff --git a/requirements-all-locked.txt b/requirements-all-locked.txt new file mode 100644 index 0000000..d6f7284 --- /dev/null +++ b/requirements-all-locked.txt @@ -0,0 +1,128 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --extra=all --output-file=requirements-all.txt --resolver=backtracking pyproject.toml +# +aiosignal==1.3.1 + # via ray +attrs==23.1.0 + # via + # jsonschema + # ray +certifi==2023.5.7 + # via + # btrdb (pyproject.toml) + # requests +charset-normalizer==3.1.0 + # via requests +click==8.1.3 + # via ray +coverage[toml]==7.2.7 + # via pytest-cov +distlib==0.3.6 + # via virtualenv +exceptiongroup==1.1.1 + # via pytest +filelock==3.12.0 + # via + # ray + # virtualenv +flake8==6.0.0 + # via pytest-flake8 +freezegun==1.2.2 + # via btrdb +frozenlist==1.3.3 + # via + # aiosignal + # ray +grpcio==1.54.2 + # via + # btrdb (pyproject.toml) + # grpcio-tools + # ray +grpcio-tools==1.54.2 + # via + # btrdb (pyproject.toml) +idna==3.4 + # via requests +iniconfig==2.0.0 + # via pytest +jsonschema==4.17.3 + # via ray +mccabe==0.7.0 + # via flake8 +msgpack==1.0.5 + # via ray +numpy==1.24.3 + # via + # btrdb + # pandas + # pyarrow + # ray +packaging==23.1 + # via + # pytest + # ray +pandas==2.0.2 + # via btrdb +platformdirs==3.5.1 + # via virtualenv +pluggy==1.0.0 + # via pytest +polars==0.18.0 + # via btrdb +protobuf==4.23.2 + # via + # grpcio-tools + # ray +pyarrow==12.0.0 + # via btrdb +pycodestyle==2.10.0 + # via flake8 +pyflakes==3.0.1 + # via flake8 +pyrsistent==0.19.3 + # via jsonschema +pytest==7.3.1 + # via + # btrdb + # pytest-cov + # pytest-flake8 +pytest-cov==4.1.0 + # via btrdb +pytest-flake8==1.1.1 + # via btrdb +python-dateutil==2.8.2 + # via + # freezegun + # pandas +pytz==2023.3 + # via + # btrdb (pyproject.toml) + # pandas +pyyaml==6.0 + # via + # btrdb (pyproject.toml) + # ray +ray==2.2.0 + # via btrdb +requests==2.31.0 + # via ray +six==1.16.0 + # via python-dateutil +tabulate==0.9.0 + # via btrdb +tomli==2.0.1 + # via + # coverage + # pytest +tzdata==2023.3 + # via pandas +urllib3==2.0.2 + # via requests +virtualenv==20.23.0 + # via ray + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/requirements-all.txt b/requirements-all.txt new file mode 100644 index 0000000..4a71eb1 --- /dev/null +++ b/requirements-all.txt @@ -0,0 +1,57 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --extra=all --output-file=requirements-all.txt --resolver=backtracking pyproject.toml +# +# Pinned versions and dependencies of dependencies were removed after the autogeneration. +certifi + # via + # btrdb (pyproject.toml) +freezegun + # via btrdb +grpcio + # via + # btrdb (pyproject.toml) + # grpcio-tools + # ray +grpcio-tools + # via + # btrdb (pyproject.toml) +numpy + # via + # btrdb + # pandas + # pyarrow + # ray +pandas + # via btrdb +polars + # via btrdb +protobuf + # via + # grpcio-tools + # ray +pyarrow + # via btrdb +pytest + # via + # btrdb + # pytest-cov + # pytest-flake8 +pytest-cov + # via btrdb +pytest-flake8 + # via btrdb +pytz + # via + # btrdb (pyproject.toml) + # pandas +pyyaml + # via + # btrdb (pyproject.toml) + # ray +ray + # via btrdb +tabulate + # via btrdb diff --git a/requirements.txt b/requirements.txt index 254b418..a921486 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,15 +1,24 @@ -# GRPC / Protobuff related +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --output-file=requirements.txt --resolver=backtracking pyproject.toml +# +# This file was modified to remove version pins. +certifi + # via btrdb (pyproject.toml) grpcio + # via + # btrdb (pyproject.toml) + # grpcio-tools grpcio-tools - -# pyarrow and other tools -pyarrow -pandas -polars - -# Time related utils + # via btrdb (pyproject.toml) +protobuf + # via grpcio-tools pytz - -# Misc libraries + # via btrdb (pyproject.toml) pyyaml -certifi + # via btrdb (pyproject.toml) + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/setup.cfg b/setup.cfg index 76684f0..dc699ef 100644 --- a/setup.cfg +++ b/setup.cfg @@ -10,38 +10,4 @@ addopts = --cov=btrdb python_files = tests/* [flake8] -flake8-ignore = - E111 - E114 - E121 - E122 - E123 - E124 - E128 - E201 - E202 - E221 - E222 - E225 - E226 - E231 - E241 - E251 - E266 - E271 - E302 - E303 - E402 - E501 - E502 - E701 - E731 - F401 - W292 - W293 - W391 - tests/* ALL - btrdb/grpcinterface/btrdb_pb2.py ALL - btrdb/__init__.py ALL - setup.py ALL - +extend-ignore = E111, E114, E121, E122, E123, E124, E128, E201, E202, E203, E221, E222, E225, E226, E231, E241, E251, E266, E271, E302, E303, E402, E501, E502,E701,E731,F401,W292,W293,W391 diff --git a/setup.py b/setup.py index 2997b26..a5b82d1 100644 --- a/setup.py +++ b/setup.py @@ -16,69 +16,70 @@ ## Imports ########################################################################## -import os import codecs +import os -from setuptools import setup -from setuptools import find_packages +from setuptools import find_packages, setup ########################################################################## ## Package Information ########################################################################## ## Basic information -NAME = "btrdb" -DESCRIPTION = "Bindings to interact with the Berkeley Tree Database using gRPC." -AUTHOR = "PingThingsIO" -EMAIL = "support@pingthings.io" -MAINTAINER = "PingThingsIO" -LICENSE = "BSD-3-Clause" -REPOSITORY = "https://github.com/PingThingsIO/btrdb-python" -PACKAGE = "btrdb" -URL = "http://btrdb.io/" -DOCS_URL = "https://btrdb.readthedocs.io/en/latest/" +NAME = "btrdb" +DESCRIPTION = "Bindings to interact with the Berkeley Tree Database using gRPC." +AUTHOR = "PingThingsIO" +EMAIL = "support@pingthings.io" +MAINTAINER = "PingThingsIO" +LICENSE = "BSD-3-Clause" +REPOSITORY = "https://github.com/PingThingsIO/btrdb-python" +PACKAGE = "btrdb" +URL = "http://btrdb.io/" +DOCS_URL = "https://btrdb.readthedocs.io/en/latest/" ## Define the keywords -KEYWORDS = ('btrdb', 'berkeley', 'timeseries', 'database', 'bindings' 'gRPC') +KEYWORDS = ("btrdb", "berkeley", "timeseries", "database", "bindings" "gRPC") ## Define the classifiers ## See https://pypi.python.org/pypi?%3Aaction=list_classifiers -CLASSIFIERS = ( - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'Intended Audience :: Information Technology', - 'Intended Audience :: Science/Research', - 'License :: OSI Approved :: BSD License', - 'Natural Language :: English', - 'Operating System :: OS Independent', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Topic :: Database', - 'Topic :: Software Development :: Libraries :: Python Modules', +CLASSIFIERS = ( + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: BSD License", + "Natural Language :: English", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Topic :: Database", + "Topic :: Software Development :: Libraries :: Python Modules", ) ## Important Paths -PROJECT = os.path.abspath(os.path.dirname(__file__)) +PROJECT = os.path.abspath(os.path.dirname(__file__)) REQUIRE_PATH = "requirements.txt" VERSION_PATH = os.path.join(PACKAGE, "version.py") PKG_DESCRIBE = "DESCRIPTION.md" ## Directories to ignore in find_packages -EXCLUDES = ( - "tests", "docs", +EXCLUDES = ( + "tests", + "docs", ) ########################################################################## ## Helper Functions ########################################################################## + def read(*parts): """ Assume UTF-8 encoding and return the contents of the file located at the absolute path from the REPOSITORY joined with *parts. """ - with codecs.open(os.path.join(PROJECT, *parts), 'rb', 'utf-8') as f: + with codecs.open(os.path.join(PROJECT, *parts), "rb", "utf-8") as f: return f.read() @@ -90,7 +91,7 @@ def get_version(path=VERSION_PATH): """ namespace = {} exec(read(path), namespace) - return namespace['get_version'](short=True) + return namespace["get_version"](short=True) def get_requires(path=REQUIRE_PATH): @@ -100,7 +101,7 @@ def get_requires(path=REQUIRE_PATH): """ for line in read(path).splitlines(): line = line.strip() - if line and not line.startswith('#'): + if line and not line.startswith("#"): yield line @@ -152,8 +153,8 @@ def get_description_type(path=PKG_DESCRIBE): }, "install_requires": list(get_requires()), "python_requires": ">=3.6, <4", - "setup_requires":["pytest-runner"], - "tests_require":["pytest"], + "setup_requires": ["pytest-runner"], + "tests_require": ["pytest"], } @@ -161,5 +162,5 @@ def get_description_type(path=PKG_DESCRIBE): ## Run setup script ########################################################################## -if __name__ == '__main__': +if __name__ == "__main__": setup(**config) diff --git a/tests/btrdb/test_base.py b/tests/btrdb/test_base.py index c44acf8..5e84786 100644 --- a/tests/btrdb/test_base.py +++ b/tests/btrdb/test_base.py @@ -16,19 +16,19 @@ ########################################################################## import os -import pytest from unittest.mock import patch -from btrdb import connect, __version__, BTRDB_ENDPOINTS, BTRDB_API_KEY -from btrdb.exceptions import ConnectionError +import pytest +from btrdb import BTRDB_API_KEY, BTRDB_ENDPOINTS, __version__, connect +from btrdb.exceptions import ConnectionError ########################################################################## ## Initialization Tests ########################################################################## -class TestConnect(object): +class TestConnect(object): def setup(self): for env in ["BTRDB_ENDPOINTS", "BTRDB_PROFILE", "BTRDB_API_KEY"]: try: @@ -36,7 +36,7 @@ def setup(self): except KeyError: pass - @patch('btrdb.credentials') + @patch("btrdb.credentials") def test_raises_err_with_no_inputs(self, mock_credentials): """ Assert ConnectionError raised if no connection arg, ENV, profile @@ -51,40 +51,38 @@ def test_raises_err_if_both_profile_and_credentials(self): Assert error is raised if both profile and credentials are sent """ with pytest.raises(ValueError): - connect("192.168.1.100:4410",None,"default") + connect("192.168.1.100:4410", None, "default") - @patch('btrdb.utils.credentials.credentials_by_profile') - @patch('btrdb.utils.credentials.credentials_by_env') - @patch('btrdb._connect') - def test_uses_args_over_env(self, mock_connect, mock_credentials_by_env, mock_credentials_by_profile): + @patch("btrdb.utils.credentials.credentials_by_profile") + @patch("btrdb.utils.credentials.credentials_by_env") + @patch("btrdb._connect") + def test_uses_args_over_env( + self, mock_connect, mock_credentials_by_env, mock_credentials_by_profile + ): """ Assert uses connect args over env """ mock_credentials_by_profile.return_value = {} - mock_credentials_by_env.return_value = { - "endpoints": "a", "apikey": "b" - } + mock_credentials_by_env.return_value = {"endpoints": "a", "apikey": "b"} connect("cat", "dog") mock_connect.assert_called_once_with(endpoints="cat", apikey="dog") - @patch('btrdb.utils.credentials.credentials_by_profile') - @patch('btrdb.utils.credentials.credentials_by_env') - @patch('btrdb._connect') - def test_uses_env_over_profile(self, mock_connect, mock_credentials_by_env, mock_credentials_by_profile): + @patch("btrdb.utils.credentials.credentials_by_profile") + @patch("btrdb.utils.credentials.credentials_by_env") + @patch("btrdb._connect") + def test_uses_env_over_profile( + self, mock_connect, mock_credentials_by_env, mock_credentials_by_profile + ): """ Assert connect uses env over profile info """ - mock_credentials_by_profile.return_value = { - "endpoints": "a", "apikey": "b" - } - mock_credentials_by_env.return_value = { - "endpoints": "c", "apikey": "d" - } + mock_credentials_by_profile.return_value = {"endpoints": "a", "apikey": "b"} + mock_credentials_by_env.return_value = {"endpoints": "c", "apikey": "d"} connect() mock_connect.assert_called_once_with(endpoints="c", apikey="d") - @patch('btrdb.utils.credentials.credentials_by_profile') - @patch('btrdb.Connection') + @patch("btrdb.utils.credentials.credentials_by_profile") + @patch("btrdb.Connection") def test_connect_with_env(self, mock_conn, mock_credentials_by_profile): """ Assert connect uses ENV variables diff --git a/tests/btrdb/test_conn.py b/tests/btrdb/test_conn.py index 5907458..4a6d95a 100644 --- a/tests/btrdb/test_conn.py +++ b/tests/btrdb/test_conn.py @@ -16,20 +16,21 @@ ########################################################################## import uuid as uuidlib +from unittest.mock import Mock, PropertyMock, call, patch + import pytest -from unittest.mock import Mock, PropertyMock, patch, call -from btrdb.conn import Connection, BTrDB +from btrdb.conn import BTrDB, Connection from btrdb.endpoint import Endpoint -from btrdb.grpcinterface import btrdb_pb2 from btrdb.exceptions import * +from btrdb.grpcinterface import btrdb_pb2 ########################################################################## ## Connection Tests ########################################################################## -class TestConnection(object): +class TestConnection(object): def test_raises_err_invalid_address(self): """ Assert ValueError is raised if address:port is invalidly formatted @@ -39,7 +40,6 @@ def test_raises_err_invalid_address(self): conn = Connection(address) assert "expecting address:port" in str(exc) - def test_raises_err_for_apikey_insecure_port(self): """ Assert error is raised if apikey used on insecure port @@ -54,8 +54,8 @@ def test_raises_err_for_apikey_insecure_port(self): ## BTrDB Tests ########################################################################## -class TestBTrDB(object): +class TestBTrDB(object): ########################################################################## ## .streams tests ########################################################################## @@ -66,65 +66,60 @@ def test_streams_raises_err_if_version_not_list(self): """ db = BTrDB(None) with pytest.raises(TypeError) as exc: - db.streams('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a', versions="2,2") + db.streams("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a", versions="2,2") assert "versions argument must be of type list" in str(exc) - def test_streams_raises_err_if_version_argument_mismatch(self): """ Assert streams raises ValueError if len(identifiers) doesnt match length of versions """ db = BTrDB(None) with pytest.raises(ValueError) as exc: - db.streams('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a', versions=[2,2]) + db.streams("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a", versions=[2, 2]) assert "versions does not match identifiers" in str(exc) - def test_streams_stores_versions(self): """ Assert streams correctly stores supplied version info """ db = BTrDB(None) - uuid1 = uuidlib.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') - uuid2 = uuidlib.UUID('17dbe387-89ea-42b6-864b-f505cdb483f5') - versions = [22,44] + uuid1 = uuidlib.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + uuid2 = uuidlib.UUID("17dbe387-89ea-42b6-864b-f505cdb483f5") + versions = [22, 44] expected = dict(zip([uuid1, uuid2], versions)) streams = db.streams(uuid1, uuid2, versions=versions) assert streams._pinned_versions == expected - - @patch('btrdb.conn.BTrDB.stream_from_uuid') + @patch("btrdb.conn.BTrDB.stream_from_uuid") def test_streams_recognizes_uuid(self, mock_func): """ Assert streams recognizes uuid strings """ db = BTrDB(None) - uuid1 = uuidlib.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uuid1 = uuidlib.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") mock_func.return_value = [1] db.streams(uuid1) mock_func.assert_called_once() assert mock_func.call_args[0][0] == uuid1 - - @patch('btrdb.conn.BTrDB.stream_from_uuid') + @patch("btrdb.conn.BTrDB.stream_from_uuid") def test_streams_recognizes_uuid_string(self, mock_func): """ Assert streams recognizes uuid strings """ db = BTrDB(None) - uuid1 = '0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a' + uuid1 = "0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a" mock_func.return_value = [1] db.streams(uuid1) mock_func.assert_called_once() assert mock_func.call_args[0][0] == uuid1 - - @patch('btrdb.conn.BTrDB.streams_in_collection') + @patch("btrdb.conn.BTrDB.streams_in_collection") def test_streams_handles_path(self, mock_func): """ Assert streams calls streams_in_collection for collection/name paths @@ -132,17 +127,16 @@ def test_streams_handles_path(self, mock_func): db = BTrDB(None) ident = "zoo/animal/dog" mock_func.return_value = [1] - db.streams(ident, '0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + db.streams(ident, "0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") mock_func.assert_called_once() - assert mock_func.call_args[0][0] == 'zoo/animal' + assert mock_func.call_args[0][0] == "zoo/animal" assert mock_func.call_args[1] == { - 'is_collection_prefix': False, - 'tags': {'name': 'dog'} + "is_collection_prefix": False, + "tags": {"name": "dog"}, } - - @patch('btrdb.conn.BTrDB.streams_in_collection') + @patch("btrdb.conn.BTrDB.streams_in_collection") def test_streams_raises_err(self, mock_func): """ Assert streams raises StreamNotFoundError @@ -154,7 +148,7 @@ def test_streams_raises_err(self, mock_func): with pytest.raises(StreamNotFoundError) as exc: db.streams(ident) - mock_func.return_value = [1,2] + mock_func.return_value = [1, 2] with pytest.raises(StreamNotFoundError) as exc: db.streams(ident) @@ -162,7 +156,6 @@ def test_streams_raises_err(self, mock_func): mock_func.return_value = [1] db.streams(ident) - def test_streams_raises_valueerror(self): """ Assert streams raises ValueError if not uuid, uuid str, or path @@ -171,7 +164,6 @@ def test_streams_raises_valueerror(self): with pytest.raises(ValueError) as exc: db.streams(11) - ########################################################################## ## other tests ########################################################################## @@ -180,7 +172,7 @@ def test_info(self): """ Assert info method returns a dict """ - serialized = b'\x18\x05*\x055.0.02\x10\n\x0elocalhost:4410' + serialized = b"\x18\x05*\x055.0.02\x10\n\x0elocalhost:4410" info = btrdb_pb2.InfoResponse.FromString(serialized) endpoint = Mock(Endpoint) @@ -191,7 +183,9 @@ def test_info(self): "majorVersion": 5, "minorVersion": 0, "build": "5.0.0", - "proxy": { "proxyEndpoints": ["localhost:4410"], }, + "proxy": { + "proxyEndpoints": ["localhost:4410"], + }, } info = conn.info() assert info == truth @@ -204,16 +198,15 @@ def test_list_collections(self): Assert list_collections method works """ endpoint = Mock(Endpoint) - endpoint.listCollections = Mock(side_effect=[iter([ - ['allen/automated'], - ['allen/bindings'] - ])]) + endpoint.listCollections = Mock( + side_effect=[iter([["allen/automated"], ["allen/bindings"]])] + ) conn = BTrDB(endpoint) - truth = ['allen/automated', 'allen/bindings'] + truth = ["allen/automated", "allen/bindings"] assert conn.list_collections() == truth - @patch('btrdb.conn.unpack_stream_descriptor') + @patch("btrdb.conn.unpack_stream_descriptor") def test_streams_in_collections_args(self, mock_util): """ Assert streams_in_collections correctly sends *collection, tags, annotations @@ -231,13 +224,17 @@ def test_streams_in_collections_args(self, mock_util): conn = BTrDB(endpoint) tags = {"unit": "volts"} annotations = {"size": "large"} - streams = conn.streams_in_collection("a", "b", is_collection_prefix=False, - tags=tags, annotations=annotations) + streams = conn.streams_in_collection( + "a", "b", is_collection_prefix=False, tags=tags, annotations=annotations + ) assert streams[0].name == "gala" assert streams[1].name == "fuji" - expected = [call('a', False, tags, annotations), call('b', False, tags, annotations)] + expected = [ + call("a", False, tags, annotations), + call("b", False, tags, annotations), + ] assert endpoint.lookupStreams.call_args_list == expected def test_streams_in_collections_no_arg(self): diff --git a/tests/btrdb/test_point.py b/tests/btrdb/test_point.py index 5085b00..b596523 100644 --- a/tests/btrdb/test_point.py +++ b/tests/btrdb/test_point.py @@ -17,23 +17,22 @@ import pytest -from btrdb.point import RawPoint, StatPoint from btrdb.grpcinterface import btrdb_pb2 - +from btrdb.point import RawPoint, StatPoint ########################################################################## ## ProtoBuff Classes ########################################################################## -RawPointProto = btrdb_pb2.RawPoint -StatPointProto = btrdb_pb2.StatPoint +RawPointProto = btrdb_pb2.RawPoint +StatPointProto = btrdb_pb2.StatPoint ########################################################################## ## RawPoint Tests ########################################################################## -class TestRawPoint(object): +class TestRawPoint(object): def test_create(self): """ Ensure we can create the object @@ -150,19 +149,19 @@ def test_repr_str(self): ## StatPoint Tests ########################################################################## -class TestStatPoint(object): +class TestStatPoint(object): def test_create(self): """ Ensure we can create the object """ - StatPoint(1,1,1,1,1,1) + StatPoint(1, 1, 1, 1, 1, 1) def test_from_proto(self): """ Assert from_proto creates a correct object """ - proto = StatPointProto(time=1,min=2,mean=3,max=4,count=5,stddev=6) + proto = StatPointProto(time=1, min=2, mean=3, max=4, count=5, stddev=6) point = StatPoint.from_proto(proto) assert point.time == proto.time assert point.min == proto.min @@ -175,7 +174,7 @@ def test_from_proto_creates_rawpoint(self): """ Assert from_proto creates a new instance of StatPoint """ - proto = StatPointProto(time=1,min=2,mean=3,max=4,count=5,stddev=6) + proto = StatPointProto(time=1, min=2, mean=3, max=4, count=5, stddev=6) point = StatPoint.from_proto(proto) assert isinstance(point, StatPoint) @@ -183,8 +182,10 @@ def test_from_proto_list(self): """ Assert from_proto_list creates valid objects """ - protos = [StatPointProto(time=1,min=2,mean=3,max=4,count=5,stddev=6), - StatPointProto(time=11,min=12,mean=13,max=14,count=15,stddev=16)] + protos = [ + StatPointProto(time=1, min=2, mean=3, max=4, count=5, stddev=6), + StatPointProto(time=11, min=12, mean=13, max=14, count=15, stddev=16), + ] points = StatPoint.from_proto_list(protos) assert points[0].time == protos[0].time assert points[0].min == protos[0].min @@ -203,8 +204,10 @@ def test_from_proto_list_returns_list_of_statpoint(self): """ Assert from_proto_list returns list of StatPoint """ - protos = [StatPointProto(time=1,min=2,mean=3,max=4,count=5,stddev=6), - StatPointProto(time=11,min=12,mean=13,max=14,count=15,stddev=16)] + protos = [ + StatPointProto(time=1, min=2, mean=3, max=4, count=5, stddev=6), + StatPointProto(time=11, min=12, mean=13, max=14, count=15, stddev=16), + ] points = StatPoint.from_proto_list(protos) assert len(points) == 2 diff --git a/tests/btrdb/test_stream.py b/tests/btrdb/test_stream.py index 321f336..36d10c4 100644 --- a/tests/btrdb/test_stream.py +++ b/tests/btrdb/test_stream.py @@ -16,28 +16,31 @@ ## Imports ########################################################################## +import datetime +import json import re import sys -import json import uuid -import pytz +from unittest.mock import Mock, PropertyMock, call, patch + import pytest -import datetime -from unittest.mock import Mock, PropertyMock, patch, call +import pytz +from btrdb import MAXIMUM_TIME, MINIMUM_TIME from btrdb.conn import BTrDB from btrdb.endpoint import Endpoint -from btrdb import MINIMUM_TIME, MAXIMUM_TIME -from btrdb.stream import Stream, StreamSet, StreamFilter, INSERT_BATCH_SIZE -from btrdb.point import RawPoint, StatPoint from btrdb.exceptions import ( BTrDBError, + InvalidCollection, InvalidOperation, + NoSuchPoint, StreamNotFoundError, InvalidCollection, NoSuchPoint, ) from btrdb.grpcinterface import btrdb_pb2 +from btrdb.point import RawPoint, StatPoint +from btrdb.stream import INSERT_BATCH_SIZE, Stream, StreamFilter, StreamSet RawPointProto = btrdb_pb2.RawPoint StatPointProto = btrdb_pb2.StatPoint diff --git a/tests/btrdb/test_transformers.py b/tests/btrdb/test_transformers.py index 56aad5a..15003c9 100644 --- a/tests/btrdb/test_transformers.py +++ b/tests/btrdb/test_transformers.py @@ -16,21 +16,22 @@ ########################################################################## import os -from io import StringIO, BytesIO - -import pytest +from io import BytesIO, StringIO from unittest.mock import Mock, PropertyMock + import numpy as np -from pandas import Series, DataFrame, Index +import pytest +from pandas import DataFrame, Index, Series -from btrdb.stream import Stream, StreamSet from btrdb.point import RawPoint, StatPoint +from btrdb.stream import Stream, StreamSet from btrdb.transformers import * ########################################################################## ## Transformer Tests ########################################################################## + @pytest.fixture def streamset(): # list of mock streams @@ -44,23 +45,105 @@ def streamset(): streams.append(stream) rows = [ - (None, RawPoint(1500000000000000000, 1.0), RawPoint(1500000000000000000, 1.0), RawPoint(1500000000000000000, 1.0)), - (RawPoint(1500000000100000000, 2.0), None, RawPoint(1500000000100000000, 2.0), RawPoint(1500000000100000000, 2.0)), - (None, RawPoint(1500000000200000000, 3.0), None, RawPoint(1500000000200000000, 3.0)), - (RawPoint(1500000000300000000, 4.0), None, RawPoint(1500000000300000000, 4.0), RawPoint(1500000000300000000, 4.0)), - (None, RawPoint(1500000000400000000, 5.0), RawPoint(1500000000400000000, 5.0), RawPoint(1500000000400000000, 5.0)), - (RawPoint(1500000000500000000, 6.0), None, None, RawPoint(1500000000500000000, 6.0)), - (None, RawPoint(1500000000600000000, 7.0), RawPoint(1500000000600000000, 7.0), RawPoint(1500000000600000000, 7.0)), - (RawPoint(1500000000700000000, 8.0), None, RawPoint(1500000000700000000, 8.0), RawPoint(1500000000700000000, 8.0)), - (None, RawPoint(1500000000800000000, 9.0), RawPoint(1500000000800000000, 9.0), RawPoint(1500000000800000000, 9.0)), - (RawPoint(1500000000900000000, 10.0), None, RawPoint(1500000000900000000, 10.0), RawPoint(1500000000900000000, 10.0) ), + ( + None, + RawPoint(1500000000000000000, 1.0), + RawPoint(1500000000000000000, 1.0), + RawPoint(1500000000000000000, 1.0), + ), + ( + RawPoint(1500000000100000000, 2.0), + None, + RawPoint(1500000000100000000, 2.0), + RawPoint(1500000000100000000, 2.0), + ), + ( + None, + RawPoint(1500000000200000000, 3.0), + None, + RawPoint(1500000000200000000, 3.0), + ), + ( + RawPoint(1500000000300000000, 4.0), + None, + RawPoint(1500000000300000000, 4.0), + RawPoint(1500000000300000000, 4.0), + ), + ( + None, + RawPoint(1500000000400000000, 5.0), + RawPoint(1500000000400000000, 5.0), + RawPoint(1500000000400000000, 5.0), + ), + ( + RawPoint(1500000000500000000, 6.0), + None, + None, + RawPoint(1500000000500000000, 6.0), + ), + ( + None, + RawPoint(1500000000600000000, 7.0), + RawPoint(1500000000600000000, 7.0), + RawPoint(1500000000600000000, 7.0), + ), + ( + RawPoint(1500000000700000000, 8.0), + None, + RawPoint(1500000000700000000, 8.0), + RawPoint(1500000000700000000, 8.0), + ), + ( + None, + RawPoint(1500000000800000000, 9.0), + RawPoint(1500000000800000000, 9.0), + RawPoint(1500000000800000000, 9.0), + ), + ( + RawPoint(1500000000900000000, 10.0), + None, + RawPoint(1500000000900000000, 10.0), + RawPoint(1500000000900000000, 10.0), + ), ] values = [ - [RawPoint(1500000000100000000, 2.0),RawPoint(1500000000300000000, 4.0),RawPoint(1500000000500000000, 6.0),RawPoint(1500000000700000000, 8.0),RawPoint(1500000000900000000, 10.0)], - [RawPoint(1500000000000000000, 1.0),RawPoint(1500000000200000000, 3.0),RawPoint(1500000000400000000, 5.0),RawPoint(1500000000600000000, 7.0),RawPoint(1500000000800000000, 9.0)], - [RawPoint(1500000000000000000, 1.0),RawPoint(1500000000100000000, 2.0),RawPoint(1500000000300000000, 4.0),RawPoint(1500000000400000000, 5.0),RawPoint(1500000000600000000, 7.0),RawPoint(1500000000700000000, 8.0),RawPoint(1500000000800000000, 9.0),RawPoint(1500000000900000000, 10.0)], - [RawPoint(1500000000000000000, 1.0),RawPoint(1500000000100000000, 2.0),RawPoint(1500000000200000000, 3.0),RawPoint(1500000000300000000, 4.0),RawPoint(1500000000400000000, 5.0),RawPoint(1500000000500000000, 6.0),RawPoint(1500000000600000000, 7.0),RawPoint(1500000000700000000, 8.0),RawPoint(1500000000800000000, 9.0),RawPoint(1500000000900000000, 10.0)] + [ + RawPoint(1500000000100000000, 2.0), + RawPoint(1500000000300000000, 4.0), + RawPoint(1500000000500000000, 6.0), + RawPoint(1500000000700000000, 8.0), + RawPoint(1500000000900000000, 10.0), + ], + [ + RawPoint(1500000000000000000, 1.0), + RawPoint(1500000000200000000, 3.0), + RawPoint(1500000000400000000, 5.0), + RawPoint(1500000000600000000, 7.0), + RawPoint(1500000000800000000, 9.0), + ], + [ + RawPoint(1500000000000000000, 1.0), + RawPoint(1500000000100000000, 2.0), + RawPoint(1500000000300000000, 4.0), + RawPoint(1500000000400000000, 5.0), + RawPoint(1500000000600000000, 7.0), + RawPoint(1500000000700000000, 8.0), + RawPoint(1500000000800000000, 9.0), + RawPoint(1500000000900000000, 10.0), + ], + [ + RawPoint(1500000000000000000, 1.0), + RawPoint(1500000000100000000, 2.0), + RawPoint(1500000000200000000, 3.0), + RawPoint(1500000000300000000, 4.0), + RawPoint(1500000000400000000, 5.0), + RawPoint(1500000000500000000, 6.0), + RawPoint(1500000000600000000, 7.0), + RawPoint(1500000000700000000, 8.0), + RawPoint(1500000000800000000, 9.0), + RawPoint(1500000000900000000, 10.0), + ], ] obj = StreamSet(streams) @@ -72,16 +155,56 @@ def streamset(): @pytest.fixture def statpoint_streamset(): rows = [ - [StatPoint(1500000000100000000, 0, 2.0, 2.5, 10, 1),StatPoint(1500000000100000000, 0, 4.0, 4.5, 11, 1),StatPoint(1500000000100000000, 0, 6.0, 6.5, 10, 1),StatPoint(1500000000100000000, 0, 8.0, 8.5, 11, 2)], - [StatPoint(1500000000300000000, 0, 3.0, 3.5, 11, 1),StatPoint(1500000000300000000, 0, 5.0, 5.5, 10, 1),StatPoint(1500000000300000000, 0, 7.0, 7.5, 10, 1),StatPoint(1500000000300000000, 0, 9.0, 9.5, 11, 2)], - [StatPoint(1500000000500000000, 0, 4.0, 4.5, 10, 1),StatPoint(1500000000500000000, 0, 6.0, 6.5, 11, 1),StatPoint(1500000000500000000, 0, 8.0, 8.5, 10, 1),StatPoint(1500000000500000000, 0, 10.0, 10.5, 11, 2)], - [StatPoint(1500000000700000000, 0, 5.0, 5.5, 11, 1),StatPoint(1500000000700000000, 0, 7.0, 7.5, 10, 1),StatPoint(1500000000700000000, 0, 9.0, 9.5, 10, 1),StatPoint(1500000000700000000, 0, 11.0, 11.5, 11, 2)], + [ + StatPoint(1500000000100000000, 0, 2.0, 2.5, 10, 1), + StatPoint(1500000000100000000, 0, 4.0, 4.5, 11, 1), + StatPoint(1500000000100000000, 0, 6.0, 6.5, 10, 1), + StatPoint(1500000000100000000, 0, 8.0, 8.5, 11, 2), + ], + [ + StatPoint(1500000000300000000, 0, 3.0, 3.5, 11, 1), + StatPoint(1500000000300000000, 0, 5.0, 5.5, 10, 1), + StatPoint(1500000000300000000, 0, 7.0, 7.5, 10, 1), + StatPoint(1500000000300000000, 0, 9.0, 9.5, 11, 2), + ], + [ + StatPoint(1500000000500000000, 0, 4.0, 4.5, 10, 1), + StatPoint(1500000000500000000, 0, 6.0, 6.5, 11, 1), + StatPoint(1500000000500000000, 0, 8.0, 8.5, 10, 1), + StatPoint(1500000000500000000, 0, 10.0, 10.5, 11, 2), + ], + [ + StatPoint(1500000000700000000, 0, 5.0, 5.5, 11, 1), + StatPoint(1500000000700000000, 0, 7.0, 7.5, 10, 1), + StatPoint(1500000000700000000, 0, 9.0, 9.5, 10, 1), + StatPoint(1500000000700000000, 0, 11.0, 11.5, 11, 2), + ], ] values = [ - [StatPoint(1500000000100000000, 0, 2.0, 2.5, 10, 1),StatPoint(1500000000300000000, 0, 4.0, 4.5, 11, 1),StatPoint(1500000000500000000, 0, 6.0, 6.5, 10, 1),StatPoint(1500000000700000000, 0, 8.0, 8.5, 11, 2)], - [StatPoint(1500000000100000000, 1, 3.0, 3.5, 11, 1),StatPoint(1500000000300000000, 1, 5.0, 5.5, 10, 1),StatPoint(1500000000500000000, 1, 7.0, 7.5, 10, 1),StatPoint(1500000000700000000, 1, 9.0, 9.5, 11, 2)], - [StatPoint(1500000000100000000, 2, 4.0, 4.5, 10, 1),StatPoint(1500000000300000000, 2, 6.0, 6.5, 11, 1),StatPoint(1500000000500000000, 2, 8.0, 8.5, 10, 1),StatPoint(1500000000700000000, 2, 10.0, 10.5, 11, 2)], - [StatPoint(1500000000100000000, 3, 5.0, 5.5, 11, 1),StatPoint(1500000000300000000, 3, 7.0, 7.5, 10, 1),StatPoint(1500000000500000000, 3, 9.0, 9.5, 10, 1),StatPoint(1500000000700000000, 3, 11.0, 11.5, 11, 2)], + [ + StatPoint(1500000000100000000, 0, 2.0, 2.5, 10, 1), + StatPoint(1500000000300000000, 0, 4.0, 4.5, 11, 1), + StatPoint(1500000000500000000, 0, 6.0, 6.5, 10, 1), + StatPoint(1500000000700000000, 0, 8.0, 8.5, 11, 2), + ], + [ + StatPoint(1500000000100000000, 1, 3.0, 3.5, 11, 1), + StatPoint(1500000000300000000, 1, 5.0, 5.5, 10, 1), + StatPoint(1500000000500000000, 1, 7.0, 7.5, 10, 1), + StatPoint(1500000000700000000, 1, 9.0, 9.5, 11, 2), + ], + [ + StatPoint(1500000000100000000, 2, 4.0, 4.5, 10, 1), + StatPoint(1500000000300000000, 2, 6.0, 6.5, 11, 1), + StatPoint(1500000000500000000, 2, 8.0, 8.5, 10, 1), + StatPoint(1500000000700000000, 2, 10.0, 10.5, 11, 2), + ], + [ + StatPoint(1500000000100000000, 3, 5.0, 5.5, 11, 1), + StatPoint(1500000000300000000, 3, 7.0, 7.5, 10, 1), + StatPoint(1500000000500000000, 3, 9.0, 9.5, 10, 1), + StatPoint(1500000000700000000, 3, 11.0, 11.5, 11, 2), + ], ] streams = [] for idx in range(4): @@ -99,16 +222,76 @@ def statpoint_streamset(): expected = { "to_dict": [ - {'time': 1500000000000000000, 'test/stream0': None, 'test/stream1': 1.0, 'test/stream2': 1.0, 'test/stream3': 1.0}, - {'time': 1500000000100000000, 'test/stream0': 2.0, 'test/stream1': None, 'test/stream2': 2.0, 'test/stream3': 2.0}, - {'time': 1500000000200000000, 'test/stream0': None, 'test/stream1': 3.0, 'test/stream2': None, 'test/stream3': 3.0}, - {'time': 1500000000300000000, 'test/stream0': 4.0, 'test/stream1': None, 'test/stream2': 4.0, 'test/stream3': 4.0}, - {'time': 1500000000400000000, 'test/stream0': None, 'test/stream1': 5.0, 'test/stream2': 5.0, 'test/stream3': 5.0}, - {'time': 1500000000500000000, 'test/stream0': 6.0, 'test/stream1': None, 'test/stream2': None, 'test/stream3': 6.0}, - {'time': 1500000000600000000, 'test/stream0': None, 'test/stream1': 7.0, 'test/stream2': 7.0, 'test/stream3': 7.0}, - {'time': 1500000000700000000, 'test/stream0': 8.0, 'test/stream1': None, 'test/stream2': 8.0, 'test/stream3': 8.0}, - {'time': 1500000000800000000, 'test/stream0': None, 'test/stream1': 9.0, 'test/stream2': 9.0, 'test/stream3': 9.0}, - {'time': 1500000000900000000, 'test/stream0': 10.0, 'test/stream1': None, 'test/stream2': 10.0, 'test/stream3': 10.0} + { + "time": 1500000000000000000, + "test/stream0": None, + "test/stream1": 1.0, + "test/stream2": 1.0, + "test/stream3": 1.0, + }, + { + "time": 1500000000100000000, + "test/stream0": 2.0, + "test/stream1": None, + "test/stream2": 2.0, + "test/stream3": 2.0, + }, + { + "time": 1500000000200000000, + "test/stream0": None, + "test/stream1": 3.0, + "test/stream2": None, + "test/stream3": 3.0, + }, + { + "time": 1500000000300000000, + "test/stream0": 4.0, + "test/stream1": None, + "test/stream2": 4.0, + "test/stream3": 4.0, + }, + { + "time": 1500000000400000000, + "test/stream0": None, + "test/stream1": 5.0, + "test/stream2": 5.0, + "test/stream3": 5.0, + }, + { + "time": 1500000000500000000, + "test/stream0": 6.0, + "test/stream1": None, + "test/stream2": None, + "test/stream3": 6.0, + }, + { + "time": 1500000000600000000, + "test/stream0": None, + "test/stream1": 7.0, + "test/stream2": 7.0, + "test/stream3": 7.0, + }, + { + "time": 1500000000700000000, + "test/stream0": 8.0, + "test/stream1": None, + "test/stream2": 8.0, + "test/stream3": 8.0, + }, + { + "time": 1500000000800000000, + "test/stream0": None, + "test/stream1": 9.0, + "test/stream2": 9.0, + "test/stream3": 9.0, + }, + { + "time": 1500000000900000000, + "test/stream0": 10.0, + "test/stream1": None, + "test/stream2": 10.0, + "test/stream3": 10.0, + }, ], "to_array": [ np.array([2.0, 4.0, 6.0, 8.0, 10.0]), @@ -142,12 +325,10 @@ def statpoint_streamset(): 1500000000800000000,,9.0,9.0,9.0 1500000000900000000,10.0,,10.0,10.0 """, - } class TestTransformers(object): - ########################################################################## ## to_dict Tests ########################################################################## @@ -157,30 +338,174 @@ def test_to_dict(self, streamset): def test_to_dict_statpoint(self, statpoint_streamset): expected = [ - {'time': 1500000000100000000, 'test/stream0': 2.0, 'test/stream1': 4.0, 'test/stream2': 6.0, 'test/stream3': 8.0}, - {'time': 1500000000300000000, 'test/stream0': 3.0, 'test/stream1': 5.0, 'test/stream2': 7.0, 'test/stream3': 9.0}, - {'time': 1500000000500000000, 'test/stream0': 4.0, 'test/stream1': 6.0, 'test/stream2': 8.0, 'test/stream3': 10.0}, - {'time': 1500000000700000000, 'test/stream0': 5.0, 'test/stream1': 7.0, 'test/stream2': 9.0, 'test/stream3': 11.0}, + { + "time": 1500000000100000000, + "test/stream0": 2.0, + "test/stream1": 4.0, + "test/stream2": 6.0, + "test/stream3": 8.0, + }, + { + "time": 1500000000300000000, + "test/stream0": 3.0, + "test/stream1": 5.0, + "test/stream2": 7.0, + "test/stream3": 9.0, + }, + { + "time": 1500000000500000000, + "test/stream0": 4.0, + "test/stream1": 6.0, + "test/stream2": 8.0, + "test/stream3": 10.0, + }, + { + "time": 1500000000700000000, + "test/stream0": 5.0, + "test/stream1": 7.0, + "test/stream2": 9.0, + "test/stream3": 11.0, + }, ] assert to_dict(statpoint_streamset) == expected def test_to_dict_statpoint_agg_count(self, statpoint_streamset): expected = [ - {'time': 1500000000100000000, 'test/stream0': 10, 'test/stream1': 11, 'test/stream2': 10, 'test/stream3': 11}, - {'time': 1500000000300000000, 'test/stream0': 11, 'test/stream1': 10, 'test/stream2': 10, 'test/stream3': 11}, - {'time': 1500000000500000000, 'test/stream0': 10, 'test/stream1': 11, 'test/stream2': 10, 'test/stream3': 11}, - {'time': 1500000000700000000, 'test/stream0': 11, 'test/stream1': 10, 'test/stream2': 10, 'test/stream3': 11}, + { + "time": 1500000000100000000, + "test/stream0": 10, + "test/stream1": 11, + "test/stream2": 10, + "test/stream3": 11, + }, + { + "time": 1500000000300000000, + "test/stream0": 11, + "test/stream1": 10, + "test/stream2": 10, + "test/stream3": 11, + }, + { + "time": 1500000000500000000, + "test/stream0": 10, + "test/stream1": 11, + "test/stream2": 10, + "test/stream3": 11, + }, + { + "time": 1500000000700000000, + "test/stream0": 11, + "test/stream1": 10, + "test/stream2": 10, + "test/stream3": 11, + }, ] - assert to_dict(statpoint_streamset, agg='count') == expected + assert to_dict(statpoint_streamset, agg="count") == expected def test_to_dict_statpoint_agg_all(self, statpoint_streamset): expected = [ - OrderedDict([('time', 1500000000100000000), ('test/stream0-min', 0), ('test/stream0-mean', 2.0), ('test/stream0-max', 2.5), ('test/stream0-count', 10), ('test/stream0-stddev', 1), ('test/stream1-min', 0), ('test/stream1-mean', 4.0), ('test/stream1-max', 4.5), ('test/stream1-count', 11), ('test/stream1-stddev', 1), ('test/stream2-min', 0), ('test/stream2-mean', 6.0), ('test/stream2-max', 6.5), ('test/stream2-count', 10), ('test/stream2-stddev', 1), ('test/stream3-min', 0), ('test/stream3-mean', 8.0), ('test/stream3-max', 8.5), ('test/stream3-count', 11), ('test/stream3-stddev', 2)]), - OrderedDict([('time', 1500000000300000000), ('test/stream0-min', 0), ('test/stream0-mean', 3.0), ('test/stream0-max', 3.5), ('test/stream0-count', 11), ('test/stream0-stddev', 1), ('test/stream1-min', 0), ('test/stream1-mean', 5.0), ('test/stream1-max', 5.5), ('test/stream1-count', 10), ('test/stream1-stddev', 1), ('test/stream2-min', 0), ('test/stream2-mean', 7.0), ('test/stream2-max', 7.5), ('test/stream2-count', 10), ('test/stream2-stddev', 1), ('test/stream3-min', 0), ('test/stream3-mean', 9.0), ('test/stream3-max', 9.5), ('test/stream3-count', 11), ('test/stream3-stddev', 2)]), - OrderedDict([('time', 1500000000500000000), ('test/stream0-min', 0), ('test/stream0-mean', 4.0), ('test/stream0-max', 4.5), ('test/stream0-count', 10), ('test/stream0-stddev', 1), ('test/stream1-min', 0), ('test/stream1-mean', 6.0), ('test/stream1-max', 6.5), ('test/stream1-count', 11), ('test/stream1-stddev', 1), ('test/stream2-min', 0), ('test/stream2-mean', 8.0), ('test/stream2-max', 8.5), ('test/stream2-count', 10), ('test/stream2-stddev', 1), ('test/stream3-min', 0), ('test/stream3-mean', 10.0), ('test/stream3-max', 10.5), ('test/stream3-count', 11), ('test/stream3-stddev', 2)]), - OrderedDict([('time', 1500000000700000000), ('test/stream0-min', 0), ('test/stream0-mean', 5.0), ('test/stream0-max', 5.5), ('test/stream0-count', 11), ('test/stream0-stddev', 1), ('test/stream1-min', 0), ('test/stream1-mean', 7.0), ('test/stream1-max', 7.5), ('test/stream1-count', 10), ('test/stream1-stddev', 1), ('test/stream2-min', 0), ('test/stream2-mean', 9.0), ('test/stream2-max', 9.5), ('test/stream2-count', 10), ('test/stream2-stddev', 1), ('test/stream3-min', 0), ('test/stream3-mean', 11.0), ('test/stream3-max', 11.5), ('test/stream3-count', 11), ('test/stream3-stddev', 2)]) + OrderedDict( + [ + ("time", 1500000000100000000), + ("test/stream0-min", 0), + ("test/stream0-mean", 2.0), + ("test/stream0-max", 2.5), + ("test/stream0-count", 10), + ("test/stream0-stddev", 1), + ("test/stream1-min", 0), + ("test/stream1-mean", 4.0), + ("test/stream1-max", 4.5), + ("test/stream1-count", 11), + ("test/stream1-stddev", 1), + ("test/stream2-min", 0), + ("test/stream2-mean", 6.0), + ("test/stream2-max", 6.5), + ("test/stream2-count", 10), + ("test/stream2-stddev", 1), + ("test/stream3-min", 0), + ("test/stream3-mean", 8.0), + ("test/stream3-max", 8.5), + ("test/stream3-count", 11), + ("test/stream3-stddev", 2), + ] + ), + OrderedDict( + [ + ("time", 1500000000300000000), + ("test/stream0-min", 0), + ("test/stream0-mean", 3.0), + ("test/stream0-max", 3.5), + ("test/stream0-count", 11), + ("test/stream0-stddev", 1), + ("test/stream1-min", 0), + ("test/stream1-mean", 5.0), + ("test/stream1-max", 5.5), + ("test/stream1-count", 10), + ("test/stream1-stddev", 1), + ("test/stream2-min", 0), + ("test/stream2-mean", 7.0), + ("test/stream2-max", 7.5), + ("test/stream2-count", 10), + ("test/stream2-stddev", 1), + ("test/stream3-min", 0), + ("test/stream3-mean", 9.0), + ("test/stream3-max", 9.5), + ("test/stream3-count", 11), + ("test/stream3-stddev", 2), + ] + ), + OrderedDict( + [ + ("time", 1500000000500000000), + ("test/stream0-min", 0), + ("test/stream0-mean", 4.0), + ("test/stream0-max", 4.5), + ("test/stream0-count", 10), + ("test/stream0-stddev", 1), + ("test/stream1-min", 0), + ("test/stream1-mean", 6.0), + ("test/stream1-max", 6.5), + ("test/stream1-count", 11), + ("test/stream1-stddev", 1), + ("test/stream2-min", 0), + ("test/stream2-mean", 8.0), + ("test/stream2-max", 8.5), + ("test/stream2-count", 10), + ("test/stream2-stddev", 1), + ("test/stream3-min", 0), + ("test/stream3-mean", 10.0), + ("test/stream3-max", 10.5), + ("test/stream3-count", 11), + ("test/stream3-stddev", 2), + ] + ), + OrderedDict( + [ + ("time", 1500000000700000000), + ("test/stream0-min", 0), + ("test/stream0-mean", 5.0), + ("test/stream0-max", 5.5), + ("test/stream0-count", 11), + ("test/stream0-stddev", 1), + ("test/stream1-min", 0), + ("test/stream1-mean", 7.0), + ("test/stream1-max", 7.5), + ("test/stream1-count", 10), + ("test/stream1-stddev", 1), + ("test/stream2-min", 0), + ("test/stream2-mean", 9.0), + ("test/stream2-max", 9.5), + ("test/stream2-count", 10), + ("test/stream2-stddev", 1), + ("test/stream3-min", 0), + ("test/stream3-mean", 11.0), + ("test/stream3-max", 11.5), + ("test/stream3-count", 11), + ("test/stream3-stddev", 2), + ] + ), ] - assert to_dict(statpoint_streamset, agg='all') == expected + assert to_dict(statpoint_streamset, agg="all") == expected ########################################################################## ## to_array Tests @@ -207,10 +532,50 @@ def test_to_array_statpoint(self, statpoint_streamset): result = to_array(statpoint_streamset, agg="min") assert result.__class__ == np.ndarray - assert (result[0] == np.array([0.0, 0.0, 0.0, 0.0,])).all() - assert (result[1] == np.array([1.0, 1.0, 1.0, 1.0, ])).all() - assert (result[2] == np.array([2.0, 2.0, 2.0, 2.0, ])).all() - assert (result[3] == np.array([3.0, 3.0, 3.0, 3.0, ])).all() + assert ( + result[0] + == np.array( + [ + 0.0, + 0.0, + 0.0, + 0.0, + ] + ) + ).all() + assert ( + result[1] + == np.array( + [ + 1.0, + 1.0, + 1.0, + 1.0, + ] + ) + ).all() + assert ( + result[2] + == np.array( + [ + 2.0, + 2.0, + 2.0, + 2.0, + ] + ) + ).all() + assert ( + result[3] + == np.array( + [ + 3.0, + 3.0, + 3.0, + 3.0, + ] + ) + ).all() ########################################################################## ## to_series Tests @@ -224,12 +589,12 @@ def test_to_series(self, streamset): expected_names = [s.collection + "/" + s.name for s in streamset] for idx, points in enumerate(streamset.values()): values, times = list(zip(*[(p.value, p.time) for p in points])) - times = Index(times, dtype='datetime64[ns]') + times = Index(times, dtype="datetime64[ns]") expected.append(Series(values, times, name=expected_names[idx])) result = to_series(streamset) for idx in range(4): - assert (result[idx] == expected[idx]).all() # verify data + assert (result[idx] == expected[idx]).all() # verify data assert result[idx].name == expected_names[idx] # verify name def test_to_series_name_lambda(self, streamset): @@ -237,7 +602,7 @@ def test_to_series_name_lambda(self, streamset): assert to_dateframe uses name lambda """ result = streamset.to_series(name_callable=lambda s: s.name) - assert [s.name for s in result] == ['stream0', 'stream1', 'stream2', 'stream3'] + assert [s.name for s in result] == ["stream0", "stream1", "stream2", "stream3"] def test_to_series_ignores_rawpoint_with_agg(self, streamset): """ @@ -252,19 +617,34 @@ def test_to_series_statpoint(self, statpoint_streamset): """ result = to_series(statpoint_streamset) values = [2.0, 4.0, 6.0, 8.0] - index = [1500000000100000000, 1500000000300000000, 1500000000500000000, 1500000000700000000] - times = Index(index, dtype='datetime64[ns]') + index = [ + 1500000000100000000, + 1500000000300000000, + 1500000000500000000, + 1500000000700000000, + ] + times = Index(index, dtype="datetime64[ns]") assert (result[0] == Series(values, index=times)).all() values = [3.0, 5.0, 7.0, 9.0] - index = [1500000000100000000, 1500000000300000000, 1500000000500000000, 1500000000700000000] - times = Index(index, dtype='datetime64[ns]') + index = [ + 1500000000100000000, + 1500000000300000000, + 1500000000500000000, + 1500000000700000000, + ] + times = Index(index, dtype="datetime64[ns]") assert (result[1] == Series(values, index=times)).all() result = to_series(statpoint_streamset, agg="max") values = [2.5, 4.5, 6.5, 8.5] - index = [1500000000100000000, 1500000000300000000, 1500000000500000000, 1500000000700000000] - times = Index(index, dtype='datetime64[ns]') + index = [ + 1500000000100000000, + 1500000000300000000, + 1500000000500000000, + 1500000000700000000, + ] + times = Index(index, dtype="datetime64[ns]") assert (result[0] == Series(values, index=times)).all() def test_to_series_index_type(self, streamset): @@ -273,7 +653,7 @@ def test_to_series_index_type(self, streamset): """ result = to_series(streamset) for series in result: - assert series.index.dtype.name == 'datetime64[ns]' + assert series.index.dtype.name == "datetime64[ns]" def test_to_series_index_as_int(self, streamset): """ @@ -281,7 +661,7 @@ def test_to_series_index_as_int(self, streamset): """ result = to_series(streamset, False) for series in result: - assert series.index.dtype.name == 'int64' + assert series.index.dtype.name == "int64" def test_to_series_raises_on_agg_all(self, statpoint_streamset): """ @@ -298,7 +678,13 @@ def test_to_dataframe(self, streamset): """ assert to_dateframe works on RawPoints """ - columns = ["time", "test/stream0", "test/stream1", "test/stream2", "test/stream3"] + columns = [ + "time", + "test/stream0", + "test/stream1", + "test/stream2", + "test/stream3", + ] df = DataFrame(expected["to_dict"], columns=columns) df.set_index("time", inplace=True) assert to_dataframe(streamset).equals(df) @@ -327,9 +713,14 @@ def test_to_dataframe_multindex(self, statpoint_streamset): """ df = statpoint_streamset.to_dataframe(agg="all") assert len(df.columns.levels) == 3 - assert list(df.columns.levels[0]) == ['test'] - assert list(df.columns.levels[1]) == ['stream0', 'stream1', 'stream2', 'stream3'] - assert list(df.columns.levels[2]) == ['count', 'max', 'mean', 'min', 'stddev'] + assert list(df.columns.levels[0]) == ["test"] + assert list(df.columns.levels[1]) == [ + "stream0", + "stream1", + "stream2", + "stream3", + ] + assert list(df.columns.levels[2]) == ["count", "max", "mean", "min", "stddev"] def test_to_dataframe_name_lambda(self, streamset): """ @@ -347,8 +738,7 @@ def test_to_dataframe_rawpoint_with_agg(self, streamset): assert to_dateframe ignores agg if RawPoint """ df = streamset.to_dataframe(agg="all") - assert df.shape == (10,4) - + assert df.shape == (10, 4) def test_to_dataframe_statpoints(self, statpoint_streamset): """ @@ -405,8 +795,8 @@ def test_to_csv_as_stringio(self, streamset): # convert back to dicts for assert due to newline issues with CSV result = [dict(row) for row in reader] for item in result: - for k,v in item.items(): - if v is '': + for k, v in item.items(): + if v is "": item[k] = None elif "." in v: item[k] = float(v) @@ -468,4 +858,4 @@ def test_to_table_raises_on_agg_all(self, statpoint_streamset): asserts to_table raises error if using "all" as agg. """ with pytest.raises(AttributeError): - to_table(statpoint_streamset, agg="all") \ No newline at end of file + to_table(statpoint_streamset, agg="all") diff --git a/tests/btrdb/utils/test_buffer.py b/tests/btrdb/utils/test_buffer.py index bd5f626..2786d5b 100644 --- a/tests/btrdb/utils/test_buffer.py +++ b/tests/btrdb/utils/test_buffer.py @@ -16,21 +16,21 @@ ########################################################################## import pytest -from btrdb.utils.buffer import PointBuffer + from btrdb.point import RawPoint +from btrdb.utils.buffer import PointBuffer ########################################################################## ## Test Constants ########################################################################## - ########################################################################## ## Initialization Tests ########################################################################## -class TestPointBuffer(object): +class TestPointBuffer(object): def test_earliest(self): """ Assert earliest returns correct key @@ -106,7 +106,6 @@ def test_is_ready(self): buffer.deactivate(1) assert buffer.is_ready(2000) == True - def test_next_key_ready_with_inactive(self): """ Assert next_key_ready returns correct key with inactive stream @@ -118,7 +117,6 @@ def test_next_key_ready_with_inactive(self): buffer.add_point(0, RawPoint(time=2000, value="leopard")) assert buffer.next_key_ready() == 1000 - # assert first key is 500 even though it was exhausted buffer = PointBuffer(3) buffer.add_point(1, RawPoint(time=500, value="leopard")) diff --git a/tests/btrdb/utils/test_conversions.py b/tests/btrdb/utils/test_conversions.py index 8cc52a1..0f3a963 100644 --- a/tests/btrdb/utils/test_conversions.py +++ b/tests/btrdb/utils/test_conversions.py @@ -15,15 +15,14 @@ ## Imports ########################################################################## -import uuid import json -import pytest -import numpy as np - +import uuid from datetime import datetime -from btrdb.utils.conversion import to_uuid -from btrdb.utils.conversion import AnnotationDecoder, AnnotationEncoder +import numpy as np +import pytest + +from btrdb.utils.conversion import AnnotationDecoder, AnnotationEncoder, to_uuid ########################################################################## ## Test Constants @@ -38,48 +37,56 @@ ## Conversion Tests ########################################################################## -class TestAnnotationJSON(object): - @pytest.mark.parametrize("obj, expected", [ - (True, "true"), - (False, "false"), - (None, "null"), - (3.14, "3.14"), - (42, "42"), - ("foo", "foo"), - ("a long walk on the beach", "a long walk on the beach"), - (['a', 'b', 'c'], '["a", "b", "c"]'), - ({'color': 'red', 'foo': 24}, '{"color": "red", "foo": 24}'), - (datetime(2018, 9, 10, 16, 30), "2018-09-10 16:30:00.000000"), - (np.datetime64(datetime(2018, 9, 10, 16, 30)), "2018-09-10 16:30:00.000000+0000"), - (EXAMPLE_UUID, EXAMPLE_UUID_STR), - ]) +class TestAnnotationJSON(object): + @pytest.mark.parametrize( + "obj, expected", + [ + (True, "true"), + (False, "false"), + (None, "null"), + (3.14, "3.14"), + (42, "42"), + ("foo", "foo"), + ("a long walk on the beach", "a long walk on the beach"), + (["a", "b", "c"], '["a", "b", "c"]'), + ({"color": "red", "foo": 24}, '{"color": "red", "foo": 24}'), + (datetime(2018, 9, 10, 16, 30), "2018-09-10 16:30:00.000000"), + ( + np.datetime64(datetime(2018, 9, 10, 16, 30)), + "2018-09-10 16:30:00.000000+0000", + ), + (EXAMPLE_UUID, EXAMPLE_UUID_STR), + ], + ) def test_annotations_encoder(self, obj, expected): msg = f"did not correctly encode type {type(obj)}" assert json.dumps(obj, cls=AnnotationEncoder, indent=None) == expected, msg - @pytest.mark.parametrize("obj, s", [ - (True, "true"), - (False, "false"), - (None, "null"), - (3.14, "3.14"), - (42, "42"), - ("foo", "foo"), - ("foo", '"foo"'), - ("a long walk on the beach", "a long walk on the beach"), - ("a long walk on the beach", '"a long walk on the beach"'), - (['a', 'b', 'c'], '["a", "b", "c"]'), - ({'color': 'red', 'foo': 24}, '{"color": "red", "foo": 24}'), - ("2018-09-10 16:30:00.000000", "2018-09-10 16:30:00.000000"), - ({'uuid': EXAMPLE_UUID_STR}, f'{{"uuid": "{EXAMPLE_UUID_STR}"}}'), - ]) + @pytest.mark.parametrize( + "obj, s", + [ + (True, "true"), + (False, "false"), + (None, "null"), + (3.14, "3.14"), + (42, "42"), + ("foo", "foo"), + ("foo", '"foo"'), + ("a long walk on the beach", "a long walk on the beach"), + ("a long walk on the beach", '"a long walk on the beach"'), + (["a", "b", "c"], '["a", "b", "c"]'), + ({"color": "red", "foo": 24}, '{"color": "red", "foo": 24}'), + ("2018-09-10 16:30:00.000000", "2018-09-10 16:30:00.000000"), + ({"uuid": EXAMPLE_UUID_STR}, f'{{"uuid": "{EXAMPLE_UUID_STR}"}}'), + ], + ) def test_annotations_decoder(self, obj, s): msg = f"did not correctly decode type {type(obj)}" assert json.loads(s, cls=AnnotationDecoder) == obj, msg class TestToUUID(object): - def test_from_bytes(self): """ Assert that `to_uuid` converts from bytes diff --git a/tests/btrdb/utils/test_credentials.py b/tests/btrdb/utils/test_credentials.py index 6bd057f..37b4811 100644 --- a/tests/btrdb/utils/test_credentials.py +++ b/tests/btrdb/utils/test_credentials.py @@ -15,21 +15,20 @@ ## Imports ########################################################################## -import pytest from unittest.mock import patch +import pytest + from btrdb.exceptions import * from btrdb.utils.credentials import * - ########################################################################## ## Tests ########################################################################## class TestLoadCredentials(object): - - @patch('builtins.open') + @patch("builtins.open") def test_raises_err_if_credentials_not_found(self, mock_open): """ Assert CredentialsFileNotFound is raised if credentials.yaml is not found @@ -40,7 +39,6 @@ def test_raises_err_if_credentials_not_found(self, mock_open): class TestLoadProfile(object): - def setup(self): for env in ["BTRDB_ENDPOINTS", "BTRDB_PROFILE", "BTRDB_API_KEY"]: try: @@ -48,14 +46,14 @@ def setup(self): except KeyError: pass - @patch('btrdb.utils.credentials.load_credentials_from_file') + @patch("btrdb.utils.credentials.load_credentials_from_file") def test_raises_err_if_profile_not_found(self, mock_credentials): """ Assert ProfileNotFound is raised if profile is asked for but not found """ mock_credentials.return_value = { "duck": { - "btrdb" : { + "btrdb": { "endpoints": "192.168.1.100:4410", "api_key": "111222333", } @@ -64,7 +62,7 @@ def test_raises_err_if_profile_not_found(self, mock_credentials): with pytest.raises(ProfileNotFound): credentials_by_profile("horse") - @patch('btrdb.utils.credentials.load_credentials_from_file') + @patch("btrdb.utils.credentials.load_credentials_from_file") def test_returns_requested_profile(self, mock_credentials): """ Assert returns correct profile @@ -75,47 +73,47 @@ def test_returns_requested_profile(self, mock_credentials): } assert credentials_by_profile("duck")["endpoints"] == "duck" - @patch('btrdb.utils.credentials.load_credentials_from_file') + @patch("btrdb.utils.credentials.load_credentials_from_file") def test_returns_default_profile(self, mock_credentials): """ Assert default profile is returned """ mock_credentials.return_value = { "duck": { - "btrdb" : { + "btrdb": { "endpoints": "192.168.1.100:4411", "api_key": "111222333", }, - "name": "duck" + "name": "duck", }, "default": { - "btrdb" : { + "btrdb": { "endpoints": "192.168.1.222:4411", "api_key": "333222111", }, - "name": "default" + "name": "default", }, } assert credentials_by_profile()["apikey"] == "333222111" - @patch('btrdb.utils.credentials.load_credentials_from_file') + @patch("btrdb.utils.credentials.load_credentials_from_file") def test_returns_no_default_profile(self, mock_credentials): """ Assert empty credentials are returned if no default profile """ mock_credentials.return_value = { "duck": { - "btrdb" : { + "btrdb": { "endpoints": "192.168.1.100:4411", "api_key": "111222333", }, - "name": "duck" + "name": "duck", }, } assert credentials_by_profile() == {} -class TestCredentials(object): +class TestCredentials(object): def setup(self): for env in ["BTRDB_ENDPOINTS", "BTRDB_PROFILE", "BTRDB_API_KEY"]: try: @@ -123,8 +121,8 @@ def setup(self): except KeyError: pass - @patch('btrdb.utils.credentials.load_credentials_from_file') - @patch('os.path.exists') + @patch("btrdb.utils.credentials.load_credentials_from_file") + @patch("os.path.exists") def test_checks_file_existence(self, mock_exists, mock_load): mock_exists.return_value = False credentials() diff --git a/tests/btrdb/utils/test_general.py b/tests/btrdb/utils/test_general.py index 0f09cf5..a13cf3c 100644 --- a/tests/btrdb/utils/test_general.py +++ b/tests/btrdb/utils/test_general.py @@ -11,41 +11,45 @@ Testing for the btrdb.utils.general module """ +from datetime import timedelta + import pytest -from datetime import timedelta -from btrdb.utils.timez import ns_delta from btrdb.utils.general import pointwidth +from btrdb.utils.timez import ns_delta class TestPointwidth(object): - - @pytest.mark.parametrize("delta, expected", [ - (timedelta(days=365), 54), - (timedelta(days=30), 51), - (timedelta(days=7), 49), - (timedelta(days=1), 46), - (timedelta(hours=4), 43), - (timedelta(minutes=15), 39), - (timedelta(seconds=30), 34), - ]) - + @pytest.mark.parametrize( + "delta, expected", + [ + (timedelta(days=365), 54), + (timedelta(days=30), 51), + (timedelta(days=7), 49), + (timedelta(days=1), 46), + (timedelta(hours=4), 43), + (timedelta(minutes=15), 39), + (timedelta(seconds=30), 34), + ], + ) def test_from_timedelta(self, delta, expected): """ Test getting the closest point width from a timedelta """ assert pointwidth.from_timedelta(delta) == expected - @pytest.mark.parametrize("nsec, expected", [ - (ns_delta(days=365), 54), - (ns_delta(days=30), 51), - (ns_delta(days=7), 49), - (ns_delta(days=1), 46), - (ns_delta(hours=12), 45), - (ns_delta(minutes=30), 40), - (ns_delta(seconds=1), 29), - ]) - + @pytest.mark.parametrize( + "nsec, expected", + [ + (ns_delta(days=365), 54), + (ns_delta(days=30), 51), + (ns_delta(days=7), 49), + (ns_delta(days=1), 46), + (ns_delta(hours=12), 45), + (ns_delta(minutes=30), 40), + (ns_delta(seconds=1), 29), + ], + ) def test_from_nanoseconds(self, nsec, expected): """ Test getting the closest point width from nanoseconds diff --git a/tests/btrdb/utils/test_timez.py b/tests/btrdb/utils/test_timez.py index 45fd3fa..7150fff 100644 --- a/tests/btrdb/utils/test_timez.py +++ b/tests/btrdb/utils/test_timez.py @@ -15,22 +15,27 @@ ## Imports ########################################################################## -import pytz -import pytest import datetime + import numpy as np +import pytest +import pytz from freezegun import freeze_time -from btrdb.utils.timez import (currently_as_ns, datetime_to_ns, ns_to_datetime, - to_nanoseconds, ns_delta) - +from btrdb.utils.timez import ( + currently_as_ns, + datetime_to_ns, + ns_delta, + ns_to_datetime, + to_nanoseconds, +) ########################################################################## ## Initialization Tests ########################################################################## -class TestCurrentlyAsNs(object): +class TestCurrentlyAsNs(object): def test_currently_as_ns(self): """ Assert currently_as_ns returns correct value @@ -41,25 +46,23 @@ def test_currently_as_ns(self): class TestDatetimeToNs(object): - def test_datetime_to_ns_naive(self): """ Assert datetime_to_ns handles naive datetime """ - dt = datetime.datetime(2018,1,1,12) + dt = datetime.datetime(2018, 1, 1, 12) localized = pytz.utc.localize(dt) expected = int(localized.timestamp() * 1e9) assert dt.tzinfo is None assert datetime_to_ns(dt) == expected - def test_datetime_to_ns_aware(self): """ Assert datetime_to_ns handles tz aware datetime """ eastern = pytz.timezone("US/Eastern") - dt = datetime.datetime(2018,1,1,17, tzinfo=eastern) + dt = datetime.datetime(2018, 1, 1, 17, tzinfo=eastern) expected = int(dt.astimezone(pytz.utc).timestamp() * 1e9) assert dt.tzinfo is not None @@ -67,7 +70,6 @@ def test_datetime_to_ns_aware(self): class TestNsToDatetime(object): - def test_ns_to_datetime_is_utc(self): """ Assert ns_to_datetime returns UTC aware datetime @@ -83,18 +85,17 @@ def test_ns_to_datetime_is_correct(self): Assert ns_to_datetime returns correct datetime """ val = 1514808000000000000 - expected = datetime.datetime(2018,1,1,12, tzinfo=pytz.UTC) + expected = datetime.datetime(2018, 1, 1, 12, tzinfo=pytz.UTC) assert ns_to_datetime(val) == expected class TestToNanoseconds(object): - def test_datetime_to_ns_naive(self): """ Assert to_nanoseconds handles naive datetime """ - dt = datetime.datetime(2018,1,1,12) + dt = datetime.datetime(2018, 1, 1, 12) localized = pytz.utc.localize(dt) expected = int(localized.timestamp() * 1e9) @@ -106,7 +107,7 @@ def test_datetime_to_ns_aware(self): Assert to_nanoseconds handles tz aware datetime """ eastern = pytz.timezone("US/Eastern") - dt = datetime.datetime(2018,1,1,17, tzinfo=eastern) + dt = datetime.datetime(2018, 1, 1, 17, tzinfo=eastern) expected = int(dt.astimezone(pytz.utc).timestamp() * 1e9) assert dt.tzinfo is not None @@ -116,7 +117,7 @@ def test_str(self): """ Assert to_nanoseconds handles RFC3339 format """ - dt = datetime.datetime(2018,1,1,12, tzinfo=pytz.utc) + dt = datetime.datetime(2018, 1, 1, 12, tzinfo=pytz.utc) expected = int(dt.timestamp() * 1e9) dt_str = "2018-1-1 12:00:00.0-0000" @@ -124,12 +125,12 @@ def test_str(self): assert to_nanoseconds(dt_str) == expected dt_str = "2018-1-1 7:00:00.0-0500" - dt = datetime.datetime(2018,1,1,12, tzinfo=pytz.timezone("US/Eastern")) + dt = datetime.datetime(2018, 1, 1, 12, tzinfo=pytz.timezone("US/Eastern")) assert dt.tzinfo is not None assert to_nanoseconds(dt_str) == expected dt_str = "2018-01-15 07:32:49" - dt = datetime.datetime(2018,1,15,7,32,49, tzinfo=pytz.utc) + dt = datetime.datetime(2018, 1, 15, 7, 32, 49, tzinfo=pytz.utc) expected = int(dt.timestamp() * 1e9) assert to_nanoseconds(dt_str) == expected @@ -173,20 +174,19 @@ def test_float(self): """ Assert to_nanoseconds handles float """ - dt = datetime.datetime(2018,1,1,12, tzinfo=pytz.utc) + dt = datetime.datetime(2018, 1, 1, 12, tzinfo=pytz.utc) expected = int(dt.timestamp() * 1e9) - dt64 = np.datetime64('2018-01-01T12:00') + dt64 = np.datetime64("2018-01-01T12:00") assert expected == to_nanoseconds(dt64) class TestToNsDelta(object): - def test_ns_delta(self): """ Assert ns_delta converts inputs properly """ - val = ns_delta(1,2,1,3,1,23,1) + val = ns_delta(1, 2, 1, 3, 1, 23, 1) assert val == 93663001023001 def test_ns_delta_precision(self): @@ -200,7 +200,8 @@ def test_returns_int(self): """ Assert ns_delta returns int if floats used for arguments """ - val = ns_delta(1.0,1.0,1.0,1.0,1.0,1.0,1) - assert val == int(1 + 1e3 + 1e6 + 1e9 + (1e9 * 60) + (1e9 * 60 * 60) + \ - (1e9 * 60 * 60 * 24) ) + val = ns_delta(1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1) + assert val == int( + 1 + 1e3 + 1e6 + 1e9 + (1e9 * 60) + (1e9 * 60 * 60) + (1e9 * 60 * 60 * 24) + ) assert isinstance(val, int) diff --git a/tests/btrdb4/test_import.py b/tests/btrdb4/test_import.py index 21c0d6c..4fd170e 100644 --- a/tests/btrdb4/test_import.py +++ b/tests/btrdb4/test_import.py @@ -21,11 +21,11 @@ ## Import Tests ########################################################################## -class TestPackage(object): +class TestPackage(object): def test_import(self): """ Assert that `import brdb4` issues warning exception """ - with pytest.raises(ImportWarning, match='not available'): + with pytest.raises(ImportWarning, match="not available"): import btrdb4 diff --git a/tests/requirements.txt b/tests/requirements.txt index 534ff57..4a71eb1 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,10 +1,57 @@ -pytest==5.3.2 -pytest-cov==2.8.1 -pytest-flake8==1.1.0 -freezegun==0.3.12 -flake8<5.0.0 - -# dependencies for conversions -numpy==1.20.3 +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --extra=all --output-file=requirements-all.txt --resolver=backtracking pyproject.toml +# +# Pinned versions and dependencies of dependencies were removed after the autogeneration. +certifi + # via + # btrdb (pyproject.toml) +freezegun + # via btrdb +grpcio + # via + # btrdb (pyproject.toml) + # grpcio-tools + # ray +grpcio-tools + # via + # btrdb (pyproject.toml) +numpy + # via + # btrdb + # pandas + # pyarrow + # ray pandas -tabulate==0.8.6 + # via btrdb +polars + # via btrdb +protobuf + # via + # grpcio-tools + # ray +pyarrow + # via btrdb +pytest + # via + # btrdb + # pytest-cov + # pytest-flake8 +pytest-cov + # via btrdb +pytest-flake8 + # via btrdb +pytz + # via + # btrdb (pyproject.toml) + # pandas +pyyaml + # via + # btrdb (pyproject.toml) + # ray +ray + # via btrdb +tabulate + # via btrdb From c7ac1775e5efe94da372386062073cbb6cb5b511 Mon Sep 17 00:00:00 2001 From: Justin Gilmer Date: Tue, 6 Jun 2023 17:46:55 -0500 Subject: [PATCH 036/131] Update pre-commit.yaml add staging to pre-commit checks --- .github/workflows/pre-commit.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml index 858e465..6567714 100644 --- a/.github/workflows/pre-commit.yaml +++ b/.github/workflows/pre-commit.yaml @@ -4,6 +4,7 @@ on: pull_request: branches: - master + - staging types: - opened - reopened From 1351e2bd3a3b5ff520135e4b81b9cd02aeed4b0b Mon Sep 17 00:00:00 2001 From: Justin Gilmer Date: Tue, 6 Jun 2023 17:48:34 -0500 Subject: [PATCH 037/131] Fix missing logging import, rerun pre-commit (#24) --- .pre-commit-config.yaml | 2 +- benchmarks/benchmark_stream_reads.py | 19 ++++++++--- benchmarks/benchmark_streamset_reads.py | 43 +++++++++++++++++++------ btrdb/conn.py | 5 ++- btrdb/endpoint.py | 2 +- btrdb/stream.py | 15 +++++---- btrdb/transformers.py | 14 ++++---- tests/btrdb/test_arrow_streams.py | 7 ++-- tests/btrdb/test_stream.py | 15 +++++---- 9 files changed, 80 insertions(+), 42 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c2da34a..6f327d1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,7 +24,7 @@ repos: hooks: - id: flake8 args: [--config=setup.cfg] - exclude: ^(btrdb/grpcinterface|tests|setup.py|btrdb4|docs) + exclude: ^(btrdb/grpcinterface|tests|setup.py|btrdb4|docs|benchmarks) - repo: local hooks: - id: pytest-check diff --git a/benchmarks/benchmark_stream_reads.py b/benchmarks/benchmark_stream_reads.py index 623695a..7c7dedc 100644 --- a/benchmarks/benchmark_stream_reads.py +++ b/benchmarks/benchmark_stream_reads.py @@ -258,7 +258,9 @@ def main(): """Run a single run of the benchmarks""" conn = btrdb.connect(profile="andy") stream1 = conn.stream_from_uuid( - list(conn.streams_in_collection("andy/7064-6684-5e6e-9e14-ff9ca7bae46e"))[0].uuid + list(conn.streams_in_collection("andy/7064-6684-5e6e-9e14-ff9ca7bae46e"))[ + 0 + ].uuid ) start = stream1.earliest()[0].time end = stream1.latest()[0].time @@ -267,10 +269,17 @@ def main(): print(f"pointwidth of: {pointwidth}") for f in [time_single_stream_arrow_raw_values, time_single_stream_raw_values]: print(f(stream1, start, end, 0)) - for f in [time_single_stream_arrow_windows_values, time_single_stream_windows_values]: + for f in [ + time_single_stream_arrow_windows_values, + time_single_stream_windows_values, + ]: print(f(stream1, start, end, width_ns=width_ns, version=0)) - for f in [time_single_stream_arrow_aligned_windows_values, time_single_stream_aligned_windows_values]: + for f in [ + time_single_stream_arrow_aligned_windows_values, + time_single_stream_aligned_windows_values, + ]: print(f(stream1, start, end, pointwidth=pointwidth, version=0)) -if __name__=="__main__": - main() \ No newline at end of file + +if __name__ == "__main__": + main() diff --git a/benchmarks/benchmark_streamset_reads.py b/benchmarks/benchmark_streamset_reads.py index 3330fd7..08296cf 100644 --- a/benchmarks/benchmark_streamset_reads.py +++ b/benchmarks/benchmark_streamset_reads.py @@ -102,7 +102,11 @@ def time_streamset_arrow_raw_values( def time_streamset_windows_values( - streamset: btrdb.stream.StreamSet, start: int, end: int, width_ns: int, version: int = 0 + streamset: btrdb.stream.StreamSet, + start: int, + end: int, + width_ns: int, + version: int = 0, ) -> Dict[str, Union[str, int, float]]: """Return the elapsed time for the streamset windows values query @@ -139,7 +143,11 @@ def time_streamset_windows_values( def time_streamset_arrow_windows_values( - streamset: btrdb.stream.StreamSet, start: int, end: int, width_ns: int, version: int = 0 + streamset: btrdb.stream.StreamSet, + start: int, + end: int, + width_ns: int, + version: int = 0, ) -> Dict[str, Union[str, int, float]]: """Return the elapsed time for the streamset arrow window values query @@ -180,7 +188,11 @@ def time_streamset_arrow_windows_values( def time_streamset_aligned_windows_values( - streamset: btrdb.stream.StreamSet, start: int, end: int, pointwidth: int, version: int = 0 + streamset: btrdb.stream.StreamSet, + start: int, + end: int, + pointwidth: int, + version: int = 0, ) -> Dict[str, Union[str, int, float]]: """Return the elapsed time for the streamset window values query @@ -217,7 +229,11 @@ def time_streamset_aligned_windows_values( def time_streamset_arrow_aligned_windows_values( - streamset: btrdb.stream.StreamSet, start: int, end: int, pointwidth: int, version: int = 0 + streamset: btrdb.stream.StreamSet, + start: int, + end: int, + pointwidth: int, + version: int = 0, ) -> Dict[str, Union[str, int, float]]: """Return the elapsed time for the streamset arrow aligned window values query @@ -264,7 +280,6 @@ def _create_streamset_result_dict( return { "n_streams": len(streamset), "total_points": point_count, - "total_time_seconds": total_time, "streamset_version": version, } @@ -274,9 +289,15 @@ def main(): """Run a single run of the benchmarks""" conn = btrdb.connect(profile="andy") stream1 = conn.stream_from_uuid( - list(conn.streams_in_collection("andy/7064-6684-5e6e-9e14-ff9ca7bae46e"))[0].uuid + list(conn.streams_in_collection("andy/7064-6684-5e6e-9e14-ff9ca7bae46e"))[ + 0 + ].uuid + ) + stream2 = conn.stream_from_uuid( + list(conn.streams_in_collection("andy/30e6-d72f-5cc7-9966-bc1579dc4a72"))[ + 0 + ].uuid ) - stream2 = conn.stream_from_uuid(list(conn.streams_in_collection("andy/30e6-d72f-5cc7-9966-bc1579dc4a72"))[0].uuid) streamset = btrdb.stream.StreamSet([stream1, stream2]) start = max(stream1.earliest()[0].time, stream2.earliest()[0].time) end = min(stream1.latest()[0].time, stream2.latest()[0].time) @@ -286,7 +307,7 @@ def main(): res_list = [] for f in [time_streamset_raw_values, time_streamset_arrow_raw_values]: res = f(streamset, start, end, 0) - res["func"] = f.__name__ + res["func"] = f.__name__ # for f in [time_streamset_windows_values, time_streamset_arrow_windows_values]: # res = f(streamset, start, end, width_ns=width_ns, version=0) # res["func"] = f.__name__ @@ -295,6 +316,8 @@ def main(): # res["func"] = res return res -if __name__=="__main__": + + +if __name__ == "__main__": results = main() - print(pandas.DataFrame(results)) \ No newline at end of file + print(pandas.DataFrame(results)) diff --git a/btrdb/conn.py b/btrdb/conn.py index 4605c11..33080bb 100644 --- a/btrdb/conn.py +++ b/btrdb/conn.py @@ -16,6 +16,7 @@ ########################################################################## import json +import logging import os import re import uuid as uuidlib @@ -113,6 +114,7 @@ def __init__(self, addrportstr, apikey=None): ) self.channel = grpc.insecure_channel(addrportstr, chan_ops) + def _is_arrow_enabled(info): info = { "majorVersion": info.majorVersion, @@ -127,6 +129,7 @@ def _is_arrow_enabled(info): else: return False + class BTrDB(object): """ The primary server connection object for communicating with a BTrDB server. @@ -135,7 +138,7 @@ class BTrDB(object): def __init__(self, endpoint): self.ep = endpoint self._executor = ThreadPoolExecutor() - self._ARROW_ENABLED = True #_is_arrow_enabled(self.ep.info()) + self._ARROW_ENABLED = True # _is_arrow_enabled(self.ep.info()) logger.debug(f"ARROW ENABLED: {self._ARROW_ENABLED}") def query(self, stmt, params=[]): diff --git a/btrdb/endpoint.py b/btrdb/endpoint.py index 3715d3f..ea736d1 100644 --- a/btrdb/endpoint.py +++ b/btrdb/endpoint.py @@ -65,7 +65,7 @@ def arrowMultiValues(self, uu_list, start, end, version_list, snap_periodNS): start=start, end=end, versionMajor=[ver for ver in version_list], - snapPeriodNs=int(snap_periodNS) + snapPeriodNs=int(snap_periodNS), ) for result in self.stub.ArrowMultiValues(params): check_proto_stat(result.stat) diff --git a/btrdb/stream.py b/btrdb/stream.py index e58c426..e816b49 100644 --- a/btrdb/stream.py +++ b/btrdb/stream.py @@ -24,7 +24,6 @@ import pyarrow as pa -import btrdb.grpcinterface.btrdb_pb2_grpc from btrdb.exceptions import ( BTrDBError, BTRDBTypeError, @@ -1169,7 +1168,7 @@ def versions(self): self._pinned_versions if self._pinned_versions else self._latest_versions() ) - def count(self, precise:bool=False): + def count(self, precise: bool = False): """ Compute the total number of points in the streams using filters. @@ -1196,10 +1195,14 @@ def count(self, precise:bool=False): start = params.get("start", MINIMUM_TIME) end = params.get("end", MAXIMUM_TIME) - versions = self._pinned_versions if self._pinned_versions else self._latest_versions() + versions = ( + self._pinned_versions if self._pinned_versions else self._latest_versions() + ) my_counts_gen = self._btrdb._executor.map( - lambda s: s.count(start, end, version=versions.get(s.uuid, 0), precise=precise), + lambda s: s.count( + start, end, version=versions.get(s.uuid, 0), precise=precise + ), self._streams, ) @@ -1655,7 +1658,7 @@ def _arrow_streamset_data(self): ) data = main_table return data - + def rows(self): """ Returns a materialized list of tuples where each tuple contains the @@ -1703,7 +1706,7 @@ def rows(self): def arrow_rows(self): """Return tuples of rows from arrow table""" raise NotImplementedError( - f"arrow_rows has not been implemented yet, please use `rows` if you need this functionality." + "arrow_rows has not been implemented yet, please use `rows` if you need this functionality." ) def insert(self, data_map: dict, merge: str = "never") -> dict: diff --git a/btrdb/transformers.py b/btrdb/transformers.py index e5ac34b..c4e634f 100644 --- a/btrdb/transformers.py +++ b/btrdb/transformers.py @@ -22,7 +22,6 @@ import pandas as pd - ########################################################################## ## Helper Functions ########################################################################## @@ -119,7 +118,7 @@ def arrow_to_series(streamset, agg="mean", name_callable=None): """ if not streamset._btrdb._ARROW_ENABLED: raise NotImplementedError( - f"arrow_to_series requires an arrow-enabled BTrDB server." + "arrow_to_series requires an arrow-enabled BTrDB server." ) arrow_df = arrow_to_dataframe( streamset=streamset, agg=agg, name_callable=name_callable @@ -150,7 +149,7 @@ def arrow_to_dataframe( """ if not streamset._btrdb._ARROW_ENABLED: raise NotImplementedError( - f"arrow_to_dataframe requires an arrow-enabled BTrDB server." + "arrow_to_dataframe requires an arrow-enabled BTrDB server." ) try: @@ -182,7 +181,6 @@ def arrow_to_dataframe( if not callable(name_callable): name_callable = lambda s: s.collection + "/" + s.name tmp_table = streamset.arrow_values() - my_cols = [c for c in tmp_table.column_names] col_names = _stream_names(streamset, name_callable) cols = [] for name in col_names: @@ -299,7 +297,7 @@ def arrow_to_polars(streamset, agg="mean", name_callable=None): """ if not streamset._btrdb._ARROW_ENABLED: raise NotImplementedError( - f"arrow_to_polars requires an arrow-enabled BTrDB server." + "arrow_to_polars requires an arrow-enabled BTrDB server." ) try: import polars as pl @@ -314,7 +312,7 @@ def arrow_to_polars(streamset, agg="mean", name_callable=None): def arrow_to_arrow_table(streamset): if not streamset._btrdb._ARROW_ENABLED: raise NotImplementedError( - f"arrow_to_arrow_table requires an arrow-enabled BTrDB server." + "arrow_to_arrow_table requires an arrow-enabled BTrDB server." ) return streamset.arrow_values() @@ -427,7 +425,7 @@ def arrow_to_numpy(streamset, agg="mean"): """ if not streamset._btrdb._ARROW_ENABLED: raise NotImplementedError( - f"arrow_to_numpy requires an arrow-enabled BTrDB server." + "arrow_to_numpy requires an arrow-enabled BTrDB server." ) arrow_df = arrow_to_dataframe(streamset=streamset, agg=agg, name_callable=None) return arrow_df.values @@ -496,7 +494,7 @@ def arrow_to_dict(streamset, agg="mean", name_callable=None): """ if not streamset._btrdb._ARROW_ENABLED: raise NotImplementedError( - f"arrow_to_dict requires an arrow-enabled BTrDB server." + "arrow_to_dict requires an arrow-enabled BTrDB server." ) arrow_df = arrow_to_dataframe( streamset=streamset, agg=agg, name_callable=name_callable diff --git a/tests/btrdb/test_arrow_streams.py b/tests/btrdb/test_arrow_streams.py index b284238..301fa78 100644 --- a/tests/btrdb/test_arrow_streams.py +++ b/tests/btrdb/test_arrow_streams.py @@ -11,7 +11,7 @@ @pytest.fixture def stream1(): - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") stream = Mock(Stream) stream.version = Mock(return_value=11) stream.uuid = Mock(return_value=uu) @@ -28,7 +28,7 @@ def stream1(): @pytest.fixture def stream2(): - uu = uuid.UUID('17dbe387-89ea-42b6-864b-f505cdb483f5') + uu = uuid.UUID("17dbe387-89ea-42b6-864b-f505cdb483f5") stream = Mock(Stream) stream.version = Mock(return_value=22) stream.uuid = Mock(return_value=uu) @@ -42,5 +42,6 @@ def stream2(): stream._btrdb._ARROW_ENABLED = Mock(return_value=True) return stream + class TestArrowStreams(object): - pass \ No newline at end of file + pass diff --git a/tests/btrdb/test_stream.py b/tests/btrdb/test_stream.py index 36d10c4..7713482 100644 --- a/tests/btrdb/test_stream.py +++ b/tests/btrdb/test_stream.py @@ -11,11 +11,6 @@ Testing package for the btrdb stream module """ import concurrent.futures - -########################################################################## -## Imports -########################################################################## - import datetime import json import re @@ -35,13 +30,16 @@ InvalidOperation, NoSuchPoint, StreamNotFoundError, - InvalidCollection, - NoSuchPoint, ) from btrdb.grpcinterface import btrdb_pb2 from btrdb.point import RawPoint, StatPoint from btrdb.stream import INSERT_BATCH_SIZE, Stream, StreamFilter, StreamSet +########################################################################## +## Imports +########################################################################## + + RawPointProto = btrdb_pb2.RawPoint StatPointProto = btrdb_pb2.StatPoint EST = pytz.timezone("America/New_York") @@ -85,6 +83,7 @@ def stream2(): stream._btrdb._ARROW_ENABLED = Mock(return_value=False) return stream + @pytest.fixture def arrow_stream3(): uu = uuid.UUID("17dbe387-89ea-42b6-864b-f505cdb483f5") @@ -101,6 +100,7 @@ def arrow_stream3(): stream._btrdb._ARROW_ENABLED = Mock(return_value=True) return stream + ########################################################################## ## Stream Tests ########################################################################## @@ -1195,6 +1195,7 @@ def test_earliest(self, stream1, stream2): RawPoint(time=10, value=1), RawPoint(time=20, value=1), ) + def test_latest(self, stream1, stream2): """ Assert latest returns correct time code From d59839d7f31ab569fa1fac2fe7ccdcbd9ce63971 Mon Sep 17 00:00:00 2001 From: Justin Gilmer Date: Tue, 6 Jun 2023 17:53:21 -0500 Subject: [PATCH 038/131] Add basic doc string to endpoint object (#25) --- btrdb/endpoint.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/btrdb/endpoint.py b/btrdb/endpoint.py index ea736d1..06ecdbb 100644 --- a/btrdb/endpoint.py +++ b/btrdb/endpoint.py @@ -38,6 +38,8 @@ class Endpoint(object): + """Server endpoint where we make specific requests.""" + def __init__(self, channel): self.stub = btrdb_pb2_grpc.BTrDBStub(channel) From f41f71166f31e6ee9bcec550ab0cbda246cd1459 Mon Sep 17 00:00:00 2001 From: Justin Gilmer Date: Mon, 12 Jun 2023 19:38:25 -0400 Subject: [PATCH 039/131] Update benchmark scripts. --- benchmarks/benchmark_stream_reads.py | 21 +++++-- benchmarks/benchmark_streamset_reads.py | 78 ++++++++++++++++--------- 2 files changed, 68 insertions(+), 31 deletions(-) diff --git a/benchmarks/benchmark_stream_reads.py b/benchmarks/benchmark_stream_reads.py index 7c7dedc..426bdfb 100644 --- a/benchmarks/benchmark_stream_reads.py +++ b/benchmarks/benchmark_stream_reads.py @@ -1,7 +1,10 @@ +import pprint import uuid from time import perf_counter from typing import Dict, Union +import pandas as pd + import btrdb @@ -267,19 +270,29 @@ def main(): width_ns = btrdb.utils.timez.ns_delta(minutes=1) pointwidth = btrdb.utils.general.pointwidth(38) print(f"pointwidth of: {pointwidth}") + res_list = [] for f in [time_single_stream_arrow_raw_values, time_single_stream_raw_values]: - print(f(stream1, start, end, 0)) + res = f(stream1, start, end, 0) + res["func"] = f.__name__ + res_list.append(res) for f in [ time_single_stream_arrow_windows_values, time_single_stream_windows_values, ]: - print(f(stream1, start, end, width_ns=width_ns, version=0)) + res = f(stream1, start, end, width_ns=width_ns, version=0) + res["func"] = f.__name__ + res_list.append(res) for f in [ time_single_stream_arrow_aligned_windows_values, time_single_stream_aligned_windows_values, ]: - print(f(stream1, start, end, pointwidth=pointwidth, version=0)) + res = f(stream1, start, end, pointwidth=pointwidth, version=0) + res["func"] = f.__name__ + res_list.append(res) + return res_list if __name__ == "__main__": - main() + result = main() + pprint.pprint(result) + print(pd.DataFrame.from_dict(result, orient="columns")) diff --git a/benchmarks/benchmark_streamset_reads.py b/benchmarks/benchmark_streamset_reads.py index 08296cf..37819ea 100644 --- a/benchmarks/benchmark_streamset_reads.py +++ b/benchmarks/benchmark_streamset_reads.py @@ -1,3 +1,4 @@ +import pprint import uuid from time import perf_counter from typing import Dict, Union @@ -33,14 +34,14 @@ def time_streamset_raw_values( The performance results of the streamset method """ streamset = streamset.filter(start=start, end=end) - versions = {s.uuid: 0 for s in streamset._streams} + versions = {s.uuid: 0 for s in streamset} streamset.pin_versions(versions) - expected_count = streamset.count(precise=True) + # minus 1 * n_streams to account for the exclusive end time + expected_count = streamset.count(precise=True) - len(streamset) tic = perf_counter() vals = streamset.values() toc = perf_counter() # print(vals) - # minus 1 to account for the exclusive end time queried_points = 0 for points in vals: print(len(points)) @@ -83,12 +84,14 @@ def time_streamset_arrow_raw_values( The performance results of the streamset method """ streamset = streamset.filter(start=start, end=end) - versions = {s.uuid: 0 for s in streamset._streams} + versions = {s.uuid: 0 for s in streamset} streamset.pin_versions(versions) - expected_count = streamset.count(precise=True) + # minus 1 * n_streams to account for the exclusive end time + expected_count = streamset.count(precise=True) - len(streamset) tic = perf_counter() vals = streamset.arrow_values() toc = perf_counter() + # TODO: These point counts and assertion needs to be column specific queried_points = len(streamset) * (vals.num_rows - 1) print(queried_points) print(expected_count) @@ -128,16 +131,20 @@ def time_streamset_windows_values( results : dict The performance results of the streamset method """ + streamset = streamset.filter(start=start, end=end) + versions = {s.uuid: 0 for s in streamset} + streamset.pin_versions(versions) + streamset = streamset.windows(width=width_ns) tic = perf_counter() - vals = streamset.windows(start, end, width=width_ns, version=version) + vals = streamset.values() toc = perf_counter() # num of statpoints - queried_points = len(vals) + queried_points = len(vals[0]) * len(streamset) assert queried_points != 0 # time in seconds to run run_time = toc - tic results = _create_streamset_result_dict( - streamset.uuid, point_count=queried_points, total_time=run_time, version=version + streamset, point_count=queried_points, total_time=run_time, version=version ) return results @@ -173,16 +180,20 @@ def time_streamset_arrow_windows_values( results : dict The performance results of the streamset method """ + streamset = streamset.filter(start=start, end=end) + versions = {s.uuid: 0 for s in streamset} + streamset.pin_versions(versions) + streamset = streamset.windows(width=width_ns) tic = perf_counter() - vals = streamset.arrow_windows(start, end, width=width_ns, version=version) + vals = streamset.arrow_values() toc = perf_counter() # num of statpoints - queried_points = vals.num_rows + queried_points = vals.num_rows * len(streamset) assert queried_points != 0 # time in seconds to run run_time = toc - tic results = _create_streamset_result_dict( - streamset.uuid, point_count=queried_points, total_time=run_time, version=version + streamset, point_count=queried_points, total_time=run_time, version=version ) return results @@ -214,16 +225,20 @@ def time_streamset_aligned_windows_values( results : dict The performance results of the streamset method """ + streamset = streamset.filter(start=start, end=end) + versions = {s.uuid: 0 for s in streamset} + streamset.pin_versions(versions) + streamset = streamset.aligned_windows(pointwidth=pointwidth) tic = perf_counter() - vals = streamset.aligned_windows(start, end, pointwidth=pointwidth, version=version) + vals = streamset.values() toc = perf_counter() # num of statpoints - queried_points = len(vals) + queried_points = len(vals[0]) * len(streamset) assert queried_points != 0 # time in seconds to run run_time = toc - tic results = _create_streamset_result_dict( - streamset.uuid, point_count=queried_points, total_time=run_time, version=version + streamset, point_count=queried_points, total_time=run_time, version=version ) return results @@ -255,18 +270,20 @@ def time_streamset_arrow_aligned_windows_values( results : dict The performance results of the streamset method """ + streamset = streamset.filter(start=start, end=end) + versions = {s.uuid: 0 for s in streamset} + streamset.pin_versions(versions) + streamset = streamset.aligned_windows(pointwidth=pointwidth) tic = perf_counter() - vals = streamset.arrow_aligned_windows( - start, end, pointwidth=pointwidth, version=version - ) + vals = streamset.arrow_values() toc = perf_counter() # num of statpoints - queried_points = vals.num_rows + queried_points = vals.num_rows * len(streamset) assert queried_points != 0 # time in seconds to run run_time = toc - tic results = _create_streamset_result_dict( - streamset.uuid, point_count=queried_points, total_time=run_time, version=version + streamset, point_count=queried_points, total_time=run_time, version=version ) return results @@ -308,16 +325,23 @@ def main(): for f in [time_streamset_raw_values, time_streamset_arrow_raw_values]: res = f(streamset, start, end, 0) res["func"] = f.__name__ - # for f in [time_streamset_windows_values, time_streamset_arrow_windows_values]: - # res = f(streamset, start, end, width_ns=width_ns, version=0) - # res["func"] = f.__name__ - # for f in [time_streamset_aligned_windows_values, time_streamset_arrow_aligned_windows_values]: - # res = f(streamset, start, end, pointwidth=pointwidth, version=0) - # res["func"] = res + res_list.append(res) + for f in [time_streamset_windows_values, time_streamset_arrow_windows_values]: + res = f(streamset, start, end, width_ns=width_ns, version=0) + res["func"] = f.__name__ + res_list.append(res) + for f in [ + time_streamset_aligned_windows_values, + time_streamset_arrow_aligned_windows_values, + ]: + res = f(streamset, start, end, pointwidth=pointwidth, version=0) + res["func"] = f.__name__ + res_list.append(res) - return res + return res_list if __name__ == "__main__": results = main() - print(pandas.DataFrame(results)) + pprint.pprint(results) + print(pandas.DataFrame.from_dict(results, orient="columns")) From c07a5beb6634650d713eccbbbbdb3206e65b53b1 Mon Sep 17 00:00:00 2001 From: Justin Gilmer Date: Tue, 13 Jun 2023 13:55:04 -0500 Subject: [PATCH 040/131] Multistream read bench insert bench (#26) * Fix multistream endpoint bugs * The streamset was passing the incorrect params to the endpoint * The endpoint does not return a `version` in its response, just `stat` and `arrowBytes` Params have been updated and a NoneType is passed around to ignore the lack of version info, which lets us use the same logic for all bytes decoding. * Add multistream benchmark methods for timesnap and no timesnap. --- benchmarks/benchmark_streamset_reads.py | 93 +++++++++++++++++++++++++ btrdb/endpoint.py | 2 +- btrdb/stream.py | 4 +- 3 files changed, 97 insertions(+), 2 deletions(-) diff --git a/benchmarks/benchmark_streamset_reads.py b/benchmarks/benchmark_streamset_reads.py index 37819ea..7c2c836 100644 --- a/benchmarks/benchmark_streamset_reads.py +++ b/benchmarks/benchmark_streamset_reads.py @@ -288,6 +288,92 @@ def time_streamset_arrow_aligned_windows_values( return results +def time_streamset_arrow_multistream_raw_values_non_timesnapped( + streamset: btrdb.stream.StreamSet, + start: int, + end: int, + version: int = 0, + sampling_frequency: int = None, +) -> Dict[str, Union[str, int, float]]: + """Use the arrow multistream endpoint that joins the stream data on-server before sending to the client. + + We make sure to set a sampling rate of 0 to ensure that we do not time snap and just perform a full-outer join on + the streams. + + Parameters + ---------- + streamset : btrdb.stream.StreamSet, required + The streamset to perform the multistream query on. + start : int, required + The start time (in nanoseconds) to query raw data from. + end : int, required + The end time (in nanoseconds) non-exclusive, to query raw data from. + version : int, optional, default=0 + The version of the stream to pin against, currently this is unused. + sampling_frequency : int, optional, ignored + The sampling frequency of the data stream in Hz + + Notes + ----- + Sampling_frequency is not used here, it will be manually set. + """ + streamset = streamset.filter(start=start, end=end, sampling_frequency=0) + versions = {s.uuid: 0 for s in streamset} + streamset = streamset.pin_versions(versions) + tic = perf_counter() + vals = streamset.arrow_values() + toc = perf_counter() + queried_points = vals.num_rows * len(streamset) + # print(vals) + # print(vals.to_pandas().describe()) + run_time = toc - tic + results = _create_streamset_result_dict( + streamset=streamset, total_time=run_time, point_count=queried_points, version=0 + ) + return results + + +def time_streamset_arrow_multistream_raw_values_timesnapped( + streamset: btrdb.stream.StreamSet, + start: int, + end: int, + sampling_frequency: int, + version: int = 0, +) -> Dict[str, Union[str, int, float]]: + """Use the arrow multistream endpoint that joins the stream data on-server before sending to the client. + + We make sure to set a sampling rate to ensure that we time snap the returned data. + + Parameters + ---------- + streamset : btrdb.stream.StreamSet, required + The streamset to perform the multistream query on. + start : int, required + The start time (in nanoseconds) to query raw data from. + end : int, required + The end time (in nanoseconds) non-exclusive, to query raw data from. + sampling_frequency : int, required + The common sampling frequency (in Hz) of the data to snap the data points to. + version : int, optional, default=0 + The version of the stream to pin against, currently this is unused. + """ + streamset = streamset.filter( + start=start, end=end, sampling_frequency=sampling_frequency + ) + versions = {s.uuid: 0 for s in streamset} + streamset = streamset.pin_versions(versions) + tic = perf_counter() + vals = streamset.arrow_values() + toc = perf_counter() + queried_points = vals.num_rows * len(streamset) + # print(vals) + run_time = toc - tic + results = _create_streamset_result_dict( + streamset=streamset, total_time=run_time, point_count=queried_points, version=0 + ) + return results + + def _create_streamset_result_dict( streamset: btrdb.stream.StreamSet, point_count: int, @@ -337,6 +423,13 @@ def main(): res = f(streamset, start, end, pointwidth=pointwidth, version=0) res["func"] = f.__name__ res_list.append(res) + for f in [ + time_streamset_arrow_multistream_raw_values_non_timesnapped, + time_streamset_arrow_multistream_raw_values_timesnapped, + ]: + res = f(streamset, start, end, sampling_frequency=2, version=0) + res["func"] = f.__name__ + res_list.append(res) return res_list diff --git a/btrdb/endpoint.py b/btrdb/endpoint.py index 06ecdbb..6eec064 100644 --- a/btrdb/endpoint.py +++ b/btrdb/endpoint.py @@ -71,7 +71,7 @@ def arrowMultiValues(self, uu_list, start, end, version_list, snap_periodNS): ) for result in self.stub.ArrowMultiValues(params): check_proto_stat(result.stat) - yield result.arrowBytes, result.versionMajor + yield result.arrowBytes, None @error_handler def arrowInsertValues(self, uu: uuid.UUID, values: bytearray, policy: str): diff --git a/btrdb/stream.py b/btrdb/stream.py index e816b49..8ded4a8 100644 --- a/btrdb/stream.py +++ b/btrdb/stream.py @@ -1837,8 +1837,10 @@ def _arrow_multivalues(self, period_ns: int): params = self._params_from_filters() versions = self.versions() params["uu_list"] = [s.uuid for s in self._streams] - params["versions"] = [versions[s.uuid] for s in self._streams] + params["version_list"] = [versions[s.uuid] for s in self._streams] params["snap_periodNS"] = period_ns + # dict.pop(key, default_return_value_if_no_key) + _ = params.pop("sampling_frequency", None) arr_bytes = self._btrdb.ep.arrowMultiValues(**params) # exhausting the generator from above bytes_materialized = list(arr_bytes) From 0a88f69a65b01f2f4da15687bb7a9da21d634b6f Mon Sep 17 00:00:00 2001 From: Justin Gilmer Date: Tue, 13 Jun 2023 18:00:33 -0500 Subject: [PATCH 041/131] Add insert benchmarking methods (#27) Benchmarking methods added for: * stream inserts using tuples of time, value data * stream inserts using pyarrow tables of timestamps, value columns * streamset inserts using a dict map of streamset stream uuids, and lists of tuples of time, value data * streamset inserts using a dict map of streamset stream uuids, and pyarrow tables of timestamps, values. --- benchmarks/benchmark_stream_inserts.py | 80 +++++++++++++++++ benchmarks/benchmark_streamset_inserts.py | 103 ++++++++++++++++++++++ 2 files changed, 183 insertions(+) create mode 100644 benchmarks/benchmark_stream_inserts.py create mode 100644 benchmarks/benchmark_streamset_inserts.py diff --git a/benchmarks/benchmark_stream_inserts.py b/benchmarks/benchmark_stream_inserts.py new file mode 100644 index 0000000..003c386 --- /dev/null +++ b/benchmarks/benchmark_stream_inserts.py @@ -0,0 +1,80 @@ +from time import perf_counter +from typing import Dict, List, Tuple, Union + +import pyarrow + +import btrdb + + +def time_stream_insert( + stream: btrdb.stream.Stream, + data: List[Tuple[int, float]], + merge_policy: str = "never", +) -> Dict[str, Union[int, float, str]]: + """Insert raw data into a single stream, where data is a List of tuples of int64 timestamps and float64 values. + + Parameters + ---------- + stream : btrdb.stream.Stream, required + The stream to insert data into. + data : List[Tuple[int, float]], required + The data to insert into stream. + merge_policy : str, optional, default = 'never' + How should the platform handle duplicated data? + Valid policies: + `never`: the default, no points are merged + `equal`: points are deduplicated if the time and value are equal + `retain`: if two points have the same timestamp, the old one is kept + `replace`: if two points have the same timestamp, the new one is kept + """ + prev_ver = stream.version() + tic = perf_counter() + new_ver = stream.insert(data, merge=merge_policy) + toc = perf_counter() + run_time = toc - tic + n_points = len(data) + result = { + "uuid": stream.uuid, + "previous_version": prev_ver, + "new_version": new_ver, + "points_to_insert": n_points, + "total_time_seconds": run_time, + "merge_policy": merge_policy, + } + return result + + +def time_stream_arrow_insert( + stream: btrdb.stream.Stream, data: pyarrow.Table, merge_policy: str = "never" +) -> Dict[str, Union[int, float, str]]: + """Insert raw data into a single stream, where data is a pyarrow Table of timestamps and float values. + + Parameters + ---------- + stream : btrdb.stream.Stream, required + The stream to insert data into. + data : pyarrow.Table, required + The table of data to insert into stream. + merge_policy : str, optional, default = 'never' + How should the platform handle duplicated data? + Valid policies: + `never`: the default, no points are merged + `equal`: points are deduplicated if the time and value are equal + `retain`: if two points have the same timestamp, the old one is kept + `replace`: if two points have the same timestamp, the new one is kept + """ + prev_ver = stream.version() + tic = perf_counter() + new_ver = stream.arrow_insert(data, merge=merge_policy) + toc = perf_counter() + run_time = toc - tic + n_points = data.num_rows + result = { + "uuid": stream.uuid, + "previous_version": prev_ver, + "new_version": new_ver, + "points_to_insert": n_points, + "total_time_seconds": run_time, + "merge_policy": merge_policy, + } + return result diff --git a/benchmarks/benchmark_streamset_inserts.py b/benchmarks/benchmark_streamset_inserts.py new file mode 100644 index 0000000..4a03e1f --- /dev/null +++ b/benchmarks/benchmark_streamset_inserts.py @@ -0,0 +1,103 @@ +import uuid +from time import perf_counter +from typing import Dict, List, Tuple, Union + +import pyarrow + +import btrdb + + +def time_streamset_inserts( + streamset: btrdb.stream.StreamSet, + data_map: Dict[uuid.UUID, List[Tuple[int, float]]], + merge_policy: str = "never", +) -> Dict[str, Union[str, int, float, List, Dict]]: + """Insert data into a streamset with a provided data map. + + Parameters + ---------- + streamset : btrdb.stream.StreamSet, required + The streams to insert data into. + data_map : Dict[uuid.UUID, List[Tuple[int, float]], required + The mappings of streamset uuids to data to insert. + merge_policy : str, optional, default = 'never' + How should the platform handle duplicated data? + Valid policies: + `never`: the default, no points are merged + `equal`: points are deduplicated if the time and value are equal + `retain`: if two points have the same timestamp, the old one is kept + `replace`: if two points have the same timestamp, the new one is kept + """ + _ensure_all_streams_in_data_map(streamset, data_map) + prev_vers = streamset.versions() + tic = perf_counter() + new_vers = streamset.insert(data_map=data_map, merge=merge_policy) + toc = perf_counter() + n_streams = len(streamset) + total_points = sum([len(v) for v in data_map.values()]) + points_insert_per_stream = {s.uuid: len(data_map[s.uuid]) for s in streamset} + total_time = toc - tic + result = { + "n_streams": n_streams, + "total_points": total_points, + "previous_versions": prev_vers, + "new_versions": new_vers, + "total_time_seconds": total_time, + "points_insert_per_stream": points_insert_per_stream, + "merge_policy": merge_policy, + } + return result + + +def time_streamset_arrow_inserts( + streamset: btrdb.stream.StreamSet, + data_map: Dict[uuid.UUID, pyarrow.Table], + merge_policy: str = "never", +) -> Dict[str, Union[str, int, float, List, Dict]]: + """Insert data into streamset using pyarrow Tables. + + Parameters + ---------- + streamset : btrdb.stream.StreamSet, required + The streams to insert data into. + data_map : Dict[uuid.UUID, pyarrow.Table], required + The mappings of streamset uuids to pyarrow tables to insert. + merge_policy : str, optional, default = 'never' + How should the platform handle duplicated data? + Valid policies: + `never`: the default, no points are merged + `equal`: points are deduplicated if the time and value are equal + `retain`: if two points have the same timestamp, the old one is kept + `replace`: if two points have the same timestamp, the new one is kept + """ + _ensure_all_streams_in_data_map(streamset, data_map) + prev_vers = streamset.versions() + tic = perf_counter() + new_vers = streamset.arrow_insert(data_map=data_map, merge=merge_policy) + toc = perf_counter() + n_streams = len(streamset) + total_points = sum([v.num_rows for v in data_map.values()]) + points_insert_per_stream = {s.uuid: data_map[s.uuid].num_rows for s in streamset} + total_time = toc - tic + result = { + "n_streams": n_streams, + "total_points": total_points, + "previous_versions": prev_vers, + "new_versions": new_vers, + "total_time_seconds": total_time, + "points_insert_per_stream": points_insert_per_stream, + "merge_policy": merge_policy, + } + return result + + +def _ensure_all_streams_in_data_map( + streamset: btrdb.stream.StreamSet, + data_map: Dict[uuid.UUID, Union[pyarrow.Table, List[Tuple[int, float]]]], +) -> None: + truthy = [s.uuid in data_map for s in streamset] + uus = [s.uuid for s in streamset] + if not all(truthy): + raise ValueError( + f"Streams in streamset are not present in the data map. Data_map keys: {data_map.keys()}, streamset uuids: {uus}" + ) From 038c14276788418adb42fe22f03bd5591dd285ed Mon Sep 17 00:00:00 2001 From: Justin Gilmer Date: Thu, 15 Jun 2023 18:54:42 -0500 Subject: [PATCH 042/131] Fix arrow inserts (#28) * Add insert benchmarking methods Benchmarking methods added for: * stream inserts using tuples of time, value data * stream inserts using pyarrow tables of timestamps, value columns * streamset inserts using a dict map of streamset stream uuids, and lists of tuples of time, value data * streamset inserts using a dict map of streamset stream uuids, and pyarrow tables of timestamps, values. * Include nullable false in pyarrow schema inserts * This was the only difference in the schemas between go and python. * also using a bytesIO stream to act as the sink for the ipc bytes. --- btrdb/stream.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/btrdb/stream.py b/btrdb/stream.py index 8ded4a8..094ddde 100644 --- a/btrdb/stream.py +++ b/btrdb/stream.py @@ -10,6 +10,8 @@ """ Module for Stream and related classes """ +import io + ########################################################################## ## Imports ########################################################################## @@ -512,8 +514,8 @@ def arrow_insert(self, data: pa.Table, merge: str = "never") -> int: logger.debug(f"tmp_table schema: {tmp_table.schema}") new_schema = pa.schema( [ - (pa.field("time", pa.timestamp(unit="ns", tz="UTC"))), - (pa.field("value", pa.float64())), + (pa.field("time", pa.timestamp(unit="ns", tz="UTC"), nullable=False)), + (pa.field("value", pa.float64(), nullable=False)), ] ) tmp_table = tmp_table.cast(new_schema) @@ -1930,10 +1932,12 @@ def _materialize_stream_as_table(arrow_bytes): def _table_slice_to_feather_bytes(table_slice: pa.Table) -> bytes: - sink = pa.BufferOutputStream() + # sink = pa.BufferOutputStream() + sink = io.BytesIO() with pa.ipc.new_stream(sink=sink, schema=table_slice.schema) as writer: - writer.write_table(table_slice) - return sink.readall() + writer.write(table_slice) + buf = sink.getvalue() + return buf def _coalesce_table_deque(tables: deque): From 5e0ce6c2f78222b81ad4f75a89a7d99ee80c0029 Mon Sep 17 00:00:00 2001 From: Andrew Chambers Date: Thu, 22 Jun 2023 19:36:19 +1200 Subject: [PATCH 043/131] Start integration test suite --- btrdb/stream.py | 9 ++---- setup.cfg | 1 - tests/btrdb/integration/conftest.py | 22 +++++++++++++ tests/btrdb/integration/test_conn.py | 29 +++++++++++++++++ tests/btrdb/integration/test_stream.py | 45 ++++++++++++++++++++++++++ tests/btrdb/test_base.py | 2 +- 6 files changed, 99 insertions(+), 9 deletions(-) create mode 100644 tests/btrdb/integration/conftest.py create mode 100644 tests/btrdb/integration/test_conn.py create mode 100644 tests/btrdb/integration/test_stream.py diff --git a/btrdb/stream.py b/btrdb/stream.py index 094ddde..0296fe6 100644 --- a/btrdb/stream.py +++ b/btrdb/stream.py @@ -737,7 +737,7 @@ def values(self, start, end, version=0): materialized.append((RawPoint.from_proto(point), version)) return materialized - def arrow_values(self, start: int, end: int, version: int = 0) -> pa.Table: + def arrow_values(self, start, end, version: int = 0) -> pa.Table: """Read raw values from BTrDB between time [a, b) in nanoseconds. RawValues queries BTrDB for the raw time series data points between @@ -771,17 +771,12 @@ def arrow_values(self, start: int, end: int, version: int = 0) -> pa.Table: raise NotImplementedError(_arrow_not_impl_str.format("arrow_values")) start = to_nanoseconds(start) end = to_nanoseconds(end) - logger.debug(f"For stream - {self.uuid} - {self.name}") arr_bytes = self._btrdb.ep.arrowRawValues( uu=self.uuid, start=start, end=end, version=version ) # exhausting the generator from above bytes_materialized = list(arr_bytes) - - logger.debug(f"Length of materialized list: {len(bytes_materialized)}") - logger.debug(f"materialized bytes[0:1]: {bytes_materialized[0:1]}") - materialized_tables = _materialize_stream_as_table(bytes_materialized) - return materialized_tables.rename_columns(["time", str(self.uuid)]) + return _materialize_stream_as_table(bytes_materialized) def aligned_windows(self, start, end, pointwidth, version=0): """ diff --git a/setup.cfg b/setup.cfg index dc699ef..3f445b3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,7 +6,6 @@ license_file = LICENSE.txt test=pytest [tool:pytest] -addopts = --cov=btrdb python_files = tests/* [flake8] diff --git a/tests/btrdb/integration/conftest.py b/tests/btrdb/integration/conftest.py new file mode 100644 index 0000000..594b9f4 --- /dev/null +++ b/tests/btrdb/integration/conftest.py @@ -0,0 +1,22 @@ +import os +import pytest +import btrdb + +@pytest.fixture +def conn(): + if os.getenv("BTRDB_INTEGRATION_TEST_PROFILE") == None: + pytest.skip("skipping because BTRDB_INTEGRATION_TEST_PROFILE is not set") + conn = btrdb.connect(profile=os.getenv("BTRDB_INTEGRATION_TEST_PROFILE")) + return conn + +def _delete_collection(conn, col): + streams = conn.streams_in_collection(col) + for stream in streams: + stream.obliterate() + +@pytest.fixture +def tmp_collection(request, conn): + col = "btrdb_python_integration_tests/" + request.node.name + "/tmp" + _delete_collection(conn, col) + yield col + _delete_collection(conn, col) diff --git a/tests/btrdb/integration/test_conn.py b/tests/btrdb/integration/test_conn.py new file mode 100644 index 0000000..15e0534 --- /dev/null +++ b/tests/btrdb/integration/test_conn.py @@ -0,0 +1,29 @@ +import pytest +import btrdb +import logging +from uuid import uuid4 as new_uuid + +def test_connection_info(conn): + info = conn.info() + logging.info(f"connection info: {info}") + +def test_create_stream(conn, tmp_collection): + uuid = new_uuid() + stream = conn.create(uuid, tmp_collection, tags={"name":"s"}) + assert stream.uuid == uuid + assert stream.name == "s" + +def test_query(conn, tmp_collection): + conn.create(new_uuid(), tmp_collection, tags={"name":"s1"}) + conn.create(new_uuid(), tmp_collection, tags={"name":"s2"}) + uuids = conn.query( + 'select name from streams where collection = $1 order by name;', [tmp_collection] + ) + assert len(uuids) == 2 + assert uuids[0]["name"] == "s1" + assert uuids[1]["name"] == "s2" + +def test_list_collections(conn, tmp_collection): + assert tmp_collection not in conn.list_collections() + stream = conn.create(new_uuid(), tmp_collection, tags={"name":"s"}) + assert tmp_collection in conn.list_collections() \ No newline at end of file diff --git a/tests/btrdb/integration/test_stream.py b/tests/btrdb/integration/test_stream.py new file mode 100644 index 0000000..e90aa46 --- /dev/null +++ b/tests/btrdb/integration/test_stream.py @@ -0,0 +1,45 @@ +import pytest +import logging + +from uuid import uuid4 as new_uuid +from btrdb.utils.timez import currently_as_ns + +try: + import pyarrow as pa +except: + pa = None + +def test_basic_insert_and_values(conn, tmp_collection): + s = conn.create(new_uuid(), tmp_collection, tags={"name":"s"}) + t = currently_as_ns() + data = [] + duration=100 + for i in range(duration): + data.append([t+i, i]) + s.insert(data) + fetched_data = s.values(start=t, end=t+duration) + assert len(fetched_data) == len(data) + for (i, (p, _)) in enumerate(fetched_data): + assert p.time == data[i][0] + assert p.value == data[i][1] + +def test_arrow_insert_and_values(conn, tmp_collection): + s = conn.create(new_uuid(), tmp_collection, tags={"name":"s"}) + t = currently_as_ns() + times = [] + values = [] + duration = 100 + for i in range(duration): + times.append(t+i) + values.append(i) + schema = pa.schema([ + pa.field('time', pa.timestamp('ns', tz='UTC'), nullable=False), + pa.field('value', pa.float64(), nullable=False)] + ) + data = pa.Table.from_arrays([ + pa.array(times), + pa.array(values), + ], schema=schema) + s.arrow_insert(data) + fetched_data = s.arrow_values(start=t, end=t+duration) + assert data == fetched_data diff --git a/tests/btrdb/test_base.py b/tests/btrdb/test_base.py index 5e84786..ac55c07 100644 --- a/tests/btrdb/test_base.py +++ b/tests/btrdb/test_base.py @@ -29,7 +29,7 @@ class TestConnect(object): - def setup(self): + def setup_method(self): for env in ["BTRDB_ENDPOINTS", "BTRDB_PROFILE", "BTRDB_API_KEY"]: try: del os.environ[env] From 9cc9b03eae7601c518260544defd17fbeab94278 Mon Sep 17 00:00:00 2001 From: Andrew Chambers Date: Thu, 22 Jun 2023 21:06:27 +1200 Subject: [PATCH 044/131] Add more streamset integration tests. --- tests/btrdb/integration/test_stream.py | 4 +- tests/btrdb/integration/test_streamset.py | 75 +++++++++++++++++++++++ 2 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 tests/btrdb/integration/test_streamset.py diff --git a/tests/btrdb/integration/test_stream.py b/tests/btrdb/integration/test_stream.py index e90aa46..52494c6 100644 --- a/tests/btrdb/integration/test_stream.py +++ b/tests/btrdb/integration/test_stream.py @@ -34,8 +34,8 @@ def test_arrow_insert_and_values(conn, tmp_collection): values.append(i) schema = pa.schema([ pa.field('time', pa.timestamp('ns', tz='UTC'), nullable=False), - pa.field('value', pa.float64(), nullable=False)] - ) + pa.field('value', pa.float64(), nullable=False), + ]) data = pa.Table.from_arrays([ pa.array(times), pa.array(values), diff --git a/tests/btrdb/integration/test_streamset.py b/tests/btrdb/integration/test_streamset.py new file mode 100644 index 0000000..87b5ba7 --- /dev/null +++ b/tests/btrdb/integration/test_streamset.py @@ -0,0 +1,75 @@ +import btrdb +import btrdb.stream +import pytest +import logging +from math import nan, isnan +from uuid import uuid4 as new_uuid +from btrdb.utils.timez import currently_as_ns + +try: + import pyarrow as pa +except: + pa = None +try: + import pandas as pd +except: + pd = None + +def test_streamset_values(conn, tmp_collection): + s1 = conn.create(new_uuid(), tmp_collection, tags={"name":"s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name":"s2"}) + t1 = [100, 105, 110, 115, 120] + t2 = [101, 106, 110, 114, 119] + d1 = [0.0, 1.0, 2.0, 3.0, 4.0] + d2 = [5.0, 6.0, 7.0, 8.0, 9.0] + s1.insert(list(zip(t1, d1))) + s2.insert(list(zip(t2, d2))) + ss = btrdb.stream.StreamSet([s1, s2]).filter(start=100, end=121) + values = ss.values() + assert len(values) == 2 + assert len(values[0]) == len(t1) + assert len(values[1]) == len(t2) + assert [(p.time, p.value) for p in values[0]] == list(zip(t1, d1)) + assert [(p.time, p.value) for p in values[1]] == list(zip(t2, d2)) + +def test_streamset_arrow_values(conn, tmp_collection): + s1 = conn.create(new_uuid(), tmp_collection, tags={"name":"s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name":"s2"}) + t1 = [100, 105, 110, 115, 120] + t2 = [101, 106, 110, 114, 119] + d1 = [0.0, 1.0, 2.0, 3.0, 4.0] + d2 = [5.0, 6.0, 7.0, 8.0, 9.0] + s1.insert(list(zip(t1, d1))) + s2.insert(list(zip(t2, d2))) + ss = btrdb.stream.StreamSet([s1, s2]).filter(start=100, end=121) + expected_times = [100, 101, 105, 106, 110, 114, 115, 119, 120] + expected_col1 = [0.0, None, 1.0, None, 2.0, None, 3.0, None, 4.0] + expected_col2 = [None, 5.0, None, 6.0, 7.0, 8.0, None, 9.0, None] + values = ss.arrow_values() + times = [t.value for t in values['time']] + col1 = [None if isnan(v) else v for v in values[tmp_collection + '/s1']] + col2 = [None if isnan(v) else v for v in values[tmp_collection + '/s2']] + assert times == expected_times + assert col1 == expected_col1 + assert col2 == expected_col2 + +def test_streamset_to_dataframe(conn, tmp_collection): + s1 = conn.create(new_uuid(), tmp_collection, tags={"name":"s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name":"s2"}) + t1 = [100, 105, 110, 115, 120] + t2 = [101, 106, 110, 114, 119] + d1 = [0.0, 1.0, 2.0, 3.0, 4.0] + d2 = [5.0, 6.0, 7.0, 8.0, 9.0] + s1.insert(list(zip(t1, d1))) + s2.insert(list(zip(t2, d2))) + ss = btrdb.stream.StreamSet([s1, s2]).filter(start=100, end=121) + values = ss.to_dataframe() + expected_times = [100, 101, 105, 106, 110, 114, 115, 119, 120] + expected_col1 = [0.0, None, 1.0, None, 2.0, None, 3.0, None, 4.0] + expected_col2 = [None, 5.0, None, 6.0, 7.0, 8.0, None, 9.0, None] + times = [t for t in values.index] + col1 = [None if isnan(v) else v for v in values[tmp_collection + '/s1']] + col2 = [None if isnan(v) else v for v in values[tmp_collection + '/s2']] + assert times == expected_times + assert col1 == expected_col1 + assert col2 == expected_col2 From caa919a33e3501732f6a2abd591a381bf3c94987 Mon Sep 17 00:00:00 2001 From: Andrew Chambers Date: Fri, 23 Jun 2023 12:50:12 +1200 Subject: [PATCH 045/131] Add support for authenticated requests without encryption. --- btrdb/conn.py | 38 ++++++++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/btrdb/conn.py b/btrdb/conn.py index 33080bb..dc48d28 100644 --- a/btrdb/conn.py +++ b/btrdb/conn.py @@ -108,12 +108,42 @@ def __init__(self, addrportstr, apikey=None): options=chan_ops, ) else: + self.channel = grpc.insecure_channel(addrportstr, chan_ops) if apikey is not None: - raise ValueError( - "cannot use an API key with an insecure (port 4410) BTrDB API. Try port 4411" + class AuthCallDetails(grpc.ClientCallDetails): + def __init__(self, apikey, client_call_details): + metadata = [] + if client_call_details.metadata is not None: + metadata = list(client_call_details.metadata) + metadata.append( + ('authorization', "Bearer " + apikey) + ) + self.method = client_call_details.method + self.timeout = client_call_details.timeout + self.credentials = client_call_details.credentials + self.wait_for_ready = client_call_details.wait_for_ready + self.compression = client_call_details.compression + self.metadata = metadata + class AuthorizationInterceptor( + grpc.UnaryUnaryClientInterceptor, + grpc.UnaryStreamClientInterceptor, + grpc.StreamUnaryClientInterceptor, + grpc.StreamStreamClientInterceptor, + ): + def __init__(self, apikey): + self.apikey = apikey + def intercept_unary_unary(self, continuation, client_call_details, request): + return continuation(AuthCallDetails(self.apikey, client_call_details), request) + def intercept_unary_stream(self, continuation, client_call_details, request): + return continuation(AuthCallDetails(self.apikey, client_call_details), request) + def intercept_stream_unary(self, continuation, client_call_details, request_iterator): + return continuation(AuthCallDetails(self.apikey, client_call_details), request) + def intercept_stream_stream(self, continuation, client_call_details, request_iterator): + return continuation(AuthCallDetails(self.apikey, client_call_details), request) + self.channel = grpc.intercept_channel( + self.channel, + AuthorizationInterceptor(apikey), ) - self.channel = grpc.insecure_channel(addrportstr, chan_ops) - def _is_arrow_enabled(info): info = { From d89d28c3311fe0649cf64e3d5563df4fd3e2d091 Mon Sep 17 00:00:00 2001 From: Justin Gilmer Date: Fri, 23 Jun 2023 15:41:19 -0500 Subject: [PATCH 046/131] Optimize logging calls (#30) Previously, the debug logging in the api would create the f-strings no matter if logging.DEBUG was the current log level or not. This can impact the performance, especially for benchmarking. Now, a cached IS_DEBUG flag is created for the stream operations, and other locations, the logger.isEnabledFor boolean is checked. Note that in the stream.py, this same function call is only executed once, and the results are cached for the rest of the logic. --- btrdb/conn.py | 54 +++++++++++++++++++++++++++++++++++------------ btrdb/endpoint.py | 13 ++++++++---- btrdb/stream.py | 45 ++++++++++++++++++++++++++------------- 3 files changed, 79 insertions(+), 33 deletions(-) diff --git a/btrdb/conn.py b/btrdb/conn.py index dc48d28..83aa5e1 100644 --- a/btrdb/conn.py +++ b/btrdb/conn.py @@ -110,20 +110,20 @@ def __init__(self, addrportstr, apikey=None): else: self.channel = grpc.insecure_channel(addrportstr, chan_ops) if apikey is not None: + class AuthCallDetails(grpc.ClientCallDetails): def __init__(self, apikey, client_call_details): metadata = [] if client_call_details.metadata is not None: metadata = list(client_call_details.metadata) - metadata.append( - ('authorization', "Bearer " + apikey) - ) + metadata.append(("authorization", "Bearer " + apikey)) self.method = client_call_details.method self.timeout = client_call_details.timeout self.credentials = client_call_details.credentials self.wait_for_ready = client_call_details.wait_for_ready self.compression = client_call_details.compression self.metadata = metadata + class AuthorizationInterceptor( grpc.UnaryUnaryClientInterceptor, grpc.UnaryStreamClientInterceptor, @@ -132,19 +132,43 @@ class AuthorizationInterceptor( ): def __init__(self, apikey): self.apikey = apikey - def intercept_unary_unary(self, continuation, client_call_details, request): - return continuation(AuthCallDetails(self.apikey, client_call_details), request) - def intercept_unary_stream(self, continuation, client_call_details, request): - return continuation(AuthCallDetails(self.apikey, client_call_details), request) - def intercept_stream_unary(self, continuation, client_call_details, request_iterator): - return continuation(AuthCallDetails(self.apikey, client_call_details), request) - def intercept_stream_stream(self, continuation, client_call_details, request_iterator): - return continuation(AuthCallDetails(self.apikey, client_call_details), request) + + def intercept_unary_unary( + self, continuation, client_call_details, request + ): + return continuation( + AuthCallDetails(self.apikey, client_call_details), request + ) + + def intercept_unary_stream( + self, continuation, client_call_details, request + ): + return continuation( + AuthCallDetails(self.apikey, client_call_details), request + ) + + def intercept_stream_unary( + self, continuation, client_call_details, request_iterator + ): + return continuation( + AuthCallDetails(self.apikey, client_call_details), + request_iterator, + ) + + def intercept_stream_stream( + self, continuation, client_call_details, request_iterator + ): + return continuation( + AuthCallDetails(self.apikey, client_call_details), + request_iterator, + ) + self.channel = grpc.intercept_channel( self.channel, AuthorizationInterceptor(apikey), ) + def _is_arrow_enabled(info): info = { "majorVersion": info.majorVersion, @@ -152,8 +176,9 @@ def _is_arrow_enabled(info): } major = info.get("majorVersion", -1) minor = info.get("minorVersion", -1) - logger.debug(f"major version: {major}") - logger.debug(f"minor version: {minor}") + if logger.isEnabledFor(logging.DEBUG): + logger.debug(f"major version: {major}") + logger.debug(f"minor version: {minor}") if major >= 5 and minor >= 30: return True else: @@ -169,7 +194,8 @@ def __init__(self, endpoint): self.ep = endpoint self._executor = ThreadPoolExecutor() self._ARROW_ENABLED = True # _is_arrow_enabled(self.ep.info()) - logger.debug(f"ARROW ENABLED: {self._ARROW_ENABLED}") + if logger.isEnabledFor(logging.DEBUG): + logger.debug(f"ARROW ENABLED: {self._ARROW_ENABLED}") def query(self, stmt, params=[]): """ diff --git a/btrdb/endpoint.py b/btrdb/endpoint.py index 6eec064..8346c4e 100644 --- a/btrdb/endpoint.py +++ b/btrdb/endpoint.py @@ -274,14 +274,19 @@ def lookupStreams(self, collection, isCollectionPrefix, tags, annotations): @error_handler def nearest(self, uu, time, version, backward): - logger.debug(f"nearest function params: {uu}\t{time}\t{version}\t{backward}") + if logger.isEnabledFor(logging.DEBUG): + logger.debug( + f"nearest function params: {uu}\t{time}\t{version}\t{backward}" + ) params = btrdb_pb2.NearestParams( uuid=uu.bytes, time=time, versionMajor=version, backward=backward ) - logger.debug(f"params from nearest: {params}") - logger.debug(f"uuid: {uu}") + if logger.isEnabledFor(logging.DEBUG): + logger.debug(f"params from nearest: {params}") + logger.debug(f"uuid: {uu}") result = self.stub.Nearest(params) - logger.debug(f"nearest, results: {result}") + if logger.isEnabledFor(logging.DEBUG): + logger.debug(f"nearest, results: {result}") check_proto_stat(result.stat) return result.value, result.versionMajor diff --git a/btrdb/stream.py b/btrdb/stream.py index 0296fe6..3698289 100644 --- a/btrdb/stream.py +++ b/btrdb/stream.py @@ -46,6 +46,7 @@ ## Module Variables ########################################################################## logger = logging.getLogger(__name__) +IS_DEBUG = logger.isEnabledFor(logging.DEBUG) INSERT_BATCH_SIZE = 50000 MINIMUM_TIME = -(16 << 56) MAXIMUM_TIME = (48 << 56) - 1 @@ -511,7 +512,8 @@ def arrow_insert(self, data: pa.Table, merge: str = "never") -> int: chunksize = INSERT_BATCH_SIZE assert isinstance(data, pa.Table) tmp_table = data.rename_columns(["time", "value"]) - logger.debug(f"tmp_table schema: {tmp_table.schema}") + if IS_DEBUG: + logger.debug(f"tmp_table schema: {tmp_table.schema}") new_schema = pa.schema( [ (pa.field("time", pa.timestamp(unit="ns", tz="UTC"), nullable=False)), @@ -536,7 +538,8 @@ def arrow_insert(self, data: pa.Table, merge: str = "never") -> int: # Process the batches as needed version = [] for tab in table_slices: - logger.debug(f"Table Slice: {tab}") + if IS_DEBUG: + logger.debug(f"Table Slice: {tab}") feather_bytes = _table_slice_to_feather_bytes(table_slice=tab) version.append( self._btrdb.ep.arrowInsertValues( @@ -730,7 +733,8 @@ def values(self, start, end, version=0): materialized = [] start = to_nanoseconds(start) end = to_nanoseconds(end) - logger.debug(f"For stream - {self.uuid} - {self.name}") + if IS_DEBUG: + logger.debug(f"For stream - {self.uuid} - {self.name}") point_windows = self._btrdb.ep.rawValues(self._uuid, start, end, version) for point_list, version in point_windows: for point in point_list: @@ -883,7 +887,8 @@ def arrow_aligned_windows( _arrow_not_impl_str.format("arrow_aligned_windows") ) - logger.debug(f"For stream - {self.uuid} - {self.name}") + if IS_DEBUG: + logger.debug(f"For stream - {self.uuid} - {self.name}") start = to_nanoseconds(start) end = to_nanoseconds(end) arr_bytes = self._btrdb.ep.arrowAlignedWindows( @@ -892,8 +897,9 @@ def arrow_aligned_windows( # exhausting the generator from above bytes_materialized = list(arr_bytes) - logger.debug(f"Length of materialized list: {len(bytes_materialized)}") - logger.debug(f"materialized bytes[0:1]: {bytes_materialized[0:1]}") + if IS_DEBUG: + logger.debug(f"Length of materialized list: {len(bytes_materialized)}") + logger.debug(f"materialized bytes[0:1]: {bytes_materialized[0:1]}") # ignore versions for now materialized_table = _materialize_stream_as_table(bytes_materialized) stream_names = [ @@ -1005,8 +1011,9 @@ def arrow_windows( # exhausting the generator from above bytes_materialized = list(arr_bytes) - logger.debug(f"Length of materialized list: {len(bytes_materialized)}") - logger.debug(f"materialized bytes[0:1]: {bytes_materialized[0:1]}") + if IS_DEBUG: + logger.debug(f"Length of materialized list: {len(bytes_materialized)}") + logger.debug(f"materialized bytes[0:1]: {bytes_materialized[0:1]}") # ignore versions for now materialized = _materialize_stream_as_table(bytes_materialized) stream_names = [ @@ -1042,11 +1049,15 @@ def nearest(self, time, version, backward=False): """ try: - logger.debug(f"checking nearest for: {self.uuid}\t\t{time}\t\t{version}") + if IS_DEBUG: + logger.debug( + f"checking nearest for: {self.uuid}\t\t{time}\t\t{version}" + ) rp, version = self._btrdb.ep.nearest( self._uuid, to_nanoseconds(time), version, backward ) - logger.debug(f"Nearest for stream: {self.uuid} - {rp}") + if IS_DEBUG: + logger.debug(f"Nearest for stream: {self.uuid} - {rp}") except BTrDBError as exc: if not isinstance(exc, NoSuchPoint): raise @@ -1223,7 +1234,8 @@ def earliest(self): params = self._params_from_filters() start = params.get("start", MINIMUM_TIME) versions = self.versions() - logger.debug(f"versions: {versions}") + if IS_DEBUG: + logger.debug(f"versions: {versions}") earliest_points_gen = self._btrdb._executor.map( lambda s: s.nearest(start, version=versions.get(s.uuid, 0), backward=False), self._streams, @@ -1842,8 +1854,9 @@ def _arrow_multivalues(self, period_ns: int): # exhausting the generator from above bytes_materialized = list(arr_bytes) - logger.debug(f"Length of materialized list: {len(bytes_materialized)}") - logger.debug(f"materialized bytes[0:1]: {bytes_materialized[0:1]}") + if IS_DEBUG: + logger.debug(f"Length of materialized list: {len(bytes_materialized)}") + logger.debug(f"materialized bytes[0:1]: {bytes_materialized[0:1]}") data = _materialize_stream_as_table(bytes_materialized) return data @@ -1919,9 +1932,11 @@ def _materialize_stream_as_table(arrow_bytes): for b, _ in arrow_bytes: with pa.ipc.open_stream(b) as reader: schema = reader.schema - logger.debug(f"schema: {schema}") + if IS_DEBUG: + logger.debug(f"schema: {schema}") table_list.append(reader.read_all()) - logger.debug(f"table list: {table_list}") + if IS_DEBUG: + logger.debug(f"table list: {table_list}") table = pa.concat_tables(table_list) return table From 744a6b887047f6f282b6a84738444d8b19d521fc Mon Sep 17 00:00:00 2001 From: Andrew Chambers Date: Mon, 26 Jun 2023 20:30:58 +1200 Subject: [PATCH 047/131] Add more arrow tests and minor refactoring. --- btrdb/endpoint.py | 35 +-- btrdb/stream.py | 286 +++++++++------------- tests/btrdb/integration/test_stream.py | 8 +- tests/btrdb/integration/test_streamset.py | 13 +- tests/btrdb/test_conn.py | 9 - 5 files changed, 147 insertions(+), 204 deletions(-) diff --git a/btrdb/endpoint.py b/btrdb/endpoint.py index 6eec064..7106541 100644 --- a/btrdb/endpoint.py +++ b/btrdb/endpoint.py @@ -24,9 +24,8 @@ # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import logging +import io import uuid - import grpc from btrdb.exceptions import BTrDBError, check_proto_stat, error_handler @@ -34,8 +33,10 @@ from btrdb.point import RawPoint from btrdb.utils.general import unpack_stream_descriptor -logger = logging.getLogger(__name__) - +try: + import pyarrow as pa +except: + pa = None class Endpoint(object): """Server endpoint where we make specific requests.""" @@ -59,8 +60,10 @@ def arrowRawValues(self, uu, start, end, version=0): ) for result in self.stub.ArrowRawValues(params): check_proto_stat(result.stat) - yield result.arrowBytes, result.versionMajor + with pa.ipc.open_stream(result.arrowBytes) as reader: + yield reader.read_all(), result.versionMajor + @error_handler def arrowMultiValues(self, uu_list, start, end, version_list, snap_periodNS): params = btrdb_pb2.ArrowMultiValuesParams( uuid=[uu.bytes for uu in uu_list], @@ -71,20 +74,25 @@ def arrowMultiValues(self, uu_list, start, end, version_list, snap_periodNS): ) for result in self.stub.ArrowMultiValues(params): check_proto_stat(result.stat) - yield result.arrowBytes, None + with pa.ipc.open_stream(result.arrowBytes) as reader: + yield reader.read_all() @error_handler - def arrowInsertValues(self, uu: uuid.UUID, values: bytearray, policy: str): + def arrowInsertValues(self, uu: uuid.UUID, values: pa.Table, policy: str): policy_map = { "never": btrdb_pb2.MergePolicy.NEVER, "equal": btrdb_pb2.MergePolicy.EQUAL, "retain": btrdb_pb2.MergePolicy.RETAIN, "replace": btrdb_pb2.MergePolicy.REPLACE, } + byte_io = io.BytesIO() + with pa.ipc.new_stream(sink=byte_io, schema=values.schema) as writer: + writer.write(values) + arrow_bytes = byte_io.getvalue() params = btrdb_pb2.ArrowInsertParams( uuid=uu.bytes, sync=False, - arrowBytes=values, + arrowBytes=arrow_bytes, merge_policy=policy_map[policy], ) result = self.stub.ArrowInsert(params) @@ -115,7 +123,8 @@ def arrowAlignedWindows(self, uu, start, end, pointwidth, version=0): ) for result in self.stub.ArrowAlignedWindows(params): check_proto_stat(result.stat) - yield result.arrowBytes, result.versionMajor + with pa.ipc.open_stream(result.arrowBytes) as reader: + yield reader.read_all(), result.versionMajor @error_handler def windows(self, uu, start, end, width, depth, version=0): @@ -143,7 +152,9 @@ def arrowWindows(self, uu, start, end, width, depth, version=0): ) for result in self.stub.ArrowWindows(params): check_proto_stat(result.stat) - yield result.arrowBytes, result.versionMajor + with pa.ipc.open_stream(result.arrowBytes) as reader: + yield reader.read_all(), result.versionMajor + @error_handler def streamInfo(self, uu, omitDescriptor, omitVersion): @@ -274,14 +285,10 @@ def lookupStreams(self, collection, isCollectionPrefix, tags, annotations): @error_handler def nearest(self, uu, time, version, backward): - logger.debug(f"nearest function params: {uu}\t{time}\t{version}\t{backward}") params = btrdb_pb2.NearestParams( uuid=uu.bytes, time=time, versionMajor=version, backward=backward ) - logger.debug(f"params from nearest: {params}") - logger.debug(f"uuid: {uu}") result = self.stub.Nearest(params) - logger.debug(f"nearest, results: {result}") check_proto_stat(result.stat) return result.value, result.versionMajor diff --git a/btrdb/stream.py b/btrdb/stream.py index 0296fe6..4f4d47e 100644 --- a/btrdb/stream.py +++ b/btrdb/stream.py @@ -56,7 +56,6 @@ RE_PATTERN = re.Pattern _arrow_not_impl_str = "The BTrDB server you are using does not support {}." -_arrow_available_str = "Pyarrow and arrow table support is available for {}, you can access this method using {}." ########################################################################## ## Stream Classes @@ -468,8 +467,6 @@ def insert(self, data, merge="never"): The version of the stream after inserting new points. """ - if self._btrdb._ARROW_ENABLED: - warnings.warn(_arrow_available_str.format("insert", "arrow_insert")) version = 0 i = 0 while i < len(data): @@ -511,7 +508,6 @@ def arrow_insert(self, data: pa.Table, merge: str = "never") -> int: chunksize = INSERT_BATCH_SIZE assert isinstance(data, pa.Table) tmp_table = data.rename_columns(["time", "value"]) - logger.debug(f"tmp_table schema: {tmp_table.schema}") new_schema = pa.schema( [ (pa.field("time", pa.timestamp(unit="ns", tz="UTC"), nullable=False)), @@ -536,11 +532,9 @@ def arrow_insert(self, data: pa.Table, merge: str = "never") -> int: # Process the batches as needed version = [] for tab in table_slices: - logger.debug(f"Table Slice: {tab}") - feather_bytes = _table_slice_to_feather_bytes(table_slice=tab) version.append( self._btrdb.ep.arrowInsertValues( - uu=self.uuid, values=feather_bytes, policy=merge + uu=self.uuid, values=tab, policy=merge ) ) return max(version) @@ -725,12 +719,9 @@ def values(self, start, end, version=0): the vector nodes. """ - if self._btrdb._ARROW_ENABLED: - warnings.warn(_arrow_available_str.format("values", "arrow_values")) materialized = [] start = to_nanoseconds(start) end = to_nanoseconds(end) - logger.debug(f"For stream - {self.uuid} - {self.name}") point_windows = self._btrdb.ep.rawValues(self._uuid, start, end, version) for point_list, version in point_windows: for point in point_list: @@ -771,12 +762,18 @@ def arrow_values(self, start, end, version: int = 0) -> pa.Table: raise NotImplementedError(_arrow_not_impl_str.format("arrow_values")) start = to_nanoseconds(start) end = to_nanoseconds(end) - arr_bytes = self._btrdb.ep.arrowRawValues( + arrow_and_versions = self._btrdb.ep.arrowRawValues( uu=self.uuid, start=start, end=end, version=version ) - # exhausting the generator from above - bytes_materialized = list(arr_bytes) - return _materialize_stream_as_table(bytes_materialized) + tables = [arrow for (arrow, _) in arrow_and_versions] + if len(tables) > 0: + return pa.concat_tables(tables) + else: + schema = pa.schema([ + pa.field('time', pa.timestamp('ns', tz='UTC'), nullable=False), + pa.field('value', pa.float64(), nullable=False), + ]) + return pa.Table.from_arrays([pa.array([]), pa.array([])], schema=schema) def aligned_windows(self, start, end, pointwidth, version=0): """ @@ -821,10 +818,6 @@ def aligned_windows(self, start, end, pointwidth, version=0): As the window-width is a power-of-two, it aligns with BTrDB internal tree data structure and is faster to execute than `windows()`. """ - if self._btrdb._ARROW_ENABLED: - warnings.warn( - _arrow_available_str.format("aligned_windows", "arrow_aligned_windows") - ) materialized = [] start = to_nanoseconds(start) end = to_nanoseconds(end) @@ -886,20 +879,21 @@ def arrow_aligned_windows( logger.debug(f"For stream - {self.uuid} - {self.name}") start = to_nanoseconds(start) end = to_nanoseconds(end) - arr_bytes = self._btrdb.ep.arrowAlignedWindows( + tables = list(self._btrdb.ep.arrowAlignedWindows( self.uuid, start=start, end=end, pointwidth=pointwidth, version=version - ) - # exhausting the generator from above - bytes_materialized = list(arr_bytes) - - logger.debug(f"Length of materialized list: {len(bytes_materialized)}") - logger.debug(f"materialized bytes[0:1]: {bytes_materialized[0:1]}") - # ignore versions for now - materialized_table = _materialize_stream_as_table(bytes_materialized) - stream_names = [ - "/".join([self.collection, self.name, prop]) for prop in _STAT_PROPERTIES - ] - return materialized_table.rename_columns(["time", *stream_names]) + )) + if len(tables) > 0: + return pa.concat_tables(tables) + else: + schema = pa.schema([ + pa.field('time', pa.timestamp('ns', tz='UTC'), nullable=False), + pa.field('mean', pa.float64(), nullable=False), + pa.field('min', pa.float64(), nullable=False), + pa.field('max', pa.float64(), nullable=False), + pa.field('count', pa.uint64(), nullable=False), + pa.field('stddev', pa.float64(), nullable=False), + ]) + return pa.Table.from_arrays([pa.array([]) for _ in range(5)], schema=schema) def windows(self, start, end, width, depth=0, version=0): """ @@ -941,8 +935,6 @@ def windows(self, start, end, width, depth=0, version=0): for depth is now 0. """ - if self._btrdb._ARROW_ENABLED: - warnings.warn(_arrow_available_str.format("windows", "arrow_windows")) materialized = [] start = to_nanoseconds(start) end = to_nanoseconds(end) @@ -994,25 +986,26 @@ def arrow_windows( raise NotImplementedError(_arrow_not_impl_str.format("arrow_windows")) start = to_nanoseconds(start) end = to_nanoseconds(end) - arr_bytes = self._btrdb.ep.arrowWindows( + tables = list(self._btrdb.ep.arrowWindows( self.uuid, start=start, end=end, width=width, depth=0, version=version, - ) - # exhausting the generator from above - bytes_materialized = list(arr_bytes) - - logger.debug(f"Length of materialized list: {len(bytes_materialized)}") - logger.debug(f"materialized bytes[0:1]: {bytes_materialized[0:1]}") - # ignore versions for now - materialized = _materialize_stream_as_table(bytes_materialized) - stream_names = [ - "/".join([self.collection, self.name, prop]) for prop in _STAT_PROPERTIES - ] - return materialized.rename_columns(["time", *stream_names]) + )) + if len(tables) > 0: + return pa.concat_tables(tables) + else: + schema = pa.schema([ + pa.field('time', pa.timestamp('ns', tz='UTC'), nullable=False), + pa.field('mean', pa.float64(), nullable=False), + pa.field('min', pa.float64(), nullable=False), + pa.field('max', pa.float64(), nullable=False), + pa.field('count', pa.uint64(), nullable=False), + pa.field('stddev', pa.float64(), nullable=False), + ]) + return pa.Table.from_arrays([pa.array([]) for _ in range(5)], schema=schema) def nearest(self, time, version, backward=False): """ @@ -1541,7 +1534,9 @@ def _streamset_data(self, as_iterators=False): params = self._params_from_filters() # sampling freq not supported for non-arrow streamset ops _ = params.pop("sampling_frequency", None) - versions = self.versions() + versions = self._pinned_versions + if versions == None: + versions = {s.uuid : 0 for s in self} if self.pointwidth is not None: # create list of stream.aligned_windows data @@ -1577,85 +1572,6 @@ def _streamset_data(self, as_iterators=False): return data - def _arrow_streamset_data(self): - params = self._params_from_filters() - versions = self.versions() - if params.get("sampling_frequency", None) is None: - _ = params.pop("sampling_frequency", None) - - if self.pointwidth is not None: - # create list of stream.aligned_windows data - params.update({"pointwidth": self.pointwidth}) - _ = params.pop("sampling_frequency", None) - # need to update params based on version of stream, use dict merge - # {**dict1, **dict2} creates a new dict which updates the key/values if there are any - # same keys in dict1 and dict2, then the second dict in the expansion (dict2 in this case) - # will update the key (similar to dict1.update(dict2)) - aligned_windows_gen = self._btrdb._executor.map( - lambda s: s.arrow_aligned_windows( - **{**params, **{"version": versions[s.uuid]}} - ), - self._streams, - ) - data = list(aligned_windows_gen) - tablex = data.pop() - if data: - for tab in data: - tablex = tablex.join(tab, "time", join_type="full outer") - data = tablex - else: - data = tablex - - elif self.width is not None and self.depth is not None: - # create list of stream.windows data (the windows method should - # prevent the possibility that only one of these is None) - _ = params.pop("sampling_frequency", None) - params.update({"width": self.width}) - windows_gen = self._btrdb._executor.map( - lambda s: s.arrow_windows( - **{**params, **{"version": versions[s.uuid]}} - ), - self._streams, - ) - data = list(windows_gen) - tablex = data.pop() - if data: - for tab in data: - tablex = tablex.join(tab, "time", join_type="full outer") - data = tablex - else: - data = tablex - - else: - # determine if we are aligning data or not - sampling_freq = params.get("sampling_frequency", None) - if sampling_freq is None: - sampling_freq = -1 - if sampling_freq > 0: - # We are getting timesnapped data - data = self._arrow_multivalues(period_ns=_to_period_ns(sampling_freq)) - elif sampling_freq == 0: - # outer-joined on time data - data = self._arrow_multivalues(period_ns=0) - else: - # getting raw value data from each stream multithreaded - # create list of stream.values - values_gen = self._btrdb._executor.map( - lambda s: s.arrow_values(**params), self._streams - ) - data = list(values_gen) - tables = deque(data) - main_table = tables.popleft() - idx = 0 - while len(tables) != 0: - idx = idx + 1 - t2 = tables.popleft() - main_table = main_table.join( - t2, "time", join_type="full outer", right_suffix=f"_{idx}" - ) - data = main_table - return data - def rows(self): """ Returns a materialized list of tuples where each tuple contains the @@ -1700,12 +1616,6 @@ def rows(self): return result - def arrow_rows(self): - """Return tuples of rows from arrow table""" - raise NotImplementedError( - "arrow_rows has not been implemented yet, please use `rows` if you need this functionality." - ) - def insert(self, data_map: dict, merge: str = "never") -> dict: """Insert new data in the form (time, value) into their mapped streams. @@ -1736,8 +1646,6 @@ def insert(self, data_map: dict, merge: str = "never") -> dict: dict[uuid, int] The versions of the stream after inserting new points. """ - if self._btrdb._ARROW_ENABLED: - warnings.warn(_arrow_available_str.format("insert", "arrow_insert")) filtered_data_map = {s.uuid: data_map[s.uuid] for s in self._streams} for key, dat in filtered_data_map.items(): data_list = [ @@ -1821,30 +1729,77 @@ def values(self): result.append([point[0] for point in stream_data]) return result - def arrow_values(self): + def arrow_values(self, name_callable=lambda s : s.collection + '/' + s.name): """Return a pyarrow table of stream values based on the streamset parameters.""" - streamset_data = self._arrow_streamset_data() - return streamset_data + params = self._params_from_filters() + versions = self._pinned_versions + if versions == None: + versions = {s.uuid : 0 for s in self} - def _arrow_multivalues(self, period_ns: int): - if not self._btrdb._ARROW_ENABLED: - raise NotImplementedError( - _arrow_not_impl_str.format("arrow_multirawvalues") + if params.get("sampling_frequency", None) is None: + _ = params.pop("sampling_frequency", None) + + if self.pointwidth is not None: + # create list of stream.aligned_windows data + params.update({"pointwidth": self.pointwidth}) + _ = params.pop("sampling_frequency", None) + # need to update params based on version of stream, use dict merge + # {**dict1, **dict2} creates a new dict which updates the key/values if there are any + # same keys in dict1 and dict2, then the second dict in the expansion (dict2 in this case) + # will update the key (similar to dict1.update(dict2)) + aligned_windows_gen = self._btrdb._executor.map( + lambda s: s.arrow_aligned_windows( + **{**params, **{"version": versions[s.uuid]}} + ), + self._streams, ) - params = self._params_from_filters() - versions = self.versions() - params["uu_list"] = [s.uuid for s in self._streams] - params["version_list"] = [versions[s.uuid] for s in self._streams] - params["snap_periodNS"] = period_ns - # dict.pop(key, default_return_value_if_no_key) - _ = params.pop("sampling_frequency", None) - arr_bytes = self._btrdb.ep.arrowMultiValues(**params) - # exhausting the generator from above - bytes_materialized = list(arr_bytes) + data = list(aligned_windows_gen) + tablex = data.pop() + if data: + for tab in data: + tablex = tablex.join(tab, "time", join_type="full outer") + data = tablex + else: + data = tablex + + elif self.width is not None and self.depth is not None: + # create list of stream.windows data (the windows method should + # prevent the possibility that only one of these is None) + _ = params.pop("sampling_frequency", None) + params.update({"width": self.width}) + windows_gen = self._btrdb._executor.map( + lambda s: s.arrow_windows( + **{**params, **{"version": versions[s.uuid]}} + ), + self._streams, + ) + data = list(windows_gen) + tablex = data.pop() + if data: + for tab in data: + tablex = tablex.join(tab, "time", join_type="full outer") + data = tablex + else: + data = tablex + else: + sampling_freq = params.pop("sampling_frequency", 0) + period_ns = 0 + if sampling_freq > 0: + period_ns = _to_period_ns(sampling_freq) + params["uu_list"] = [s.uuid for s in self._streams] + params["version_list"] = [versions[s.uuid] for s in self._streams] + params["snap_periodNS"] = period_ns + table = list(self._btrdb.ep.arrowMultiValues(**params)) + if len(table) > 0: + data = pa.concat_tables(table) + data = data.rename_columns(["time"] + [name_callable(s) for s in self._streams]) + else: + schema = pa.schema( + [pa.field('time', pa.timestamp('ns', tz='UTC'), nullable=False)] + + [pa.field(name_callable(s), pa.float64(), nullable=False) for s in self._streams], + ) + data = pa.Table.from_arrays([pa.array([]) for i in range(1+len(self._streams))], schema=schema) - logger.debug(f"Length of materialized list: {len(bytes_materialized)}") - logger.debug(f"materialized bytes[0:1]: {bytes_materialized[0:1]}") - data = _materialize_stream_as_table(bytes_materialized) return data def __repr__(self): @@ -1906,35 +1861,12 @@ def __init__( if self.start is not None and self.end is not None and self.start >= self.end: raise BTRDBValueError("`start` must be strictly less than `end` argument") - def _to_period_ns(fs: int): """Convert sampling rate to sampling period in ns.""" period = 1 / fs period_ns = period * 1e9 return int(period_ns) - -def _materialize_stream_as_table(arrow_bytes): - table_list = [] - for b, _ in arrow_bytes: - with pa.ipc.open_stream(b) as reader: - schema = reader.schema - logger.debug(f"schema: {schema}") - table_list.append(reader.read_all()) - logger.debug(f"table list: {table_list}") - table = pa.concat_tables(table_list) - return table - - -def _table_slice_to_feather_bytes(table_slice: pa.Table) -> bytes: - # sink = pa.BufferOutputStream() - sink = io.BytesIO() - with pa.ipc.new_stream(sink=sink, schema=table_slice.schema) as writer: - writer.write(table_slice) - buf = sink.getvalue() - return buf - - def _coalesce_table_deque(tables: deque): main_table = tables.popleft() idx = 0 diff --git a/tests/btrdb/integration/test_stream.py b/tests/btrdb/integration/test_stream.py index 52494c6..ab3857c 100644 --- a/tests/btrdb/integration/test_stream.py +++ b/tests/btrdb/integration/test_stream.py @@ -9,7 +9,7 @@ except: pa = None -def test_basic_insert_and_values(conn, tmp_collection): +def test_grpc_insert_and_values(conn, tmp_collection): s = conn.create(new_uuid(), tmp_collection, tags={"name":"s"}) t = currently_as_ns() data = [] @@ -43,3 +43,9 @@ def test_arrow_insert_and_values(conn, tmp_collection): s.arrow_insert(data) fetched_data = s.arrow_values(start=t, end=t+duration) assert data == fetched_data + +def test_arrow_empty_values(conn, tmp_collection): + s = conn.create(new_uuid(), tmp_collection, tags={"name":"s"}) + data = s.arrow_values(start=100, end=1000) + assert len(data['time']) == 0 + assert len(data['value']) == 0 diff --git a/tests/btrdb/integration/test_streamset.py b/tests/btrdb/integration/test_streamset.py index 87b5ba7..e01cae8 100644 --- a/tests/btrdb/integration/test_streamset.py +++ b/tests/btrdb/integration/test_streamset.py @@ -47,12 +47,19 @@ def test_streamset_arrow_values(conn, tmp_collection): expected_col2 = [None, 5.0, None, 6.0, 7.0, 8.0, None, 9.0, None] values = ss.arrow_values() times = [t.value for t in values['time']] - col1 = [None if isnan(v) else v for v in values[tmp_collection + '/s1']] - col2 = [None if isnan(v) else v for v in values[tmp_collection + '/s2']] + col1 = [None if isnan(v.as_py()) else v.as_py() for v in values[tmp_collection + '/s1']] + col2 = [None if isnan(v.as_py()) else v.as_py() for v in values[tmp_collection + '/s2']] assert times == expected_times assert col1 == expected_col1 assert col2 == expected_col2 +def test_streamset_empty_arrow_values(conn, tmp_collection): + s = conn.create(new_uuid(), tmp_collection, tags={"name":"s"}) + ss = btrdb.stream.StreamSet([s]).filter(start=100, end=200) + values = ss.arrow_values() + assert [t.value for t in values['time']] == [] + assert [v for v in values[tmp_collection + '/s']] == [] + def test_streamset_to_dataframe(conn, tmp_collection): s1 = conn.create(new_uuid(), tmp_collection, tags={"name":"s1"}) s2 = conn.create(new_uuid(), tmp_collection, tags={"name":"s2"}) @@ -72,4 +79,4 @@ def test_streamset_to_dataframe(conn, tmp_collection): col2 = [None if isnan(v) else v for v in values[tmp_collection + '/s2']] assert times == expected_times assert col1 == expected_col1 - assert col2 == expected_col2 + assert col2 == expected_col2 \ No newline at end of file diff --git a/tests/btrdb/test_conn.py b/tests/btrdb/test_conn.py index 4a6d95a..340416d 100644 --- a/tests/btrdb/test_conn.py +++ b/tests/btrdb/test_conn.py @@ -40,15 +40,6 @@ def test_raises_err_invalid_address(self): conn = Connection(address) assert "expecting address:port" in str(exc) - def test_raises_err_for_apikey_insecure_port(self): - """ - Assert error is raised if apikey used on insecure port - """ - address = "127.0.0.1:4410" - with pytest.raises(ValueError) as exc: - conn = Connection(address, apikey="abcd") - assert "cannot use an API key with an insecure" in str(exc) - ########################################################################## ## BTrDB Tests From a40ef4f6de47b5305ceb07dc682fdc5fc8766e69 Mon Sep 17 00:00:00 2001 From: Andrew Chambers Date: Sat, 1 Jul 2023 13:04:29 +1200 Subject: [PATCH 048/131] More integration test cases --- btrdb/stream.py | 4 +--- tests/btrdb/integration/test_stream.py | 30 ++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/btrdb/stream.py b/btrdb/stream.py index 0f4d00e..a0a8a45 100644 --- a/btrdb/stream.py +++ b/btrdb/stream.py @@ -396,9 +396,7 @@ def annotations(self, refresh=False): Returns a stream's annotations Annotations returns the annotations of the stream (and the annotation - version). It will always require a round trip to the server. If you are - ok with stale data and want a higher performance version, use - Stream.CachedAnnotations(). + version). Do not modify the resulting map. diff --git a/tests/btrdb/integration/test_stream.py b/tests/btrdb/integration/test_stream.py index ab3857c..d4a246b 100644 --- a/tests/btrdb/integration/test_stream.py +++ b/tests/btrdb/integration/test_stream.py @@ -49,3 +49,33 @@ def test_arrow_empty_values(conn, tmp_collection): data = s.arrow_values(start=100, end=1000) assert len(data['time']) == 0 assert len(data['value']) == 0 + +def test_stream_annotation_update(conn, tmp_collection): + s = conn.create(new_uuid(), tmp_collection, tags={"name":"s"}, annotations={"foo":"bar"}) + annotations1, version1 = s.annotations() + assert version1 == 0 + assert annotations1["foo"] == "bar" + s.update(annotations={"foo":"baz"}) + annotations2, version2 = s.annotations() + assert version2 > version1 + assert annotations2["foo"] == "baz" + s.update(annotations={}, replace=True) + annotations3, _ = s.annotations() + assert len(annotations3) == 0 + +def test_create_count_obliterate_concurrency_bug(conn, tmp_collection): + # It is unclear if this was the cause of a bug, but it is an attempt + # at recreating an obliteration and metadata crash. + from concurrent.futures import ThreadPoolExecutor + n_streams = 10 + def create_stream(i): + return conn.create(new_uuid(), tmp_collection, tags={"name":f"s{i}"}) + def points_in_stream(s): + return s.count() + def obliterate_stream(s): + s.obliterate() + with ThreadPoolExecutor() as executor: + for i in range(2): + streams = list(executor.map(create_stream, range(n_streams))) + assert sum(executor.map(points_in_stream, streams)) == 0 + _ = list(executor.map(obliterate_stream, streams)) From 79e74ea13484f8d0703cddf16e7b4afa47131aab Mon Sep 17 00:00:00 2001 From: Andrew Chambers Date: Sat, 1 Jul 2023 14:38:56 +1200 Subject: [PATCH 049/131] Restructure tests. --- .../conftest.py | 0 .../test_conn.py | 0 tests/btrdb_integration/test_regressions.py | 17 +++++++++++++++++ .../test_stream.py | 17 ----------------- .../test_streamset.py | 0 5 files changed, 17 insertions(+), 17 deletions(-) rename tests/{btrdb/integration => btrdb_integration}/conftest.py (100%) rename tests/{btrdb/integration => btrdb_integration}/test_conn.py (100%) create mode 100644 tests/btrdb_integration/test_regressions.py rename tests/{btrdb/integration => btrdb_integration}/test_stream.py (73%) rename tests/{btrdb/integration => btrdb_integration}/test_streamset.py (100%) diff --git a/tests/btrdb/integration/conftest.py b/tests/btrdb_integration/conftest.py similarity index 100% rename from tests/btrdb/integration/conftest.py rename to tests/btrdb_integration/conftest.py diff --git a/tests/btrdb/integration/test_conn.py b/tests/btrdb_integration/test_conn.py similarity index 100% rename from tests/btrdb/integration/test_conn.py rename to tests/btrdb_integration/test_conn.py diff --git a/tests/btrdb_integration/test_regressions.py b/tests/btrdb_integration/test_regressions.py new file mode 100644 index 0000000..7ed3810 --- /dev/null +++ b/tests/btrdb_integration/test_regressions.py @@ -0,0 +1,17 @@ +from uuid import uuid4 as new_uuid + +def test_create_count_obliterate_concurrency_bug(conn, tmp_collection): + from concurrent.futures import ThreadPoolExecutor + n_streams = 10 + def create_stream(i): + return conn.create(new_uuid(), tmp_collection, tags={"name":f"s{i}"}) + def points_in_stream(s): + return s.count() + def obliterate_stream(s): + s.obliterate() + with ThreadPoolExecutor() as executor: + for i in range(2): + pmap = lambda f , it : list(executor.map(f, it, timeout=30)) + streams = pmap(create_stream, range(n_streams)) + assert sum(pmap(points_in_stream, streams)) == 0 + pmap(obliterate_stream, streams) diff --git a/tests/btrdb/integration/test_stream.py b/tests/btrdb_integration/test_stream.py similarity index 73% rename from tests/btrdb/integration/test_stream.py rename to tests/btrdb_integration/test_stream.py index d4a246b..bba1529 100644 --- a/tests/btrdb/integration/test_stream.py +++ b/tests/btrdb_integration/test_stream.py @@ -62,20 +62,3 @@ def test_stream_annotation_update(conn, tmp_collection): s.update(annotations={}, replace=True) annotations3, _ = s.annotations() assert len(annotations3) == 0 - -def test_create_count_obliterate_concurrency_bug(conn, tmp_collection): - # It is unclear if this was the cause of a bug, but it is an attempt - # at recreating an obliteration and metadata crash. - from concurrent.futures import ThreadPoolExecutor - n_streams = 10 - def create_stream(i): - return conn.create(new_uuid(), tmp_collection, tags={"name":f"s{i}"}) - def points_in_stream(s): - return s.count() - def obliterate_stream(s): - s.obliterate() - with ThreadPoolExecutor() as executor: - for i in range(2): - streams = list(executor.map(create_stream, range(n_streams))) - assert sum(executor.map(points_in_stream, streams)) == 0 - _ = list(executor.map(obliterate_stream, streams)) diff --git a/tests/btrdb/integration/test_streamset.py b/tests/btrdb_integration/test_streamset.py similarity index 100% rename from tests/btrdb/integration/test_streamset.py rename to tests/btrdb_integration/test_streamset.py From e8d6869f14d4f6d4ac4e44bff672adbeb15ac00d Mon Sep 17 00:00:00 2001 From: Andrew Chambers Date: Sat, 1 Jul 2023 15:03:30 +1200 Subject: [PATCH 050/131] Mark new failing tests as expected failures for now. --- tests/btrdb_integration/test_regressions.py | 8 +++++++- tests/btrdb_integration/test_stream.py | 2 ++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/btrdb_integration/test_regressions.py b/tests/btrdb_integration/test_regressions.py index 7ed3810..026fdd3 100644 --- a/tests/btrdb_integration/test_regressions.py +++ b/tests/btrdb_integration/test_regressions.py @@ -1,6 +1,12 @@ +import pytest from uuid import uuid4 as new_uuid +@pytest.mark.xfail def test_create_count_obliterate_concurrency_bug(conn, tmp_collection): + # This mark this test as failed, but don't run it as it + # currently crashes btrdb, it needs to be resolved and + # enabled. + assert False from concurrent.futures import ThreadPoolExecutor n_streams = 10 def create_stream(i): @@ -11,7 +17,7 @@ def obliterate_stream(s): s.obliterate() with ThreadPoolExecutor() as executor: for i in range(2): - pmap = lambda f , it : list(executor.map(f, it, timeout=30)) + pmap = lambda f, it : list(executor.map(f, it, timeout=30)) streams = pmap(create_stream, range(n_streams)) assert sum(pmap(points_in_stream, streams)) == 0 pmap(obliterate_stream, streams) diff --git a/tests/btrdb_integration/test_stream.py b/tests/btrdb_integration/test_stream.py index bba1529..3db250f 100644 --- a/tests/btrdb_integration/test_stream.py +++ b/tests/btrdb_integration/test_stream.py @@ -50,7 +50,9 @@ def test_arrow_empty_values(conn, tmp_collection): assert len(data['time']) == 0 assert len(data['value']) == 0 +@pytest.mark.xfail def test_stream_annotation_update(conn, tmp_collection): + # XXX marked as expected failure until someone has time to investigate. s = conn.create(new_uuid(), tmp_collection, tags={"name":"s"}, annotations={"foo":"bar"}) annotations1, version1 = s.annotations() assert version1 == 0 From 78901f1bfbabfe204026249bd5cce18f04aeb607 Mon Sep 17 00:00:00 2001 From: Andrew Chambers Date: Thu, 6 Jul 2023 16:52:29 +1200 Subject: [PATCH 051/131] Disable gzip compression, it is very slow. --- btrdb/conn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/btrdb/conn.py b/btrdb/conn.py index 83aa5e1..5c86491 100644 --- a/btrdb/conn.py +++ b/btrdb/conn.py @@ -60,7 +60,7 @@ def __init__(self, addrportstr, apikey=None): """ addrport = addrportstr.split(":", 2) - chan_ops = [("grpc.default_compression_algorithm", CompressionAlgorithm.gzip)] + chan_ops = [] if len(addrport) != 2: raise ValueError("expecting address:port") From 0654465c72090614a8e9b8de1a4ddbb754833623 Mon Sep 17 00:00:00 2001 From: Andrew Chambers Date: Wed, 12 Jul 2023 14:20:28 +1200 Subject: [PATCH 052/131] Reenable test, server has been fixed. --- tests/btrdb_integration/test_regressions.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/btrdb_integration/test_regressions.py b/tests/btrdb_integration/test_regressions.py index 026fdd3..7ca4df7 100644 --- a/tests/btrdb_integration/test_regressions.py +++ b/tests/btrdb_integration/test_regressions.py @@ -1,12 +1,7 @@ import pytest from uuid import uuid4 as new_uuid -@pytest.mark.xfail def test_create_count_obliterate_concurrency_bug(conn, tmp_collection): - # This mark this test as failed, but don't run it as it - # currently crashes btrdb, it needs to be resolved and - # enabled. - assert False from concurrent.futures import ThreadPoolExecutor n_streams = 10 def create_stream(i): From 915bf5f430b54ea50d17f7a7a6b7848991bb25f8 Mon Sep 17 00:00:00 2001 From: Justin Gilmer Date: Wed, 19 Jul 2023 15:52:16 -0500 Subject: [PATCH 053/131] Update pandas testing and fix flake8 issues (#31) * Update pandas testing and fix flake8 issues * Update stream logic for unpacking arrow tables, update integration tests. * add init.py for integration tests. * Add additional tests for arrow methods vs their old api counterparts. * Add tests for timesnap boundary conditions. (#32) * Add more integration tests. * Add additional integration tests, modify the name_callable ability of the arrow_values. * remove extraneous prints. * Include retry logic. * Update statpoint order in arrow, fix some bugs with the arrow methods. * Update testing to account for NaNs. * Update github action versions. * Update tests, add in a test for duplicate values. * Remove empty test, remove extraneous prints --------- Co-authored-by: andrewchambers --- .github/workflows/release.yaml | 10 +- btrdb/conn.py | 56 ++- btrdb/endpoint.py | 8 +- btrdb/exceptions.py | 49 +++ btrdb/stream.py | 320 ++++++++++++--- btrdb/transformers.py | 130 +++--- tests/btrdb/test_stream.py | 5 + tests/btrdb_integration/__init__.py | 0 tests/btrdb_integration/conftest.py | 46 +++ tests/btrdb_integration/test_conn.py | 23 +- tests/btrdb_integration/test_regressions.py | 14 +- tests/btrdb_integration/test_stream.py | 334 +++++++++++++-- tests/btrdb_integration/test_streamset.py | 427 ++++++++++++++++++-- 13 files changed, 1223 insertions(+), 199 deletions(-) create mode 100644 tests/btrdb_integration/__init__.py diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 8946559..207f62f 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -18,9 +18,9 @@ jobs: os: [ubuntu-latest, macos-latest, windows-latest] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} ${{ matrix.os }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -39,7 +39,7 @@ jobs: if: startsWith(github.ref, 'refs/tags/') runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Create Release id: create_release uses: actions/create-release@v1 @@ -59,9 +59,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: '3.8' - name: Install dependencies diff --git a/btrdb/conn.py b/btrdb/conn.py index 5c86491..06fbb3c 100644 --- a/btrdb/conn.py +++ b/btrdb/conn.py @@ -26,7 +26,7 @@ import grpc from grpc._cython.cygrpc import CompressionAlgorithm -from btrdb.exceptions import InvalidOperation, StreamNotFoundError +from btrdb.exceptions import InvalidOperation, StreamNotFoundError, retry from btrdb.stream import Stream, StreamSet from btrdb.utils.conversion import to_uuid from btrdb.utils.general import unpack_stream_descriptor @@ -179,7 +179,7 @@ def _is_arrow_enabled(info): if logger.isEnabledFor(logging.DEBUG): logger.debug(f"major version: {major}") logger.debug(f"minor version: {minor}") - if major >= 5 and minor >= 30: + if major >= 5 and minor >= 29: return True else: return False @@ -193,11 +193,23 @@ class BTrDB(object): def __init__(self, endpoint): self.ep = endpoint self._executor = ThreadPoolExecutor() - self._ARROW_ENABLED = True # _is_arrow_enabled(self.ep.info()) + try: + self._ARROW_ENABLED = _is_arrow_enabled(self.ep.info()) + except Exception: + self._ARROW_ENABLED = False if logger.isEnabledFor(logging.DEBUG): logger.debug(f"ARROW ENABLED: {self._ARROW_ENABLED}") - def query(self, stmt, params=[]): + @retry + def query( + self, + stmt, + params=None, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Performs a SQL query on the database metadata and returns a list of dictionaries from the resulting cursor. @@ -227,6 +239,8 @@ def query(self, stmt, params=[]): See https://btrdb.readthedocs.io/en/latest/ for more info. """ + if params is None: + params = list() return [ json.loads(row.decode("utf-8")) for page in self.ep.sql_query(stmt, params) @@ -317,7 +331,18 @@ def stream_from_uuid(self, uuid): """ return Stream(self, to_uuid(uuid)) - def create(self, uuid, collection, tags=None, annotations=None): + @retry + def create( + self, + uuid, + collection, + tags=None, + annotations=None, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Tells BTrDB to create a new stream with UUID `uuid` in `collection` with specified `tags` and `annotations`. @@ -379,8 +404,17 @@ def list_collections(self, starts_with=""): """ return [c for some in self.ep.listCollections(starts_with) for c in some] + @retry def streams_in_collection( - self, *collection, is_collection_prefix=True, tags=None, annotations=None + self, + *collection, + is_collection_prefix=True, + tags=None, + annotations=None, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, ): """ Search for streams matching given parameters @@ -436,7 +470,15 @@ def streams_in_collection( return result - def collection_metadata(self, prefix): + @retry + def collection_metadata( + self, + prefix, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Gives statistics about metadata for collections that match a prefix. diff --git a/btrdb/endpoint.py b/btrdb/endpoint.py index 7106541..79a3b72 100644 --- a/btrdb/endpoint.py +++ b/btrdb/endpoint.py @@ -25,8 +25,8 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import io +import typing import uuid -import grpc from btrdb.exceptions import BTrDBError, check_proto_stat, error_handler from btrdb.grpcinterface import btrdb_pb2, btrdb_pb2_grpc @@ -35,9 +35,10 @@ try: import pyarrow as pa -except: +except ImportError: pa = None + class Endpoint(object): """Server endpoint where we make specific requests.""" @@ -155,7 +156,6 @@ def arrowWindows(self, uu, start, end, width, depth, version=0): with pa.ipc.open_stream(result.arrowBytes) as reader: yield reader.read_all(), result.versionMajor - @error_handler def streamInfo(self, uu, omitDescriptor, omitVersion): params = btrdb_pb2.StreamInfoParams( @@ -381,7 +381,7 @@ def generateCSV( yield result.row @error_handler - def sql_query(self, stmt, params=[]): + def sql_query(self, stmt, params: typing.List): request = btrdb_pb2.SQLQueryParams(query=stmt, params=params) for page in self.stub.SQLQuery(request): check_proto_stat(page.stat) diff --git a/btrdb/exceptions.py b/btrdb/exceptions.py index fe13716..9777679 100644 --- a/btrdb/exceptions.py +++ b/btrdb/exceptions.py @@ -12,14 +12,63 @@ ########################################################################## import inspect +import logging +import time from functools import wraps from grpc import RpcError +########################################################################## +## Module Variables and Constants +########################################################################## +MAX_RETRIES = 5 +INITIAL_RETRY_DELAY = 3 # time to wait before first retry in seconds +RETRY_BACKOFF_FACTOR = ( + 4 # exponential factor by which retry backoff increases between tries +) + + ########################################################################## ## Decorators ########################################################################## +# config logging for auto-retry +logging.basicConfig( + format="%(asctime)s:%(levelname)s:%(message)s", + level=logging.INFO, + datefmt="%m/%d/%Y %I:%M:%S", +) + + +def retry(fn, *args, **kwargs): + """ + decorates functions and retries it in the event of BTrDBErrors or RpcErrors + """ + + @wraps(fn) + def retry_func(*args, **kwargs): + # don't retry if arg is set to False + if not kwargs.get("auto_retry"): + return fn(*args, **kwargs) + + total_retries = kwargs.get("retries") or MAX_RETRIES + retries = total_retries + delay = kwargs.get("retry_delay") or INITIAL_RETRY_DELAY + backoff = kwargs.get("retry_backoff") or RETRY_BACKOFF_FACTOR + while retries > 0: + try: + return fn(*args, **kwargs) + except (BTrDBError, RpcError) as e: + msg = f"ERROR: {e}, attempting retry {total_retries-retries+1}/{total_retries} in {delay} seconds..." + logging.info(msg) + time.sleep(delay) + retries -= 1 + delay *= backoff + err = e + handle_grpc_error(err) + + return retry_func + def consume_generator(fn, *args, **kwargs): # when a generator is passed back to the calling function, it may encounter an error diff --git a/btrdb/stream.py b/btrdb/stream.py index a0a8a45..0eed275 100644 --- a/btrdb/stream.py +++ b/btrdb/stream.py @@ -34,6 +34,7 @@ InvalidOperation, NoSuchPoint, StreamNotFoundError, + retry, ) from btrdb.point import RawPoint, StatPoint from btrdb.transformers import _STAT_PROPERTIES, StreamSetTransformer @@ -304,7 +305,15 @@ def collection(self): self.refresh_metadata() return self._collection - def earliest(self, version=0): + @retry + def earliest( + self, + version=0, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Returns the first point of data in the stream. Returns None if error encountered during lookup or no values in stream. @@ -325,7 +334,15 @@ def earliest(self, version=0): start = MINIMUM_TIME return self.nearest(start, version=version, backward=False) - def latest(self, version=0): + @retry + def latest( + self, + version=0, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Returns last point of data in the stream. Returns None if error encountered during lookup or no values in stream. @@ -346,7 +363,15 @@ def latest(self, version=0): start = MAXIMUM_TIME return self.nearest(start, version=version, backward=True) - def current(self, version=0): + @retry + def current( + self, + version=0, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Returns the point that is closest to the current timestamp, e.g. the latest point in the stream up until now. Note that no future values will be returned. @@ -367,7 +392,15 @@ def current(self, version=0): start = currently_as_ns() return self.nearest(start, version, backward=True) - def tags(self, refresh=False): + @retry + def tags( + self, + refresh=False, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Returns the stream's tags. @@ -391,7 +424,15 @@ def tags(self, refresh=False): return deepcopy(self._tags) - def annotations(self, refresh=False): + @retry + def annotations( + self, + refresh=False, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Returns a stream's annotations @@ -418,7 +459,14 @@ def annotations(self, refresh=False): return deepcopy(self._annotations), deepcopy(self._property_version) - def version(self): + @retry + def version( + self, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Returns the current data version of the stream. @@ -532,9 +580,7 @@ def arrow_insert(self, data: pa.Table, merge: str = "never") -> int: version = [] for tab in table_slices: version.append( - self._btrdb.ep.arrowInsertValues( - uu=self.uuid, values=tab, policy=merge - ) + self._btrdb.ep.arrowInsertValues(uu=self.uuid, values=tab, policy=merge) ) return max(version) @@ -575,6 +621,7 @@ def _update_annotations(self, annotations, encoder, replace): removals=removals, ) + @retry def update( self, tags=None, @@ -582,6 +629,10 @@ def update( collection=None, encoder=AnnotationEncoder, replace=False, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, ): """ Updates metadata including tags, annotations, and collection as an @@ -655,7 +706,16 @@ def update( return self._property_version - def delete(self, start, end): + @retry + def delete( + self, + start, + end, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ "Delete" all points between [`start`, `end`) @@ -683,7 +743,17 @@ def delete(self, start, end): self._uuid, to_nanoseconds(start), to_nanoseconds(end) ) - def values(self, start, end, version=0): + @retry + def values( + self, + start, + end, + version=0, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Read raw values from BTrDB between time [a, b) in nanoseconds. @@ -727,7 +797,17 @@ def values(self, start, end, version=0): materialized.append((RawPoint.from_proto(point), version)) return materialized - def arrow_values(self, start, end, version: int = 0) -> pa.Table: + @retry + def arrow_values( + self, + start, + end, + version: int = 0, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ) -> pa.Table: """Read raw values from BTrDB between time [a, b) in nanoseconds. RawValues queries BTrDB for the raw time series data points between @@ -764,17 +844,31 @@ def arrow_values(self, start, end, version: int = 0) -> pa.Table: arrow_and_versions = self._btrdb.ep.arrowRawValues( uu=self.uuid, start=start, end=end, version=version ) - tables = [arrow for (arrow, _) in arrow_and_versions] + tables = list(arrow_and_versions) if len(tables) > 0: - return pa.concat_tables(tables) + tabs, ver = zip(*tables) + return pa.concat_tables(tabs) else: - schema = pa.schema([ - pa.field('time', pa.timestamp('ns', tz='UTC'), nullable=False), - pa.field('value', pa.float64(), nullable=False), - ]) + schema = pa.schema( + [ + pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False), + pa.field("value", pa.float64(), nullable=False), + ] + ) return pa.Table.from_arrays([pa.array([]), pa.array([])], schema=schema) - def aligned_windows(self, start, end, pointwidth, version=0): + @retry + def aligned_windows( + self, + start, + end, + pointwidth, + version=0, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Read statistical aggregates of windows of data from BTrDB. @@ -828,8 +922,17 @@ def aligned_windows(self, start, end, pointwidth, version=0): materialized.append((StatPoint.from_proto(point), version)) return tuple(materialized) + @retry def arrow_aligned_windows( - self, start: int, end: int, pointwidth: int, version: int = 0 + self, + start: int, + end: int, + pointwidth: int, + version: int = 0, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, ) -> pa.Table: """Read statistical aggregates of windows of data from BTrDB. @@ -879,23 +982,40 @@ def arrow_aligned_windows( logger.debug(f"For stream - {self.uuid} - {self.name}") start = to_nanoseconds(start) end = to_nanoseconds(end) - tables = list(self._btrdb.ep.arrowAlignedWindows( - self.uuid, start=start, end=end, pointwidth=pointwidth, version=version - )) + tables = list( + self._btrdb.ep.arrowAlignedWindows( + self.uuid, start=start, end=end, pointwidth=pointwidth, version=version + ) + ) if len(tables) > 0: - return pa.concat_tables(tables) + tabs, ver = zip(*tables) + return pa.concat_tables(tabs) else: - schema = pa.schema([ - pa.field('time', pa.timestamp('ns', tz='UTC'), nullable=False), - pa.field('mean', pa.float64(), nullable=False), - pa.field('min', pa.float64(), nullable=False), - pa.field('max', pa.float64(), nullable=False), - pa.field('count', pa.uint64(), nullable=False), - pa.field('stddev', pa.float64(), nullable=False), - ]) + schema = pa.schema( + [ + pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False), + pa.field("min", pa.float64(), nullable=False), + pa.field("mean", pa.float64(), nullable=False), + pa.field("max", pa.float64(), nullable=False), + pa.field("count", pa.uint64(), nullable=False), + pa.field("stddev", pa.float64(), nullable=False), + ] + ) return pa.Table.from_arrays([pa.array([]) for _ in range(5)], schema=schema) - def windows(self, start, end, width, depth=0, version=0): + @retry + def windows( + self, + start, + end, + width, + depth=0, + version=0, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Read arbitrarily-sized windows of data from BTrDB. StatPoint objects will be returned representing the data for each window. @@ -945,8 +1065,17 @@ def windows(self, start, end, width, depth=0, version=0): return tuple(materialized) + @retry def arrow_windows( - self, start: int, end: int, width: int, version: int = 0 + self, + start: int, + end: int, + width: int, + version: int = 0, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, ) -> pa.Table: """Read arbitrarily-sized windows of data from BTrDB. @@ -986,28 +1115,43 @@ def arrow_windows( raise NotImplementedError(_arrow_not_impl_str.format("arrow_windows")) start = to_nanoseconds(start) end = to_nanoseconds(end) - tables = list(self._btrdb.ep.arrowWindows( - self.uuid, - start=start, - end=end, - width=width, - depth=0, - version=version, - )) + tables = list( + self._btrdb.ep.arrowWindows( + self.uuid, + start=start, + end=end, + width=width, + depth=0, + version=version, + ) + ) if len(tables) > 0: - return pa.concat_tables(tables) + tabs, ver = zip(*tables) + return pa.concat_tables(tabs) else: - schema = pa.schema([ - pa.field('time', pa.timestamp('ns', tz='UTC'), nullable=False), - pa.field('mean', pa.float64(), nullable=False), - pa.field('min', pa.float64(), nullable=False), - pa.field('max', pa.float64(), nullable=False), - pa.field('count', pa.uint64(), nullable=False), - pa.field('stddev', pa.float64(), nullable=False), - ]) + schema = pa.schema( + [ + pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False), + pa.field("min", pa.float64(), nullable=False), + pa.field("mean", pa.float64(), nullable=False), + pa.field("max", pa.float64(), nullable=False), + pa.field("count", pa.uint64(), nullable=False), + pa.field("stddev", pa.float64(), nullable=False), + ] + ) return pa.Table.from_arrays([pa.array([]) for _ in range(5)], schema=schema) - def nearest(self, time, version, backward=False): + @retry + def nearest( + self, + time, + version, + backward=False, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Finds the closest point in the stream to a specified time. @@ -1051,7 +1195,14 @@ def nearest(self, time, version, backward=False): return RawPoint.from_proto(rp), version - def obliterate(self): + @retry + def obliterate( + self, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Obliterate deletes a stream from the BTrDB server. An exception will be raised if the stream could not be found. @@ -1064,7 +1215,14 @@ def obliterate(self): """ self._btrdb.ep.obliterate(self._uuid) - def flush(self): + @retry + def flush( + self, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Flush writes the stream buffers out to persistent storage. @@ -1540,8 +1698,8 @@ def _streamset_data(self, as_iterators=False): # sampling freq not supported for non-arrow streamset ops _ = params.pop("sampling_frequency", None) versions = self._pinned_versions - if versions == None: - versions = {s.uuid : 0 for s in self} + if versions is None: + versions = {s.uuid: 0 for s in self} if self.pointwidth is not None: # create list of stream.aligned_windows data @@ -1734,12 +1892,14 @@ def values(self): result.append([point[0] for point in stream_data]) return result - def arrow_values(self, name_callable=lambda s : s.collection + '/' + s.name): + def arrow_values( + self, + ): """Return a pyarrow table of stream values based on the streamset parameters.""" params = self._params_from_filters() versions = self._pinned_versions - if versions == None: - versions = {s.uuid : 0 for s in self} + if versions is None: + versions = {s.uuid: 0 for s in self} if params.get("sampling_frequency", None) is None: _ = params.pop("sampling_frequency", None) @@ -1758,10 +1918,20 @@ def arrow_values(self, name_callable=lambda s : s.collection + '/' + s.name): ), self._streams, ) + stream_uus = [str(s.uuid) for s in self._streams] data = list(aligned_windows_gen) - tablex = data.pop() + tablex = data.pop(0) + uu = stream_uus.pop(0) + tab_columns = [ + c if c == "time" else uu + "/" + c for c in tablex.column_names + ] + tablex = tablex.rename_columns(tab_columns) if data: - for tab in data: + for tab, uu in zip(data, stream_uus): + tab_columns = [ + c if c == "time" else uu + "/" + c for c in tab.column_names + ] + tab = tab.rename_columns(tab_columns) tablex = tablex.join(tab, "time", join_type="full outer") data = tablex else: @@ -1778,10 +1948,20 @@ def arrow_values(self, name_callable=lambda s : s.collection + '/' + s.name): ), self._streams, ) + stream_uus = [str(s.uuid) for s in self._streams] data = list(windows_gen) - tablex = data.pop() + tablex = data.pop(0) + uu = stream_uus.pop(0) + tab_columns = [ + c if c == "time" else uu + "/" + c for c in tablex.column_names + ] + tablex = tablex.rename_columns(tab_columns) if data: - for tab in data: + for tab, uu in zip(data, stream_uus): + tab_columns = [ + c if c == "time" else uu + "/" + c for c in tab.column_names + ] + tab = tab.rename_columns(tab_columns) tablex = tablex.join(tab, "time", join_type="full outer") data = tablex else: @@ -1797,13 +1977,17 @@ def arrow_values(self, name_callable=lambda s : s.collection + '/' + s.name): table = list(self._btrdb.ep.arrowMultiValues(**params)) if len(table) > 0: data = pa.concat_tables(table) - data = data.rename_columns(["time"] + [name_callable(s) for s in self._streams]) else: schema = pa.schema( - [pa.field('time', pa.timestamp('ns', tz='UTC'), nullable=False)] - + [pa.field(name_callable(s), pa.float64(), nullable=False) for s in self._streams], + [pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False)] + + [ + pa.field(str(s.uuid), pa.float64(), nullable=False) + for s in self._streams + ], + ) + data = pa.Table.from_arrays( + [pa.array([]) for i in range(1 + len(self._streams))], schema=schema ) - data = pa.Table.from_arrays([pa.array([]) for i in range(1+len(self._streams))], schema=schema) return data def __repr__(self): @@ -1865,12 +2049,14 @@ def __init__( if self.start is not None and self.end is not None and self.start >= self.end: raise BTRDBValueError("`start` must be strictly less than `end` argument") + def _to_period_ns(fs: int): """Convert sampling rate to sampling period in ns.""" period = 1 / fs period_ns = period * 1e9 return int(period_ns) + def _coalesce_table_deque(tables: deque): main_table = tables.popleft() idx = 0 diff --git a/btrdb/transformers.py b/btrdb/transformers.py index c4e634f..b2aaa28 100644 --- a/btrdb/transformers.py +++ b/btrdb/transformers.py @@ -18,9 +18,11 @@ import contextlib import csv from collections import OrderedDict +from typing import Sequence from warnings import warn import pandas as pd +import pyarrow ########################################################################## ## Helper Functions @@ -107,9 +109,9 @@ def arrow_to_series(streamset, agg="mean", name_callable=None): Parameters ---------- - agg : str, default: "mean" - Specify the StatPoint field (e.g. aggregating function) to create the Series - from. Must be one of "min", "mean", "max", "count", or "stddev". This + agg : List[str], default: ["mean"] + Specify the StatPoint field or fields (e.g. aggregating function) to create the Series + from. Must be one or more of "min", "mean", "max", "count", or "stddev". This argument is ignored if RawPoint values are passed into the function. name_callable : lambda, default: lambda s: s.collection + "/" + s.name @@ -120,6 +122,12 @@ def arrow_to_series(streamset, agg="mean", name_callable=None): raise NotImplementedError( "arrow_to_series requires an arrow-enabled BTrDB server." ) + if agg is None: + agg = ["mean"] + if not isinstance(agg, list): + raise ValueError( + f"Expected argument 'agg' to be a list of strings, was provided type - {type(agg)} with values - {agg}" + ) arrow_df = arrow_to_dataframe( streamset=streamset, agg=agg, name_callable=name_callable ) @@ -127,7 +135,7 @@ def arrow_to_series(streamset, agg="mean", name_callable=None): def arrow_to_dataframe( - streamset, columns=None, agg="mean", name_callable=None + streamset, columns=None, agg=None, name_callable=None ) -> pd.DataFrame: """ Returns a Pandas DataFrame object indexed by time and using the values of a @@ -138,9 +146,9 @@ def arrow_to_dataframe( columns: sequence column names to use for DataFrame. Deprecated and not compatible with name_callable. - agg : str, default: "mean" - Specify the StatPoint field (e.g. aggregating function) to create the Series - from. Must be one of "min", "mean", "max", "count", "stddev", or "all". This + agg : List[str], default: ["mean"] + Specify the StatPoint fields (e.g. aggregating function) to create the dataframe + from. Must be one or more of "min", "mean", "max", "count", "stddev", or "all". This argument is ignored if not using StatPoints. name_callable : lambda, default: lambda s: s.collection + "/" + s.name @@ -167,45 +175,50 @@ def arrow_to_dataframe( stacklevel=2, ) - # TODO: allow this at some future point - if agg == "all" and name_callable is not None: - raise AttributeError( - "cannot provide name_callable when using 'all' as aggregate at this time" + if agg is None: + agg = ["mean"] + if not isinstance(agg, list): + raise ValueError( + f"Argument 'agg' not provided as a list of strings was provided type - {type(agg)} with values - {agg}" ) - # do not allow agg="all" with RawPoints - if agg == "all" and streamset.allow_window: + if "all" in agg and streamset.allow_window: agg = "" + elif any(["all" in val for val in agg]) and not streamset.allow_window: + agg = ["min", "mean", "max", "count", "stddev"] + else: + agg = agg # default arg values if not callable(name_callable): name_callable = lambda s: s.collection + "/" + s.name + # format is: uuid/stat_type tmp_table = streamset.arrow_values() col_names = _stream_names(streamset, name_callable) - cols = [] - for name in col_names: - for prop in _STAT_PROPERTIES: - cols.append(name + "/" + prop) - if agg == "all": - tmp = tmp_table.select(["time", *cols]) - elif not streamset.allow_window: - usable_cols = [val for val in cols if agg in val] + col_names_map = {str(s.uuid): c for s, c in zip(streamset, col_names)} + updated_table_columns = [] + for old_col in tmp_table.column_names: + if old_col == "time": + updated_table_columns.append("time") + else: + for uu, new_name in col_names_map.items(): + if uu in old_col: + updated_table_columns.append(old_col.replace(uu, new_name)) + else: + continue + tmp_table = tmp_table.rename_columns(updated_table_columns) + if not streamset.allow_window: + usable_cols = [] + for column_str in tmp_table.column_names: + for agg_name in agg: + if agg_name in column_str: + usable_cols.append(column_str) tmp = tmp_table.select(["time", *usable_cols]) else: tmp = tmp_table - df = tmp.to_pandas() - if not df.empty: - df = df.set_index("time") - - if agg == "all" and not streamset.allow_window: - stream_names = [ - [s.collection, s.name, prop] - for s in streamset._streams - for prop in _STAT_PROPERTIES - ] - df.columns = pd.MultiIndex.from_tuples(stream_names) - else: - df.columns = columns if columns else _stream_names(streamset, name_callable) + df = tmp.to_pandas(date_as_object=False, types_mapper=pd.ArrowDtype) + df = df.set_index("time") + df.index = pd.DatetimeIndex(df.index, tz="UTC") return df @@ -257,7 +270,7 @@ def to_dataframe(streamset, columns=None, agg="mean", name_callable=None): if not callable(name_callable): name_callable = lambda s: s.collection + "/" + s.name - df = pd.DataFrame(to_dict(streamset, agg=agg)) + df = pd.DataFrame(to_dict(streamset, agg=agg, name_callable=name_callable)) if not df.empty: df = df.set_index("time") @@ -275,21 +288,21 @@ def to_dataframe(streamset, columns=None, agg="mean", name_callable=None): return df -def arrow_to_polars(streamset, agg="mean", name_callable=None): +def arrow_to_polars(streamset, agg=None, name_callable=None): """ Returns a Polars DataFrame object with time as a column and the values of a stream for each additional column from an arrow table. Parameters ---------- - agg : str, default: "mean" - Specify the StatPoint field (e.g. aggregating function) to create the Series - from. Must be one of "min", "mean", "max", "count", "stddev", or "all". This + agg : List[str], default: ["mean"] + Specify the StatPoint field or fields (e.g. aggregating function) to create the dataframe + from. Must be one or multiple of "min", "mean", "max", "count", "stddev", or "all". This argument is ignored if not using StatPoints. name_callable : lambda, default: lambda s: s.collection + "/" + s.name Specify a callable that can be used to determine the series name given a - Stream object. This is not compatible with agg == "all" at this time + Stream object. Notes ----- @@ -303,10 +316,16 @@ def arrow_to_polars(streamset, agg="mean", name_callable=None): import polars as pl except ImportError: raise ImportError("Please install polars to use this transformation function.") + if agg is None: + agg = ["mean"] + if not isinstance(agg, list): + raise ValueError( + f"Expected argument 'agg' to be a list of strings, was provided type - {type(agg)} with values - {agg}" + ) arrow_df = arrow_to_dataframe( streamset=streamset, agg=agg, name_callable=name_callable ) - return pl.from_pandas(arrow_df, include_index=True) + return pl.from_pandas(arrow_df, include_index=True, nan_to_null=False) def arrow_to_arrow_table(streamset): @@ -352,7 +371,7 @@ def to_polars(streamset, agg="mean", name_callable=None): if not callable(name_callable): name_callable = lambda s: s.collection + "/" + s.name - df = streamset.to_dataframe(agg=agg) + df = streamset.to_dataframe(agg=agg, name_callable=name_callable) else: df = pd.DataFrame(to_dict(streamset, agg=agg, name_callable=name_callable)) @@ -362,6 +381,7 @@ def to_polars(streamset, agg="mean", name_callable=None): else: df = df.set_index("time") + df.index = pd.DatetimeIndex(df.index, tz="UTC") if agg == "all" and streamset.allow_window: stream_names = [ [s.collection, s.name, prop] @@ -372,7 +392,7 @@ def to_polars(streamset, agg="mean", name_callable=None): else: df.columns = _stream_names(streamset, name_callable) - return pl.from_pandas(df.reset_index()) + return pl.from_pandas(df.reset_index(), nan_to_null=False) def to_array(streamset, agg="mean"): @@ -409,19 +429,21 @@ def to_array(streamset, agg="mean"): return np.array(results, dtype=object) -def arrow_to_numpy(streamset, agg="mean"): +def arrow_to_numpy(streamset, agg=None): """Return a multidimensional array in the numpy format. Parameters ---------- - agg : str, default: "mean" - Specify the StatPoint field (e.g. aggregating function) to return for the - arrays. Must be one of "min", "mean", "max", "count", or "stddev". This + agg : List[str], default: ["mean"] + Specify the StatPoint field or fields (e.g. aggregating function) to return for the + arrays. Must be one or more of "min", "mean", "max", "count", or "stddev". This argument is ignored if RawPoint values are passed into the function. Notes ----- This method first converts to a pandas data frame then to a numpy array. + + This method requires the btrdb server to have arrow support enabled. """ if not streamset._btrdb._ARROW_ENABLED: raise NotImplementedError( @@ -475,16 +497,16 @@ def to_dict(streamset, agg="mean", name_callable=None): return data -def arrow_to_dict(streamset, agg="mean", name_callable=None): +def arrow_to_dict(streamset, agg=None, name_callable=None): """ Returns a list of dicts for each time code with the appropriate stream data attached. Parameters ---------- - agg : str, default: "mean" - Specify the StatPoint field (e.g. aggregating function) to constrain dict - keys. Must be one of "min", "mean", "max", "count", or "stddev". This + agg : List[str], default: ["mean"] + Specify the StatPoint field or fields (e.g. aggregating function) to constrain dict + keys. Must be one or more of "min", "mean", "max", "count", or "stddev". This argument is ignored if RawPoint values are passed into the function. name_callable : lambda, default: lambda s: s.collection + "/" + s.name @@ -496,6 +518,12 @@ def arrow_to_dict(streamset, agg="mean", name_callable=None): raise NotImplementedError( "arrow_to_dict requires an arrow-enabled BTrDB server." ) + if agg is None: + agg = ["mean"] + if not isinstance(agg, list): + raise ValueError( + f"Expected a list of strings for 'agg', was provided type - {type(agg)} with values {agg}." + ) arrow_df = arrow_to_dataframe( streamset=streamset, agg=agg, name_callable=name_callable ) diff --git a/tests/btrdb/test_stream.py b/tests/btrdb/test_stream.py index 7713482..39b82e7 100644 --- a/tests/btrdb/test_stream.py +++ b/tests/btrdb/test_stream.py @@ -134,6 +134,7 @@ def test_refresh_metadata(self): uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) endpoint.streamInfo = Mock(return_value=("koala", 42, {}, {}, None)) + endpoint.info = Mock(return_value={"majorVersion": 5, "minorVersion": 30}) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) stream.refresh_metadata() @@ -171,6 +172,7 @@ def test_refresh_metadata_deserializes_annotations(self): endpoint = Mock(Endpoint) endpoint.streamInfo = Mock(return_value=("koala", 42, {}, serialized, None)) + endpoint.info = Mock(return_value={"majorVersion": 5, "minorVersion": 30}) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) stream.refresh_metadata() @@ -204,6 +206,7 @@ def test_update_arguments(self): """ uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) + endpoint.info = Mock(return_value={"majorVersion": 5, "minorVersion": 30}) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) # no arguments @@ -232,6 +235,7 @@ def test_update_tags(self): """ uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) + endpoint.info = Mock(return_value={"majorVersion": 5, "minorVersion": 30}) endpoint.streamInfo = Mock(return_value=("koala", 42, {}, {}, None)) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) @@ -250,6 +254,7 @@ def test_update_collection(self): uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) endpoint.streamInfo = Mock(return_value=("koala", 42, {}, {}, None)) + endpoint.info = Mock(return_value={"majorVersion": 5, "minorVersion": 30}) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) collection = "giraffe" diff --git a/tests/btrdb_integration/__init__.py b/tests/btrdb_integration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/btrdb_integration/conftest.py b/tests/btrdb_integration/conftest.py index 594b9f4..5e3a7f4 100644 --- a/tests/btrdb_integration/conftest.py +++ b/tests/btrdb_integration/conftest.py @@ -1,7 +1,51 @@ import os + +import pyarrow as pa import pytest + import btrdb + +@pytest.fixture +def single_stream_values_arrow_schema(): + schema = pa.schema( + [ + pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False), + pa.field("value", pa.float64(), nullable=False), + ] + ) + return schema + + +@pytest.fixture +def single_stream_windows_all_stats_arrow_schema(): + schema = pa.schema( + [ + pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False), + pa.field("min", pa.float64(), nullable=False), + pa.field("mean", pa.float64(), nullable=False), + pa.field("max", pa.float64(), nullable=False), + pa.field("count", pa.uint64(), nullable=False), + pa.field("stddev", pa.float64(), nullable=False), + ] + ) + return schema + + +def single_stream_windows_mean_stddev_count_stats_arrow_schema(): + schema = ( + pa.schema( + [ + pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False), + pa.field("mean", pa.float64(), nullable=False), + pa.field("count", pa.uint64(), nullable=False), + pa.field("stddev", pa.float64(), nullable=False), + ] + ), + ) + return schema + + @pytest.fixture def conn(): if os.getenv("BTRDB_INTEGRATION_TEST_PROFILE") == None: @@ -9,11 +53,13 @@ def conn(): conn = btrdb.connect(profile=os.getenv("BTRDB_INTEGRATION_TEST_PROFILE")) return conn + def _delete_collection(conn, col): streams = conn.streams_in_collection(col) for stream in streams: stream.obliterate() + @pytest.fixture def tmp_collection(request, conn): col = "btrdb_python_integration_tests/" + request.node.name + "/tmp" diff --git a/tests/btrdb_integration/test_conn.py b/tests/btrdb_integration/test_conn.py index 15e0534..5d30c4f 100644 --- a/tests/btrdb_integration/test_conn.py +++ b/tests/btrdb_integration/test_conn.py @@ -1,29 +1,36 @@ -import pytest -import btrdb import logging from uuid import uuid4 as new_uuid +import pytest + +import btrdb + + def test_connection_info(conn): info = conn.info() logging.info(f"connection info: {info}") + def test_create_stream(conn, tmp_collection): uuid = new_uuid() - stream = conn.create(uuid, tmp_collection, tags={"name":"s"}) + stream = conn.create(uuid, tmp_collection, tags={"name": "s"}) assert stream.uuid == uuid assert stream.name == "s" + def test_query(conn, tmp_collection): - conn.create(new_uuid(), tmp_collection, tags={"name":"s1"}) - conn.create(new_uuid(), tmp_collection, tags={"name":"s2"}) + conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) uuids = conn.query( - 'select name from streams where collection = $1 order by name;', [tmp_collection] + "select name from streams where collection = $1 order by name;", + [tmp_collection], ) assert len(uuids) == 2 assert uuids[0]["name"] == "s1" assert uuids[1]["name"] == "s2" + def test_list_collections(conn, tmp_collection): assert tmp_collection not in conn.list_collections() - stream = conn.create(new_uuid(), tmp_collection, tags={"name":"s"}) - assert tmp_collection in conn.list_collections() \ No newline at end of file + stream = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) + assert tmp_collection in conn.list_collections() diff --git a/tests/btrdb_integration/test_regressions.py b/tests/btrdb_integration/test_regressions.py index 7ca4df7..db6fd2d 100644 --- a/tests/btrdb_integration/test_regressions.py +++ b/tests/btrdb_integration/test_regressions.py @@ -1,18 +1,26 @@ -import pytest from uuid import uuid4 as new_uuid +import pytest + + +@pytest.mark.xfail def test_create_count_obliterate_concurrency_bug(conn, tmp_collection): from concurrent.futures import ThreadPoolExecutor + n_streams = 10 + def create_stream(i): - return conn.create(new_uuid(), tmp_collection, tags={"name":f"s{i}"}) + return conn.create(new_uuid(), tmp_collection, tags={"name": f"s{i}"}) + def points_in_stream(s): return s.count() + def obliterate_stream(s): s.obliterate() + with ThreadPoolExecutor() as executor: for i in range(2): - pmap = lambda f, it : list(executor.map(f, it, timeout=30)) + pmap = lambda f, it: list(executor.map(f, it, timeout=30)) streams = pmap(create_stream, range(n_streams)) assert sum(pmap(points_in_stream, streams)) == 0 pmap(obliterate_stream, streams) diff --git a/tests/btrdb_integration/test_stream.py b/tests/btrdb_integration/test_stream.py index 3db250f..e65c71d 100644 --- a/tests/btrdb_integration/test_stream.py +++ b/tests/btrdb_integration/test_stream.py @@ -1,63 +1,345 @@ -import pytest import logging - from uuid import uuid4 as new_uuid -from btrdb.utils.timez import currently_as_ns + +import numpy as np +import pandas as pd +import pytest + +import btrdb.utils.timez +from btrdb.utils.timez import currently_as_ns, ns_delta try: import pyarrow as pa -except: +except ImportError: pa = None + +def test_stream_methods(conn, tmp_collection): + s1 = conn.create( + new_uuid(), + tmp_collection, + tags={"name": "s1", "unit": "foo"}, + annotations={"test": "bar"}, + ) + + def test_grpc_insert_and_values(conn, tmp_collection): - s = conn.create(new_uuid(), tmp_collection, tags={"name":"s"}) + s = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) t = currently_as_ns() data = [] - duration=100 + duration = 100 for i in range(duration): - data.append([t+i, i]) + data.append([t + i, i]) s.insert(data) - fetched_data = s.values(start=t, end=t+duration) + fetched_data = s.values(start=t, end=t + duration) assert len(fetched_data) == len(data) - for (i, (p, _)) in enumerate(fetched_data): + for i, (p, _) in enumerate(fetched_data): assert p.time == data[i][0] assert p.value == data[i][1] -def test_arrow_insert_and_values(conn, tmp_collection): - s = conn.create(new_uuid(), tmp_collection, tags={"name":"s"}) + +def test_arrow_insert_and_values( + conn, tmp_collection, single_stream_values_arrow_schema +): + s = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) t = currently_as_ns() times = [] values = [] duration = 100 for i in range(duration): - times.append(t+i) + times.append(t + i) values.append(i) - schema = pa.schema([ - pa.field('time', pa.timestamp('ns', tz='UTC'), nullable=False), - pa.field('value', pa.float64(), nullable=False), - ]) - data = pa.Table.from_arrays([ - pa.array(times), - pa.array(values), - ], schema=schema) + data = pa.Table.from_arrays( + [ + pa.array(times), + pa.array(values), + ], + schema=single_stream_values_arrow_schema, + ) s.arrow_insert(data) - fetched_data = s.arrow_values(start=t, end=t+duration) + fetched_data = s.arrow_values(start=t, end=t + duration) assert data == fetched_data + +@pytest.mark.parametrize( + "merge_policy,duplicates_expected", + [("never", True), ("equal", True), ("retain", False), ("replace", False)], +) +def test_arrow_insert_duplicate_timestamps( + conn, + tmp_collection, + single_stream_values_arrow_schema, + merge_policy, + duplicates_expected, +): + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + t1 = [100, 105, 110, 110, 120] + d1 = [0.0, 1.0, 2.0, 3.0, 4.0] + s1.insert(list(zip(t1, d1)), merge=merge_policy) + df = s1.arrow_values(start=100, end=121).to_pandas().set_index("time") + assert any(df.index.duplicated().tolist()) == duplicates_expected + + +def test_arrow_values_table_schema( + conn, tmp_collection, single_stream_values_arrow_schema +): + s = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) + t = currently_as_ns() + times = [] + values = [] + duration = 100 + for i in range(duration): + times.append(t + i) + values.append(i) + data = pa.Table.from_arrays( + [ + pa.array(times), + pa.array(values), + ], + schema=single_stream_values_arrow_schema, + ) + s.arrow_insert(data) + fetched_data = s.arrow_values(start=t, end=t + duration) + assert single_stream_values_arrow_schema.equals(fetched_data.schema) + + +def test_arrow_values_table_schema( + conn, tmp_collection, single_stream_values_arrow_schema +): + s = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) + t = currently_as_ns() + times = [] + values = [] + duration = 100 + for i in range(duration): + times.append(t + i) + values.append(i) + data = pa.Table.from_arrays( + [ + pa.array(times), + pa.array(values), + ], + schema=single_stream_values_arrow_schema, + ) + s.arrow_insert(data) + fetched_data = s.arrow_values(start=t, end=t + duration) + assert single_stream_values_arrow_schema.equals(fetched_data.schema) + + +def test_arrow_windows( + conn, + tmp_collection, + single_stream_windows_all_stats_arrow_schema, + single_stream_values_arrow_schema, +): + rng = np.random.default_rng(seed=7) + s = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) + t = currently_as_ns() + times = [] + values = [] + duration = btrdb.utils.timez.ns_delta(minutes=1) + nvals = 100 + for i in range(nvals): + times.append(t + i * duration) + values.append(i + rng.random()) + data = pa.Table.from_arrays( + [ + pa.array(times), + pa.array(values), + ], + schema=single_stream_values_arrow_schema, + ) + # group these 5 at a time to go with the 5 min windows from below + mean_dat = np.mean(np.asarray(values).reshape(-1, 5), axis=1) + min_dat = np.min(np.asarray(values).reshape(-1, 5), axis=1) + max_dat = np.max(np.asarray(values).reshape(-1, 5), axis=1) + count_dat = [np.asarray(values).reshape(-1, 5).shape[1]] * int( + np.asarray(values).shape[0] / 5 + ) + stddev_dat = np.std(np.asarray(values).reshape(-1, 5), axis=1) + time_dat = [t for i, t in enumerate(times) if i % 5 == 0] + window_table = pa.Table.from_arrays( + [time_dat, min_dat, mean_dat, max_dat, count_dat, stddev_dat], + schema=single_stream_windows_all_stats_arrow_schema, + ) + s.arrow_insert(data) + # 5 min windows + fetched_data = s.arrow_windows( + start=t, end=t + duration * nvals + 1, width=duration * 5 + ) + fetched_df = fetched_data.to_pandas() + fetched_df["time"] = fetched_df["time"].astype(int) + window_df = window_table.to_pandas() + window_df["time"] = window_df["time"].astype(int) + assert np.allclose(fetched_df.values, window_df.values, rtol=0, atol=1e-9) + + +@pytest.mark.xfail +def test_aligned_windows_no_flush( + conn, + tmp_collection, + single_stream_windows_all_stats_arrow_schema, + single_stream_values_arrow_schema, +): + rng = np.random.default_rng(seed=7) + s = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) + t = currently_as_ns() + times = [] + values = [] + duration = btrdb.utils.timez.ns_delta(minutes=1) + nvals = 100 + for i in range(nvals): + times.append(t + i * duration) + values.append(i + rng.random()) + data = pa.Table.from_arrays( + [ + pa.array(times), + pa.array(values), + ], + schema=single_stream_values_arrow_schema, + ) + s.arrow_insert(data) + fetched_data = s.arrow_aligned_windows( + start=t, + end=t + duration * nvals, + pointwidth=btrdb.utils.general.pointwidth.from_nanoseconds(duration * 6), + ) + fetched_df = fetched_data.to_pandas() + assert fetched_df["stddev"].isna().sum() == 0 + + +def test_arrow_aligned_windows_vs_aligned_windows( + conn, + tmp_collection, + single_stream_windows_all_stats_arrow_schema, + single_stream_values_arrow_schema, +): + rng = np.random.default_rng(seed=7) + s = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) + t = currently_as_ns() + times = [] + values = [] + duration = btrdb.utils.timez.ns_delta(minutes=1) + nvals = 100 + for i in range(nvals): + times.append(t + i * duration) + values.append(i + rng.random()) + data = pa.Table.from_arrays( + [ + pa.array(times), + pa.array(values), + ], + schema=single_stream_values_arrow_schema, + ) + s.arrow_insert(data) + s.flush() + fetched_data = s.arrow_aligned_windows( + start=t, + end=t + duration * nvals, + pointwidth=btrdb.utils.general.pointwidth.from_nanoseconds(duration * 6), + ) + fetched_df = fetched_data.to_pandas() + fetched_df["time"] = fetched_df["time"].astype(int) + fetched_df["count"] = fetched_df["count"].astype(int) + + other_method = s.aligned_windows( + start=t, + end=t + duration * nvals, + pointwidth=btrdb.utils.general.pointwidth.from_nanoseconds(duration * 6), + ) + other_method_data = [] + for statpt, ver in other_method: + tmp = dict() + tmp["time"] = statpt.time + tmp["min"] = statpt.min + tmp["mean"] = statpt.mean + tmp["max"] = statpt.max + tmp["count"] = statpt.count + tmp["stddev"] = statpt.stddev + other_method_data.append(tmp) + other_method_df = pd.DataFrame(other_method_data) + assert fetched_data.schema.equals(single_stream_windows_all_stats_arrow_schema) + assert fetched_df.equals(other_method_df) + + +def compare_arrow_windows_vs_windows( + conn, + tmp_collection, + single_stream_windows_all_stats_arrow_schema, + single_stream_values_arrow_schema, +): + rng = np.random.default_rng(seed=7) + s = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) + t = currently_as_ns() + times = [] + values = [] + duration = btrdb.utils.timez.ns_delta(minutes=1) + nvals = 100 + for i in range(nvals): + times.append(t + i * duration) + values.append(i + rng.random()) + data = pa.Table.from_arrays( + [ + pa.array(times), + pa.array(values), + ], + schema=single_stream_values_arrow_schema, + ) + s.arrow_insert(data) + fetched_data = s.arrow_windows( + start=t, + end=t + duration * nvals, + width=duration * 5, + ) + fetched_df = fetched_data.to_pandas() + fetched_df["time"] = fetched_df["time"].astype(int) + fetched_df["count"] = fetched_df["count"].astype(int) + + other_method = s.windows(start=t, end=t + duration * nvals, width=duration * 5) + other_method_data = [] + for statpt, ver in other_method: + tmp = dict() + tmp["time"] = statpt.time + tmp["min"] = statpt.min + tmp["mean"] = statpt.mean + tmp["max"] = statpt.max + tmp["count"] = statpt.count + tmp["stddev"] = statpt.stddev + other_method_data.append(tmp) + other_method_df = pd.DataFrame(other_method_data) + assert fetched_data.schema.equals(single_stream_windows_all_stats_arrow_schema) + assert fetched_df.equals(other_method_df) + + def test_arrow_empty_values(conn, tmp_collection): - s = conn.create(new_uuid(), tmp_collection, tags={"name":"s"}) + s = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) data = s.arrow_values(start=100, end=1000) - assert len(data['time']) == 0 - assert len(data['value']) == 0 + assert len(data["time"]) == 0 + assert len(data["value"]) == 0 + + +def test_arrow_empty_values_schema(conn, tmp_collection): + s = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) + data = s.arrow_values(start=100, end=1000) + schema = pa.schema( + [ + pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False), + pa.field("value", pa.float64(), nullable=False), + ] + ) + assert schema.equals(data.schema) + @pytest.mark.xfail def test_stream_annotation_update(conn, tmp_collection): # XXX marked as expected failure until someone has time to investigate. - s = conn.create(new_uuid(), tmp_collection, tags={"name":"s"}, annotations={"foo":"bar"}) + s = conn.create( + new_uuid(), tmp_collection, tags={"name": "s"}, annotations={"foo": "bar"} + ) annotations1, version1 = s.annotations() assert version1 == 0 assert annotations1["foo"] == "bar" - s.update(annotations={"foo":"baz"}) + s.update(annotations={"foo": "baz"}) annotations2, version2 = s.annotations() assert version2 > version1 assert annotations2["foo"] == "baz" diff --git a/tests/btrdb_integration/test_streamset.py b/tests/btrdb_integration/test_streamset.py index e01cae8..a132f1b 100644 --- a/tests/btrdb_integration/test_streamset.py +++ b/tests/btrdb_integration/test_streamset.py @@ -1,23 +1,33 @@ -import btrdb -import btrdb.stream -import pytest import logging -from math import nan, isnan +from math import isnan, nan from uuid import uuid4 as new_uuid + +import numpy as np +import pytest +from numpy import testing as np_test + +import btrdb +import btrdb.stream from btrdb.utils.timez import currently_as_ns try: import pyarrow as pa -except: +except ImportError: pa = None try: import pandas as pd -except: +except ImportError: pd = None +try: + import polars as pl + from polars import testing as pl_test +except ImportError: + pl = None + def test_streamset_values(conn, tmp_collection): - s1 = conn.create(new_uuid(), tmp_collection, tags={"name":"s1"}) - s2 = conn.create(new_uuid(), tmp_collection, tags={"name":"s2"}) + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) t1 = [100, 105, 110, 115, 120] t2 = [101, 106, 110, 114, 119] d1 = [0.0, 1.0, 2.0, 3.0, 4.0] @@ -32,9 +42,10 @@ def test_streamset_values(conn, tmp_collection): assert [(p.time, p.value) for p in values[0]] == list(zip(t1, d1)) assert [(p.time, p.value) for p in values[1]] == list(zip(t2, d2)) + def test_streamset_arrow_values(conn, tmp_collection): - s1 = conn.create(new_uuid(), tmp_collection, tags={"name":"s1"}) - s2 = conn.create(new_uuid(), tmp_collection, tags={"name":"s2"}) + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) t1 = [100, 105, 110, 115, 120] t2 = [101, 106, 110, 114, 119] d1 = [0.0, 1.0, 2.0, 3.0, 4.0] @@ -43,26 +54,157 @@ def test_streamset_arrow_values(conn, tmp_collection): s2.insert(list(zip(t2, d2))) ss = btrdb.stream.StreamSet([s1, s2]).filter(start=100, end=121) expected_times = [100, 101, 105, 106, 110, 114, 115, 119, 120] - expected_col1 = [0.0, None, 1.0, None, 2.0, None, 3.0, None, 4.0] - expected_col2 = [None, 5.0, None, 6.0, 7.0, 8.0, None, 9.0, None] + expected_col1 = [0.0, np.NaN, 1.0, np.NaN, 2.0, np.NaN, 3.0, np.NaN, 4.0] + expected_col2 = [np.NaN, 5.0, np.NaN, 6.0, 7.0, 8.0, np.NaN, 9.0, np.NaN] + expected_schema = pa.schema( + [ + pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False), + pa.field(str(s1.uuid), pa.float64(), nullable=False), + pa.field(str(s2.uuid), pa.float64(), nullable=False), + ] + ) values = ss.arrow_values() - times = [t.value for t in values['time']] - col1 = [None if isnan(v.as_py()) else v.as_py() for v in values[tmp_collection + '/s1']] - col2 = [None if isnan(v.as_py()) else v.as_py() for v in values[tmp_collection + '/s2']] + times = [t.value for t in values["time"]] + col1 = [np.NaN if isnan(v.as_py()) else v.as_py() for v in values[str(s1.uuid)]] + col2 = [np.NaN if isnan(v.as_py()) else v.as_py() for v in values[str(s2.uuid)]] assert times == expected_times assert col1 == expected_col1 assert col2 == expected_col2 + assert expected_schema.equals(values.schema) + + +@pytest.mark.parametrize( + "name_callable", + [(None), (lambda s: str(s.uuid)), (lambda s: s.name + "/" + s.collection)], + ids=["empty", "uu_as_str", "name_collection"], +) +def test_streamset_arrow_windows_vs_windows(conn, tmp_collection, name_callable): + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + s3 = conn.create(new_uuid(), tmp_collection, tags={"name": "s3"}) + t1 = [100, 105, 110, 115, 120] + t2 = [101, 106, 110, 114, 119] + d1 = [0.0, 1.0, 2.0, 3.0, 4.0] + d2 = [5.0, 6.0, 7.0, 8.0, 9.0] + d3 = [1.0, 9.0, 44.0, 8.0, 9.0] + s1.insert(list(zip(t1, d1))) + s2.insert(list(zip(t2, d2))) + s3.insert(list(zip(t2, d3))) + ss = ( + btrdb.stream.StreamSet([s1, s2, s3]) + .filter(start=100, end=121) + .windows(width=btrdb.utils.timez.ns_delta(nanoseconds=10)) + ) + values_arrow = ss.arrow_to_dataframe(name_callable=name_callable) + values_prev = ss.to_dataframe(name_callable=name_callable).convert_dtypes( + dtype_backend="pyarrow" + ) + values_prev.index = pd.DatetimeIndex(values_prev.index, tz="UTC") + col_map = {old_col: old_col + "/mean" for old_col in values_prev.columns} + values_prev = values_prev.rename(columns=col_map) + (values_arrow) + (values_prev) + assert values_arrow.equals(values_prev) + + +def test_streamset_arrow_windows_vs_windows_agg_all(conn, tmp_collection): + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + s3 = conn.create(new_uuid(), tmp_collection, tags={"name": "s3"}) + t1 = [100, 105, 110, 115, 120] + t2 = [101, 106, 110, 114, 119] + d1 = [0.0, 1.0, 2.0, 3.0, 4.0] + d2 = [5.0, 6.0, 7.0, 8.0, 9.0] + d3 = [1.0, 9.0, 44.0, 8.0, 9.0] + s1.insert(list(zip(t1, d1))) + s2.insert(list(zip(t2, d2))) + s3.insert(list(zip(t2, d3))) + ss = ( + btrdb.stream.StreamSet([s1, s2, s3]) + .filter(start=100, end=121) + .windows(width=btrdb.utils.timez.ns_delta(nanoseconds=10)) + ) + values_arrow = ss.arrow_to_dataframe(name_callable=None, agg=["all"]) + values_prev = ss.to_dataframe(name_callable=None, agg="all") + values_prev = values_prev.apply(lambda x: x.astype(str(x.dtype) + "[pyarrow]")) + values_prev = values_prev.apply( + lambda x: x.astype("uint64[pyarrow]") if "count" in x.name else x + ) + values_prev.index = pd.DatetimeIndex(values_prev.index, tz="UTC") + values_prev = values_prev.convert_dtypes(dtype_backend="pyarrow") + new_cols = ["/".join(old_col) for old_col in values_prev.columns] + values_prev.columns = new_cols + assert values_arrow.equals(values_prev) + + with pytest.raises( + AttributeError, + match="cannot provide name_callable when using 'all' as aggregate at this time", + ): + values_prev = ss.to_dataframe(name_callable=lambda x: str(x.uuid), agg="all") + other_arrow_df = ss.arrow_to_dataframe( + agg=["all", "mean", "trash"], name_callable=lambda x: str(x.uuid) + ) + assert ( + len(other_arrow_df.filter(regex="[min,mean,max,count,stddev]").columns) == 3 * 5 + ) + + +@pytest.mark.parametrize( + "name_callable", + [(None), (lambda s: str(s.uuid)), (lambda s: s.name + "/" + s.collection)], + ids=["empty", "uu_as_str", "name_collection"], +) +def test_streamset_arrow_aligned_windows_vs_aligned_windows( + conn, tmp_collection, name_callable +): + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + s3 = conn.create(new_uuid(), tmp_collection, tags={"name": "s3"}) + t1 = [100, 105, 110, 115, 120] + t2 = [101, 106, 110, 114, 119] + d1 = [0.0, 1.0, 2.0, 3.0, 4.0] + d2 = [5.0, 6.0, 7.0, 8.0, 9.0] + d3 = [1.0, 9.0, 44.0, 8.0, 9.0] + s1.insert(list(zip(t1, d1))) + s2.insert(list(zip(t2, d2))) + s3.insert(list(zip(t2, d3))) + ss = ( + btrdb.stream.StreamSet([s1, s2, s3]) + .filter(start=100, end=121) + .windows(width=btrdb.utils.general.pointwidth.from_nanoseconds(10)) + ) + values_arrow = ss.arrow_to_dataframe(name_callable=name_callable) + values_prev = ss.to_dataframe( + name_callable=name_callable + ) # .convert_dtypes(dtype_backend='pyarrow') + values_prev = values_prev.apply(lambda x: x.astype(str(x.dtype) + "[pyarrow]")) + values_prev = values_prev.apply( + lambda x: x.astype("uint64[pyarrow]") if "count" in x.name else x + ) + values_prev.index = pd.DatetimeIndex(values_prev.index, tz="UTC") + col_map = {old_col: old_col + "/mean" for old_col in values_prev.columns} + values_prev = values_prev.rename(columns=col_map) + assert values_arrow.equals(values_prev) + def test_streamset_empty_arrow_values(conn, tmp_collection): - s = conn.create(new_uuid(), tmp_collection, tags={"name":"s"}) + s = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) ss = btrdb.stream.StreamSet([s]).filter(start=100, end=200) values = ss.arrow_values() - assert [t.value for t in values['time']] == [] - assert [v for v in values[tmp_collection + '/s']] == [] + expected_schema = pa.schema( + [ + pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False), + pa.field(str(s.uuid), pa.float64(), nullable=False), + ] + ) + assert [t.value for t in values["time"]] == [] + assert [v for v in values[str(s.uuid)]] == [] + assert expected_schema.equals(values.schema) + def test_streamset_to_dataframe(conn, tmp_collection): - s1 = conn.create(new_uuid(), tmp_collection, tags={"name":"s1"}) - s2 = conn.create(new_uuid(), tmp_collection, tags={"name":"s2"}) + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) t1 = [100, 105, 110, 115, 120] t2 = [101, 106, 110, 114, 119] d1 = [0.0, 1.0, 2.0, 3.0, 4.0] @@ -72,11 +214,240 @@ def test_streamset_to_dataframe(conn, tmp_collection): ss = btrdb.stream.StreamSet([s1, s2]).filter(start=100, end=121) values = ss.to_dataframe() expected_times = [100, 101, 105, 106, 110, 114, 115, 119, 120] - expected_col1 = [0.0, None, 1.0, None, 2.0, None, 3.0, None, 4.0] - expected_col2 = [None, 5.0, None, 6.0, 7.0, 8.0, None, 9.0, None] - times = [t for t in values.index] - col1 = [None if isnan(v) else v for v in values[tmp_collection + '/s1']] - col2 = [None if isnan(v) else v for v in values[tmp_collection + '/s2']] - assert times == expected_times - assert col1 == expected_col1 - assert col2 == expected_col2 \ No newline at end of file + expected_col1 = [0.0, np.NaN, 1.0, np.NaN, 2.0, np.NaN, 3.0, np.NaN, 4.0] + expected_col2 = [np.NaN, 5.0, np.NaN, 6.0, 7.0, 8.0, np.NaN, 9.0, np.NaN] + expected_dat = { + tmp_collection + "/s1": expected_col1, + tmp_collection + "/s2": expected_col2, + } + expected_df = pd.DataFrame(expected_dat, index=expected_times) + assert values.equals(expected_df) + + +def test_arrow_streamset_to_dataframe(conn, tmp_collection): + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + t1 = [100, 105, 110, 115, 120] + t2 = [101, 106, 110, 114, 119] + d1 = [0.0, 1.0, 2.0, 3.0, 4.0] + d2 = [5.0, 6.0, 7.0, 8.0, 9.0] + s1.insert(list(zip(t1, d1))) + s2.insert(list(zip(t2, d2))) + ss = btrdb.stream.StreamSet([s1, s2]).filter(start=100, end=121) + values = ss.arrow_to_dataframe() + expected_times = [100, 101, 105, 106, 110, 114, 115, 119, 120] + expected_times = [ + pa.scalar(v, type=pa.timestamp("ns", tz="UTC")).as_py() for v in expected_times + ] + expected_col1 = pa.array( + [0.0, np.NaN, 1.0, np.NaN, 2.0, np.NaN, 3.0, np.NaN, 4.0], mask=[False] * 9 + ) + expected_col2 = pa.array( + [np.NaN, 5.0, np.NaN, 6.0, 7.0, 8.0, np.NaN, 9.0, np.NaN], mask=[False] * 9 + ) + expected_dat = { + "time": expected_times, + tmp_collection + "/s1": expected_col1, + tmp_collection + "/s2": expected_col2, + } + schema = pa.schema( + fields=[ + pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False), + pa.field(tmp_collection + "/s1", type=pa.float64(), nullable=False), + pa.field(tmp_collection + "/s2", type=pa.float64(), nullable=False), + ] + ) + expected_table = pa.Table.from_pydict(expected_dat, schema=schema) + expected_df = expected_table.to_pandas( + timestamp_as_object=False, types_mapper=pd.ArrowDtype + ) + expected_df.set_index("time", inplace=True) + expected_df.index = pd.DatetimeIndex(expected_df.index, tz="UTC") + np_test.assert_array_equal( + values.values.astype(float), expected_df.values.astype(float) + ) + + +def test_arrow_streamset_to_polars(conn, tmp_collection): + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + t1 = [100, 105, 110, 115, 120] + t2 = [101, 106, 110, 114, 119] + d1 = [0.0, 1.0, 2.0, 3.0, 4.0] + d2 = [5.0, 6.0, 7.0, 8.0, 9.0] + s1.insert(list(zip(t1, d1))) + s2.insert(list(zip(t2, d2))) + ss = btrdb.stream.StreamSet([s1, s2]).filter(start=100, end=121) + values = ss.arrow_to_polars() + expected_times = [100, 101, 105, 106, 110, 114, 115, 119, 120] + expected_times = [ + pa.scalar(v, type=pa.timestamp("ns", tz="UTC")).as_py() for v in expected_times + ] + expected_col1 = [0.0, np.NaN, 1.0, np.NaN, 2.0, np.NaN, 3.0, np.NaN, 4.0] + expected_col2 = [np.NaN, 5.0, np.NaN, 6.0, 7.0, 8.0, np.NaN, 9.0, np.NaN] + expected_dat = { + tmp_collection + "/s1": expected_col1, + tmp_collection + "/s2": expected_col2, + } + expected_df = pd.DataFrame( + expected_dat, index=pd.DatetimeIndex(expected_times) + ).reset_index(names="time") + expected_df_pl = pl.from_pandas(expected_df, nan_to_null=False) + pl_test.assert_frame_equal(values, expected_df_pl) + + +@pytest.mark.parametrize( + "name_callable", + [(None), (lambda s: str(s.uuid)), (lambda s: s.name + "/" + s.collection)], + ids=["empty", "uu_as_str", "name_collection"], +) +def test_streamset_arrow_polars_vs_old_to_polars(conn, tmp_collection, name_callable): + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + t1 = [100, 105, 110, 115, 120] + t2 = [101, 106, 110, 114, 119] + d1 = [0.0, 1.0, 2.0, 3.0, 4.0] + d2 = [5.0, 6.0, 7.0, 8.0, 9.0] + s1.insert(list(zip(t1, d1))) + s2.insert(list(zip(t2, d2))) + ss = btrdb.stream.StreamSet([s1, s2]).filter(start=100, end=121) + values_arrow = ss.arrow_to_polars() + values_non_arrow = ss.to_polars() + expected_times = [100, 101, 105, 106, 110, 114, 115, 119, 120] + expected_times = [ + pa.scalar(v, type=pa.timestamp("ns", tz="UTC")).as_py() for v in expected_times + ] + expected_col1 = [0.0, np.NaN, 1.0, np.NaN, 2.0, np.NaN, 3.0, np.NaN, 4.0] + expected_col2 = [np.NaN, 5.0, np.NaN, 6.0, 7.0, 8.0, np.NaN, 9.0, np.NaN] + expected_dat = { + tmp_collection + "/s1": expected_col1, + tmp_collection + "/s2": expected_col2, + } + expected_df = pd.DataFrame( + expected_dat, index=pd.DatetimeIndex(expected_times, tz="UTC") + ).reset_index(names="time") + expected_df_pl = pl.from_pandas(expected_df, nan_to_null=False) + pl_test.assert_frame_equal(values_arrow, expected_df_pl) + pl_test.assert_frame_equal(values_non_arrow, expected_df_pl) + pl_test.assert_frame_equal(values_non_arrow, values_arrow) + + +@pytest.mark.parametrize( + "name_callable", + [(None), (lambda s: str(s.uuid)), (lambda s: s.name + "/" + s.collection)], + ids=["empty", "uu_as_str", "name_collection"], +) +def test_streamset_windows_arrow_polars_vs_old_to_polars( + conn, tmp_collection, name_callable +): + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + s3 = conn.create(new_uuid(), tmp_collection, tags={"name": "s3"}) + t1 = [100, 105, 110, 115, 120] + t2 = [101, 106, 110, 114, 119] + d1 = [0.0, 1.0, 2.0, 3.0, 4.0] + d2 = [5.0, 6.0, 7.0, 8.0, 9.0] + d3 = [1.0, 9.0, 44.0, 8.0, 9.0] + s1.insert(list(zip(t1, d1))) + s2.insert(list(zip(t2, d2))) + s3.insert(list(zip(t2, d3))) + ss = ( + btrdb.stream.StreamSet([s1, s2, s3]) + .filter(start=100, end=121) + .windows(width=btrdb.utils.timez.ns_delta(nanoseconds=10)) + ) + values_arrow_pl = ss.arrow_to_polars(name_callable=name_callable) + values_non_arrow_pl = ss.to_polars(name_callable=name_callable) + new_names = { + old_col: str(old_col) + "/" + "mean" + for old_col in values_non_arrow_pl.select(pl.col(pl.Float64)).columns + } + new_names["time"] = "time" + values_non_arrow_pl = values_non_arrow_pl.rename(mapping=new_names) + assert values_arrow_pl.frame_equal(values_non_arrow_pl) + + +def test_streamset_windows_aggregates_filter(conn, tmp_collection): + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + s3 = conn.create(new_uuid(), tmp_collection, tags={"name": "s3"}) + t1 = [100, 105, 110, 115, 120] + t2 = [101, 106, 110, 114, 119] + d1 = [0.0, 1.0, 2.0, 3.0, 4.0] + d2 = [5.0, 6.0, 7.0, 8.0, 9.0] + d3 = [1.0, 9.0, 44.0, 8.0, 9.0] + s1.insert(list(zip(t1, d1))) + s2.insert(list(zip(t2, d2))) + s3.insert(list(zip(t2, d3))) + ss = ( + btrdb.stream.StreamSet([s1, s2, s3]) + .filter(start=100, end=121) + .windows(width=btrdb.utils.timez.ns_delta(nanoseconds=10)) + ) + values_arrow_df = ss.arrow_to_dataframe(agg=["mean", "stddev"]) + values_non_arrow_df = ss.to_dataframe(agg="all") + values_non_arrow_df.index = pd.DatetimeIndex(values_non_arrow_df.index, tz="UTC") + values_non_arrow_df = values_non_arrow_df.apply( + lambda x: x.astype(str(x.dtype) + "[pyarrow]") + ) + values_non_arrow_df = values_non_arrow_df.apply( + lambda x: x.astype("uint64[pyarrow]") if "count" in x.name else x + ) + new_cols = ["/".join(old_col) for old_col in values_non_arrow_df.columns] + values_non_arrow_df.columns = new_cols + cols_to_drop = [ + col + for col in values_non_arrow_df.columns + if ("count" in col) or ("min" in col) or ("max" in col) + ] + values_non_arrow_df.drop(columns=cols_to_drop, inplace=True) + assert values_arrow_df.equals(values_non_arrow_df) + + +def test_timesnap_backward_extends_range(conn, tmp_collection): + sec = 10**9 + tv1 = [ + [int(0.5 * sec), 0.5], + [2 * sec, 2.0], + ] + tv2 = [ + [int(0.5 * sec) - 1, 0.5], + [2 * sec, 2.0], + ] + tv3 = [ + [1 * sec, 1.0], + [2 * sec, 2.0], + ] + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + s3 = conn.create(new_uuid(), tmp_collection, tags={"name": "s3"}) + s1.insert(tv1) + s2.insert(tv2) + s3.insert(tv3) + ss = btrdb.stream.StreamSet([s1, s2, s3]).filter( + start=1 * sec, end=3 * sec, sampling_frequency=1 + ) + values = ss.arrow_values() + assert [1 * sec, 2 * sec] == [t.value for t in values["time"]] + assert [0.5, 2.0] == [v.as_py() for v in values[str(s1.uuid)]] + assert [np.NaN, 2.0] == [ + np.NaN if isnan(v.as_py()) else v.as_py() for v in values[str(s2.uuid)] + ] + assert [1.0, 2.0] == [v.as_py() for v in values[str(s3.uuid)]] + + +def test_timesnap_forward_restricts_range(conn, tmp_collection): + sec = 10**9 + tv = [ + [1 * sec, 1.0], + [2 * sec, 2.0], + [int(2.75 * sec), 2.75], + ] + s = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) + s.insert(tv) + ss = btrdb.stream.StreamSet([s]).filter(start=1 * sec, sampling_frequency=1) + values = ss.filter(end=int(3.0 * sec)).arrow_values() + assert [1 * sec, 2 * sec] == [t.value for t in values["time"]] + assert [1.0, 2.0] == [v.as_py() for v in values[str(s.uuid)]] + # Same result if skipping past end instead of to end. + assert values == ss.filter(end=int(2.9 * sec)).arrow_values() From 0beef06c7e13c2c57894b026ec7b4dd522405c1b Mon Sep 17 00:00:00 2001 From: Justin Gilmer Date: Wed, 19 Jul 2023 18:19:39 -0500 Subject: [PATCH 054/131] Update docs for arrow (#35) * Update docs, add in final enhanced edits. --- btrdb/conn.py | 61 +++++++++ btrdb/stream.py | 224 +++++++++++++++++++++++++++++-- btrdb/transformers.py | 23 +++- docs/requirements.txt | 1 + docs/source/api/transformers.rst | 19 ++- docs/source/conf.py | 6 +- 6 files changed, 316 insertions(+), 18 deletions(-) diff --git a/btrdb/conn.py b/btrdb/conn.py index 06fbb3c..6a0c4a7 100644 --- a/btrdb/conn.py +++ b/btrdb/conn.py @@ -404,6 +404,67 @@ def list_collections(self, starts_with=""): """ return [c for some in self.ep.listCollections(starts_with) for c in some] + def _list_unique_tags_annotations(self, key, collection): + """ + Returns a SQL statement and parameters to get list of tags or annotations. + """ + if key == "annotations": + query = "select distinct({}) as {} from streams".format( + "skeys(annotations)", "annotations" + ) + else: + query = "select distinct({}) as {} from streams".format(key, key) + params = [] + if isinstance(collection, str): + params.append("{}%".format(collection)) + query = " where ".join([query, """collection like $1"""]) + return [metadata[key] for metadata in self.query(query, params)] + + def list_unique_annotations(self, collection=None): + """ + Returns a list of annotation keys used in a given collection prefix. + + Parameters + ------- + collection: str + Prefix of the collection to filter. + + Returns + ------- + annotations: list[str] + """ + return self._list_unique_tags_annotations("annotations", collection) + + def list_unique_names(self, collection=None): + """ + Returns a list of names used in a given collection prefix. + + Parameters + ------- + collection: str + Prefix of the collection to filter. + + Returns + ------- + names: list[str] + """ + return self._list_unique_tags_annotations("name", collection) + + def list_unique_units(self, collection=None): + """ + Returns a list of units used in a given collection prefix. + + Parameters + ------- + collection: str + Prefix of the collection to filter. + + Returns + ------- + units: list[str] + """ + return self._list_unique_tags_annotations("unit", collection) + @retry def streams_in_collection( self, diff --git a/btrdb/stream.py b/btrdb/stream.py index 0eed275..b302af4 100644 --- a/btrdb/stream.py +++ b/btrdb/stream.py @@ -10,7 +10,6 @@ """ Module for Stream and related classes """ -import io ########################################################################## ## Imports @@ -117,7 +116,7 @@ def refresh_metadata(self): ) = ep.streamInfo(self._uuid, False, True) self._known_to_exist = True - # deserialize annoation values + # deserialize annotation values self._annotations = { key: json.loads(val, cls=AnnotationDecoder) for key, val in self._annotations.items() @@ -323,6 +322,17 @@ def earliest( version : int, default: 0 Specify the version of the stream to query; if zero, queries the latest stream state rather than pinning to a version. + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- @@ -352,6 +362,17 @@ def latest( version : int, default: 0 Specify the version of the stream to query; if zero, queries the latest stream state rather than pinning to a version. + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- @@ -382,6 +403,17 @@ def current( version : int, default: 0 Specify the version of the stream to query; if zero, queries the latest stream state rather than pinning to a version. + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- @@ -409,9 +441,20 @@ def tags( Parameters ---------- - refresh: bool + refresh: bool, default: False Indicates whether a round trip to the server should be implemented regardless of whether there is a local copy. + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- @@ -443,9 +486,20 @@ def annotations( Parameters ---------- - refresh: bool + refresh: bool, default: False Indicates whether a round trip to the server should be implemented regardless of whether there is a local copy. + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- @@ -476,7 +530,17 @@ def version( Parameters ---------- - None + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- @@ -549,6 +613,9 @@ def arrow_insert(self, data: pa.Table, merge: str = "never") -> int: int The version of the stream after inserting new points. + Notes + ----- + ARROW ENABLED SERVERS REQUIRED - CHANGE ME FOR FINAL """ if not self._btrdb._ARROW_ENABLED: raise NotImplementedError(_arrow_not_impl_str.format("arrow_insert")) @@ -674,6 +741,17 @@ def update( Replace all annotations or tags with the specified dictionaries instead of performing the normal upsert operation. Specifying True is the only way to remove annotation keys. + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- @@ -732,6 +810,17 @@ def delete( end : int or datetime like object The end time in nanoseconds for the range to be deleted. (see :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- @@ -771,6 +860,17 @@ def values( :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) version: int The version of the stream to be queried + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------ @@ -822,6 +922,17 @@ def arrow_values( :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) version: int The version of the stream to be queried + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------ @@ -836,6 +947,8 @@ def arrow_values( from a sensor). This is the lowest level of data with the finest time granularity. In the tree data structure of BTrDB, this data is stored in the vector nodes. + + ARROW ENABLED SERVERS REQUIRED - CHANGE ME FOR FINAL """ if not self._btrdb._ARROW_ENABLED: raise NotImplementedError(_arrow_not_impl_str.format("arrow_values")) @@ -898,6 +1011,17 @@ def aligned_windows( Specify the number of ns between data points (2**pointwidth) version : int Version of the stream to query + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- @@ -952,16 +1076,27 @@ def arrow_aligned_windows( Parameters ---------- - start : int or datetime like object + start : int or datetime like object, required The start time in nanoseconds for the range to be queried. (see :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) - end : int or datetime like object + end : int or datetime like object, required The end time in nanoseconds for the range to be queried. (see :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) - pointwidth : int + pointwidth : int, required Specify the number of ns between data points (2**pointwidth) version : int Version of the stream to query + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- @@ -972,6 +1107,8 @@ def arrow_aligned_windows( ----- As the window-width is a power-of-two, it aligns with BTrDB internal tree data structure and is faster to execute than `windows()`. + + ARROW ENABLED SERVERS REQUIRED - CHANGE ME FOR FINAL """ if not self._btrdb._ARROW_ENABLED: raise NotImplementedError( @@ -1032,6 +1169,17 @@ def windows( The number of nanoseconds in each window. version : int The version of the stream to query. + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- @@ -1054,6 +1202,7 @@ def windows( parameter previously available has been deprecated. The only valid value for depth is now 0. + ARROW ENABLED SERVERS REQUIRED - CHANGE ME FOR FINAL """ materialized = [] start = to_nanoseconds(start) @@ -1091,6 +1240,17 @@ def arrow_windows( The number of nanoseconds in each window. version : int, default=0, optional The version of the stream to query. + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- @@ -1110,6 +1270,7 @@ def arrow_windows( end = start + width * floordiv(end - start, width). The `depth` parameter previously available has been deprecated. The only valid value for depth is now 0. + ARROW ENABLED SERVERS REQUIRED - CHANGE ME FOR FINAL """ if not self._btrdb._ARROW_ENABLED: raise NotImplementedError(_arrow_not_impl_str.format("arrow_windows")) @@ -1170,6 +1331,17 @@ def nearest( Version of the stream to use in search backward : boolean True to search backwards from time, else false for forward + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- @@ -1207,11 +1379,24 @@ def obliterate( Obliterate deletes a stream from the BTrDB server. An exception will be raised if the stream could not be found. + Parameters + ---------- + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False + Raises ------ BTrDBError [404] stream does not exist The stream could not be found in BTrDB - """ self._btrdb.ep.obliterate(self._uuid) @@ -1226,6 +1411,19 @@ def flush( """ Flush writes the stream buffers out to persistent storage. + Parameters + ---------- + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False """ self._btrdb.ep.flush(self._uuid) @@ -1848,6 +2046,7 @@ def arrow_insert(self, data_map: dict, merge: str = "never") -> dict: Notes ----- BTrDB expects datetimes to be in UTC+0. + ARROW ENABLED SERVERS REQUIRED - CHANGE ME FOR FINAL Returns ------- @@ -1895,7 +2094,12 @@ def values(self): def arrow_values( self, ): - """Return a pyarrow table of stream values based on the streamset parameters.""" + """Return a pyarrow table of stream values based on the streamset parameters. + + Notes + ----- + ARROW ENABLED SERVERS REQUIRED - CHANGE ME FOR FINAL + """ params = self._params_from_filters() versions = self._pinned_versions if versions is None: diff --git a/btrdb/transformers.py b/btrdb/transformers.py index b2aaa28..4acbc40 100644 --- a/btrdb/transformers.py +++ b/btrdb/transformers.py @@ -117,6 +117,10 @@ def arrow_to_series(streamset, agg="mean", name_callable=None): name_callable : lambda, default: lambda s: s.collection + "/" + s.name Specify a callable that can be used to determine the series name given a Stream object. + + Notes + ----- + ARROW ENABLED SERVERS REQUIRED - CHANGE ME FOR FINAL """ if not streamset._btrdb._ARROW_ENABLED: raise NotImplementedError( @@ -153,7 +157,11 @@ def arrow_to_dataframe( name_callable : lambda, default: lambda s: s.collection + "/" + s.name Specify a callable that can be used to determine the series name given a - Stream object. This is not compatible with agg == "all" at this time + Stream object. + + Notes + ----- + ARROW ENABLED SERVERS REQUIRED - CHANGE ME FOR FINAL """ if not streamset._btrdb._ARROW_ENABLED: raise NotImplementedError( @@ -306,7 +314,7 @@ def arrow_to_polars(streamset, agg=None, name_callable=None): Notes ----- - This requires a BTrDB server that has arrow support enabled. + ARROW ENABLED SERVERS REQUIRED - CHANGE ME FOR FINAL """ if not streamset._btrdb._ARROW_ENABLED: raise NotImplementedError( @@ -329,6 +337,12 @@ def arrow_to_polars(streamset, agg=None, name_callable=None): def arrow_to_arrow_table(streamset): + """Return a pyarrow table of data. + + Notes + ----- + ARROW ENABLED SERVERS REQUIRED - CHANGE ME FOR FINAL + """ if not streamset._btrdb._ARROW_ENABLED: raise NotImplementedError( "arrow_to_arrow_table requires an arrow-enabled BTrDB server." @@ -443,7 +457,7 @@ def arrow_to_numpy(streamset, agg=None): ----- This method first converts to a pandas data frame then to a numpy array. - This method requires the btrdb server to have arrow support enabled. + ARROW ENABLED SERVERS REQUIRED - CHANGE ME FOR FINAL """ if not streamset._btrdb._ARROW_ENABLED: raise NotImplementedError( @@ -513,6 +527,9 @@ def arrow_to_dict(streamset, agg=None, name_callable=None): Specify a callable that can be used to determine the series name given a Stream object. + Notes + ----- + ARROW ENABLED SERVERS REQUIRED - CHANGE ME FOR FINAL """ if not streamset._btrdb._ARROW_ENABLED: raise NotImplementedError( diff --git a/docs/requirements.txt b/docs/requirements.txt index 4da8205..a3bb1f5 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,3 @@ alabaster>=0.7.12 Sphinx>=1.7 +sphinx-rtd-theme diff --git a/docs/source/api/transformers.rst b/docs/source/api/transformers.rst index 289a1ea..facbbc4 100644 --- a/docs/source/api/transformers.rst +++ b/docs/source/api/transformers.rst @@ -1,17 +1,30 @@ btrdb.transformers ========================= -A number of tranformation and serialization functions have been developed so +A number of transformation and serialization functions have been developed so you can use the data in the format of your choice. These functions are provided -in the :code:`btrdb.utils.transformers` module but are also available directly -off the the :code:`StreamSet` class. +in the :code:`StreamSet` class. .. automodule:: btrdb.transformers +.. autofunction:: arrow_to_dict + +.. autofunction:: arrow_to_numpy + +.. autofunction:: arrow_to_series + +.. autofunction:: arrow_to_dataframe + +.. autofunction:: arrow_to_polars + +.. autofunction:: arrow_to_arrow_table + .. autofunction:: to_dict .. autofunction:: to_array +.. autofunction:: to_polars + .. autofunction:: to_series .. autofunction:: to_dataframe diff --git a/docs/source/conf.py b/docs/source/conf.py index 9801730..a2ce391 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -51,6 +51,7 @@ "sphinx.ext.todo", "sphinx.ext.githubpages", "sphinx.ext.intersphinx", + "numpydoc", ] # Add any paths that contain templates here, relative to this directory. @@ -70,7 +71,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -86,7 +87,8 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = "alabaster" +# html_theme = "alabaster" +html_theme = "sphinx_rtd_theme" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the From 0c646d44d4b1461ee998f385bb23957a893fb90c Mon Sep 17 00:00:00 2001 From: Justin Gilmer Date: Thu, 20 Jul 2023 14:50:16 -0500 Subject: [PATCH 055/131] Only enable arrow-endpoints when version >= 5.30 (#36) Once we have a v5.30tag of the server with arrow/multistream, we can merge this and complete the ticket. --- btrdb/conn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/btrdb/conn.py b/btrdb/conn.py index 6a0c4a7..75e78c2 100644 --- a/btrdb/conn.py +++ b/btrdb/conn.py @@ -179,7 +179,7 @@ def _is_arrow_enabled(info): if logger.isEnabledFor(logging.DEBUG): logger.debug(f"major version: {major}") logger.debug(f"minor version: {minor}") - if major >= 5 and minor >= 29: + if major >= 5 and minor >= 30: return True else: return False From c5290170e4ee1359370e3feae138f315f79d1f76 Mon Sep 17 00:00:00 2001 From: Justin Gilmer Date: Thu, 20 Jul 2023 15:27:54 -0500 Subject: [PATCH 056/131] Update arrow notes, small doc changes. (#38) --- btrdb/conn.py | 69 ++++++++++++++++++++++++++++++++++++++----- btrdb/stream.py | 24 +++++++-------- btrdb/transformers.py | 12 ++++---- 3 files changed, 79 insertions(+), 26 deletions(-) diff --git a/btrdb/conn.py b/btrdb/conn.py index 75e78c2..ccfea5d 100644 --- a/btrdb/conn.py +++ b/btrdb/conn.py @@ -53,10 +53,10 @@ def __init__(self, addrportstr, apikey=None): Parameters ---------- - addrportstr: str + addrportstr: str, required The address of the cluster to connect to, e.g 123.123.123:4411 - apikey: str - The option API key to authenticate requests + apikey: str, optional + The optional API key to authenticate requests """ addrport = addrportstr.split(":", 2) @@ -224,6 +224,17 @@ def query( a list of parameter values to be sanitized and interpolated into the SQL statement. Using parameters forces value/type checking and is considered a best practice at the very least. + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- @@ -348,8 +359,25 @@ def create( Parameters ---------- - uuid: UUID + uuid: UUID, required The uuid of the requested stream. + collection: str, required + The collection string prefix that the stream will belong to. + tags: dict, required + The tags-level immutable metadata key:value pairs. + annotations: dict, optional + The mutable metadata of the stream, key:value pairs + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- @@ -376,7 +404,7 @@ def create( def info(self): """ - Returns information about the connected BTrDB srerver. + Returns information about the connected BTrDB server. Returns ------- @@ -397,9 +425,14 @@ def list_collections(self, starts_with=""): Returns a list of collection paths using the `starts_with` argument for filtering. + Parameters + ---------- + starts_with: str, optional, default = '' + Filter collections that start with the string provided, if none passed, will list all collections. + Returns ------- - collection paths: list[str] + collections: List[str] """ return [c for some in self.ep.listCollections(starts_with) for c in some] @@ -492,6 +525,17 @@ def streams_in_collection( The tags to identify the stream. annotations: Dict[str, str] The annotations to identify the stream. + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------ @@ -546,8 +590,19 @@ def collection_metadata( Parameters ---------- - prefix: str + prefix: str, required A prefix of the collection names to look at + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- diff --git a/btrdb/stream.py b/btrdb/stream.py index b302af4..b777bf5 100644 --- a/btrdb/stream.py +++ b/btrdb/stream.py @@ -100,7 +100,7 @@ def __init__(self, btrdb, uuid, **db_values): def refresh_metadata(self): """ - Refreshes the locally cached meta data for a stream + Refreshes the locally cached metadata for a stream from the server. Queries the BTrDB server for all stream metadata including collection, annotation, and tags. This method requires a round trip to the server. @@ -576,7 +576,6 @@ def insert(self, data, merge="never"): ------- int The version of the stream after inserting new points. - """ version = 0 i = 0 @@ -615,7 +614,7 @@ def arrow_insert(self, data: pa.Table, merge: str = "never") -> int: Notes ----- - ARROW ENABLED SERVERS REQUIRED - CHANGE ME FOR FINAL + This method is available for commercial customers with arrow-enabled servers. """ if not self._btrdb._ARROW_ENABLED: raise NotImplementedError(_arrow_not_impl_str.format("arrow_insert")) @@ -758,7 +757,6 @@ def update( int The version of the metadata (separate from the version of the data) also known as the "property version". - """ if tags is None and annotations is None and collection is None: raise BTRDBValueError( @@ -826,7 +824,6 @@ def delete( ------- int The version of the new stream created - """ return self._btrdb.ep.deleteRange( self._uuid, to_nanoseconds(start), to_nanoseconds(end) @@ -948,7 +945,7 @@ def arrow_values( granularity. In the tree data structure of BTrDB, this data is stored in the vector nodes. - ARROW ENABLED SERVERS REQUIRED - CHANGE ME FOR FINAL + This method is available for commercial customers with arrow-enabled servers. """ if not self._btrdb._ARROW_ENABLED: raise NotImplementedError(_arrow_not_impl_str.format("arrow_values")) @@ -1108,7 +1105,7 @@ def arrow_aligned_windows( As the window-width is a power-of-two, it aligns with BTrDB internal tree data structure and is faster to execute than `windows()`. - ARROW ENABLED SERVERS REQUIRED - CHANGE ME FOR FINAL + This method is available for commercial customers with arrow-enabled servers. """ if not self._btrdb._ARROW_ENABLED: raise NotImplementedError( @@ -1202,7 +1199,7 @@ def windows( parameter previously available has been deprecated. The only valid value for depth is now 0. - ARROW ENABLED SERVERS REQUIRED - CHANGE ME FOR FINAL + This method is available for commercial customers with arrow-enabled servers. """ materialized = [] start = to_nanoseconds(start) @@ -1270,7 +1267,7 @@ def arrow_windows( end = start + width * floordiv(end - start, width). The `depth` parameter previously available has been deprecated. The only valid value for depth is now 0. - ARROW ENABLED SERVERS REQUIRED - CHANGE ME FOR FINAL + This method is available for commercial customers with arrow-enabled servers. """ if not self._btrdb._ARROW_ENABLED: raise NotImplementedError(_arrow_not_impl_str.format("arrow_windows")) @@ -1827,12 +1824,12 @@ def windows(self, width, depth=0): Windows returns arbitrary precision windows from BTrDB. It is slower than aligned_windows, but still significantly faster than values. Each returned window will be width nanoseconds long. start is inclusive, but - end is exclusive (e.g if end < start+width you will get no results). + end is exclusive (e.g. if end < start+width you will get no results). That is, results will be returned for all windows that start at a time less than the end timestamp. If (end - start) is not a multiple of width, then end will be decreased to the greatest value less than end such that (end - start) is a multiple of width (i.e., we set end = start - + width * floordiv(end - start, width). The `depth` parameter previously + + width * floordiv(end - start, width)). The `depth` parameter previously available has been deprecated. The only valid value for depth is now 0. """ @@ -2046,7 +2043,8 @@ def arrow_insert(self, data_map: dict, merge: str = "never") -> dict: Notes ----- BTrDB expects datetimes to be in UTC+0. - ARROW ENABLED SERVERS REQUIRED - CHANGE ME FOR FINAL + + This method is available for commercial customers with arrow-enabled servers. Returns ------- @@ -2098,7 +2096,7 @@ def arrow_values( Notes ----- - ARROW ENABLED SERVERS REQUIRED - CHANGE ME FOR FINAL + This method is available for commercial customers with arrow-enabled servers. """ params = self._params_from_filters() versions = self._pinned_versions diff --git a/btrdb/transformers.py b/btrdb/transformers.py index 4acbc40..4939215 100644 --- a/btrdb/transformers.py +++ b/btrdb/transformers.py @@ -120,7 +120,7 @@ def arrow_to_series(streamset, agg="mean", name_callable=None): Notes ----- - ARROW ENABLED SERVERS REQUIRED - CHANGE ME FOR FINAL + This method is available for commercial customers with arrow-enabled servers. """ if not streamset._btrdb._ARROW_ENABLED: raise NotImplementedError( @@ -161,7 +161,7 @@ def arrow_to_dataframe( Notes ----- - ARROW ENABLED SERVERS REQUIRED - CHANGE ME FOR FINAL + This method is available for commercial customers with arrow-enabled servers. """ if not streamset._btrdb._ARROW_ENABLED: raise NotImplementedError( @@ -314,7 +314,7 @@ def arrow_to_polars(streamset, agg=None, name_callable=None): Notes ----- - ARROW ENABLED SERVERS REQUIRED - CHANGE ME FOR FINAL + This method is available for commercial customers with arrow-enabled servers. """ if not streamset._btrdb._ARROW_ENABLED: raise NotImplementedError( @@ -341,7 +341,7 @@ def arrow_to_arrow_table(streamset): Notes ----- - ARROW ENABLED SERVERS REQUIRED - CHANGE ME FOR FINAL + This method is available for commercial customers with arrow-enabled servers. """ if not streamset._btrdb._ARROW_ENABLED: raise NotImplementedError( @@ -457,7 +457,7 @@ def arrow_to_numpy(streamset, agg=None): ----- This method first converts to a pandas data frame then to a numpy array. - ARROW ENABLED SERVERS REQUIRED - CHANGE ME FOR FINAL + This method is available for commercial customers with arrow-enabled servers. """ if not streamset._btrdb._ARROW_ENABLED: raise NotImplementedError( @@ -529,7 +529,7 @@ def arrow_to_dict(streamset, agg=None, name_callable=None): Notes ----- - ARROW ENABLED SERVERS REQUIRED - CHANGE ME FOR FINAL + This method is available for commercial customers with arrow-enabled servers. """ if not streamset._btrdb._ARROW_ENABLED: raise NotImplementedError( From 1900a307f06b09b9f6a83a106d3154d8c4bfbd9e Mon Sep 17 00:00:00 2001 From: Alexander Young Date: Thu, 20 Jul 2023 16:36:46 -0500 Subject: [PATCH 057/131] Merge Arrow into Main for Release (#37) * Threadpool executor (#22) * Release v5.15.0 * update protobuf to v4.22.3 * Add threaded streamset calls Using concurrent.futures.ThreadPoolExecutor * Blacken code * Update for failing tests * Ignore flake8 as part of testing pytest-flake8 seems to have issues with the later versions of flake8 https://github.com/tholo/pytest-flake8/issues/92 * Update .gitignore * Update ignore and remove extra print. * Remove idea folder (pycharm) --------- Co-authored-by: David Konigsberg <72822263+davidkonigsberg@users.noreply.github.com> Co-authored-by: Jeff Lin <42981468+jleifnf@users.noreply.github.com> * Threaded arrow (#23) * Release v5.15.0 * update protobuf to v4.22.3 * Add threaded streamset calls Using concurrent.futures.ThreadPoolExecutor * Blacken code * Update for failing tests * Ignore flake8 as part of testing pytest-flake8 seems to have issues with the later versions of flake8 https://github.com/tholo/pytest-flake8/issues/92 * Update .gitignore * Update proto definitions. * Update endpoint to support arrow methods * Support arrow endpoints * Additional arrow updates * Update transformers, add polars conversion * Update .gitignore * Update ignore and remove extra print. * Remove idea folder (pycharm) * Update requirements.txt * Update btrdb/transformers.py * Update the way to check for arrow-enabled btrdb This has not been "turned on" yet though, since we dont know the version number this will be enabled for. The method is currently commented out, but can be re-enabled pretty easily. * Use IPC streams to send the arrow bytes for insert Instead of writing out feather files to an `io.BytesIO` stream and then sending the feather files over the wire, this creates a buffered outputstream and then sends that data back as bytes to btrdb. * Create arrow specific stream methods. * Update test conn object to support minor version * Update tests and migrate arrow code. * Arrow and standard streamset insert * Create basic arrow to dataframe transformer * Support multirawvalues, arrow transformers * Multivalue arrow queries, in progress * Update stream filter to properly filter for sampling frequency * Update arrow values queries for multivalues * Update param passing for sampling frequency * Update index passing, and ignore depth * benchmark raw values queries for arrow and current api * Add aligned windows and run func * Streamset read benchmarks (WIP) In addition: * update streamset.count to support the `precise` boolean flag. * Update mock return value for versionMajor * In progress validation of stream benchs --------- Co-authored-by: David Konigsberg <72822263+davidkonigsberg@users.noreply.github.com> Co-authored-by: Jeff Lin <42981468+jleifnf@users.noreply.github.com> * Add 3.10 python to the testing matrix (#21) * Add 3.10 python to the testing matrix * Fix yaml parsing * Update requirements to support 3.10 * Use pip-tools `pip-compile` cli tool to generate requirements.txt files from the updated pyproject.toml file * Include pyproject.toml with basic features to support proper extra deps * Support different ways to install btrdb from pip * `btrdb, btrdb[data], btrdb[all], btrdb[testing], btrdb[ray]` * Update transformers.py to build up a numpy array when the subarrays are not the same size (number of entries) * This converts the main array's dtype to `object` * tests still pass with this change * recompile the btrdb proto files with latest protobuf and grpc plugins * Create multiple requirements.txt files for easier updating in the future as well as a locked version with pinned dependencies * Ignore protoc generated flake errors * Update test requirements * Include pre-commit and setup. * Pre-commit lints. * Update pre-commit.yaml add staging to pre-commit checks * Fix missing logging import, rerun pre-commit (#24) * Add basic doc string to endpoint object (#25) * Update benchmark scripts. * Multistream read bench insert bench (#26) * Fix multistream endpoint bugs * The streamset was passing the incorrect params to the endpoint * The endpoint does not return a `version` in its response, just `stat` and `arrowBytes` Params have been updated and a NoneType is passed around to ignore the lack of version info, which lets us use the same logic for all bytes decoding. * Add multistream benchmark methods for timesnap and no timesnap. * Add insert benchmarking methods (#27) Benchmarking methods added for: * stream inserts using tuples of time, value data * stream inserts using pyarrow tables of timestamps, value columns * streamset inserts using a dict map of streamset stream uuids, and lists of tuples of time, value data * streamset inserts using a dict map of streamset stream uuids, and pyarrow tables of timestamps, values. * Fix arrow inserts (#28) * Add insert benchmarking methods Benchmarking methods added for: * stream inserts using tuples of time, value data * stream inserts using pyarrow tables of timestamps, value columns * streamset inserts using a dict map of streamset stream uuids, and lists of tuples of time, value data * streamset inserts using a dict map of streamset stream uuids, and pyarrow tables of timestamps, values. * Include nullable false in pyarrow schema inserts * This was the only difference in the schemas between go and python. * also using a bytesIO stream to act as the sink for the ipc bytes. * Start integration test suite * Add more streamset integration tests. * Add support for authenticated requests without encryption. * Optimize logging calls (#30) Previously, the debug logging in the api would create the f-strings no matter if logging.DEBUG was the current log level or not. This can impact the performance, especially for benchmarking. Now, a cached IS_DEBUG flag is created for the stream operations, and other locations, the logger.isEnabledFor boolean is checked. Note that in the stream.py, this same function call is only executed once, and the results are cached for the rest of the logic. * Add more arrow tests and minor refactoring. * More integration test cases * Restructure tests. * Mark new failing tests as expected failures for now. * Disable gzip compression, it is very slow. * Reenable test, server has been fixed. * Update pandas testing and fix flake8 issues (#31) * Update pandas testing and fix flake8 issues * Update stream logic for unpacking arrow tables, update integration tests. * add init.py for integration tests. * Add additional tests for arrow methods vs their old api counterparts. * Add tests for timesnap boundary conditions. (#32) * Add more integration tests. * Add additional integration tests, modify the name_callable ability of the arrow_values. * remove extraneous prints. * Include retry logic. * Update statpoint order in arrow, fix some bugs with the arrow methods. * Update testing to account for NaNs. * Update github action versions. * Update tests, add in a test for duplicate values. * Remove empty test, remove extraneous prints --------- Co-authored-by: andrewchambers * Update docs for arrow (#35) * Update docs, add in final enhanced edits. * Only enable arrow-endpoints when version >= 5.30 (#36) Once we have a v5.30tag of the server with arrow/multistream, we can merge this and complete the ticket. * Update arrow notes, small doc changes. (#38) --------- Co-authored-by: Justin Gilmer Co-authored-by: David Konigsberg <72822263+davidkonigsberg@users.noreply.github.com> Co-authored-by: Jeff Lin <42981468+jleifnf@users.noreply.github.com> Co-authored-by: Andrew Chambers Co-authored-by: andrewchambers --- .github/workflows/pre-commit.yaml | 39 + .github/workflows/release.yaml | 12 +- .gitignore | 10 + .pre-commit-config.yaml | 35 + MANIFEST.in | 2 +- benchmarks/benchmark_stream_inserts.py | 80 + benchmarks/benchmark_stream_reads.py | 298 ++ benchmarks/benchmark_streamset_inserts.py | 103 + benchmarks/benchmark_streamset_reads.py | 440 ++ btrdb/__init__.py | 18 +- btrdb/conn.py | 330 +- btrdb/endpoint.py | 136 +- btrdb/exceptions.py | 159 +- btrdb/grpcinterface/README.md | 6 + btrdb/grpcinterface/btrdb.proto | 51 + btrdb/grpcinterface/btrdb_pb2.py | 3694 +---------------- btrdb/grpcinterface/btrdb_pb2.pyi | 720 ++++ btrdb/grpcinterface/btrdb_pb2_grpc.py | 1204 ++++-- btrdb/point.py | 29 +- btrdb/stream.py | 1210 +++++- btrdb/transformers.py | 411 +- btrdb/utils/buffer.py | 9 +- btrdb/utils/conversion.py | 12 +- btrdb/utils/credentials.py | 40 +- btrdb/utils/general.py | 13 +- btrdb/utils/ray.py | 37 +- btrdb/utils/timez.py | 25 +- btrdb/version.py | 17 +- btrdb4/__init__.py | 8 +- docs/requirements.txt | 3 +- docs/source/api/transformers.rst | 19 +- docs/source/conf.py | 96 +- docs/source/installing.rst | 2 - docs/source/working/multiprocessing.rst | 2 +- docs/source/working/ray.rst | 4 +- .../source/working/stream-manage-metadata.rst | 2 - pyproject.toml | 76 + release.sh | 2 +- requirements-all-locked.txt | 128 + requirements-all.txt | 57 + requirements.txt | 30 +- setup.cfg | 37 +- setup.py | 73 +- tests/btrdb/test_arrow_streams.py | 47 + tests/btrdb/test_base.py | 50 +- tests/btrdb/test_conn.py | 89 +- tests/btrdb/test_point.py | 29 +- tests/btrdb/test_stream.py | 748 ++-- tests/btrdb/test_transformers.py | 546 ++- tests/btrdb/utils/test_buffer.py | 8 +- tests/btrdb/utils/test_conversions.py | 81 +- tests/btrdb/utils/test_credentials.py | 36 +- tests/btrdb/utils/test_general.py | 50 +- tests/btrdb/utils/test_timez.py | 51 +- tests/btrdb4/test_import.py | 4 +- tests/btrdb_integration/__init__.py | 0 tests/btrdb_integration/conftest.py | 68 + tests/btrdb_integration/test_conn.py | 36 + tests/btrdb_integration/test_regressions.py | 26 + tests/btrdb_integration/test_stream.py | 348 ++ tests/btrdb_integration/test_streamset.py | 453 ++ tests/requirements.txt | 65 +- 62 files changed, 7396 insertions(+), 5018 deletions(-) create mode 100644 .github/workflows/pre-commit.yaml create mode 100644 .pre-commit-config.yaml create mode 100644 benchmarks/benchmark_stream_inserts.py create mode 100644 benchmarks/benchmark_stream_reads.py create mode 100644 benchmarks/benchmark_streamset_inserts.py create mode 100644 benchmarks/benchmark_streamset_reads.py create mode 100644 btrdb/grpcinterface/README.md create mode 100644 btrdb/grpcinterface/btrdb_pb2.pyi create mode 100644 pyproject.toml create mode 100644 requirements-all-locked.txt create mode 100644 requirements-all.txt create mode 100644 tests/btrdb/test_arrow_streams.py create mode 100644 tests/btrdb_integration/__init__.py create mode 100644 tests/btrdb_integration/conftest.py create mode 100644 tests/btrdb_integration/test_conn.py create mode 100644 tests/btrdb_integration/test_regressions.py create mode 100644 tests/btrdb_integration/test_stream.py create mode 100644 tests/btrdb_integration/test_streamset.py diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml new file mode 100644 index 0000000..6567714 --- /dev/null +++ b/.github/workflows/pre-commit.yaml @@ -0,0 +1,39 @@ +name: pre-commit + +on: + pull_request: + branches: + - master + - staging + types: + - opened + - reopened + - ready_for_review + - synchronize + +env: + SKIP: pytest-check + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + fetch-depth: 0 # get full git history + - uses: actions/setup-python@v3 + with: + cache: 'pip' + - name: Install pre-commit + run: | + pip install pre-commit + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v21 + with: + token: ${{ secrets.GITHUB_TOKEN }} + - name: Run pre-commit + uses: pre-commit/action@v2.0.3 + with: + extra_args: --files ${{ steps.changed-files.outputs.all_changed_files }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 61de84e..207f62f 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -14,13 +14,13 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: [3.7, 3.8, 3.9] + python-version: [3.7, 3.8, 3.9, '3.10'] os: [ubuntu-latest, macos-latest, windows-latest] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} ${{ matrix.os }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -39,7 +39,7 @@ jobs: if: startsWith(github.ref, 'refs/tags/') runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Create Release id: create_release uses: actions/create-release@v1 @@ -59,9 +59,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: '3.8' - name: Install dependencies diff --git a/.gitignore b/.gitignore index e7c86be..4629a15 100644 --- a/.gitignore +++ b/.gitignore @@ -118,3 +118,13 @@ dmypy.json # Pyre type checker .pyre/ + +# arrow parquet files +*.parquet + +.idea +.idea/misc.xml +.idea/vcs.xml +.idea/inspectionProfiles/profiles_settings.xml +.idea/inspectionProfiles/Project_Default.xml +/.idea/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..6f327d1 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,35 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + exclude: ^(setup.cfg|btrdb/grpcinterface) +- repo: https://github.com/psf/black + rev: 23.3.0 + hooks: + - id: black-jupyter + args: [--line-length=88] + exclude: btrdb/grpcinterface/.*\.py +- repo: https://github.com/pycqa/isort + rev: 5.11.5 + hooks: + - id: isort + name: isort (python) + args: [--profile=black, --line-length=88] + exclude: btrdb/grpcinterface/.*\.py +- repo: https://github.com/PyCQA/flake8 + rev: 6.0.0 + hooks: + - id: flake8 + args: [--config=setup.cfg] + exclude: ^(btrdb/grpcinterface|tests|setup.py|btrdb4|docs|benchmarks) +- repo: local + hooks: + - id: pytest-check + name: pytest-check + entry: pytest + language: system + pass_filenames: false + always_run: true diff --git a/MANIFEST.in b/MANIFEST.in index 89c5b46..7f67c08 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -20,4 +20,4 @@ global-exclude *.py[co] global-exclude .ipynb_checkpoints global-exclude .DS_Store global-exclude .env -global-exclude .coverage.* \ No newline at end of file +global-exclude .coverage.* diff --git a/benchmarks/benchmark_stream_inserts.py b/benchmarks/benchmark_stream_inserts.py new file mode 100644 index 0000000..003c386 --- /dev/null +++ b/benchmarks/benchmark_stream_inserts.py @@ -0,0 +1,80 @@ +from time import perf_counter +from typing import Dict, List, Tuple, Union + +import pyarrow + +import btrdb + + +def time_stream_insert( + stream: btrdb.stream.Stream, + data: List[Tuple[int, float]], + merge_policy: str = "never", +) -> Dict[str, Union[int, float, str]]: + """Insert raw data into a single stream, where data is a List of tuples of int64 timestamps and float64 values. + + Parameters + ---------- + stream : btrdb.stream.Stream, required + The stream to insert data into. + data : List[Tuple[int, float]], required + The data to insert into stream. + merge_policy : str, optional, default = 'never' + How should the platform handle duplicated data? + Valid policies: + `never`: the default, no points are merged + `equal`: points are deduplicated if the time and value are equal + `retain`: if two points have the same timestamp, the old one is kept + `replace`: if two points have the same timestamp, the new one is kept + """ + prev_ver = stream.version() + tic = perf_counter() + new_ver = stream.insert(data, merge=merge_policy) + toc = perf_counter() + run_time = toc - tic + n_points = len(data) + result = { + "uuid": stream.uuid, + "previous_version": prev_ver, + "new_version": new_ver, + "points_to_insert": n_points, + "total_time_seconds": run_time, + "merge_policy": merge_policy, + } + return result + + +def time_stream_arrow_insert( + stream: btrdb.stream.Stream, data: pyarrow.Table, merge_policy: str = "never" +) -> Dict[str, Union[int, float, str]]: + """Insert raw data into a single stream, where data is a pyarrow Table of timestamps and float values. + + Parameters + ---------- + stream : btrdb.stream.Stream, required + The stream to insert data into. + data : pyarrow.Table, required + The table of data to insert into stream. + merge_policy : str, optional, default = 'never' + How should the platform handle duplicated data? + Valid policies: + `never`: the default, no points are merged + `equal`: points are deduplicated if the time and value are equal + `retain`: if two points have the same timestamp, the old one is kept + `replace`: if two points have the same timestamp, the new one is kept + """ + prev_ver = stream.version() + tic = perf_counter() + new_ver = stream.arrow_insert(data, merge=merge_policy) + toc = perf_counter() + run_time = toc - tic + n_points = data.num_rows + result = { + "uuid": stream.uuid, + "previous_version": prev_ver, + "new_version": new_ver, + "points_to_insert": n_points, + "total_time_seconds": run_time, + "merge_policy": merge_policy, + } + return result diff --git a/benchmarks/benchmark_stream_reads.py b/benchmarks/benchmark_stream_reads.py new file mode 100644 index 0000000..426bdfb --- /dev/null +++ b/benchmarks/benchmark_stream_reads.py @@ -0,0 +1,298 @@ +import pprint +import uuid +from time import perf_counter +from typing import Dict, Union + +import pandas as pd + +import btrdb + + +def time_single_stream_raw_values( + stream: btrdb.stream.Stream, start: int, end: int, version: int = 0 +) -> Dict[str, Union[int, str]]: + """Return the elapsed time for the stream raw values query + + Parameters + ---------- + stream : btrdb.Stream, required + The data stream to return raw values. + start : int, required + The start time (in nanoseconds) to return raw values (inclusive). + end : int, required + The end time (in nanoseconds) to return raw values (exclusive) + version : int, optional, default : 0 + The version of the stream to query for points. + + Notes + ----- + The data points returned will be [start, end) + + Returns + ------- + results : dict + The performance results of the stream method + """ + expected_count = stream.count(start, end, version=version, precise=True) + tic = perf_counter() + vals = stream.values(start, end, version=version) + toc = perf_counter() + # minus 1 to account for the exclusive end time + queried_points = len(vals) + assert queried_points == expected_count + # time in seconds to run + run_time = toc - tic + results = _create_stream_result_dict( + stream.uuid, point_count=queried_points, total_time=run_time, version=version + ) + return results + + +def time_single_stream_arrow_raw_values( + stream: btrdb.stream.Stream, start: int, end: int, version: int = 0 +) -> Dict[str, Union[str, int, float]]: + """Return the elapsed time for the stream arrow raw values query + + Parameters + ---------- + stream : btrdb.Stream, required + The data stream to return the raw data as an arrow table. + start : int, required + The start time (in nanoseconds) to return raw values (inclusive). + end : int, required + The end time (in nanoseconds) to return raw values (exclusive) + version : int, optional, default : 0 + The version of the stream to query for points. + + Notes + ----- + The data points returned will be [start, end) + + Returns + ------- + results : dict + The performance results of the stream method + """ + # minus 1 to account for the exclusive end time for the values query + expected_count = stream.count(start, end, version=version, precise=True) + tic = perf_counter() + vals = stream.arrow_values(start, end, version=version) + toc = perf_counter() + # num of rows + queried_points = vals.num_rows + assert queried_points == expected_count + # time in seconds to run + run_time = toc - tic + results = _create_stream_result_dict( + stream.uuid, point_count=queried_points, total_time=run_time, version=version + ) + return results + + +def time_single_stream_windows_values( + stream: btrdb.stream.Stream, start: int, end: int, width_ns: int, version: int = 0 +) -> Dict[str, Union[str, int, float]]: + """Return the elapsed time for the stream windows values query + + Parameters + ---------- + stream : btrdb.Stream, required + The data stream to return the windowed data as a list of statpoints + start : int, required + The start time (in nanoseconds) to return statpoint values + end : int, required + The end time (in nanoseconds) to return statpoint values + width_ns : int, required + The window width (in nanoseconds) for the statpoints + version : int, optional, default : 0 + The version of the stream to query for points. + + Returns + ------- + results : dict + The performance results of the stream method + """ + tic = perf_counter() + vals = stream.windows(start, end, width=width_ns, version=version) + toc = perf_counter() + # num of statpoints + queried_points = len(vals) + assert queried_points != 0 + # time in seconds to run + run_time = toc - tic + results = _create_stream_result_dict( + stream.uuid, point_count=queried_points, total_time=run_time, version=version + ) + return results + + +def time_single_stream_arrow_windows_values( + stream: btrdb.stream.Stream, start: int, end: int, width_ns: int, version: int = 0 +) -> Dict[str, Union[str, int, float]]: + """Return the elapsed time for the stream arrow window values query + + Parameters + ---------- + stream : btrdb.Stream, required + The data stream to return the windowed data as an arrow table. + start : int, required + The start time (in nanoseconds) to return statpoint values + end : int, required + The end time (in nanoseconds) to return statpoint values + width_ns : int, required + The window width (in nanoseconds) for the statpoints + version : int, optional, default : 0 + The version of the stream to query for points. + + Notes + ----- + The data points returned will be [start, end) + + Returns + ------- + results : dict + The performance results of the stream method + """ + tic = perf_counter() + vals = stream.arrow_windows(start, end, width=width_ns, version=version) + toc = perf_counter() + # num of statpoints + queried_points = vals.num_rows + assert queried_points != 0 + # time in seconds to run + run_time = toc - tic + results = _create_stream_result_dict( + stream.uuid, point_count=queried_points, total_time=run_time, version=version + ) + return results + + +def time_single_stream_aligned_windows_values( + stream: btrdb.stream.Stream, start: int, end: int, pointwidth: int, version: int = 0 +) -> Dict[str, Union[str, int, float]]: + """Return the elapsed time for the stream window values query + + Parameters + ---------- + stream : btrdb.Stream, required + The data stream to return the windowed data as a list of statpoints + start : int, required + The start time (in nanoseconds) to return statpoint values + end : int, required + The end time (in nanoseconds) to return statpoint values + pointwidth : int, required + The level of the tree to return statpoints (the exponent k in 2**k) + version : int, optional, default : 0 + The version of the stream to query for points. + + Returns + ------- + results : dict + The performance results of the stream method + """ + tic = perf_counter() + vals = stream.aligned_windows(start, end, pointwidth=pointwidth, version=version) + toc = perf_counter() + # num of statpoints + queried_points = len(vals) + assert queried_points != 0 + # time in seconds to run + run_time = toc - tic + results = _create_stream_result_dict( + stream.uuid, point_count=queried_points, total_time=run_time, version=version + ) + return results + + +def time_single_stream_arrow_aligned_windows_values( + stream: btrdb.stream.Stream, start: int, end: int, pointwidth: int, version: int = 0 +) -> Dict[str, Union[str, int, float]]: + """Return the elapsed time for the stream arrow aligned window values query + + Parameters + ---------- + stream : btrdb.Stream, required + The data stream to return the windowed data as an arrow table. + start : int, required + The start time (in nanoseconds) to return statpoint values + end : int, required + The end time (in nanoseconds) to return statpoint values + pointwidth : int, required + The level of the tree to return statpoints (the exponent k in 2**k) + version : int, optional, default : 0 + The version of the stream to query for points. + + Returns + ------- + results : dict + The performance results of the stream method + """ + tic = perf_counter() + vals = stream.arrow_aligned_windows( + start, end, pointwidth=pointwidth, version=version + ) + toc = perf_counter() + # num of statpoints + queried_points = vals.num_rows + assert queried_points != 0 + # time in seconds to run + run_time = toc - tic + results = _create_stream_result_dict( + stream.uuid, point_count=queried_points, total_time=run_time, version=version + ) + return results + + +def _create_stream_result_dict( + uu: uuid.UUID, + point_count: int, + total_time: float, + version: int, +) -> Dict[str, Union[str, int, float]]: + return { + "uuid": str(uu), + "total_points": point_count, + "total_time_seconds": total_time, + "stream_version": version, + } + + +def main(): + """Run a single run of the benchmarks""" + conn = btrdb.connect(profile="andy") + stream1 = conn.stream_from_uuid( + list(conn.streams_in_collection("andy/7064-6684-5e6e-9e14-ff9ca7bae46e"))[ + 0 + ].uuid + ) + start = stream1.earliest()[0].time + end = stream1.latest()[0].time + width_ns = btrdb.utils.timez.ns_delta(minutes=1) + pointwidth = btrdb.utils.general.pointwidth(38) + print(f"pointwidth of: {pointwidth}") + res_list = [] + for f in [time_single_stream_arrow_raw_values, time_single_stream_raw_values]: + res = f(stream1, start, end, 0) + res["func"] = f.__name__ + res_list.append(res) + for f in [ + time_single_stream_arrow_windows_values, + time_single_stream_windows_values, + ]: + res = f(stream1, start, end, width_ns=width_ns, version=0) + res["func"] = f.__name__ + res_list.append(res) + for f in [ + time_single_stream_arrow_aligned_windows_values, + time_single_stream_aligned_windows_values, + ]: + res = f(stream1, start, end, pointwidth=pointwidth, version=0) + res["func"] = f.__name__ + res_list.append(res) + return res_list + + +if __name__ == "__main__": + result = main() + pprint.pprint(result) + print(pd.DataFrame.from_dict(result, orient="columns")) diff --git a/benchmarks/benchmark_streamset_inserts.py b/benchmarks/benchmark_streamset_inserts.py new file mode 100644 index 0000000..4a03e1f --- /dev/null +++ b/benchmarks/benchmark_streamset_inserts.py @@ -0,0 +1,103 @@ +import uuid +from time import perf_counter +from typing import Dict, List, Tuple, Union + +import pyarrow + +import btrdb + + +def time_streamset_inserts( + streamset: btrdb.stream.StreamSet, + data_map: Dict[uuid.UUID, List[Tuple[int, float]]], + merge_policy: str = "never", +) -> Dict[str, Union[str, int, float, List, Dict]]: + """Insert data into a streamset with a provided data map. + + Parameters + ---------- + streamset : btrdb.stream.StreamSet, required + The streams to insert data into. + data_map : Dict[uuid.UUID, List[Tuple[int, float]], required + The mappings of streamset uuids to data to insert. + merge_policy : str, optional, default = 'never' + How should the platform handle duplicated data? + Valid policies: + `never`: the default, no points are merged + `equal`: points are deduplicated if the time and value are equal + `retain`: if two points have the same timestamp, the old one is kept + `replace`: if two points have the same timestamp, the new one is kept + """ + _ensure_all_streams_in_data_map(streamset, data_map) + prev_vers = streamset.versions() + tic = perf_counter() + new_vers = streamset.insert(data_map=data_map, merge=merge_policy) + toc = perf_counter() + n_streams = len(streamset) + total_points = sum([len(v) for v in data_map.values()]) + points_insert_per_stream = {s.uuid: len(data_map[s.uuid]) for s in streamset} + total_time = toc - tic + result = { + "n_streams": n_streams, + "total_points": total_points, + "previous_versions": prev_vers, + "new_versions": new_vers, + "total_time_seconds": total_time, + "points_insert_per_stream": points_insert_per_stream, + "merge_policy": merge_policy, + } + return result + + +def time_streamset_arrow_inserts( + streamset: btrdb.stream.StreamSet, + data_map: Dict[uuid.UUID, pyarrow.Table], + merge_policy: str = "never", +) -> Dict[str, Union[str, int, float, List, Dict]]: + """Insert data into streamset using pyarrow Tables. + + Parameters + ---------- + streamset : btrdb.stream.StreamSet, required + The streams to insert data into. + data_map : Dict[uuid.UUID, pyarrow.Table], required + The mappings of streamset uuids to pyarrow tables to insert. + merge_policy : str, optional, default = 'never' + How should the platform handle duplicated data? + Valid policies: + `never`: the default, no points are merged + `equal`: points are deduplicated if the time and value are equal + `retain`: if two points have the same timestamp, the old one is kept + `replace`: if two points have the same timestamp, the new one is kept + """ + _ensure_all_streams_in_data_map(streamset, data_map) + prev_vers = streamset.versions() + tic = perf_counter() + new_vers = streamset.arrow_insert(data_map=data_map, merge=merge_policy) + toc = perf_counter() + n_streams = len(streamset) + total_points = sum([v.num_rows for v in data_map.values()]) + points_insert_per_stream = {s.uuid: data_map[s.uuid].num_rows for s in streamset} + total_time = toc - tic + result = { + "n_streams": n_streams, + "total_points": total_points, + "previous_versions": prev_vers, + "new_versions": new_vers, + "total_time_seconds": total_time, + "points_insert_per_stream": points_insert_per_stream, + "merge_policy": merge_policy, + } + return result + + +def _ensure_all_streams_in_data_map( + streamset: btrdb.stream.StreamSet, + data_map: Dict[uuid.UUID, Union[pyarrow.Table, List[Tuple[int, float]]]], +) -> None: + truthy = [s.uuid in data_map for s in streamset] + uus = [s.uuid for s in streamset] + if not all(truthy): + raise ValueError( + f"Streams in streamset are not present in the data map. Data_map keys: {data_map.keys()}, streamset uuids: {uus}" + ) diff --git a/benchmarks/benchmark_streamset_reads.py b/benchmarks/benchmark_streamset_reads.py new file mode 100644 index 0000000..7c2c836 --- /dev/null +++ b/benchmarks/benchmark_streamset_reads.py @@ -0,0 +1,440 @@ +import pprint +import uuid +from time import perf_counter +from typing import Dict, Union + +import pandas + +import btrdb + + +def time_streamset_raw_values( + streamset: btrdb.stream.StreamSet, start: int, end: int, version: int = 0 +) -> Dict[str, Union[int, str]]: + """Return the elapsed time for the streamset raw values query + + Parameters + ---------- + streamset : btrdb.StreamSet, required + The data streamset to return raw values. + start : int, required + The start time (in nanoseconds) to return raw values (inclusive). + end : int, required + The end time (in nanoseconds) to return raw values (exclusive) + version : int, optional, default : 0 + The version of the streamset to query for points. + + Notes + ----- + The data points returned will be [start, end) + + Returns + ------- + results : dict + The performance results of the streamset method + """ + streamset = streamset.filter(start=start, end=end) + versions = {s.uuid: 0 for s in streamset} + streamset.pin_versions(versions) + # minus 1 * n_streams to account for the exclusive end time + expected_count = streamset.count(precise=True) - len(streamset) + tic = perf_counter() + vals = streamset.values() + toc = perf_counter() + # print(vals) + queried_points = 0 + for points in vals: + print(len(points)) + queried_points += len(points) + expected_count = streamset.count(precise=True) + print(queried_points) + print(expected_count) + assert queried_points == expected_count + # time in seconds to run + run_time = toc - tic + res = _create_streamset_result_dict( + streamset, point_count=queried_points, total_time=run_time, version=version + ) + return res + + +def time_streamset_arrow_raw_values( + streamset: btrdb.stream.StreamSet, start: int, end: int, version: int = 0 +) -> Dict[str, Union[str, int, float]]: + """Return the elapsed time for the streamset arrow raw values query + + Parameters + ---------- + streamset : btrdb.StreamSet, required + The data streamset to return the raw data as an arrow table. + start : int, required + The start time (in nanoseconds) to return raw values (inclusive). + end : int, required + The end time (in nanoseconds) to return raw values (exclusive) + version : int, optional, default : 0 + The version of the streamset to query for points. + + Notes + ----- + The data points returned will be [start, end) + + Returns + ------- + results : dict + The performance results of the streamset method + """ + streamset = streamset.filter(start=start, end=end) + versions = {s.uuid: 0 for s in streamset} + streamset.pin_versions(versions) + # minus 1 * n_streams to account for the exclusive end time + expected_count = streamset.count(precise=True) - len(streamset) + tic = perf_counter() + vals = streamset.arrow_values() + toc = perf_counter() + # TODO: These point counts and assertion needs to be column specific + queried_points = len(streamset) * (vals.num_rows - 1) + print(queried_points) + print(expected_count) + assert queried_points == expected_count + # time in seconds to run + run_time = toc - tic + res = _create_streamset_result_dict( + streamset, point_count=queried_points, total_time=run_time, version=version + ) + return res + + +def time_streamset_windows_values( + streamset: btrdb.stream.StreamSet, + start: int, + end: int, + width_ns: int, + version: int = 0, +) -> Dict[str, Union[str, int, float]]: + """Return the elapsed time for the streamset windows values query + + Parameters + ---------- + streamset : btrdb.StreamSet, required + The data streamset to return the windowed data as a list of statpoints + start : int, required + The start time (in nanoseconds) to return statpoint values + end : int, required + The end time (in nanoseconds) to return statpoint values + width_ns : int, required + The window width (in nanoseconds) for the statpoints + version : int, optional, default : 0 + The version of the streamset to query for points. + + Returns + ------- + results : dict + The performance results of the streamset method + """ + streamset = streamset.filter(start=start, end=end) + versions = {s.uuid: 0 for s in streamset} + streamset.pin_versions(versions) + streamset = streamset.windows(width=width_ns) + tic = perf_counter() + vals = streamset.values() + toc = perf_counter() + # num of statpoints + queried_points = len(vals[0]) * len(streamset) + assert queried_points != 0 + # time in seconds to run + run_time = toc - tic + results = _create_streamset_result_dict( + streamset, point_count=queried_points, total_time=run_time, version=version + ) + return results + + +def time_streamset_arrow_windows_values( + streamset: btrdb.stream.StreamSet, + start: int, + end: int, + width_ns: int, + version: int = 0, +) -> Dict[str, Union[str, int, float]]: + """Return the elapsed time for the streamset arrow window values query + + Parameters + ---------- + streamset : btrdb.StreamSet, required + The data streamset to return the windowed data as an arrow table. + start : int, required + The start time (in nanoseconds) to return statpoint values + end : int, required + The end time (in nanoseconds) to return statpoint values + width_ns : int, required + The window width (in nanoseconds) for the statpoints + version : int, optional, default : 0 + The version of the streamset to query for points. + + Notes + ----- + The data points returned will be [start, end) + + Returns + ------- + results : dict + The performance results of the streamset method + """ + streamset = streamset.filter(start=start, end=end) + versions = {s.uuid: 0 for s in streamset} + streamset.pin_versions(versions) + streamset = streamset.windows(width=width_ns) + tic = perf_counter() + vals = streamset.arrow_values() + toc = perf_counter() + # num of statpoints + queried_points = vals.num_rows * len(streamset) + assert queried_points != 0 + # time in seconds to run + run_time = toc - tic + results = _create_streamset_result_dict( + streamset, point_count=queried_points, total_time=run_time, version=version + ) + return results + + +def time_streamset_aligned_windows_values( + streamset: btrdb.stream.StreamSet, + start: int, + end: int, + pointwidth: int, + version: int = 0, +) -> Dict[str, Union[str, int, float]]: + """Return the elapsed time for the streamset window values query + + Parameters + ---------- + streamset : btrdb.StreamSet, required + The data streamset to return the windowed data as a list of statpoints + start : int, required + The start time (in nanoseconds) to return statpoint values + end : int, required + The end time (in nanoseconds) to return statpoint values + pointwidth : int, required + The level of the tree to return statpoints (the exponent k in 2**k) + version : int, optional, default : 0 + The version of the streamset to query for points. + + Returns + ------- + results : dict + The performance results of the streamset method + """ + streamset = streamset.filter(start=start, end=end) + versions = {s.uuid: 0 for s in streamset} + streamset.pin_versions(versions) + streamset = streamset.aligned_windows(pointwidth=pointwidth) + tic = perf_counter() + vals = streamset.values() + toc = perf_counter() + # num of statpoints + queried_points = len(vals[0]) * len(streamset) + assert queried_points != 0 + # time in seconds to run + run_time = toc - tic + results = _create_streamset_result_dict( + streamset, point_count=queried_points, total_time=run_time, version=version + ) + return results + + +def time_streamset_arrow_aligned_windows_values( + streamset: btrdb.stream.StreamSet, + start: int, + end: int, + pointwidth: int, + version: int = 0, +) -> Dict[str, Union[str, int, float]]: + """Return the elapsed time for the streamset arrow aligned window values query + + Parameters + ---------- + streamset : btrdb.StreamSet, required + The data streamset to return the windowed data as an arrow table. + start : int, required + The start time (in nanoseconds) to return statpoint values + end : int, required + The end time (in nanoseconds) to return statpoint values + pointwidth : int, required + The level of the tree to return statpoints (the exponent k in 2**k) + version : int, optional, default : 0 + The version of the streamset to query for points. + + Returns + ------- + results : dict + The performance results of the streamset method + """ + streamset = streamset.filter(start=start, end=end) + versions = {s.uuid: 0 for s in streamset} + streamset.pin_versions(versions) + streamset = streamset.aligned_windows(pointwidth=pointwidth) + tic = perf_counter() + vals = streamset.arrow_values() + toc = perf_counter() + # num of statpoints + queried_points = vals.num_rows * len(streamset) + assert queried_points != 0 + # time in seconds to run + run_time = toc - tic + results = _create_streamset_result_dict( + streamset, point_count=queried_points, total_time=run_time, version=version + ) + return results + + +def time_streamset_arrow_multistream_raw_values_non_timesnapped( + streamset: btrdb.stream.StreamSet, + start: int, + end: int, + version: int = 0, + sampling_frequency: int = None, +) -> Dict[str, Union[str, int, float]]: + """Use the arrow multistream endpoint that joins the stream data on-server before sending to the client. + + We make sure to set a sampling rate of 0 to ensure that we do not time snap and just perform a full-outer join on + the streams. + + Parameters + ---------- + streamset : btrdb.stream.StreamSet, required + The streamset to perform the multistream query on. + start : int, required + The start time (in nanoseconds) to query raw data from. + end : int, required + The end time (in nanoseconds) non-exclusive, to query raw data from. + version : int, optional, default=0 + The version of the stream to pin against, currently this is unused. + sampling_frequency : int, optional, ignored + The sampling frequency of the data stream in Hz + + Notes + ----- + Sampling_frequency is not used here, it will be manually set. + """ + streamset = streamset.filter(start=start, end=end, sampling_frequency=0) + versions = {s.uuid: 0 for s in streamset} + streamset = streamset.pin_versions(versions) + tic = perf_counter() + vals = streamset.arrow_values() + toc = perf_counter() + queried_points = vals.num_rows * len(streamset) + # print(vals) + # print(vals.to_pandas().describe()) + run_time = toc - tic + results = _create_streamset_result_dict( + streamset=streamset, total_time=run_time, point_count=queried_points, version=0 + ) + return results + + +def time_streamset_arrow_multistream_raw_values_timesnapped( + streamset: btrdb.stream.StreamSet, + start: int, + end: int, + sampling_frequency: int, + version: int = 0, +) -> Dict[str, Union[str, int, float]]: + """Use the arrow multistream endpoint that joins the stream data on-server before sending to the client. + + We make sure to set a sampling rate to ensure that we time snap the returned data. + + Parameters + ---------- + streamset : btrdb.stream.StreamSet, required + The streamset to perform the multistream query on. + start : int, required + The start time (in nanoseconds) to query raw data from. + end : int, required + The end time (in nanoseconds) non-exclusive, to query raw data from. + sampling_frequency : int, required + The common sampling frequency (in Hz) of the data to snap the data points to. + version : int, optional, default=0 + The version of the stream to pin against, currently this is unused. + """ + streamset = streamset.filter( + start=start, end=end, sampling_frequency=sampling_frequency + ) + versions = {s.uuid: 0 for s in streamset} + streamset = streamset.pin_versions(versions) + tic = perf_counter() + vals = streamset.arrow_values() + toc = perf_counter() + queried_points = vals.num_rows * len(streamset) + # print(vals) + run_time = toc - tic + results = _create_streamset_result_dict( + streamset=streamset, total_time=run_time, point_count=queried_points, version=0 + ) + return results + + +def _create_streamset_result_dict( + streamset: btrdb.stream.StreamSet, + point_count: int, + total_time: float, + version: int, +) -> Dict[str, Union[str, int, float]]: + return { + "n_streams": len(streamset), + "total_points": point_count, + "total_time_seconds": total_time, + "streamset_version": version, + } + + +def main(): + """Run a single run of the benchmarks""" + conn = btrdb.connect(profile="andy") + stream1 = conn.stream_from_uuid( + list(conn.streams_in_collection("andy/7064-6684-5e6e-9e14-ff9ca7bae46e"))[ + 0 + ].uuid + ) + stream2 = conn.stream_from_uuid( + list(conn.streams_in_collection("andy/30e6-d72f-5cc7-9966-bc1579dc4a72"))[ + 0 + ].uuid + ) + streamset = btrdb.stream.StreamSet([stream1, stream2]) + start = max(stream1.earliest()[0].time, stream2.earliest()[0].time) + end = min(stream1.latest()[0].time, stream2.latest()[0].time) + width_ns = btrdb.utils.timez.ns_delta(minutes=1) + pointwidth = btrdb.utils.general.pointwidth(38) + print(f"pointwidth of: {pointwidth}") + res_list = [] + for f in [time_streamset_raw_values, time_streamset_arrow_raw_values]: + res = f(streamset, start, end, 0) + res["func"] = f.__name__ + res_list.append(res) + for f in [time_streamset_windows_values, time_streamset_arrow_windows_values]: + res = f(streamset, start, end, width_ns=width_ns, version=0) + res["func"] = f.__name__ + res_list.append(res) + for f in [ + time_streamset_aligned_windows_values, + time_streamset_arrow_aligned_windows_values, + ]: + res = f(streamset, start, end, pointwidth=pointwidth, version=0) + res["func"] = f.__name__ + res_list.append(res) + for f in [ + time_streamset_arrow_multistream_raw_values_non_timesnapped, + time_streamset_arrow_multistream_raw_values_timesnapped, + ]: + res = f(streamset, start, end, sampling_frequency=2, version=0) + res["func"] = f.__name__ + res_list.append(res) + + return res_list + + +if __name__ == "__main__": + results = main() + pprint.pprint(results) + print(pandas.DataFrame.from_dict(results, orient="columns")) diff --git a/btrdb/__init__.py b/btrdb/__init__.py index cd14662..efcb686 100644 --- a/btrdb/__init__.py +++ b/btrdb/__init__.py @@ -15,14 +15,15 @@ ## Imports ########################################################################## -from btrdb.conn import Connection, BTrDB +from warnings import warn + +from btrdb.conn import BTrDB, Connection from btrdb.endpoint import Endpoint from btrdb.exceptions import ConnectionError -from btrdb.version import get_version -from btrdb.utils.credentials import credentials_by_profile, credentials +from btrdb.stream import MAXIMUM_TIME, MINIMUM_TIME +from btrdb.utils.credentials import credentials, credentials_by_profile from btrdb.utils.ray import register_serializer -from btrdb.stream import MINIMUM_TIME, MAXIMUM_TIME -from warnings import warn +from btrdb.version import get_version ########################################################################## ## Module Variables @@ -39,9 +40,11 @@ ## Functions ########################################################################## + def _connect(endpoints=None, apikey=None): return BTrDB(Endpoint(Connection(endpoints, apikey=apikey).channel)) + def connect(conn_str=None, apikey=None, profile=None, shareable=False): """ Connect to a BTrDB server. @@ -78,7 +81,9 @@ def connect(conn_str=None, apikey=None, profile=None, shareable=False): # check shareable flag and register custom serializer if necessary if shareable: - warn("a shareable connection is potentially insecure; other users of the same cluster may be able to access your API key") + warn( + "a shareable connection is potentially insecure; other users of the same cluster may be able to access your API key" + ) register_serializer(conn_str=conn_str, apikey=apikey, profile=profile) # use specific profile if requested @@ -91,4 +96,3 @@ def connect(conn_str=None, apikey=None, profile=None, shareable=False): return _connect(**creds) raise ConnectionError("Could not determine credentials to use.") - diff --git a/btrdb/conn.py b/btrdb/conn.py index bb79f2f..ccfea5d 100644 --- a/btrdb/conn.py +++ b/btrdb/conn.py @@ -15,19 +15,21 @@ ## Imports ########################################################################## +import json +import logging import os import re -import json -import certifi import uuid as uuidlib +from concurrent.futures import ThreadPoolExecutor +import certifi import grpc from grpc._cython.cygrpc import CompressionAlgorithm +from btrdb.exceptions import InvalidOperation, StreamNotFoundError, retry from btrdb.stream import Stream, StreamSet -from btrdb.utils.general import unpack_stream_descriptor from btrdb.utils.conversion import to_uuid -from btrdb.exceptions import StreamNotFoundError, InvalidOperation +from btrdb.utils.general import unpack_stream_descriptor ########################################################################## ## Module Variables @@ -41,23 +43,24 @@ ########################################################################## ## Classes ########################################################################## +logger = logging.getLogger(__name__) -class Connection(object): +class Connection(object): def __init__(self, addrportstr, apikey=None): """ Connects to a BTrDB server Parameters ---------- - addrportstr: str + addrportstr: str, required The address of the cluster to connect to, e.g 123.123.123:4411 - apikey: str - The option API key to authenticate requests + apikey: str, optional + The optional API key to authenticate requests """ addrport = addrportstr.split(":", 2) - chan_ops = [('grpc.default_compression_algorithm', CompressionAlgorithm.gzip)] + chan_ops = [] if len(addrport) != 2: raise ValueError("expecting address:port") @@ -82,7 +85,10 @@ def __init__(self, addrportstr, apikey=None): except Exception: if env_bundle != "": # The user has given us something but we can't use it, we need to make noise - raise Exception("BTRDB_CA_BUNDLE(%s) env is defined but could not read file" % ca_bundle) + raise Exception( + "BTRDB_CA_BUNDLE(%s) env is defined but could not read file" + % ca_bundle + ) else: contents = None @@ -90,23 +96,93 @@ def __init__(self, addrportstr, apikey=None): self.channel = grpc.secure_channel( addrportstr, grpc.ssl_channel_credentials(contents), - options=chan_ops + options=chan_ops, ) else: self.channel = grpc.secure_channel( addrportstr, grpc.composite_channel_credentials( grpc.ssl_channel_credentials(contents), - grpc.access_token_call_credentials(apikey) + grpc.access_token_call_credentials(apikey), ), - options=chan_ops + options=chan_ops, ) else: - if apikey is not None: - raise ValueError("cannot use an API key with an insecure (port 4410) BTrDB API. Try port 4411") self.channel = grpc.insecure_channel(addrportstr, chan_ops) + if apikey is not None: + + class AuthCallDetails(grpc.ClientCallDetails): + def __init__(self, apikey, client_call_details): + metadata = [] + if client_call_details.metadata is not None: + metadata = list(client_call_details.metadata) + metadata.append(("authorization", "Bearer " + apikey)) + self.method = client_call_details.method + self.timeout = client_call_details.timeout + self.credentials = client_call_details.credentials + self.wait_for_ready = client_call_details.wait_for_ready + self.compression = client_call_details.compression + self.metadata = metadata + + class AuthorizationInterceptor( + grpc.UnaryUnaryClientInterceptor, + grpc.UnaryStreamClientInterceptor, + grpc.StreamUnaryClientInterceptor, + grpc.StreamStreamClientInterceptor, + ): + def __init__(self, apikey): + self.apikey = apikey + + def intercept_unary_unary( + self, continuation, client_call_details, request + ): + return continuation( + AuthCallDetails(self.apikey, client_call_details), request + ) + + def intercept_unary_stream( + self, continuation, client_call_details, request + ): + return continuation( + AuthCallDetails(self.apikey, client_call_details), request + ) + + def intercept_stream_unary( + self, continuation, client_call_details, request_iterator + ): + return continuation( + AuthCallDetails(self.apikey, client_call_details), + request_iterator, + ) + + def intercept_stream_stream( + self, continuation, client_call_details, request_iterator + ): + return continuation( + AuthCallDetails(self.apikey, client_call_details), + request_iterator, + ) + + self.channel = grpc.intercept_channel( + self.channel, + AuthorizationInterceptor(apikey), + ) +def _is_arrow_enabled(info): + info = { + "majorVersion": info.majorVersion, + "minorVersion": info.minorVersion, + } + major = info.get("majorVersion", -1) + minor = info.get("minorVersion", -1) + if logger.isEnabledFor(logging.DEBUG): + logger.debug(f"major version: {major}") + logger.debug(f"minor version: {minor}") + if major >= 5 and minor >= 30: + return True + else: + return False class BTrDB(object): @@ -116,8 +192,24 @@ class BTrDB(object): def __init__(self, endpoint): self.ep = endpoint - - def query(self, stmt, params=[]): + self._executor = ThreadPoolExecutor() + try: + self._ARROW_ENABLED = _is_arrow_enabled(self.ep.info()) + except Exception: + self._ARROW_ENABLED = False + if logger.isEnabledFor(logging.DEBUG): + logger.debug(f"ARROW ENABLED: {self._ARROW_ENABLED}") + + @retry + def query( + self, + stmt, + params=None, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Performs a SQL query on the database metadata and returns a list of dictionaries from the resulting cursor. @@ -132,6 +224,17 @@ def query(self, stmt, params=[]): a list of parameter values to be sanitized and interpolated into the SQL statement. Using parameters forces value/type checking and is considered a best practice at the very least. + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- @@ -147,13 +250,14 @@ def query(self, stmt, params=[]): See https://btrdb.readthedocs.io/en/latest/ for more info. """ + if params is None: + params = list() return [ json.loads(row.decode("utf-8")) for page in self.ep.sql_query(stmt, params) for row in page ] - def streams(self, *identifiers, versions=None, is_collection_prefix=False): """ Returns a StreamSet object with BTrDB streams from the supplied @@ -196,20 +300,23 @@ def streams(self, *identifiers, versions=None, is_collection_prefix=False): found = self.streams_in_collection( "/".join(parts[:-1]), is_collection_prefix=is_collection_prefix, - tags={"name": parts[-1]} + tags={"name": parts[-1]}, ) if len(found) == 1: streams.append(found[0]) continue raise StreamNotFoundError(f"Could not identify stream `{ident}`") - raise ValueError(f"Could not identify stream based on `{ident}`. Identifier must be UUID or collection/name.") - + raise ValueError( + f"Could not identify stream based on `{ident}`. Identifier must be UUID or collection/name." + ) obj = StreamSet(streams) if versions: - version_dict = {streams[idx].uuid: versions[idx] for idx in range(len(versions))} + version_dict = { + streams[idx].uuid: versions[idx] for idx in range(len(versions)) + } obj.pin_versions(version_dict) return obj @@ -235,14 +342,42 @@ def stream_from_uuid(self, uuid): """ return Stream(self, to_uuid(uuid)) - def create(self, uuid, collection, tags=None, annotations=None): + @retry + def create( + self, + uuid, + collection, + tags=None, + annotations=None, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Tells BTrDB to create a new stream with UUID `uuid` in `collection` with specified `tags` and `annotations`. Parameters ---------- - uuid: UUID + uuid: UUID, required The uuid of the requested stream. + collection: str, required + The collection string prefix that the stream will belong to. + tags: dict, required + The tags-level immutable metadata key:value pairs. + annotations: dict, optional + The mutable metadata of the stream, key:value pairs + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- @@ -257,17 +392,19 @@ def create(self, uuid, collection, tags=None, annotations=None): annotations = {} self.ep.create(uuid, collection, tags, annotations) - return Stream(self, uuid, + return Stream( + self, + uuid, known_to_exist=True, collection=collection, tags=tags.copy(), annotations=annotations.copy(), - property_version=0 + property_version=0, ) def info(self): """ - Returns information about the connected BTrDB srerver. + Returns information about the connected BTrDB server. Returns ------- @@ -278,8 +415,9 @@ def info(self): info = self.ep.info() return { "majorVersion": info.majorVersion, + "minorVersion": info.minorVersion, "build": info.build, - "proxy": { "proxyEndpoints": [ep for ep in info.proxy.proxyEndpoints] }, + "proxy": {"proxyEndpoints": [ep for ep in info.proxy.proxyEndpoints]}, } def list_collections(self, starts_with=""): @@ -287,14 +425,91 @@ def list_collections(self, starts_with=""): Returns a list of collection paths using the `starts_with` argument for filtering. + Parameters + ---------- + starts_with: str, optional, default = '' + Filter collections that start with the string provided, if none passed, will list all collections. + Returns ------- - collection paths: list[str] + collections: List[str] """ return [c for some in self.ep.listCollections(starts_with) for c in some] - def streams_in_collection(self, *collection, is_collection_prefix=True, tags=None, annotations=None): + def _list_unique_tags_annotations(self, key, collection): + """ + Returns a SQL statement and parameters to get list of tags or annotations. + """ + if key == "annotations": + query = "select distinct({}) as {} from streams".format( + "skeys(annotations)", "annotations" + ) + else: + query = "select distinct({}) as {} from streams".format(key, key) + params = [] + if isinstance(collection, str): + params.append("{}%".format(collection)) + query = " where ".join([query, """collection like $1"""]) + return [metadata[key] for metadata in self.query(query, params)] + + def list_unique_annotations(self, collection=None): + """ + Returns a list of annotation keys used in a given collection prefix. + + Parameters + ------- + collection: str + Prefix of the collection to filter. + + Returns + ------- + annotations: list[str] + """ + return self._list_unique_tags_annotations("annotations", collection) + + def list_unique_names(self, collection=None): + """ + Returns a list of names used in a given collection prefix. + + Parameters + ------- + collection: str + Prefix of the collection to filter. + + Returns + ------- + names: list[str] + """ + return self._list_unique_tags_annotations("name", collection) + + def list_unique_units(self, collection=None): + """ + Returns a list of units used in a given collection prefix. + + Parameters + ------- + collection: str + Prefix of the collection to filter. + + Returns + ------- + units: list[str] + """ + return self._list_unique_tags_annotations("unit", collection) + + @retry + def streams_in_collection( + self, + *collection, + is_collection_prefix=True, + tags=None, + annotations=None, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Search for streams matching given parameters @@ -310,6 +525,17 @@ def streams_in_collection(self, *collection, is_collection_prefix=True, tags=Non The tags to identify the stream. annotations: Dict[str, str] The annotations to identify the stream. + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------ @@ -329,28 +555,54 @@ def streams_in_collection(self, *collection, is_collection_prefix=True, tags=Non collection = [None] for item in collection: - streams = self.ep.lookupStreams(item, is_collection_prefix, tags, annotations) + streams = self.ep.lookupStreams( + item, is_collection_prefix, tags, annotations + ) for desclist in streams: for desc in desclist: tagsanns = unpack_stream_descriptor(desc) - result.append(Stream( - self, uuidlib.UUID(bytes = desc.uuid), - known_to_exist=True, collection=desc.collection, - tags=tagsanns[0], annotations=tagsanns[1], - property_version=desc.propertyVersion - )) + result.append( + Stream( + self, + uuidlib.UUID(bytes=desc.uuid), + known_to_exist=True, + collection=desc.collection, + tags=tagsanns[0], + annotations=tagsanns[1], + property_version=desc.propertyVersion, + ) + ) return result - def collection_metadata(self, prefix): + @retry + def collection_metadata( + self, + prefix, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Gives statistics about metadata for collections that match a prefix. Parameters ---------- - prefix: str + prefix: str, required A prefix of the collection names to look at + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- diff --git a/btrdb/endpoint.py b/btrdb/endpoint.py index de5378e..79a3b72 100644 --- a/btrdb/endpoint.py +++ b/btrdb/endpoint.py @@ -24,15 +24,24 @@ # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import io +import typing +import uuid -from btrdb.grpcinterface import btrdb_pb2 -from btrdb.grpcinterface import btrdb_pb2_grpc +from btrdb.exceptions import BTrDBError, check_proto_stat, error_handler +from btrdb.grpcinterface import btrdb_pb2, btrdb_pb2_grpc from btrdb.point import RawPoint -from btrdb.exceptions import BTrDBError, error_handler, check_proto_stat from btrdb.utils.general import unpack_stream_descriptor +try: + import pyarrow as pa +except ImportError: + pa = None + class Endpoint(object): + """Server endpoint where we make specific requests.""" + def __init__(self, channel): self.stub = btrdb_pb2_grpc.BTrDBStub(channel) @@ -45,6 +54,52 @@ def rawValues(self, uu, start, end, version=0): check_proto_stat(result.stat) yield result.values, result.versionMajor + @error_handler + def arrowRawValues(self, uu, start, end, version=0): + params = btrdb_pb2.RawValuesParams( + uuid=uu.bytes, start=start, end=end, versionMajor=version + ) + for result in self.stub.ArrowRawValues(params): + check_proto_stat(result.stat) + with pa.ipc.open_stream(result.arrowBytes) as reader: + yield reader.read_all(), result.versionMajor + + @error_handler + def arrowMultiValues(self, uu_list, start, end, version_list, snap_periodNS): + params = btrdb_pb2.ArrowMultiValuesParams( + uuid=[uu.bytes for uu in uu_list], + start=start, + end=end, + versionMajor=[ver for ver in version_list], + snapPeriodNs=int(snap_periodNS), + ) + for result in self.stub.ArrowMultiValues(params): + check_proto_stat(result.stat) + with pa.ipc.open_stream(result.arrowBytes) as reader: + yield reader.read_all() + + @error_handler + def arrowInsertValues(self, uu: uuid.UUID, values: pa.Table, policy: str): + policy_map = { + "never": btrdb_pb2.MergePolicy.NEVER, + "equal": btrdb_pb2.MergePolicy.EQUAL, + "retain": btrdb_pb2.MergePolicy.RETAIN, + "replace": btrdb_pb2.MergePolicy.REPLACE, + } + byte_io = io.BytesIO() + with pa.ipc.new_stream(sink=byte_io, schema=values.schema) as writer: + writer.write(values) + arrow_bytes = byte_io.getvalue() + params = btrdb_pb2.ArrowInsertParams( + uuid=uu.bytes, + sync=False, + arrowBytes=arrow_bytes, + merge_policy=policy_map[policy], + ) + result = self.stub.ArrowInsert(params) + check_proto_stat(result.stat) + return result.versionMajor + @error_handler def alignedWindows(self, uu, start, end, pointwidth, version=0): params = btrdb_pb2.AlignedWindowsParams( @@ -58,6 +113,20 @@ def alignedWindows(self, uu, start, end, pointwidth, version=0): check_proto_stat(result.stat) yield result.values, result.versionMajor + @error_handler + def arrowAlignedWindows(self, uu, start, end, pointwidth, version=0): + params = btrdb_pb2.AlignedWindowsParams( + uuid=uu.bytes, + start=start, + end=end, + versionMajor=version, + pointWidth=int(pointwidth), + ) + for result in self.stub.ArrowAlignedWindows(params): + check_proto_stat(result.stat) + with pa.ipc.open_stream(result.arrowBytes) as reader: + yield reader.read_all(), result.versionMajor + @error_handler def windows(self, uu, start, end, width, depth, version=0): params = btrdb_pb2.WindowsParams( @@ -72,6 +141,21 @@ def windows(self, uu, start, end, width, depth, version=0): check_proto_stat(result.stat) yield result.values, result.versionMajor + @error_handler + def arrowWindows(self, uu, start, end, width, depth, version=0): + params = btrdb_pb2.WindowsParams( + uuid=uu.bytes, + start=start, + end=end, + versionMajor=version, + width=width, + depth=depth, + ) + for result in self.stub.ArrowWindows(params): + check_proto_stat(result.stat) + with pa.ipc.open_stream(result.arrowBytes) as reader: + yield reader.read_all(), result.versionMajor + @error_handler def streamInfo(self, uu, omitDescriptor, omitVersion): params = btrdb_pb2.StreamInfoParams( @@ -81,7 +165,13 @@ def streamInfo(self, uu, omitDescriptor, omitVersion): desc = result.descriptor check_proto_stat(result.stat) tagsanns = unpack_stream_descriptor(desc) - return desc.collection, desc.propertyVersion, tagsanns[0], tagsanns[1], result.versionMajor + return ( + desc.collection, + desc.propertyVersion, + tagsanns[0], + tagsanns[1], + result.versionMajor, + ) @error_handler def obliterate(self, uu): @@ -135,11 +225,11 @@ def setStreamTags(self, uu, expected, tags, collection): def create(self, uu, collection, tags, annotations): tagkvlist = [] for k, v in tags.items(): - kv = btrdb_pb2.KeyOptValue(key = k, val = btrdb_pb2.OptValue(value=v)) + kv = btrdb_pb2.KeyOptValue(key=k, val=btrdb_pb2.OptValue(value=v)) tagkvlist.append(kv) annkvlist = [] for k, v in annotations.items(): - kv = btrdb_pb2.KeyOptValue(key = k, val = btrdb_pb2.OptValue(value=v)) + kv = btrdb_pb2.KeyOptValue(key=k, val=btrdb_pb2.OptValue(value=v)) annkvlist.append(kv) params = btrdb_pb2.CreateParams( uuid=uu.bytes, collection=collection, tags=tagkvlist, annotations=annkvlist @@ -201,7 +291,7 @@ def nearest(self, uu, time, version, backward): result = self.stub.Nearest(params) check_proto_stat(result.stat) return result.value, result.versionMajor - + @error_handler def changes(self, uu, fromVersion, toVersion, resolution): params = btrdb_pb2.ChangesParams( @@ -268,24 +358,30 @@ def getMetadataUsage(self, prefix): return result.tags, result.annotations @error_handler - def generateCSV(self, queryType, start, end, width, depth, includeVersions, *streams): - protoStreams = [btrdb_pb2.StreamCSVConfig(version = stream[0], - label = stream[1], - uuid = stream[2].bytes) - for stream in streams] - params = btrdb_pb2.GenerateCSVParams(queryType = queryType.to_proto(), - startTime = start, - endTime = end, - windowSize = width, - depth = depth, - includeVersions = includeVersions, - streams = protoStreams) + def generateCSV( + self, queryType, start, end, width, depth, includeVersions, *streams + ): + protoStreams = [ + btrdb_pb2.StreamCSVConfig( + version=stream[0], label=stream[1], uuid=stream[2].bytes + ) + for stream in streams + ] + params = btrdb_pb2.GenerateCSVParams( + queryType=queryType.to_proto(), + startTime=start, + endTime=end, + windowSize=width, + depth=depth, + includeVersions=includeVersions, + streams=protoStreams, + ) for result in self.stub.GenerateCSV(params): check_proto_stat(result.stat) yield result.row @error_handler - def sql_query(self, stmt, params=[]): + def sql_query(self, stmt, params: typing.List): request = btrdb_pb2.SQLQueryParams(query=stmt, params=params) for page in self.stub.SQLQuery(request): check_proto_stat(page.stat) diff --git a/btrdb/exceptions.py b/btrdb/exceptions.py index 2007503..9777679 100644 --- a/btrdb/exceptions.py +++ b/btrdb/exceptions.py @@ -12,13 +12,64 @@ ########################################################################## import inspect -from grpc import RpcError +import logging +import time from functools import wraps +from grpc import RpcError + +########################################################################## +## Module Variables and Constants +########################################################################## +MAX_RETRIES = 5 +INITIAL_RETRY_DELAY = 3 # time to wait before first retry in seconds +RETRY_BACKOFF_FACTOR = ( + 4 # exponential factor by which retry backoff increases between tries +) + + ########################################################################## ## Decorators ########################################################################## +# config logging for auto-retry +logging.basicConfig( + format="%(asctime)s:%(levelname)s:%(message)s", + level=logging.INFO, + datefmt="%m/%d/%Y %I:%M:%S", +) + + +def retry(fn, *args, **kwargs): + """ + decorates functions and retries it in the event of BTrDBErrors or RpcErrors + """ + + @wraps(fn) + def retry_func(*args, **kwargs): + # don't retry if arg is set to False + if not kwargs.get("auto_retry"): + return fn(*args, **kwargs) + + total_retries = kwargs.get("retries") or MAX_RETRIES + retries = total_retries + delay = kwargs.get("retry_delay") or INITIAL_RETRY_DELAY + backoff = kwargs.get("retry_backoff") or RETRY_BACKOFF_FACTOR + while retries > 0: + try: + return fn(*args, **kwargs) + except (BTrDBError, RpcError) as e: + msg = f"ERROR: {e}, attempting retry {total_retries-retries+1}/{total_retries} in {delay} seconds..." + logging.info(msg) + time.sleep(delay) + retries -= 1 + delay *= backoff + err = e + handle_grpc_error(err) + + return retry_func + + def consume_generator(fn, *args, **kwargs): # when a generator is passed back to the calling function, it may encounter an error # when trying to call next(), in that case we want to yield an Exception @@ -27,6 +78,7 @@ def consume_generator(fn, *args, **kwargs): except RpcError as e: handle_grpc_error(e) + def error_handler(fn): """ decorates endpoint functions and checks for grpc.RpcErrors @@ -35,6 +87,7 @@ def error_handler(fn): ---------- fn: function """ + # allows input func to keep its name and metadata @wraps(fn) def wrap(*args, **kwargs): @@ -44,12 +97,15 @@ def wrap(*args, **kwargs): return fn(*args, **kwargs) except RpcError as e: handle_grpc_error(e) + return wrap + ########################################################################## ## gRPC error handling ########################################################################## + # NOTE: this function relies on matching strings and isn't really ideal. # this is more of a band-aid solution while we figure out how to send # better errors from btrdb-server @@ -92,141 +148,186 @@ def check_proto_stat(stat): raise BTRDBServerError(stat.msg) raise BTrDBError(stat.msg) + ########################################################################## ## BTrDB Exceptions ########################################################################## + class BTrDBError(Exception): """ The primary exception for BTrDB errors. """ + pass + class ConnectionError(BTrDBError): """ Raised when an error occurrs while trying to establish a connection with BTrDB. """ + pass + class StreamNotFoundError(BTrDBError): """ Raised when attempting to perform an operation on a stream that does not exist in the specified BTrDB allocation. """ + pass + class CredentialsFileNotFound(FileNotFoundError, BTrDBError): """ Raised when a credentials file could not be found. """ + pass + class ProfileNotFound(BTrDBError): """ Raised when a requested profile could not be found in the credentials file. """ + pass + class BTRDBServerError(BTrDBError): """ Raised when an error occurs with btrdb-server. """ + pass + class BTRDBTypeError(TypeError, BTrDBError): """ Raised when attempting to perform an operation with an invalid type. """ + pass + class InvalidOperation(BTrDBError): """ Raised when an invalid BTrDB operation has been requested. """ + pass + class StreamExists(InvalidOperation): """ Raised when create() has been attempted and the uuid already exists. """ + pass + class AmbiguousStream(InvalidOperation): """ Raised when create() has been attempted and uuid is different, but collection and tags already exist """ + pass + class PermissionDenied(InvalidOperation): """ Raised when user does not have permission to perform an operation. """ + pass + class BTRDBValueError(ValueError, BTrDBError): """ Raised when an invalid value has been passed to a BTrDB operation. """ + pass + class InvalidCollection(BTRDBValueError): """ Raised when a collection name is invalid. It is either too long or not a valid string. """ + pass + class InvalidTagKey(BTRDBValueError): """ Raised when a tag key is invalid. Must be one of ("name", "unit", "ingress", "distiller"). """ + pass + class InvalidTagValue(BTRDBValueError): """ Raised when a tag value is invalid. It is either too long or not a valid string. """ + pass + class InvalidTimeRange(BTRDBValueError): """ Raised when insert data contains a timestamp outside the range of (btrdb.MINIMUM_TIME, btrdb.MAXIMUM_TIME) """ + pass + class InvalidPointWidth(BTRDBValueError): """ Raised when attempting to use a pointwidth that is not a whole number between 0 and 64 (exclusive). """ + pass + class BadValue(BTRDBValueError): """ Raised when attempting to insert data that contains non-float values such as None. """ + pass + class RecycledUUID(BTRDBValueError): """ Raised when attempting to create a stream with a uuid that matches a previously deleted stream. """ + pass + class BadSQLValue(BTRDBValueError): """ Raised when invalid parameters have been passed to metadata db. """ + pass + class VersionNotAvailable(BTRDBValueError): """ Raised when querying a stream at a pruned or otherwise invalid version number. """ + pass + class NoSuchPoint(BTRDBValueError): """ Raised when asking for next/previous point and there isn't one. """ + pass @@ -248,35 +349,35 @@ class NoSuchPoint(BTRDBValueError): 425: BadValue, 429: RecycledUUID, 441: BadSQLValue, - 450: VersionNotAvailable + 450: VersionNotAvailable, } # All of these raise BTRDBServerError BTRDB_SERVER_ERRORS = [ - 402, # ContextError - 403, # InsertFailure - 405, # WrongEndpoint - 414, # InsertTooBig - 421, # WrongArgs - 423, # AnnotationVersionMismatch - 424, # FaultInjectionDisabled - 426, # ResourceDepleted - 427, # InvalidVersions - 431, # ObliterateDisabled - 432, # CephError - 433, # NodeExisted - 434, # JournalError - 438, # InvalidParameter - 440, # MetadataConnectionError - 452, # OverlappingTrimRange - 453, # LockFailed - 454, # NoLeafNode - 455, # TierCacheError - 456, # StreamEmpty - 457, # TieredStorageBackendError - 458, # TieredStorageOutOfBounds - 459, # TieredStorageTemporaryError - 500, # InvariantFailure - 501, # NotImplemented - 502, # UnsupportedRollback -] \ No newline at end of file + 402, # ContextError + 403, # InsertFailure + 405, # WrongEndpoint + 414, # InsertTooBig + 421, # WrongArgs + 423, # AnnotationVersionMismatch + 424, # FaultInjectionDisabled + 426, # ResourceDepleted + 427, # InvalidVersions + 431, # ObliterateDisabled + 432, # CephError + 433, # NodeExisted + 434, # JournalError + 438, # InvalidParameter + 440, # MetadataConnectionError + 452, # OverlappingTrimRange + 453, # LockFailed + 454, # NoLeafNode + 455, # TierCacheError + 456, # StreamEmpty + 457, # TieredStorageBackendError + 458, # TieredStorageOutOfBounds + 459, # TieredStorageTemporaryError + 500, # InvariantFailure + 501, # NotImplemented + 502, # UnsupportedRollback +] diff --git a/btrdb/grpcinterface/README.md b/btrdb/grpcinterface/README.md new file mode 100644 index 0000000..24987ee --- /dev/null +++ b/btrdb/grpcinterface/README.md @@ -0,0 +1,6 @@ +To create new protobuf files in this folder: +```commandline +python -m grpc_tools.protoc --proto_path=./ --python_out=./ --pyi_out=. --grpc_python_out=./ btrdb.proto +``` + +Make sure the proto file is newest. And that you run the above command from within this folder. diff --git a/btrdb/grpcinterface/btrdb.proto b/btrdb/grpcinterface/btrdb.proto index 93fdff2..884a71c 100644 --- a/btrdb/grpcinterface/btrdb.proto +++ b/btrdb/grpcinterface/btrdb.proto @@ -4,8 +4,12 @@ package v5api; service BTrDB { rpc RawValues(RawValuesParams) returns (stream RawValuesResponse); + rpc ArrowRawValues(RawValuesParams) returns (stream ArrowRawValuesResponse); + rpc ArrowMultiValues(ArrowMultiValuesParams) returns (stream ArrowMultiValuesResponse); rpc AlignedWindows(AlignedWindowsParams) returns (stream AlignedWindowsResponse); + rpc ArrowAlignedWindows(AlignedWindowsParams) returns (stream ArrowAlignedWindowsResponse); rpc Windows(WindowsParams) returns (stream WindowsResponse); + rpc ArrowWindows(WindowsParams) returns (stream ArrowWindowsResponse); rpc StreamInfo(StreamInfoParams) returns (StreamInfoResponse); rpc SetStreamAnnotations(SetStreamAnnotationsParams) returns (SetStreamAnnotationsResponse); rpc SetStreamTags(SetStreamTagsParams) returns (SetStreamTagsResponse); @@ -15,6 +19,7 @@ service BTrDB { rpc Nearest(NearestParams) returns (NearestResponse); rpc Changes(ChangesParams) returns (stream ChangesResponse); rpc Insert(InsertParams) returns (InsertResponse); + rpc ArrowInsert(ArrowInsertParams) returns (InsertResponse); rpc Delete(DeleteParams) returns (DeleteResponse); rpc Info(InfoParams) returns (InfoResponse); rpc FaultInject(FaultInjectParams) returns (FaultInjectResponse); @@ -39,6 +44,27 @@ message RawValuesResponse { uint64 versionMinor = 3; repeated RawPoint values = 4; } +message ArrowRawValuesResponse { + Status stat = 1; + uint64 versionMajor = 2; + uint64 versionMinor = 3; + bytes arrowBytes = 4; +} +message ArrowMultiValuesParams { + repeated bytes uuid = 1; + repeated uint64 versionMajor = 2; + sfixed64 start = 3; + sfixed64 end = 4; + int64 snapPeriodNs = 5; +} +message ArrowMultiValuesResponse { + Status stat = 1; + bytes arrowBytes = 2; +} +message RawPointVec { + sfixed64 time = 1; + repeated double value = 2; +}; message AlignedWindowsParams { bytes uuid = 1; sfixed64 start = 2; @@ -52,6 +78,12 @@ message AlignedWindowsResponse { uint64 versionMinor = 3; repeated StatPoint values = 4; } +message ArrowAlignedWindowsResponse { + Status stat = 1; + uint64 versionMajor = 2; + uint64 versionMinor = 3; + bytes arrowBytes = 4; +} message WindowsParams { bytes uuid = 1; sfixed64 start = 2; @@ -66,6 +98,12 @@ message WindowsResponse { uint64 versionMinor = 3; repeated StatPoint values = 4; } +message ArrowWindowsResponse { + Status stat = 1; + uint64 versionMajor = 2; + uint64 versionMinor = 3; + bytes arrowBytes = 4; +} message StreamInfoParams { bytes uuid = 1; bool omitVersion = 2; @@ -175,12 +213,25 @@ enum MergePolicy { RETAIN = 2; // When timestamps are equal, keep old value REPLACE = 3; // When timestamps are equal, keep new value } +message RoundSpec { + oneof spec { + int32 bits = 2; + } +} message InsertParams { bytes uuid = 1; bool sync = 2; MergePolicy merge_policy = 4; + RoundSpec rounding=5; repeated RawPoint values = 3; } +message ArrowInsertParams { + bytes uuid = 1; + bool sync = 2; + MergePolicy merge_policy = 3; + RoundSpec rounding = 4; + bytes arrowBytes = 5; +} message InsertResponse { Status stat = 1; uint64 versionMajor = 2; diff --git a/btrdb/grpcinterface/btrdb_pb2.py b/btrdb/grpcinterface/btrdb_pb2.py index 8104a34..bed2b0d 100644 --- a/btrdb/grpcinterface/btrdb_pb2.py +++ b/btrdb/grpcinterface/btrdb_pb2.py @@ -1,12 +1,10 @@ +# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: btrdb.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf.internal import enum_type_wrapper +"""Generated protocol buffer code.""" +from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection +from google.protobuf import descriptor_pool as _descriptor_pool from google.protobuf import symbol_database as _symbol_database # @@protoc_insertion_point(imports) @@ -15,3542 +13,152 @@ -DESCRIPTOR = _descriptor.FileDescriptor( - name='btrdb.proto', - package='v5api', - syntax='proto3', - serialized_options=None, - serialized_pb=_b('\n\x0b\x62trdb.proto\x12\x05v5api\"Q\n\x0fRawValuesParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\"}\n\x11RawValuesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x1f\n\x06values\x18\x04 \x03(\x0b\x32\x0f.v5api.RawPoint\"j\n\x14\x41lignedWindowsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\x12\x12\n\npointWidth\x18\x05 \x01(\r\"\x83\x01\n\x16\x41lignedWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12 \n\x06values\x18\x04 \x03(\x0b\x32\x10.v5api.StatPoint\"m\n\rWindowsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\x12\r\n\x05width\x18\x05 \x01(\x04\x12\r\n\x05\x64\x65pth\x18\x06 \x01(\r\"|\n\x0fWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12 \n\x06values\x18\x04 \x03(\x0b\x32\x10.v5api.StatPoint\"h\n\x10StreamInfoParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x13\n\x0bomitVersion\x18\x02 \x01(\x08\x12\x16\n\x0eomitDescriptor\x18\x03 \x01(\x08\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"\x8a\x01\n\x12StreamInfoResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12+\n\ndescriptor\x18\x04 \x01(\x0b\x32\x17.v5api.StreamDescriptor\"\x98\x01\n\x10StreamDescriptor\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x12\n\ncollection\x18\x02 \x01(\t\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x17\n\x0fpropertyVersion\x18\x05 \x01(\x04\"\x82\x01\n\x1aSetStreamAnnotationsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x1f\n\x17\x65xpectedPropertyVersion\x18\x02 \x01(\x04\x12#\n\x07\x63hanges\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x10\n\x08removals\x18\x04 \x03(\t\";\n\x1cSetStreamAnnotationsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"\x8a\x01\n\x13SetStreamTagsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x1f\n\x17\x65xpectedPropertyVersion\x18\x02 \x01(\x04\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x12\n\ncollection\x18\x04 \x01(\t\x12\x0e\n\x06remove\x18\x05 \x03(\t\"4\n\x15SetStreamTagsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"{\n\x0c\x43reateParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x12\n\ncollection\x18\x02 \x01(\t\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\"-\n\x0e\x43reateResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"@\n\x13MetadataUsageParams\x12\x0e\n\x06prefix\x18\x01 \x01(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"y\n\x15MetadataUsageResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x1d\n\x04tags\x18\x02 \x03(\x0b\x32\x0f.v5api.KeyCount\x12$\n\x0b\x61nnotations\x18\x03 \x03(\x0b\x32\x0f.v5api.KeyCount\"&\n\x08KeyCount\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05\x63ount\x18\x02 \x01(\x04\"B\n\x15ListCollectionsParams\x12\x0e\n\x06prefix\x18\x01 \x01(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"K\n\x17ListCollectionsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x13\n\x0b\x63ollections\x18\x02 \x03(\t\"\xab\x01\n\x13LookupStreamsParams\x12\x12\n\ncollection\x18\x01 \x01(\t\x12\x1a\n\x12isCollectionPrefix\x18\x02 \x01(\x08\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"^\n\x15LookupStreamsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12(\n\x07results\x18\x02 \x03(\x0b\x32\x17.v5api.StreamDescriptor\"S\n\rNearestParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x0c\n\x04time\x18\x02 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x03 \x01(\x04\x12\x10\n\x08\x62\x61\x63kward\x18\x04 \x01(\x08\"z\n\x0fNearestResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x1e\n\x05value\x18\x04 \x01(\x0b\x32\x0f.v5api.RawPoint\"U\n\rChangesParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x11\n\tfromMajor\x18\x02 \x01(\x04\x12\x0f\n\x07toMajor\x18\x03 \x01(\x04\x12\x12\n\nresolution\x18\x04 \x01(\r\"\x7f\n\x0f\x43hangesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12#\n\x06ranges\x18\x04 \x03(\x0b\x32\x13.v5api.ChangedRange\"u\n\x0cInsertParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x0c\n\x04sync\x18\x02 \x01(\x08\x12(\n\x0cmerge_policy\x18\x04 \x01(\x0e\x32\x12.v5api.MergePolicy\x12\x1f\n\x06values\x18\x03 \x03(\x0b\x32\x0f.v5api.RawPoint\"Y\n\x0eInsertResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\"8\n\x0c\x44\x65leteParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\"Y\n\x0e\x44\x65leteResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\"\x0c\n\nInfoParams\"\xa2\x01\n\x0cInfoResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x19\n\x04mash\x18\x02 \x01(\x0b\x32\x0b.v5api.Mash\x12\x14\n\x0cmajorVersion\x18\x03 \x01(\r\x12\x14\n\x0cminorVersion\x18\x04 \x01(\r\x12\r\n\x05\x62uild\x18\x05 \x01(\t\x12\x1f\n\x05proxy\x18\x06 \x01(\x0b\x32\x10.v5api.ProxyInfo\"#\n\tProxyInfo\x12\x16\n\x0eproxyEndpoints\x18\x01 \x03(\t\"1\n\x11\x46\x61ultInjectParams\x12\x0c\n\x04type\x18\x01 \x01(\x04\x12\x0e\n\x06params\x18\x02 \x01(\x0c\">\n\x13\x46\x61ultInjectResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\n\n\x02rv\x18\x02 \x01(\x0c\"\x1b\n\x0b\x46lushParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"X\n\rFlushResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\" \n\x10ObliterateParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"1\n\x12ObliterateResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"\'\n\x08RawPoint\x12\x0c\n\x04time\x18\x01 \x01(\x10\x12\r\n\x05value\x18\x02 \x01(\x01\"`\n\tStatPoint\x12\x0c\n\x04time\x18\x01 \x01(\x10\x12\x0b\n\x03min\x18\x02 \x01(\x01\x12\x0c\n\x04mean\x18\x03 \x01(\x01\x12\x0b\n\x03max\x18\x04 \x01(\x01\x12\r\n\x05\x63ount\x18\x05 \x01(\x06\x12\x0e\n\x06stddev\x18\x06 \x01(\x01\"*\n\x0c\x43hangedRange\x12\r\n\x05start\x18\x01 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x02 \x01(\x10\">\n\x06Status\x12\x0c\n\x04\x63ode\x18\x01 \x01(\r\x12\x0b\n\x03msg\x18\x02 \x01(\t\x12\x19\n\x04mash\x18\x03 \x01(\x0b\x32\x0b.v5api.Mash\"\x98\x01\n\x04Mash\x12\x10\n\x08revision\x18\x01 \x01(\x03\x12\x0e\n\x06leader\x18\x02 \x01(\t\x12\x16\n\x0eleaderRevision\x18\x03 \x01(\x03\x12\x13\n\x0btotalWeight\x18\x04 \x01(\x03\x12\x0f\n\x07healthy\x18\x05 \x01(\x08\x12\x10\n\x08unmapped\x18\x06 \x01(\x01\x12\x1e\n\x07members\x18\x07 \x03(\x0b\x32\r.v5api.Member\"\xc3\x01\n\x06Member\x12\x0c\n\x04hash\x18\x01 \x01(\r\x12\x10\n\x08nodename\x18\x02 \x01(\t\x12\n\n\x02up\x18\x03 \x01(\x08\x12\n\n\x02in\x18\x04 \x01(\x08\x12\x0f\n\x07\x65nabled\x18\x05 \x01(\x08\x12\r\n\x05start\x18\x06 \x01(\x03\x12\x0b\n\x03\x65nd\x18\x07 \x01(\x03\x12\x0e\n\x06weight\x18\x08 \x01(\x03\x12\x16\n\x0ereadPreference\x18\t \x01(\x01\x12\x15\n\rhttpEndpoints\x18\n \x01(\t\x12\x15\n\rgrpcEndpoints\x18\x0b \x01(\t\"8\n\x0bKeyOptValue\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x1c\n\x03val\x18\x02 \x01(\x0b\x32\x0f.v5api.OptValue\"\x19\n\x08OptValue\x12\r\n\x05value\x18\x01 \x01(\t\"&\n\x08KeyValue\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"?\n\x0fStreamCSVConfig\x12\x0f\n\x07version\x18\x01 \x01(\x04\x12\r\n\x05label\x18\x02 \x01(\t\x12\x0c\n\x04uuid\x18\x03 \x01(\x0c\"\x9d\x02\n\x11GenerateCSVParams\x12\x35\n\tqueryType\x18\x01 \x01(\x0e\x32\".v5api.GenerateCSVParams.QueryType\x12\x11\n\tstartTime\x18\x02 \x01(\x03\x12\x0f\n\x07\x65ndTime\x18\x03 \x01(\x03\x12\x12\n\nwindowSize\x18\x04 \x01(\x04\x12\r\n\x05\x64\x65pth\x18\x05 \x01(\r\x12\x17\n\x0fincludeVersions\x18\x06 \x01(\x08\x12\'\n\x07streams\x18\x07 \x03(\x0b\x32\x16.v5api.StreamCSVConfig\"H\n\tQueryType\x12\x19\n\x15\x41LIGNED_WINDOWS_QUERY\x10\x00\x12\x11\n\rWINDOWS_QUERY\x10\x01\x12\r\n\tRAW_QUERY\x10\x02\"Q\n\x13GenerateCSVResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x10\n\x08isHeader\x18\x02 \x01(\x08\x12\x0b\n\x03row\x18\x03 \x03(\t\"J\n\x0eSQLQueryParams\x12\r\n\x05query\x18\x01 \x01(\t\x12\x0e\n\x06params\x18\x02 \x03(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"D\n\x10SQLQueryResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x13\n\x0bSQLQueryRow\x18\x02 \x03(\x0c\"\x14\n\x04Role\x12\x0c\n\x04name\x18\x01 \x01(\t\"\x94\x01\n\x19SetCompactionConfigParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x18\n\x10\x43ompactedVersion\x18\x02 \x01(\x04\x12>\n\x17reducedResolutionRanges\x18\x03 \x03(\x0b\x32\x1d.v5api.ReducedResolutionRange\x12\x0f\n\x07unused0\x18\x04 \x01(\x04\":\n\x1bSetCompactionConfigResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\")\n\x19GetCompactionConfigParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"\xc1\x01\n\x1bGetCompactionConfigResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x1a\n\x12LatestMajorVersion\x18\x02 \x01(\x04\x12\x18\n\x10\x43ompactedVersion\x18\x03 \x01(\x04\x12>\n\x17reducedResolutionRanges\x18\x04 \x03(\x0b\x32\x1d.v5api.ReducedResolutionRange\x12\x0f\n\x07unused0\x18\x05 \x01(\x04\"H\n\x16ReducedResolutionRange\x12\r\n\x05Start\x18\x01 \x01(\x03\x12\x0b\n\x03\x45nd\x18\x02 \x01(\x03\x12\x12\n\nResolution\x18\x03 \x01(\r*<\n\x0bMergePolicy\x12\t\n\x05NEVER\x10\x00\x12\t\n\x05\x45QUAL\x10\x01\x12\n\n\x06RETAIN\x10\x02\x12\x0b\n\x07REPLACE\x10\x03\x32\xb3\n\n\x05\x42TrDB\x12?\n\tRawValues\x12\x16.v5api.RawValuesParams\x1a\x18.v5api.RawValuesResponse0\x01\x12N\n\x0e\x41lignedWindows\x12\x1b.v5api.AlignedWindowsParams\x1a\x1d.v5api.AlignedWindowsResponse0\x01\x12\x39\n\x07Windows\x12\x14.v5api.WindowsParams\x1a\x16.v5api.WindowsResponse0\x01\x12@\n\nStreamInfo\x12\x17.v5api.StreamInfoParams\x1a\x19.v5api.StreamInfoResponse\x12^\n\x14SetStreamAnnotations\x12!.v5api.SetStreamAnnotationsParams\x1a#.v5api.SetStreamAnnotationsResponse\x12I\n\rSetStreamTags\x12\x1a.v5api.SetStreamTagsParams\x1a\x1c.v5api.SetStreamTagsResponse\x12\x34\n\x06\x43reate\x12\x13.v5api.CreateParams\x1a\x15.v5api.CreateResponse\x12Q\n\x0fListCollections\x12\x1c.v5api.ListCollectionsParams\x1a\x1e.v5api.ListCollectionsResponse0\x01\x12K\n\rLookupStreams\x12\x1a.v5api.LookupStreamsParams\x1a\x1c.v5api.LookupStreamsResponse0\x01\x12\x37\n\x07Nearest\x12\x14.v5api.NearestParams\x1a\x16.v5api.NearestResponse\x12\x39\n\x07\x43hanges\x12\x14.v5api.ChangesParams\x1a\x16.v5api.ChangesResponse0\x01\x12\x34\n\x06Insert\x12\x13.v5api.InsertParams\x1a\x15.v5api.InsertResponse\x12\x34\n\x06\x44\x65lete\x12\x13.v5api.DeleteParams\x1a\x15.v5api.DeleteResponse\x12.\n\x04Info\x12\x11.v5api.InfoParams\x1a\x13.v5api.InfoResponse\x12\x43\n\x0b\x46\x61ultInject\x12\x18.v5api.FaultInjectParams\x1a\x1a.v5api.FaultInjectResponse\x12\x31\n\x05\x46lush\x12\x12.v5api.FlushParams\x1a\x14.v5api.FlushResponse\x12@\n\nObliterate\x12\x17.v5api.ObliterateParams\x1a\x19.v5api.ObliterateResponse\x12L\n\x10GetMetadataUsage\x12\x1a.v5api.MetadataUsageParams\x1a\x1c.v5api.MetadataUsageResponse\x12\x45\n\x0bGenerateCSV\x12\x18.v5api.GenerateCSVParams\x1a\x1a.v5api.GenerateCSVResponse0\x01\x12<\n\x08SQLQuery\x12\x15.v5api.SQLQueryParams\x1a\x17.v5api.SQLQueryResponse0\x01\x62\x06proto3') -) - -_MERGEPOLICY = _descriptor.EnumDescriptor( - name='MergePolicy', - full_name='v5api.MergePolicy', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='NEVER', index=0, number=0, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='EQUAL', index=1, number=1, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='RETAIN', index=2, number=2, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='REPLACE', index=3, number=3, - serialized_options=None, - type=None), - ], - containing_type=None, - serialized_options=None, - serialized_start=5490, - serialized_end=5550, -) -_sym_db.RegisterEnumDescriptor(_MERGEPOLICY) - -MergePolicy = enum_type_wrapper.EnumTypeWrapper(_MERGEPOLICY) -NEVER = 0 -EQUAL = 1 -RETAIN = 2 -REPLACE = 3 - - -_GENERATECSVPARAMS_QUERYTYPE = _descriptor.EnumDescriptor( - name='QueryType', - full_name='v5api.GenerateCSVParams.QueryType', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='ALIGNED_WINDOWS_QUERY', index=0, number=0, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='WINDOWS_QUERY', index=1, number=1, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='RAW_QUERY', index=2, number=2, - serialized_options=None, - type=None), - ], - containing_type=None, - serialized_options=None, - serialized_start=4641, - serialized_end=4713, -) -_sym_db.RegisterEnumDescriptor(_GENERATECSVPARAMS_QUERYTYPE) - - -_RAWVALUESPARAMS = _descriptor.Descriptor( - name='RawValuesParams', - full_name='v5api.RawValuesParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.RawValuesParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='start', full_name='v5api.RawValuesParams.start', index=1, - number=2, type=16, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='end', full_name='v5api.RawValuesParams.end', index=2, - number=3, type=16, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMajor', full_name='v5api.RawValuesParams.versionMajor', index=3, - number=4, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=22, - serialized_end=103, -) - - -_RAWVALUESRESPONSE = _descriptor.Descriptor( - name='RawValuesResponse', - full_name='v5api.RawValuesResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.RawValuesResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMajor', full_name='v5api.RawValuesResponse.versionMajor', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMinor', full_name='v5api.RawValuesResponse.versionMinor', index=2, - number=3, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='values', full_name='v5api.RawValuesResponse.values', index=3, - number=4, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=105, - serialized_end=230, -) - - -_ALIGNEDWINDOWSPARAMS = _descriptor.Descriptor( - name='AlignedWindowsParams', - full_name='v5api.AlignedWindowsParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.AlignedWindowsParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='start', full_name='v5api.AlignedWindowsParams.start', index=1, - number=2, type=16, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='end', full_name='v5api.AlignedWindowsParams.end', index=2, - number=3, type=16, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMajor', full_name='v5api.AlignedWindowsParams.versionMajor', index=3, - number=4, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='pointWidth', full_name='v5api.AlignedWindowsParams.pointWidth', index=4, - number=5, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=232, - serialized_end=338, -) - - -_ALIGNEDWINDOWSRESPONSE = _descriptor.Descriptor( - name='AlignedWindowsResponse', - full_name='v5api.AlignedWindowsResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.AlignedWindowsResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMajor', full_name='v5api.AlignedWindowsResponse.versionMajor', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMinor', full_name='v5api.AlignedWindowsResponse.versionMinor', index=2, - number=3, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='values', full_name='v5api.AlignedWindowsResponse.values', index=3, - number=4, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=341, - serialized_end=472, -) - - -_WINDOWSPARAMS = _descriptor.Descriptor( - name='WindowsParams', - full_name='v5api.WindowsParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.WindowsParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='start', full_name='v5api.WindowsParams.start', index=1, - number=2, type=16, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='end', full_name='v5api.WindowsParams.end', index=2, - number=3, type=16, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMajor', full_name='v5api.WindowsParams.versionMajor', index=3, - number=4, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='width', full_name='v5api.WindowsParams.width', index=4, - number=5, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='depth', full_name='v5api.WindowsParams.depth', index=5, - number=6, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=474, - serialized_end=583, -) - - -_WINDOWSRESPONSE = _descriptor.Descriptor( - name='WindowsResponse', - full_name='v5api.WindowsResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.WindowsResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMajor', full_name='v5api.WindowsResponse.versionMajor', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMinor', full_name='v5api.WindowsResponse.versionMinor', index=2, - number=3, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='values', full_name='v5api.WindowsResponse.values', index=3, - number=4, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=585, - serialized_end=709, -) - - -_STREAMINFOPARAMS = _descriptor.Descriptor( - name='StreamInfoParams', - full_name='v5api.StreamInfoParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.StreamInfoParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='omitVersion', full_name='v5api.StreamInfoParams.omitVersion', index=1, - number=2, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='omitDescriptor', full_name='v5api.StreamInfoParams.omitDescriptor', index=2, - number=3, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='role', full_name='v5api.StreamInfoParams.role', index=3, - number=100, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=711, - serialized_end=815, -) - - -_STREAMINFORESPONSE = _descriptor.Descriptor( - name='StreamInfoResponse', - full_name='v5api.StreamInfoResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.StreamInfoResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMajor', full_name='v5api.StreamInfoResponse.versionMajor', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMinor', full_name='v5api.StreamInfoResponse.versionMinor', index=2, - number=3, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='descriptor', full_name='v5api.StreamInfoResponse.descriptor', index=3, - number=4, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=818, - serialized_end=956, -) - - -_STREAMDESCRIPTOR = _descriptor.Descriptor( - name='StreamDescriptor', - full_name='v5api.StreamDescriptor', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.StreamDescriptor.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='collection', full_name='v5api.StreamDescriptor.collection', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='tags', full_name='v5api.StreamDescriptor.tags', index=2, - number=3, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='annotations', full_name='v5api.StreamDescriptor.annotations', index=3, - number=4, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='propertyVersion', full_name='v5api.StreamDescriptor.propertyVersion', index=4, - number=5, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=959, - serialized_end=1111, -) - - -_SETSTREAMANNOTATIONSPARAMS = _descriptor.Descriptor( - name='SetStreamAnnotationsParams', - full_name='v5api.SetStreamAnnotationsParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.SetStreamAnnotationsParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='expectedPropertyVersion', full_name='v5api.SetStreamAnnotationsParams.expectedPropertyVersion', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='changes', full_name='v5api.SetStreamAnnotationsParams.changes', index=2, - number=3, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='removals', full_name='v5api.SetStreamAnnotationsParams.removals', index=3, - number=4, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1114, - serialized_end=1244, -) - - -_SETSTREAMANNOTATIONSRESPONSE = _descriptor.Descriptor( - name='SetStreamAnnotationsResponse', - full_name='v5api.SetStreamAnnotationsResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.SetStreamAnnotationsResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1246, - serialized_end=1305, -) - - -_SETSTREAMTAGSPARAMS = _descriptor.Descriptor( - name='SetStreamTagsParams', - full_name='v5api.SetStreamTagsParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.SetStreamTagsParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='expectedPropertyVersion', full_name='v5api.SetStreamTagsParams.expectedPropertyVersion', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='tags', full_name='v5api.SetStreamTagsParams.tags', index=2, - number=3, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='collection', full_name='v5api.SetStreamTagsParams.collection', index=3, - number=4, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='remove', full_name='v5api.SetStreamTagsParams.remove', index=4, - number=5, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1308, - serialized_end=1446, -) - - -_SETSTREAMTAGSRESPONSE = _descriptor.Descriptor( - name='SetStreamTagsResponse', - full_name='v5api.SetStreamTagsResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.SetStreamTagsResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1448, - serialized_end=1500, -) - - -_CREATEPARAMS = _descriptor.Descriptor( - name='CreateParams', - full_name='v5api.CreateParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.CreateParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='collection', full_name='v5api.CreateParams.collection', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='tags', full_name='v5api.CreateParams.tags', index=2, - number=3, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='annotations', full_name='v5api.CreateParams.annotations', index=3, - number=4, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1502, - serialized_end=1625, -) - - -_CREATERESPONSE = _descriptor.Descriptor( - name='CreateResponse', - full_name='v5api.CreateResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.CreateResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1627, - serialized_end=1672, -) - - -_METADATAUSAGEPARAMS = _descriptor.Descriptor( - name='MetadataUsageParams', - full_name='v5api.MetadataUsageParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='prefix', full_name='v5api.MetadataUsageParams.prefix', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='role', full_name='v5api.MetadataUsageParams.role', index=1, - number=100, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1674, - serialized_end=1738, -) - - -_METADATAUSAGERESPONSE = _descriptor.Descriptor( - name='MetadataUsageResponse', - full_name='v5api.MetadataUsageResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.MetadataUsageResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='tags', full_name='v5api.MetadataUsageResponse.tags', index=1, - number=2, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='annotations', full_name='v5api.MetadataUsageResponse.annotations', index=2, - number=3, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1740, - serialized_end=1861, -) - - -_KEYCOUNT = _descriptor.Descriptor( - name='KeyCount', - full_name='v5api.KeyCount', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='key', full_name='v5api.KeyCount.key', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='count', full_name='v5api.KeyCount.count', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1863, - serialized_end=1901, -) - - -_LISTCOLLECTIONSPARAMS = _descriptor.Descriptor( - name='ListCollectionsParams', - full_name='v5api.ListCollectionsParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='prefix', full_name='v5api.ListCollectionsParams.prefix', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='role', full_name='v5api.ListCollectionsParams.role', index=1, - number=100, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1903, - serialized_end=1969, -) - - -_LISTCOLLECTIONSRESPONSE = _descriptor.Descriptor( - name='ListCollectionsResponse', - full_name='v5api.ListCollectionsResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.ListCollectionsResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='collections', full_name='v5api.ListCollectionsResponse.collections', index=1, - number=2, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1971, - serialized_end=2046, -) - - -_LOOKUPSTREAMSPARAMS = _descriptor.Descriptor( - name='LookupStreamsParams', - full_name='v5api.LookupStreamsParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='collection', full_name='v5api.LookupStreamsParams.collection', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='isCollectionPrefix', full_name='v5api.LookupStreamsParams.isCollectionPrefix', index=1, - number=2, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='tags', full_name='v5api.LookupStreamsParams.tags', index=2, - number=3, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='annotations', full_name='v5api.LookupStreamsParams.annotations', index=3, - number=4, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='role', full_name='v5api.LookupStreamsParams.role', index=4, - number=100, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=2049, - serialized_end=2220, -) - - -_LOOKUPSTREAMSRESPONSE = _descriptor.Descriptor( - name='LookupStreamsResponse', - full_name='v5api.LookupStreamsResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.LookupStreamsResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='results', full_name='v5api.LookupStreamsResponse.results', index=1, - number=2, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=2222, - serialized_end=2316, -) - - -_NEARESTPARAMS = _descriptor.Descriptor( - name='NearestParams', - full_name='v5api.NearestParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.NearestParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='time', full_name='v5api.NearestParams.time', index=1, - number=2, type=16, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMajor', full_name='v5api.NearestParams.versionMajor', index=2, - number=3, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='backward', full_name='v5api.NearestParams.backward', index=3, - number=4, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=2318, - serialized_end=2401, -) - - -_NEARESTRESPONSE = _descriptor.Descriptor( - name='NearestResponse', - full_name='v5api.NearestResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.NearestResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMajor', full_name='v5api.NearestResponse.versionMajor', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMinor', full_name='v5api.NearestResponse.versionMinor', index=2, - number=3, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='value', full_name='v5api.NearestResponse.value', index=3, - number=4, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=2403, - serialized_end=2525, -) - - -_CHANGESPARAMS = _descriptor.Descriptor( - name='ChangesParams', - full_name='v5api.ChangesParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.ChangesParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='fromMajor', full_name='v5api.ChangesParams.fromMajor', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='toMajor', full_name='v5api.ChangesParams.toMajor', index=2, - number=3, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='resolution', full_name='v5api.ChangesParams.resolution', index=3, - number=4, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=2527, - serialized_end=2612, -) - - -_CHANGESRESPONSE = _descriptor.Descriptor( - name='ChangesResponse', - full_name='v5api.ChangesResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.ChangesResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMajor', full_name='v5api.ChangesResponse.versionMajor', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMinor', full_name='v5api.ChangesResponse.versionMinor', index=2, - number=3, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='ranges', full_name='v5api.ChangesResponse.ranges', index=3, - number=4, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=2614, - serialized_end=2741, -) - - -_INSERTPARAMS = _descriptor.Descriptor( - name='InsertParams', - full_name='v5api.InsertParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.InsertParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='sync', full_name='v5api.InsertParams.sync', index=1, - number=2, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='merge_policy', full_name='v5api.InsertParams.merge_policy', index=2, - number=4, type=14, cpp_type=8, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='values', full_name='v5api.InsertParams.values', index=3, - number=3, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=2743, - serialized_end=2860, -) - - -_INSERTRESPONSE = _descriptor.Descriptor( - name='InsertResponse', - full_name='v5api.InsertResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.InsertResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMajor', full_name='v5api.InsertResponse.versionMajor', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMinor', full_name='v5api.InsertResponse.versionMinor', index=2, - number=3, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=2862, - serialized_end=2951, -) - - -_DELETEPARAMS = _descriptor.Descriptor( - name='DeleteParams', - full_name='v5api.DeleteParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.DeleteParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='start', full_name='v5api.DeleteParams.start', index=1, - number=2, type=16, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='end', full_name='v5api.DeleteParams.end', index=2, - number=3, type=16, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=2953, - serialized_end=3009, -) - - -_DELETERESPONSE = _descriptor.Descriptor( - name='DeleteResponse', - full_name='v5api.DeleteResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.DeleteResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMajor', full_name='v5api.DeleteResponse.versionMajor', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMinor', full_name='v5api.DeleteResponse.versionMinor', index=2, - number=3, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3011, - serialized_end=3100, -) - - -_INFOPARAMS = _descriptor.Descriptor( - name='InfoParams', - full_name='v5api.InfoParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3102, - serialized_end=3114, -) - - -_INFORESPONSE = _descriptor.Descriptor( - name='InfoResponse', - full_name='v5api.InfoResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.InfoResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='mash', full_name='v5api.InfoResponse.mash', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='majorVersion', full_name='v5api.InfoResponse.majorVersion', index=2, - number=3, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='minorVersion', full_name='v5api.InfoResponse.minorVersion', index=3, - number=4, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='build', full_name='v5api.InfoResponse.build', index=4, - number=5, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='proxy', full_name='v5api.InfoResponse.proxy', index=5, - number=6, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3117, - serialized_end=3279, -) - - -_PROXYINFO = _descriptor.Descriptor( - name='ProxyInfo', - full_name='v5api.ProxyInfo', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='proxyEndpoints', full_name='v5api.ProxyInfo.proxyEndpoints', index=0, - number=1, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3281, - serialized_end=3316, -) - - -_FAULTINJECTPARAMS = _descriptor.Descriptor( - name='FaultInjectParams', - full_name='v5api.FaultInjectParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='type', full_name='v5api.FaultInjectParams.type', index=0, - number=1, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='params', full_name='v5api.FaultInjectParams.params', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3318, - serialized_end=3367, -) - - -_FAULTINJECTRESPONSE = _descriptor.Descriptor( - name='FaultInjectResponse', - full_name='v5api.FaultInjectResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.FaultInjectResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='rv', full_name='v5api.FaultInjectResponse.rv', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3369, - serialized_end=3431, -) - - -_FLUSHPARAMS = _descriptor.Descriptor( - name='FlushParams', - full_name='v5api.FlushParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.FlushParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3433, - serialized_end=3460, -) - - -_FLUSHRESPONSE = _descriptor.Descriptor( - name='FlushResponse', - full_name='v5api.FlushResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.FlushResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMajor', full_name='v5api.FlushResponse.versionMajor', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='versionMinor', full_name='v5api.FlushResponse.versionMinor', index=2, - number=3, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3462, - serialized_end=3550, -) - - -_OBLITERATEPARAMS = _descriptor.Descriptor( - name='ObliterateParams', - full_name='v5api.ObliterateParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.ObliterateParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3552, - serialized_end=3584, -) - - -_OBLITERATERESPONSE = _descriptor.Descriptor( - name='ObliterateResponse', - full_name='v5api.ObliterateResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.ObliterateResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3586, - serialized_end=3635, -) - - -_RAWPOINT = _descriptor.Descriptor( - name='RawPoint', - full_name='v5api.RawPoint', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='time', full_name='v5api.RawPoint.time', index=0, - number=1, type=16, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='value', full_name='v5api.RawPoint.value', index=1, - number=2, type=1, cpp_type=5, label=1, - has_default_value=False, default_value=float(0), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3637, - serialized_end=3676, -) - - -_STATPOINT = _descriptor.Descriptor( - name='StatPoint', - full_name='v5api.StatPoint', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='time', full_name='v5api.StatPoint.time', index=0, - number=1, type=16, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='min', full_name='v5api.StatPoint.min', index=1, - number=2, type=1, cpp_type=5, label=1, - has_default_value=False, default_value=float(0), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='mean', full_name='v5api.StatPoint.mean', index=2, - number=3, type=1, cpp_type=5, label=1, - has_default_value=False, default_value=float(0), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='max', full_name='v5api.StatPoint.max', index=3, - number=4, type=1, cpp_type=5, label=1, - has_default_value=False, default_value=float(0), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='count', full_name='v5api.StatPoint.count', index=4, - number=5, type=6, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='stddev', full_name='v5api.StatPoint.stddev', index=5, - number=6, type=1, cpp_type=5, label=1, - has_default_value=False, default_value=float(0), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3678, - serialized_end=3774, -) - - -_CHANGEDRANGE = _descriptor.Descriptor( - name='ChangedRange', - full_name='v5api.ChangedRange', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='start', full_name='v5api.ChangedRange.start', index=0, - number=1, type=16, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='end', full_name='v5api.ChangedRange.end', index=1, - number=2, type=16, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3776, - serialized_end=3818, -) - - -_STATUS = _descriptor.Descriptor( - name='Status', - full_name='v5api.Status', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='code', full_name='v5api.Status.code', index=0, - number=1, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='msg', full_name='v5api.Status.msg', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='mash', full_name='v5api.Status.mash', index=2, - number=3, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3820, - serialized_end=3882, -) - - -_MASH = _descriptor.Descriptor( - name='Mash', - full_name='v5api.Mash', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='revision', full_name='v5api.Mash.revision', index=0, - number=1, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='leader', full_name='v5api.Mash.leader', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='leaderRevision', full_name='v5api.Mash.leaderRevision', index=2, - number=3, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='totalWeight', full_name='v5api.Mash.totalWeight', index=3, - number=4, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='healthy', full_name='v5api.Mash.healthy', index=4, - number=5, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='unmapped', full_name='v5api.Mash.unmapped', index=5, - number=6, type=1, cpp_type=5, label=1, - has_default_value=False, default_value=float(0), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='members', full_name='v5api.Mash.members', index=6, - number=7, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=3885, - serialized_end=4037, -) - - -_MEMBER = _descriptor.Descriptor( - name='Member', - full_name='v5api.Member', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='hash', full_name='v5api.Member.hash', index=0, - number=1, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='nodename', full_name='v5api.Member.nodename', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='up', full_name='v5api.Member.up', index=2, - number=3, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='in', full_name='v5api.Member.in', index=3, - number=4, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='enabled', full_name='v5api.Member.enabled', index=4, - number=5, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='start', full_name='v5api.Member.start', index=5, - number=6, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='end', full_name='v5api.Member.end', index=6, - number=7, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='weight', full_name='v5api.Member.weight', index=7, - number=8, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='readPreference', full_name='v5api.Member.readPreference', index=8, - number=9, type=1, cpp_type=5, label=1, - has_default_value=False, default_value=float(0), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='httpEndpoints', full_name='v5api.Member.httpEndpoints', index=9, - number=10, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='grpcEndpoints', full_name='v5api.Member.grpcEndpoints', index=10, - number=11, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=4040, - serialized_end=4235, -) - - -_KEYOPTVALUE = _descriptor.Descriptor( - name='KeyOptValue', - full_name='v5api.KeyOptValue', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='key', full_name='v5api.KeyOptValue.key', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='val', full_name='v5api.KeyOptValue.val', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=4237, - serialized_end=4293, -) - - -_OPTVALUE = _descriptor.Descriptor( - name='OptValue', - full_name='v5api.OptValue', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='value', full_name='v5api.OptValue.value', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=4295, - serialized_end=4320, -) - - -_KEYVALUE = _descriptor.Descriptor( - name='KeyValue', - full_name='v5api.KeyValue', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='key', full_name='v5api.KeyValue.key', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='value', full_name='v5api.KeyValue.value', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=4322, - serialized_end=4360, -) - - -_STREAMCSVCONFIG = _descriptor.Descriptor( - name='StreamCSVConfig', - full_name='v5api.StreamCSVConfig', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='version', full_name='v5api.StreamCSVConfig.version', index=0, - number=1, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='label', full_name='v5api.StreamCSVConfig.label', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.StreamCSVConfig.uuid', index=2, - number=3, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=4362, - serialized_end=4425, -) - - -_GENERATECSVPARAMS = _descriptor.Descriptor( - name='GenerateCSVParams', - full_name='v5api.GenerateCSVParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='queryType', full_name='v5api.GenerateCSVParams.queryType', index=0, - number=1, type=14, cpp_type=8, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='startTime', full_name='v5api.GenerateCSVParams.startTime', index=1, - number=2, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='endTime', full_name='v5api.GenerateCSVParams.endTime', index=2, - number=3, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='windowSize', full_name='v5api.GenerateCSVParams.windowSize', index=3, - number=4, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='depth', full_name='v5api.GenerateCSVParams.depth', index=4, - number=5, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='includeVersions', full_name='v5api.GenerateCSVParams.includeVersions', index=5, - number=6, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='streams', full_name='v5api.GenerateCSVParams.streams', index=6, - number=7, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - _GENERATECSVPARAMS_QUERYTYPE, - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=4428, - serialized_end=4713, -) - - -_GENERATECSVRESPONSE = _descriptor.Descriptor( - name='GenerateCSVResponse', - full_name='v5api.GenerateCSVResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.GenerateCSVResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='isHeader', full_name='v5api.GenerateCSVResponse.isHeader', index=1, - number=2, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='row', full_name='v5api.GenerateCSVResponse.row', index=2, - number=3, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=4715, - serialized_end=4796, -) - - -_SQLQUERYPARAMS = _descriptor.Descriptor( - name='SQLQueryParams', - full_name='v5api.SQLQueryParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='query', full_name='v5api.SQLQueryParams.query', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='params', full_name='v5api.SQLQueryParams.params', index=1, - number=2, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='role', full_name='v5api.SQLQueryParams.role', index=2, - number=100, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=4798, - serialized_end=4872, -) - - -_SQLQUERYRESPONSE = _descriptor.Descriptor( - name='SQLQueryResponse', - full_name='v5api.SQLQueryResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.SQLQueryResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='SQLQueryRow', full_name='v5api.SQLQueryResponse.SQLQueryRow', index=1, - number=2, type=12, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=4874, - serialized_end=4942, -) - - -_ROLE = _descriptor.Descriptor( - name='Role', - full_name='v5api.Role', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='v5api.Role.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=4944, - serialized_end=4964, -) - - -_SETCOMPACTIONCONFIGPARAMS = _descriptor.Descriptor( - name='SetCompactionConfigParams', - full_name='v5api.SetCompactionConfigParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.SetCompactionConfigParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='CompactedVersion', full_name='v5api.SetCompactionConfigParams.CompactedVersion', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='reducedResolutionRanges', full_name='v5api.SetCompactionConfigParams.reducedResolutionRanges', index=2, - number=3, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='unused0', full_name='v5api.SetCompactionConfigParams.unused0', index=3, - number=4, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=4967, - serialized_end=5115, -) - - -_SETCOMPACTIONCONFIGRESPONSE = _descriptor.Descriptor( - name='SetCompactionConfigResponse', - full_name='v5api.SetCompactionConfigResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.SetCompactionConfigResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=5117, - serialized_end=5175, -) - - -_GETCOMPACTIONCONFIGPARAMS = _descriptor.Descriptor( - name='GetCompactionConfigParams', - full_name='v5api.GetCompactionConfigParams', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uuid', full_name='v5api.GetCompactionConfigParams.uuid', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=5177, - serialized_end=5218, -) - - -_GETCOMPACTIONCONFIGRESPONSE = _descriptor.Descriptor( - name='GetCompactionConfigResponse', - full_name='v5api.GetCompactionConfigResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='stat', full_name='v5api.GetCompactionConfigResponse.stat', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='LatestMajorVersion', full_name='v5api.GetCompactionConfigResponse.LatestMajorVersion', index=1, - number=2, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='CompactedVersion', full_name='v5api.GetCompactionConfigResponse.CompactedVersion', index=2, - number=3, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='reducedResolutionRanges', full_name='v5api.GetCompactionConfigResponse.reducedResolutionRanges', index=3, - number=4, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='unused0', full_name='v5api.GetCompactionConfigResponse.unused0', index=4, - number=5, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=5221, - serialized_end=5414, -) - - -_REDUCEDRESOLUTIONRANGE = _descriptor.Descriptor( - name='ReducedResolutionRange', - full_name='v5api.ReducedResolutionRange', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='Start', full_name='v5api.ReducedResolutionRange.Start', index=0, - number=1, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='End', full_name='v5api.ReducedResolutionRange.End', index=1, - number=2, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='Resolution', full_name='v5api.ReducedResolutionRange.Resolution', index=2, - number=3, type=13, cpp_type=3, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=5416, - serialized_end=5488, -) - -_RAWVALUESRESPONSE.fields_by_name['stat'].message_type = _STATUS -_RAWVALUESRESPONSE.fields_by_name['values'].message_type = _RAWPOINT -_ALIGNEDWINDOWSRESPONSE.fields_by_name['stat'].message_type = _STATUS -_ALIGNEDWINDOWSRESPONSE.fields_by_name['values'].message_type = _STATPOINT -_WINDOWSRESPONSE.fields_by_name['stat'].message_type = _STATUS -_WINDOWSRESPONSE.fields_by_name['values'].message_type = _STATPOINT -_STREAMINFOPARAMS.fields_by_name['role'].message_type = _ROLE -_STREAMINFORESPONSE.fields_by_name['stat'].message_type = _STATUS -_STREAMINFORESPONSE.fields_by_name['descriptor'].message_type = _STREAMDESCRIPTOR -_STREAMDESCRIPTOR.fields_by_name['tags'].message_type = _KEYOPTVALUE -_STREAMDESCRIPTOR.fields_by_name['annotations'].message_type = _KEYOPTVALUE -_SETSTREAMANNOTATIONSPARAMS.fields_by_name['changes'].message_type = _KEYOPTVALUE -_SETSTREAMANNOTATIONSRESPONSE.fields_by_name['stat'].message_type = _STATUS -_SETSTREAMTAGSPARAMS.fields_by_name['tags'].message_type = _KEYOPTVALUE -_SETSTREAMTAGSRESPONSE.fields_by_name['stat'].message_type = _STATUS -_CREATEPARAMS.fields_by_name['tags'].message_type = _KEYOPTVALUE -_CREATEPARAMS.fields_by_name['annotations'].message_type = _KEYOPTVALUE -_CREATERESPONSE.fields_by_name['stat'].message_type = _STATUS -_METADATAUSAGEPARAMS.fields_by_name['role'].message_type = _ROLE -_METADATAUSAGERESPONSE.fields_by_name['stat'].message_type = _STATUS -_METADATAUSAGERESPONSE.fields_by_name['tags'].message_type = _KEYCOUNT -_METADATAUSAGERESPONSE.fields_by_name['annotations'].message_type = _KEYCOUNT -_LISTCOLLECTIONSPARAMS.fields_by_name['role'].message_type = _ROLE -_LISTCOLLECTIONSRESPONSE.fields_by_name['stat'].message_type = _STATUS -_LOOKUPSTREAMSPARAMS.fields_by_name['tags'].message_type = _KEYOPTVALUE -_LOOKUPSTREAMSPARAMS.fields_by_name['annotations'].message_type = _KEYOPTVALUE -_LOOKUPSTREAMSPARAMS.fields_by_name['role'].message_type = _ROLE -_LOOKUPSTREAMSRESPONSE.fields_by_name['stat'].message_type = _STATUS -_LOOKUPSTREAMSRESPONSE.fields_by_name['results'].message_type = _STREAMDESCRIPTOR -_NEARESTRESPONSE.fields_by_name['stat'].message_type = _STATUS -_NEARESTRESPONSE.fields_by_name['value'].message_type = _RAWPOINT -_CHANGESRESPONSE.fields_by_name['stat'].message_type = _STATUS -_CHANGESRESPONSE.fields_by_name['ranges'].message_type = _CHANGEDRANGE -_INSERTPARAMS.fields_by_name['merge_policy'].enum_type = _MERGEPOLICY -_INSERTPARAMS.fields_by_name['values'].message_type = _RAWPOINT -_INSERTRESPONSE.fields_by_name['stat'].message_type = _STATUS -_DELETERESPONSE.fields_by_name['stat'].message_type = _STATUS -_INFORESPONSE.fields_by_name['stat'].message_type = _STATUS -_INFORESPONSE.fields_by_name['mash'].message_type = _MASH -_INFORESPONSE.fields_by_name['proxy'].message_type = _PROXYINFO -_FAULTINJECTRESPONSE.fields_by_name['stat'].message_type = _STATUS -_FLUSHRESPONSE.fields_by_name['stat'].message_type = _STATUS -_OBLITERATERESPONSE.fields_by_name['stat'].message_type = _STATUS -_STATUS.fields_by_name['mash'].message_type = _MASH -_MASH.fields_by_name['members'].message_type = _MEMBER -_KEYOPTVALUE.fields_by_name['val'].message_type = _OPTVALUE -_GENERATECSVPARAMS.fields_by_name['queryType'].enum_type = _GENERATECSVPARAMS_QUERYTYPE -_GENERATECSVPARAMS.fields_by_name['streams'].message_type = _STREAMCSVCONFIG -_GENERATECSVPARAMS_QUERYTYPE.containing_type = _GENERATECSVPARAMS -_GENERATECSVRESPONSE.fields_by_name['stat'].message_type = _STATUS -_SQLQUERYPARAMS.fields_by_name['role'].message_type = _ROLE -_SQLQUERYRESPONSE.fields_by_name['stat'].message_type = _STATUS -_SETCOMPACTIONCONFIGPARAMS.fields_by_name['reducedResolutionRanges'].message_type = _REDUCEDRESOLUTIONRANGE -_SETCOMPACTIONCONFIGRESPONSE.fields_by_name['stat'].message_type = _STATUS -_GETCOMPACTIONCONFIGRESPONSE.fields_by_name['stat'].message_type = _STATUS -_GETCOMPACTIONCONFIGRESPONSE.fields_by_name['reducedResolutionRanges'].message_type = _REDUCEDRESOLUTIONRANGE -DESCRIPTOR.message_types_by_name['RawValuesParams'] = _RAWVALUESPARAMS -DESCRIPTOR.message_types_by_name['RawValuesResponse'] = _RAWVALUESRESPONSE -DESCRIPTOR.message_types_by_name['AlignedWindowsParams'] = _ALIGNEDWINDOWSPARAMS -DESCRIPTOR.message_types_by_name['AlignedWindowsResponse'] = _ALIGNEDWINDOWSRESPONSE -DESCRIPTOR.message_types_by_name['WindowsParams'] = _WINDOWSPARAMS -DESCRIPTOR.message_types_by_name['WindowsResponse'] = _WINDOWSRESPONSE -DESCRIPTOR.message_types_by_name['StreamInfoParams'] = _STREAMINFOPARAMS -DESCRIPTOR.message_types_by_name['StreamInfoResponse'] = _STREAMINFORESPONSE -DESCRIPTOR.message_types_by_name['StreamDescriptor'] = _STREAMDESCRIPTOR -DESCRIPTOR.message_types_by_name['SetStreamAnnotationsParams'] = _SETSTREAMANNOTATIONSPARAMS -DESCRIPTOR.message_types_by_name['SetStreamAnnotationsResponse'] = _SETSTREAMANNOTATIONSRESPONSE -DESCRIPTOR.message_types_by_name['SetStreamTagsParams'] = _SETSTREAMTAGSPARAMS -DESCRIPTOR.message_types_by_name['SetStreamTagsResponse'] = _SETSTREAMTAGSRESPONSE -DESCRIPTOR.message_types_by_name['CreateParams'] = _CREATEPARAMS -DESCRIPTOR.message_types_by_name['CreateResponse'] = _CREATERESPONSE -DESCRIPTOR.message_types_by_name['MetadataUsageParams'] = _METADATAUSAGEPARAMS -DESCRIPTOR.message_types_by_name['MetadataUsageResponse'] = _METADATAUSAGERESPONSE -DESCRIPTOR.message_types_by_name['KeyCount'] = _KEYCOUNT -DESCRIPTOR.message_types_by_name['ListCollectionsParams'] = _LISTCOLLECTIONSPARAMS -DESCRIPTOR.message_types_by_name['ListCollectionsResponse'] = _LISTCOLLECTIONSRESPONSE -DESCRIPTOR.message_types_by_name['LookupStreamsParams'] = _LOOKUPSTREAMSPARAMS -DESCRIPTOR.message_types_by_name['LookupStreamsResponse'] = _LOOKUPSTREAMSRESPONSE -DESCRIPTOR.message_types_by_name['NearestParams'] = _NEARESTPARAMS -DESCRIPTOR.message_types_by_name['NearestResponse'] = _NEARESTRESPONSE -DESCRIPTOR.message_types_by_name['ChangesParams'] = _CHANGESPARAMS -DESCRIPTOR.message_types_by_name['ChangesResponse'] = _CHANGESRESPONSE -DESCRIPTOR.message_types_by_name['InsertParams'] = _INSERTPARAMS -DESCRIPTOR.message_types_by_name['InsertResponse'] = _INSERTRESPONSE -DESCRIPTOR.message_types_by_name['DeleteParams'] = _DELETEPARAMS -DESCRIPTOR.message_types_by_name['DeleteResponse'] = _DELETERESPONSE -DESCRIPTOR.message_types_by_name['InfoParams'] = _INFOPARAMS -DESCRIPTOR.message_types_by_name['InfoResponse'] = _INFORESPONSE -DESCRIPTOR.message_types_by_name['ProxyInfo'] = _PROXYINFO -DESCRIPTOR.message_types_by_name['FaultInjectParams'] = _FAULTINJECTPARAMS -DESCRIPTOR.message_types_by_name['FaultInjectResponse'] = _FAULTINJECTRESPONSE -DESCRIPTOR.message_types_by_name['FlushParams'] = _FLUSHPARAMS -DESCRIPTOR.message_types_by_name['FlushResponse'] = _FLUSHRESPONSE -DESCRIPTOR.message_types_by_name['ObliterateParams'] = _OBLITERATEPARAMS -DESCRIPTOR.message_types_by_name['ObliterateResponse'] = _OBLITERATERESPONSE -DESCRIPTOR.message_types_by_name['RawPoint'] = _RAWPOINT -DESCRIPTOR.message_types_by_name['StatPoint'] = _STATPOINT -DESCRIPTOR.message_types_by_name['ChangedRange'] = _CHANGEDRANGE -DESCRIPTOR.message_types_by_name['Status'] = _STATUS -DESCRIPTOR.message_types_by_name['Mash'] = _MASH -DESCRIPTOR.message_types_by_name['Member'] = _MEMBER -DESCRIPTOR.message_types_by_name['KeyOptValue'] = _KEYOPTVALUE -DESCRIPTOR.message_types_by_name['OptValue'] = _OPTVALUE -DESCRIPTOR.message_types_by_name['KeyValue'] = _KEYVALUE -DESCRIPTOR.message_types_by_name['StreamCSVConfig'] = _STREAMCSVCONFIG -DESCRIPTOR.message_types_by_name['GenerateCSVParams'] = _GENERATECSVPARAMS -DESCRIPTOR.message_types_by_name['GenerateCSVResponse'] = _GENERATECSVRESPONSE -DESCRIPTOR.message_types_by_name['SQLQueryParams'] = _SQLQUERYPARAMS -DESCRIPTOR.message_types_by_name['SQLQueryResponse'] = _SQLQUERYRESPONSE -DESCRIPTOR.message_types_by_name['Role'] = _ROLE -DESCRIPTOR.message_types_by_name['SetCompactionConfigParams'] = _SETCOMPACTIONCONFIGPARAMS -DESCRIPTOR.message_types_by_name['SetCompactionConfigResponse'] = _SETCOMPACTIONCONFIGRESPONSE -DESCRIPTOR.message_types_by_name['GetCompactionConfigParams'] = _GETCOMPACTIONCONFIGPARAMS -DESCRIPTOR.message_types_by_name['GetCompactionConfigResponse'] = _GETCOMPACTIONCONFIGRESPONSE -DESCRIPTOR.message_types_by_name['ReducedResolutionRange'] = _REDUCEDRESOLUTIONRANGE -DESCRIPTOR.enum_types_by_name['MergePolicy'] = _MERGEPOLICY -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -RawValuesParams = _reflection.GeneratedProtocolMessageType('RawValuesParams', (_message.Message,), dict( - DESCRIPTOR = _RAWVALUESPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.RawValuesParams) - )) -_sym_db.RegisterMessage(RawValuesParams) - -RawValuesResponse = _reflection.GeneratedProtocolMessageType('RawValuesResponse', (_message.Message,), dict( - DESCRIPTOR = _RAWVALUESRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.RawValuesResponse) - )) -_sym_db.RegisterMessage(RawValuesResponse) - -AlignedWindowsParams = _reflection.GeneratedProtocolMessageType('AlignedWindowsParams', (_message.Message,), dict( - DESCRIPTOR = _ALIGNEDWINDOWSPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.AlignedWindowsParams) - )) -_sym_db.RegisterMessage(AlignedWindowsParams) - -AlignedWindowsResponse = _reflection.GeneratedProtocolMessageType('AlignedWindowsResponse', (_message.Message,), dict( - DESCRIPTOR = _ALIGNEDWINDOWSRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.AlignedWindowsResponse) - )) -_sym_db.RegisterMessage(AlignedWindowsResponse) - -WindowsParams = _reflection.GeneratedProtocolMessageType('WindowsParams', (_message.Message,), dict( - DESCRIPTOR = _WINDOWSPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.WindowsParams) - )) -_sym_db.RegisterMessage(WindowsParams) - -WindowsResponse = _reflection.GeneratedProtocolMessageType('WindowsResponse', (_message.Message,), dict( - DESCRIPTOR = _WINDOWSRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.WindowsResponse) - )) -_sym_db.RegisterMessage(WindowsResponse) - -StreamInfoParams = _reflection.GeneratedProtocolMessageType('StreamInfoParams', (_message.Message,), dict( - DESCRIPTOR = _STREAMINFOPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.StreamInfoParams) - )) -_sym_db.RegisterMessage(StreamInfoParams) - -StreamInfoResponse = _reflection.GeneratedProtocolMessageType('StreamInfoResponse', (_message.Message,), dict( - DESCRIPTOR = _STREAMINFORESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.StreamInfoResponse) - )) -_sym_db.RegisterMessage(StreamInfoResponse) - -StreamDescriptor = _reflection.GeneratedProtocolMessageType('StreamDescriptor', (_message.Message,), dict( - DESCRIPTOR = _STREAMDESCRIPTOR, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.StreamDescriptor) - )) -_sym_db.RegisterMessage(StreamDescriptor) - -SetStreamAnnotationsParams = _reflection.GeneratedProtocolMessageType('SetStreamAnnotationsParams', (_message.Message,), dict( - DESCRIPTOR = _SETSTREAMANNOTATIONSPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.SetStreamAnnotationsParams) - )) -_sym_db.RegisterMessage(SetStreamAnnotationsParams) - -SetStreamAnnotationsResponse = _reflection.GeneratedProtocolMessageType('SetStreamAnnotationsResponse', (_message.Message,), dict( - DESCRIPTOR = _SETSTREAMANNOTATIONSRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.SetStreamAnnotationsResponse) - )) -_sym_db.RegisterMessage(SetStreamAnnotationsResponse) - -SetStreamTagsParams = _reflection.GeneratedProtocolMessageType('SetStreamTagsParams', (_message.Message,), dict( - DESCRIPTOR = _SETSTREAMTAGSPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.SetStreamTagsParams) - )) -_sym_db.RegisterMessage(SetStreamTagsParams) - -SetStreamTagsResponse = _reflection.GeneratedProtocolMessageType('SetStreamTagsResponse', (_message.Message,), dict( - DESCRIPTOR = _SETSTREAMTAGSRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.SetStreamTagsResponse) - )) -_sym_db.RegisterMessage(SetStreamTagsResponse) - -CreateParams = _reflection.GeneratedProtocolMessageType('CreateParams', (_message.Message,), dict( - DESCRIPTOR = _CREATEPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.CreateParams) - )) -_sym_db.RegisterMessage(CreateParams) - -CreateResponse = _reflection.GeneratedProtocolMessageType('CreateResponse', (_message.Message,), dict( - DESCRIPTOR = _CREATERESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.CreateResponse) - )) -_sym_db.RegisterMessage(CreateResponse) - -MetadataUsageParams = _reflection.GeneratedProtocolMessageType('MetadataUsageParams', (_message.Message,), dict( - DESCRIPTOR = _METADATAUSAGEPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.MetadataUsageParams) - )) -_sym_db.RegisterMessage(MetadataUsageParams) - -MetadataUsageResponse = _reflection.GeneratedProtocolMessageType('MetadataUsageResponse', (_message.Message,), dict( - DESCRIPTOR = _METADATAUSAGERESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.MetadataUsageResponse) - )) -_sym_db.RegisterMessage(MetadataUsageResponse) - -KeyCount = _reflection.GeneratedProtocolMessageType('KeyCount', (_message.Message,), dict( - DESCRIPTOR = _KEYCOUNT, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.KeyCount) - )) -_sym_db.RegisterMessage(KeyCount) - -ListCollectionsParams = _reflection.GeneratedProtocolMessageType('ListCollectionsParams', (_message.Message,), dict( - DESCRIPTOR = _LISTCOLLECTIONSPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.ListCollectionsParams) - )) -_sym_db.RegisterMessage(ListCollectionsParams) - -ListCollectionsResponse = _reflection.GeneratedProtocolMessageType('ListCollectionsResponse', (_message.Message,), dict( - DESCRIPTOR = _LISTCOLLECTIONSRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.ListCollectionsResponse) - )) -_sym_db.RegisterMessage(ListCollectionsResponse) - -LookupStreamsParams = _reflection.GeneratedProtocolMessageType('LookupStreamsParams', (_message.Message,), dict( - DESCRIPTOR = _LOOKUPSTREAMSPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.LookupStreamsParams) - )) -_sym_db.RegisterMessage(LookupStreamsParams) - -LookupStreamsResponse = _reflection.GeneratedProtocolMessageType('LookupStreamsResponse', (_message.Message,), dict( - DESCRIPTOR = _LOOKUPSTREAMSRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.LookupStreamsResponse) - )) -_sym_db.RegisterMessage(LookupStreamsResponse) - -NearestParams = _reflection.GeneratedProtocolMessageType('NearestParams', (_message.Message,), dict( - DESCRIPTOR = _NEARESTPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.NearestParams) - )) -_sym_db.RegisterMessage(NearestParams) - -NearestResponse = _reflection.GeneratedProtocolMessageType('NearestResponse', (_message.Message,), dict( - DESCRIPTOR = _NEARESTRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.NearestResponse) - )) -_sym_db.RegisterMessage(NearestResponse) - -ChangesParams = _reflection.GeneratedProtocolMessageType('ChangesParams', (_message.Message,), dict( - DESCRIPTOR = _CHANGESPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.ChangesParams) - )) -_sym_db.RegisterMessage(ChangesParams) - -ChangesResponse = _reflection.GeneratedProtocolMessageType('ChangesResponse', (_message.Message,), dict( - DESCRIPTOR = _CHANGESRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.ChangesResponse) - )) -_sym_db.RegisterMessage(ChangesResponse) - -InsertParams = _reflection.GeneratedProtocolMessageType('InsertParams', (_message.Message,), dict( - DESCRIPTOR = _INSERTPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.InsertParams) - )) -_sym_db.RegisterMessage(InsertParams) - -InsertResponse = _reflection.GeneratedProtocolMessageType('InsertResponse', (_message.Message,), dict( - DESCRIPTOR = _INSERTRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.InsertResponse) - )) -_sym_db.RegisterMessage(InsertResponse) - -DeleteParams = _reflection.GeneratedProtocolMessageType('DeleteParams', (_message.Message,), dict( - DESCRIPTOR = _DELETEPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.DeleteParams) - )) -_sym_db.RegisterMessage(DeleteParams) - -DeleteResponse = _reflection.GeneratedProtocolMessageType('DeleteResponse', (_message.Message,), dict( - DESCRIPTOR = _DELETERESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.DeleteResponse) - )) -_sym_db.RegisterMessage(DeleteResponse) - -InfoParams = _reflection.GeneratedProtocolMessageType('InfoParams', (_message.Message,), dict( - DESCRIPTOR = _INFOPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.InfoParams) - )) -_sym_db.RegisterMessage(InfoParams) - -InfoResponse = _reflection.GeneratedProtocolMessageType('InfoResponse', (_message.Message,), dict( - DESCRIPTOR = _INFORESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.InfoResponse) - )) -_sym_db.RegisterMessage(InfoResponse) - -ProxyInfo = _reflection.GeneratedProtocolMessageType('ProxyInfo', (_message.Message,), dict( - DESCRIPTOR = _PROXYINFO, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.ProxyInfo) - )) -_sym_db.RegisterMessage(ProxyInfo) - -FaultInjectParams = _reflection.GeneratedProtocolMessageType('FaultInjectParams', (_message.Message,), dict( - DESCRIPTOR = _FAULTINJECTPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.FaultInjectParams) - )) -_sym_db.RegisterMessage(FaultInjectParams) - -FaultInjectResponse = _reflection.GeneratedProtocolMessageType('FaultInjectResponse', (_message.Message,), dict( - DESCRIPTOR = _FAULTINJECTRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.FaultInjectResponse) - )) -_sym_db.RegisterMessage(FaultInjectResponse) - -FlushParams = _reflection.GeneratedProtocolMessageType('FlushParams', (_message.Message,), dict( - DESCRIPTOR = _FLUSHPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.FlushParams) - )) -_sym_db.RegisterMessage(FlushParams) - -FlushResponse = _reflection.GeneratedProtocolMessageType('FlushResponse', (_message.Message,), dict( - DESCRIPTOR = _FLUSHRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.FlushResponse) - )) -_sym_db.RegisterMessage(FlushResponse) - -ObliterateParams = _reflection.GeneratedProtocolMessageType('ObliterateParams', (_message.Message,), dict( - DESCRIPTOR = _OBLITERATEPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.ObliterateParams) - )) -_sym_db.RegisterMessage(ObliterateParams) - -ObliterateResponse = _reflection.GeneratedProtocolMessageType('ObliterateResponse', (_message.Message,), dict( - DESCRIPTOR = _OBLITERATERESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.ObliterateResponse) - )) -_sym_db.RegisterMessage(ObliterateResponse) - -RawPoint = _reflection.GeneratedProtocolMessageType('RawPoint', (_message.Message,), dict( - DESCRIPTOR = _RAWPOINT, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.RawPoint) - )) -_sym_db.RegisterMessage(RawPoint) - -StatPoint = _reflection.GeneratedProtocolMessageType('StatPoint', (_message.Message,), dict( - DESCRIPTOR = _STATPOINT, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.StatPoint) - )) -_sym_db.RegisterMessage(StatPoint) - -ChangedRange = _reflection.GeneratedProtocolMessageType('ChangedRange', (_message.Message,), dict( - DESCRIPTOR = _CHANGEDRANGE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.ChangedRange) - )) -_sym_db.RegisterMessage(ChangedRange) - -Status = _reflection.GeneratedProtocolMessageType('Status', (_message.Message,), dict( - DESCRIPTOR = _STATUS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.Status) - )) -_sym_db.RegisterMessage(Status) - -Mash = _reflection.GeneratedProtocolMessageType('Mash', (_message.Message,), dict( - DESCRIPTOR = _MASH, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.Mash) - )) -_sym_db.RegisterMessage(Mash) - -Member = _reflection.GeneratedProtocolMessageType('Member', (_message.Message,), dict( - DESCRIPTOR = _MEMBER, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.Member) - )) -_sym_db.RegisterMessage(Member) - -KeyOptValue = _reflection.GeneratedProtocolMessageType('KeyOptValue', (_message.Message,), dict( - DESCRIPTOR = _KEYOPTVALUE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.KeyOptValue) - )) -_sym_db.RegisterMessage(KeyOptValue) - -OptValue = _reflection.GeneratedProtocolMessageType('OptValue', (_message.Message,), dict( - DESCRIPTOR = _OPTVALUE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.OptValue) - )) -_sym_db.RegisterMessage(OptValue) - -KeyValue = _reflection.GeneratedProtocolMessageType('KeyValue', (_message.Message,), dict( - DESCRIPTOR = _KEYVALUE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.KeyValue) - )) -_sym_db.RegisterMessage(KeyValue) - -StreamCSVConfig = _reflection.GeneratedProtocolMessageType('StreamCSVConfig', (_message.Message,), dict( - DESCRIPTOR = _STREAMCSVCONFIG, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.StreamCSVConfig) - )) -_sym_db.RegisterMessage(StreamCSVConfig) - -GenerateCSVParams = _reflection.GeneratedProtocolMessageType('GenerateCSVParams', (_message.Message,), dict( - DESCRIPTOR = _GENERATECSVPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.GenerateCSVParams) - )) -_sym_db.RegisterMessage(GenerateCSVParams) - -GenerateCSVResponse = _reflection.GeneratedProtocolMessageType('GenerateCSVResponse', (_message.Message,), dict( - DESCRIPTOR = _GENERATECSVRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.GenerateCSVResponse) - )) -_sym_db.RegisterMessage(GenerateCSVResponse) - -SQLQueryParams = _reflection.GeneratedProtocolMessageType('SQLQueryParams', (_message.Message,), dict( - DESCRIPTOR = _SQLQUERYPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.SQLQueryParams) - )) -_sym_db.RegisterMessage(SQLQueryParams) - -SQLQueryResponse = _reflection.GeneratedProtocolMessageType('SQLQueryResponse', (_message.Message,), dict( - DESCRIPTOR = _SQLQUERYRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.SQLQueryResponse) - )) -_sym_db.RegisterMessage(SQLQueryResponse) - -Role = _reflection.GeneratedProtocolMessageType('Role', (_message.Message,), dict( - DESCRIPTOR = _ROLE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.Role) - )) -_sym_db.RegisterMessage(Role) - -SetCompactionConfigParams = _reflection.GeneratedProtocolMessageType('SetCompactionConfigParams', (_message.Message,), dict( - DESCRIPTOR = _SETCOMPACTIONCONFIGPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.SetCompactionConfigParams) - )) -_sym_db.RegisterMessage(SetCompactionConfigParams) - -SetCompactionConfigResponse = _reflection.GeneratedProtocolMessageType('SetCompactionConfigResponse', (_message.Message,), dict( - DESCRIPTOR = _SETCOMPACTIONCONFIGRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.SetCompactionConfigResponse) - )) -_sym_db.RegisterMessage(SetCompactionConfigResponse) - -GetCompactionConfigParams = _reflection.GeneratedProtocolMessageType('GetCompactionConfigParams', (_message.Message,), dict( - DESCRIPTOR = _GETCOMPACTIONCONFIGPARAMS, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.GetCompactionConfigParams) - )) -_sym_db.RegisterMessage(GetCompactionConfigParams) - -GetCompactionConfigResponse = _reflection.GeneratedProtocolMessageType('GetCompactionConfigResponse', (_message.Message,), dict( - DESCRIPTOR = _GETCOMPACTIONCONFIGRESPONSE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.GetCompactionConfigResponse) - )) -_sym_db.RegisterMessage(GetCompactionConfigResponse) - -ReducedResolutionRange = _reflection.GeneratedProtocolMessageType('ReducedResolutionRange', (_message.Message,), dict( - DESCRIPTOR = _REDUCEDRESOLUTIONRANGE, - __module__ = 'btrdb_pb2' - # @@protoc_insertion_point(class_scope:v5api.ReducedResolutionRange) - )) -_sym_db.RegisterMessage(ReducedResolutionRange) - - - -_BTRDB = _descriptor.ServiceDescriptor( - name='BTrDB', - full_name='v5api.BTrDB', - file=DESCRIPTOR, - index=0, - serialized_options=None, - serialized_start=5553, - serialized_end=6884, - methods=[ - _descriptor.MethodDescriptor( - name='RawValues', - full_name='v5api.BTrDB.RawValues', - index=0, - containing_service=None, - input_type=_RAWVALUESPARAMS, - output_type=_RAWVALUESRESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='AlignedWindows', - full_name='v5api.BTrDB.AlignedWindows', - index=1, - containing_service=None, - input_type=_ALIGNEDWINDOWSPARAMS, - output_type=_ALIGNEDWINDOWSRESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='Windows', - full_name='v5api.BTrDB.Windows', - index=2, - containing_service=None, - input_type=_WINDOWSPARAMS, - output_type=_WINDOWSRESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='StreamInfo', - full_name='v5api.BTrDB.StreamInfo', - index=3, - containing_service=None, - input_type=_STREAMINFOPARAMS, - output_type=_STREAMINFORESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='SetStreamAnnotations', - full_name='v5api.BTrDB.SetStreamAnnotations', - index=4, - containing_service=None, - input_type=_SETSTREAMANNOTATIONSPARAMS, - output_type=_SETSTREAMANNOTATIONSRESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='SetStreamTags', - full_name='v5api.BTrDB.SetStreamTags', - index=5, - containing_service=None, - input_type=_SETSTREAMTAGSPARAMS, - output_type=_SETSTREAMTAGSRESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='Create', - full_name='v5api.BTrDB.Create', - index=6, - containing_service=None, - input_type=_CREATEPARAMS, - output_type=_CREATERESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='ListCollections', - full_name='v5api.BTrDB.ListCollections', - index=7, - containing_service=None, - input_type=_LISTCOLLECTIONSPARAMS, - output_type=_LISTCOLLECTIONSRESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='LookupStreams', - full_name='v5api.BTrDB.LookupStreams', - index=8, - containing_service=None, - input_type=_LOOKUPSTREAMSPARAMS, - output_type=_LOOKUPSTREAMSRESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='Nearest', - full_name='v5api.BTrDB.Nearest', - index=9, - containing_service=None, - input_type=_NEARESTPARAMS, - output_type=_NEARESTRESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='Changes', - full_name='v5api.BTrDB.Changes', - index=10, - containing_service=None, - input_type=_CHANGESPARAMS, - output_type=_CHANGESRESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='Insert', - full_name='v5api.BTrDB.Insert', - index=11, - containing_service=None, - input_type=_INSERTPARAMS, - output_type=_INSERTRESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='Delete', - full_name='v5api.BTrDB.Delete', - index=12, - containing_service=None, - input_type=_DELETEPARAMS, - output_type=_DELETERESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='Info', - full_name='v5api.BTrDB.Info', - index=13, - containing_service=None, - input_type=_INFOPARAMS, - output_type=_INFORESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='FaultInject', - full_name='v5api.BTrDB.FaultInject', - index=14, - containing_service=None, - input_type=_FAULTINJECTPARAMS, - output_type=_FAULTINJECTRESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='Flush', - full_name='v5api.BTrDB.Flush', - index=15, - containing_service=None, - input_type=_FLUSHPARAMS, - output_type=_FLUSHRESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='Obliterate', - full_name='v5api.BTrDB.Obliterate', - index=16, - containing_service=None, - input_type=_OBLITERATEPARAMS, - output_type=_OBLITERATERESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='GetMetadataUsage', - full_name='v5api.BTrDB.GetMetadataUsage', - index=17, - containing_service=None, - input_type=_METADATAUSAGEPARAMS, - output_type=_METADATAUSAGERESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='GenerateCSV', - full_name='v5api.BTrDB.GenerateCSV', - index=18, - containing_service=None, - input_type=_GENERATECSVPARAMS, - output_type=_GENERATECSVRESPONSE, - serialized_options=None, - ), - _descriptor.MethodDescriptor( - name='SQLQuery', - full_name='v5api.BTrDB.SQLQuery', - index=19, - containing_service=None, - input_type=_SQLQUERYPARAMS, - output_type=_SQLQUERYRESPONSE, - serialized_options=None, - ), -]) -_sym_db.RegisterServiceDescriptor(_BTRDB) - -DESCRIPTOR.services_by_name['BTrDB'] = _BTRDB +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0b\x62trdb.proto\x12\x05v5api\"Q\n\x0fRawValuesParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\"}\n\x11RawValuesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x1f\n\x06values\x18\x04 \x03(\x0b\x32\x0f.v5api.RawPoint\"u\n\x16\x41rrowRawValuesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x12\n\narrowBytes\x18\x04 \x01(\x0c\"n\n\x16\x41rrowMultiValuesParams\x12\x0c\n\x04uuid\x18\x01 \x03(\x0c\x12\x14\n\x0cversionMajor\x18\x02 \x03(\x04\x12\r\n\x05start\x18\x03 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x04 \x01(\x10\x12\x14\n\x0csnapPeriodNs\x18\x05 \x01(\x03\"K\n\x18\x41rrowMultiValuesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x12\n\narrowBytes\x18\x02 \x01(\x0c\"*\n\x0bRawPointVec\x12\x0c\n\x04time\x18\x01 \x01(\x10\x12\r\n\x05value\x18\x02 \x03(\x01\"j\n\x14\x41lignedWindowsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\x12\x12\n\npointWidth\x18\x05 \x01(\r\"\x83\x01\n\x16\x41lignedWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12 \n\x06values\x18\x04 \x03(\x0b\x32\x10.v5api.StatPoint\"z\n\x1b\x41rrowAlignedWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x12\n\narrowBytes\x18\x04 \x01(\x0c\"m\n\rWindowsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\x12\r\n\x05width\x18\x05 \x01(\x04\x12\r\n\x05\x64\x65pth\x18\x06 \x01(\r\"|\n\x0fWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12 \n\x06values\x18\x04 \x03(\x0b\x32\x10.v5api.StatPoint\"s\n\x14\x41rrowWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x12\n\narrowBytes\x18\x04 \x01(\x0c\"h\n\x10StreamInfoParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x13\n\x0bomitVersion\x18\x02 \x01(\x08\x12\x16\n\x0eomitDescriptor\x18\x03 \x01(\x08\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"\x8a\x01\n\x12StreamInfoResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12+\n\ndescriptor\x18\x04 \x01(\x0b\x32\x17.v5api.StreamDescriptor\"\x98\x01\n\x10StreamDescriptor\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x12\n\ncollection\x18\x02 \x01(\t\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x17\n\x0fpropertyVersion\x18\x05 \x01(\x04\"\x82\x01\n\x1aSetStreamAnnotationsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x1f\n\x17\x65xpectedPropertyVersion\x18\x02 \x01(\x04\x12#\n\x07\x63hanges\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x10\n\x08removals\x18\x04 \x03(\t\";\n\x1cSetStreamAnnotationsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"\x8a\x01\n\x13SetStreamTagsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x1f\n\x17\x65xpectedPropertyVersion\x18\x02 \x01(\x04\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x12\n\ncollection\x18\x04 \x01(\t\x12\x0e\n\x06remove\x18\x05 \x03(\t\"4\n\x15SetStreamTagsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"{\n\x0c\x43reateParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x12\n\ncollection\x18\x02 \x01(\t\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\"-\n\x0e\x43reateResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"@\n\x13MetadataUsageParams\x12\x0e\n\x06prefix\x18\x01 \x01(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"y\n\x15MetadataUsageResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x1d\n\x04tags\x18\x02 \x03(\x0b\x32\x0f.v5api.KeyCount\x12$\n\x0b\x61nnotations\x18\x03 \x03(\x0b\x32\x0f.v5api.KeyCount\"&\n\x08KeyCount\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05\x63ount\x18\x02 \x01(\x04\"B\n\x15ListCollectionsParams\x12\x0e\n\x06prefix\x18\x01 \x01(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"K\n\x17ListCollectionsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x13\n\x0b\x63ollections\x18\x02 \x03(\t\"\xab\x01\n\x13LookupStreamsParams\x12\x12\n\ncollection\x18\x01 \x01(\t\x12\x1a\n\x12isCollectionPrefix\x18\x02 \x01(\x08\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"^\n\x15LookupStreamsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12(\n\x07results\x18\x02 \x03(\x0b\x32\x17.v5api.StreamDescriptor\"S\n\rNearestParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x0c\n\x04time\x18\x02 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x03 \x01(\x04\x12\x10\n\x08\x62\x61\x63kward\x18\x04 \x01(\x08\"z\n\x0fNearestResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x1e\n\x05value\x18\x04 \x01(\x0b\x32\x0f.v5api.RawPoint\"U\n\rChangesParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x11\n\tfromMajor\x18\x02 \x01(\x04\x12\x0f\n\x07toMajor\x18\x03 \x01(\x04\x12\x12\n\nresolution\x18\x04 \x01(\r\"\x7f\n\x0f\x43hangesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12#\n\x06ranges\x18\x04 \x03(\x0b\x32\x13.v5api.ChangedRange\"#\n\tRoundSpec\x12\x0e\n\x04\x62its\x18\x02 \x01(\x05H\x00\x42\x06\n\x04spec\"\x99\x01\n\x0cInsertParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x0c\n\x04sync\x18\x02 \x01(\x08\x12(\n\x0cmerge_policy\x18\x04 \x01(\x0e\x32\x12.v5api.MergePolicy\x12\"\n\x08rounding\x18\x05 \x01(\x0b\x32\x10.v5api.RoundSpec\x12\x1f\n\x06values\x18\x03 \x03(\x0b\x32\x0f.v5api.RawPoint\"\x91\x01\n\x11\x41rrowInsertParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x0c\n\x04sync\x18\x02 \x01(\x08\x12(\n\x0cmerge_policy\x18\x03 \x01(\x0e\x32\x12.v5api.MergePolicy\x12\"\n\x08rounding\x18\x04 \x01(\x0b\x32\x10.v5api.RoundSpec\x12\x12\n\narrowBytes\x18\x05 \x01(\x0c\"Y\n\x0eInsertResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\"8\n\x0c\x44\x65leteParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\"Y\n\x0e\x44\x65leteResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\"\x0c\n\nInfoParams\"\xa2\x01\n\x0cInfoResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x19\n\x04mash\x18\x02 \x01(\x0b\x32\x0b.v5api.Mash\x12\x14\n\x0cmajorVersion\x18\x03 \x01(\r\x12\x14\n\x0cminorVersion\x18\x04 \x01(\r\x12\r\n\x05\x62uild\x18\x05 \x01(\t\x12\x1f\n\x05proxy\x18\x06 \x01(\x0b\x32\x10.v5api.ProxyInfo\"#\n\tProxyInfo\x12\x16\n\x0eproxyEndpoints\x18\x01 \x03(\t\"1\n\x11\x46\x61ultInjectParams\x12\x0c\n\x04type\x18\x01 \x01(\x04\x12\x0e\n\x06params\x18\x02 \x01(\x0c\">\n\x13\x46\x61ultInjectResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\n\n\x02rv\x18\x02 \x01(\x0c\"\x1b\n\x0b\x46lushParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"X\n\rFlushResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\" \n\x10ObliterateParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"1\n\x12ObliterateResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"\'\n\x08RawPoint\x12\x0c\n\x04time\x18\x01 \x01(\x10\x12\r\n\x05value\x18\x02 \x01(\x01\"`\n\tStatPoint\x12\x0c\n\x04time\x18\x01 \x01(\x10\x12\x0b\n\x03min\x18\x02 \x01(\x01\x12\x0c\n\x04mean\x18\x03 \x01(\x01\x12\x0b\n\x03max\x18\x04 \x01(\x01\x12\r\n\x05\x63ount\x18\x05 \x01(\x06\x12\x0e\n\x06stddev\x18\x06 \x01(\x01\"*\n\x0c\x43hangedRange\x12\r\n\x05start\x18\x01 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x02 \x01(\x10\">\n\x06Status\x12\x0c\n\x04\x63ode\x18\x01 \x01(\r\x12\x0b\n\x03msg\x18\x02 \x01(\t\x12\x19\n\x04mash\x18\x03 \x01(\x0b\x32\x0b.v5api.Mash\"\x98\x01\n\x04Mash\x12\x10\n\x08revision\x18\x01 \x01(\x03\x12\x0e\n\x06leader\x18\x02 \x01(\t\x12\x16\n\x0eleaderRevision\x18\x03 \x01(\x03\x12\x13\n\x0btotalWeight\x18\x04 \x01(\x03\x12\x0f\n\x07healthy\x18\x05 \x01(\x08\x12\x10\n\x08unmapped\x18\x06 \x01(\x01\x12\x1e\n\x07members\x18\x07 \x03(\x0b\x32\r.v5api.Member\"\xc3\x01\n\x06Member\x12\x0c\n\x04hash\x18\x01 \x01(\r\x12\x10\n\x08nodename\x18\x02 \x01(\t\x12\n\n\x02up\x18\x03 \x01(\x08\x12\n\n\x02in\x18\x04 \x01(\x08\x12\x0f\n\x07\x65nabled\x18\x05 \x01(\x08\x12\r\n\x05start\x18\x06 \x01(\x03\x12\x0b\n\x03\x65nd\x18\x07 \x01(\x03\x12\x0e\n\x06weight\x18\x08 \x01(\x03\x12\x16\n\x0ereadPreference\x18\t \x01(\x01\x12\x15\n\rhttpEndpoints\x18\n \x01(\t\x12\x15\n\rgrpcEndpoints\x18\x0b \x01(\t\"8\n\x0bKeyOptValue\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x1c\n\x03val\x18\x02 \x01(\x0b\x32\x0f.v5api.OptValue\"\x19\n\x08OptValue\x12\r\n\x05value\x18\x01 \x01(\t\"&\n\x08KeyValue\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"?\n\x0fStreamCSVConfig\x12\x0f\n\x07version\x18\x01 \x01(\x04\x12\r\n\x05label\x18\x02 \x01(\t\x12\x0c\n\x04uuid\x18\x03 \x01(\x0c\"\x9d\x02\n\x11GenerateCSVParams\x12\x35\n\tqueryType\x18\x01 \x01(\x0e\x32\".v5api.GenerateCSVParams.QueryType\x12\x11\n\tstartTime\x18\x02 \x01(\x03\x12\x0f\n\x07\x65ndTime\x18\x03 \x01(\x03\x12\x12\n\nwindowSize\x18\x04 \x01(\x04\x12\r\n\x05\x64\x65pth\x18\x05 \x01(\r\x12\x17\n\x0fincludeVersions\x18\x06 \x01(\x08\x12\'\n\x07streams\x18\x07 \x03(\x0b\x32\x16.v5api.StreamCSVConfig\"H\n\tQueryType\x12\x19\n\x15\x41LIGNED_WINDOWS_QUERY\x10\x00\x12\x11\n\rWINDOWS_QUERY\x10\x01\x12\r\n\tRAW_QUERY\x10\x02\"Q\n\x13GenerateCSVResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x10\n\x08isHeader\x18\x02 \x01(\x08\x12\x0b\n\x03row\x18\x03 \x03(\t\"J\n\x0eSQLQueryParams\x12\r\n\x05query\x18\x01 \x01(\t\x12\x0e\n\x06params\x18\x02 \x03(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"D\n\x10SQLQueryResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x13\n\x0bSQLQueryRow\x18\x02 \x03(\x0c\"\x14\n\x04Role\x12\x0c\n\x04name\x18\x01 \x01(\t\"\x94\x01\n\x19SetCompactionConfigParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x18\n\x10\x43ompactedVersion\x18\x02 \x01(\x04\x12>\n\x17reducedResolutionRanges\x18\x03 \x03(\x0b\x32\x1d.v5api.ReducedResolutionRange\x12\x0f\n\x07unused0\x18\x04 \x01(\x04\":\n\x1bSetCompactionConfigResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\")\n\x19GetCompactionConfigParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"\xc1\x01\n\x1bGetCompactionConfigResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x1a\n\x12LatestMajorVersion\x18\x02 \x01(\x04\x12\x18\n\x10\x43ompactedVersion\x18\x03 \x01(\x04\x12>\n\x17reducedResolutionRanges\x18\x04 \x03(\x0b\x32\x1d.v5api.ReducedResolutionRange\x12\x0f\n\x07unused0\x18\x05 \x01(\x04\"H\n\x16ReducedResolutionRange\x12\r\n\x05Start\x18\x01 \x01(\x03\x12\x0b\n\x03\x45nd\x18\x02 \x01(\x03\x12\x12\n\nResolution\x18\x03 \x01(\r*<\n\x0bMergePolicy\x12\t\n\x05NEVER\x10\x00\x12\t\n\x05\x45QUAL\x10\x01\x12\n\n\x06RETAIN\x10\x02\x12\x0b\n\x07REPLACE\x10\x03\x32\xb3\r\n\x05\x42TrDB\x12?\n\tRawValues\x12\x16.v5api.RawValuesParams\x1a\x18.v5api.RawValuesResponse0\x01\x12I\n\x0e\x41rrowRawValues\x12\x16.v5api.RawValuesParams\x1a\x1d.v5api.ArrowRawValuesResponse0\x01\x12T\n\x10\x41rrowMultiValues\x12\x1d.v5api.ArrowMultiValuesParams\x1a\x1f.v5api.ArrowMultiValuesResponse0\x01\x12N\n\x0e\x41lignedWindows\x12\x1b.v5api.AlignedWindowsParams\x1a\x1d.v5api.AlignedWindowsResponse0\x01\x12X\n\x13\x41rrowAlignedWindows\x12\x1b.v5api.AlignedWindowsParams\x1a\".v5api.ArrowAlignedWindowsResponse0\x01\x12\x39\n\x07Windows\x12\x14.v5api.WindowsParams\x1a\x16.v5api.WindowsResponse0\x01\x12\x43\n\x0c\x41rrowWindows\x12\x14.v5api.WindowsParams\x1a\x1b.v5api.ArrowWindowsResponse0\x01\x12@\n\nStreamInfo\x12\x17.v5api.StreamInfoParams\x1a\x19.v5api.StreamInfoResponse\x12^\n\x14SetStreamAnnotations\x12!.v5api.SetStreamAnnotationsParams\x1a#.v5api.SetStreamAnnotationsResponse\x12I\n\rSetStreamTags\x12\x1a.v5api.SetStreamTagsParams\x1a\x1c.v5api.SetStreamTagsResponse\x12\x34\n\x06\x43reate\x12\x13.v5api.CreateParams\x1a\x15.v5api.CreateResponse\x12Q\n\x0fListCollections\x12\x1c.v5api.ListCollectionsParams\x1a\x1e.v5api.ListCollectionsResponse0\x01\x12K\n\rLookupStreams\x12\x1a.v5api.LookupStreamsParams\x1a\x1c.v5api.LookupStreamsResponse0\x01\x12\x37\n\x07Nearest\x12\x14.v5api.NearestParams\x1a\x16.v5api.NearestResponse\x12\x39\n\x07\x43hanges\x12\x14.v5api.ChangesParams\x1a\x16.v5api.ChangesResponse0\x01\x12\x34\n\x06Insert\x12\x13.v5api.InsertParams\x1a\x15.v5api.InsertResponse\x12>\n\x0b\x41rrowInsert\x12\x18.v5api.ArrowInsertParams\x1a\x15.v5api.InsertResponse\x12\x34\n\x06\x44\x65lete\x12\x13.v5api.DeleteParams\x1a\x15.v5api.DeleteResponse\x12.\n\x04Info\x12\x11.v5api.InfoParams\x1a\x13.v5api.InfoResponse\x12\x43\n\x0b\x46\x61ultInject\x12\x18.v5api.FaultInjectParams\x1a\x1a.v5api.FaultInjectResponse\x12\x31\n\x05\x46lush\x12\x12.v5api.FlushParams\x1a\x14.v5api.FlushResponse\x12@\n\nObliterate\x12\x17.v5api.ObliterateParams\x1a\x19.v5api.ObliterateResponse\x12L\n\x10GetMetadataUsage\x12\x1a.v5api.MetadataUsageParams\x1a\x1c.v5api.MetadataUsageResponse\x12\x45\n\x0bGenerateCSV\x12\x18.v5api.GenerateCSVParams\x1a\x1a.v5api.GenerateCSVResponse0\x01\x12<\n\x08SQLQuery\x12\x15.v5api.SQLQueryParams\x1a\x17.v5api.SQLQueryResponse0\x01\x62\x06proto3') + +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'btrdb_pb2', globals()) +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + _MERGEPOLICY._serialized_start=6305 + _MERGEPOLICY._serialized_end=6365 + _RAWVALUESPARAMS._serialized_start=22 + _RAWVALUESPARAMS._serialized_end=103 + _RAWVALUESRESPONSE._serialized_start=105 + _RAWVALUESRESPONSE._serialized_end=230 + _ARROWRAWVALUESRESPONSE._serialized_start=232 + _ARROWRAWVALUESRESPONSE._serialized_end=349 + _ARROWMULTIVALUESPARAMS._serialized_start=351 + _ARROWMULTIVALUESPARAMS._serialized_end=461 + _ARROWMULTIVALUESRESPONSE._serialized_start=463 + _ARROWMULTIVALUESRESPONSE._serialized_end=538 + _RAWPOINTVEC._serialized_start=540 + _RAWPOINTVEC._serialized_end=582 + _ALIGNEDWINDOWSPARAMS._serialized_start=584 + _ALIGNEDWINDOWSPARAMS._serialized_end=690 + _ALIGNEDWINDOWSRESPONSE._serialized_start=693 + _ALIGNEDWINDOWSRESPONSE._serialized_end=824 + _ARROWALIGNEDWINDOWSRESPONSE._serialized_start=826 + _ARROWALIGNEDWINDOWSRESPONSE._serialized_end=948 + _WINDOWSPARAMS._serialized_start=950 + _WINDOWSPARAMS._serialized_end=1059 + _WINDOWSRESPONSE._serialized_start=1061 + _WINDOWSRESPONSE._serialized_end=1185 + _ARROWWINDOWSRESPONSE._serialized_start=1187 + _ARROWWINDOWSRESPONSE._serialized_end=1302 + _STREAMINFOPARAMS._serialized_start=1304 + _STREAMINFOPARAMS._serialized_end=1408 + _STREAMINFORESPONSE._serialized_start=1411 + _STREAMINFORESPONSE._serialized_end=1549 + _STREAMDESCRIPTOR._serialized_start=1552 + _STREAMDESCRIPTOR._serialized_end=1704 + _SETSTREAMANNOTATIONSPARAMS._serialized_start=1707 + _SETSTREAMANNOTATIONSPARAMS._serialized_end=1837 + _SETSTREAMANNOTATIONSRESPONSE._serialized_start=1839 + _SETSTREAMANNOTATIONSRESPONSE._serialized_end=1898 + _SETSTREAMTAGSPARAMS._serialized_start=1901 + _SETSTREAMTAGSPARAMS._serialized_end=2039 + _SETSTREAMTAGSRESPONSE._serialized_start=2041 + _SETSTREAMTAGSRESPONSE._serialized_end=2093 + _CREATEPARAMS._serialized_start=2095 + _CREATEPARAMS._serialized_end=2218 + _CREATERESPONSE._serialized_start=2220 + _CREATERESPONSE._serialized_end=2265 + _METADATAUSAGEPARAMS._serialized_start=2267 + _METADATAUSAGEPARAMS._serialized_end=2331 + _METADATAUSAGERESPONSE._serialized_start=2333 + _METADATAUSAGERESPONSE._serialized_end=2454 + _KEYCOUNT._serialized_start=2456 + _KEYCOUNT._serialized_end=2494 + _LISTCOLLECTIONSPARAMS._serialized_start=2496 + _LISTCOLLECTIONSPARAMS._serialized_end=2562 + _LISTCOLLECTIONSRESPONSE._serialized_start=2564 + _LISTCOLLECTIONSRESPONSE._serialized_end=2639 + _LOOKUPSTREAMSPARAMS._serialized_start=2642 + _LOOKUPSTREAMSPARAMS._serialized_end=2813 + _LOOKUPSTREAMSRESPONSE._serialized_start=2815 + _LOOKUPSTREAMSRESPONSE._serialized_end=2909 + _NEARESTPARAMS._serialized_start=2911 + _NEARESTPARAMS._serialized_end=2994 + _NEARESTRESPONSE._serialized_start=2996 + _NEARESTRESPONSE._serialized_end=3118 + _CHANGESPARAMS._serialized_start=3120 + _CHANGESPARAMS._serialized_end=3205 + _CHANGESRESPONSE._serialized_start=3207 + _CHANGESRESPONSE._serialized_end=3334 + _ROUNDSPEC._serialized_start=3336 + _ROUNDSPEC._serialized_end=3371 + _INSERTPARAMS._serialized_start=3374 + _INSERTPARAMS._serialized_end=3527 + _ARROWINSERTPARAMS._serialized_start=3530 + _ARROWINSERTPARAMS._serialized_end=3675 + _INSERTRESPONSE._serialized_start=3677 + _INSERTRESPONSE._serialized_end=3766 + _DELETEPARAMS._serialized_start=3768 + _DELETEPARAMS._serialized_end=3824 + _DELETERESPONSE._serialized_start=3826 + _DELETERESPONSE._serialized_end=3915 + _INFOPARAMS._serialized_start=3917 + _INFOPARAMS._serialized_end=3929 + _INFORESPONSE._serialized_start=3932 + _INFORESPONSE._serialized_end=4094 + _PROXYINFO._serialized_start=4096 + _PROXYINFO._serialized_end=4131 + _FAULTINJECTPARAMS._serialized_start=4133 + _FAULTINJECTPARAMS._serialized_end=4182 + _FAULTINJECTRESPONSE._serialized_start=4184 + _FAULTINJECTRESPONSE._serialized_end=4246 + _FLUSHPARAMS._serialized_start=4248 + _FLUSHPARAMS._serialized_end=4275 + _FLUSHRESPONSE._serialized_start=4277 + _FLUSHRESPONSE._serialized_end=4365 + _OBLITERATEPARAMS._serialized_start=4367 + _OBLITERATEPARAMS._serialized_end=4399 + _OBLITERATERESPONSE._serialized_start=4401 + _OBLITERATERESPONSE._serialized_end=4450 + _RAWPOINT._serialized_start=4452 + _RAWPOINT._serialized_end=4491 + _STATPOINT._serialized_start=4493 + _STATPOINT._serialized_end=4589 + _CHANGEDRANGE._serialized_start=4591 + _CHANGEDRANGE._serialized_end=4633 + _STATUS._serialized_start=4635 + _STATUS._serialized_end=4697 + _MASH._serialized_start=4700 + _MASH._serialized_end=4852 + _MEMBER._serialized_start=4855 + _MEMBER._serialized_end=5050 + _KEYOPTVALUE._serialized_start=5052 + _KEYOPTVALUE._serialized_end=5108 + _OPTVALUE._serialized_start=5110 + _OPTVALUE._serialized_end=5135 + _KEYVALUE._serialized_start=5137 + _KEYVALUE._serialized_end=5175 + _STREAMCSVCONFIG._serialized_start=5177 + _STREAMCSVCONFIG._serialized_end=5240 + _GENERATECSVPARAMS._serialized_start=5243 + _GENERATECSVPARAMS._serialized_end=5528 + _GENERATECSVPARAMS_QUERYTYPE._serialized_start=5456 + _GENERATECSVPARAMS_QUERYTYPE._serialized_end=5528 + _GENERATECSVRESPONSE._serialized_start=5530 + _GENERATECSVRESPONSE._serialized_end=5611 + _SQLQUERYPARAMS._serialized_start=5613 + _SQLQUERYPARAMS._serialized_end=5687 + _SQLQUERYRESPONSE._serialized_start=5689 + _SQLQUERYRESPONSE._serialized_end=5757 + _ROLE._serialized_start=5759 + _ROLE._serialized_end=5779 + _SETCOMPACTIONCONFIGPARAMS._serialized_start=5782 + _SETCOMPACTIONCONFIGPARAMS._serialized_end=5930 + _SETCOMPACTIONCONFIGRESPONSE._serialized_start=5932 + _SETCOMPACTIONCONFIGRESPONSE._serialized_end=5990 + _GETCOMPACTIONCONFIGPARAMS._serialized_start=5992 + _GETCOMPACTIONCONFIGPARAMS._serialized_end=6033 + _GETCOMPACTIONCONFIGRESPONSE._serialized_start=6036 + _GETCOMPACTIONCONFIGRESPONSE._serialized_end=6229 + _REDUCEDRESOLUTIONRANGE._serialized_start=6231 + _REDUCEDRESOLUTIONRANGE._serialized_end=6303 + _BTRDB._serialized_start=6368 + _BTRDB._serialized_end=8083 # @@protoc_insertion_point(module_scope) diff --git a/btrdb/grpcinterface/btrdb_pb2.pyi b/btrdb/grpcinterface/btrdb_pb2.pyi new file mode 100644 index 0000000..ff9b6fc --- /dev/null +++ b/btrdb/grpcinterface/btrdb_pb2.pyi @@ -0,0 +1,720 @@ +from google.protobuf.internal import containers as _containers +from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from typing import ClassVar as _ClassVar, Iterable as _Iterable, Mapping as _Mapping, Optional as _Optional, Union as _Union + +DESCRIPTOR: _descriptor.FileDescriptor +EQUAL: MergePolicy +NEVER: MergePolicy +REPLACE: MergePolicy +RETAIN: MergePolicy + +class AlignedWindowsParams(_message.Message): + __slots__ = ["end", "pointWidth", "start", "uuid", "versionMajor"] + END_FIELD_NUMBER: _ClassVar[int] + POINTWIDTH_FIELD_NUMBER: _ClassVar[int] + START_FIELD_NUMBER: _ClassVar[int] + UUID_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + end: int + pointWidth: int + start: int + uuid: bytes + versionMajor: int + def __init__(self, uuid: _Optional[bytes] = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., versionMajor: _Optional[int] = ..., pointWidth: _Optional[int] = ...) -> None: ... + +class AlignedWindowsResponse(_message.Message): + __slots__ = ["stat", "values", "versionMajor", "versionMinor"] + STAT_FIELD_NUMBER: _ClassVar[int] + VALUES_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + stat: Status + values: _containers.RepeatedCompositeFieldContainer[StatPoint] + versionMajor: int + versionMinor: int + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., values: _Optional[_Iterable[_Union[StatPoint, _Mapping]]] = ...) -> None: ... + +class ArrowAlignedWindowsResponse(_message.Message): + __slots__ = ["arrowBytes", "stat", "versionMajor", "versionMinor"] + ARROWBYTES_FIELD_NUMBER: _ClassVar[int] + STAT_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + arrowBytes: bytes + stat: Status + versionMajor: int + versionMinor: int + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., arrowBytes: _Optional[bytes] = ...) -> None: ... + +class ArrowInsertParams(_message.Message): + __slots__ = ["arrowBytes", "merge_policy", "rounding", "sync", "uuid"] + ARROWBYTES_FIELD_NUMBER: _ClassVar[int] + MERGE_POLICY_FIELD_NUMBER: _ClassVar[int] + ROUNDING_FIELD_NUMBER: _ClassVar[int] + SYNC_FIELD_NUMBER: _ClassVar[int] + UUID_FIELD_NUMBER: _ClassVar[int] + arrowBytes: bytes + merge_policy: MergePolicy + rounding: RoundSpec + sync: bool + uuid: bytes + def __init__(self, uuid: _Optional[bytes] = ..., sync: bool = ..., merge_policy: _Optional[_Union[MergePolicy, str]] = ..., rounding: _Optional[_Union[RoundSpec, _Mapping]] = ..., arrowBytes: _Optional[bytes] = ...) -> None: ... + +class ArrowMultiValuesParams(_message.Message): + __slots__ = ["end", "snapPeriodNs", "start", "uuid", "versionMajor"] + END_FIELD_NUMBER: _ClassVar[int] + SNAPPERIODNS_FIELD_NUMBER: _ClassVar[int] + START_FIELD_NUMBER: _ClassVar[int] + UUID_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + end: int + snapPeriodNs: int + start: int + uuid: _containers.RepeatedScalarFieldContainer[bytes] + versionMajor: _containers.RepeatedScalarFieldContainer[int] + def __init__(self, uuid: _Optional[_Iterable[bytes]] = ..., versionMajor: _Optional[_Iterable[int]] = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., snapPeriodNs: _Optional[int] = ...) -> None: ... + +class ArrowMultiValuesResponse(_message.Message): + __slots__ = ["arrowBytes", "stat"] + ARROWBYTES_FIELD_NUMBER: _ClassVar[int] + STAT_FIELD_NUMBER: _ClassVar[int] + arrowBytes: bytes + stat: Status + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., arrowBytes: _Optional[bytes] = ...) -> None: ... + +class ArrowRawValuesResponse(_message.Message): + __slots__ = ["arrowBytes", "stat", "versionMajor", "versionMinor"] + ARROWBYTES_FIELD_NUMBER: _ClassVar[int] + STAT_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + arrowBytes: bytes + stat: Status + versionMajor: int + versionMinor: int + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., arrowBytes: _Optional[bytes] = ...) -> None: ... + +class ArrowWindowsResponse(_message.Message): + __slots__ = ["arrowBytes", "stat", "versionMajor", "versionMinor"] + ARROWBYTES_FIELD_NUMBER: _ClassVar[int] + STAT_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + arrowBytes: bytes + stat: Status + versionMajor: int + versionMinor: int + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., arrowBytes: _Optional[bytes] = ...) -> None: ... + +class ChangedRange(_message.Message): + __slots__ = ["end", "start"] + END_FIELD_NUMBER: _ClassVar[int] + START_FIELD_NUMBER: _ClassVar[int] + end: int + start: int + def __init__(self, start: _Optional[int] = ..., end: _Optional[int] = ...) -> None: ... + +class ChangesParams(_message.Message): + __slots__ = ["fromMajor", "resolution", "toMajor", "uuid"] + FROMMAJOR_FIELD_NUMBER: _ClassVar[int] + RESOLUTION_FIELD_NUMBER: _ClassVar[int] + TOMAJOR_FIELD_NUMBER: _ClassVar[int] + UUID_FIELD_NUMBER: _ClassVar[int] + fromMajor: int + resolution: int + toMajor: int + uuid: bytes + def __init__(self, uuid: _Optional[bytes] = ..., fromMajor: _Optional[int] = ..., toMajor: _Optional[int] = ..., resolution: _Optional[int] = ...) -> None: ... + +class ChangesResponse(_message.Message): + __slots__ = ["ranges", "stat", "versionMajor", "versionMinor"] + RANGES_FIELD_NUMBER: _ClassVar[int] + STAT_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + ranges: _containers.RepeatedCompositeFieldContainer[ChangedRange] + stat: Status + versionMajor: int + versionMinor: int + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., ranges: _Optional[_Iterable[_Union[ChangedRange, _Mapping]]] = ...) -> None: ... + +class CreateParams(_message.Message): + __slots__ = ["annotations", "collection", "tags", "uuid"] + ANNOTATIONS_FIELD_NUMBER: _ClassVar[int] + COLLECTION_FIELD_NUMBER: _ClassVar[int] + TAGS_FIELD_NUMBER: _ClassVar[int] + UUID_FIELD_NUMBER: _ClassVar[int] + annotations: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + collection: str + tags: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + uuid: bytes + def __init__(self, uuid: _Optional[bytes] = ..., collection: _Optional[str] = ..., tags: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., annotations: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ...) -> None: ... + +class CreateResponse(_message.Message): + __slots__ = ["stat"] + STAT_FIELD_NUMBER: _ClassVar[int] + stat: Status + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ...) -> None: ... + +class DeleteParams(_message.Message): + __slots__ = ["end", "start", "uuid"] + END_FIELD_NUMBER: _ClassVar[int] + START_FIELD_NUMBER: _ClassVar[int] + UUID_FIELD_NUMBER: _ClassVar[int] + end: int + start: int + uuid: bytes + def __init__(self, uuid: _Optional[bytes] = ..., start: _Optional[int] = ..., end: _Optional[int] = ...) -> None: ... + +class DeleteResponse(_message.Message): + __slots__ = ["stat", "versionMajor", "versionMinor"] + STAT_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + stat: Status + versionMajor: int + versionMinor: int + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ...) -> None: ... + +class FaultInjectParams(_message.Message): + __slots__ = ["params", "type"] + PARAMS_FIELD_NUMBER: _ClassVar[int] + TYPE_FIELD_NUMBER: _ClassVar[int] + params: bytes + type: int + def __init__(self, type: _Optional[int] = ..., params: _Optional[bytes] = ...) -> None: ... + +class FaultInjectResponse(_message.Message): + __slots__ = ["rv", "stat"] + RV_FIELD_NUMBER: _ClassVar[int] + STAT_FIELD_NUMBER: _ClassVar[int] + rv: bytes + stat: Status + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., rv: _Optional[bytes] = ...) -> None: ... + +class FlushParams(_message.Message): + __slots__ = ["uuid"] + UUID_FIELD_NUMBER: _ClassVar[int] + uuid: bytes + def __init__(self, uuid: _Optional[bytes] = ...) -> None: ... + +class FlushResponse(_message.Message): + __slots__ = ["stat", "versionMajor", "versionMinor"] + STAT_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + stat: Status + versionMajor: int + versionMinor: int + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ...) -> None: ... + +class GenerateCSVParams(_message.Message): + __slots__ = ["depth", "endTime", "includeVersions", "queryType", "startTime", "streams", "windowSize"] + class QueryType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = [] + ALIGNED_WINDOWS_QUERY: GenerateCSVParams.QueryType + DEPTH_FIELD_NUMBER: _ClassVar[int] + ENDTIME_FIELD_NUMBER: _ClassVar[int] + INCLUDEVERSIONS_FIELD_NUMBER: _ClassVar[int] + QUERYTYPE_FIELD_NUMBER: _ClassVar[int] + RAW_QUERY: GenerateCSVParams.QueryType + STARTTIME_FIELD_NUMBER: _ClassVar[int] + STREAMS_FIELD_NUMBER: _ClassVar[int] + WINDOWSIZE_FIELD_NUMBER: _ClassVar[int] + WINDOWS_QUERY: GenerateCSVParams.QueryType + depth: int + endTime: int + includeVersions: bool + queryType: GenerateCSVParams.QueryType + startTime: int + streams: _containers.RepeatedCompositeFieldContainer[StreamCSVConfig] + windowSize: int + def __init__(self, queryType: _Optional[_Union[GenerateCSVParams.QueryType, str]] = ..., startTime: _Optional[int] = ..., endTime: _Optional[int] = ..., windowSize: _Optional[int] = ..., depth: _Optional[int] = ..., includeVersions: bool = ..., streams: _Optional[_Iterable[_Union[StreamCSVConfig, _Mapping]]] = ...) -> None: ... + +class GenerateCSVResponse(_message.Message): + __slots__ = ["isHeader", "row", "stat"] + ISHEADER_FIELD_NUMBER: _ClassVar[int] + ROW_FIELD_NUMBER: _ClassVar[int] + STAT_FIELD_NUMBER: _ClassVar[int] + isHeader: bool + row: _containers.RepeatedScalarFieldContainer[str] + stat: Status + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., isHeader: bool = ..., row: _Optional[_Iterable[str]] = ...) -> None: ... + +class GetCompactionConfigParams(_message.Message): + __slots__ = ["uuid"] + UUID_FIELD_NUMBER: _ClassVar[int] + uuid: bytes + def __init__(self, uuid: _Optional[bytes] = ...) -> None: ... + +class GetCompactionConfigResponse(_message.Message): + __slots__ = ["CompactedVersion", "LatestMajorVersion", "reducedResolutionRanges", "stat", "unused0"] + COMPACTEDVERSION_FIELD_NUMBER: _ClassVar[int] + CompactedVersion: int + LATESTMAJORVERSION_FIELD_NUMBER: _ClassVar[int] + LatestMajorVersion: int + REDUCEDRESOLUTIONRANGES_FIELD_NUMBER: _ClassVar[int] + STAT_FIELD_NUMBER: _ClassVar[int] + UNUSED0_FIELD_NUMBER: _ClassVar[int] + reducedResolutionRanges: _containers.RepeatedCompositeFieldContainer[ReducedResolutionRange] + stat: Status + unused0: int + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., LatestMajorVersion: _Optional[int] = ..., CompactedVersion: _Optional[int] = ..., reducedResolutionRanges: _Optional[_Iterable[_Union[ReducedResolutionRange, _Mapping]]] = ..., unused0: _Optional[int] = ...) -> None: ... + +class InfoParams(_message.Message): + __slots__ = [] + def __init__(self) -> None: ... + +class InfoResponse(_message.Message): + __slots__ = ["build", "majorVersion", "mash", "minorVersion", "proxy", "stat"] + BUILD_FIELD_NUMBER: _ClassVar[int] + MAJORVERSION_FIELD_NUMBER: _ClassVar[int] + MASH_FIELD_NUMBER: _ClassVar[int] + MINORVERSION_FIELD_NUMBER: _ClassVar[int] + PROXY_FIELD_NUMBER: _ClassVar[int] + STAT_FIELD_NUMBER: _ClassVar[int] + build: str + majorVersion: int + mash: Mash + minorVersion: int + proxy: ProxyInfo + stat: Status + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., mash: _Optional[_Union[Mash, _Mapping]] = ..., majorVersion: _Optional[int] = ..., minorVersion: _Optional[int] = ..., build: _Optional[str] = ..., proxy: _Optional[_Union[ProxyInfo, _Mapping]] = ...) -> None: ... + +class InsertParams(_message.Message): + __slots__ = ["merge_policy", "rounding", "sync", "uuid", "values"] + MERGE_POLICY_FIELD_NUMBER: _ClassVar[int] + ROUNDING_FIELD_NUMBER: _ClassVar[int] + SYNC_FIELD_NUMBER: _ClassVar[int] + UUID_FIELD_NUMBER: _ClassVar[int] + VALUES_FIELD_NUMBER: _ClassVar[int] + merge_policy: MergePolicy + rounding: RoundSpec + sync: bool + uuid: bytes + values: _containers.RepeatedCompositeFieldContainer[RawPoint] + def __init__(self, uuid: _Optional[bytes] = ..., sync: bool = ..., merge_policy: _Optional[_Union[MergePolicy, str]] = ..., rounding: _Optional[_Union[RoundSpec, _Mapping]] = ..., values: _Optional[_Iterable[_Union[RawPoint, _Mapping]]] = ...) -> None: ... + +class InsertResponse(_message.Message): + __slots__ = ["stat", "versionMajor", "versionMinor"] + STAT_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + stat: Status + versionMajor: int + versionMinor: int + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ...) -> None: ... + +class KeyCount(_message.Message): + __slots__ = ["count", "key"] + COUNT_FIELD_NUMBER: _ClassVar[int] + KEY_FIELD_NUMBER: _ClassVar[int] + count: int + key: str + def __init__(self, key: _Optional[str] = ..., count: _Optional[int] = ...) -> None: ... + +class KeyOptValue(_message.Message): + __slots__ = ["key", "val"] + KEY_FIELD_NUMBER: _ClassVar[int] + VAL_FIELD_NUMBER: _ClassVar[int] + key: str + val: OptValue + def __init__(self, key: _Optional[str] = ..., val: _Optional[_Union[OptValue, _Mapping]] = ...) -> None: ... + +class KeyValue(_message.Message): + __slots__ = ["key", "value"] + KEY_FIELD_NUMBER: _ClassVar[int] + VALUE_FIELD_NUMBER: _ClassVar[int] + key: str + value: str + def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ... + +class ListCollectionsParams(_message.Message): + __slots__ = ["prefix", "role"] + PREFIX_FIELD_NUMBER: _ClassVar[int] + ROLE_FIELD_NUMBER: _ClassVar[int] + prefix: str + role: Role + def __init__(self, prefix: _Optional[str] = ..., role: _Optional[_Union[Role, _Mapping]] = ...) -> None: ... + +class ListCollectionsResponse(_message.Message): + __slots__ = ["collections", "stat"] + COLLECTIONS_FIELD_NUMBER: _ClassVar[int] + STAT_FIELD_NUMBER: _ClassVar[int] + collections: _containers.RepeatedScalarFieldContainer[str] + stat: Status + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., collections: _Optional[_Iterable[str]] = ...) -> None: ... + +class LookupStreamsParams(_message.Message): + __slots__ = ["annotations", "collection", "isCollectionPrefix", "role", "tags"] + ANNOTATIONS_FIELD_NUMBER: _ClassVar[int] + COLLECTION_FIELD_NUMBER: _ClassVar[int] + ISCOLLECTIONPREFIX_FIELD_NUMBER: _ClassVar[int] + ROLE_FIELD_NUMBER: _ClassVar[int] + TAGS_FIELD_NUMBER: _ClassVar[int] + annotations: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + collection: str + isCollectionPrefix: bool + role: Role + tags: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + def __init__(self, collection: _Optional[str] = ..., isCollectionPrefix: bool = ..., tags: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., annotations: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., role: _Optional[_Union[Role, _Mapping]] = ...) -> None: ... + +class LookupStreamsResponse(_message.Message): + __slots__ = ["results", "stat"] + RESULTS_FIELD_NUMBER: _ClassVar[int] + STAT_FIELD_NUMBER: _ClassVar[int] + results: _containers.RepeatedCompositeFieldContainer[StreamDescriptor] + stat: Status + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., results: _Optional[_Iterable[_Union[StreamDescriptor, _Mapping]]] = ...) -> None: ... + +class Mash(_message.Message): + __slots__ = ["healthy", "leader", "leaderRevision", "members", "revision", "totalWeight", "unmapped"] + HEALTHY_FIELD_NUMBER: _ClassVar[int] + LEADERREVISION_FIELD_NUMBER: _ClassVar[int] + LEADER_FIELD_NUMBER: _ClassVar[int] + MEMBERS_FIELD_NUMBER: _ClassVar[int] + REVISION_FIELD_NUMBER: _ClassVar[int] + TOTALWEIGHT_FIELD_NUMBER: _ClassVar[int] + UNMAPPED_FIELD_NUMBER: _ClassVar[int] + healthy: bool + leader: str + leaderRevision: int + members: _containers.RepeatedCompositeFieldContainer[Member] + revision: int + totalWeight: int + unmapped: float + def __init__(self, revision: _Optional[int] = ..., leader: _Optional[str] = ..., leaderRevision: _Optional[int] = ..., totalWeight: _Optional[int] = ..., healthy: bool = ..., unmapped: _Optional[float] = ..., members: _Optional[_Iterable[_Union[Member, _Mapping]]] = ...) -> None: ... + +class Member(_message.Message): + __slots__ = ["enabled", "end", "grpcEndpoints", "hash", "httpEndpoints", "nodename", "readPreference", "start", "up", "weight"] + ENABLED_FIELD_NUMBER: _ClassVar[int] + END_FIELD_NUMBER: _ClassVar[int] + GRPCENDPOINTS_FIELD_NUMBER: _ClassVar[int] + HASH_FIELD_NUMBER: _ClassVar[int] + HTTPENDPOINTS_FIELD_NUMBER: _ClassVar[int] + IN_FIELD_NUMBER: _ClassVar[int] + NODENAME_FIELD_NUMBER: _ClassVar[int] + READPREFERENCE_FIELD_NUMBER: _ClassVar[int] + START_FIELD_NUMBER: _ClassVar[int] + UP_FIELD_NUMBER: _ClassVar[int] + WEIGHT_FIELD_NUMBER: _ClassVar[int] + enabled: bool + end: int + grpcEndpoints: str + hash: int + httpEndpoints: str + nodename: str + readPreference: float + start: int + up: bool + weight: int + def __init__(self, hash: _Optional[int] = ..., nodename: _Optional[str] = ..., up: bool = ..., enabled: bool = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., weight: _Optional[int] = ..., readPreference: _Optional[float] = ..., httpEndpoints: _Optional[str] = ..., grpcEndpoints: _Optional[str] = ..., **kwargs) -> None: ... + +class MetadataUsageParams(_message.Message): + __slots__ = ["prefix", "role"] + PREFIX_FIELD_NUMBER: _ClassVar[int] + ROLE_FIELD_NUMBER: _ClassVar[int] + prefix: str + role: Role + def __init__(self, prefix: _Optional[str] = ..., role: _Optional[_Union[Role, _Mapping]] = ...) -> None: ... + +class MetadataUsageResponse(_message.Message): + __slots__ = ["annotations", "stat", "tags"] + ANNOTATIONS_FIELD_NUMBER: _ClassVar[int] + STAT_FIELD_NUMBER: _ClassVar[int] + TAGS_FIELD_NUMBER: _ClassVar[int] + annotations: _containers.RepeatedCompositeFieldContainer[KeyCount] + stat: Status + tags: _containers.RepeatedCompositeFieldContainer[KeyCount] + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., tags: _Optional[_Iterable[_Union[KeyCount, _Mapping]]] = ..., annotations: _Optional[_Iterable[_Union[KeyCount, _Mapping]]] = ...) -> None: ... + +class NearestParams(_message.Message): + __slots__ = ["backward", "time", "uuid", "versionMajor"] + BACKWARD_FIELD_NUMBER: _ClassVar[int] + TIME_FIELD_NUMBER: _ClassVar[int] + UUID_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + backward: bool + time: int + uuid: bytes + versionMajor: int + def __init__(self, uuid: _Optional[bytes] = ..., time: _Optional[int] = ..., versionMajor: _Optional[int] = ..., backward: bool = ...) -> None: ... + +class NearestResponse(_message.Message): + __slots__ = ["stat", "value", "versionMajor", "versionMinor"] + STAT_FIELD_NUMBER: _ClassVar[int] + VALUE_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + stat: Status + value: RawPoint + versionMajor: int + versionMinor: int + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., value: _Optional[_Union[RawPoint, _Mapping]] = ...) -> None: ... + +class ObliterateParams(_message.Message): + __slots__ = ["uuid"] + UUID_FIELD_NUMBER: _ClassVar[int] + uuid: bytes + def __init__(self, uuid: _Optional[bytes] = ...) -> None: ... + +class ObliterateResponse(_message.Message): + __slots__ = ["stat"] + STAT_FIELD_NUMBER: _ClassVar[int] + stat: Status + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ...) -> None: ... + +class OptValue(_message.Message): + __slots__ = ["value"] + VALUE_FIELD_NUMBER: _ClassVar[int] + value: str + def __init__(self, value: _Optional[str] = ...) -> None: ... + +class ProxyInfo(_message.Message): + __slots__ = ["proxyEndpoints"] + PROXYENDPOINTS_FIELD_NUMBER: _ClassVar[int] + proxyEndpoints: _containers.RepeatedScalarFieldContainer[str] + def __init__(self, proxyEndpoints: _Optional[_Iterable[str]] = ...) -> None: ... + +class RawPoint(_message.Message): + __slots__ = ["time", "value"] + TIME_FIELD_NUMBER: _ClassVar[int] + VALUE_FIELD_NUMBER: _ClassVar[int] + time: int + value: float + def __init__(self, time: _Optional[int] = ..., value: _Optional[float] = ...) -> None: ... + +class RawPointVec(_message.Message): + __slots__ = ["time", "value"] + TIME_FIELD_NUMBER: _ClassVar[int] + VALUE_FIELD_NUMBER: _ClassVar[int] + time: int + value: _containers.RepeatedScalarFieldContainer[float] + def __init__(self, time: _Optional[int] = ..., value: _Optional[_Iterable[float]] = ...) -> None: ... + +class RawValuesParams(_message.Message): + __slots__ = ["end", "start", "uuid", "versionMajor"] + END_FIELD_NUMBER: _ClassVar[int] + START_FIELD_NUMBER: _ClassVar[int] + UUID_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + end: int + start: int + uuid: bytes + versionMajor: int + def __init__(self, uuid: _Optional[bytes] = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., versionMajor: _Optional[int] = ...) -> None: ... + +class RawValuesResponse(_message.Message): + __slots__ = ["stat", "values", "versionMajor", "versionMinor"] + STAT_FIELD_NUMBER: _ClassVar[int] + VALUES_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + stat: Status + values: _containers.RepeatedCompositeFieldContainer[RawPoint] + versionMajor: int + versionMinor: int + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., values: _Optional[_Iterable[_Union[RawPoint, _Mapping]]] = ...) -> None: ... + +class ReducedResolutionRange(_message.Message): + __slots__ = ["End", "Resolution", "Start"] + END_FIELD_NUMBER: _ClassVar[int] + End: int + RESOLUTION_FIELD_NUMBER: _ClassVar[int] + Resolution: int + START_FIELD_NUMBER: _ClassVar[int] + Start: int + def __init__(self, Start: _Optional[int] = ..., End: _Optional[int] = ..., Resolution: _Optional[int] = ...) -> None: ... + +class Role(_message.Message): + __slots__ = ["name"] + NAME_FIELD_NUMBER: _ClassVar[int] + name: str + def __init__(self, name: _Optional[str] = ...) -> None: ... + +class RoundSpec(_message.Message): + __slots__ = ["bits"] + BITS_FIELD_NUMBER: _ClassVar[int] + bits: int + def __init__(self, bits: _Optional[int] = ...) -> None: ... + +class SQLQueryParams(_message.Message): + __slots__ = ["params", "query", "role"] + PARAMS_FIELD_NUMBER: _ClassVar[int] + QUERY_FIELD_NUMBER: _ClassVar[int] + ROLE_FIELD_NUMBER: _ClassVar[int] + params: _containers.RepeatedScalarFieldContainer[str] + query: str + role: Role + def __init__(self, query: _Optional[str] = ..., params: _Optional[_Iterable[str]] = ..., role: _Optional[_Union[Role, _Mapping]] = ...) -> None: ... + +class SQLQueryResponse(_message.Message): + __slots__ = ["SQLQueryRow", "stat"] + SQLQUERYROW_FIELD_NUMBER: _ClassVar[int] + SQLQueryRow: _containers.RepeatedScalarFieldContainer[bytes] + STAT_FIELD_NUMBER: _ClassVar[int] + stat: Status + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., SQLQueryRow: _Optional[_Iterable[bytes]] = ...) -> None: ... + +class SetCompactionConfigParams(_message.Message): + __slots__ = ["CompactedVersion", "reducedResolutionRanges", "unused0", "uuid"] + COMPACTEDVERSION_FIELD_NUMBER: _ClassVar[int] + CompactedVersion: int + REDUCEDRESOLUTIONRANGES_FIELD_NUMBER: _ClassVar[int] + UNUSED0_FIELD_NUMBER: _ClassVar[int] + UUID_FIELD_NUMBER: _ClassVar[int] + reducedResolutionRanges: _containers.RepeatedCompositeFieldContainer[ReducedResolutionRange] + unused0: int + uuid: bytes + def __init__(self, uuid: _Optional[bytes] = ..., CompactedVersion: _Optional[int] = ..., reducedResolutionRanges: _Optional[_Iterable[_Union[ReducedResolutionRange, _Mapping]]] = ..., unused0: _Optional[int] = ...) -> None: ... + +class SetCompactionConfigResponse(_message.Message): + __slots__ = ["stat"] + STAT_FIELD_NUMBER: _ClassVar[int] + stat: Status + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ...) -> None: ... + +class SetStreamAnnotationsParams(_message.Message): + __slots__ = ["changes", "expectedPropertyVersion", "removals", "uuid"] + CHANGES_FIELD_NUMBER: _ClassVar[int] + EXPECTEDPROPERTYVERSION_FIELD_NUMBER: _ClassVar[int] + REMOVALS_FIELD_NUMBER: _ClassVar[int] + UUID_FIELD_NUMBER: _ClassVar[int] + changes: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + expectedPropertyVersion: int + removals: _containers.RepeatedScalarFieldContainer[str] + uuid: bytes + def __init__(self, uuid: _Optional[bytes] = ..., expectedPropertyVersion: _Optional[int] = ..., changes: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., removals: _Optional[_Iterable[str]] = ...) -> None: ... + +class SetStreamAnnotationsResponse(_message.Message): + __slots__ = ["stat"] + STAT_FIELD_NUMBER: _ClassVar[int] + stat: Status + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ...) -> None: ... + +class SetStreamTagsParams(_message.Message): + __slots__ = ["collection", "expectedPropertyVersion", "remove", "tags", "uuid"] + COLLECTION_FIELD_NUMBER: _ClassVar[int] + EXPECTEDPROPERTYVERSION_FIELD_NUMBER: _ClassVar[int] + REMOVE_FIELD_NUMBER: _ClassVar[int] + TAGS_FIELD_NUMBER: _ClassVar[int] + UUID_FIELD_NUMBER: _ClassVar[int] + collection: str + expectedPropertyVersion: int + remove: _containers.RepeatedScalarFieldContainer[str] + tags: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + uuid: bytes + def __init__(self, uuid: _Optional[bytes] = ..., expectedPropertyVersion: _Optional[int] = ..., tags: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., collection: _Optional[str] = ..., remove: _Optional[_Iterable[str]] = ...) -> None: ... + +class SetStreamTagsResponse(_message.Message): + __slots__ = ["stat"] + STAT_FIELD_NUMBER: _ClassVar[int] + stat: Status + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ...) -> None: ... + +class StatPoint(_message.Message): + __slots__ = ["count", "max", "mean", "min", "stddev", "time"] + COUNT_FIELD_NUMBER: _ClassVar[int] + MAX_FIELD_NUMBER: _ClassVar[int] + MEAN_FIELD_NUMBER: _ClassVar[int] + MIN_FIELD_NUMBER: _ClassVar[int] + STDDEV_FIELD_NUMBER: _ClassVar[int] + TIME_FIELD_NUMBER: _ClassVar[int] + count: int + max: float + mean: float + min: float + stddev: float + time: int + def __init__(self, time: _Optional[int] = ..., min: _Optional[float] = ..., mean: _Optional[float] = ..., max: _Optional[float] = ..., count: _Optional[int] = ..., stddev: _Optional[float] = ...) -> None: ... + +class Status(_message.Message): + __slots__ = ["code", "mash", "msg"] + CODE_FIELD_NUMBER: _ClassVar[int] + MASH_FIELD_NUMBER: _ClassVar[int] + MSG_FIELD_NUMBER: _ClassVar[int] + code: int + mash: Mash + msg: str + def __init__(self, code: _Optional[int] = ..., msg: _Optional[str] = ..., mash: _Optional[_Union[Mash, _Mapping]] = ...) -> None: ... + +class StreamCSVConfig(_message.Message): + __slots__ = ["label", "uuid", "version"] + LABEL_FIELD_NUMBER: _ClassVar[int] + UUID_FIELD_NUMBER: _ClassVar[int] + VERSION_FIELD_NUMBER: _ClassVar[int] + label: str + uuid: bytes + version: int + def __init__(self, version: _Optional[int] = ..., label: _Optional[str] = ..., uuid: _Optional[bytes] = ...) -> None: ... + +class StreamDescriptor(_message.Message): + __slots__ = ["annotations", "collection", "propertyVersion", "tags", "uuid"] + ANNOTATIONS_FIELD_NUMBER: _ClassVar[int] + COLLECTION_FIELD_NUMBER: _ClassVar[int] + PROPERTYVERSION_FIELD_NUMBER: _ClassVar[int] + TAGS_FIELD_NUMBER: _ClassVar[int] + UUID_FIELD_NUMBER: _ClassVar[int] + annotations: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + collection: str + propertyVersion: int + tags: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + uuid: bytes + def __init__(self, uuid: _Optional[bytes] = ..., collection: _Optional[str] = ..., tags: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., annotations: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., propertyVersion: _Optional[int] = ...) -> None: ... + +class StreamInfoParams(_message.Message): + __slots__ = ["omitDescriptor", "omitVersion", "role", "uuid"] + OMITDESCRIPTOR_FIELD_NUMBER: _ClassVar[int] + OMITVERSION_FIELD_NUMBER: _ClassVar[int] + ROLE_FIELD_NUMBER: _ClassVar[int] + UUID_FIELD_NUMBER: _ClassVar[int] + omitDescriptor: bool + omitVersion: bool + role: Role + uuid: bytes + def __init__(self, uuid: _Optional[bytes] = ..., omitVersion: bool = ..., omitDescriptor: bool = ..., role: _Optional[_Union[Role, _Mapping]] = ...) -> None: ... + +class StreamInfoResponse(_message.Message): + __slots__ = ["descriptor", "stat", "versionMajor", "versionMinor"] + DESCRIPTOR_FIELD_NUMBER: _ClassVar[int] + STAT_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + descriptor: StreamDescriptor + stat: Status + versionMajor: int + versionMinor: int + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., descriptor: _Optional[_Union[StreamDescriptor, _Mapping]] = ...) -> None: ... + +class WindowsParams(_message.Message): + __slots__ = ["depth", "end", "start", "uuid", "versionMajor", "width"] + DEPTH_FIELD_NUMBER: _ClassVar[int] + END_FIELD_NUMBER: _ClassVar[int] + START_FIELD_NUMBER: _ClassVar[int] + UUID_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + WIDTH_FIELD_NUMBER: _ClassVar[int] + depth: int + end: int + start: int + uuid: bytes + versionMajor: int + width: int + def __init__(self, uuid: _Optional[bytes] = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., versionMajor: _Optional[int] = ..., width: _Optional[int] = ..., depth: _Optional[int] = ...) -> None: ... + +class WindowsResponse(_message.Message): + __slots__ = ["stat", "values", "versionMajor", "versionMinor"] + STAT_FIELD_NUMBER: _ClassVar[int] + VALUES_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + stat: Status + values: _containers.RepeatedCompositeFieldContainer[StatPoint] + versionMajor: int + versionMinor: int + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., values: _Optional[_Iterable[_Union[StatPoint, _Mapping]]] = ...) -> None: ... + +class MergePolicy(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = [] diff --git a/btrdb/grpcinterface/btrdb_pb2_grpc.py b/btrdb/grpcinterface/btrdb_pb2_grpc.py index 89b9ce0..52a72bd 100644 --- a/btrdb/grpcinterface/btrdb_pb2_grpc.py +++ b/btrdb/grpcinterface/btrdb_pb2_grpc.py @@ -1,370 +1,860 @@ # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" import grpc import btrdb.grpcinterface.btrdb_pb2 as btrdb__pb2 class BTrDBStub(object): - # missing associated documentation comment in .proto file - pass - - def __init__(self, channel): - """Constructor. - - Args: - channel: A grpc.Channel. - """ - self.RawValues = channel.unary_stream( - '/v5api.BTrDB/RawValues', - request_serializer=btrdb__pb2.RawValuesParams.SerializeToString, - response_deserializer=btrdb__pb2.RawValuesResponse.FromString, - ) - self.AlignedWindows = channel.unary_stream( - '/v5api.BTrDB/AlignedWindows', - request_serializer=btrdb__pb2.AlignedWindowsParams.SerializeToString, - response_deserializer=btrdb__pb2.AlignedWindowsResponse.FromString, - ) - self.Windows = channel.unary_stream( - '/v5api.BTrDB/Windows', - request_serializer=btrdb__pb2.WindowsParams.SerializeToString, - response_deserializer=btrdb__pb2.WindowsResponse.FromString, - ) - self.StreamInfo = channel.unary_unary( - '/v5api.BTrDB/StreamInfo', - request_serializer=btrdb__pb2.StreamInfoParams.SerializeToString, - response_deserializer=btrdb__pb2.StreamInfoResponse.FromString, - ) - self.SetStreamAnnotations = channel.unary_unary( - '/v5api.BTrDB/SetStreamAnnotations', - request_serializer=btrdb__pb2.SetStreamAnnotationsParams.SerializeToString, - response_deserializer=btrdb__pb2.SetStreamAnnotationsResponse.FromString, - ) - self.SetStreamTags = channel.unary_unary( - '/v5api.BTrDB/SetStreamTags', - request_serializer=btrdb__pb2.SetStreamTagsParams.SerializeToString, - response_deserializer=btrdb__pb2.SetStreamTagsResponse.FromString, - ) - self.Create = channel.unary_unary( - '/v5api.BTrDB/Create', - request_serializer=btrdb__pb2.CreateParams.SerializeToString, - response_deserializer=btrdb__pb2.CreateResponse.FromString, - ) - self.ListCollections = channel.unary_stream( - '/v5api.BTrDB/ListCollections', - request_serializer=btrdb__pb2.ListCollectionsParams.SerializeToString, - response_deserializer=btrdb__pb2.ListCollectionsResponse.FromString, - ) - self.LookupStreams = channel.unary_stream( - '/v5api.BTrDB/LookupStreams', - request_serializer=btrdb__pb2.LookupStreamsParams.SerializeToString, - response_deserializer=btrdb__pb2.LookupStreamsResponse.FromString, - ) - self.Nearest = channel.unary_unary( - '/v5api.BTrDB/Nearest', - request_serializer=btrdb__pb2.NearestParams.SerializeToString, - response_deserializer=btrdb__pb2.NearestResponse.FromString, - ) - self.Changes = channel.unary_stream( - '/v5api.BTrDB/Changes', - request_serializer=btrdb__pb2.ChangesParams.SerializeToString, - response_deserializer=btrdb__pb2.ChangesResponse.FromString, - ) - self.Insert = channel.unary_unary( - '/v5api.BTrDB/Insert', - request_serializer=btrdb__pb2.InsertParams.SerializeToString, - response_deserializer=btrdb__pb2.InsertResponse.FromString, - ) - self.Delete = channel.unary_unary( - '/v5api.BTrDB/Delete', - request_serializer=btrdb__pb2.DeleteParams.SerializeToString, - response_deserializer=btrdb__pb2.DeleteResponse.FromString, - ) - self.Info = channel.unary_unary( - '/v5api.BTrDB/Info', - request_serializer=btrdb__pb2.InfoParams.SerializeToString, - response_deserializer=btrdb__pb2.InfoResponse.FromString, - ) - self.FaultInject = channel.unary_unary( - '/v5api.BTrDB/FaultInject', - request_serializer=btrdb__pb2.FaultInjectParams.SerializeToString, - response_deserializer=btrdb__pb2.FaultInjectResponse.FromString, - ) - self.Flush = channel.unary_unary( - '/v5api.BTrDB/Flush', - request_serializer=btrdb__pb2.FlushParams.SerializeToString, - response_deserializer=btrdb__pb2.FlushResponse.FromString, - ) - self.Obliterate = channel.unary_unary( - '/v5api.BTrDB/Obliterate', - request_serializer=btrdb__pb2.ObliterateParams.SerializeToString, - response_deserializer=btrdb__pb2.ObliterateResponse.FromString, - ) - self.GetMetadataUsage = channel.unary_unary( - '/v5api.BTrDB/GetMetadataUsage', - request_serializer=btrdb__pb2.MetadataUsageParams.SerializeToString, - response_deserializer=btrdb__pb2.MetadataUsageResponse.FromString, - ) - self.GenerateCSV = channel.unary_stream( - '/v5api.BTrDB/GenerateCSV', - request_serializer=btrdb__pb2.GenerateCSVParams.SerializeToString, - response_deserializer=btrdb__pb2.GenerateCSVResponse.FromString, - ) - self.SQLQuery = channel.unary_stream( - '/v5api.BTrDB/SQLQuery', - request_serializer=btrdb__pb2.SQLQueryParams.SerializeToString, - response_deserializer=btrdb__pb2.SQLQueryResponse.FromString, - ) + """Missing associated documentation comment in .proto file.""" + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.RawValues = channel.unary_stream( + '/v5api.BTrDB/RawValues', + request_serializer=btrdb__pb2.RawValuesParams.SerializeToString, + response_deserializer=btrdb__pb2.RawValuesResponse.FromString, + ) + self.ArrowRawValues = channel.unary_stream( + '/v5api.BTrDB/ArrowRawValues', + request_serializer=btrdb__pb2.RawValuesParams.SerializeToString, + response_deserializer=btrdb__pb2.ArrowRawValuesResponse.FromString, + ) + self.ArrowMultiValues = channel.unary_stream( + '/v5api.BTrDB/ArrowMultiValues', + request_serializer=btrdb__pb2.ArrowMultiValuesParams.SerializeToString, + response_deserializer=btrdb__pb2.ArrowMultiValuesResponse.FromString, + ) + self.AlignedWindows = channel.unary_stream( + '/v5api.BTrDB/AlignedWindows', + request_serializer=btrdb__pb2.AlignedWindowsParams.SerializeToString, + response_deserializer=btrdb__pb2.AlignedWindowsResponse.FromString, + ) + self.ArrowAlignedWindows = channel.unary_stream( + '/v5api.BTrDB/ArrowAlignedWindows', + request_serializer=btrdb__pb2.AlignedWindowsParams.SerializeToString, + response_deserializer=btrdb__pb2.ArrowAlignedWindowsResponse.FromString, + ) + self.Windows = channel.unary_stream( + '/v5api.BTrDB/Windows', + request_serializer=btrdb__pb2.WindowsParams.SerializeToString, + response_deserializer=btrdb__pb2.WindowsResponse.FromString, + ) + self.ArrowWindows = channel.unary_stream( + '/v5api.BTrDB/ArrowWindows', + request_serializer=btrdb__pb2.WindowsParams.SerializeToString, + response_deserializer=btrdb__pb2.ArrowWindowsResponse.FromString, + ) + self.StreamInfo = channel.unary_unary( + '/v5api.BTrDB/StreamInfo', + request_serializer=btrdb__pb2.StreamInfoParams.SerializeToString, + response_deserializer=btrdb__pb2.StreamInfoResponse.FromString, + ) + self.SetStreamAnnotations = channel.unary_unary( + '/v5api.BTrDB/SetStreamAnnotations', + request_serializer=btrdb__pb2.SetStreamAnnotationsParams.SerializeToString, + response_deserializer=btrdb__pb2.SetStreamAnnotationsResponse.FromString, + ) + self.SetStreamTags = channel.unary_unary( + '/v5api.BTrDB/SetStreamTags', + request_serializer=btrdb__pb2.SetStreamTagsParams.SerializeToString, + response_deserializer=btrdb__pb2.SetStreamTagsResponse.FromString, + ) + self.Create = channel.unary_unary( + '/v5api.BTrDB/Create', + request_serializer=btrdb__pb2.CreateParams.SerializeToString, + response_deserializer=btrdb__pb2.CreateResponse.FromString, + ) + self.ListCollections = channel.unary_stream( + '/v5api.BTrDB/ListCollections', + request_serializer=btrdb__pb2.ListCollectionsParams.SerializeToString, + response_deserializer=btrdb__pb2.ListCollectionsResponse.FromString, + ) + self.LookupStreams = channel.unary_stream( + '/v5api.BTrDB/LookupStreams', + request_serializer=btrdb__pb2.LookupStreamsParams.SerializeToString, + response_deserializer=btrdb__pb2.LookupStreamsResponse.FromString, + ) + self.Nearest = channel.unary_unary( + '/v5api.BTrDB/Nearest', + request_serializer=btrdb__pb2.NearestParams.SerializeToString, + response_deserializer=btrdb__pb2.NearestResponse.FromString, + ) + self.Changes = channel.unary_stream( + '/v5api.BTrDB/Changes', + request_serializer=btrdb__pb2.ChangesParams.SerializeToString, + response_deserializer=btrdb__pb2.ChangesResponse.FromString, + ) + self.Insert = channel.unary_unary( + '/v5api.BTrDB/Insert', + request_serializer=btrdb__pb2.InsertParams.SerializeToString, + response_deserializer=btrdb__pb2.InsertResponse.FromString, + ) + self.ArrowInsert = channel.unary_unary( + '/v5api.BTrDB/ArrowInsert', + request_serializer=btrdb__pb2.ArrowInsertParams.SerializeToString, + response_deserializer=btrdb__pb2.InsertResponse.FromString, + ) + self.Delete = channel.unary_unary( + '/v5api.BTrDB/Delete', + request_serializer=btrdb__pb2.DeleteParams.SerializeToString, + response_deserializer=btrdb__pb2.DeleteResponse.FromString, + ) + self.Info = channel.unary_unary( + '/v5api.BTrDB/Info', + request_serializer=btrdb__pb2.InfoParams.SerializeToString, + response_deserializer=btrdb__pb2.InfoResponse.FromString, + ) + self.FaultInject = channel.unary_unary( + '/v5api.BTrDB/FaultInject', + request_serializer=btrdb__pb2.FaultInjectParams.SerializeToString, + response_deserializer=btrdb__pb2.FaultInjectResponse.FromString, + ) + self.Flush = channel.unary_unary( + '/v5api.BTrDB/Flush', + request_serializer=btrdb__pb2.FlushParams.SerializeToString, + response_deserializer=btrdb__pb2.FlushResponse.FromString, + ) + self.Obliterate = channel.unary_unary( + '/v5api.BTrDB/Obliterate', + request_serializer=btrdb__pb2.ObliterateParams.SerializeToString, + response_deserializer=btrdb__pb2.ObliterateResponse.FromString, + ) + self.GetMetadataUsage = channel.unary_unary( + '/v5api.BTrDB/GetMetadataUsage', + request_serializer=btrdb__pb2.MetadataUsageParams.SerializeToString, + response_deserializer=btrdb__pb2.MetadataUsageResponse.FromString, + ) + self.GenerateCSV = channel.unary_stream( + '/v5api.BTrDB/GenerateCSV', + request_serializer=btrdb__pb2.GenerateCSVParams.SerializeToString, + response_deserializer=btrdb__pb2.GenerateCSVResponse.FromString, + ) + self.SQLQuery = channel.unary_stream( + '/v5api.BTrDB/SQLQuery', + request_serializer=btrdb__pb2.SQLQueryParams.SerializeToString, + response_deserializer=btrdb__pb2.SQLQueryResponse.FromString, + ) class BTrDBServicer(object): - # missing associated documentation comment in .proto file - pass - - def RawValues(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def AlignedWindows(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Windows(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def StreamInfo(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def SetStreamAnnotations(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def SetStreamTags(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Create(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def ListCollections(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def LookupStreams(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Nearest(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Changes(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Insert(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Delete(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Info(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def FaultInject(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Flush(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Obliterate(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def GetMetadataUsage(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def GenerateCSV(self, request, context): - # missing associated documentation comment in .proto file - pass - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def SQLQuery(self, request, context): - """rpc SetCompactionConfig(SetCompactionConfigParams) returns (SetCompactionConfigResponse); - rpc GetCompactionConfig(GetCompactionConfigParams) returns (GetCompactionConfigResponse); - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') + """Missing associated documentation comment in .proto file.""" + + def RawValues(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def ArrowRawValues(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def ArrowMultiValues(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def AlignedWindows(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def ArrowAlignedWindows(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Windows(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def ArrowWindows(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def StreamInfo(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def SetStreamAnnotations(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def SetStreamTags(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Create(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def ListCollections(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def LookupStreams(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Nearest(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Changes(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Insert(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def ArrowInsert(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Delete(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Info(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def FaultInject(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Flush(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Obliterate(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetMetadataUsage(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GenerateCSV(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def SQLQuery(self, request, context): + """rpc SetCompactionConfig(SetCompactionConfigParams) returns (SetCompactionConfigResponse); + rpc GetCompactionConfig(GetCompactionConfigParams) returns (GetCompactionConfigResponse); + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') def add_BTrDBServicer_to_server(servicer, server): - rpc_method_handlers = { - 'RawValues': grpc.unary_stream_rpc_method_handler( - servicer.RawValues, - request_deserializer=btrdb__pb2.RawValuesParams.FromString, - response_serializer=btrdb__pb2.RawValuesResponse.SerializeToString, - ), - 'AlignedWindows': grpc.unary_stream_rpc_method_handler( - servicer.AlignedWindows, - request_deserializer=btrdb__pb2.AlignedWindowsParams.FromString, - response_serializer=btrdb__pb2.AlignedWindowsResponse.SerializeToString, - ), - 'Windows': grpc.unary_stream_rpc_method_handler( - servicer.Windows, - request_deserializer=btrdb__pb2.WindowsParams.FromString, - response_serializer=btrdb__pb2.WindowsResponse.SerializeToString, - ), - 'StreamInfo': grpc.unary_unary_rpc_method_handler( - servicer.StreamInfo, - request_deserializer=btrdb__pb2.StreamInfoParams.FromString, - response_serializer=btrdb__pb2.StreamInfoResponse.SerializeToString, - ), - 'SetStreamAnnotations': grpc.unary_unary_rpc_method_handler( - servicer.SetStreamAnnotations, - request_deserializer=btrdb__pb2.SetStreamAnnotationsParams.FromString, - response_serializer=btrdb__pb2.SetStreamAnnotationsResponse.SerializeToString, - ), - 'SetStreamTags': grpc.unary_unary_rpc_method_handler( - servicer.SetStreamTags, - request_deserializer=btrdb__pb2.SetStreamTagsParams.FromString, - response_serializer=btrdb__pb2.SetStreamTagsResponse.SerializeToString, - ), - 'Create': grpc.unary_unary_rpc_method_handler( - servicer.Create, - request_deserializer=btrdb__pb2.CreateParams.FromString, - response_serializer=btrdb__pb2.CreateResponse.SerializeToString, - ), - 'ListCollections': grpc.unary_stream_rpc_method_handler( - servicer.ListCollections, - request_deserializer=btrdb__pb2.ListCollectionsParams.FromString, - response_serializer=btrdb__pb2.ListCollectionsResponse.SerializeToString, - ), - 'LookupStreams': grpc.unary_stream_rpc_method_handler( - servicer.LookupStreams, - request_deserializer=btrdb__pb2.LookupStreamsParams.FromString, - response_serializer=btrdb__pb2.LookupStreamsResponse.SerializeToString, - ), - 'Nearest': grpc.unary_unary_rpc_method_handler( - servicer.Nearest, - request_deserializer=btrdb__pb2.NearestParams.FromString, - response_serializer=btrdb__pb2.NearestResponse.SerializeToString, - ), - 'Changes': grpc.unary_stream_rpc_method_handler( - servicer.Changes, - request_deserializer=btrdb__pb2.ChangesParams.FromString, - response_serializer=btrdb__pb2.ChangesResponse.SerializeToString, - ), - 'Insert': grpc.unary_unary_rpc_method_handler( - servicer.Insert, - request_deserializer=btrdb__pb2.InsertParams.FromString, - response_serializer=btrdb__pb2.InsertResponse.SerializeToString, - ), - 'Delete': grpc.unary_unary_rpc_method_handler( - servicer.Delete, - request_deserializer=btrdb__pb2.DeleteParams.FromString, - response_serializer=btrdb__pb2.DeleteResponse.SerializeToString, - ), - 'Info': grpc.unary_unary_rpc_method_handler( - servicer.Info, - request_deserializer=btrdb__pb2.InfoParams.FromString, - response_serializer=btrdb__pb2.InfoResponse.SerializeToString, - ), - 'FaultInject': grpc.unary_unary_rpc_method_handler( - servicer.FaultInject, - request_deserializer=btrdb__pb2.FaultInjectParams.FromString, - response_serializer=btrdb__pb2.FaultInjectResponse.SerializeToString, - ), - 'Flush': grpc.unary_unary_rpc_method_handler( - servicer.Flush, - request_deserializer=btrdb__pb2.FlushParams.FromString, - response_serializer=btrdb__pb2.FlushResponse.SerializeToString, - ), - 'Obliterate': grpc.unary_unary_rpc_method_handler( - servicer.Obliterate, - request_deserializer=btrdb__pb2.ObliterateParams.FromString, - response_serializer=btrdb__pb2.ObliterateResponse.SerializeToString, - ), - 'GetMetadataUsage': grpc.unary_unary_rpc_method_handler( - servicer.GetMetadataUsage, - request_deserializer=btrdb__pb2.MetadataUsageParams.FromString, - response_serializer=btrdb__pb2.MetadataUsageResponse.SerializeToString, - ), - 'GenerateCSV': grpc.unary_stream_rpc_method_handler( - servicer.GenerateCSV, - request_deserializer=btrdb__pb2.GenerateCSVParams.FromString, - response_serializer=btrdb__pb2.GenerateCSVResponse.SerializeToString, - ), - 'SQLQuery': grpc.unary_stream_rpc_method_handler( - servicer.SQLQuery, - request_deserializer=btrdb__pb2.SQLQueryParams.FromString, - response_serializer=btrdb__pb2.SQLQueryResponse.SerializeToString, - ), - } - generic_handler = grpc.method_handlers_generic_handler( - 'v5api.BTrDB', rpc_method_handlers) - server.add_generic_rpc_handlers((generic_handler,)) + rpc_method_handlers = { + 'RawValues': grpc.unary_stream_rpc_method_handler( + servicer.RawValues, + request_deserializer=btrdb__pb2.RawValuesParams.FromString, + response_serializer=btrdb__pb2.RawValuesResponse.SerializeToString, + ), + 'ArrowRawValues': grpc.unary_stream_rpc_method_handler( + servicer.ArrowRawValues, + request_deserializer=btrdb__pb2.RawValuesParams.FromString, + response_serializer=btrdb__pb2.ArrowRawValuesResponse.SerializeToString, + ), + 'ArrowMultiValues': grpc.unary_stream_rpc_method_handler( + servicer.ArrowMultiValues, + request_deserializer=btrdb__pb2.ArrowMultiValuesParams.FromString, + response_serializer=btrdb__pb2.ArrowMultiValuesResponse.SerializeToString, + ), + 'AlignedWindows': grpc.unary_stream_rpc_method_handler( + servicer.AlignedWindows, + request_deserializer=btrdb__pb2.AlignedWindowsParams.FromString, + response_serializer=btrdb__pb2.AlignedWindowsResponse.SerializeToString, + ), + 'ArrowAlignedWindows': grpc.unary_stream_rpc_method_handler( + servicer.ArrowAlignedWindows, + request_deserializer=btrdb__pb2.AlignedWindowsParams.FromString, + response_serializer=btrdb__pb2.ArrowAlignedWindowsResponse.SerializeToString, + ), + 'Windows': grpc.unary_stream_rpc_method_handler( + servicer.Windows, + request_deserializer=btrdb__pb2.WindowsParams.FromString, + response_serializer=btrdb__pb2.WindowsResponse.SerializeToString, + ), + 'ArrowWindows': grpc.unary_stream_rpc_method_handler( + servicer.ArrowWindows, + request_deserializer=btrdb__pb2.WindowsParams.FromString, + response_serializer=btrdb__pb2.ArrowWindowsResponse.SerializeToString, + ), + 'StreamInfo': grpc.unary_unary_rpc_method_handler( + servicer.StreamInfo, + request_deserializer=btrdb__pb2.StreamInfoParams.FromString, + response_serializer=btrdb__pb2.StreamInfoResponse.SerializeToString, + ), + 'SetStreamAnnotations': grpc.unary_unary_rpc_method_handler( + servicer.SetStreamAnnotations, + request_deserializer=btrdb__pb2.SetStreamAnnotationsParams.FromString, + response_serializer=btrdb__pb2.SetStreamAnnotationsResponse.SerializeToString, + ), + 'SetStreamTags': grpc.unary_unary_rpc_method_handler( + servicer.SetStreamTags, + request_deserializer=btrdb__pb2.SetStreamTagsParams.FromString, + response_serializer=btrdb__pb2.SetStreamTagsResponse.SerializeToString, + ), + 'Create': grpc.unary_unary_rpc_method_handler( + servicer.Create, + request_deserializer=btrdb__pb2.CreateParams.FromString, + response_serializer=btrdb__pb2.CreateResponse.SerializeToString, + ), + 'ListCollections': grpc.unary_stream_rpc_method_handler( + servicer.ListCollections, + request_deserializer=btrdb__pb2.ListCollectionsParams.FromString, + response_serializer=btrdb__pb2.ListCollectionsResponse.SerializeToString, + ), + 'LookupStreams': grpc.unary_stream_rpc_method_handler( + servicer.LookupStreams, + request_deserializer=btrdb__pb2.LookupStreamsParams.FromString, + response_serializer=btrdb__pb2.LookupStreamsResponse.SerializeToString, + ), + 'Nearest': grpc.unary_unary_rpc_method_handler( + servicer.Nearest, + request_deserializer=btrdb__pb2.NearestParams.FromString, + response_serializer=btrdb__pb2.NearestResponse.SerializeToString, + ), + 'Changes': grpc.unary_stream_rpc_method_handler( + servicer.Changes, + request_deserializer=btrdb__pb2.ChangesParams.FromString, + response_serializer=btrdb__pb2.ChangesResponse.SerializeToString, + ), + 'Insert': grpc.unary_unary_rpc_method_handler( + servicer.Insert, + request_deserializer=btrdb__pb2.InsertParams.FromString, + response_serializer=btrdb__pb2.InsertResponse.SerializeToString, + ), + 'ArrowInsert': grpc.unary_unary_rpc_method_handler( + servicer.ArrowInsert, + request_deserializer=btrdb__pb2.ArrowInsertParams.FromString, + response_serializer=btrdb__pb2.InsertResponse.SerializeToString, + ), + 'Delete': grpc.unary_unary_rpc_method_handler( + servicer.Delete, + request_deserializer=btrdb__pb2.DeleteParams.FromString, + response_serializer=btrdb__pb2.DeleteResponse.SerializeToString, + ), + 'Info': grpc.unary_unary_rpc_method_handler( + servicer.Info, + request_deserializer=btrdb__pb2.InfoParams.FromString, + response_serializer=btrdb__pb2.InfoResponse.SerializeToString, + ), + 'FaultInject': grpc.unary_unary_rpc_method_handler( + servicer.FaultInject, + request_deserializer=btrdb__pb2.FaultInjectParams.FromString, + response_serializer=btrdb__pb2.FaultInjectResponse.SerializeToString, + ), + 'Flush': grpc.unary_unary_rpc_method_handler( + servicer.Flush, + request_deserializer=btrdb__pb2.FlushParams.FromString, + response_serializer=btrdb__pb2.FlushResponse.SerializeToString, + ), + 'Obliterate': grpc.unary_unary_rpc_method_handler( + servicer.Obliterate, + request_deserializer=btrdb__pb2.ObliterateParams.FromString, + response_serializer=btrdb__pb2.ObliterateResponse.SerializeToString, + ), + 'GetMetadataUsage': grpc.unary_unary_rpc_method_handler( + servicer.GetMetadataUsage, + request_deserializer=btrdb__pb2.MetadataUsageParams.FromString, + response_serializer=btrdb__pb2.MetadataUsageResponse.SerializeToString, + ), + 'GenerateCSV': grpc.unary_stream_rpc_method_handler( + servicer.GenerateCSV, + request_deserializer=btrdb__pb2.GenerateCSVParams.FromString, + response_serializer=btrdb__pb2.GenerateCSVResponse.SerializeToString, + ), + 'SQLQuery': grpc.unary_stream_rpc_method_handler( + servicer.SQLQuery, + request_deserializer=btrdb__pb2.SQLQueryParams.FromString, + response_serializer=btrdb__pb2.SQLQueryResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'v5api.BTrDB', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + + + # This class is part of an EXPERIMENTAL API. +class BTrDB(object): + """Missing associated documentation comment in .proto file.""" + + @staticmethod + def RawValues(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream(request, target, '/v5api.BTrDB/RawValues', + btrdb__pb2.RawValuesParams.SerializeToString, + btrdb__pb2.RawValuesResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def ArrowRawValues(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream(request, target, '/v5api.BTrDB/ArrowRawValues', + btrdb__pb2.RawValuesParams.SerializeToString, + btrdb__pb2.ArrowRawValuesResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def ArrowMultiValues(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream(request, target, '/v5api.BTrDB/ArrowMultiValues', + btrdb__pb2.ArrowMultiValuesParams.SerializeToString, + btrdb__pb2.ArrowMultiValuesResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def AlignedWindows(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream(request, target, '/v5api.BTrDB/AlignedWindows', + btrdb__pb2.AlignedWindowsParams.SerializeToString, + btrdb__pb2.AlignedWindowsResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def ArrowAlignedWindows(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream(request, target, '/v5api.BTrDB/ArrowAlignedWindows', + btrdb__pb2.AlignedWindowsParams.SerializeToString, + btrdb__pb2.ArrowAlignedWindowsResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Windows(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream(request, target, '/v5api.BTrDB/Windows', + btrdb__pb2.WindowsParams.SerializeToString, + btrdb__pb2.WindowsResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def ArrowWindows(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream(request, target, '/v5api.BTrDB/ArrowWindows', + btrdb__pb2.WindowsParams.SerializeToString, + btrdb__pb2.ArrowWindowsResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def StreamInfo(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/v5api.BTrDB/StreamInfo', + btrdb__pb2.StreamInfoParams.SerializeToString, + btrdb__pb2.StreamInfoResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def SetStreamAnnotations(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/v5api.BTrDB/SetStreamAnnotations', + btrdb__pb2.SetStreamAnnotationsParams.SerializeToString, + btrdb__pb2.SetStreamAnnotationsResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def SetStreamTags(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/v5api.BTrDB/SetStreamTags', + btrdb__pb2.SetStreamTagsParams.SerializeToString, + btrdb__pb2.SetStreamTagsResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Create(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/v5api.BTrDB/Create', + btrdb__pb2.CreateParams.SerializeToString, + btrdb__pb2.CreateResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def ListCollections(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream(request, target, '/v5api.BTrDB/ListCollections', + btrdb__pb2.ListCollectionsParams.SerializeToString, + btrdb__pb2.ListCollectionsResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def LookupStreams(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream(request, target, '/v5api.BTrDB/LookupStreams', + btrdb__pb2.LookupStreamsParams.SerializeToString, + btrdb__pb2.LookupStreamsResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Nearest(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/v5api.BTrDB/Nearest', + btrdb__pb2.NearestParams.SerializeToString, + btrdb__pb2.NearestResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Changes(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream(request, target, '/v5api.BTrDB/Changes', + btrdb__pb2.ChangesParams.SerializeToString, + btrdb__pb2.ChangesResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Insert(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/v5api.BTrDB/Insert', + btrdb__pb2.InsertParams.SerializeToString, + btrdb__pb2.InsertResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def ArrowInsert(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/v5api.BTrDB/ArrowInsert', + btrdb__pb2.ArrowInsertParams.SerializeToString, + btrdb__pb2.InsertResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Delete(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/v5api.BTrDB/Delete', + btrdb__pb2.DeleteParams.SerializeToString, + btrdb__pb2.DeleteResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Info(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/v5api.BTrDB/Info', + btrdb__pb2.InfoParams.SerializeToString, + btrdb__pb2.InfoResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def FaultInject(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/v5api.BTrDB/FaultInject', + btrdb__pb2.FaultInjectParams.SerializeToString, + btrdb__pb2.FaultInjectResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Flush(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/v5api.BTrDB/Flush', + btrdb__pb2.FlushParams.SerializeToString, + btrdb__pb2.FlushResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Obliterate(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/v5api.BTrDB/Obliterate', + btrdb__pb2.ObliterateParams.SerializeToString, + btrdb__pb2.ObliterateResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def GetMetadataUsage(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/v5api.BTrDB/GetMetadataUsage', + btrdb__pb2.MetadataUsageParams.SerializeToString, + btrdb__pb2.MetadataUsageResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def GenerateCSV(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream(request, target, '/v5api.BTrDB/GenerateCSV', + btrdb__pb2.GenerateCSVParams.SerializeToString, + btrdb__pb2.GenerateCSVResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def SQLQuery(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_stream(request, target, '/v5api.BTrDB/SQLQuery', + btrdb__pb2.SQLQueryParams.SerializeToString, + btrdb__pb2.SQLQueryResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/btrdb/point.py b/btrdb/point.py index fa767f9..00cb7b0 100644 --- a/btrdb/point.py +++ b/btrdb/point.py @@ -17,11 +17,11 @@ from btrdb.grpcinterface import btrdb_pb2 - ########################################################################## ## Point Classes ########################################################################## + class RawPoint(object): """ A point of data representing a single position within a time series. Each @@ -75,7 +75,7 @@ def __getitem__(self, index): @staticmethod def to_proto(point): - return btrdb_pb2.RawPoint(time = point[0], value = point[1]) + return btrdb_pb2.RawPoint(time=point[0], value=point[1]) @staticmethod def to_proto_list(points): @@ -148,7 +148,9 @@ def __init__(self, time, minv, meanv, maxv, count, stddev): @classmethod def from_proto(cls, proto): - return cls(proto.time, proto.min, proto.mean, proto.max, proto.count, proto.stddev) + return cls( + proto.time, proto.min, proto.mean, proto.max, proto.count, proto.stddev + ) @classmethod def from_proto_list(cls, proto_list): @@ -214,12 +216,7 @@ def __getitem__(self, index): def __repr__(self): return "StatPoint({}, {}, {}, {}, {}, {})".format( - self.time, - self.min, - self.mean, - self.max, - self.count, - self.stddev + self.time, self.min, self.mean, self.max, self.count, self.stddev ) def __str__(self): @@ -230,9 +227,11 @@ def __eq__(self, other): if not hasattr(other, attr): return False - return self.time == other.time and \ - self.min == other.min and \ - self.mean == other.mean and \ - self.max == other.max and \ - self.count == other.count and \ - self.stddev == other.stddev + return ( + self.time == other.time + and self.min == other.min + and self.mean == other.mean + and self.max == other.max + and self.count == other.count + and self.stddev == other.stddev + ) diff --git a/btrdb/stream.py b/btrdb/stream.py index 39d86fe..b777bf5 100644 --- a/btrdb/stream.py +++ b/btrdb/stream.py @@ -14,34 +14,39 @@ ########################################################################## ## Imports ########################################################################## - -import re import json +import logging +import re import uuid as uuidlib -from copy import deepcopy +import warnings +from collections import deque from collections.abc import Sequence +from copy import deepcopy + +import pyarrow as pa -from btrdb.utils.buffer import PointBuffer -from btrdb.point import RawPoint, StatPoint -from btrdb.transformers import StreamSetTransformer -from btrdb.utils.timez import currently_as_ns, to_nanoseconds -from btrdb.utils.conversion import AnnotationEncoder, AnnotationDecoder -from btrdb.utils.general import pointwidth as pw from btrdb.exceptions import ( BTrDBError, BTRDBTypeError, BTRDBValueError, - InvalidOperation, InvalidCollection, + InvalidOperation, + NoSuchPoint, StreamNotFoundError, - NoSuchPoint + retry, ) - +from btrdb.point import RawPoint, StatPoint +from btrdb.transformers import _STAT_PROPERTIES, StreamSetTransformer +from btrdb.utils.buffer import PointBuffer +from btrdb.utils.conversion import AnnotationDecoder, AnnotationEncoder +from btrdb.utils.general import pointwidth as pw +from btrdb.utils.timez import currently_as_ns, to_nanoseconds ########################################################################## ## Module Variables ########################################################################## - +logger = logging.getLogger(__name__) +IS_DEBUG = logger.isEnabledFor(logging.DEBUG) INSERT_BATCH_SIZE = 50000 MINIMUM_TIME = -(16 << 56) MAXIMUM_TIME = (48 << 56) - 1 @@ -51,11 +56,13 @@ except Exception: RE_PATTERN = re.Pattern +_arrow_not_impl_str = "The BTrDB server you are using does not support {}." ########################################################################## ## Stream Classes ########################################################################## + class Stream(object): """ An object that represents a specific time series stream in the BTrDB database. @@ -72,30 +79,44 @@ class Stream(object): """ def __init__(self, btrdb, uuid, **db_values): - db_args = ('known_to_exist', 'collection', 'tags', 'annotations', 'property_version') + db_args = ( + "known_to_exist", + "collection", + "tags", + "annotations", + "property_version", + ) for key in db_args: value = db_values.pop(key, None) setattr(self, "_{}".format(key), value) if db_values: bad_keys = ", ".join(db_values.keys()) - raise BTRDBTypeError("got unexpected db_values argument(s) '{}'".format(bad_keys)) + raise BTRDBTypeError( + "got unexpected db_values argument(s) '{}'".format(bad_keys) + ) self._btrdb = btrdb self._uuid = uuid def refresh_metadata(self): """ - Refreshes the locally cached meta data for a stream + Refreshes the locally cached metadata for a stream from the server. Queries the BTrDB server for all stream metadata including collection, annotation, and tags. This method requires a round trip to the server. """ ep = self._btrdb.ep - self._collection, self._property_version, self._tags, self._annotations, _ = ep.streamInfo(self._uuid, False, True) + ( + self._collection, + self._property_version, + self._tags, + self._annotations, + _, + ) = ep.streamInfo(self._uuid, False, True) self._known_to_exist = True - # deserialize annoation values + # deserialize annotation values self._annotations = { key: json.loads(val, cls=AnnotationDecoder) for key, val in self._annotations.items() @@ -131,7 +152,14 @@ def exists(self): return False raise bte - def count(self, start=MINIMUM_TIME, end=MAXIMUM_TIME, pointwidth=62, precise=False, version=0): + def count( + self, + start=MINIMUM_TIME, + end=MAXIMUM_TIME, + pointwidth=62, + precise=False, + version=0, + ): """ Compute the total number of points in the stream @@ -176,15 +204,17 @@ def count(self, start=MINIMUM_TIME, end=MAXIMUM_TIME, pointwidth=62, precise=Fal """ if not precise: - pointwidth = min(pointwidth, pw.from_nanoseconds(to_nanoseconds(end) - to_nanoseconds(start))-1) + pointwidth = min( + pointwidth, + pw.from_nanoseconds(to_nanoseconds(end) - to_nanoseconds(start)) - 1, + ) points = self.aligned_windows(start, end, pointwidth, version) - return sum([point.count for point, _ in points]) + return sum([point.count for point, _ in points]) depth = 0 width = to_nanoseconds(end) - to_nanoseconds(start) points = self.windows(start, end, width, depth, version) - return sum([point.count for point, _ in points]) - + return sum([point.count for point, _ in points]) @property def btrdb(self): @@ -206,7 +236,7 @@ def btrdb(self): @property def uuid(self): """ - Returns the stream's UUID. The stream may nor may not exist + Returns the stream's UUID. The stream may or may not exist yet, depending on how the stream object was obtained. Returns @@ -274,7 +304,15 @@ def collection(self): self.refresh_metadata() return self._collection - def earliest(self, version=0): + @retry + def earliest( + self, + version=0, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Returns the first point of data in the stream. Returns None if error encountered during lookup or no values in stream. @@ -284,6 +322,17 @@ def earliest(self, version=0): version : int, default: 0 Specify the version of the stream to query; if zero, queries the latest stream state rather than pinning to a version. + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- @@ -295,7 +344,15 @@ def earliest(self, version=0): start = MINIMUM_TIME return self.nearest(start, version=version, backward=False) - def latest(self, version=0): + @retry + def latest( + self, + version=0, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Returns last point of data in the stream. Returns None if error encountered during lookup or no values in stream. @@ -305,6 +362,17 @@ def latest(self, version=0): version : int, default: 0 Specify the version of the stream to query; if zero, queries the latest stream state rather than pinning to a version. + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- @@ -316,7 +384,15 @@ def latest(self, version=0): start = MAXIMUM_TIME return self.nearest(start, version=version, backward=True) - def current(self, version=0): + @retry + def current( + self, + version=0, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Returns the point that is closest to the current timestamp, e.g. the latest point in the stream up until now. Note that no future values will be returned. @@ -327,6 +403,17 @@ def current(self, version=0): version : int, default: 0 Specify the version of the stream to query; if zero, queries the latest stream state rather than pinning to a version. + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- @@ -337,7 +424,15 @@ def current(self, version=0): start = currently_as_ns() return self.nearest(start, version, backward=True) - def tags(self, refresh=False): + @retry + def tags( + self, + refresh=False, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Returns the stream's tags. @@ -346,9 +441,20 @@ def tags(self, refresh=False): Parameters ---------- - refresh: bool + refresh: bool, default: False Indicates whether a round trip to the server should be implemented regardless of whether there is a local copy. + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- @@ -361,22 +467,39 @@ def tags(self, refresh=False): return deepcopy(self._tags) - def annotations(self, refresh=False): + @retry + def annotations( + self, + refresh=False, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Returns a stream's annotations Annotations returns the annotations of the stream (and the annotation - version). It will always require a round trip to the server. If you are - ok with stale data and want a higher performance version, use - Stream.CachedAnnotations(). + version). Do not modify the resulting map. Parameters ---------- - refresh: bool + refresh: bool, default: False Indicates whether a round trip to the server should be implemented regardless of whether there is a local copy. + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- @@ -390,7 +513,14 @@ def annotations(self, refresh=False): return deepcopy(self._annotations), deepcopy(self._property_version) - def version(self): + @retry + def version( + self, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Returns the current data version of the stream. @@ -400,7 +530,17 @@ def version(self): Parameters ---------- - None + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- @@ -410,7 +550,7 @@ def version(self): """ return self._btrdb.ep.streamInfo(self._uuid, True, False)[4] - def insert(self, data, merge='never'): + def insert(self, data, merge="never"): """ Insert new data in the form (time, value) into the series. @@ -436,27 +576,93 @@ def insert(self, data, merge='never'): ------- int The version of the stream after inserting new points. - """ - i = 0 version = 0 + i = 0 while i < len(data): - thisBatch = data[i:i + INSERT_BATCH_SIZE] + thisBatch = data[i : i + INSERT_BATCH_SIZE] version = self._btrdb.ep.insert(self._uuid, thisBatch, merge) i += INSERT_BATCH_SIZE return version + def arrow_insert(self, data: pa.Table, merge: str = "never") -> int: + """ + Insert new data in the form of a pyarrow Table with (time, value) columns. + + Inserts a table of new (time, value) columns into the stream. The values + in the table need not be sorted by time. If the arrays are larger than + appropriate, this function will automatically chunk the inserts. As a + consequence, the insert is not necessarily atomic, but can be used with + a very large array. + + Parameters + ---------- + data: pyarrow.Table, required + A pyarrow table with a schema of time:Timestamp[ns, tz=UTC], value:float64 + This schema will be validated and converted if necessary. + merge: str + A string describing the merge policy. Valid policies are: + - 'never': the default, no points are merged + - 'equal': points are deduplicated if the time and value are equal + - 'retain': if two points have the same timestamp, the old one is kept + - 'replace': if two points have the same timestamp, the new one is kept + + Returns + ------- + int + The version of the stream after inserting new points. + + Notes + ----- + This method is available for commercial customers with arrow-enabled servers. + """ + if not self._btrdb._ARROW_ENABLED: + raise NotImplementedError(_arrow_not_impl_str.format("arrow_insert")) + chunksize = INSERT_BATCH_SIZE + assert isinstance(data, pa.Table) + tmp_table = data.rename_columns(["time", "value"]) + new_schema = pa.schema( + [ + (pa.field("time", pa.timestamp(unit="ns", tz="UTC"), nullable=False)), + (pa.field("value", pa.float64(), nullable=False)), + ] + ) + tmp_table = tmp_table.cast(new_schema) + num_rows = tmp_table.num_rows + + # Calculate the number of batches based on the chunk size + num_batches = num_rows // chunksize + if num_rows % chunksize != 0 or num_batches == 0: + num_batches = num_batches + 1 + + table_slices = [] + + for i in range(num_batches): + start_idx = i * chunksize + t = tmp_table.slice(offset=start_idx, length=chunksize) + table_slices.append(t) + + # Process the batches as needed + version = [] + for tab in table_slices: + version.append( + self._btrdb.ep.arrowInsertValues(uu=self.uuid, values=tab, policy=merge) + ) + return max(version) + def _update_tags_collection(self, tags, collection): tags = self.tags() if tags is None else tags collection = self.collection if collection is None else collection if collection is None: - raise BTRDBValueError("collection must be provided to update tags or collection") + raise BTRDBValueError( + "collection must be provided to update tags or collection" + ) self._btrdb.ep.setStreamTags( uu=self.uuid, expected=self._property_version, tags=tags, - collection=collection + collection=collection, ) def _update_annotations(self, annotations, encoder, replace): @@ -470,16 +676,30 @@ def _update_annotations(self, annotations, encoder, replace): removals = [] if replace: - removals = [i for i in self._annotations.keys() if i not in annotations.keys()] + removals = [ + i for i in self._annotations.keys() if i not in annotations.keys() + ] self._btrdb.ep.setStreamAnnotations( uu=self.uuid, expected=self._property_version, changes=serialized, - removals=removals + removals=removals, ) - def update(self, tags=None, annotations=None, collection=None, encoder=AnnotationEncoder, replace=False): + @retry + def update( + self, + tags=None, + annotations=None, + collection=None, + encoder=AnnotationEncoder, + replace=False, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Updates metadata including tags, annotations, and collection as an UPSERT operation. @@ -520,16 +740,28 @@ def update(self, tags=None, annotations=None, collection=None, encoder=Annotatio Replace all annotations or tags with the specified dictionaries instead of performing the normal upsert operation. Specifying True is the only way to remove annotation keys. + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- int The version of the metadata (separate from the version of the data) also known as the "property version". - """ if tags is None and annotations is None and collection is None: - raise BTRDBValueError("you must supply a tags, annotations, or collection argument") + raise BTRDBValueError( + "you must supply a tags, annotations, or collection argument" + ) if tags is not None and isinstance(tags, dict) is False: raise BTRDBTypeError("tags must be of type dict") @@ -550,7 +782,16 @@ def update(self, tags=None, annotations=None, collection=None, encoder=Annotatio return self._property_version - def delete(self, start, end): + @retry + def delete( + self, + start, + end, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ "Delete" all points between [`start`, `end`) @@ -567,17 +808,38 @@ def delete(self, start, end): end : int or datetime like object The end time in nanoseconds for the range to be deleted. (see :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- int The version of the new stream created - """ - return self._btrdb.ep.deleteRange(self._uuid, to_nanoseconds(start), - to_nanoseconds(end)) + return self._btrdb.ep.deleteRange( + self._uuid, to_nanoseconds(start), to_nanoseconds(end) + ) - def values(self, start, end, version=0): + @retry + def values( + self, + start, + end, + version=0, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Read raw values from BTrDB between time [a, b) in nanoseconds. @@ -595,6 +857,17 @@ def values(self, start, end, version=0): :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) version: int The version of the stream to be queried + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------ @@ -615,14 +888,97 @@ def values(self, start, end, version=0): materialized = [] start = to_nanoseconds(start) end = to_nanoseconds(end) - point_windows = self._btrdb.ep.rawValues(self._uuid, start, end, version) for point_list, version in point_windows: for point in point_list: materialized.append((RawPoint.from_proto(point), version)) return materialized - def aligned_windows(self, start, end, pointwidth, version=0): + @retry + def arrow_values( + self, + start, + end, + version: int = 0, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ) -> pa.Table: + """Read raw values from BTrDB between time [a, b) in nanoseconds. + + RawValues queries BTrDB for the raw time series data points between + `start` and `end` time, both in nanoseconds since the Epoch for the + specified stream `version`. + + start : int or datetime like object + The start time in nanoseconds for the range to be queried. (see + :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) + end : int or datetime like object + The end time in nanoseconds for the range to be queried. (see + :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) + version: int + The version of the stream to be queried + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False + + Returns + ------ + pyarrow.Table + A pyarrow table of the raw values with time and value columns. + + + Notes + ----- + Note that the raw data points are the original values at the sensor's + native sampling rate (assuming the time series represents measurements + from a sensor). This is the lowest level of data with the finest time + granularity. In the tree data structure of BTrDB, this data is stored in + the vector nodes. + + This method is available for commercial customers with arrow-enabled servers. + """ + if not self._btrdb._ARROW_ENABLED: + raise NotImplementedError(_arrow_not_impl_str.format("arrow_values")) + start = to_nanoseconds(start) + end = to_nanoseconds(end) + arrow_and_versions = self._btrdb.ep.arrowRawValues( + uu=self.uuid, start=start, end=end, version=version + ) + tables = list(arrow_and_versions) + if len(tables) > 0: + tabs, ver = zip(*tables) + return pa.concat_tables(tabs) + else: + schema = pa.schema( + [ + pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False), + pa.field("value", pa.float64(), nullable=False), + ] + ) + return pa.Table.from_arrays([pa.array([]), pa.array([])], schema=schema) + + @retry + def aligned_windows( + self, + start, + end, + pointwidth, + version=0, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Read statistical aggregates of windows of data from BTrDB. @@ -652,6 +1008,17 @@ def aligned_windows(self, start, end, pointwidth, version=0): Specify the number of ns between data points (2**pointwidth) version : int Version of the stream to query + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- @@ -668,15 +1035,121 @@ def aligned_windows(self, start, end, pointwidth, version=0): materialized = [] start = to_nanoseconds(start) end = to_nanoseconds(end) - - windows = self._btrdb.ep.alignedWindows(self._uuid, start, end, pointwidth, version) + windows = self._btrdb.ep.alignedWindows( + self._uuid, start, end, pointwidth, version + ) for stat_points, version in windows: for point in stat_points: materialized.append((StatPoint.from_proto(point), version)) - return tuple(materialized) - def windows(self, start, end, width, depth=0, version=0): + @retry + def arrow_aligned_windows( + self, + start: int, + end: int, + pointwidth: int, + version: int = 0, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ) -> pa.Table: + """Read statistical aggregates of windows of data from BTrDB. + + Query BTrDB for aggregates (or roll ups or windows) of the time series + with `version` between time `start` (inclusive) and `end` (exclusive) in + nanoseconds [start, end). Each point returned is a statistical aggregate of all the + raw data within a window of width 2**`pointwidth` nanoseconds. These + statistical aggregates currently include the mean, minimum, and maximum + of the data and the count of data points composing the window. + + Note that `start` is inclusive, but `end` is exclusive. That is, results + will be returned for all windows that start in the interval [start, end). + If end < start+2^pointwidth you will not get any results. If start and + end are not powers of two, the bottom pointwidth bits will be cleared. + Each window will contain statistical summaries of the window. + Statistical points with count == 0 will be omitted. + + Parameters + ---------- + start : int or datetime like object, required + The start time in nanoseconds for the range to be queried. (see + :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) + end : int or datetime like object, required + The end time in nanoseconds for the range to be queried. (see + :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) + pointwidth : int, required + Specify the number of ns between data points (2**pointwidth) + version : int + Version of the stream to query + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False + + Returns + ------- + pyarrow.Table + Returns a pyarrow table containing the windows of data. + + Notes + ----- + As the window-width is a power-of-two, it aligns with BTrDB internal + tree data structure and is faster to execute than `windows()`. + + This method is available for commercial customers with arrow-enabled servers. + """ + if not self._btrdb._ARROW_ENABLED: + raise NotImplementedError( + _arrow_not_impl_str.format("arrow_aligned_windows") + ) + + if IS_DEBUG: + logger.debug(f"For stream - {self.uuid} - {self.name}") + start = to_nanoseconds(start) + end = to_nanoseconds(end) + tables = list( + self._btrdb.ep.arrowAlignedWindows( + self.uuid, start=start, end=end, pointwidth=pointwidth, version=version + ) + ) + if len(tables) > 0: + tabs, ver = zip(*tables) + return pa.concat_tables(tabs) + else: + schema = pa.schema( + [ + pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False), + pa.field("min", pa.float64(), nullable=False), + pa.field("mean", pa.float64(), nullable=False), + pa.field("max", pa.float64(), nullable=False), + pa.field("count", pa.uint64(), nullable=False), + pa.field("stddev", pa.float64(), nullable=False), + ] + ) + return pa.Table.from_arrays([pa.array([]) for _ in range(5)], schema=schema) + + @retry + def windows( + self, + start, + end, + width, + depth=0, + version=0, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Read arbitrarily-sized windows of data from BTrDB. StatPoint objects will be returned representing the data for each window. @@ -693,6 +1166,17 @@ def windows(self, start, end, width, depth=0, version=0): The number of nanoseconds in each window. version : int The version of the stream to query. + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- @@ -715,11 +1199,11 @@ def windows(self, start, end, width, depth=0, version=0): parameter previously available has been deprecated. The only valid value for depth is now 0. + This method is available for commercial customers with arrow-enabled servers. """ materialized = [] start = to_nanoseconds(start) end = to_nanoseconds(end) - windows = self._btrdb.ep.windows(self._uuid, start, end, width, depth, version) for stat_points, version in windows: for point in stat_points: @@ -727,7 +1211,105 @@ def windows(self, start, end, width, depth=0, version=0): return tuple(materialized) - def nearest(self, time, version, backward=False): + @retry + def arrow_windows( + self, + start: int, + end: int, + width: int, + version: int = 0, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ) -> pa.Table: + """Read arbitrarily-sized windows of data from BTrDB. + + Parameters + ---------- + start : int or datetime like object, required + The start time in nanoseconds for the range to be queried. (see + :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) + end : int or datetime like object, required + The end time in nanoseconds for the range to be queried. (see + :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) + width : int, required + The number of nanoseconds in each window. + version : int, default=0, optional + The version of the stream to query. + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False + + Returns + ------- + pyarrow.Table + Returns a pyarrow Table containing windows of data. + + Notes + ----- + Windows returns arbitrary precision windows from BTrDB. It is slower + than AlignedWindows, but still significantly faster than RawValues. Each + returned window will be `width` nanoseconds long. `start` is inclusive, + but `end` is exclusive (e.g if end < start+width you will get no + results). That is, results will be returned for all windows that start + at a time less than the end timestamp. If (`end` - `start`) is not a + multiple of width, then end will be decreased to the greatest value less + than end such that (end - start) is a multiple of `width` (i.e., we set + end = start + width * floordiv(end - start, width). The `depth` + parameter previously available has been deprecated. The only valid value + for depth is now 0. + This method is available for commercial customers with arrow-enabled servers. + """ + if not self._btrdb._ARROW_ENABLED: + raise NotImplementedError(_arrow_not_impl_str.format("arrow_windows")) + start = to_nanoseconds(start) + end = to_nanoseconds(end) + tables = list( + self._btrdb.ep.arrowWindows( + self.uuid, + start=start, + end=end, + width=width, + depth=0, + version=version, + ) + ) + if len(tables) > 0: + tabs, ver = zip(*tables) + return pa.concat_tables(tabs) + else: + schema = pa.schema( + [ + pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False), + pa.field("min", pa.float64(), nullable=False), + pa.field("mean", pa.float64(), nullable=False), + pa.field("max", pa.float64(), nullable=False), + pa.field("count", pa.uint64(), nullable=False), + pa.field("stddev", pa.float64(), nullable=False), + ] + ) + return pa.Table.from_arrays([pa.array([]) for _ in range(5)], schema=schema) + + @retry + def nearest( + self, + time, + version, + backward=False, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Finds the closest point in the stream to a specified time. @@ -746,6 +1328,17 @@ def nearest(self, time, version, backward=False): Version of the stream to use in search backward : boolean True to search backwards from time, else false for forward + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False Returns ------- @@ -755,8 +1348,15 @@ def nearest(self, time, version, backward=False): """ try: - rp, version = self._btrdb.ep.nearest(self._uuid, - to_nanoseconds(time), version, backward) + if IS_DEBUG: + logger.debug( + f"checking nearest for: {self.uuid}\t\t{time}\t\t{version}" + ) + rp, version = self._btrdb.ep.nearest( + self._uuid, to_nanoseconds(time), version, backward + ) + if IS_DEBUG: + logger.debug(f"Nearest for stream: {self.uuid} - {rp}") except BTrDBError as exc: if not isinstance(exc, NoSuchPoint): raise @@ -764,35 +1364,75 @@ def nearest(self, time, version, backward=False): return RawPoint.from_proto(rp), version - - def obliterate(self): + @retry + def obliterate( + self, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Obliterate deletes a stream from the BTrDB server. An exception will be raised if the stream could not be found. + Parameters + ---------- + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False + Raises ------ BTrDBError [404] stream does not exist The stream could not be found in BTrDB - """ self._btrdb.ep.obliterate(self._uuid) - def flush(self): + @retry + def flush( + self, + auto_retry=False, + retries=5, + retry_delay=3, + retry_backoff=4, + ): """ Flush writes the stream buffers out to persistent storage. + Parameters + ---------- + auto_retry: bool, default: False + Whether to retry this request in the event of an error + retries: int, default: 5 + Number of times to retry this request if there is an error. Will + be ignored if auto_retry is False + retry_delay: int, default: 3 + initial time to wait before retrying function call if there is an error. + Will be ignored if auto_retry is False + retry_backoff: int, default: 4 + Exponential factor by which the backoff increases between retries. + Will be ignored if auto_retry is False """ self._btrdb.ep.flush(self._uuid) def __repr__(self): - return "".format(self.collection, - self.name) + return "".format(self.collection, self.name) + ########################################################################## ## StreamSet Classes ########################################################################## + class StreamSetBase(Sequence): """ A lighweight wrapper around a list of stream objects @@ -800,20 +1440,30 @@ class StreamSetBase(Sequence): def __init__(self, streams): self._streams = streams + if len(self._streams) < 1: + raise ValueError( + f"Trying to create streamset with an empty list of streams {self._streams}." + ) + try: + self._btrdb = self._streams[0]._btrdb + except Exception as e: + warnings.warn(f"Issue setting btrdb object from stream: {e}") self._pinned_versions = None self.filters = [] self.pointwidth = None self.width = None - self.depth = None + self.depth = 0 @property def allow_window(self): - return not bool(self.pointwidth or (self.width and self.depth)) + return not bool(self.pointwidth or (self.width and self.depth == 0)) def _latest_versions(self): - return {s.uuid: s.version() for s in self._streams} - + uuid_ver_tups = self._btrdb._executor.map( + lambda s: (s.uuid, s.version()), self._streams + ) + return {uu: v for uu, v in uuid_ver_tups} def pin_versions(self, versions=None): """ @@ -841,7 +1491,6 @@ def pin_versions(self, versions=None): if not isinstance(key, uuidlib.UUID): raise BTRDBTypeError("version keys must be type UUID") - self._pinned_versions = self._latest_versions() if not versions else versions return self @@ -862,9 +1511,11 @@ def versions(self): A dict containing the stream UUID and version ints as key/values """ - return self._pinned_versions if self._pinned_versions else self._latest_versions() + return ( + self._pinned_versions if self._pinned_versions else self._latest_versions() + ) - def count(self): + def count(self, precise: bool = False): """ Compute the total number of points in the streams using filters. @@ -875,13 +1526,12 @@ def count(self): Note that this helper method sums the counts of all StatPoints returned by ``aligned_windows``. Because of this the start and end timestamps - may be adjusted if they are not powers of 2. You can also set the - pointwidth property for smaller windows of time to ensure that the - count granularity is captured appropriately. + may be adjusted if they are not powers of 2. Parameters ---------- - None + precise : bool, default = False + Use statpoint counts using aligned_windows which trades accuracy for speed. Returns ------- @@ -891,14 +1541,19 @@ def count(self): params = self._params_from_filters() start = params.get("start", MINIMUM_TIME) end = params.get("end", MAXIMUM_TIME) - versions = self._pinned_versions if self._pinned_versions else {} - count = 0 - for s in self._streams: - version = versions.get(s.uuid, 0) - count += s.count(start, end, version=version) + versions = ( + self._pinned_versions if self._pinned_versions else self._latest_versions() + ) - return count + my_counts_gen = self._btrdb._executor.map( + lambda s: s.count( + start, end, version=versions.get(s.uuid, 0), precise=precise + ), + self._streams, + ) + + return sum(list(my_counts_gen)) def earliest(self): """ @@ -918,10 +1573,13 @@ def earliest(self): params = self._params_from_filters() start = params.get("start", MINIMUM_TIME) versions = self.versions() - - for s in self._streams: - version = versions.get(s.uuid, 0) - point, _ = s.nearest(start, version=version, backward=False) + if IS_DEBUG: + logger.debug(f"versions: {versions}") + earliest_points_gen = self._btrdb._executor.map( + lambda s: s.nearest(start, version=versions.get(s.uuid, 0), backward=False), + self._streams, + ) + for point, _ in earliest_points_gen: earliest.append(point) return tuple(earliest) @@ -944,10 +1602,11 @@ def latest(self): params = self._params_from_filters() start = params.get("end", MAXIMUM_TIME) versions = self.versions() - - for s in self._streams: - version = versions.get(s.uuid, 0) - point, _ = s.nearest(start, version=version, backward=True) + latest_points_gen = self._btrdb._executor.map( + lambda s: s.nearest(start, version=versions.get(s.uuid, 0), backward=True), + self._streams, + ) + for point, _ in latest_points_gen: latest.append(point) return tuple(latest) @@ -955,13 +1614,9 @@ def latest(self): def current(self): """ Returns the points of data in the streams closest to the current timestamp. If - the current timestamp is outside of the filtered range of data, a ValueError is + the current timestamp is outside the filtered range of data, a ValueError is raised. - Parameters - ---------- - None - Returns ------- tuple @@ -974,17 +1629,31 @@ def current(self): start = params.get("start", None) if (end is not None and end <= now) or (start is not None and start > now): - raise BTRDBValueError("current time is not included in filtered stream range") + raise BTRDBValueError( + "current time is not included in filtered stream range" + ) - for s in self._streams: - version = self.versions()[s.uuid] - point, _ = s.nearest(now, version=version, backward=True) + versions = self.versions() + latest_points_gen = self._btrdb._executor.map( + lambda s: (s.nearest(now, version=versions.get(s.uuid, 0), backward=True)), + self._streams, + ) + for point in latest_points_gen: latest.append(point) return tuple(latest) - def filter(self, start=None, end=None, collection=None, name=None, unit=None, - tags=None, annotations=None): + def filter( + self, + start=None, + end=None, + collection=None, + name=None, + unit=None, + tags=None, + annotations=None, + sampling_frequency=None, + ): """ Provides a new StreamSet instance containing stored query parameters and stream objects that match filtering criteria. @@ -1019,58 +1688,90 @@ def filter(self, start=None, end=None, collection=None, name=None, unit=None, key/value pairs for filtering streams based on tags annotations : dict key/value pairs for filtering streams based on annotations + sampling_frequency : int + The sampling frequency of the data streams in Hz, set this if you want timesnapped values. Returns ------- StreamSet a new instance cloned from the original with filters applied + Notes + ----- + If you set `sampling_frequency` to a non-zero value, the stream data returned will be aligned to a + grid of timestamps based on the period of the sampling frequency. For example, a sampling rate of 30hz will + have a sampling period of 1/30hz -> ~33_333_333 ns per sample. Leave sampling_frequency as None, or set to 0 to + prevent time alignment. You should **not** use aligned data for frequency-based analysis. """ obj = self.clone() - if start is not None or end is not None: - obj.filters.append(StreamFilter(start, end)) + if start is not None or end is not None or sampling_frequency is not None: + obj.filters.append( + StreamFilter( + start=start, end=end, sampling_frequency=sampling_frequency + ) + ) # filter by collection if collection is not None: if isinstance(collection, RE_PATTERN): - obj._streams = [s for s in obj._streams for m in [collection.search(s.collection)] if m] + obj._streams = [ + s + for s in obj._streams + for m in [collection.search(s.collection)] + if m + ] elif isinstance(collection, str): - obj._streams = [s for s in obj._streams if s.collection.lower() == collection.lower()] + obj._streams = [ + s + for s in obj._streams + if s.collection.lower() == collection.lower() + ] else: raise BTRDBTypeError("collection must be string or compiled regex") # filter by name if name is not None: if isinstance(name, RE_PATTERN): - obj._streams = [s for s in obj._streams for m in [name.search(s.name)] if m] + obj._streams = [ + s for s in obj._streams for m in [name.search(s.name)] if m + ] elif isinstance(name, str): - obj._streams = [s for s in obj._streams if s.name.lower() == name.lower()] + obj._streams = [ + s for s in obj._streams if s.name.lower() == name.lower() + ] else: raise BTRDBTypeError("name must be string or compiled regex") # filter by unit if unit is not None: if isinstance(unit, RE_PATTERN): - obj._streams = [s for s in obj._streams for m in [unit.search(s.tags()["unit"])] if m] + obj._streams = [ + s + for s in obj._streams + for m in [unit.search(s.tags()["unit"])] + if m + ] elif isinstance(unit, str): - obj._streams = [s for s in obj._streams if s.tags().get("unit", "").lower() == unit.lower()] + obj._streams = [ + s + for s in obj._streams + if s.tags().get("unit", "").lower() == unit.lower() + ] else: raise BTRDBTypeError("unit must be string or compiled regex") # filter by tags if tags: # filters if the subset of the tags matches the given tags - obj._streams = [ - s for s in obj._streams - if tags.items() <= s.tags().items() - ] + obj._streams = [s for s in obj._streams if tags.items() <= s.tags().items()] # filter by annotations if annotations: # filters if the subset of the annotations matches the given annotations obj._streams = [ - s for s in obj._streams + s + for s in obj._streams if annotations.items() <= s.annotations()[0].items() ] @@ -1091,14 +1792,14 @@ def clone(self): Returns a new copy of the instance """ - protected = ('_streams', ) + protected = ("_streams", "_btrdb") clone = self.__class__(self._streams) for attr, val in self.__dict__.items(): if attr not in protected: setattr(clone, attr, deepcopy(val)) return clone - def windows(self, width, depth): + def windows(self, width, depth=0): """ Stores the request for a windowing operation when the query is eventually materialized. @@ -1123,12 +1824,12 @@ def windows(self, width, depth): Windows returns arbitrary precision windows from BTrDB. It is slower than aligned_windows, but still significantly faster than values. Each returned window will be width nanoseconds long. start is inclusive, but - end is exclusive (e.g if end < start+width you will get no results). + end is exclusive (e.g. if end < start+width you will get no results). That is, results will be returned for all windows that start at a time less than the end timestamp. If (end - start) is not a multiple of width, then end will be decreased to the greatest value less than end such that (end - start) is a multiple of width (i.e., we set end = start - + width * floordiv(end - start, width). The `depth` parameter previously + + width * floordiv(end - start, width)). The `depth` parameter previously available has been deprecated. The only valid value for depth is now 0. """ @@ -1137,7 +1838,11 @@ def windows(self, width, depth): # TODO: refactor keeping in mind how exception is raised self.width = int(width) - self.depth = int(depth) + if depth != 0: + warnings.warn( + f"The only valid value for `depth` is now 0. You provided: {depth}. Overriding this value to 0." + ) + self.depth = 0 return self def aligned_windows(self, pointwidth): @@ -1185,35 +1890,46 @@ def _streamset_data(self, as_iterators=False): Returns each single stream's data as an iterator. Defaults to False. """ params = self._params_from_filters() - versions = self.versions() - data = [] + # sampling freq not supported for non-arrow streamset ops + _ = params.pop("sampling_frequency", None) + versions = self._pinned_versions + if versions is None: + versions = {s.uuid: 0 for s in self} if self.pointwidth is not None: # create list of stream.aligned_windows data params.update({"pointwidth": self.pointwidth}) - for s in self._streams: - params.update({"version": versions[s.uuid]}) - data.append(s.aligned_windows(**params)) - + # need to update params based on version of stream, use dict merge + aligned_windows_gen = self._btrdb._executor.map( + lambda s: s.aligned_windows( + **{**params, **{"version": versions[s.uuid]}} + ), + self._streams, + ) + data = list(aligned_windows_gen) elif self.width is not None and self.depth is not None: # create list of stream.windows data (the windows method should # prevent the possibility that only one of these is None) params.update({"width": self.width, "depth": self.depth}) - for s in self._streams: - params.update({"version": versions[s.uuid]}) - data.append(s.windows(**params)) + windows_gen = self._btrdb._executor.map( + lambda s: s.windows(**{**params, **{"version": versions[s.uuid]}}), + self._streams, + ) + data = list(windows_gen) else: # create list of stream.values - data = [s.values(**params) for s in self._streams] + values_gen = self._btrdb._executor.map( + lambda s: s.values(**params), self._streams + ) + data = list(values_gen) if as_iterators: return [iter(ii) for ii in data] return data - def rows(self): """ Returns a materialized list of tuples where each tuple contains the @@ -1240,7 +1956,6 @@ def rows(self): # add next values from streams into buffer for stream_idx, data in enumerate(streamset_data): - if buffer.active[stream_idx]: try: point, _ = next(data) @@ -1259,6 +1974,94 @@ def rows(self): return result + def insert(self, data_map: dict, merge: str = "never") -> dict: + """Insert new data in the form (time, value) into their mapped streams. + + The times in the dataframe need not be sorted by time. If the point counts are larger than + appropriate, this function will automatically chunk the inserts. As a + consequence, the insert is not necessarily atomic, but can be used with + a very large array. + + Parameters + ---------- + data_map: dict[uuid, pandas.DataFrame] + A dictionary mapping stream uuids to insert data into and their value as a + pandas dataframe containing two columns, one named "time" which contains int64 utc+0 timestamps + and a "value" column containing float64 measurements. These columns will be typecast into these types. + merge: str + A string describing the merge policy. Valid policies are: + - 'never': the default, no points are merged + - 'equal': points are deduplicated if the time and value are equal + - 'retain': if two points have the same timestamp, the old one is kept + - 'replace': if two points have the same timestamp, the new one is kept + + Notes + ----- + You MUST convert your datetimes into utc+0 yourself. BTrDB expects utc+0 datetimes. + + Returns + ------- + dict[uuid, int] + The versions of the stream after inserting new points. + """ + filtered_data_map = {s.uuid: data_map[s.uuid] for s in self._streams} + for key, dat in filtered_data_map.items(): + data_list = [ + (times, vals) + for times, vals in zip( + dat["time"].astype(int).values, dat["value"].astype(float).values + ) + ] + filtered_data_map[key] = data_list + versions_gen = self._btrdb._executor.map( + lambda s: (s.uuid, s.insert(data=filtered_data_map[s.uuid], merge=merge)), + self._streams, + ) + versions = {uu: ver for uu, ver in versions_gen} + return versions + + def arrow_insert(self, data_map: dict, merge: str = "never") -> dict: + """Insert new data in the form (time, value) into their mapped streams using pyarrow tables. + + The times in the arrow table need not be sorted by time. If the point counts are larger than + appropriate, this function will automatically chunk the inserts. As a + consequence, the insert is not necessarily atomic, but can be used with + a very large array. + + Parameters + ---------- + data_map: dict[uuid, pyarrow.Table] + A dictionary keyed on stream uuids and mapped to pyarrow tables with a schema + of time:Timestamp[ns, tz=UTC], value:float64. This schema will be validated and converted if necessary. + merge: str + A string describing the merge policy. Valid policies are: + - 'never': the default, no points are merged + - 'equal': points are deduplicated if the time and value are equal + - 'retain': if two points have the same timestamp, the old one is kept + - 'replace': if two points have the same timestamp, the new one is kept + + Notes + ----- + BTrDB expects datetimes to be in UTC+0. + + This method is available for commercial customers with arrow-enabled servers. + + Returns + ------- + dict[uuid, int] + The versions of the stream after inserting new points. + """ + filtered_data_map = {s.uuid: data_map[s.uuid] for s in self._streams} + versions_gen = self._btrdb._executor.map( + lambda s: ( + s.uuid, + s.arrow_insert(data=filtered_data_map[s.uuid], merge=merge), + ), + self._streams, + ) + versions = {uu: ver for uu, ver in versions_gen} + return versions + def _params_from_filters(self): params = {} for filter in self.filters: @@ -1266,6 +2069,8 @@ def _params_from_filters(self): params["start"] = filter.start if filter.end is not None: params["end"] = filter.end + if filter.sampling_frequency is not None: + params["sampling_frequency"] = filter.sampling_frequency return params def values_iter(self): @@ -1282,14 +2087,114 @@ def values(self): streamset_data = self._streamset_data() for stream_data in streamset_data: result.append([point[0] for point in stream_data]) - return result + def arrow_values( + self, + ): + """Return a pyarrow table of stream values based on the streamset parameters. + + Notes + ----- + This method is available for commercial customers with arrow-enabled servers. + """ + params = self._params_from_filters() + versions = self._pinned_versions + if versions is None: + versions = {s.uuid: 0 for s in self} + + if params.get("sampling_frequency", None) is None: + _ = params.pop("sampling_frequency", None) + + if self.pointwidth is not None: + # create list of stream.aligned_windows data + params.update({"pointwidth": self.pointwidth}) + _ = params.pop("sampling_frequency", None) + # need to update params based on version of stream, use dict merge + # {**dict1, **dict2} creates a new dict which updates the key/values if there are any + # same keys in dict1 and dict2, then the second dict in the expansion (dict2 in this case) + # will update the key (similar to dict1.update(dict2)) + aligned_windows_gen = self._btrdb._executor.map( + lambda s: s.arrow_aligned_windows( + **{**params, **{"version": versions[s.uuid]}} + ), + self._streams, + ) + stream_uus = [str(s.uuid) for s in self._streams] + data = list(aligned_windows_gen) + tablex = data.pop(0) + uu = stream_uus.pop(0) + tab_columns = [ + c if c == "time" else uu + "/" + c for c in tablex.column_names + ] + tablex = tablex.rename_columns(tab_columns) + if data: + for tab, uu in zip(data, stream_uus): + tab_columns = [ + c if c == "time" else uu + "/" + c for c in tab.column_names + ] + tab = tab.rename_columns(tab_columns) + tablex = tablex.join(tab, "time", join_type="full outer") + data = tablex + else: + data = tablex + + elif self.width is not None and self.depth is not None: + # create list of stream.windows data (the windows method should + # prevent the possibility that only one of these is None) + _ = params.pop("sampling_frequency", None) + params.update({"width": self.width}) + windows_gen = self._btrdb._executor.map( + lambda s: s.arrow_windows( + **{**params, **{"version": versions[s.uuid]}} + ), + self._streams, + ) + stream_uus = [str(s.uuid) for s in self._streams] + data = list(windows_gen) + tablex = data.pop(0) + uu = stream_uus.pop(0) + tab_columns = [ + c if c == "time" else uu + "/" + c for c in tablex.column_names + ] + tablex = tablex.rename_columns(tab_columns) + if data: + for tab, uu in zip(data, stream_uus): + tab_columns = [ + c if c == "time" else uu + "/" + c for c in tab.column_names + ] + tab = tab.rename_columns(tab_columns) + tablex = tablex.join(tab, "time", join_type="full outer") + data = tablex + else: + data = tablex + else: + sampling_freq = params.pop("sampling_frequency", 0) + period_ns = 0 + if sampling_freq > 0: + period_ns = _to_period_ns(sampling_freq) + params["uu_list"] = [s.uuid for s in self._streams] + params["version_list"] = [versions[s.uuid] for s in self._streams] + params["snap_periodNS"] = period_ns + table = list(self._btrdb.ep.arrowMultiValues(**params)) + if len(table) > 0: + data = pa.concat_tables(table) + else: + schema = pa.schema( + [pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False)] + + [ + pa.field(str(s.uuid), pa.float64(), nullable=False) + for s in self._streams + ], + ) + data = pa.Table.from_arrays( + [pa.array([]) for i in range(1 + len(self._streams))], schema=schema + ) + return data + def __repr__(self): token = "stream" if len(self) == 1 else "streams" - return "<{}({} {})>".format( - self.__class__.__name__, len(self._streams), token - ) + return "<{}({} {})>".format(self.__class__.__name__, len(self._streams), token) def __str__(self): token = "stream" if len(self) == 1 else "streams" @@ -1317,6 +2222,7 @@ class StreamSet(StreamSetBase, StreamSetTransformer): """ Public class for a collection of streams """ + pass @@ -1324,16 +2230,42 @@ class StreamSet(StreamSetBase, StreamSetTransformer): ## Utility Classes ########################################################################## + class StreamFilter(object): """ Object for storing requested filtering options """ - def __init__(self, start=None, end=None): + + def __init__( + self, start: int = None, end: int = None, sampling_frequency: int = None + ): self.start = to_nanoseconds(start) if start else None self.end = to_nanoseconds(end) if end else None + self.sampling_frequency = ( + int(sampling_frequency) if sampling_frequency else None + ) if self.start is None and self.end is None: raise BTRDBValueError("A valid `start` or `end` must be supplied") if self.start is not None and self.end is not None and self.start >= self.end: raise BTRDBValueError("`start` must be strictly less than `end` argument") + + +def _to_period_ns(fs: int): + """Convert sampling rate to sampling period in ns.""" + period = 1 / fs + period_ns = period * 1e9 + return int(period_ns) + + +def _coalesce_table_deque(tables: deque): + main_table = tables.popleft() + idx = 0 + while len(tables) != 0: + idx = idx + 1 + t2 = tables.popleft() + main_table = main_table.join( + t2, "time", join_type="full outer", right_suffix=f"_{idx}" + ) + return main_table diff --git a/btrdb/transformers.py b/btrdb/transformers.py index 1cf346c..4939215 100644 --- a/btrdb/transformers.py +++ b/btrdb/transformers.py @@ -15,20 +15,26 @@ ## Imports ########################################################################## -import csv import contextlib +import csv from collections import OrderedDict +from typing import Sequence from warnings import warn +import pandas as pd +import pyarrow + ########################################################################## ## Helper Functions ########################################################################## -_STAT_PROPERTIES = ('min', 'mean', 'max', 'count', 'stddev') +_STAT_PROPERTIES = ("min", "mean", "max", "count", "stddev") + def _get_time_from_row(row): for item in row: - if item: return item.time + if item: + return item.time raise Exception("Row contains no data") @@ -38,15 +44,14 @@ def _stream_names(streamset, func): before sending a collection of streams (dataframe, etc.) back to the user. """ - return tuple( - func(s) for s in streamset._streams - ) + return tuple(func(s) for s in streamset._streams) ########################################################################## ## Transform Functions ########################################################################## + def to_series(streamset, datetime64_index=True, agg="mean", name_callable=None): """ Returns a list of Pandas Series objects indexed by time @@ -63,7 +68,7 @@ def to_series(streamset, datetime64_index=True, agg="mean", name_callable=None): argument is ignored if RawPoint values are passed into the function. name_callable : lambda, default: lambda s: s.collection + "/" + s.name - Sprecify a callable that can be used to determine the series name given a + Specify a callable that can be used to determine the series name given a Stream object. """ @@ -77,8 +82,7 @@ def to_series(streamset, datetime64_index=True, agg="mean", name_callable=None): raise AttributeError("cannot use 'all' as aggregate at this time") if not callable(name_callable): - name_callable = lambda s: s.collection + "/" + s.name - + name_callable = lambda s: s.collection + "/" + s.name result = [] stream_names = _stream_names(streamset, name_callable) @@ -93,14 +97,139 @@ def to_series(streamset, datetime64_index=True, agg="mean", name_callable=None): values.append(getattr(point, agg)) if datetime64_index: - times = pd.Index(times, dtype='datetime64[ns]') + times = pd.Index(times, dtype="datetime64[ns]") - result.append(pd.Series( - data=values, index=times, name=stream_names[idx] - )) + result.append(pd.Series(data=values, index=times, name=stream_names[idx])) return result +def arrow_to_series(streamset, agg="mean", name_callable=None): + """ + Returns a list of Pandas Series objects indexed by time + + Parameters + ---------- + agg : List[str], default: ["mean"] + Specify the StatPoint field or fields (e.g. aggregating function) to create the Series + from. Must be one or more of "min", "mean", "max", "count", or "stddev". This + argument is ignored if RawPoint values are passed into the function. + + name_callable : lambda, default: lambda s: s.collection + "/" + s.name + Specify a callable that can be used to determine the series name given a + Stream object. + + Notes + ----- + This method is available for commercial customers with arrow-enabled servers. + """ + if not streamset._btrdb._ARROW_ENABLED: + raise NotImplementedError( + "arrow_to_series requires an arrow-enabled BTrDB server." + ) + if agg is None: + agg = ["mean"] + if not isinstance(agg, list): + raise ValueError( + f"Expected argument 'agg' to be a list of strings, was provided type - {type(agg)} with values - {agg}" + ) + arrow_df = arrow_to_dataframe( + streamset=streamset, agg=agg, name_callable=name_callable + ) + return [arrow_df[col] for col in arrow_df] + + +def arrow_to_dataframe( + streamset, columns=None, agg=None, name_callable=None +) -> pd.DataFrame: + """ + Returns a Pandas DataFrame object indexed by time and using the values of a + stream for each column. + + Parameters + ---------- + columns: sequence + column names to use for DataFrame. Deprecated and not compatible with name_callable. + + agg : List[str], default: ["mean"] + Specify the StatPoint fields (e.g. aggregating function) to create the dataframe + from. Must be one or more of "min", "mean", "max", "count", "stddev", or "all". This + argument is ignored if not using StatPoints. + + name_callable : lambda, default: lambda s: s.collection + "/" + s.name + Specify a callable that can be used to determine the series name given a + Stream object. + + Notes + ----- + This method is available for commercial customers with arrow-enabled servers. + """ + if not streamset._btrdb._ARROW_ENABLED: + raise NotImplementedError( + "arrow_to_dataframe requires an arrow-enabled BTrDB server." + ) + + try: + import pandas as pd + import pyarrow as pa + except ImportError as err: + raise ImportError( + f"Please install Pandas and pyarrow to use this transformation function. ErrorMessage: {err}" + ) + # deprecation warning added in v5.8 + if columns: + warn( + "the columns argument is deprecated and will be removed in a future release", + DeprecationWarning, + stacklevel=2, + ) + + if agg is None: + agg = ["mean"] + if not isinstance(agg, list): + raise ValueError( + f"Argument 'agg' not provided as a list of strings was provided type - {type(agg)} with values - {agg}" + ) + # do not allow agg="all" with RawPoints + if "all" in agg and streamset.allow_window: + agg = "" + elif any(["all" in val for val in agg]) and not streamset.allow_window: + agg = ["min", "mean", "max", "count", "stddev"] + else: + agg = agg + + # default arg values + if not callable(name_callable): + name_callable = lambda s: s.collection + "/" + s.name + # format is: uuid/stat_type + tmp_table = streamset.arrow_values() + col_names = _stream_names(streamset, name_callable) + col_names_map = {str(s.uuid): c for s, c in zip(streamset, col_names)} + updated_table_columns = [] + for old_col in tmp_table.column_names: + if old_col == "time": + updated_table_columns.append("time") + else: + for uu, new_name in col_names_map.items(): + if uu in old_col: + updated_table_columns.append(old_col.replace(uu, new_name)) + else: + continue + tmp_table = tmp_table.rename_columns(updated_table_columns) + if not streamset.allow_window: + usable_cols = [] + for column_str in tmp_table.column_names: + for agg_name in agg: + if agg_name in column_str: + usable_cols.append(column_str) + tmp = tmp_table.select(["time", *usable_cols]) + else: + tmp = tmp_table + df = tmp.to_pandas(date_as_object=False, types_mapper=pd.ArrowDtype) + df = df.set_index("time") + df.index = pd.DatetimeIndex(df.index, tz="UTC") + return df + + def to_dataframe(streamset, columns=None, agg="mean", name_callable=None): """ Returns a Pandas DataFrame object indexed by time and using the values of a @@ -117,7 +246,7 @@ def to_dataframe(streamset, columns=None, agg="mean", name_callable=None): argument is ignored if not using StatPoints. name_callable : lambda, default: lambda s: s.collection + "/" + s.name - Sprecify a callable that can be used to determine the series name given a + Specify a callable that can be used to determine the series name given a Stream object. This is not compatible with agg == "all" at this time @@ -129,35 +258,157 @@ def to_dataframe(streamset, columns=None, agg="mean", name_callable=None): # deprecation warning added in v5.8 if columns: - warn("the columns argument is deprecated and will be removed in a future release", DeprecationWarning, stacklevel=2) + warn( + "the columns argument is deprecated and will be removed in a future release", + DeprecationWarning, + stacklevel=2, + ) # TODO: allow this at some future point if agg == "all" and name_callable is not None: - raise AttributeError("cannot provide name_callable when using 'all' as aggregate at this time") + raise AttributeError( + "cannot provide name_callable when using 'all' as aggregate at this time" + ) # do not allow agg="all" with RawPoints if agg == "all" and streamset.allow_window: - agg="" + agg = "" # default arg values if not callable(name_callable): - name_callable = lambda s: s.collection + "/" + s.name - + name_callable = lambda s: s.collection + "/" + s.name - df = pd.DataFrame(to_dict(streamset,agg=agg)) + df = pd.DataFrame(to_dict(streamset, agg=agg, name_callable=name_callable)) if not df.empty: df = df.set_index("time") if agg == "all" and not streamset.allow_window: - stream_names = [[s.collection, s.name, prop] for s in streamset._streams for prop in _STAT_PROPERTIES] - df.columns=pd.MultiIndex.from_tuples(stream_names) + stream_names = [ + [s.collection, s.name, prop] + for s in streamset._streams + for prop in _STAT_PROPERTIES + ] + df.columns = pd.MultiIndex.from_tuples(stream_names) else: - df.columns = columns if columns else _stream_names(streamset, name_callable) + df.columns = columns if columns else _stream_names(streamset, name_callable) return df +def arrow_to_polars(streamset, agg=None, name_callable=None): + """ + Returns a Polars DataFrame object with time as a column and the values of a + stream for each additional column from an arrow table. + + Parameters + ---------- + agg : List[str], default: ["mean"] + Specify the StatPoint field or fields (e.g. aggregating function) to create the dataframe + from. Must be one or multiple of "min", "mean", "max", "count", "stddev", or "all". This + argument is ignored if not using StatPoints. + + name_callable : lambda, default: lambda s: s.collection + "/" + s.name + Specify a callable that can be used to determine the series name given a + Stream object. + + Notes + ----- + This method is available for commercial customers with arrow-enabled servers. + """ + if not streamset._btrdb._ARROW_ENABLED: + raise NotImplementedError( + "arrow_to_polars requires an arrow-enabled BTrDB server." + ) + try: + import polars as pl + except ImportError: + raise ImportError("Please install polars to use this transformation function.") + if agg is None: + agg = ["mean"] + if not isinstance(agg, list): + raise ValueError( + f"Expected argument 'agg' to be a list of strings, was provided type - {type(agg)} with values - {agg}" + ) + arrow_df = arrow_to_dataframe( + streamset=streamset, agg=agg, name_callable=name_callable + ) + return pl.from_pandas(arrow_df, include_index=True, nan_to_null=False) + + +def arrow_to_arrow_table(streamset): + """Return a pyarrow table of data. + + Notes + ----- + This method is available for commercial customers with arrow-enabled servers. + """ + if not streamset._btrdb._ARROW_ENABLED: + raise NotImplementedError( + "arrow_to_arrow_table requires an arrow-enabled BTrDB server." + ) + return streamset.arrow_values() + + +def to_polars(streamset, agg="mean", name_callable=None): + """ + Returns a Polars DataFrame object with time as a column and the values of a + stream for each additional column. + + Parameters + ---------- + agg : str, default: "mean" + Specify the StatPoint field (e.g. aggregating function) to create the Series + from. Must be one of "min", "mean", "max", "count", "stddev", or "all". This + argument is ignored if not using StatPoints. + + name_callable : lambda, default: lambda s: s.collection + "/" + s.name + Specify a callable that can be used to determine the series name given a + Stream object. This is not compatible with agg == "all" at this time + """ + try: + import polars as pl + except ImportError: + raise ImportError("Please install polars to use this transformation function.") + + # TODO: allow this at some future point + if agg == "all" and name_callable is not None: + raise AttributeError( + "cannot provide name_callable when using 'all' as aggregate at this time" + ) + + # do not allow agg="all" with RawPoints + if agg == "all" and not streamset.allow_window: + agg = "" + + # default arg values + if not callable(name_callable): + name_callable = lambda s: s.collection + "/" + s.name + + df = streamset.to_dataframe(agg=agg, name_callable=name_callable) + else: + df = pd.DataFrame(to_dict(streamset, agg=agg, name_callable=name_callable)) + + if not df.empty: + if df.index.name == "time": + pass + else: + df = df.set_index("time") + + df.index = pd.DatetimeIndex(df.index, tz="UTC") + if agg == "all" and streamset.allow_window: + stream_names = [ + [s.collection, s.name, prop] + for s in streamset._streams + for prop in _STAT_PROPERTIES + ] + df.columns = pd.MultiIndex.from_tuples(stream_names) + else: + df.columns = _stream_names(streamset, name_callable) + + return pl.from_pandas(df.reset_index(), nan_to_null=False) + + def to_array(streamset, agg="mean"): """ Returns a multidimensional numpy array (similar to a list of lists) containing point @@ -188,8 +439,32 @@ def to_array(streamset, agg="mean"): segment.append(point.value) else: segment.append(getattr(point, agg)) - results.append(segment) - return np.array(results) + results.append(np.array(segment)) + return np.array(results, dtype=object) + + +def arrow_to_numpy(streamset, agg=None): + """Return a multidimensional array in the numpy format. + + Parameters + ---------- + agg : List[str], default: ["mean"] + Specify the StatPoint field or fields (e.g. aggregating function) to return for the + arrays. Must be one or more of "min", "mean", "max", "count", or "stddev". This + argument is ignored if RawPoint values are passed into the function. + + Notes + ----- + This method first converts to a pandas data frame then to a numpy array. + + This method is available for commercial customers with arrow-enabled servers. + """ + if not streamset._btrdb._ARROW_ENABLED: + raise NotImplementedError( + "arrow_to_numpy requires an arrow-enabled BTrDB server." + ) + arrow_df = arrow_to_dataframe(streamset=streamset, agg=agg, name_callable=None) + return arrow_df.values def to_dict(streamset, agg="mean", name_callable=None): @@ -205,34 +480,76 @@ def to_dict(streamset, agg="mean", name_callable=None): argument is ignored if RawPoint values are passed into the function. name_callable : lambda, default: lambda s: s.collection + "/" + s.name - Sprecify a callable that can be used to determine the series name given a + Specify a callable that can be used to determine the series name given a Stream object. """ if not callable(name_callable): - name_callable = lambda s: s.collection + "/" + s.name + name_callable = lambda s: s.collection + "/" + s.name data = [] stream_names = _stream_names(streamset, name_callable) for row in streamset.rows(): - item = OrderedDict({ - "time": _get_time_from_row(row), - }) + item = OrderedDict( + { + "time": _get_time_from_row(row), + } + ) for idx, col in enumerate(stream_names): if row[idx].__class__.__name__ == "RawPoint": item[col] = row[idx].value if row[idx] else None else: if agg == "all": for stat in _STAT_PROPERTIES: - item["{}-{}".format(col, stat)] = getattr(row[idx], stat) if row[idx] else None + item["{}-{}".format(col, stat)] = ( + getattr(row[idx], stat) if row[idx] else None + ) else: item[col] = getattr(row[idx], agg) if row[idx] else None data.append(item) return data -def to_csv(streamset, fobj, dialect=None, fieldnames=None, agg="mean", name_callable=None): +def arrow_to_dict(streamset, agg=None, name_callable=None): + """ + Returns a list of dicts for each time code with the appropriate + stream data attached. + + Parameters + ---------- + agg : List[str], default: ["mean"] + Specify the StatPoint field or fields (e.g. aggregating function) to constrain dict + keys. Must be one or more of "min", "mean", "max", "count", or "stddev". This + argument is ignored if RawPoint values are passed into the function. + + name_callable : lambda, default: lambda s: s.collection + "/" + s.name + Specify a callable that can be used to determine the series name given a + Stream object. + + Notes + ----- + This method is available for commercial customers with arrow-enabled servers. + """ + if not streamset._btrdb._ARROW_ENABLED: + raise NotImplementedError( + "arrow_to_dict requires an arrow-enabled BTrDB server." + ) + if agg is None: + agg = ["mean"] + if not isinstance(agg, list): + raise ValueError( + f"Expected a list of strings for 'agg', was provided type - {type(agg)} with values {agg}." + ) + arrow_df = arrow_to_dataframe( + streamset=streamset, agg=agg, name_callable=name_callable + ) + return arrow_df.to_dict(orient="index") + + +def to_csv( + streamset, fobj, dialect=None, fieldnames=None, agg="mean", name_callable=None +): """ Saves stream data as a CSV file. @@ -255,7 +572,7 @@ def to_csv(streamset, fobj, dialect=None, fieldnames=None, agg="mean", name_call This argument is ignored if RawPoint values are passed into the function. name_callable : lambda, default: lambda s: s.collection + "/" + s.name - Sprecify a callable that can be used to determine the series name given a + Specify a callable that can be used to determine the series name given a Stream object. """ @@ -264,12 +581,12 @@ def to_csv(streamset, fobj, dialect=None, fieldnames=None, agg="mean", name_call raise AttributeError("cannot use 'all' as aggregate at this time") if not callable(name_callable): - name_callable = lambda s: s.collection + "/" + s.name + name_callable = lambda s: s.collection + "/" + s.name @contextlib.contextmanager def open_path_or_file(path_or_file): if isinstance(path_or_file, str): - f = file_to_close = open(path_or_file, 'w', newline='') + f = file_to_close = open(path_or_file, "w", newline="") else: f = path_or_file file_to_close = None @@ -303,37 +620,53 @@ def to_table(streamset, agg="mean", name_callable=None): argument is ignored if RawPoint values are passed into the function. name_callable : lambda, default: lambda s: s.collection + "/" + s.name - Sprecify a callable that can be used to determine the column name given a + Specify a callable that can be used to determine the column name given a Stream object. """ try: from tabulate import tabulate except ImportError: - raise ImportError("Please install tabulate to use this transformation function.") + raise ImportError( + "Please install tabulate to use this transformation function." + ) # TODO: allow this at some future point if agg == "all": raise AttributeError("cannot use 'all' as aggregate at this time") if not callable(name_callable): - name_callable = lambda s: s.collection + "/" + s.name + name_callable = lambda s: s.collection + "/" + s.name - return tabulate(streamset.to_dict(agg=agg, name_callable=name_callable), headers="keys") + return tabulate( + streamset.to_dict(agg=agg, name_callable=name_callable), headers="keys" + ) ########################################################################## ## Transform Classes ########################################################################## + class StreamSetTransformer(object): """ Base class for StreamSet or Stream transformations """ + to_dict = to_dict + arrow_to_dict = arrow_to_dict + to_array = to_array + arrow_to_numpy = arrow_to_numpy + to_series = to_series + arrow_to_series = arrow_to_series + to_dataframe = to_dataframe + arrow_to_dataframe = arrow_to_dataframe + + to_polars = to_polars + arrow_to_polars = arrow_to_polars to_csv = to_csv to_table = to_table diff --git a/btrdb/utils/buffer.py b/btrdb/utils/buffer.py index 02b2fc5..d06f7b5 100644 --- a/btrdb/utils/buffer.py +++ b/btrdb/utils/buffer.py @@ -17,20 +17,18 @@ from collections import defaultdict - ########################################################################## ## Classes ########################################################################## -class PointBuffer(defaultdict): +class PointBuffer(defaultdict): def __init__(self, length, *args, **kwargs): super().__init__(lambda: [None] * length, *args, **kwargs) self.num_streams = length self.active = [True] * length self.last_known_time = [None] * length - def add_point(self, stream_index, point): self.last_known_time[stream_index] = max( (self.last_known_time[stream_index] or -1), point.time @@ -53,7 +51,6 @@ def is_ready(self, key): """ # check each stream value to see if its valid/ready for idx, latest_time in enumerate(self.last_known_time): - # return False if stream is active and we are waiting on val if self.active[idx] and (latest_time is None or key > latest_time): return False @@ -61,9 +58,7 @@ def is_ready(self, key): return True def next_key_ready(self): - """ - - """ + """ """ keys = list(self.keys()) keys.sort() for key in keys: diff --git a/btrdb/utils/conversion.py b/btrdb/utils/conversion.py index a1b979f..687be95 100644 --- a/btrdb/utils/conversion.py +++ b/btrdb/utils/conversion.py @@ -17,12 +17,12 @@ ## Imports ########################################################################## -import uuid import json -import pytz - +import uuid from datetime import datetime +import pytz + try: import numpy as np except ImportError: @@ -36,6 +36,7 @@ ## Classes ########################################################################## + class AnnotationEncoder(json.JSONEncoder): """Default JSON encoder class for saving stream annotations""" @@ -87,6 +88,7 @@ def decode(self, s): ## Functions ########################################################################## + def to_uuid(obj): """ Converts argument to UUID @@ -104,6 +106,4 @@ def to_uuid(obj): if isinstance(obj, str): return uuid.UUID(obj) - raise TypeError("Cannot convert object to UUID ({})".format( - obj.__class__.__name__) - ) + raise TypeError("Cannot convert object to UUID ({})".format(obj.__class__.__name__)) diff --git a/btrdb/utils/credentials.py b/btrdb/utils/credentials.py index e7d2510..d0c8263 100644 --- a/btrdb/utils/credentials.py +++ b/btrdb/utils/credentials.py @@ -16,10 +16,11 @@ ########################################################################## import os -import yaml from functools import wraps -from btrdb.exceptions import ProfileNotFound, CredentialsFileNotFound +import yaml + +from btrdb.exceptions import CredentialsFileNotFound, ProfileNotFound ########################################################################## ## Module Variables @@ -27,21 +28,27 @@ CONFIG_DIR = ".predictivegrid" CREDENTIALS_FILENAME = "credentials.yaml" -CREDENTIALS_PATH = os.path.join(os.path.expanduser("~"), CONFIG_DIR, CREDENTIALS_FILENAME) +CREDENTIALS_PATH = os.path.join( + os.path.expanduser("~"), CONFIG_DIR, CREDENTIALS_FILENAME +) ########################################################################## ## Functions ########################################################################## + def filter_none(f): """ decorator for removing dict items with None as value """ + @wraps(f) def inner(*args, **kwargs): - return { k: v for k, v in f(*args, **kwargs).items() if v is not None } + return {k: v for k, v in f(*args, **kwargs).items() if v is not None} + return inner + def load_credentials_from_file(): """ Returns a dict of the credentials file contents @@ -52,10 +59,13 @@ def load_credentials_from_file(): A dictionary of profile connection information """ try: - with open(CREDENTIALS_PATH, 'r') as stream: + with open(CREDENTIALS_PATH, "r") as stream: return yaml.safe_load(stream) except FileNotFoundError as exc: - raise CredentialsFileNotFound("Cound not find `{}`".format(CREDENTIALS_PATH)) from exc + raise CredentialsFileNotFound( + "Cound not find `{}`".format(CREDENTIALS_PATH) + ) from exc + @filter_none def credentials_by_profile(name=None): @@ -82,14 +92,16 @@ def credentials_by_profile(name=None): The requested profile could not be found in the credentials file """ if not name: - name = os.environ.get("BTRDB_PROFILE", 'default') + name = os.environ.get("BTRDB_PROFILE", "default") # load from credentials yaml file creds = load_credentials_from_file() if name not in creds.keys(): - if name == 'default': + if name == "default": return {} - raise ProfileNotFound("Profile `{}` not found in credentials file.".format(name)) + raise ProfileNotFound( + "Profile `{}` not found in credentials file.".format(name) + ) # rename api_key if needed and return fragment = creds[name].get("btrdb", {}) @@ -97,6 +109,7 @@ def credentials_by_profile(name=None): fragment["apikey"] = fragment.pop("api_key") return fragment + @filter_none def credentials_by_env(): """ @@ -110,7 +123,7 @@ def credentials_by_env(): """ return { "endpoints": os.environ.get("BTRDB_ENDPOINTS", None), - "apikey": os.environ.get("BTRDB_API_KEY", None), + "apikey": os.environ.get("BTRDB_API_KEY", None), } @@ -131,7 +144,12 @@ def credentials(endpoints=None, apikey=None): """ creds = {} - credentials_by_arg = filter_none(lambda: { "endpoints": endpoints, "apikey": apikey, }) + credentials_by_arg = filter_none( + lambda: { + "endpoints": endpoints, + "apikey": apikey, + } + ) pipeline = [credentials_by_env, credentials_by_arg] if os.path.exists(CREDENTIALS_PATH): pipeline.insert(0, credentials_by_profile) diff --git a/btrdb/utils/general.py b/btrdb/utils/general.py index 15ce1b8..6344fe6 100644 --- a/btrdb/utils/general.py +++ b/btrdb/utils/general.py @@ -36,6 +36,7 @@ def unpack_stream_descriptor(desc): ## Pointwidth Helpers ########################################################################## + class pointwidth(object): """ A representation of a period of time described by the BTrDB tree (and used in @@ -58,7 +59,7 @@ def from_timedelta(cls, delta): specified duration. Because pointwidths are in powers of 2, be sure to check that the returned real duration is sufficient. """ - return cls.from_nanoseconds(int(delta.total_seconds()*1e9)) + return cls.from_nanoseconds(int(delta.total_seconds() * 1e9)) @classmethod def from_nanoseconds(cls, nsec): @@ -117,10 +118,10 @@ def years(self): return self.nanoseconds / 3.154e16 def decr(self): - return pointwidth(self-1) + return pointwidth(self - 1) def incr(self): - return pointwidth(self+1) + return pointwidth(self + 1) def __int__(self): return self._pointwidth @@ -141,10 +142,10 @@ def __ge__(self, other): return int(self) >= int(other) def __add__(self, other): - return pointwidth(int(self)+int(other)) + return pointwidth(int(self) + int(other)) def __sub__(self, other): - return pointwidth(int(self)-int(other)) + return pointwidth(int(self) - int(other)) def __str__(self): """ @@ -180,4 +181,4 @@ def __str__(self): return "{:0.0f} nanoseconds".format(self.nanoseconds) def __repr__(self): - return "".format(int(self)) \ No newline at end of file + return "".format(int(self)) diff --git a/btrdb/utils/ray.py b/btrdb/utils/ray.py index e83ec2d..b1bbb66 100644 --- a/btrdb/utils/ray.py +++ b/btrdb/utils/ray.py @@ -1,6 +1,7 @@ +from functools import partial + import btrdb from btrdb.conn import BTrDB -from functools import partial def register_serializer(conn_str=None, apikey=None, profile=None): @@ -28,20 +29,35 @@ def register_serializer(conn_str=None, apikey=None, profile=None): import semver except ImportError: raise ImportError("must pip install semver to register custom serializer") - - assert ray.is_initialized(), "Need to call ray.init() before registering custom serializer" + + assert ( + ray.is_initialized() + ), "Need to call ray.init() before registering custom serializer" ver = semver.VersionInfo.parse(ray.__version__) if ver.major == 0: ray.register_custom_serializer( - BTrDB, serializer=btrdb_serializer, - deserializer=partial(btrdb_deserializer, conn_str=conn_str, apikey=apikey, profile=profile)) - elif ver.major == 1 and ver.minor in range(2, 13): # as of May 2022, latest Ray version is at 1.12 + BTrDB, + serializer=btrdb_serializer, + deserializer=partial( + btrdb_deserializer, conn_str=conn_str, apikey=apikey, profile=profile + ), + ) + elif ver.major == 1 and ver.minor in range( + 2, 13 + ): # as of May 2022, latest Ray version is at 1.12 # TODO: check different versions of ray? ray.util.register_serializer( - BTrDB, serializer=btrdb_serializer, - deserializer=partial(btrdb_deserializer, conn_str=conn_str, apikey=apikey, profile=profile)) + BTrDB, + serializer=btrdb_serializer, + deserializer=partial( + btrdb_deserializer, conn_str=conn_str, apikey=apikey, profile=profile + ), + ) else: - raise Exception("Ray version %s does not have custom serialization. Please upgrade to >= 1.4.0" % ray.__version__) + raise Exception( + "Ray version %s does not have custom serialization. Please upgrade to >= 1.4.0" + % ray.__version__ + ) def btrdb_serializer(_): @@ -50,10 +66,11 @@ def btrdb_serializer(_): """ return None + def btrdb_deserializer(_, conn_str=None, apikey=None, profile=None): """ deserialize function - + Parameters ---------- conn_str: str, default=None diff --git a/btrdb/utils/timez.py b/btrdb/utils/timez.py index cc311dc..32cec9e 100644 --- a/btrdb/utils/timez.py +++ b/btrdb/utils/timez.py @@ -18,27 +18,25 @@ ########################################################################## from datetime import datetime - -from operator import mul from decimal import Decimal +from operator import mul import pytz - ########################################################################## ## Module Variables ########################################################################## DATETIME_FORMATS = ( "%Y-%m-%d %H:%M:%S.%f%z", # most common RFC3339 nanoseconds - "%Y-%m-%d %H:%M:%S.%f", # expects UTC default timezone - "%Y-%m-%dT%H:%M:%S.%fZ", # JSON encoding, UTC timezone - "%Y-%m-%dT%H:%M:%SZ", # JSON encoding, UTC timezone + "%Y-%m-%d %H:%M:%S.%f", # expects UTC default timezone + "%Y-%m-%dT%H:%M:%S.%fZ", # JSON encoding, UTC timezone + "%Y-%m-%dT%H:%M:%SZ", # JSON encoding, UTC timezone "%Y-%m-%dT%H:%M:%S.%f%z", # less common JSON-ish encoding - "%Y-%m-%dT%H:%M:%S.%f", # for completeness, UTC default timezone - "%Y-%m-%d %H:%M:%S%z", # human readable date time with TZ - "%Y-%m-%d %H:%M:%S", # human readable date time UTC default - "%Y-%m-%d", # helper to get midnight on a particular date + "%Y-%m-%dT%H:%M:%S.%f", # for completeness, UTC default timezone + "%Y-%m-%d %H:%M:%S%z", # human readable date time with TZ + "%Y-%m-%d %H:%M:%S", # human readable date time UTC default + "%Y-%m-%d", # helper to get midnight on a particular date ) @@ -46,6 +44,7 @@ ## Functions ########################################################################## + def currently_as_ns(): """ Returns the current UTC time as nanoseconds since epoch @@ -141,6 +140,7 @@ def to_nanoseconds(val): try: import numpy as np + if isinstance(val, np.datetime64): val = val.astype(datetime) except ImportError: @@ -174,8 +174,9 @@ def to_nanoseconds(val): raise TypeError("only int, float, str, datetime, and datetime64 are allowed") -def ns_delta(days=0, hours=0, minutes=0, seconds=0, milliseconds=0, \ - microseconds=0, nanoseconds=0): +def ns_delta( + days=0, hours=0, minutes=0, seconds=0, milliseconds=0, microseconds=0, nanoseconds=0 +): """ Similar to `timedelta`, ns_delta represents a span of time but as the total number of nanoseconds. diff --git a/btrdb/version.py b/btrdb/version.py index 2592e90..492de74 100644 --- a/btrdb/version.py +++ b/btrdb/version.py @@ -15,20 +15,23 @@ ## Module Info ########################################################################## -__version_info__ = { 'major': 5, 'minor': 28, 'micro': 1, 'releaselevel': 'final'} +__version_info__ = {"major": 5, "minor": 28, "micro": 1, "releaselevel": "final"} ########################################################################## ## Helper Functions ########################################################################## + def get_version(short=False): """ Prints the version. """ - assert __version_info__['releaselevel'] in ('alpha', 'beta', 'final') - vers = ["%(major)i.%(minor)i" % __version_info__, ] - if __version_info__['micro']: + assert __version_info__["releaselevel"] in ("alpha", "beta", "final") + vers = [ + "%(major)i.%(minor)i" % __version_info__, + ] + if __version_info__["micro"]: vers.append(".%(micro)i" % __version_info__) - if __version_info__['releaselevel'] != 'final' and not short: - vers.append('%s%i' % (__version_info__['releaselevel'][0])) - return ''.join(vers) + if __version_info__["releaselevel"] != "final" and not short: + vers.append("%s%i" % (__version_info__["releaselevel"][0])) + return "".join(vers) diff --git a/btrdb4/__init__.py b/btrdb4/__init__.py index 85af01b..ee79a41 100644 --- a/btrdb4/__init__.py +++ b/btrdb4/__init__.py @@ -21,11 +21,13 @@ ## Warning ########################################################################## -MESSAGE = ("The btrdb4 package is not available in this version. To install " - "with pip use `pip install btrdb==4.*`") +MESSAGE = ( + "The btrdb4 package is not available in this version. To install " + "with pip use `pip install btrdb==4.*`" +) # raise exception on ImportWarning -warnings.simplefilter('error', ImportWarning) +warnings.simplefilter("error", ImportWarning) # warn user of version issue warnings.warn(MESSAGE, ImportWarning) diff --git a/docs/requirements.txt b/docs/requirements.txt index 074ee76..a3bb1f5 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,3 @@ alabaster>=0.7.12 -Sphinx>=1.7 \ No newline at end of file +Sphinx>=1.7 +sphinx-rtd-theme diff --git a/docs/source/api/transformers.rst b/docs/source/api/transformers.rst index 289a1ea..facbbc4 100644 --- a/docs/source/api/transformers.rst +++ b/docs/source/api/transformers.rst @@ -1,17 +1,30 @@ btrdb.transformers ========================= -A number of tranformation and serialization functions have been developed so +A number of transformation and serialization functions have been developed so you can use the data in the format of your choice. These functions are provided -in the :code:`btrdb.utils.transformers` module but are also available directly -off the the :code:`StreamSet` class. +in the :code:`StreamSet` class. .. automodule:: btrdb.transformers +.. autofunction:: arrow_to_dict + +.. autofunction:: arrow_to_numpy + +.. autofunction:: arrow_to_series + +.. autofunction:: arrow_to_dataframe + +.. autofunction:: arrow_to_polars + +.. autofunction:: arrow_to_arrow_table + .. autofunction:: to_dict .. autofunction:: to_array +.. autofunction:: to_polars + .. autofunction:: to_series .. autofunction:: to_dataframe diff --git a/docs/source/conf.py b/docs/source/conf.py index 79f76de..a2ce391 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -18,21 +18,22 @@ import os import sys -sys.path.insert(0, os.path.abspath('_themes')) -sys.path.insert(0, os.path.abspath('../..')) + +sys.path.insert(0, os.path.abspath("_themes")) +sys.path.insert(0, os.path.abspath("../..")) from btrdb.version import get_version # -- Project information ----------------------------------------------------- -project = 'btrdb' -copyright = '2021, Ping Things, Inc.' -author = 'PingThingsIO' +project = "btrdb" +copyright = "2021, Ping Things, Inc." +author = "PingThingsIO" # The short X.Y version version = get_version() # The full version, including alpha/beta/rc tags -release = 'v' + get_version() +release = "v" + get_version() # -- General configuration --------------------------------------------------- @@ -45,36 +46,37 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.napoleon', - 'sphinx.ext.todo', - 'sphinx.ext.githubpages', - 'sphinx.ext.intersphinx', + "sphinx.ext.autodoc", + "sphinx.ext.napoleon", + "sphinx.ext.todo", + "sphinx.ext.githubpages", + "sphinx.ext.intersphinx", + "numpydoc", ] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] -source_suffix = '.rst' +source_suffix = ".rst" # The master toctree document. -master_doc = 'index' +master_doc = "index" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] # The name of the Pygments (syntax highlighting) style to use. pygments_style = None @@ -85,31 +87,30 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'alabaster' +# html_theme = "alabaster" +html_theme = "sphinx_rtd_theme" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. # html_theme_options = { - 'show_powered_by': False, - 'github_user': 'PingThingsIO', - 'github_repo': 'btrdb-python', - 'travis_button': False, - 'github_banner': False, - 'show_related': False, - 'note_bg': '#FFF59C', - 'description': 'A midweight library to converse with the BTrDB database.', - 'extra_nav_links': { - "btrdb": "http://btrdb.io" - }, - 'show_relbars': True, + "show_powered_by": False, + "github_user": "PingThingsIO", + "github_repo": "btrdb-python", + "travis_button": False, + "github_banner": False, + "show_related": False, + "note_bg": "#FFF59C", + "description": "A midweight library to converse with the BTrDB database.", + "extra_nav_links": {"btrdb": "http://btrdb.io"}, + "show_relbars": True, } # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # Custom sidebar templates, must be a dictionary that maps document names # to template names. @@ -131,7 +132,7 @@ # -- Options for HTMLHelp output --------------------------------------------- # Output file base name for HTML help builder. -htmlhelp_basename = 'btrdb-pythondoc' +htmlhelp_basename = "btrdb-pythondoc" # -- Options for LaTeX output ------------------------------------------------ @@ -140,15 +141,12 @@ # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. # # 'preamble': '', - # Latex figure (float) alignment # # 'figure_align': 'htbp', @@ -158,8 +156,13 @@ # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'btrdb-python.tex', 'btrdb-python Documentation', - 'PingThingsIO', 'manual'), + ( + master_doc, + "btrdb-python.tex", + "btrdb-python Documentation", + "PingThingsIO", + "manual", + ), ] @@ -167,10 +170,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'btrdb-python', 'btrdb-python Documentation', - [author], 1) -] +man_pages = [(master_doc, "btrdb-python", "btrdb-python Documentation", [author], 1)] # -- Options for Texinfo output ---------------------------------------------- @@ -179,9 +179,15 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'btrdb-python', 'btrdb-python Documentation', - author, 'btrdb-python', 'One line description of project.', - 'Miscellaneous'), + ( + master_doc, + "btrdb-python", + "btrdb-python Documentation", + author, + "btrdb-python", + "One line description of project.", + "Miscellaneous", + ), ] @@ -200,7 +206,7 @@ # epub_uid = '' # A list of files that should not be packed into the epub file. -epub_exclude_files = ['search.html'] +epub_exclude_files = ["search.html"] # -- Extension configuration ------------------------------------------------- @@ -210,4 +216,4 @@ # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = True -numfig = True \ No newline at end of file +numfig = True diff --git a/docs/source/installing.rst b/docs/source/installing.rst index 394c068..5b82210 100644 --- a/docs/source/installing.rst +++ b/docs/source/installing.rst @@ -41,5 +41,3 @@ like to install the version 4 bindings you will need to use `pip` as shown above .. code-block:: bash $ conda install -c pingthings btrdb - - diff --git a/docs/source/working/multiprocessing.rst b/docs/source/working/multiprocessing.rst index 0aa6a67..c474154 100644 --- a/docs/source/working/multiprocessing.rst +++ b/docs/source/working/multiprocessing.rst @@ -78,4 +78,4 @@ Let's break this down quickly since this is a very common design pattern. First The ``stream_quality`` function is our parallelizable function (e.g. computing the data quality for multiple streams at a time). Depending on how long the ``data_quality`` function takes to compute, we may also want to parallelize ``(stream, start, end)`` tuples. -If you would like features like a connection pool object (as other databases have) or multiprocessing helpers, please leave us a note in our GitHub issues! \ No newline at end of file +If you would like features like a connection pool object (as other databases have) or multiprocessing helpers, please leave us a note in our GitHub issues! diff --git a/docs/source/working/ray.rst b/docs/source/working/ray.rst index 98a7fbd..f13fc80 100644 --- a/docs/source/working/ray.rst +++ b/docs/source/working/ray.rst @@ -18,7 +18,7 @@ Setting up the ray serializer ray.init() conn_params = {"profile": "profile_name"} - + # register serializer with the connection parameters register_serializer(**conn_params) @@ -55,4 +55,4 @@ Setting up the ray serializer >>(pid=28482) (RawPoint(1533210100000000000, 0.0), 0) # output of test_streamset >>(pid=28481) (RawPoint(1533210100000000000, 0.0), RawPoint(1533210100000000000, 0.0)) - >>(pid=28481) StreamSet with 2 streams \ No newline at end of file + >>(pid=28481) StreamSet with 2 streams diff --git a/docs/source/working/stream-manage-metadata.rst b/docs/source/working/stream-manage-metadata.rst index 9452253..75b7fd6 100644 --- a/docs/source/working/stream-manage-metadata.rst +++ b/docs/source/working/stream-manage-metadata.rst @@ -109,5 +109,3 @@ If you would like to remove any keys from your annotations you must use the `rep annotations, _ = stream.anotations() del annotations["key_to_delete"] stream.update(annotations=annotations, replace=True) - - diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..066ed3b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,76 @@ +[project] +name = "btrdb" +version = "5.28.1" +authors = [ + {name="PingThingsIO", email="support@pingthings.io"}, +] +description = "Bindings to interact with the Berkeley Tree Database using gRPC." +readme = "README.md" +license = {file="LICENSE.txt"} +requires-python = ">=3.7,<=3.10" +classifiers = [ + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'Intended Audience :: Information Technology', + 'Intended Audience :: Science/Research', + 'License :: OSI Approved :: BSD License', + 'Natural Language :: English', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Topic :: Database', + 'Topic :: Software Development :: Libraries :: Python Modules', +] +keywords = [ + 'btrdb', 'berkeley', 'timeseries', 'database', 'bindings', 'gRPC', +] +dependencies = [ + "grpcio", + "grpcio-tools", + "pytz", + "pyyaml", + "certifi", +] + +[project.optional-dependencies] +data = [ + "pyarrow", + "pandas>=2.0", + "polars", + "numpy", + "tabulate", +] +ray = [ + "btrdb[data]", + "ray" +] +testing = [ + "btrdb[data]", + "pytest", + "freezegun", + "pytest-cov", + "pytest-flake8" +] +all = [ + "btrdb[data]", + "btrdb[ray]", + "btrdb[testing]", +] + +[project.urls] +"Homepage" = "https://btrdb.io" +"Docs" = "https://btrdb.readthedocs.io" +"Repository" = "https://github.com/pingthingsio/btrdb-python.git" + +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.exclude-package-data] +drop = [ + "tests", + "docs" +] diff --git a/release.sh b/release.sh index 9104416..d630e0e 100755 --- a/release.sh +++ b/release.sh @@ -44,4 +44,4 @@ git tag v$1.$2.$3 git push origin v$1.$2.$3 sleep 10 -git push \ No newline at end of file +git push diff --git a/requirements-all-locked.txt b/requirements-all-locked.txt new file mode 100644 index 0000000..d6f7284 --- /dev/null +++ b/requirements-all-locked.txt @@ -0,0 +1,128 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --extra=all --output-file=requirements-all.txt --resolver=backtracking pyproject.toml +# +aiosignal==1.3.1 + # via ray +attrs==23.1.0 + # via + # jsonschema + # ray +certifi==2023.5.7 + # via + # btrdb (pyproject.toml) + # requests +charset-normalizer==3.1.0 + # via requests +click==8.1.3 + # via ray +coverage[toml]==7.2.7 + # via pytest-cov +distlib==0.3.6 + # via virtualenv +exceptiongroup==1.1.1 + # via pytest +filelock==3.12.0 + # via + # ray + # virtualenv +flake8==6.0.0 + # via pytest-flake8 +freezegun==1.2.2 + # via btrdb +frozenlist==1.3.3 + # via + # aiosignal + # ray +grpcio==1.54.2 + # via + # btrdb (pyproject.toml) + # grpcio-tools + # ray +grpcio-tools==1.54.2 + # via + # btrdb (pyproject.toml) +idna==3.4 + # via requests +iniconfig==2.0.0 + # via pytest +jsonschema==4.17.3 + # via ray +mccabe==0.7.0 + # via flake8 +msgpack==1.0.5 + # via ray +numpy==1.24.3 + # via + # btrdb + # pandas + # pyarrow + # ray +packaging==23.1 + # via + # pytest + # ray +pandas==2.0.2 + # via btrdb +platformdirs==3.5.1 + # via virtualenv +pluggy==1.0.0 + # via pytest +polars==0.18.0 + # via btrdb +protobuf==4.23.2 + # via + # grpcio-tools + # ray +pyarrow==12.0.0 + # via btrdb +pycodestyle==2.10.0 + # via flake8 +pyflakes==3.0.1 + # via flake8 +pyrsistent==0.19.3 + # via jsonschema +pytest==7.3.1 + # via + # btrdb + # pytest-cov + # pytest-flake8 +pytest-cov==4.1.0 + # via btrdb +pytest-flake8==1.1.1 + # via btrdb +python-dateutil==2.8.2 + # via + # freezegun + # pandas +pytz==2023.3 + # via + # btrdb (pyproject.toml) + # pandas +pyyaml==6.0 + # via + # btrdb (pyproject.toml) + # ray +ray==2.2.0 + # via btrdb +requests==2.31.0 + # via ray +six==1.16.0 + # via python-dateutil +tabulate==0.9.0 + # via btrdb +tomli==2.0.1 + # via + # coverage + # pytest +tzdata==2023.3 + # via pandas +urllib3==2.0.2 + # via requests +virtualenv==20.23.0 + # via ray + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/requirements-all.txt b/requirements-all.txt new file mode 100644 index 0000000..4a71eb1 --- /dev/null +++ b/requirements-all.txt @@ -0,0 +1,57 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --extra=all --output-file=requirements-all.txt --resolver=backtracking pyproject.toml +# +# Pinned versions and dependencies of dependencies were removed after the autogeneration. +certifi + # via + # btrdb (pyproject.toml) +freezegun + # via btrdb +grpcio + # via + # btrdb (pyproject.toml) + # grpcio-tools + # ray +grpcio-tools + # via + # btrdb (pyproject.toml) +numpy + # via + # btrdb + # pandas + # pyarrow + # ray +pandas + # via btrdb +polars + # via btrdb +protobuf + # via + # grpcio-tools + # ray +pyarrow + # via btrdb +pytest + # via + # btrdb + # pytest-cov + # pytest-flake8 +pytest-cov + # via btrdb +pytest-flake8 + # via btrdb +pytz + # via + # btrdb (pyproject.toml) + # pandas +pyyaml + # via + # btrdb (pyproject.toml) + # ray +ray + # via btrdb +tabulate + # via btrdb diff --git a/requirements.txt b/requirements.txt index 8b35bcf..a921486 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,24 @@ -# GRPC / Protobuff related -grpcio>=1.19.0,<=1.48.2 -grpcio-tools>=1.19.0,<=1.48.2 - -# Time related utils +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --output-file=requirements.txt --resolver=backtracking pyproject.toml +# +# This file was modified to remove version pins. +certifi + # via btrdb (pyproject.toml) +grpcio + # via + # btrdb (pyproject.toml) + # grpcio-tools +grpcio-tools + # via btrdb (pyproject.toml) +protobuf + # via grpcio-tools pytz - -# Misc libraries + # via btrdb (pyproject.toml) pyyaml -certifi + # via btrdb (pyproject.toml) + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/setup.cfg b/setup.cfg index cded2bd..3f445b3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,38 +6,7 @@ license_file = LICENSE.txt test=pytest [tool:pytest] -addopts = --cov=btrdb --flake8 python_files = tests/* -flake8-ignore = - E111 - E114 - E121 - E122 - E123 - E124 - E128 - E201 - E202 - E221 - E222 - E225 - E226 - E231 - E241 - E251 - E266 - E271 - E302 - E303 - E402 - E501 - E502 - E701 - E731 - F401 - W292 - W293 - W391 - tests/* ALL - btrdb/grpcinterface/btrdb_pb2.py UnusedImport - btrdb/__init__.py UnusedImport + +[flake8] +extend-ignore = E111, E114, E121, E122, E123, E124, E128, E201, E202, E203, E221, E222, E225, E226, E231, E241, E251, E266, E271, E302, E303, E402, E501, E502,E701,E731,F401,W292,W293,W391 diff --git a/setup.py b/setup.py index 2997b26..a5b82d1 100644 --- a/setup.py +++ b/setup.py @@ -16,69 +16,70 @@ ## Imports ########################################################################## -import os import codecs +import os -from setuptools import setup -from setuptools import find_packages +from setuptools import find_packages, setup ########################################################################## ## Package Information ########################################################################## ## Basic information -NAME = "btrdb" -DESCRIPTION = "Bindings to interact with the Berkeley Tree Database using gRPC." -AUTHOR = "PingThingsIO" -EMAIL = "support@pingthings.io" -MAINTAINER = "PingThingsIO" -LICENSE = "BSD-3-Clause" -REPOSITORY = "https://github.com/PingThingsIO/btrdb-python" -PACKAGE = "btrdb" -URL = "http://btrdb.io/" -DOCS_URL = "https://btrdb.readthedocs.io/en/latest/" +NAME = "btrdb" +DESCRIPTION = "Bindings to interact with the Berkeley Tree Database using gRPC." +AUTHOR = "PingThingsIO" +EMAIL = "support@pingthings.io" +MAINTAINER = "PingThingsIO" +LICENSE = "BSD-3-Clause" +REPOSITORY = "https://github.com/PingThingsIO/btrdb-python" +PACKAGE = "btrdb" +URL = "http://btrdb.io/" +DOCS_URL = "https://btrdb.readthedocs.io/en/latest/" ## Define the keywords -KEYWORDS = ('btrdb', 'berkeley', 'timeseries', 'database', 'bindings' 'gRPC') +KEYWORDS = ("btrdb", "berkeley", "timeseries", "database", "bindings" "gRPC") ## Define the classifiers ## See https://pypi.python.org/pypi?%3Aaction=list_classifiers -CLASSIFIERS = ( - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'Intended Audience :: Information Technology', - 'Intended Audience :: Science/Research', - 'License :: OSI Approved :: BSD License', - 'Natural Language :: English', - 'Operating System :: OS Independent', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Topic :: Database', - 'Topic :: Software Development :: Libraries :: Python Modules', +CLASSIFIERS = ( + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: BSD License", + "Natural Language :: English", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Topic :: Database", + "Topic :: Software Development :: Libraries :: Python Modules", ) ## Important Paths -PROJECT = os.path.abspath(os.path.dirname(__file__)) +PROJECT = os.path.abspath(os.path.dirname(__file__)) REQUIRE_PATH = "requirements.txt" VERSION_PATH = os.path.join(PACKAGE, "version.py") PKG_DESCRIBE = "DESCRIPTION.md" ## Directories to ignore in find_packages -EXCLUDES = ( - "tests", "docs", +EXCLUDES = ( + "tests", + "docs", ) ########################################################################## ## Helper Functions ########################################################################## + def read(*parts): """ Assume UTF-8 encoding and return the contents of the file located at the absolute path from the REPOSITORY joined with *parts. """ - with codecs.open(os.path.join(PROJECT, *parts), 'rb', 'utf-8') as f: + with codecs.open(os.path.join(PROJECT, *parts), "rb", "utf-8") as f: return f.read() @@ -90,7 +91,7 @@ def get_version(path=VERSION_PATH): """ namespace = {} exec(read(path), namespace) - return namespace['get_version'](short=True) + return namespace["get_version"](short=True) def get_requires(path=REQUIRE_PATH): @@ -100,7 +101,7 @@ def get_requires(path=REQUIRE_PATH): """ for line in read(path).splitlines(): line = line.strip() - if line and not line.startswith('#'): + if line and not line.startswith("#"): yield line @@ -152,8 +153,8 @@ def get_description_type(path=PKG_DESCRIBE): }, "install_requires": list(get_requires()), "python_requires": ">=3.6, <4", - "setup_requires":["pytest-runner"], - "tests_require":["pytest"], + "setup_requires": ["pytest-runner"], + "tests_require": ["pytest"], } @@ -161,5 +162,5 @@ def get_description_type(path=PKG_DESCRIBE): ## Run setup script ########################################################################## -if __name__ == '__main__': +if __name__ == "__main__": setup(**config) diff --git a/tests/btrdb/test_arrow_streams.py b/tests/btrdb/test_arrow_streams.py new file mode 100644 index 0000000..301fa78 --- /dev/null +++ b/tests/btrdb/test_arrow_streams.py @@ -0,0 +1,47 @@ +import concurrent +import uuid +from unittest.mock import Mock, PropertyMock, patch + +import pytest + +import btrdb +from btrdb.point import RawPoint +from btrdb.stream import Stream + + +@pytest.fixture +def stream1(): + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + stream = Mock(Stream) + stream.version = Mock(return_value=11) + stream.uuid = Mock(return_value=uu) + stream.nearest = Mock(return_value=(RawPoint(time=10, value=1), 11)) + type(stream).collection = PropertyMock(return_value="fruits/apple") + type(stream).name = PropertyMock(return_value="gala") + stream.tags = Mock(return_value={"name": "gala", "unit": "volts"}) + stream.annotations = Mock(return_value=({"owner": "ABC", "color": "red"}, 11)) + stream._btrdb = Mock() + stream._btrdb._executor = concurrent.futures.ThreadPoolExecutor() + stream._btrdb._ARROW_ENABLED = Mock(return_value=True) + return stream + + +@pytest.fixture +def stream2(): + uu = uuid.UUID("17dbe387-89ea-42b6-864b-f505cdb483f5") + stream = Mock(Stream) + stream.version = Mock(return_value=22) + stream.uuid = Mock(return_value=uu) + stream.nearest = Mock(return_value=(RawPoint(time=20, value=1), 22)) + type(stream).collection = PropertyMock(return_value="fruits/orange") + type(stream).name = PropertyMock(return_value="blood") + stream.tags = Mock(return_value={"name": "blood", "unit": "amps"}) + stream.annotations = Mock(return_value=({"owner": "ABC", "color": "orange"}, 22)) + stream._btrdb = Mock() + stream._btrdb._executor = Mock() + stream._btrdb._ARROW_ENABLED = Mock(return_value=True) + return stream + + +class TestArrowStreams(object): + pass diff --git a/tests/btrdb/test_base.py b/tests/btrdb/test_base.py index c44acf8..ac55c07 100644 --- a/tests/btrdb/test_base.py +++ b/tests/btrdb/test_base.py @@ -16,27 +16,27 @@ ########################################################################## import os -import pytest from unittest.mock import patch -from btrdb import connect, __version__, BTRDB_ENDPOINTS, BTRDB_API_KEY -from btrdb.exceptions import ConnectionError +import pytest +from btrdb import BTRDB_API_KEY, BTRDB_ENDPOINTS, __version__, connect +from btrdb.exceptions import ConnectionError ########################################################################## ## Initialization Tests ########################################################################## -class TestConnect(object): - def setup(self): +class TestConnect(object): + def setup_method(self): for env in ["BTRDB_ENDPOINTS", "BTRDB_PROFILE", "BTRDB_API_KEY"]: try: del os.environ[env] except KeyError: pass - @patch('btrdb.credentials') + @patch("btrdb.credentials") def test_raises_err_with_no_inputs(self, mock_credentials): """ Assert ConnectionError raised if no connection arg, ENV, profile @@ -51,40 +51,38 @@ def test_raises_err_if_both_profile_and_credentials(self): Assert error is raised if both profile and credentials are sent """ with pytest.raises(ValueError): - connect("192.168.1.100:4410",None,"default") + connect("192.168.1.100:4410", None, "default") - @patch('btrdb.utils.credentials.credentials_by_profile') - @patch('btrdb.utils.credentials.credentials_by_env') - @patch('btrdb._connect') - def test_uses_args_over_env(self, mock_connect, mock_credentials_by_env, mock_credentials_by_profile): + @patch("btrdb.utils.credentials.credentials_by_profile") + @patch("btrdb.utils.credentials.credentials_by_env") + @patch("btrdb._connect") + def test_uses_args_over_env( + self, mock_connect, mock_credentials_by_env, mock_credentials_by_profile + ): """ Assert uses connect args over env """ mock_credentials_by_profile.return_value = {} - mock_credentials_by_env.return_value = { - "endpoints": "a", "apikey": "b" - } + mock_credentials_by_env.return_value = {"endpoints": "a", "apikey": "b"} connect("cat", "dog") mock_connect.assert_called_once_with(endpoints="cat", apikey="dog") - @patch('btrdb.utils.credentials.credentials_by_profile') - @patch('btrdb.utils.credentials.credentials_by_env') - @patch('btrdb._connect') - def test_uses_env_over_profile(self, mock_connect, mock_credentials_by_env, mock_credentials_by_profile): + @patch("btrdb.utils.credentials.credentials_by_profile") + @patch("btrdb.utils.credentials.credentials_by_env") + @patch("btrdb._connect") + def test_uses_env_over_profile( + self, mock_connect, mock_credentials_by_env, mock_credentials_by_profile + ): """ Assert connect uses env over profile info """ - mock_credentials_by_profile.return_value = { - "endpoints": "a", "apikey": "b" - } - mock_credentials_by_env.return_value = { - "endpoints": "c", "apikey": "d" - } + mock_credentials_by_profile.return_value = {"endpoints": "a", "apikey": "b"} + mock_credentials_by_env.return_value = {"endpoints": "c", "apikey": "d"} connect() mock_connect.assert_called_once_with(endpoints="c", apikey="d") - @patch('btrdb.utils.credentials.credentials_by_profile') - @patch('btrdb.Connection') + @patch("btrdb.utils.credentials.credentials_by_profile") + @patch("btrdb.Connection") def test_connect_with_env(self, mock_conn, mock_credentials_by_profile): """ Assert connect uses ENV variables diff --git a/tests/btrdb/test_conn.py b/tests/btrdb/test_conn.py index f863187..340416d 100644 --- a/tests/btrdb/test_conn.py +++ b/tests/btrdb/test_conn.py @@ -16,20 +16,21 @@ ########################################################################## import uuid as uuidlib +from unittest.mock import Mock, PropertyMock, call, patch + import pytest -from unittest.mock import Mock, PropertyMock, patch, call -from btrdb.conn import Connection, BTrDB +from btrdb.conn import BTrDB, Connection from btrdb.endpoint import Endpoint -from btrdb.grpcinterface import btrdb_pb2 from btrdb.exceptions import * +from btrdb.grpcinterface import btrdb_pb2 ########################################################################## ## Connection Tests ########################################################################## -class TestConnection(object): +class TestConnection(object): def test_raises_err_invalid_address(self): """ Assert ValueError is raised if address:port is invalidly formatted @@ -40,22 +41,12 @@ def test_raises_err_invalid_address(self): assert "expecting address:port" in str(exc) - def test_raises_err_for_apikey_insecure_port(self): - """ - Assert error is raised if apikey used on insecure port - """ - address = "127.0.0.1:4410" - with pytest.raises(ValueError) as exc: - conn = Connection(address, apikey="abcd") - assert "cannot use an API key with an insecure" in str(exc) - - ########################################################################## ## BTrDB Tests ########################################################################## -class TestBTrDB(object): +class TestBTrDB(object): ########################################################################## ## .streams tests ########################################################################## @@ -66,65 +57,60 @@ def test_streams_raises_err_if_version_not_list(self): """ db = BTrDB(None) with pytest.raises(TypeError) as exc: - db.streams('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a', versions="2,2") + db.streams("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a", versions="2,2") assert "versions argument must be of type list" in str(exc) - def test_streams_raises_err_if_version_argument_mismatch(self): """ Assert streams raises ValueError if len(identifiers) doesnt match length of versions """ db = BTrDB(None) with pytest.raises(ValueError) as exc: - db.streams('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a', versions=[2,2]) + db.streams("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a", versions=[2, 2]) assert "versions does not match identifiers" in str(exc) - def test_streams_stores_versions(self): """ Assert streams correctly stores supplied version info """ db = BTrDB(None) - uuid1 = uuidlib.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') - uuid2 = uuidlib.UUID('17dbe387-89ea-42b6-864b-f505cdb483f5') - versions = [22,44] + uuid1 = uuidlib.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + uuid2 = uuidlib.UUID("17dbe387-89ea-42b6-864b-f505cdb483f5") + versions = [22, 44] expected = dict(zip([uuid1, uuid2], versions)) streams = db.streams(uuid1, uuid2, versions=versions) assert streams._pinned_versions == expected - - @patch('btrdb.conn.BTrDB.stream_from_uuid') + @patch("btrdb.conn.BTrDB.stream_from_uuid") def test_streams_recognizes_uuid(self, mock_func): """ Assert streams recognizes uuid strings """ db = BTrDB(None) - uuid1 = uuidlib.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uuid1 = uuidlib.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") mock_func.return_value = [1] db.streams(uuid1) mock_func.assert_called_once() assert mock_func.call_args[0][0] == uuid1 - - @patch('btrdb.conn.BTrDB.stream_from_uuid') + @patch("btrdb.conn.BTrDB.stream_from_uuid") def test_streams_recognizes_uuid_string(self, mock_func): """ Assert streams recognizes uuid strings """ db = BTrDB(None) - uuid1 = '0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a' + uuid1 = "0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a" mock_func.return_value = [1] db.streams(uuid1) mock_func.assert_called_once() assert mock_func.call_args[0][0] == uuid1 - - @patch('btrdb.conn.BTrDB.streams_in_collection') + @patch("btrdb.conn.BTrDB.streams_in_collection") def test_streams_handles_path(self, mock_func): """ Assert streams calls streams_in_collection for collection/name paths @@ -132,17 +118,16 @@ def test_streams_handles_path(self, mock_func): db = BTrDB(None) ident = "zoo/animal/dog" mock_func.return_value = [1] - db.streams(ident, '0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + db.streams(ident, "0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") mock_func.assert_called_once() - assert mock_func.call_args[0][0] == 'zoo/animal' + assert mock_func.call_args[0][0] == "zoo/animal" assert mock_func.call_args[1] == { - 'is_collection_prefix': False, - 'tags': {'name': 'dog'} + "is_collection_prefix": False, + "tags": {"name": "dog"}, } - - @patch('btrdb.conn.BTrDB.streams_in_collection') + @patch("btrdb.conn.BTrDB.streams_in_collection") def test_streams_raises_err(self, mock_func): """ Assert streams raises StreamNotFoundError @@ -154,7 +139,7 @@ def test_streams_raises_err(self, mock_func): with pytest.raises(StreamNotFoundError) as exc: db.streams(ident) - mock_func.return_value = [1,2] + mock_func.return_value = [1, 2] with pytest.raises(StreamNotFoundError) as exc: db.streams(ident) @@ -162,7 +147,6 @@ def test_streams_raises_err(self, mock_func): mock_func.return_value = [1] db.streams(ident) - def test_streams_raises_valueerror(self): """ Assert streams raises ValueError if not uuid, uuid str, or path @@ -171,7 +155,6 @@ def test_streams_raises_valueerror(self): with pytest.raises(ValueError) as exc: db.streams(11) - ########################################################################## ## other tests ########################################################################## @@ -180,7 +163,7 @@ def test_info(self): """ Assert info method returns a dict """ - serialized = b'\x18\x05*\x055.0.02\x10\n\x0elocalhost:4410' + serialized = b"\x18\x05*\x055.0.02\x10\n\x0elocalhost:4410" info = btrdb_pb2.InfoResponse.FromString(serialized) endpoint = Mock(Endpoint) @@ -189,8 +172,11 @@ def test_info(self): truth = { "majorVersion": 5, + "minorVersion": 0, "build": "5.0.0", - "proxy": { "proxyEndpoints": ["localhost:4410"], }, + "proxy": { + "proxyEndpoints": ["localhost:4410"], + }, } info = conn.info() assert info == truth @@ -203,16 +189,15 @@ def test_list_collections(self): Assert list_collections method works """ endpoint = Mock(Endpoint) - endpoint.listCollections = Mock(side_effect=[iter([ - ['allen/automated'], - ['allen/bindings'] - ])]) + endpoint.listCollections = Mock( + side_effect=[iter([["allen/automated"], ["allen/bindings"]])] + ) conn = BTrDB(endpoint) - truth = ['allen/automated', 'allen/bindings'] + truth = ["allen/automated", "allen/bindings"] assert conn.list_collections() == truth - @patch('btrdb.conn.unpack_stream_descriptor') + @patch("btrdb.conn.unpack_stream_descriptor") def test_streams_in_collections_args(self, mock_util): """ Assert streams_in_collections correctly sends *collection, tags, annotations @@ -230,13 +215,17 @@ def test_streams_in_collections_args(self, mock_util): conn = BTrDB(endpoint) tags = {"unit": "volts"} annotations = {"size": "large"} - streams = conn.streams_in_collection("a", "b", is_collection_prefix=False, - tags=tags, annotations=annotations) + streams = conn.streams_in_collection( + "a", "b", is_collection_prefix=False, tags=tags, annotations=annotations + ) assert streams[0].name == "gala" assert streams[1].name == "fuji" - expected = [call('a', False, tags, annotations), call('b', False, tags, annotations)] + expected = [ + call("a", False, tags, annotations), + call("b", False, tags, annotations), + ] assert endpoint.lookupStreams.call_args_list == expected def test_streams_in_collections_no_arg(self): diff --git a/tests/btrdb/test_point.py b/tests/btrdb/test_point.py index 5085b00..b596523 100644 --- a/tests/btrdb/test_point.py +++ b/tests/btrdb/test_point.py @@ -17,23 +17,22 @@ import pytest -from btrdb.point import RawPoint, StatPoint from btrdb.grpcinterface import btrdb_pb2 - +from btrdb.point import RawPoint, StatPoint ########################################################################## ## ProtoBuff Classes ########################################################################## -RawPointProto = btrdb_pb2.RawPoint -StatPointProto = btrdb_pb2.StatPoint +RawPointProto = btrdb_pb2.RawPoint +StatPointProto = btrdb_pb2.StatPoint ########################################################################## ## RawPoint Tests ########################################################################## -class TestRawPoint(object): +class TestRawPoint(object): def test_create(self): """ Ensure we can create the object @@ -150,19 +149,19 @@ def test_repr_str(self): ## StatPoint Tests ########################################################################## -class TestStatPoint(object): +class TestStatPoint(object): def test_create(self): """ Ensure we can create the object """ - StatPoint(1,1,1,1,1,1) + StatPoint(1, 1, 1, 1, 1, 1) def test_from_proto(self): """ Assert from_proto creates a correct object """ - proto = StatPointProto(time=1,min=2,mean=3,max=4,count=5,stddev=6) + proto = StatPointProto(time=1, min=2, mean=3, max=4, count=5, stddev=6) point = StatPoint.from_proto(proto) assert point.time == proto.time assert point.min == proto.min @@ -175,7 +174,7 @@ def test_from_proto_creates_rawpoint(self): """ Assert from_proto creates a new instance of StatPoint """ - proto = StatPointProto(time=1,min=2,mean=3,max=4,count=5,stddev=6) + proto = StatPointProto(time=1, min=2, mean=3, max=4, count=5, stddev=6) point = StatPoint.from_proto(proto) assert isinstance(point, StatPoint) @@ -183,8 +182,10 @@ def test_from_proto_list(self): """ Assert from_proto_list creates valid objects """ - protos = [StatPointProto(time=1,min=2,mean=3,max=4,count=5,stddev=6), - StatPointProto(time=11,min=12,mean=13,max=14,count=15,stddev=16)] + protos = [ + StatPointProto(time=1, min=2, mean=3, max=4, count=5, stddev=6), + StatPointProto(time=11, min=12, mean=13, max=14, count=15, stddev=16), + ] points = StatPoint.from_proto_list(protos) assert points[0].time == protos[0].time assert points[0].min == protos[0].min @@ -203,8 +204,10 @@ def test_from_proto_list_returns_list_of_statpoint(self): """ Assert from_proto_list returns list of StatPoint """ - protos = [StatPointProto(time=1,min=2,mean=3,max=4,count=5,stddev=6), - StatPointProto(time=11,min=12,mean=13,max=14,count=15,stddev=16)] + protos = [ + StatPointProto(time=1, min=2, mean=3, max=4, count=5, stddev=6), + StatPointProto(time=11, min=12, mean=13, max=14, count=15, stddev=16), + ] points = StatPoint.from_proto_list(protos) assert len(points) == 2 diff --git a/tests/btrdb/test_stream.py b/tests/btrdb/test_stream.py index 3ba9461..39b82e7 100644 --- a/tests/btrdb/test_stream.py +++ b/tests/btrdb/test_stream.py @@ -10,46 +10,49 @@ """ Testing package for the btrdb stream module """ - -########################################################################## -## Imports -########################################################################## - +import concurrent.futures +import datetime +import json import re import sys -import json import uuid -import pytz +from unittest.mock import Mock, PropertyMock, call, patch + import pytest -import datetime -from unittest.mock import Mock, PropertyMock, patch, call +import pytz +from btrdb import MAXIMUM_TIME, MINIMUM_TIME from btrdb.conn import BTrDB from btrdb.endpoint import Endpoint -from btrdb import MINIMUM_TIME, MAXIMUM_TIME -from btrdb.stream import Stream, StreamSet, StreamFilter, INSERT_BATCH_SIZE -from btrdb.point import RawPoint, StatPoint from btrdb.exceptions import ( BTrDBError, + InvalidCollection, InvalidOperation, + NoSuchPoint, StreamNotFoundError, - InvalidCollection, - NoSuchPoint ) from btrdb.grpcinterface import btrdb_pb2 +from btrdb.point import RawPoint, StatPoint +from btrdb.stream import INSERT_BATCH_SIZE, Stream, StreamFilter, StreamSet + +########################################################################## +## Imports +########################################################################## + -RawPointProto = btrdb_pb2.RawPoint -StatPointProto = btrdb_pb2.StatPoint -EST = pytz.timezone('America/New_York') +RawPointProto = btrdb_pb2.RawPoint +StatPointProto = btrdb_pb2.StatPoint +EST = pytz.timezone("America/New_York") ########################################################################## ## Fixtures ########################################################################## + @pytest.fixture def stream1(): - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") stream = Mock(Stream) stream.version = Mock(return_value=11) stream.uuid = Mock(return_value=uu) @@ -58,12 +61,15 @@ def stream1(): type(stream).name = PropertyMock(return_value="gala") stream.tags = Mock(return_value={"name": "gala", "unit": "volts"}) stream.annotations = Mock(return_value=({"owner": "ABC", "color": "red"}, 11)) + stream._btrdb = Mock() + stream._btrdb._executor = concurrent.futures.ThreadPoolExecutor() + stream._btrdb._ARROW_ENABLED = Mock(return_value=False) return stream @pytest.fixture def stream2(): - uu = uuid.UUID('17dbe387-89ea-42b6-864b-f505cdb483f5') + uu = uuid.UUID("17dbe387-89ea-42b6-864b-f505cdb483f5") stream = Mock(Stream) stream.version = Mock(return_value=22) stream.uuid = Mock(return_value=uu) @@ -72,6 +78,26 @@ def stream2(): type(stream).name = PropertyMock(return_value="blood") stream.tags = Mock(return_value={"name": "blood", "unit": "amps"}) stream.annotations = Mock(return_value=({"owner": "ABC", "color": "orange"}, 22)) + stream._btrdb = Mock() + stream._btrdb._executor = Mock() + stream._btrdb._ARROW_ENABLED = Mock(return_value=False) + return stream + + +@pytest.fixture +def arrow_stream3(): + uu = uuid.UUID("17dbe387-89ea-42b6-864b-f505cdb483f5") + stream = Mock(Stream) + stream.version = Mock(return_value=22) + stream.uuid = Mock(return_value=uu) + stream.nearest = Mock(return_value=(RawPoint(time=20, value=1), 22)) + type(stream).collection = PropertyMock(return_value="fruits/orange") + type(stream).name = PropertyMock(return_value="blood") + stream.tags = Mock(return_value={"name": "blood", "unit": "amps"}) + stream.annotations = Mock(return_value=({"owner": "ABC", "color": "orange"}, 22)) + stream._btrdb = Mock() + stream._btrdb._executor = Mock() + stream._btrdb._ARROW_ENABLED = Mock(return_value=True) return stream @@ -79,15 +105,14 @@ def stream2(): ## Stream Tests ########################################################################## -class TestStream(object): +class TestStream(object): def test_create(self): """ Assert we can create the object """ Stream(None, "FAKE") - def test_repr_str(self): """ Assert the repr and str output are correct @@ -98,64 +123,61 @@ def test_repr_str(self): stream._collection = COLLECTION stream._tags = {"name": NAME} - expected = "".format( - COLLECTION, NAME - ) + expected = "".format(COLLECTION, NAME) assert stream.__repr__() == expected assert stream.__str__() == expected - def test_refresh_metadata(self): """ Assert refresh_metadata calls Endpoint.streamInfo """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) endpoint.streamInfo = Mock(return_value=("koala", 42, {}, {}, None)) + endpoint.info = Mock(return_value={"majorVersion": 5, "minorVersion": 30}) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) stream.refresh_metadata() stream._btrdb.ep.streamInfo.assert_called_once_with(uu, False, True) - def test_refresh_metadata_deserializes_annotations(self): """ Assert refresh_metadata deserializes annotation values """ uu = uuid.uuid4() serialized = { - 'acronym': 'VPHM', - 'description': 'El Segundo PMU 42 Ean', - 'devacronym': 'PMU!EL_SEG_PMU_42', - 'enabled': 'true', - 'id': '76932ae4-09bc-472c-8dc6-64fea68d2797', - 'phase': 'A', - 'label': 'null', - 'frequency': '30', - 'control': '2019-11-07 13:21:23.000000-0500', + "acronym": "VPHM", + "description": "El Segundo PMU 42 Ean", + "devacronym": "PMU!EL_SEG_PMU_42", + "enabled": "true", + "id": "76932ae4-09bc-472c-8dc6-64fea68d2797", + "phase": "A", + "label": "null", + "frequency": "30", + "control": "2019-11-07 13:21:23.000000-0500", "calibrate": '{"racf": 1.8, "pacf": 0.005}', } expected = { - 'acronym': 'VPHM', - 'description': 'El Segundo PMU 42 Ean', - 'devacronym': 'PMU!EL_SEG_PMU_42', - 'enabled': True, - 'id': '76932ae4-09bc-472c-8dc6-64fea68d2797', - 'phase': 'A', - 'label': None, - 'frequency': 30, - 'control': '2019-11-07 13:21:23.000000-0500', + "acronym": "VPHM", + "description": "El Segundo PMU 42 Ean", + "devacronym": "PMU!EL_SEG_PMU_42", + "enabled": True, + "id": "76932ae4-09bc-472c-8dc6-64fea68d2797", + "phase": "A", + "label": None, + "frequency": 30, + "control": "2019-11-07 13:21:23.000000-0500", "calibrate": {"racf": 1.8, "pacf": 0.005}, } endpoint = Mock(Endpoint) endpoint.streamInfo = Mock(return_value=("koala", 42, {}, serialized, None)) + endpoint.info = Mock(return_value={"majorVersion": 5, "minorVersion": 30}) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) stream.refresh_metadata() assert stream.annotations()[0] == expected - def test_stream_name_property(self): """ Assert name property comes from tags @@ -165,7 +187,6 @@ def test_stream_name_property(self): stream._tags = {"name": name} assert stream.name == name - def test_stream_unit_property(self): """ Assert unit property comes from tags @@ -175,7 +196,6 @@ def test_stream_unit_property(self): stream._tags = {"unit": unit} assert stream.unit == unit - ########################################################################## ## update tests ########################################################################## @@ -184,8 +204,9 @@ def test_update_arguments(self): """ Assert update raises errors on invalid arguments """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) + endpoint.info = Mock(return_value={"majorVersion": 5, "minorVersion": 30}) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) # no arguments @@ -208,46 +229,47 @@ def test_update_arguments(self): stream.update(collection=42) assert "collection must be of type string" in str(exc) - def test_update_tags(self): """ Assert update calls correct Endpoint methods for tags update """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) + endpoint.info = Mock(return_value={"majorVersion": 5, "minorVersion": 30}) endpoint.streamInfo = Mock(return_value=("koala", 42, {}, {}, None)) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) tags = {"cat": "dog"} stream.update(tags=tags) - stream._btrdb.ep.setStreamTags.assert_called_once_with(uu=uu, expected=42, - tags=tags, collection="koala") + stream._btrdb.ep.setStreamTags.assert_called_once_with( + uu=uu, expected=42, tags=tags, collection="koala" + ) stream._btrdb.ep.setStreamAnnotations.assert_not_called() - def test_update_collection(self): """ Assert update calls correct Endpoint methods for collection update """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) endpoint.streamInfo = Mock(return_value=("koala", 42, {}, {}, None)) + endpoint.info = Mock(return_value={"majorVersion": 5, "minorVersion": 30}) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) collection = "giraffe" stream.update(collection=collection) - stream._btrdb.ep.setStreamTags.assert_called_once_with(uu=uu, expected=42, - tags=stream.tags(), collection=collection) + stream._btrdb.ep.setStreamTags.assert_called_once_with( + uu=uu, expected=42, tags=stream.tags(), collection=collection + ) stream._btrdb.ep.setStreamAnnotations.assert_not_called() - def test_update_annotations(self): """ Assert update calls correct Endpoint methods for annotations update """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) endpoint.streamInfo = Mock(return_value=("koala", 42, {}, {}, None)) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) @@ -258,7 +280,7 @@ def test_update_annotations(self): "description": "El Segundo PMU 42 Ean", "devacronym": "PMU!EL_SEG_PMU_42", "enabled": True, - "id": uuid.UUID('76932ae4-09bc-472c-8dc6-64fea68d2797'), + "id": uuid.UUID("76932ae4-09bc-472c-8dc6-64fea68d2797"), "phase": "A", "label": None, "frequency": 30, @@ -272,22 +294,21 @@ def test_update_annotations(self): uu=uu, expected=42, changes={ - 'acronym': 'VPHM', - 'description': 'El Segundo PMU 42 Ean', - 'devacronym': 'PMU!EL_SEG_PMU_42', - 'enabled': 'true', - 'id': '76932ae4-09bc-472c-8dc6-64fea68d2797', - 'phase': 'A', - 'label': 'null', - 'frequency': '30', - 'control': '2019-11-07 13:21:23.000000-0500', + "acronym": "VPHM", + "description": "El Segundo PMU 42 Ean", + "devacronym": "PMU!EL_SEG_PMU_42", + "enabled": "true", + "id": "76932ae4-09bc-472c-8dc6-64fea68d2797", + "phase": "A", + "label": "null", + "frequency": "30", + "control": "2019-11-07 13:21:23.000000-0500", "calibrate": '{"racf": 1.8, "pacf": 0.005}', }, removals=[], ) stream._btrdb.ep.setStreamTags.assert_not_called() - def test_update_annotations_nested_conversions(self): """ Assert update correctly encodes nested annotation data @@ -304,21 +325,23 @@ def test_update_annotations_nested_conversions(self): "num": 12, "float": 1.3, "string": "the quick brown fox is 12", - } - } + }, + }, } - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) - endpoint.streamInfo = Mock(return_value=("koala", 42, {}, {"foo": "42 Cherry Hill"}, None)) + endpoint.streamInfo = Mock( + return_value=("koala", 42, {}, {"foo": "42 Cherry Hill"}, None) + ) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) stream.refresh_metadata() stream.update(annotations=annotations) # spot check a nested value for Python 3.4 and 3.5 compatability - changes = stream._btrdb.ep.setStreamAnnotations.call_args[1]['changes'] - assert changes['nested'].__class__ == str - assert json.loads(changes['nested']) == annotations['nested'] + changes = stream._btrdb.ep.setStreamAnnotations.call_args[1]["changes"] + assert changes["nested"].__class__ == str + assert json.loads(changes["nested"]) == annotations["nested"] # check all args if Python > 3.5 if sys.version_info[0] > 3.5: @@ -326,18 +349,18 @@ def test_update_annotations_nested_conversions(self): uu=uu, expected=42, changes={ - 'num': '10', - 'float': '1.3', - 'string': '"the quick brown fox is 10"', - 'nested': '{"num": 11, "float": 1.3, "string": "the quick brown fox is 11", "nested": {"num": 12, "float": 1.3, "string": "the quick brown fox is 12"}}' - } + "num": "10", + "float": "1.3", + "string": '"the quick brown fox is 10"', + "nested": '{"num": 11, "float": 1.3, "string": "the quick brown fox is 11", "nested": {"num": 12, "float": 1.3, "string": "the quick brown fox is 12"}}', + }, ) def test_update_annotations_no_encoder(self): """ Assert update annotations works with None as encoder argument """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) endpoint.streamInfo = Mock(return_value=("koala", 42, {}, {}, None)) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) @@ -361,12 +384,17 @@ def test_update_annotations_replace(self): Assert that replace argument will add proper keys to removals array in endpoint call. """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) - endpoint.streamInfo = Mock(return_value=("koala", 42, {}, {"phase": "A", "source": "PJM"}, None)) + endpoint.streamInfo = Mock( + return_value=("koala", 42, {}, {"phase": "A", "source": "PJM"}, None) + ) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) - annotations = {"foo": "this is a string", "phase": "A", } + annotations = { + "foo": "this is a string", + "phase": "A", + } stream.refresh_metadata() @@ -396,45 +424,42 @@ def test_exists_cached_value(self): """ Assert exists first uses cached value """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") stream = Stream(btrdb=BTrDB(Mock(Endpoint)), uuid=uu, known_to_exist=True) stream.refresh_metadata = Mock() assert stream.exists() stream.refresh_metadata.assert_not_called() - def test_exists(self): """ Assert exists refreshes data if value is unknown """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") stream = Stream(btrdb=BTrDB(Mock(Endpoint)), uuid=uu) stream.refresh_metadata = Mock(return_value=True) assert stream.exists() assert stream.refresh_metadata.call_count == 1 - def test_exists_returns_false_on_404(self): """ Assert exists returns False on 404 error """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") stream = Stream(btrdb=BTrDB(Mock(Endpoint)), uuid=uu) - stream.refresh_metadata = Mock(side_effect=StreamNotFoundError( - "stream not found with provided uuid" - )) + stream.refresh_metadata = Mock( + side_effect=StreamNotFoundError("stream not found with provided uuid") + ) assert stream.exists() == False assert stream.refresh_metadata.call_count == 1 - def test_exists_passes_other_errors(self): """ Assert exists does not keep non 404 errors trapped """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") stream = Stream(btrdb=BTrDB(Mock(Endpoint)), uuid=uu) stream.refresh_metadata = Mock(side_effect=ValueError()) @@ -442,7 +467,6 @@ def test_exists_passes_other_errors(self): stream.exists() assert stream.refresh_metadata.call_count == 1 - ########################################################################## ## tag/annotation tests ########################################################################## @@ -451,34 +475,33 @@ def test_tags_returns_copy(self): """ Assert tags returns a copy of the tags dict """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") token = {"cat": "dog"} stream = Stream(btrdb=BTrDB(Mock(Endpoint)), uuid=uu, tags=token) assert stream.tags() is not token assert stream.tags() == token - def test_tags_returns_cached_values(self): """ Assert tags returns a copy of the tags dict """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") token = {"cat": "dog"} - stream = Stream(btrdb=BTrDB(Mock(Endpoint)), uuid=uu, tags=token, - property_version=42) + stream = Stream( + btrdb=BTrDB(Mock(Endpoint)), uuid=uu, tags=token, property_version=42 + ) stream.refresh_metadata = Mock() assert stream.tags(refresh=False) == token stream.refresh_metadata.assert_not_called() - def test_tags_forces_refresh_if_requested(self): """ Assert tags calls refresh_metadata if requested even though a cached copy is available """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") token = {"cat": "dog"} stream = Stream(btrdb=BTrDB(Mock(Endpoint)), uuid=uu, tags=token) stream.refresh_metadata = Mock() @@ -486,42 +509,41 @@ def test_tags_forces_refresh_if_requested(self): stream.tags(refresh=True) assert stream.refresh_metadata.call_count == 1 - def test_annotations_returns_copy_of_value(self): """ Assert annotations returns a copy of the annotations dict """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") token = {"cat": "dog"} - stream = Stream(btrdb=BTrDB(Mock(Endpoint)), uuid=uu, annotations=token, - property_version=42) + stream = Stream( + btrdb=BTrDB(Mock(Endpoint)), uuid=uu, annotations=token, property_version=42 + ) stream.refresh_metadata = Mock() assert stream.annotations(refresh=False)[0] == token assert stream.annotations(refresh=False)[0] is not token assert stream.annotations(refresh=False)[1] == 42 - def test_annotations_returns_cached_values(self): """ Assert annotations returns a copy of the annotations dict """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") token = {"cat": "dog"} - stream = Stream(btrdb=BTrDB(Mock(Endpoint)), uuid=uu, annotations=token, - property_version=42) + stream = Stream( + btrdb=BTrDB(Mock(Endpoint)), uuid=uu, annotations=token, property_version=42 + ) stream.refresh_metadata = Mock() assert stream.annotations(refresh=False)[0] == token stream.refresh_metadata.assert_not_called() - def test_annotations_forces_refresh_if_requested(self): """ Assert annotations calls refresh_metadata if requested even though a cached copy is available """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") token = {"cat": "dog"} stream = Stream(btrdb=BTrDB(Mock(Endpoint)), uuid=uu, annotations=token) stream.refresh_metadata = Mock() @@ -529,7 +551,6 @@ def test_annotations_forces_refresh_if_requested(self): stream.annotations(refresh=True) assert stream.refresh_metadata.call_count == 1 - ########################################################################## ## windowing tests ########################################################################## @@ -538,15 +559,29 @@ def test_windows(self): """ Assert windows returns tuples of data from Endpoint.windows """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) windows = [ - [(StatPointProto(time=1,min=2,mean=3,max=4,count=5,stddev=6), StatPointProto(time=2,min=3,mean=4,max=5,count=6,stddev=7)), 42], - [(StatPointProto(time=3,min=4,mean=5,max=6,count=7,stddev=8), StatPointProto(time=4,min=5,mean=6,max=7,count=8,stddev=9)), 42], + [ + ( + StatPointProto(time=1, min=2, mean=3, max=4, count=5, stddev=6), + StatPointProto(time=2, min=3, mean=4, max=5, count=6, stddev=7), + ), + 42, + ], + [ + ( + StatPointProto(time=3, min=4, mean=5, max=6, count=7, stddev=8), + StatPointProto(time=4, min=5, mean=6, max=7, count=8, stddev=9), + ), + 42, + ], ] expected = ( - (StatPoint(time=1,minv=2.0,meanv=3.0,maxv=4.0,count=5,stddev=6.0), 42), (StatPoint(time=2,minv=3.0,meanv=4.0,maxv=5.0,count=6,stddev=7.0), 42), - (StatPoint(time=3,minv=4.0,meanv=5.0,maxv=6.0,count=7,stddev=8.0), 42), (StatPoint(time=4,minv=5.0,meanv=6.0,maxv=7.0,count=8,stddev=9.0), 42), + (StatPoint(time=1, minv=2.0, meanv=3.0, maxv=4.0, count=5, stddev=6.0), 42), + (StatPoint(time=2, minv=3.0, meanv=4.0, maxv=5.0, count=6, stddev=7.0), 42), + (StatPoint(time=3, minv=4.0, meanv=5.0, maxv=6.0, count=7, stddev=8.0), 42), + (StatPoint(time=4, minv=5.0, meanv=6.0, maxv=7.0, count=8, stddev=9.0), 42), ) endpoint.windows = Mock(return_value=windows) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) @@ -556,24 +591,35 @@ def test_windows(self): assert result == expected assert isinstance(result, tuple) assert isinstance(result[0], tuple) - stream._btrdb.ep.windows.assert_called_once_with( - uu, 100, 500, 2, 0, 0 - ) - + stream._btrdb.ep.windows.assert_called_once_with(uu, 100, 500, 2, 0, 0) def test_aligned_windows(self): """ Assert windows returns tuples of data from Endpoint.alignedWindows """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) windows = [ - [(StatPointProto(time=1,min=2,mean=3,max=4,count=5,stddev=6), StatPointProto(time=2,min=3,mean=4,max=5,count=6,stddev=7)), 42], - [(StatPointProto(time=3,min=4,mean=5,max=6,count=7,stddev=8), StatPointProto(time=4,min=5,mean=6,max=7,count=8,stddev=9)), 42], + [ + ( + StatPointProto(time=1, min=2, mean=3, max=4, count=5, stddev=6), + StatPointProto(time=2, min=3, mean=4, max=5, count=6, stddev=7), + ), + 42, + ], + [ + ( + StatPointProto(time=3, min=4, mean=5, max=6, count=7, stddev=8), + StatPointProto(time=4, min=5, mean=6, max=7, count=8, stddev=9), + ), + 42, + ], ] expected = ( - (StatPoint(time=1,minv=2.0,meanv=3.0,maxv=4.0,count=5,stddev=6.0), 42), (StatPoint(time=2,minv=3.0,meanv=4.0,maxv=5.0,count=6,stddev=7.0), 42), - (StatPoint(time=3,minv=4.0,meanv=5.0,maxv=6.0,count=7,stddev=8.0), 42), (StatPoint(time=4,minv=5.0,meanv=6.0,maxv=7.0,count=8,stddev=9.0), 42), + (StatPoint(time=1, minv=2.0, meanv=3.0, maxv=4.0, count=5, stddev=6.0), 42), + (StatPoint(time=2, minv=3.0, meanv=4.0, maxv=5.0, count=6, stddev=7.0), 42), + (StatPoint(time=3, minv=4.0, meanv=5.0, maxv=6.0, count=7, stddev=8.0), 42), + (StatPoint(time=4, minv=5.0, meanv=6.0, maxv=7.0, count=8, stddev=9.0), 42), ) endpoint.alignedWindows = Mock(return_value=windows) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) @@ -583,20 +629,29 @@ def test_aligned_windows(self): assert result == expected assert isinstance(result, tuple) assert isinstance(result[0], tuple) - stream._btrdb.ep.alignedWindows.assert_called_once_with( - uu, 100, 500, 1, 0 - ) - + stream._btrdb.ep.alignedWindows.assert_called_once_with(uu, 100, 500, 1, 0) def test_count(self): """ Test that stream count method uses aligned windows """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) windows = [ - [(StatPointProto(time=1,min=2,mean=3,max=4,count=5,stddev=6), StatPointProto(time=2,min=3,mean=4,max=5,count=6,stddev=7)), 42], - [(StatPointProto(time=3,min=4,mean=5,max=6,count=7,stddev=8), StatPointProto(time=4,min=5,mean=6,max=7,count=8,stddev=9)), 42], + [ + ( + StatPointProto(time=1, min=2, mean=3, max=4, count=5, stddev=6), + StatPointProto(time=2, min=3, mean=4, max=5, count=6, stddev=7), + ), + 42, + ], + [ + ( + StatPointProto(time=3, min=4, mean=5, max=6, count=7, stddev=8), + StatPointProto(time=4, min=5, mean=6, max=7, count=8, stddev=9), + ), + 42, + ], ] endpoint.alignedWindows = Mock(return_value=windows) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) @@ -609,7 +664,6 @@ def test_count(self): stream.count(10, 1000, 8, version=1200) stream._btrdb.ep.alignedWindows.assert_called_with(uu, 10, 1000, 8, 1200) - ########################################################################## ## earliest/latest tests ########################################################################## @@ -618,7 +672,7 @@ def test_earliest(self): """ Assert earliest calls Endpoint.nearest """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) endpoint.nearest = Mock(return_value=(RawPointProto(time=100, value=1.0), 42)) @@ -627,12 +681,11 @@ def test_earliest(self): assert (point, ver) == (RawPoint(100, 1.0), 42) endpoint.nearest.assert_called_once_with(uu, MINIMUM_TIME, 0, False) - def test_earliest_swallows_exception(self): """ Assert earliest returns None when endpoint throws exception """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) endpoint.nearest = Mock(side_effect=NoSuchPoint("next point does not exist")) @@ -640,12 +693,11 @@ def test_earliest_swallows_exception(self): assert stream.earliest() is None endpoint.nearest.assert_called_once_with(uu, MINIMUM_TIME, 0, False) - def test_earliest_passes_exception(self): """ Assert earliest reraises non 401 exception """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) endpoint.nearest = Mock(side_effect=BTrDBError("empty")) @@ -655,12 +707,11 @@ def test_earliest_passes_exception(self): assert exc.value.args[0] == "empty" endpoint.nearest.assert_called_once_with(uu, MINIMUM_TIME, 0, False) - def test_latest(self): """ Assert latest calls Endpoint.nearest """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) endpoint.nearest = Mock(return_value=(RawPointProto(time=100, value=1.0), 42)) @@ -669,12 +720,11 @@ def test_latest(self): assert (point, ver) == (RawPoint(100, 1.0), 42) endpoint.nearest.assert_called_once_with(uu, MAXIMUM_TIME, 0, True) - def test_latest_swallows_exception(self): """ Assert latest returns None when endpoint throws exception """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) endpoint.nearest = Mock(side_effect=NoSuchPoint("empty")) @@ -682,12 +732,11 @@ def test_latest_swallows_exception(self): assert stream.latest() is None endpoint.nearest.assert_called_once_with(uu, MAXIMUM_TIME, 0, True) - def test_latest_passes_exception(self): """ Assert latest reraises non 401 exception """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) endpoint.nearest = Mock(side_effect=BTrDBError("empty")) @@ -697,13 +746,12 @@ def test_latest_passes_exception(self): assert exc.value.args[0] == "empty" endpoint.nearest.assert_called_once_with(uu, MAXIMUM_TIME, 0, True) - @patch("btrdb.stream.currently_as_ns") def test_currently(self, mocked): """ Assert currently calls Endpoint.nearest """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) endpoint.nearest = Mock(return_value=(RawPointProto(time=100, value=1.0), 42)) @@ -714,13 +762,12 @@ def test_currently(self, mocked): assert (point, ver) == (RawPoint(100, 1.0), 42) endpoint.nearest.assert_called_once_with(uu, ns_fake_time, 0, True) - @patch("btrdb.stream.currently_as_ns") def test_currently_swallows_exception(self, mocked): """ Assert currently returns None when endpoint throws exception """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) endpoint.nearest = Mock(side_effect=NoSuchPoint("empty")) @@ -730,13 +777,12 @@ def test_currently_swallows_exception(self, mocked): assert stream.current() is None endpoint.nearest.assert_called_once_with(uu, ns_fake_time, 0, True) - @patch("btrdb.stream.currently_as_ns") def test_currently_passes_exception(self, mocked): """ Assert currently reraises non 401 exception """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) endpoint.nearest = Mock(side_effect=BTrDBError("empty")) @@ -748,7 +794,6 @@ def test_currently_passes_exception(self, mocked): assert exc.value.args[0] == "empty" endpoint.nearest.assert_called_once_with(uu, ns_fake_time, 0, True) - ########################################################################## ## misc tests ########################################################################## @@ -757,7 +802,7 @@ def test_version(self): """ Assert version calls and returns correct value from streamInfo """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) endpoint.streamInfo = Mock(return_value=("", 0, {}, {}, 42)) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) @@ -765,30 +810,36 @@ def test_version(self): assert stream.version() == 42 stream._btrdb.ep.streamInfo.assert_called_once_with(uu, True, False) - def test_insert(self): """ Assert insert batches data to endpoint insert and returns version """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) - endpoint.insert = Mock(side_effect=[1,2,3]) + endpoint.insert = Mock(side_effect=[1, 2, 3]) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) - data = list(zip(range(10000,120000), map(float, range(110000)))) + data = list(zip(range(10000, 120000), map(float, range(110000)))) version = stream.insert(data) - assert stream._btrdb.ep.insert.call_args_list[0][0][1] == data[:INSERT_BATCH_SIZE] - assert stream._btrdb.ep.insert.call_args_list[1][0][1] == data[INSERT_BATCH_SIZE:2*INSERT_BATCH_SIZE] - assert stream._btrdb.ep.insert.call_args_list[2][0][1] == data[2*INSERT_BATCH_SIZE:] + assert ( + stream._btrdb.ep.insert.call_args_list[0][0][1] == data[:INSERT_BATCH_SIZE] + ) + assert ( + stream._btrdb.ep.insert.call_args_list[1][0][1] + == data[INSERT_BATCH_SIZE : 2 * INSERT_BATCH_SIZE] + ) + assert ( + stream._btrdb.ep.insert.call_args_list[2][0][1] + == data[2 * INSERT_BATCH_SIZE :] + ) assert version == 3 - def test_nearest(self): """ Assert nearest calls Endpoint.nearest with correct arguments """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) endpoint.nearest = Mock(return_value=(RawPointProto(time=100, value=1.0), 42)) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) @@ -798,12 +849,11 @@ def test_nearest(self): assert point == RawPoint(100, 1.0) assert version == 42 - def test_nearest_swallows_exception(self): """ Assert nearest returns None when endpoint throws 401 exception """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) endpoint.nearest = Mock(side_effect=NoSuchPoint("next point does not exist")) @@ -811,12 +861,11 @@ def test_nearest_swallows_exception(self): assert stream.nearest(0, 0, False) is None endpoint.nearest.assert_called_once_with(uu, 0, 0, False) - def test_nearest_passes_exception(self): """ Assert nearest reraises non 401 exception """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) endpoint.nearest = Mock(side_effect=BTrDBError("foo")) @@ -826,45 +875,41 @@ def test_nearest_passes_exception(self): assert exc.value.args[0] == "foo" endpoint.nearest.assert_called_once_with(uu, 0, 0, False) - def test_delete_range(self): """ Assert delete_range calls Endpoint.deleteRange with correct arguments """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") stream = Stream(btrdb=BTrDB(Mock(Endpoint)), uuid=uu) stream.delete(5, 10) stream._btrdb.ep.deleteRange.assert_called_once_with(uu, 5, 10) - def test_flush(self): """ Assert flush calls Endpoint.flush with UUID """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") stream = Stream(btrdb=BTrDB(Mock(Endpoint)), uuid=uu) stream.flush() stream._btrdb.ep.flush.assert_called_once_with(uu) - def test_obliterate(self): """ Assert obliterate calls Endpoint.obliterate with UUID """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") stream = Stream(btrdb=BTrDB(Mock(Endpoint)), uuid=uu) stream.obliterate() stream._btrdb.ep.obliterate.assert_called_once_with(uu) - def test_obliterate_allows_error(self): """ Assert obliterate raises error if stream not found. (does not swallow error) """ - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") endpoint = Mock(Endpoint) endpoint.obliterate = Mock(side_effect=StreamNotFoundError()) stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) @@ -874,20 +919,26 @@ def test_obliterate_allows_error(self): stream._btrdb.ep.obliterate.assert_called_once_with(uu) - - ########################################################################## ## StreamSet Tests ########################################################################## -class TestStreamSet(object): +class TestStreamSet(object): def test_create(self): """ Assert we can create the object """ - StreamSet([]) + StreamSet([1]) + @pytest.mark.parametrize( + "empty_container", [(tuple()), (dict()), (list()), (set())] + ) + def test_create_error_with_no_objects(self, empty_container): + with pytest.raises( + ValueError, match="Trying to create streamset with an empty list of streams" + ): + StreamSet(empty_container) ########################################################################## ## builtin / magic methods tests @@ -897,7 +948,7 @@ def test_repr(self): """ Assert StreamSet instance repr output """ - data = [11,22,"dog","cat"] + data = [11, 22, "dog", "cat"] streams = StreamSet(data) expected = "" assert streams.__repr__() == expected @@ -907,12 +958,11 @@ def test_repr(self): expected = "" assert streams.__repr__() == expected - def test_str(self): """ Assert StreamSet instance str output """ - data = [11,22,"dog","cat"] + data = [11, 22, "dog", "cat"] streams = StreamSet(data) expected = "StreamSet with 4 streams" assert str(streams) == expected @@ -922,41 +972,37 @@ def test_str(self): expected = "StreamSet with 1 stream" assert str(streams) == expected - def test_subscriptable(self): """ Assert StreamSet instance is subscriptable """ - data = [11,22,"dog","cat"] + data = [11, 22, "dog", "cat"] streams = StreamSet(data) for index, val in enumerate(data): assert streams[index] == val - def test_len(self): """ Assert StreamSet instance support len """ - data = [11,22,"dog","cat"] + data = [11, 22, "dog", "cat"] streams = StreamSet(data) assert len(streams) == len(data) - def test_iter(self): """ Assert StreamSet instance support iteration """ - data = [11,22,"dog","cat"] + data = [11, 22, "dog", "cat"] streams = StreamSet(data) for index, stream in enumerate(streams): assert data[index] == stream - def test_indexing(self): """ Assert StreamSet instance supports indexing """ - data = [11,22,"dog","cat"] + data = [11, 22, "dog", "cat"] streams = StreamSet(data) # verify index lookup @@ -967,7 +1013,6 @@ def test_indexing(self): # verify slicing works assert streams[:2] == data[:2] - def test_mapping(self): """ Assert StreamSet instance support key mapping @@ -990,34 +1035,30 @@ def test_mapping(self): streams[missing] assert str(missing) in str(e) - def test_contains(self): """ Assert StreamSet instance supports contains """ - data = [11,22,"dog","cat"] + data = [11, 22, "dog", "cat"] streams = StreamSet(data) assert "dog" in streams - def test_reverse(self): """ Assert StreamSet instance supports reversal """ - data = [11,22,"dog","cat"] + data = [11, 22, "dog", "cat"] streams = StreamSet(data) assert list(reversed(streams)) == list(reversed(data)) - def test_to_list(self): """ Assert StreamSet instance cast to list """ - data = [11,22,"dog","cat"] + data = [11, 22, "dog", "cat"] streams = StreamSet(data) assert list(streams) == data - ########################################################################## ## allow_window tests ########################################################################## @@ -1026,18 +1067,16 @@ def test_allow_window(self): """ Assert allow_window returns False if window already requested """ - streams = StreamSet([1,2,3]) + streams = StreamSet([1, 2, 3]) assert streams.allow_window == True streams.windows(30, 4) assert streams.allow_window == False - - streams = StreamSet([1,2,3]) + streams = StreamSet([1, 2, 3]) streams.aligned_windows(30) assert streams.allow_window == False - ########################################################################## ## _latest_versions tests ########################################################################## @@ -1047,13 +1086,9 @@ def test_latest_versions(self, stream1, stream2): Assert _latest_versions returns correct values """ streams = StreamSet([stream1, stream2]) - expected = { - stream1.uuid: stream1.version(), - stream2.uuid: stream2.version() - } + expected = {stream1.uuid: stream1.version(), stream2.uuid: stream2.version()} assert streams._latest_versions() == expected - ########################################################################## ## pin_versions tests ########################################################################## @@ -1063,50 +1098,40 @@ def test_pin_versions_returns_self(self, stream1): Assert pin_versions returns self """ streams = StreamSet([stream1]) - expected = { - stream1.uuid(): stream1.version() - } - result = streams.pin_versions(expected) + expected = {stream1.uuid(): stream1.version()} + result = streams.pin_versions(expected) assert streams is result - def test_pin_versions_with_argument(self): """ Assert pin_versions uses supplied version numbers """ - streams = StreamSet([1,2]) - expected = [3,4] + streams = StreamSet([1, 2]) + expected = [3, 4] assert streams.pin_versions(expected) == streams assert streams._pinned_versions == expected - def test_pin_versions_with_argument(self): """ Assert pin_versions uses supplied version numbers """ - streams = StreamSet([1,2]) - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') - expected = { - uu: 42 - } + streams = StreamSet([1, 2]) + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + expected = {uu: 42} assert streams.pin_versions(expected) == streams assert streams._pinned_versions == expected - def test_pin_versions_no_argument(self, stream1, stream2): """ Assert pin_versions uses latest version numbers """ streams = StreamSet([stream1, stream2]) - uu = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') - expected = { - uu: 42 - } + uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + expected = {uu: 42} assert streams.pin_versions(expected) == streams assert streams._pinned_versions == expected - def test_pin_versions_raise_on_non_dict(self): """ Assert pin_versions raises if versions argument is not dict @@ -1118,7 +1143,6 @@ def test_pin_versions_raise_on_non_dict(self): streams.pin_versions(expected) == streams assert "dict" in str(e).lower() - def test_pin_versions_raise_on_non_uuid_key(self): """ Assert pin_versions raises if versions argument dict does not have UUID @@ -1131,7 +1155,6 @@ def test_pin_versions_raise_on_non_uuid_key(self): streams.pin_versions(expected) == streams assert "uuid" in str(e).lower() - ########################################################################## ## versions tests ########################################################################## @@ -1141,37 +1164,29 @@ def test_versions_no_pin(self, stream1, stream2): Assert versions returns correctly if pin_versions not called """ streams = StreamSet([stream1, stream2]) - expected = { - stream1.uuid: stream1.version(), - stream2.uuid: stream2.version() - } + expected = {stream1.uuid: stream1.version(), stream2.uuid: stream2.version()} assert streams.versions() == expected - def test_versions_with_pin(self, stream1, stream2): """ Assert versions returns correctly if pin_versions called """ - uu1 = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu1 = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") s1 = Mock(Stream) s1.version = Mock(return_value=11) s1.uuid = Mock(return_value=uu1) - uu2 = uuid.UUID('17dbe387-89ea-42b6-864b-f505cdb483f5') + uu2 = uuid.UUID("17dbe387-89ea-42b6-864b-f505cdb483f5") s2 = Mock(Stream) s2.version = Mock(return_value=22) s2.uuid = Mock(return_value=uu2) streams = StreamSet([stream1, stream2]) - expected = { - stream1.uuid(): 88, - stream2.uuid(): 99 - } + expected = {stream1.uuid(): 88, stream2.uuid(): 99} streams.pin_versions(expected) assert streams.versions() == expected - ########################################################################## ## earliest/latest tests ########################################################################## @@ -1181,22 +1196,27 @@ def test_earliest(self, stream1, stream2): Assert earliest returns correct time code """ streams = StreamSet([stream1, stream2]) - assert streams.latest() == (RawPoint(time=10, value=1), RawPoint(time=20, value=1)) - + assert streams.earliest() == ( + RawPoint(time=10, value=1), + RawPoint(time=20, value=1), + ) def test_latest(self, stream1, stream2): """ Assert latest returns correct time code """ streams = StreamSet([stream1, stream2]) - assert streams.latest() == (RawPoint(time=10, value=1), RawPoint(time=20, value=1)) + assert streams.latest() == ( + RawPoint(time=10, value=1), + RawPoint(time=20, value=1), + ) @patch("btrdb.stream.currently_as_ns") def test_current(self, mocked, stream1, stream2): """ Assert current calls nearest with the current time """ - mocked.return_value=15 + mocked.return_value = 15 streams = StreamSet([stream1, stream2]) streams.current() stream1.nearest.assert_called_once_with(15, version=11, backward=True) @@ -1207,52 +1227,84 @@ def test_currently_out_of_range(self, mocked): """ Assert currently raises an exception if it is not filtered """ - mocked.return_value=15 + mocked.return_value = 15 streams = StreamSet([stream1, stream2]) - with pytest.raises(ValueError, match="current time is not included in filtered stream range"): + with pytest.raises( + ValueError, match="current time is not included in filtered stream range" + ): streams.filter(start=20, end=30).current() - with pytest.raises(ValueError, match="current time is not included in filtered stream range"): + with pytest.raises( + ValueError, match="current time is not included in filtered stream range" + ): streams.filter(start=0, end=10).current() def test_count(self): """ Test the stream set count method """ - uu1 = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') - uu2 = uuid.UUID('4dadf38d-52a5-4b7a-ada9-a5d563f9538c') + uu1 = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + uu2 = uuid.UUID("4dadf38d-52a5-4b7a-ada9-a5d563f9538c") endpoint = Mock(Endpoint) + endpoint.streamInfo = Mock(return_value=[-1, -1, -1, -1, 0]) windows = [ - [(StatPointProto(time=1,min=2,mean=3,max=4,count=5,stddev=6), StatPointProto(time=2,min=3,mean=4,max=5,count=6,stddev=7)), 42], - [(StatPointProto(time=3,min=4,mean=5,max=6,count=7,stddev=8), StatPointProto(time=4,min=5,mean=6,max=7,count=8,stddev=9)), 42], + [ + ( + StatPointProto(time=1, min=2, mean=3, max=4, count=5, stddev=6), + StatPointProto(time=2, min=3, mean=4, max=5, count=6, stddev=7), + ), + 42, + ], + [ + ( + StatPointProto(time=3, min=4, mean=5, max=6, count=7, stddev=8), + StatPointProto(time=4, min=5, mean=6, max=7, count=8, stddev=9), + ), + 42, + ], ] endpoint.alignedWindows = Mock(return_value=windows) - streams = StreamSet([ - Stream(btrdb=BTrDB(endpoint), uuid=uu1), - Stream(btrdb=BTrDB(endpoint), uuid=uu2), - ]) + streams = StreamSet( + [ + Stream(btrdb=BTrDB(endpoint), uuid=uu1), + Stream(btrdb=BTrDB(endpoint), uuid=uu2), + ] + ) assert streams.count() == 52 endpoint.alignedWindows.assert_any_call(uu1, MINIMUM_TIME, MAXIMUM_TIME, 60, 0) endpoint.alignedWindows.assert_any_call(uu2, MINIMUM_TIME, MAXIMUM_TIME, 60, 0) - def test_count_filtered(self): """ Test the stream set count method with filters """ - uu1 = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') - uu2 = uuid.UUID('4dadf38d-52a5-4b7a-ada9-a5d563f9538c') + uu1 = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + uu2 = uuid.UUID("4dadf38d-52a5-4b7a-ada9-a5d563f9538c") endpoint = Mock(Endpoint) endpoint.alignedWindows = Mock(return_value=[]) - streams = StreamSet([ - Stream(btrdb=BTrDB(endpoint), uuid=uu1), - Stream(btrdb=BTrDB(endpoint), uuid=uu2), - ]) + streams = StreamSet( + [ + Stream(btrdb=BTrDB(endpoint), uuid=uu1), + Stream(btrdb=BTrDB(endpoint), uuid=uu2), + ] + ) windows = [ - [(StatPointProto(time=1,min=2,mean=3,max=4,count=5,stddev=6), StatPointProto(time=2,min=3,mean=4,max=5,count=6,stddev=7)), 42], - [(StatPointProto(time=3,min=4,mean=5,max=6,count=7,stddev=8), StatPointProto(time=4,min=5,mean=6,max=7,count=8,stddev=9)), 42], + [ + ( + StatPointProto(time=1, min=2, mean=3, max=4, count=5, stddev=6), + StatPointProto(time=2, min=3, mean=4, max=5, count=6, stddev=7), + ), + 42, + ], + [ + ( + StatPointProto(time=3, min=4, mean=5, max=6, count=7, stddev=8), + StatPointProto(time=4, min=5, mean=6, max=7, count=8, stddev=9), + ), + 42, + ], ] endpoint.alignedWindows = Mock(return_value=windows) @@ -1263,8 +1315,6 @@ def test_count_filtered(self): endpoint.alignedWindows.assert_any_call(uu1, 10, 1000, 8, 42) endpoint.alignedWindows.assert_any_call(uu2, 10, 1000, 8, 99) - - ########################################################################## ## filter tests ########################################################################## @@ -1282,7 +1332,6 @@ def test_filter(self, stream1): assert streams.filters[0].end == end assert isinstance(streams.filters[0], StreamFilter) - def test_filter_returns_new_instance(self, stream1): """ Assert filter returns new instance @@ -1294,7 +1343,6 @@ def test_filter_returns_new_instance(self, stream1): assert other is not streams assert isinstance(other, streams.__class__) - def test_filter_collection(self, stream1, stream2): """ Assert filter collection works as intended @@ -1331,7 +1379,6 @@ def test_filter_collection(self, stream1, stream2): other = streams.filter(collection=re.compile(r"region\.")) assert other._streams == [stream1] - def test_filter_name(self, stream1, stream2): """ Assert filter name works as intended @@ -1366,7 +1413,6 @@ def test_filter_name(self, stream1, stream2): other = streams.filter(name=re.compile(r"region\.")) assert other._streams == [stream1] - def test_filter_unit(self, stream1, stream2): """ Assert filter unit works as intended @@ -1391,7 +1437,6 @@ def test_filter_unit(self, stream1, stream2): other = streams.filter(unit=re.compile("meters")) assert other._streams == [] - def test_filter_tags(self, stream1, stream2): """ Assert filter annotations works as intended @@ -1410,7 +1455,6 @@ def test_filter_tags(self, stream1, stream2): other = streams.filter(tags={"unit": "volts"}) assert other._streams == [stream1, stream2] - def test_filter_annotations(self, stream1, stream2): """ Assert filter annotations works as intended @@ -1429,7 +1473,6 @@ def test_filter_annotations(self, stream1, stream2): other = streams.filter(annotations={"owner": "ABC", "color": "red"}) assert other._streams == [stream1] - ########################################################################## ## clone tests ########################################################################## @@ -1444,12 +1487,11 @@ def test_clone_returns_new_object(self, stream1, stream2): assert id(clone) != id(streams) assert clone._streams is streams._streams - ########################################################################## ## windows tests ########################################################################## - def test_windows_raises_valueerror(self, stream1): + def test_windows_raises_valueerror_and_warning(self, stream1): """ Assert that raises ValueError if arguments not castable to int """ @@ -1458,10 +1500,8 @@ def test_windows_raises_valueerror(self, stream1): streams.windows("invalid", 42) assert "literal" in str(exc).lower() - with pytest.raises(ValueError) as exc: + with pytest.warns(Warning, match="Overriding") as exc: streams.windows(42, "invalid") - assert "literal" in str(exc).lower() - def test_windows_raises_if_not_allowed(self, stream1): """ @@ -1474,7 +1514,6 @@ def test_windows_raises_if_not_allowed(self, stream1): streams.windows(10, 20) assert "window operation is already requested" in str(exc).lower() - def test_windows_returns_self(self, stream1): """ Assert windows returns self @@ -1483,7 +1522,6 @@ def test_windows_returns_self(self, stream1): result = streams.windows(10, 20) assert result is streams - def test_windows_stores_values(self, stream1): """ Assert windows stores values @@ -1492,21 +1530,24 @@ def test_windows_stores_values(self, stream1): result = streams.windows(10, 20) # assert stores values - assert streams.width == 10 - assert streams.depth == 20 - + assert result.width == 10 + assert result.depth == 0 def test_windows_values_and_calls_to_endpoint(self): """ assert windows result and endpoint calls are correct """ endpoint = Mock(Endpoint) - window1 = [[(StatPointProto(time=1,min=2,mean=3,max=4,count=5,stddev=6),), 11]] - window2 = [[(StatPointProto(time=2,min=3,mean=4,max=5,count=6,stddev=7),), 12]] - endpoint.windows = Mock(side_effect=[ window1, window2 ]) + window1 = [ + [(StatPointProto(time=1, min=2, mean=3, max=4, count=5, stddev=6),), 11] + ] + window2 = [ + [(StatPointProto(time=2, min=3, mean=4, max=5, count=6, stddev=7),), 12] + ] + endpoint.windows = Mock(side_effect=[window1, window2]) - uu1 = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') - uu2 = uuid.UUID('5d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu1 = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + uu2 = uuid.UUID("5d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") s1 = Stream(btrdb=BTrDB(endpoint), uuid=uu1) s2 = Stream(btrdb=BTrDB(endpoint), uuid=uu2) versions = {uu1: 11, uu2: 12} @@ -1514,12 +1555,16 @@ def test_windows_values_and_calls_to_endpoint(self): start, end, width, depth = 1, 100, 1000, 25 streams = StreamSet([s1, s2]) streams.pin_versions(versions) - values = streams.filter(start=start, end=end).windows(width=width, depth=depth).values() + values = ( + streams.filter(start=start, end=end) + .windows(width=width, depth=depth) + .values() + ) # assert endpoint calls have correct arguments, version expected = [ - call(uu1, start, end, width, depth, versions[uu1]), - call(uu2, start, end, width, depth, versions[uu2]) + call(uu1, start, end, width, 0, versions[uu1]), + call(uu2, start, end, width, 0, versions[uu2]), ] assert endpoint.windows.call_args_list == expected @@ -1530,18 +1575,21 @@ def test_windows_values_and_calls_to_endpoint(self): ] assert values == expected - def test_windows_rows_and_calls_to_endpoint(self): """ assert windows rows result and endpoint calls are correct """ endpoint = Mock(Endpoint) - window1 = [[(StatPointProto(time=1,min=2,mean=3,max=4,count=5,stddev=6),), 11]] - window2 = [[(StatPointProto(time=2,min=3,mean=4,max=5,count=6,stddev=7),), 12]] - endpoint.windows = Mock(side_effect=[ window1, window2 ]) + window1 = [ + [(StatPointProto(time=1, min=2, mean=3, max=4, count=5, stddev=6),), 11] + ] + window2 = [ + [(StatPointProto(time=2, min=3, mean=4, max=5, count=6, stddev=7),), 12] + ] + endpoint.windows = Mock(side_effect=[window1, window2]) - uu1 = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') - uu2 = uuid.UUID('5d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu1 = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + uu2 = uuid.UUID("5d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") s1 = Stream(btrdb=BTrDB(endpoint), uuid=uu1) s2 = Stream(btrdb=BTrDB(endpoint), uuid=uu2) versions = {uu1: 11, uu2: 12} @@ -1549,12 +1597,16 @@ def test_windows_rows_and_calls_to_endpoint(self): start, end, width, depth = 1, 100, 1000, 25 streams = StreamSet([s1, s2]) streams.pin_versions(versions) - rows = streams.filter(start=start, end=end).windows(width=width, depth=depth).rows() + rows = ( + streams.filter(start=start, end=end) + .windows(width=width, depth=depth) + .rows() + ) # assert endpoint calls have correct arguments, version expected = [ - call(uu1, start, end, width, depth, versions[uu1]), - call(uu2, start, end, width, depth, versions[uu2]) + call(uu1, start, end, width, 0, versions[uu1]), + call(uu2, start, end, width, 0, versions[uu2]), ] assert endpoint.windows.call_args_list == expected @@ -1565,7 +1617,6 @@ def test_windows_rows_and_calls_to_endpoint(self): ] assert rows == expected - ########################################################################## ## aligned_windows tests ########################################################################## @@ -1579,7 +1630,6 @@ def test_aligned_windows_raises_valueerror(self, stream1): streams.aligned_windows("invalid") assert "literal" in str(exc).lower() - def test_aligned_windows_raises_if_not_allowed(self, stream1): """ Assert that aligned_windows raises Exception if not allowed @@ -1591,7 +1641,6 @@ def test_aligned_windows_raises_if_not_allowed(self, stream1): streams.aligned_windows(20) assert "window operation is already requested" in str(exc).lower() - def test_aligned_windows(self, stream1): """ Assert aligned_windows stores objects @@ -1600,7 +1649,6 @@ def test_aligned_windows(self, stream1): result = streams.aligned_windows(20) assert streams.pointwidth == 20 - def test_aligned_windows_returns_self(self, stream1): """ Assert aligned_windows returns self @@ -1609,18 +1657,21 @@ def test_aligned_windows_returns_self(self, stream1): result = streams.aligned_windows(20) assert result is streams - def test_aligned_windows_values_and_calls_to_endpoint(self): """ assert aligned_windows result and endpoint calls are correct """ endpoint = Mock(Endpoint) - window1 = [[(StatPointProto(time=1,min=2,mean=3,max=4,count=5,stddev=6),), 11]] - window2 = [[(StatPointProto(time=2,min=3,mean=4,max=5,count=6,stddev=7),), 12]] - endpoint.alignedWindows = Mock(side_effect=[ window1, window2 ]) + window1 = [ + [(StatPointProto(time=1, min=2, mean=3, max=4, count=5, stddev=6),), 11] + ] + window2 = [ + [(StatPointProto(time=2, min=3, mean=4, max=5, count=6, stddev=7),), 12] + ] + endpoint.alignedWindows = Mock(side_effect=[window1, window2]) - uu1 = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') - uu2 = uuid.UUID('5d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu1 = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + uu2 = uuid.UUID("5d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") s1 = Stream(btrdb=BTrDB(endpoint), uuid=uu1) s2 = Stream(btrdb=BTrDB(endpoint), uuid=uu2) versions = {uu1: 11, uu2: 12} @@ -1628,12 +1679,16 @@ def test_aligned_windows_values_and_calls_to_endpoint(self): start, end, pointwidth = 1, 100, 25 streams = StreamSet([s1, s2]) streams.pin_versions(versions) - values = streams.filter(start=start, end=end).aligned_windows(pointwidth=pointwidth).values() + values = ( + streams.filter(start=start, end=end) + .aligned_windows(pointwidth=pointwidth) + .values() + ) # assert endpoint calls have correct arguments, version expected = [ call(uu1, start, end, pointwidth, versions[uu1]), - call(uu2, start, end, pointwidth, versions[uu2]) + call(uu2, start, end, pointwidth, versions[uu2]), ] assert endpoint.alignedWindows.call_args_list == expected @@ -1644,18 +1699,21 @@ def test_aligned_windows_values_and_calls_to_endpoint(self): ] assert values == expected - def test_aligned_windows_rows_and_calls_to_endpoint(self): """ assert aligned_windows rows result and endpoint calls are correct """ endpoint = Mock(Endpoint) - window1 = [[(StatPointProto(time=1,min=2,mean=3,max=4,count=5,stddev=6),), 11]] - window2 = [[(StatPointProto(time=2,min=3,mean=4,max=5,count=6,stddev=7),), 12]] - endpoint.alignedWindows = Mock(side_effect=[ window1, window2 ]) + window1 = [ + [(StatPointProto(time=1, min=2, mean=3, max=4, count=5, stddev=6),), 11] + ] + window2 = [ + [(StatPointProto(time=2, min=3, mean=4, max=5, count=6, stddev=7),), 12] + ] + endpoint.alignedWindows = Mock(side_effect=[window1, window2]) - uu1 = uuid.UUID('0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') - uu2 = uuid.UUID('5d22a53b-e2ef-4e0a-ab89-b2d48fb2592a') + uu1 = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + uu2 = uuid.UUID("5d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") s1 = Stream(btrdb=BTrDB(endpoint), uuid=uu1) s2 = Stream(btrdb=BTrDB(endpoint), uuid=uu2) versions = {uu1: 11, uu2: 12} @@ -1663,12 +1721,16 @@ def test_aligned_windows_rows_and_calls_to_endpoint(self): start, end, pointwidth = 1, 100, 25 streams = StreamSet([s1, s2]) streams.pin_versions(versions) - rows = streams.filter(start=start, end=end).aligned_windows(pointwidth=pointwidth).rows() + rows = ( + streams.filter(start=start, end=end) + .aligned_windows(pointwidth=pointwidth) + .rows() + ) # assert endpoint calls have correct arguments, version expected = [ call(uu1, start, end, pointwidth, versions[uu1]), - call(uu2, start, end, pointwidth, versions[uu2]) + call(uu2, start, end, pointwidth, versions[uu2]), ] assert endpoint.alignedWindows.call_args_list == expected @@ -1679,8 +1741,6 @@ def test_aligned_windows_rows_and_calls_to_endpoint(self): ] assert rows == expected - - ########################################################################## ## rows tests ########################################################################## @@ -1689,13 +1749,21 @@ def test_rows(self, stream1, stream2): """ Assert rows returns correct values """ - stream1.values = Mock(return_value=iter([ - (RawPoint(time=1, value=1), 1), (RawPoint(time=2, value=2), 1), - (RawPoint(time=3, value=3), 1), (RawPoint(time=4, value=4), 1), - ])) - stream2.values = Mock(return_value=iter([ - (RawPoint(time=1, value=1), 2), (RawPoint(time=3, value=3), 2) - ])) + stream1.values = Mock( + return_value=iter( + [ + (RawPoint(time=1, value=1), 1), + (RawPoint(time=2, value=2), 1), + (RawPoint(time=3, value=3), 1), + (RawPoint(time=4, value=4), 1), + ] + ) + ) + stream2.values = Mock( + return_value=iter( + [(RawPoint(time=1, value=1), 2), (RawPoint(time=3, value=3), 2)] + ) + ) streams = StreamSet([stream1, stream2]) rows = iter(streams.rows()) @@ -1705,7 +1773,6 @@ def test_rows(self, stream1, stream2): assert next(rows) == (RawPoint(time=3, value=3), RawPoint(time=3, value=3)) assert next(rows) == (RawPoint(time=4, value=4), None) - ########################################################################## ## _params_from_filters tests ########################################################################## @@ -1723,7 +1790,6 @@ def test_params_from_filters_latest_value_wins(self, stream1): streams = streams.filter(start=2) assert streams._params_from_filters() == {"start": 2} - def test_params_from_filters_works(self, stream1): """ Assert _params_from_filters returns correct values @@ -1737,7 +1803,6 @@ def test_params_from_filters_works(self, stream1): streams = streams.filter(start=9, end=10) assert streams._params_from_filters() == {"start": 9, "end": 10} - ########################################################################## ## values tests ########################################################################## @@ -1747,13 +1812,16 @@ def test_values(self, stream1, stream2): Assert values returns correct data """ stream1_values = [ - (RawPoint(time=1, value=1), 1), (RawPoint(time=2, value=2), 1), - (RawPoint(time=3, value=3), 1), (RawPoint(time=4, value=4), 1), + (RawPoint(time=1, value=1), 1), + (RawPoint(time=2, value=2), 1), + (RawPoint(time=3, value=3), 1), + (RawPoint(time=4, value=4), 1), ] stream1.values = Mock(return_value=iter(stream1_values)) stream2_values = [ - (RawPoint(time=1, value=1), 2), (RawPoint(time=3, value=3), 2) + (RawPoint(time=1, value=1), 2), + (RawPoint(time=3, value=3), 2), ] stream2.values = Mock(return_value=iter(stream2_values)) @@ -1768,20 +1836,20 @@ def test_values(self, stream1, stream2): ## StreamFilter Tests ########################################################################## -class TestStreamFilter(object): +class TestStreamFilter(object): def test_create(self): """ Assert we can create the object """ - StreamFilter(0,1) + StreamFilter(0, 1) def test_start_larger_or_equal(self): """ Assert we raise ValueError if start is greater than/equal to end """ with pytest.raises(ValueError): - StreamFilter(1,1) + StreamFilter(1, 1) def test_start_valid(self): """ diff --git a/tests/btrdb/test_transformers.py b/tests/btrdb/test_transformers.py index 5cdc0a5..15003c9 100644 --- a/tests/btrdb/test_transformers.py +++ b/tests/btrdb/test_transformers.py @@ -16,21 +16,22 @@ ########################################################################## import os -from io import StringIO, BytesIO - -import pytest +from io import BytesIO, StringIO from unittest.mock import Mock, PropertyMock + import numpy as np -from pandas import Series, DataFrame, Index +import pytest +from pandas import DataFrame, Index, Series -from btrdb.stream import Stream, StreamSet from btrdb.point import RawPoint, StatPoint +from btrdb.stream import Stream, StreamSet from btrdb.transformers import * ########################################################################## ## Transformer Tests ########################################################################## + @pytest.fixture def streamset(): # list of mock streams @@ -39,26 +40,110 @@ def streamset(): stream = Mock(Stream) type(stream).collection = PropertyMock(return_value="test") type(stream).name = PropertyMock(return_value="stream{}".format(idx)) + stream._btrdb = Mock() + stream._btrdb._ARROW_ENABLED = Mock(return_value=False) streams.append(stream) rows = [ - (None, RawPoint(1500000000000000000, 1.0), RawPoint(1500000000000000000, 1.0), RawPoint(1500000000000000000, 1.0)), - (RawPoint(1500000000100000000, 2.0), None, RawPoint(1500000000100000000, 2.0), RawPoint(1500000000100000000, 2.0)), - (None, RawPoint(1500000000200000000, 3.0), None, RawPoint(1500000000200000000, 3.0)), - (RawPoint(1500000000300000000, 4.0), None, RawPoint(1500000000300000000, 4.0), RawPoint(1500000000300000000, 4.0)), - (None, RawPoint(1500000000400000000, 5.0), RawPoint(1500000000400000000, 5.0), RawPoint(1500000000400000000, 5.0)), - (RawPoint(1500000000500000000, 6.0), None, None, RawPoint(1500000000500000000, 6.0)), - (None, RawPoint(1500000000600000000, 7.0), RawPoint(1500000000600000000, 7.0), RawPoint(1500000000600000000, 7.0)), - (RawPoint(1500000000700000000, 8.0), None, RawPoint(1500000000700000000, 8.0), RawPoint(1500000000700000000, 8.0)), - (None, RawPoint(1500000000800000000, 9.0), RawPoint(1500000000800000000, 9.0), RawPoint(1500000000800000000, 9.0)), - (RawPoint(1500000000900000000, 10.0), None, RawPoint(1500000000900000000, 10.0), RawPoint(1500000000900000000, 10.0) ), + ( + None, + RawPoint(1500000000000000000, 1.0), + RawPoint(1500000000000000000, 1.0), + RawPoint(1500000000000000000, 1.0), + ), + ( + RawPoint(1500000000100000000, 2.0), + None, + RawPoint(1500000000100000000, 2.0), + RawPoint(1500000000100000000, 2.0), + ), + ( + None, + RawPoint(1500000000200000000, 3.0), + None, + RawPoint(1500000000200000000, 3.0), + ), + ( + RawPoint(1500000000300000000, 4.0), + None, + RawPoint(1500000000300000000, 4.0), + RawPoint(1500000000300000000, 4.0), + ), + ( + None, + RawPoint(1500000000400000000, 5.0), + RawPoint(1500000000400000000, 5.0), + RawPoint(1500000000400000000, 5.0), + ), + ( + RawPoint(1500000000500000000, 6.0), + None, + None, + RawPoint(1500000000500000000, 6.0), + ), + ( + None, + RawPoint(1500000000600000000, 7.0), + RawPoint(1500000000600000000, 7.0), + RawPoint(1500000000600000000, 7.0), + ), + ( + RawPoint(1500000000700000000, 8.0), + None, + RawPoint(1500000000700000000, 8.0), + RawPoint(1500000000700000000, 8.0), + ), + ( + None, + RawPoint(1500000000800000000, 9.0), + RawPoint(1500000000800000000, 9.0), + RawPoint(1500000000800000000, 9.0), + ), + ( + RawPoint(1500000000900000000, 10.0), + None, + RawPoint(1500000000900000000, 10.0), + RawPoint(1500000000900000000, 10.0), + ), ] values = [ - [RawPoint(1500000000100000000, 2.0),RawPoint(1500000000300000000, 4.0),RawPoint(1500000000500000000, 6.0),RawPoint(1500000000700000000, 8.0),RawPoint(1500000000900000000, 10.0)], - [RawPoint(1500000000000000000, 1.0),RawPoint(1500000000200000000, 3.0),RawPoint(1500000000400000000, 5.0),RawPoint(1500000000600000000, 7.0),RawPoint(1500000000800000000, 9.0)], - [RawPoint(1500000000000000000, 1.0),RawPoint(1500000000100000000, 2.0),RawPoint(1500000000300000000, 4.0),RawPoint(1500000000400000000, 5.0),RawPoint(1500000000600000000, 7.0),RawPoint(1500000000700000000, 8.0),RawPoint(1500000000800000000, 9.0),RawPoint(1500000000900000000, 10.0)], - [RawPoint(1500000000000000000, 1.0),RawPoint(1500000000100000000, 2.0),RawPoint(1500000000200000000, 3.0),RawPoint(1500000000300000000, 4.0),RawPoint(1500000000400000000, 5.0),RawPoint(1500000000500000000, 6.0),RawPoint(1500000000600000000, 7.0),RawPoint(1500000000700000000, 8.0),RawPoint(1500000000800000000, 9.0),RawPoint(1500000000900000000, 10.0)] + [ + RawPoint(1500000000100000000, 2.0), + RawPoint(1500000000300000000, 4.0), + RawPoint(1500000000500000000, 6.0), + RawPoint(1500000000700000000, 8.0), + RawPoint(1500000000900000000, 10.0), + ], + [ + RawPoint(1500000000000000000, 1.0), + RawPoint(1500000000200000000, 3.0), + RawPoint(1500000000400000000, 5.0), + RawPoint(1500000000600000000, 7.0), + RawPoint(1500000000800000000, 9.0), + ], + [ + RawPoint(1500000000000000000, 1.0), + RawPoint(1500000000100000000, 2.0), + RawPoint(1500000000300000000, 4.0), + RawPoint(1500000000400000000, 5.0), + RawPoint(1500000000600000000, 7.0), + RawPoint(1500000000700000000, 8.0), + RawPoint(1500000000800000000, 9.0), + RawPoint(1500000000900000000, 10.0), + ], + [ + RawPoint(1500000000000000000, 1.0), + RawPoint(1500000000100000000, 2.0), + RawPoint(1500000000200000000, 3.0), + RawPoint(1500000000300000000, 4.0), + RawPoint(1500000000400000000, 5.0), + RawPoint(1500000000500000000, 6.0), + RawPoint(1500000000600000000, 7.0), + RawPoint(1500000000700000000, 8.0), + RawPoint(1500000000800000000, 9.0), + RawPoint(1500000000900000000, 10.0), + ], ] obj = StreamSet(streams) @@ -70,16 +155,56 @@ def streamset(): @pytest.fixture def statpoint_streamset(): rows = [ - [StatPoint(1500000000100000000, 0, 2.0, 2.5, 10, 1),StatPoint(1500000000100000000, 0, 4.0, 4.5, 11, 1),StatPoint(1500000000100000000, 0, 6.0, 6.5, 10, 1),StatPoint(1500000000100000000, 0, 8.0, 8.5, 11, 2)], - [StatPoint(1500000000300000000, 0, 3.0, 3.5, 11, 1),StatPoint(1500000000300000000, 0, 5.0, 5.5, 10, 1),StatPoint(1500000000300000000, 0, 7.0, 7.5, 10, 1),StatPoint(1500000000300000000, 0, 9.0, 9.5, 11, 2)], - [StatPoint(1500000000500000000, 0, 4.0, 4.5, 10, 1),StatPoint(1500000000500000000, 0, 6.0, 6.5, 11, 1),StatPoint(1500000000500000000, 0, 8.0, 8.5, 10, 1),StatPoint(1500000000500000000, 0, 10.0, 10.5, 11, 2)], - [StatPoint(1500000000700000000, 0, 5.0, 5.5, 11, 1),StatPoint(1500000000700000000, 0, 7.0, 7.5, 10, 1),StatPoint(1500000000700000000, 0, 9.0, 9.5, 10, 1),StatPoint(1500000000700000000, 0, 11.0, 11.5, 11, 2)], + [ + StatPoint(1500000000100000000, 0, 2.0, 2.5, 10, 1), + StatPoint(1500000000100000000, 0, 4.0, 4.5, 11, 1), + StatPoint(1500000000100000000, 0, 6.0, 6.5, 10, 1), + StatPoint(1500000000100000000, 0, 8.0, 8.5, 11, 2), + ], + [ + StatPoint(1500000000300000000, 0, 3.0, 3.5, 11, 1), + StatPoint(1500000000300000000, 0, 5.0, 5.5, 10, 1), + StatPoint(1500000000300000000, 0, 7.0, 7.5, 10, 1), + StatPoint(1500000000300000000, 0, 9.0, 9.5, 11, 2), + ], + [ + StatPoint(1500000000500000000, 0, 4.0, 4.5, 10, 1), + StatPoint(1500000000500000000, 0, 6.0, 6.5, 11, 1), + StatPoint(1500000000500000000, 0, 8.0, 8.5, 10, 1), + StatPoint(1500000000500000000, 0, 10.0, 10.5, 11, 2), + ], + [ + StatPoint(1500000000700000000, 0, 5.0, 5.5, 11, 1), + StatPoint(1500000000700000000, 0, 7.0, 7.5, 10, 1), + StatPoint(1500000000700000000, 0, 9.0, 9.5, 10, 1), + StatPoint(1500000000700000000, 0, 11.0, 11.5, 11, 2), + ], ] values = [ - [StatPoint(1500000000100000000, 0, 2.0, 2.5, 10, 1),StatPoint(1500000000300000000, 0, 4.0, 4.5, 11, 1),StatPoint(1500000000500000000, 0, 6.0, 6.5, 10, 1),StatPoint(1500000000700000000, 0, 8.0, 8.5, 11, 2)], - [StatPoint(1500000000100000000, 1, 3.0, 3.5, 11, 1),StatPoint(1500000000300000000, 1, 5.0, 5.5, 10, 1),StatPoint(1500000000500000000, 1, 7.0, 7.5, 10, 1),StatPoint(1500000000700000000, 1, 9.0, 9.5, 11, 2)], - [StatPoint(1500000000100000000, 2, 4.0, 4.5, 10, 1),StatPoint(1500000000300000000, 2, 6.0, 6.5, 11, 1),StatPoint(1500000000500000000, 2, 8.0, 8.5, 10, 1),StatPoint(1500000000700000000, 2, 10.0, 10.5, 11, 2)], - [StatPoint(1500000000100000000, 3, 5.0, 5.5, 11, 1),StatPoint(1500000000300000000, 3, 7.0, 7.5, 10, 1),StatPoint(1500000000500000000, 3, 9.0, 9.5, 10, 1),StatPoint(1500000000700000000, 3, 11.0, 11.5, 11, 2)], + [ + StatPoint(1500000000100000000, 0, 2.0, 2.5, 10, 1), + StatPoint(1500000000300000000, 0, 4.0, 4.5, 11, 1), + StatPoint(1500000000500000000, 0, 6.0, 6.5, 10, 1), + StatPoint(1500000000700000000, 0, 8.0, 8.5, 11, 2), + ], + [ + StatPoint(1500000000100000000, 1, 3.0, 3.5, 11, 1), + StatPoint(1500000000300000000, 1, 5.0, 5.5, 10, 1), + StatPoint(1500000000500000000, 1, 7.0, 7.5, 10, 1), + StatPoint(1500000000700000000, 1, 9.0, 9.5, 11, 2), + ], + [ + StatPoint(1500000000100000000, 2, 4.0, 4.5, 10, 1), + StatPoint(1500000000300000000, 2, 6.0, 6.5, 11, 1), + StatPoint(1500000000500000000, 2, 8.0, 8.5, 10, 1), + StatPoint(1500000000700000000, 2, 10.0, 10.5, 11, 2), + ], + [ + StatPoint(1500000000100000000, 3, 5.0, 5.5, 11, 1), + StatPoint(1500000000300000000, 3, 7.0, 7.5, 10, 1), + StatPoint(1500000000500000000, 3, 9.0, 9.5, 10, 1), + StatPoint(1500000000700000000, 3, 11.0, 11.5, 11, 2), + ], ] streams = [] for idx in range(4): @@ -97,16 +222,76 @@ def statpoint_streamset(): expected = { "to_dict": [ - {'time': 1500000000000000000, 'test/stream0': None, 'test/stream1': 1.0, 'test/stream2': 1.0, 'test/stream3': 1.0}, - {'time': 1500000000100000000, 'test/stream0': 2.0, 'test/stream1': None, 'test/stream2': 2.0, 'test/stream3': 2.0}, - {'time': 1500000000200000000, 'test/stream0': None, 'test/stream1': 3.0, 'test/stream2': None, 'test/stream3': 3.0}, - {'time': 1500000000300000000, 'test/stream0': 4.0, 'test/stream1': None, 'test/stream2': 4.0, 'test/stream3': 4.0}, - {'time': 1500000000400000000, 'test/stream0': None, 'test/stream1': 5.0, 'test/stream2': 5.0, 'test/stream3': 5.0}, - {'time': 1500000000500000000, 'test/stream0': 6.0, 'test/stream1': None, 'test/stream2': None, 'test/stream3': 6.0}, - {'time': 1500000000600000000, 'test/stream0': None, 'test/stream1': 7.0, 'test/stream2': 7.0, 'test/stream3': 7.0}, - {'time': 1500000000700000000, 'test/stream0': 8.0, 'test/stream1': None, 'test/stream2': 8.0, 'test/stream3': 8.0}, - {'time': 1500000000800000000, 'test/stream0': None, 'test/stream1': 9.0, 'test/stream2': 9.0, 'test/stream3': 9.0}, - {'time': 1500000000900000000, 'test/stream0': 10.0, 'test/stream1': None, 'test/stream2': 10.0, 'test/stream3': 10.0} + { + "time": 1500000000000000000, + "test/stream0": None, + "test/stream1": 1.0, + "test/stream2": 1.0, + "test/stream3": 1.0, + }, + { + "time": 1500000000100000000, + "test/stream0": 2.0, + "test/stream1": None, + "test/stream2": 2.0, + "test/stream3": 2.0, + }, + { + "time": 1500000000200000000, + "test/stream0": None, + "test/stream1": 3.0, + "test/stream2": None, + "test/stream3": 3.0, + }, + { + "time": 1500000000300000000, + "test/stream0": 4.0, + "test/stream1": None, + "test/stream2": 4.0, + "test/stream3": 4.0, + }, + { + "time": 1500000000400000000, + "test/stream0": None, + "test/stream1": 5.0, + "test/stream2": 5.0, + "test/stream3": 5.0, + }, + { + "time": 1500000000500000000, + "test/stream0": 6.0, + "test/stream1": None, + "test/stream2": None, + "test/stream3": 6.0, + }, + { + "time": 1500000000600000000, + "test/stream0": None, + "test/stream1": 7.0, + "test/stream2": 7.0, + "test/stream3": 7.0, + }, + { + "time": 1500000000700000000, + "test/stream0": 8.0, + "test/stream1": None, + "test/stream2": 8.0, + "test/stream3": 8.0, + }, + { + "time": 1500000000800000000, + "test/stream0": None, + "test/stream1": 9.0, + "test/stream2": 9.0, + "test/stream3": 9.0, + }, + { + "time": 1500000000900000000, + "test/stream0": 10.0, + "test/stream1": None, + "test/stream2": 10.0, + "test/stream3": 10.0, + }, ], "to_array": [ np.array([2.0, 4.0, 6.0, 8.0, 10.0]), @@ -140,12 +325,10 @@ def statpoint_streamset(): 1500000000800000000,,9.0,9.0,9.0 1500000000900000000,10.0,,10.0,10.0 """, - } class TestTransformers(object): - ########################################################################## ## to_dict Tests ########################################################################## @@ -155,30 +338,174 @@ def test_to_dict(self, streamset): def test_to_dict_statpoint(self, statpoint_streamset): expected = [ - {'time': 1500000000100000000, 'test/stream0': 2.0, 'test/stream1': 4.0, 'test/stream2': 6.0, 'test/stream3': 8.0}, - {'time': 1500000000300000000, 'test/stream0': 3.0, 'test/stream1': 5.0, 'test/stream2': 7.0, 'test/stream3': 9.0}, - {'time': 1500000000500000000, 'test/stream0': 4.0, 'test/stream1': 6.0, 'test/stream2': 8.0, 'test/stream3': 10.0}, - {'time': 1500000000700000000, 'test/stream0': 5.0, 'test/stream1': 7.0, 'test/stream2': 9.0, 'test/stream3': 11.0}, + { + "time": 1500000000100000000, + "test/stream0": 2.0, + "test/stream1": 4.0, + "test/stream2": 6.0, + "test/stream3": 8.0, + }, + { + "time": 1500000000300000000, + "test/stream0": 3.0, + "test/stream1": 5.0, + "test/stream2": 7.0, + "test/stream3": 9.0, + }, + { + "time": 1500000000500000000, + "test/stream0": 4.0, + "test/stream1": 6.0, + "test/stream2": 8.0, + "test/stream3": 10.0, + }, + { + "time": 1500000000700000000, + "test/stream0": 5.0, + "test/stream1": 7.0, + "test/stream2": 9.0, + "test/stream3": 11.0, + }, ] assert to_dict(statpoint_streamset) == expected def test_to_dict_statpoint_agg_count(self, statpoint_streamset): expected = [ - {'time': 1500000000100000000, 'test/stream0': 10, 'test/stream1': 11, 'test/stream2': 10, 'test/stream3': 11}, - {'time': 1500000000300000000, 'test/stream0': 11, 'test/stream1': 10, 'test/stream2': 10, 'test/stream3': 11}, - {'time': 1500000000500000000, 'test/stream0': 10, 'test/stream1': 11, 'test/stream2': 10, 'test/stream3': 11}, - {'time': 1500000000700000000, 'test/stream0': 11, 'test/stream1': 10, 'test/stream2': 10, 'test/stream3': 11}, + { + "time": 1500000000100000000, + "test/stream0": 10, + "test/stream1": 11, + "test/stream2": 10, + "test/stream3": 11, + }, + { + "time": 1500000000300000000, + "test/stream0": 11, + "test/stream1": 10, + "test/stream2": 10, + "test/stream3": 11, + }, + { + "time": 1500000000500000000, + "test/stream0": 10, + "test/stream1": 11, + "test/stream2": 10, + "test/stream3": 11, + }, + { + "time": 1500000000700000000, + "test/stream0": 11, + "test/stream1": 10, + "test/stream2": 10, + "test/stream3": 11, + }, ] - assert to_dict(statpoint_streamset, agg='count') == expected + assert to_dict(statpoint_streamset, agg="count") == expected def test_to_dict_statpoint_agg_all(self, statpoint_streamset): expected = [ - OrderedDict([('time', 1500000000100000000), ('test/stream0-min', 0), ('test/stream0-mean', 2.0), ('test/stream0-max', 2.5), ('test/stream0-count', 10), ('test/stream0-stddev', 1), ('test/stream1-min', 0), ('test/stream1-mean', 4.0), ('test/stream1-max', 4.5), ('test/stream1-count', 11), ('test/stream1-stddev', 1), ('test/stream2-min', 0), ('test/stream2-mean', 6.0), ('test/stream2-max', 6.5), ('test/stream2-count', 10), ('test/stream2-stddev', 1), ('test/stream3-min', 0), ('test/stream3-mean', 8.0), ('test/stream3-max', 8.5), ('test/stream3-count', 11), ('test/stream3-stddev', 2)]), - OrderedDict([('time', 1500000000300000000), ('test/stream0-min', 0), ('test/stream0-mean', 3.0), ('test/stream0-max', 3.5), ('test/stream0-count', 11), ('test/stream0-stddev', 1), ('test/stream1-min', 0), ('test/stream1-mean', 5.0), ('test/stream1-max', 5.5), ('test/stream1-count', 10), ('test/stream1-stddev', 1), ('test/stream2-min', 0), ('test/stream2-mean', 7.0), ('test/stream2-max', 7.5), ('test/stream2-count', 10), ('test/stream2-stddev', 1), ('test/stream3-min', 0), ('test/stream3-mean', 9.0), ('test/stream3-max', 9.5), ('test/stream3-count', 11), ('test/stream3-stddev', 2)]), - OrderedDict([('time', 1500000000500000000), ('test/stream0-min', 0), ('test/stream0-mean', 4.0), ('test/stream0-max', 4.5), ('test/stream0-count', 10), ('test/stream0-stddev', 1), ('test/stream1-min', 0), ('test/stream1-mean', 6.0), ('test/stream1-max', 6.5), ('test/stream1-count', 11), ('test/stream1-stddev', 1), ('test/stream2-min', 0), ('test/stream2-mean', 8.0), ('test/stream2-max', 8.5), ('test/stream2-count', 10), ('test/stream2-stddev', 1), ('test/stream3-min', 0), ('test/stream3-mean', 10.0), ('test/stream3-max', 10.5), ('test/stream3-count', 11), ('test/stream3-stddev', 2)]), - OrderedDict([('time', 1500000000700000000), ('test/stream0-min', 0), ('test/stream0-mean', 5.0), ('test/stream0-max', 5.5), ('test/stream0-count', 11), ('test/stream0-stddev', 1), ('test/stream1-min', 0), ('test/stream1-mean', 7.0), ('test/stream1-max', 7.5), ('test/stream1-count', 10), ('test/stream1-stddev', 1), ('test/stream2-min', 0), ('test/stream2-mean', 9.0), ('test/stream2-max', 9.5), ('test/stream2-count', 10), ('test/stream2-stddev', 1), ('test/stream3-min', 0), ('test/stream3-mean', 11.0), ('test/stream3-max', 11.5), ('test/stream3-count', 11), ('test/stream3-stddev', 2)]) + OrderedDict( + [ + ("time", 1500000000100000000), + ("test/stream0-min", 0), + ("test/stream0-mean", 2.0), + ("test/stream0-max", 2.5), + ("test/stream0-count", 10), + ("test/stream0-stddev", 1), + ("test/stream1-min", 0), + ("test/stream1-mean", 4.0), + ("test/stream1-max", 4.5), + ("test/stream1-count", 11), + ("test/stream1-stddev", 1), + ("test/stream2-min", 0), + ("test/stream2-mean", 6.0), + ("test/stream2-max", 6.5), + ("test/stream2-count", 10), + ("test/stream2-stddev", 1), + ("test/stream3-min", 0), + ("test/stream3-mean", 8.0), + ("test/stream3-max", 8.5), + ("test/stream3-count", 11), + ("test/stream3-stddev", 2), + ] + ), + OrderedDict( + [ + ("time", 1500000000300000000), + ("test/stream0-min", 0), + ("test/stream0-mean", 3.0), + ("test/stream0-max", 3.5), + ("test/stream0-count", 11), + ("test/stream0-stddev", 1), + ("test/stream1-min", 0), + ("test/stream1-mean", 5.0), + ("test/stream1-max", 5.5), + ("test/stream1-count", 10), + ("test/stream1-stddev", 1), + ("test/stream2-min", 0), + ("test/stream2-mean", 7.0), + ("test/stream2-max", 7.5), + ("test/stream2-count", 10), + ("test/stream2-stddev", 1), + ("test/stream3-min", 0), + ("test/stream3-mean", 9.0), + ("test/stream3-max", 9.5), + ("test/stream3-count", 11), + ("test/stream3-stddev", 2), + ] + ), + OrderedDict( + [ + ("time", 1500000000500000000), + ("test/stream0-min", 0), + ("test/stream0-mean", 4.0), + ("test/stream0-max", 4.5), + ("test/stream0-count", 10), + ("test/stream0-stddev", 1), + ("test/stream1-min", 0), + ("test/stream1-mean", 6.0), + ("test/stream1-max", 6.5), + ("test/stream1-count", 11), + ("test/stream1-stddev", 1), + ("test/stream2-min", 0), + ("test/stream2-mean", 8.0), + ("test/stream2-max", 8.5), + ("test/stream2-count", 10), + ("test/stream2-stddev", 1), + ("test/stream3-min", 0), + ("test/stream3-mean", 10.0), + ("test/stream3-max", 10.5), + ("test/stream3-count", 11), + ("test/stream3-stddev", 2), + ] + ), + OrderedDict( + [ + ("time", 1500000000700000000), + ("test/stream0-min", 0), + ("test/stream0-mean", 5.0), + ("test/stream0-max", 5.5), + ("test/stream0-count", 11), + ("test/stream0-stddev", 1), + ("test/stream1-min", 0), + ("test/stream1-mean", 7.0), + ("test/stream1-max", 7.5), + ("test/stream1-count", 10), + ("test/stream1-stddev", 1), + ("test/stream2-min", 0), + ("test/stream2-mean", 9.0), + ("test/stream2-max", 9.5), + ("test/stream2-count", 10), + ("test/stream2-stddev", 1), + ("test/stream3-min", 0), + ("test/stream3-mean", 11.0), + ("test/stream3-max", 11.5), + ("test/stream3-count", 11), + ("test/stream3-stddev", 2), + ] + ), ] - assert to_dict(statpoint_streamset, agg='all') == expected + assert to_dict(statpoint_streamset, agg="all") == expected ########################################################################## ## to_array Tests @@ -205,10 +532,50 @@ def test_to_array_statpoint(self, statpoint_streamset): result = to_array(statpoint_streamset, agg="min") assert result.__class__ == np.ndarray - assert (result[0] == np.array([0.0, 0.0, 0.0, 0.0,])).all() - assert (result[1] == np.array([1.0, 1.0, 1.0, 1.0, ])).all() - assert (result[2] == np.array([2.0, 2.0, 2.0, 2.0, ])).all() - assert (result[3] == np.array([3.0, 3.0, 3.0, 3.0, ])).all() + assert ( + result[0] + == np.array( + [ + 0.0, + 0.0, + 0.0, + 0.0, + ] + ) + ).all() + assert ( + result[1] + == np.array( + [ + 1.0, + 1.0, + 1.0, + 1.0, + ] + ) + ).all() + assert ( + result[2] + == np.array( + [ + 2.0, + 2.0, + 2.0, + 2.0, + ] + ) + ).all() + assert ( + result[3] + == np.array( + [ + 3.0, + 3.0, + 3.0, + 3.0, + ] + ) + ).all() ########################################################################## ## to_series Tests @@ -222,12 +589,12 @@ def test_to_series(self, streamset): expected_names = [s.collection + "/" + s.name for s in streamset] for idx, points in enumerate(streamset.values()): values, times = list(zip(*[(p.value, p.time) for p in points])) - times = Index(times, dtype='datetime64[ns]') + times = Index(times, dtype="datetime64[ns]") expected.append(Series(values, times, name=expected_names[idx])) result = to_series(streamset) for idx in range(4): - assert (result[idx] == expected[idx]).all() # verify data + assert (result[idx] == expected[idx]).all() # verify data assert result[idx].name == expected_names[idx] # verify name def test_to_series_name_lambda(self, streamset): @@ -235,7 +602,7 @@ def test_to_series_name_lambda(self, streamset): assert to_dateframe uses name lambda """ result = streamset.to_series(name_callable=lambda s: s.name) - assert [s.name for s in result] == ['stream0', 'stream1', 'stream2', 'stream3'] + assert [s.name for s in result] == ["stream0", "stream1", "stream2", "stream3"] def test_to_series_ignores_rawpoint_with_agg(self, streamset): """ @@ -250,19 +617,34 @@ def test_to_series_statpoint(self, statpoint_streamset): """ result = to_series(statpoint_streamset) values = [2.0, 4.0, 6.0, 8.0] - index = [1500000000100000000, 1500000000300000000, 1500000000500000000, 1500000000700000000] - times = Index(index, dtype='datetime64[ns]') + index = [ + 1500000000100000000, + 1500000000300000000, + 1500000000500000000, + 1500000000700000000, + ] + times = Index(index, dtype="datetime64[ns]") assert (result[0] == Series(values, index=times)).all() values = [3.0, 5.0, 7.0, 9.0] - index = [1500000000100000000, 1500000000300000000, 1500000000500000000, 1500000000700000000] - times = Index(index, dtype='datetime64[ns]') + index = [ + 1500000000100000000, + 1500000000300000000, + 1500000000500000000, + 1500000000700000000, + ] + times = Index(index, dtype="datetime64[ns]") assert (result[1] == Series(values, index=times)).all() result = to_series(statpoint_streamset, agg="max") values = [2.5, 4.5, 6.5, 8.5] - index = [1500000000100000000, 1500000000300000000, 1500000000500000000, 1500000000700000000] - times = Index(index, dtype='datetime64[ns]') + index = [ + 1500000000100000000, + 1500000000300000000, + 1500000000500000000, + 1500000000700000000, + ] + times = Index(index, dtype="datetime64[ns]") assert (result[0] == Series(values, index=times)).all() def test_to_series_index_type(self, streamset): @@ -271,7 +653,7 @@ def test_to_series_index_type(self, streamset): """ result = to_series(streamset) for series in result: - assert series.index.dtype.name == 'datetime64[ns]' + assert series.index.dtype.name == "datetime64[ns]" def test_to_series_index_as_int(self, streamset): """ @@ -279,7 +661,7 @@ def test_to_series_index_as_int(self, streamset): """ result = to_series(streamset, False) for series in result: - assert series.index.dtype.name == 'int64' + assert series.index.dtype.name == "int64" def test_to_series_raises_on_agg_all(self, statpoint_streamset): """ @@ -296,7 +678,13 @@ def test_to_dataframe(self, streamset): """ assert to_dateframe works on RawPoints """ - columns = ["time", "test/stream0", "test/stream1", "test/stream2", "test/stream3"] + columns = [ + "time", + "test/stream0", + "test/stream1", + "test/stream2", + "test/stream3", + ] df = DataFrame(expected["to_dict"], columns=columns) df.set_index("time", inplace=True) assert to_dataframe(streamset).equals(df) @@ -325,9 +713,14 @@ def test_to_dataframe_multindex(self, statpoint_streamset): """ df = statpoint_streamset.to_dataframe(agg="all") assert len(df.columns.levels) == 3 - assert list(df.columns.levels[0]) == ['test'] - assert list(df.columns.levels[1]) == ['stream0', 'stream1', 'stream2', 'stream3'] - assert list(df.columns.levels[2]) == ['count', 'max', 'mean', 'min', 'stddev'] + assert list(df.columns.levels[0]) == ["test"] + assert list(df.columns.levels[1]) == [ + "stream0", + "stream1", + "stream2", + "stream3", + ] + assert list(df.columns.levels[2]) == ["count", "max", "mean", "min", "stddev"] def test_to_dataframe_name_lambda(self, streamset): """ @@ -345,8 +738,7 @@ def test_to_dataframe_rawpoint_with_agg(self, streamset): assert to_dateframe ignores agg if RawPoint """ df = streamset.to_dataframe(agg="all") - assert df.shape == (10,4) - + assert df.shape == (10, 4) def test_to_dataframe_statpoints(self, statpoint_streamset): """ @@ -403,8 +795,8 @@ def test_to_csv_as_stringio(self, streamset): # convert back to dicts for assert due to newline issues with CSV result = [dict(row) for row in reader] for item in result: - for k,v in item.items(): - if v is '': + for k, v in item.items(): + if v is "": item[k] = None elif "." in v: item[k] = float(v) @@ -466,4 +858,4 @@ def test_to_table_raises_on_agg_all(self, statpoint_streamset): asserts to_table raises error if using "all" as agg. """ with pytest.raises(AttributeError): - to_table(statpoint_streamset, agg="all") \ No newline at end of file + to_table(statpoint_streamset, agg="all") diff --git a/tests/btrdb/utils/test_buffer.py b/tests/btrdb/utils/test_buffer.py index bd5f626..2786d5b 100644 --- a/tests/btrdb/utils/test_buffer.py +++ b/tests/btrdb/utils/test_buffer.py @@ -16,21 +16,21 @@ ########################################################################## import pytest -from btrdb.utils.buffer import PointBuffer + from btrdb.point import RawPoint +from btrdb.utils.buffer import PointBuffer ########################################################################## ## Test Constants ########################################################################## - ########################################################################## ## Initialization Tests ########################################################################## -class TestPointBuffer(object): +class TestPointBuffer(object): def test_earliest(self): """ Assert earliest returns correct key @@ -106,7 +106,6 @@ def test_is_ready(self): buffer.deactivate(1) assert buffer.is_ready(2000) == True - def test_next_key_ready_with_inactive(self): """ Assert next_key_ready returns correct key with inactive stream @@ -118,7 +117,6 @@ def test_next_key_ready_with_inactive(self): buffer.add_point(0, RawPoint(time=2000, value="leopard")) assert buffer.next_key_ready() == 1000 - # assert first key is 500 even though it was exhausted buffer = PointBuffer(3) buffer.add_point(1, RawPoint(time=500, value="leopard")) diff --git a/tests/btrdb/utils/test_conversions.py b/tests/btrdb/utils/test_conversions.py index 8cc52a1..0f3a963 100644 --- a/tests/btrdb/utils/test_conversions.py +++ b/tests/btrdb/utils/test_conversions.py @@ -15,15 +15,14 @@ ## Imports ########################################################################## -import uuid import json -import pytest -import numpy as np - +import uuid from datetime import datetime -from btrdb.utils.conversion import to_uuid -from btrdb.utils.conversion import AnnotationDecoder, AnnotationEncoder +import numpy as np +import pytest + +from btrdb.utils.conversion import AnnotationDecoder, AnnotationEncoder, to_uuid ########################################################################## ## Test Constants @@ -38,48 +37,56 @@ ## Conversion Tests ########################################################################## -class TestAnnotationJSON(object): - @pytest.mark.parametrize("obj, expected", [ - (True, "true"), - (False, "false"), - (None, "null"), - (3.14, "3.14"), - (42, "42"), - ("foo", "foo"), - ("a long walk on the beach", "a long walk on the beach"), - (['a', 'b', 'c'], '["a", "b", "c"]'), - ({'color': 'red', 'foo': 24}, '{"color": "red", "foo": 24}'), - (datetime(2018, 9, 10, 16, 30), "2018-09-10 16:30:00.000000"), - (np.datetime64(datetime(2018, 9, 10, 16, 30)), "2018-09-10 16:30:00.000000+0000"), - (EXAMPLE_UUID, EXAMPLE_UUID_STR), - ]) +class TestAnnotationJSON(object): + @pytest.mark.parametrize( + "obj, expected", + [ + (True, "true"), + (False, "false"), + (None, "null"), + (3.14, "3.14"), + (42, "42"), + ("foo", "foo"), + ("a long walk on the beach", "a long walk on the beach"), + (["a", "b", "c"], '["a", "b", "c"]'), + ({"color": "red", "foo": 24}, '{"color": "red", "foo": 24}'), + (datetime(2018, 9, 10, 16, 30), "2018-09-10 16:30:00.000000"), + ( + np.datetime64(datetime(2018, 9, 10, 16, 30)), + "2018-09-10 16:30:00.000000+0000", + ), + (EXAMPLE_UUID, EXAMPLE_UUID_STR), + ], + ) def test_annotations_encoder(self, obj, expected): msg = f"did not correctly encode type {type(obj)}" assert json.dumps(obj, cls=AnnotationEncoder, indent=None) == expected, msg - @pytest.mark.parametrize("obj, s", [ - (True, "true"), - (False, "false"), - (None, "null"), - (3.14, "3.14"), - (42, "42"), - ("foo", "foo"), - ("foo", '"foo"'), - ("a long walk on the beach", "a long walk on the beach"), - ("a long walk on the beach", '"a long walk on the beach"'), - (['a', 'b', 'c'], '["a", "b", "c"]'), - ({'color': 'red', 'foo': 24}, '{"color": "red", "foo": 24}'), - ("2018-09-10 16:30:00.000000", "2018-09-10 16:30:00.000000"), - ({'uuid': EXAMPLE_UUID_STR}, f'{{"uuid": "{EXAMPLE_UUID_STR}"}}'), - ]) + @pytest.mark.parametrize( + "obj, s", + [ + (True, "true"), + (False, "false"), + (None, "null"), + (3.14, "3.14"), + (42, "42"), + ("foo", "foo"), + ("foo", '"foo"'), + ("a long walk on the beach", "a long walk on the beach"), + ("a long walk on the beach", '"a long walk on the beach"'), + (["a", "b", "c"], '["a", "b", "c"]'), + ({"color": "red", "foo": 24}, '{"color": "red", "foo": 24}'), + ("2018-09-10 16:30:00.000000", "2018-09-10 16:30:00.000000"), + ({"uuid": EXAMPLE_UUID_STR}, f'{{"uuid": "{EXAMPLE_UUID_STR}"}}'), + ], + ) def test_annotations_decoder(self, obj, s): msg = f"did not correctly decode type {type(obj)}" assert json.loads(s, cls=AnnotationDecoder) == obj, msg class TestToUUID(object): - def test_from_bytes(self): """ Assert that `to_uuid` converts from bytes diff --git a/tests/btrdb/utils/test_credentials.py b/tests/btrdb/utils/test_credentials.py index 6bd057f..37b4811 100644 --- a/tests/btrdb/utils/test_credentials.py +++ b/tests/btrdb/utils/test_credentials.py @@ -15,21 +15,20 @@ ## Imports ########################################################################## -import pytest from unittest.mock import patch +import pytest + from btrdb.exceptions import * from btrdb.utils.credentials import * - ########################################################################## ## Tests ########################################################################## class TestLoadCredentials(object): - - @patch('builtins.open') + @patch("builtins.open") def test_raises_err_if_credentials_not_found(self, mock_open): """ Assert CredentialsFileNotFound is raised if credentials.yaml is not found @@ -40,7 +39,6 @@ def test_raises_err_if_credentials_not_found(self, mock_open): class TestLoadProfile(object): - def setup(self): for env in ["BTRDB_ENDPOINTS", "BTRDB_PROFILE", "BTRDB_API_KEY"]: try: @@ -48,14 +46,14 @@ def setup(self): except KeyError: pass - @patch('btrdb.utils.credentials.load_credentials_from_file') + @patch("btrdb.utils.credentials.load_credentials_from_file") def test_raises_err_if_profile_not_found(self, mock_credentials): """ Assert ProfileNotFound is raised if profile is asked for but not found """ mock_credentials.return_value = { "duck": { - "btrdb" : { + "btrdb": { "endpoints": "192.168.1.100:4410", "api_key": "111222333", } @@ -64,7 +62,7 @@ def test_raises_err_if_profile_not_found(self, mock_credentials): with pytest.raises(ProfileNotFound): credentials_by_profile("horse") - @patch('btrdb.utils.credentials.load_credentials_from_file') + @patch("btrdb.utils.credentials.load_credentials_from_file") def test_returns_requested_profile(self, mock_credentials): """ Assert returns correct profile @@ -75,47 +73,47 @@ def test_returns_requested_profile(self, mock_credentials): } assert credentials_by_profile("duck")["endpoints"] == "duck" - @patch('btrdb.utils.credentials.load_credentials_from_file') + @patch("btrdb.utils.credentials.load_credentials_from_file") def test_returns_default_profile(self, mock_credentials): """ Assert default profile is returned """ mock_credentials.return_value = { "duck": { - "btrdb" : { + "btrdb": { "endpoints": "192.168.1.100:4411", "api_key": "111222333", }, - "name": "duck" + "name": "duck", }, "default": { - "btrdb" : { + "btrdb": { "endpoints": "192.168.1.222:4411", "api_key": "333222111", }, - "name": "default" + "name": "default", }, } assert credentials_by_profile()["apikey"] == "333222111" - @patch('btrdb.utils.credentials.load_credentials_from_file') + @patch("btrdb.utils.credentials.load_credentials_from_file") def test_returns_no_default_profile(self, mock_credentials): """ Assert empty credentials are returned if no default profile """ mock_credentials.return_value = { "duck": { - "btrdb" : { + "btrdb": { "endpoints": "192.168.1.100:4411", "api_key": "111222333", }, - "name": "duck" + "name": "duck", }, } assert credentials_by_profile() == {} -class TestCredentials(object): +class TestCredentials(object): def setup(self): for env in ["BTRDB_ENDPOINTS", "BTRDB_PROFILE", "BTRDB_API_KEY"]: try: @@ -123,8 +121,8 @@ def setup(self): except KeyError: pass - @patch('btrdb.utils.credentials.load_credentials_from_file') - @patch('os.path.exists') + @patch("btrdb.utils.credentials.load_credentials_from_file") + @patch("os.path.exists") def test_checks_file_existence(self, mock_exists, mock_load): mock_exists.return_value = False credentials() diff --git a/tests/btrdb/utils/test_general.py b/tests/btrdb/utils/test_general.py index 0f09cf5..a13cf3c 100644 --- a/tests/btrdb/utils/test_general.py +++ b/tests/btrdb/utils/test_general.py @@ -11,41 +11,45 @@ Testing for the btrdb.utils.general module """ +from datetime import timedelta + import pytest -from datetime import timedelta -from btrdb.utils.timez import ns_delta from btrdb.utils.general import pointwidth +from btrdb.utils.timez import ns_delta class TestPointwidth(object): - - @pytest.mark.parametrize("delta, expected", [ - (timedelta(days=365), 54), - (timedelta(days=30), 51), - (timedelta(days=7), 49), - (timedelta(days=1), 46), - (timedelta(hours=4), 43), - (timedelta(minutes=15), 39), - (timedelta(seconds=30), 34), - ]) - + @pytest.mark.parametrize( + "delta, expected", + [ + (timedelta(days=365), 54), + (timedelta(days=30), 51), + (timedelta(days=7), 49), + (timedelta(days=1), 46), + (timedelta(hours=4), 43), + (timedelta(minutes=15), 39), + (timedelta(seconds=30), 34), + ], + ) def test_from_timedelta(self, delta, expected): """ Test getting the closest point width from a timedelta """ assert pointwidth.from_timedelta(delta) == expected - @pytest.mark.parametrize("nsec, expected", [ - (ns_delta(days=365), 54), - (ns_delta(days=30), 51), - (ns_delta(days=7), 49), - (ns_delta(days=1), 46), - (ns_delta(hours=12), 45), - (ns_delta(minutes=30), 40), - (ns_delta(seconds=1), 29), - ]) - + @pytest.mark.parametrize( + "nsec, expected", + [ + (ns_delta(days=365), 54), + (ns_delta(days=30), 51), + (ns_delta(days=7), 49), + (ns_delta(days=1), 46), + (ns_delta(hours=12), 45), + (ns_delta(minutes=30), 40), + (ns_delta(seconds=1), 29), + ], + ) def test_from_nanoseconds(self, nsec, expected): """ Test getting the closest point width from nanoseconds diff --git a/tests/btrdb/utils/test_timez.py b/tests/btrdb/utils/test_timez.py index 45fd3fa..7150fff 100644 --- a/tests/btrdb/utils/test_timez.py +++ b/tests/btrdb/utils/test_timez.py @@ -15,22 +15,27 @@ ## Imports ########################################################################## -import pytz -import pytest import datetime + import numpy as np +import pytest +import pytz from freezegun import freeze_time -from btrdb.utils.timez import (currently_as_ns, datetime_to_ns, ns_to_datetime, - to_nanoseconds, ns_delta) - +from btrdb.utils.timez import ( + currently_as_ns, + datetime_to_ns, + ns_delta, + ns_to_datetime, + to_nanoseconds, +) ########################################################################## ## Initialization Tests ########################################################################## -class TestCurrentlyAsNs(object): +class TestCurrentlyAsNs(object): def test_currently_as_ns(self): """ Assert currently_as_ns returns correct value @@ -41,25 +46,23 @@ def test_currently_as_ns(self): class TestDatetimeToNs(object): - def test_datetime_to_ns_naive(self): """ Assert datetime_to_ns handles naive datetime """ - dt = datetime.datetime(2018,1,1,12) + dt = datetime.datetime(2018, 1, 1, 12) localized = pytz.utc.localize(dt) expected = int(localized.timestamp() * 1e9) assert dt.tzinfo is None assert datetime_to_ns(dt) == expected - def test_datetime_to_ns_aware(self): """ Assert datetime_to_ns handles tz aware datetime """ eastern = pytz.timezone("US/Eastern") - dt = datetime.datetime(2018,1,1,17, tzinfo=eastern) + dt = datetime.datetime(2018, 1, 1, 17, tzinfo=eastern) expected = int(dt.astimezone(pytz.utc).timestamp() * 1e9) assert dt.tzinfo is not None @@ -67,7 +70,6 @@ def test_datetime_to_ns_aware(self): class TestNsToDatetime(object): - def test_ns_to_datetime_is_utc(self): """ Assert ns_to_datetime returns UTC aware datetime @@ -83,18 +85,17 @@ def test_ns_to_datetime_is_correct(self): Assert ns_to_datetime returns correct datetime """ val = 1514808000000000000 - expected = datetime.datetime(2018,1,1,12, tzinfo=pytz.UTC) + expected = datetime.datetime(2018, 1, 1, 12, tzinfo=pytz.UTC) assert ns_to_datetime(val) == expected class TestToNanoseconds(object): - def test_datetime_to_ns_naive(self): """ Assert to_nanoseconds handles naive datetime """ - dt = datetime.datetime(2018,1,1,12) + dt = datetime.datetime(2018, 1, 1, 12) localized = pytz.utc.localize(dt) expected = int(localized.timestamp() * 1e9) @@ -106,7 +107,7 @@ def test_datetime_to_ns_aware(self): Assert to_nanoseconds handles tz aware datetime """ eastern = pytz.timezone("US/Eastern") - dt = datetime.datetime(2018,1,1,17, tzinfo=eastern) + dt = datetime.datetime(2018, 1, 1, 17, tzinfo=eastern) expected = int(dt.astimezone(pytz.utc).timestamp() * 1e9) assert dt.tzinfo is not None @@ -116,7 +117,7 @@ def test_str(self): """ Assert to_nanoseconds handles RFC3339 format """ - dt = datetime.datetime(2018,1,1,12, tzinfo=pytz.utc) + dt = datetime.datetime(2018, 1, 1, 12, tzinfo=pytz.utc) expected = int(dt.timestamp() * 1e9) dt_str = "2018-1-1 12:00:00.0-0000" @@ -124,12 +125,12 @@ def test_str(self): assert to_nanoseconds(dt_str) == expected dt_str = "2018-1-1 7:00:00.0-0500" - dt = datetime.datetime(2018,1,1,12, tzinfo=pytz.timezone("US/Eastern")) + dt = datetime.datetime(2018, 1, 1, 12, tzinfo=pytz.timezone("US/Eastern")) assert dt.tzinfo is not None assert to_nanoseconds(dt_str) == expected dt_str = "2018-01-15 07:32:49" - dt = datetime.datetime(2018,1,15,7,32,49, tzinfo=pytz.utc) + dt = datetime.datetime(2018, 1, 15, 7, 32, 49, tzinfo=pytz.utc) expected = int(dt.timestamp() * 1e9) assert to_nanoseconds(dt_str) == expected @@ -173,20 +174,19 @@ def test_float(self): """ Assert to_nanoseconds handles float """ - dt = datetime.datetime(2018,1,1,12, tzinfo=pytz.utc) + dt = datetime.datetime(2018, 1, 1, 12, tzinfo=pytz.utc) expected = int(dt.timestamp() * 1e9) - dt64 = np.datetime64('2018-01-01T12:00') + dt64 = np.datetime64("2018-01-01T12:00") assert expected == to_nanoseconds(dt64) class TestToNsDelta(object): - def test_ns_delta(self): """ Assert ns_delta converts inputs properly """ - val = ns_delta(1,2,1,3,1,23,1) + val = ns_delta(1, 2, 1, 3, 1, 23, 1) assert val == 93663001023001 def test_ns_delta_precision(self): @@ -200,7 +200,8 @@ def test_returns_int(self): """ Assert ns_delta returns int if floats used for arguments """ - val = ns_delta(1.0,1.0,1.0,1.0,1.0,1.0,1) - assert val == int(1 + 1e3 + 1e6 + 1e9 + (1e9 * 60) + (1e9 * 60 * 60) + \ - (1e9 * 60 * 60 * 24) ) + val = ns_delta(1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1) + assert val == int( + 1 + 1e3 + 1e6 + 1e9 + (1e9 * 60) + (1e9 * 60 * 60) + (1e9 * 60 * 60 * 24) + ) assert isinstance(val, int) diff --git a/tests/btrdb4/test_import.py b/tests/btrdb4/test_import.py index 21c0d6c..4fd170e 100644 --- a/tests/btrdb4/test_import.py +++ b/tests/btrdb4/test_import.py @@ -21,11 +21,11 @@ ## Import Tests ########################################################################## -class TestPackage(object): +class TestPackage(object): def test_import(self): """ Assert that `import brdb4` issues warning exception """ - with pytest.raises(ImportWarning, match='not available'): + with pytest.raises(ImportWarning, match="not available"): import btrdb4 diff --git a/tests/btrdb_integration/__init__.py b/tests/btrdb_integration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/btrdb_integration/conftest.py b/tests/btrdb_integration/conftest.py new file mode 100644 index 0000000..5e3a7f4 --- /dev/null +++ b/tests/btrdb_integration/conftest.py @@ -0,0 +1,68 @@ +import os + +import pyarrow as pa +import pytest + +import btrdb + + +@pytest.fixture +def single_stream_values_arrow_schema(): + schema = pa.schema( + [ + pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False), + pa.field("value", pa.float64(), nullable=False), + ] + ) + return schema + + +@pytest.fixture +def single_stream_windows_all_stats_arrow_schema(): + schema = pa.schema( + [ + pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False), + pa.field("min", pa.float64(), nullable=False), + pa.field("mean", pa.float64(), nullable=False), + pa.field("max", pa.float64(), nullable=False), + pa.field("count", pa.uint64(), nullable=False), + pa.field("stddev", pa.float64(), nullable=False), + ] + ) + return schema + + +def single_stream_windows_mean_stddev_count_stats_arrow_schema(): + schema = ( + pa.schema( + [ + pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False), + pa.field("mean", pa.float64(), nullable=False), + pa.field("count", pa.uint64(), nullable=False), + pa.field("stddev", pa.float64(), nullable=False), + ] + ), + ) + return schema + + +@pytest.fixture +def conn(): + if os.getenv("BTRDB_INTEGRATION_TEST_PROFILE") == None: + pytest.skip("skipping because BTRDB_INTEGRATION_TEST_PROFILE is not set") + conn = btrdb.connect(profile=os.getenv("BTRDB_INTEGRATION_TEST_PROFILE")) + return conn + + +def _delete_collection(conn, col): + streams = conn.streams_in_collection(col) + for stream in streams: + stream.obliterate() + + +@pytest.fixture +def tmp_collection(request, conn): + col = "btrdb_python_integration_tests/" + request.node.name + "/tmp" + _delete_collection(conn, col) + yield col + _delete_collection(conn, col) diff --git a/tests/btrdb_integration/test_conn.py b/tests/btrdb_integration/test_conn.py new file mode 100644 index 0000000..5d30c4f --- /dev/null +++ b/tests/btrdb_integration/test_conn.py @@ -0,0 +1,36 @@ +import logging +from uuid import uuid4 as new_uuid + +import pytest + +import btrdb + + +def test_connection_info(conn): + info = conn.info() + logging.info(f"connection info: {info}") + + +def test_create_stream(conn, tmp_collection): + uuid = new_uuid() + stream = conn.create(uuid, tmp_collection, tags={"name": "s"}) + assert stream.uuid == uuid + assert stream.name == "s" + + +def test_query(conn, tmp_collection): + conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + uuids = conn.query( + "select name from streams where collection = $1 order by name;", + [tmp_collection], + ) + assert len(uuids) == 2 + assert uuids[0]["name"] == "s1" + assert uuids[1]["name"] == "s2" + + +def test_list_collections(conn, tmp_collection): + assert tmp_collection not in conn.list_collections() + stream = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) + assert tmp_collection in conn.list_collections() diff --git a/tests/btrdb_integration/test_regressions.py b/tests/btrdb_integration/test_regressions.py new file mode 100644 index 0000000..db6fd2d --- /dev/null +++ b/tests/btrdb_integration/test_regressions.py @@ -0,0 +1,26 @@ +from uuid import uuid4 as new_uuid + +import pytest + + +@pytest.mark.xfail +def test_create_count_obliterate_concurrency_bug(conn, tmp_collection): + from concurrent.futures import ThreadPoolExecutor + + n_streams = 10 + + def create_stream(i): + return conn.create(new_uuid(), tmp_collection, tags={"name": f"s{i}"}) + + def points_in_stream(s): + return s.count() + + def obliterate_stream(s): + s.obliterate() + + with ThreadPoolExecutor() as executor: + for i in range(2): + pmap = lambda f, it: list(executor.map(f, it, timeout=30)) + streams = pmap(create_stream, range(n_streams)) + assert sum(pmap(points_in_stream, streams)) == 0 + pmap(obliterate_stream, streams) diff --git a/tests/btrdb_integration/test_stream.py b/tests/btrdb_integration/test_stream.py new file mode 100644 index 0000000..e65c71d --- /dev/null +++ b/tests/btrdb_integration/test_stream.py @@ -0,0 +1,348 @@ +import logging +from uuid import uuid4 as new_uuid + +import numpy as np +import pandas as pd +import pytest + +import btrdb.utils.timez +from btrdb.utils.timez import currently_as_ns, ns_delta + +try: + import pyarrow as pa +except ImportError: + pa = None + + +def test_stream_methods(conn, tmp_collection): + s1 = conn.create( + new_uuid(), + tmp_collection, + tags={"name": "s1", "unit": "foo"}, + annotations={"test": "bar"}, + ) + + +def test_grpc_insert_and_values(conn, tmp_collection): + s = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) + t = currently_as_ns() + data = [] + duration = 100 + for i in range(duration): + data.append([t + i, i]) + s.insert(data) + fetched_data = s.values(start=t, end=t + duration) + assert len(fetched_data) == len(data) + for i, (p, _) in enumerate(fetched_data): + assert p.time == data[i][0] + assert p.value == data[i][1] + + +def test_arrow_insert_and_values( + conn, tmp_collection, single_stream_values_arrow_schema +): + s = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) + t = currently_as_ns() + times = [] + values = [] + duration = 100 + for i in range(duration): + times.append(t + i) + values.append(i) + data = pa.Table.from_arrays( + [ + pa.array(times), + pa.array(values), + ], + schema=single_stream_values_arrow_schema, + ) + s.arrow_insert(data) + fetched_data = s.arrow_values(start=t, end=t + duration) + assert data == fetched_data + + +@pytest.mark.parametrize( + "merge_policy,duplicates_expected", + [("never", True), ("equal", True), ("retain", False), ("replace", False)], +) +def test_arrow_insert_duplicate_timestamps( + conn, + tmp_collection, + single_stream_values_arrow_schema, + merge_policy, + duplicates_expected, +): + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + t1 = [100, 105, 110, 110, 120] + d1 = [0.0, 1.0, 2.0, 3.0, 4.0] + s1.insert(list(zip(t1, d1)), merge=merge_policy) + df = s1.arrow_values(start=100, end=121).to_pandas().set_index("time") + assert any(df.index.duplicated().tolist()) == duplicates_expected + + +def test_arrow_values_table_schema( + conn, tmp_collection, single_stream_values_arrow_schema +): + s = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) + t = currently_as_ns() + times = [] + values = [] + duration = 100 + for i in range(duration): + times.append(t + i) + values.append(i) + data = pa.Table.from_arrays( + [ + pa.array(times), + pa.array(values), + ], + schema=single_stream_values_arrow_schema, + ) + s.arrow_insert(data) + fetched_data = s.arrow_values(start=t, end=t + duration) + assert single_stream_values_arrow_schema.equals(fetched_data.schema) + + +def test_arrow_values_table_schema( + conn, tmp_collection, single_stream_values_arrow_schema +): + s = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) + t = currently_as_ns() + times = [] + values = [] + duration = 100 + for i in range(duration): + times.append(t + i) + values.append(i) + data = pa.Table.from_arrays( + [ + pa.array(times), + pa.array(values), + ], + schema=single_stream_values_arrow_schema, + ) + s.arrow_insert(data) + fetched_data = s.arrow_values(start=t, end=t + duration) + assert single_stream_values_arrow_schema.equals(fetched_data.schema) + + +def test_arrow_windows( + conn, + tmp_collection, + single_stream_windows_all_stats_arrow_schema, + single_stream_values_arrow_schema, +): + rng = np.random.default_rng(seed=7) + s = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) + t = currently_as_ns() + times = [] + values = [] + duration = btrdb.utils.timez.ns_delta(minutes=1) + nvals = 100 + for i in range(nvals): + times.append(t + i * duration) + values.append(i + rng.random()) + data = pa.Table.from_arrays( + [ + pa.array(times), + pa.array(values), + ], + schema=single_stream_values_arrow_schema, + ) + # group these 5 at a time to go with the 5 min windows from below + mean_dat = np.mean(np.asarray(values).reshape(-1, 5), axis=1) + min_dat = np.min(np.asarray(values).reshape(-1, 5), axis=1) + max_dat = np.max(np.asarray(values).reshape(-1, 5), axis=1) + count_dat = [np.asarray(values).reshape(-1, 5).shape[1]] * int( + np.asarray(values).shape[0] / 5 + ) + stddev_dat = np.std(np.asarray(values).reshape(-1, 5), axis=1) + time_dat = [t for i, t in enumerate(times) if i % 5 == 0] + window_table = pa.Table.from_arrays( + [time_dat, min_dat, mean_dat, max_dat, count_dat, stddev_dat], + schema=single_stream_windows_all_stats_arrow_schema, + ) + s.arrow_insert(data) + # 5 min windows + fetched_data = s.arrow_windows( + start=t, end=t + duration * nvals + 1, width=duration * 5 + ) + fetched_df = fetched_data.to_pandas() + fetched_df["time"] = fetched_df["time"].astype(int) + window_df = window_table.to_pandas() + window_df["time"] = window_df["time"].astype(int) + assert np.allclose(fetched_df.values, window_df.values, rtol=0, atol=1e-9) + + +@pytest.mark.xfail +def test_aligned_windows_no_flush( + conn, + tmp_collection, + single_stream_windows_all_stats_arrow_schema, + single_stream_values_arrow_schema, +): + rng = np.random.default_rng(seed=7) + s = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) + t = currently_as_ns() + times = [] + values = [] + duration = btrdb.utils.timez.ns_delta(minutes=1) + nvals = 100 + for i in range(nvals): + times.append(t + i * duration) + values.append(i + rng.random()) + data = pa.Table.from_arrays( + [ + pa.array(times), + pa.array(values), + ], + schema=single_stream_values_arrow_schema, + ) + s.arrow_insert(data) + fetched_data = s.arrow_aligned_windows( + start=t, + end=t + duration * nvals, + pointwidth=btrdb.utils.general.pointwidth.from_nanoseconds(duration * 6), + ) + fetched_df = fetched_data.to_pandas() + assert fetched_df["stddev"].isna().sum() == 0 + + +def test_arrow_aligned_windows_vs_aligned_windows( + conn, + tmp_collection, + single_stream_windows_all_stats_arrow_schema, + single_stream_values_arrow_schema, +): + rng = np.random.default_rng(seed=7) + s = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) + t = currently_as_ns() + times = [] + values = [] + duration = btrdb.utils.timez.ns_delta(minutes=1) + nvals = 100 + for i in range(nvals): + times.append(t + i * duration) + values.append(i + rng.random()) + data = pa.Table.from_arrays( + [ + pa.array(times), + pa.array(values), + ], + schema=single_stream_values_arrow_schema, + ) + s.arrow_insert(data) + s.flush() + fetched_data = s.arrow_aligned_windows( + start=t, + end=t + duration * nvals, + pointwidth=btrdb.utils.general.pointwidth.from_nanoseconds(duration * 6), + ) + fetched_df = fetched_data.to_pandas() + fetched_df["time"] = fetched_df["time"].astype(int) + fetched_df["count"] = fetched_df["count"].astype(int) + + other_method = s.aligned_windows( + start=t, + end=t + duration * nvals, + pointwidth=btrdb.utils.general.pointwidth.from_nanoseconds(duration * 6), + ) + other_method_data = [] + for statpt, ver in other_method: + tmp = dict() + tmp["time"] = statpt.time + tmp["min"] = statpt.min + tmp["mean"] = statpt.mean + tmp["max"] = statpt.max + tmp["count"] = statpt.count + tmp["stddev"] = statpt.stddev + other_method_data.append(tmp) + other_method_df = pd.DataFrame(other_method_data) + assert fetched_data.schema.equals(single_stream_windows_all_stats_arrow_schema) + assert fetched_df.equals(other_method_df) + + +def compare_arrow_windows_vs_windows( + conn, + tmp_collection, + single_stream_windows_all_stats_arrow_schema, + single_stream_values_arrow_schema, +): + rng = np.random.default_rng(seed=7) + s = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) + t = currently_as_ns() + times = [] + values = [] + duration = btrdb.utils.timez.ns_delta(minutes=1) + nvals = 100 + for i in range(nvals): + times.append(t + i * duration) + values.append(i + rng.random()) + data = pa.Table.from_arrays( + [ + pa.array(times), + pa.array(values), + ], + schema=single_stream_values_arrow_schema, + ) + s.arrow_insert(data) + fetched_data = s.arrow_windows( + start=t, + end=t + duration * nvals, + width=duration * 5, + ) + fetched_df = fetched_data.to_pandas() + fetched_df["time"] = fetched_df["time"].astype(int) + fetched_df["count"] = fetched_df["count"].astype(int) + + other_method = s.windows(start=t, end=t + duration * nvals, width=duration * 5) + other_method_data = [] + for statpt, ver in other_method: + tmp = dict() + tmp["time"] = statpt.time + tmp["min"] = statpt.min + tmp["mean"] = statpt.mean + tmp["max"] = statpt.max + tmp["count"] = statpt.count + tmp["stddev"] = statpt.stddev + other_method_data.append(tmp) + other_method_df = pd.DataFrame(other_method_data) + assert fetched_data.schema.equals(single_stream_windows_all_stats_arrow_schema) + assert fetched_df.equals(other_method_df) + + +def test_arrow_empty_values(conn, tmp_collection): + s = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) + data = s.arrow_values(start=100, end=1000) + assert len(data["time"]) == 0 + assert len(data["value"]) == 0 + + +def test_arrow_empty_values_schema(conn, tmp_collection): + s = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) + data = s.arrow_values(start=100, end=1000) + schema = pa.schema( + [ + pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False), + pa.field("value", pa.float64(), nullable=False), + ] + ) + assert schema.equals(data.schema) + + +@pytest.mark.xfail +def test_stream_annotation_update(conn, tmp_collection): + # XXX marked as expected failure until someone has time to investigate. + s = conn.create( + new_uuid(), tmp_collection, tags={"name": "s"}, annotations={"foo": "bar"} + ) + annotations1, version1 = s.annotations() + assert version1 == 0 + assert annotations1["foo"] == "bar" + s.update(annotations={"foo": "baz"}) + annotations2, version2 = s.annotations() + assert version2 > version1 + assert annotations2["foo"] == "baz" + s.update(annotations={}, replace=True) + annotations3, _ = s.annotations() + assert len(annotations3) == 0 diff --git a/tests/btrdb_integration/test_streamset.py b/tests/btrdb_integration/test_streamset.py new file mode 100644 index 0000000..a132f1b --- /dev/null +++ b/tests/btrdb_integration/test_streamset.py @@ -0,0 +1,453 @@ +import logging +from math import isnan, nan +from uuid import uuid4 as new_uuid + +import numpy as np +import pytest +from numpy import testing as np_test + +import btrdb +import btrdb.stream +from btrdb.utils.timez import currently_as_ns + +try: + import pyarrow as pa +except ImportError: + pa = None +try: + import pandas as pd +except ImportError: + pd = None +try: + import polars as pl + from polars import testing as pl_test +except ImportError: + pl = None + + +def test_streamset_values(conn, tmp_collection): + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + t1 = [100, 105, 110, 115, 120] + t2 = [101, 106, 110, 114, 119] + d1 = [0.0, 1.0, 2.0, 3.0, 4.0] + d2 = [5.0, 6.0, 7.0, 8.0, 9.0] + s1.insert(list(zip(t1, d1))) + s2.insert(list(zip(t2, d2))) + ss = btrdb.stream.StreamSet([s1, s2]).filter(start=100, end=121) + values = ss.values() + assert len(values) == 2 + assert len(values[0]) == len(t1) + assert len(values[1]) == len(t2) + assert [(p.time, p.value) for p in values[0]] == list(zip(t1, d1)) + assert [(p.time, p.value) for p in values[1]] == list(zip(t2, d2)) + + +def test_streamset_arrow_values(conn, tmp_collection): + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + t1 = [100, 105, 110, 115, 120] + t2 = [101, 106, 110, 114, 119] + d1 = [0.0, 1.0, 2.0, 3.0, 4.0] + d2 = [5.0, 6.0, 7.0, 8.0, 9.0] + s1.insert(list(zip(t1, d1))) + s2.insert(list(zip(t2, d2))) + ss = btrdb.stream.StreamSet([s1, s2]).filter(start=100, end=121) + expected_times = [100, 101, 105, 106, 110, 114, 115, 119, 120] + expected_col1 = [0.0, np.NaN, 1.0, np.NaN, 2.0, np.NaN, 3.0, np.NaN, 4.0] + expected_col2 = [np.NaN, 5.0, np.NaN, 6.0, 7.0, 8.0, np.NaN, 9.0, np.NaN] + expected_schema = pa.schema( + [ + pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False), + pa.field(str(s1.uuid), pa.float64(), nullable=False), + pa.field(str(s2.uuid), pa.float64(), nullable=False), + ] + ) + values = ss.arrow_values() + times = [t.value for t in values["time"]] + col1 = [np.NaN if isnan(v.as_py()) else v.as_py() for v in values[str(s1.uuid)]] + col2 = [np.NaN if isnan(v.as_py()) else v.as_py() for v in values[str(s2.uuid)]] + assert times == expected_times + assert col1 == expected_col1 + assert col2 == expected_col2 + assert expected_schema.equals(values.schema) + + +@pytest.mark.parametrize( + "name_callable", + [(None), (lambda s: str(s.uuid)), (lambda s: s.name + "/" + s.collection)], + ids=["empty", "uu_as_str", "name_collection"], +) +def test_streamset_arrow_windows_vs_windows(conn, tmp_collection, name_callable): + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + s3 = conn.create(new_uuid(), tmp_collection, tags={"name": "s3"}) + t1 = [100, 105, 110, 115, 120] + t2 = [101, 106, 110, 114, 119] + d1 = [0.0, 1.0, 2.0, 3.0, 4.0] + d2 = [5.0, 6.0, 7.0, 8.0, 9.0] + d3 = [1.0, 9.0, 44.0, 8.0, 9.0] + s1.insert(list(zip(t1, d1))) + s2.insert(list(zip(t2, d2))) + s3.insert(list(zip(t2, d3))) + ss = ( + btrdb.stream.StreamSet([s1, s2, s3]) + .filter(start=100, end=121) + .windows(width=btrdb.utils.timez.ns_delta(nanoseconds=10)) + ) + values_arrow = ss.arrow_to_dataframe(name_callable=name_callable) + values_prev = ss.to_dataframe(name_callable=name_callable).convert_dtypes( + dtype_backend="pyarrow" + ) + values_prev.index = pd.DatetimeIndex(values_prev.index, tz="UTC") + col_map = {old_col: old_col + "/mean" for old_col in values_prev.columns} + values_prev = values_prev.rename(columns=col_map) + (values_arrow) + (values_prev) + assert values_arrow.equals(values_prev) + + +def test_streamset_arrow_windows_vs_windows_agg_all(conn, tmp_collection): + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + s3 = conn.create(new_uuid(), tmp_collection, tags={"name": "s3"}) + t1 = [100, 105, 110, 115, 120] + t2 = [101, 106, 110, 114, 119] + d1 = [0.0, 1.0, 2.0, 3.0, 4.0] + d2 = [5.0, 6.0, 7.0, 8.0, 9.0] + d3 = [1.0, 9.0, 44.0, 8.0, 9.0] + s1.insert(list(zip(t1, d1))) + s2.insert(list(zip(t2, d2))) + s3.insert(list(zip(t2, d3))) + ss = ( + btrdb.stream.StreamSet([s1, s2, s3]) + .filter(start=100, end=121) + .windows(width=btrdb.utils.timez.ns_delta(nanoseconds=10)) + ) + values_arrow = ss.arrow_to_dataframe(name_callable=None, agg=["all"]) + values_prev = ss.to_dataframe(name_callable=None, agg="all") + values_prev = values_prev.apply(lambda x: x.astype(str(x.dtype) + "[pyarrow]")) + values_prev = values_prev.apply( + lambda x: x.astype("uint64[pyarrow]") if "count" in x.name else x + ) + values_prev.index = pd.DatetimeIndex(values_prev.index, tz="UTC") + values_prev = values_prev.convert_dtypes(dtype_backend="pyarrow") + new_cols = ["/".join(old_col) for old_col in values_prev.columns] + values_prev.columns = new_cols + assert values_arrow.equals(values_prev) + + with pytest.raises( + AttributeError, + match="cannot provide name_callable when using 'all' as aggregate at this time", + ): + values_prev = ss.to_dataframe(name_callable=lambda x: str(x.uuid), agg="all") + other_arrow_df = ss.arrow_to_dataframe( + agg=["all", "mean", "trash"], name_callable=lambda x: str(x.uuid) + ) + assert ( + len(other_arrow_df.filter(regex="[min,mean,max,count,stddev]").columns) == 3 * 5 + ) + + +@pytest.mark.parametrize( + "name_callable", + [(None), (lambda s: str(s.uuid)), (lambda s: s.name + "/" + s.collection)], + ids=["empty", "uu_as_str", "name_collection"], +) +def test_streamset_arrow_aligned_windows_vs_aligned_windows( + conn, tmp_collection, name_callable +): + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + s3 = conn.create(new_uuid(), tmp_collection, tags={"name": "s3"}) + t1 = [100, 105, 110, 115, 120] + t2 = [101, 106, 110, 114, 119] + d1 = [0.0, 1.0, 2.0, 3.0, 4.0] + d2 = [5.0, 6.0, 7.0, 8.0, 9.0] + d3 = [1.0, 9.0, 44.0, 8.0, 9.0] + s1.insert(list(zip(t1, d1))) + s2.insert(list(zip(t2, d2))) + s3.insert(list(zip(t2, d3))) + ss = ( + btrdb.stream.StreamSet([s1, s2, s3]) + .filter(start=100, end=121) + .windows(width=btrdb.utils.general.pointwidth.from_nanoseconds(10)) + ) + values_arrow = ss.arrow_to_dataframe(name_callable=name_callable) + values_prev = ss.to_dataframe( + name_callable=name_callable + ) # .convert_dtypes(dtype_backend='pyarrow') + values_prev = values_prev.apply(lambda x: x.astype(str(x.dtype) + "[pyarrow]")) + values_prev = values_prev.apply( + lambda x: x.astype("uint64[pyarrow]") if "count" in x.name else x + ) + values_prev.index = pd.DatetimeIndex(values_prev.index, tz="UTC") + col_map = {old_col: old_col + "/mean" for old_col in values_prev.columns} + values_prev = values_prev.rename(columns=col_map) + assert values_arrow.equals(values_prev) + + +def test_streamset_empty_arrow_values(conn, tmp_collection): + s = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) + ss = btrdb.stream.StreamSet([s]).filter(start=100, end=200) + values = ss.arrow_values() + expected_schema = pa.schema( + [ + pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False), + pa.field(str(s.uuid), pa.float64(), nullable=False), + ] + ) + assert [t.value for t in values["time"]] == [] + assert [v for v in values[str(s.uuid)]] == [] + assert expected_schema.equals(values.schema) + + +def test_streamset_to_dataframe(conn, tmp_collection): + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + t1 = [100, 105, 110, 115, 120] + t2 = [101, 106, 110, 114, 119] + d1 = [0.0, 1.0, 2.0, 3.0, 4.0] + d2 = [5.0, 6.0, 7.0, 8.0, 9.0] + s1.insert(list(zip(t1, d1))) + s2.insert(list(zip(t2, d2))) + ss = btrdb.stream.StreamSet([s1, s2]).filter(start=100, end=121) + values = ss.to_dataframe() + expected_times = [100, 101, 105, 106, 110, 114, 115, 119, 120] + expected_col1 = [0.0, np.NaN, 1.0, np.NaN, 2.0, np.NaN, 3.0, np.NaN, 4.0] + expected_col2 = [np.NaN, 5.0, np.NaN, 6.0, 7.0, 8.0, np.NaN, 9.0, np.NaN] + expected_dat = { + tmp_collection + "/s1": expected_col1, + tmp_collection + "/s2": expected_col2, + } + expected_df = pd.DataFrame(expected_dat, index=expected_times) + assert values.equals(expected_df) + + +def test_arrow_streamset_to_dataframe(conn, tmp_collection): + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + t1 = [100, 105, 110, 115, 120] + t2 = [101, 106, 110, 114, 119] + d1 = [0.0, 1.0, 2.0, 3.0, 4.0] + d2 = [5.0, 6.0, 7.0, 8.0, 9.0] + s1.insert(list(zip(t1, d1))) + s2.insert(list(zip(t2, d2))) + ss = btrdb.stream.StreamSet([s1, s2]).filter(start=100, end=121) + values = ss.arrow_to_dataframe() + expected_times = [100, 101, 105, 106, 110, 114, 115, 119, 120] + expected_times = [ + pa.scalar(v, type=pa.timestamp("ns", tz="UTC")).as_py() for v in expected_times + ] + expected_col1 = pa.array( + [0.0, np.NaN, 1.0, np.NaN, 2.0, np.NaN, 3.0, np.NaN, 4.0], mask=[False] * 9 + ) + expected_col2 = pa.array( + [np.NaN, 5.0, np.NaN, 6.0, 7.0, 8.0, np.NaN, 9.0, np.NaN], mask=[False] * 9 + ) + expected_dat = { + "time": expected_times, + tmp_collection + "/s1": expected_col1, + tmp_collection + "/s2": expected_col2, + } + schema = pa.schema( + fields=[ + pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False), + pa.field(tmp_collection + "/s1", type=pa.float64(), nullable=False), + pa.field(tmp_collection + "/s2", type=pa.float64(), nullable=False), + ] + ) + expected_table = pa.Table.from_pydict(expected_dat, schema=schema) + expected_df = expected_table.to_pandas( + timestamp_as_object=False, types_mapper=pd.ArrowDtype + ) + expected_df.set_index("time", inplace=True) + expected_df.index = pd.DatetimeIndex(expected_df.index, tz="UTC") + np_test.assert_array_equal( + values.values.astype(float), expected_df.values.astype(float) + ) + + +def test_arrow_streamset_to_polars(conn, tmp_collection): + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + t1 = [100, 105, 110, 115, 120] + t2 = [101, 106, 110, 114, 119] + d1 = [0.0, 1.0, 2.0, 3.0, 4.0] + d2 = [5.0, 6.0, 7.0, 8.0, 9.0] + s1.insert(list(zip(t1, d1))) + s2.insert(list(zip(t2, d2))) + ss = btrdb.stream.StreamSet([s1, s2]).filter(start=100, end=121) + values = ss.arrow_to_polars() + expected_times = [100, 101, 105, 106, 110, 114, 115, 119, 120] + expected_times = [ + pa.scalar(v, type=pa.timestamp("ns", tz="UTC")).as_py() for v in expected_times + ] + expected_col1 = [0.0, np.NaN, 1.0, np.NaN, 2.0, np.NaN, 3.0, np.NaN, 4.0] + expected_col2 = [np.NaN, 5.0, np.NaN, 6.0, 7.0, 8.0, np.NaN, 9.0, np.NaN] + expected_dat = { + tmp_collection + "/s1": expected_col1, + tmp_collection + "/s2": expected_col2, + } + expected_df = pd.DataFrame( + expected_dat, index=pd.DatetimeIndex(expected_times) + ).reset_index(names="time") + expected_df_pl = pl.from_pandas(expected_df, nan_to_null=False) + pl_test.assert_frame_equal(values, expected_df_pl) + + +@pytest.mark.parametrize( + "name_callable", + [(None), (lambda s: str(s.uuid)), (lambda s: s.name + "/" + s.collection)], + ids=["empty", "uu_as_str", "name_collection"], +) +def test_streamset_arrow_polars_vs_old_to_polars(conn, tmp_collection, name_callable): + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + t1 = [100, 105, 110, 115, 120] + t2 = [101, 106, 110, 114, 119] + d1 = [0.0, 1.0, 2.0, 3.0, 4.0] + d2 = [5.0, 6.0, 7.0, 8.0, 9.0] + s1.insert(list(zip(t1, d1))) + s2.insert(list(zip(t2, d2))) + ss = btrdb.stream.StreamSet([s1, s2]).filter(start=100, end=121) + values_arrow = ss.arrow_to_polars() + values_non_arrow = ss.to_polars() + expected_times = [100, 101, 105, 106, 110, 114, 115, 119, 120] + expected_times = [ + pa.scalar(v, type=pa.timestamp("ns", tz="UTC")).as_py() for v in expected_times + ] + expected_col1 = [0.0, np.NaN, 1.0, np.NaN, 2.0, np.NaN, 3.0, np.NaN, 4.0] + expected_col2 = [np.NaN, 5.0, np.NaN, 6.0, 7.0, 8.0, np.NaN, 9.0, np.NaN] + expected_dat = { + tmp_collection + "/s1": expected_col1, + tmp_collection + "/s2": expected_col2, + } + expected_df = pd.DataFrame( + expected_dat, index=pd.DatetimeIndex(expected_times, tz="UTC") + ).reset_index(names="time") + expected_df_pl = pl.from_pandas(expected_df, nan_to_null=False) + pl_test.assert_frame_equal(values_arrow, expected_df_pl) + pl_test.assert_frame_equal(values_non_arrow, expected_df_pl) + pl_test.assert_frame_equal(values_non_arrow, values_arrow) + + +@pytest.mark.parametrize( + "name_callable", + [(None), (lambda s: str(s.uuid)), (lambda s: s.name + "/" + s.collection)], + ids=["empty", "uu_as_str", "name_collection"], +) +def test_streamset_windows_arrow_polars_vs_old_to_polars( + conn, tmp_collection, name_callable +): + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + s3 = conn.create(new_uuid(), tmp_collection, tags={"name": "s3"}) + t1 = [100, 105, 110, 115, 120] + t2 = [101, 106, 110, 114, 119] + d1 = [0.0, 1.0, 2.0, 3.0, 4.0] + d2 = [5.0, 6.0, 7.0, 8.0, 9.0] + d3 = [1.0, 9.0, 44.0, 8.0, 9.0] + s1.insert(list(zip(t1, d1))) + s2.insert(list(zip(t2, d2))) + s3.insert(list(zip(t2, d3))) + ss = ( + btrdb.stream.StreamSet([s1, s2, s3]) + .filter(start=100, end=121) + .windows(width=btrdb.utils.timez.ns_delta(nanoseconds=10)) + ) + values_arrow_pl = ss.arrow_to_polars(name_callable=name_callable) + values_non_arrow_pl = ss.to_polars(name_callable=name_callable) + new_names = { + old_col: str(old_col) + "/" + "mean" + for old_col in values_non_arrow_pl.select(pl.col(pl.Float64)).columns + } + new_names["time"] = "time" + values_non_arrow_pl = values_non_arrow_pl.rename(mapping=new_names) + assert values_arrow_pl.frame_equal(values_non_arrow_pl) + + +def test_streamset_windows_aggregates_filter(conn, tmp_collection): + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + s3 = conn.create(new_uuid(), tmp_collection, tags={"name": "s3"}) + t1 = [100, 105, 110, 115, 120] + t2 = [101, 106, 110, 114, 119] + d1 = [0.0, 1.0, 2.0, 3.0, 4.0] + d2 = [5.0, 6.0, 7.0, 8.0, 9.0] + d3 = [1.0, 9.0, 44.0, 8.0, 9.0] + s1.insert(list(zip(t1, d1))) + s2.insert(list(zip(t2, d2))) + s3.insert(list(zip(t2, d3))) + ss = ( + btrdb.stream.StreamSet([s1, s2, s3]) + .filter(start=100, end=121) + .windows(width=btrdb.utils.timez.ns_delta(nanoseconds=10)) + ) + values_arrow_df = ss.arrow_to_dataframe(agg=["mean", "stddev"]) + values_non_arrow_df = ss.to_dataframe(agg="all") + values_non_arrow_df.index = pd.DatetimeIndex(values_non_arrow_df.index, tz="UTC") + values_non_arrow_df = values_non_arrow_df.apply( + lambda x: x.astype(str(x.dtype) + "[pyarrow]") + ) + values_non_arrow_df = values_non_arrow_df.apply( + lambda x: x.astype("uint64[pyarrow]") if "count" in x.name else x + ) + new_cols = ["/".join(old_col) for old_col in values_non_arrow_df.columns] + values_non_arrow_df.columns = new_cols + cols_to_drop = [ + col + for col in values_non_arrow_df.columns + if ("count" in col) or ("min" in col) or ("max" in col) + ] + values_non_arrow_df.drop(columns=cols_to_drop, inplace=True) + assert values_arrow_df.equals(values_non_arrow_df) + + +def test_timesnap_backward_extends_range(conn, tmp_collection): + sec = 10**9 + tv1 = [ + [int(0.5 * sec), 0.5], + [2 * sec, 2.0], + ] + tv2 = [ + [int(0.5 * sec) - 1, 0.5], + [2 * sec, 2.0], + ] + tv3 = [ + [1 * sec, 1.0], + [2 * sec, 2.0], + ] + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + s3 = conn.create(new_uuid(), tmp_collection, tags={"name": "s3"}) + s1.insert(tv1) + s2.insert(tv2) + s3.insert(tv3) + ss = btrdb.stream.StreamSet([s1, s2, s3]).filter( + start=1 * sec, end=3 * sec, sampling_frequency=1 + ) + values = ss.arrow_values() + assert [1 * sec, 2 * sec] == [t.value for t in values["time"]] + assert [0.5, 2.0] == [v.as_py() for v in values[str(s1.uuid)]] + assert [np.NaN, 2.0] == [ + np.NaN if isnan(v.as_py()) else v.as_py() for v in values[str(s2.uuid)] + ] + assert [1.0, 2.0] == [v.as_py() for v in values[str(s3.uuid)]] + + +def test_timesnap_forward_restricts_range(conn, tmp_collection): + sec = 10**9 + tv = [ + [1 * sec, 1.0], + [2 * sec, 2.0], + [int(2.75 * sec), 2.75], + ] + s = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) + s.insert(tv) + ss = btrdb.stream.StreamSet([s]).filter(start=1 * sec, sampling_frequency=1) + values = ss.filter(end=int(3.0 * sec)).arrow_values() + assert [1 * sec, 2 * sec] == [t.value for t in values["time"]] + assert [1.0, 2.0] == [v.as_py() for v in values[str(s.uuid)]] + # Same result if skipping past end instead of to end. + assert values == ss.filter(end=int(2.9 * sec)).arrow_values() diff --git a/tests/requirements.txt b/tests/requirements.txt index 534ff57..4a71eb1 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,10 +1,57 @@ -pytest==5.3.2 -pytest-cov==2.8.1 -pytest-flake8==1.1.0 -freezegun==0.3.12 -flake8<5.0.0 - -# dependencies for conversions -numpy==1.20.3 +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --extra=all --output-file=requirements-all.txt --resolver=backtracking pyproject.toml +# +# Pinned versions and dependencies of dependencies were removed after the autogeneration. +certifi + # via + # btrdb (pyproject.toml) +freezegun + # via btrdb +grpcio + # via + # btrdb (pyproject.toml) + # grpcio-tools + # ray +grpcio-tools + # via + # btrdb (pyproject.toml) +numpy + # via + # btrdb + # pandas + # pyarrow + # ray pandas -tabulate==0.8.6 + # via btrdb +polars + # via btrdb +protobuf + # via + # grpcio-tools + # ray +pyarrow + # via btrdb +pytest + # via + # btrdb + # pytest-cov + # pytest-flake8 +pytest-cov + # via btrdb +pytest-flake8 + # via btrdb +pytz + # via + # btrdb (pyproject.toml) + # pandas +pyyaml + # via + # btrdb (pyproject.toml) + # ray +ray + # via btrdb +tabulate + # via btrdb From 2cc7f7410fe0f2e66f65a308cf33ced7b90be8df Mon Sep 17 00:00:00 2001 From: Alexander Young Date: Thu, 20 Jul 2023 16:40:01 -0500 Subject: [PATCH 058/131] Release v5.30.0 --- btrdb/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/btrdb/version.py b/btrdb/version.py index 492de74..e23f223 100644 --- a/btrdb/version.py +++ b/btrdb/version.py @@ -15,7 +15,7 @@ ## Module Info ########################################################################## -__version_info__ = {"major": 5, "minor": 28, "micro": 1, "releaselevel": "final"} +__version_info__ = { 'major': 5, 'minor': 30, 'micro': 0, 'releaselevel': 'final'} ########################################################################## ## Helper Functions From e982316a4c6b52b0c5af88dc46444a3e16342030 Mon Sep 17 00:00:00 2001 From: Justin Gilmer Date: Thu, 20 Jul 2023 17:08:52 -0500 Subject: [PATCH 059/131] Justin gilmer patch 1 (#39) * Update release.yaml * Update pyproject.toml --- .github/workflows/release.yaml | 2 +- pyproject.toml | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 207f62f..3223859 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -70,7 +70,7 @@ jobs: pip install setuptools wheel - name: Build and publish run: | - python setup.py sdist bdist_wheel + pip wheel . - name: Publish package uses: pypa/gh-action-pypi-publish@master with: diff --git a/pyproject.toml b/pyproject.toml index 066ed3b..f7d54bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,10 @@ dependencies = [ "pytz", "pyyaml", "certifi", + "pyarrow", + "polars", + "numpy", + "pandas>=2.0", ] [project.optional-dependencies] From 8aff5e53d1f8f946d2d6d9b4504909a95f4d6026 Mon Sep 17 00:00:00 2001 From: Justin Gilmer Date: Thu, 20 Jul 2023 17:20:45 -0500 Subject: [PATCH 060/131] Update release.yaml --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 3223859..27e85dc 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -70,7 +70,7 @@ jobs: pip install setuptools wheel - name: Build and publish run: | - pip wheel . + pip wheel . -w dist - name: Publish package uses: pypa/gh-action-pypi-publish@master with: From cc465687efdf0e69d10ec70885951db3572f31ce Mon Sep 17 00:00:00 2001 From: Alexander Young Date: Thu, 20 Jul 2023 17:22:14 -0500 Subject: [PATCH 061/131] Change publish to use support version --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 27e85dc..e9a2f89 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -72,7 +72,7 @@ jobs: run: | pip wheel . -w dist - name: Publish package - uses: pypa/gh-action-pypi-publish@master + uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ password: ${{ secrets.PYPI_DEPLOYMENT_TOKEN }} From f597b9c68b97bc5c09116c5fc310a161a45df56e Mon Sep 17 00:00:00 2001 From: Justin Gilmer Date: Thu, 20 Jul 2023 17:33:41 -0500 Subject: [PATCH 062/131] Update release.yaml --- .github/workflows/release.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index e9a2f89..027d682 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -67,10 +67,10 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install setuptools wheel + pip install setuptools wheel build - name: Build and publish run: | - pip wheel . -w dist + python -m build --sdist --wheel --outdir dist/ . - name: Publish package uses: pypa/gh-action-pypi-publish@release/v1 with: From 4cdb7743e6c4bc6b6875881b2ce1da51125793e7 Mon Sep 17 00:00:00 2001 From: Justin Gilmer Date: Thu, 20 Jul 2023 17:44:19 -0500 Subject: [PATCH 063/131] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f7d54bc..6c92415 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "btrdb" -version = "5.28.1" +version = "5.30.1" authors = [ {name="PingThingsIO", email="support@pingthings.io"}, ] From a3a4a659100091022f58af97be25c3abcec38139 Mon Sep 17 00:00:00 2001 From: Alexander Young Date: Thu, 20 Jul 2023 17:45:11 -0500 Subject: [PATCH 064/131] Release v5.30.1 --- btrdb/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/btrdb/version.py b/btrdb/version.py index e23f223..552f48f 100644 --- a/btrdb/version.py +++ b/btrdb/version.py @@ -15,7 +15,7 @@ ## Module Info ########################################################################## -__version_info__ = { 'major': 5, 'minor': 30, 'micro': 0, 'releaselevel': 'final'} +__version_info__ = { 'major': 5, 'minor': 30, 'micro': 1, 'releaselevel': 'final'} ########################################################################## ## Helper Functions From 89aa50dc98256ddba570da3b2675c289ea18c189 Mon Sep 17 00:00:00 2001 From: Justin Gilmer Date: Fri, 21 Jul 2023 13:12:41 -0500 Subject: [PATCH 065/131] Update readthedocs to new yaml for testing. (#40) --- .readthedocs.yaml | 32 ++++++++++++++++++++++++++++++++ docs/requirements.txt | 2 ++ docs/source/conf.py | 3 ++- 3 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 .readthedocs.yaml diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..816f276 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,32 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the OS, Python version and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.10" + # You can also specify other tool versions: + # nodejs: "19" + # rust: "1.64" + # golang: "1.19" + +# Build documentation in the "docs/" directory with Sphinx +sphinx: + configuration: docs/source/conf.py + +# Optionally build your docs in additional formats such as PDF and ePub +formats: + - pdf +# - epub + +# Optional but recommended, declare the Python requirements required +# to build your documentation +# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +python: + install: + - requirements: docs/requirements.txt diff --git a/docs/requirements.txt b/docs/requirements.txt index a3bb1f5..49e71d8 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,5 @@ alabaster>=0.7.12 Sphinx>=1.7 sphinx-rtd-theme +numpydoc +pydata-sphinx-theme diff --git a/docs/source/conf.py b/docs/source/conf.py index a2ce391..ccb0c8b 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -88,7 +88,8 @@ # a list of builtin themes. # # html_theme = "alabaster" -html_theme = "sphinx_rtd_theme" +# html_theme = "sphinx_rtd_theme" +html_theme = "pydata_sphinx_theme" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the From 3235404602049812315e0a9f66a95a17c09185d7 Mon Sep 17 00:00:00 2001 From: Taite Nazifi <135669716+pingt8@users.noreply.github.com> Date: Fri, 21 Jul 2023 11:24:00 -0700 Subject: [PATCH 066/131] fix: patch up stream object type and other bugs (#33) * fix: patch up stream object type and other bugs * fix: resolve depth errors in stream window * fix: resolve remaining test warnings * fix: resolve test imports * chore: add pre-commit install to readme --- README.md | 26 ++--- btrdb/conn.py | 10 +- btrdb/stream.py | 20 +++- tests/btrdb/test_conn.py | 64 ++++++++++-- tests/btrdb/test_stream.py | 140 ++++++++++++++++++-------- tests/btrdb/test_transformers.py | 4 +- tests/btrdb/utils/test_credentials.py | 4 +- 7 files changed, 194 insertions(+), 74 deletions(-) diff --git a/README.md b/README.md index 7a808fe..9a04042 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,10 @@ # BTrDB Bindings for Python -These are BTrDB Bindings for Python allowing you painless and productive access to the Berkeley Tree Database (BTrDB). BTrDB is a time series database focusing on blazing speed with respect to univariate time series data at the nanosecond scale. - +These are BTrDB Bindings for Python allowing you painless and productive access to the Berkeley Tree Database (BTrDB). BTrDB is a time series database focusing on blazing speed with respect to univariate time series data at the nanosecond scale. ## Sample Code -Our goal is to make BTrDB as easy to use as possible, focusing on integration with other tools and the productivity of our users. In keeping with this we continue to add new features such as easy transformation to numpy arrays, pandas Series, etc. See the sample code below and then checkout our [documentation](https://btrdb.readthedocs.io/en/latest/) for more in depth instructions. - +Our goal is to make BTrDB as easy to use as possible, focusing on integration with other tools and the productivity of our users. In keeping with this we continue to add new features such as easy transformation to numpy arrays, pandas Series, etc. See the sample code below and then checkout our [documentation](https://btrdb.readthedocs.io/en/latest/) for more in depth instructions. import btrdb @@ -40,8 +38,6 @@ Our goal is to make BTrDB as easy to use as possible, focusing on integration wi >> StatPoint(1500000000300000000, 4.0, 5.0, 6.0, 3, 0.816496580927726) >> StatPoint(1500000000600000000, 7.0, 8.0, 9.0, 3, 0.816496580927726) - - You can also easily work with a group of streams for when you need to evaluate data across multiple time series or serialize to disk. from btrdb.utils.timez import to_nanoseconds @@ -69,17 +65,15 @@ You can also easily work with a group of streams for when you need to evaluate d 8 1500000000800000000 NaN 9.0 9 1500000000900000000 10.0 NaN - ## Installation -See our documentation on [installing](https://btrdb.readthedocs.io/en/latest/installing.html) the bindings for more detailed instructions. However, to quickly get started using the latest available versions you can use `pip` to install from pypi with `conda` support coming in the near future. +See our documentation on [installing](https://btrdb.readthedocs.io/en/latest/installing.html) the bindings for more detailed instructions. However, to quickly get started using the latest available versions you can use `pip` to install from pypi with `conda` support coming in the near future. $ pip install btrdb - ## Tests -This project includes a suite of automated tests based upon [pytest](https://docs.pytest.org/en/latest/). For your convenience, a `Makefile` has been provided with a target for evaluating the test suite. Use the following command to run the tests. +This project includes a suite of automated tests based upon [pytest](https://docs.pytest.org/en/latest/). For your convenience, a `Makefile` has been provided with a target for evaluating the test suite. Use the following command to run the tests. $ make test @@ -89,13 +83,13 @@ Note that the test suite has additional dependencies that must be installed for ## Releases -This codebase uses github actions to control the release process. To create a new release of the software, run `release.sh` with arguments for the new version as shown below. Make sure you are in the master branch when running this script. +This codebase uses github actions to control the release process. To create a new release of the software, run `release.sh` with arguments for the new version as shown below. Make sure you are in the master branch when running this script. ``` ./release.sh 5 11 4 ``` -This will tag and push the current commit and github actions will run the test suite, build the package, and push it to pypi. If any issues are encountered with the automated tests, the build will fail and you will have a tag with no corresponding release. +This will tag and push the current commit and github actions will run the test suite, build the package, and push it to pypi. If any issues are encountered with the automated tests, the build will fail and you will have a tag with no corresponding release. After a release is created, you can manually edit the release description through github. @@ -111,4 +105,10 @@ Note that the documentation also requires Sphix and other dependencies to succes ## Versioning -This codebases uses a form of [Semantic Versioning](http://semver.org/) to structure version numbers. In general, the major version number will track with the BTrDB codebase to transparently maintain version compatibility. Planned features between major versions will increment the minor version while any special releases (bug fixes, etc.) will increment the patch number. +This codebases uses a form of [Semantic Versioning](http://semver.org/) to structure version numbers. In general, the major version number will track with the BTrDB codebase to transparently maintain version compatibility. Planned features between major versions will increment the minor version while any special releases (bug fixes, etc.) will increment the patch number. + +## Pre-commit hooks + +`pip install pre-commit` and then run +`pre-commit run --all-files` to comb through changes and make sure they are formatted correctly. +`pre-commit install` and then run `git commit` to automatically run the pre-commit hooks on every commit. diff --git a/btrdb/conn.py b/btrdb/conn.py index ccfea5d..09f8721 100644 --- a/btrdb/conn.py +++ b/btrdb/conn.py @@ -21,6 +21,7 @@ import re import uuid as uuidlib from concurrent.futures import ThreadPoolExecutor +from typing import List import certifi import grpc @@ -280,8 +281,7 @@ def streams(self, *identifiers, versions=None, is_collection_prefix=False): if versions and len(versions) != len(identifiers): raise ValueError("number of versions does not match identifiers") - - streams = [] + streams: List[Stream] = [] for ident in identifiers: if isinstance(ident, uuidlib.UUID): streams.append(self.stream_from_uuid(ident)) @@ -302,7 +302,10 @@ def streams(self, *identifiers, versions=None, is_collection_prefix=False): is_collection_prefix=is_collection_prefix, tags={"name": parts[-1]}, ) - if len(found) == 1: + if isinstance(found, Stream): + streams.append(found) + continue + if isinstance(found, list) and len(found) == 1: streams.append(found[0]) continue raise StreamNotFoundError(f"Could not identify stream `{ident}`") @@ -310,7 +313,6 @@ def streams(self, *identifiers, versions=None, is_collection_prefix=False): raise ValueError( f"Could not identify stream based on `{ident}`. Identifier must be UUID or collection/name." ) - obj = StreamSet(streams) if versions: diff --git a/btrdb/stream.py b/btrdb/stream.py index b777bf5..a7a840a 100644 --- a/btrdb/stream.py +++ b/btrdb/stream.py @@ -22,6 +22,7 @@ from collections import deque from collections.abc import Sequence from copy import deepcopy +from typing import List import pyarrow as pa @@ -1438,8 +1439,14 @@ class StreamSetBase(Sequence): A lighweight wrapper around a list of stream objects """ - def __init__(self, streams): - self._streams = streams + def __init__(self, streams: List[Stream]): + self._streams: List[Stream] = [] + for stream in streams: + if not isinstance(stream, Stream): + raise BTRDBTypeError( + f"streams must be of type Stream {stream}, {type(stream)}" + ) + self._streams.append(stream) if len(self._streams) < 1: raise ValueError( f"Trying to create streamset with an empty list of streams {self._streams}." @@ -2214,6 +2221,15 @@ def __getitem__(self, item): return self._streams[item] + def __contains__(self, item): + if isinstance(item, str): + for stream in self._streams: + if str(stream.uuid()) == item: + return True + return False + + return item in self._streams + def __len__(self): return len(self._streams) diff --git a/tests/btrdb/test_conn.py b/tests/btrdb/test_conn.py index 340416d..ed00aa9 100644 --- a/tests/btrdb/test_conn.py +++ b/tests/btrdb/test_conn.py @@ -24,6 +24,54 @@ from btrdb.endpoint import Endpoint from btrdb.exceptions import * from btrdb.grpcinterface import btrdb_pb2 +from btrdb.stream import Stream + +########################################################################## +## Fixtures +########################################################################## + + +@pytest.fixture +def stream1(): + uu = uuidlib.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + stream = Mock(Stream) + stream.version = Mock(return_value=11) + stream.uuid = Mock(return_value=uu) + type(stream).collection = PropertyMock(return_value="fruits/apple") + type(stream).name = PropertyMock(return_value="gala") + stream.tags = Mock(return_value={"name": "gala", "unit": "volts"}) + stream.annotations = Mock(return_value=({"owner": "ABC", "color": "red"}, 11)) + stream._btrdb = Mock() + return stream + + +@pytest.fixture +def stream2(): + uu = uuidlib.UUID("17dbe387-89ea-42b6-864b-f505cdb483f5") + stream = Mock(Stream) + stream.version = Mock(return_value=22) + stream.uuid = Mock(return_value=uu) + type(stream).collection = PropertyMock(return_value="fruits/orange") + type(stream).name = PropertyMock(return_value="blood") + stream.tags = Mock(return_value={"name": "blood", "unit": "amps"}) + stream.annotations = Mock(return_value=({"owner": "ABC", "color": "orange"}, 22)) + stream._btrdb = Mock() + return stream + + +@pytest.fixture +def stream3(): + uu = uuidlib.UUID("17dbe387-89ea-42b6-864b-e2ef0d22a53b") + stream = Mock(Stream) + stream.version = Mock(return_value=33) + stream.uuid = Mock(return_value=uu) + type(stream).collection = PropertyMock(return_value="fruits/banana") + type(stream).name = PropertyMock(return_value="yellow") + stream.tags = Mock(return_value={"name": "yellow", "unit": "watts"}) + stream.annotations = Mock(return_value=({"owner": "ABC", "color": "yellow"}, 33)) + stream._btrdb = Mock() + return stream + ########################################################################## ## Connection Tests @@ -91,7 +139,7 @@ def test_streams_recognizes_uuid(self, mock_func): """ db = BTrDB(None) uuid1 = uuidlib.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") - mock_func.return_value = [1] + mock_func.return_value = Stream(db, uuid1) db.streams(uuid1) mock_func.assert_called_once() @@ -104,7 +152,7 @@ def test_streams_recognizes_uuid_string(self, mock_func): """ db = BTrDB(None) uuid1 = "0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a" - mock_func.return_value = [1] + mock_func.return_value = Stream(db, uuid1) db.streams(uuid1) mock_func.assert_called_once() @@ -117,7 +165,9 @@ def test_streams_handles_path(self, mock_func): """ db = BTrDB(None) ident = "zoo/animal/dog" - mock_func.return_value = [1] + mock_func.return_value = [ + Stream(db, "0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a"), + ] db.streams(ident, "0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") mock_func.assert_called_once() @@ -139,12 +189,10 @@ def test_streams_raises_err(self, mock_func): with pytest.raises(StreamNotFoundError) as exc: db.streams(ident) - mock_func.return_value = [1, 2] - with pytest.raises(StreamNotFoundError) as exc: - db.streams(ident) - # check that does not raise if one returned - mock_func.return_value = [1] + mock_func.return_value = [ + Stream(db, ident), + ] db.streams(ident) def test_streams_raises_valueerror(self): diff --git a/tests/btrdb/test_stream.py b/tests/btrdb/test_stream.py index 39b82e7..4fede74 100644 --- a/tests/btrdb/test_stream.py +++ b/tests/btrdb/test_stream.py @@ -84,11 +84,28 @@ def stream2(): return stream +@pytest.fixture +def stream3(): + uu = uuid.UUID("17dbe387-89ea-42b6-864b-e2ef0d22a53b") + stream = Mock(Stream) + stream.version = Mock(return_value=33) + stream.uuid = Mock(return_value=uu) + stream.nearest = Mock(return_value=(RawPoint(time=30, value=1), 33)) + type(stream).collection = PropertyMock(return_value="fruits/banana") + type(stream).name = PropertyMock(return_value="yellow") + stream.tags = Mock(return_value={"name": "yellow", "unit": "watts"}) + stream.annotations = Mock(return_value=({"owner": "ABC", "color": "yellow"}, 3)) + stream._btrdb = Mock() + stream._btrdb._executor = Mock() + stream._btrdb._ARROW_ENABLED = Mock(return_value=False) + return stream + + @pytest.fixture def arrow_stream3(): uu = uuid.UUID("17dbe387-89ea-42b6-864b-f505cdb483f5") stream = Mock(Stream) - stream.version = Mock(return_value=22) + stream.version = Mock(return_value=33) stream.uuid = Mock(return_value=uu) stream.nearest = Mock(return_value=(RawPoint(time=20, value=1), 22)) type(stream).collection = PropertyMock(return_value="fruits/orange") @@ -111,7 +128,7 @@ def test_create(self): """ Assert we can create the object """ - Stream(None, "FAKE") + Stream(None, "FAKE_UUID") def test_repr_str(self): """ @@ -929,7 +946,7 @@ def test_create(self): """ Assert we can create the object """ - StreamSet([1]) + StreamSet([Stream(None, "1"), Stream(None, "2"), Stream(None, "3")]) @pytest.mark.parametrize( "empty_container", [(tuple()), (dict()), (list()), (set())] @@ -948,12 +965,17 @@ def test_repr(self): """ Assert StreamSet instance repr output """ - data = [11, 22, "dog", "cat"] + data = [ + Stream(None, "11"), + Stream(None, "22"), + Stream(None, "dog"), + Stream(None, "cat"), + ] streams = StreamSet(data) expected = "" assert streams.__repr__() == expected - data = [1] + data = [Stream(None, "1")] streams = StreamSet(data) expected = "" assert streams.__repr__() == expected @@ -962,12 +984,17 @@ def test_str(self): """ Assert StreamSet instance str output """ - data = [11, 22, "dog", "cat"] + data = [ + Stream(None, "11"), + Stream(None, "22"), + Stream(None, "dog"), + Stream(None, "cat"), + ] streams = StreamSet(data) expected = "StreamSet with 4 streams" assert str(streams) == expected - data = [1] + data = [Stream(None, "1")] streams = StreamSet(data) expected = "StreamSet with 1 stream" assert str(streams) == expected @@ -976,7 +1003,12 @@ def test_subscriptable(self): """ Assert StreamSet instance is subscriptable """ - data = [11, 22, "dog", "cat"] + data = [ + Stream(None, "11"), + Stream(None, "22"), + Stream(None, "dog"), + Stream(None, "cat"), + ] streams = StreamSet(data) for index, val in enumerate(data): assert streams[index] == val @@ -985,7 +1017,12 @@ def test_len(self): """ Assert StreamSet instance support len """ - data = [11, 22, "dog", "cat"] + data = [ + Stream(None, "11"), + Stream(None, "22"), + Stream(None, "dog"), + Stream(None, "cat"), + ] streams = StreamSet(data) assert len(streams) == len(data) @@ -993,7 +1030,12 @@ def test_iter(self): """ Assert StreamSet instance support iteration """ - data = [11, 22, "dog", "cat"] + data = [ + Stream(None, "11"), + Stream(None, "22"), + Stream(None, "dog"), + Stream(None, "cat"), + ] streams = StreamSet(data) for index, stream in enumerate(streams): assert data[index] == stream @@ -1002,7 +1044,12 @@ def test_indexing(self): """ Assert StreamSet instance supports indexing """ - data = [11, 22, "dog", "cat"] + data = [ + Stream(None, "11"), + Stream(None, "22"), + Stream(None, "dog"), + Stream(None, "cat"), + ] streams = StreamSet(data) # verify index lookup @@ -1035,19 +1082,24 @@ def test_mapping(self): streams[missing] assert str(missing) in str(e) - def test_contains(self): + def test_contains(self, stream1, stream2, stream3): """ Assert StreamSet instance supports contains """ - data = [11, 22, "dog", "cat"] + data = [stream1, stream2, stream3] streams = StreamSet(data) - assert "dog" in streams + assert "0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a" in streams def test_reverse(self): """ Assert StreamSet instance supports reversal """ - data = [11, 22, "dog", "cat"] + data = [ + Stream(None, "11"), + Stream(None, "22"), + Stream(None, "dog"), + Stream(None, "cat"), + ] streams = StreamSet(data) assert list(reversed(streams)) == list(reversed(data)) @@ -1055,7 +1107,12 @@ def test_to_list(self): """ Assert StreamSet instance cast to list """ - data = [11, 22, "dog", "cat"] + data = [ + Stream(None, "11"), + Stream(None, "22"), + Stream(None, "dog"), + Stream(None, "cat"), + ] streams = StreamSet(data) assert list(streams) == data @@ -1063,17 +1120,17 @@ def test_to_list(self): ## allow_window tests ########################################################################## - def test_allow_window(self): + def test_allow_window(self, stream1, stream2, stream3): """ Assert allow_window returns False if window already requested """ - streams = StreamSet([1, 2, 3]) + streams = StreamSet([stream1, stream2, stream3]) assert streams.allow_window == True - streams.windows(30, 4) + streams.windows(30) assert streams.allow_window == False - streams = StreamSet([1, 2, 3]) + streams = StreamSet([stream1, stream2, stream3]) streams.aligned_windows(30) assert streams.allow_window == False @@ -1102,20 +1159,20 @@ def test_pin_versions_returns_self(self, stream1): result = streams.pin_versions(expected) assert streams is result - def test_pin_versions_with_argument(self): + def test_pin_versions_with_argument(self, stream1, stream2): """ Assert pin_versions uses supplied version numbers """ - streams = StreamSet([1, 2]) + streams = StreamSet([stream1, stream2]) expected = [3, 4] assert streams.pin_versions(expected) == streams assert streams._pinned_versions == expected - def test_pin_versions_with_argument(self): + def test_pin_versions_with_argument(self, stream1, stream2): """ Assert pin_versions uses supplied version numbers """ - streams = StreamSet([1, 2]) + streams = StreamSet([stream1, stream2]) uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") expected = {uu: 42} assert streams.pin_versions(expected) == streams @@ -1132,11 +1189,11 @@ def test_pin_versions_no_argument(self, stream1, stream2): assert streams.pin_versions(expected) == streams assert streams._pinned_versions == expected - def test_pin_versions_raise_on_non_dict(self): + def test_pin_versions_raise_on_non_dict(self, stream1): """ Assert pin_versions raises if versions argument is not dict """ - streams = StreamSet([1]) + streams = StreamSet([stream1]) expected = "INVALID DATA" with pytest.raises(TypeError) as e: @@ -1148,7 +1205,7 @@ def test_pin_versions_raise_on_non_uuid_key(self): Assert pin_versions raises if versions argument dict does not have UUID for keys """ - streams = StreamSet([1]) + streams = StreamSet([Stream(None, "1")]) expected = {"uuid": 42} with pytest.raises(TypeError) as e: @@ -1223,7 +1280,7 @@ def test_current(self, mocked, stream1, stream2): stream2.nearest.assert_called_once_with(15, version=22, backward=True) @patch("btrdb.stream.currently_as_ns") - def test_currently_out_of_range(self, mocked): + def test_currently_out_of_range(self, mocked, stream1, stream2): """ Assert currently raises an exception if it is not filtered """ @@ -1485,7 +1542,8 @@ def test_clone_returns_new_object(self, stream1, stream2): clone = streams.clone() assert id(clone) != id(streams) - assert clone._streams is streams._streams + assert clone._streams is not streams._streams + assert clone._streams == streams._streams ########################################################################## ## windows tests @@ -1508,10 +1566,10 @@ def test_windows_raises_if_not_allowed(self, stream1): Assert windows raises Exception if not allowed """ streams = StreamSet([stream1]) - streams.windows(8, 22) + streams.windows(8) with pytest.raises(Exception) as exc: - streams.windows(10, 20) + streams.windows(10) assert "window operation is already requested" in str(exc).lower() def test_windows_returns_self(self, stream1): @@ -1519,7 +1577,7 @@ def test_windows_returns_self(self, stream1): Assert windows returns self """ streams = StreamSet([stream1]) - result = streams.windows(10, 20) + result = streams.windows(10) assert result is streams def test_windows_stores_values(self, stream1): @@ -1527,7 +1585,7 @@ def test_windows_stores_values(self, stream1): Assert windows stores values """ streams = StreamSet([stream1]) - result = streams.windows(10, 20) + result = streams.windows(10) # assert stores values assert result.width == 10 @@ -1552,13 +1610,11 @@ def test_windows_values_and_calls_to_endpoint(self): s2 = Stream(btrdb=BTrDB(endpoint), uuid=uu2) versions = {uu1: 11, uu2: 12} - start, end, width, depth = 1, 100, 1000, 25 + start, end, width = 1, 100, 1000 streams = StreamSet([s1, s2]) streams.pin_versions(versions) values = ( - streams.filter(start=start, end=end) - .windows(width=width, depth=depth) - .values() + streams.filter(start=start, end=end).windows(width=width, depth=0).values() ) # assert endpoint calls have correct arguments, version @@ -1594,14 +1650,10 @@ def test_windows_rows_and_calls_to_endpoint(self): s2 = Stream(btrdb=BTrDB(endpoint), uuid=uu2) versions = {uu1: 11, uu2: 12} - start, end, width, depth = 1, 100, 1000, 25 + start, end, width = 1, 100, 1000 streams = StreamSet([s1, s2]) streams.pin_versions(versions) - rows = ( - streams.filter(start=start, end=end) - .windows(width=width, depth=depth) - .rows() - ) + rows = streams.filter(start=start, end=end).windows(width=width, depth=0).rows() # assert endpoint calls have correct arguments, version expected = [ @@ -1635,7 +1687,7 @@ def test_aligned_windows_raises_if_not_allowed(self, stream1): Assert that aligned_windows raises Exception if not allowed """ streams = StreamSet([stream1]) - streams.windows(8, 22) + streams.windows(8) with pytest.raises(Exception) as exc: streams.aligned_windows(20) diff --git a/tests/btrdb/test_transformers.py b/tests/btrdb/test_transformers.py index 15003c9..833de69 100644 --- a/tests/btrdb/test_transformers.py +++ b/tests/btrdb/test_transformers.py @@ -23,6 +23,7 @@ import pytest from pandas import DataFrame, Index, Series +from btrdb.conn import BTrDB from btrdb.point import RawPoint, StatPoint from btrdb.stream import Stream, StreamSet from btrdb.transformers import * @@ -211,6 +212,7 @@ def statpoint_streamset(): stream = Mock(Stream) type(stream).collection = PropertyMock(return_value="test") type(stream).name = PropertyMock(return_value="stream{}".format(idx)) + stream._btrdb = Mock() streams.append(stream) obj = StreamSet(streams) @@ -796,7 +798,7 @@ def test_to_csv_as_stringio(self, streamset): result = [dict(row) for row in reader] for item in result: for k, v in item.items(): - if v is "": + if v == "": item[k] = None elif "." in v: item[k] = float(v) diff --git a/tests/btrdb/utils/test_credentials.py b/tests/btrdb/utils/test_credentials.py index 37b4811..7a49018 100644 --- a/tests/btrdb/utils/test_credentials.py +++ b/tests/btrdb/utils/test_credentials.py @@ -39,7 +39,7 @@ def test_raises_err_if_credentials_not_found(self, mock_open): class TestLoadProfile(object): - def setup(self): + def setup_method(self): for env in ["BTRDB_ENDPOINTS", "BTRDB_PROFILE", "BTRDB_API_KEY"]: try: del os.environ[env] @@ -114,7 +114,7 @@ def test_returns_no_default_profile(self, mock_credentials): class TestCredentials(object): - def setup(self): + def setup_method(self): for env in ["BTRDB_ENDPOINTS", "BTRDB_PROFILE", "BTRDB_API_KEY"]: try: del os.environ[env] From 8a3788fd1b02aea0de732e40208f376549148062 Mon Sep 17 00:00:00 2001 From: Justin Gilmer Date: Fri, 21 Jul 2023 15:49:36 -0400 Subject: [PATCH 067/131] Trying to fix readthedocs --- .readthedocs.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 816f276..2095729 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -30,3 +30,5 @@ formats: python: install: - requirements: docs/requirements.txt + - method: pip + path: . From 56f75fe9faa56f9d69acb4ddc0ca34040ca9da78 Mon Sep 17 00:00:00 2001 From: Justin Gilmer Date: Mon, 24 Jul 2023 11:34:34 -0500 Subject: [PATCH 068/131] Converting pandas index takes very long, add in arrow_table. (#41) * Converting pandas index takes very long, add in arrow_table. * Bump pyproject toml version number for pip build. --- btrdb/transformers.py | 7 +++---- pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/btrdb/transformers.py b/btrdb/transformers.py index 4939215..9bcf7b5 100644 --- a/btrdb/transformers.py +++ b/btrdb/transformers.py @@ -224,10 +224,7 @@ def arrow_to_dataframe( tmp = tmp_table.select(["time", *usable_cols]) else: tmp = tmp_table - df = tmp.to_pandas(date_as_object=False, types_mapper=pd.ArrowDtype) - df = df.set_index("time") - df.index = pd.DatetimeIndex(df.index, tz="UTC") - return df + return tmp.to_pandas(date_as_object=False, types_mapper=pd.ArrowDtype) def to_dataframe(streamset, columns=None, agg="mean", name_callable=None): @@ -668,5 +665,7 @@ class StreamSetTransformer(object): to_polars = to_polars arrow_to_polars = arrow_to_polars + arrow_to_arrow_table = arrow_to_arrow_table + to_csv = to_csv to_table = to_table diff --git a/pyproject.toml b/pyproject.toml index 6c92415..f42ddf0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "btrdb" -version = "5.30.1" +version = "5.30.2" authors = [ {name="PingThingsIO", email="support@pingthings.io"}, ] From e7f1db63ba5c6586e43aa819688fccd2a1d9a5c9 Mon Sep 17 00:00:00 2001 From: Alexander Young Date: Mon, 24 Jul 2023 11:51:18 -0500 Subject: [PATCH 069/131] Release v5.30.2 (#42) * Release v5.30.2 * Update btrdb/version.py * small precommit fix. --------- Co-authored-by: Justin Gilmer --- btrdb/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/btrdb/version.py b/btrdb/version.py index 552f48f..e98a93b 100644 --- a/btrdb/version.py +++ b/btrdb/version.py @@ -15,7 +15,7 @@ ## Module Info ########################################################################## -__version_info__ = { 'major': 5, 'minor': 30, 'micro': 1, 'releaselevel': 'final'} +__version_info__ = {"major": 5, "minor": 30, "micro": 2, "releaselevel": "final"} ########################################################################## ## Helper Functions From 0d12d5814c623907d685a230a0897ab618e3989c Mon Sep 17 00:00:00 2001 From: Alexander Young Date: Fri, 25 Aug 2023 16:20:02 -0500 Subject: [PATCH 070/131] Have release script update pyproject.toml file (#48) --- release.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/release.sh b/release.sh index d630e0e..31f849e 100755 --- a/release.sh +++ b/release.sh @@ -37,11 +37,12 @@ echo "Setting version to v$1.$2.$3" VERSION_CODE="__version_info__ = { 'major': $1, 'minor': $2, 'micro': $3, 'releaselevel': 'final'}" sed -i.bak "s/^__version_info__.*$/${VERSION_CODE}/g" btrdb/version.py +sed -i.bak "s/^version.*$/version\ = \"$1.$2.$3\"/g" pyproject.toml git add btrdb/version.py +git add pyproject.toml git commit -m "Release v$1.$2.$3" git tag v$1.$2.$3 git push origin v$1.$2.$3 -sleep 10 git push From be0b757504a82d0e75bb3631d8d67bf0f1455a49 Mon Sep 17 00:00:00 2001 From: Justin Gilmer Date: Fri, 25 Aug 2023 16:20:28 -0500 Subject: [PATCH 071/131] Provide option to sort the arrow tables (#47) * Provide option to sort the arrow tables Previously, the arrow endpoints are not guaranteed to present their data in a sorted order, this PR lets the user set that for single stream calls, and sorts by time for the streamset operations by default. Streamset transformers like streamset.to_dataframe in the old version of the api used a PointBuffer that would sort the values by time before returning to the user. The single stream arrow methods now have a boolean argument `sorted` to let the user specify if they want to sort the returned table on the 'time' column or not. This is False by default. The streamset methods though, will be sorted by default and the user wont be able to switch that. We can change that later if needed. * Only sort window queries. * Update sorted parameter to sort_time. --- btrdb/stream.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/btrdb/stream.py b/btrdb/stream.py index b777bf5..caf95ab 100644 --- a/btrdb/stream.py +++ b/btrdb/stream.py @@ -917,7 +917,7 @@ def arrow_values( end : int or datetime like object The end time in nanoseconds for the range to be queried. (see :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) - version: int + version: int, default: 0 The version of the stream to be queried auto_retry: bool, default: False Whether to retry this request in the event of an error @@ -931,6 +931,7 @@ def arrow_values( Exponential factor by which the backoff increases between retries. Will be ignored if auto_retry is False + Returns ------ pyarrow.Table @@ -1049,6 +1050,7 @@ def arrow_aligned_windows( start: int, end: int, pointwidth: int, + sort_time: bool = False, version: int = 0, auto_retry=False, retries=5, @@ -1081,7 +1083,9 @@ def arrow_aligned_windows( :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) pointwidth : int, required Specify the number of ns between data points (2**pointwidth) - version : int + sort_time : bool, default: False + Should the table be sorted on the 'time' column? + version : int, default: 0 Version of the stream to query auto_retry: bool, default: False Whether to retry this request in the event of an error @@ -1123,7 +1127,10 @@ def arrow_aligned_windows( ) if len(tables) > 0: tabs, ver = zip(*tables) - return pa.concat_tables(tabs) + if sort_time: + return pa.concat_tables(tabs).sort_by("time") + else: + return pa.concat_tables(tabs) else: schema = pa.schema( [ @@ -1217,6 +1224,7 @@ def arrow_windows( start: int, end: int, width: int, + sort_time: bool = False, version: int = 0, auto_retry=False, retries=5, @@ -1235,6 +1243,8 @@ def arrow_windows( :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) width : int, required The number of nanoseconds in each window. + sort_time : bool, default: False + Should the table be sorted on the 'time' column. version : int, default=0, optional The version of the stream to query. auto_retry: bool, default: False @@ -1285,7 +1295,10 @@ def arrow_windows( ) if len(tables) > 0: tabs, ver = zip(*tables) - return pa.concat_tables(tabs) + if sort_time: + return pa.concat_tables(tabs).sort_by("time") + else: + return pa.concat_tables(tabs) else: schema = pa.schema( [ @@ -2094,6 +2107,8 @@ def arrow_values( ): """Return a pyarrow table of stream values based on the streamset parameters. + This data will be sorted by the 'time' column. + Notes ----- This method is available for commercial customers with arrow-enabled servers. @@ -2138,6 +2153,7 @@ def arrow_values( data = tablex else: data = tablex + data = data.sort_by("time") elif self.width is not None and self.depth is not None: # create list of stream.windows data (the windows method should @@ -2168,6 +2184,7 @@ def arrow_values( data = tablex else: data = tablex + data = data.sort_by("time") else: sampling_freq = params.pop("sampling_frequency", 0) period_ns = 0 From 334ed893b125547712e70068d4fe6190046cb710 Mon Sep 17 00:00:00 2001 From: Justin Gilmer Date: Wed, 30 Aug 2023 10:26:18 -0500 Subject: [PATCH 072/131] Remove 4MB limit for gRPC message payloads (#49) Currently our logic chunks data to send over gRPC according to numbers of rows of data, this led to large streamsets in the multivalue api erroring out while trying to recv this data. This PR sets the limit for the client to receive from the server to be unlimited for the time being. This will allow arbitrarly-sized streamsets to have successful multivalue queries. In the future we should update our logic to better handle these size limits when sending from the server, but this is a patch fix for now. --- btrdb/conn.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/btrdb/conn.py b/btrdb/conn.py index ccfea5d..0d712e2 100644 --- a/btrdb/conn.py +++ b/btrdb/conn.py @@ -60,7 +60,10 @@ def __init__(self, addrportstr, apikey=None): """ addrport = addrportstr.split(":", 2) - chan_ops = [] + # 100MB size limit ~ 2500 streams for 5000 points with each point being 64bit + # 500MB size limit ~ 13K streams for 5000 points + # -1 size limit = no limit of size to send + chan_ops = [("grpc.max_receive_message_length", -1)] if len(addrport) != 2: raise ValueError("expecting address:port") From 178379692f54bb28103357d920e6e7b32639e065 Mon Sep 17 00:00:00 2001 From: Justin Gilmer Date: Mon, 11 Sep 2023 11:21:50 -0500 Subject: [PATCH 073/131] Update documentation for arrow methods (#50) * Include documentation for arrow methods * Update copyright * Update broken links * Add Dash minimal example * Add page for arrow queries * Add changelog to main codebase and to api docs * Remove any broken links * Update installation documentation --- CHANGELOG.md | 40 +++++++++ docs/source/api-index.rst | 16 ++++ docs/source/api/conn.rst | 1 + docs/source/api/streams.rst | 2 + docs/source/api/transformers.rst | 1 + docs/source/changelog.rst | 80 +++++++++++++++++ docs/source/concepts.rst | 48 ++++++---- docs/source/conf.py | 4 +- docs/source/explained.rst | 1 - docs/source/index.rst | 25 +----- docs/source/installing.rst | 27 +++--- docs/source/quick-start.rst | 8 ++ docs/source/working.rst | 3 + docs/source/working/arrow-enabled-queries.rst | 32 +++++++ docs/source/working/dash.rst | 89 +++++++++++++++++++ docs/source/working/multistream.rst | 6 ++ docs/source/working/server.rst | 2 +- 17 files changed, 330 insertions(+), 55 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 docs/source/api-index.rst create mode 100644 docs/source/changelog.rst create mode 100644 docs/source/working/arrow-enabled-queries.rst create mode 100644 docs/source/working/dash.rst create mode 100644 docs/source/working/multistream.rst diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..3f6d54f --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,40 @@ +# Changelog +## 5.30.2 +### What's Changed +* Update readthedocs to new yaml for testing. by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/40 +* Converting pandas index takes very long, add in arrow_table. by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/41 + + +**Full Changelog**: https://github.com/PingThingsIO/btrdb-python/compare/v5.30.1...v5.30.2 + +## 5.30.1 +### What's Changed +* Small version bump for pypi release + + +**Full Changelog**: https://github.com/PingThingsIO/btrdb-python/compare/v5.30.0...v5.30.1 + + +## 5.30.0 +### What's Changed +* Merge Arrow support into Main for Release by @youngale-pingthings in https://github.com/PingThingsIO/btrdb-python/pull/37 + * This PR contains many changes that support the commercial Arrow data fetches and inserts + * `arrow_` prefixed methods for `Stream` Objects: + * `insert, aligned_windows, windows, values` + * `arrow_` prefixed methods for StreamSet` objects: + * `insert, values, to_dataframe, to_polars, to_arrow_table, to_numpy, to_dict, to_series` +* Justin gilmer patch 1 by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/39 + + +**Full Changelog**: https://github.com/PingThingsIO/btrdb-python/compare/v5.28.1...v5.30.0 + + +## 5.28.1 +### What's Changed +* Upgrade ray versions by @jleifnf in https://github.com/PingThingsIO/btrdb-python/pull/15 +* Release v5.28.1 and Update Python by @youngale-pingthings in https://github.com/PingThingsIO/btrdb-python/pull/17 + +### New Contributors +* @jleifnf made their first contribution in https://github.com/PingThingsIO/btrdb-python/pull/15 + +**Full Changelog**: https://github.com/PingThingsIO/btrdb-python/compare/v5.15.1...v5.28.1 diff --git a/docs/source/api-index.rst b/docs/source/api-index.rst new file mode 100644 index 0000000..72b478f --- /dev/null +++ b/docs/source/api-index.rst @@ -0,0 +1,16 @@ + +.. _API REF: + +API Reference +------------- + +.. toctree:: + :maxdepth: 2 + + api/package + api/conn + api/streams + api/points + api/exceptions + api/transformers + api/utils-timez diff --git a/docs/source/api/conn.rst b/docs/source/api/conn.rst index c577eaa..62ba065 100644 --- a/docs/source/api/conn.rst +++ b/docs/source/api/conn.rst @@ -1,6 +1,7 @@ btrdb.conn ============= +.. _Conn info: .. automodule:: btrdb.conn .. autoclass:: BTrDB diff --git a/docs/source/api/streams.rst b/docs/source/api/streams.rst index 6c55473..7137d36 100644 --- a/docs/source/api/streams.rst +++ b/docs/source/api/streams.rst @@ -1,10 +1,12 @@ btrdb.stream ============== +.. _StreamGeneralDocs: .. automodule:: btrdb.stream .. autoclass:: Stream :members: +.. _StreamSet API: .. autoclass:: StreamSetBase :members: diff --git a/docs/source/api/transformers.rst b/docs/source/api/transformers.rst index facbbc4..830e0b5 100644 --- a/docs/source/api/transformers.rst +++ b/docs/source/api/transformers.rst @@ -1,3 +1,4 @@ +.. _TransformersDocs: btrdb.transformers ========================= diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst new file mode 100644 index 0000000..8a6881d --- /dev/null +++ b/docs/source/changelog.rst @@ -0,0 +1,80 @@ +.. _changelog: +Changelog +========= + +5.30.2 +------ +What’s Changed +^^^^^^^^^^^^^^ +- Update readthedocs to new yaml for testing. by @justinGilmer in + https://github.com/PingThingsIO/btrdb-python/pull/40 +- Converting pandas index takes very long, fix this in arrow_table. by + @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/41 + +**Full Changelog**: +`5.30.2 `_ + +.. _section-1: + +5.30.1 +------ +.. _whats-changed-1: + +What’s Changed +^^^^^^^^^^^^^^ + +- Small version bump for pypi release + +**Full Changelog**: +`5.30.1 `_ + +.. _section-2: + +5.30.0 +------ +.. _whats-changed-2: + +What’s Changed +^^^^^^^^^^^^^^ + +- Merge Arrow support into Main for Release by @youngale-pingthings in + https://github.com/PingThingsIO/btrdb-python/pull/37 + + - This PR contains many changes that support the commercial Arrow + data fetches and inserts + - ``arrow_`` prefixed methods for ``Stream`` Objects: + + - ``insert, aligned_windows, windows, values`` + + - ``arrow_`` prefixed methods for StreamSet\` objects: + + - ``insert, values, to_dataframe, to_polars, to_arrow_table, to_numpy, to_dict, to_series`` + +- Justin gilmer patch 1 by @justinGilmer in + https://github.com/PingThingsIO/btrdb-python/pull/39 + +**Full Changelog**: +`5.30.0 `_ + +.. _section-3: + +5.28.1 +------ +.. _whats-changed-3: + +What’s Changed +^^^^^^^^^^^^^^ + +- Upgrade ray versions by @jleifnf in + https://github.com/PingThingsIO/btrdb-python/pull/15 +- Release v5.28.1 and Update Python by @youngale-pingthings in + https://github.com/PingThingsIO/btrdb-python/pull/17 + +New Contributors +^^^^^^^^^^^^^^^^ + +- @jleifnf made their first contribution in + https://github.com/PingThingsIO/btrdb-python/pull/15 + +**Full Changelog**: +`5.28.1 `_ diff --git a/docs/source/concepts.rst b/docs/source/concepts.rst index 09c6675..4685556 100644 --- a/docs/source/concepts.rst +++ b/docs/source/concepts.rst @@ -22,6 +22,7 @@ BTrDB focuses on univariate data which opens a host of benefits and is one of the reasons BTrDB is able to process incredibly large amounts of data quickly and easily. +.. _Points Described: Points ------------ Points of data within a time series make up the smallest objects you will be @@ -40,10 +41,10 @@ value within the stream. # view time and value of a single point in the stream point.time - >> 1547241923338098176 + >>> 1547241923338098176 point.value - >> 120.5 + >>> 120.5 StatPoint ^^^^^^^^^^^^ @@ -63,30 +64,37 @@ not need to read the underlying points to return these statistics! # view aggregate values for points in a stream point.time - >> 1547241923338098176 + >>> 1547241923338098176 point.min - >> 42.1 + >>> 42.1 point.mean - >> 78.477 + >>> 78.477 point.max - >> 122.4 + >>> 122.4 point.count - >> 18600 + >>> 18600 point.stddev - >> 3.4 + >>> 3.4 + + +Tabular Data +------------ +In addition to working with the :code:`RawPoint` or :code:`StatPoint` objects, newer versions of the platform now natively support some tabular data formats as well. +This is enabled for commercial customers and are available using the :code:`stream.arrow_` or :code:`streamset.arrow_` methods. +Refer to the :ref:`arrow enabled queries page ` and the :ref:`API docs ` Streams ------------ Streams represent a single series of time/value pairs. As such, the database can hold an almost unlimited amount of individual streams. Each stream has a -`collection` which is similar to a "path" or grouping for multiple streams. Each -steam will also have a `name` as well as a `uuid` which is guaranteed to be unique +:code:`collection` which is similar to a "path" or grouping for multiple streams. Each +steam will also have a :code:`name` as well as a :code:`uuid` which is guaranteed to be unique across streams. BTrDB data is versioned such that changes to a given stream (time series) will @@ -109,26 +117,26 @@ metadata. # retrieve stream's UUID stream.uuid - >> UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + >>> UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") # retrieve stream's current version stream.version() - >> 244 + >>> 244 # retrieve stream tags stream.tags() - >> {'name': 'L1MAG', 'unit': 'volts', 'ingress': ''} + >>> {'name': 'L1MAG', 'unit': 'volts', 'ingress': ''} # retrieve stream annotations stream.annotations() - >> {'poc': 'Salvatore McFesterson', 'region': 'northwest', 'state': 'WA'} + >>> {'poc': 'Salvatore McFesterson', 'region': 'northwest', 'state': 'WA'} # loop through points in the stream for point, _ in stream.values(end=1547241923338098176, version=133): print(point) - >> RawPoint(1500000000100000000, 2.4) - >> RawPoint(1500000000200000000, 2.8) - >> RawPoint(1500000000300000000, 3.6) + >>> RawPoint(1500000000100000000, 2.4) + >>> RawPoint(1500000000200000000, 2.8) + >>> RawPoint(1500000000300000000, 3.6) ... @@ -166,3 +174,9 @@ our API docs to see the full list of features provided to you. 7 1500000000700000000 8.0 NaN 8 1500000000800000000 NaN 9.0 9 1500000000900000000 10.0 NaN + + +Apache-Arrow Accelerated Methods +-------------------------------- + +* Refer to :ref:`Arrow Page` diff --git a/docs/source/conf.py b/docs/source/conf.py index ccb0c8b..1cc1ebf 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -27,7 +27,7 @@ # -- Project information ----------------------------------------------------- project = "btrdb" -copyright = "2021, Ping Things, Inc." +copyright = "2023, Ping Things, Inc." author = "PingThingsIO" # The short X.Y version @@ -104,7 +104,7 @@ "show_related": False, "note_bg": "#FFF59C", "description": "A midweight library to converse with the BTrDB database.", - "extra_nav_links": {"btrdb": "http://btrdb.io"}, + "extra_nav_links": {"btrdb": "http://btrdb-python.readthedocs.io"}, "show_relbars": True, } diff --git a/docs/source/explained.rst b/docs/source/explained.rst index 6ace50e..68bdfe7 100644 --- a/docs/source/explained.rst +++ b/docs/source/explained.rst @@ -126,6 +126,5 @@ The original version of this page can be found at: This page is written based on the following sources: -- `Homepage `_ - `Whitepaper `_ - `Code `_ diff --git a/docs/source/index.rst b/docs/source/index.rst index da1b71e..3f1076b 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -42,7 +42,7 @@ your appetite. import btrdb from btrdb.utils.timez import to_nanoseconds - # establish connection to server + # establish connection to a server conn = btrdb.connect("192.168.1.101:4410") # search for streams and view metadata @@ -75,27 +75,8 @@ in Github! concepts working explained - - -API Reference -------------- - -.. toctree:: - :maxdepth: 2 - - api/package - api/conn - api/streams - api/points - api/exceptions - api/transformers - api/utils-timez - - -Maintainers ------------ - -* :doc:`maintainers/anaconda` + api-index + changelog diff --git a/docs/source/installing.rst b/docs/source/installing.rst index 5b82210..71cab8e 100644 --- a/docs/source/installing.rst +++ b/docs/source/installing.rst @@ -13,13 +13,24 @@ We recommend using pip to install btrdb-python on all platforms: $ pip install btrdb +With :code:`btrdb>=5.30.2`, there are now extra dependencies that can be installed with ``pip``. +We recommend installing the :code:`data` extra dependencies (the second option in the code block below). + + +.. code-block:: shell-session + + $ pip install "btrdb>=5.30.2" # standard btrdb + $ pip install "btrdb[data]>=5.30.2" # btrdb with data science packages included (recommended) + $ pip install "btrdb[all]>=5.30.2" # btrdb with testing, data science and all other optional packages + + To get a specific version of btrdb-python supply the version number. The major -version of this library is pegged to the major version of the BTrDB database as -in the 4.x bindings are best used to speak to a 4.x BTrDB database. +version of this library is tied to the major version of the BTrDB database as +in the 4.X bindings are best used to speak to a 4.X BTrDB database, the 5.X bindings for 5.X platform.. .. code-block:: bash - $ pip install btrdb==4.11.2 + $ pip install "btrdb[data]==5.30.2" To upgrade using pip: @@ -32,12 +43,4 @@ To upgrade using pip: Installing with Anaconda ------------------------ -If you'd like to use Anaconda, you'll need to download the library from the pingthings -channel as shown below. - -Note however that only the version 5 bindings are available in Anaconda Cloud. If you'd -like to install the version 4 bindings you will need to use `pip` as shown above. - -.. code-block:: bash - - $ conda install -c pingthings btrdb +We recommend installing using ``pip``. diff --git a/docs/source/quick-start.rst b/docs/source/quick-start.rst index bc4af5f..8901177 100644 --- a/docs/source/quick-start.rst +++ b/docs/source/quick-start.rst @@ -20,6 +20,14 @@ Connecting to a server is easy with the supplied :code:`connect` function from t # connect with API key conn = btrdb.connect("192.168.1.101:4411", apikey="123456789123456789") +Get Platform Information +^^^^^^^^^^^^^^^^^^^^^^^^ +.. code-block:: python + + conn.info() + +Refer to :ref:`the connection API documentation page. ` + Retrieving a Stream ---------------------- diff --git a/docs/source/working.rst b/docs/source/working.rst index 8773907..cc33d2c 100644 --- a/docs/source/working.rst +++ b/docs/source/working.rst @@ -15,4 +15,7 @@ to interact with the BTrDB database. working/stream-view-data working/streamsets working/multiprocessing + working/multistream + working/arrow-enabled-queries + working/dash working/ray diff --git a/docs/source/working/arrow-enabled-queries.rst b/docs/source/working/arrow-enabled-queries.rst new file mode 100644 index 0000000..8a40a5e --- /dev/null +++ b/docs/source/working/arrow-enabled-queries.rst @@ -0,0 +1,32 @@ +.. -*- mode: rst -*- + +.. _Arrow Page: + +Arrow-enabled Queries +===================== + +In more recent deployments of the BTrDB platform (>=5.30.0), commercial customers also have access to additional accelerated functionality for data fetching and inserting. + +Also, most :code:`StreamSet` based value queries (:code:`AlignedWindows`, :code:`Windows`, :code:`Values`) are multithreaded by default. +This leads to decent performance improvements for fetching and inserting data using the standard :code:`StreamSet` api without any edits by the user. +Refer to :ref:`The StreamSet API ` + +In addition to these improvements to the standard API, commercial customers also have access to additional accelerated data fetching and inserting methods that can dramatically speed up their workflows. + +Apache Arrow Data Format +^^^^^^^^^^^^^^^^^^^^^^^^ + +While keeping our standard API consistent with our :code:`Point` and :code:`StatPoint` :ref:`python object model `, we have also created additional methods that will provide this same type of data, but in a tabular format by default. +Leveraging the `language-agnostic columnar data format Arrow `_, we can transmit our timeseries data in a format that is already optimized for data analytics with well-defined schemas that take the guesswork out of the data types, timezones, etc. +To learn more about these methods, please refer to the :ref:`arrow_ prefixed methods for both Stream and StreamSet objects ` and the :ref:`StreamSet transformer methods `. + +.. _ArrowMultistreamDocs: +True Multistream Support +^^^^^^^^^^^^^^^^^^^^^^^^ + +Until now, there has not been a true multistream query support, our previous api and with the new edits, emulates multistream support with :code:`StreamSet`s and using multithreading. +However, this will still only scale to an amount of streams based on the amount of threads that the python threadpool logic can support. + +Due to this, raw data queries for :code:`StreamSet`s using our arrow api :code:`StreamSet.filter(start=X, end=Y,).arrow_values()` will now perform true multistream queries. +The platform, instead of the python client, will now quickly grab all the stream data for all streams in your streamset, and then package that back to the python client in an :code:`arrow` table! +This leads to data fetch speedups on the order of 10-50x based on the amount and kind of streams. diff --git a/docs/source/working/dash.rst b/docs/source/working/dash.rst new file mode 100644 index 0000000..b6bf10d --- /dev/null +++ b/docs/source/working/dash.rst @@ -0,0 +1,89 @@ +.. -*- mode: rst -*- + +Working with Dash and Plotly +============================ +From `Plotly's getting start guide: `_ "The plotly Python library is an interactive, open-source plotting library that supports over 40 unique chart types covering a wide range of statistical, financial, geographic, scientific, and 3-dimensional use-cases." + +These tools are usable in jupyter notebooks and can also be ran as their own standalone apps using plotly-dash. + + +Below are two examples using the standard API and the Arrow enabled API to retrieve data as a pandas.DataFrame, and then plotting the results. +These examples are based off of the `minimal dash app `_. + +Non-Multistream API +------------------- + +.. code-block:: python + + from dash import Dash, html, dcc, callback, Output, Input + import plotly.express as px + import pandas as pd + import btrdb + + conn = btrdb.connect() + streams = conn.streams_in_collection("YOUR_COLLECTION_HERE") + streamset = conn.streams(*[s.uuid for s in streams]) + latest = streamset.latest() + end = min([pt.time for pt in latest]) + start = end - btrdb.utils.timez.ns_delta(minutes=5) + + df = streamset.filter(start=start, end=end).to_dataframe() + + app = Dash(__name__) + + app.layout = html.Div([ + html.H1(children='Title of Dash App', style={'textAlign':'center'}), + dcc.Dropdown(df.columns, id='dropdown-selection'), + dcc.Graph(id='graph-content') + ]) + + @callback( + Output('graph-content', 'figure'), + Input('dropdown-selection', 'value') + ) + def update_graph(value): + dff = df[value] + return px.line(dff, x=dff.index, y=value) + + if __name__ == '__main__': + app.run(debug=True) + + + +Multistream API +--------------- +.. code-block:: python + + from dash import Dash, html, dcc, callback, Output, Input + import plotly.express as px + import pandas as pd + import btrdb + + conn = btrdb.connect() + streams = conn.streams_in_collection("YOUR_COLLECTION_HERE") + streamset = conn.streams(*[s.uuid for s in streams]) + latest = streamset.latest() + end = min([pt.time for pt in latest]) + start = end - btrdb.utils.timez.ns_delta(minutes=5) + + df = streamset.filter(start=start, end=end).arrow_to_dataframe() + df = df.set_index('time') + + app = Dash(__name__) + + app.layout = html.Div([ + html.H1(children='Title of Dash App', style={'textAlign':'center'}), + dcc.Dropdown(df.columns, id='dropdown-selection'), + dcc.Graph(id='graph-content') + ]) + + @callback( + Output('graph-content', 'figure'), + Input('dropdown-selection', 'value') + ) + def update_graph(value): + dff = df[value] + return px.line(dff, x=dff.index, y=value) + + if __name__ == '__main__': + app.run(debug=True) diff --git a/docs/source/working/multistream.rst b/docs/source/working/multistream.rst new file mode 100644 index 0000000..77548ff --- /dev/null +++ b/docs/source/working/multistream.rst @@ -0,0 +1,6 @@ +.. -*- mode: rst -*- + +Multistream Queries +=================== + +Refer to :ref:`ArrowMultistreamDocs`. diff --git a/docs/source/working/server.rst b/docs/source/working/server.rst index cd479c0..4eaf1b4 100644 --- a/docs/source/working/server.rst +++ b/docs/source/working/server.rst @@ -14,7 +14,7 @@ an API key will raise an exception. Connecting to servers --------------------------- -The btrdb library comes with a high level :code:`connnect` function to interface +The btrdb library comes with a high level :code:`connect` function to interface with a BTrDB server. Upon successfully connecting, you will be returned a :code:`BTrDB` object which is the starting point for all of your server interactions. From 78046dccf827eca62b01d3953758569508cd1a48 Mon Sep 17 00:00:00 2001 From: Justin Gilmer Date: Fri, 15 Sep 2023 12:31:41 -0500 Subject: [PATCH 074/131] Update staging branch with latest `master` changes (#52) --- .github/workflows/release.yaml | 6 +- .readthedocs.yaml | 34 +++++++ CHANGELOG.md | 40 +++++++++ btrdb/conn.py | 6 +- btrdb/stream.py | 25 +++++- btrdb/transformers.py | 7 +- btrdb/version.py | 2 +- docs/requirements.txt | 2 + docs/source/api-index.rst | 16 ++++ docs/source/api/conn.rst | 1 + docs/source/api/streams.rst | 2 + docs/source/api/transformers.rst | 1 + docs/source/changelog.rst | 80 +++++++++++++++++ docs/source/concepts.rst | 48 ++++++---- docs/source/conf.py | 7 +- docs/source/explained.rst | 1 - docs/source/index.rst | 25 +----- docs/source/installing.rst | 27 +++--- docs/source/quick-start.rst | 8 ++ docs/source/working.rst | 3 + docs/source/working/arrow-enabled-queries.rst | 32 +++++++ docs/source/working/dash.rst | 89 +++++++++++++++++++ docs/source/working/multistream.rst | 6 ++ docs/source/working/server.rst | 2 +- pyproject.toml | 6 +- release.sh | 3 +- 26 files changed, 408 insertions(+), 71 deletions(-) create mode 100644 .readthedocs.yaml create mode 100644 CHANGELOG.md create mode 100644 docs/source/api-index.rst create mode 100644 docs/source/changelog.rst create mode 100644 docs/source/working/arrow-enabled-queries.rst create mode 100644 docs/source/working/dash.rst create mode 100644 docs/source/working/multistream.rst diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 207f62f..027d682 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -67,12 +67,12 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install setuptools wheel + pip install setuptools wheel build - name: Build and publish run: | - python setup.py sdist bdist_wheel + python -m build --sdist --wheel --outdir dist/ . - name: Publish package - uses: pypa/gh-action-pypi-publish@master + uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ password: ${{ secrets.PYPI_DEPLOYMENT_TOKEN }} diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..2095729 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,34 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the OS, Python version and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.10" + # You can also specify other tool versions: + # nodejs: "19" + # rust: "1.64" + # golang: "1.19" + +# Build documentation in the "docs/" directory with Sphinx +sphinx: + configuration: docs/source/conf.py + +# Optionally build your docs in additional formats such as PDF and ePub +formats: + - pdf +# - epub + +# Optional but recommended, declare the Python requirements required +# to build your documentation +# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +python: + install: + - requirements: docs/requirements.txt + - method: pip + path: . diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..3f6d54f --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,40 @@ +# Changelog +## 5.30.2 +### What's Changed +* Update readthedocs to new yaml for testing. by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/40 +* Converting pandas index takes very long, add in arrow_table. by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/41 + + +**Full Changelog**: https://github.com/PingThingsIO/btrdb-python/compare/v5.30.1...v5.30.2 + +## 5.30.1 +### What's Changed +* Small version bump for pypi release + + +**Full Changelog**: https://github.com/PingThingsIO/btrdb-python/compare/v5.30.0...v5.30.1 + + +## 5.30.0 +### What's Changed +* Merge Arrow support into Main for Release by @youngale-pingthings in https://github.com/PingThingsIO/btrdb-python/pull/37 + * This PR contains many changes that support the commercial Arrow data fetches and inserts + * `arrow_` prefixed methods for `Stream` Objects: + * `insert, aligned_windows, windows, values` + * `arrow_` prefixed methods for StreamSet` objects: + * `insert, values, to_dataframe, to_polars, to_arrow_table, to_numpy, to_dict, to_series` +* Justin gilmer patch 1 by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/39 + + +**Full Changelog**: https://github.com/PingThingsIO/btrdb-python/compare/v5.28.1...v5.30.0 + + +## 5.28.1 +### What's Changed +* Upgrade ray versions by @jleifnf in https://github.com/PingThingsIO/btrdb-python/pull/15 +* Release v5.28.1 and Update Python by @youngale-pingthings in https://github.com/PingThingsIO/btrdb-python/pull/17 + +### New Contributors +* @jleifnf made their first contribution in https://github.com/PingThingsIO/btrdb-python/pull/15 + +**Full Changelog**: https://github.com/PingThingsIO/btrdb-python/compare/v5.15.1...v5.28.1 diff --git a/btrdb/conn.py b/btrdb/conn.py index 09f8721..a77eced 100644 --- a/btrdb/conn.py +++ b/btrdb/conn.py @@ -61,7 +61,10 @@ def __init__(self, addrportstr, apikey=None): """ addrport = addrportstr.split(":", 2) - chan_ops = [] + # 100MB size limit ~ 2500 streams for 5000 points with each point being 64bit + # 500MB size limit ~ 13K streams for 5000 points + # -1 size limit = no limit of size to send + chan_ops = [("grpc.max_receive_message_length", -1)] if len(addrport) != 2: raise ValueError("expecting address:port") @@ -313,6 +316,7 @@ def streams(self, *identifiers, versions=None, is_collection_prefix=False): raise ValueError( f"Could not identify stream based on `{ident}`. Identifier must be UUID or collection/name." ) + obj = StreamSet(streams) if versions: diff --git a/btrdb/stream.py b/btrdb/stream.py index a7a840a..7a02ac2 100644 --- a/btrdb/stream.py +++ b/btrdb/stream.py @@ -918,7 +918,7 @@ def arrow_values( end : int or datetime like object The end time in nanoseconds for the range to be queried. (see :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) - version: int + version: int, default: 0 The version of the stream to be queried auto_retry: bool, default: False Whether to retry this request in the event of an error @@ -932,6 +932,7 @@ def arrow_values( Exponential factor by which the backoff increases between retries. Will be ignored if auto_retry is False + Returns ------ pyarrow.Table @@ -1050,6 +1051,7 @@ def arrow_aligned_windows( start: int, end: int, pointwidth: int, + sort_time: bool = False, version: int = 0, auto_retry=False, retries=5, @@ -1082,7 +1084,9 @@ def arrow_aligned_windows( :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) pointwidth : int, required Specify the number of ns between data points (2**pointwidth) - version : int + sort_time : bool, default: False + Should the table be sorted on the 'time' column? + version : int, default: 0 Version of the stream to query auto_retry: bool, default: False Whether to retry this request in the event of an error @@ -1124,7 +1128,10 @@ def arrow_aligned_windows( ) if len(tables) > 0: tabs, ver = zip(*tables) - return pa.concat_tables(tabs) + if sort_time: + return pa.concat_tables(tabs).sort_by("time") + else: + return pa.concat_tables(tabs) else: schema = pa.schema( [ @@ -1218,6 +1225,7 @@ def arrow_windows( start: int, end: int, width: int, + sort_time: bool = False, version: int = 0, auto_retry=False, retries=5, @@ -1236,6 +1244,8 @@ def arrow_windows( :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) width : int, required The number of nanoseconds in each window. + sort_time : bool, default: False + Should the table be sorted on the 'time' column. version : int, default=0, optional The version of the stream to query. auto_retry: bool, default: False @@ -1286,7 +1296,10 @@ def arrow_windows( ) if len(tables) > 0: tabs, ver = zip(*tables) - return pa.concat_tables(tabs) + if sort_time: + return pa.concat_tables(tabs).sort_by("time") + else: + return pa.concat_tables(tabs) else: schema = pa.schema( [ @@ -2101,6 +2114,8 @@ def arrow_values( ): """Return a pyarrow table of stream values based on the streamset parameters. + This data will be sorted by the 'time' column. + Notes ----- This method is available for commercial customers with arrow-enabled servers. @@ -2145,6 +2160,7 @@ def arrow_values( data = tablex else: data = tablex + data = data.sort_by("time") elif self.width is not None and self.depth is not None: # create list of stream.windows data (the windows method should @@ -2175,6 +2191,7 @@ def arrow_values( data = tablex else: data = tablex + data = data.sort_by("time") else: sampling_freq = params.pop("sampling_frequency", 0) period_ns = 0 diff --git a/btrdb/transformers.py b/btrdb/transformers.py index 4939215..9bcf7b5 100644 --- a/btrdb/transformers.py +++ b/btrdb/transformers.py @@ -224,10 +224,7 @@ def arrow_to_dataframe( tmp = tmp_table.select(["time", *usable_cols]) else: tmp = tmp_table - df = tmp.to_pandas(date_as_object=False, types_mapper=pd.ArrowDtype) - df = df.set_index("time") - df.index = pd.DatetimeIndex(df.index, tz="UTC") - return df + return tmp.to_pandas(date_as_object=False, types_mapper=pd.ArrowDtype) def to_dataframe(streamset, columns=None, agg="mean", name_callable=None): @@ -668,5 +665,7 @@ class StreamSetTransformer(object): to_polars = to_polars arrow_to_polars = arrow_to_polars + arrow_to_arrow_table = arrow_to_arrow_table + to_csv = to_csv to_table = to_table diff --git a/btrdb/version.py b/btrdb/version.py index 492de74..e98a93b 100644 --- a/btrdb/version.py +++ b/btrdb/version.py @@ -15,7 +15,7 @@ ## Module Info ########################################################################## -__version_info__ = {"major": 5, "minor": 28, "micro": 1, "releaselevel": "final"} +__version_info__ = {"major": 5, "minor": 30, "micro": 2, "releaselevel": "final"} ########################################################################## ## Helper Functions diff --git a/docs/requirements.txt b/docs/requirements.txt index a3bb1f5..49e71d8 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,5 @@ alabaster>=0.7.12 Sphinx>=1.7 sphinx-rtd-theme +numpydoc +pydata-sphinx-theme diff --git a/docs/source/api-index.rst b/docs/source/api-index.rst new file mode 100644 index 0000000..72b478f --- /dev/null +++ b/docs/source/api-index.rst @@ -0,0 +1,16 @@ + +.. _API REF: + +API Reference +------------- + +.. toctree:: + :maxdepth: 2 + + api/package + api/conn + api/streams + api/points + api/exceptions + api/transformers + api/utils-timez diff --git a/docs/source/api/conn.rst b/docs/source/api/conn.rst index c577eaa..62ba065 100644 --- a/docs/source/api/conn.rst +++ b/docs/source/api/conn.rst @@ -1,6 +1,7 @@ btrdb.conn ============= +.. _Conn info: .. automodule:: btrdb.conn .. autoclass:: BTrDB diff --git a/docs/source/api/streams.rst b/docs/source/api/streams.rst index 6c55473..7137d36 100644 --- a/docs/source/api/streams.rst +++ b/docs/source/api/streams.rst @@ -1,10 +1,12 @@ btrdb.stream ============== +.. _StreamGeneralDocs: .. automodule:: btrdb.stream .. autoclass:: Stream :members: +.. _StreamSet API: .. autoclass:: StreamSetBase :members: diff --git a/docs/source/api/transformers.rst b/docs/source/api/transformers.rst index facbbc4..830e0b5 100644 --- a/docs/source/api/transformers.rst +++ b/docs/source/api/transformers.rst @@ -1,3 +1,4 @@ +.. _TransformersDocs: btrdb.transformers ========================= diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst new file mode 100644 index 0000000..8a6881d --- /dev/null +++ b/docs/source/changelog.rst @@ -0,0 +1,80 @@ +.. _changelog: +Changelog +========= + +5.30.2 +------ +What’s Changed +^^^^^^^^^^^^^^ +- Update readthedocs to new yaml for testing. by @justinGilmer in + https://github.com/PingThingsIO/btrdb-python/pull/40 +- Converting pandas index takes very long, fix this in arrow_table. by + @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/41 + +**Full Changelog**: +`5.30.2 `_ + +.. _section-1: + +5.30.1 +------ +.. _whats-changed-1: + +What’s Changed +^^^^^^^^^^^^^^ + +- Small version bump for pypi release + +**Full Changelog**: +`5.30.1 `_ + +.. _section-2: + +5.30.0 +------ +.. _whats-changed-2: + +What’s Changed +^^^^^^^^^^^^^^ + +- Merge Arrow support into Main for Release by @youngale-pingthings in + https://github.com/PingThingsIO/btrdb-python/pull/37 + + - This PR contains many changes that support the commercial Arrow + data fetches and inserts + - ``arrow_`` prefixed methods for ``Stream`` Objects: + + - ``insert, aligned_windows, windows, values`` + + - ``arrow_`` prefixed methods for StreamSet\` objects: + + - ``insert, values, to_dataframe, to_polars, to_arrow_table, to_numpy, to_dict, to_series`` + +- Justin gilmer patch 1 by @justinGilmer in + https://github.com/PingThingsIO/btrdb-python/pull/39 + +**Full Changelog**: +`5.30.0 `_ + +.. _section-3: + +5.28.1 +------ +.. _whats-changed-3: + +What’s Changed +^^^^^^^^^^^^^^ + +- Upgrade ray versions by @jleifnf in + https://github.com/PingThingsIO/btrdb-python/pull/15 +- Release v5.28.1 and Update Python by @youngale-pingthings in + https://github.com/PingThingsIO/btrdb-python/pull/17 + +New Contributors +^^^^^^^^^^^^^^^^ + +- @jleifnf made their first contribution in + https://github.com/PingThingsIO/btrdb-python/pull/15 + +**Full Changelog**: +`5.28.1 `_ diff --git a/docs/source/concepts.rst b/docs/source/concepts.rst index 09c6675..4685556 100644 --- a/docs/source/concepts.rst +++ b/docs/source/concepts.rst @@ -22,6 +22,7 @@ BTrDB focuses on univariate data which opens a host of benefits and is one of the reasons BTrDB is able to process incredibly large amounts of data quickly and easily. +.. _Points Described: Points ------------ Points of data within a time series make up the smallest objects you will be @@ -40,10 +41,10 @@ value within the stream. # view time and value of a single point in the stream point.time - >> 1547241923338098176 + >>> 1547241923338098176 point.value - >> 120.5 + >>> 120.5 StatPoint ^^^^^^^^^^^^ @@ -63,30 +64,37 @@ not need to read the underlying points to return these statistics! # view aggregate values for points in a stream point.time - >> 1547241923338098176 + >>> 1547241923338098176 point.min - >> 42.1 + >>> 42.1 point.mean - >> 78.477 + >>> 78.477 point.max - >> 122.4 + >>> 122.4 point.count - >> 18600 + >>> 18600 point.stddev - >> 3.4 + >>> 3.4 + + +Tabular Data +------------ +In addition to working with the :code:`RawPoint` or :code:`StatPoint` objects, newer versions of the platform now natively support some tabular data formats as well. +This is enabled for commercial customers and are available using the :code:`stream.arrow_` or :code:`streamset.arrow_` methods. +Refer to the :ref:`arrow enabled queries page ` and the :ref:`API docs ` Streams ------------ Streams represent a single series of time/value pairs. As such, the database can hold an almost unlimited amount of individual streams. Each stream has a -`collection` which is similar to a "path" or grouping for multiple streams. Each -steam will also have a `name` as well as a `uuid` which is guaranteed to be unique +:code:`collection` which is similar to a "path" or grouping for multiple streams. Each +steam will also have a :code:`name` as well as a :code:`uuid` which is guaranteed to be unique across streams. BTrDB data is versioned such that changes to a given stream (time series) will @@ -109,26 +117,26 @@ metadata. # retrieve stream's UUID stream.uuid - >> UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + >>> UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") # retrieve stream's current version stream.version() - >> 244 + >>> 244 # retrieve stream tags stream.tags() - >> {'name': 'L1MAG', 'unit': 'volts', 'ingress': ''} + >>> {'name': 'L1MAG', 'unit': 'volts', 'ingress': ''} # retrieve stream annotations stream.annotations() - >> {'poc': 'Salvatore McFesterson', 'region': 'northwest', 'state': 'WA'} + >>> {'poc': 'Salvatore McFesterson', 'region': 'northwest', 'state': 'WA'} # loop through points in the stream for point, _ in stream.values(end=1547241923338098176, version=133): print(point) - >> RawPoint(1500000000100000000, 2.4) - >> RawPoint(1500000000200000000, 2.8) - >> RawPoint(1500000000300000000, 3.6) + >>> RawPoint(1500000000100000000, 2.4) + >>> RawPoint(1500000000200000000, 2.8) + >>> RawPoint(1500000000300000000, 3.6) ... @@ -166,3 +174,9 @@ our API docs to see the full list of features provided to you. 7 1500000000700000000 8.0 NaN 8 1500000000800000000 NaN 9.0 9 1500000000900000000 10.0 NaN + + +Apache-Arrow Accelerated Methods +-------------------------------- + +* Refer to :ref:`Arrow Page` diff --git a/docs/source/conf.py b/docs/source/conf.py index a2ce391..1cc1ebf 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -27,7 +27,7 @@ # -- Project information ----------------------------------------------------- project = "btrdb" -copyright = "2021, Ping Things, Inc." +copyright = "2023, Ping Things, Inc." author = "PingThingsIO" # The short X.Y version @@ -88,7 +88,8 @@ # a list of builtin themes. # # html_theme = "alabaster" -html_theme = "sphinx_rtd_theme" +# html_theme = "sphinx_rtd_theme" +html_theme = "pydata_sphinx_theme" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -103,7 +104,7 @@ "show_related": False, "note_bg": "#FFF59C", "description": "A midweight library to converse with the BTrDB database.", - "extra_nav_links": {"btrdb": "http://btrdb.io"}, + "extra_nav_links": {"btrdb": "http://btrdb-python.readthedocs.io"}, "show_relbars": True, } diff --git a/docs/source/explained.rst b/docs/source/explained.rst index 6ace50e..68bdfe7 100644 --- a/docs/source/explained.rst +++ b/docs/source/explained.rst @@ -126,6 +126,5 @@ The original version of this page can be found at: This page is written based on the following sources: -- `Homepage `_ - `Whitepaper `_ - `Code `_ diff --git a/docs/source/index.rst b/docs/source/index.rst index da1b71e..3f1076b 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -42,7 +42,7 @@ your appetite. import btrdb from btrdb.utils.timez import to_nanoseconds - # establish connection to server + # establish connection to a server conn = btrdb.connect("192.168.1.101:4410") # search for streams and view metadata @@ -75,27 +75,8 @@ in Github! concepts working explained - - -API Reference -------------- - -.. toctree:: - :maxdepth: 2 - - api/package - api/conn - api/streams - api/points - api/exceptions - api/transformers - api/utils-timez - - -Maintainers ------------ - -* :doc:`maintainers/anaconda` + api-index + changelog diff --git a/docs/source/installing.rst b/docs/source/installing.rst index 5b82210..71cab8e 100644 --- a/docs/source/installing.rst +++ b/docs/source/installing.rst @@ -13,13 +13,24 @@ We recommend using pip to install btrdb-python on all platforms: $ pip install btrdb +With :code:`btrdb>=5.30.2`, there are now extra dependencies that can be installed with ``pip``. +We recommend installing the :code:`data` extra dependencies (the second option in the code block below). + + +.. code-block:: shell-session + + $ pip install "btrdb>=5.30.2" # standard btrdb + $ pip install "btrdb[data]>=5.30.2" # btrdb with data science packages included (recommended) + $ pip install "btrdb[all]>=5.30.2" # btrdb with testing, data science and all other optional packages + + To get a specific version of btrdb-python supply the version number. The major -version of this library is pegged to the major version of the BTrDB database as -in the 4.x bindings are best used to speak to a 4.x BTrDB database. +version of this library is tied to the major version of the BTrDB database as +in the 4.X bindings are best used to speak to a 4.X BTrDB database, the 5.X bindings for 5.X platform.. .. code-block:: bash - $ pip install btrdb==4.11.2 + $ pip install "btrdb[data]==5.30.2" To upgrade using pip: @@ -32,12 +43,4 @@ To upgrade using pip: Installing with Anaconda ------------------------ -If you'd like to use Anaconda, you'll need to download the library from the pingthings -channel as shown below. - -Note however that only the version 5 bindings are available in Anaconda Cloud. If you'd -like to install the version 4 bindings you will need to use `pip` as shown above. - -.. code-block:: bash - - $ conda install -c pingthings btrdb +We recommend installing using ``pip``. diff --git a/docs/source/quick-start.rst b/docs/source/quick-start.rst index bc4af5f..8901177 100644 --- a/docs/source/quick-start.rst +++ b/docs/source/quick-start.rst @@ -20,6 +20,14 @@ Connecting to a server is easy with the supplied :code:`connect` function from t # connect with API key conn = btrdb.connect("192.168.1.101:4411", apikey="123456789123456789") +Get Platform Information +^^^^^^^^^^^^^^^^^^^^^^^^ +.. code-block:: python + + conn.info() + +Refer to :ref:`the connection API documentation page. ` + Retrieving a Stream ---------------------- diff --git a/docs/source/working.rst b/docs/source/working.rst index 8773907..cc33d2c 100644 --- a/docs/source/working.rst +++ b/docs/source/working.rst @@ -15,4 +15,7 @@ to interact with the BTrDB database. working/stream-view-data working/streamsets working/multiprocessing + working/multistream + working/arrow-enabled-queries + working/dash working/ray diff --git a/docs/source/working/arrow-enabled-queries.rst b/docs/source/working/arrow-enabled-queries.rst new file mode 100644 index 0000000..8a40a5e --- /dev/null +++ b/docs/source/working/arrow-enabled-queries.rst @@ -0,0 +1,32 @@ +.. -*- mode: rst -*- + +.. _Arrow Page: + +Arrow-enabled Queries +===================== + +In more recent deployments of the BTrDB platform (>=5.30.0), commercial customers also have access to additional accelerated functionality for data fetching and inserting. + +Also, most :code:`StreamSet` based value queries (:code:`AlignedWindows`, :code:`Windows`, :code:`Values`) are multithreaded by default. +This leads to decent performance improvements for fetching and inserting data using the standard :code:`StreamSet` api without any edits by the user. +Refer to :ref:`The StreamSet API ` + +In addition to these improvements to the standard API, commercial customers also have access to additional accelerated data fetching and inserting methods that can dramatically speed up their workflows. + +Apache Arrow Data Format +^^^^^^^^^^^^^^^^^^^^^^^^ + +While keeping our standard API consistent with our :code:`Point` and :code:`StatPoint` :ref:`python object model `, we have also created additional methods that will provide this same type of data, but in a tabular format by default. +Leveraging the `language-agnostic columnar data format Arrow `_, we can transmit our timeseries data in a format that is already optimized for data analytics with well-defined schemas that take the guesswork out of the data types, timezones, etc. +To learn more about these methods, please refer to the :ref:`arrow_ prefixed methods for both Stream and StreamSet objects ` and the :ref:`StreamSet transformer methods `. + +.. _ArrowMultistreamDocs: +True Multistream Support +^^^^^^^^^^^^^^^^^^^^^^^^ + +Until now, there has not been a true multistream query support, our previous api and with the new edits, emulates multistream support with :code:`StreamSet`s and using multithreading. +However, this will still only scale to an amount of streams based on the amount of threads that the python threadpool logic can support. + +Due to this, raw data queries for :code:`StreamSet`s using our arrow api :code:`StreamSet.filter(start=X, end=Y,).arrow_values()` will now perform true multistream queries. +The platform, instead of the python client, will now quickly grab all the stream data for all streams in your streamset, and then package that back to the python client in an :code:`arrow` table! +This leads to data fetch speedups on the order of 10-50x based on the amount and kind of streams. diff --git a/docs/source/working/dash.rst b/docs/source/working/dash.rst new file mode 100644 index 0000000..b6bf10d --- /dev/null +++ b/docs/source/working/dash.rst @@ -0,0 +1,89 @@ +.. -*- mode: rst -*- + +Working with Dash and Plotly +============================ +From `Plotly's getting start guide: `_ "The plotly Python library is an interactive, open-source plotting library that supports over 40 unique chart types covering a wide range of statistical, financial, geographic, scientific, and 3-dimensional use-cases." + +These tools are usable in jupyter notebooks and can also be ran as their own standalone apps using plotly-dash. + + +Below are two examples using the standard API and the Arrow enabled API to retrieve data as a pandas.DataFrame, and then plotting the results. +These examples are based off of the `minimal dash app `_. + +Non-Multistream API +------------------- + +.. code-block:: python + + from dash import Dash, html, dcc, callback, Output, Input + import plotly.express as px + import pandas as pd + import btrdb + + conn = btrdb.connect() + streams = conn.streams_in_collection("YOUR_COLLECTION_HERE") + streamset = conn.streams(*[s.uuid for s in streams]) + latest = streamset.latest() + end = min([pt.time for pt in latest]) + start = end - btrdb.utils.timez.ns_delta(minutes=5) + + df = streamset.filter(start=start, end=end).to_dataframe() + + app = Dash(__name__) + + app.layout = html.Div([ + html.H1(children='Title of Dash App', style={'textAlign':'center'}), + dcc.Dropdown(df.columns, id='dropdown-selection'), + dcc.Graph(id='graph-content') + ]) + + @callback( + Output('graph-content', 'figure'), + Input('dropdown-selection', 'value') + ) + def update_graph(value): + dff = df[value] + return px.line(dff, x=dff.index, y=value) + + if __name__ == '__main__': + app.run(debug=True) + + + +Multistream API +--------------- +.. code-block:: python + + from dash import Dash, html, dcc, callback, Output, Input + import plotly.express as px + import pandas as pd + import btrdb + + conn = btrdb.connect() + streams = conn.streams_in_collection("YOUR_COLLECTION_HERE") + streamset = conn.streams(*[s.uuid for s in streams]) + latest = streamset.latest() + end = min([pt.time for pt in latest]) + start = end - btrdb.utils.timez.ns_delta(minutes=5) + + df = streamset.filter(start=start, end=end).arrow_to_dataframe() + df = df.set_index('time') + + app = Dash(__name__) + + app.layout = html.Div([ + html.H1(children='Title of Dash App', style={'textAlign':'center'}), + dcc.Dropdown(df.columns, id='dropdown-selection'), + dcc.Graph(id='graph-content') + ]) + + @callback( + Output('graph-content', 'figure'), + Input('dropdown-selection', 'value') + ) + def update_graph(value): + dff = df[value] + return px.line(dff, x=dff.index, y=value) + + if __name__ == '__main__': + app.run(debug=True) diff --git a/docs/source/working/multistream.rst b/docs/source/working/multistream.rst new file mode 100644 index 0000000..77548ff --- /dev/null +++ b/docs/source/working/multistream.rst @@ -0,0 +1,6 @@ +.. -*- mode: rst -*- + +Multistream Queries +=================== + +Refer to :ref:`ArrowMultistreamDocs`. diff --git a/docs/source/working/server.rst b/docs/source/working/server.rst index cd479c0..4eaf1b4 100644 --- a/docs/source/working/server.rst +++ b/docs/source/working/server.rst @@ -14,7 +14,7 @@ an API key will raise an exception. Connecting to servers --------------------------- -The btrdb library comes with a high level :code:`connnect` function to interface +The btrdb library comes with a high level :code:`connect` function to interface with a BTrDB server. Upon successfully connecting, you will be returned a :code:`BTrDB` object which is the starting point for all of your server interactions. diff --git a/pyproject.toml b/pyproject.toml index 066ed3b..f42ddf0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "btrdb" -version = "5.28.1" +version = "5.30.2" authors = [ {name="PingThingsIO", email="support@pingthings.io"}, ] @@ -33,6 +33,10 @@ dependencies = [ "pytz", "pyyaml", "certifi", + "pyarrow", + "polars", + "numpy", + "pandas>=2.0", ] [project.optional-dependencies] diff --git a/release.sh b/release.sh index d630e0e..31f849e 100755 --- a/release.sh +++ b/release.sh @@ -37,11 +37,12 @@ echo "Setting version to v$1.$2.$3" VERSION_CODE="__version_info__ = { 'major': $1, 'minor': $2, 'micro': $3, 'releaselevel': 'final'}" sed -i.bak "s/^__version_info__.*$/${VERSION_CODE}/g" btrdb/version.py +sed -i.bak "s/^version.*$/version\ = \"$1.$2.$3\"/g" pyproject.toml git add btrdb/version.py +git add pyproject.toml git commit -m "Release v$1.$2.$3" git tag v$1.$2.$3 git push origin v$1.$2.$3 -sleep 10 git push From 3ea962cca2dd16338dc7f6c68c7d3ac426b7c7cd Mon Sep 17 00:00:00 2001 From: Justin Gilmer Date: Fri, 22 Sep 2023 13:40:59 -0400 Subject: [PATCH 075/131] Add integration tests to check for misbehaving sampling frequencies. (#53) * Add integration tests to check for misbehaving sampling frequencies. * fix int conversion of float. --- btrdb/stream.py | 6 ++--- tests/btrdb_integration/test_streamset.py | 28 ++++++++++++++++++++++- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/btrdb/stream.py b/btrdb/stream.py index 7a02ac2..7343def 100644 --- a/btrdb/stream.py +++ b/btrdb/stream.py @@ -1708,7 +1708,7 @@ def filter( key/value pairs for filtering streams based on tags annotations : dict key/value pairs for filtering streams based on annotations - sampling_frequency : int + sampling_frequency : float The sampling frequency of the data streams in Hz, set this if you want timesnapped values. Returns @@ -2274,9 +2274,7 @@ def __init__( ): self.start = to_nanoseconds(start) if start else None self.end = to_nanoseconds(end) if end else None - self.sampling_frequency = ( - int(sampling_frequency) if sampling_frequency else None - ) + self.sampling_frequency = sampling_frequency if sampling_frequency else None if self.start is None and self.end is None: raise BTRDBValueError("A valid `start` or `end` must be supplied") diff --git a/tests/btrdb_integration/test_streamset.py b/tests/btrdb_integration/test_streamset.py index a132f1b..583f128 100644 --- a/tests/btrdb_integration/test_streamset.py +++ b/tests/btrdb_integration/test_streamset.py @@ -1,4 +1,4 @@ -import logging +import random from math import isnan, nan from uuid import uuid4 as new_uuid @@ -451,3 +451,29 @@ def test_timesnap_forward_restricts_range(conn, tmp_collection): assert [1.0, 2.0] == [v.as_py() for v in values[str(s.uuid)]] # Same result if skipping past end instead of to end. assert values == ss.filter(end=int(2.9 * sec)).arrow_values() + + +@pytest.mark.parametrize("freq", [(None), (30), (15), (1), (0.1)]) +def test_timesnap_with_different_sampling_frequencies(freq, conn, tmp_collection): + # 30hz data + data_insert_freq = 30 + period_for_data = int(1 / data_insert_freq * 1e9) + stop = btrdb.utils.timez.currently_as_ns() + start = stop - btrdb.utils.timez.ns_delta(minutes=10) + t1 = [i for i in range(start, stop - period_for_data, period_for_data)] + v1 = [ + random.random() for _ in range(start, stop - period_for_data, period_for_data) + ] + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + stset = btrdb.stream.StreamSet([s1, s2]) + data_map = {s.uuid: pa.table([t1, v1], names=["time", "value"]) for s in stset} + stset.arrow_insert(data_map=data_map) + df = stset.filter( + start=start, end=stop, sampling_frequency=freq + ).arrow_to_dataframe() + total_points = df.shape[0] * df.shape[1] + total_raw_pts = len(v1) * len(stset) + expected_frac_of_pts = 1 if freq is None else freq / data_insert_freq + actual_frac_of_pts = total_points / total_raw_pts + assert np.isclose(expected_frac_of_pts, actual_frac_of_pts, rtol=0.001) From 2f32815ddcab69930e01c03ecbeb994de571760e Mon Sep 17 00:00:00 2001 From: Justin Gilmer Date: Mon, 25 Sep 2023 12:33:58 -0400 Subject: [PATCH 076/131] Update from staging (#54) * Threadpool executor (#22) * Release v5.15.0 * update protobuf to v4.22.3 * Add threaded streamset calls Using concurrent.futures.ThreadPoolExecutor * Blacken code * Update for failing tests * Ignore flake8 as part of testing pytest-flake8 seems to have issues with the later versions of flake8 https://github.com/tholo/pytest-flake8/issues/92 * Update .gitignore * Update ignore and remove extra print. * Remove idea folder (pycharm) --------- Co-authored-by: David Konigsberg <72822263+davidkonigsberg@users.noreply.github.com> Co-authored-by: Jeff Lin <42981468+jleifnf@users.noreply.github.com> * Threaded arrow (#23) * Release v5.15.0 * update protobuf to v4.22.3 * Add threaded streamset calls Using concurrent.futures.ThreadPoolExecutor * Blacken code * Update for failing tests * Ignore flake8 as part of testing pytest-flake8 seems to have issues with the later versions of flake8 https://github.com/tholo/pytest-flake8/issues/92 * Update .gitignore * Update proto definitions. * Update endpoint to support arrow methods * Support arrow endpoints * Additional arrow updates * Update transformers, add polars conversion * Update .gitignore * Update ignore and remove extra print. * Remove idea folder (pycharm) * Update requirements.txt * Update btrdb/transformers.py * Update the way to check for arrow-enabled btrdb This has not been "turned on" yet though, since we dont know the version number this will be enabled for. The method is currently commented out, but can be re-enabled pretty easily. * Use IPC streams to send the arrow bytes for insert Instead of writing out feather files to an `io.BytesIO` stream and then sending the feather files over the wire, this creates a buffered outputstream and then sends that data back as bytes to btrdb. * Create arrow specific stream methods. * Update test conn object to support minor version * Update tests and migrate arrow code. * Arrow and standard streamset insert * Create basic arrow to dataframe transformer * Support multirawvalues, arrow transformers * Multivalue arrow queries, in progress * Update stream filter to properly filter for sampling frequency * Update arrow values queries for multivalues * Update param passing for sampling frequency * Update index passing, and ignore depth * benchmark raw values queries for arrow and current api * Add aligned windows and run func * Streamset read benchmarks (WIP) In addition: * update streamset.count to support the `precise` boolean flag. * Update mock return value for versionMajor * In progress validation of stream benchs --------- Co-authored-by: David Konigsberg <72822263+davidkonigsberg@users.noreply.github.com> Co-authored-by: Jeff Lin <42981468+jleifnf@users.noreply.github.com> * Add 3.10 python to the testing matrix (#21) * Add 3.10 python to the testing matrix * Fix yaml parsing * Update requirements to support 3.10 * Use pip-tools `pip-compile` cli tool to generate requirements.txt files from the updated pyproject.toml file * Include pyproject.toml with basic features to support proper extra deps * Support different ways to install btrdb from pip * `btrdb, btrdb[data], btrdb[all], btrdb[testing], btrdb[ray]` * Update transformers.py to build up a numpy array when the subarrays are not the same size (number of entries) * This converts the main array's dtype to `object` * tests still pass with this change * recompile the btrdb proto files with latest protobuf and grpc plugins * Create multiple requirements.txt files for easier updating in the future as well as a locked version with pinned dependencies * Ignore protoc generated flake errors * Update test requirements * Include pre-commit and setup. * Pre-commit lints. * Update pre-commit.yaml add staging to pre-commit checks * Fix missing logging import, rerun pre-commit (#24) * Add basic doc string to endpoint object (#25) * Update benchmark scripts. * Multistream read bench insert bench (#26) * Fix multistream endpoint bugs * The streamset was passing the incorrect params to the endpoint * The endpoint does not return a `version` in its response, just `stat` and `arrowBytes` Params have been updated and a NoneType is passed around to ignore the lack of version info, which lets us use the same logic for all bytes decoding. * Add multistream benchmark methods for timesnap and no timesnap. * Add insert benchmarking methods (#27) Benchmarking methods added for: * stream inserts using tuples of time, value data * stream inserts using pyarrow tables of timestamps, value columns * streamset inserts using a dict map of streamset stream uuids, and lists of tuples of time, value data * streamset inserts using a dict map of streamset stream uuids, and pyarrow tables of timestamps, values. * Fix arrow inserts (#28) * Add insert benchmarking methods Benchmarking methods added for: * stream inserts using tuples of time, value data * stream inserts using pyarrow tables of timestamps, value columns * streamset inserts using a dict map of streamset stream uuids, and lists of tuples of time, value data * streamset inserts using a dict map of streamset stream uuids, and pyarrow tables of timestamps, values. * Include nullable false in pyarrow schema inserts * This was the only difference in the schemas between go and python. * also using a bytesIO stream to act as the sink for the ipc bytes. * Start integration test suite * Add more streamset integration tests. * Add support for authenticated requests without encryption. * Optimize logging calls (#30) Previously, the debug logging in the api would create the f-strings no matter if logging.DEBUG was the current log level or not. This can impact the performance, especially for benchmarking. Now, a cached IS_DEBUG flag is created for the stream operations, and other locations, the logger.isEnabledFor boolean is checked. Note that in the stream.py, this same function call is only executed once, and the results are cached for the rest of the logic. * Add more arrow tests and minor refactoring. * More integration test cases * Restructure tests. * Mark new failing tests as expected failures for now. * Disable gzip compression, it is very slow. * Reenable test, server has been fixed. * Update pandas testing and fix flake8 issues (#31) * Update pandas testing and fix flake8 issues * Update stream logic for unpacking arrow tables, update integration tests. * add init.py for integration tests. * Add additional tests for arrow methods vs their old api counterparts. * Add tests for timesnap boundary conditions. (#32) * Add more integration tests. * Add additional integration tests, modify the name_callable ability of the arrow_values. * remove extraneous prints. * Include retry logic. * Update statpoint order in arrow, fix some bugs with the arrow methods. * Update testing to account for NaNs. * Update github action versions. * Update tests, add in a test for duplicate values. * Remove empty test, remove extraneous prints --------- Co-authored-by: andrewchambers * Update docs for arrow (#35) * Update docs, add in final enhanced edits. * Only enable arrow-endpoints when version >= 5.30 (#36) Once we have a v5.30tag of the server with arrow/multistream, we can merge this and complete the ticket. * Update arrow notes, small doc changes. (#38) * fix: patch up stream object type and other bugs (#33) * fix: patch up stream object type and other bugs * fix: resolve depth errors in stream window * fix: resolve remaining test warnings * fix: resolve test imports * chore: add pre-commit install to readme * Update staging branch with latest `master` changes (#52) --------- Co-authored-by: David Konigsberg <72822263+davidkonigsberg@users.noreply.github.com> Co-authored-by: Jeff Lin <42981468+jleifnf@users.noreply.github.com> Co-authored-by: Andrew Chambers Co-authored-by: andrewchambers Co-authored-by: Taite Nazifi <135669716+pingt8@users.noreply.github.com> --- README.md | 26 ++--- btrdb/conn.py | 9 +- btrdb/stream.py | 20 +++- tests/btrdb/test_conn.py | 64 ++++++++++-- tests/btrdb/test_stream.py | 140 ++++++++++++++++++-------- tests/btrdb/test_transformers.py | 4 +- tests/btrdb/utils/test_credentials.py | 4 +- 7 files changed, 194 insertions(+), 73 deletions(-) diff --git a/README.md b/README.md index 7a808fe..9a04042 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,10 @@ # BTrDB Bindings for Python -These are BTrDB Bindings for Python allowing you painless and productive access to the Berkeley Tree Database (BTrDB). BTrDB is a time series database focusing on blazing speed with respect to univariate time series data at the nanosecond scale. - +These are BTrDB Bindings for Python allowing you painless and productive access to the Berkeley Tree Database (BTrDB). BTrDB is a time series database focusing on blazing speed with respect to univariate time series data at the nanosecond scale. ## Sample Code -Our goal is to make BTrDB as easy to use as possible, focusing on integration with other tools and the productivity of our users. In keeping with this we continue to add new features such as easy transformation to numpy arrays, pandas Series, etc. See the sample code below and then checkout our [documentation](https://btrdb.readthedocs.io/en/latest/) for more in depth instructions. - +Our goal is to make BTrDB as easy to use as possible, focusing on integration with other tools and the productivity of our users. In keeping with this we continue to add new features such as easy transformation to numpy arrays, pandas Series, etc. See the sample code below and then checkout our [documentation](https://btrdb.readthedocs.io/en/latest/) for more in depth instructions. import btrdb @@ -40,8 +38,6 @@ Our goal is to make BTrDB as easy to use as possible, focusing on integration wi >> StatPoint(1500000000300000000, 4.0, 5.0, 6.0, 3, 0.816496580927726) >> StatPoint(1500000000600000000, 7.0, 8.0, 9.0, 3, 0.816496580927726) - - You can also easily work with a group of streams for when you need to evaluate data across multiple time series or serialize to disk. from btrdb.utils.timez import to_nanoseconds @@ -69,17 +65,15 @@ You can also easily work with a group of streams for when you need to evaluate d 8 1500000000800000000 NaN 9.0 9 1500000000900000000 10.0 NaN - ## Installation -See our documentation on [installing](https://btrdb.readthedocs.io/en/latest/installing.html) the bindings for more detailed instructions. However, to quickly get started using the latest available versions you can use `pip` to install from pypi with `conda` support coming in the near future. +See our documentation on [installing](https://btrdb.readthedocs.io/en/latest/installing.html) the bindings for more detailed instructions. However, to quickly get started using the latest available versions you can use `pip` to install from pypi with `conda` support coming in the near future. $ pip install btrdb - ## Tests -This project includes a suite of automated tests based upon [pytest](https://docs.pytest.org/en/latest/). For your convenience, a `Makefile` has been provided with a target for evaluating the test suite. Use the following command to run the tests. +This project includes a suite of automated tests based upon [pytest](https://docs.pytest.org/en/latest/). For your convenience, a `Makefile` has been provided with a target for evaluating the test suite. Use the following command to run the tests. $ make test @@ -89,13 +83,13 @@ Note that the test suite has additional dependencies that must be installed for ## Releases -This codebase uses github actions to control the release process. To create a new release of the software, run `release.sh` with arguments for the new version as shown below. Make sure you are in the master branch when running this script. +This codebase uses github actions to control the release process. To create a new release of the software, run `release.sh` with arguments for the new version as shown below. Make sure you are in the master branch when running this script. ``` ./release.sh 5 11 4 ``` -This will tag and push the current commit and github actions will run the test suite, build the package, and push it to pypi. If any issues are encountered with the automated tests, the build will fail and you will have a tag with no corresponding release. +This will tag and push the current commit and github actions will run the test suite, build the package, and push it to pypi. If any issues are encountered with the automated tests, the build will fail and you will have a tag with no corresponding release. After a release is created, you can manually edit the release description through github. @@ -111,4 +105,10 @@ Note that the documentation also requires Sphix and other dependencies to succes ## Versioning -This codebases uses a form of [Semantic Versioning](http://semver.org/) to structure version numbers. In general, the major version number will track with the BTrDB codebase to transparently maintain version compatibility. Planned features between major versions will increment the minor version while any special releases (bug fixes, etc.) will increment the patch number. +This codebases uses a form of [Semantic Versioning](http://semver.org/) to structure version numbers. In general, the major version number will track with the BTrDB codebase to transparently maintain version compatibility. Planned features between major versions will increment the minor version while any special releases (bug fixes, etc.) will increment the patch number. + +## Pre-commit hooks + +`pip install pre-commit` and then run +`pre-commit run --all-files` to comb through changes and make sure they are formatted correctly. +`pre-commit install` and then run `git commit` to automatically run the pre-commit hooks on every commit. diff --git a/btrdb/conn.py b/btrdb/conn.py index 0d712e2..a77eced 100644 --- a/btrdb/conn.py +++ b/btrdb/conn.py @@ -21,6 +21,7 @@ import re import uuid as uuidlib from concurrent.futures import ThreadPoolExecutor +from typing import List import certifi import grpc @@ -283,8 +284,7 @@ def streams(self, *identifiers, versions=None, is_collection_prefix=False): if versions and len(versions) != len(identifiers): raise ValueError("number of versions does not match identifiers") - - streams = [] + streams: List[Stream] = [] for ident in identifiers: if isinstance(ident, uuidlib.UUID): streams.append(self.stream_from_uuid(ident)) @@ -305,7 +305,10 @@ def streams(self, *identifiers, versions=None, is_collection_prefix=False): is_collection_prefix=is_collection_prefix, tags={"name": parts[-1]}, ) - if len(found) == 1: + if isinstance(found, Stream): + streams.append(found) + continue + if isinstance(found, list) and len(found) == 1: streams.append(found[0]) continue raise StreamNotFoundError(f"Could not identify stream `{ident}`") diff --git a/btrdb/stream.py b/btrdb/stream.py index caf95ab..7a02ac2 100644 --- a/btrdb/stream.py +++ b/btrdb/stream.py @@ -22,6 +22,7 @@ from collections import deque from collections.abc import Sequence from copy import deepcopy +from typing import List import pyarrow as pa @@ -1451,8 +1452,14 @@ class StreamSetBase(Sequence): A lighweight wrapper around a list of stream objects """ - def __init__(self, streams): - self._streams = streams + def __init__(self, streams: List[Stream]): + self._streams: List[Stream] = [] + for stream in streams: + if not isinstance(stream, Stream): + raise BTRDBTypeError( + f"streams must be of type Stream {stream}, {type(stream)}" + ) + self._streams.append(stream) if len(self._streams) < 1: raise ValueError( f"Trying to create streamset with an empty list of streams {self._streams}." @@ -2231,6 +2238,15 @@ def __getitem__(self, item): return self._streams[item] + def __contains__(self, item): + if isinstance(item, str): + for stream in self._streams: + if str(stream.uuid()) == item: + return True + return False + + return item in self._streams + def __len__(self): return len(self._streams) diff --git a/tests/btrdb/test_conn.py b/tests/btrdb/test_conn.py index 340416d..ed00aa9 100644 --- a/tests/btrdb/test_conn.py +++ b/tests/btrdb/test_conn.py @@ -24,6 +24,54 @@ from btrdb.endpoint import Endpoint from btrdb.exceptions import * from btrdb.grpcinterface import btrdb_pb2 +from btrdb.stream import Stream + +########################################################################## +## Fixtures +########################################################################## + + +@pytest.fixture +def stream1(): + uu = uuidlib.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + stream = Mock(Stream) + stream.version = Mock(return_value=11) + stream.uuid = Mock(return_value=uu) + type(stream).collection = PropertyMock(return_value="fruits/apple") + type(stream).name = PropertyMock(return_value="gala") + stream.tags = Mock(return_value={"name": "gala", "unit": "volts"}) + stream.annotations = Mock(return_value=({"owner": "ABC", "color": "red"}, 11)) + stream._btrdb = Mock() + return stream + + +@pytest.fixture +def stream2(): + uu = uuidlib.UUID("17dbe387-89ea-42b6-864b-f505cdb483f5") + stream = Mock(Stream) + stream.version = Mock(return_value=22) + stream.uuid = Mock(return_value=uu) + type(stream).collection = PropertyMock(return_value="fruits/orange") + type(stream).name = PropertyMock(return_value="blood") + stream.tags = Mock(return_value={"name": "blood", "unit": "amps"}) + stream.annotations = Mock(return_value=({"owner": "ABC", "color": "orange"}, 22)) + stream._btrdb = Mock() + return stream + + +@pytest.fixture +def stream3(): + uu = uuidlib.UUID("17dbe387-89ea-42b6-864b-e2ef0d22a53b") + stream = Mock(Stream) + stream.version = Mock(return_value=33) + stream.uuid = Mock(return_value=uu) + type(stream).collection = PropertyMock(return_value="fruits/banana") + type(stream).name = PropertyMock(return_value="yellow") + stream.tags = Mock(return_value={"name": "yellow", "unit": "watts"}) + stream.annotations = Mock(return_value=({"owner": "ABC", "color": "yellow"}, 33)) + stream._btrdb = Mock() + return stream + ########################################################################## ## Connection Tests @@ -91,7 +139,7 @@ def test_streams_recognizes_uuid(self, mock_func): """ db = BTrDB(None) uuid1 = uuidlib.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") - mock_func.return_value = [1] + mock_func.return_value = Stream(db, uuid1) db.streams(uuid1) mock_func.assert_called_once() @@ -104,7 +152,7 @@ def test_streams_recognizes_uuid_string(self, mock_func): """ db = BTrDB(None) uuid1 = "0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a" - mock_func.return_value = [1] + mock_func.return_value = Stream(db, uuid1) db.streams(uuid1) mock_func.assert_called_once() @@ -117,7 +165,9 @@ def test_streams_handles_path(self, mock_func): """ db = BTrDB(None) ident = "zoo/animal/dog" - mock_func.return_value = [1] + mock_func.return_value = [ + Stream(db, "0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a"), + ] db.streams(ident, "0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") mock_func.assert_called_once() @@ -139,12 +189,10 @@ def test_streams_raises_err(self, mock_func): with pytest.raises(StreamNotFoundError) as exc: db.streams(ident) - mock_func.return_value = [1, 2] - with pytest.raises(StreamNotFoundError) as exc: - db.streams(ident) - # check that does not raise if one returned - mock_func.return_value = [1] + mock_func.return_value = [ + Stream(db, ident), + ] db.streams(ident) def test_streams_raises_valueerror(self): diff --git a/tests/btrdb/test_stream.py b/tests/btrdb/test_stream.py index 39b82e7..4fede74 100644 --- a/tests/btrdb/test_stream.py +++ b/tests/btrdb/test_stream.py @@ -84,11 +84,28 @@ def stream2(): return stream +@pytest.fixture +def stream3(): + uu = uuid.UUID("17dbe387-89ea-42b6-864b-e2ef0d22a53b") + stream = Mock(Stream) + stream.version = Mock(return_value=33) + stream.uuid = Mock(return_value=uu) + stream.nearest = Mock(return_value=(RawPoint(time=30, value=1), 33)) + type(stream).collection = PropertyMock(return_value="fruits/banana") + type(stream).name = PropertyMock(return_value="yellow") + stream.tags = Mock(return_value={"name": "yellow", "unit": "watts"}) + stream.annotations = Mock(return_value=({"owner": "ABC", "color": "yellow"}, 3)) + stream._btrdb = Mock() + stream._btrdb._executor = Mock() + stream._btrdb._ARROW_ENABLED = Mock(return_value=False) + return stream + + @pytest.fixture def arrow_stream3(): uu = uuid.UUID("17dbe387-89ea-42b6-864b-f505cdb483f5") stream = Mock(Stream) - stream.version = Mock(return_value=22) + stream.version = Mock(return_value=33) stream.uuid = Mock(return_value=uu) stream.nearest = Mock(return_value=(RawPoint(time=20, value=1), 22)) type(stream).collection = PropertyMock(return_value="fruits/orange") @@ -111,7 +128,7 @@ def test_create(self): """ Assert we can create the object """ - Stream(None, "FAKE") + Stream(None, "FAKE_UUID") def test_repr_str(self): """ @@ -929,7 +946,7 @@ def test_create(self): """ Assert we can create the object """ - StreamSet([1]) + StreamSet([Stream(None, "1"), Stream(None, "2"), Stream(None, "3")]) @pytest.mark.parametrize( "empty_container", [(tuple()), (dict()), (list()), (set())] @@ -948,12 +965,17 @@ def test_repr(self): """ Assert StreamSet instance repr output """ - data = [11, 22, "dog", "cat"] + data = [ + Stream(None, "11"), + Stream(None, "22"), + Stream(None, "dog"), + Stream(None, "cat"), + ] streams = StreamSet(data) expected = "" assert streams.__repr__() == expected - data = [1] + data = [Stream(None, "1")] streams = StreamSet(data) expected = "" assert streams.__repr__() == expected @@ -962,12 +984,17 @@ def test_str(self): """ Assert StreamSet instance str output """ - data = [11, 22, "dog", "cat"] + data = [ + Stream(None, "11"), + Stream(None, "22"), + Stream(None, "dog"), + Stream(None, "cat"), + ] streams = StreamSet(data) expected = "StreamSet with 4 streams" assert str(streams) == expected - data = [1] + data = [Stream(None, "1")] streams = StreamSet(data) expected = "StreamSet with 1 stream" assert str(streams) == expected @@ -976,7 +1003,12 @@ def test_subscriptable(self): """ Assert StreamSet instance is subscriptable """ - data = [11, 22, "dog", "cat"] + data = [ + Stream(None, "11"), + Stream(None, "22"), + Stream(None, "dog"), + Stream(None, "cat"), + ] streams = StreamSet(data) for index, val in enumerate(data): assert streams[index] == val @@ -985,7 +1017,12 @@ def test_len(self): """ Assert StreamSet instance support len """ - data = [11, 22, "dog", "cat"] + data = [ + Stream(None, "11"), + Stream(None, "22"), + Stream(None, "dog"), + Stream(None, "cat"), + ] streams = StreamSet(data) assert len(streams) == len(data) @@ -993,7 +1030,12 @@ def test_iter(self): """ Assert StreamSet instance support iteration """ - data = [11, 22, "dog", "cat"] + data = [ + Stream(None, "11"), + Stream(None, "22"), + Stream(None, "dog"), + Stream(None, "cat"), + ] streams = StreamSet(data) for index, stream in enumerate(streams): assert data[index] == stream @@ -1002,7 +1044,12 @@ def test_indexing(self): """ Assert StreamSet instance supports indexing """ - data = [11, 22, "dog", "cat"] + data = [ + Stream(None, "11"), + Stream(None, "22"), + Stream(None, "dog"), + Stream(None, "cat"), + ] streams = StreamSet(data) # verify index lookup @@ -1035,19 +1082,24 @@ def test_mapping(self): streams[missing] assert str(missing) in str(e) - def test_contains(self): + def test_contains(self, stream1, stream2, stream3): """ Assert StreamSet instance supports contains """ - data = [11, 22, "dog", "cat"] + data = [stream1, stream2, stream3] streams = StreamSet(data) - assert "dog" in streams + assert "0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a" in streams def test_reverse(self): """ Assert StreamSet instance supports reversal """ - data = [11, 22, "dog", "cat"] + data = [ + Stream(None, "11"), + Stream(None, "22"), + Stream(None, "dog"), + Stream(None, "cat"), + ] streams = StreamSet(data) assert list(reversed(streams)) == list(reversed(data)) @@ -1055,7 +1107,12 @@ def test_to_list(self): """ Assert StreamSet instance cast to list """ - data = [11, 22, "dog", "cat"] + data = [ + Stream(None, "11"), + Stream(None, "22"), + Stream(None, "dog"), + Stream(None, "cat"), + ] streams = StreamSet(data) assert list(streams) == data @@ -1063,17 +1120,17 @@ def test_to_list(self): ## allow_window tests ########################################################################## - def test_allow_window(self): + def test_allow_window(self, stream1, stream2, stream3): """ Assert allow_window returns False if window already requested """ - streams = StreamSet([1, 2, 3]) + streams = StreamSet([stream1, stream2, stream3]) assert streams.allow_window == True - streams.windows(30, 4) + streams.windows(30) assert streams.allow_window == False - streams = StreamSet([1, 2, 3]) + streams = StreamSet([stream1, stream2, stream3]) streams.aligned_windows(30) assert streams.allow_window == False @@ -1102,20 +1159,20 @@ def test_pin_versions_returns_self(self, stream1): result = streams.pin_versions(expected) assert streams is result - def test_pin_versions_with_argument(self): + def test_pin_versions_with_argument(self, stream1, stream2): """ Assert pin_versions uses supplied version numbers """ - streams = StreamSet([1, 2]) + streams = StreamSet([stream1, stream2]) expected = [3, 4] assert streams.pin_versions(expected) == streams assert streams._pinned_versions == expected - def test_pin_versions_with_argument(self): + def test_pin_versions_with_argument(self, stream1, stream2): """ Assert pin_versions uses supplied version numbers """ - streams = StreamSet([1, 2]) + streams = StreamSet([stream1, stream2]) uu = uuid.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") expected = {uu: 42} assert streams.pin_versions(expected) == streams @@ -1132,11 +1189,11 @@ def test_pin_versions_no_argument(self, stream1, stream2): assert streams.pin_versions(expected) == streams assert streams._pinned_versions == expected - def test_pin_versions_raise_on_non_dict(self): + def test_pin_versions_raise_on_non_dict(self, stream1): """ Assert pin_versions raises if versions argument is not dict """ - streams = StreamSet([1]) + streams = StreamSet([stream1]) expected = "INVALID DATA" with pytest.raises(TypeError) as e: @@ -1148,7 +1205,7 @@ def test_pin_versions_raise_on_non_uuid_key(self): Assert pin_versions raises if versions argument dict does not have UUID for keys """ - streams = StreamSet([1]) + streams = StreamSet([Stream(None, "1")]) expected = {"uuid": 42} with pytest.raises(TypeError) as e: @@ -1223,7 +1280,7 @@ def test_current(self, mocked, stream1, stream2): stream2.nearest.assert_called_once_with(15, version=22, backward=True) @patch("btrdb.stream.currently_as_ns") - def test_currently_out_of_range(self, mocked): + def test_currently_out_of_range(self, mocked, stream1, stream2): """ Assert currently raises an exception if it is not filtered """ @@ -1485,7 +1542,8 @@ def test_clone_returns_new_object(self, stream1, stream2): clone = streams.clone() assert id(clone) != id(streams) - assert clone._streams is streams._streams + assert clone._streams is not streams._streams + assert clone._streams == streams._streams ########################################################################## ## windows tests @@ -1508,10 +1566,10 @@ def test_windows_raises_if_not_allowed(self, stream1): Assert windows raises Exception if not allowed """ streams = StreamSet([stream1]) - streams.windows(8, 22) + streams.windows(8) with pytest.raises(Exception) as exc: - streams.windows(10, 20) + streams.windows(10) assert "window operation is already requested" in str(exc).lower() def test_windows_returns_self(self, stream1): @@ -1519,7 +1577,7 @@ def test_windows_returns_self(self, stream1): Assert windows returns self """ streams = StreamSet([stream1]) - result = streams.windows(10, 20) + result = streams.windows(10) assert result is streams def test_windows_stores_values(self, stream1): @@ -1527,7 +1585,7 @@ def test_windows_stores_values(self, stream1): Assert windows stores values """ streams = StreamSet([stream1]) - result = streams.windows(10, 20) + result = streams.windows(10) # assert stores values assert result.width == 10 @@ -1552,13 +1610,11 @@ def test_windows_values_and_calls_to_endpoint(self): s2 = Stream(btrdb=BTrDB(endpoint), uuid=uu2) versions = {uu1: 11, uu2: 12} - start, end, width, depth = 1, 100, 1000, 25 + start, end, width = 1, 100, 1000 streams = StreamSet([s1, s2]) streams.pin_versions(versions) values = ( - streams.filter(start=start, end=end) - .windows(width=width, depth=depth) - .values() + streams.filter(start=start, end=end).windows(width=width, depth=0).values() ) # assert endpoint calls have correct arguments, version @@ -1594,14 +1650,10 @@ def test_windows_rows_and_calls_to_endpoint(self): s2 = Stream(btrdb=BTrDB(endpoint), uuid=uu2) versions = {uu1: 11, uu2: 12} - start, end, width, depth = 1, 100, 1000, 25 + start, end, width = 1, 100, 1000 streams = StreamSet([s1, s2]) streams.pin_versions(versions) - rows = ( - streams.filter(start=start, end=end) - .windows(width=width, depth=depth) - .rows() - ) + rows = streams.filter(start=start, end=end).windows(width=width, depth=0).rows() # assert endpoint calls have correct arguments, version expected = [ @@ -1635,7 +1687,7 @@ def test_aligned_windows_raises_if_not_allowed(self, stream1): Assert that aligned_windows raises Exception if not allowed """ streams = StreamSet([stream1]) - streams.windows(8, 22) + streams.windows(8) with pytest.raises(Exception) as exc: streams.aligned_windows(20) diff --git a/tests/btrdb/test_transformers.py b/tests/btrdb/test_transformers.py index 15003c9..833de69 100644 --- a/tests/btrdb/test_transformers.py +++ b/tests/btrdb/test_transformers.py @@ -23,6 +23,7 @@ import pytest from pandas import DataFrame, Index, Series +from btrdb.conn import BTrDB from btrdb.point import RawPoint, StatPoint from btrdb.stream import Stream, StreamSet from btrdb.transformers import * @@ -211,6 +212,7 @@ def statpoint_streamset(): stream = Mock(Stream) type(stream).collection = PropertyMock(return_value="test") type(stream).name = PropertyMock(return_value="stream{}".format(idx)) + stream._btrdb = Mock() streams.append(stream) obj = StreamSet(streams) @@ -796,7 +798,7 @@ def test_to_csv_as_stringio(self, streamset): result = [dict(row) for row in reader] for item in result: for k, v in item.items(): - if v is "": + if v == "": item[k] = None elif "." in v: item[k] = float(v) diff --git a/tests/btrdb/utils/test_credentials.py b/tests/btrdb/utils/test_credentials.py index 37b4811..7a49018 100644 --- a/tests/btrdb/utils/test_credentials.py +++ b/tests/btrdb/utils/test_credentials.py @@ -39,7 +39,7 @@ def test_raises_err_if_credentials_not_found(self, mock_open): class TestLoadProfile(object): - def setup(self): + def setup_method(self): for env in ["BTRDB_ENDPOINTS", "BTRDB_PROFILE", "BTRDB_API_KEY"]: try: del os.environ[env] @@ -114,7 +114,7 @@ def test_returns_no_default_profile(self, mock_credentials): class TestCredentials(object): - def setup(self): + def setup_method(self): for env in ["BTRDB_ENDPOINTS", "BTRDB_PROFILE", "BTRDB_API_KEY"]: try: del os.environ[env] From c82f5d26598df33de8130e12221f937b38da4c02 Mon Sep 17 00:00:00 2001 From: Justin Gilmer Date: Mon, 25 Sep 2023 11:37:17 -0500 Subject: [PATCH 077/131] Update README.md Change readthedocs url --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9a04042..af73c28 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ These are BTrDB Bindings for Python allowing you painless and productive access ## Sample Code -Our goal is to make BTrDB as easy to use as possible, focusing on integration with other tools and the productivity of our users. In keeping with this we continue to add new features such as easy transformation to numpy arrays, pandas Series, etc. See the sample code below and then checkout our [documentation](https://btrdb.readthedocs.io/en/latest/) for more in depth instructions. +Our goal is to make BTrDB as easy to use as possible, focusing on integration with other tools and the productivity of our users. In keeping with this we continue to add new features such as easy transformation to numpy arrays, pandas Series, etc. See the sample code below and then checkout our [documentation](https://btrdb-python.readthedocs.io/en/latest/) for more in depth instructions. import btrdb @@ -67,7 +67,7 @@ You can also easily work with a group of streams for when you need to evaluate d ## Installation -See our documentation on [installing](https://btrdb.readthedocs.io/en/latest/installing.html) the bindings for more detailed instructions. However, to quickly get started using the latest available versions you can use `pip` to install from pypi with `conda` support coming in the near future. +See our documentation on [installing](https://btrdb-python.readthedocs.io/en/latest/installing.html) the bindings for more detailed instructions. However, to quickly get started using the latest available versions you can use `pip` to install from pypi with `conda` support coming in the near future. $ pip install btrdb @@ -99,7 +99,7 @@ The project documentation is written in reStructuredText and is built using Sphi $ make html -This will build the HTML documentation locally in `docs/build`, which can be viewed using `open docs/build/index.html`. Other formats (PDF, epub, etc) can be built using `docs/Makefile`. The documentation is automatically built on every GitHub release and hosted on [Read The Docs](https://btrdb.readthedocs.io/en/latest/). +This will build the HTML documentation locally in `docs/build`, which can be viewed using `open docs/build/index.html`. Other formats (PDF, epub, etc) can be built using `docs/Makefile`. The documentation is automatically built on every GitHub release and hosted on [Read The Docs](https://btrdb-python.readthedocs.io/en/latest/). Note that the documentation also requires Sphix and other dependencies to successfully build: `pip install -r docs/requirements.txt`. From 2eed67473663a1e5d5f78a4da579a7b218572ee0 Mon Sep 17 00:00:00 2001 From: Alexander Young Date: Mon, 25 Sep 2023 13:19:01 -0500 Subject: [PATCH 078/131] Release v5.31.0 --- btrdb/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/btrdb/version.py b/btrdb/version.py index e98a93b..d657ab4 100644 --- a/btrdb/version.py +++ b/btrdb/version.py @@ -15,7 +15,7 @@ ## Module Info ########################################################################## -__version_info__ = {"major": 5, "minor": 30, "micro": 2, "releaselevel": "final"} +__version_info__ = {"major": 5, "minor": 31, "micro": 0, "releaselevel": "final"} ########################################################################## ## Helper Functions diff --git a/pyproject.toml b/pyproject.toml index f42ddf0..80f656d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "btrdb" -version = "5.30.2" +version = "5.31.0" authors = [ {name="PingThingsIO", email="support@pingthings.io"}, ] From 6f10939ff50891af2cf7571f218762de1f7582df Mon Sep 17 00:00:00 2001 From: Justin Gilmer Date: Tue, 26 Sep 2023 13:11:05 -0400 Subject: [PATCH 079/131] Fix integration tests where the time column is not automatically set as the index. (#56) --- tests/btrdb_integration/test_streamset.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/tests/btrdb_integration/test_streamset.py b/tests/btrdb_integration/test_streamset.py index a132f1b..6e83a45 100644 --- a/tests/btrdb_integration/test_streamset.py +++ b/tests/btrdb_integration/test_streamset.py @@ -96,14 +96,17 @@ def test_streamset_arrow_windows_vs_windows(conn, tmp_collection, name_callable) .windows(width=btrdb.utils.timez.ns_delta(nanoseconds=10)) ) values_arrow = ss.arrow_to_dataframe(name_callable=name_callable) + values_arrow.set_index("time", inplace=True) + values_arrow.index = pd.DatetimeIndex(values_arrow.index) values_prev = ss.to_dataframe(name_callable=name_callable).convert_dtypes( dtype_backend="pyarrow" ) - values_prev.index = pd.DatetimeIndex(values_prev.index, tz="UTC") + values_prev.index = pd.to_datetime(values_prev.index, utc=True) + values_prev = values_prev.convert_dtypes( + dtype_backend="pyarrow", + ) col_map = {old_col: old_col + "/mean" for old_col in values_prev.columns} values_prev = values_prev.rename(columns=col_map) - (values_arrow) - (values_prev) assert values_arrow.equals(values_prev) @@ -125,6 +128,8 @@ def test_streamset_arrow_windows_vs_windows_agg_all(conn, tmp_collection): .windows(width=btrdb.utils.timez.ns_delta(nanoseconds=10)) ) values_arrow = ss.arrow_to_dataframe(name_callable=None, agg=["all"]) + values_arrow.set_index("time", inplace=True) + values_arrow.index = pd.DatetimeIndex(values_arrow.index) values_prev = ss.to_dataframe(name_callable=None, agg="all") values_prev = values_prev.apply(lambda x: x.astype(str(x.dtype) + "[pyarrow]")) values_prev = values_prev.apply( @@ -145,7 +150,12 @@ def test_streamset_arrow_windows_vs_windows_agg_all(conn, tmp_collection): agg=["all", "mean", "trash"], name_callable=lambda x: str(x.uuid) ) assert ( - len(other_arrow_df.filter(regex="[min,mean,max,count,stddev]").columns) == 3 * 5 + len( + other_arrow_df.filter( + regex=".*\/[min,mean,max,count,stddev]", axis=1 + ).columns + ) + == 3 * 5 ) @@ -174,6 +184,8 @@ def test_streamset_arrow_aligned_windows_vs_aligned_windows( .windows(width=btrdb.utils.general.pointwidth.from_nanoseconds(10)) ) values_arrow = ss.arrow_to_dataframe(name_callable=name_callable) + values_arrow.set_index("time", inplace=True) + values_arrow.index = pd.DatetimeIndex(values_arrow.index) values_prev = ss.to_dataframe( name_callable=name_callable ) # .convert_dtypes(dtype_backend='pyarrow') @@ -235,6 +247,7 @@ def test_arrow_streamset_to_dataframe(conn, tmp_collection): s2.insert(list(zip(t2, d2))) ss = btrdb.stream.StreamSet([s1, s2]).filter(start=100, end=121) values = ss.arrow_to_dataframe() + values.set_index("time", inplace=True) expected_times = [100, 101, 105, 106, 110, 114, 115, 119, 120] expected_times = [ pa.scalar(v, type=pa.timestamp("ns", tz="UTC")).as_py() for v in expected_times @@ -385,6 +398,8 @@ def test_streamset_windows_aggregates_filter(conn, tmp_collection): .windows(width=btrdb.utils.timez.ns_delta(nanoseconds=10)) ) values_arrow_df = ss.arrow_to_dataframe(agg=["mean", "stddev"]) + values_arrow_df.set_index("time", inplace=True) + values_arrow_df.index = pd.DatetimeIndex(values_arrow_df.index) values_non_arrow_df = ss.to_dataframe(agg="all") values_non_arrow_df.index = pd.DatetimeIndex(values_non_arrow_df.index, tz="UTC") values_non_arrow_df = values_non_arrow_df.apply( From b484d7bc540d951dcd5732708b23ae50996d80a4 Mon Sep 17 00:00:00 2001 From: Justin Gilmer Date: Fri, 29 Sep 2023 11:47:45 -0400 Subject: [PATCH 080/131] Update test for sampling frequency. --- tests/btrdb_integration/test_streamset.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/btrdb_integration/test_streamset.py b/tests/btrdb_integration/test_streamset.py index 9f1b571..d173818 100644 --- a/tests/btrdb_integration/test_streamset.py +++ b/tests/btrdb_integration/test_streamset.py @@ -152,7 +152,7 @@ def test_streamset_arrow_windows_vs_windows_agg_all(conn, tmp_collection): assert ( len( other_arrow_df.filter( - regex=".*\/[min,mean,max,count,stddev]", axis=1 + regex=r".*\/[min,mean,max,count,stddev]", axis=1 ).columns ) == 3 * 5 @@ -487,6 +487,7 @@ def test_timesnap_with_different_sampling_frequencies(freq, conn, tmp_collection df = stset.filter( start=start, end=stop, sampling_frequency=freq ).arrow_to_dataframe() + df.set_index("time", inplace=True) total_points = df.shape[0] * df.shape[1] total_raw_pts = len(v1) * len(stset) expected_frac_of_pts = 1 if freq is None else freq / data_insert_freq From 9b7fb1ea39cce3b875682174f5d5452567863792 Mon Sep 17 00:00:00 2001 From: Jeff Lin <42981468+jleifnf@users.noreply.github.com> Date: Fri, 29 Sep 2023 13:56:39 -0700 Subject: [PATCH 081/131] Fix build deprecation warnings in using pyproject.toml and setuptool (#57) * update setuptools parameters for strict standards starting 2023-Oct-30 * remove invalid url for homepage of the project * remove invalid url for homepage of the project * update the docs url in pyproject.toml --- pyproject.toml | 6 ++++-- setup.cfg | 4 ++-- setup.py | 3 +-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 80f656d..d3c9f94 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,6 +4,9 @@ version = "5.31.0" authors = [ {name="PingThingsIO", email="support@pingthings.io"}, ] +maintainers = [ + {name="PingThingsIO", email="support@pingthings.io"}, +] description = "Bindings to interact with the Berkeley Tree Database using gRPC." readme = "README.md" license = {file="LICENSE.txt"} @@ -65,8 +68,7 @@ all = [ ] [project.urls] -"Homepage" = "https://btrdb.io" -"Docs" = "https://btrdb.readthedocs.io" +"Docs" = "https://btrdb-python.readthedocs.io/" "Repository" = "https://github.com/pingthingsio/btrdb-python.git" [build-system] diff --git a/setup.cfg b/setup.cfg index 3f445b3..8d1166f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] -description-file = DESCRIPTION.md -license_file = LICENSE.txt +description_file = DESCRIPTION.md +license_files = LICENSE.txt [aliases] test=pytest diff --git a/setup.py b/setup.py index a5b82d1..df6ad76 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ REPOSITORY = "https://github.com/PingThingsIO/btrdb-python" PACKAGE = "btrdb" URL = "http://btrdb.io/" -DOCS_URL = "https://btrdb.readthedocs.io/en/latest/" +DOCS_URL = "https://btrdb-python.readthedocs.io/" ## Define the keywords KEYWORDS = ("btrdb", "berkeley", "timeseries", "database", "bindings" "gRPC") @@ -133,7 +133,6 @@ def get_description_type(path=PKG_DESCRIBE): "license": LICENSE, "author": AUTHOR, "author_email": EMAIL, - "url": URL, "maintainer": MAINTAINER, "maintainer_email": EMAIL, "project_urls": { From 8c4fed56c70269f2ab78cf0794f1d06e894c3bde Mon Sep 17 00:00:00 2001 From: Jeff Lin <42981468+jleifnf@users.noreply.github.com> Date: Tue, 3 Oct 2023 14:11:37 -0700 Subject: [PATCH 082/131] remove deprecated columns arg from to_dataframe and arrow_to_dataframe (#59) --- btrdb/transformers.py | 29 +++-------------------------- tests/btrdb/test_transformers.py | 21 +++++++-------------- 2 files changed, 10 insertions(+), 40 deletions(-) diff --git a/btrdb/transformers.py b/btrdb/transformers.py index 9bcf7b5..f01787f 100644 --- a/btrdb/transformers.py +++ b/btrdb/transformers.py @@ -138,18 +138,13 @@ def arrow_to_series(streamset, agg="mean", name_callable=None): return [arrow_df[col] for col in arrow_df] -def arrow_to_dataframe( - streamset, columns=None, agg=None, name_callable=None -) -> pd.DataFrame: +def arrow_to_dataframe(streamset, agg=None, name_callable=None) -> pd.DataFrame: """ Returns a Pandas DataFrame object indexed by time and using the values of a stream for each column. Parameters ---------- - columns: sequence - column names to use for DataFrame. Deprecated and not compatible with name_callable. - agg : List[str], default: ["mean"] Specify the StatPoint fields (e.g. aggregating function) to create the dataframe from. Must be one or more of "min", "mean", "max", "count", "stddev", or "all". This @@ -175,13 +170,6 @@ def arrow_to_dataframe( raise ImportError( f"Please install Pandas and pyarrow to use this transformation function. ErrorMessage: {err}" ) - # deprecation warning added in v5.8 - if columns: - warn( - "the columns argument is deprecated and will be removed in a future release", - DeprecationWarning, - stacklevel=2, - ) if agg is None: agg = ["mean"] @@ -227,16 +215,13 @@ def arrow_to_dataframe( return tmp.to_pandas(date_as_object=False, types_mapper=pd.ArrowDtype) -def to_dataframe(streamset, columns=None, agg="mean", name_callable=None): +def to_dataframe(streamset, agg="mean", name_callable=None): """ Returns a Pandas DataFrame object indexed by time and using the values of a stream for each column. Parameters ---------- - columns: sequence - column names to use for DataFrame. Deprecated and not compatible with name_callable. - agg : str, default: "mean" Specify the StatPoint field (e.g. aggregating function) to create the Series from. Must be one of "min", "mean", "max", "count", "stddev", or "all". This @@ -253,14 +238,6 @@ def to_dataframe(streamset, columns=None, agg="mean", name_callable=None): except ImportError: raise ImportError("Please install Pandas to use this transformation function.") - # deprecation warning added in v5.8 - if columns: - warn( - "the columns argument is deprecated and will be removed in a future release", - DeprecationWarning, - stacklevel=2, - ) - # TODO: allow this at some future point if agg == "all" and name_callable is not None: raise AttributeError( @@ -288,7 +265,7 @@ def to_dataframe(streamset, columns=None, agg="mean", name_callable=None): ] df.columns = pd.MultiIndex.from_tuples(stream_names) else: - df.columns = columns if columns else _stream_names(streamset, name_callable) + df.columns = _stream_names(streamset, name_callable) return df diff --git a/tests/btrdb/test_transformers.py b/tests/btrdb/test_transformers.py index 833de69..c7ce719 100644 --- a/tests/btrdb/test_transformers.py +++ b/tests/btrdb/test_transformers.py @@ -601,7 +601,7 @@ def test_to_series(self, streamset): def test_to_series_name_lambda(self, streamset): """ - assert to_dateframe uses name lambda + assert to_series uses name lambda """ result = streamset.to_series(name_callable=lambda s: s.name) assert [s.name for s in result] == ["stream0", "stream1", "stream2", "stream3"] @@ -691,23 +691,16 @@ def test_to_dataframe(self, streamset): df.set_index("time", inplace=True) assert to_dataframe(streamset).equals(df) - def test_to_dataframe_column_issues_warning(self, statpoint_streamset): + def test_to_dataframe_column_issues_error(self, statpoint_streamset): """ - assert to_dateframe with column argument issues warning + assert to_dateframe with column argument issues error """ columns = ["test/cats", "test/dogs", "test/horses", "test/fishes"] - with pytest.deprecated_call(): + with pytest.raises(TypeError) as unexpected_key_err: statpoint_streamset.to_dataframe(columns=columns) - - def test_to_dataframe_column(self, statpoint_streamset): - """ - assert to_dateframe with column argument actually renames columns - """ - columns = ["test/cats", "test/dogs", "test/horses", "test/fishes"] - with pytest.deprecated_call(): - df = statpoint_streamset.to_dataframe(columns=columns) - - assert df.columns.tolist() == columns + assert "got an unexpected keyword argument 'columns'" in str( + unexpected_key_err.value + ) def test_to_dataframe_multindex(self, statpoint_streamset): """ From 19edda0b6025282d307f5863acc24c733d4566f1 Mon Sep 17 00:00:00 2001 From: Justin Gilmer Date: Thu, 16 Nov 2023 17:16:39 -0500 Subject: [PATCH 083/131] Subscription endpoint (#63) Co-authored-by: Andrew Chambers --- btrdb/endpoint.py | 22 + btrdb/grpcinterface/btrdb.proto | 17 + btrdb/grpcinterface/btrdb_pb2.py | 297 +++---- btrdb/grpcinterface/btrdb_pb2.pyi | 1030 +++++++++++++------------ btrdb/grpcinterface/btrdb_pb2_grpc.py | 33 + 5 files changed, 754 insertions(+), 645 deletions(-) diff --git a/btrdb/endpoint.py b/btrdb/endpoint.py index 79a3b72..b526849 100644 --- a/btrdb/endpoint.py +++ b/btrdb/endpoint.py @@ -386,3 +386,25 @@ def sql_query(self, stmt, params: typing.List): for page in self.stub.SQLQuery(request): check_proto_stat(page.stat) yield page.SQLQueryRow + + @error_handler + def subscribe(self, update_queue): + def updates(): + while True: + update = update_queue.get() + if update is None: + return + (to_add, to_remove) = update + if len(to_add) != 0: + yield btrdb_pb2.SubscriptionUpdate( + op=0, uuid=[uu.bytes for uu in to_add] + ) + if len(to_remove) != 0: + yield btrdb_pb2.SubscriptionUpdate( + op=1, uuid=[uu.bytes for uu in to_remove] + ) + + for response in self.stub.Subscribe(updates()): + check_proto_stat(response.stat) + with pa.ipc.open_stream(response.arrowBytes) as reader: + yield uuid.UUID(bytes=response.uuid), reader.read_all() diff --git a/btrdb/grpcinterface/btrdb.proto b/btrdb/grpcinterface/btrdb.proto index 884a71c..61d2050 100644 --- a/btrdb/grpcinterface/btrdb.proto +++ b/btrdb/grpcinterface/btrdb.proto @@ -28,6 +28,7 @@ service BTrDB { rpc GetMetadataUsage(MetadataUsageParams) returns (MetadataUsageResponse); rpc GenerateCSV(GenerateCSVParams) returns (stream GenerateCSVResponse); rpc SQLQuery(SQLQueryParams) returns (stream SQLQueryResponse); + rpc Subscribe(stream SubscriptionUpdate) returns (stream SubscriptionResp); //rpc SetCompactionConfig(SetCompactionConfigParams) returns (SetCompactionConfigResponse); //rpc GetCompactionConfig(GetCompactionConfigParams) returns (GetCompactionConfigResponse); } @@ -426,3 +427,19 @@ message ReducedResolutionRange { int64 End = 2; uint32 Resolution = 3; } + +enum SubscriptionUpdateOp { + ADD_UUIDS = 0; + REMOVE_UUIDS = 1; +} + +message SubscriptionUpdate { + SubscriptionUpdateOp op = 1; + repeated bytes uuid = 2; +} + +message SubscriptionResp { + Status stat = 1; + bytes uuid = 2; + bytes arrowBytes = 3; +} diff --git a/btrdb/grpcinterface/btrdb_pb2.py b/btrdb/grpcinterface/btrdb_pb2.py index bed2b0d..58b6732 100644 --- a/btrdb/grpcinterface/btrdb_pb2.py +++ b/btrdb/grpcinterface/btrdb_pb2.py @@ -2,10 +2,10 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: btrdb.proto """Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -13,152 +13,157 @@ +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0b\x62trdb.proto\x12\x05v5api\"Q\n\x0fRawValuesParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\"}\n\x11RawValuesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x1f\n\x06values\x18\x04 \x03(\x0b\x32\x0f.v5api.RawPoint\"u\n\x16\x41rrowRawValuesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x12\n\narrowBytes\x18\x04 \x01(\x0c\"n\n\x16\x41rrowMultiValuesParams\x12\x0c\n\x04uuid\x18\x01 \x03(\x0c\x12\x14\n\x0cversionMajor\x18\x02 \x03(\x04\x12\r\n\x05start\x18\x03 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x04 \x01(\x10\x12\x14\n\x0csnapPeriodNs\x18\x05 \x01(\x03\"K\n\x18\x41rrowMultiValuesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x12\n\narrowBytes\x18\x02 \x01(\x0c\"*\n\x0bRawPointVec\x12\x0c\n\x04time\x18\x01 \x01(\x10\x12\r\n\x05value\x18\x02 \x03(\x01\"j\n\x14\x41lignedWindowsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\x12\x12\n\npointWidth\x18\x05 \x01(\r\"\x83\x01\n\x16\x41lignedWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12 \n\x06values\x18\x04 \x03(\x0b\x32\x10.v5api.StatPoint\"z\n\x1b\x41rrowAlignedWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x12\n\narrowBytes\x18\x04 \x01(\x0c\"m\n\rWindowsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\x12\r\n\x05width\x18\x05 \x01(\x04\x12\r\n\x05\x64\x65pth\x18\x06 \x01(\r\"|\n\x0fWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12 \n\x06values\x18\x04 \x03(\x0b\x32\x10.v5api.StatPoint\"s\n\x14\x41rrowWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x12\n\narrowBytes\x18\x04 \x01(\x0c\"h\n\x10StreamInfoParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x13\n\x0bomitVersion\x18\x02 \x01(\x08\x12\x16\n\x0eomitDescriptor\x18\x03 \x01(\x08\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"\x8a\x01\n\x12StreamInfoResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12+\n\ndescriptor\x18\x04 \x01(\x0b\x32\x17.v5api.StreamDescriptor\"\x98\x01\n\x10StreamDescriptor\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x12\n\ncollection\x18\x02 \x01(\t\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x17\n\x0fpropertyVersion\x18\x05 \x01(\x04\"\x82\x01\n\x1aSetStreamAnnotationsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x1f\n\x17\x65xpectedPropertyVersion\x18\x02 \x01(\x04\x12#\n\x07\x63hanges\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x10\n\x08removals\x18\x04 \x03(\t\";\n\x1cSetStreamAnnotationsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"\x8a\x01\n\x13SetStreamTagsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x1f\n\x17\x65xpectedPropertyVersion\x18\x02 \x01(\x04\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x12\n\ncollection\x18\x04 \x01(\t\x12\x0e\n\x06remove\x18\x05 \x03(\t\"4\n\x15SetStreamTagsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"{\n\x0c\x43reateParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x12\n\ncollection\x18\x02 \x01(\t\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\"-\n\x0e\x43reateResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"@\n\x13MetadataUsageParams\x12\x0e\n\x06prefix\x18\x01 \x01(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"y\n\x15MetadataUsageResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x1d\n\x04tags\x18\x02 \x03(\x0b\x32\x0f.v5api.KeyCount\x12$\n\x0b\x61nnotations\x18\x03 \x03(\x0b\x32\x0f.v5api.KeyCount\"&\n\x08KeyCount\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05\x63ount\x18\x02 \x01(\x04\"B\n\x15ListCollectionsParams\x12\x0e\n\x06prefix\x18\x01 \x01(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"K\n\x17ListCollectionsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x13\n\x0b\x63ollections\x18\x02 \x03(\t\"\xab\x01\n\x13LookupStreamsParams\x12\x12\n\ncollection\x18\x01 \x01(\t\x12\x1a\n\x12isCollectionPrefix\x18\x02 \x01(\x08\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"^\n\x15LookupStreamsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12(\n\x07results\x18\x02 \x03(\x0b\x32\x17.v5api.StreamDescriptor\"S\n\rNearestParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x0c\n\x04time\x18\x02 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x03 \x01(\x04\x12\x10\n\x08\x62\x61\x63kward\x18\x04 \x01(\x08\"z\n\x0fNearestResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x1e\n\x05value\x18\x04 \x01(\x0b\x32\x0f.v5api.RawPoint\"U\n\rChangesParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x11\n\tfromMajor\x18\x02 \x01(\x04\x12\x0f\n\x07toMajor\x18\x03 \x01(\x04\x12\x12\n\nresolution\x18\x04 \x01(\r\"\x7f\n\x0f\x43hangesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12#\n\x06ranges\x18\x04 \x03(\x0b\x32\x13.v5api.ChangedRange\"#\n\tRoundSpec\x12\x0e\n\x04\x62its\x18\x02 \x01(\x05H\x00\x42\x06\n\x04spec\"\x99\x01\n\x0cInsertParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x0c\n\x04sync\x18\x02 \x01(\x08\x12(\n\x0cmerge_policy\x18\x04 \x01(\x0e\x32\x12.v5api.MergePolicy\x12\"\n\x08rounding\x18\x05 \x01(\x0b\x32\x10.v5api.RoundSpec\x12\x1f\n\x06values\x18\x03 \x03(\x0b\x32\x0f.v5api.RawPoint\"\x91\x01\n\x11\x41rrowInsertParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x0c\n\x04sync\x18\x02 \x01(\x08\x12(\n\x0cmerge_policy\x18\x03 \x01(\x0e\x32\x12.v5api.MergePolicy\x12\"\n\x08rounding\x18\x04 \x01(\x0b\x32\x10.v5api.RoundSpec\x12\x12\n\narrowBytes\x18\x05 \x01(\x0c\"Y\n\x0eInsertResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\"8\n\x0c\x44\x65leteParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\"Y\n\x0e\x44\x65leteResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\"\x0c\n\nInfoParams\"\xa2\x01\n\x0cInfoResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x19\n\x04mash\x18\x02 \x01(\x0b\x32\x0b.v5api.Mash\x12\x14\n\x0cmajorVersion\x18\x03 \x01(\r\x12\x14\n\x0cminorVersion\x18\x04 \x01(\r\x12\r\n\x05\x62uild\x18\x05 \x01(\t\x12\x1f\n\x05proxy\x18\x06 \x01(\x0b\x32\x10.v5api.ProxyInfo\"#\n\tProxyInfo\x12\x16\n\x0eproxyEndpoints\x18\x01 \x03(\t\"1\n\x11\x46\x61ultInjectParams\x12\x0c\n\x04type\x18\x01 \x01(\x04\x12\x0e\n\x06params\x18\x02 \x01(\x0c\">\n\x13\x46\x61ultInjectResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\n\n\x02rv\x18\x02 \x01(\x0c\"\x1b\n\x0b\x46lushParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"X\n\rFlushResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\" \n\x10ObliterateParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"1\n\x12ObliterateResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"\'\n\x08RawPoint\x12\x0c\n\x04time\x18\x01 \x01(\x10\x12\r\n\x05value\x18\x02 \x01(\x01\"`\n\tStatPoint\x12\x0c\n\x04time\x18\x01 \x01(\x10\x12\x0b\n\x03min\x18\x02 \x01(\x01\x12\x0c\n\x04mean\x18\x03 \x01(\x01\x12\x0b\n\x03max\x18\x04 \x01(\x01\x12\r\n\x05\x63ount\x18\x05 \x01(\x06\x12\x0e\n\x06stddev\x18\x06 \x01(\x01\"*\n\x0c\x43hangedRange\x12\r\n\x05start\x18\x01 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x02 \x01(\x10\">\n\x06Status\x12\x0c\n\x04\x63ode\x18\x01 \x01(\r\x12\x0b\n\x03msg\x18\x02 \x01(\t\x12\x19\n\x04mash\x18\x03 \x01(\x0b\x32\x0b.v5api.Mash\"\x98\x01\n\x04Mash\x12\x10\n\x08revision\x18\x01 \x01(\x03\x12\x0e\n\x06leader\x18\x02 \x01(\t\x12\x16\n\x0eleaderRevision\x18\x03 \x01(\x03\x12\x13\n\x0btotalWeight\x18\x04 \x01(\x03\x12\x0f\n\x07healthy\x18\x05 \x01(\x08\x12\x10\n\x08unmapped\x18\x06 \x01(\x01\x12\x1e\n\x07members\x18\x07 \x03(\x0b\x32\r.v5api.Member\"\xc3\x01\n\x06Member\x12\x0c\n\x04hash\x18\x01 \x01(\r\x12\x10\n\x08nodename\x18\x02 \x01(\t\x12\n\n\x02up\x18\x03 \x01(\x08\x12\n\n\x02in\x18\x04 \x01(\x08\x12\x0f\n\x07\x65nabled\x18\x05 \x01(\x08\x12\r\n\x05start\x18\x06 \x01(\x03\x12\x0b\n\x03\x65nd\x18\x07 \x01(\x03\x12\x0e\n\x06weight\x18\x08 \x01(\x03\x12\x16\n\x0ereadPreference\x18\t \x01(\x01\x12\x15\n\rhttpEndpoints\x18\n \x01(\t\x12\x15\n\rgrpcEndpoints\x18\x0b \x01(\t\"8\n\x0bKeyOptValue\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x1c\n\x03val\x18\x02 \x01(\x0b\x32\x0f.v5api.OptValue\"\x19\n\x08OptValue\x12\r\n\x05value\x18\x01 \x01(\t\"&\n\x08KeyValue\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"?\n\x0fStreamCSVConfig\x12\x0f\n\x07version\x18\x01 \x01(\x04\x12\r\n\x05label\x18\x02 \x01(\t\x12\x0c\n\x04uuid\x18\x03 \x01(\x0c\"\x9d\x02\n\x11GenerateCSVParams\x12\x35\n\tqueryType\x18\x01 \x01(\x0e\x32\".v5api.GenerateCSVParams.QueryType\x12\x11\n\tstartTime\x18\x02 \x01(\x03\x12\x0f\n\x07\x65ndTime\x18\x03 \x01(\x03\x12\x12\n\nwindowSize\x18\x04 \x01(\x04\x12\r\n\x05\x64\x65pth\x18\x05 \x01(\r\x12\x17\n\x0fincludeVersions\x18\x06 \x01(\x08\x12\'\n\x07streams\x18\x07 \x03(\x0b\x32\x16.v5api.StreamCSVConfig\"H\n\tQueryType\x12\x19\n\x15\x41LIGNED_WINDOWS_QUERY\x10\x00\x12\x11\n\rWINDOWS_QUERY\x10\x01\x12\r\n\tRAW_QUERY\x10\x02\"Q\n\x13GenerateCSVResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x10\n\x08isHeader\x18\x02 \x01(\x08\x12\x0b\n\x03row\x18\x03 \x03(\t\"J\n\x0eSQLQueryParams\x12\r\n\x05query\x18\x01 \x01(\t\x12\x0e\n\x06params\x18\x02 \x03(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"D\n\x10SQLQueryResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x13\n\x0bSQLQueryRow\x18\x02 \x03(\x0c\"\x14\n\x04Role\x12\x0c\n\x04name\x18\x01 \x01(\t\"\x94\x01\n\x19SetCompactionConfigParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x18\n\x10\x43ompactedVersion\x18\x02 \x01(\x04\x12>\n\x17reducedResolutionRanges\x18\x03 \x03(\x0b\x32\x1d.v5api.ReducedResolutionRange\x12\x0f\n\x07unused0\x18\x04 \x01(\x04\":\n\x1bSetCompactionConfigResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\")\n\x19GetCompactionConfigParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"\xc1\x01\n\x1bGetCompactionConfigResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x1a\n\x12LatestMajorVersion\x18\x02 \x01(\x04\x12\x18\n\x10\x43ompactedVersion\x18\x03 \x01(\x04\x12>\n\x17reducedResolutionRanges\x18\x04 \x03(\x0b\x32\x1d.v5api.ReducedResolutionRange\x12\x0f\n\x07unused0\x18\x05 \x01(\x04\"H\n\x16ReducedResolutionRange\x12\r\n\x05Start\x18\x01 \x01(\x03\x12\x0b\n\x03\x45nd\x18\x02 \x01(\x03\x12\x12\n\nResolution\x18\x03 \x01(\r\"K\n\x12SubscriptionUpdate\x12\'\n\x02op\x18\x01 \x01(\x0e\x32\x1b.v5api.SubscriptionUpdateOp\x12\x0c\n\x04uuid\x18\x02 \x03(\x0c\"Q\n\x10SubscriptionResp\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x0c\n\x04uuid\x18\x02 \x01(\x0c\x12\x12\n\narrowBytes\x18\x03 \x01(\x0c*<\n\x0bMergePolicy\x12\t\n\x05NEVER\x10\x00\x12\t\n\x05\x45QUAL\x10\x01\x12\n\n\x06RETAIN\x10\x02\x12\x0b\n\x07REPLACE\x10\x03*7\n\x14SubscriptionUpdateOp\x12\r\n\tADD_UUIDS\x10\x00\x12\x10\n\x0cREMOVE_UUIDS\x10\x01\x32\xf8\r\n\x05\x42TrDB\x12?\n\tRawValues\x12\x16.v5api.RawValuesParams\x1a\x18.v5api.RawValuesResponse0\x01\x12I\n\x0e\x41rrowRawValues\x12\x16.v5api.RawValuesParams\x1a\x1d.v5api.ArrowRawValuesResponse0\x01\x12T\n\x10\x41rrowMultiValues\x12\x1d.v5api.ArrowMultiValuesParams\x1a\x1f.v5api.ArrowMultiValuesResponse0\x01\x12N\n\x0e\x41lignedWindows\x12\x1b.v5api.AlignedWindowsParams\x1a\x1d.v5api.AlignedWindowsResponse0\x01\x12X\n\x13\x41rrowAlignedWindows\x12\x1b.v5api.AlignedWindowsParams\x1a\".v5api.ArrowAlignedWindowsResponse0\x01\x12\x39\n\x07Windows\x12\x14.v5api.WindowsParams\x1a\x16.v5api.WindowsResponse0\x01\x12\x43\n\x0c\x41rrowWindows\x12\x14.v5api.WindowsParams\x1a\x1b.v5api.ArrowWindowsResponse0\x01\x12@\n\nStreamInfo\x12\x17.v5api.StreamInfoParams\x1a\x19.v5api.StreamInfoResponse\x12^\n\x14SetStreamAnnotations\x12!.v5api.SetStreamAnnotationsParams\x1a#.v5api.SetStreamAnnotationsResponse\x12I\n\rSetStreamTags\x12\x1a.v5api.SetStreamTagsParams\x1a\x1c.v5api.SetStreamTagsResponse\x12\x34\n\x06\x43reate\x12\x13.v5api.CreateParams\x1a\x15.v5api.CreateResponse\x12Q\n\x0fListCollections\x12\x1c.v5api.ListCollectionsParams\x1a\x1e.v5api.ListCollectionsResponse0\x01\x12K\n\rLookupStreams\x12\x1a.v5api.LookupStreamsParams\x1a\x1c.v5api.LookupStreamsResponse0\x01\x12\x37\n\x07Nearest\x12\x14.v5api.NearestParams\x1a\x16.v5api.NearestResponse\x12\x39\n\x07\x43hanges\x12\x14.v5api.ChangesParams\x1a\x16.v5api.ChangesResponse0\x01\x12\x34\n\x06Insert\x12\x13.v5api.InsertParams\x1a\x15.v5api.InsertResponse\x12>\n\x0b\x41rrowInsert\x12\x18.v5api.ArrowInsertParams\x1a\x15.v5api.InsertResponse\x12\x34\n\x06\x44\x65lete\x12\x13.v5api.DeleteParams\x1a\x15.v5api.DeleteResponse\x12.\n\x04Info\x12\x11.v5api.InfoParams\x1a\x13.v5api.InfoResponse\x12\x43\n\x0b\x46\x61ultInject\x12\x18.v5api.FaultInjectParams\x1a\x1a.v5api.FaultInjectResponse\x12\x31\n\x05\x46lush\x12\x12.v5api.FlushParams\x1a\x14.v5api.FlushResponse\x12@\n\nObliterate\x12\x17.v5api.ObliterateParams\x1a\x19.v5api.ObliterateResponse\x12L\n\x10GetMetadataUsage\x12\x1a.v5api.MetadataUsageParams\x1a\x1c.v5api.MetadataUsageResponse\x12\x45\n\x0bGenerateCSV\x12\x18.v5api.GenerateCSVParams\x1a\x1a.v5api.GenerateCSVResponse0\x01\x12<\n\x08SQLQuery\x12\x15.v5api.SQLQueryParams\x1a\x17.v5api.SQLQueryResponse0\x01\x12\x43\n\tSubscribe\x12\x19.v5api.SubscriptionUpdate\x1a\x17.v5api.SubscriptionResp(\x01\x30\x01\x62\x06proto3') -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0b\x62trdb.proto\x12\x05v5api\"Q\n\x0fRawValuesParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\"}\n\x11RawValuesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x1f\n\x06values\x18\x04 \x03(\x0b\x32\x0f.v5api.RawPoint\"u\n\x16\x41rrowRawValuesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x12\n\narrowBytes\x18\x04 \x01(\x0c\"n\n\x16\x41rrowMultiValuesParams\x12\x0c\n\x04uuid\x18\x01 \x03(\x0c\x12\x14\n\x0cversionMajor\x18\x02 \x03(\x04\x12\r\n\x05start\x18\x03 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x04 \x01(\x10\x12\x14\n\x0csnapPeriodNs\x18\x05 \x01(\x03\"K\n\x18\x41rrowMultiValuesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x12\n\narrowBytes\x18\x02 \x01(\x0c\"*\n\x0bRawPointVec\x12\x0c\n\x04time\x18\x01 \x01(\x10\x12\r\n\x05value\x18\x02 \x03(\x01\"j\n\x14\x41lignedWindowsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\x12\x12\n\npointWidth\x18\x05 \x01(\r\"\x83\x01\n\x16\x41lignedWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12 \n\x06values\x18\x04 \x03(\x0b\x32\x10.v5api.StatPoint\"z\n\x1b\x41rrowAlignedWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x12\n\narrowBytes\x18\x04 \x01(\x0c\"m\n\rWindowsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\x12\r\n\x05width\x18\x05 \x01(\x04\x12\r\n\x05\x64\x65pth\x18\x06 \x01(\r\"|\n\x0fWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12 \n\x06values\x18\x04 \x03(\x0b\x32\x10.v5api.StatPoint\"s\n\x14\x41rrowWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x12\n\narrowBytes\x18\x04 \x01(\x0c\"h\n\x10StreamInfoParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x13\n\x0bomitVersion\x18\x02 \x01(\x08\x12\x16\n\x0eomitDescriptor\x18\x03 \x01(\x08\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"\x8a\x01\n\x12StreamInfoResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12+\n\ndescriptor\x18\x04 \x01(\x0b\x32\x17.v5api.StreamDescriptor\"\x98\x01\n\x10StreamDescriptor\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x12\n\ncollection\x18\x02 \x01(\t\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x17\n\x0fpropertyVersion\x18\x05 \x01(\x04\"\x82\x01\n\x1aSetStreamAnnotationsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x1f\n\x17\x65xpectedPropertyVersion\x18\x02 \x01(\x04\x12#\n\x07\x63hanges\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x10\n\x08removals\x18\x04 \x03(\t\";\n\x1cSetStreamAnnotationsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"\x8a\x01\n\x13SetStreamTagsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x1f\n\x17\x65xpectedPropertyVersion\x18\x02 \x01(\x04\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x12\n\ncollection\x18\x04 \x01(\t\x12\x0e\n\x06remove\x18\x05 \x03(\t\"4\n\x15SetStreamTagsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"{\n\x0c\x43reateParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x12\n\ncollection\x18\x02 \x01(\t\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\"-\n\x0e\x43reateResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"@\n\x13MetadataUsageParams\x12\x0e\n\x06prefix\x18\x01 \x01(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"y\n\x15MetadataUsageResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x1d\n\x04tags\x18\x02 \x03(\x0b\x32\x0f.v5api.KeyCount\x12$\n\x0b\x61nnotations\x18\x03 \x03(\x0b\x32\x0f.v5api.KeyCount\"&\n\x08KeyCount\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05\x63ount\x18\x02 \x01(\x04\"B\n\x15ListCollectionsParams\x12\x0e\n\x06prefix\x18\x01 \x01(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"K\n\x17ListCollectionsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x13\n\x0b\x63ollections\x18\x02 \x03(\t\"\xab\x01\n\x13LookupStreamsParams\x12\x12\n\ncollection\x18\x01 \x01(\t\x12\x1a\n\x12isCollectionPrefix\x18\x02 \x01(\x08\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"^\n\x15LookupStreamsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12(\n\x07results\x18\x02 \x03(\x0b\x32\x17.v5api.StreamDescriptor\"S\n\rNearestParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x0c\n\x04time\x18\x02 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x03 \x01(\x04\x12\x10\n\x08\x62\x61\x63kward\x18\x04 \x01(\x08\"z\n\x0fNearestResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x1e\n\x05value\x18\x04 \x01(\x0b\x32\x0f.v5api.RawPoint\"U\n\rChangesParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x11\n\tfromMajor\x18\x02 \x01(\x04\x12\x0f\n\x07toMajor\x18\x03 \x01(\x04\x12\x12\n\nresolution\x18\x04 \x01(\r\"\x7f\n\x0f\x43hangesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12#\n\x06ranges\x18\x04 \x03(\x0b\x32\x13.v5api.ChangedRange\"#\n\tRoundSpec\x12\x0e\n\x04\x62its\x18\x02 \x01(\x05H\x00\x42\x06\n\x04spec\"\x99\x01\n\x0cInsertParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x0c\n\x04sync\x18\x02 \x01(\x08\x12(\n\x0cmerge_policy\x18\x04 \x01(\x0e\x32\x12.v5api.MergePolicy\x12\"\n\x08rounding\x18\x05 \x01(\x0b\x32\x10.v5api.RoundSpec\x12\x1f\n\x06values\x18\x03 \x03(\x0b\x32\x0f.v5api.RawPoint\"\x91\x01\n\x11\x41rrowInsertParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x0c\n\x04sync\x18\x02 \x01(\x08\x12(\n\x0cmerge_policy\x18\x03 \x01(\x0e\x32\x12.v5api.MergePolicy\x12\"\n\x08rounding\x18\x04 \x01(\x0b\x32\x10.v5api.RoundSpec\x12\x12\n\narrowBytes\x18\x05 \x01(\x0c\"Y\n\x0eInsertResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\"8\n\x0c\x44\x65leteParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\"Y\n\x0e\x44\x65leteResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\"\x0c\n\nInfoParams\"\xa2\x01\n\x0cInfoResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x19\n\x04mash\x18\x02 \x01(\x0b\x32\x0b.v5api.Mash\x12\x14\n\x0cmajorVersion\x18\x03 \x01(\r\x12\x14\n\x0cminorVersion\x18\x04 \x01(\r\x12\r\n\x05\x62uild\x18\x05 \x01(\t\x12\x1f\n\x05proxy\x18\x06 \x01(\x0b\x32\x10.v5api.ProxyInfo\"#\n\tProxyInfo\x12\x16\n\x0eproxyEndpoints\x18\x01 \x03(\t\"1\n\x11\x46\x61ultInjectParams\x12\x0c\n\x04type\x18\x01 \x01(\x04\x12\x0e\n\x06params\x18\x02 \x01(\x0c\">\n\x13\x46\x61ultInjectResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\n\n\x02rv\x18\x02 \x01(\x0c\"\x1b\n\x0b\x46lushParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"X\n\rFlushResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\" \n\x10ObliterateParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"1\n\x12ObliterateResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"\'\n\x08RawPoint\x12\x0c\n\x04time\x18\x01 \x01(\x10\x12\r\n\x05value\x18\x02 \x01(\x01\"`\n\tStatPoint\x12\x0c\n\x04time\x18\x01 \x01(\x10\x12\x0b\n\x03min\x18\x02 \x01(\x01\x12\x0c\n\x04mean\x18\x03 \x01(\x01\x12\x0b\n\x03max\x18\x04 \x01(\x01\x12\r\n\x05\x63ount\x18\x05 \x01(\x06\x12\x0e\n\x06stddev\x18\x06 \x01(\x01\"*\n\x0c\x43hangedRange\x12\r\n\x05start\x18\x01 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x02 \x01(\x10\">\n\x06Status\x12\x0c\n\x04\x63ode\x18\x01 \x01(\r\x12\x0b\n\x03msg\x18\x02 \x01(\t\x12\x19\n\x04mash\x18\x03 \x01(\x0b\x32\x0b.v5api.Mash\"\x98\x01\n\x04Mash\x12\x10\n\x08revision\x18\x01 \x01(\x03\x12\x0e\n\x06leader\x18\x02 \x01(\t\x12\x16\n\x0eleaderRevision\x18\x03 \x01(\x03\x12\x13\n\x0btotalWeight\x18\x04 \x01(\x03\x12\x0f\n\x07healthy\x18\x05 \x01(\x08\x12\x10\n\x08unmapped\x18\x06 \x01(\x01\x12\x1e\n\x07members\x18\x07 \x03(\x0b\x32\r.v5api.Member\"\xc3\x01\n\x06Member\x12\x0c\n\x04hash\x18\x01 \x01(\r\x12\x10\n\x08nodename\x18\x02 \x01(\t\x12\n\n\x02up\x18\x03 \x01(\x08\x12\n\n\x02in\x18\x04 \x01(\x08\x12\x0f\n\x07\x65nabled\x18\x05 \x01(\x08\x12\r\n\x05start\x18\x06 \x01(\x03\x12\x0b\n\x03\x65nd\x18\x07 \x01(\x03\x12\x0e\n\x06weight\x18\x08 \x01(\x03\x12\x16\n\x0ereadPreference\x18\t \x01(\x01\x12\x15\n\rhttpEndpoints\x18\n \x01(\t\x12\x15\n\rgrpcEndpoints\x18\x0b \x01(\t\"8\n\x0bKeyOptValue\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x1c\n\x03val\x18\x02 \x01(\x0b\x32\x0f.v5api.OptValue\"\x19\n\x08OptValue\x12\r\n\x05value\x18\x01 \x01(\t\"&\n\x08KeyValue\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"?\n\x0fStreamCSVConfig\x12\x0f\n\x07version\x18\x01 \x01(\x04\x12\r\n\x05label\x18\x02 \x01(\t\x12\x0c\n\x04uuid\x18\x03 \x01(\x0c\"\x9d\x02\n\x11GenerateCSVParams\x12\x35\n\tqueryType\x18\x01 \x01(\x0e\x32\".v5api.GenerateCSVParams.QueryType\x12\x11\n\tstartTime\x18\x02 \x01(\x03\x12\x0f\n\x07\x65ndTime\x18\x03 \x01(\x03\x12\x12\n\nwindowSize\x18\x04 \x01(\x04\x12\r\n\x05\x64\x65pth\x18\x05 \x01(\r\x12\x17\n\x0fincludeVersions\x18\x06 \x01(\x08\x12\'\n\x07streams\x18\x07 \x03(\x0b\x32\x16.v5api.StreamCSVConfig\"H\n\tQueryType\x12\x19\n\x15\x41LIGNED_WINDOWS_QUERY\x10\x00\x12\x11\n\rWINDOWS_QUERY\x10\x01\x12\r\n\tRAW_QUERY\x10\x02\"Q\n\x13GenerateCSVResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x10\n\x08isHeader\x18\x02 \x01(\x08\x12\x0b\n\x03row\x18\x03 \x03(\t\"J\n\x0eSQLQueryParams\x12\r\n\x05query\x18\x01 \x01(\t\x12\x0e\n\x06params\x18\x02 \x03(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"D\n\x10SQLQueryResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x13\n\x0bSQLQueryRow\x18\x02 \x03(\x0c\"\x14\n\x04Role\x12\x0c\n\x04name\x18\x01 \x01(\t\"\x94\x01\n\x19SetCompactionConfigParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x18\n\x10\x43ompactedVersion\x18\x02 \x01(\x04\x12>\n\x17reducedResolutionRanges\x18\x03 \x03(\x0b\x32\x1d.v5api.ReducedResolutionRange\x12\x0f\n\x07unused0\x18\x04 \x01(\x04\":\n\x1bSetCompactionConfigResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\")\n\x19GetCompactionConfigParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"\xc1\x01\n\x1bGetCompactionConfigResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x1a\n\x12LatestMajorVersion\x18\x02 \x01(\x04\x12\x18\n\x10\x43ompactedVersion\x18\x03 \x01(\x04\x12>\n\x17reducedResolutionRanges\x18\x04 \x03(\x0b\x32\x1d.v5api.ReducedResolutionRange\x12\x0f\n\x07unused0\x18\x05 \x01(\x04\"H\n\x16ReducedResolutionRange\x12\r\n\x05Start\x18\x01 \x01(\x03\x12\x0b\n\x03\x45nd\x18\x02 \x01(\x03\x12\x12\n\nResolution\x18\x03 \x01(\r*<\n\x0bMergePolicy\x12\t\n\x05NEVER\x10\x00\x12\t\n\x05\x45QUAL\x10\x01\x12\n\n\x06RETAIN\x10\x02\x12\x0b\n\x07REPLACE\x10\x03\x32\xb3\r\n\x05\x42TrDB\x12?\n\tRawValues\x12\x16.v5api.RawValuesParams\x1a\x18.v5api.RawValuesResponse0\x01\x12I\n\x0e\x41rrowRawValues\x12\x16.v5api.RawValuesParams\x1a\x1d.v5api.ArrowRawValuesResponse0\x01\x12T\n\x10\x41rrowMultiValues\x12\x1d.v5api.ArrowMultiValuesParams\x1a\x1f.v5api.ArrowMultiValuesResponse0\x01\x12N\n\x0e\x41lignedWindows\x12\x1b.v5api.AlignedWindowsParams\x1a\x1d.v5api.AlignedWindowsResponse0\x01\x12X\n\x13\x41rrowAlignedWindows\x12\x1b.v5api.AlignedWindowsParams\x1a\".v5api.ArrowAlignedWindowsResponse0\x01\x12\x39\n\x07Windows\x12\x14.v5api.WindowsParams\x1a\x16.v5api.WindowsResponse0\x01\x12\x43\n\x0c\x41rrowWindows\x12\x14.v5api.WindowsParams\x1a\x1b.v5api.ArrowWindowsResponse0\x01\x12@\n\nStreamInfo\x12\x17.v5api.StreamInfoParams\x1a\x19.v5api.StreamInfoResponse\x12^\n\x14SetStreamAnnotations\x12!.v5api.SetStreamAnnotationsParams\x1a#.v5api.SetStreamAnnotationsResponse\x12I\n\rSetStreamTags\x12\x1a.v5api.SetStreamTagsParams\x1a\x1c.v5api.SetStreamTagsResponse\x12\x34\n\x06\x43reate\x12\x13.v5api.CreateParams\x1a\x15.v5api.CreateResponse\x12Q\n\x0fListCollections\x12\x1c.v5api.ListCollectionsParams\x1a\x1e.v5api.ListCollectionsResponse0\x01\x12K\n\rLookupStreams\x12\x1a.v5api.LookupStreamsParams\x1a\x1c.v5api.LookupStreamsResponse0\x01\x12\x37\n\x07Nearest\x12\x14.v5api.NearestParams\x1a\x16.v5api.NearestResponse\x12\x39\n\x07\x43hanges\x12\x14.v5api.ChangesParams\x1a\x16.v5api.ChangesResponse0\x01\x12\x34\n\x06Insert\x12\x13.v5api.InsertParams\x1a\x15.v5api.InsertResponse\x12>\n\x0b\x41rrowInsert\x12\x18.v5api.ArrowInsertParams\x1a\x15.v5api.InsertResponse\x12\x34\n\x06\x44\x65lete\x12\x13.v5api.DeleteParams\x1a\x15.v5api.DeleteResponse\x12.\n\x04Info\x12\x11.v5api.InfoParams\x1a\x13.v5api.InfoResponse\x12\x43\n\x0b\x46\x61ultInject\x12\x18.v5api.FaultInjectParams\x1a\x1a.v5api.FaultInjectResponse\x12\x31\n\x05\x46lush\x12\x12.v5api.FlushParams\x1a\x14.v5api.FlushResponse\x12@\n\nObliterate\x12\x17.v5api.ObliterateParams\x1a\x19.v5api.ObliterateResponse\x12L\n\x10GetMetadataUsage\x12\x1a.v5api.MetadataUsageParams\x1a\x1c.v5api.MetadataUsageResponse\x12\x45\n\x0bGenerateCSV\x12\x18.v5api.GenerateCSVParams\x1a\x1a.v5api.GenerateCSVResponse0\x01\x12<\n\x08SQLQuery\x12\x15.v5api.SQLQueryParams\x1a\x17.v5api.SQLQueryResponse0\x01\x62\x06proto3') - -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'btrdb_pb2', globals()) +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'btrdb_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: - DESCRIPTOR._options = None - _MERGEPOLICY._serialized_start=6305 - _MERGEPOLICY._serialized_end=6365 - _RAWVALUESPARAMS._serialized_start=22 - _RAWVALUESPARAMS._serialized_end=103 - _RAWVALUESRESPONSE._serialized_start=105 - _RAWVALUESRESPONSE._serialized_end=230 - _ARROWRAWVALUESRESPONSE._serialized_start=232 - _ARROWRAWVALUESRESPONSE._serialized_end=349 - _ARROWMULTIVALUESPARAMS._serialized_start=351 - _ARROWMULTIVALUESPARAMS._serialized_end=461 - _ARROWMULTIVALUESRESPONSE._serialized_start=463 - _ARROWMULTIVALUESRESPONSE._serialized_end=538 - _RAWPOINTVEC._serialized_start=540 - _RAWPOINTVEC._serialized_end=582 - _ALIGNEDWINDOWSPARAMS._serialized_start=584 - _ALIGNEDWINDOWSPARAMS._serialized_end=690 - _ALIGNEDWINDOWSRESPONSE._serialized_start=693 - _ALIGNEDWINDOWSRESPONSE._serialized_end=824 - _ARROWALIGNEDWINDOWSRESPONSE._serialized_start=826 - _ARROWALIGNEDWINDOWSRESPONSE._serialized_end=948 - _WINDOWSPARAMS._serialized_start=950 - _WINDOWSPARAMS._serialized_end=1059 - _WINDOWSRESPONSE._serialized_start=1061 - _WINDOWSRESPONSE._serialized_end=1185 - _ARROWWINDOWSRESPONSE._serialized_start=1187 - _ARROWWINDOWSRESPONSE._serialized_end=1302 - _STREAMINFOPARAMS._serialized_start=1304 - _STREAMINFOPARAMS._serialized_end=1408 - _STREAMINFORESPONSE._serialized_start=1411 - _STREAMINFORESPONSE._serialized_end=1549 - _STREAMDESCRIPTOR._serialized_start=1552 - _STREAMDESCRIPTOR._serialized_end=1704 - _SETSTREAMANNOTATIONSPARAMS._serialized_start=1707 - _SETSTREAMANNOTATIONSPARAMS._serialized_end=1837 - _SETSTREAMANNOTATIONSRESPONSE._serialized_start=1839 - _SETSTREAMANNOTATIONSRESPONSE._serialized_end=1898 - _SETSTREAMTAGSPARAMS._serialized_start=1901 - _SETSTREAMTAGSPARAMS._serialized_end=2039 - _SETSTREAMTAGSRESPONSE._serialized_start=2041 - _SETSTREAMTAGSRESPONSE._serialized_end=2093 - _CREATEPARAMS._serialized_start=2095 - _CREATEPARAMS._serialized_end=2218 - _CREATERESPONSE._serialized_start=2220 - _CREATERESPONSE._serialized_end=2265 - _METADATAUSAGEPARAMS._serialized_start=2267 - _METADATAUSAGEPARAMS._serialized_end=2331 - _METADATAUSAGERESPONSE._serialized_start=2333 - _METADATAUSAGERESPONSE._serialized_end=2454 - _KEYCOUNT._serialized_start=2456 - _KEYCOUNT._serialized_end=2494 - _LISTCOLLECTIONSPARAMS._serialized_start=2496 - _LISTCOLLECTIONSPARAMS._serialized_end=2562 - _LISTCOLLECTIONSRESPONSE._serialized_start=2564 - _LISTCOLLECTIONSRESPONSE._serialized_end=2639 - _LOOKUPSTREAMSPARAMS._serialized_start=2642 - _LOOKUPSTREAMSPARAMS._serialized_end=2813 - _LOOKUPSTREAMSRESPONSE._serialized_start=2815 - _LOOKUPSTREAMSRESPONSE._serialized_end=2909 - _NEARESTPARAMS._serialized_start=2911 - _NEARESTPARAMS._serialized_end=2994 - _NEARESTRESPONSE._serialized_start=2996 - _NEARESTRESPONSE._serialized_end=3118 - _CHANGESPARAMS._serialized_start=3120 - _CHANGESPARAMS._serialized_end=3205 - _CHANGESRESPONSE._serialized_start=3207 - _CHANGESRESPONSE._serialized_end=3334 - _ROUNDSPEC._serialized_start=3336 - _ROUNDSPEC._serialized_end=3371 - _INSERTPARAMS._serialized_start=3374 - _INSERTPARAMS._serialized_end=3527 - _ARROWINSERTPARAMS._serialized_start=3530 - _ARROWINSERTPARAMS._serialized_end=3675 - _INSERTRESPONSE._serialized_start=3677 - _INSERTRESPONSE._serialized_end=3766 - _DELETEPARAMS._serialized_start=3768 - _DELETEPARAMS._serialized_end=3824 - _DELETERESPONSE._serialized_start=3826 - _DELETERESPONSE._serialized_end=3915 - _INFOPARAMS._serialized_start=3917 - _INFOPARAMS._serialized_end=3929 - _INFORESPONSE._serialized_start=3932 - _INFORESPONSE._serialized_end=4094 - _PROXYINFO._serialized_start=4096 - _PROXYINFO._serialized_end=4131 - _FAULTINJECTPARAMS._serialized_start=4133 - _FAULTINJECTPARAMS._serialized_end=4182 - _FAULTINJECTRESPONSE._serialized_start=4184 - _FAULTINJECTRESPONSE._serialized_end=4246 - _FLUSHPARAMS._serialized_start=4248 - _FLUSHPARAMS._serialized_end=4275 - _FLUSHRESPONSE._serialized_start=4277 - _FLUSHRESPONSE._serialized_end=4365 - _OBLITERATEPARAMS._serialized_start=4367 - _OBLITERATEPARAMS._serialized_end=4399 - _OBLITERATERESPONSE._serialized_start=4401 - _OBLITERATERESPONSE._serialized_end=4450 - _RAWPOINT._serialized_start=4452 - _RAWPOINT._serialized_end=4491 - _STATPOINT._serialized_start=4493 - _STATPOINT._serialized_end=4589 - _CHANGEDRANGE._serialized_start=4591 - _CHANGEDRANGE._serialized_end=4633 - _STATUS._serialized_start=4635 - _STATUS._serialized_end=4697 - _MASH._serialized_start=4700 - _MASH._serialized_end=4852 - _MEMBER._serialized_start=4855 - _MEMBER._serialized_end=5050 - _KEYOPTVALUE._serialized_start=5052 - _KEYOPTVALUE._serialized_end=5108 - _OPTVALUE._serialized_start=5110 - _OPTVALUE._serialized_end=5135 - _KEYVALUE._serialized_start=5137 - _KEYVALUE._serialized_end=5175 - _STREAMCSVCONFIG._serialized_start=5177 - _STREAMCSVCONFIG._serialized_end=5240 - _GENERATECSVPARAMS._serialized_start=5243 - _GENERATECSVPARAMS._serialized_end=5528 - _GENERATECSVPARAMS_QUERYTYPE._serialized_start=5456 - _GENERATECSVPARAMS_QUERYTYPE._serialized_end=5528 - _GENERATECSVRESPONSE._serialized_start=5530 - _GENERATECSVRESPONSE._serialized_end=5611 - _SQLQUERYPARAMS._serialized_start=5613 - _SQLQUERYPARAMS._serialized_end=5687 - _SQLQUERYRESPONSE._serialized_start=5689 - _SQLQUERYRESPONSE._serialized_end=5757 - _ROLE._serialized_start=5759 - _ROLE._serialized_end=5779 - _SETCOMPACTIONCONFIGPARAMS._serialized_start=5782 - _SETCOMPACTIONCONFIGPARAMS._serialized_end=5930 - _SETCOMPACTIONCONFIGRESPONSE._serialized_start=5932 - _SETCOMPACTIONCONFIGRESPONSE._serialized_end=5990 - _GETCOMPACTIONCONFIGPARAMS._serialized_start=5992 - _GETCOMPACTIONCONFIGPARAMS._serialized_end=6033 - _GETCOMPACTIONCONFIGRESPONSE._serialized_start=6036 - _GETCOMPACTIONCONFIGRESPONSE._serialized_end=6229 - _REDUCEDRESOLUTIONRANGE._serialized_start=6231 - _REDUCEDRESOLUTIONRANGE._serialized_end=6303 - _BTRDB._serialized_start=6368 - _BTRDB._serialized_end=8083 + _globals['_MERGEPOLICY']._serialized_start=6465 + _globals['_MERGEPOLICY']._serialized_end=6525 + _globals['_SUBSCRIPTIONUPDATEOP']._serialized_start=6527 + _globals['_SUBSCRIPTIONUPDATEOP']._serialized_end=6582 + _globals['_RAWVALUESPARAMS']._serialized_start=22 + _globals['_RAWVALUESPARAMS']._serialized_end=103 + _globals['_RAWVALUESRESPONSE']._serialized_start=105 + _globals['_RAWVALUESRESPONSE']._serialized_end=230 + _globals['_ARROWRAWVALUESRESPONSE']._serialized_start=232 + _globals['_ARROWRAWVALUESRESPONSE']._serialized_end=349 + _globals['_ARROWMULTIVALUESPARAMS']._serialized_start=351 + _globals['_ARROWMULTIVALUESPARAMS']._serialized_end=461 + _globals['_ARROWMULTIVALUESRESPONSE']._serialized_start=463 + _globals['_ARROWMULTIVALUESRESPONSE']._serialized_end=538 + _globals['_RAWPOINTVEC']._serialized_start=540 + _globals['_RAWPOINTVEC']._serialized_end=582 + _globals['_ALIGNEDWINDOWSPARAMS']._serialized_start=584 + _globals['_ALIGNEDWINDOWSPARAMS']._serialized_end=690 + _globals['_ALIGNEDWINDOWSRESPONSE']._serialized_start=693 + _globals['_ALIGNEDWINDOWSRESPONSE']._serialized_end=824 + _globals['_ARROWALIGNEDWINDOWSRESPONSE']._serialized_start=826 + _globals['_ARROWALIGNEDWINDOWSRESPONSE']._serialized_end=948 + _globals['_WINDOWSPARAMS']._serialized_start=950 + _globals['_WINDOWSPARAMS']._serialized_end=1059 + _globals['_WINDOWSRESPONSE']._serialized_start=1061 + _globals['_WINDOWSRESPONSE']._serialized_end=1185 + _globals['_ARROWWINDOWSRESPONSE']._serialized_start=1187 + _globals['_ARROWWINDOWSRESPONSE']._serialized_end=1302 + _globals['_STREAMINFOPARAMS']._serialized_start=1304 + _globals['_STREAMINFOPARAMS']._serialized_end=1408 + _globals['_STREAMINFORESPONSE']._serialized_start=1411 + _globals['_STREAMINFORESPONSE']._serialized_end=1549 + _globals['_STREAMDESCRIPTOR']._serialized_start=1552 + _globals['_STREAMDESCRIPTOR']._serialized_end=1704 + _globals['_SETSTREAMANNOTATIONSPARAMS']._serialized_start=1707 + _globals['_SETSTREAMANNOTATIONSPARAMS']._serialized_end=1837 + _globals['_SETSTREAMANNOTATIONSRESPONSE']._serialized_start=1839 + _globals['_SETSTREAMANNOTATIONSRESPONSE']._serialized_end=1898 + _globals['_SETSTREAMTAGSPARAMS']._serialized_start=1901 + _globals['_SETSTREAMTAGSPARAMS']._serialized_end=2039 + _globals['_SETSTREAMTAGSRESPONSE']._serialized_start=2041 + _globals['_SETSTREAMTAGSRESPONSE']._serialized_end=2093 + _globals['_CREATEPARAMS']._serialized_start=2095 + _globals['_CREATEPARAMS']._serialized_end=2218 + _globals['_CREATERESPONSE']._serialized_start=2220 + _globals['_CREATERESPONSE']._serialized_end=2265 + _globals['_METADATAUSAGEPARAMS']._serialized_start=2267 + _globals['_METADATAUSAGEPARAMS']._serialized_end=2331 + _globals['_METADATAUSAGERESPONSE']._serialized_start=2333 + _globals['_METADATAUSAGERESPONSE']._serialized_end=2454 + _globals['_KEYCOUNT']._serialized_start=2456 + _globals['_KEYCOUNT']._serialized_end=2494 + _globals['_LISTCOLLECTIONSPARAMS']._serialized_start=2496 + _globals['_LISTCOLLECTIONSPARAMS']._serialized_end=2562 + _globals['_LISTCOLLECTIONSRESPONSE']._serialized_start=2564 + _globals['_LISTCOLLECTIONSRESPONSE']._serialized_end=2639 + _globals['_LOOKUPSTREAMSPARAMS']._serialized_start=2642 + _globals['_LOOKUPSTREAMSPARAMS']._serialized_end=2813 + _globals['_LOOKUPSTREAMSRESPONSE']._serialized_start=2815 + _globals['_LOOKUPSTREAMSRESPONSE']._serialized_end=2909 + _globals['_NEARESTPARAMS']._serialized_start=2911 + _globals['_NEARESTPARAMS']._serialized_end=2994 + _globals['_NEARESTRESPONSE']._serialized_start=2996 + _globals['_NEARESTRESPONSE']._serialized_end=3118 + _globals['_CHANGESPARAMS']._serialized_start=3120 + _globals['_CHANGESPARAMS']._serialized_end=3205 + _globals['_CHANGESRESPONSE']._serialized_start=3207 + _globals['_CHANGESRESPONSE']._serialized_end=3334 + _globals['_ROUNDSPEC']._serialized_start=3336 + _globals['_ROUNDSPEC']._serialized_end=3371 + _globals['_INSERTPARAMS']._serialized_start=3374 + _globals['_INSERTPARAMS']._serialized_end=3527 + _globals['_ARROWINSERTPARAMS']._serialized_start=3530 + _globals['_ARROWINSERTPARAMS']._serialized_end=3675 + _globals['_INSERTRESPONSE']._serialized_start=3677 + _globals['_INSERTRESPONSE']._serialized_end=3766 + _globals['_DELETEPARAMS']._serialized_start=3768 + _globals['_DELETEPARAMS']._serialized_end=3824 + _globals['_DELETERESPONSE']._serialized_start=3826 + _globals['_DELETERESPONSE']._serialized_end=3915 + _globals['_INFOPARAMS']._serialized_start=3917 + _globals['_INFOPARAMS']._serialized_end=3929 + _globals['_INFORESPONSE']._serialized_start=3932 + _globals['_INFORESPONSE']._serialized_end=4094 + _globals['_PROXYINFO']._serialized_start=4096 + _globals['_PROXYINFO']._serialized_end=4131 + _globals['_FAULTINJECTPARAMS']._serialized_start=4133 + _globals['_FAULTINJECTPARAMS']._serialized_end=4182 + _globals['_FAULTINJECTRESPONSE']._serialized_start=4184 + _globals['_FAULTINJECTRESPONSE']._serialized_end=4246 + _globals['_FLUSHPARAMS']._serialized_start=4248 + _globals['_FLUSHPARAMS']._serialized_end=4275 + _globals['_FLUSHRESPONSE']._serialized_start=4277 + _globals['_FLUSHRESPONSE']._serialized_end=4365 + _globals['_OBLITERATEPARAMS']._serialized_start=4367 + _globals['_OBLITERATEPARAMS']._serialized_end=4399 + _globals['_OBLITERATERESPONSE']._serialized_start=4401 + _globals['_OBLITERATERESPONSE']._serialized_end=4450 + _globals['_RAWPOINT']._serialized_start=4452 + _globals['_RAWPOINT']._serialized_end=4491 + _globals['_STATPOINT']._serialized_start=4493 + _globals['_STATPOINT']._serialized_end=4589 + _globals['_CHANGEDRANGE']._serialized_start=4591 + _globals['_CHANGEDRANGE']._serialized_end=4633 + _globals['_STATUS']._serialized_start=4635 + _globals['_STATUS']._serialized_end=4697 + _globals['_MASH']._serialized_start=4700 + _globals['_MASH']._serialized_end=4852 + _globals['_MEMBER']._serialized_start=4855 + _globals['_MEMBER']._serialized_end=5050 + _globals['_KEYOPTVALUE']._serialized_start=5052 + _globals['_KEYOPTVALUE']._serialized_end=5108 + _globals['_OPTVALUE']._serialized_start=5110 + _globals['_OPTVALUE']._serialized_end=5135 + _globals['_KEYVALUE']._serialized_start=5137 + _globals['_KEYVALUE']._serialized_end=5175 + _globals['_STREAMCSVCONFIG']._serialized_start=5177 + _globals['_STREAMCSVCONFIG']._serialized_end=5240 + _globals['_GENERATECSVPARAMS']._serialized_start=5243 + _globals['_GENERATECSVPARAMS']._serialized_end=5528 + _globals['_GENERATECSVPARAMS_QUERYTYPE']._serialized_start=5456 + _globals['_GENERATECSVPARAMS_QUERYTYPE']._serialized_end=5528 + _globals['_GENERATECSVRESPONSE']._serialized_start=5530 + _globals['_GENERATECSVRESPONSE']._serialized_end=5611 + _globals['_SQLQUERYPARAMS']._serialized_start=5613 + _globals['_SQLQUERYPARAMS']._serialized_end=5687 + _globals['_SQLQUERYRESPONSE']._serialized_start=5689 + _globals['_SQLQUERYRESPONSE']._serialized_end=5757 + _globals['_ROLE']._serialized_start=5759 + _globals['_ROLE']._serialized_end=5779 + _globals['_SETCOMPACTIONCONFIGPARAMS']._serialized_start=5782 + _globals['_SETCOMPACTIONCONFIGPARAMS']._serialized_end=5930 + _globals['_SETCOMPACTIONCONFIGRESPONSE']._serialized_start=5932 + _globals['_SETCOMPACTIONCONFIGRESPONSE']._serialized_end=5990 + _globals['_GETCOMPACTIONCONFIGPARAMS']._serialized_start=5992 + _globals['_GETCOMPACTIONCONFIGPARAMS']._serialized_end=6033 + _globals['_GETCOMPACTIONCONFIGRESPONSE']._serialized_start=6036 + _globals['_GETCOMPACTIONCONFIGRESPONSE']._serialized_end=6229 + _globals['_REDUCEDRESOLUTIONRANGE']._serialized_start=6231 + _globals['_REDUCEDRESOLUTIONRANGE']._serialized_end=6303 + _globals['_SUBSCRIPTIONUPDATE']._serialized_start=6305 + _globals['_SUBSCRIPTIONUPDATE']._serialized_end=6380 + _globals['_SUBSCRIPTIONRESP']._serialized_start=6382 + _globals['_SUBSCRIPTIONRESP']._serialized_end=6463 + _globals['_BTRDB']._serialized_start=6585 + _globals['_BTRDB']._serialized_end=8369 # @@protoc_insertion_point(module_scope) diff --git a/btrdb/grpcinterface/btrdb_pb2.pyi b/btrdb/grpcinterface/btrdb_pb2.pyi index ff9b6fc..9573a2c 100644 --- a/btrdb/grpcinterface/btrdb_pb2.pyi +++ b/btrdb/grpcinterface/btrdb_pb2.pyi @@ -5,332 +5,289 @@ from google.protobuf import message as _message from typing import ClassVar as _ClassVar, Iterable as _Iterable, Mapping as _Mapping, Optional as _Optional, Union as _Union DESCRIPTOR: _descriptor.FileDescriptor -EQUAL: MergePolicy + +class MergePolicy(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = [] + NEVER: _ClassVar[MergePolicy] + EQUAL: _ClassVar[MergePolicy] + RETAIN: _ClassVar[MergePolicy] + REPLACE: _ClassVar[MergePolicy] + +class SubscriptionUpdateOp(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = [] + ADD_UUIDS: _ClassVar[SubscriptionUpdateOp] + REMOVE_UUIDS: _ClassVar[SubscriptionUpdateOp] NEVER: MergePolicy -REPLACE: MergePolicy +EQUAL: MergePolicy RETAIN: MergePolicy +REPLACE: MergePolicy +ADD_UUIDS: SubscriptionUpdateOp +REMOVE_UUIDS: SubscriptionUpdateOp -class AlignedWindowsParams(_message.Message): - __slots__ = ["end", "pointWidth", "start", "uuid", "versionMajor"] - END_FIELD_NUMBER: _ClassVar[int] - POINTWIDTH_FIELD_NUMBER: _ClassVar[int] - START_FIELD_NUMBER: _ClassVar[int] +class RawValuesParams(_message.Message): + __slots__ = ["uuid", "start", "end", "versionMajor"] UUID_FIELD_NUMBER: _ClassVar[int] + START_FIELD_NUMBER: _ClassVar[int] + END_FIELD_NUMBER: _ClassVar[int] VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] - end: int - pointWidth: int - start: int uuid: bytes + start: int + end: int versionMajor: int - def __init__(self, uuid: _Optional[bytes] = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., versionMajor: _Optional[int] = ..., pointWidth: _Optional[int] = ...) -> None: ... + def __init__(self, uuid: _Optional[bytes] = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., versionMajor: _Optional[int] = ...) -> None: ... -class AlignedWindowsResponse(_message.Message): - __slots__ = ["stat", "values", "versionMajor", "versionMinor"] +class RawValuesResponse(_message.Message): + __slots__ = ["stat", "versionMajor", "versionMinor", "values"] STAT_FIELD_NUMBER: _ClassVar[int] - VALUES_FIELD_NUMBER: _ClassVar[int] VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + VALUES_FIELD_NUMBER: _ClassVar[int] stat: Status - values: _containers.RepeatedCompositeFieldContainer[StatPoint] versionMajor: int versionMinor: int - def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., values: _Optional[_Iterable[_Union[StatPoint, _Mapping]]] = ...) -> None: ... + values: _containers.RepeatedCompositeFieldContainer[RawPoint] + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., values: _Optional[_Iterable[_Union[RawPoint, _Mapping]]] = ...) -> None: ... -class ArrowAlignedWindowsResponse(_message.Message): - __slots__ = ["arrowBytes", "stat", "versionMajor", "versionMinor"] - ARROWBYTES_FIELD_NUMBER: _ClassVar[int] +class ArrowRawValuesResponse(_message.Message): + __slots__ = ["stat", "versionMajor", "versionMinor", "arrowBytes"] STAT_FIELD_NUMBER: _ClassVar[int] VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] - arrowBytes: bytes + ARROWBYTES_FIELD_NUMBER: _ClassVar[int] stat: Status versionMajor: int versionMinor: int - def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., arrowBytes: _Optional[bytes] = ...) -> None: ... - -class ArrowInsertParams(_message.Message): - __slots__ = ["arrowBytes", "merge_policy", "rounding", "sync", "uuid"] - ARROWBYTES_FIELD_NUMBER: _ClassVar[int] - MERGE_POLICY_FIELD_NUMBER: _ClassVar[int] - ROUNDING_FIELD_NUMBER: _ClassVar[int] - SYNC_FIELD_NUMBER: _ClassVar[int] - UUID_FIELD_NUMBER: _ClassVar[int] arrowBytes: bytes - merge_policy: MergePolicy - rounding: RoundSpec - sync: bool - uuid: bytes - def __init__(self, uuid: _Optional[bytes] = ..., sync: bool = ..., merge_policy: _Optional[_Union[MergePolicy, str]] = ..., rounding: _Optional[_Union[RoundSpec, _Mapping]] = ..., arrowBytes: _Optional[bytes] = ...) -> None: ... + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., arrowBytes: _Optional[bytes] = ...) -> None: ... class ArrowMultiValuesParams(_message.Message): - __slots__ = ["end", "snapPeriodNs", "start", "uuid", "versionMajor"] - END_FIELD_NUMBER: _ClassVar[int] - SNAPPERIODNS_FIELD_NUMBER: _ClassVar[int] - START_FIELD_NUMBER: _ClassVar[int] + __slots__ = ["uuid", "versionMajor", "start", "end", "snapPeriodNs"] UUID_FIELD_NUMBER: _ClassVar[int] VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] - end: int - snapPeriodNs: int - start: int + START_FIELD_NUMBER: _ClassVar[int] + END_FIELD_NUMBER: _ClassVar[int] + SNAPPERIODNS_FIELD_NUMBER: _ClassVar[int] uuid: _containers.RepeatedScalarFieldContainer[bytes] versionMajor: _containers.RepeatedScalarFieldContainer[int] + start: int + end: int + snapPeriodNs: int def __init__(self, uuid: _Optional[_Iterable[bytes]] = ..., versionMajor: _Optional[_Iterable[int]] = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., snapPeriodNs: _Optional[int] = ...) -> None: ... class ArrowMultiValuesResponse(_message.Message): - __slots__ = ["arrowBytes", "stat"] - ARROWBYTES_FIELD_NUMBER: _ClassVar[int] + __slots__ = ["stat", "arrowBytes"] STAT_FIELD_NUMBER: _ClassVar[int] - arrowBytes: bytes + ARROWBYTES_FIELD_NUMBER: _ClassVar[int] stat: Status + arrowBytes: bytes def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., arrowBytes: _Optional[bytes] = ...) -> None: ... -class ArrowRawValuesResponse(_message.Message): - __slots__ = ["arrowBytes", "stat", "versionMajor", "versionMinor"] - ARROWBYTES_FIELD_NUMBER: _ClassVar[int] +class RawPointVec(_message.Message): + __slots__ = ["time", "value"] + TIME_FIELD_NUMBER: _ClassVar[int] + VALUE_FIELD_NUMBER: _ClassVar[int] + time: int + value: _containers.RepeatedScalarFieldContainer[float] + def __init__(self, time: _Optional[int] = ..., value: _Optional[_Iterable[float]] = ...) -> None: ... + +class AlignedWindowsParams(_message.Message): + __slots__ = ["uuid", "start", "end", "versionMajor", "pointWidth"] + UUID_FIELD_NUMBER: _ClassVar[int] + START_FIELD_NUMBER: _ClassVar[int] + END_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + POINTWIDTH_FIELD_NUMBER: _ClassVar[int] + uuid: bytes + start: int + end: int + versionMajor: int + pointWidth: int + def __init__(self, uuid: _Optional[bytes] = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., versionMajor: _Optional[int] = ..., pointWidth: _Optional[int] = ...) -> None: ... + +class AlignedWindowsResponse(_message.Message): + __slots__ = ["stat", "versionMajor", "versionMinor", "values"] STAT_FIELD_NUMBER: _ClassVar[int] VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] - arrowBytes: bytes + VALUES_FIELD_NUMBER: _ClassVar[int] stat: Status versionMajor: int versionMinor: int - def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., arrowBytes: _Optional[bytes] = ...) -> None: ... + values: _containers.RepeatedCompositeFieldContainer[StatPoint] + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., values: _Optional[_Iterable[_Union[StatPoint, _Mapping]]] = ...) -> None: ... -class ArrowWindowsResponse(_message.Message): - __slots__ = ["arrowBytes", "stat", "versionMajor", "versionMinor"] - ARROWBYTES_FIELD_NUMBER: _ClassVar[int] +class ArrowAlignedWindowsResponse(_message.Message): + __slots__ = ["stat", "versionMajor", "versionMinor", "arrowBytes"] STAT_FIELD_NUMBER: _ClassVar[int] VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] - arrowBytes: bytes + ARROWBYTES_FIELD_NUMBER: _ClassVar[int] stat: Status versionMajor: int versionMinor: int + arrowBytes: bytes def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., arrowBytes: _Optional[bytes] = ...) -> None: ... -class ChangedRange(_message.Message): - __slots__ = ["end", "start"] - END_FIELD_NUMBER: _ClassVar[int] +class WindowsParams(_message.Message): + __slots__ = ["uuid", "start", "end", "versionMajor", "width", "depth"] + UUID_FIELD_NUMBER: _ClassVar[int] START_FIELD_NUMBER: _ClassVar[int] - end: int + END_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + WIDTH_FIELD_NUMBER: _ClassVar[int] + DEPTH_FIELD_NUMBER: _ClassVar[int] + uuid: bytes start: int - def __init__(self, start: _Optional[int] = ..., end: _Optional[int] = ...) -> None: ... + end: int + versionMajor: int + width: int + depth: int + def __init__(self, uuid: _Optional[bytes] = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., versionMajor: _Optional[int] = ..., width: _Optional[int] = ..., depth: _Optional[int] = ...) -> None: ... -class ChangesParams(_message.Message): - __slots__ = ["fromMajor", "resolution", "toMajor", "uuid"] - FROMMAJOR_FIELD_NUMBER: _ClassVar[int] - RESOLUTION_FIELD_NUMBER: _ClassVar[int] - TOMAJOR_FIELD_NUMBER: _ClassVar[int] +class WindowsResponse(_message.Message): + __slots__ = ["stat", "versionMajor", "versionMinor", "values"] + STAT_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + VALUES_FIELD_NUMBER: _ClassVar[int] + stat: Status + versionMajor: int + versionMinor: int + values: _containers.RepeatedCompositeFieldContainer[StatPoint] + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., values: _Optional[_Iterable[_Union[StatPoint, _Mapping]]] = ...) -> None: ... + +class ArrowWindowsResponse(_message.Message): + __slots__ = ["stat", "versionMajor", "versionMinor", "arrowBytes"] + STAT_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + ARROWBYTES_FIELD_NUMBER: _ClassVar[int] + stat: Status + versionMajor: int + versionMinor: int + arrowBytes: bytes + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., arrowBytes: _Optional[bytes] = ...) -> None: ... + +class StreamInfoParams(_message.Message): + __slots__ = ["uuid", "omitVersion", "omitDescriptor", "role"] UUID_FIELD_NUMBER: _ClassVar[int] - fromMajor: int - resolution: int - toMajor: int + OMITVERSION_FIELD_NUMBER: _ClassVar[int] + OMITDESCRIPTOR_FIELD_NUMBER: _ClassVar[int] + ROLE_FIELD_NUMBER: _ClassVar[int] uuid: bytes - def __init__(self, uuid: _Optional[bytes] = ..., fromMajor: _Optional[int] = ..., toMajor: _Optional[int] = ..., resolution: _Optional[int] = ...) -> None: ... + omitVersion: bool + omitDescriptor: bool + role: Role + def __init__(self, uuid: _Optional[bytes] = ..., omitVersion: bool = ..., omitDescriptor: bool = ..., role: _Optional[_Union[Role, _Mapping]] = ...) -> None: ... -class ChangesResponse(_message.Message): - __slots__ = ["ranges", "stat", "versionMajor", "versionMinor"] - RANGES_FIELD_NUMBER: _ClassVar[int] +class StreamInfoResponse(_message.Message): + __slots__ = ["stat", "versionMajor", "versionMinor", "descriptor"] STAT_FIELD_NUMBER: _ClassVar[int] VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] - ranges: _containers.RepeatedCompositeFieldContainer[ChangedRange] + DESCRIPTOR_FIELD_NUMBER: _ClassVar[int] stat: Status versionMajor: int versionMinor: int - def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., ranges: _Optional[_Iterable[_Union[ChangedRange, _Mapping]]] = ...) -> None: ... + descriptor: StreamDescriptor + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., descriptor: _Optional[_Union[StreamDescriptor, _Mapping]] = ...) -> None: ... -class CreateParams(_message.Message): - __slots__ = ["annotations", "collection", "tags", "uuid"] - ANNOTATIONS_FIELD_NUMBER: _ClassVar[int] +class StreamDescriptor(_message.Message): + __slots__ = ["uuid", "collection", "tags", "annotations", "propertyVersion"] + UUID_FIELD_NUMBER: _ClassVar[int] COLLECTION_FIELD_NUMBER: _ClassVar[int] TAGS_FIELD_NUMBER: _ClassVar[int] - UUID_FIELD_NUMBER: _ClassVar[int] - annotations: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + ANNOTATIONS_FIELD_NUMBER: _ClassVar[int] + PROPERTYVERSION_FIELD_NUMBER: _ClassVar[int] + uuid: bytes collection: str tags: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + annotations: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + propertyVersion: int + def __init__(self, uuid: _Optional[bytes] = ..., collection: _Optional[str] = ..., tags: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., annotations: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., propertyVersion: _Optional[int] = ...) -> None: ... + +class SetStreamAnnotationsParams(_message.Message): + __slots__ = ["uuid", "expectedPropertyVersion", "changes", "removals"] + UUID_FIELD_NUMBER: _ClassVar[int] + EXPECTEDPROPERTYVERSION_FIELD_NUMBER: _ClassVar[int] + CHANGES_FIELD_NUMBER: _ClassVar[int] + REMOVALS_FIELD_NUMBER: _ClassVar[int] uuid: bytes - def __init__(self, uuid: _Optional[bytes] = ..., collection: _Optional[str] = ..., tags: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., annotations: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ...) -> None: ... + expectedPropertyVersion: int + changes: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + removals: _containers.RepeatedScalarFieldContainer[str] + def __init__(self, uuid: _Optional[bytes] = ..., expectedPropertyVersion: _Optional[int] = ..., changes: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., removals: _Optional[_Iterable[str]] = ...) -> None: ... -class CreateResponse(_message.Message): +class SetStreamAnnotationsResponse(_message.Message): __slots__ = ["stat"] STAT_FIELD_NUMBER: _ClassVar[int] stat: Status def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ...) -> None: ... -class DeleteParams(_message.Message): - __slots__ = ["end", "start", "uuid"] - END_FIELD_NUMBER: _ClassVar[int] - START_FIELD_NUMBER: _ClassVar[int] +class SetStreamTagsParams(_message.Message): + __slots__ = ["uuid", "expectedPropertyVersion", "tags", "collection", "remove"] UUID_FIELD_NUMBER: _ClassVar[int] - end: int - start: int + EXPECTEDPROPERTYVERSION_FIELD_NUMBER: _ClassVar[int] + TAGS_FIELD_NUMBER: _ClassVar[int] + COLLECTION_FIELD_NUMBER: _ClassVar[int] + REMOVE_FIELD_NUMBER: _ClassVar[int] uuid: bytes - def __init__(self, uuid: _Optional[bytes] = ..., start: _Optional[int] = ..., end: _Optional[int] = ...) -> None: ... + expectedPropertyVersion: int + tags: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + collection: str + remove: _containers.RepeatedScalarFieldContainer[str] + def __init__(self, uuid: _Optional[bytes] = ..., expectedPropertyVersion: _Optional[int] = ..., tags: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., collection: _Optional[str] = ..., remove: _Optional[_Iterable[str]] = ...) -> None: ... -class DeleteResponse(_message.Message): - __slots__ = ["stat", "versionMajor", "versionMinor"] +class SetStreamTagsResponse(_message.Message): + __slots__ = ["stat"] STAT_FIELD_NUMBER: _ClassVar[int] - VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] - VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] stat: Status - versionMajor: int - versionMinor: int - def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ...) -> None: ... + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ...) -> None: ... -class FaultInjectParams(_message.Message): - __slots__ = ["params", "type"] - PARAMS_FIELD_NUMBER: _ClassVar[int] - TYPE_FIELD_NUMBER: _ClassVar[int] - params: bytes - type: int - def __init__(self, type: _Optional[int] = ..., params: _Optional[bytes] = ...) -> None: ... +class CreateParams(_message.Message): + __slots__ = ["uuid", "collection", "tags", "annotations"] + UUID_FIELD_NUMBER: _ClassVar[int] + COLLECTION_FIELD_NUMBER: _ClassVar[int] + TAGS_FIELD_NUMBER: _ClassVar[int] + ANNOTATIONS_FIELD_NUMBER: _ClassVar[int] + uuid: bytes + collection: str + tags: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + annotations: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + def __init__(self, uuid: _Optional[bytes] = ..., collection: _Optional[str] = ..., tags: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., annotations: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ...) -> None: ... -class FaultInjectResponse(_message.Message): - __slots__ = ["rv", "stat"] - RV_FIELD_NUMBER: _ClassVar[int] +class CreateResponse(_message.Message): + __slots__ = ["stat"] STAT_FIELD_NUMBER: _ClassVar[int] - rv: bytes stat: Status - def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., rv: _Optional[bytes] = ...) -> None: ... + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ...) -> None: ... -class FlushParams(_message.Message): - __slots__ = ["uuid"] - UUID_FIELD_NUMBER: _ClassVar[int] - uuid: bytes - def __init__(self, uuid: _Optional[bytes] = ...) -> None: ... +class MetadataUsageParams(_message.Message): + __slots__ = ["prefix", "role"] + PREFIX_FIELD_NUMBER: _ClassVar[int] + ROLE_FIELD_NUMBER: _ClassVar[int] + prefix: str + role: Role + def __init__(self, prefix: _Optional[str] = ..., role: _Optional[_Union[Role, _Mapping]] = ...) -> None: ... -class FlushResponse(_message.Message): - __slots__ = ["stat", "versionMajor", "versionMinor"] +class MetadataUsageResponse(_message.Message): + __slots__ = ["stat", "tags", "annotations"] STAT_FIELD_NUMBER: _ClassVar[int] - VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] - VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + TAGS_FIELD_NUMBER: _ClassVar[int] + ANNOTATIONS_FIELD_NUMBER: _ClassVar[int] stat: Status - versionMajor: int - versionMinor: int - def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ...) -> None: ... - -class GenerateCSVParams(_message.Message): - __slots__ = ["depth", "endTime", "includeVersions", "queryType", "startTime", "streams", "windowSize"] - class QueryType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): - __slots__ = [] - ALIGNED_WINDOWS_QUERY: GenerateCSVParams.QueryType - DEPTH_FIELD_NUMBER: _ClassVar[int] - ENDTIME_FIELD_NUMBER: _ClassVar[int] - INCLUDEVERSIONS_FIELD_NUMBER: _ClassVar[int] - QUERYTYPE_FIELD_NUMBER: _ClassVar[int] - RAW_QUERY: GenerateCSVParams.QueryType - STARTTIME_FIELD_NUMBER: _ClassVar[int] - STREAMS_FIELD_NUMBER: _ClassVar[int] - WINDOWSIZE_FIELD_NUMBER: _ClassVar[int] - WINDOWS_QUERY: GenerateCSVParams.QueryType - depth: int - endTime: int - includeVersions: bool - queryType: GenerateCSVParams.QueryType - startTime: int - streams: _containers.RepeatedCompositeFieldContainer[StreamCSVConfig] - windowSize: int - def __init__(self, queryType: _Optional[_Union[GenerateCSVParams.QueryType, str]] = ..., startTime: _Optional[int] = ..., endTime: _Optional[int] = ..., windowSize: _Optional[int] = ..., depth: _Optional[int] = ..., includeVersions: bool = ..., streams: _Optional[_Iterable[_Union[StreamCSVConfig, _Mapping]]] = ...) -> None: ... - -class GenerateCSVResponse(_message.Message): - __slots__ = ["isHeader", "row", "stat"] - ISHEADER_FIELD_NUMBER: _ClassVar[int] - ROW_FIELD_NUMBER: _ClassVar[int] - STAT_FIELD_NUMBER: _ClassVar[int] - isHeader: bool - row: _containers.RepeatedScalarFieldContainer[str] - stat: Status - def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., isHeader: bool = ..., row: _Optional[_Iterable[str]] = ...) -> None: ... - -class GetCompactionConfigParams(_message.Message): - __slots__ = ["uuid"] - UUID_FIELD_NUMBER: _ClassVar[int] - uuid: bytes - def __init__(self, uuid: _Optional[bytes] = ...) -> None: ... - -class GetCompactionConfigResponse(_message.Message): - __slots__ = ["CompactedVersion", "LatestMajorVersion", "reducedResolutionRanges", "stat", "unused0"] - COMPACTEDVERSION_FIELD_NUMBER: _ClassVar[int] - CompactedVersion: int - LATESTMAJORVERSION_FIELD_NUMBER: _ClassVar[int] - LatestMajorVersion: int - REDUCEDRESOLUTIONRANGES_FIELD_NUMBER: _ClassVar[int] - STAT_FIELD_NUMBER: _ClassVar[int] - UNUSED0_FIELD_NUMBER: _ClassVar[int] - reducedResolutionRanges: _containers.RepeatedCompositeFieldContainer[ReducedResolutionRange] - stat: Status - unused0: int - def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., LatestMajorVersion: _Optional[int] = ..., CompactedVersion: _Optional[int] = ..., reducedResolutionRanges: _Optional[_Iterable[_Union[ReducedResolutionRange, _Mapping]]] = ..., unused0: _Optional[int] = ...) -> None: ... - -class InfoParams(_message.Message): - __slots__ = [] - def __init__(self) -> None: ... - -class InfoResponse(_message.Message): - __slots__ = ["build", "majorVersion", "mash", "minorVersion", "proxy", "stat"] - BUILD_FIELD_NUMBER: _ClassVar[int] - MAJORVERSION_FIELD_NUMBER: _ClassVar[int] - MASH_FIELD_NUMBER: _ClassVar[int] - MINORVERSION_FIELD_NUMBER: _ClassVar[int] - PROXY_FIELD_NUMBER: _ClassVar[int] - STAT_FIELD_NUMBER: _ClassVar[int] - build: str - majorVersion: int - mash: Mash - minorVersion: int - proxy: ProxyInfo - stat: Status - def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., mash: _Optional[_Union[Mash, _Mapping]] = ..., majorVersion: _Optional[int] = ..., minorVersion: _Optional[int] = ..., build: _Optional[str] = ..., proxy: _Optional[_Union[ProxyInfo, _Mapping]] = ...) -> None: ... - -class InsertParams(_message.Message): - __slots__ = ["merge_policy", "rounding", "sync", "uuid", "values"] - MERGE_POLICY_FIELD_NUMBER: _ClassVar[int] - ROUNDING_FIELD_NUMBER: _ClassVar[int] - SYNC_FIELD_NUMBER: _ClassVar[int] - UUID_FIELD_NUMBER: _ClassVar[int] - VALUES_FIELD_NUMBER: _ClassVar[int] - merge_policy: MergePolicy - rounding: RoundSpec - sync: bool - uuid: bytes - values: _containers.RepeatedCompositeFieldContainer[RawPoint] - def __init__(self, uuid: _Optional[bytes] = ..., sync: bool = ..., merge_policy: _Optional[_Union[MergePolicy, str]] = ..., rounding: _Optional[_Union[RoundSpec, _Mapping]] = ..., values: _Optional[_Iterable[_Union[RawPoint, _Mapping]]] = ...) -> None: ... - -class InsertResponse(_message.Message): - __slots__ = ["stat", "versionMajor", "versionMinor"] - STAT_FIELD_NUMBER: _ClassVar[int] - VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] - VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] - stat: Status - versionMajor: int - versionMinor: int - def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ...) -> None: ... + tags: _containers.RepeatedCompositeFieldContainer[KeyCount] + annotations: _containers.RepeatedCompositeFieldContainer[KeyCount] + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., tags: _Optional[_Iterable[_Union[KeyCount, _Mapping]]] = ..., annotations: _Optional[_Iterable[_Union[KeyCount, _Mapping]]] = ...) -> None: ... class KeyCount(_message.Message): - __slots__ = ["count", "key"] - COUNT_FIELD_NUMBER: _ClassVar[int] + __slots__ = ["key", "count"] KEY_FIELD_NUMBER: _ClassVar[int] - count: int + COUNT_FIELD_NUMBER: _ClassVar[int] key: str + count: int def __init__(self, key: _Optional[str] = ..., count: _Optional[int] = ...) -> None: ... -class KeyOptValue(_message.Message): - __slots__ = ["key", "val"] - KEY_FIELD_NUMBER: _ClassVar[int] - VAL_FIELD_NUMBER: _ClassVar[int] - key: str - val: OptValue - def __init__(self, key: _Optional[str] = ..., val: _Optional[_Union[OptValue, _Mapping]] = ...) -> None: ... - -class KeyValue(_message.Message): - __slots__ = ["key", "value"] - KEY_FIELD_NUMBER: _ClassVar[int] - VALUE_FIELD_NUMBER: _ClassVar[int] - key: str - value: str - def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ... - class ListCollectionsParams(_message.Message): __slots__ = ["prefix", "role"] PREFIX_FIELD_NUMBER: _ClassVar[int] @@ -340,381 +297,456 @@ class ListCollectionsParams(_message.Message): def __init__(self, prefix: _Optional[str] = ..., role: _Optional[_Union[Role, _Mapping]] = ...) -> None: ... class ListCollectionsResponse(_message.Message): - __slots__ = ["collections", "stat"] - COLLECTIONS_FIELD_NUMBER: _ClassVar[int] + __slots__ = ["stat", "collections"] STAT_FIELD_NUMBER: _ClassVar[int] - collections: _containers.RepeatedScalarFieldContainer[str] + COLLECTIONS_FIELD_NUMBER: _ClassVar[int] stat: Status + collections: _containers.RepeatedScalarFieldContainer[str] def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., collections: _Optional[_Iterable[str]] = ...) -> None: ... class LookupStreamsParams(_message.Message): - __slots__ = ["annotations", "collection", "isCollectionPrefix", "role", "tags"] - ANNOTATIONS_FIELD_NUMBER: _ClassVar[int] + __slots__ = ["collection", "isCollectionPrefix", "tags", "annotations", "role"] COLLECTION_FIELD_NUMBER: _ClassVar[int] ISCOLLECTIONPREFIX_FIELD_NUMBER: _ClassVar[int] - ROLE_FIELD_NUMBER: _ClassVar[int] TAGS_FIELD_NUMBER: _ClassVar[int] - annotations: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + ANNOTATIONS_FIELD_NUMBER: _ClassVar[int] + ROLE_FIELD_NUMBER: _ClassVar[int] collection: str isCollectionPrefix: bool - role: Role tags: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + annotations: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + role: Role def __init__(self, collection: _Optional[str] = ..., isCollectionPrefix: bool = ..., tags: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., annotations: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., role: _Optional[_Union[Role, _Mapping]] = ...) -> None: ... class LookupStreamsResponse(_message.Message): - __slots__ = ["results", "stat"] - RESULTS_FIELD_NUMBER: _ClassVar[int] + __slots__ = ["stat", "results"] STAT_FIELD_NUMBER: _ClassVar[int] - results: _containers.RepeatedCompositeFieldContainer[StreamDescriptor] + RESULTS_FIELD_NUMBER: _ClassVar[int] stat: Status + results: _containers.RepeatedCompositeFieldContainer[StreamDescriptor] def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., results: _Optional[_Iterable[_Union[StreamDescriptor, _Mapping]]] = ...) -> None: ... -class Mash(_message.Message): - __slots__ = ["healthy", "leader", "leaderRevision", "members", "revision", "totalWeight", "unmapped"] - HEALTHY_FIELD_NUMBER: _ClassVar[int] - LEADERREVISION_FIELD_NUMBER: _ClassVar[int] - LEADER_FIELD_NUMBER: _ClassVar[int] - MEMBERS_FIELD_NUMBER: _ClassVar[int] - REVISION_FIELD_NUMBER: _ClassVar[int] - TOTALWEIGHT_FIELD_NUMBER: _ClassVar[int] - UNMAPPED_FIELD_NUMBER: _ClassVar[int] - healthy: bool - leader: str - leaderRevision: int - members: _containers.RepeatedCompositeFieldContainer[Member] - revision: int - totalWeight: int - unmapped: float - def __init__(self, revision: _Optional[int] = ..., leader: _Optional[str] = ..., leaderRevision: _Optional[int] = ..., totalWeight: _Optional[int] = ..., healthy: bool = ..., unmapped: _Optional[float] = ..., members: _Optional[_Iterable[_Union[Member, _Mapping]]] = ...) -> None: ... - -class Member(_message.Message): - __slots__ = ["enabled", "end", "grpcEndpoints", "hash", "httpEndpoints", "nodename", "readPreference", "start", "up", "weight"] - ENABLED_FIELD_NUMBER: _ClassVar[int] - END_FIELD_NUMBER: _ClassVar[int] - GRPCENDPOINTS_FIELD_NUMBER: _ClassVar[int] - HASH_FIELD_NUMBER: _ClassVar[int] - HTTPENDPOINTS_FIELD_NUMBER: _ClassVar[int] - IN_FIELD_NUMBER: _ClassVar[int] - NODENAME_FIELD_NUMBER: _ClassVar[int] - READPREFERENCE_FIELD_NUMBER: _ClassVar[int] - START_FIELD_NUMBER: _ClassVar[int] - UP_FIELD_NUMBER: _ClassVar[int] - WEIGHT_FIELD_NUMBER: _ClassVar[int] - enabled: bool - end: int - grpcEndpoints: str - hash: int - httpEndpoints: str - nodename: str - readPreference: float - start: int - up: bool - weight: int - def __init__(self, hash: _Optional[int] = ..., nodename: _Optional[str] = ..., up: bool = ..., enabled: bool = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., weight: _Optional[int] = ..., readPreference: _Optional[float] = ..., httpEndpoints: _Optional[str] = ..., grpcEndpoints: _Optional[str] = ..., **kwargs) -> None: ... - -class MetadataUsageParams(_message.Message): - __slots__ = ["prefix", "role"] - PREFIX_FIELD_NUMBER: _ClassVar[int] - ROLE_FIELD_NUMBER: _ClassVar[int] - prefix: str - role: Role - def __init__(self, prefix: _Optional[str] = ..., role: _Optional[_Union[Role, _Mapping]] = ...) -> None: ... - -class MetadataUsageResponse(_message.Message): - __slots__ = ["annotations", "stat", "tags"] - ANNOTATIONS_FIELD_NUMBER: _ClassVar[int] - STAT_FIELD_NUMBER: _ClassVar[int] - TAGS_FIELD_NUMBER: _ClassVar[int] - annotations: _containers.RepeatedCompositeFieldContainer[KeyCount] - stat: Status - tags: _containers.RepeatedCompositeFieldContainer[KeyCount] - def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., tags: _Optional[_Iterable[_Union[KeyCount, _Mapping]]] = ..., annotations: _Optional[_Iterable[_Union[KeyCount, _Mapping]]] = ...) -> None: ... - class NearestParams(_message.Message): - __slots__ = ["backward", "time", "uuid", "versionMajor"] - BACKWARD_FIELD_NUMBER: _ClassVar[int] - TIME_FIELD_NUMBER: _ClassVar[int] + __slots__ = ["uuid", "time", "versionMajor", "backward"] UUID_FIELD_NUMBER: _ClassVar[int] + TIME_FIELD_NUMBER: _ClassVar[int] VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] - backward: bool - time: int + BACKWARD_FIELD_NUMBER: _ClassVar[int] uuid: bytes + time: int versionMajor: int + backward: bool def __init__(self, uuid: _Optional[bytes] = ..., time: _Optional[int] = ..., versionMajor: _Optional[int] = ..., backward: bool = ...) -> None: ... class NearestResponse(_message.Message): - __slots__ = ["stat", "value", "versionMajor", "versionMinor"] + __slots__ = ["stat", "versionMajor", "versionMinor", "value"] STAT_FIELD_NUMBER: _ClassVar[int] - VALUE_FIELD_NUMBER: _ClassVar[int] VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + VALUE_FIELD_NUMBER: _ClassVar[int] stat: Status - value: RawPoint versionMajor: int versionMinor: int + value: RawPoint def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., value: _Optional[_Union[RawPoint, _Mapping]] = ...) -> None: ... -class ObliterateParams(_message.Message): - __slots__ = ["uuid"] +class ChangesParams(_message.Message): + __slots__ = ["uuid", "fromMajor", "toMajor", "resolution"] UUID_FIELD_NUMBER: _ClassVar[int] + FROMMAJOR_FIELD_NUMBER: _ClassVar[int] + TOMAJOR_FIELD_NUMBER: _ClassVar[int] + RESOLUTION_FIELD_NUMBER: _ClassVar[int] uuid: bytes - def __init__(self, uuid: _Optional[bytes] = ...) -> None: ... + fromMajor: int + toMajor: int + resolution: int + def __init__(self, uuid: _Optional[bytes] = ..., fromMajor: _Optional[int] = ..., toMajor: _Optional[int] = ..., resolution: _Optional[int] = ...) -> None: ... -class ObliterateResponse(_message.Message): - __slots__ = ["stat"] +class ChangesResponse(_message.Message): + __slots__ = ["stat", "versionMajor", "versionMinor", "ranges"] STAT_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + RANGES_FIELD_NUMBER: _ClassVar[int] stat: Status - def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ...) -> None: ... + versionMajor: int + versionMinor: int + ranges: _containers.RepeatedCompositeFieldContainer[ChangedRange] + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., ranges: _Optional[_Iterable[_Union[ChangedRange, _Mapping]]] = ...) -> None: ... -class OptValue(_message.Message): - __slots__ = ["value"] - VALUE_FIELD_NUMBER: _ClassVar[int] - value: str - def __init__(self, value: _Optional[str] = ...) -> None: ... +class RoundSpec(_message.Message): + __slots__ = ["bits"] + BITS_FIELD_NUMBER: _ClassVar[int] + bits: int + def __init__(self, bits: _Optional[int] = ...) -> None: ... -class ProxyInfo(_message.Message): - __slots__ = ["proxyEndpoints"] - PROXYENDPOINTS_FIELD_NUMBER: _ClassVar[int] - proxyEndpoints: _containers.RepeatedScalarFieldContainer[str] - def __init__(self, proxyEndpoints: _Optional[_Iterable[str]] = ...) -> None: ... +class InsertParams(_message.Message): + __slots__ = ["uuid", "sync", "merge_policy", "rounding", "values"] + UUID_FIELD_NUMBER: _ClassVar[int] + SYNC_FIELD_NUMBER: _ClassVar[int] + MERGE_POLICY_FIELD_NUMBER: _ClassVar[int] + ROUNDING_FIELD_NUMBER: _ClassVar[int] + VALUES_FIELD_NUMBER: _ClassVar[int] + uuid: bytes + sync: bool + merge_policy: MergePolicy + rounding: RoundSpec + values: _containers.RepeatedCompositeFieldContainer[RawPoint] + def __init__(self, uuid: _Optional[bytes] = ..., sync: bool = ..., merge_policy: _Optional[_Union[MergePolicy, str]] = ..., rounding: _Optional[_Union[RoundSpec, _Mapping]] = ..., values: _Optional[_Iterable[_Union[RawPoint, _Mapping]]] = ...) -> None: ... -class RawPoint(_message.Message): - __slots__ = ["time", "value"] - TIME_FIELD_NUMBER: _ClassVar[int] - VALUE_FIELD_NUMBER: _ClassVar[int] - time: int - value: float - def __init__(self, time: _Optional[int] = ..., value: _Optional[float] = ...) -> None: ... - -class RawPointVec(_message.Message): - __slots__ = ["time", "value"] - TIME_FIELD_NUMBER: _ClassVar[int] - VALUE_FIELD_NUMBER: _ClassVar[int] - time: int - value: _containers.RepeatedScalarFieldContainer[float] - def __init__(self, time: _Optional[int] = ..., value: _Optional[_Iterable[float]] = ...) -> None: ... - -class RawValuesParams(_message.Message): - __slots__ = ["end", "start", "uuid", "versionMajor"] - END_FIELD_NUMBER: _ClassVar[int] - START_FIELD_NUMBER: _ClassVar[int] +class ArrowInsertParams(_message.Message): + __slots__ = ["uuid", "sync", "merge_policy", "rounding", "arrowBytes"] UUID_FIELD_NUMBER: _ClassVar[int] - VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] - end: int - start: int + SYNC_FIELD_NUMBER: _ClassVar[int] + MERGE_POLICY_FIELD_NUMBER: _ClassVar[int] + ROUNDING_FIELD_NUMBER: _ClassVar[int] + ARROWBYTES_FIELD_NUMBER: _ClassVar[int] uuid: bytes - versionMajor: int - def __init__(self, uuid: _Optional[bytes] = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., versionMajor: _Optional[int] = ...) -> None: ... + sync: bool + merge_policy: MergePolicy + rounding: RoundSpec + arrowBytes: bytes + def __init__(self, uuid: _Optional[bytes] = ..., sync: bool = ..., merge_policy: _Optional[_Union[MergePolicy, str]] = ..., rounding: _Optional[_Union[RoundSpec, _Mapping]] = ..., arrowBytes: _Optional[bytes] = ...) -> None: ... -class RawValuesResponse(_message.Message): - __slots__ = ["stat", "values", "versionMajor", "versionMinor"] +class InsertResponse(_message.Message): + __slots__ = ["stat", "versionMajor", "versionMinor"] STAT_FIELD_NUMBER: _ClassVar[int] - VALUES_FIELD_NUMBER: _ClassVar[int] VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] stat: Status - values: _containers.RepeatedCompositeFieldContainer[RawPoint] versionMajor: int versionMinor: int - def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., values: _Optional[_Iterable[_Union[RawPoint, _Mapping]]] = ...) -> None: ... + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ...) -> None: ... -class ReducedResolutionRange(_message.Message): - __slots__ = ["End", "Resolution", "Start"] - END_FIELD_NUMBER: _ClassVar[int] - End: int - RESOLUTION_FIELD_NUMBER: _ClassVar[int] - Resolution: int +class DeleteParams(_message.Message): + __slots__ = ["uuid", "start", "end"] + UUID_FIELD_NUMBER: _ClassVar[int] START_FIELD_NUMBER: _ClassVar[int] - Start: int - def __init__(self, Start: _Optional[int] = ..., End: _Optional[int] = ..., Resolution: _Optional[int] = ...) -> None: ... - -class Role(_message.Message): - __slots__ = ["name"] - NAME_FIELD_NUMBER: _ClassVar[int] - name: str - def __init__(self, name: _Optional[str] = ...) -> None: ... + END_FIELD_NUMBER: _ClassVar[int] + uuid: bytes + start: int + end: int + def __init__(self, uuid: _Optional[bytes] = ..., start: _Optional[int] = ..., end: _Optional[int] = ...) -> None: ... -class RoundSpec(_message.Message): - __slots__ = ["bits"] - BITS_FIELD_NUMBER: _ClassVar[int] - bits: int - def __init__(self, bits: _Optional[int] = ...) -> None: ... +class DeleteResponse(_message.Message): + __slots__ = ["stat", "versionMajor", "versionMinor"] + STAT_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + stat: Status + versionMajor: int + versionMinor: int + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ...) -> None: ... -class SQLQueryParams(_message.Message): - __slots__ = ["params", "query", "role"] - PARAMS_FIELD_NUMBER: _ClassVar[int] - QUERY_FIELD_NUMBER: _ClassVar[int] - ROLE_FIELD_NUMBER: _ClassVar[int] - params: _containers.RepeatedScalarFieldContainer[str] - query: str - role: Role - def __init__(self, query: _Optional[str] = ..., params: _Optional[_Iterable[str]] = ..., role: _Optional[_Union[Role, _Mapping]] = ...) -> None: ... +class InfoParams(_message.Message): + __slots__ = [] + def __init__(self) -> None: ... -class SQLQueryResponse(_message.Message): - __slots__ = ["SQLQueryRow", "stat"] - SQLQUERYROW_FIELD_NUMBER: _ClassVar[int] - SQLQueryRow: _containers.RepeatedScalarFieldContainer[bytes] +class InfoResponse(_message.Message): + __slots__ = ["stat", "mash", "majorVersion", "minorVersion", "build", "proxy"] STAT_FIELD_NUMBER: _ClassVar[int] + MASH_FIELD_NUMBER: _ClassVar[int] + MAJORVERSION_FIELD_NUMBER: _ClassVar[int] + MINORVERSION_FIELD_NUMBER: _ClassVar[int] + BUILD_FIELD_NUMBER: _ClassVar[int] + PROXY_FIELD_NUMBER: _ClassVar[int] stat: Status - def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., SQLQueryRow: _Optional[_Iterable[bytes]] = ...) -> None: ... + mash: Mash + majorVersion: int + minorVersion: int + build: str + proxy: ProxyInfo + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., mash: _Optional[_Union[Mash, _Mapping]] = ..., majorVersion: _Optional[int] = ..., minorVersion: _Optional[int] = ..., build: _Optional[str] = ..., proxy: _Optional[_Union[ProxyInfo, _Mapping]] = ...) -> None: ... -class SetCompactionConfigParams(_message.Message): - __slots__ = ["CompactedVersion", "reducedResolutionRanges", "unused0", "uuid"] - COMPACTEDVERSION_FIELD_NUMBER: _ClassVar[int] - CompactedVersion: int - REDUCEDRESOLUTIONRANGES_FIELD_NUMBER: _ClassVar[int] - UNUSED0_FIELD_NUMBER: _ClassVar[int] - UUID_FIELD_NUMBER: _ClassVar[int] - reducedResolutionRanges: _containers.RepeatedCompositeFieldContainer[ReducedResolutionRange] - unused0: int - uuid: bytes - def __init__(self, uuid: _Optional[bytes] = ..., CompactedVersion: _Optional[int] = ..., reducedResolutionRanges: _Optional[_Iterable[_Union[ReducedResolutionRange, _Mapping]]] = ..., unused0: _Optional[int] = ...) -> None: ... +class ProxyInfo(_message.Message): + __slots__ = ["proxyEndpoints"] + PROXYENDPOINTS_FIELD_NUMBER: _ClassVar[int] + proxyEndpoints: _containers.RepeatedScalarFieldContainer[str] + def __init__(self, proxyEndpoints: _Optional[_Iterable[str]] = ...) -> None: ... -class SetCompactionConfigResponse(_message.Message): - __slots__ = ["stat"] +class FaultInjectParams(_message.Message): + __slots__ = ["type", "params"] + TYPE_FIELD_NUMBER: _ClassVar[int] + PARAMS_FIELD_NUMBER: _ClassVar[int] + type: int + params: bytes + def __init__(self, type: _Optional[int] = ..., params: _Optional[bytes] = ...) -> None: ... + +class FaultInjectResponse(_message.Message): + __slots__ = ["stat", "rv"] STAT_FIELD_NUMBER: _ClassVar[int] + RV_FIELD_NUMBER: _ClassVar[int] stat: Status - def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ...) -> None: ... + rv: bytes + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., rv: _Optional[bytes] = ...) -> None: ... -class SetStreamAnnotationsParams(_message.Message): - __slots__ = ["changes", "expectedPropertyVersion", "removals", "uuid"] - CHANGES_FIELD_NUMBER: _ClassVar[int] - EXPECTEDPROPERTYVERSION_FIELD_NUMBER: _ClassVar[int] - REMOVALS_FIELD_NUMBER: _ClassVar[int] +class FlushParams(_message.Message): + __slots__ = ["uuid"] UUID_FIELD_NUMBER: _ClassVar[int] - changes: _containers.RepeatedCompositeFieldContainer[KeyOptValue] - expectedPropertyVersion: int - removals: _containers.RepeatedScalarFieldContainer[str] uuid: bytes - def __init__(self, uuid: _Optional[bytes] = ..., expectedPropertyVersion: _Optional[int] = ..., changes: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., removals: _Optional[_Iterable[str]] = ...) -> None: ... + def __init__(self, uuid: _Optional[bytes] = ...) -> None: ... -class SetStreamAnnotationsResponse(_message.Message): - __slots__ = ["stat"] +class FlushResponse(_message.Message): + __slots__ = ["stat", "versionMajor", "versionMinor"] STAT_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] stat: Status - def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ...) -> None: ... + versionMajor: int + versionMinor: int + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ...) -> None: ... -class SetStreamTagsParams(_message.Message): - __slots__ = ["collection", "expectedPropertyVersion", "remove", "tags", "uuid"] - COLLECTION_FIELD_NUMBER: _ClassVar[int] - EXPECTEDPROPERTYVERSION_FIELD_NUMBER: _ClassVar[int] - REMOVE_FIELD_NUMBER: _ClassVar[int] - TAGS_FIELD_NUMBER: _ClassVar[int] +class ObliterateParams(_message.Message): + __slots__ = ["uuid"] UUID_FIELD_NUMBER: _ClassVar[int] - collection: str - expectedPropertyVersion: int - remove: _containers.RepeatedScalarFieldContainer[str] - tags: _containers.RepeatedCompositeFieldContainer[KeyOptValue] uuid: bytes - def __init__(self, uuid: _Optional[bytes] = ..., expectedPropertyVersion: _Optional[int] = ..., tags: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., collection: _Optional[str] = ..., remove: _Optional[_Iterable[str]] = ...) -> None: ... + def __init__(self, uuid: _Optional[bytes] = ...) -> None: ... -class SetStreamTagsResponse(_message.Message): +class ObliterateResponse(_message.Message): __slots__ = ["stat"] STAT_FIELD_NUMBER: _ClassVar[int] stat: Status def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ...) -> None: ... +class RawPoint(_message.Message): + __slots__ = ["time", "value"] + TIME_FIELD_NUMBER: _ClassVar[int] + VALUE_FIELD_NUMBER: _ClassVar[int] + time: int + value: float + def __init__(self, time: _Optional[int] = ..., value: _Optional[float] = ...) -> None: ... + class StatPoint(_message.Message): - __slots__ = ["count", "max", "mean", "min", "stddev", "time"] - COUNT_FIELD_NUMBER: _ClassVar[int] - MAX_FIELD_NUMBER: _ClassVar[int] - MEAN_FIELD_NUMBER: _ClassVar[int] + __slots__ = ["time", "min", "mean", "max", "count", "stddev"] + TIME_FIELD_NUMBER: _ClassVar[int] MIN_FIELD_NUMBER: _ClassVar[int] + MEAN_FIELD_NUMBER: _ClassVar[int] + MAX_FIELD_NUMBER: _ClassVar[int] + COUNT_FIELD_NUMBER: _ClassVar[int] STDDEV_FIELD_NUMBER: _ClassVar[int] - TIME_FIELD_NUMBER: _ClassVar[int] - count: int - max: float - mean: float + time: int min: float + mean: float + max: float + count: int stddev: float - time: int def __init__(self, time: _Optional[int] = ..., min: _Optional[float] = ..., mean: _Optional[float] = ..., max: _Optional[float] = ..., count: _Optional[int] = ..., stddev: _Optional[float] = ...) -> None: ... +class ChangedRange(_message.Message): + __slots__ = ["start", "end"] + START_FIELD_NUMBER: _ClassVar[int] + END_FIELD_NUMBER: _ClassVar[int] + start: int + end: int + def __init__(self, start: _Optional[int] = ..., end: _Optional[int] = ...) -> None: ... + class Status(_message.Message): - __slots__ = ["code", "mash", "msg"] + __slots__ = ["code", "msg", "mash"] CODE_FIELD_NUMBER: _ClassVar[int] - MASH_FIELD_NUMBER: _ClassVar[int] MSG_FIELD_NUMBER: _ClassVar[int] + MASH_FIELD_NUMBER: _ClassVar[int] code: int - mash: Mash msg: str + mash: Mash def __init__(self, code: _Optional[int] = ..., msg: _Optional[str] = ..., mash: _Optional[_Union[Mash, _Mapping]] = ...) -> None: ... +class Mash(_message.Message): + __slots__ = ["revision", "leader", "leaderRevision", "totalWeight", "healthy", "unmapped", "members"] + REVISION_FIELD_NUMBER: _ClassVar[int] + LEADER_FIELD_NUMBER: _ClassVar[int] + LEADERREVISION_FIELD_NUMBER: _ClassVar[int] + TOTALWEIGHT_FIELD_NUMBER: _ClassVar[int] + HEALTHY_FIELD_NUMBER: _ClassVar[int] + UNMAPPED_FIELD_NUMBER: _ClassVar[int] + MEMBERS_FIELD_NUMBER: _ClassVar[int] + revision: int + leader: str + leaderRevision: int + totalWeight: int + healthy: bool + unmapped: float + members: _containers.RepeatedCompositeFieldContainer[Member] + def __init__(self, revision: _Optional[int] = ..., leader: _Optional[str] = ..., leaderRevision: _Optional[int] = ..., totalWeight: _Optional[int] = ..., healthy: bool = ..., unmapped: _Optional[float] = ..., members: _Optional[_Iterable[_Union[Member, _Mapping]]] = ...) -> None: ... + +class Member(_message.Message): + __slots__ = ["hash", "nodename", "up", "enabled", "start", "end", "weight", "readPreference", "httpEndpoints", "grpcEndpoints"] + HASH_FIELD_NUMBER: _ClassVar[int] + NODENAME_FIELD_NUMBER: _ClassVar[int] + UP_FIELD_NUMBER: _ClassVar[int] + IN_FIELD_NUMBER: _ClassVar[int] + ENABLED_FIELD_NUMBER: _ClassVar[int] + START_FIELD_NUMBER: _ClassVar[int] + END_FIELD_NUMBER: _ClassVar[int] + WEIGHT_FIELD_NUMBER: _ClassVar[int] + READPREFERENCE_FIELD_NUMBER: _ClassVar[int] + HTTPENDPOINTS_FIELD_NUMBER: _ClassVar[int] + GRPCENDPOINTS_FIELD_NUMBER: _ClassVar[int] + hash: int + nodename: str + up: bool + enabled: bool + start: int + end: int + weight: int + readPreference: float + httpEndpoints: str + grpcEndpoints: str + def __init__(self, hash: _Optional[int] = ..., nodename: _Optional[str] = ..., up: bool = ..., enabled: bool = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., weight: _Optional[int] = ..., readPreference: _Optional[float] = ..., httpEndpoints: _Optional[str] = ..., grpcEndpoints: _Optional[str] = ..., **kwargs) -> None: ... + +class KeyOptValue(_message.Message): + __slots__ = ["key", "val"] + KEY_FIELD_NUMBER: _ClassVar[int] + VAL_FIELD_NUMBER: _ClassVar[int] + key: str + val: OptValue + def __init__(self, key: _Optional[str] = ..., val: _Optional[_Union[OptValue, _Mapping]] = ...) -> None: ... + +class OptValue(_message.Message): + __slots__ = ["value"] + VALUE_FIELD_NUMBER: _ClassVar[int] + value: str + def __init__(self, value: _Optional[str] = ...) -> None: ... + +class KeyValue(_message.Message): + __slots__ = ["key", "value"] + KEY_FIELD_NUMBER: _ClassVar[int] + VALUE_FIELD_NUMBER: _ClassVar[int] + key: str + value: str + def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ... + class StreamCSVConfig(_message.Message): - __slots__ = ["label", "uuid", "version"] + __slots__ = ["version", "label", "uuid"] + VERSION_FIELD_NUMBER: _ClassVar[int] LABEL_FIELD_NUMBER: _ClassVar[int] UUID_FIELD_NUMBER: _ClassVar[int] - VERSION_FIELD_NUMBER: _ClassVar[int] + version: int label: str uuid: bytes - version: int def __init__(self, version: _Optional[int] = ..., label: _Optional[str] = ..., uuid: _Optional[bytes] = ...) -> None: ... -class StreamDescriptor(_message.Message): - __slots__ = ["annotations", "collection", "propertyVersion", "tags", "uuid"] - ANNOTATIONS_FIELD_NUMBER: _ClassVar[int] - COLLECTION_FIELD_NUMBER: _ClassVar[int] - PROPERTYVERSION_FIELD_NUMBER: _ClassVar[int] - TAGS_FIELD_NUMBER: _ClassVar[int] - UUID_FIELD_NUMBER: _ClassVar[int] - annotations: _containers.RepeatedCompositeFieldContainer[KeyOptValue] - collection: str - propertyVersion: int - tags: _containers.RepeatedCompositeFieldContainer[KeyOptValue] - uuid: bytes - def __init__(self, uuid: _Optional[bytes] = ..., collection: _Optional[str] = ..., tags: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., annotations: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., propertyVersion: _Optional[int] = ...) -> None: ... +class GenerateCSVParams(_message.Message): + __slots__ = ["queryType", "startTime", "endTime", "windowSize", "depth", "includeVersions", "streams"] + class QueryType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = [] + ALIGNED_WINDOWS_QUERY: _ClassVar[GenerateCSVParams.QueryType] + WINDOWS_QUERY: _ClassVar[GenerateCSVParams.QueryType] + RAW_QUERY: _ClassVar[GenerateCSVParams.QueryType] + ALIGNED_WINDOWS_QUERY: GenerateCSVParams.QueryType + WINDOWS_QUERY: GenerateCSVParams.QueryType + RAW_QUERY: GenerateCSVParams.QueryType + QUERYTYPE_FIELD_NUMBER: _ClassVar[int] + STARTTIME_FIELD_NUMBER: _ClassVar[int] + ENDTIME_FIELD_NUMBER: _ClassVar[int] + WINDOWSIZE_FIELD_NUMBER: _ClassVar[int] + DEPTH_FIELD_NUMBER: _ClassVar[int] + INCLUDEVERSIONS_FIELD_NUMBER: _ClassVar[int] + STREAMS_FIELD_NUMBER: _ClassVar[int] + queryType: GenerateCSVParams.QueryType + startTime: int + endTime: int + windowSize: int + depth: int + includeVersions: bool + streams: _containers.RepeatedCompositeFieldContainer[StreamCSVConfig] + def __init__(self, queryType: _Optional[_Union[GenerateCSVParams.QueryType, str]] = ..., startTime: _Optional[int] = ..., endTime: _Optional[int] = ..., windowSize: _Optional[int] = ..., depth: _Optional[int] = ..., includeVersions: bool = ..., streams: _Optional[_Iterable[_Union[StreamCSVConfig, _Mapping]]] = ...) -> None: ... -class StreamInfoParams(_message.Message): - __slots__ = ["omitDescriptor", "omitVersion", "role", "uuid"] - OMITDESCRIPTOR_FIELD_NUMBER: _ClassVar[int] - OMITVERSION_FIELD_NUMBER: _ClassVar[int] +class GenerateCSVResponse(_message.Message): + __slots__ = ["stat", "isHeader", "row"] + STAT_FIELD_NUMBER: _ClassVar[int] + ISHEADER_FIELD_NUMBER: _ClassVar[int] + ROW_FIELD_NUMBER: _ClassVar[int] + stat: Status + isHeader: bool + row: _containers.RepeatedScalarFieldContainer[str] + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., isHeader: bool = ..., row: _Optional[_Iterable[str]] = ...) -> None: ... + +class SQLQueryParams(_message.Message): + __slots__ = ["query", "params", "role"] + QUERY_FIELD_NUMBER: _ClassVar[int] + PARAMS_FIELD_NUMBER: _ClassVar[int] ROLE_FIELD_NUMBER: _ClassVar[int] - UUID_FIELD_NUMBER: _ClassVar[int] - omitDescriptor: bool - omitVersion: bool + query: str + params: _containers.RepeatedScalarFieldContainer[str] role: Role + def __init__(self, query: _Optional[str] = ..., params: _Optional[_Iterable[str]] = ..., role: _Optional[_Union[Role, _Mapping]] = ...) -> None: ... + +class SQLQueryResponse(_message.Message): + __slots__ = ["stat", "SQLQueryRow"] + STAT_FIELD_NUMBER: _ClassVar[int] + SQLQUERYROW_FIELD_NUMBER: _ClassVar[int] + stat: Status + SQLQueryRow: _containers.RepeatedScalarFieldContainer[bytes] + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., SQLQueryRow: _Optional[_Iterable[bytes]] = ...) -> None: ... + +class Role(_message.Message): + __slots__ = ["name"] + NAME_FIELD_NUMBER: _ClassVar[int] + name: str + def __init__(self, name: _Optional[str] = ...) -> None: ... + +class SetCompactionConfigParams(_message.Message): + __slots__ = ["uuid", "CompactedVersion", "reducedResolutionRanges", "unused0"] + UUID_FIELD_NUMBER: _ClassVar[int] + COMPACTEDVERSION_FIELD_NUMBER: _ClassVar[int] + REDUCEDRESOLUTIONRANGES_FIELD_NUMBER: _ClassVar[int] + UNUSED0_FIELD_NUMBER: _ClassVar[int] uuid: bytes - def __init__(self, uuid: _Optional[bytes] = ..., omitVersion: bool = ..., omitDescriptor: bool = ..., role: _Optional[_Union[Role, _Mapping]] = ...) -> None: ... + CompactedVersion: int + reducedResolutionRanges: _containers.RepeatedCompositeFieldContainer[ReducedResolutionRange] + unused0: int + def __init__(self, uuid: _Optional[bytes] = ..., CompactedVersion: _Optional[int] = ..., reducedResolutionRanges: _Optional[_Iterable[_Union[ReducedResolutionRange, _Mapping]]] = ..., unused0: _Optional[int] = ...) -> None: ... -class StreamInfoResponse(_message.Message): - __slots__ = ["descriptor", "stat", "versionMajor", "versionMinor"] - DESCRIPTOR_FIELD_NUMBER: _ClassVar[int] +class SetCompactionConfigResponse(_message.Message): + __slots__ = ["stat"] STAT_FIELD_NUMBER: _ClassVar[int] - VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] - VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] - descriptor: StreamDescriptor stat: Status - versionMajor: int - versionMinor: int - def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., descriptor: _Optional[_Union[StreamDescriptor, _Mapping]] = ...) -> None: ... + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ...) -> None: ... -class WindowsParams(_message.Message): - __slots__ = ["depth", "end", "start", "uuid", "versionMajor", "width"] - DEPTH_FIELD_NUMBER: _ClassVar[int] - END_FIELD_NUMBER: _ClassVar[int] - START_FIELD_NUMBER: _ClassVar[int] +class GetCompactionConfigParams(_message.Message): + __slots__ = ["uuid"] UUID_FIELD_NUMBER: _ClassVar[int] - VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] - WIDTH_FIELD_NUMBER: _ClassVar[int] - depth: int - end: int - start: int uuid: bytes - versionMajor: int - width: int - def __init__(self, uuid: _Optional[bytes] = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., versionMajor: _Optional[int] = ..., width: _Optional[int] = ..., depth: _Optional[int] = ...) -> None: ... + def __init__(self, uuid: _Optional[bytes] = ...) -> None: ... -class WindowsResponse(_message.Message): - __slots__ = ["stat", "values", "versionMajor", "versionMinor"] +class GetCompactionConfigResponse(_message.Message): + __slots__ = ["stat", "LatestMajorVersion", "CompactedVersion", "reducedResolutionRanges", "unused0"] STAT_FIELD_NUMBER: _ClassVar[int] - VALUES_FIELD_NUMBER: _ClassVar[int] - VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] - VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + LATESTMAJORVERSION_FIELD_NUMBER: _ClassVar[int] + COMPACTEDVERSION_FIELD_NUMBER: _ClassVar[int] + REDUCEDRESOLUTIONRANGES_FIELD_NUMBER: _ClassVar[int] + UNUSED0_FIELD_NUMBER: _ClassVar[int] stat: Status - values: _containers.RepeatedCompositeFieldContainer[StatPoint] - versionMajor: int - versionMinor: int - def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., values: _Optional[_Iterable[_Union[StatPoint, _Mapping]]] = ...) -> None: ... + LatestMajorVersion: int + CompactedVersion: int + reducedResolutionRanges: _containers.RepeatedCompositeFieldContainer[ReducedResolutionRange] + unused0: int + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., LatestMajorVersion: _Optional[int] = ..., CompactedVersion: _Optional[int] = ..., reducedResolutionRanges: _Optional[_Iterable[_Union[ReducedResolutionRange, _Mapping]]] = ..., unused0: _Optional[int] = ...) -> None: ... -class MergePolicy(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): - __slots__ = [] +class ReducedResolutionRange(_message.Message): + __slots__ = ["Start", "End", "Resolution"] + START_FIELD_NUMBER: _ClassVar[int] + END_FIELD_NUMBER: _ClassVar[int] + RESOLUTION_FIELD_NUMBER: _ClassVar[int] + Start: int + End: int + Resolution: int + def __init__(self, Start: _Optional[int] = ..., End: _Optional[int] = ..., Resolution: _Optional[int] = ...) -> None: ... + +class SubscriptionUpdate(_message.Message): + __slots__ = ["op", "uuid"] + OP_FIELD_NUMBER: _ClassVar[int] + UUID_FIELD_NUMBER: _ClassVar[int] + op: SubscriptionUpdateOp + uuid: _containers.RepeatedScalarFieldContainer[bytes] + def __init__(self, op: _Optional[_Union[SubscriptionUpdateOp, str]] = ..., uuid: _Optional[_Iterable[bytes]] = ...) -> None: ... + +class SubscriptionResp(_message.Message): + __slots__ = ["stat", "uuid", "arrowBytes"] + STAT_FIELD_NUMBER: _ClassVar[int] + UUID_FIELD_NUMBER: _ClassVar[int] + ARROWBYTES_FIELD_NUMBER: _ClassVar[int] + stat: Status + uuid: bytes + arrowBytes: bytes + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., uuid: _Optional[bytes] = ..., arrowBytes: _Optional[bytes] = ...) -> None: ... diff --git a/btrdb/grpcinterface/btrdb_pb2_grpc.py b/btrdb/grpcinterface/btrdb_pb2_grpc.py index 52a72bd..d05d9cf 100644 --- a/btrdb/grpcinterface/btrdb_pb2_grpc.py +++ b/btrdb/grpcinterface/btrdb_pb2_grpc.py @@ -139,6 +139,11 @@ def __init__(self, channel): request_serializer=btrdb__pb2.SQLQueryParams.SerializeToString, response_deserializer=btrdb__pb2.SQLQueryResponse.FromString, ) + self.Subscribe = channel.stream_stream( + '/v5api.BTrDB/Subscribe', + request_serializer=btrdb__pb2.SubscriptionUpdate.SerializeToString, + response_deserializer=btrdb__pb2.SubscriptionResp.FromString, + ) class BTrDBServicer(object): @@ -289,6 +294,12 @@ def GenerateCSV(self, request, context): raise NotImplementedError('Method not implemented!') def SQLQuery(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Subscribe(self, request_iterator, context): """rpc SetCompactionConfig(SetCompactionConfigParams) returns (SetCompactionConfigResponse); rpc GetCompactionConfig(GetCompactionConfigParams) returns (GetCompactionConfigResponse); """ @@ -424,6 +435,11 @@ def add_BTrDBServicer_to_server(servicer, server): request_deserializer=btrdb__pb2.SQLQueryParams.FromString, response_serializer=btrdb__pb2.SQLQueryResponse.SerializeToString, ), + 'Subscribe': grpc.stream_stream_rpc_method_handler( + servicer.Subscribe, + request_deserializer=btrdb__pb2.SubscriptionUpdate.FromString, + response_serializer=btrdb__pb2.SubscriptionResp.SerializeToString, + ), } generic_handler = grpc.method_handlers_generic_handler( 'v5api.BTrDB', rpc_method_handlers) @@ -858,3 +874,20 @@ def SQLQuery(request, btrdb__pb2.SQLQueryResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Subscribe(request_iterator, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.stream_stream(request_iterator, target, '/v5api.BTrDB/Subscribe', + btrdb__pb2.SubscriptionUpdate.SerializeToString, + btrdb__pb2.SubscriptionResp.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) From 87a789ae5494c833c86d216d57887db2222e6471 Mon Sep 17 00:00:00 2001 From: Jeff Lin <42981468+jleifnf@users.noreply.github.com> Date: Mon, 27 Nov 2023 10:16:20 -0800 Subject: [PATCH 084/131] `arrow_to_dataframe` return with `time` as index (#62) --- btrdb/transformers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/btrdb/transformers.py b/btrdb/transformers.py index f01787f..0273e2a 100644 --- a/btrdb/transformers.py +++ b/btrdb/transformers.py @@ -212,7 +212,9 @@ def arrow_to_dataframe(streamset, agg=None, name_callable=None) -> pd.DataFrame: tmp = tmp_table.select(["time", *usable_cols]) else: tmp = tmp_table - return tmp.to_pandas(date_as_object=False, types_mapper=pd.ArrowDtype) + return tmp.to_pandas(date_as_object=False, types_mapper=pd.ArrowDtype).set_index( + "time" + ) def to_dataframe(streamset, agg="mean", name_callable=None): From f7afb11434c4f8a16b34f736c5e1ff324450d6fa Mon Sep 17 00:00:00 2001 From: Justin Gilmer Date: Mon, 27 Nov 2023 17:20:08 -0500 Subject: [PATCH 085/131] =?UTF-8?q?Update=20changelog,=20and=20update=20st?= =?UTF-8?q?ream=20logic=20to=20sort=20arrow=20tables=20joined=E2=80=A6=20(?= =?UTF-8?q?#58)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 8 ++++++++ btrdb/stream.py | 20 +++----------------- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f6d54f..c61644d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,12 @@ # Changelog + +## 5.31.1 +### What's Changed +* Sort tables by time by default for any `pyarrow` tables. by @justinGilmer in +* Fix deprecation warnings for pip installations. by @jleifnf in + +## 5.31.0 + ## 5.30.2 ### What's Changed * Update readthedocs to new yaml for testing. by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/40 diff --git a/btrdb/stream.py b/btrdb/stream.py index 7343def..6bcd8c3 100644 --- a/btrdb/stream.py +++ b/btrdb/stream.py @@ -1051,7 +1051,6 @@ def arrow_aligned_windows( start: int, end: int, pointwidth: int, - sort_time: bool = False, version: int = 0, auto_retry=False, retries=5, @@ -1084,8 +1083,6 @@ def arrow_aligned_windows( :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) pointwidth : int, required Specify the number of ns between data points (2**pointwidth) - sort_time : bool, default: False - Should the table be sorted on the 'time' column? version : int, default: 0 Version of the stream to query auto_retry: bool, default: False @@ -1128,10 +1125,7 @@ def arrow_aligned_windows( ) if len(tables) > 0: tabs, ver = zip(*tables) - if sort_time: - return pa.concat_tables(tabs).sort_by("time") - else: - return pa.concat_tables(tabs) + return pa.concat_tables(tabs).sort_by("time") else: schema = pa.schema( [ @@ -1225,7 +1219,6 @@ def arrow_windows( start: int, end: int, width: int, - sort_time: bool = False, version: int = 0, auto_retry=False, retries=5, @@ -1244,8 +1237,6 @@ def arrow_windows( :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) width : int, required The number of nanoseconds in each window. - sort_time : bool, default: False - Should the table be sorted on the 'time' column. version : int, default=0, optional The version of the stream to query. auto_retry: bool, default: False @@ -1296,10 +1287,7 @@ def arrow_windows( ) if len(tables) > 0: tabs, ver = zip(*tables) - if sort_time: - return pa.concat_tables(tabs).sort_by("time") - else: - return pa.concat_tables(tabs) + return pa.concat_tables(tabs).sort_by("time") else: schema = pa.schema( [ @@ -2160,7 +2148,6 @@ def arrow_values( data = tablex else: data = tablex - data = data.sort_by("time") elif self.width is not None and self.depth is not None: # create list of stream.windows data (the windows method should @@ -2191,7 +2178,6 @@ def arrow_values( data = tablex else: data = tablex - data = data.sort_by("time") else: sampling_freq = params.pop("sampling_frequency", 0) period_ns = 0 @@ -2214,7 +2200,7 @@ def arrow_values( data = pa.Table.from_arrays( [pa.array([]) for i in range(1 + len(self._streams))], schema=schema ) - return data + return data.sort_by("time") def __repr__(self): token = "stream" if len(self) == 1 else "streams" From 9048df0d36f68f0be9c695e54e4a4dbb7bdebf2c Mon Sep 17 00:00:00 2001 From: Jeff Lin <42981468+jleifnf@users.noreply.github.com> Date: Mon, 27 Nov 2023 18:17:55 -0800 Subject: [PATCH 086/131] fix NoneType error for earliest/latest for empty streams (#64) --- btrdb/stream.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/btrdb/stream.py b/btrdb/stream.py index 6bcd8c3..c468c42 100644 --- a/btrdb/stream.py +++ b/btrdb/stream.py @@ -1587,8 +1587,11 @@ def earliest(self): lambda s: s.nearest(start, version=versions.get(s.uuid, 0), backward=False), self._streams, ) - for point, _ in earliest_points_gen: - earliest.append(point) + for point in earliest_points_gen: + if point is not None: + earliest.append(point[0]) + else: + earliest.append(None) return tuple(earliest) @@ -1614,8 +1617,11 @@ def latest(self): lambda s: s.nearest(start, version=versions.get(s.uuid, 0), backward=True), self._streams, ) - for point, _ in latest_points_gen: - latest.append(point) + for point in latest_points_gen: + if point is not None: + latest.append(point[0]) + else: + latest.append(None) return tuple(latest) From f54e5f2b7f2cf82903233ec7f0ea9d6ac05ab37f Mon Sep 17 00:00:00 2001 From: Jeff Lin <42981468+jleifnf@users.noreply.github.com> Date: Mon, 27 Nov 2023 18:22:12 -0800 Subject: [PATCH 087/131] FutureWarning for streams_in_collection to return StreamSet (#60) Co-authored-by: Justin Gilmer --- btrdb/conn.py | 8 +++++++- tests/btrdb/test_conn.py | 2 ++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/btrdb/conn.py b/btrdb/conn.py index a77eced..83542d5 100644 --- a/btrdb/conn.py +++ b/btrdb/conn.py @@ -22,6 +22,7 @@ import uuid as uuidlib from concurrent.futures import ThreadPoolExecutor from typing import List +from warnings import warn import certifi import grpc @@ -578,7 +579,12 @@ def streams_in_collection( property_version=desc.propertyVersion, ) ) - + # TODO: In future release update this method to return a streamset object. + warn( + "StreamSet will be returned in a future release.", + FutureWarning, + stacklevel=2, + ) return result @retry diff --git a/tests/btrdb/test_conn.py b/tests/btrdb/test_conn.py index ed00aa9..66034b3 100644 --- a/tests/btrdb/test_conn.py +++ b/tests/btrdb/test_conn.py @@ -98,6 +98,8 @@ class TestBTrDB(object): ########################################################################## ## .streams tests ########################################################################## + # TODO: remove this when removing future warnings from `streams_in_collection` + pytestmark = pytest.mark.filterwarnings("ignore::FutureWarning") def test_streams_raises_err_if_version_not_list(self): """ From 534fe51b0cee5b49bb57916208aadb1404b62b61 Mon Sep 17 00:00:00 2001 From: Jeff Lin <42981468+jleifnf@users.noreply.github.com> Date: Mon, 18 Dec 2023 10:47:42 -0800 Subject: [PATCH 088/131] Update pre-commit versions and add coverage report to gh actions (#68) --- .github/workflows/release.yaml | 5 ++++- .pre-commit-config.yaml | 8 ++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 027d682..33bec25 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -30,7 +30,10 @@ jobs: pip install -r tests/requirements.txt - name: Test with pytest run: | - pytest + pytest --cov-report= --cov=btrdb # suppress coverage report here + - name: Coverage Report + run: | + coverage report -m release: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6f327d1..9c35795 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,26 +1,26 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: check-yaml - id: end-of-file-fixer - id: trailing-whitespace exclude: ^(setup.cfg|btrdb/grpcinterface) - repo: https://github.com/psf/black - rev: 23.3.0 + rev: 23.12.0 hooks: - id: black-jupyter args: [--line-length=88] exclude: btrdb/grpcinterface/.*\.py - repo: https://github.com/pycqa/isort - rev: 5.11.5 + rev: 5.13.2 hooks: - id: isort name: isort (python) args: [--profile=black, --line-length=88] exclude: btrdb/grpcinterface/.*\.py - repo: https://github.com/PyCQA/flake8 - rev: 6.0.0 + rev: 6.1.0 hooks: - id: flake8 args: [--config=setup.cfg] From 13399e8b709ab4cdc2b8f7709b4653be7b121b7f Mon Sep 17 00:00:00 2001 From: Jeff Lin <42981468+jleifnf@users.noreply.github.com> Date: Mon, 18 Dec 2023 10:52:40 -0800 Subject: [PATCH 089/131] Add to_timedelta method for pointwidth class (#69) --- btrdb/utils/general.py | 7 +++++++ tests/btrdb/utils/test_general.py | 16 ++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/btrdb/utils/general.py b/btrdb/utils/general.py index 6344fe6..e93a73f 100644 --- a/btrdb/utils/general.py +++ b/btrdb/utils/general.py @@ -12,6 +12,7 @@ """ General utilities for btrdb bindings """ +from datetime import timedelta ########################################################################## ## Functions @@ -123,6 +124,12 @@ def decr(self): def incr(self): return pointwidth(self + 1) + def to_timedelta(self): + """ + Returns the timedelta of the pointwidth. + """ + return timedelta(microseconds=self.microseconds) + def __int__(self): return self._pointwidth diff --git a/tests/btrdb/utils/test_general.py b/tests/btrdb/utils/test_general.py index a13cf3c..094c9d2 100644 --- a/tests/btrdb/utils/test_general.py +++ b/tests/btrdb/utils/test_general.py @@ -114,3 +114,19 @@ def test_incr(self): Test incrementing a pointwidth """ assert pointwidth(23).incr() == 24 + + @pytest.mark.parametrize( + "pw, expected", + [ + (54, timedelta(days=208, seconds=43198, microseconds=509482)), + (51, timedelta(days=26, seconds=5399, microseconds=813685)), + (49, timedelta(days=6, seconds=44549, microseconds=953421)), + (46, timedelta(seconds=70368, microseconds=744178)), + (43, timedelta(seconds=8796, microseconds=93022)), + (39, timedelta(seconds=549, microseconds=755814)), + (34, timedelta(seconds=17, microseconds=179869)), + (30, timedelta(seconds=1, microseconds=73742)), + ], + ) + def test_to_timedelta(self, pw, expected): + assert pointwidth(pw).to_timedelta() == expected From cf6e0b250a7a1fd0e4a24102df05132f330ccfea Mon Sep 17 00:00:00 2001 From: Jeff Lin <42981468+jleifnf@users.noreply.github.com> Date: Mon, 18 Dec 2023 15:13:10 -0800 Subject: [PATCH 090/131] Method to get first and last timestamps from aligned_windows (#70) Co-authored-by: Justin Gilmer --- btrdb/utils/general.py | 49 +++++++++++++++++++++++++++++-- tests/btrdb/utils/test_general.py | 30 +++++++++++++++++++ 2 files changed, 77 insertions(+), 2 deletions(-) diff --git a/btrdb/utils/general.py b/btrdb/utils/general.py index e93a73f..d438fce 100644 --- a/btrdb/utils/general.py +++ b/btrdb/utils/general.py @@ -14,11 +14,12 @@ """ from datetime import timedelta +from btrdb.utils.timez import to_nanoseconds + + ########################################################################## ## Functions ########################################################################## - - def unpack_stream_descriptor(desc): """ Returns dicts for tags and annotations found in supplied stream @@ -75,6 +76,50 @@ def from_nanoseconds(cls, nsec): break return cls(pos) + def for_aligned_windows(self, start, end): + """ + Returns the aligned_windows's first and last timestamps, as well as the number of windows, + based on a given pointwidth set. + + Parameters + ---------- + start : int or datetime like object + The start time in nanoseconds for the range to be queried. (see + :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) + + end : int or datetime like object + The end time in nanoseconds for the range to be queried. (see + :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) + + + Returns + ------- + aligned_start: int + First timestamp would be returned by `aligned_windows` that is inclusive of specified start + timestamp. + aligned_end: int + Last timestamp would be returned by `aligned_windows` that is inclusive of specified end + timestamp. + n_windows: int + The number of windows would be returned by `aligned_windows`. + + Examples + -------- + Querying `aligned_windows` of pointwidth of 30, spaning 1 day (~24 hours). + + >>> start, end = "2016-03-01", "2016-03-02" + >>> pointwidth(30).for_aligned_windows(start, end) + (1456790399996657664, 1456876798632525824, 80466) + # output timestamp's `strftime` is: + # ['2016-02-29 23:59:59.996657664', '2016-03-01 23:59:58.632525824'] + + """ + start, end = to_nanoseconds(start), to_nanoseconds(end) + aligned_start = start - (start % self.nanoseconds) + n_windows = (end - aligned_start) // self.nanoseconds + aligned_end = aligned_start + (self.nanoseconds * (n_windows - 1)) + return aligned_start, aligned_end, n_windows + def __init__(self, p): self._pointwidth = int(p) diff --git a/tests/btrdb/utils/test_general.py b/tests/btrdb/utils/test_general.py index 094c9d2..0edb86e 100644 --- a/tests/btrdb/utils/test_general.py +++ b/tests/btrdb/utils/test_general.py @@ -56,6 +56,36 @@ def test_from_nanoseconds(self, nsec, expected): """ assert pointwidth.from_nanoseconds(nsec) == expected + @pytest.mark.parametrize( + "pw_time_range, expected", + [ + ( + (52, 1560127027000000000, 1702702800000000000), + (1558245471070191616, 1697857059518676992, 32), + ), + ( + (48, 1560127027000000000, 1702702800000000000), + (1559934320930455552, 1702360659146047488, 507), + ), + ( + (44, 1456876800000000000, 1464652800000000000), + (1456861702896222208, 1464619856941809664, 442), + ), + ( + (38, 1456876800000000000, 1464652800000000000), + (1456876546303197184, 1464652292534829056, 28289), + ), + ( + (32, 1456876800000000000, 1464652800000000000), + (1456876799706267648, 1464652795046002688, 1810491), + ), + ], + ) + def test_for_aligned_windows(self, pw_time_range, expected): + pw, start, end = pw_time_range + result = pointwidth(pw).for_aligned_windows(start, end) + assert result == expected + def test_time_conversions(self): """ Test standard pointwidth time conversions From b7f6032cc3abec4b612e57c480dee2819104330f Mon Sep 17 00:00:00 2001 From: Justin Gilmer Date: Tue, 19 Dec 2023 19:40:01 -0500 Subject: [PATCH 091/131] Remove non-required dependencies, migrate to 'data' optional dependency (#71) --- btrdb/endpoint.py | 12 ++++++ btrdb/stream.py | 23 +++++++++++- btrdb/transformers.py | 85 ++++++++++++++++++++++++++----------------- pyproject.toml | 4 -- 4 files changed, 85 insertions(+), 39 deletions(-) diff --git a/btrdb/endpoint.py b/btrdb/endpoint.py index b526849..801a6e7 100644 --- a/btrdb/endpoint.py +++ b/btrdb/endpoint.py @@ -38,6 +38,8 @@ except ImportError: pa = None +_ARROW_IMPORT_MSG = """Package pyarrow required, please pip install.""" + class Endpoint(object): """Server endpoint where we make specific requests.""" @@ -56,6 +58,8 @@ def rawValues(self, uu, start, end, version=0): @error_handler def arrowRawValues(self, uu, start, end, version=0): + if pa is None: + raise ImportError(_ARROW_IMPORT_MSG) params = btrdb_pb2.RawValuesParams( uuid=uu.bytes, start=start, end=end, versionMajor=version ) @@ -66,6 +70,8 @@ def arrowRawValues(self, uu, start, end, version=0): @error_handler def arrowMultiValues(self, uu_list, start, end, version_list, snap_periodNS): + if pa is None: + raise ImportError(_ARROW_IMPORT_MSG) params = btrdb_pb2.ArrowMultiValuesParams( uuid=[uu.bytes for uu in uu_list], start=start, @@ -80,6 +86,8 @@ def arrowMultiValues(self, uu_list, start, end, version_list, snap_periodNS): @error_handler def arrowInsertValues(self, uu: uuid.UUID, values: pa.Table, policy: str): + if pa is None: + raise ImportError(_ARROW_IMPORT_MSG) policy_map = { "never": btrdb_pb2.MergePolicy.NEVER, "equal": btrdb_pb2.MergePolicy.EQUAL, @@ -115,6 +123,8 @@ def alignedWindows(self, uu, start, end, pointwidth, version=0): @error_handler def arrowAlignedWindows(self, uu, start, end, pointwidth, version=0): + if pa is None: + raise ImportError(_ARROW_IMPORT_MSG) params = btrdb_pb2.AlignedWindowsParams( uuid=uu.bytes, start=start, @@ -143,6 +153,8 @@ def windows(self, uu, start, end, width, depth, version=0): @error_handler def arrowWindows(self, uu, start, end, width, depth, version=0): + if pa is None: + raise ImportError(_ARROW_IMPORT_MSG) params = btrdb_pb2.WindowsParams( uuid=uu.bytes, start=start, diff --git a/btrdb/stream.py b/btrdb/stream.py index c468c42..2100c09 100644 --- a/btrdb/stream.py +++ b/btrdb/stream.py @@ -22,9 +22,15 @@ from collections import deque from collections.abc import Sequence from copy import deepcopy -from typing import List +from typing import TYPE_CHECKING, List -import pyarrow as pa +try: + import pyarrow as pa +except ImportError: + pa = None + +if TYPE_CHECKING: + import pyarrow as pa from btrdb.exceptions import ( BTrDBError, @@ -52,6 +58,9 @@ MINIMUM_TIME = -(16 << 56) MAXIMUM_TIME = (48 << 56) - 1 + +_ARROW_IMPORT_MSG = """Package pyarrow required, please pip install.""" + try: RE_PATTERN = re._pattern_type except Exception: @@ -619,6 +628,8 @@ def arrow_insert(self, data: pa.Table, merge: str = "never") -> int: """ if not self._btrdb._ARROW_ENABLED: raise NotImplementedError(_arrow_not_impl_str.format("arrow_insert")) + if pa is None: + raise ImportError(_ARROW_IMPORT_MSG) chunksize = INSERT_BATCH_SIZE assert isinstance(data, pa.Table) tmp_table = data.rename_columns(["time", "value"]) @@ -951,6 +962,8 @@ def arrow_values( """ if not self._btrdb._ARROW_ENABLED: raise NotImplementedError(_arrow_not_impl_str.format("arrow_values")) + if pa is None: + raise ImportError(_ARROW_IMPORT_MSG) start = to_nanoseconds(start) end = to_nanoseconds(end) arrow_and_versions = self._btrdb.ep.arrowRawValues( @@ -1113,6 +1126,8 @@ def arrow_aligned_windows( raise NotImplementedError( _arrow_not_impl_str.format("arrow_aligned_windows") ) + if pa is None: + raise ImportError(_ARROW_IMPORT_MSG) if IS_DEBUG: logger.debug(f"For stream - {self.uuid} - {self.name}") @@ -1273,6 +1288,8 @@ def arrow_windows( """ if not self._btrdb._ARROW_ENABLED: raise NotImplementedError(_arrow_not_impl_str.format("arrow_windows")) + if pa is None: + raise ImportError(_ARROW_IMPORT_MSG) start = to_nanoseconds(start) end = to_nanoseconds(end) tables = list( @@ -2114,6 +2131,8 @@ def arrow_values( ----- This method is available for commercial customers with arrow-enabled servers. """ + if pa is None: + raise ImportError(_ARROW_IMPORT_MSG) params = self._params_from_filters() versions = self._pinned_versions if versions is None: diff --git a/btrdb/transformers.py b/btrdb/transformers.py index 0273e2a..a57320c 100644 --- a/btrdb/transformers.py +++ b/btrdb/transformers.py @@ -18,11 +18,34 @@ import contextlib import csv from collections import OrderedDict -from typing import Sequence -from warnings import warn - -import pandas as pd -import pyarrow +from typing import TYPE_CHECKING + +try: + import pyarrow as pa +except ImportError: + pa = None +try: + import polars as pl +except ImportError: + pl = None +try: + import pandas as pd +except ImportError: + pd = None +try: + import numpy as np +except ImportError: + np = None + +if TYPE_CHECKING: + import numpy as np + import pandas as pd + import polars as pl + import pyarrow as pa + +_IMPORT_ERR_MSG = ( + """Package(s) expected, but not found. Please pip install the following: {}""" +) ########################################################################## ## Helper Functions @@ -72,10 +95,8 @@ def to_series(streamset, datetime64_index=True, agg="mean", name_callable=None): Stream object. """ - try: - import pandas as pd - except ImportError: - raise ImportError("Please install Pandas to use this transformation function.") + if pd is None: + raise ImportError(_IMPORT_ERR_MSG.format("pandas")) # TODO: allow this at some future point if agg == "all": @@ -126,6 +147,8 @@ def arrow_to_series(streamset, agg="mean", name_callable=None): raise NotImplementedError( "arrow_to_series requires an arrow-enabled BTrDB server." ) + if pa is None or pd is None: + raise ImportError(_IMPORT_ERR_MSG.format(",".join(["pyarrow", "pandas"]))) if agg is None: agg = ["mean"] if not isinstance(agg, list): @@ -162,14 +185,8 @@ def arrow_to_dataframe(streamset, agg=None, name_callable=None) -> pd.DataFrame: raise NotImplementedError( "arrow_to_dataframe requires an arrow-enabled BTrDB server." ) - - try: - import pandas as pd - import pyarrow as pa - except ImportError as err: - raise ImportError( - f"Please install Pandas and pyarrow to use this transformation function. ErrorMessage: {err}" - ) + if pa is None or pd is None: + raise ImportError(_IMPORT_ERR_MSG.format(",".join(["pyarrow", "pandas"]))) if agg is None: agg = ["mean"] @@ -235,10 +252,8 @@ def to_dataframe(streamset, agg="mean", name_callable=None): """ - try: - import pandas as pd - except ImportError: - raise ImportError("Please install Pandas to use this transformation function.") + if pd is None: + raise ImportError(_IMPORT_ERR_MSG.format("pandas")) # TODO: allow this at some future point if agg == "all" and name_callable is not None: @@ -296,10 +311,10 @@ def arrow_to_polars(streamset, agg=None, name_callable=None): raise NotImplementedError( "arrow_to_polars requires an arrow-enabled BTrDB server." ) - try: - import polars as pl - except ImportError: - raise ImportError("Please install polars to use this transformation function.") + if pa is None or pd is None or pl is None: + raise ImportError( + _IMPORT_ERR_MSG.format(",".join(["pyarrow", "pandas", "polars"])) + ) if agg is None: agg = ["mean"] if not isinstance(agg, list): @@ -323,6 +338,8 @@ def arrow_to_arrow_table(streamset): raise NotImplementedError( "arrow_to_arrow_table requires an arrow-enabled BTrDB server." ) + if pa is None: + raise ImportError(_IMPORT_ERR_MSG.format("pyarrow")) return streamset.arrow_values() @@ -342,10 +359,8 @@ def to_polars(streamset, agg="mean", name_callable=None): Specify a callable that can be used to determine the series name given a Stream object. This is not compatible with agg == "all" at this time """ - try: - import polars as pl - except ImportError: - raise ImportError("Please install polars to use this transformation function.") + if pl is None or pd is None: + raise ImportError(_IMPORT_ERR_MSG.format(",".join(["polars", "pandas"]))) # TODO: allow this at some future point if agg == "all" and name_callable is not None: @@ -398,10 +413,8 @@ def to_array(streamset, agg="mean"): argument is ignored if RawPoint values are passed into the function. """ - try: - import numpy as np - except ImportError: - raise ImportError("Please install Numpy to use this transformation function.") + if np is None: + raise ImportError(_IMPORT_ERR_MSG.format("numpy")) # TODO: allow this at some future point if agg == "all": @@ -439,6 +452,10 @@ def arrow_to_numpy(streamset, agg=None): raise NotImplementedError( "arrow_to_numpy requires an arrow-enabled BTrDB server." ) + if np is None or pa is None or pd is None: + raise ImportError( + _IMPORT_ERR_MSG.format(",".join(["numpy", "pyarrow", "pandas"])) + ) arrow_df = arrow_to_dataframe(streamset=streamset, agg=agg, name_callable=None) return arrow_df.values @@ -511,6 +528,8 @@ def arrow_to_dict(streamset, agg=None, name_callable=None): raise NotImplementedError( "arrow_to_dict requires an arrow-enabled BTrDB server." ) + if pa is None or pd is None: + raise ImportError(_IMPORT_ERR_MSG.format(",".join(["pyarrow", "pandas"]))) if agg is None: agg = ["mean"] if not isinstance(agg, list): diff --git a/pyproject.toml b/pyproject.toml index d3c9f94..5bd1745 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,10 +36,6 @@ dependencies = [ "pytz", "pyyaml", "certifi", - "pyarrow", - "polars", - "numpy", - "pandas>=2.0", ] [project.optional-dependencies] From 0857da9d86d57efc53c9ed52182ef87eea5a9c8a Mon Sep 17 00:00:00 2001 From: andrewchambers Date: Wed, 17 Jan 2024 04:11:23 +0900 Subject: [PATCH 092/131] Add option for specifying the schema of the returned data. (#51) Co-authored-by: Justin Gilmer --- btrdb/conn.py | 2 +- btrdb/endpoint.py | 29 ++- btrdb/grpcinterface/btrdb.proto | 10 +- btrdb/grpcinterface/btrdb_pb2.py | 289 +++++++++++----------- btrdb/grpcinterface/btrdb_pb2.pyi | 162 ++++++------ btrdb/grpcinterface/btrdb_pb2_grpc.py | 6 +- btrdb/stream.py | 78 ++++-- btrdb/transformers.py | 21 +- tests/btrdb_integration/test_stream.py | 32 ++- tests/btrdb_integration/test_streamset.py | 58 +++-- 10 files changed, 410 insertions(+), 277 deletions(-) diff --git a/btrdb/conn.py b/btrdb/conn.py index 83542d5..51b29f7 100644 --- a/btrdb/conn.py +++ b/btrdb/conn.py @@ -406,7 +406,7 @@ def create( collection=collection, tags=tags.copy(), annotations=annotations.copy(), - property_version=0, + property_version=1, ) def info(self): diff --git a/btrdb/endpoint.py b/btrdb/endpoint.py index 801a6e7..9bb5e8d 100644 --- a/btrdb/endpoint.py +++ b/btrdb/endpoint.py @@ -28,7 +28,7 @@ import typing import uuid -from btrdb.exceptions import BTrDBError, check_proto_stat, error_handler +from btrdb.exceptions import check_proto_stat, error_handler from btrdb.grpcinterface import btrdb_pb2, btrdb_pb2_grpc from btrdb.point import RawPoint from btrdb.utils.general import unpack_stream_descriptor @@ -57,11 +57,21 @@ def rawValues(self, uu, start, end, version=0): yield result.values, result.versionMajor @error_handler - def arrowRawValues(self, uu, start, end, version=0): + def arrowRawValues(self, uu, start, end, version=0, schema=None): if pa is None: raise ImportError(_ARROW_IMPORT_MSG) - params = btrdb_pb2.RawValuesParams( - uuid=uu.bytes, start=start, end=end, versionMajor=version + templateBytes = b"" + if schema is not None: + byte_io = io.BytesIO() + with pa.ipc.new_stream(sink=byte_io, schema=schema) as _: + pass + templateBytes = byte_io.getvalue() + params = btrdb_pb2.ArrowRawValuesParams( + uuid=uu.bytes, + start=start, + end=end, + versionMajor=version, + templateBytes=templateBytes, ) for result in self.stub.ArrowRawValues(params): check_proto_stat(result.stat) @@ -69,15 +79,24 @@ def arrowRawValues(self, uu, start, end, version=0): yield reader.read_all(), result.versionMajor @error_handler - def arrowMultiValues(self, uu_list, start, end, version_list, snap_periodNS): + def arrowMultiValues( + self, uu_list, start, end, version_list, snap_periodNS=None, schema=None + ): if pa is None: raise ImportError(_ARROW_IMPORT_MSG) + templateBytes = b"" + if schema is not None: + byte_io = io.BytesIO() + with pa.ipc.new_stream(sink=byte_io, schema=schema) as _: + pass + templateBytes = byte_io.getvalue() params = btrdb_pb2.ArrowMultiValuesParams( uuid=[uu.bytes for uu in uu_list], start=start, end=end, versionMajor=[ver for ver in version_list], snapPeriodNs=int(snap_periodNS), + templateBytes=templateBytes, ) for result in self.stub.ArrowMultiValues(params): check_proto_stat(result.stat) diff --git a/btrdb/grpcinterface/btrdb.proto b/btrdb/grpcinterface/btrdb.proto index 61d2050..1e147d6 100644 --- a/btrdb/grpcinterface/btrdb.proto +++ b/btrdb/grpcinterface/btrdb.proto @@ -4,7 +4,7 @@ package v5api; service BTrDB { rpc RawValues(RawValuesParams) returns (stream RawValuesResponse); - rpc ArrowRawValues(RawValuesParams) returns (stream ArrowRawValuesResponse); + rpc ArrowRawValues(ArrowRawValuesParams) returns (stream ArrowRawValuesResponse); rpc ArrowMultiValues(ArrowMultiValuesParams) returns (stream ArrowMultiValuesResponse); rpc AlignedWindows(AlignedWindowsParams) returns (stream AlignedWindowsResponse); rpc ArrowAlignedWindows(AlignedWindowsParams) returns (stream ArrowAlignedWindowsResponse); @@ -45,6 +45,13 @@ message RawValuesResponse { uint64 versionMinor = 3; repeated RawPoint values = 4; } +message ArrowRawValuesParams { + bytes uuid = 1; + sfixed64 start = 2; + sfixed64 end = 3; + uint64 versionMajor = 4; + bytes templateBytes = 5; +} message ArrowRawValuesResponse { Status stat = 1; uint64 versionMajor = 2; @@ -57,6 +64,7 @@ message ArrowMultiValuesParams { sfixed64 start = 3; sfixed64 end = 4; int64 snapPeriodNs = 5; + bytes templateBytes = 6; } message ArrowMultiValuesResponse { Status stat = 1; diff --git a/btrdb/grpcinterface/btrdb_pb2.py b/btrdb/grpcinterface/btrdb_pb2.py index 58b6732..9dcef1a 100644 --- a/btrdb/grpcinterface/btrdb_pb2.py +++ b/btrdb/grpcinterface/btrdb_pb2.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: btrdb.proto +# Protobuf Python Version: 4.25.0 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -13,157 +14,159 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0b\x62trdb.proto\x12\x05v5api\"Q\n\x0fRawValuesParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\"}\n\x11RawValuesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x1f\n\x06values\x18\x04 \x03(\x0b\x32\x0f.v5api.RawPoint\"u\n\x16\x41rrowRawValuesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x12\n\narrowBytes\x18\x04 \x01(\x0c\"n\n\x16\x41rrowMultiValuesParams\x12\x0c\n\x04uuid\x18\x01 \x03(\x0c\x12\x14\n\x0cversionMajor\x18\x02 \x03(\x04\x12\r\n\x05start\x18\x03 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x04 \x01(\x10\x12\x14\n\x0csnapPeriodNs\x18\x05 \x01(\x03\"K\n\x18\x41rrowMultiValuesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x12\n\narrowBytes\x18\x02 \x01(\x0c\"*\n\x0bRawPointVec\x12\x0c\n\x04time\x18\x01 \x01(\x10\x12\r\n\x05value\x18\x02 \x03(\x01\"j\n\x14\x41lignedWindowsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\x12\x12\n\npointWidth\x18\x05 \x01(\r\"\x83\x01\n\x16\x41lignedWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12 \n\x06values\x18\x04 \x03(\x0b\x32\x10.v5api.StatPoint\"z\n\x1b\x41rrowAlignedWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x12\n\narrowBytes\x18\x04 \x01(\x0c\"m\n\rWindowsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\x12\r\n\x05width\x18\x05 \x01(\x04\x12\r\n\x05\x64\x65pth\x18\x06 \x01(\r\"|\n\x0fWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12 \n\x06values\x18\x04 \x03(\x0b\x32\x10.v5api.StatPoint\"s\n\x14\x41rrowWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x12\n\narrowBytes\x18\x04 \x01(\x0c\"h\n\x10StreamInfoParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x13\n\x0bomitVersion\x18\x02 \x01(\x08\x12\x16\n\x0eomitDescriptor\x18\x03 \x01(\x08\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"\x8a\x01\n\x12StreamInfoResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12+\n\ndescriptor\x18\x04 \x01(\x0b\x32\x17.v5api.StreamDescriptor\"\x98\x01\n\x10StreamDescriptor\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x12\n\ncollection\x18\x02 \x01(\t\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x17\n\x0fpropertyVersion\x18\x05 \x01(\x04\"\x82\x01\n\x1aSetStreamAnnotationsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x1f\n\x17\x65xpectedPropertyVersion\x18\x02 \x01(\x04\x12#\n\x07\x63hanges\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x10\n\x08removals\x18\x04 \x03(\t\";\n\x1cSetStreamAnnotationsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"\x8a\x01\n\x13SetStreamTagsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x1f\n\x17\x65xpectedPropertyVersion\x18\x02 \x01(\x04\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x12\n\ncollection\x18\x04 \x01(\t\x12\x0e\n\x06remove\x18\x05 \x03(\t\"4\n\x15SetStreamTagsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"{\n\x0c\x43reateParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x12\n\ncollection\x18\x02 \x01(\t\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\"-\n\x0e\x43reateResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"@\n\x13MetadataUsageParams\x12\x0e\n\x06prefix\x18\x01 \x01(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"y\n\x15MetadataUsageResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x1d\n\x04tags\x18\x02 \x03(\x0b\x32\x0f.v5api.KeyCount\x12$\n\x0b\x61nnotations\x18\x03 \x03(\x0b\x32\x0f.v5api.KeyCount\"&\n\x08KeyCount\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05\x63ount\x18\x02 \x01(\x04\"B\n\x15ListCollectionsParams\x12\x0e\n\x06prefix\x18\x01 \x01(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"K\n\x17ListCollectionsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x13\n\x0b\x63ollections\x18\x02 \x03(\t\"\xab\x01\n\x13LookupStreamsParams\x12\x12\n\ncollection\x18\x01 \x01(\t\x12\x1a\n\x12isCollectionPrefix\x18\x02 \x01(\x08\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"^\n\x15LookupStreamsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12(\n\x07results\x18\x02 \x03(\x0b\x32\x17.v5api.StreamDescriptor\"S\n\rNearestParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x0c\n\x04time\x18\x02 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x03 \x01(\x04\x12\x10\n\x08\x62\x61\x63kward\x18\x04 \x01(\x08\"z\n\x0fNearestResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x1e\n\x05value\x18\x04 \x01(\x0b\x32\x0f.v5api.RawPoint\"U\n\rChangesParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x11\n\tfromMajor\x18\x02 \x01(\x04\x12\x0f\n\x07toMajor\x18\x03 \x01(\x04\x12\x12\n\nresolution\x18\x04 \x01(\r\"\x7f\n\x0f\x43hangesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12#\n\x06ranges\x18\x04 \x03(\x0b\x32\x13.v5api.ChangedRange\"#\n\tRoundSpec\x12\x0e\n\x04\x62its\x18\x02 \x01(\x05H\x00\x42\x06\n\x04spec\"\x99\x01\n\x0cInsertParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x0c\n\x04sync\x18\x02 \x01(\x08\x12(\n\x0cmerge_policy\x18\x04 \x01(\x0e\x32\x12.v5api.MergePolicy\x12\"\n\x08rounding\x18\x05 \x01(\x0b\x32\x10.v5api.RoundSpec\x12\x1f\n\x06values\x18\x03 \x03(\x0b\x32\x0f.v5api.RawPoint\"\x91\x01\n\x11\x41rrowInsertParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x0c\n\x04sync\x18\x02 \x01(\x08\x12(\n\x0cmerge_policy\x18\x03 \x01(\x0e\x32\x12.v5api.MergePolicy\x12\"\n\x08rounding\x18\x04 \x01(\x0b\x32\x10.v5api.RoundSpec\x12\x12\n\narrowBytes\x18\x05 \x01(\x0c\"Y\n\x0eInsertResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\"8\n\x0c\x44\x65leteParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\"Y\n\x0e\x44\x65leteResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\"\x0c\n\nInfoParams\"\xa2\x01\n\x0cInfoResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x19\n\x04mash\x18\x02 \x01(\x0b\x32\x0b.v5api.Mash\x12\x14\n\x0cmajorVersion\x18\x03 \x01(\r\x12\x14\n\x0cminorVersion\x18\x04 \x01(\r\x12\r\n\x05\x62uild\x18\x05 \x01(\t\x12\x1f\n\x05proxy\x18\x06 \x01(\x0b\x32\x10.v5api.ProxyInfo\"#\n\tProxyInfo\x12\x16\n\x0eproxyEndpoints\x18\x01 \x03(\t\"1\n\x11\x46\x61ultInjectParams\x12\x0c\n\x04type\x18\x01 \x01(\x04\x12\x0e\n\x06params\x18\x02 \x01(\x0c\">\n\x13\x46\x61ultInjectResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\n\n\x02rv\x18\x02 \x01(\x0c\"\x1b\n\x0b\x46lushParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"X\n\rFlushResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\" \n\x10ObliterateParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"1\n\x12ObliterateResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"\'\n\x08RawPoint\x12\x0c\n\x04time\x18\x01 \x01(\x10\x12\r\n\x05value\x18\x02 \x01(\x01\"`\n\tStatPoint\x12\x0c\n\x04time\x18\x01 \x01(\x10\x12\x0b\n\x03min\x18\x02 \x01(\x01\x12\x0c\n\x04mean\x18\x03 \x01(\x01\x12\x0b\n\x03max\x18\x04 \x01(\x01\x12\r\n\x05\x63ount\x18\x05 \x01(\x06\x12\x0e\n\x06stddev\x18\x06 \x01(\x01\"*\n\x0c\x43hangedRange\x12\r\n\x05start\x18\x01 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x02 \x01(\x10\">\n\x06Status\x12\x0c\n\x04\x63ode\x18\x01 \x01(\r\x12\x0b\n\x03msg\x18\x02 \x01(\t\x12\x19\n\x04mash\x18\x03 \x01(\x0b\x32\x0b.v5api.Mash\"\x98\x01\n\x04Mash\x12\x10\n\x08revision\x18\x01 \x01(\x03\x12\x0e\n\x06leader\x18\x02 \x01(\t\x12\x16\n\x0eleaderRevision\x18\x03 \x01(\x03\x12\x13\n\x0btotalWeight\x18\x04 \x01(\x03\x12\x0f\n\x07healthy\x18\x05 \x01(\x08\x12\x10\n\x08unmapped\x18\x06 \x01(\x01\x12\x1e\n\x07members\x18\x07 \x03(\x0b\x32\r.v5api.Member\"\xc3\x01\n\x06Member\x12\x0c\n\x04hash\x18\x01 \x01(\r\x12\x10\n\x08nodename\x18\x02 \x01(\t\x12\n\n\x02up\x18\x03 \x01(\x08\x12\n\n\x02in\x18\x04 \x01(\x08\x12\x0f\n\x07\x65nabled\x18\x05 \x01(\x08\x12\r\n\x05start\x18\x06 \x01(\x03\x12\x0b\n\x03\x65nd\x18\x07 \x01(\x03\x12\x0e\n\x06weight\x18\x08 \x01(\x03\x12\x16\n\x0ereadPreference\x18\t \x01(\x01\x12\x15\n\rhttpEndpoints\x18\n \x01(\t\x12\x15\n\rgrpcEndpoints\x18\x0b \x01(\t\"8\n\x0bKeyOptValue\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x1c\n\x03val\x18\x02 \x01(\x0b\x32\x0f.v5api.OptValue\"\x19\n\x08OptValue\x12\r\n\x05value\x18\x01 \x01(\t\"&\n\x08KeyValue\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"?\n\x0fStreamCSVConfig\x12\x0f\n\x07version\x18\x01 \x01(\x04\x12\r\n\x05label\x18\x02 \x01(\t\x12\x0c\n\x04uuid\x18\x03 \x01(\x0c\"\x9d\x02\n\x11GenerateCSVParams\x12\x35\n\tqueryType\x18\x01 \x01(\x0e\x32\".v5api.GenerateCSVParams.QueryType\x12\x11\n\tstartTime\x18\x02 \x01(\x03\x12\x0f\n\x07\x65ndTime\x18\x03 \x01(\x03\x12\x12\n\nwindowSize\x18\x04 \x01(\x04\x12\r\n\x05\x64\x65pth\x18\x05 \x01(\r\x12\x17\n\x0fincludeVersions\x18\x06 \x01(\x08\x12\'\n\x07streams\x18\x07 \x03(\x0b\x32\x16.v5api.StreamCSVConfig\"H\n\tQueryType\x12\x19\n\x15\x41LIGNED_WINDOWS_QUERY\x10\x00\x12\x11\n\rWINDOWS_QUERY\x10\x01\x12\r\n\tRAW_QUERY\x10\x02\"Q\n\x13GenerateCSVResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x10\n\x08isHeader\x18\x02 \x01(\x08\x12\x0b\n\x03row\x18\x03 \x03(\t\"J\n\x0eSQLQueryParams\x12\r\n\x05query\x18\x01 \x01(\t\x12\x0e\n\x06params\x18\x02 \x03(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"D\n\x10SQLQueryResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x13\n\x0bSQLQueryRow\x18\x02 \x03(\x0c\"\x14\n\x04Role\x12\x0c\n\x04name\x18\x01 \x01(\t\"\x94\x01\n\x19SetCompactionConfigParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x18\n\x10\x43ompactedVersion\x18\x02 \x01(\x04\x12>\n\x17reducedResolutionRanges\x18\x03 \x03(\x0b\x32\x1d.v5api.ReducedResolutionRange\x12\x0f\n\x07unused0\x18\x04 \x01(\x04\":\n\x1bSetCompactionConfigResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\")\n\x19GetCompactionConfigParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"\xc1\x01\n\x1bGetCompactionConfigResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x1a\n\x12LatestMajorVersion\x18\x02 \x01(\x04\x12\x18\n\x10\x43ompactedVersion\x18\x03 \x01(\x04\x12>\n\x17reducedResolutionRanges\x18\x04 \x03(\x0b\x32\x1d.v5api.ReducedResolutionRange\x12\x0f\n\x07unused0\x18\x05 \x01(\x04\"H\n\x16ReducedResolutionRange\x12\r\n\x05Start\x18\x01 \x01(\x03\x12\x0b\n\x03\x45nd\x18\x02 \x01(\x03\x12\x12\n\nResolution\x18\x03 \x01(\r\"K\n\x12SubscriptionUpdate\x12\'\n\x02op\x18\x01 \x01(\x0e\x32\x1b.v5api.SubscriptionUpdateOp\x12\x0c\n\x04uuid\x18\x02 \x03(\x0c\"Q\n\x10SubscriptionResp\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x0c\n\x04uuid\x18\x02 \x01(\x0c\x12\x12\n\narrowBytes\x18\x03 \x01(\x0c*<\n\x0bMergePolicy\x12\t\n\x05NEVER\x10\x00\x12\t\n\x05\x45QUAL\x10\x01\x12\n\n\x06RETAIN\x10\x02\x12\x0b\n\x07REPLACE\x10\x03*7\n\x14SubscriptionUpdateOp\x12\r\n\tADD_UUIDS\x10\x00\x12\x10\n\x0cREMOVE_UUIDS\x10\x01\x32\xf8\r\n\x05\x42TrDB\x12?\n\tRawValues\x12\x16.v5api.RawValuesParams\x1a\x18.v5api.RawValuesResponse0\x01\x12I\n\x0e\x41rrowRawValues\x12\x16.v5api.RawValuesParams\x1a\x1d.v5api.ArrowRawValuesResponse0\x01\x12T\n\x10\x41rrowMultiValues\x12\x1d.v5api.ArrowMultiValuesParams\x1a\x1f.v5api.ArrowMultiValuesResponse0\x01\x12N\n\x0e\x41lignedWindows\x12\x1b.v5api.AlignedWindowsParams\x1a\x1d.v5api.AlignedWindowsResponse0\x01\x12X\n\x13\x41rrowAlignedWindows\x12\x1b.v5api.AlignedWindowsParams\x1a\".v5api.ArrowAlignedWindowsResponse0\x01\x12\x39\n\x07Windows\x12\x14.v5api.WindowsParams\x1a\x16.v5api.WindowsResponse0\x01\x12\x43\n\x0c\x41rrowWindows\x12\x14.v5api.WindowsParams\x1a\x1b.v5api.ArrowWindowsResponse0\x01\x12@\n\nStreamInfo\x12\x17.v5api.StreamInfoParams\x1a\x19.v5api.StreamInfoResponse\x12^\n\x14SetStreamAnnotations\x12!.v5api.SetStreamAnnotationsParams\x1a#.v5api.SetStreamAnnotationsResponse\x12I\n\rSetStreamTags\x12\x1a.v5api.SetStreamTagsParams\x1a\x1c.v5api.SetStreamTagsResponse\x12\x34\n\x06\x43reate\x12\x13.v5api.CreateParams\x1a\x15.v5api.CreateResponse\x12Q\n\x0fListCollections\x12\x1c.v5api.ListCollectionsParams\x1a\x1e.v5api.ListCollectionsResponse0\x01\x12K\n\rLookupStreams\x12\x1a.v5api.LookupStreamsParams\x1a\x1c.v5api.LookupStreamsResponse0\x01\x12\x37\n\x07Nearest\x12\x14.v5api.NearestParams\x1a\x16.v5api.NearestResponse\x12\x39\n\x07\x43hanges\x12\x14.v5api.ChangesParams\x1a\x16.v5api.ChangesResponse0\x01\x12\x34\n\x06Insert\x12\x13.v5api.InsertParams\x1a\x15.v5api.InsertResponse\x12>\n\x0b\x41rrowInsert\x12\x18.v5api.ArrowInsertParams\x1a\x15.v5api.InsertResponse\x12\x34\n\x06\x44\x65lete\x12\x13.v5api.DeleteParams\x1a\x15.v5api.DeleteResponse\x12.\n\x04Info\x12\x11.v5api.InfoParams\x1a\x13.v5api.InfoResponse\x12\x43\n\x0b\x46\x61ultInject\x12\x18.v5api.FaultInjectParams\x1a\x1a.v5api.FaultInjectResponse\x12\x31\n\x05\x46lush\x12\x12.v5api.FlushParams\x1a\x14.v5api.FlushResponse\x12@\n\nObliterate\x12\x17.v5api.ObliterateParams\x1a\x19.v5api.ObliterateResponse\x12L\n\x10GetMetadataUsage\x12\x1a.v5api.MetadataUsageParams\x1a\x1c.v5api.MetadataUsageResponse\x12\x45\n\x0bGenerateCSV\x12\x18.v5api.GenerateCSVParams\x1a\x1a.v5api.GenerateCSVResponse0\x01\x12<\n\x08SQLQuery\x12\x15.v5api.SQLQueryParams\x1a\x17.v5api.SQLQueryResponse0\x01\x12\x43\n\tSubscribe\x12\x19.v5api.SubscriptionUpdate\x1a\x17.v5api.SubscriptionResp(\x01\x30\x01\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0b\x62trdb.proto\x12\x05v5api\"Q\n\x0fRawValuesParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\"}\n\x11RawValuesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x1f\n\x06values\x18\x04 \x03(\x0b\x32\x0f.v5api.RawPoint\"m\n\x14\x41rrowRawValuesParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\x12\x15\n\rtemplateBytes\x18\x05 \x01(\x0c\"u\n\x16\x41rrowRawValuesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x12\n\narrowBytes\x18\x04 \x01(\x0c\"\x85\x01\n\x16\x41rrowMultiValuesParams\x12\x0c\n\x04uuid\x18\x01 \x03(\x0c\x12\x14\n\x0cversionMajor\x18\x02 \x03(\x04\x12\r\n\x05start\x18\x03 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x04 \x01(\x10\x12\x14\n\x0csnapPeriodNs\x18\x05 \x01(\x03\x12\x15\n\rtemplateBytes\x18\x06 \x01(\x0c\"K\n\x18\x41rrowMultiValuesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x12\n\narrowBytes\x18\x02 \x01(\x0c\"*\n\x0bRawPointVec\x12\x0c\n\x04time\x18\x01 \x01(\x10\x12\r\n\x05value\x18\x02 \x03(\x01\"j\n\x14\x41lignedWindowsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\x12\x12\n\npointWidth\x18\x05 \x01(\r\"\x83\x01\n\x16\x41lignedWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12 \n\x06values\x18\x04 \x03(\x0b\x32\x10.v5api.StatPoint\"z\n\x1b\x41rrowAlignedWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x12\n\narrowBytes\x18\x04 \x01(\x0c\"m\n\rWindowsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\x12\r\n\x05width\x18\x05 \x01(\x04\x12\r\n\x05\x64\x65pth\x18\x06 \x01(\r\"|\n\x0fWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12 \n\x06values\x18\x04 \x03(\x0b\x32\x10.v5api.StatPoint\"s\n\x14\x41rrowWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x12\n\narrowBytes\x18\x04 \x01(\x0c\"h\n\x10StreamInfoParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x13\n\x0bomitVersion\x18\x02 \x01(\x08\x12\x16\n\x0eomitDescriptor\x18\x03 \x01(\x08\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"\x8a\x01\n\x12StreamInfoResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12+\n\ndescriptor\x18\x04 \x01(\x0b\x32\x17.v5api.StreamDescriptor\"\x98\x01\n\x10StreamDescriptor\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x12\n\ncollection\x18\x02 \x01(\t\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x17\n\x0fpropertyVersion\x18\x05 \x01(\x04\"\x82\x01\n\x1aSetStreamAnnotationsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x1f\n\x17\x65xpectedPropertyVersion\x18\x02 \x01(\x04\x12#\n\x07\x63hanges\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x10\n\x08removals\x18\x04 \x03(\t\";\n\x1cSetStreamAnnotationsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"\x8a\x01\n\x13SetStreamTagsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x1f\n\x17\x65xpectedPropertyVersion\x18\x02 \x01(\x04\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x12\n\ncollection\x18\x04 \x01(\t\x12\x0e\n\x06remove\x18\x05 \x03(\t\"4\n\x15SetStreamTagsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"{\n\x0c\x43reateParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x12\n\ncollection\x18\x02 \x01(\t\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\"-\n\x0e\x43reateResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"@\n\x13MetadataUsageParams\x12\x0e\n\x06prefix\x18\x01 \x01(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"y\n\x15MetadataUsageResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x1d\n\x04tags\x18\x02 \x03(\x0b\x32\x0f.v5api.KeyCount\x12$\n\x0b\x61nnotations\x18\x03 \x03(\x0b\x32\x0f.v5api.KeyCount\"&\n\x08KeyCount\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05\x63ount\x18\x02 \x01(\x04\"B\n\x15ListCollectionsParams\x12\x0e\n\x06prefix\x18\x01 \x01(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"K\n\x17ListCollectionsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x13\n\x0b\x63ollections\x18\x02 \x03(\t\"\xab\x01\n\x13LookupStreamsParams\x12\x12\n\ncollection\x18\x01 \x01(\t\x12\x1a\n\x12isCollectionPrefix\x18\x02 \x01(\x08\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"^\n\x15LookupStreamsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12(\n\x07results\x18\x02 \x03(\x0b\x32\x17.v5api.StreamDescriptor\"S\n\rNearestParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x0c\n\x04time\x18\x02 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x03 \x01(\x04\x12\x10\n\x08\x62\x61\x63kward\x18\x04 \x01(\x08\"z\n\x0fNearestResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x1e\n\x05value\x18\x04 \x01(\x0b\x32\x0f.v5api.RawPoint\"U\n\rChangesParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x11\n\tfromMajor\x18\x02 \x01(\x04\x12\x0f\n\x07toMajor\x18\x03 \x01(\x04\x12\x12\n\nresolution\x18\x04 \x01(\r\"\x7f\n\x0f\x43hangesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12#\n\x06ranges\x18\x04 \x03(\x0b\x32\x13.v5api.ChangedRange\"#\n\tRoundSpec\x12\x0e\n\x04\x62its\x18\x02 \x01(\x05H\x00\x42\x06\n\x04spec\"\x99\x01\n\x0cInsertParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x0c\n\x04sync\x18\x02 \x01(\x08\x12(\n\x0cmerge_policy\x18\x04 \x01(\x0e\x32\x12.v5api.MergePolicy\x12\"\n\x08rounding\x18\x05 \x01(\x0b\x32\x10.v5api.RoundSpec\x12\x1f\n\x06values\x18\x03 \x03(\x0b\x32\x0f.v5api.RawPoint\"\x91\x01\n\x11\x41rrowInsertParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x0c\n\x04sync\x18\x02 \x01(\x08\x12(\n\x0cmerge_policy\x18\x03 \x01(\x0e\x32\x12.v5api.MergePolicy\x12\"\n\x08rounding\x18\x04 \x01(\x0b\x32\x10.v5api.RoundSpec\x12\x12\n\narrowBytes\x18\x05 \x01(\x0c\"Y\n\x0eInsertResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\"8\n\x0c\x44\x65leteParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\"Y\n\x0e\x44\x65leteResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\"\x0c\n\nInfoParams\"\xa2\x01\n\x0cInfoResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x19\n\x04mash\x18\x02 \x01(\x0b\x32\x0b.v5api.Mash\x12\x14\n\x0cmajorVersion\x18\x03 \x01(\r\x12\x14\n\x0cminorVersion\x18\x04 \x01(\r\x12\r\n\x05\x62uild\x18\x05 \x01(\t\x12\x1f\n\x05proxy\x18\x06 \x01(\x0b\x32\x10.v5api.ProxyInfo\"#\n\tProxyInfo\x12\x16\n\x0eproxyEndpoints\x18\x01 \x03(\t\"1\n\x11\x46\x61ultInjectParams\x12\x0c\n\x04type\x18\x01 \x01(\x04\x12\x0e\n\x06params\x18\x02 \x01(\x0c\">\n\x13\x46\x61ultInjectResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\n\n\x02rv\x18\x02 \x01(\x0c\"\x1b\n\x0b\x46lushParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"X\n\rFlushResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\" \n\x10ObliterateParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"1\n\x12ObliterateResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"\'\n\x08RawPoint\x12\x0c\n\x04time\x18\x01 \x01(\x10\x12\r\n\x05value\x18\x02 \x01(\x01\"`\n\tStatPoint\x12\x0c\n\x04time\x18\x01 \x01(\x10\x12\x0b\n\x03min\x18\x02 \x01(\x01\x12\x0c\n\x04mean\x18\x03 \x01(\x01\x12\x0b\n\x03max\x18\x04 \x01(\x01\x12\r\n\x05\x63ount\x18\x05 \x01(\x06\x12\x0e\n\x06stddev\x18\x06 \x01(\x01\"*\n\x0c\x43hangedRange\x12\r\n\x05start\x18\x01 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x02 \x01(\x10\">\n\x06Status\x12\x0c\n\x04\x63ode\x18\x01 \x01(\r\x12\x0b\n\x03msg\x18\x02 \x01(\t\x12\x19\n\x04mash\x18\x03 \x01(\x0b\x32\x0b.v5api.Mash\"\x98\x01\n\x04Mash\x12\x10\n\x08revision\x18\x01 \x01(\x03\x12\x0e\n\x06leader\x18\x02 \x01(\t\x12\x16\n\x0eleaderRevision\x18\x03 \x01(\x03\x12\x13\n\x0btotalWeight\x18\x04 \x01(\x03\x12\x0f\n\x07healthy\x18\x05 \x01(\x08\x12\x10\n\x08unmapped\x18\x06 \x01(\x01\x12\x1e\n\x07members\x18\x07 \x03(\x0b\x32\r.v5api.Member\"\xc3\x01\n\x06Member\x12\x0c\n\x04hash\x18\x01 \x01(\r\x12\x10\n\x08nodename\x18\x02 \x01(\t\x12\n\n\x02up\x18\x03 \x01(\x08\x12\n\n\x02in\x18\x04 \x01(\x08\x12\x0f\n\x07\x65nabled\x18\x05 \x01(\x08\x12\r\n\x05start\x18\x06 \x01(\x03\x12\x0b\n\x03\x65nd\x18\x07 \x01(\x03\x12\x0e\n\x06weight\x18\x08 \x01(\x03\x12\x16\n\x0ereadPreference\x18\t \x01(\x01\x12\x15\n\rhttpEndpoints\x18\n \x01(\t\x12\x15\n\rgrpcEndpoints\x18\x0b \x01(\t\"8\n\x0bKeyOptValue\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x1c\n\x03val\x18\x02 \x01(\x0b\x32\x0f.v5api.OptValue\"\x19\n\x08OptValue\x12\r\n\x05value\x18\x01 \x01(\t\"&\n\x08KeyValue\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"?\n\x0fStreamCSVConfig\x12\x0f\n\x07version\x18\x01 \x01(\x04\x12\r\n\x05label\x18\x02 \x01(\t\x12\x0c\n\x04uuid\x18\x03 \x01(\x0c\"\x9d\x02\n\x11GenerateCSVParams\x12\x35\n\tqueryType\x18\x01 \x01(\x0e\x32\".v5api.GenerateCSVParams.QueryType\x12\x11\n\tstartTime\x18\x02 \x01(\x03\x12\x0f\n\x07\x65ndTime\x18\x03 \x01(\x03\x12\x12\n\nwindowSize\x18\x04 \x01(\x04\x12\r\n\x05\x64\x65pth\x18\x05 \x01(\r\x12\x17\n\x0fincludeVersions\x18\x06 \x01(\x08\x12\'\n\x07streams\x18\x07 \x03(\x0b\x32\x16.v5api.StreamCSVConfig\"H\n\tQueryType\x12\x19\n\x15\x41LIGNED_WINDOWS_QUERY\x10\x00\x12\x11\n\rWINDOWS_QUERY\x10\x01\x12\r\n\tRAW_QUERY\x10\x02\"Q\n\x13GenerateCSVResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x10\n\x08isHeader\x18\x02 \x01(\x08\x12\x0b\n\x03row\x18\x03 \x03(\t\"J\n\x0eSQLQueryParams\x12\r\n\x05query\x18\x01 \x01(\t\x12\x0e\n\x06params\x18\x02 \x03(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"D\n\x10SQLQueryResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x13\n\x0bSQLQueryRow\x18\x02 \x03(\x0c\"\x14\n\x04Role\x12\x0c\n\x04name\x18\x01 \x01(\t\"\x94\x01\n\x19SetCompactionConfigParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x18\n\x10\x43ompactedVersion\x18\x02 \x01(\x04\x12>\n\x17reducedResolutionRanges\x18\x03 \x03(\x0b\x32\x1d.v5api.ReducedResolutionRange\x12\x0f\n\x07unused0\x18\x04 \x01(\x04\":\n\x1bSetCompactionConfigResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\")\n\x19GetCompactionConfigParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"\xc1\x01\n\x1bGetCompactionConfigResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x1a\n\x12LatestMajorVersion\x18\x02 \x01(\x04\x12\x18\n\x10\x43ompactedVersion\x18\x03 \x01(\x04\x12>\n\x17reducedResolutionRanges\x18\x04 \x03(\x0b\x32\x1d.v5api.ReducedResolutionRange\x12\x0f\n\x07unused0\x18\x05 \x01(\x04\"H\n\x16ReducedResolutionRange\x12\r\n\x05Start\x18\x01 \x01(\x03\x12\x0b\n\x03\x45nd\x18\x02 \x01(\x03\x12\x12\n\nResolution\x18\x03 \x01(\r\"K\n\x12SubscriptionUpdate\x12\'\n\x02op\x18\x01 \x01(\x0e\x32\x1b.v5api.SubscriptionUpdateOp\x12\x0c\n\x04uuid\x18\x02 \x03(\x0c\"Q\n\x10SubscriptionResp\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x0c\n\x04uuid\x18\x02 \x01(\x0c\x12\x12\n\narrowBytes\x18\x03 \x01(\x0c*<\n\x0bMergePolicy\x12\t\n\x05NEVER\x10\x00\x12\t\n\x05\x45QUAL\x10\x01\x12\n\n\x06RETAIN\x10\x02\x12\x0b\n\x07REPLACE\x10\x03*7\n\x14SubscriptionUpdateOp\x12\r\n\tADD_UUIDS\x10\x00\x12\x10\n\x0cREMOVE_UUIDS\x10\x01\x32\xfd\r\n\x05\x42TrDB\x12?\n\tRawValues\x12\x16.v5api.RawValuesParams\x1a\x18.v5api.RawValuesResponse0\x01\x12N\n\x0e\x41rrowRawValues\x12\x1b.v5api.ArrowRawValuesParams\x1a\x1d.v5api.ArrowRawValuesResponse0\x01\x12T\n\x10\x41rrowMultiValues\x12\x1d.v5api.ArrowMultiValuesParams\x1a\x1f.v5api.ArrowMultiValuesResponse0\x01\x12N\n\x0e\x41lignedWindows\x12\x1b.v5api.AlignedWindowsParams\x1a\x1d.v5api.AlignedWindowsResponse0\x01\x12X\n\x13\x41rrowAlignedWindows\x12\x1b.v5api.AlignedWindowsParams\x1a\".v5api.ArrowAlignedWindowsResponse0\x01\x12\x39\n\x07Windows\x12\x14.v5api.WindowsParams\x1a\x16.v5api.WindowsResponse0\x01\x12\x43\n\x0c\x41rrowWindows\x12\x14.v5api.WindowsParams\x1a\x1b.v5api.ArrowWindowsResponse0\x01\x12@\n\nStreamInfo\x12\x17.v5api.StreamInfoParams\x1a\x19.v5api.StreamInfoResponse\x12^\n\x14SetStreamAnnotations\x12!.v5api.SetStreamAnnotationsParams\x1a#.v5api.SetStreamAnnotationsResponse\x12I\n\rSetStreamTags\x12\x1a.v5api.SetStreamTagsParams\x1a\x1c.v5api.SetStreamTagsResponse\x12\x34\n\x06\x43reate\x12\x13.v5api.CreateParams\x1a\x15.v5api.CreateResponse\x12Q\n\x0fListCollections\x12\x1c.v5api.ListCollectionsParams\x1a\x1e.v5api.ListCollectionsResponse0\x01\x12K\n\rLookupStreams\x12\x1a.v5api.LookupStreamsParams\x1a\x1c.v5api.LookupStreamsResponse0\x01\x12\x37\n\x07Nearest\x12\x14.v5api.NearestParams\x1a\x16.v5api.NearestResponse\x12\x39\n\x07\x43hanges\x12\x14.v5api.ChangesParams\x1a\x16.v5api.ChangesResponse0\x01\x12\x34\n\x06Insert\x12\x13.v5api.InsertParams\x1a\x15.v5api.InsertResponse\x12>\n\x0b\x41rrowInsert\x12\x18.v5api.ArrowInsertParams\x1a\x15.v5api.InsertResponse\x12\x34\n\x06\x44\x65lete\x12\x13.v5api.DeleteParams\x1a\x15.v5api.DeleteResponse\x12.\n\x04Info\x12\x11.v5api.InfoParams\x1a\x13.v5api.InfoResponse\x12\x43\n\x0b\x46\x61ultInject\x12\x18.v5api.FaultInjectParams\x1a\x1a.v5api.FaultInjectResponse\x12\x31\n\x05\x46lush\x12\x12.v5api.FlushParams\x1a\x14.v5api.FlushResponse\x12@\n\nObliterate\x12\x17.v5api.ObliterateParams\x1a\x19.v5api.ObliterateResponse\x12L\n\x10GetMetadataUsage\x12\x1a.v5api.MetadataUsageParams\x1a\x1c.v5api.MetadataUsageResponse\x12\x45\n\x0bGenerateCSV\x12\x18.v5api.GenerateCSVParams\x1a\x1a.v5api.GenerateCSVResponse0\x01\x12<\n\x08SQLQuery\x12\x15.v5api.SQLQueryParams\x1a\x17.v5api.SQLQueryResponse0\x01\x12\x43\n\tSubscribe\x12\x19.v5api.SubscriptionUpdate\x1a\x17.v5api.SubscriptionResp(\x01\x30\x01\x62\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'btrdb_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None - _globals['_MERGEPOLICY']._serialized_start=6465 - _globals['_MERGEPOLICY']._serialized_end=6525 - _globals['_SUBSCRIPTIONUPDATEOP']._serialized_start=6527 - _globals['_SUBSCRIPTIONUPDATEOP']._serialized_end=6582 + _globals['_MERGEPOLICY']._serialized_start=6600 + _globals['_MERGEPOLICY']._serialized_end=6660 + _globals['_SUBSCRIPTIONUPDATEOP']._serialized_start=6662 + _globals['_SUBSCRIPTIONUPDATEOP']._serialized_end=6717 _globals['_RAWVALUESPARAMS']._serialized_start=22 _globals['_RAWVALUESPARAMS']._serialized_end=103 _globals['_RAWVALUESRESPONSE']._serialized_start=105 _globals['_RAWVALUESRESPONSE']._serialized_end=230 - _globals['_ARROWRAWVALUESRESPONSE']._serialized_start=232 - _globals['_ARROWRAWVALUESRESPONSE']._serialized_end=349 - _globals['_ARROWMULTIVALUESPARAMS']._serialized_start=351 - _globals['_ARROWMULTIVALUESPARAMS']._serialized_end=461 - _globals['_ARROWMULTIVALUESRESPONSE']._serialized_start=463 - _globals['_ARROWMULTIVALUESRESPONSE']._serialized_end=538 - _globals['_RAWPOINTVEC']._serialized_start=540 - _globals['_RAWPOINTVEC']._serialized_end=582 - _globals['_ALIGNEDWINDOWSPARAMS']._serialized_start=584 - _globals['_ALIGNEDWINDOWSPARAMS']._serialized_end=690 - _globals['_ALIGNEDWINDOWSRESPONSE']._serialized_start=693 - _globals['_ALIGNEDWINDOWSRESPONSE']._serialized_end=824 - _globals['_ARROWALIGNEDWINDOWSRESPONSE']._serialized_start=826 - _globals['_ARROWALIGNEDWINDOWSRESPONSE']._serialized_end=948 - _globals['_WINDOWSPARAMS']._serialized_start=950 - _globals['_WINDOWSPARAMS']._serialized_end=1059 - _globals['_WINDOWSRESPONSE']._serialized_start=1061 - _globals['_WINDOWSRESPONSE']._serialized_end=1185 - _globals['_ARROWWINDOWSRESPONSE']._serialized_start=1187 - _globals['_ARROWWINDOWSRESPONSE']._serialized_end=1302 - _globals['_STREAMINFOPARAMS']._serialized_start=1304 - _globals['_STREAMINFOPARAMS']._serialized_end=1408 - _globals['_STREAMINFORESPONSE']._serialized_start=1411 - _globals['_STREAMINFORESPONSE']._serialized_end=1549 - _globals['_STREAMDESCRIPTOR']._serialized_start=1552 - _globals['_STREAMDESCRIPTOR']._serialized_end=1704 - _globals['_SETSTREAMANNOTATIONSPARAMS']._serialized_start=1707 - _globals['_SETSTREAMANNOTATIONSPARAMS']._serialized_end=1837 - _globals['_SETSTREAMANNOTATIONSRESPONSE']._serialized_start=1839 - _globals['_SETSTREAMANNOTATIONSRESPONSE']._serialized_end=1898 - _globals['_SETSTREAMTAGSPARAMS']._serialized_start=1901 - _globals['_SETSTREAMTAGSPARAMS']._serialized_end=2039 - _globals['_SETSTREAMTAGSRESPONSE']._serialized_start=2041 - _globals['_SETSTREAMTAGSRESPONSE']._serialized_end=2093 - _globals['_CREATEPARAMS']._serialized_start=2095 - _globals['_CREATEPARAMS']._serialized_end=2218 - _globals['_CREATERESPONSE']._serialized_start=2220 - _globals['_CREATERESPONSE']._serialized_end=2265 - _globals['_METADATAUSAGEPARAMS']._serialized_start=2267 - _globals['_METADATAUSAGEPARAMS']._serialized_end=2331 - _globals['_METADATAUSAGERESPONSE']._serialized_start=2333 - _globals['_METADATAUSAGERESPONSE']._serialized_end=2454 - _globals['_KEYCOUNT']._serialized_start=2456 - _globals['_KEYCOUNT']._serialized_end=2494 - _globals['_LISTCOLLECTIONSPARAMS']._serialized_start=2496 - _globals['_LISTCOLLECTIONSPARAMS']._serialized_end=2562 - _globals['_LISTCOLLECTIONSRESPONSE']._serialized_start=2564 - _globals['_LISTCOLLECTIONSRESPONSE']._serialized_end=2639 - _globals['_LOOKUPSTREAMSPARAMS']._serialized_start=2642 - _globals['_LOOKUPSTREAMSPARAMS']._serialized_end=2813 - _globals['_LOOKUPSTREAMSRESPONSE']._serialized_start=2815 - _globals['_LOOKUPSTREAMSRESPONSE']._serialized_end=2909 - _globals['_NEARESTPARAMS']._serialized_start=2911 - _globals['_NEARESTPARAMS']._serialized_end=2994 - _globals['_NEARESTRESPONSE']._serialized_start=2996 - _globals['_NEARESTRESPONSE']._serialized_end=3118 - _globals['_CHANGESPARAMS']._serialized_start=3120 - _globals['_CHANGESPARAMS']._serialized_end=3205 - _globals['_CHANGESRESPONSE']._serialized_start=3207 - _globals['_CHANGESRESPONSE']._serialized_end=3334 - _globals['_ROUNDSPEC']._serialized_start=3336 - _globals['_ROUNDSPEC']._serialized_end=3371 - _globals['_INSERTPARAMS']._serialized_start=3374 - _globals['_INSERTPARAMS']._serialized_end=3527 - _globals['_ARROWINSERTPARAMS']._serialized_start=3530 - _globals['_ARROWINSERTPARAMS']._serialized_end=3675 - _globals['_INSERTRESPONSE']._serialized_start=3677 - _globals['_INSERTRESPONSE']._serialized_end=3766 - _globals['_DELETEPARAMS']._serialized_start=3768 - _globals['_DELETEPARAMS']._serialized_end=3824 - _globals['_DELETERESPONSE']._serialized_start=3826 - _globals['_DELETERESPONSE']._serialized_end=3915 - _globals['_INFOPARAMS']._serialized_start=3917 - _globals['_INFOPARAMS']._serialized_end=3929 - _globals['_INFORESPONSE']._serialized_start=3932 - _globals['_INFORESPONSE']._serialized_end=4094 - _globals['_PROXYINFO']._serialized_start=4096 - _globals['_PROXYINFO']._serialized_end=4131 - _globals['_FAULTINJECTPARAMS']._serialized_start=4133 - _globals['_FAULTINJECTPARAMS']._serialized_end=4182 - _globals['_FAULTINJECTRESPONSE']._serialized_start=4184 - _globals['_FAULTINJECTRESPONSE']._serialized_end=4246 - _globals['_FLUSHPARAMS']._serialized_start=4248 - _globals['_FLUSHPARAMS']._serialized_end=4275 - _globals['_FLUSHRESPONSE']._serialized_start=4277 - _globals['_FLUSHRESPONSE']._serialized_end=4365 - _globals['_OBLITERATEPARAMS']._serialized_start=4367 - _globals['_OBLITERATEPARAMS']._serialized_end=4399 - _globals['_OBLITERATERESPONSE']._serialized_start=4401 - _globals['_OBLITERATERESPONSE']._serialized_end=4450 - _globals['_RAWPOINT']._serialized_start=4452 - _globals['_RAWPOINT']._serialized_end=4491 - _globals['_STATPOINT']._serialized_start=4493 - _globals['_STATPOINT']._serialized_end=4589 - _globals['_CHANGEDRANGE']._serialized_start=4591 - _globals['_CHANGEDRANGE']._serialized_end=4633 - _globals['_STATUS']._serialized_start=4635 - _globals['_STATUS']._serialized_end=4697 - _globals['_MASH']._serialized_start=4700 - _globals['_MASH']._serialized_end=4852 - _globals['_MEMBER']._serialized_start=4855 - _globals['_MEMBER']._serialized_end=5050 - _globals['_KEYOPTVALUE']._serialized_start=5052 - _globals['_KEYOPTVALUE']._serialized_end=5108 - _globals['_OPTVALUE']._serialized_start=5110 - _globals['_OPTVALUE']._serialized_end=5135 - _globals['_KEYVALUE']._serialized_start=5137 - _globals['_KEYVALUE']._serialized_end=5175 - _globals['_STREAMCSVCONFIG']._serialized_start=5177 - _globals['_STREAMCSVCONFIG']._serialized_end=5240 - _globals['_GENERATECSVPARAMS']._serialized_start=5243 - _globals['_GENERATECSVPARAMS']._serialized_end=5528 - _globals['_GENERATECSVPARAMS_QUERYTYPE']._serialized_start=5456 - _globals['_GENERATECSVPARAMS_QUERYTYPE']._serialized_end=5528 - _globals['_GENERATECSVRESPONSE']._serialized_start=5530 - _globals['_GENERATECSVRESPONSE']._serialized_end=5611 - _globals['_SQLQUERYPARAMS']._serialized_start=5613 - _globals['_SQLQUERYPARAMS']._serialized_end=5687 - _globals['_SQLQUERYRESPONSE']._serialized_start=5689 - _globals['_SQLQUERYRESPONSE']._serialized_end=5757 - _globals['_ROLE']._serialized_start=5759 - _globals['_ROLE']._serialized_end=5779 - _globals['_SETCOMPACTIONCONFIGPARAMS']._serialized_start=5782 - _globals['_SETCOMPACTIONCONFIGPARAMS']._serialized_end=5930 - _globals['_SETCOMPACTIONCONFIGRESPONSE']._serialized_start=5932 - _globals['_SETCOMPACTIONCONFIGRESPONSE']._serialized_end=5990 - _globals['_GETCOMPACTIONCONFIGPARAMS']._serialized_start=5992 - _globals['_GETCOMPACTIONCONFIGPARAMS']._serialized_end=6033 - _globals['_GETCOMPACTIONCONFIGRESPONSE']._serialized_start=6036 - _globals['_GETCOMPACTIONCONFIGRESPONSE']._serialized_end=6229 - _globals['_REDUCEDRESOLUTIONRANGE']._serialized_start=6231 - _globals['_REDUCEDRESOLUTIONRANGE']._serialized_end=6303 - _globals['_SUBSCRIPTIONUPDATE']._serialized_start=6305 - _globals['_SUBSCRIPTIONUPDATE']._serialized_end=6380 - _globals['_SUBSCRIPTIONRESP']._serialized_start=6382 - _globals['_SUBSCRIPTIONRESP']._serialized_end=6463 - _globals['_BTRDB']._serialized_start=6585 - _globals['_BTRDB']._serialized_end=8369 + _globals['_ARROWRAWVALUESPARAMS']._serialized_start=232 + _globals['_ARROWRAWVALUESPARAMS']._serialized_end=341 + _globals['_ARROWRAWVALUESRESPONSE']._serialized_start=343 + _globals['_ARROWRAWVALUESRESPONSE']._serialized_end=460 + _globals['_ARROWMULTIVALUESPARAMS']._serialized_start=463 + _globals['_ARROWMULTIVALUESPARAMS']._serialized_end=596 + _globals['_ARROWMULTIVALUESRESPONSE']._serialized_start=598 + _globals['_ARROWMULTIVALUESRESPONSE']._serialized_end=673 + _globals['_RAWPOINTVEC']._serialized_start=675 + _globals['_RAWPOINTVEC']._serialized_end=717 + _globals['_ALIGNEDWINDOWSPARAMS']._serialized_start=719 + _globals['_ALIGNEDWINDOWSPARAMS']._serialized_end=825 + _globals['_ALIGNEDWINDOWSRESPONSE']._serialized_start=828 + _globals['_ALIGNEDWINDOWSRESPONSE']._serialized_end=959 + _globals['_ARROWALIGNEDWINDOWSRESPONSE']._serialized_start=961 + _globals['_ARROWALIGNEDWINDOWSRESPONSE']._serialized_end=1083 + _globals['_WINDOWSPARAMS']._serialized_start=1085 + _globals['_WINDOWSPARAMS']._serialized_end=1194 + _globals['_WINDOWSRESPONSE']._serialized_start=1196 + _globals['_WINDOWSRESPONSE']._serialized_end=1320 + _globals['_ARROWWINDOWSRESPONSE']._serialized_start=1322 + _globals['_ARROWWINDOWSRESPONSE']._serialized_end=1437 + _globals['_STREAMINFOPARAMS']._serialized_start=1439 + _globals['_STREAMINFOPARAMS']._serialized_end=1543 + _globals['_STREAMINFORESPONSE']._serialized_start=1546 + _globals['_STREAMINFORESPONSE']._serialized_end=1684 + _globals['_STREAMDESCRIPTOR']._serialized_start=1687 + _globals['_STREAMDESCRIPTOR']._serialized_end=1839 + _globals['_SETSTREAMANNOTATIONSPARAMS']._serialized_start=1842 + _globals['_SETSTREAMANNOTATIONSPARAMS']._serialized_end=1972 + _globals['_SETSTREAMANNOTATIONSRESPONSE']._serialized_start=1974 + _globals['_SETSTREAMANNOTATIONSRESPONSE']._serialized_end=2033 + _globals['_SETSTREAMTAGSPARAMS']._serialized_start=2036 + _globals['_SETSTREAMTAGSPARAMS']._serialized_end=2174 + _globals['_SETSTREAMTAGSRESPONSE']._serialized_start=2176 + _globals['_SETSTREAMTAGSRESPONSE']._serialized_end=2228 + _globals['_CREATEPARAMS']._serialized_start=2230 + _globals['_CREATEPARAMS']._serialized_end=2353 + _globals['_CREATERESPONSE']._serialized_start=2355 + _globals['_CREATERESPONSE']._serialized_end=2400 + _globals['_METADATAUSAGEPARAMS']._serialized_start=2402 + _globals['_METADATAUSAGEPARAMS']._serialized_end=2466 + _globals['_METADATAUSAGERESPONSE']._serialized_start=2468 + _globals['_METADATAUSAGERESPONSE']._serialized_end=2589 + _globals['_KEYCOUNT']._serialized_start=2591 + _globals['_KEYCOUNT']._serialized_end=2629 + _globals['_LISTCOLLECTIONSPARAMS']._serialized_start=2631 + _globals['_LISTCOLLECTIONSPARAMS']._serialized_end=2697 + _globals['_LISTCOLLECTIONSRESPONSE']._serialized_start=2699 + _globals['_LISTCOLLECTIONSRESPONSE']._serialized_end=2774 + _globals['_LOOKUPSTREAMSPARAMS']._serialized_start=2777 + _globals['_LOOKUPSTREAMSPARAMS']._serialized_end=2948 + _globals['_LOOKUPSTREAMSRESPONSE']._serialized_start=2950 + _globals['_LOOKUPSTREAMSRESPONSE']._serialized_end=3044 + _globals['_NEARESTPARAMS']._serialized_start=3046 + _globals['_NEARESTPARAMS']._serialized_end=3129 + _globals['_NEARESTRESPONSE']._serialized_start=3131 + _globals['_NEARESTRESPONSE']._serialized_end=3253 + _globals['_CHANGESPARAMS']._serialized_start=3255 + _globals['_CHANGESPARAMS']._serialized_end=3340 + _globals['_CHANGESRESPONSE']._serialized_start=3342 + _globals['_CHANGESRESPONSE']._serialized_end=3469 + _globals['_ROUNDSPEC']._serialized_start=3471 + _globals['_ROUNDSPEC']._serialized_end=3506 + _globals['_INSERTPARAMS']._serialized_start=3509 + _globals['_INSERTPARAMS']._serialized_end=3662 + _globals['_ARROWINSERTPARAMS']._serialized_start=3665 + _globals['_ARROWINSERTPARAMS']._serialized_end=3810 + _globals['_INSERTRESPONSE']._serialized_start=3812 + _globals['_INSERTRESPONSE']._serialized_end=3901 + _globals['_DELETEPARAMS']._serialized_start=3903 + _globals['_DELETEPARAMS']._serialized_end=3959 + _globals['_DELETERESPONSE']._serialized_start=3961 + _globals['_DELETERESPONSE']._serialized_end=4050 + _globals['_INFOPARAMS']._serialized_start=4052 + _globals['_INFOPARAMS']._serialized_end=4064 + _globals['_INFORESPONSE']._serialized_start=4067 + _globals['_INFORESPONSE']._serialized_end=4229 + _globals['_PROXYINFO']._serialized_start=4231 + _globals['_PROXYINFO']._serialized_end=4266 + _globals['_FAULTINJECTPARAMS']._serialized_start=4268 + _globals['_FAULTINJECTPARAMS']._serialized_end=4317 + _globals['_FAULTINJECTRESPONSE']._serialized_start=4319 + _globals['_FAULTINJECTRESPONSE']._serialized_end=4381 + _globals['_FLUSHPARAMS']._serialized_start=4383 + _globals['_FLUSHPARAMS']._serialized_end=4410 + _globals['_FLUSHRESPONSE']._serialized_start=4412 + _globals['_FLUSHRESPONSE']._serialized_end=4500 + _globals['_OBLITERATEPARAMS']._serialized_start=4502 + _globals['_OBLITERATEPARAMS']._serialized_end=4534 + _globals['_OBLITERATERESPONSE']._serialized_start=4536 + _globals['_OBLITERATERESPONSE']._serialized_end=4585 + _globals['_RAWPOINT']._serialized_start=4587 + _globals['_RAWPOINT']._serialized_end=4626 + _globals['_STATPOINT']._serialized_start=4628 + _globals['_STATPOINT']._serialized_end=4724 + _globals['_CHANGEDRANGE']._serialized_start=4726 + _globals['_CHANGEDRANGE']._serialized_end=4768 + _globals['_STATUS']._serialized_start=4770 + _globals['_STATUS']._serialized_end=4832 + _globals['_MASH']._serialized_start=4835 + _globals['_MASH']._serialized_end=4987 + _globals['_MEMBER']._serialized_start=4990 + _globals['_MEMBER']._serialized_end=5185 + _globals['_KEYOPTVALUE']._serialized_start=5187 + _globals['_KEYOPTVALUE']._serialized_end=5243 + _globals['_OPTVALUE']._serialized_start=5245 + _globals['_OPTVALUE']._serialized_end=5270 + _globals['_KEYVALUE']._serialized_start=5272 + _globals['_KEYVALUE']._serialized_end=5310 + _globals['_STREAMCSVCONFIG']._serialized_start=5312 + _globals['_STREAMCSVCONFIG']._serialized_end=5375 + _globals['_GENERATECSVPARAMS']._serialized_start=5378 + _globals['_GENERATECSVPARAMS']._serialized_end=5663 + _globals['_GENERATECSVPARAMS_QUERYTYPE']._serialized_start=5591 + _globals['_GENERATECSVPARAMS_QUERYTYPE']._serialized_end=5663 + _globals['_GENERATECSVRESPONSE']._serialized_start=5665 + _globals['_GENERATECSVRESPONSE']._serialized_end=5746 + _globals['_SQLQUERYPARAMS']._serialized_start=5748 + _globals['_SQLQUERYPARAMS']._serialized_end=5822 + _globals['_SQLQUERYRESPONSE']._serialized_start=5824 + _globals['_SQLQUERYRESPONSE']._serialized_end=5892 + _globals['_ROLE']._serialized_start=5894 + _globals['_ROLE']._serialized_end=5914 + _globals['_SETCOMPACTIONCONFIGPARAMS']._serialized_start=5917 + _globals['_SETCOMPACTIONCONFIGPARAMS']._serialized_end=6065 + _globals['_SETCOMPACTIONCONFIGRESPONSE']._serialized_start=6067 + _globals['_SETCOMPACTIONCONFIGRESPONSE']._serialized_end=6125 + _globals['_GETCOMPACTIONCONFIGPARAMS']._serialized_start=6127 + _globals['_GETCOMPACTIONCONFIGPARAMS']._serialized_end=6168 + _globals['_GETCOMPACTIONCONFIGRESPONSE']._serialized_start=6171 + _globals['_GETCOMPACTIONCONFIGRESPONSE']._serialized_end=6364 + _globals['_REDUCEDRESOLUTIONRANGE']._serialized_start=6366 + _globals['_REDUCEDRESOLUTIONRANGE']._serialized_end=6438 + _globals['_SUBSCRIPTIONUPDATE']._serialized_start=6440 + _globals['_SUBSCRIPTIONUPDATE']._serialized_end=6515 + _globals['_SUBSCRIPTIONRESP']._serialized_start=6517 + _globals['_SUBSCRIPTIONRESP']._serialized_end=6598 + _globals['_BTRDB']._serialized_start=6720 + _globals['_BTRDB']._serialized_end=8509 # @@protoc_insertion_point(module_scope) diff --git a/btrdb/grpcinterface/btrdb_pb2.pyi b/btrdb/grpcinterface/btrdb_pb2.pyi index 9573a2c..cc6320c 100644 --- a/btrdb/grpcinterface/btrdb_pb2.pyi +++ b/btrdb/grpcinterface/btrdb_pb2.pyi @@ -7,14 +7,14 @@ from typing import ClassVar as _ClassVar, Iterable as _Iterable, Mapping as _Map DESCRIPTOR: _descriptor.FileDescriptor class MergePolicy(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): - __slots__ = [] + __slots__ = () NEVER: _ClassVar[MergePolicy] EQUAL: _ClassVar[MergePolicy] RETAIN: _ClassVar[MergePolicy] REPLACE: _ClassVar[MergePolicy] class SubscriptionUpdateOp(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): - __slots__ = [] + __slots__ = () ADD_UUIDS: _ClassVar[SubscriptionUpdateOp] REMOVE_UUIDS: _ClassVar[SubscriptionUpdateOp] NEVER: MergePolicy @@ -25,7 +25,7 @@ ADD_UUIDS: SubscriptionUpdateOp REMOVE_UUIDS: SubscriptionUpdateOp class RawValuesParams(_message.Message): - __slots__ = ["uuid", "start", "end", "versionMajor"] + __slots__ = ("uuid", "start", "end", "versionMajor") UUID_FIELD_NUMBER: _ClassVar[int] START_FIELD_NUMBER: _ClassVar[int] END_FIELD_NUMBER: _ClassVar[int] @@ -37,7 +37,7 @@ class RawValuesParams(_message.Message): def __init__(self, uuid: _Optional[bytes] = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., versionMajor: _Optional[int] = ...) -> None: ... class RawValuesResponse(_message.Message): - __slots__ = ["stat", "versionMajor", "versionMinor", "values"] + __slots__ = ("stat", "versionMajor", "versionMinor", "values") STAT_FIELD_NUMBER: _ClassVar[int] VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] @@ -48,8 +48,22 @@ class RawValuesResponse(_message.Message): values: _containers.RepeatedCompositeFieldContainer[RawPoint] def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., values: _Optional[_Iterable[_Union[RawPoint, _Mapping]]] = ...) -> None: ... +class ArrowRawValuesParams(_message.Message): + __slots__ = ("uuid", "start", "end", "versionMajor", "templateBytes") + UUID_FIELD_NUMBER: _ClassVar[int] + START_FIELD_NUMBER: _ClassVar[int] + END_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + TEMPLATEBYTES_FIELD_NUMBER: _ClassVar[int] + uuid: bytes + start: int + end: int + versionMajor: int + templateBytes: bytes + def __init__(self, uuid: _Optional[bytes] = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., versionMajor: _Optional[int] = ..., templateBytes: _Optional[bytes] = ...) -> None: ... + class ArrowRawValuesResponse(_message.Message): - __slots__ = ["stat", "versionMajor", "versionMinor", "arrowBytes"] + __slots__ = ("stat", "versionMajor", "versionMinor", "arrowBytes") STAT_FIELD_NUMBER: _ClassVar[int] VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] @@ -61,21 +75,23 @@ class ArrowRawValuesResponse(_message.Message): def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., arrowBytes: _Optional[bytes] = ...) -> None: ... class ArrowMultiValuesParams(_message.Message): - __slots__ = ["uuid", "versionMajor", "start", "end", "snapPeriodNs"] + __slots__ = ("uuid", "versionMajor", "start", "end", "snapPeriodNs", "templateBytes") UUID_FIELD_NUMBER: _ClassVar[int] VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] START_FIELD_NUMBER: _ClassVar[int] END_FIELD_NUMBER: _ClassVar[int] SNAPPERIODNS_FIELD_NUMBER: _ClassVar[int] + TEMPLATEBYTES_FIELD_NUMBER: _ClassVar[int] uuid: _containers.RepeatedScalarFieldContainer[bytes] versionMajor: _containers.RepeatedScalarFieldContainer[int] start: int end: int snapPeriodNs: int - def __init__(self, uuid: _Optional[_Iterable[bytes]] = ..., versionMajor: _Optional[_Iterable[int]] = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., snapPeriodNs: _Optional[int] = ...) -> None: ... + templateBytes: bytes + def __init__(self, uuid: _Optional[_Iterable[bytes]] = ..., versionMajor: _Optional[_Iterable[int]] = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., snapPeriodNs: _Optional[int] = ..., templateBytes: _Optional[bytes] = ...) -> None: ... class ArrowMultiValuesResponse(_message.Message): - __slots__ = ["stat", "arrowBytes"] + __slots__ = ("stat", "arrowBytes") STAT_FIELD_NUMBER: _ClassVar[int] ARROWBYTES_FIELD_NUMBER: _ClassVar[int] stat: Status @@ -83,7 +99,7 @@ class ArrowMultiValuesResponse(_message.Message): def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., arrowBytes: _Optional[bytes] = ...) -> None: ... class RawPointVec(_message.Message): - __slots__ = ["time", "value"] + __slots__ = ("time", "value") TIME_FIELD_NUMBER: _ClassVar[int] VALUE_FIELD_NUMBER: _ClassVar[int] time: int @@ -91,7 +107,7 @@ class RawPointVec(_message.Message): def __init__(self, time: _Optional[int] = ..., value: _Optional[_Iterable[float]] = ...) -> None: ... class AlignedWindowsParams(_message.Message): - __slots__ = ["uuid", "start", "end", "versionMajor", "pointWidth"] + __slots__ = ("uuid", "start", "end", "versionMajor", "pointWidth") UUID_FIELD_NUMBER: _ClassVar[int] START_FIELD_NUMBER: _ClassVar[int] END_FIELD_NUMBER: _ClassVar[int] @@ -105,7 +121,7 @@ class AlignedWindowsParams(_message.Message): def __init__(self, uuid: _Optional[bytes] = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., versionMajor: _Optional[int] = ..., pointWidth: _Optional[int] = ...) -> None: ... class AlignedWindowsResponse(_message.Message): - __slots__ = ["stat", "versionMajor", "versionMinor", "values"] + __slots__ = ("stat", "versionMajor", "versionMinor", "values") STAT_FIELD_NUMBER: _ClassVar[int] VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] @@ -117,7 +133,7 @@ class AlignedWindowsResponse(_message.Message): def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., values: _Optional[_Iterable[_Union[StatPoint, _Mapping]]] = ...) -> None: ... class ArrowAlignedWindowsResponse(_message.Message): - __slots__ = ["stat", "versionMajor", "versionMinor", "arrowBytes"] + __slots__ = ("stat", "versionMajor", "versionMinor", "arrowBytes") STAT_FIELD_NUMBER: _ClassVar[int] VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] @@ -129,7 +145,7 @@ class ArrowAlignedWindowsResponse(_message.Message): def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., arrowBytes: _Optional[bytes] = ...) -> None: ... class WindowsParams(_message.Message): - __slots__ = ["uuid", "start", "end", "versionMajor", "width", "depth"] + __slots__ = ("uuid", "start", "end", "versionMajor", "width", "depth") UUID_FIELD_NUMBER: _ClassVar[int] START_FIELD_NUMBER: _ClassVar[int] END_FIELD_NUMBER: _ClassVar[int] @@ -145,7 +161,7 @@ class WindowsParams(_message.Message): def __init__(self, uuid: _Optional[bytes] = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., versionMajor: _Optional[int] = ..., width: _Optional[int] = ..., depth: _Optional[int] = ...) -> None: ... class WindowsResponse(_message.Message): - __slots__ = ["stat", "versionMajor", "versionMinor", "values"] + __slots__ = ("stat", "versionMajor", "versionMinor", "values") STAT_FIELD_NUMBER: _ClassVar[int] VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] @@ -157,7 +173,7 @@ class WindowsResponse(_message.Message): def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., values: _Optional[_Iterable[_Union[StatPoint, _Mapping]]] = ...) -> None: ... class ArrowWindowsResponse(_message.Message): - __slots__ = ["stat", "versionMajor", "versionMinor", "arrowBytes"] + __slots__ = ("stat", "versionMajor", "versionMinor", "arrowBytes") STAT_FIELD_NUMBER: _ClassVar[int] VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] @@ -169,7 +185,7 @@ class ArrowWindowsResponse(_message.Message): def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., arrowBytes: _Optional[bytes] = ...) -> None: ... class StreamInfoParams(_message.Message): - __slots__ = ["uuid", "omitVersion", "omitDescriptor", "role"] + __slots__ = ("uuid", "omitVersion", "omitDescriptor", "role") UUID_FIELD_NUMBER: _ClassVar[int] OMITVERSION_FIELD_NUMBER: _ClassVar[int] OMITDESCRIPTOR_FIELD_NUMBER: _ClassVar[int] @@ -181,7 +197,7 @@ class StreamInfoParams(_message.Message): def __init__(self, uuid: _Optional[bytes] = ..., omitVersion: bool = ..., omitDescriptor: bool = ..., role: _Optional[_Union[Role, _Mapping]] = ...) -> None: ... class StreamInfoResponse(_message.Message): - __slots__ = ["stat", "versionMajor", "versionMinor", "descriptor"] + __slots__ = ("stat", "versionMajor", "versionMinor", "descriptor") STAT_FIELD_NUMBER: _ClassVar[int] VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] @@ -193,7 +209,7 @@ class StreamInfoResponse(_message.Message): def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., descriptor: _Optional[_Union[StreamDescriptor, _Mapping]] = ...) -> None: ... class StreamDescriptor(_message.Message): - __slots__ = ["uuid", "collection", "tags", "annotations", "propertyVersion"] + __slots__ = ("uuid", "collection", "tags", "annotations", "propertyVersion") UUID_FIELD_NUMBER: _ClassVar[int] COLLECTION_FIELD_NUMBER: _ClassVar[int] TAGS_FIELD_NUMBER: _ClassVar[int] @@ -207,7 +223,7 @@ class StreamDescriptor(_message.Message): def __init__(self, uuid: _Optional[bytes] = ..., collection: _Optional[str] = ..., tags: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., annotations: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., propertyVersion: _Optional[int] = ...) -> None: ... class SetStreamAnnotationsParams(_message.Message): - __slots__ = ["uuid", "expectedPropertyVersion", "changes", "removals"] + __slots__ = ("uuid", "expectedPropertyVersion", "changes", "removals") UUID_FIELD_NUMBER: _ClassVar[int] EXPECTEDPROPERTYVERSION_FIELD_NUMBER: _ClassVar[int] CHANGES_FIELD_NUMBER: _ClassVar[int] @@ -219,13 +235,13 @@ class SetStreamAnnotationsParams(_message.Message): def __init__(self, uuid: _Optional[bytes] = ..., expectedPropertyVersion: _Optional[int] = ..., changes: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., removals: _Optional[_Iterable[str]] = ...) -> None: ... class SetStreamAnnotationsResponse(_message.Message): - __slots__ = ["stat"] + __slots__ = ("stat",) STAT_FIELD_NUMBER: _ClassVar[int] stat: Status def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ...) -> None: ... class SetStreamTagsParams(_message.Message): - __slots__ = ["uuid", "expectedPropertyVersion", "tags", "collection", "remove"] + __slots__ = ("uuid", "expectedPropertyVersion", "tags", "collection", "remove") UUID_FIELD_NUMBER: _ClassVar[int] EXPECTEDPROPERTYVERSION_FIELD_NUMBER: _ClassVar[int] TAGS_FIELD_NUMBER: _ClassVar[int] @@ -239,13 +255,13 @@ class SetStreamTagsParams(_message.Message): def __init__(self, uuid: _Optional[bytes] = ..., expectedPropertyVersion: _Optional[int] = ..., tags: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., collection: _Optional[str] = ..., remove: _Optional[_Iterable[str]] = ...) -> None: ... class SetStreamTagsResponse(_message.Message): - __slots__ = ["stat"] + __slots__ = ("stat",) STAT_FIELD_NUMBER: _ClassVar[int] stat: Status def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ...) -> None: ... class CreateParams(_message.Message): - __slots__ = ["uuid", "collection", "tags", "annotations"] + __slots__ = ("uuid", "collection", "tags", "annotations") UUID_FIELD_NUMBER: _ClassVar[int] COLLECTION_FIELD_NUMBER: _ClassVar[int] TAGS_FIELD_NUMBER: _ClassVar[int] @@ -257,13 +273,13 @@ class CreateParams(_message.Message): def __init__(self, uuid: _Optional[bytes] = ..., collection: _Optional[str] = ..., tags: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., annotations: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ...) -> None: ... class CreateResponse(_message.Message): - __slots__ = ["stat"] + __slots__ = ("stat",) STAT_FIELD_NUMBER: _ClassVar[int] stat: Status def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ...) -> None: ... class MetadataUsageParams(_message.Message): - __slots__ = ["prefix", "role"] + __slots__ = ("prefix", "role") PREFIX_FIELD_NUMBER: _ClassVar[int] ROLE_FIELD_NUMBER: _ClassVar[int] prefix: str @@ -271,7 +287,7 @@ class MetadataUsageParams(_message.Message): def __init__(self, prefix: _Optional[str] = ..., role: _Optional[_Union[Role, _Mapping]] = ...) -> None: ... class MetadataUsageResponse(_message.Message): - __slots__ = ["stat", "tags", "annotations"] + __slots__ = ("stat", "tags", "annotations") STAT_FIELD_NUMBER: _ClassVar[int] TAGS_FIELD_NUMBER: _ClassVar[int] ANNOTATIONS_FIELD_NUMBER: _ClassVar[int] @@ -281,7 +297,7 @@ class MetadataUsageResponse(_message.Message): def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., tags: _Optional[_Iterable[_Union[KeyCount, _Mapping]]] = ..., annotations: _Optional[_Iterable[_Union[KeyCount, _Mapping]]] = ...) -> None: ... class KeyCount(_message.Message): - __slots__ = ["key", "count"] + __slots__ = ("key", "count") KEY_FIELD_NUMBER: _ClassVar[int] COUNT_FIELD_NUMBER: _ClassVar[int] key: str @@ -289,7 +305,7 @@ class KeyCount(_message.Message): def __init__(self, key: _Optional[str] = ..., count: _Optional[int] = ...) -> None: ... class ListCollectionsParams(_message.Message): - __slots__ = ["prefix", "role"] + __slots__ = ("prefix", "role") PREFIX_FIELD_NUMBER: _ClassVar[int] ROLE_FIELD_NUMBER: _ClassVar[int] prefix: str @@ -297,7 +313,7 @@ class ListCollectionsParams(_message.Message): def __init__(self, prefix: _Optional[str] = ..., role: _Optional[_Union[Role, _Mapping]] = ...) -> None: ... class ListCollectionsResponse(_message.Message): - __slots__ = ["stat", "collections"] + __slots__ = ("stat", "collections") STAT_FIELD_NUMBER: _ClassVar[int] COLLECTIONS_FIELD_NUMBER: _ClassVar[int] stat: Status @@ -305,7 +321,7 @@ class ListCollectionsResponse(_message.Message): def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., collections: _Optional[_Iterable[str]] = ...) -> None: ... class LookupStreamsParams(_message.Message): - __slots__ = ["collection", "isCollectionPrefix", "tags", "annotations", "role"] + __slots__ = ("collection", "isCollectionPrefix", "tags", "annotations", "role") COLLECTION_FIELD_NUMBER: _ClassVar[int] ISCOLLECTIONPREFIX_FIELD_NUMBER: _ClassVar[int] TAGS_FIELD_NUMBER: _ClassVar[int] @@ -319,7 +335,7 @@ class LookupStreamsParams(_message.Message): def __init__(self, collection: _Optional[str] = ..., isCollectionPrefix: bool = ..., tags: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., annotations: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., role: _Optional[_Union[Role, _Mapping]] = ...) -> None: ... class LookupStreamsResponse(_message.Message): - __slots__ = ["stat", "results"] + __slots__ = ("stat", "results") STAT_FIELD_NUMBER: _ClassVar[int] RESULTS_FIELD_NUMBER: _ClassVar[int] stat: Status @@ -327,7 +343,7 @@ class LookupStreamsResponse(_message.Message): def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., results: _Optional[_Iterable[_Union[StreamDescriptor, _Mapping]]] = ...) -> None: ... class NearestParams(_message.Message): - __slots__ = ["uuid", "time", "versionMajor", "backward"] + __slots__ = ("uuid", "time", "versionMajor", "backward") UUID_FIELD_NUMBER: _ClassVar[int] TIME_FIELD_NUMBER: _ClassVar[int] VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] @@ -339,7 +355,7 @@ class NearestParams(_message.Message): def __init__(self, uuid: _Optional[bytes] = ..., time: _Optional[int] = ..., versionMajor: _Optional[int] = ..., backward: bool = ...) -> None: ... class NearestResponse(_message.Message): - __slots__ = ["stat", "versionMajor", "versionMinor", "value"] + __slots__ = ("stat", "versionMajor", "versionMinor", "value") STAT_FIELD_NUMBER: _ClassVar[int] VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] @@ -351,7 +367,7 @@ class NearestResponse(_message.Message): def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., value: _Optional[_Union[RawPoint, _Mapping]] = ...) -> None: ... class ChangesParams(_message.Message): - __slots__ = ["uuid", "fromMajor", "toMajor", "resolution"] + __slots__ = ("uuid", "fromMajor", "toMajor", "resolution") UUID_FIELD_NUMBER: _ClassVar[int] FROMMAJOR_FIELD_NUMBER: _ClassVar[int] TOMAJOR_FIELD_NUMBER: _ClassVar[int] @@ -363,7 +379,7 @@ class ChangesParams(_message.Message): def __init__(self, uuid: _Optional[bytes] = ..., fromMajor: _Optional[int] = ..., toMajor: _Optional[int] = ..., resolution: _Optional[int] = ...) -> None: ... class ChangesResponse(_message.Message): - __slots__ = ["stat", "versionMajor", "versionMinor", "ranges"] + __slots__ = ("stat", "versionMajor", "versionMinor", "ranges") STAT_FIELD_NUMBER: _ClassVar[int] VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] @@ -375,13 +391,13 @@ class ChangesResponse(_message.Message): def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., ranges: _Optional[_Iterable[_Union[ChangedRange, _Mapping]]] = ...) -> None: ... class RoundSpec(_message.Message): - __slots__ = ["bits"] + __slots__ = ("bits",) BITS_FIELD_NUMBER: _ClassVar[int] bits: int def __init__(self, bits: _Optional[int] = ...) -> None: ... class InsertParams(_message.Message): - __slots__ = ["uuid", "sync", "merge_policy", "rounding", "values"] + __slots__ = ("uuid", "sync", "merge_policy", "rounding", "values") UUID_FIELD_NUMBER: _ClassVar[int] SYNC_FIELD_NUMBER: _ClassVar[int] MERGE_POLICY_FIELD_NUMBER: _ClassVar[int] @@ -395,7 +411,7 @@ class InsertParams(_message.Message): def __init__(self, uuid: _Optional[bytes] = ..., sync: bool = ..., merge_policy: _Optional[_Union[MergePolicy, str]] = ..., rounding: _Optional[_Union[RoundSpec, _Mapping]] = ..., values: _Optional[_Iterable[_Union[RawPoint, _Mapping]]] = ...) -> None: ... class ArrowInsertParams(_message.Message): - __slots__ = ["uuid", "sync", "merge_policy", "rounding", "arrowBytes"] + __slots__ = ("uuid", "sync", "merge_policy", "rounding", "arrowBytes") UUID_FIELD_NUMBER: _ClassVar[int] SYNC_FIELD_NUMBER: _ClassVar[int] MERGE_POLICY_FIELD_NUMBER: _ClassVar[int] @@ -409,7 +425,7 @@ class ArrowInsertParams(_message.Message): def __init__(self, uuid: _Optional[bytes] = ..., sync: bool = ..., merge_policy: _Optional[_Union[MergePolicy, str]] = ..., rounding: _Optional[_Union[RoundSpec, _Mapping]] = ..., arrowBytes: _Optional[bytes] = ...) -> None: ... class InsertResponse(_message.Message): - __slots__ = ["stat", "versionMajor", "versionMinor"] + __slots__ = ("stat", "versionMajor", "versionMinor") STAT_FIELD_NUMBER: _ClassVar[int] VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] @@ -419,7 +435,7 @@ class InsertResponse(_message.Message): def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ...) -> None: ... class DeleteParams(_message.Message): - __slots__ = ["uuid", "start", "end"] + __slots__ = ("uuid", "start", "end") UUID_FIELD_NUMBER: _ClassVar[int] START_FIELD_NUMBER: _ClassVar[int] END_FIELD_NUMBER: _ClassVar[int] @@ -429,7 +445,7 @@ class DeleteParams(_message.Message): def __init__(self, uuid: _Optional[bytes] = ..., start: _Optional[int] = ..., end: _Optional[int] = ...) -> None: ... class DeleteResponse(_message.Message): - __slots__ = ["stat", "versionMajor", "versionMinor"] + __slots__ = ("stat", "versionMajor", "versionMinor") STAT_FIELD_NUMBER: _ClassVar[int] VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] @@ -439,11 +455,11 @@ class DeleteResponse(_message.Message): def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ...) -> None: ... class InfoParams(_message.Message): - __slots__ = [] + __slots__ = () def __init__(self) -> None: ... class InfoResponse(_message.Message): - __slots__ = ["stat", "mash", "majorVersion", "minorVersion", "build", "proxy"] + __slots__ = ("stat", "mash", "majorVersion", "minorVersion", "build", "proxy") STAT_FIELD_NUMBER: _ClassVar[int] MASH_FIELD_NUMBER: _ClassVar[int] MAJORVERSION_FIELD_NUMBER: _ClassVar[int] @@ -459,13 +475,13 @@ class InfoResponse(_message.Message): def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., mash: _Optional[_Union[Mash, _Mapping]] = ..., majorVersion: _Optional[int] = ..., minorVersion: _Optional[int] = ..., build: _Optional[str] = ..., proxy: _Optional[_Union[ProxyInfo, _Mapping]] = ...) -> None: ... class ProxyInfo(_message.Message): - __slots__ = ["proxyEndpoints"] + __slots__ = ("proxyEndpoints",) PROXYENDPOINTS_FIELD_NUMBER: _ClassVar[int] proxyEndpoints: _containers.RepeatedScalarFieldContainer[str] def __init__(self, proxyEndpoints: _Optional[_Iterable[str]] = ...) -> None: ... class FaultInjectParams(_message.Message): - __slots__ = ["type", "params"] + __slots__ = ("type", "params") TYPE_FIELD_NUMBER: _ClassVar[int] PARAMS_FIELD_NUMBER: _ClassVar[int] type: int @@ -473,7 +489,7 @@ class FaultInjectParams(_message.Message): def __init__(self, type: _Optional[int] = ..., params: _Optional[bytes] = ...) -> None: ... class FaultInjectResponse(_message.Message): - __slots__ = ["stat", "rv"] + __slots__ = ("stat", "rv") STAT_FIELD_NUMBER: _ClassVar[int] RV_FIELD_NUMBER: _ClassVar[int] stat: Status @@ -481,13 +497,13 @@ class FaultInjectResponse(_message.Message): def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., rv: _Optional[bytes] = ...) -> None: ... class FlushParams(_message.Message): - __slots__ = ["uuid"] + __slots__ = ("uuid",) UUID_FIELD_NUMBER: _ClassVar[int] uuid: bytes def __init__(self, uuid: _Optional[bytes] = ...) -> None: ... class FlushResponse(_message.Message): - __slots__ = ["stat", "versionMajor", "versionMinor"] + __slots__ = ("stat", "versionMajor", "versionMinor") STAT_FIELD_NUMBER: _ClassVar[int] VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] @@ -497,19 +513,19 @@ class FlushResponse(_message.Message): def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ...) -> None: ... class ObliterateParams(_message.Message): - __slots__ = ["uuid"] + __slots__ = ("uuid",) UUID_FIELD_NUMBER: _ClassVar[int] uuid: bytes def __init__(self, uuid: _Optional[bytes] = ...) -> None: ... class ObliterateResponse(_message.Message): - __slots__ = ["stat"] + __slots__ = ("stat",) STAT_FIELD_NUMBER: _ClassVar[int] stat: Status def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ...) -> None: ... class RawPoint(_message.Message): - __slots__ = ["time", "value"] + __slots__ = ("time", "value") TIME_FIELD_NUMBER: _ClassVar[int] VALUE_FIELD_NUMBER: _ClassVar[int] time: int @@ -517,7 +533,7 @@ class RawPoint(_message.Message): def __init__(self, time: _Optional[int] = ..., value: _Optional[float] = ...) -> None: ... class StatPoint(_message.Message): - __slots__ = ["time", "min", "mean", "max", "count", "stddev"] + __slots__ = ("time", "min", "mean", "max", "count", "stddev") TIME_FIELD_NUMBER: _ClassVar[int] MIN_FIELD_NUMBER: _ClassVar[int] MEAN_FIELD_NUMBER: _ClassVar[int] @@ -533,7 +549,7 @@ class StatPoint(_message.Message): def __init__(self, time: _Optional[int] = ..., min: _Optional[float] = ..., mean: _Optional[float] = ..., max: _Optional[float] = ..., count: _Optional[int] = ..., stddev: _Optional[float] = ...) -> None: ... class ChangedRange(_message.Message): - __slots__ = ["start", "end"] + __slots__ = ("start", "end") START_FIELD_NUMBER: _ClassVar[int] END_FIELD_NUMBER: _ClassVar[int] start: int @@ -541,7 +557,7 @@ class ChangedRange(_message.Message): def __init__(self, start: _Optional[int] = ..., end: _Optional[int] = ...) -> None: ... class Status(_message.Message): - __slots__ = ["code", "msg", "mash"] + __slots__ = ("code", "msg", "mash") CODE_FIELD_NUMBER: _ClassVar[int] MSG_FIELD_NUMBER: _ClassVar[int] MASH_FIELD_NUMBER: _ClassVar[int] @@ -551,7 +567,7 @@ class Status(_message.Message): def __init__(self, code: _Optional[int] = ..., msg: _Optional[str] = ..., mash: _Optional[_Union[Mash, _Mapping]] = ...) -> None: ... class Mash(_message.Message): - __slots__ = ["revision", "leader", "leaderRevision", "totalWeight", "healthy", "unmapped", "members"] + __slots__ = ("revision", "leader", "leaderRevision", "totalWeight", "healthy", "unmapped", "members") REVISION_FIELD_NUMBER: _ClassVar[int] LEADER_FIELD_NUMBER: _ClassVar[int] LEADERREVISION_FIELD_NUMBER: _ClassVar[int] @@ -569,7 +585,7 @@ class Mash(_message.Message): def __init__(self, revision: _Optional[int] = ..., leader: _Optional[str] = ..., leaderRevision: _Optional[int] = ..., totalWeight: _Optional[int] = ..., healthy: bool = ..., unmapped: _Optional[float] = ..., members: _Optional[_Iterable[_Union[Member, _Mapping]]] = ...) -> None: ... class Member(_message.Message): - __slots__ = ["hash", "nodename", "up", "enabled", "start", "end", "weight", "readPreference", "httpEndpoints", "grpcEndpoints"] + __slots__ = ("hash", "nodename", "up", "enabled", "start", "end", "weight", "readPreference", "httpEndpoints", "grpcEndpoints") HASH_FIELD_NUMBER: _ClassVar[int] NODENAME_FIELD_NUMBER: _ClassVar[int] UP_FIELD_NUMBER: _ClassVar[int] @@ -594,7 +610,7 @@ class Member(_message.Message): def __init__(self, hash: _Optional[int] = ..., nodename: _Optional[str] = ..., up: bool = ..., enabled: bool = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., weight: _Optional[int] = ..., readPreference: _Optional[float] = ..., httpEndpoints: _Optional[str] = ..., grpcEndpoints: _Optional[str] = ..., **kwargs) -> None: ... class KeyOptValue(_message.Message): - __slots__ = ["key", "val"] + __slots__ = ("key", "val") KEY_FIELD_NUMBER: _ClassVar[int] VAL_FIELD_NUMBER: _ClassVar[int] key: str @@ -602,13 +618,13 @@ class KeyOptValue(_message.Message): def __init__(self, key: _Optional[str] = ..., val: _Optional[_Union[OptValue, _Mapping]] = ...) -> None: ... class OptValue(_message.Message): - __slots__ = ["value"] + __slots__ = ("value",) VALUE_FIELD_NUMBER: _ClassVar[int] value: str def __init__(self, value: _Optional[str] = ...) -> None: ... class KeyValue(_message.Message): - __slots__ = ["key", "value"] + __slots__ = ("key", "value") KEY_FIELD_NUMBER: _ClassVar[int] VALUE_FIELD_NUMBER: _ClassVar[int] key: str @@ -616,7 +632,7 @@ class KeyValue(_message.Message): def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ... class StreamCSVConfig(_message.Message): - __slots__ = ["version", "label", "uuid"] + __slots__ = ("version", "label", "uuid") VERSION_FIELD_NUMBER: _ClassVar[int] LABEL_FIELD_NUMBER: _ClassVar[int] UUID_FIELD_NUMBER: _ClassVar[int] @@ -626,9 +642,9 @@ class StreamCSVConfig(_message.Message): def __init__(self, version: _Optional[int] = ..., label: _Optional[str] = ..., uuid: _Optional[bytes] = ...) -> None: ... class GenerateCSVParams(_message.Message): - __slots__ = ["queryType", "startTime", "endTime", "windowSize", "depth", "includeVersions", "streams"] + __slots__ = ("queryType", "startTime", "endTime", "windowSize", "depth", "includeVersions", "streams") class QueryType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): - __slots__ = [] + __slots__ = () ALIGNED_WINDOWS_QUERY: _ClassVar[GenerateCSVParams.QueryType] WINDOWS_QUERY: _ClassVar[GenerateCSVParams.QueryType] RAW_QUERY: _ClassVar[GenerateCSVParams.QueryType] @@ -652,7 +668,7 @@ class GenerateCSVParams(_message.Message): def __init__(self, queryType: _Optional[_Union[GenerateCSVParams.QueryType, str]] = ..., startTime: _Optional[int] = ..., endTime: _Optional[int] = ..., windowSize: _Optional[int] = ..., depth: _Optional[int] = ..., includeVersions: bool = ..., streams: _Optional[_Iterable[_Union[StreamCSVConfig, _Mapping]]] = ...) -> None: ... class GenerateCSVResponse(_message.Message): - __slots__ = ["stat", "isHeader", "row"] + __slots__ = ("stat", "isHeader", "row") STAT_FIELD_NUMBER: _ClassVar[int] ISHEADER_FIELD_NUMBER: _ClassVar[int] ROW_FIELD_NUMBER: _ClassVar[int] @@ -662,7 +678,7 @@ class GenerateCSVResponse(_message.Message): def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., isHeader: bool = ..., row: _Optional[_Iterable[str]] = ...) -> None: ... class SQLQueryParams(_message.Message): - __slots__ = ["query", "params", "role"] + __slots__ = ("query", "params", "role") QUERY_FIELD_NUMBER: _ClassVar[int] PARAMS_FIELD_NUMBER: _ClassVar[int] ROLE_FIELD_NUMBER: _ClassVar[int] @@ -672,7 +688,7 @@ class SQLQueryParams(_message.Message): def __init__(self, query: _Optional[str] = ..., params: _Optional[_Iterable[str]] = ..., role: _Optional[_Union[Role, _Mapping]] = ...) -> None: ... class SQLQueryResponse(_message.Message): - __slots__ = ["stat", "SQLQueryRow"] + __slots__ = ("stat", "SQLQueryRow") STAT_FIELD_NUMBER: _ClassVar[int] SQLQUERYROW_FIELD_NUMBER: _ClassVar[int] stat: Status @@ -680,13 +696,13 @@ class SQLQueryResponse(_message.Message): def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., SQLQueryRow: _Optional[_Iterable[bytes]] = ...) -> None: ... class Role(_message.Message): - __slots__ = ["name"] + __slots__ = ("name",) NAME_FIELD_NUMBER: _ClassVar[int] name: str def __init__(self, name: _Optional[str] = ...) -> None: ... class SetCompactionConfigParams(_message.Message): - __slots__ = ["uuid", "CompactedVersion", "reducedResolutionRanges", "unused0"] + __slots__ = ("uuid", "CompactedVersion", "reducedResolutionRanges", "unused0") UUID_FIELD_NUMBER: _ClassVar[int] COMPACTEDVERSION_FIELD_NUMBER: _ClassVar[int] REDUCEDRESOLUTIONRANGES_FIELD_NUMBER: _ClassVar[int] @@ -698,19 +714,19 @@ class SetCompactionConfigParams(_message.Message): def __init__(self, uuid: _Optional[bytes] = ..., CompactedVersion: _Optional[int] = ..., reducedResolutionRanges: _Optional[_Iterable[_Union[ReducedResolutionRange, _Mapping]]] = ..., unused0: _Optional[int] = ...) -> None: ... class SetCompactionConfigResponse(_message.Message): - __slots__ = ["stat"] + __slots__ = ("stat",) STAT_FIELD_NUMBER: _ClassVar[int] stat: Status def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ...) -> None: ... class GetCompactionConfigParams(_message.Message): - __slots__ = ["uuid"] + __slots__ = ("uuid",) UUID_FIELD_NUMBER: _ClassVar[int] uuid: bytes def __init__(self, uuid: _Optional[bytes] = ...) -> None: ... class GetCompactionConfigResponse(_message.Message): - __slots__ = ["stat", "LatestMajorVersion", "CompactedVersion", "reducedResolutionRanges", "unused0"] + __slots__ = ("stat", "LatestMajorVersion", "CompactedVersion", "reducedResolutionRanges", "unused0") STAT_FIELD_NUMBER: _ClassVar[int] LATESTMAJORVERSION_FIELD_NUMBER: _ClassVar[int] COMPACTEDVERSION_FIELD_NUMBER: _ClassVar[int] @@ -724,7 +740,7 @@ class GetCompactionConfigResponse(_message.Message): def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., LatestMajorVersion: _Optional[int] = ..., CompactedVersion: _Optional[int] = ..., reducedResolutionRanges: _Optional[_Iterable[_Union[ReducedResolutionRange, _Mapping]]] = ..., unused0: _Optional[int] = ...) -> None: ... class ReducedResolutionRange(_message.Message): - __slots__ = ["Start", "End", "Resolution"] + __slots__ = ("Start", "End", "Resolution") START_FIELD_NUMBER: _ClassVar[int] END_FIELD_NUMBER: _ClassVar[int] RESOLUTION_FIELD_NUMBER: _ClassVar[int] @@ -734,7 +750,7 @@ class ReducedResolutionRange(_message.Message): def __init__(self, Start: _Optional[int] = ..., End: _Optional[int] = ..., Resolution: _Optional[int] = ...) -> None: ... class SubscriptionUpdate(_message.Message): - __slots__ = ["op", "uuid"] + __slots__ = ("op", "uuid") OP_FIELD_NUMBER: _ClassVar[int] UUID_FIELD_NUMBER: _ClassVar[int] op: SubscriptionUpdateOp @@ -742,7 +758,7 @@ class SubscriptionUpdate(_message.Message): def __init__(self, op: _Optional[_Union[SubscriptionUpdateOp, str]] = ..., uuid: _Optional[_Iterable[bytes]] = ...) -> None: ... class SubscriptionResp(_message.Message): - __slots__ = ["stat", "uuid", "arrowBytes"] + __slots__ = ("stat", "uuid", "arrowBytes") STAT_FIELD_NUMBER: _ClassVar[int] UUID_FIELD_NUMBER: _ClassVar[int] ARROWBYTES_FIELD_NUMBER: _ClassVar[int] diff --git a/btrdb/grpcinterface/btrdb_pb2_grpc.py b/btrdb/grpcinterface/btrdb_pb2_grpc.py index d05d9cf..d0db693 100644 --- a/btrdb/grpcinterface/btrdb_pb2_grpc.py +++ b/btrdb/grpcinterface/btrdb_pb2_grpc.py @@ -21,7 +21,7 @@ def __init__(self, channel): ) self.ArrowRawValues = channel.unary_stream( '/v5api.BTrDB/ArrowRawValues', - request_serializer=btrdb__pb2.RawValuesParams.SerializeToString, + request_serializer=btrdb__pb2.ArrowRawValuesParams.SerializeToString, response_deserializer=btrdb__pb2.ArrowRawValuesResponse.FromString, ) self.ArrowMultiValues = channel.unary_stream( @@ -317,7 +317,7 @@ def add_BTrDBServicer_to_server(servicer, server): ), 'ArrowRawValues': grpc.unary_stream_rpc_method_handler( servicer.ArrowRawValues, - request_deserializer=btrdb__pb2.RawValuesParams.FromString, + request_deserializer=btrdb__pb2.ArrowRawValuesParams.FromString, response_serializer=btrdb__pb2.ArrowRawValuesResponse.SerializeToString, ), 'ArrowMultiValues': grpc.unary_stream_rpc_method_handler( @@ -479,7 +479,7 @@ def ArrowRawValues(request, timeout=None, metadata=None): return grpc.experimental.unary_stream(request, target, '/v5api.BTrDB/ArrowRawValues', - btrdb__pb2.RawValuesParams.SerializeToString, + btrdb__pb2.ArrowRawValuesParams.SerializeToString, btrdb__pb2.ArrowRawValuesResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/btrdb/stream.py b/btrdb/stream.py index 2100c09..ccc50eb 100644 --- a/btrdb/stream.py +++ b/btrdb/stream.py @@ -916,6 +916,7 @@ def arrow_values( retries=5, retry_delay=3, retry_backoff=4, + schema=None, ) -> pa.Table: """Read raw values from BTrDB between time [a, b) in nanoseconds. @@ -931,6 +932,9 @@ def arrow_values( :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) version: int, default: 0 The version of the stream to be queried + schema: pyarrow.Schema + Optional arrow schema the server will cast the returned data to before sending it over + the network. You can use this to change the timestamp format, column names or data sizes. auto_retry: bool, default: False Whether to retry this request in the event of an error retries: int, default: 5 @@ -967,19 +971,20 @@ def arrow_values( start = to_nanoseconds(start) end = to_nanoseconds(end) arrow_and_versions = self._btrdb.ep.arrowRawValues( - uu=self.uuid, start=start, end=end, version=version + uu=self.uuid, start=start, end=end, version=version, schema=schema ) tables = list(arrow_and_versions) if len(tables) > 0: tabs, ver = zip(*tables) return pa.concat_tables(tabs) else: - schema = pa.schema( - [ - pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False), - pa.field("value", pa.float64(), nullable=False), - ] - ) + if schema is None: + schema = pa.schema( + [ + pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False), + pa.field("value", pa.float64(), nullable=False), + ] + ) return pa.Table.from_arrays([pa.array([]), pa.array([])], schema=schema) @retry @@ -1140,7 +1145,9 @@ def arrow_aligned_windows( ) if len(tables) > 0: tabs, ver = zip(*tables) - return pa.concat_tables(tabs).sort_by("time") + # assume that time column is the first column returned + time_col = tabs[0].column_names[0] + return pa.concat_tables(tabs).sort_by(time_col) else: schema = pa.schema( [ @@ -1304,7 +1311,8 @@ def arrow_windows( ) if len(tables) > 0: tabs, ver = zip(*tables) - return pa.concat_tables(tabs).sort_by("time") + time_col = tabs[0].column_names[0] + return pa.concat_tables(tabs).sort_by(time_col) else: schema = pa.schema( [ @@ -1684,6 +1692,7 @@ def filter( tags=None, annotations=None, sampling_frequency=None, + schema=None, ): """ Provides a new StreamSet instance containing stored query parameters and @@ -1721,6 +1730,9 @@ def filter( key/value pairs for filtering streams based on annotations sampling_frequency : float The sampling frequency of the data streams in Hz, set this if you want timesnapped values. + schema: pyarrow.Schema + Optional arrow schema the server will cast the returned data to before sending it over + the network. You can use this to change the timestamp format, column names or data sizes. Returns ------- @@ -1736,10 +1748,18 @@ def filter( """ obj = self.clone() - if start is not None or end is not None or sampling_frequency is not None: + if ( + start is not None + or end is not None + or sampling_frequency is not None + or schema is not None + ): obj.filters.append( StreamFilter( - start=start, end=end, sampling_frequency=sampling_frequency + start=start, + end=end, + sampling_frequency=sampling_frequency, + schema=schema, ) ) @@ -2102,6 +2122,8 @@ def _params_from_filters(self): params["end"] = filter.end if filter.sampling_frequency is not None: params["sampling_frequency"] = filter.sampling_frequency + if filter.schema is not None: + params["schema"] = filter.schema return params def values_iter(self): @@ -2142,6 +2164,10 @@ def arrow_values( _ = params.pop("sampling_frequency", None) if self.pointwidth is not None: + if params.pop("schema", None) is not None: + raise NotImplementedError( + "aligned windows queries do not yet support an arrow schema" + ) # create list of stream.aligned_windows data params.update({"pointwidth": self.pointwidth}) _ = params.pop("sampling_frequency", None) @@ -2177,6 +2203,10 @@ def arrow_values( elif self.width is not None and self.depth is not None: # create list of stream.windows data (the windows method should # prevent the possibility that only one of these is None) + if params.pop("schema", None) is not None: + raise NotImplementedError( + "windows queries do not yet support an arrow schema" + ) _ = params.pop("sampling_frequency", None) params.update({"width": self.width}) windows_gen = self._btrdb._executor.map( @@ -2215,17 +2245,20 @@ def arrow_values( if len(table) > 0: data = pa.concat_tables(table) else: - schema = pa.schema( - [pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False)] - + [ - pa.field(str(s.uuid), pa.float64(), nullable=False) - for s in self._streams - ], - ) + schema = params.pop("schema", None) + if schema is None: + schema = pa.schema( + [pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False)] + + [ + pa.field(str(s.uuid), pa.float64(), nullable=False) + for s in self._streams + ], + ) data = pa.Table.from_arrays( [pa.array([]) for i in range(1 + len(self._streams))], schema=schema ) - return data.sort_by("time") + time_col = data.column_names[0] + return data.sort_by(time_col) def __repr__(self): token = "stream" if len(self) == 1 else "streams" @@ -2281,10 +2314,15 @@ class StreamFilter(object): """ def __init__( - self, start: int = None, end: int = None, sampling_frequency: int = None + self, + start: int = None, + end: int = None, + sampling_frequency: int = None, + schema=None, ): self.start = to_nanoseconds(start) if start else None self.end = to_nanoseconds(end) if end else None + self.schema = schema self.sampling_frequency = sampling_frequency if sampling_frequency else None if self.start is None and self.end is None: diff --git a/btrdb/transformers.py b/btrdb/transformers.py index a57320c..662dee0 100644 --- a/btrdb/transformers.py +++ b/btrdb/transformers.py @@ -118,7 +118,7 @@ def to_series(streamset, datetime64_index=True, agg="mean", name_callable=None): values.append(getattr(point, agg)) if datetime64_index: - times = pd.Index(times, dtype="datetime64[ns]") + times = pd.Index(times, dtype="datetime64[ns]", name="time") result.append(pd.Series(data=values, index=times, name=stream_names[idx])) return result @@ -210,9 +210,11 @@ def arrow_to_dataframe(streamset, agg=None, name_callable=None) -> pd.DataFrame: col_names = _stream_names(streamset, name_callable) col_names_map = {str(s.uuid): c for s, c in zip(streamset, col_names)} updated_table_columns = [] + # assume time col is the first column + time_col = tmp_table.column_names[0] for old_col in tmp_table.column_names: - if old_col == "time": - updated_table_columns.append("time") + if old_col == time_col: + updated_table_columns.append(time_col) else: for uu, new_name in col_names_map.items(): if uu in old_col: @@ -226,12 +228,14 @@ def arrow_to_dataframe(streamset, agg=None, name_callable=None) -> pd.DataFrame: for agg_name in agg: if agg_name in column_str: usable_cols.append(column_str) - tmp = tmp_table.select(["time", *usable_cols]) + tmp = tmp_table.select([time_col, *usable_cols]) else: tmp = tmp_table - return tmp.to_pandas(date_as_object=False, types_mapper=pd.ArrowDtype).set_index( - "time" + tmp_df = tmp.to_pandas(date_as_object=False, types_mapper=pd.ArrowDtype).set_index( + time_col ) + tmp_df.index.name = time_col + return tmp_df def to_dataframe(streamset, agg="mean", name_callable=None): @@ -273,6 +277,7 @@ def to_dataframe(streamset, agg="mean", name_callable=None): if not df.empty: df = df.set_index("time") + df.index.name = "time" if agg == "all" and not streamset.allow_window: stream_names = [ @@ -386,7 +391,7 @@ def to_polars(streamset, agg="mean", name_callable=None): else: df = df.set_index("time") - df.index = pd.DatetimeIndex(df.index, tz="UTC") + df.index = pd.DatetimeIndex(df.index, tz="UTC", name="time") if agg == "all" and streamset.allow_window: stream_names = [ [s.collection, s.name, prop] @@ -397,7 +402,7 @@ def to_polars(streamset, agg="mean", name_callable=None): else: df.columns = _stream_names(streamset, name_callable) - return pl.from_pandas(df.reset_index(), nan_to_null=False) + return pl.from_pandas(df, nan_to_null=False, include_index=True) def to_array(streamset, agg="mean"): diff --git a/tests/btrdb_integration/test_stream.py b/tests/btrdb_integration/test_stream.py index e65c71d..5fba2a1 100644 --- a/tests/btrdb_integration/test_stream.py +++ b/tests/btrdb_integration/test_stream.py @@ -61,6 +61,24 @@ def test_arrow_insert_and_values( assert data == fetched_data +def test_arrow_values_template_schema(conn, tmp_collection): + s = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) + t = currently_as_ns() + times = [100, 200, 300, 400] + values = [1.0, 2.0, 3.0, 4.0] + s.insert(list(zip(times, values))) + schema = pa.schema( + [ + pa.field("t", pa.int64(), nullable=False), + pa.field("v", pa.float32(), nullable=False), + ] + ) + expected = pa.Table.from_arrays([pa.array(times), pa.array(values)], schema=schema) + fetched_data = s.arrow_values(start=times[0], end=times[-1] + 1, schema=schema) + assert expected == fetched_data + assert expected.schema.equals(fetched_data.schema) + + @pytest.mark.parametrize( "merge_policy,duplicates_expected", [("never", True), ("equal", True), ("retain", False), ("replace", False)], @@ -103,7 +121,7 @@ def test_arrow_values_table_schema( assert single_stream_values_arrow_schema.equals(fetched_data.schema) -def test_arrow_values_table_schema( +def test_arrow_values_table_schema_2( conn, tmp_collection, single_stream_values_arrow_schema ): s = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) @@ -330,19 +348,21 @@ def test_arrow_empty_values_schema(conn, tmp_collection): assert schema.equals(data.schema) -@pytest.mark.xfail def test_stream_annotation_update(conn, tmp_collection): - # XXX marked as expected failure until someone has time to investigate. s = conn.create( - new_uuid(), tmp_collection, tags={"name": "s"}, annotations={"foo": "bar"} + new_uuid(), + tmp_collection + "foo/", + tags={"name": "s"}, + annotations={"foo": "bar"}, ) annotations1, version1 = s.annotations() - assert version1 == 0 + assert version1 == 1 assert annotations1["foo"] == "bar" s.update(annotations={"foo": "baz"}) annotations2, version2 = s.annotations() assert version2 > version1 assert annotations2["foo"] == "baz" s.update(annotations={}, replace=True) - annotations3, _ = s.annotations() + annotations3, version3 = s.annotations() assert len(annotations3) == 0 + assert version3 > version2 diff --git a/tests/btrdb_integration/test_streamset.py b/tests/btrdb_integration/test_streamset.py index d173818..f74e7d3 100644 --- a/tests/btrdb_integration/test_streamset.py +++ b/tests/btrdb_integration/test_streamset.py @@ -73,6 +73,36 @@ def test_streamset_arrow_values(conn, tmp_collection): assert expected_schema.equals(values.schema) +def test_streamset_template_schema(conn, tmp_collection): + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + t1 = [100, 105, 110, 115, 120] + t2 = [101, 106, 110, 114, 119] + d1 = [0.0, 1.0, 2.0, 3.0, 4.0] + d2 = [5.0, 6.0, 7.0, 8.0, 9.0] + s1.insert(list(zip(t1, d1))) + s2.insert(list(zip(t2, d2))) + schema = pa.schema( + [ + pa.field("t", pa.timestamp("ns", tz="UTC"), nullable=False), + pa.field("a", pa.float64(), nullable=False), + pa.field("b", pa.float32(), nullable=False), + ] + ) + ss = btrdb.stream.StreamSet([s1, s2]).filter(start=100, end=121, schema=schema) + expected_times = [100, 101, 105, 106, 110, 114, 115, 119, 120] + expected_col1 = [0.0, np.NaN, 1.0, np.NaN, 2.0, np.NaN, 3.0, np.NaN, 4.0] + expected_col2 = [np.NaN, 5.0, np.NaN, 6.0, 7.0, 8.0, np.NaN, 9.0, np.NaN] + values = ss.arrow_values() + times = [t.value for t in values["t"]] + col1 = [np.NaN if isnan(v.as_py()) else v.as_py() for v in values["a"]] + col2 = [np.NaN if isnan(v.as_py()) else v.as_py() for v in values["b"]] + assert times == expected_times + assert col1 == expected_col1 + assert col2 == expected_col2 + assert schema.equals(values.schema) + + @pytest.mark.parametrize( "name_callable", [(None), (lambda s: str(s.uuid)), (lambda s: s.name + "/" + s.collection)], @@ -96,7 +126,6 @@ def test_streamset_arrow_windows_vs_windows(conn, tmp_collection, name_callable) .windows(width=btrdb.utils.timez.ns_delta(nanoseconds=10)) ) values_arrow = ss.arrow_to_dataframe(name_callable=name_callable) - values_arrow.set_index("time", inplace=True) values_arrow.index = pd.DatetimeIndex(values_arrow.index) values_prev = ss.to_dataframe(name_callable=name_callable).convert_dtypes( dtype_backend="pyarrow" @@ -128,7 +157,6 @@ def test_streamset_arrow_windows_vs_windows_agg_all(conn, tmp_collection): .windows(width=btrdb.utils.timez.ns_delta(nanoseconds=10)) ) values_arrow = ss.arrow_to_dataframe(name_callable=None, agg=["all"]) - values_arrow.set_index("time", inplace=True) values_arrow.index = pd.DatetimeIndex(values_arrow.index) values_prev = ss.to_dataframe(name_callable=None, agg="all") values_prev = values_prev.apply(lambda x: x.astype(str(x.dtype) + "[pyarrow]")) @@ -184,7 +212,6 @@ def test_streamset_arrow_aligned_windows_vs_aligned_windows( .windows(width=btrdb.utils.general.pointwidth.from_nanoseconds(10)) ) values_arrow = ss.arrow_to_dataframe(name_callable=name_callable) - values_arrow.set_index("time", inplace=True) values_arrow.index = pd.DatetimeIndex(values_arrow.index) values_prev = ss.to_dataframe( name_callable=name_callable @@ -247,7 +274,6 @@ def test_arrow_streamset_to_dataframe(conn, tmp_collection): s2.insert(list(zip(t2, d2))) ss = btrdb.stream.StreamSet([s1, s2]).filter(start=100, end=121) values = ss.arrow_to_dataframe() - values.set_index("time", inplace=True) expected_times = [100, 101, 105, 106, 110, 114, 115, 119, 120] expected_times = [ pa.scalar(v, type=pa.timestamp("ns", tz="UTC")).as_py() for v in expected_times @@ -270,11 +296,11 @@ def test_arrow_streamset_to_dataframe(conn, tmp_collection): pa.field(tmp_collection + "/s2", type=pa.float64(), nullable=False), ] ) - expected_table = pa.Table.from_pydict(expected_dat, schema=schema) + expected_table = pa.Table.from_pydict(mapping=expected_dat, schema=schema) expected_df = expected_table.to_pandas( - timestamp_as_object=False, types_mapper=pd.ArrowDtype - ) - expected_df.set_index("time", inplace=True) + timestamp_as_object=False, + types_mapper=pd.ArrowDtype, + ).set_index("time") expected_df.index = pd.DatetimeIndex(expected_df.index, tz="UTC") np_test.assert_array_equal( values.values.astype(float), expected_df.values.astype(float) @@ -303,9 +329,9 @@ def test_arrow_streamset_to_polars(conn, tmp_collection): tmp_collection + "/s2": expected_col2, } expected_df = pd.DataFrame( - expected_dat, index=pd.DatetimeIndex(expected_times) - ).reset_index(names="time") - expected_df_pl = pl.from_pandas(expected_df, nan_to_null=False) + expected_dat, index=pd.DatetimeIndex(expected_times, name="time") + ) + expected_df_pl = pl.from_pandas(expected_df, nan_to_null=False, include_index=True) pl_test.assert_frame_equal(values, expected_df_pl) @@ -337,9 +363,9 @@ def test_streamset_arrow_polars_vs_old_to_polars(conn, tmp_collection, name_call tmp_collection + "/s2": expected_col2, } expected_df = pd.DataFrame( - expected_dat, index=pd.DatetimeIndex(expected_times, tz="UTC") - ).reset_index(names="time") - expected_df_pl = pl.from_pandas(expected_df, nan_to_null=False) + expected_dat, index=pd.DatetimeIndex(expected_times, tz="UTC", name="time") + ) + expected_df_pl = pl.from_pandas(expected_df, nan_to_null=False, include_index=True) pl_test.assert_frame_equal(values_arrow, expected_df_pl) pl_test.assert_frame_equal(values_non_arrow, expected_df_pl) pl_test.assert_frame_equal(values_non_arrow, values_arrow) @@ -377,7 +403,7 @@ def test_streamset_windows_arrow_polars_vs_old_to_polars( } new_names["time"] = "time" values_non_arrow_pl = values_non_arrow_pl.rename(mapping=new_names) - assert values_arrow_pl.frame_equal(values_non_arrow_pl) + assert values_arrow_pl.equals(values_non_arrow_pl) def test_streamset_windows_aggregates_filter(conn, tmp_collection): @@ -398,7 +424,6 @@ def test_streamset_windows_aggregates_filter(conn, tmp_collection): .windows(width=btrdb.utils.timez.ns_delta(nanoseconds=10)) ) values_arrow_df = ss.arrow_to_dataframe(agg=["mean", "stddev"]) - values_arrow_df.set_index("time", inplace=True) values_arrow_df.index = pd.DatetimeIndex(values_arrow_df.index) values_non_arrow_df = ss.to_dataframe(agg="all") values_non_arrow_df.index = pd.DatetimeIndex(values_non_arrow_df.index, tz="UTC") @@ -487,7 +512,6 @@ def test_timesnap_with_different_sampling_frequencies(freq, conn, tmp_collection df = stset.filter( start=start, end=stop, sampling_frequency=freq ).arrow_to_dataframe() - df.set_index("time", inplace=True) total_points = df.shape[0] * df.shape[1] total_raw_pts = len(v1) * len(stset) expected_frac_of_pts = 1 if freq is None else freq / data_insert_freq From d88292fab3c5bd3604f3d0894442c8f9c9fb9dda Mon Sep 17 00:00:00 2001 From: Justin Gilmer Date: Tue, 16 Jan 2024 14:11:35 -0500 Subject: [PATCH 093/131] Update exception handling to support RpcErrors. (#72) --- btrdb/exceptions.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/btrdb/exceptions.py b/btrdb/exceptions.py index 9777679..2ce0c13 100644 --- a/btrdb/exceptions.py +++ b/btrdb/exceptions.py @@ -117,18 +117,21 @@ def handle_grpc_error(err): Parameters ---------- - err: grpc.RpcError + err: Union[grpc.RpcError, btrdb.BTrDBError] """ details = err.details() - if details == "[404] stream does not exist": - raise StreamNotFoundError("Stream not found with provided uuid") from None - elif details == "failed to connect to all addresses": - raise ConnectionError("Failed to connect to BTrDB") from None - elif any(str(e) in err.details() for e in BTRDB_SERVER_ERRORS): - raise BTRDBServerError("An error has occured with btrdb-server") from None - elif str(err.code()) == "StatusCode.PERMISSION_DENIED": - raise PermissionDenied(details) from None - raise BTrDBError(details) from None + if isinstance(err, BTrDBError): + if details == "[404] stream does not exist": + raise StreamNotFoundError("Stream not found with provided uuid") from None + elif details == "failed to connect to all addresses": + raise ConnectionError("Failed to connect to BTrDB") from None + elif any(str(e) in err.details() for e in BTRDB_SERVER_ERRORS): + raise BTRDBServerError("An error has occured with btrdb-server") from None + elif str(err.code()) == "StatusCode.PERMISSION_DENIED": + raise PermissionDenied(details) from None + raise BTrDBError(details) from None + else: + raise err def check_proto_stat(stat): From 7e69f1db35d2ed25750fa40c88a2edd0fa102617 Mon Sep 17 00:00:00 2001 From: Justin Gilmer <16173348+justinGilmer@users.noreply.github.com> Date: Wed, 7 Feb 2024 15:24:47 -0500 Subject: [PATCH 094/131] Include 3.11 for testing. (#74) --- .github/workflows/release.yaml | 71 ++++++++++++++++------------------ 1 file changed, 34 insertions(+), 37 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 33bec25..31929c9 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -6,35 +6,33 @@ on: - synchronize push: tags: - - 'v[0-9]+.[0-9]+.[0-9]+*' + - "v[0-9]+.[0-9]+.[0-9]+*" jobs: - test-suite: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: [3.7, 3.8, 3.9, '3.10'] + python-version: [3.7, 3.8, 3.9, "3.10", "3.11"] os: [ubuntu-latest, macos-latest, windows-latest] steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} ${{ matrix.os }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install -r tests/requirements.txt - - name: Test with pytest - run: | - pytest --cov-report= --cov=btrdb # suppress coverage report here - - name: Coverage Report - run: | - coverage report -m - + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} ${{ matrix.os }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r tests/requirements.txt + - name: Test with pytest + run: | + pytest --cov-report= --cov=btrdb # suppress coverage report here + - name: Coverage Report + run: | + coverage report -m release: needs: @@ -54,7 +52,6 @@ jobs: draft: false prerelease: false - deploy: needs: - release @@ -62,20 +59,20 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.8' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel build - - name: Build and publish - run: | + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.8" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel build + - name: Build and publish + run: | python -m build --sdist --wheel --outdir dist/ . - - name: Publish package - uses: pypa/gh-action-pypi-publish@release/v1 - with: - user: __token__ - password: ${{ secrets.PYPI_DEPLOYMENT_TOKEN }} + - name: Publish package + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.PYPI_DEPLOYMENT_TOKEN }} From bdd7b688292c1f2feae8f4b98c0984588fc73621 Mon Sep 17 00:00:00 2001 From: Jeff Lin <42981468+jleifnf@users.noreply.github.com> Date: Wed, 7 Feb 2024 14:39:33 -0800 Subject: [PATCH 095/131] Improve arrow_to_dataframe for large amounts of columns (#73) Co-authored-by: Justin Gilmer <16173348+justinGilmer@users.noreply.github.com> --- btrdb/transformers.py | 52 +++++++++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/btrdb/transformers.py b/btrdb/transformers.py index 662dee0..0ff01fb 100644 --- a/btrdb/transformers.py +++ b/btrdb/transformers.py @@ -181,6 +181,19 @@ def arrow_to_dataframe(streamset, agg=None, name_callable=None) -> pd.DataFrame: ----- This method is available for commercial customers with arrow-enabled servers. """ + + def _rename(col_name, col_names_map): + if col_name == "time": + return col_name + col_name_parts = col_name.split("/") + if len(col_name_parts) == 1: + uuid = col_name_parts[0] + new_col_name = col_names_map[uuid] + elif len(col_name_parts) == 2: + uuid, _ = col_name_parts + new_col_name = col_name.replace(uuid, col_names_map[uuid]) + return new_col_name + if not streamset._btrdb._ARROW_ENABLED: raise NotImplementedError( "arrow_to_dataframe requires an arrow-enabled BTrDB server." @@ -206,35 +219,30 @@ def arrow_to_dataframe(streamset, agg=None, name_callable=None) -> pd.DataFrame: if not callable(name_callable): name_callable = lambda s: s.collection + "/" + s.name # format is: uuid/stat_type - tmp_table = streamset.arrow_values() + tmp = streamset.arrow_values() + # assume time col is the first column + time_col = tmp.column_names[0] + + tmp_df = tmp.to_pandas( + date_as_object=False, + types_mapper=pd.ArrowDtype, + split_blocks=True, + self_destruct=True, + ).set_index(time_col) + tmp_df.index.name = time_col col_names = _stream_names(streamset, name_callable) col_names_map = {str(s.uuid): c for s, c in zip(streamset, col_names)} - updated_table_columns = [] - # assume time col is the first column - time_col = tmp_table.column_names[0] - for old_col in tmp_table.column_names: - if old_col == time_col: - updated_table_columns.append(time_col) - else: - for uu, new_name in col_names_map.items(): - if uu in old_col: - updated_table_columns.append(old_col.replace(uu, new_name)) - else: - continue - tmp_table = tmp_table.rename_columns(updated_table_columns) + tmp_df.rename( + columns=lambda col_name: _rename(col_name, col_names_map), inplace=True + ) if not streamset.allow_window: usable_cols = [] - for column_str in tmp_table.column_names: + for column_str in tmp_df.columns: for agg_name in agg: if agg_name in column_str: usable_cols.append(column_str) - tmp = tmp_table.select([time_col, *usable_cols]) - else: - tmp = tmp_table - tmp_df = tmp.to_pandas(date_as_object=False, types_mapper=pd.ArrowDtype).set_index( - time_col - ) - tmp_df.index.name = time_col + tmp_df = tmp_df.loc[:, usable_cols] + return tmp_df From 22423dc6d583e7d779dd268831c86451f00f04fd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Feb 2024 12:39:48 -0500 Subject: [PATCH 096/131] Bump tj-actions/changed-files from 21 to 41 in /.github/workflows (#75) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/pre-commit.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml index 6567714..7ca805a 100644 --- a/.github/workflows/pre-commit.yaml +++ b/.github/workflows/pre-commit.yaml @@ -30,7 +30,7 @@ jobs: pip install pre-commit - name: Get changed files id: changed-files - uses: tj-actions/changed-files@v21 + uses: tj-actions/changed-files@v41 with: token: ${{ secrets.GITHUB_TOKEN }} - name: Run pre-commit From 4efbbb629ce6f3a0170b1bfb56ee16e84bce890b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 23 Feb 2024 12:31:22 -0500 Subject: [PATCH 097/131] Bump urllib3 from 2.0.2 to 2.0.7 (#76) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements-all-locked.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-all-locked.txt b/requirements-all-locked.txt index d6f7284..6ab93bc 100644 --- a/requirements-all-locked.txt +++ b/requirements-all-locked.txt @@ -119,7 +119,7 @@ tomli==2.0.1 # pytest tzdata==2023.3 # via pandas -urllib3==2.0.2 +urllib3==2.0.7 # via requests virtualenv==20.23.0 # via ray From b590961096b567977b49df3dd73b100a6c736db8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 23 Feb 2024 12:31:47 -0500 Subject: [PATCH 098/131] Bump certifi from 2023.5.7 to 2023.7.22 (#77) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements-all-locked.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-all-locked.txt b/requirements-all-locked.txt index 6ab93bc..e5bfd27 100644 --- a/requirements-all-locked.txt +++ b/requirements-all-locked.txt @@ -10,7 +10,7 @@ attrs==23.1.0 # via # jsonschema # ray -certifi==2023.5.7 +certifi==2023.7.22 # via # btrdb (pyproject.toml) # requests From 66e7e542979711f33da19602fb741d27b807e156 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 23 Feb 2024 12:32:11 -0500 Subject: [PATCH 099/131] Bump pyarrow from 12.0.0 to 14.0.1 (#78) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements-all-locked.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-all-locked.txt b/requirements-all-locked.txt index e5bfd27..05ba631 100644 --- a/requirements-all-locked.txt +++ b/requirements-all-locked.txt @@ -76,7 +76,7 @@ protobuf==4.23.2 # via # grpcio-tools # ray -pyarrow==12.0.0 +pyarrow==14.0.1 # via btrdb pycodestyle==2.10.0 # via flake8 From bf9ce48931734e2b8658b90de60548efdf707c10 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 23 Feb 2024 12:32:29 -0500 Subject: [PATCH 100/131] Bump grpcio from 1.54.2 to 1.54.3 (#79) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements-all-locked.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-all-locked.txt b/requirements-all-locked.txt index 05ba631..5cf5c57 100644 --- a/requirements-all-locked.txt +++ b/requirements-all-locked.txt @@ -36,7 +36,7 @@ frozenlist==1.3.3 # via # aiosignal # ray -grpcio==1.54.2 +grpcio==1.54.3 # via # btrdb (pyproject.toml) # grpcio-tools From 405fb6f0aabe30ca3337478ca2a1811b4084fc59 Mon Sep 17 00:00:00 2001 From: Justin Gilmer <16173348+justinGilmer@users.noreply.github.com> Date: Fri, 23 Feb 2024 15:12:49 -0500 Subject: [PATCH 101/131] Test new join (#80) --- btrdb/stream.py | 120 ++++++++++++++++------ tests/btrdb_integration/test_streamset.py | 41 +++++++- 2 files changed, 127 insertions(+), 34 deletions(-) diff --git a/btrdb/stream.py b/btrdb/stream.py index ccc50eb..be0f62e 100644 --- a/btrdb/stream.py +++ b/btrdb/stream.py @@ -17,12 +17,16 @@ import json import logging import re +import uuid import uuid as uuidlib import warnings from collections import deque from collections.abc import Sequence from copy import deepcopy -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING, Dict, List + +import pyarrow +import pyarrow.compute as pc try: import pyarrow as pa @@ -2183,22 +2187,10 @@ def arrow_values( ) stream_uus = [str(s.uuid) for s in self._streams] data = list(aligned_windows_gen) - tablex = data.pop(0) - uu = stream_uus.pop(0) - tab_columns = [ - c if c == "time" else uu + "/" + c for c in tablex.column_names - ] - tablex = tablex.rename_columns(tab_columns) - if data: - for tab, uu in zip(data, stream_uus): - tab_columns = [ - c if c == "time" else uu + "/" + c for c in tab.column_names - ] - tab = tab.rename_columns(tab_columns) - tablex = tablex.join(tab, "time", join_type="full outer") - data = tablex - else: - data = tablex + table_joined = _merge_pyarrow_tables( + {uu: tab for uu, tab in zip(stream_uus, data)} + ) + data = table_joined elif self.width is not None and self.depth is not None: # create list of stream.windows data (the windows method should @@ -2217,22 +2209,10 @@ def arrow_values( ) stream_uus = [str(s.uuid) for s in self._streams] data = list(windows_gen) - tablex = data.pop(0) - uu = stream_uus.pop(0) - tab_columns = [ - c if c == "time" else uu + "/" + c for c in tablex.column_names - ] - tablex = tablex.rename_columns(tab_columns) - if data: - for tab, uu in zip(data, stream_uus): - tab_columns = [ - c if c == "time" else uu + "/" + c for c in tab.column_names - ] - tab = tab.rename_columns(tab_columns) - tablex = tablex.join(tab, "time", join_type="full outer") - data = tablex - else: - data = tablex + table_joined = _merge_pyarrow_tables( + {uu: tab for uu, tab in zip(stream_uus, data)} + ) + data = table_joined else: sampling_freq = params.pop("sampling_frequency", 0) period_ns = 0 @@ -2349,3 +2329,77 @@ def _coalesce_table_deque(tables: deque): t2, "time", join_type="full outer", right_suffix=f"_{idx}" ) return main_table + + +def _extract_unique_times(stream_map: Dict[uuid.UUID, pa.Table]) -> pa.Array: + """Extracts and returns unique 'time' values from all tables.""" + time_arrays = [ + table.column("time").combine_chunks() + for table in stream_map.values() + if table.num_rows > 0 + ] + if not time_arrays: # Check if the list is empty + return pa.array([], type=pa.timestamp("ns")) + + all_times = pa.concat_arrays(time_arrays) + return pc.unique(all_times).sort() + + +def _build_combined_schema( + stream_map: Dict[uuid.UUID, pa.Table], unique_times: pa.Array +) -> pa.Schema: + """Constructs a combined schema for the merged table, ensuring unique column names.""" + combined_schema = [("time", unique_times.type)] + # Use a list comprehension to flatten the loop into a single iterable over all columns + combined_schema += [ + pa.field( + f"{uu}/{col_name}", + table.column(col_name).type if table.num_rows > 0 else pa.null(), + ) + for uu, table in stream_map.items() + for col_name in table.column_names + if col_name != "time" + ] + + return pa.schema(combined_schema) + + +def _merge_pyarrow_tables(stream_map: Dict[uuid.UUID, pa.Table]) -> pa.Table: + """Merges PyArrow tables based on 'time' values into a single table.""" + unique_times = _extract_unique_times(stream_map) + combined_schema = _build_combined_schema(stream_map, unique_times) + + none_data = [None] * len(unique_times) + preallocated_data = { + field.name: pa.array(none_data, type=field.type) for field in combined_schema + } + preallocated_data["time"] = unique_times + + for uu, table in stream_map.items(): + if table.num_rows > 0: + time_indices = pc.index_in( + preallocated_data["time"], + value_set=table.column("time"), + skip_nulls=True, + ) + for col_name in table.column_names: + if col_name == "time": + continue + combined_col_name = f"{str(uu)}/{col_name}" + preallocated_data[combined_col_name] = pc.take( + table.column(col_name), indices=time_indices + ) + else: + # For empty tables, ensure their columns are represented with all nulls + for col_name in combined_schema.names: + if ( + col_name.startswith(f"{str(uu)}/") + and col_name not in preallocated_data + ): + field_type = combined_schema.field(col_name).type + preallocated_data[col_name] = pa.array( + [None] * preallocated_data["time"].length(), type=field_type + ) + + arrays = [preallocated_data[col] for col in combined_schema.names] + return pa.Table.from_arrays(arrays=arrays, schema=combined_schema) diff --git a/tests/btrdb_integration/test_streamset.py b/tests/btrdb_integration/test_streamset.py index f74e7d3..5a0fbec 100644 --- a/tests/btrdb_integration/test_streamset.py +++ b/tests/btrdb_integration/test_streamset.py @@ -209,7 +209,46 @@ def test_streamset_arrow_aligned_windows_vs_aligned_windows( ss = ( btrdb.stream.StreamSet([s1, s2, s3]) .filter(start=100, end=121) - .windows(width=btrdb.utils.general.pointwidth.from_nanoseconds(10)) + .aligned_windows(pointwidth=btrdb.utils.general.pointwidth.from_nanoseconds(10)) + ) + values_arrow = ss.arrow_to_dataframe(name_callable=name_callable) + values_arrow.index = pd.DatetimeIndex(values_arrow.index) + values_prev = ss.to_dataframe( + name_callable=name_callable + ) # .convert_dtypes(dtype_backend='pyarrow') + values_prev = values_prev.apply(lambda x: x.astype(str(x.dtype) + "[pyarrow]")) + values_prev = values_prev.apply( + lambda x: x.astype("uint64[pyarrow]") if "count" in x.name else x + ) + values_prev.index = pd.DatetimeIndex(values_prev.index, tz="UTC") + col_map = {old_col: old_col + "/mean" for old_col in values_prev.columns} + values_prev = values_prev.rename(columns=col_map) + assert values_arrow.equals(values_prev) + + +@pytest.mark.parametrize( + "name_callable", + [(None), (lambda s: str(s.uuid)), (lambda s: s.name + "/" + s.collection)], + ids=["empty", "uu_as_str", "name_collection"], +) +def test_streamset_arrow_aligned_windows_join_logic( + conn, tmp_collection, name_callable +): + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + s3 = conn.create(new_uuid(), tmp_collection, tags={"name": "s3"}) + t1 = [100, 105, 110, 115, 120] + t2 = [101, 106, 110, 132, 140] + d1 = [0.0, 1.0, 2.0, 3.0, 4.0] + d2 = [5.0, 6.0, 7.0, 8.0, 9.0] + d3 = [1.0, 9.0, 44.0, 8.0, 9.0] + s1.insert(list(zip(t1, d1))) + s2.insert(list(zip(t2, d2))) + s3.insert(list(zip(t2, d3))) + ss = ( + btrdb.stream.StreamSet([s1, s2, s3]) + .filter(start=100, end=141) + .aligned_windows(pointwidth=btrdb.utils.general.pointwidth.from_nanoseconds(8)) ) values_arrow = ss.arrow_to_dataframe(name_callable=name_callable) values_arrow.index = pd.DatetimeIndex(values_arrow.index) From 78dbfe50b05c3318cccea344f9fcc053afee59ef Mon Sep 17 00:00:00 2001 From: Justin Gilmer <16173348+justinGilmer@users.noreply.github.com> Date: Mon, 25 Mar 2024 19:22:51 -0400 Subject: [PATCH 102/131] Initial docstrings and a new test. (#82) --- btrdb/__init__.py | 29 +- btrdb/conn.py | 311 ++++++-- btrdb/endpoint.py | 2 +- btrdb/exceptions.py | 2 +- btrdb/stream.py | 670 ++++++++++++++---- btrdb/transformers.py | 226 +++++- btrdb/utils/timez.py | 12 +- docs/requirements.txt | 3 + docs/source/api/streams.rst | 5 +- docs/source/concepts.rst | 149 ++-- docs/source/conf.py | 35 +- docs/source/explained.rst | 2 +- docs/source/index.rst | 12 +- docs/source/installing.rst | 8 +- docs/source/quick-start.rst | 202 ++++-- docs/source/working/arrow-enabled-queries.rst | 4 +- .../source/working/stream-manage-metadata.rst | 2 +- docs/source/working/stream-query-manage.rst | 10 +- docs/source/working/stream-view-data.rst | 64 +- docs/source/working/streamsets.rst | 12 +- tests/btrdb/test_base.py | 39 +- tests/btrdb/test_conn.py | 124 ++-- tests/btrdb_integration/test_conn.py | 79 ++- 23 files changed, 1473 insertions(+), 529 deletions(-) diff --git a/btrdb/__init__.py b/btrdb/__init__.py index efcb686..1584b3b 100644 --- a/btrdb/__init__.py +++ b/btrdb/__init__.py @@ -52,18 +52,18 @@ def connect(conn_str=None, apikey=None, profile=None, shareable=False): Parameters ---------- conn_str: str, default=None - The address and port of the cluster to connect to, e.g. `192.168.1.1:4411`. - If set to None, will look in the environment variable `$BTRDB_ENDPOINTS` + The address and port of the cluster to connect to, e.g. ``192.168.1.1:4411``. + If set to None, will look in the environment variable ``$BTRDB_ENDPOINTS`` (recommended). apikey: str, default=None The API key used to authenticate requests (optional). If None, the key - is looked up from the environment variable `$BTRDB_API_KEY`. + is looked up from the environment variable ``$BTRDB_API_KEY``. profile: str, default=None The name of a profile containing the required connection information as found in the user's predictive grid credentials file - `~/.predictivegrid/credentials.yaml`. + ``~/.predictivegrid/credentials.yaml``. shareable: bool, default=False - Whether or not the connection can be "shared" in a distributed setting such + Whether the connection can be "shared" in a distributed setting such as Ray workers. If set to True, the connection can be serialized and sent to other workers so that data can be retrieved in parallel; **however**, this is less secure because it is possible for other users of the Ray cluster to @@ -74,6 +74,25 @@ def connect(conn_str=None, apikey=None, profile=None, shareable=False): db : BTrDB An instance of the BTrDB context to directly interact with the database. + Examples + -------- + This example looks for the env variables: ``BTRDB_ENDPOINTS`` and ``BTRDB_API_KEY``. + + >>> conn = btrdb.connect() + + + + Connect to the platform by looking for the relevant platform profile + in ``${HOME}/.predictivegrid/credentials.yaml`` if the file is present. + + >>> conn = btrdb.connect(profile='test') + + + If you provide incorrect credentials, you will get an error. + + >>> conn = btrdb.connect(conn_str="192.168.1.1:4411", apikey="NONSENSICAL_API_KEY") + + """ # do not allow user to provide both address and profile if conn_str and profile: diff --git a/btrdb/conn.py b/btrdb/conn.py index 51b29f7..0e9b5fc 100644 --- a/btrdb/conn.py +++ b/btrdb/conn.py @@ -21,14 +21,14 @@ import re import uuid as uuidlib from concurrent.futures import ThreadPoolExecutor -from typing import List +from typing import List, Tuple, Union from warnings import warn import certifi import grpc from grpc._cython.cygrpc import CompressionAlgorithm -from btrdb.exceptions import InvalidOperation, StreamNotFoundError, retry +from btrdb.exceptions import BTrDBError, InvalidOperation, StreamNotFoundError, retry from btrdb.stream import Stream, StreamSet from btrdb.utils.conversion import to_uuid from btrdb.utils.general import unpack_stream_descriptor @@ -55,11 +55,15 @@ def __init__(self, addrportstr, apikey=None): Parameters ---------- - addrportstr: str, required + addrportstr : str, required The address of the cluster to connect to, e.g 123.123.123:4411 - apikey: str, optional + apikey : str, optional The optional API key to authenticate requests + Notes + ----- + The ``btrdb.connect`` method is a helper function to make connecting to the platform easier + usually that will be sufficient for most users. """ addrport = addrportstr.split(":", 2) # 100MB size limit ~ 2500 streams for 5000 points with each point being 64bit @@ -197,6 +201,10 @@ class BTrDB(object): def __init__(self, endpoint): self.ep = endpoint + try: + _ = self.ep.info() + except Exception as err: + raise BTrDBError(f"Could not connect to the database, error message: {err}") self._executor = ThreadPoolExecutor() try: self._ARROW_ENABLED = _is_arrow_enabled(self.ep.info()) @@ -208,8 +216,8 @@ def __init__(self, endpoint): @retry def query( self, - stmt, - params=None, + stmt: str, + params: Union[Tuple[str], List[str]] = None, auto_retry=False, retries=5, retry_delay=3, @@ -221,23 +229,23 @@ def query( Parameters ---------- - stmt: str + stmt : str a SQL statement to be executed on the BTrDB metadata. Available tables are noted below. To sanitize inputs use a `$1` style parameter such as `select * from streams where name = $1 or name = $2`. - params: list or tuple + params : list or tuple a list of parameter values to be sanitized and interpolated into the SQL statement. Using parameters forces value/type checking and is considered a best practice at the very least. - auto_retry: bool, default: False + auto_retry : bool, default: False Whether to retry this request in the event of an error - retries: int, default: 5 + retries : int, default: 5 Number of times to retry this request if there is an error. Will be ignored if auto_retry is False - retry_delay: int, default: 3 + retry_delay : int, default: 3 initial time to wait before retrying function call if there is an error. Will be ignored if auto_retry is False - retry_backoff: int, default: 4 + retry_backoff : int, default: 4 Exponential factor by which the backoff increases between retries. Will be ignored if auto_retry is False @@ -250,10 +258,48 @@ def query( Notes ------- Parameters will be inserted into the SQL statement as noted by the - paramter number such as `$1`, `$2`, or `$3`. The `streams` table is + parameter number such as `$1`, `$2`, or `$3`. The `streams` table is available for `SELECT` statements only. See https://btrdb.readthedocs.io/en/latest/ for more info. + + The following are the queryable columns in the postgres ``streams`` table. + + +------------------+------------------------+-----------+ + | Column | Type | Nullable | + +==================+========================+===========+ + | uuid | uuid | not null | + +------------------+------------------------+-----------+ + | collection | character varying(256) | not null | + +------------------+------------------------+-----------+ + | name | character varying(256) | not null | + +------------------+------------------------+-----------+ + | unit | character varying(256) | not null | + +------------------+------------------------+-----------+ + | ingress | character varying(256) | not null | + +------------------+------------------------+-----------+ + | property_version | bigint | not null | + +------------------+------------------------+-----------+ + | annotations | hstore | | + +------------------+------------------------+-----------+ + + Examples + -------- + Count all streams in the platform. + + >>> conn = btrdb.connect() + >>> conn.query("SELECT COUNT(uuid) FROM streams") + [{'count': ...}] + + Count all streams in the collection ``foo/bar`` by passing in the variable as a parameter. + + >>> conn.query("SELECT COUNT(uuid) FROM streams WHERE collection=$1::text", params=["foo/bar"]) + [{'count': ...}] + + Count all streams in the platform that has a non-null entry for the metadata annotation ``foo``. + + >>> conn.query("SELECT COUNT(uuid) FROM streams WHERE annotations->$1::text IS NOT NULL", params=["foo"]) + [{'count': ...}] """ if params is None: params = list() @@ -267,18 +313,56 @@ def streams(self, *identifiers, versions=None, is_collection_prefix=False): """ Returns a StreamSet object with BTrDB streams from the supplied identifiers. If any streams cannot be found matching the identifier - than StreamNotFoundError will be returned. + then a ``StreamNotFoundError`` will be returned. Parameters ---------- - identifiers: str or UUID + identifiers : str or UUID a single item or iterable of items which can be used to query for - streams. identiers are expected to be UUID as string, UUID as UUID, + streams. Identifiers are expected to be UUID as string, UUID as UUID, or collection/name string. - versions: list[int] + versions : list[int] a single or iterable of version numbers to match the identifiers + is_collection_prefix : bool, default=False + If providing a collection string, is that string just a prefix, or the entire collection name? + This will impact how many streams are returned. + + + Returns + ------- + :class:`StreamSet` + Collection of streams. + + Examples + -------- + With a sequence of uuids. + + >>> conn = btrdb.connect() + >>> conn.streams(identifiers=list_of_uuids) + + + With a sequence of uuids and version numbers. + Here we are using version 0 to use the latest data points. + + >>> conn.streams(identifiers=list_of_uuids, versions=[0 for _ in list_of_uuids]) + + + Filtering by ``collection`` prefix ``"foo"`` where multiple collections exist like the following: + ``foo/bar``, ``foo/baz``, ``foo/bar/new``, and ``foo``. + If we set `is_collection_prefix`` to ``True``, this will return all streams that exist in the collections defined above. + It is similar to a regex pattern ``^foo.*`` for matching purposes. + + >>> conn.streams(identifiers="foo", is_collection_prefix=True) + + + If you set ``is_collection_prefix`` to ``False``, this will assume that the string identifier you provide is the full collection name. + Matching like the regex here: ``^foo`` + + >>> conn.streams(identifiers="foo", is_collection_prefix=False) + + """ if versions is not None and not isinstance(versions, list): raise TypeError("versions argument must be of type list") @@ -338,7 +422,7 @@ def stream_from_uuid(self, uuid): Parameters ---------- - uuid: UUID + uuid : UUID The uuid of the requested stream. Returns @@ -346,6 +430,16 @@ def stream_from_uuid(self, uuid): Stream instance of Stream class or None + Examples + -------- + + >>> import btrdb + >>> conn = btrdb.connect() + >>> uuid = "f98f4b4e-9fab-46b5-8a80-f282059d69b1" + >>> stream = conn.stream_from_uuid(uuid) + >>> stream + + """ return Stream(self, to_uuid(uuid)) @@ -366,23 +460,23 @@ def create( Parameters ---------- - uuid: UUID, required + uuid : UUID, required The uuid of the requested stream. - collection: str, required + collection : str, required The collection string prefix that the stream will belong to. - tags: dict, required - The tags-level immutable metadata key:value pairs. - annotations: dict, optional + tags : dict, required + The tags-level metadata key:value pairs. + annotations : dict, optional The mutable metadata of the stream, key:value pairs - auto_retry: bool, default: False + auto_retry : bool, default: False Whether to retry this request in the event of an error - retries: int, default: 5 + retries : int, default: 5 Number of times to retry this request if there is an error. Will be ignored if auto_retry is False - retry_delay: int, default: 3 + retry_delay : int, default: 3 initial time to wait before retrying function call if there is an error. Will be ignored if auto_retry is False - retry_backoff: int, default: 4 + retry_backoff : int, default: 4 Exponential factor by which the backoff increases between retries. Will be ignored if auto_retry is False @@ -390,6 +484,18 @@ def create( ------- Stream instance of Stream class + + + Examples + -------- + >>> import btrdb + >>> from uuid import uuid4 # this generates a random uuid + >>> conn = btrdb.connect() + >>> collection = "new/stream/collection" + >>> tags = {"name":"foo", "unit":"V"} + >>> annotations = {"bar": "baz"} + >>> s = conn.create(uuid=uuid4(), tags=tags, annotations=annotations, collection=collection) + """ if tags is None: @@ -411,12 +517,23 @@ def create( def info(self): """ - Returns information about the connected BTrDB server. + Returns information about the platform proxy server. Returns ------- dict - server connection and status information + Proxy server connection and status information + + Examples + -------- + >>> conn = btrdb.connect() + >>> conn.info() + { + .. 'majorVersion': 5, + .. 'minorVersion': 8, + .. 'build': ..., + .. 'proxy': ..., + } """ info = self.ep.info() @@ -434,13 +551,27 @@ def list_collections(self, starts_with=""): Parameters ---------- - starts_with: str, optional, default = '' + starts_with : str, optional, default: '' Filter collections that start with the string provided, if none passed, will list all collections. Returns ------- collections: List[str] + Examples + -------- + + Assuming we have the following collections in the platform: + ``foo``, ``bar``, ``foo/baz``, ``bar/baz`` + + >>> conn = btrdb.connect() + >>> conn.list_collections().sort() + ["bar", "bar/baz", "foo", "foo/bar"] + + >>> conn.list_collections(starts_with="foo") + ["foo", "foo/bar"] + + """ return [c for some in self.ep.listCollections(starts_with) for c in some] @@ -465,13 +596,25 @@ def list_unique_annotations(self, collection=None): Returns a list of annotation keys used in a given collection prefix. Parameters - ------- - collection: str + ---------- + collection : str Prefix of the collection to filter. Returns ------- - annotations: list[str] + annotations : list[str] + + Notes + ----- + This query treats the ``collection`` string as a prefix, so ``collection="foo"`` will match with the following wildcard syntax ``foo%``. + If you only want to filter for a single collection, you will need to provide the full collection, if there are other collections + that match the ``foo%`` pattern, you might need to use a custom SQL query using ``conn.query``. + + Examples + -------- + >>> conn.list_unique_annotations(collection="sunshine/PMU1") + ['foo', 'location', 'impedance'] + """ return self._list_unique_tags_annotations("annotations", collection) @@ -480,13 +623,28 @@ def list_unique_names(self, collection=None): Returns a list of names used in a given collection prefix. Parameters - ------- - collection: str + ---------- + collection : str Prefix of the collection to filter. Returns ------- - names: list[str] + names : list[str] + + Examples + -------- + Can specify a full ``collection`` name. + + >>> conn.list_unique_names(collection="sunshine/PMU1") + ['C1ANG', 'C1MAG', 'C2ANG', 'C2MAG', 'C3ANG', 'C3MAG', 'L1ANG', 'L1MAG', 'L2ANG', 'L2MAG', 'L3ANG', 'L3MAG', 'LSTATE'] + + And also provide a ``collection`` prefix. + + >>> conn.list_unique_names(collection="sunshine/") + ['C1ANG', 'C1MAG', 'C2ANG', 'C2MAG', 'C3ANG', 'C3MAG', 'L1ANG', 'L1MAG', 'L2ANG', 'L2MAG', 'L3ANG', 'L3MAG', 'LSTATE'] + + + """ return self._list_unique_tags_annotations("name", collection) @@ -495,13 +653,21 @@ def list_unique_units(self, collection=None): Returns a list of units used in a given collection prefix. Parameters - ------- - collection: str + ---------- + collection : str Prefix of the collection to filter. Returns ------- - units: list[str] + units : list[str] + + + Examples + -------- + + >>> conn.list_unique_units(collection="sunshine/PMU1") + ['amps', 'deg', 'mask', 'volts'] + """ return self._list_unique_tags_annotations("unit", collection) @@ -524,31 +690,54 @@ def streams_in_collection( Parameters ---------- - collection: str + collection : str collections to use when searching for streams, case sensitive. - is_collection_prefix: bool + is_collection_prefix : bool Whether the collection is a prefix. - tags: Dict[str, str] + tags : Dict[str, str] The tags to identify the stream. - annotations: Dict[str, str] + annotations : Dict[str, str] The annotations to identify the stream. - auto_retry: bool, default: False + auto_retry : bool, default: False Whether to retry this request in the event of an error - retries: int, default: 5 + retries : int, default: 5 Number of times to retry this request if there is an error. Will be ignored if auto_retry is False - retry_delay: int, default: 3 + retry_delay : int, default: 3 initial time to wait before retrying function call if there is an error. Will be ignored if auto_retry is False - retry_backoff: int, default: 4 + retry_backoff : int, default: 4 Exponential factor by which the backoff increases between retries. Will be ignored if auto_retry is False Returns ------ - list - A list of stream objects found with the provided search arguments. + list[Stream] + A list of ``Stream`` objects found with the provided search arguments. + + + .. note:: + In a future release, the default return value of this function will be a ``StreamSet`` + + Examples + -------- + + >>> conn = btrdb.connect() + >>> conn.streams_in_collection(collection="foo", is_collection_prefix=True) + [, , ] + + >>> conn.streams_in_collection(collection="foo", is_collection_prefix=False) + [, ] + + >>> conn.streams_in_collection(collection="foo", + ... is_collection_prefix=False, tags={"unit":"Volts"}) + [] + + >>> conn.streams_in_collection(collection="foo", + ... is_collection_prefix=False, tags={"unit":"UNKNOWN"}) + [] """ result = [] @@ -581,7 +770,7 @@ def streams_in_collection( ) # TODO: In future release update this method to return a streamset object. warn( - "StreamSet will be returned in a future release.", + "StreamSet will be the default return object for ``streams_in_collection`` in a future release.", FutureWarning, stacklevel=2, ) @@ -598,21 +787,21 @@ def collection_metadata( ): """ Gives statistics about metadata for collections that match a - prefix. + ``prefix``. Parameters ---------- - prefix: str, required + prefix : str, required A prefix of the collection names to look at - auto_retry: bool, default: False + auto_retry : bool, default: False Whether to retry this request in the event of an error - retries: int, default: 5 + retries : int, default: 5 Number of times to retry this request if there is an error. Will be ignored if auto_retry is False - retry_delay: int, default: 3 + retry_delay : int, default: 3 initial time to wait before retrying function call if there is an error. Will be ignored if auto_retry is False - retry_backoff: int, default: 4 + retry_backoff : int, default: 4 Exponential factor by which the backoff increases between retries. Will be ignored if auto_retry is False @@ -622,6 +811,16 @@ def collection_metadata( A tuple of dictionaries containing metadata on the streams in the provided collection. + Examples + -------- + >>> conn.collection_metadata("sunshine/PMU1") + ({'name': 0, 'unit': 0, 'ingress': 0, 'distiller': 0}, + .. {'foo': 1, 'impedance': 12, 'location': 12}) + + >>> conn.collection_metadata("sunshine/") + ({'name': 0, 'unit': 0, 'ingress': 0, 'distiller': 0}, + .. {'foo': 1, 'impedance': 72, 'location': 72}) + """ ep = self.ep tags, annotations = ep.getMetadataUsage(prefix) diff --git a/btrdb/endpoint.py b/btrdb/endpoint.py index 9bb5e8d..c632bbb 100644 --- a/btrdb/endpoint.py +++ b/btrdb/endpoint.py @@ -38,7 +38,7 @@ except ImportError: pa = None -_ARROW_IMPORT_MSG = """Package pyarrow required, please pip install.""" +_ARROW_IMPORT_MSG = """Package `pyarrow` required, please pip install.""" class Endpoint(object): diff --git a/btrdb/exceptions.py b/btrdb/exceptions.py index 2ce0c13..8507f7c 100644 --- a/btrdb/exceptions.py +++ b/btrdb/exceptions.py @@ -148,7 +148,7 @@ def check_proto_stat(stat): if code in BTRDB_ERRORS: raise BTRDB_ERRORS[code](stat.msg) elif code in BTRDB_SERVER_ERRORS: - raise BTRDBServerError(stat.msg) + raise BTRDBServerError(str(code) + ": " + stat.msg) raise BTrDBError(stat.msg) diff --git a/btrdb/stream.py b/btrdb/stream.py index be0f62e..e9d4ac5 100644 --- a/btrdb/stream.py +++ b/btrdb/stream.py @@ -153,6 +153,18 @@ def exists(self): ------- bool Indicates whether stream is extant in the BTrDB server. + + Examples + -------- + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream = conn.stream_from_uuid("...") + >>> stream.uuid + UUID('...') + >>> stream.exists() + True + + """ if self._known_to_exist: @@ -178,15 +190,7 @@ def count( Compute the total number of points in the stream Counts the number of points in the specified window and version. By - default returns the latest total count of points in the stream. This - helper method sums the counts of all StatPoints returned by - ``aligned_windows``. Because of this, note that the start and end - timestamps may be adjusted if they are not powers of 2. For smaller - windows of time, you may also need to adjust the pointwidth to ensure - that the count granularity is captured appropriately. - - Alternatively you can set the precise argument to True which will - give an exact count to the nanosecond but may be slower to execute. + default, returns the latest total count of points in the stream. Parameters ---------- @@ -215,6 +219,32 @@ def count( ------- int The total number of points in the stream for the specified window. + + + .. note:: + + This helper method sums the counts of all StatPoints returned by + ``aligned_windows``. Because of this, note that the start and end + timestamps may be adjusted if they are not powers of 2. For smaller + windows of time, you may also need to adjust the ``pointwidth`` to ensure + that the count granularity is captured appropriately. + + Alternatively you can set the ``precise`` argument to ``True`` which will + give an exact count to the nanosecond but may be slower to execute. + + + + Examples + -------- + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream = conn.stream_from_uuid("...") + >>> stream.count() + 1234 + >>> stream.count(start=1500000000000000000, end=1603680000000000000, pointwidth=55) + 567 + >>> stream.count(start=1500000000000000000, end=1603680000000000000, precise=True) + 789 """ if not precise: @@ -233,16 +263,26 @@ def count( @property def btrdb(self): """ - Returns the stream's BTrDB object. + Returns the stream's BTrDB object. - Parameters - ---------- - None + Parameters + ---------- + None - Returns - ------- - BTrDB - The BTrDB database object. + Returns + ------- + BTrDB + The BTrDB database object. + + Examples + -------- + + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream = conn.stream_from_uuid("...") + >>> btrdb_obj = stream.btrdb + >>> btrdb_obj + """ return self._btrdb @@ -278,6 +318,14 @@ def name(self): str The name of the stream. + Examples + -------- + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream = conn.stream_from_uuid("...") + >>> stream.name + 'foo' + """ return self.tags()["name"] @@ -293,6 +341,14 @@ def unit(self): str The unit for values of the stream. + Examples + -------- + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream = conn.stream_from_uuid("...") + >>> stream.unit + 'volts' + """ return self.tags()["unit"] @@ -311,6 +367,14 @@ def collection(self): str the collection of the stream + Examples + -------- + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream = conn.stream_from_uuid("...") + >>> stream.collection + 'foo/bar' + """ if self._collection is not None: return self._collection @@ -354,6 +418,20 @@ def earliest( The first data point in the stream and the version of the stream the value was retrieved at (tuple(RawPoint, int)). + + Examples + -------- + Get the earliest point for a stream using ``version`` ``0``. + + >>> stream.earliest(version=0) + (, 1234567) + + Extract just the ``RawPoint`` data. + + >>> pt, _ = stream.earliest(version=0) + >>> print(pt.time, pt.value) + 1547241923338098176 123.7 + """ start = MINIMUM_TIME return self.nearest(start, version=version, backward=False) @@ -394,6 +472,18 @@ def latest( The last data point in the stream and the version of the stream the value was retrieved at (tuple(RawPoint, int)). + Examples + -------- + Get the latest point for a stream using ``version`` ``0``. + + >>> stream.latest(version=0) + (, 1234567) + + Extract just the ``RawPoint`` data. + + >>> pt, _ = stream.latest(version=0) + >>> print(pt.time, pt.value) + 1547241923338098176 123.7 """ start = MAXIMUM_TIME return self.nearest(start, version=version, backward=True) @@ -410,7 +500,7 @@ def current( """ Returns the point that is closest to the current timestamp, e.g. the latest point in the stream up until now. Note that no future values will be returned. - Returns None if errors during lookup or there are no values before now. + Returns None if errors occur during lookup or there are no values before now. Parameters ---------- @@ -521,6 +611,25 @@ def annotations( A tuple containing a dictionary of annotations and an integer representing the version of the metadata (tuple(dict, int)). + + .. note:: + + This ``version`` is not the same as the ``stream.version``. + + Examples + -------- + Accessing a streams annotations. + + >>> stream.annotations() + ({"foo":"bar", "baz":"bazaar"}, 231) + + Extract the version and metadata separately. + + >>> annotations, metadata_version = stream.annotations() + >>> annotations + {"foo":"bar", "baz":"bazaar"} + >>> metadata_version + 231 """ if refresh or self._annotations is None: self.refresh_metadata() @@ -538,9 +647,11 @@ def version( """ Returns the current data version of the stream. - Version returns the current data version of the stream. This is not - cached, it queries each time. Take care that you do not intorduce races - in your code by assuming this function will always return the same vaue + .. warning:: + + Version returns the current data version of the stream. This is not + cached, it queries each time. Take care that you do not introduce races + in your code by assuming this function will always return the same value. Parameters ---------- @@ -590,6 +701,18 @@ def insert(self, data, merge="never"): ------- int The version of the stream after inserting new points. + + + Examples + -------- + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream = conn.stream_from_uuid("...") + >>> data = [(1500000000000000000, 1.0), (1500000000100000000, 2.0)] + >>> stream.insert(data) + 1234 + >>> stream.insert(data, merge="replace") + 1235 """ version = 0 i = 0 @@ -626,9 +749,39 @@ def arrow_insert(self, data: pa.Table, merge: str = "never") -> int: int The version of the stream after inserting new points. - Notes - ----- - This method is available for commercial customers with arrow-enabled servers. + + .. note:: + + This method is available for commercial customers with arrow-enabled servers. + + Examples + -------- + Assuming we have a sequence of ``times`` and ``values`` where ``times`` are in nanoseconds. + Insert the data as a pyarrow table, and if there are duplicate timestamps already in the database, + replace with the new ones in ``payload``. + + >>> conn = btrdb.connect() + >>> import pyarrow as pa + >>> for t, v in zip(times, vals): + ... print(t,v) + 1500000000000000000 1.0 + 1500000000100000000 2.0 + 1500000000200000000 3.0 + 1500000000300000000 4.0 + 1500000000400000000 5.0 + 1500000000500000000 6.0 + 1500000000600000000 7.0 + 1500000000700000000 8.0 + 1500000000800000000 9.0 + 1500000000900000000 10.0 + >>> schema = pa.schema( + ... [ + ... pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False), + ... pa.field("value", pa.float64(), nullable=False), + ... ] + ... ) + >>> payload = pa.Table.from_arrays([times, vals], schema=schema) + >>> version = stream.arrow_insert(payload, merge="replace") """ if not self._btrdb._ARROW_ENABLED: raise NotImplementedError(_arrow_not_impl_str.format("arrow_insert")) @@ -667,6 +820,9 @@ def arrow_insert(self, data: pa.Table, merge: str = "never") -> int: return max(version) def _update_tags_collection(self, tags, collection): + """ + :meta private: + """ tags = self.tags() if tags is None else tags collection = self.collection if collection is None else collection if collection is None: @@ -682,6 +838,9 @@ def _update_tags_collection(self, tags, collection): ) def _update_annotations(self, annotations, encoder, replace): + """ + :meta private: + """ # make a copy of the annotations to prevent accidental mutable object mutation serialized = deepcopy(annotations) if encoder is not None: @@ -727,14 +886,10 @@ def update( None, they will remain unchanged in the database. To delete either tags or annotations, you must specify exactly which - keys and values you want set for the field and set `replace=True`. For - example: + keys and values you want set for the field and set ``replace=True``. - >>> annotations, _ = stream.anotations() - >>> del annotations["key_to_delete"] - >>> stream.update(annotations=annotations, replace=True) - This ensures that all of the keys and values for the annotations are + This ensures that all the keys and values for the annotations are preserved except for the key to be deleted. Parameters @@ -750,11 +905,11 @@ def update( Specify a new collection for the stream. If None, the collection will remain unchanged. encoder : json.JSONEncoder or None - JSON encoder class to use for annotation serialization. Set to None + JSON encoder class to use for annotation serialization. Set to ``None`` to prevent JSON encoding of the annotations. replace : bool, default: False Replace all annotations or tags with the specified dictionaries - instead of performing the normal upsert operation. Specifying True + instead of performing the normal upsert operation. Specifying ``True`` is the only way to remove annotation keys. auto_retry: bool, default: False Whether to retry this request in the event of an error @@ -773,6 +928,18 @@ def update( int The version of the metadata (separate from the version of the data) also known as the "property version". + + + Examples + -------- + >>> annotations, _ = stream.anotations() + >>> del annotations["key_to_delete"] + >>> stream.update(annotations=annotations, replace=True) + 12345 + >>> annotations, _ = stream.annotations() + >>> "key_to_delete" in annotations + False + """ if tags is None and annotations is None and collection is None: raise BTRDBValueError( @@ -809,21 +976,25 @@ def delete( retry_backoff=4, ): """ - "Delete" all points between [`start`, `end`) + "Delete" all points between [``start``, ``end``) + + "Delete" all points between ``start`` (inclusive) and ``end`` (exclusive), + both in nanoseconds. + + .. note:: - "Delete" all points between `start` (inclusive) and `end` (exclusive), - both in nanoseconds. As BTrDB has persistent multiversioning, the - deleted points will still exist as part of an older version of the - stream. + As ``BTrDB`` has persistent multiversioning, the + deleted points will still exist as part of an older version of the + stream. Parameters ---------- start : int or datetime like object The start time in nanoseconds for the range to be deleted. (see - :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) + :func:``btrdb.utils.timez.to_nanoseconds`` for valid input types) end : int or datetime like object The end time in nanoseconds for the range to be deleted. (see - :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) + :func:``btrdb.utils.timez.to_nanoseconds`` for valid input types) auto_retry: bool, default: False Whether to retry this request in the event of an error retries: int, default: 5 @@ -840,6 +1011,19 @@ def delete( ------- int The version of the new stream created + + + Examples + -------- + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream = conn.stream_from_uuid("...") + >>> start = 1500000000000000000 + >>> end = 1500000001000000000 + >>> stream.delete(start, end) + 1234 + >>> stream.count(start=start, end=end) + 0 """ return self._btrdb.ep.deleteRange( self._uuid, to_nanoseconds(start), to_nanoseconds(end) @@ -892,13 +1076,13 @@ def values( version (list(tuple(RawPoint,int))). - Notes - ----- - Note that the raw data points are the original values at the sensor's - native sampling rate (assuming the time series represents measurements - from a sensor). This is the lowest level of data with the finest time - granularity. In the tree data structure of BTrDB, this data is stored in - the vector nodes. + .. note:: + + Note that the raw data points are the original values at the sensor's + native sampling rate (assuming the time series represents measurements + from a sensor). This is the lowest level of data with the finest time + granularity. In the tree data structure of BTrDB, this data is stored in + the vector nodes. """ materialized = [] @@ -958,15 +1142,20 @@ def arrow_values( A pyarrow table of the raw values with time and value columns. - Notes - ----- - Note that the raw data points are the original values at the sensor's - native sampling rate (assuming the time series represents measurements - from a sensor). This is the lowest level of data with the finest time - granularity. In the tree data structure of BTrDB, this data is stored in - the vector nodes. + .. note:: + + Note that the raw data points are the original values at the sensor's + native sampling rate (assuming the time series represents measurements + from a sensor). This is the lowest level of data with the finest time + granularity. In the tree data structure of BTrDB, this data is stored in + the vector nodes. + + + .. note:: + + This method is available for commercial customers with arrow-enabled servers. + - This method is available for commercial customers with arrow-enabled servers. """ if not self._btrdb._ARROW_ENABLED: raise NotImplementedError(_arrow_not_impl_str.format("arrow_values")) @@ -1007,16 +1196,18 @@ def aligned_windows( Read statistical aggregates of windows of data from BTrDB. Query BTrDB for aggregates (or roll ups or windows) of the time series - with `version` between time `start` (inclusive) and `end` (exclusive) in + with `version` between time ``start`` (inclusive) and ``end`` (exclusive) in nanoseconds. Each point returned is a statistical aggregate of all the - raw data within a window of width 2**`pointwidth` nanoseconds. These + raw data within a window of width ``2**pointwidth`` nanoseconds. These statistical aggregates currently include the mean, minimum, and maximum of the data and the count of data points composing the window. - Note that `start` is inclusive, but `end` is exclusive. That is, results + .. note:: + + ``start`` is inclusive, but ``end`` is exclusive. That is, results will be returned for all windows that start in the interval [start, end). If end < start+2^pointwidth you will not get any results. If start and - end are not powers of two, the bottom pointwidth bits will be cleared. + end are not powers of two, the bottom ``pointwidth`` bits will be cleared. Each window will contain statistical summaries of the window. Statistical points with count == 0 will be omitted. @@ -1051,10 +1242,12 @@ def aligned_windows( containing data tuples. Each data tuple contains a StatPoint and the stream version. - Notes - ----- - As the window-width is a power-of-two, it aligns with BTrDB internal - tree data structure and is faster to execute than `windows()`. + + .. note:: + + As the window-width is a power-of-two, it aligns with ``BTrDB`` internal + tree data structure and is faster to execute than ``windows()``. + """ materialized = [] start = to_nanoseconds(start) @@ -1088,21 +1281,24 @@ def arrow_aligned_windows( statistical aggregates currently include the mean, minimum, and maximum of the data and the count of data points composing the window. - Note that `start` is inclusive, but `end` is exclusive. That is, results - will be returned for all windows that start in the interval [start, end). - If end < start+2^pointwidth you will not get any results. If start and - end are not powers of two, the bottom pointwidth bits will be cleared. - Each window will contain statistical summaries of the window. - Statistical points with count == 0 will be omitted. + + .. note:: + + ``start`` is inclusive, but ``end`` is exclusive. That is, results + will be returned for all windows that start in the interval [start, end). + If end < start+2^pointwidth you will not get any results. If start and + end are not powers of two, the bottom ``pointwidth`` bits will be cleared. + Each window will contain statistical summaries of the window. + Statistical points with ``count == 0`` will be omitted. Parameters ---------- start : int or datetime like object, required The start time in nanoseconds for the range to be queried. (see - :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) + :func:``btrdb.utils.timez.to_nanoseconds`` for valid input types) end : int or datetime like object, required The end time in nanoseconds for the range to be queried. (see - :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) + :func:``btrdb.utils.timez.to_nanoseconds`` for valid input types) pointwidth : int, required Specify the number of ns between data points (2**pointwidth) version : int, default: 0 @@ -1124,12 +1320,15 @@ def arrow_aligned_windows( pyarrow.Table Returns a pyarrow table containing the windows of data. - Notes - ----- - As the window-width is a power-of-two, it aligns with BTrDB internal - tree data structure and is faster to execute than `windows()`. - This method is available for commercial customers with arrow-enabled servers. + .. note:: + + As the window-width is a power-of-two, it aligns with BTrDB internal + tree data structure and is faster to execute than `windows()`. + + .. note:: + + This method is available for commercial customers with arrow-enabled servers. """ if not self._btrdb._ARROW_ENABLED: raise NotImplementedError( @@ -1213,21 +1412,21 @@ def windows( containing data tuples. Each data tuple contains a StatPoint and the stream version (tuple(tuple(StatPoint, int), ...)). - Notes - ----- - Windows returns arbitrary precision windows from BTrDB. It is slower - than AlignedWindows, but still significantly faster than RawValues. Each - returned window will be `width` nanoseconds long. `start` is inclusive, - but `end` is exclusive (e.g if end < start+width you will get no - results). That is, results will be returned for all windows that start - at a time less than the end timestamp. If (`end` - `start`) is not a - multiple of width, then end will be decreased to the greatest value less - than end such that (end - start) is a multiple of `width` (i.e., we set - end = start + width * floordiv(end - start, width). The `depth` - parameter previously available has been deprecated. The only valid value - for depth is now 0. - This method is available for commercial customers with arrow-enabled servers. + .. note:: + + ``windows`` returns arbitrary precision windows from BTrDB. It is slower + than ``aligned_windows``, but can be significantly faster than raw value queries (``values``). Each + returned window will be ``width`` nanoseconds long. ``start`` is inclusive, + but ``end`` is exclusive (e.g if ``end < start+width`` you will get no + results). That is, results will be returned for all windows that start + at a time less than the end timestamp. If (``end`` - ``start``) is not a + multiple of ``width``, then ``end`` will be decreased to the greatest value less + than ``end`` such that (``end`` - ``start``) is a multiple of ``width`` (i.e., we set + ``end = start + width * floordiv(end - start, width)``. The ``depth`` + parameter previously available has been deprecated. The only valid value + for ``depth`` is now ``0``. + """ materialized = [] start = to_nanoseconds(start) @@ -1282,20 +1481,24 @@ def arrow_windows( pyarrow.Table Returns a pyarrow Table containing windows of data. - Notes - ----- - Windows returns arbitrary precision windows from BTrDB. It is slower - than AlignedWindows, but still significantly faster than RawValues. Each - returned window will be `width` nanoseconds long. `start` is inclusive, - but `end` is exclusive (e.g if end < start+width you will get no - results). That is, results will be returned for all windows that start - at a time less than the end timestamp. If (`end` - `start`) is not a - multiple of width, then end will be decreased to the greatest value less - than end such that (end - start) is a multiple of `width` (i.e., we set - end = start + width * floordiv(end - start, width). The `depth` - parameter previously available has been deprecated. The only valid value - for depth is now 0. - This method is available for commercial customers with arrow-enabled servers. + + .. note:: + + ``windows`` returns arbitrary precision windows from ``BTrDB``. It is slower + than ``aligned_windows``, but still significantly faster than RawValues. Each + returned window will be ``width`` nanoseconds long. ``start`` is inclusive, + but ``end`` is exclusive (e.g if end < start+width you will get no + results). That is, results will be returned for all windows that start + at a time less than the ``end`` timestamp. If (``end`` - ``start``) is not a + multiple of ``width``, then ``end`` will be decreased to the greatest value less + than ``end`` such that (``end`` - ``start``) is a multiple of `width` (i.e., we set + ``end = start + width * floordiv(end - start, width)``. The ``depth`` + parameter previously available has been deprecated. The only valid value + for ``depth`` is now 0. + + .. note:: + + This method is available for commercial customers with arrow-enabled servers. """ if not self._btrdb._ARROW_ENABLED: raise NotImplementedError(_arrow_not_impl_str.format("arrow_windows")) @@ -1344,17 +1547,17 @@ def nearest( """ Finds the closest point in the stream to a specified time. - Return the point nearest to the specified `time` in nanoseconds since - Epoch in the stream with `version` while specifying whether to search - forward or backward in time. If `backward` is false, the returned point - will be >= `time`. If backward is true, the returned point will be < - `time`. The version of the stream used to satisfy the query is returned. + Return the point nearest to the specified ``time`` in nanoseconds since + Epoch in the stream with ``version`` while specifying whether to search + forward or backward in time. If ``backward`` is ``false``, the returned point + will be >= ``time``. If ``backward`` is ``true``, the returned point will be < + ``time``. The ``version`` of the ``stream`` used to satisfy the query is returned. Parameters ---------- time : int or datetime like object The time (in nanoseconds since Epoch) to search near (see - :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) + :func:``btrdb.utils.timez.to_nanoseconds`` for valid input types) version : int Version of the stream to use in search backward : boolean @@ -1497,6 +1700,9 @@ def allow_window(self): return not bool(self.pointwidth or (self.width and self.depth == 0)) def _latest_versions(self): + """ + :meta private: + """ uuid_ver_tups = self._btrdb._executor.map( lambda s: (s.uuid, s.version()), self._streams ) @@ -1506,7 +1712,7 @@ def pin_versions(self, versions=None): """ Saves the stream versions that future materializations should use. If no pin is requested then the first materialization will automatically - pin the return versions. Versions can also be supplied through a dict + pin the return versions. Versions can also be supplied through a ``dict`` object with key:UUID, value:stream.version(). Parameters @@ -1519,6 +1725,10 @@ def pin_versions(self, versions=None): StreamSet Returns self + Examples + -------- + >>> version_map = {s.uuid: 0 for s in streamset} + >>> pinned_streamset = streamset.pin_versions(versions=version_map) """ if versions is not None: if not isinstance(versions, dict): @@ -1547,6 +1757,18 @@ def versions(self): dict A dict containing the stream UUID and version ints as key/values + + Examples + -------- + A pinned vs non-pinned streamset + + >>> streamset = btrdb.stream.StreamSet([stream1, stream2]) + >>> version_map = {s.uuid: 0 for s streamset} + >>> pinned_streamset = streamset.pin_versions(versions=version_map) + >>> pinned_streamset.versions() + {UUID('fa42f64a-a851-408f-aa7e-88a85b3d295c'): 0, UUID('18e5527a-ed13-424d-bb97-3e06a763609e'): 0} + >>> streamset.versions() + {UUID('fa42f64a-a851-408f-aa7e-88a85b3d295c'): 34532, UUID('18e5527a-ed13-424d-bb97-3e06a763609e'): 12345} """ return ( self._pinned_versions if self._pinned_versions else self._latest_versions() @@ -1561,10 +1783,6 @@ def count(self, precise: bool = False): all points in the streams. The count is modified by start and end filters or by pinning versions. - Note that this helper method sums the counts of all StatPoints returned - by ``aligned_windows``. Because of this the start and end timestamps - may be adjusted if they are not powers of 2. - Parameters ---------- precise : bool, default = False @@ -1574,6 +1792,27 @@ def count(self, precise: bool = False): ------- int The total number of points in all streams for the specified filters. + + .. note:: + + Note that this helper method sums the counts of all StatPoints returned + by ``aligned_windows``. Because of this the start and end timestamps + may be adjusted if they are not powers of 2. + + Examples + -------- + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream1 = conn.stream_from_uuid("...") + >>> stream2 = conn.stream_from_uuid("...") + >>> streamset = btrdb.stream.StreamSet([stream1, stream2]) + >>> streamset.count() + 2345 + >>> filtered_streamset = streamset.filter(start=1500000000000000000, end=1500000001000000000) + >>> filtered_streamset.count(precise=True) + 734 + >>> streamset.filter(start=1500000000000000000, end=1500000001000000000).count(precise=True) + 734 """ params = self._params_from_filters() start = params.get("start", MINIMUM_TIME) @@ -1605,6 +1844,15 @@ def earliest(self): tuple The earliest points of data found among all streams + Examples + -------- + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream1 = conn.stream_from_uuid("...") + >>> stream2 = conn.stream_from_uuid("...") + >>> streamset = btrdb.stream.StreamSet([stream1, stream2]) + >>> streamset.earliest() + (, ) """ earliest = [] params = self._params_from_filters() @@ -1637,6 +1885,15 @@ def latest(self): tuple The latest points of data found among all streams + Examples + -------- + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream1 = conn.stream_from_uuid("...") + >>> stream2 = conn.stream_from_uuid("...") + >>> streamset = btrdb.stream.StreamSet([stream1, stream2]) + >>> streamset.earliest() + (, ) """ latest = [] params = self._params_from_filters() @@ -1664,6 +1921,16 @@ def current(self): ------- tuple The latest points of data found among all streams + + Examples + -------- + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream1 = conn.stream_from_uuid("...") + >>> stream2 = conn.stream_from_uuid("...") + >>> streamset = btrdb.stream.StreamSet([stream1, stream2]) + >>> streamset.current() + (, ) """ latest = [] params = self._params_from_filters() @@ -1677,12 +1944,15 @@ def current(self): ) versions = self.versions() - latest_points_gen = self._btrdb._executor.map( + current_points_gen = self._btrdb._executor.map( lambda s: (s.nearest(now, version=versions.get(s.uuid, 0), backward=True)), self._streams, ) - for point in latest_points_gen: - latest.append(point) + for point in current_points_gen: + if point is not None: + latest.append(point[0]) + else: + latest.append(None) return tuple(latest) @@ -1743,12 +2013,37 @@ def filter( StreamSet a new instance cloned from the original with filters applied - Notes - ----- - If you set `sampling_frequency` to a non-zero value, the stream data returned will be aligned to a - grid of timestamps based on the period of the sampling frequency. For example, a sampling rate of 30hz will - have a sampling period of 1/30hz -> ~33_333_333 ns per sample. Leave sampling_frequency as None, or set to 0 to - prevent time alignment. You should **not** use aligned data for frequency-based analysis. + + .. note:: + + If you set ``sampling_frequency`` to a non-zero value, the stream data returned will be aligned to a + grid of timestamps based on the period of the sampling frequency. For example, a sampling rate of 30hz will + have a sampling period of 1/30hz -> ~33_333_333 ns per sample. Leave ``sampling_frequency`` as ``None``, or set to ``0`` to + prevent time alignment. You should **not** use aligned data for frequency-based analysis. + + + Examples + -------- + create a streamset and apply a few filters + + >>> streamset = btrdb.stream.StreamSet(list_of_streams) + >>> print(f"Total streams: {len(streamset)}") + Total streams: 89 + + >>> streamset.filter(units="Volts") + >>> print(f"Total streams: {len(streamset)}") + Total streams: 89 + + >>> filtered_streamset = streamset.filter(units="Volts") + >>> print(f"Total streams: {len(filtered_streamset)}") + Total streams: 23 + + >>> multiple_filters_streamset = (streamset.filter(unit="Volts") + >>> .filter(name="Sensor 1") + >>> .filter(annotations={"phase":"A"}) + >>> ) + >>> print(f"Total streams: {len(multiple_filters_streamset)}") + Total streams: 1 """ obj = self.clone() @@ -1874,19 +2169,35 @@ def windows(self, width, depth=0): Returns self - Notes - ----- - Windows returns arbitrary precision windows from BTrDB. It is slower - than aligned_windows, but still significantly faster than values. Each - returned window will be width nanoseconds long. start is inclusive, but - end is exclusive (e.g. if end < start+width you will get no results). - That is, results will be returned for all windows that start at a time - less than the end timestamp. If (end - start) is not a multiple of - width, then end will be decreased to the greatest value less than end - such that (end - start) is a multiple of width (i.e., we set end = start - + width * floordiv(end - start, width)). The `depth` parameter previously - available has been deprecated. The only valid value for depth is now 0. + .. note:: + + ``windows`` returns arbitrary precision windows from BTrDB. It is slower + than ``aligned_windows``, but can be significantly faster than values. + Each returned window will be ``width`` nanoseconds long. + ``start`` is inclusive, but ``end`` is exclusive ( ``[start, end) ) + (e.g. if end < start+width you will get no results). + That is, results will be returned for all windows that start at a time + less than the end timestamp. If (``end`` - ``start``) is not a multiple of + ``width``, then ``end`` will be decreased to the greatest value less than ``end`` + such that (end - start) is a multiple of width (i.e., we set end = start + + width * floordiv(end - start, width)). The ``depth`` parameter previously + available has been deprecated. The only valid value for ``depth`` is now ``0``. + Examples + -------- + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream1 = conn.stream_from_uuid("...") + >>> stream2 = conn.stream_from_uuid("...") + >>> streamset = btrdb.stream.StreamSet([stream1, stream2]) + >>> streamset.windows(width=1000000000) + + >>> streamset.windows(width=1000000000, depth=0) + + >>> streamset.aligned_windows(pointwidth=30) + Traceback (most recent call last): + ... + btrdb.exceptions.InvalidOperation: A window operation is already requested """ if not self.allow_window: raise InvalidOperation("A window operation is already requested") @@ -1915,18 +2226,32 @@ def aligned_windows(self, pointwidth): StreamSet Returns self - Notes - ----- - `aligned_windows` reads power-of-two aligned windows from BTrDB. It is - faster than Windows(). Each returned window will be 2^pointwidth - nanoseconds long, starting at start. Note that start is inclusive, but - end is exclusive. That is, results will be returned for all windows that - start in the interval [start, end). If end < start+2^pointwidth you will - not get any results. If start and end are not powers of two, the bottom - pointwidth bits will be cleared. Each window will contain statistical - summaries of the window. Statistical points with count == 0 will be - omitted. + .. note:: + + ``aligned_windows`` reads power-of-two aligned windows from BTrDB. It is + faster than ``windows()``. Each returned window will be 2^``pointwidth`` + nanoseconds long, beginning at ``start``. Note that start is inclusive, but + end is exclusive. That is, results will be returned for all windows that + start in the interval ``[start, end)``. If ``end`` < ``start``+2^``pointwidth`` you will + not get any results. If ``start`` and ``end`` are not powers of two, the bottom + ``pointwidth`` bits will be cleared. Each window will contain statistical + summaries of the window. Statistical points with ``count`` == 0 will be + omitted. + + Examples + -------- + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream1 = conn.stream_from_uuid("...") + >>> stream2 = conn.stream_from_uuid("...") + >>> streamset = btrdb.stream.StreamSet([stream1, stream2]) + >>> streamset.aligned_windows(pointwidth=30) + + >>> streamset.windows(width=1000000000) + Traceback (most recent call last): + ... + btrdb.exceptions.InvalidOperation: A window operation is already requested """ if not self.allow_window: raise InvalidOperation("A window operation is already requested") @@ -1943,6 +2268,8 @@ def _streamset_data(self, as_iterators=False): ---------- as_iterators : bool Returns each single stream's data as an iterator. Defaults to False. + + :meta private: """ params = self._params_from_filters() # sampling freq not supported for non-arrow streamset ops @@ -2001,6 +2328,20 @@ def rows(self): A list of tuples containing a RawPoint (or StatPoint) and the stream version (list(tuple(RawPoint, int))). + Examples + -------- + >>> for row in streams.rows(): + >>> print(row) + (None, RawPoint(1500000000000000000, 1.0), RawPoint(1500000000000000000, 1.0), RawPoint(1500000000000000000, 1.0)) + (RawPoint(1500000000100000000, 2.0), None, RawPoint(1500000000100000000, 2.0), RawPoint(1500000000100000000, 2.0)) + (None, RawPoint(1500000000200000000, 3.0), None, RawPoint(1500000000200000000, 3.0)) + (RawPoint(1500000000300000000, 4.0), None, RawPoint(1500000000300000000, 4.0), RawPoint(1500000000300000000, 4.0)) + (None, RawPoint(1500000000400000000, 5.0), RawPoint(1500000000400000000, 5.0), RawPoint(1500000000400000000, 5.0)) + (RawPoint(1500000000500000000, 6.0), None, None, RawPoint(1500000000500000000, 6.0)) + (None, RawPoint(1500000000600000000, 7.0), RawPoint(1500000000600000000, 7.0), RawPoint(1500000000600000000, 7.0)) + (RawPoint(1500000000700000000, 8.0), None, RawPoint(1500000000700000000, 8.0), RawPoint(1500000000700000000, 8.0)) + (None, RawPoint(1500000000800000000, 9.0), RawPoint(1500000000800000000, 9.0), RawPoint(1500000000800000000, 9.0)) + (RawPoint(1500000000900000000, 10.0), None, RawPoint(1500000000900000000, 10.0), RawPoint(1500000000900000000, 10.0)) """ result = [] streamset_data = self._streamset_data(as_iterators=True) @@ -2050,14 +2391,34 @@ def insert(self, data_map: dict, merge: str = "never") -> dict: - 'retain': if two points have the same timestamp, the old one is kept - 'replace': if two points have the same timestamp, the new one is kept - Notes - ----- - You MUST convert your datetimes into utc+0 yourself. BTrDB expects utc+0 datetimes. Returns ------- dict[uuid, int] The versions of the stream after inserting new points. + + + .. note:: + + You MUST convert your datetimes into UTC+0 **yourself**. BTrDB expects UTC+0 datetimes. + + + Examples + -------- + >>> import pandas as pd + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream1 = conn.stream_from_uuid("...") + >>> stream2 = conn.stream_from_uuid("...") + >>> streamset = btrdb.stream.StreamSet([stream1, stream2]) + >>> data_map = { + ... stream1.uuid: pd.DataFrame({'time': [1500000000000000000, 1500000000100000000], 'value': [1.0, 2.0]}), + ... stream2.uuid: pd.DataFrame({'time': [1500000000000000000, 1500000000100000000], 'value': [3.0, 4.0]}) + ... } + >>> streamset.insert(data_map) + {UUID('...'): 1234, UUID('...'): 5678} + >>> streamset.insert(data_map, merge='replace') + {UUID('...'): 1235, UUID('...'): 5679} """ filtered_data_map = {s.uuid: data_map[s.uuid] for s in self._streams} for key, dat in filtered_data_map.items(): @@ -2118,6 +2479,9 @@ def arrow_insert(self, data_map: dict, merge: str = "never") -> dict: return versions def _params_from_filters(self): + """ + :meta private: + """ params = {} for filter in self.filters: if filter.start is not None: @@ -2133,6 +2497,8 @@ def _params_from_filters(self): def values_iter(self): """ Must return context object which would then close server cursor on __exit__ + + :meta private: """ raise NotImplementedError() @@ -2153,9 +2519,9 @@ def arrow_values( This data will be sorted by the 'time' column. - Notes - ----- - This method is available for commercial customers with arrow-enabled servers. + .. note:: + + This method is available for commercial customers with arrow-enabled servers. """ if pa is None: raise ImportError(_ARROW_IMPORT_MSG) diff --git a/btrdb/transformers.py b/btrdb/transformers.py index 0ff01fb..dcdee4f 100644 --- a/btrdb/transformers.py +++ b/btrdb/transformers.py @@ -94,6 +94,11 @@ def to_series(streamset, datetime64_index=True, agg="mean", name_callable=None): Specify a callable that can be used to determine the series name given a Stream object. + + .. note:: + + This method does **not** use the ``arrow`` -accelerated endpoints for faster and more efficient data retrieval. + """ if pd is None: raise ImportError(_IMPORT_ERR_MSG.format("pandas")) @@ -124,7 +129,7 @@ def to_series(streamset, datetime64_index=True, agg="mean", name_callable=None): return result -def arrow_to_series(streamset, agg="mean", name_callable=None): +def arrow_to_series(streamset, agg=None, name_callable=None): """ Returns a list of Pandas Series objects indexed by time @@ -139,9 +144,82 @@ def arrow_to_series(streamset, agg="mean", name_callable=None): Specify a callable that can be used to determine the series name given a Stream object. - Notes - ----- - This method is available for commercial customers with arrow-enabled servers. + + Returns + ------- + List[pandas.Series] + + + .. note:: + + This method is available for commercial customers with arrow-enabled servers. + + + .. note:: + + If you are not performing a ``window`` or ``aligned_window`` query, the ``agg`` parameter will be ignored. + + Examples + -------- + Return a list of series of raw data per stream. + + >>> conn = btrdb.connect() + >>> s1 = conn.stream_from_uuid('c9fd8735-5ec5-4141-9a51-d23e1b2dfa42') + >>> s2 = conn.stream_from_uuid('9173fa70-87ab-4ac8-ac08-4fd63b910cae' + >>> streamset = btrdb.stream.StreamSet([s1,s2]) + >>> streamset.filter(start=1500000000000000000, end=1500000000900000001).arrow_to_series(agg=None) + [time + 2017-07-14 02:40:00+00:00 1.0 + 2017-07-14 02:40:00.100000+00:00 2.0 + 2017-07-14 02:40:00.200000+00:00 3.0 + 2017-07-14 02:40:00.300000+00:00 4.0 + 2017-07-14 02:40:00.400000+00:00 5.0 + 2017-07-14 02:40:00.500000+00:00 6.0 + 2017-07-14 02:40:00.600000+00:00 7.0 + 2017-07-14 02:40:00.700000+00:00 8.0 + 2017-07-14 02:40:00.800000+00:00 9.0 + 2017-07-14 02:40:00.900000+00:00 10.0 + Name: new/stream/collection/foo, dtype: double[pyarrow], + time + 2017-07-14 02:40:00+00:00 1.0 + 2017-07-14 02:40:00.100000+00:00 2.0 + 2017-07-14 02:40:00.200000+00:00 3.0 + 2017-07-14 02:40:00.300000+00:00 4.0 + 2017-07-14 02:40:00.400000+00:00 5.0 + 2017-07-14 02:40:00.500000+00:00 6.0 + 2017-07-14 02:40:00.600000+00:00 7.0 + 2017-07-14 02:40:00.700000+00:00 8.0 + 2017-07-14 02:40:00.800000+00:00 9.0 + 2017-07-14 02:40:00.900000+00:00 10.0 + Name: new/stream/bar, dtype: double[pyarrow]] + + + A window query of 0.5seconds long. + + >>> streamset.filter(start=1500000000000000000, end=1500000000900000001) + ... .windows(width=int(0.5 * 10**9)) + ... .arrow_to_series(agg=["mean", "count"]) + [time + 2017-07-14 02:40:00+00:00 2.5 + 2017-07-14 02:40:00.400000+00:00 6.5 + Name: new/stream/collection/foo/mean, dtype: double[pyarrow], + ... + time + 2017-07-14 02:40:00+00:00 4 + 2017-07-14 02:40:00.400000+00:00 4 + Name: new/stream/collection/foo/count, dtype: uint64[pyarrow], + ... + time + 2017-07-14 02:40:00+00:00 2.5 + 2017-07-14 02:40:00.400000+00:00 6.5 + Name: new/stream/bar/mean, dtype: double[pyarrow], + ... + time + 2017-07-14 02:40:00+00:00 4 + 2017-07-14 02:40:00.400000+00:00 4 + Name: new/stream/bar/count, dtype: uint64[pyarrow]] + + """ if not streamset._btrdb._ARROW_ENABLED: raise NotImplementedError( @@ -177,9 +255,74 @@ def arrow_to_dataframe(streamset, agg=None, name_callable=None) -> pd.DataFrame: Specify a callable that can be used to determine the series name given a Stream object. - Notes - ----- - This method is available for commercial customers with arrow-enabled servers. + + .. note:: + + This method is available for commercial customers with arrow-enabled servers. + + + Examples + -------- + + >>> conn = btrdb.connect() + >>> s1 = conn.stream_from_uuid('c9fd8735-5ec5-4141-9a51-d23e1b2dfa42') + >>> s2 = conn.stream_from_uuid('9173fa70-87ab-4ac8-ac08-4fd63b910cae' + >>> streamset = btrdb.stream.StreamSet([s1,s2]) + >>> streamset.filter(start=1500000000000000000, end=1500000000900000001).arrow_to_dataframe() + new/stream/collection/foo new/stream/bar + time + 2017-07-14 02:40:00+00:00 1.0 1.0 + 2017-07-14 02:40:00.100000+00:00 2.0 2.0 + 2017-07-14 02:40:00.200000+00:00 3.0 3.0 + 2017-07-14 02:40:00.300000+00:00 4.0 4.0 + 2017-07-14 02:40:00.400000+00:00 5.0 5.0 + 2017-07-14 02:40:00.500000+00:00 6.0 6.0 + 2017-07-14 02:40:00.600000+00:00 7.0 7.0 + 2017-07-14 02:40:00.700000+00:00 8.0 8.0 + 2017-07-14 02:40:00.800000+00:00 9.0 9.0 + 2017-07-14 02:40:00.900000+00:00 10.0 10.0 + + + Use the stream uuids as their column names instead, using a lambda function. + + >>> streamset.filter(start=1500000000000000000, end=1500000000900000001) + ... .arrow_to_dataframe( + ... name_callable=lambda s: str(s.uuid) + ... ) + c9fd8735-5ec5-4141-9a51-d23e1b2dfa42 9173fa70-87ab-4ac8-ac08-4fd63b910cae + time + 2017-07-14 02:40:00+00:00 1.0 1.0 + 2017-07-14 02:40:00.100000+00:00 2.0 2.0 + 2017-07-14 02:40:00.200000+00:00 3.0 3.0 + 2017-07-14 02:40:00.300000+00:00 4.0 4.0 + 2017-07-14 02:40:00.400000+00:00 5.0 5.0 + 2017-07-14 02:40:00.500000+00:00 6.0 6.0 + 2017-07-14 02:40:00.600000+00:00 7.0 7.0 + 2017-07-14 02:40:00.700000+00:00 8.0 8.0 + 2017-07-14 02:40:00.800000+00:00 9.0 9.0 + 2017-07-14 02:40:00.900000+00:00 10.0 10.0 + + + A window query, with a window width of 0.4 seconds, and only showing the ``mean`` statpoint. + + >>> streamset.filter(start=1500000000000000000, end=1500000000900000001) + ... .windows(width=int(0.4*10**9)) + ... .arrow_to_dataframe(agg=["mean"]) + new/stream/collection/foo/mean new/stream/bar/mean + time + 2017-07-14 02:40:00+00:00 2.5 2.5 + 2017-07-14 02:40:00.400000+00:00 6.5 6.5 + + + A window query, with a window width of 0.4 seconds, and only showing the ``mean`` and ``count`` statpoints. + + >>> streamset.filter(start=1500000000000000000, end=1500000000900000001) + ... .windows(width=int(0.4*10**9)) + ... .arrow_to_dataframe(agg=["mean", "count"]) + new/stream/collection/foo/mean new/stream/collection/foo/count new/stream/bar/mean new/stream/bar/count + time + 2017-07-14 02:40:00+00:00 2.5 4 2.5 4 + 2017-07-14 02:40:00.400000+00:00 6.5 4 6.5 4 """ def _rename(col_name, col_names_map): @@ -263,6 +406,10 @@ def to_dataframe(streamset, agg="mean", name_callable=None): Stream object. This is not compatible with agg == "all" at this time + .. note:: + + This method does **not** use the ``arrow`` -accelerated endpoints for faster and more efficient data retrieval. + """ if pd is None: raise ImportError(_IMPORT_ERR_MSG.format("pandas")) @@ -316,9 +463,11 @@ def arrow_to_polars(streamset, agg=None, name_callable=None): Specify a callable that can be used to determine the series name given a Stream object. - Notes - ----- - This method is available for commercial customers with arrow-enabled servers. + + .. note:: + + This method is available for commercial customers with arrow-enabled servers. + """ if not streamset._btrdb._ARROW_ENABLED: raise NotImplementedError( @@ -343,9 +492,11 @@ def arrow_to_polars(streamset, agg=None, name_callable=None): def arrow_to_arrow_table(streamset): """Return a pyarrow table of data. - Notes - ----- - This method is available for commercial customers with arrow-enabled servers. + + .. note:: + + This method is available for commercial customers with arrow-enabled servers. + """ if not streamset._btrdb._ARROW_ENABLED: raise NotImplementedError( @@ -371,6 +522,13 @@ def to_polars(streamset, agg="mean", name_callable=None): name_callable : lambda, default: lambda s: s.collection + "/" + s.name Specify a callable that can be used to determine the series name given a Stream object. This is not compatible with agg == "all" at this time + + + + .. note:: + + This method does **not** use the ``arrow`` -accelerated endpoints for faster and more efficient data retrieval. + """ if pl is None or pd is None: raise ImportError(_IMPORT_ERR_MSG.format(",".join(["polars", "pandas"]))) @@ -425,6 +583,11 @@ def to_array(streamset, agg="mean"): arrays. Must be one of "min", "mean", "max", "count", or "stddev". This argument is ignored if RawPoint values are passed into the function. + + .. note:: + + This method does **not** use the ``arrow`` -accelerated endpoints for faster and more efficient data retrieval. + """ if np is None: raise ImportError(_IMPORT_ERR_MSG.format("numpy")) @@ -455,11 +618,16 @@ def arrow_to_numpy(streamset, agg=None): arrays. Must be one or more of "min", "mean", "max", "count", or "stddev". This argument is ignored if RawPoint values are passed into the function. - Notes - ----- - This method first converts to a pandas data frame then to a numpy array. - This method is available for commercial customers with arrow-enabled servers. + .. note:: + + This method first converts to a pandas data frame then to a numpy array. + + + .. note:: + + This method is available for commercial customers with arrow-enabled servers. + """ if not streamset._btrdb._ARROW_ENABLED: raise NotImplementedError( @@ -489,6 +657,11 @@ def to_dict(streamset, agg="mean", name_callable=None): Specify a callable that can be used to determine the series name given a Stream object. + + .. note:: + + This method does **not** use the ``arrow`` -accelerated endpoints for faster and more efficient data retrieval. + """ if not callable(name_callable): name_callable = lambda s: s.collection + "/" + s.name @@ -533,9 +706,11 @@ def arrow_to_dict(streamset, agg=None, name_callable=None): Specify a callable that can be used to determine the series name given a Stream object. - Notes - ----- - This method is available for commercial customers with arrow-enabled servers. + + .. note:: + + This method is available for commercial customers with arrow-enabled servers. + """ if not streamset._btrdb._ARROW_ENABLED: raise NotImplementedError( @@ -582,6 +757,12 @@ def to_csv( name_callable : lambda, default: lambda s: s.collection + "/" + s.name Specify a callable that can be used to determine the series name given a Stream object. + + + .. note:: + + This method does **not** use the ``arrow`` -accelerated endpoints for faster and more efficient data retrieval. + """ # TODO: allow this at some future point @@ -631,6 +812,11 @@ def to_table(streamset, agg="mean", name_callable=None): Specify a callable that can be used to determine the column name given a Stream object. + + .. note:: + + This method does **not** use the ``arrow`` -accelerated endpoints for faster and more efficient data retrieval. + """ try: from tabulate import tabulate diff --git a/btrdb/utils/timez.py b/btrdb/utils/timez.py index 32cec9e..3c0428a 100644 --- a/btrdb/utils/timez.py +++ b/btrdb/utils/timez.py @@ -55,7 +55,7 @@ def currently_as_ns(): def ns_to_datetime(ns): """ - Converts nanoseconds to a datetime object (UTC) + Converts nanoseconds to a naive datetime object (UTC+0) Parameters ---------- @@ -178,7 +178,7 @@ def ns_delta( days=0, hours=0, minutes=0, seconds=0, milliseconds=0, microseconds=0, nanoseconds=0 ): """ - Similar to `timedelta`, ns_delta represents a span of time but as + Similar to ``timedelta``, ``ns_delta`` represents a span of time but as the total number of nanoseconds. Parameters @@ -202,6 +202,14 @@ def ns_delta( ------- amount of time in nanoseconds : int + + Examples + -------- + 1 minute time delta should be 60 billion nanoseconds + + >>> deltaT = ns_delta(minutes=1) + >>> deltaT == 1 * 60 * 10**9 # 1 minute * 60 seconds * 1billion nanoseconds / second + True """ MICROSECOND = 1000 MILLISECOND = MICROSECOND * 1000 diff --git a/docs/requirements.txt b/docs/requirements.txt index 49e71d8..2e6e1f1 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,8 @@ alabaster>=0.7.12 Sphinx>=1.7 +ipython sphinx-rtd-theme +sphinx-copybutton +sphinx-design numpydoc pydata-sphinx-theme diff --git a/docs/source/api/streams.rst b/docs/source/api/streams.rst index 7137d36..7707bbc 100644 --- a/docs/source/api/streams.rst +++ b/docs/source/api/streams.rst @@ -1,6 +1,7 @@ btrdb.stream ============== + .. _StreamGeneralDocs: .. automodule:: btrdb.stream @@ -8,5 +9,7 @@ btrdb.stream :members: .. _StreamSet API: -.. autoclass:: StreamSetBase +.. autoclass:: StreamSet + :show-inheritance: + :inherited-members: :members: diff --git a/docs/source/concepts.rst b/docs/source/concepts.rst index 4685556..2392034 100644 --- a/docs/source/concepts.rst +++ b/docs/source/concepts.rst @@ -11,7 +11,7 @@ of their behavior which will allow you to use them effectively. .. note:: - Data requests are fully materialized at this time. A future release will include the option to process data using generators to save on memory usage. + Data requests are fully materialized at this time. A future release will include the option to process data using generators to save on memory usage. BTrDB Server @@ -38,48 +38,46 @@ value within the stream. .. code-block:: python - # view time and value of a single point in the stream + >>> # view time and value of a single point in the stream + >>> point.time + 1547241923338098176 - point.time - >>> 1547241923338098176 - - point.value - >>> 120.5 + >>> point.value + 120.5 StatPoint ^^^^^^^^^^^^ The StatPoint provides statistics about multiple points and gives -aggregation values such as `min`, `max`, `mean`, and `stddev` (standard deviation). +aggregation values such as :code:`min`, :code:`max`, :code:`mean`, :code:`count` and :code:`stddev` (standard deviation). This is most useful when you don't need to touch every individual value such as when you only need the count of the values over a range of time. These statistical queries execute in time proportional to the number of results, not the number of underlying points (i.e logarithmic time) and so you can attain valuable data in a fraction of the time when compared with retrieving -all of the individual values. Due to the internal data structures, BTrDB does +all of the individual values. Due to the internal data structures, :code:`BTrDB` does not need to read the underlying points to return these statistics! .. code-block:: python - # view aggregate values for points in a stream - - point.time - >>> 1547241923338098176 + >>> # view aggregate values for points in a stream + >>> point.time + 1547241923338098176 - point.min - >>> 42.1 + >>>point.min + 42.1 - point.mean - >>> 78.477 + >>> point.mean + 78.477 - point.max - >>> 122.4 + >>> point.max + 122.4 - point.count - >>> 18600 + >>> point.count + 18600 - point.stddev - >>> 3.4 + >>> point.stddev + 3.4 Tabular Data @@ -91,52 +89,55 @@ Refer to the :ref:`arrow enabled queries page ` and the :ref:`API do Streams ------------ -Streams represent a single series of time/value pairs. As such, the database +:code:`Stream` s represent a single series of time/value pairs. As such, the database can hold an almost unlimited amount of individual streams. Each stream has a :code:`collection` which is similar to a "path" or grouping for multiple streams. Each steam will also have a :code:`name` as well as a :code:`uuid` which is guaranteed to be unique across streams. -BTrDB data is versioned such that changes to a given stream (time series) will +:code:`BTrDB` data is versioned such that changes to a given stream (time series) will result in a new version for the stream. In this manner, you can pin your interactions to a specific version ensuring the values do not change over the course of your -interactions. If you want to work with the most recent version/data then -specify a version of zero (the default). +interactions. + +.. note:: + + If you want to work with the most recent version/data then specify a version of :code:`0` (the default). Each stream has a number of attributes and methods available and these are documented -within the API section of this publication. But the most common interactions -by users are to access the UUID, tags, annotations, version, and underlying data. +within the :ref:`API Reference ` section of this publication. But the most common interactions +by users are to access the :code:`UUID`, :code:`tags`, :code:`annotations`, :code:`version`, and underlying data. -Each stream uses a UUID as its unique identifier which can also be used when querying -for streams. Metadata is provided by tags and annotations which are both provided -as dictionaries of data. Tags are used internally and have very specific keys -while annotations are more free-form and can be used by you to store your own +Each stream uses a :code:`UUID` as its unique identifier which can also be used when querying +for streams. Metadata is provided by :code:`tags` and :code:`annotations` which are both provided +as dictionaries of data. :code:`tags` are used internally and have very specific keys +while :code:`annotations` are more free-form and can be used by you to store your own metadata. .. code-block:: python - # retrieve stream's UUID - stream.uuid - >>> UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + >>> # retrieve stream's UUID + >>> stream.uuid + UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") - # retrieve stream's current version - stream.version() - >>> 244 + >>> # retrieve stream's current version + >>> stream.version() + 244 - # retrieve stream tags - stream.tags() - >>> {'name': 'L1MAG', 'unit': 'volts', 'ingress': ''} + >>> # retrieve stream tags + >>> stream.tags() + {'name': 'L1MAG', 'unit': 'volts', 'ingress': ''} - # retrieve stream annotations - stream.annotations() - >>> {'poc': 'Salvatore McFesterson', 'region': 'northwest', 'state': 'WA'} + >>> # retrieve stream annotations + >>> stream.annotations() + ({'poc': 'Salvatore McFesterson', 'region': 'northwest', 'state': 'WA'}, 23) - # loop through points in the stream - for point, _ in stream.values(end=1547241923338098176, version=133): - print(point) - >>> RawPoint(1500000000100000000, 2.4) - >>> RawPoint(1500000000200000000, 2.8) - >>> RawPoint(1500000000300000000, 3.6) + >>> # loop through points in the stream + >>> for point, _ in stream.values(end=1547241923338098176, version=133): + >>> print(point) + RawPoint(1500000000100000000, 2.4) + RawPoint(1500000000200000000, 2.8) + RawPoint(1500000000300000000, 3.6) ... @@ -144,8 +145,8 @@ StreamSets ------------ Often you will want to query and work with multiple streams instead of just an -individual stream - StreamSets allow you to do this effectively. It is a light -wrapper around a list of Stream objects with convenience methods provided to +individual stream - :code:`StreamSets` allow you to do this effectively. It is a light +wrapper around a list of :code:`Stream` objects with convenience methods provided to help you work with multiple streams of data. As an example, you can filter the stream data with a single method call and then @@ -153,17 +154,45 @@ easily transform the data into other data types such as a pandas DataFrame or to disk as a CSV file. See the examples below for a quick sample and then visit our API docs to see the full list of features provided to you. +.. note:: + + :code:`StreamSet` methods that filter and operate on the :code:`StreamSet` object (like :code:`StreamSet.filter` ) return new copies of the :code:`StreamSet` itself rather than modifying in place. Similar to how most :code:`pandas.DataFrame` methods return a new :code:`DataFrame` object. This lets you compose multiple functions in a single call, which can improve readability, but can be tricky if you are not expecting this behavior. + +Lets explore a common use-case, filtering a streamset. + +.. code-block:: python + + >>> # create a streamset and apply a few filters + >>> streamset = btrdb.stream.StreamSet(list_of_streams) + >>> print(f"Total streams: {len(streamset)}") + Total streams: 89 + + >>> streamset.filter(units="Volts") + >>> print(f"Total streams: {len(streamset)}") + Total streams: 89 + + >>> filtered_streamset = streamset.filter(units="Volts") + >>> print(f"Total streams: {len(filtered_streamset)}") + Total streams: 23 + + >>> multiple_filters_streamset = (streamset.filter(unit="Volts") + >>> .filter(name="Sensor 1") + >>> .filter(annotations={"phase":"A"}) + >>> ) + >>> print(f"Total streams: {len(multiple_filters_streamset)}") + Total streams: 1 + .. code-block:: python - # establish database connection and query for streams by UUID - db = connect() - uuid_list = ["0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a", ...] - streams = db.streams(*uuid_list) + >>> # establish database connection and query for streams by UUID + >>> db = connect() + >>> uuid_list = ["0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a", ...] + >>> streams = db.streams(*uuid_list) - streams.filter(start=1500000000000000000).to_csv("data.csv") + >>> streams.filter(start=1500000000000000000).to_csv("data.csv") - streams.filter(start=1500000000000000000).to_dataframe() - >> time NW/stream0 NW/stream1 + >>> streams.filter(start=1500000000000000000).to_dataframe() + time NW/stream0 NW/stream1 0 1500000000000000000 NaN 1.0 1 1500000000100000000 2.0 NaN 2 1500000000200000000 NaN 3.0 diff --git a/docs/source/conf.py b/docs/source/conf.py index 1cc1ebf..5db17d4 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -27,7 +27,7 @@ # -- Project information ----------------------------------------------------- project = "btrdb" -copyright = "2023, Ping Things, Inc." +copyright = "2024, Ping Things, Inc." author = "PingThingsIO" # The short X.Y version @@ -48,10 +48,18 @@ extensions = [ "sphinx.ext.autodoc", "sphinx.ext.napoleon", + "sphinx.ext.autosummary", + # 'sphinx.ext.inheritance_diagram', + # 'sphinx.ext.intersphinx', + "sphinx.ext.ifconfig", + # 'IPython.sphinxext.ipython_console_highlighting', + # 'IPython.sphinxext.ipython_directive', "sphinx.ext.todo", "sphinx.ext.githubpages", - "sphinx.ext.intersphinx", + # "sphinx.ext.intersphinx", "numpydoc", + "sphinx_copybutton", + "sphinx_design", ] # Add any paths that contain templates here, relative to this directory. @@ -96,16 +104,19 @@ # documentation. # html_theme_options = { - "show_powered_by": False, - "github_user": "PingThingsIO", - "github_repo": "btrdb-python", - "travis_button": False, - "github_banner": False, - "show_related": False, - "note_bg": "#FFF59C", - "description": "A midweight library to converse with the BTrDB database.", - "extra_nav_links": {"btrdb": "http://btrdb-python.readthedocs.io"}, - "show_relbars": True, + # "show_powered_by": False, + # "github_user": "PingThingsIO", + # "github_repo": "btrdb-python", + # "travis_button": False, + # "github_banner": False, + # "show_related": False, + # "note_bg": "#FFF59C", + # "description": "A midweight library to converse with the BTrDB database.", + # "extra_nav_links": {"btrdb": "http://btrdb-python.readthedocs.io"}, + # "show_relbars": True, + "show_toc_level": 2, + "navigation_depth": 4, # Adjust the depth of the sidebar TOC + "show_nav_level": 2, # Initially shown levels of the TOC } # Add any paths that contain custom static files (such as style sheets) here, diff --git a/docs/source/explained.rst b/docs/source/explained.rst index 68bdfe7..d40eea7 100644 --- a/docs/source/explained.rst +++ b/docs/source/explained.rst @@ -5,7 +5,7 @@ BTrDB Explained **A next-gen timeseries database for dense, streaming telemetry.** -**Problem**: Existing timeseries databases are poorly equipped for a new generation of ultra-fafst sensor telemetry. Specifically, millions of high-precision power meters are to be deployed through the power grid to help analyze and prevent blackouts. Thus, new software must be built to facilitate the storage and analysis of its data. +**Problem**: Existing timeseries databases are poorly equipped for a new generation of ultra-fast sensor telemetry. Specifically, millions of high-precision power meters are to be deployed through the power grid to help analyze and prevent blackouts. Thus, new software must be built to facilitate the storage and analysis of its data. **Baseline**: We need 1.4M inserts/second and 5x that in reads if we are to support 1000 `micro-synchrophasors`_ per server node. No timeseries database can do this. diff --git a/docs/source/index.rst b/docs/source/index.rst index 3f1076b..5320d6f 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -16,15 +16,6 @@ Welcome to btrdb docs! .. image:: https://img.shields.io/pypi/v/btrdb.svg :target: https://pypi.python.org/project/btrdb/ -.. note:: - - Starting with the 5.0 release, btrdb-python will be Python 3 only! This decision - was not made lightly but is necessary to keep compatibility with underlying - packages. - - In addition, this software is only compatible with version 5.x of the BTrDB - server. To communicate with a 4.x server, please install an earlier version - of this software. Welcome to btrdb-python's documentation. We provide Python access to the Berkeley Tree Database (BTrBD) along with some select convenience methods. If you are @@ -59,6 +50,9 @@ your appetite. for point, _ in stream.values(start, end): print(point.time, point.value) + # return the data as an arrow table instead + data = stream.arrow_values(start, end) + User Guide ---------- diff --git a/docs/source/installing.rst b/docs/source/installing.rst index 71cab8e..99a873b 100644 --- a/docs/source/installing.rst +++ b/docs/source/installing.rst @@ -1,13 +1,13 @@ Installing ======================== -The btrdb package has only a few requirements and is relatively easy to install. +The :code:`btrdb` package has only a few requirements and is relatively easy to install. A number of installation options are available as detailed below. Installing with pip ------------------- -We recommend using pip to install btrdb-python on all platforms: +We recommend using :code:`pip` to install :code:`btrdb-python` on all platforms: .. code-block:: bash @@ -24,7 +24,7 @@ We recommend installing the :code:`data` extra dependencies (the second option i $ pip install "btrdb[all]>=5.30.2" # btrdb with testing, data science and all other optional packages -To get a specific version of btrdb-python supply the version number. The major +To get a specific version of :code:`btrdb-python` supply the version number. The major version of this library is tied to the major version of the BTrDB database as in the 4.X bindings are best used to speak to a 4.X BTrDB database, the 5.X bindings for 5.X platform.. @@ -43,4 +43,4 @@ To upgrade using pip: Installing with Anaconda ------------------------ -We recommend installing using ``pip``. +We recommend installing using :code:`pip`. diff --git a/docs/source/quick-start.rst b/docs/source/quick-start.rst index 8901177..7e70ce8 100644 --- a/docs/source/quick-start.rst +++ b/docs/source/quick-start.rst @@ -9,22 +9,22 @@ Connecting to a server is easy with the supplied :code:`connect` function from t .. code-block:: python - import btrdb - - # connect without credentials - conn = btrdb.connect("192.168.1.101:4410") - - # connect using TLS - conn = btrdb.connect("192.168.1.101:4411") - - # connect with API key - conn = btrdb.connect("192.168.1.101:4411", apikey="123456789123456789") + >>> import btrdb + >>> # connect with API key + >>> conn = btrdb.connect("192.168.1.101:4411", apikey="123456789123456789") + >>> conn + Get Platform Information ^^^^^^^^^^^^^^^^^^^^^^^^ .. code-block:: python - conn.info() + >>> conn.info() + {'majorVersion': ..., + 'minorVersion': ..., + 'build': ..., + 'proxy': {...}} + Refer to :ref:`the connection API documentation page. ` @@ -32,8 +32,8 @@ Refer to :ref:`the connection API documentation page. ` Retrieving a Stream ---------------------- -In order to interact with data, you'll need to obtain or create a :code:`Stream` object. A -number of options are available to get existing streams. +In order to interact with data, you'll need to obtain or create a :code:`Stream` object. +A number of options are available to get existing streams. Find streams by collection ^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -43,9 +43,9 @@ collection you can use the :code:`streams_in_collection` method. .. code-block:: python - streams = conn.streams_in_collection("USEAST_NOC1/90807") - for stream in streams: - print(stream.uuid, stream.name) + >>> streams = conn.streams_in_collection("USEAST_NOC1/90807") + >>> for stream in streams: + >>> print(stream.uuid, stream.name) Find stream by UUID ^^^^^^^^^^^^^^^^^^^^^ @@ -55,7 +55,7 @@ would like to retrieve. For convenience, this method accepts instances of either .. code-block:: python - stream = conn.stream_from_uuid("07d28a44-4991-492d-b9c5-2d8cec5aa6d4") + >>> stream = conn.stream_from_uuid("07d28a44-4991-492d-b9c5-2d8cec5aa6d4") @@ -69,42 +69,65 @@ if needed. .. code-block:: python - start = datetime(2018,1,1,12,30, tzinfo=timezone.utc) - start = start.timestamp() * 1e9 - end = start + (3600 * 1e9) + >>> start = datetime(2018,1,1,12,30, tzinfo=timezone.utc) + >>> start = start.timestamp() * 1e9 + >>> end = start + (3600 * 1e9) - for point, _ in stream.values(start, end): - print(point.time, point.value) + >>> for point, _ in stream.values(start, end): + >>> print(point.time, point.value) Some convenience functions are available to make it easier to deal with converting to nanoseconds. .. code-block:: python - from btrdb.utils.timez import to_nanoseconds, currently_as_ns + >>> from btrdb.utils.timez import to_nanoseconds, currently_as_ns - start = to_nanoseconds(datetime(2018,1,1, tzinfo=timezone.utc)) - end = currently_as_ns() + >>> start = to_nanoseconds(datetime(2018,1,1, tzinfo=timezone.utc)) + >>> end = currently_as_ns() - for point, _ in stream.values(start, end): - print(point.time, point.value) + >>> for point, _ in stream.values(start, end): + >>> print(point.time, point.value) You can also view windows of data at arbitrary levels of detail. One such windowing feature is shown below. .. code-block:: python - # query for windows of data 10,000 nanoseconds wide using a depth of zero - # which is accurate to the nanosecond. - params = { - "start": 1500000000000000000, - "end": 1500000000010000000, - "width": 2000000, - "depth": 0, - } - for window in stream.windows(**params): - for point, version in window: - print(point, version) + >>> # query for windows of data 10,000 nanoseconds wide using a depth of zero + >>> # which is accurate to the nanosecond. + >>> params = { + ... "start": 1500000000000000000, + ... "end": 1500000000010000000, + ... "width": 2000000, + ... "depth": 0, + ... } + >>> for window in stream.windows(**params): + >>> for point, version in window: + >>> print(point, version) + + +Return data as :code:`arrow` tables +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Instead of returning data a :code:`RawPoint` at a time, which can be more computationally intensive, there is now the ability to return the data in a tabular format from the start, which can drastically save on run time as well as facilitate interoperability with many more data-science driven tools. +`Apache Arrow is a language agnostic columnar data schema `_ that has become a defacto standard for in-memory data analytics. +All data retrieval methods in :code:`BTrDB` now have corresponding :code:`arrow-` prepended methods that natively return :code:`pyarrow` data tables. + +.. code-block:: python + + >>> s.arrow_values(start=1500000000000000000, end=1500000002000000001).to_pandas() + time value + 0 2017-07-14 02:40:00+00:00 1.0 + 1 2017-07-14 02:40:00.100000+00:00 2.0 + 2 2017-07-14 02:40:00.200000+00:00 3.0 + 3 2017-07-14 02:40:00.300000+00:00 4.0 + 4 2017-07-14 02:40:00.400000+00:00 5.0 + 5 2017-07-14 02:40:00.500000+00:00 6.0 + 6 2017-07-14 02:40:00.600000+00:00 7.0 + 7 2017-07-14 02:40:00.700000+00:00 8.0 + 8 2017-07-14 02:40:00.800000+00:00 9.0 + 9 2017-07-14 02:40:00.900000+00:00 10.0 + Using StreamSets -------------------- @@ -118,33 +141,45 @@ level of the individual :code:`Stream` object. Aside from being useful to see concurrent data across streams, you can also easily transform the data to other data structures or even serialize the data to disk in one operation. -Some quick examples are shown below but please review the API docs for the full +Some quick examples are shown below but please review the :ref:`API docs ` for the full list of features. + +.. note:: + + In the following examples, notice that the end time is **not** inclusive of the data that is present at :code:`end` . :code:`start` is **inclusive** while :code:`end` is **exclusive**. This is the case for **all** :code:`BTrDB` data query operations. + + .. math:: + [start, end) + + .. code-block:: python - streams = db.streams(*uuid_list) - - # serialize data to disk as CSV - streams.filter(start=1500000000000000000).to_csv("data.csv") - - # convert data to a pandas DataFrame - streams.filter(start=1500000000000000000).to_dataframe() - >> time NW/stream0 NW/stream1 - 0 1500000000000000000 NaN 1.0 - 1 1500000000100000000 2.0 NaN - 2 1500000000200000000 NaN 3.0 - 3 1500000000300000000 4.0 NaN - 4 1500000000400000000 NaN 5.0 - 5 1500000000500000000 6.0 NaN - 6 1500000000600000000 NaN 7.0 - 7 1500000000700000000 8.0 NaN - 8 1500000000800000000 NaN 9.0 - 9 1500000000900000000 10.0 NaN - - # materialize the streams' data - streams.filter(start=1500000000000000000).values() - >> [[RawPoint(1500000000100000000, 2.0), + >>> streams = db.streams(*uuid_list) + + >>> # serialize data to disk as CSV + >>> streams.filter(start=1500000000000000000, end=1500000000900000000).to_csv("data.csv") + + >>> # convert data to a pandas DataFrame + >>> streams.filter(start=1500000000000000000, end=1500000000900000000).to_dataframe() + nw/stream0 nw/stream1 + time + 1500000000000000000 nan 1.0 + 1500000000100000000 2.0 nan + 1500000000200000000 nan 3.0 + 1500000000300000000 4.0 nan + 1500000000400000000 nan 5.0 + 1500000000500000000 6.0 nan + 1500000000600000000 nan 7.0 + 1500000000700000000 8.0 nan + 1500000000800000000 nan 9.0 + + + + + >>> # materialize the streams' data + >>> streams.filter(start=1500000000000000000, end=1500000000900000000).values() + [[RawPoint(1500000000100000000, 2.0), RawPoint(1500000000300000000, 4.0), RawPoint(1500000000500000000, 6.0), RawPoint(1500000000700000000, 8.0), @@ -153,3 +188,48 @@ list of features. RawPoint(1500000000200000000, 3.0), RawPoint(1500000000400000000, 5.0), ... + + + + + + +Return data as :code:`arrow` tables +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:code:`StreamSets` are also able to return :code:`arrow` tables for the group of streams they represent. +This is especially convenient and is usually **much** faster than using the traditional :code:`RawPoint` -based data representation. +We recommend using the :code:`arrow` functions whenever possible. + +.. code-block:: python + + >>> # convert data to a pandas DataFrame, using pyarrow + >>> streams.filter(start=1500000000000000000, end=1500000000900000000) + ... .arrow_to_dataframe() + NW/stream0 NW/stream1 + time + 2017-07-14 02:40:00+00:00 NaN 1.0 + 2017-07-14 02:40:00.100000+00:00 2.0 NaN + 2017-07-14 02:40:00.200000+00:00 NaN 3.0 + 2017-07-14 02:40:00.300000+00:00 4.0 NaN + 2017-07-14 02:40:00.400000+00:00 NaN 5.0 + 2017-07-14 02:40:00.500000+00:00 6.0 NaN + 2017-07-14 02:40:00.600000+00:00 NaN 7.0 + 2017-07-14 02:40:00.700000+00:00 8.0 NaN + 2017-07-14 02:40:00.800000+00:00 NaN 9.0 + + + >>> # materialize the streams' data as an arrow table + >>> streams.filter(start=1500000000000000000, end=1500000000900000000).arrow_values() + pyarrow.Table + time: timestamp[ns, tz=UTC] not null + b29204f4-6c13-4ec7-a149-88e2ff950a72: double not null + 99a0d0b0-e24f-4875-b7d8-eae0036f2149: double not null + ---- + time: [ + ... [2017-07-14 02:40:00.000000000Z,2017-07-14 02:40:00.100000000Z, + ... 2017-07-14 02:40:00.200000000Z,2017-07-14 02:40:00.300000000Z, + ... 2017-07-14 02:40:00.400000000Z,2017-07-14 02:40:00.500000000Z, + ... 2017-07-14 02:40:00.600000000Z,2017-07-14 02:40:00.700000000Z, + ... 2017-07-14 02:40:00.800000000Z]] + b29204f4-6c13-4ec7-a149-88e2ff950a72: [[nan,2,nan,4,nan,6,nan,8,nan]] + 99a0d0b0-e24f-4875-b7d8-eae0036f2149: [[1,nan,3,nan,5,nan,7,nan,9]] diff --git a/docs/source/working/arrow-enabled-queries.rst b/docs/source/working/arrow-enabled-queries.rst index 8a40a5e..fa531c9 100644 --- a/docs/source/working/arrow-enabled-queries.rst +++ b/docs/source/working/arrow-enabled-queries.rst @@ -24,9 +24,9 @@ To learn more about these methods, please refer to the :ref:`arrow_ prefixed met True Multistream Support ^^^^^^^^^^^^^^^^^^^^^^^^ -Until now, there has not been a true multistream query support, our previous api and with the new edits, emulates multistream support with :code:`StreamSet`s and using multithreading. +Until now, there has not been a true multistream query support, our previous api and with the new edits, emulates multistream support with :code:`StreamSet` s and using multithreading. However, this will still only scale to an amount of streams based on the amount of threads that the python threadpool logic can support. -Due to this, raw data queries for :code:`StreamSet`s using our arrow api :code:`StreamSet.filter(start=X, end=Y,).arrow_values()` will now perform true multistream queries. +Due to this, raw data queries for :code:`StreamSet` s using our arrow api :code:`StreamSet.filter(start=X, end=Y,).arrow_values()` will now perform true multistream queries. The platform, instead of the python client, will now quickly grab all the stream data for all streams in your streamset, and then package that back to the python client in an :code:`arrow` table! This leads to data fetch speedups on the order of 10-50x based on the amount and kind of streams. diff --git a/docs/source/working/stream-manage-metadata.rst b/docs/source/working/stream-manage-metadata.rst index 75b7fd6..7d90655 100644 --- a/docs/source/working/stream-manage-metadata.rst +++ b/docs/source/working/stream-manage-metadata.rst @@ -102,7 +102,7 @@ that the data has already been changed by another user or process. annotations=annotations ) -If you would like to remove any keys from your annotations you must use the `replace=True` keyword argument. This will ensure that the annotations dictionary you provide completely replaces the existing values rather than perform an UPSERT operation. The example below shows how you could remove an existing key from the annotations dictionary. +If you would like to remove any keys from your annotations you must use the :code:`replace=True` keyword argument. This will ensure that the annotations dictionary you provide completely replaces the existing values rather than perform an UPSERT operation. The example below shows how you could remove an existing key from the annotations dictionary. .. code-block:: python diff --git a/docs/source/working/stream-query-manage.rst b/docs/source/working/stream-query-manage.rst index d0f83c6..9476381 100644 --- a/docs/source/working/stream-query-manage.rst +++ b/docs/source/working/stream-query-manage.rst @@ -75,15 +75,15 @@ detail. Querying Metadata ----------------- Finally, you can query for metadata using standard SQL although at the moment, only the -`streams` table is available. SQL queries can be submitted using the `query` -method which accepts both a `stmt` and `params` argument. The `stmt` should -contain the SQL you'd like executed with parameter placeholders such as `$1` or -`$2` as shown below. +:code:`streams` table is available. SQL queries can be submitted using the :code:`query` +method which accepts both a :code:`stmt` and :code:`params` argument. The :code:`stmt` should +contain the SQL you'd like executed with parameter placeholders such as :code:`$1` or +:code:`$2` as shown below. .. code-block:: python conn = btrdb.connect() - stmt = "select uuid from streams where name = $1 or name = $2" + stmt = "SELECT uuid FROM streams WHERE name = $1 OR name = $2" params = ["Boston_1", "Boston_2"] for row in conn.query(stmt, params): diff --git a/docs/source/working/stream-view-data.rst b/docs/source/working/stream-view-data.rst index 915e75e..b9c41c7 100644 --- a/docs/source/working/stream-view-data.rst +++ b/docs/source/working/stream-view-data.rst @@ -45,7 +45,7 @@ you query for all the data using the :code:`Stream.values` method due to the mem consumption implied). Each of these three methods returns a tuple containing a RawPoint and the data version number. The exact timestamp can be obtained from the RawPoint. Keep in mind that all of these methods accept a :code:`version` argument so that -you can ask for the earliest, latest, or nearest point from a previous version of the stream. +you can ask for the :code:`earliest`, :code:`latest`, or :code:`nearest` point from a previous version of the stream. .. code-block:: python @@ -86,7 +86,7 @@ aggregate of all the raw data within a window of width 2^pointwidth nanoseconds. Note that :code:`start` is inclusive, but :code:`end` is exclusive. That is, results -will be returned for all windows that start in the interval [start, end). +will be returned for all windows that start in the interval :math:`[start, end)`. If end < start+2^pointwidth you will not get any results. If start and end are not powers of two, the bottom pointwidth bits will be cleared. Each window will contain statistical summaries of the window. Statistical points @@ -150,37 +150,37 @@ of sample depths are provided below. +-------+-------------+---------------------------+-----------------+ As usual when querying data from BTrDB, the :code:`start` time is inclusive -while the :code:`end` time is exclusive. Note that if your last window spans +while the :code:`end` time is exclusive. **Note**: that if your last window spans across the end time then it will not be included in the results. .. code-block:: python - start = 1500000000000000000 - end = 1500000001000000000 - - # view underlying data for comparison - for point, _ in stream.values(start=start, end=end): - print(point) - >> RawPoint(1500000000000000000, 1.0) - >> RawPoint(1500000000100000000, 2.0) - >> RawPoint(1500000000200000000, 3.0) - >> RawPoint(1500000000300000000, 4.0) - >> RawPoint(1500000000400000000, 5.0) - >> RawPoint(1500000000500000000, 6.0) - >> RawPoint(1500000000600000000, 7.0) - >> RawPoint(1500000000700000000, 8.0) - >> RawPoint(1500000000800000000, 9.0) - >> RawPoint(1500000000900000000, 10.0) - - # each window spans 300 milleseconds - width = 300000000 - - # request a precision of roughly 1 millesecond - depth = 20 - - # view windowed data - for point, _ in stream.windows(start=start, end=end, - width=width, depth=depth): - >> StatPoint(1500000000000000000, 1.0, 2.0, 3.0, 3, 0.816496580927726) - >> StatPoint(1500000000300000000, 4.0, 5.0, 6.0, 3, 0.816496580927726) - >> StatPoint(1500000000600000000, 7.0, 8.0, 9.0, 3, 0.816496580927726) + >>> start = 1500000000000000000 + >>> end = 1500000001000000000 + + >>> # view underlying data for comparison + >>> for point, _ in stream.values(start=start, end=end): + >>> print(point) + RawPoint(1500000000000000000, 1.0) + RawPoint(1500000000100000000, 2.0) + RawPoint(1500000000200000000, 3.0) + RawPoint(1500000000300000000, 4.0) + RawPoint(1500000000400000000, 5.0) + RawPoint(1500000000500000000, 6.0) + RawPoint(1500000000600000000, 7.0) + RawPoint(1500000000700000000, 8.0) + RawPoint(1500000000800000000, 9.0) + RawPoint(1500000000900000000, 10.0) + + >>> # each window spans 300 milliseconds + >>> width = 300000000 + + >>> # request a precision of roughly 1 millisecond + >>> depth = 20 + + >>> # view windowed data + >>> for point, _ in stream.windows(start=start, end=end, + ... width=width, depth=depth): + StatPoint(1500000000000000000, 1.0, 2.0, 3.0, 3, 0.816496580927726) + StatPoint(1500000000300000000, 4.0, 5.0, 6.0, 3, 0.816496580927726) + StatPoint(1500000000600000000, 7.0, 8.0, 9.0, 3, 0.816496580927726) diff --git a/docs/source/working/streamsets.rst b/docs/source/working/streamsets.rst index 8a9d807..da6c1d0 100644 --- a/docs/source/working/streamsets.rst +++ b/docs/source/working/streamsets.rst @@ -38,8 +38,8 @@ Filtering To apply query parameters to your request, you should use the :code:`filter` method to supply a :code:`start` or :code:`end` argument. -Keep in mind that :code:`filter` will return a new object so you can keep -multiple filtered StreamSets in memory while you explore your data. The +Keep in mind that :code:`filter` will **return a new object so you can keep +multiple filtered StreamSets in memory while you explore your data**. The :code:`filter` method may be called multiple times but only the final values will be used when it is time to fulfill the request by the server. @@ -83,8 +83,8 @@ re.search to choose the streams to include. Retrieving Data ---------------- -There are two options available when you are ready to process the data from the -server. Both options are fully materialized but are organized in different ways +There are three options available when you are ready to process the data from the +server. All options are fully materialized but are organized in different ways according to what is more convenient for you. StreamSet.values() @@ -131,6 +131,10 @@ consists of 4 streams. >> RawPoint(1500000000900000000, 10.0)]] +.. attention:: + + + StreamSet.rows() ^^^^^^^^^^^^^^^^^^ By contrast, the :code:`rows` method aligns data by time rather than by stream. diff --git a/tests/btrdb/test_base.py b/tests/btrdb/test_base.py index ac55c07..0fddc90 100644 --- a/tests/btrdb/test_base.py +++ b/tests/btrdb/test_base.py @@ -16,11 +16,11 @@ ########################################################################## import os -from unittest.mock import patch +from unittest.mock import MagicMock, Mock, patch import pytest -from btrdb import BTRDB_API_KEY, BTRDB_ENDPOINTS, __version__, connect +from btrdb import BTRDB_API_KEY, BTRDB_ENDPOINTS, Endpoint, __version__, connect from btrdb.exceptions import ConnectionError ########################################################################## @@ -81,21 +81,34 @@ def test_uses_env_over_profile( connect() mock_connect.assert_called_once_with(endpoints="c", apikey="d") + # @patch("btrdb.utils.credentials.credentials_by_env") + @patch("btrdb.utils.credentials.credentials_by_env") @patch("btrdb.utils.credentials.credentials_by_profile") @patch("btrdb.Connection") - def test_connect_with_env(self, mock_conn, mock_credentials_by_profile): + def test_connect_with_env( + self, mock_conn, mock_credentials_by_profile, mock_credentials_by_env + ): """ Assert connect uses ENV variables """ - mock_credentials_by_profile.return_value = {} - address = "127.0.0.1:4410" - os.environ[BTRDB_ENDPOINTS] = address + with patch( + "btrdb.endpoint.Endpoint.info", return_value=Mock(Endpoint) + ) as mock_ep_info: + mock_ep_info.return_value = MagicMock() + mock_credentials_by_profile.return_value = {} + address = "127.0.0.1:4410" + mock_credentials_by_env.return_value = {"endpoints": address} + os.environ[BTRDB_ENDPOINTS] = address - btrdb = connect() - mock_conn.assert_called_once_with(address, apikey=None) - mock_conn.reset_mock() + btrdb = connect() + mock_conn.assert_called_once_with(address, apikey=None) + mock_conn.reset_mock() - apikey = "abcd" - os.environ[BTRDB_API_KEY] = apikey - btrdb = connect() - mock_conn.assert_called_once_with(address, apikey=apikey) + apikey = "abcd" + os.environ[BTRDB_API_KEY] = apikey + mock_credentials_by_env.return_value = { + "endpoints": address, + "apikey": apikey, + } + btrdb = connect() + mock_conn.assert_called_once_with(address, apikey=apikey) diff --git a/tests/btrdb/test_conn.py b/tests/btrdb/test_conn.py index 66034b3..3720ef2 100644 --- a/tests/btrdb/test_conn.py +++ b/tests/btrdb/test_conn.py @@ -16,7 +16,7 @@ ########################################################################## import uuid as uuidlib -from unittest.mock import Mock, PropertyMock, call, patch +from unittest.mock import MagicMock, Mock, PropertyMock, call, patch import pytest @@ -41,7 +41,7 @@ def stream1(): type(stream).name = PropertyMock(return_value="gala") stream.tags = Mock(return_value={"name": "gala", "unit": "volts"}) stream.annotations = Mock(return_value=({"owner": "ABC", "color": "red"}, 11)) - stream._btrdb = Mock() + stream._btrdb = MagicMock() return stream @@ -55,7 +55,7 @@ def stream2(): type(stream).name = PropertyMock(return_value="blood") stream.tags = Mock(return_value={"name": "blood", "unit": "amps"}) stream.annotations = Mock(return_value=({"owner": "ABC", "color": "orange"}, 22)) - stream._btrdb = Mock() + stream._btrdb = MagicMock() return stream @@ -69,7 +69,7 @@ def stream3(): type(stream).name = PropertyMock(return_value="yellow") stream.tags = Mock(return_value={"name": "yellow", "unit": "watts"}) stream.annotations = Mock(return_value=({"owner": "ABC", "color": "yellow"}, 33)) - stream._btrdb = Mock() + stream._btrdb = MagicMock() return stream @@ -105,105 +105,113 @@ def test_streams_raises_err_if_version_not_list(self): """ Assert streams raises TypeError if versions is not list """ - db = BTrDB(None) - with pytest.raises(TypeError) as exc: - db.streams("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a", versions="2,2") + with patch("btrdb.endpoint.Endpoint", return_value=MagicMock()) as ep: + db = BTrDB(ep) + with pytest.raises(TypeError) as exc: + db.streams("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a", versions="2,2") - assert "versions argument must be of type list" in str(exc) + assert "versions argument must be of type list" in str(exc) def test_streams_raises_err_if_version_argument_mismatch(self): """ Assert streams raises ValueError if len(identifiers) doesnt match length of versions """ - db = BTrDB(None) - with pytest.raises(ValueError) as exc: - db.streams("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a", versions=[2, 2]) + with patch("btrdb.endpoint.Endpoint", return_value=MagicMock()) as ep: + db = BTrDB(ep) + with pytest.raises(ValueError) as exc: + db.streams("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a", versions=[2, 2]) - assert "versions does not match identifiers" in str(exc) + assert "versions does not match identifiers" in str(exc) def test_streams_stores_versions(self): """ Assert streams correctly stores supplied version info """ - db = BTrDB(None) - uuid1 = uuidlib.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") - uuid2 = uuidlib.UUID("17dbe387-89ea-42b6-864b-f505cdb483f5") - versions = [22, 44] - expected = dict(zip([uuid1, uuid2], versions)) + with patch("btrdb.endpoint.Endpoint", return_value=MagicMock()) as ep: + db = BTrDB(ep) + uuid1 = uuidlib.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + uuid2 = uuidlib.UUID("17dbe387-89ea-42b6-864b-f505cdb483f5") + versions = [22, 44] + expected = dict(zip([uuid1, uuid2], versions)) - streams = db.streams(uuid1, uuid2, versions=versions) - assert streams._pinned_versions == expected + streams = db.streams(uuid1, uuid2, versions=versions) + assert streams._pinned_versions == expected @patch("btrdb.conn.BTrDB.stream_from_uuid") def test_streams_recognizes_uuid(self, mock_func): """ Assert streams recognizes uuid strings """ - db = BTrDB(None) - uuid1 = uuidlib.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") - mock_func.return_value = Stream(db, uuid1) - db.streams(uuid1) + with patch("btrdb.endpoint.Endpoint", return_value=MagicMock()) as ep: + db = BTrDB(ep) + uuid1 = uuidlib.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + mock_func.return_value = Stream(db, uuid1) + db.streams(uuid1) - mock_func.assert_called_once() - assert mock_func.call_args[0][0] == uuid1 + mock_func.assert_called_once() + assert mock_func.call_args[0][0] == uuid1 @patch("btrdb.conn.BTrDB.stream_from_uuid") def test_streams_recognizes_uuid_string(self, mock_func): """ Assert streams recognizes uuid strings """ - db = BTrDB(None) - uuid1 = "0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a" - mock_func.return_value = Stream(db, uuid1) - db.streams(uuid1) + with patch("btrdb.endpoint.Endpoint", return_value=MagicMock()) as ep: + db = BTrDB(ep) + uuid1 = "0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a" + mock_func.return_value = Stream(db, uuid1) + db.streams(uuid1) - mock_func.assert_called_once() - assert mock_func.call_args[0][0] == uuid1 + mock_func.assert_called_once() + assert mock_func.call_args[0][0] == uuid1 @patch("btrdb.conn.BTrDB.streams_in_collection") def test_streams_handles_path(self, mock_func): """ Assert streams calls streams_in_collection for collection/name paths """ - db = BTrDB(None) - ident = "zoo/animal/dog" - mock_func.return_value = [ - Stream(db, "0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a"), - ] - db.streams(ident, "0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") - - mock_func.assert_called_once() - assert mock_func.call_args[0][0] == "zoo/animal" - assert mock_func.call_args[1] == { - "is_collection_prefix": False, - "tags": {"name": "dog"}, - } + with patch("btrdb.endpoint.Endpoint", return_value=MagicMock()) as ep: + db = BTrDB(ep) + ident = "zoo/animal/dog" + mock_func.return_value = [ + Stream(db, "0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a"), + ] + db.streams(ident, "0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + + mock_func.assert_called_once() + assert mock_func.call_args[0][0] == "zoo/animal" + assert mock_func.call_args[1] == { + "is_collection_prefix": False, + "tags": {"name": "dog"}, + } @patch("btrdb.conn.BTrDB.streams_in_collection") def test_streams_raises_err(self, mock_func): """ Assert streams raises StreamNotFoundError """ - db = BTrDB(None) - ident = "zoo/animal/dog" - - mock_func.return_value = [] - with pytest.raises(StreamNotFoundError) as exc: + with patch("btrdb.endpoint.Endpoint", return_value=MagicMock()) as ep: + db = BTrDB(ep) + ident = "zoo/animal/dog" + + mock_func.return_value = [] + with pytest.raises(StreamNotFoundError) as exc: + db.streams(ident) + + # check that does not raise if one returned + mock_func.return_value = [ + Stream(db, ident), + ] db.streams(ident) - # check that does not raise if one returned - mock_func.return_value = [ - Stream(db, ident), - ] - db.streams(ident) - def test_streams_raises_valueerror(self): """ Assert streams raises ValueError if not uuid, uuid str, or path """ - db = BTrDB(None) - with pytest.raises(ValueError) as exc: - db.streams(11) + with patch("btrdb.endpoint.Endpoint", return_value=MagicMock()) as ep: + db = BTrDB(ep) + with pytest.raises(ValueError) as exc: + db.streams(11) ########################################################################## ## other tests diff --git a/tests/btrdb_integration/test_conn.py b/tests/btrdb_integration/test_conn.py index 5d30c4f..bc6c9ac 100644 --- a/tests/btrdb_integration/test_conn.py +++ b/tests/btrdb_integration/test_conn.py @@ -1,36 +1,57 @@ import logging +import os from uuid import uuid4 as new_uuid import pytest import btrdb - - -def test_connection_info(conn): - info = conn.info() - logging.info(f"connection info: {info}") - - -def test_create_stream(conn, tmp_collection): - uuid = new_uuid() - stream = conn.create(uuid, tmp_collection, tags={"name": "s"}) - assert stream.uuid == uuid - assert stream.name == "s" - - -def test_query(conn, tmp_collection): - conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) - conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) - uuids = conn.query( - "select name from streams where collection = $1 order by name;", - [tmp_collection], +from btrdb.exceptions import BTrDBError + + +@pytest.mark.skipif( + os.getenv("BTRDB_INTEGRATION_TEST_PROFILE") is None, + reason="Need an integration test server to run integration tests.", +) +class TestConnection: + def test_connection_info(self, conn): + info = conn.info() + logging.info(f"connection info: {info}") + + def test_incorrect_connect( + self, + ): + err_msg = r"""Could not connect to the database, error message: <_InactiveRpcError of RPC that terminated with:\n\tstatus = StatusCode.UNAUTHENTICATED\n\tdetails = "invalid api key"\n""" + with pytest.raises(BTrDBError, match=err_msg): + conn = btrdb.connect(conn_str="127.0.0.1:4410", apikey="BOGUS_KEY") + + @pytest.mark.xfail( + reason="Should return BTrDBError, but returns GRPCError instead, FIXME" ) - assert len(uuids) == 2 - assert uuids[0]["name"] == "s1" - assert uuids[1]["name"] == "s2" - - -def test_list_collections(conn, tmp_collection): - assert tmp_collection not in conn.list_collections() - stream = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) - assert tmp_collection in conn.list_collections() + def test_from_uuid(self, conn): + # todo, investigate this, the stat code returned is 0, which is why its not being returned as a btrdb error + uu = new_uuid() + stream = conn.stream_from_uuid(uu) + with pytest.raises(btrdb.exceptions.BTrDBError): + print(stream) + + def test_create_stream(self, conn, tmp_collection): + uuid = new_uuid() + stream = conn.create(uuid, tmp_collection, tags={"name": "s"}) + assert stream.uuid == uuid + assert stream.name == "s" + + def test_query(self, conn, tmp_collection): + conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + uuids = conn.query( + "select name from streams where collection = $1 order by name;", + [tmp_collection], + ) + assert len(uuids) == 2 + assert uuids[0]["name"] == "s1" + assert uuids[1]["name"] == "s2" + + def test_list_collections(self, conn, tmp_collection): + assert tmp_collection not in conn.list_collections() + stream = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) + assert tmp_collection in conn.list_collections() From 6c157d25877bdb2261904cb3adace48d92c27ace Mon Sep 17 00:00:00 2001 From: Justin Gilmer <16173348+justinGilmer@users.noreply.github.com> Date: Mon, 25 Mar 2024 19:39:08 -0400 Subject: [PATCH 103/131] Update changelog for next release. --- CHANGELOG.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c61644d..451cb3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,28 @@ # Changelog +## 5.32.0 - YYYY-MM-DD + +### What's Changed +- Initial docstring overhaul and a new test for better documentation and test coverage. by @JustinGilmer in #82 +- Test new join functionality for improved data manipulation. by @JustinGilmer in #80 +- Improve `arrow_to_dataframe` function for handling large amounts of columns, enhancing performance and usability. by @Jefflinf in #73 +- Expand testing to include Python 3.11, ensuring compatibility and stability. by @JustinGilmer in #74 +- Update exception handling to better support `RpcErrors`, improving error management and debugging. by @JustinGilmer in #72 +- Introduce an option for specifying the schema of the returned data, allowing for more flexibility in data handling. by @TODO in #51 +- Remove non-required dependencies and migrate to 'data' optional dependency for a lighter package and easier installation. by @JustinGilmer in #71 +- New method to get first and last timestamps from `aligned_windows`, enhancing data analysis capabilities. by @Jefflinf in #70 +- Add `to_timedelta` method for `pointwidth` class, providing more options for time-based data manipulation. by @Jefflinf in #69 + +### Fixed +- Fix `NoneType` error for `earliest/latest` for empty streams, ensuring reliability and error handling. by @Jefflinf in #64 +- Correct integration tests where the time column is not automatically set as the index, improving test accuracy and reliability. by @JustinGilmer in #56 + +### Deprecated +- FutureWarning for `streams_in_collection` to return `StreamSet` in the future, preparing users for upcoming API changes. by @Jefflinf in #60 + +**Full Changelog**: [GitHub compare view](https://github.com/PingThingsIO/btrdb-python/compare/v5.31.1...v5.32.0) + + ## 5.31.1 ### What's Changed * Sort tables by time by default for any `pyarrow` tables. by @justinGilmer in From 60511538811c6acae35c162ab33fbcbafe9e1116 Mon Sep 17 00:00:00 2001 From: Justin Gilmer <16173348+justinGilmer@users.noreply.github.com> Date: Tue, 26 Mar 2024 16:12:04 -0400 Subject: [PATCH 104/131] Update github actions dependencies to recent versions. --- .github/workflows/pre-commit.yaml | 6 +++--- .github/workflows/release.yaml | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml index 6567714..98e35c2 100644 --- a/.github/workflows/pre-commit.yaml +++ b/.github/workflows/pre-commit.yaml @@ -18,11 +18,11 @@ jobs: pre-commit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: token: ${{ secrets.GITHUB_TOKEN }} fetch-depth: 0 # get full git history - - uses: actions/setup-python@v3 + - uses: actions/setup-python@v5 with: cache: 'pip' - name: Install pre-commit @@ -30,7 +30,7 @@ jobs: pip install pre-commit - name: Get changed files id: changed-files - uses: tj-actions/changed-files@v21 + uses: tj-actions/changed-files@v43 with: token: ${{ secrets.GITHUB_TOKEN }} - name: Run pre-commit diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 31929c9..15dc856 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -17,9 +17,9 @@ jobs: os: [ubuntu-latest, macos-latest, windows-latest] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} ${{ matrix.os }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -40,7 +40,7 @@ jobs: if: startsWith(github.ref, 'refs/tags/') runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Create Release id: create_release uses: actions/create-release@v1 @@ -59,11 +59,11 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: "3.8" + python-version: "3.10" - name: Install dependencies run: | python -m pip install --upgrade pip From e095afe8c5ff18d3b3ad3ebdd007b8b1c231398e Mon Sep 17 00:00:00 2001 From: Justin Gilmer <16173348+justinGilmer@users.noreply.github.com> Date: Tue, 26 Mar 2024 16:12:35 -0400 Subject: [PATCH 105/131] Further changelog refinements. --- CHANGELOG.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 451cb3d..9d0c080 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ### What's Changed - Initial docstring overhaul and a new test for better documentation and test coverage. by @JustinGilmer in #82 -- Test new join functionality for improved data manipulation. by @JustinGilmer in #80 +- Test new join functionality for improved data loading for windowed queries. by @JustinGilmer in #80 - Improve `arrow_to_dataframe` function for handling large amounts of columns, enhancing performance and usability. by @Jefflinf in #73 - Expand testing to include Python 3.11, ensuring compatibility and stability. by @JustinGilmer in #74 - Update exception handling to better support `RpcErrors`, improving error management and debugging. by @JustinGilmer in #72 @@ -20,16 +20,20 @@ ### Deprecated - FutureWarning for `streams_in_collection` to return `StreamSet` in the future, preparing users for upcoming API changes. by @Jefflinf in #60 -**Full Changelog**: [GitHub compare view](https://github.com/PingThingsIO/btrdb-python/compare/v5.31.1...v5.32.0) +**Full Changelog**: [GitHub compare view](https://github.com/PingThingsIO/btrdb-python/compare/v5.31.0...v5.32.0) -## 5.31.1 -### What's Changed +## 5.31.0 +## What's Changed +* Release v5.30.2 by @youngale-pingthings in https://github.com/PingThingsIO/btrdb-python/pull/42 +* Have release script update pyproject.toml file by @youngale-pingthings in https://github.com/PingThingsIO/btrdb-python/pull/48 +* Provide option to sort the arrow tables by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/47 +* [sc-25841] Remove 4MB limit for gRPC message payloads by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/49 +* Update documentation for arrow methods by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/50 +* Update from staging by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/54 * Sort tables by time by default for any `pyarrow` tables. by @justinGilmer in * Fix deprecation warnings for pip installations. by @jleifnf in -## 5.31.0 - ## 5.30.2 ### What's Changed * Update readthedocs to new yaml for testing. by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/40 From 4648eb53d0f6f8233ca8ccbccab91b6f78add03f Mon Sep 17 00:00:00 2001 From: Justin Gilmer <16173348+justinGilmer@users.noreply.github.com> Date: Tue, 26 Mar 2024 16:14:01 -0400 Subject: [PATCH 106/131] Update precommit deps. --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9c35795..2d8e501 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ repos: - id: trailing-whitespace exclude: ^(setup.cfg|btrdb/grpcinterface) - repo: https://github.com/psf/black - rev: 23.12.0 + rev: 24.3.0 hooks: - id: black-jupyter args: [--line-length=88] @@ -20,7 +20,7 @@ repos: args: [--profile=black, --line-length=88] exclude: btrdb/grpcinterface/.*\.py - repo: https://github.com/PyCQA/flake8 - rev: 6.1.0 + rev: 7.0.0 hooks: - id: flake8 args: [--config=setup.cfg] From cccdfe0e5863cd685756bfed22f31b78737cc99e Mon Sep 17 00:00:00 2001 From: Alexander Young Date: Tue, 26 Mar 2024 15:37:00 -0500 Subject: [PATCH 107/131] Release v5.32.0 --- btrdb/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/btrdb/version.py b/btrdb/version.py index d657ab4..2c31b69 100644 --- a/btrdb/version.py +++ b/btrdb/version.py @@ -15,7 +15,7 @@ ## Module Info ########################################################################## -__version_info__ = {"major": 5, "minor": 31, "micro": 0, "releaselevel": "final"} +__version_info__ = { 'major': 5, 'minor': 32, 'micro': 0, 'releaselevel': 'final'} ########################################################################## ## Helper Functions diff --git a/pyproject.toml b/pyproject.toml index 5bd1745..a57516f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "btrdb" -version = "5.31.0" +version = "5.32.0" authors = [ {name="PingThingsIO", email="support@pingthings.io"}, ] From 0b6ed67007b9233915c4d361d0287510d4dbd55c Mon Sep 17 00:00:00 2001 From: Alexander Young Date: Tue, 26 Mar 2024 15:37:30 -0500 Subject: [PATCH 108/131] Add changes to release script --- release.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/release.sh b/release.sh index 31f849e..3302da0 100755 --- a/release.sh +++ b/release.sh @@ -42,7 +42,5 @@ sed -i.bak "s/^version.*$/version\ = \"$1.$2.$3\"/g" pyproject.toml git add btrdb/version.py git add pyproject.toml git commit -m "Release v$1.$2.$3" -git tag v$1.$2.$3 -git push origin v$1.$2.$3 -git push +echo "Now make a PR and merge it into main, then make tag :)" From 462925e2d653be6dd9e21cd3826cc070a023c77d Mon Sep 17 00:00:00 2001 From: Alexander Young Date: Tue, 26 Mar 2024 15:40:08 -0500 Subject: [PATCH 109/131] Switch to double quotes on release --- btrdb/version.py | 2 +- release.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/btrdb/version.py b/btrdb/version.py index 2c31b69..0d1010f 100644 --- a/btrdb/version.py +++ b/btrdb/version.py @@ -15,7 +15,7 @@ ## Module Info ########################################################################## -__version_info__ = { 'major': 5, 'minor': 32, 'micro': 0, 'releaselevel': 'final'} +__version_info__ = { "major": 5, "minor": 32, "micro": 0, "releaselevel": "final"} ########################################################################## ## Helper Functions diff --git a/release.sh b/release.sh index 3302da0..c6f8a35 100755 --- a/release.sh +++ b/release.sh @@ -35,7 +35,7 @@ fi echo "Setting version to v$1.$2.$3" -VERSION_CODE="__version_info__ = { 'major': $1, 'minor': $2, 'micro': $3, 'releaselevel': 'final'}" +VERSION_CODE="__version_info__ = { \"major\": $1, \"minor\": $2, \"micro\": $3, \"releaselevel\": \"final\"}" sed -i.bak "s/^__version_info__.*$/${VERSION_CODE}/g" btrdb/version.py sed -i.bak "s/^version.*$/version\ = \"$1.$2.$3\"/g" pyproject.toml From cb3c82098935168ec532c102cb3a54708cdc05a0 Mon Sep 17 00:00:00 2001 From: Alexander Young Date: Tue, 26 Mar 2024 15:41:26 -0500 Subject: [PATCH 110/131] Remove preceding space in release --- btrdb/version.py | 2 +- release.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/btrdb/version.py b/btrdb/version.py index 0d1010f..7bf90da 100644 --- a/btrdb/version.py +++ b/btrdb/version.py @@ -15,7 +15,7 @@ ## Module Info ########################################################################## -__version_info__ = { "major": 5, "minor": 32, "micro": 0, "releaselevel": "final"} +__version_info__ = {"major": 5, "minor": 32, "micro": 0, "releaselevel": "final"} ########################################################################## ## Helper Functions diff --git a/release.sh b/release.sh index c6f8a35..b63f28c 100755 --- a/release.sh +++ b/release.sh @@ -35,7 +35,7 @@ fi echo "Setting version to v$1.$2.$3" -VERSION_CODE="__version_info__ = { \"major\": $1, \"minor\": $2, \"micro\": $3, \"releaselevel\": \"final\"}" +VERSION_CODE="__version_info__ = {\"major\": $1, \"minor\": $2, \"micro\": $3, \"releaselevel\": \"final\"}" sed -i.bak "s/^__version_info__.*$/${VERSION_CODE}/g" btrdb/version.py sed -i.bak "s/^version.*$/version\ = \"$1.$2.$3\"/g" pyproject.toml From e1fc14363e4f325315430704fc017ab0feac199d Mon Sep 17 00:00:00 2001 From: Justin Gilmer <16173348+justinGilmer@users.noreply.github.com> Date: Tue, 26 Mar 2024 16:45:56 -0400 Subject: [PATCH 111/131] Update from latest staging. preparing for new release (#83) Co-authored-by: Jeff Lin <42981468+jleifnf@users.noreply.github.com> Co-authored-by: Andrew Chambers Co-authored-by: andrewchambers Co-authored-by: Alexander Young --- .github/workflows/pre-commit.yaml | 6 +- .github/workflows/release.yaml | 70 +- .pre-commit-config.yaml | 8 +- CHANGELOG.md | 35 + btrdb/__init__.py | 29 +- btrdb/conn.py | 319 ++++- btrdb/endpoint.py | 63 +- btrdb/exceptions.py | 25 +- btrdb/grpcinterface/btrdb.proto | 27 +- btrdb/grpcinterface/btrdb_pb2.py | 300 ++--- btrdb/grpcinterface/btrdb_pb2.pyi | 1062 +++++++++-------- btrdb/grpcinterface/btrdb_pb2_grpc.py | 39 +- btrdb/stream.py | 917 ++++++++++---- btrdb/transformers.py | 395 ++++-- btrdb/utils/general.py | 56 +- btrdb/utils/timez.py | 12 +- btrdb/version.py | 2 +- docs/requirements.txt | 3 + docs/source/api/streams.rst | 5 +- docs/source/concepts.rst | 149 ++- docs/source/conf.py | 35 +- docs/source/explained.rst | 2 +- docs/source/index.rst | 12 +- docs/source/installing.rst | 8 +- docs/source/quick-start.rst | 202 +++- docs/source/working/arrow-enabled-queries.rst | 4 +- .../source/working/stream-manage-metadata.rst | 2 +- docs/source/working/stream-query-manage.rst | 10 +- docs/source/working/stream-view-data.rst | 64 +- docs/source/working/streamsets.rst | 12 +- pyproject.toml | 12 +- release.sh | 6 +- setup.cfg | 4 +- setup.py | 3 +- tests/btrdb/test_base.py | 39 +- tests/btrdb/test_conn.py | 126 +- tests/btrdb/test_transformers.py | 21 +- tests/btrdb/utils/test_general.py | 46 + tests/btrdb_integration/test_conn.py | 79 +- tests/btrdb_integration/test_stream.py | 32 +- tests/btrdb_integration/test_streamset.py | 99 +- 41 files changed, 2890 insertions(+), 1450 deletions(-) diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml index 7ca805a..98e35c2 100644 --- a/.github/workflows/pre-commit.yaml +++ b/.github/workflows/pre-commit.yaml @@ -18,11 +18,11 @@ jobs: pre-commit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: token: ${{ secrets.GITHUB_TOKEN }} fetch-depth: 0 # get full git history - - uses: actions/setup-python@v3 + - uses: actions/setup-python@v5 with: cache: 'pip' - name: Install pre-commit @@ -30,7 +30,7 @@ jobs: pip install pre-commit - name: Get changed files id: changed-files - uses: tj-actions/changed-files@v41 + uses: tj-actions/changed-files@v43 with: token: ${{ secrets.GITHUB_TOKEN }} - name: Run pre-commit diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 027d682..15dc856 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -6,32 +6,33 @@ on: - synchronize push: tags: - - 'v[0-9]+.[0-9]+.[0-9]+*' + - "v[0-9]+.[0-9]+.[0-9]+*" jobs: - test-suite: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: [3.7, 3.8, 3.9, '3.10'] + python-version: [3.7, 3.8, 3.9, "3.10", "3.11"] os: [ubuntu-latest, macos-latest, windows-latest] steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} ${{ matrix.os }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install -r tests/requirements.txt - - name: Test with pytest - run: | - pytest - + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} ${{ matrix.os }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r tests/requirements.txt + - name: Test with pytest + run: | + pytest --cov-report= --cov=btrdb # suppress coverage report here + - name: Coverage Report + run: | + coverage report -m release: needs: @@ -39,7 +40,7 @@ jobs: if: startsWith(github.ref, 'refs/tags/') runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Create Release id: create_release uses: actions/create-release@v1 @@ -51,7 +52,6 @@ jobs: draft: false prerelease: false - deploy: needs: - release @@ -59,20 +59,20 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.8' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel build - - name: Build and publish - run: | + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel build + - name: Build and publish + run: | python -m build --sdist --wheel --outdir dist/ . - - name: Publish package - uses: pypa/gh-action-pypi-publish@release/v1 - with: - user: __token__ - password: ${{ secrets.PYPI_DEPLOYMENT_TOKEN }} + - name: Publish package + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.PYPI_DEPLOYMENT_TOKEN }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6f327d1..2d8e501 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,26 +1,26 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: check-yaml - id: end-of-file-fixer - id: trailing-whitespace exclude: ^(setup.cfg|btrdb/grpcinterface) - repo: https://github.com/psf/black - rev: 23.3.0 + rev: 24.3.0 hooks: - id: black-jupyter args: [--line-length=88] exclude: btrdb/grpcinterface/.*\.py - repo: https://github.com/pycqa/isort - rev: 5.11.5 + rev: 5.13.2 hooks: - id: isort name: isort (python) args: [--profile=black, --line-length=88] exclude: btrdb/grpcinterface/.*\.py - repo: https://github.com/PyCQA/flake8 - rev: 6.0.0 + rev: 7.0.0 hooks: - id: flake8 args: [--config=setup.cfg] diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f6d54f..9d0c080 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,39 @@ # Changelog + +## 5.32.0 - YYYY-MM-DD + +### What's Changed +- Initial docstring overhaul and a new test for better documentation and test coverage. by @JustinGilmer in #82 +- Test new join functionality for improved data loading for windowed queries. by @JustinGilmer in #80 +- Improve `arrow_to_dataframe` function for handling large amounts of columns, enhancing performance and usability. by @Jefflinf in #73 +- Expand testing to include Python 3.11, ensuring compatibility and stability. by @JustinGilmer in #74 +- Update exception handling to better support `RpcErrors`, improving error management and debugging. by @JustinGilmer in #72 +- Introduce an option for specifying the schema of the returned data, allowing for more flexibility in data handling. by @TODO in #51 +- Remove non-required dependencies and migrate to 'data' optional dependency for a lighter package and easier installation. by @JustinGilmer in #71 +- New method to get first and last timestamps from `aligned_windows`, enhancing data analysis capabilities. by @Jefflinf in #70 +- Add `to_timedelta` method for `pointwidth` class, providing more options for time-based data manipulation. by @Jefflinf in #69 + +### Fixed +- Fix `NoneType` error for `earliest/latest` for empty streams, ensuring reliability and error handling. by @Jefflinf in #64 +- Correct integration tests where the time column is not automatically set as the index, improving test accuracy and reliability. by @JustinGilmer in #56 + +### Deprecated +- FutureWarning for `streams_in_collection` to return `StreamSet` in the future, preparing users for upcoming API changes. by @Jefflinf in #60 + +**Full Changelog**: [GitHub compare view](https://github.com/PingThingsIO/btrdb-python/compare/v5.31.0...v5.32.0) + + +## 5.31.0 +## What's Changed +* Release v5.30.2 by @youngale-pingthings in https://github.com/PingThingsIO/btrdb-python/pull/42 +* Have release script update pyproject.toml file by @youngale-pingthings in https://github.com/PingThingsIO/btrdb-python/pull/48 +* Provide option to sort the arrow tables by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/47 +* [sc-25841] Remove 4MB limit for gRPC message payloads by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/49 +* Update documentation for arrow methods by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/50 +* Update from staging by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/54 +* Sort tables by time by default for any `pyarrow` tables. by @justinGilmer in +* Fix deprecation warnings for pip installations. by @jleifnf in + ## 5.30.2 ### What's Changed * Update readthedocs to new yaml for testing. by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/40 diff --git a/btrdb/__init__.py b/btrdb/__init__.py index efcb686..1584b3b 100644 --- a/btrdb/__init__.py +++ b/btrdb/__init__.py @@ -52,18 +52,18 @@ def connect(conn_str=None, apikey=None, profile=None, shareable=False): Parameters ---------- conn_str: str, default=None - The address and port of the cluster to connect to, e.g. `192.168.1.1:4411`. - If set to None, will look in the environment variable `$BTRDB_ENDPOINTS` + The address and port of the cluster to connect to, e.g. ``192.168.1.1:4411``. + If set to None, will look in the environment variable ``$BTRDB_ENDPOINTS`` (recommended). apikey: str, default=None The API key used to authenticate requests (optional). If None, the key - is looked up from the environment variable `$BTRDB_API_KEY`. + is looked up from the environment variable ``$BTRDB_API_KEY``. profile: str, default=None The name of a profile containing the required connection information as found in the user's predictive grid credentials file - `~/.predictivegrid/credentials.yaml`. + ``~/.predictivegrid/credentials.yaml``. shareable: bool, default=False - Whether or not the connection can be "shared" in a distributed setting such + Whether the connection can be "shared" in a distributed setting such as Ray workers. If set to True, the connection can be serialized and sent to other workers so that data can be retrieved in parallel; **however**, this is less secure because it is possible for other users of the Ray cluster to @@ -74,6 +74,25 @@ def connect(conn_str=None, apikey=None, profile=None, shareable=False): db : BTrDB An instance of the BTrDB context to directly interact with the database. + Examples + -------- + This example looks for the env variables: ``BTRDB_ENDPOINTS`` and ``BTRDB_API_KEY``. + + >>> conn = btrdb.connect() + + + + Connect to the platform by looking for the relevant platform profile + in ``${HOME}/.predictivegrid/credentials.yaml`` if the file is present. + + >>> conn = btrdb.connect(profile='test') + + + If you provide incorrect credentials, you will get an error. + + >>> conn = btrdb.connect(conn_str="192.168.1.1:4411", apikey="NONSENSICAL_API_KEY") + + """ # do not allow user to provide both address and profile if conn_str and profile: diff --git a/btrdb/conn.py b/btrdb/conn.py index a77eced..0e9b5fc 100644 --- a/btrdb/conn.py +++ b/btrdb/conn.py @@ -21,13 +21,14 @@ import re import uuid as uuidlib from concurrent.futures import ThreadPoolExecutor -from typing import List +from typing import List, Tuple, Union +from warnings import warn import certifi import grpc from grpc._cython.cygrpc import CompressionAlgorithm -from btrdb.exceptions import InvalidOperation, StreamNotFoundError, retry +from btrdb.exceptions import BTrDBError, InvalidOperation, StreamNotFoundError, retry from btrdb.stream import Stream, StreamSet from btrdb.utils.conversion import to_uuid from btrdb.utils.general import unpack_stream_descriptor @@ -54,11 +55,15 @@ def __init__(self, addrportstr, apikey=None): Parameters ---------- - addrportstr: str, required + addrportstr : str, required The address of the cluster to connect to, e.g 123.123.123:4411 - apikey: str, optional + apikey : str, optional The optional API key to authenticate requests + Notes + ----- + The ``btrdb.connect`` method is a helper function to make connecting to the platform easier + usually that will be sufficient for most users. """ addrport = addrportstr.split(":", 2) # 100MB size limit ~ 2500 streams for 5000 points with each point being 64bit @@ -196,6 +201,10 @@ class BTrDB(object): def __init__(self, endpoint): self.ep = endpoint + try: + _ = self.ep.info() + except Exception as err: + raise BTrDBError(f"Could not connect to the database, error message: {err}") self._executor = ThreadPoolExecutor() try: self._ARROW_ENABLED = _is_arrow_enabled(self.ep.info()) @@ -207,8 +216,8 @@ def __init__(self, endpoint): @retry def query( self, - stmt, - params=None, + stmt: str, + params: Union[Tuple[str], List[str]] = None, auto_retry=False, retries=5, retry_delay=3, @@ -220,23 +229,23 @@ def query( Parameters ---------- - stmt: str + stmt : str a SQL statement to be executed on the BTrDB metadata. Available tables are noted below. To sanitize inputs use a `$1` style parameter such as `select * from streams where name = $1 or name = $2`. - params: list or tuple + params : list or tuple a list of parameter values to be sanitized and interpolated into the SQL statement. Using parameters forces value/type checking and is considered a best practice at the very least. - auto_retry: bool, default: False + auto_retry : bool, default: False Whether to retry this request in the event of an error - retries: int, default: 5 + retries : int, default: 5 Number of times to retry this request if there is an error. Will be ignored if auto_retry is False - retry_delay: int, default: 3 + retry_delay : int, default: 3 initial time to wait before retrying function call if there is an error. Will be ignored if auto_retry is False - retry_backoff: int, default: 4 + retry_backoff : int, default: 4 Exponential factor by which the backoff increases between retries. Will be ignored if auto_retry is False @@ -249,10 +258,48 @@ def query( Notes ------- Parameters will be inserted into the SQL statement as noted by the - paramter number such as `$1`, `$2`, or `$3`. The `streams` table is + parameter number such as `$1`, `$2`, or `$3`. The `streams` table is available for `SELECT` statements only. See https://btrdb.readthedocs.io/en/latest/ for more info. + + The following are the queryable columns in the postgres ``streams`` table. + + +------------------+------------------------+-----------+ + | Column | Type | Nullable | + +==================+========================+===========+ + | uuid | uuid | not null | + +------------------+------------------------+-----------+ + | collection | character varying(256) | not null | + +------------------+------------------------+-----------+ + | name | character varying(256) | not null | + +------------------+------------------------+-----------+ + | unit | character varying(256) | not null | + +------------------+------------------------+-----------+ + | ingress | character varying(256) | not null | + +------------------+------------------------+-----------+ + | property_version | bigint | not null | + +------------------+------------------------+-----------+ + | annotations | hstore | | + +------------------+------------------------+-----------+ + + Examples + -------- + Count all streams in the platform. + + >>> conn = btrdb.connect() + >>> conn.query("SELECT COUNT(uuid) FROM streams") + [{'count': ...}] + + Count all streams in the collection ``foo/bar`` by passing in the variable as a parameter. + + >>> conn.query("SELECT COUNT(uuid) FROM streams WHERE collection=$1::text", params=["foo/bar"]) + [{'count': ...}] + + Count all streams in the platform that has a non-null entry for the metadata annotation ``foo``. + + >>> conn.query("SELECT COUNT(uuid) FROM streams WHERE annotations->$1::text IS NOT NULL", params=["foo"]) + [{'count': ...}] """ if params is None: params = list() @@ -266,18 +313,56 @@ def streams(self, *identifiers, versions=None, is_collection_prefix=False): """ Returns a StreamSet object with BTrDB streams from the supplied identifiers. If any streams cannot be found matching the identifier - than StreamNotFoundError will be returned. + then a ``StreamNotFoundError`` will be returned. Parameters ---------- - identifiers: str or UUID + identifiers : str or UUID a single item or iterable of items which can be used to query for - streams. identiers are expected to be UUID as string, UUID as UUID, + streams. Identifiers are expected to be UUID as string, UUID as UUID, or collection/name string. - versions: list[int] + versions : list[int] a single or iterable of version numbers to match the identifiers + is_collection_prefix : bool, default=False + If providing a collection string, is that string just a prefix, or the entire collection name? + This will impact how many streams are returned. + + + Returns + ------- + :class:`StreamSet` + Collection of streams. + + Examples + -------- + With a sequence of uuids. + + >>> conn = btrdb.connect() + >>> conn.streams(identifiers=list_of_uuids) + + + With a sequence of uuids and version numbers. + Here we are using version 0 to use the latest data points. + + >>> conn.streams(identifiers=list_of_uuids, versions=[0 for _ in list_of_uuids]) + + + Filtering by ``collection`` prefix ``"foo"`` where multiple collections exist like the following: + ``foo/bar``, ``foo/baz``, ``foo/bar/new``, and ``foo``. + If we set `is_collection_prefix`` to ``True``, this will return all streams that exist in the collections defined above. + It is similar to a regex pattern ``^foo.*`` for matching purposes. + + >>> conn.streams(identifiers="foo", is_collection_prefix=True) + + + If you set ``is_collection_prefix`` to ``False``, this will assume that the string identifier you provide is the full collection name. + Matching like the regex here: ``^foo`` + + >>> conn.streams(identifiers="foo", is_collection_prefix=False) + + """ if versions is not None and not isinstance(versions, list): raise TypeError("versions argument must be of type list") @@ -337,7 +422,7 @@ def stream_from_uuid(self, uuid): Parameters ---------- - uuid: UUID + uuid : UUID The uuid of the requested stream. Returns @@ -345,6 +430,16 @@ def stream_from_uuid(self, uuid): Stream instance of Stream class or None + Examples + -------- + + >>> import btrdb + >>> conn = btrdb.connect() + >>> uuid = "f98f4b4e-9fab-46b5-8a80-f282059d69b1" + >>> stream = conn.stream_from_uuid(uuid) + >>> stream + + """ return Stream(self, to_uuid(uuid)) @@ -365,23 +460,23 @@ def create( Parameters ---------- - uuid: UUID, required + uuid : UUID, required The uuid of the requested stream. - collection: str, required + collection : str, required The collection string prefix that the stream will belong to. - tags: dict, required - The tags-level immutable metadata key:value pairs. - annotations: dict, optional + tags : dict, required + The tags-level metadata key:value pairs. + annotations : dict, optional The mutable metadata of the stream, key:value pairs - auto_retry: bool, default: False + auto_retry : bool, default: False Whether to retry this request in the event of an error - retries: int, default: 5 + retries : int, default: 5 Number of times to retry this request if there is an error. Will be ignored if auto_retry is False - retry_delay: int, default: 3 + retry_delay : int, default: 3 initial time to wait before retrying function call if there is an error. Will be ignored if auto_retry is False - retry_backoff: int, default: 4 + retry_backoff : int, default: 4 Exponential factor by which the backoff increases between retries. Will be ignored if auto_retry is False @@ -389,6 +484,18 @@ def create( ------- Stream instance of Stream class + + + Examples + -------- + >>> import btrdb + >>> from uuid import uuid4 # this generates a random uuid + >>> conn = btrdb.connect() + >>> collection = "new/stream/collection" + >>> tags = {"name":"foo", "unit":"V"} + >>> annotations = {"bar": "baz"} + >>> s = conn.create(uuid=uuid4(), tags=tags, annotations=annotations, collection=collection) + """ if tags is None: @@ -405,17 +512,28 @@ def create( collection=collection, tags=tags.copy(), annotations=annotations.copy(), - property_version=0, + property_version=1, ) def info(self): """ - Returns information about the connected BTrDB server. + Returns information about the platform proxy server. Returns ------- dict - server connection and status information + Proxy server connection and status information + + Examples + -------- + >>> conn = btrdb.connect() + >>> conn.info() + { + .. 'majorVersion': 5, + .. 'minorVersion': 8, + .. 'build': ..., + .. 'proxy': ..., + } """ info = self.ep.info() @@ -433,13 +551,27 @@ def list_collections(self, starts_with=""): Parameters ---------- - starts_with: str, optional, default = '' + starts_with : str, optional, default: '' Filter collections that start with the string provided, if none passed, will list all collections. Returns ------- collections: List[str] + Examples + -------- + + Assuming we have the following collections in the platform: + ``foo``, ``bar``, ``foo/baz``, ``bar/baz`` + + >>> conn = btrdb.connect() + >>> conn.list_collections().sort() + ["bar", "bar/baz", "foo", "foo/bar"] + + >>> conn.list_collections(starts_with="foo") + ["foo", "foo/bar"] + + """ return [c for some in self.ep.listCollections(starts_with) for c in some] @@ -464,13 +596,25 @@ def list_unique_annotations(self, collection=None): Returns a list of annotation keys used in a given collection prefix. Parameters - ------- - collection: str + ---------- + collection : str Prefix of the collection to filter. Returns ------- - annotations: list[str] + annotations : list[str] + + Notes + ----- + This query treats the ``collection`` string as a prefix, so ``collection="foo"`` will match with the following wildcard syntax ``foo%``. + If you only want to filter for a single collection, you will need to provide the full collection, if there are other collections + that match the ``foo%`` pattern, you might need to use a custom SQL query using ``conn.query``. + + Examples + -------- + >>> conn.list_unique_annotations(collection="sunshine/PMU1") + ['foo', 'location', 'impedance'] + """ return self._list_unique_tags_annotations("annotations", collection) @@ -479,13 +623,28 @@ def list_unique_names(self, collection=None): Returns a list of names used in a given collection prefix. Parameters - ------- - collection: str + ---------- + collection : str Prefix of the collection to filter. Returns ------- - names: list[str] + names : list[str] + + Examples + -------- + Can specify a full ``collection`` name. + + >>> conn.list_unique_names(collection="sunshine/PMU1") + ['C1ANG', 'C1MAG', 'C2ANG', 'C2MAG', 'C3ANG', 'C3MAG', 'L1ANG', 'L1MAG', 'L2ANG', 'L2MAG', 'L3ANG', 'L3MAG', 'LSTATE'] + + And also provide a ``collection`` prefix. + + >>> conn.list_unique_names(collection="sunshine/") + ['C1ANG', 'C1MAG', 'C2ANG', 'C2MAG', 'C3ANG', 'C3MAG', 'L1ANG', 'L1MAG', 'L2ANG', 'L2MAG', 'L3ANG', 'L3MAG', 'LSTATE'] + + + """ return self._list_unique_tags_annotations("name", collection) @@ -494,13 +653,21 @@ def list_unique_units(self, collection=None): Returns a list of units used in a given collection prefix. Parameters - ------- - collection: str + ---------- + collection : str Prefix of the collection to filter. Returns ------- - units: list[str] + units : list[str] + + + Examples + -------- + + >>> conn.list_unique_units(collection="sunshine/PMU1") + ['amps', 'deg', 'mask', 'volts'] + """ return self._list_unique_tags_annotations("unit", collection) @@ -523,31 +690,54 @@ def streams_in_collection( Parameters ---------- - collection: str + collection : str collections to use when searching for streams, case sensitive. - is_collection_prefix: bool + is_collection_prefix : bool Whether the collection is a prefix. - tags: Dict[str, str] + tags : Dict[str, str] The tags to identify the stream. - annotations: Dict[str, str] + annotations : Dict[str, str] The annotations to identify the stream. - auto_retry: bool, default: False + auto_retry : bool, default: False Whether to retry this request in the event of an error - retries: int, default: 5 + retries : int, default: 5 Number of times to retry this request if there is an error. Will be ignored if auto_retry is False - retry_delay: int, default: 3 + retry_delay : int, default: 3 initial time to wait before retrying function call if there is an error. Will be ignored if auto_retry is False - retry_backoff: int, default: 4 + retry_backoff : int, default: 4 Exponential factor by which the backoff increases between retries. Will be ignored if auto_retry is False Returns ------ - list - A list of stream objects found with the provided search arguments. + list[Stream] + A list of ``Stream`` objects found with the provided search arguments. + + + .. note:: + + In a future release, the default return value of this function will be a ``StreamSet`` + + Examples + -------- + >>> conn = btrdb.connect() + >>> conn.streams_in_collection(collection="foo", is_collection_prefix=True) + [, , ] + + >>> conn.streams_in_collection(collection="foo", is_collection_prefix=False) + [, ] + + >>> conn.streams_in_collection(collection="foo", + ... is_collection_prefix=False, tags={"unit":"Volts"}) + [] + + >>> conn.streams_in_collection(collection="foo", + ... is_collection_prefix=False, tags={"unit":"UNKNOWN"}) + [] """ result = [] @@ -578,7 +768,12 @@ def streams_in_collection( property_version=desc.propertyVersion, ) ) - + # TODO: In future release update this method to return a streamset object. + warn( + "StreamSet will be the default return object for ``streams_in_collection`` in a future release.", + FutureWarning, + stacklevel=2, + ) return result @retry @@ -592,21 +787,21 @@ def collection_metadata( ): """ Gives statistics about metadata for collections that match a - prefix. + ``prefix``. Parameters ---------- - prefix: str, required + prefix : str, required A prefix of the collection names to look at - auto_retry: bool, default: False + auto_retry : bool, default: False Whether to retry this request in the event of an error - retries: int, default: 5 + retries : int, default: 5 Number of times to retry this request if there is an error. Will be ignored if auto_retry is False - retry_delay: int, default: 3 + retry_delay : int, default: 3 initial time to wait before retrying function call if there is an error. Will be ignored if auto_retry is False - retry_backoff: int, default: 4 + retry_backoff : int, default: 4 Exponential factor by which the backoff increases between retries. Will be ignored if auto_retry is False @@ -616,6 +811,16 @@ def collection_metadata( A tuple of dictionaries containing metadata on the streams in the provided collection. + Examples + -------- + >>> conn.collection_metadata("sunshine/PMU1") + ({'name': 0, 'unit': 0, 'ingress': 0, 'distiller': 0}, + .. {'foo': 1, 'impedance': 12, 'location': 12}) + + >>> conn.collection_metadata("sunshine/") + ({'name': 0, 'unit': 0, 'ingress': 0, 'distiller': 0}, + .. {'foo': 1, 'impedance': 72, 'location': 72}) + """ ep = self.ep tags, annotations = ep.getMetadataUsage(prefix) diff --git a/btrdb/endpoint.py b/btrdb/endpoint.py index 79a3b72..c632bbb 100644 --- a/btrdb/endpoint.py +++ b/btrdb/endpoint.py @@ -28,7 +28,7 @@ import typing import uuid -from btrdb.exceptions import BTrDBError, check_proto_stat, error_handler +from btrdb.exceptions import check_proto_stat, error_handler from btrdb.grpcinterface import btrdb_pb2, btrdb_pb2_grpc from btrdb.point import RawPoint from btrdb.utils.general import unpack_stream_descriptor @@ -38,6 +38,8 @@ except ImportError: pa = None +_ARROW_IMPORT_MSG = """Package `pyarrow` required, please pip install.""" + class Endpoint(object): """Server endpoint where we make specific requests.""" @@ -55,9 +57,21 @@ def rawValues(self, uu, start, end, version=0): yield result.values, result.versionMajor @error_handler - def arrowRawValues(self, uu, start, end, version=0): - params = btrdb_pb2.RawValuesParams( - uuid=uu.bytes, start=start, end=end, versionMajor=version + def arrowRawValues(self, uu, start, end, version=0, schema=None): + if pa is None: + raise ImportError(_ARROW_IMPORT_MSG) + templateBytes = b"" + if schema is not None: + byte_io = io.BytesIO() + with pa.ipc.new_stream(sink=byte_io, schema=schema) as _: + pass + templateBytes = byte_io.getvalue() + params = btrdb_pb2.ArrowRawValuesParams( + uuid=uu.bytes, + start=start, + end=end, + versionMajor=version, + templateBytes=templateBytes, ) for result in self.stub.ArrowRawValues(params): check_proto_stat(result.stat) @@ -65,13 +79,24 @@ def arrowRawValues(self, uu, start, end, version=0): yield reader.read_all(), result.versionMajor @error_handler - def arrowMultiValues(self, uu_list, start, end, version_list, snap_periodNS): + def arrowMultiValues( + self, uu_list, start, end, version_list, snap_periodNS=None, schema=None + ): + if pa is None: + raise ImportError(_ARROW_IMPORT_MSG) + templateBytes = b"" + if schema is not None: + byte_io = io.BytesIO() + with pa.ipc.new_stream(sink=byte_io, schema=schema) as _: + pass + templateBytes = byte_io.getvalue() params = btrdb_pb2.ArrowMultiValuesParams( uuid=[uu.bytes for uu in uu_list], start=start, end=end, versionMajor=[ver for ver in version_list], snapPeriodNs=int(snap_periodNS), + templateBytes=templateBytes, ) for result in self.stub.ArrowMultiValues(params): check_proto_stat(result.stat) @@ -80,6 +105,8 @@ def arrowMultiValues(self, uu_list, start, end, version_list, snap_periodNS): @error_handler def arrowInsertValues(self, uu: uuid.UUID, values: pa.Table, policy: str): + if pa is None: + raise ImportError(_ARROW_IMPORT_MSG) policy_map = { "never": btrdb_pb2.MergePolicy.NEVER, "equal": btrdb_pb2.MergePolicy.EQUAL, @@ -115,6 +142,8 @@ def alignedWindows(self, uu, start, end, pointwidth, version=0): @error_handler def arrowAlignedWindows(self, uu, start, end, pointwidth, version=0): + if pa is None: + raise ImportError(_ARROW_IMPORT_MSG) params = btrdb_pb2.AlignedWindowsParams( uuid=uu.bytes, start=start, @@ -143,6 +172,8 @@ def windows(self, uu, start, end, width, depth, version=0): @error_handler def arrowWindows(self, uu, start, end, width, depth, version=0): + if pa is None: + raise ImportError(_ARROW_IMPORT_MSG) params = btrdb_pb2.WindowsParams( uuid=uu.bytes, start=start, @@ -386,3 +417,25 @@ def sql_query(self, stmt, params: typing.List): for page in self.stub.SQLQuery(request): check_proto_stat(page.stat) yield page.SQLQueryRow + + @error_handler + def subscribe(self, update_queue): + def updates(): + while True: + update = update_queue.get() + if update is None: + return + (to_add, to_remove) = update + if len(to_add) != 0: + yield btrdb_pb2.SubscriptionUpdate( + op=0, uuid=[uu.bytes for uu in to_add] + ) + if len(to_remove) != 0: + yield btrdb_pb2.SubscriptionUpdate( + op=1, uuid=[uu.bytes for uu in to_remove] + ) + + for response in self.stub.Subscribe(updates()): + check_proto_stat(response.stat) + with pa.ipc.open_stream(response.arrowBytes) as reader: + yield uuid.UUID(bytes=response.uuid), reader.read_all() diff --git a/btrdb/exceptions.py b/btrdb/exceptions.py index 9777679..8507f7c 100644 --- a/btrdb/exceptions.py +++ b/btrdb/exceptions.py @@ -117,18 +117,21 @@ def handle_grpc_error(err): Parameters ---------- - err: grpc.RpcError + err: Union[grpc.RpcError, btrdb.BTrDBError] """ details = err.details() - if details == "[404] stream does not exist": - raise StreamNotFoundError("Stream not found with provided uuid") from None - elif details == "failed to connect to all addresses": - raise ConnectionError("Failed to connect to BTrDB") from None - elif any(str(e) in err.details() for e in BTRDB_SERVER_ERRORS): - raise BTRDBServerError("An error has occured with btrdb-server") from None - elif str(err.code()) == "StatusCode.PERMISSION_DENIED": - raise PermissionDenied(details) from None - raise BTrDBError(details) from None + if isinstance(err, BTrDBError): + if details == "[404] stream does not exist": + raise StreamNotFoundError("Stream not found with provided uuid") from None + elif details == "failed to connect to all addresses": + raise ConnectionError("Failed to connect to BTrDB") from None + elif any(str(e) in err.details() for e in BTRDB_SERVER_ERRORS): + raise BTRDBServerError("An error has occured with btrdb-server") from None + elif str(err.code()) == "StatusCode.PERMISSION_DENIED": + raise PermissionDenied(details) from None + raise BTrDBError(details) from None + else: + raise err def check_proto_stat(stat): @@ -145,7 +148,7 @@ def check_proto_stat(stat): if code in BTRDB_ERRORS: raise BTRDB_ERRORS[code](stat.msg) elif code in BTRDB_SERVER_ERRORS: - raise BTRDBServerError(stat.msg) + raise BTRDBServerError(str(code) + ": " + stat.msg) raise BTrDBError(stat.msg) diff --git a/btrdb/grpcinterface/btrdb.proto b/btrdb/grpcinterface/btrdb.proto index 884a71c..1e147d6 100644 --- a/btrdb/grpcinterface/btrdb.proto +++ b/btrdb/grpcinterface/btrdb.proto @@ -4,7 +4,7 @@ package v5api; service BTrDB { rpc RawValues(RawValuesParams) returns (stream RawValuesResponse); - rpc ArrowRawValues(RawValuesParams) returns (stream ArrowRawValuesResponse); + rpc ArrowRawValues(ArrowRawValuesParams) returns (stream ArrowRawValuesResponse); rpc ArrowMultiValues(ArrowMultiValuesParams) returns (stream ArrowMultiValuesResponse); rpc AlignedWindows(AlignedWindowsParams) returns (stream AlignedWindowsResponse); rpc ArrowAlignedWindows(AlignedWindowsParams) returns (stream ArrowAlignedWindowsResponse); @@ -28,6 +28,7 @@ service BTrDB { rpc GetMetadataUsage(MetadataUsageParams) returns (MetadataUsageResponse); rpc GenerateCSV(GenerateCSVParams) returns (stream GenerateCSVResponse); rpc SQLQuery(SQLQueryParams) returns (stream SQLQueryResponse); + rpc Subscribe(stream SubscriptionUpdate) returns (stream SubscriptionResp); //rpc SetCompactionConfig(SetCompactionConfigParams) returns (SetCompactionConfigResponse); //rpc GetCompactionConfig(GetCompactionConfigParams) returns (GetCompactionConfigResponse); } @@ -44,6 +45,13 @@ message RawValuesResponse { uint64 versionMinor = 3; repeated RawPoint values = 4; } +message ArrowRawValuesParams { + bytes uuid = 1; + sfixed64 start = 2; + sfixed64 end = 3; + uint64 versionMajor = 4; + bytes templateBytes = 5; +} message ArrowRawValuesResponse { Status stat = 1; uint64 versionMajor = 2; @@ -56,6 +64,7 @@ message ArrowMultiValuesParams { sfixed64 start = 3; sfixed64 end = 4; int64 snapPeriodNs = 5; + bytes templateBytes = 6; } message ArrowMultiValuesResponse { Status stat = 1; @@ -426,3 +435,19 @@ message ReducedResolutionRange { int64 End = 2; uint32 Resolution = 3; } + +enum SubscriptionUpdateOp { + ADD_UUIDS = 0; + REMOVE_UUIDS = 1; +} + +message SubscriptionUpdate { + SubscriptionUpdateOp op = 1; + repeated bytes uuid = 2; +} + +message SubscriptionResp { + Status stat = 1; + bytes uuid = 2; + bytes arrowBytes = 3; +} diff --git a/btrdb/grpcinterface/btrdb_pb2.py b/btrdb/grpcinterface/btrdb_pb2.py index bed2b0d..9dcef1a 100644 --- a/btrdb/grpcinterface/btrdb_pb2.py +++ b/btrdb/grpcinterface/btrdb_pb2.py @@ -1,11 +1,12 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: btrdb.proto +# Protobuf Python Version: 4.25.0 """Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -13,152 +14,159 @@ +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0b\x62trdb.proto\x12\x05v5api\"Q\n\x0fRawValuesParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\"}\n\x11RawValuesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x1f\n\x06values\x18\x04 \x03(\x0b\x32\x0f.v5api.RawPoint\"m\n\x14\x41rrowRawValuesParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\x12\x15\n\rtemplateBytes\x18\x05 \x01(\x0c\"u\n\x16\x41rrowRawValuesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x12\n\narrowBytes\x18\x04 \x01(\x0c\"\x85\x01\n\x16\x41rrowMultiValuesParams\x12\x0c\n\x04uuid\x18\x01 \x03(\x0c\x12\x14\n\x0cversionMajor\x18\x02 \x03(\x04\x12\r\n\x05start\x18\x03 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x04 \x01(\x10\x12\x14\n\x0csnapPeriodNs\x18\x05 \x01(\x03\x12\x15\n\rtemplateBytes\x18\x06 \x01(\x0c\"K\n\x18\x41rrowMultiValuesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x12\n\narrowBytes\x18\x02 \x01(\x0c\"*\n\x0bRawPointVec\x12\x0c\n\x04time\x18\x01 \x01(\x10\x12\r\n\x05value\x18\x02 \x03(\x01\"j\n\x14\x41lignedWindowsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\x12\x12\n\npointWidth\x18\x05 \x01(\r\"\x83\x01\n\x16\x41lignedWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12 \n\x06values\x18\x04 \x03(\x0b\x32\x10.v5api.StatPoint\"z\n\x1b\x41rrowAlignedWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x12\n\narrowBytes\x18\x04 \x01(\x0c\"m\n\rWindowsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\x12\r\n\x05width\x18\x05 \x01(\x04\x12\r\n\x05\x64\x65pth\x18\x06 \x01(\r\"|\n\x0fWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12 \n\x06values\x18\x04 \x03(\x0b\x32\x10.v5api.StatPoint\"s\n\x14\x41rrowWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x12\n\narrowBytes\x18\x04 \x01(\x0c\"h\n\x10StreamInfoParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x13\n\x0bomitVersion\x18\x02 \x01(\x08\x12\x16\n\x0eomitDescriptor\x18\x03 \x01(\x08\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"\x8a\x01\n\x12StreamInfoResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12+\n\ndescriptor\x18\x04 \x01(\x0b\x32\x17.v5api.StreamDescriptor\"\x98\x01\n\x10StreamDescriptor\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x12\n\ncollection\x18\x02 \x01(\t\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x17\n\x0fpropertyVersion\x18\x05 \x01(\x04\"\x82\x01\n\x1aSetStreamAnnotationsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x1f\n\x17\x65xpectedPropertyVersion\x18\x02 \x01(\x04\x12#\n\x07\x63hanges\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x10\n\x08removals\x18\x04 \x03(\t\";\n\x1cSetStreamAnnotationsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"\x8a\x01\n\x13SetStreamTagsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x1f\n\x17\x65xpectedPropertyVersion\x18\x02 \x01(\x04\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x12\n\ncollection\x18\x04 \x01(\t\x12\x0e\n\x06remove\x18\x05 \x03(\t\"4\n\x15SetStreamTagsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"{\n\x0c\x43reateParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x12\n\ncollection\x18\x02 \x01(\t\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\"-\n\x0e\x43reateResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"@\n\x13MetadataUsageParams\x12\x0e\n\x06prefix\x18\x01 \x01(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"y\n\x15MetadataUsageResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x1d\n\x04tags\x18\x02 \x03(\x0b\x32\x0f.v5api.KeyCount\x12$\n\x0b\x61nnotations\x18\x03 \x03(\x0b\x32\x0f.v5api.KeyCount\"&\n\x08KeyCount\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05\x63ount\x18\x02 \x01(\x04\"B\n\x15ListCollectionsParams\x12\x0e\n\x06prefix\x18\x01 \x01(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"K\n\x17ListCollectionsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x13\n\x0b\x63ollections\x18\x02 \x03(\t\"\xab\x01\n\x13LookupStreamsParams\x12\x12\n\ncollection\x18\x01 \x01(\t\x12\x1a\n\x12isCollectionPrefix\x18\x02 \x01(\x08\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"^\n\x15LookupStreamsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12(\n\x07results\x18\x02 \x03(\x0b\x32\x17.v5api.StreamDescriptor\"S\n\rNearestParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x0c\n\x04time\x18\x02 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x03 \x01(\x04\x12\x10\n\x08\x62\x61\x63kward\x18\x04 \x01(\x08\"z\n\x0fNearestResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x1e\n\x05value\x18\x04 \x01(\x0b\x32\x0f.v5api.RawPoint\"U\n\rChangesParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x11\n\tfromMajor\x18\x02 \x01(\x04\x12\x0f\n\x07toMajor\x18\x03 \x01(\x04\x12\x12\n\nresolution\x18\x04 \x01(\r\"\x7f\n\x0f\x43hangesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12#\n\x06ranges\x18\x04 \x03(\x0b\x32\x13.v5api.ChangedRange\"#\n\tRoundSpec\x12\x0e\n\x04\x62its\x18\x02 \x01(\x05H\x00\x42\x06\n\x04spec\"\x99\x01\n\x0cInsertParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x0c\n\x04sync\x18\x02 \x01(\x08\x12(\n\x0cmerge_policy\x18\x04 \x01(\x0e\x32\x12.v5api.MergePolicy\x12\"\n\x08rounding\x18\x05 \x01(\x0b\x32\x10.v5api.RoundSpec\x12\x1f\n\x06values\x18\x03 \x03(\x0b\x32\x0f.v5api.RawPoint\"\x91\x01\n\x11\x41rrowInsertParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x0c\n\x04sync\x18\x02 \x01(\x08\x12(\n\x0cmerge_policy\x18\x03 \x01(\x0e\x32\x12.v5api.MergePolicy\x12\"\n\x08rounding\x18\x04 \x01(\x0b\x32\x10.v5api.RoundSpec\x12\x12\n\narrowBytes\x18\x05 \x01(\x0c\"Y\n\x0eInsertResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\"8\n\x0c\x44\x65leteParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\"Y\n\x0e\x44\x65leteResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\"\x0c\n\nInfoParams\"\xa2\x01\n\x0cInfoResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x19\n\x04mash\x18\x02 \x01(\x0b\x32\x0b.v5api.Mash\x12\x14\n\x0cmajorVersion\x18\x03 \x01(\r\x12\x14\n\x0cminorVersion\x18\x04 \x01(\r\x12\r\n\x05\x62uild\x18\x05 \x01(\t\x12\x1f\n\x05proxy\x18\x06 \x01(\x0b\x32\x10.v5api.ProxyInfo\"#\n\tProxyInfo\x12\x16\n\x0eproxyEndpoints\x18\x01 \x03(\t\"1\n\x11\x46\x61ultInjectParams\x12\x0c\n\x04type\x18\x01 \x01(\x04\x12\x0e\n\x06params\x18\x02 \x01(\x0c\">\n\x13\x46\x61ultInjectResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\n\n\x02rv\x18\x02 \x01(\x0c\"\x1b\n\x0b\x46lushParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"X\n\rFlushResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\" \n\x10ObliterateParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"1\n\x12ObliterateResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"\'\n\x08RawPoint\x12\x0c\n\x04time\x18\x01 \x01(\x10\x12\r\n\x05value\x18\x02 \x01(\x01\"`\n\tStatPoint\x12\x0c\n\x04time\x18\x01 \x01(\x10\x12\x0b\n\x03min\x18\x02 \x01(\x01\x12\x0c\n\x04mean\x18\x03 \x01(\x01\x12\x0b\n\x03max\x18\x04 \x01(\x01\x12\r\n\x05\x63ount\x18\x05 \x01(\x06\x12\x0e\n\x06stddev\x18\x06 \x01(\x01\"*\n\x0c\x43hangedRange\x12\r\n\x05start\x18\x01 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x02 \x01(\x10\">\n\x06Status\x12\x0c\n\x04\x63ode\x18\x01 \x01(\r\x12\x0b\n\x03msg\x18\x02 \x01(\t\x12\x19\n\x04mash\x18\x03 \x01(\x0b\x32\x0b.v5api.Mash\"\x98\x01\n\x04Mash\x12\x10\n\x08revision\x18\x01 \x01(\x03\x12\x0e\n\x06leader\x18\x02 \x01(\t\x12\x16\n\x0eleaderRevision\x18\x03 \x01(\x03\x12\x13\n\x0btotalWeight\x18\x04 \x01(\x03\x12\x0f\n\x07healthy\x18\x05 \x01(\x08\x12\x10\n\x08unmapped\x18\x06 \x01(\x01\x12\x1e\n\x07members\x18\x07 \x03(\x0b\x32\r.v5api.Member\"\xc3\x01\n\x06Member\x12\x0c\n\x04hash\x18\x01 \x01(\r\x12\x10\n\x08nodename\x18\x02 \x01(\t\x12\n\n\x02up\x18\x03 \x01(\x08\x12\n\n\x02in\x18\x04 \x01(\x08\x12\x0f\n\x07\x65nabled\x18\x05 \x01(\x08\x12\r\n\x05start\x18\x06 \x01(\x03\x12\x0b\n\x03\x65nd\x18\x07 \x01(\x03\x12\x0e\n\x06weight\x18\x08 \x01(\x03\x12\x16\n\x0ereadPreference\x18\t \x01(\x01\x12\x15\n\rhttpEndpoints\x18\n \x01(\t\x12\x15\n\rgrpcEndpoints\x18\x0b \x01(\t\"8\n\x0bKeyOptValue\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x1c\n\x03val\x18\x02 \x01(\x0b\x32\x0f.v5api.OptValue\"\x19\n\x08OptValue\x12\r\n\x05value\x18\x01 \x01(\t\"&\n\x08KeyValue\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"?\n\x0fStreamCSVConfig\x12\x0f\n\x07version\x18\x01 \x01(\x04\x12\r\n\x05label\x18\x02 \x01(\t\x12\x0c\n\x04uuid\x18\x03 \x01(\x0c\"\x9d\x02\n\x11GenerateCSVParams\x12\x35\n\tqueryType\x18\x01 \x01(\x0e\x32\".v5api.GenerateCSVParams.QueryType\x12\x11\n\tstartTime\x18\x02 \x01(\x03\x12\x0f\n\x07\x65ndTime\x18\x03 \x01(\x03\x12\x12\n\nwindowSize\x18\x04 \x01(\x04\x12\r\n\x05\x64\x65pth\x18\x05 \x01(\r\x12\x17\n\x0fincludeVersions\x18\x06 \x01(\x08\x12\'\n\x07streams\x18\x07 \x03(\x0b\x32\x16.v5api.StreamCSVConfig\"H\n\tQueryType\x12\x19\n\x15\x41LIGNED_WINDOWS_QUERY\x10\x00\x12\x11\n\rWINDOWS_QUERY\x10\x01\x12\r\n\tRAW_QUERY\x10\x02\"Q\n\x13GenerateCSVResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x10\n\x08isHeader\x18\x02 \x01(\x08\x12\x0b\n\x03row\x18\x03 \x03(\t\"J\n\x0eSQLQueryParams\x12\r\n\x05query\x18\x01 \x01(\t\x12\x0e\n\x06params\x18\x02 \x03(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"D\n\x10SQLQueryResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x13\n\x0bSQLQueryRow\x18\x02 \x03(\x0c\"\x14\n\x04Role\x12\x0c\n\x04name\x18\x01 \x01(\t\"\x94\x01\n\x19SetCompactionConfigParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x18\n\x10\x43ompactedVersion\x18\x02 \x01(\x04\x12>\n\x17reducedResolutionRanges\x18\x03 \x03(\x0b\x32\x1d.v5api.ReducedResolutionRange\x12\x0f\n\x07unused0\x18\x04 \x01(\x04\":\n\x1bSetCompactionConfigResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\")\n\x19GetCompactionConfigParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"\xc1\x01\n\x1bGetCompactionConfigResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x1a\n\x12LatestMajorVersion\x18\x02 \x01(\x04\x12\x18\n\x10\x43ompactedVersion\x18\x03 \x01(\x04\x12>\n\x17reducedResolutionRanges\x18\x04 \x03(\x0b\x32\x1d.v5api.ReducedResolutionRange\x12\x0f\n\x07unused0\x18\x05 \x01(\x04\"H\n\x16ReducedResolutionRange\x12\r\n\x05Start\x18\x01 \x01(\x03\x12\x0b\n\x03\x45nd\x18\x02 \x01(\x03\x12\x12\n\nResolution\x18\x03 \x01(\r\"K\n\x12SubscriptionUpdate\x12\'\n\x02op\x18\x01 \x01(\x0e\x32\x1b.v5api.SubscriptionUpdateOp\x12\x0c\n\x04uuid\x18\x02 \x03(\x0c\"Q\n\x10SubscriptionResp\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x0c\n\x04uuid\x18\x02 \x01(\x0c\x12\x12\n\narrowBytes\x18\x03 \x01(\x0c*<\n\x0bMergePolicy\x12\t\n\x05NEVER\x10\x00\x12\t\n\x05\x45QUAL\x10\x01\x12\n\n\x06RETAIN\x10\x02\x12\x0b\n\x07REPLACE\x10\x03*7\n\x14SubscriptionUpdateOp\x12\r\n\tADD_UUIDS\x10\x00\x12\x10\n\x0cREMOVE_UUIDS\x10\x01\x32\xfd\r\n\x05\x42TrDB\x12?\n\tRawValues\x12\x16.v5api.RawValuesParams\x1a\x18.v5api.RawValuesResponse0\x01\x12N\n\x0e\x41rrowRawValues\x12\x1b.v5api.ArrowRawValuesParams\x1a\x1d.v5api.ArrowRawValuesResponse0\x01\x12T\n\x10\x41rrowMultiValues\x12\x1d.v5api.ArrowMultiValuesParams\x1a\x1f.v5api.ArrowMultiValuesResponse0\x01\x12N\n\x0e\x41lignedWindows\x12\x1b.v5api.AlignedWindowsParams\x1a\x1d.v5api.AlignedWindowsResponse0\x01\x12X\n\x13\x41rrowAlignedWindows\x12\x1b.v5api.AlignedWindowsParams\x1a\".v5api.ArrowAlignedWindowsResponse0\x01\x12\x39\n\x07Windows\x12\x14.v5api.WindowsParams\x1a\x16.v5api.WindowsResponse0\x01\x12\x43\n\x0c\x41rrowWindows\x12\x14.v5api.WindowsParams\x1a\x1b.v5api.ArrowWindowsResponse0\x01\x12@\n\nStreamInfo\x12\x17.v5api.StreamInfoParams\x1a\x19.v5api.StreamInfoResponse\x12^\n\x14SetStreamAnnotations\x12!.v5api.SetStreamAnnotationsParams\x1a#.v5api.SetStreamAnnotationsResponse\x12I\n\rSetStreamTags\x12\x1a.v5api.SetStreamTagsParams\x1a\x1c.v5api.SetStreamTagsResponse\x12\x34\n\x06\x43reate\x12\x13.v5api.CreateParams\x1a\x15.v5api.CreateResponse\x12Q\n\x0fListCollections\x12\x1c.v5api.ListCollectionsParams\x1a\x1e.v5api.ListCollectionsResponse0\x01\x12K\n\rLookupStreams\x12\x1a.v5api.LookupStreamsParams\x1a\x1c.v5api.LookupStreamsResponse0\x01\x12\x37\n\x07Nearest\x12\x14.v5api.NearestParams\x1a\x16.v5api.NearestResponse\x12\x39\n\x07\x43hanges\x12\x14.v5api.ChangesParams\x1a\x16.v5api.ChangesResponse0\x01\x12\x34\n\x06Insert\x12\x13.v5api.InsertParams\x1a\x15.v5api.InsertResponse\x12>\n\x0b\x41rrowInsert\x12\x18.v5api.ArrowInsertParams\x1a\x15.v5api.InsertResponse\x12\x34\n\x06\x44\x65lete\x12\x13.v5api.DeleteParams\x1a\x15.v5api.DeleteResponse\x12.\n\x04Info\x12\x11.v5api.InfoParams\x1a\x13.v5api.InfoResponse\x12\x43\n\x0b\x46\x61ultInject\x12\x18.v5api.FaultInjectParams\x1a\x1a.v5api.FaultInjectResponse\x12\x31\n\x05\x46lush\x12\x12.v5api.FlushParams\x1a\x14.v5api.FlushResponse\x12@\n\nObliterate\x12\x17.v5api.ObliterateParams\x1a\x19.v5api.ObliterateResponse\x12L\n\x10GetMetadataUsage\x12\x1a.v5api.MetadataUsageParams\x1a\x1c.v5api.MetadataUsageResponse\x12\x45\n\x0bGenerateCSV\x12\x18.v5api.GenerateCSVParams\x1a\x1a.v5api.GenerateCSVResponse0\x01\x12<\n\x08SQLQuery\x12\x15.v5api.SQLQueryParams\x1a\x17.v5api.SQLQueryResponse0\x01\x12\x43\n\tSubscribe\x12\x19.v5api.SubscriptionUpdate\x1a\x17.v5api.SubscriptionResp(\x01\x30\x01\x62\x06proto3') -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0b\x62trdb.proto\x12\x05v5api\"Q\n\x0fRawValuesParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\"}\n\x11RawValuesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x1f\n\x06values\x18\x04 \x03(\x0b\x32\x0f.v5api.RawPoint\"u\n\x16\x41rrowRawValuesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x12\n\narrowBytes\x18\x04 \x01(\x0c\"n\n\x16\x41rrowMultiValuesParams\x12\x0c\n\x04uuid\x18\x01 \x03(\x0c\x12\x14\n\x0cversionMajor\x18\x02 \x03(\x04\x12\r\n\x05start\x18\x03 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x04 \x01(\x10\x12\x14\n\x0csnapPeriodNs\x18\x05 \x01(\x03\"K\n\x18\x41rrowMultiValuesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x12\n\narrowBytes\x18\x02 \x01(\x0c\"*\n\x0bRawPointVec\x12\x0c\n\x04time\x18\x01 \x01(\x10\x12\r\n\x05value\x18\x02 \x03(\x01\"j\n\x14\x41lignedWindowsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\x12\x12\n\npointWidth\x18\x05 \x01(\r\"\x83\x01\n\x16\x41lignedWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12 \n\x06values\x18\x04 \x03(\x0b\x32\x10.v5api.StatPoint\"z\n\x1b\x41rrowAlignedWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x12\n\narrowBytes\x18\x04 \x01(\x0c\"m\n\rWindowsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x04 \x01(\x04\x12\r\n\x05width\x18\x05 \x01(\x04\x12\r\n\x05\x64\x65pth\x18\x06 \x01(\r\"|\n\x0fWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12 \n\x06values\x18\x04 \x03(\x0b\x32\x10.v5api.StatPoint\"s\n\x14\x41rrowWindowsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x12\n\narrowBytes\x18\x04 \x01(\x0c\"h\n\x10StreamInfoParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x13\n\x0bomitVersion\x18\x02 \x01(\x08\x12\x16\n\x0eomitDescriptor\x18\x03 \x01(\x08\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"\x8a\x01\n\x12StreamInfoResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12+\n\ndescriptor\x18\x04 \x01(\x0b\x32\x17.v5api.StreamDescriptor\"\x98\x01\n\x10StreamDescriptor\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x12\n\ncollection\x18\x02 \x01(\t\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x17\n\x0fpropertyVersion\x18\x05 \x01(\x04\"\x82\x01\n\x1aSetStreamAnnotationsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x1f\n\x17\x65xpectedPropertyVersion\x18\x02 \x01(\x04\x12#\n\x07\x63hanges\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x10\n\x08removals\x18\x04 \x03(\t\";\n\x1cSetStreamAnnotationsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"\x8a\x01\n\x13SetStreamTagsParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x1f\n\x17\x65xpectedPropertyVersion\x18\x02 \x01(\x04\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x12\n\ncollection\x18\x04 \x01(\t\x12\x0e\n\x06remove\x18\x05 \x03(\t\"4\n\x15SetStreamTagsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"{\n\x0c\x43reateParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x12\n\ncollection\x18\x02 \x01(\t\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\"-\n\x0e\x43reateResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"@\n\x13MetadataUsageParams\x12\x0e\n\x06prefix\x18\x01 \x01(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"y\n\x15MetadataUsageResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x1d\n\x04tags\x18\x02 \x03(\x0b\x32\x0f.v5api.KeyCount\x12$\n\x0b\x61nnotations\x18\x03 \x03(\x0b\x32\x0f.v5api.KeyCount\"&\n\x08KeyCount\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05\x63ount\x18\x02 \x01(\x04\"B\n\x15ListCollectionsParams\x12\x0e\n\x06prefix\x18\x01 \x01(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"K\n\x17ListCollectionsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x13\n\x0b\x63ollections\x18\x02 \x03(\t\"\xab\x01\n\x13LookupStreamsParams\x12\x12\n\ncollection\x18\x01 \x01(\t\x12\x1a\n\x12isCollectionPrefix\x18\x02 \x01(\x08\x12 \n\x04tags\x18\x03 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\'\n\x0b\x61nnotations\x18\x04 \x03(\x0b\x32\x12.v5api.KeyOptValue\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"^\n\x15LookupStreamsResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12(\n\x07results\x18\x02 \x03(\x0b\x32\x17.v5api.StreamDescriptor\"S\n\rNearestParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x0c\n\x04time\x18\x02 \x01(\x10\x12\x14\n\x0cversionMajor\x18\x03 \x01(\x04\x12\x10\n\x08\x62\x61\x63kward\x18\x04 \x01(\x08\"z\n\x0fNearestResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12\x1e\n\x05value\x18\x04 \x01(\x0b\x32\x0f.v5api.RawPoint\"U\n\rChangesParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x11\n\tfromMajor\x18\x02 \x01(\x04\x12\x0f\n\x07toMajor\x18\x03 \x01(\x04\x12\x12\n\nresolution\x18\x04 \x01(\r\"\x7f\n\x0f\x43hangesResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\x12#\n\x06ranges\x18\x04 \x03(\x0b\x32\x13.v5api.ChangedRange\"#\n\tRoundSpec\x12\x0e\n\x04\x62its\x18\x02 \x01(\x05H\x00\x42\x06\n\x04spec\"\x99\x01\n\x0cInsertParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x0c\n\x04sync\x18\x02 \x01(\x08\x12(\n\x0cmerge_policy\x18\x04 \x01(\x0e\x32\x12.v5api.MergePolicy\x12\"\n\x08rounding\x18\x05 \x01(\x0b\x32\x10.v5api.RoundSpec\x12\x1f\n\x06values\x18\x03 \x03(\x0b\x32\x0f.v5api.RawPoint\"\x91\x01\n\x11\x41rrowInsertParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x0c\n\x04sync\x18\x02 \x01(\x08\x12(\n\x0cmerge_policy\x18\x03 \x01(\x0e\x32\x12.v5api.MergePolicy\x12\"\n\x08rounding\x18\x04 \x01(\x0b\x32\x10.v5api.RoundSpec\x12\x12\n\narrowBytes\x18\x05 \x01(\x0c\"Y\n\x0eInsertResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\"8\n\x0c\x44\x65leteParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\r\n\x05start\x18\x02 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x03 \x01(\x10\"Y\n\x0e\x44\x65leteResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\"\x0c\n\nInfoParams\"\xa2\x01\n\x0cInfoResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x19\n\x04mash\x18\x02 \x01(\x0b\x32\x0b.v5api.Mash\x12\x14\n\x0cmajorVersion\x18\x03 \x01(\r\x12\x14\n\x0cminorVersion\x18\x04 \x01(\r\x12\r\n\x05\x62uild\x18\x05 \x01(\t\x12\x1f\n\x05proxy\x18\x06 \x01(\x0b\x32\x10.v5api.ProxyInfo\"#\n\tProxyInfo\x12\x16\n\x0eproxyEndpoints\x18\x01 \x03(\t\"1\n\x11\x46\x61ultInjectParams\x12\x0c\n\x04type\x18\x01 \x01(\x04\x12\x0e\n\x06params\x18\x02 \x01(\x0c\">\n\x13\x46\x61ultInjectResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\n\n\x02rv\x18\x02 \x01(\x0c\"\x1b\n\x0b\x46lushParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"X\n\rFlushResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x14\n\x0cversionMajor\x18\x02 \x01(\x04\x12\x14\n\x0cversionMinor\x18\x03 \x01(\x04\" \n\x10ObliterateParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"1\n\x12ObliterateResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\"\'\n\x08RawPoint\x12\x0c\n\x04time\x18\x01 \x01(\x10\x12\r\n\x05value\x18\x02 \x01(\x01\"`\n\tStatPoint\x12\x0c\n\x04time\x18\x01 \x01(\x10\x12\x0b\n\x03min\x18\x02 \x01(\x01\x12\x0c\n\x04mean\x18\x03 \x01(\x01\x12\x0b\n\x03max\x18\x04 \x01(\x01\x12\r\n\x05\x63ount\x18\x05 \x01(\x06\x12\x0e\n\x06stddev\x18\x06 \x01(\x01\"*\n\x0c\x43hangedRange\x12\r\n\x05start\x18\x01 \x01(\x10\x12\x0b\n\x03\x65nd\x18\x02 \x01(\x10\">\n\x06Status\x12\x0c\n\x04\x63ode\x18\x01 \x01(\r\x12\x0b\n\x03msg\x18\x02 \x01(\t\x12\x19\n\x04mash\x18\x03 \x01(\x0b\x32\x0b.v5api.Mash\"\x98\x01\n\x04Mash\x12\x10\n\x08revision\x18\x01 \x01(\x03\x12\x0e\n\x06leader\x18\x02 \x01(\t\x12\x16\n\x0eleaderRevision\x18\x03 \x01(\x03\x12\x13\n\x0btotalWeight\x18\x04 \x01(\x03\x12\x0f\n\x07healthy\x18\x05 \x01(\x08\x12\x10\n\x08unmapped\x18\x06 \x01(\x01\x12\x1e\n\x07members\x18\x07 \x03(\x0b\x32\r.v5api.Member\"\xc3\x01\n\x06Member\x12\x0c\n\x04hash\x18\x01 \x01(\r\x12\x10\n\x08nodename\x18\x02 \x01(\t\x12\n\n\x02up\x18\x03 \x01(\x08\x12\n\n\x02in\x18\x04 \x01(\x08\x12\x0f\n\x07\x65nabled\x18\x05 \x01(\x08\x12\r\n\x05start\x18\x06 \x01(\x03\x12\x0b\n\x03\x65nd\x18\x07 \x01(\x03\x12\x0e\n\x06weight\x18\x08 \x01(\x03\x12\x16\n\x0ereadPreference\x18\t \x01(\x01\x12\x15\n\rhttpEndpoints\x18\n \x01(\t\x12\x15\n\rgrpcEndpoints\x18\x0b \x01(\t\"8\n\x0bKeyOptValue\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x1c\n\x03val\x18\x02 \x01(\x0b\x32\x0f.v5api.OptValue\"\x19\n\x08OptValue\x12\r\n\x05value\x18\x01 \x01(\t\"&\n\x08KeyValue\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"?\n\x0fStreamCSVConfig\x12\x0f\n\x07version\x18\x01 \x01(\x04\x12\r\n\x05label\x18\x02 \x01(\t\x12\x0c\n\x04uuid\x18\x03 \x01(\x0c\"\x9d\x02\n\x11GenerateCSVParams\x12\x35\n\tqueryType\x18\x01 \x01(\x0e\x32\".v5api.GenerateCSVParams.QueryType\x12\x11\n\tstartTime\x18\x02 \x01(\x03\x12\x0f\n\x07\x65ndTime\x18\x03 \x01(\x03\x12\x12\n\nwindowSize\x18\x04 \x01(\x04\x12\r\n\x05\x64\x65pth\x18\x05 \x01(\r\x12\x17\n\x0fincludeVersions\x18\x06 \x01(\x08\x12\'\n\x07streams\x18\x07 \x03(\x0b\x32\x16.v5api.StreamCSVConfig\"H\n\tQueryType\x12\x19\n\x15\x41LIGNED_WINDOWS_QUERY\x10\x00\x12\x11\n\rWINDOWS_QUERY\x10\x01\x12\r\n\tRAW_QUERY\x10\x02\"Q\n\x13GenerateCSVResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x10\n\x08isHeader\x18\x02 \x01(\x08\x12\x0b\n\x03row\x18\x03 \x03(\t\"J\n\x0eSQLQueryParams\x12\r\n\x05query\x18\x01 \x01(\t\x12\x0e\n\x06params\x18\x02 \x03(\t\x12\x19\n\x04role\x18\x64 \x01(\x0b\x32\x0b.v5api.Role\"D\n\x10SQLQueryResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x13\n\x0bSQLQueryRow\x18\x02 \x03(\x0c\"\x14\n\x04Role\x12\x0c\n\x04name\x18\x01 \x01(\t\"\x94\x01\n\x19SetCompactionConfigParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\x12\x18\n\x10\x43ompactedVersion\x18\x02 \x01(\x04\x12>\n\x17reducedResolutionRanges\x18\x03 \x03(\x0b\x32\x1d.v5api.ReducedResolutionRange\x12\x0f\n\x07unused0\x18\x04 \x01(\x04\":\n\x1bSetCompactionConfigResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\")\n\x19GetCompactionConfigParams\x12\x0c\n\x04uuid\x18\x01 \x01(\x0c\"\xc1\x01\n\x1bGetCompactionConfigResponse\x12\x1b\n\x04stat\x18\x01 \x01(\x0b\x32\r.v5api.Status\x12\x1a\n\x12LatestMajorVersion\x18\x02 \x01(\x04\x12\x18\n\x10\x43ompactedVersion\x18\x03 \x01(\x04\x12>\n\x17reducedResolutionRanges\x18\x04 \x03(\x0b\x32\x1d.v5api.ReducedResolutionRange\x12\x0f\n\x07unused0\x18\x05 \x01(\x04\"H\n\x16ReducedResolutionRange\x12\r\n\x05Start\x18\x01 \x01(\x03\x12\x0b\n\x03\x45nd\x18\x02 \x01(\x03\x12\x12\n\nResolution\x18\x03 \x01(\r*<\n\x0bMergePolicy\x12\t\n\x05NEVER\x10\x00\x12\t\n\x05\x45QUAL\x10\x01\x12\n\n\x06RETAIN\x10\x02\x12\x0b\n\x07REPLACE\x10\x03\x32\xb3\r\n\x05\x42TrDB\x12?\n\tRawValues\x12\x16.v5api.RawValuesParams\x1a\x18.v5api.RawValuesResponse0\x01\x12I\n\x0e\x41rrowRawValues\x12\x16.v5api.RawValuesParams\x1a\x1d.v5api.ArrowRawValuesResponse0\x01\x12T\n\x10\x41rrowMultiValues\x12\x1d.v5api.ArrowMultiValuesParams\x1a\x1f.v5api.ArrowMultiValuesResponse0\x01\x12N\n\x0e\x41lignedWindows\x12\x1b.v5api.AlignedWindowsParams\x1a\x1d.v5api.AlignedWindowsResponse0\x01\x12X\n\x13\x41rrowAlignedWindows\x12\x1b.v5api.AlignedWindowsParams\x1a\".v5api.ArrowAlignedWindowsResponse0\x01\x12\x39\n\x07Windows\x12\x14.v5api.WindowsParams\x1a\x16.v5api.WindowsResponse0\x01\x12\x43\n\x0c\x41rrowWindows\x12\x14.v5api.WindowsParams\x1a\x1b.v5api.ArrowWindowsResponse0\x01\x12@\n\nStreamInfo\x12\x17.v5api.StreamInfoParams\x1a\x19.v5api.StreamInfoResponse\x12^\n\x14SetStreamAnnotations\x12!.v5api.SetStreamAnnotationsParams\x1a#.v5api.SetStreamAnnotationsResponse\x12I\n\rSetStreamTags\x12\x1a.v5api.SetStreamTagsParams\x1a\x1c.v5api.SetStreamTagsResponse\x12\x34\n\x06\x43reate\x12\x13.v5api.CreateParams\x1a\x15.v5api.CreateResponse\x12Q\n\x0fListCollections\x12\x1c.v5api.ListCollectionsParams\x1a\x1e.v5api.ListCollectionsResponse0\x01\x12K\n\rLookupStreams\x12\x1a.v5api.LookupStreamsParams\x1a\x1c.v5api.LookupStreamsResponse0\x01\x12\x37\n\x07Nearest\x12\x14.v5api.NearestParams\x1a\x16.v5api.NearestResponse\x12\x39\n\x07\x43hanges\x12\x14.v5api.ChangesParams\x1a\x16.v5api.ChangesResponse0\x01\x12\x34\n\x06Insert\x12\x13.v5api.InsertParams\x1a\x15.v5api.InsertResponse\x12>\n\x0b\x41rrowInsert\x12\x18.v5api.ArrowInsertParams\x1a\x15.v5api.InsertResponse\x12\x34\n\x06\x44\x65lete\x12\x13.v5api.DeleteParams\x1a\x15.v5api.DeleteResponse\x12.\n\x04Info\x12\x11.v5api.InfoParams\x1a\x13.v5api.InfoResponse\x12\x43\n\x0b\x46\x61ultInject\x12\x18.v5api.FaultInjectParams\x1a\x1a.v5api.FaultInjectResponse\x12\x31\n\x05\x46lush\x12\x12.v5api.FlushParams\x1a\x14.v5api.FlushResponse\x12@\n\nObliterate\x12\x17.v5api.ObliterateParams\x1a\x19.v5api.ObliterateResponse\x12L\n\x10GetMetadataUsage\x12\x1a.v5api.MetadataUsageParams\x1a\x1c.v5api.MetadataUsageResponse\x12\x45\n\x0bGenerateCSV\x12\x18.v5api.GenerateCSVParams\x1a\x1a.v5api.GenerateCSVResponse0\x01\x12<\n\x08SQLQuery\x12\x15.v5api.SQLQueryParams\x1a\x17.v5api.SQLQueryResponse0\x01\x62\x06proto3') - -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'btrdb_pb2', globals()) +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'btrdb_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: - DESCRIPTOR._options = None - _MERGEPOLICY._serialized_start=6305 - _MERGEPOLICY._serialized_end=6365 - _RAWVALUESPARAMS._serialized_start=22 - _RAWVALUESPARAMS._serialized_end=103 - _RAWVALUESRESPONSE._serialized_start=105 - _RAWVALUESRESPONSE._serialized_end=230 - _ARROWRAWVALUESRESPONSE._serialized_start=232 - _ARROWRAWVALUESRESPONSE._serialized_end=349 - _ARROWMULTIVALUESPARAMS._serialized_start=351 - _ARROWMULTIVALUESPARAMS._serialized_end=461 - _ARROWMULTIVALUESRESPONSE._serialized_start=463 - _ARROWMULTIVALUESRESPONSE._serialized_end=538 - _RAWPOINTVEC._serialized_start=540 - _RAWPOINTVEC._serialized_end=582 - _ALIGNEDWINDOWSPARAMS._serialized_start=584 - _ALIGNEDWINDOWSPARAMS._serialized_end=690 - _ALIGNEDWINDOWSRESPONSE._serialized_start=693 - _ALIGNEDWINDOWSRESPONSE._serialized_end=824 - _ARROWALIGNEDWINDOWSRESPONSE._serialized_start=826 - _ARROWALIGNEDWINDOWSRESPONSE._serialized_end=948 - _WINDOWSPARAMS._serialized_start=950 - _WINDOWSPARAMS._serialized_end=1059 - _WINDOWSRESPONSE._serialized_start=1061 - _WINDOWSRESPONSE._serialized_end=1185 - _ARROWWINDOWSRESPONSE._serialized_start=1187 - _ARROWWINDOWSRESPONSE._serialized_end=1302 - _STREAMINFOPARAMS._serialized_start=1304 - _STREAMINFOPARAMS._serialized_end=1408 - _STREAMINFORESPONSE._serialized_start=1411 - _STREAMINFORESPONSE._serialized_end=1549 - _STREAMDESCRIPTOR._serialized_start=1552 - _STREAMDESCRIPTOR._serialized_end=1704 - _SETSTREAMANNOTATIONSPARAMS._serialized_start=1707 - _SETSTREAMANNOTATIONSPARAMS._serialized_end=1837 - _SETSTREAMANNOTATIONSRESPONSE._serialized_start=1839 - _SETSTREAMANNOTATIONSRESPONSE._serialized_end=1898 - _SETSTREAMTAGSPARAMS._serialized_start=1901 - _SETSTREAMTAGSPARAMS._serialized_end=2039 - _SETSTREAMTAGSRESPONSE._serialized_start=2041 - _SETSTREAMTAGSRESPONSE._serialized_end=2093 - _CREATEPARAMS._serialized_start=2095 - _CREATEPARAMS._serialized_end=2218 - _CREATERESPONSE._serialized_start=2220 - _CREATERESPONSE._serialized_end=2265 - _METADATAUSAGEPARAMS._serialized_start=2267 - _METADATAUSAGEPARAMS._serialized_end=2331 - _METADATAUSAGERESPONSE._serialized_start=2333 - _METADATAUSAGERESPONSE._serialized_end=2454 - _KEYCOUNT._serialized_start=2456 - _KEYCOUNT._serialized_end=2494 - _LISTCOLLECTIONSPARAMS._serialized_start=2496 - _LISTCOLLECTIONSPARAMS._serialized_end=2562 - _LISTCOLLECTIONSRESPONSE._serialized_start=2564 - _LISTCOLLECTIONSRESPONSE._serialized_end=2639 - _LOOKUPSTREAMSPARAMS._serialized_start=2642 - _LOOKUPSTREAMSPARAMS._serialized_end=2813 - _LOOKUPSTREAMSRESPONSE._serialized_start=2815 - _LOOKUPSTREAMSRESPONSE._serialized_end=2909 - _NEARESTPARAMS._serialized_start=2911 - _NEARESTPARAMS._serialized_end=2994 - _NEARESTRESPONSE._serialized_start=2996 - _NEARESTRESPONSE._serialized_end=3118 - _CHANGESPARAMS._serialized_start=3120 - _CHANGESPARAMS._serialized_end=3205 - _CHANGESRESPONSE._serialized_start=3207 - _CHANGESRESPONSE._serialized_end=3334 - _ROUNDSPEC._serialized_start=3336 - _ROUNDSPEC._serialized_end=3371 - _INSERTPARAMS._serialized_start=3374 - _INSERTPARAMS._serialized_end=3527 - _ARROWINSERTPARAMS._serialized_start=3530 - _ARROWINSERTPARAMS._serialized_end=3675 - _INSERTRESPONSE._serialized_start=3677 - _INSERTRESPONSE._serialized_end=3766 - _DELETEPARAMS._serialized_start=3768 - _DELETEPARAMS._serialized_end=3824 - _DELETERESPONSE._serialized_start=3826 - _DELETERESPONSE._serialized_end=3915 - _INFOPARAMS._serialized_start=3917 - _INFOPARAMS._serialized_end=3929 - _INFORESPONSE._serialized_start=3932 - _INFORESPONSE._serialized_end=4094 - _PROXYINFO._serialized_start=4096 - _PROXYINFO._serialized_end=4131 - _FAULTINJECTPARAMS._serialized_start=4133 - _FAULTINJECTPARAMS._serialized_end=4182 - _FAULTINJECTRESPONSE._serialized_start=4184 - _FAULTINJECTRESPONSE._serialized_end=4246 - _FLUSHPARAMS._serialized_start=4248 - _FLUSHPARAMS._serialized_end=4275 - _FLUSHRESPONSE._serialized_start=4277 - _FLUSHRESPONSE._serialized_end=4365 - _OBLITERATEPARAMS._serialized_start=4367 - _OBLITERATEPARAMS._serialized_end=4399 - _OBLITERATERESPONSE._serialized_start=4401 - _OBLITERATERESPONSE._serialized_end=4450 - _RAWPOINT._serialized_start=4452 - _RAWPOINT._serialized_end=4491 - _STATPOINT._serialized_start=4493 - _STATPOINT._serialized_end=4589 - _CHANGEDRANGE._serialized_start=4591 - _CHANGEDRANGE._serialized_end=4633 - _STATUS._serialized_start=4635 - _STATUS._serialized_end=4697 - _MASH._serialized_start=4700 - _MASH._serialized_end=4852 - _MEMBER._serialized_start=4855 - _MEMBER._serialized_end=5050 - _KEYOPTVALUE._serialized_start=5052 - _KEYOPTVALUE._serialized_end=5108 - _OPTVALUE._serialized_start=5110 - _OPTVALUE._serialized_end=5135 - _KEYVALUE._serialized_start=5137 - _KEYVALUE._serialized_end=5175 - _STREAMCSVCONFIG._serialized_start=5177 - _STREAMCSVCONFIG._serialized_end=5240 - _GENERATECSVPARAMS._serialized_start=5243 - _GENERATECSVPARAMS._serialized_end=5528 - _GENERATECSVPARAMS_QUERYTYPE._serialized_start=5456 - _GENERATECSVPARAMS_QUERYTYPE._serialized_end=5528 - _GENERATECSVRESPONSE._serialized_start=5530 - _GENERATECSVRESPONSE._serialized_end=5611 - _SQLQUERYPARAMS._serialized_start=5613 - _SQLQUERYPARAMS._serialized_end=5687 - _SQLQUERYRESPONSE._serialized_start=5689 - _SQLQUERYRESPONSE._serialized_end=5757 - _ROLE._serialized_start=5759 - _ROLE._serialized_end=5779 - _SETCOMPACTIONCONFIGPARAMS._serialized_start=5782 - _SETCOMPACTIONCONFIGPARAMS._serialized_end=5930 - _SETCOMPACTIONCONFIGRESPONSE._serialized_start=5932 - _SETCOMPACTIONCONFIGRESPONSE._serialized_end=5990 - _GETCOMPACTIONCONFIGPARAMS._serialized_start=5992 - _GETCOMPACTIONCONFIGPARAMS._serialized_end=6033 - _GETCOMPACTIONCONFIGRESPONSE._serialized_start=6036 - _GETCOMPACTIONCONFIGRESPONSE._serialized_end=6229 - _REDUCEDRESOLUTIONRANGE._serialized_start=6231 - _REDUCEDRESOLUTIONRANGE._serialized_end=6303 - _BTRDB._serialized_start=6368 - _BTRDB._serialized_end=8083 + _globals['_MERGEPOLICY']._serialized_start=6600 + _globals['_MERGEPOLICY']._serialized_end=6660 + _globals['_SUBSCRIPTIONUPDATEOP']._serialized_start=6662 + _globals['_SUBSCRIPTIONUPDATEOP']._serialized_end=6717 + _globals['_RAWVALUESPARAMS']._serialized_start=22 + _globals['_RAWVALUESPARAMS']._serialized_end=103 + _globals['_RAWVALUESRESPONSE']._serialized_start=105 + _globals['_RAWVALUESRESPONSE']._serialized_end=230 + _globals['_ARROWRAWVALUESPARAMS']._serialized_start=232 + _globals['_ARROWRAWVALUESPARAMS']._serialized_end=341 + _globals['_ARROWRAWVALUESRESPONSE']._serialized_start=343 + _globals['_ARROWRAWVALUESRESPONSE']._serialized_end=460 + _globals['_ARROWMULTIVALUESPARAMS']._serialized_start=463 + _globals['_ARROWMULTIVALUESPARAMS']._serialized_end=596 + _globals['_ARROWMULTIVALUESRESPONSE']._serialized_start=598 + _globals['_ARROWMULTIVALUESRESPONSE']._serialized_end=673 + _globals['_RAWPOINTVEC']._serialized_start=675 + _globals['_RAWPOINTVEC']._serialized_end=717 + _globals['_ALIGNEDWINDOWSPARAMS']._serialized_start=719 + _globals['_ALIGNEDWINDOWSPARAMS']._serialized_end=825 + _globals['_ALIGNEDWINDOWSRESPONSE']._serialized_start=828 + _globals['_ALIGNEDWINDOWSRESPONSE']._serialized_end=959 + _globals['_ARROWALIGNEDWINDOWSRESPONSE']._serialized_start=961 + _globals['_ARROWALIGNEDWINDOWSRESPONSE']._serialized_end=1083 + _globals['_WINDOWSPARAMS']._serialized_start=1085 + _globals['_WINDOWSPARAMS']._serialized_end=1194 + _globals['_WINDOWSRESPONSE']._serialized_start=1196 + _globals['_WINDOWSRESPONSE']._serialized_end=1320 + _globals['_ARROWWINDOWSRESPONSE']._serialized_start=1322 + _globals['_ARROWWINDOWSRESPONSE']._serialized_end=1437 + _globals['_STREAMINFOPARAMS']._serialized_start=1439 + _globals['_STREAMINFOPARAMS']._serialized_end=1543 + _globals['_STREAMINFORESPONSE']._serialized_start=1546 + _globals['_STREAMINFORESPONSE']._serialized_end=1684 + _globals['_STREAMDESCRIPTOR']._serialized_start=1687 + _globals['_STREAMDESCRIPTOR']._serialized_end=1839 + _globals['_SETSTREAMANNOTATIONSPARAMS']._serialized_start=1842 + _globals['_SETSTREAMANNOTATIONSPARAMS']._serialized_end=1972 + _globals['_SETSTREAMANNOTATIONSRESPONSE']._serialized_start=1974 + _globals['_SETSTREAMANNOTATIONSRESPONSE']._serialized_end=2033 + _globals['_SETSTREAMTAGSPARAMS']._serialized_start=2036 + _globals['_SETSTREAMTAGSPARAMS']._serialized_end=2174 + _globals['_SETSTREAMTAGSRESPONSE']._serialized_start=2176 + _globals['_SETSTREAMTAGSRESPONSE']._serialized_end=2228 + _globals['_CREATEPARAMS']._serialized_start=2230 + _globals['_CREATEPARAMS']._serialized_end=2353 + _globals['_CREATERESPONSE']._serialized_start=2355 + _globals['_CREATERESPONSE']._serialized_end=2400 + _globals['_METADATAUSAGEPARAMS']._serialized_start=2402 + _globals['_METADATAUSAGEPARAMS']._serialized_end=2466 + _globals['_METADATAUSAGERESPONSE']._serialized_start=2468 + _globals['_METADATAUSAGERESPONSE']._serialized_end=2589 + _globals['_KEYCOUNT']._serialized_start=2591 + _globals['_KEYCOUNT']._serialized_end=2629 + _globals['_LISTCOLLECTIONSPARAMS']._serialized_start=2631 + _globals['_LISTCOLLECTIONSPARAMS']._serialized_end=2697 + _globals['_LISTCOLLECTIONSRESPONSE']._serialized_start=2699 + _globals['_LISTCOLLECTIONSRESPONSE']._serialized_end=2774 + _globals['_LOOKUPSTREAMSPARAMS']._serialized_start=2777 + _globals['_LOOKUPSTREAMSPARAMS']._serialized_end=2948 + _globals['_LOOKUPSTREAMSRESPONSE']._serialized_start=2950 + _globals['_LOOKUPSTREAMSRESPONSE']._serialized_end=3044 + _globals['_NEARESTPARAMS']._serialized_start=3046 + _globals['_NEARESTPARAMS']._serialized_end=3129 + _globals['_NEARESTRESPONSE']._serialized_start=3131 + _globals['_NEARESTRESPONSE']._serialized_end=3253 + _globals['_CHANGESPARAMS']._serialized_start=3255 + _globals['_CHANGESPARAMS']._serialized_end=3340 + _globals['_CHANGESRESPONSE']._serialized_start=3342 + _globals['_CHANGESRESPONSE']._serialized_end=3469 + _globals['_ROUNDSPEC']._serialized_start=3471 + _globals['_ROUNDSPEC']._serialized_end=3506 + _globals['_INSERTPARAMS']._serialized_start=3509 + _globals['_INSERTPARAMS']._serialized_end=3662 + _globals['_ARROWINSERTPARAMS']._serialized_start=3665 + _globals['_ARROWINSERTPARAMS']._serialized_end=3810 + _globals['_INSERTRESPONSE']._serialized_start=3812 + _globals['_INSERTRESPONSE']._serialized_end=3901 + _globals['_DELETEPARAMS']._serialized_start=3903 + _globals['_DELETEPARAMS']._serialized_end=3959 + _globals['_DELETERESPONSE']._serialized_start=3961 + _globals['_DELETERESPONSE']._serialized_end=4050 + _globals['_INFOPARAMS']._serialized_start=4052 + _globals['_INFOPARAMS']._serialized_end=4064 + _globals['_INFORESPONSE']._serialized_start=4067 + _globals['_INFORESPONSE']._serialized_end=4229 + _globals['_PROXYINFO']._serialized_start=4231 + _globals['_PROXYINFO']._serialized_end=4266 + _globals['_FAULTINJECTPARAMS']._serialized_start=4268 + _globals['_FAULTINJECTPARAMS']._serialized_end=4317 + _globals['_FAULTINJECTRESPONSE']._serialized_start=4319 + _globals['_FAULTINJECTRESPONSE']._serialized_end=4381 + _globals['_FLUSHPARAMS']._serialized_start=4383 + _globals['_FLUSHPARAMS']._serialized_end=4410 + _globals['_FLUSHRESPONSE']._serialized_start=4412 + _globals['_FLUSHRESPONSE']._serialized_end=4500 + _globals['_OBLITERATEPARAMS']._serialized_start=4502 + _globals['_OBLITERATEPARAMS']._serialized_end=4534 + _globals['_OBLITERATERESPONSE']._serialized_start=4536 + _globals['_OBLITERATERESPONSE']._serialized_end=4585 + _globals['_RAWPOINT']._serialized_start=4587 + _globals['_RAWPOINT']._serialized_end=4626 + _globals['_STATPOINT']._serialized_start=4628 + _globals['_STATPOINT']._serialized_end=4724 + _globals['_CHANGEDRANGE']._serialized_start=4726 + _globals['_CHANGEDRANGE']._serialized_end=4768 + _globals['_STATUS']._serialized_start=4770 + _globals['_STATUS']._serialized_end=4832 + _globals['_MASH']._serialized_start=4835 + _globals['_MASH']._serialized_end=4987 + _globals['_MEMBER']._serialized_start=4990 + _globals['_MEMBER']._serialized_end=5185 + _globals['_KEYOPTVALUE']._serialized_start=5187 + _globals['_KEYOPTVALUE']._serialized_end=5243 + _globals['_OPTVALUE']._serialized_start=5245 + _globals['_OPTVALUE']._serialized_end=5270 + _globals['_KEYVALUE']._serialized_start=5272 + _globals['_KEYVALUE']._serialized_end=5310 + _globals['_STREAMCSVCONFIG']._serialized_start=5312 + _globals['_STREAMCSVCONFIG']._serialized_end=5375 + _globals['_GENERATECSVPARAMS']._serialized_start=5378 + _globals['_GENERATECSVPARAMS']._serialized_end=5663 + _globals['_GENERATECSVPARAMS_QUERYTYPE']._serialized_start=5591 + _globals['_GENERATECSVPARAMS_QUERYTYPE']._serialized_end=5663 + _globals['_GENERATECSVRESPONSE']._serialized_start=5665 + _globals['_GENERATECSVRESPONSE']._serialized_end=5746 + _globals['_SQLQUERYPARAMS']._serialized_start=5748 + _globals['_SQLQUERYPARAMS']._serialized_end=5822 + _globals['_SQLQUERYRESPONSE']._serialized_start=5824 + _globals['_SQLQUERYRESPONSE']._serialized_end=5892 + _globals['_ROLE']._serialized_start=5894 + _globals['_ROLE']._serialized_end=5914 + _globals['_SETCOMPACTIONCONFIGPARAMS']._serialized_start=5917 + _globals['_SETCOMPACTIONCONFIGPARAMS']._serialized_end=6065 + _globals['_SETCOMPACTIONCONFIGRESPONSE']._serialized_start=6067 + _globals['_SETCOMPACTIONCONFIGRESPONSE']._serialized_end=6125 + _globals['_GETCOMPACTIONCONFIGPARAMS']._serialized_start=6127 + _globals['_GETCOMPACTIONCONFIGPARAMS']._serialized_end=6168 + _globals['_GETCOMPACTIONCONFIGRESPONSE']._serialized_start=6171 + _globals['_GETCOMPACTIONCONFIGRESPONSE']._serialized_end=6364 + _globals['_REDUCEDRESOLUTIONRANGE']._serialized_start=6366 + _globals['_REDUCEDRESOLUTIONRANGE']._serialized_end=6438 + _globals['_SUBSCRIPTIONUPDATE']._serialized_start=6440 + _globals['_SUBSCRIPTIONUPDATE']._serialized_end=6515 + _globals['_SUBSCRIPTIONRESP']._serialized_start=6517 + _globals['_SUBSCRIPTIONRESP']._serialized_end=6598 + _globals['_BTRDB']._serialized_start=6720 + _globals['_BTRDB']._serialized_end=8509 # @@protoc_insertion_point(module_scope) diff --git a/btrdb/grpcinterface/btrdb_pb2.pyi b/btrdb/grpcinterface/btrdb_pb2.pyi index ff9b6fc..cc6320c 100644 --- a/btrdb/grpcinterface/btrdb_pb2.pyi +++ b/btrdb/grpcinterface/btrdb_pb2.pyi @@ -5,334 +5,307 @@ from google.protobuf import message as _message from typing import ClassVar as _ClassVar, Iterable as _Iterable, Mapping as _Mapping, Optional as _Optional, Union as _Union DESCRIPTOR: _descriptor.FileDescriptor -EQUAL: MergePolicy + +class MergePolicy(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = () + NEVER: _ClassVar[MergePolicy] + EQUAL: _ClassVar[MergePolicy] + RETAIN: _ClassVar[MergePolicy] + REPLACE: _ClassVar[MergePolicy] + +class SubscriptionUpdateOp(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = () + ADD_UUIDS: _ClassVar[SubscriptionUpdateOp] + REMOVE_UUIDS: _ClassVar[SubscriptionUpdateOp] NEVER: MergePolicy -REPLACE: MergePolicy +EQUAL: MergePolicy RETAIN: MergePolicy +REPLACE: MergePolicy +ADD_UUIDS: SubscriptionUpdateOp +REMOVE_UUIDS: SubscriptionUpdateOp -class AlignedWindowsParams(_message.Message): - __slots__ = ["end", "pointWidth", "start", "uuid", "versionMajor"] - END_FIELD_NUMBER: _ClassVar[int] - POINTWIDTH_FIELD_NUMBER: _ClassVar[int] - START_FIELD_NUMBER: _ClassVar[int] +class RawValuesParams(_message.Message): + __slots__ = ("uuid", "start", "end", "versionMajor") UUID_FIELD_NUMBER: _ClassVar[int] + START_FIELD_NUMBER: _ClassVar[int] + END_FIELD_NUMBER: _ClassVar[int] VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] - end: int - pointWidth: int - start: int uuid: bytes + start: int + end: int versionMajor: int - def __init__(self, uuid: _Optional[bytes] = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., versionMajor: _Optional[int] = ..., pointWidth: _Optional[int] = ...) -> None: ... + def __init__(self, uuid: _Optional[bytes] = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., versionMajor: _Optional[int] = ...) -> None: ... -class AlignedWindowsResponse(_message.Message): - __slots__ = ["stat", "values", "versionMajor", "versionMinor"] +class RawValuesResponse(_message.Message): + __slots__ = ("stat", "versionMajor", "versionMinor", "values") STAT_FIELD_NUMBER: _ClassVar[int] - VALUES_FIELD_NUMBER: _ClassVar[int] VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + VALUES_FIELD_NUMBER: _ClassVar[int] stat: Status - values: _containers.RepeatedCompositeFieldContainer[StatPoint] versionMajor: int versionMinor: int - def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., values: _Optional[_Iterable[_Union[StatPoint, _Mapping]]] = ...) -> None: ... + values: _containers.RepeatedCompositeFieldContainer[RawPoint] + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., values: _Optional[_Iterable[_Union[RawPoint, _Mapping]]] = ...) -> None: ... -class ArrowAlignedWindowsResponse(_message.Message): - __slots__ = ["arrowBytes", "stat", "versionMajor", "versionMinor"] - ARROWBYTES_FIELD_NUMBER: _ClassVar[int] +class ArrowRawValuesParams(_message.Message): + __slots__ = ("uuid", "start", "end", "versionMajor", "templateBytes") + UUID_FIELD_NUMBER: _ClassVar[int] + START_FIELD_NUMBER: _ClassVar[int] + END_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + TEMPLATEBYTES_FIELD_NUMBER: _ClassVar[int] + uuid: bytes + start: int + end: int + versionMajor: int + templateBytes: bytes + def __init__(self, uuid: _Optional[bytes] = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., versionMajor: _Optional[int] = ..., templateBytes: _Optional[bytes] = ...) -> None: ... + +class ArrowRawValuesResponse(_message.Message): + __slots__ = ("stat", "versionMajor", "versionMinor", "arrowBytes") STAT_FIELD_NUMBER: _ClassVar[int] VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] - arrowBytes: bytes + ARROWBYTES_FIELD_NUMBER: _ClassVar[int] stat: Status versionMajor: int versionMinor: int - def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., arrowBytes: _Optional[bytes] = ...) -> None: ... - -class ArrowInsertParams(_message.Message): - __slots__ = ["arrowBytes", "merge_policy", "rounding", "sync", "uuid"] - ARROWBYTES_FIELD_NUMBER: _ClassVar[int] - MERGE_POLICY_FIELD_NUMBER: _ClassVar[int] - ROUNDING_FIELD_NUMBER: _ClassVar[int] - SYNC_FIELD_NUMBER: _ClassVar[int] - UUID_FIELD_NUMBER: _ClassVar[int] arrowBytes: bytes - merge_policy: MergePolicy - rounding: RoundSpec - sync: bool - uuid: bytes - def __init__(self, uuid: _Optional[bytes] = ..., sync: bool = ..., merge_policy: _Optional[_Union[MergePolicy, str]] = ..., rounding: _Optional[_Union[RoundSpec, _Mapping]] = ..., arrowBytes: _Optional[bytes] = ...) -> None: ... + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., arrowBytes: _Optional[bytes] = ...) -> None: ... class ArrowMultiValuesParams(_message.Message): - __slots__ = ["end", "snapPeriodNs", "start", "uuid", "versionMajor"] - END_FIELD_NUMBER: _ClassVar[int] - SNAPPERIODNS_FIELD_NUMBER: _ClassVar[int] - START_FIELD_NUMBER: _ClassVar[int] + __slots__ = ("uuid", "versionMajor", "start", "end", "snapPeriodNs", "templateBytes") UUID_FIELD_NUMBER: _ClassVar[int] VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] - end: int - snapPeriodNs: int - start: int + START_FIELD_NUMBER: _ClassVar[int] + END_FIELD_NUMBER: _ClassVar[int] + SNAPPERIODNS_FIELD_NUMBER: _ClassVar[int] + TEMPLATEBYTES_FIELD_NUMBER: _ClassVar[int] uuid: _containers.RepeatedScalarFieldContainer[bytes] versionMajor: _containers.RepeatedScalarFieldContainer[int] - def __init__(self, uuid: _Optional[_Iterable[bytes]] = ..., versionMajor: _Optional[_Iterable[int]] = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., snapPeriodNs: _Optional[int] = ...) -> None: ... + start: int + end: int + snapPeriodNs: int + templateBytes: bytes + def __init__(self, uuid: _Optional[_Iterable[bytes]] = ..., versionMajor: _Optional[_Iterable[int]] = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., snapPeriodNs: _Optional[int] = ..., templateBytes: _Optional[bytes] = ...) -> None: ... class ArrowMultiValuesResponse(_message.Message): - __slots__ = ["arrowBytes", "stat"] - ARROWBYTES_FIELD_NUMBER: _ClassVar[int] + __slots__ = ("stat", "arrowBytes") STAT_FIELD_NUMBER: _ClassVar[int] - arrowBytes: bytes + ARROWBYTES_FIELD_NUMBER: _ClassVar[int] stat: Status + arrowBytes: bytes def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., arrowBytes: _Optional[bytes] = ...) -> None: ... -class ArrowRawValuesResponse(_message.Message): - __slots__ = ["arrowBytes", "stat", "versionMajor", "versionMinor"] - ARROWBYTES_FIELD_NUMBER: _ClassVar[int] +class RawPointVec(_message.Message): + __slots__ = ("time", "value") + TIME_FIELD_NUMBER: _ClassVar[int] + VALUE_FIELD_NUMBER: _ClassVar[int] + time: int + value: _containers.RepeatedScalarFieldContainer[float] + def __init__(self, time: _Optional[int] = ..., value: _Optional[_Iterable[float]] = ...) -> None: ... + +class AlignedWindowsParams(_message.Message): + __slots__ = ("uuid", "start", "end", "versionMajor", "pointWidth") + UUID_FIELD_NUMBER: _ClassVar[int] + START_FIELD_NUMBER: _ClassVar[int] + END_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + POINTWIDTH_FIELD_NUMBER: _ClassVar[int] + uuid: bytes + start: int + end: int + versionMajor: int + pointWidth: int + def __init__(self, uuid: _Optional[bytes] = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., versionMajor: _Optional[int] = ..., pointWidth: _Optional[int] = ...) -> None: ... + +class AlignedWindowsResponse(_message.Message): + __slots__ = ("stat", "versionMajor", "versionMinor", "values") STAT_FIELD_NUMBER: _ClassVar[int] VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] - arrowBytes: bytes + VALUES_FIELD_NUMBER: _ClassVar[int] stat: Status versionMajor: int versionMinor: int - def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., arrowBytes: _Optional[bytes] = ...) -> None: ... + values: _containers.RepeatedCompositeFieldContainer[StatPoint] + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., values: _Optional[_Iterable[_Union[StatPoint, _Mapping]]] = ...) -> None: ... -class ArrowWindowsResponse(_message.Message): - __slots__ = ["arrowBytes", "stat", "versionMajor", "versionMinor"] - ARROWBYTES_FIELD_NUMBER: _ClassVar[int] +class ArrowAlignedWindowsResponse(_message.Message): + __slots__ = ("stat", "versionMajor", "versionMinor", "arrowBytes") STAT_FIELD_NUMBER: _ClassVar[int] VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] - arrowBytes: bytes + ARROWBYTES_FIELD_NUMBER: _ClassVar[int] stat: Status versionMajor: int versionMinor: int + arrowBytes: bytes def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., arrowBytes: _Optional[bytes] = ...) -> None: ... -class ChangedRange(_message.Message): - __slots__ = ["end", "start"] - END_FIELD_NUMBER: _ClassVar[int] +class WindowsParams(_message.Message): + __slots__ = ("uuid", "start", "end", "versionMajor", "width", "depth") + UUID_FIELD_NUMBER: _ClassVar[int] START_FIELD_NUMBER: _ClassVar[int] - end: int + END_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + WIDTH_FIELD_NUMBER: _ClassVar[int] + DEPTH_FIELD_NUMBER: _ClassVar[int] + uuid: bytes start: int - def __init__(self, start: _Optional[int] = ..., end: _Optional[int] = ...) -> None: ... + end: int + versionMajor: int + width: int + depth: int + def __init__(self, uuid: _Optional[bytes] = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., versionMajor: _Optional[int] = ..., width: _Optional[int] = ..., depth: _Optional[int] = ...) -> None: ... -class ChangesParams(_message.Message): - __slots__ = ["fromMajor", "resolution", "toMajor", "uuid"] - FROMMAJOR_FIELD_NUMBER: _ClassVar[int] - RESOLUTION_FIELD_NUMBER: _ClassVar[int] - TOMAJOR_FIELD_NUMBER: _ClassVar[int] +class WindowsResponse(_message.Message): + __slots__ = ("stat", "versionMajor", "versionMinor", "values") + STAT_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + VALUES_FIELD_NUMBER: _ClassVar[int] + stat: Status + versionMajor: int + versionMinor: int + values: _containers.RepeatedCompositeFieldContainer[StatPoint] + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., values: _Optional[_Iterable[_Union[StatPoint, _Mapping]]] = ...) -> None: ... + +class ArrowWindowsResponse(_message.Message): + __slots__ = ("stat", "versionMajor", "versionMinor", "arrowBytes") + STAT_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + ARROWBYTES_FIELD_NUMBER: _ClassVar[int] + stat: Status + versionMajor: int + versionMinor: int + arrowBytes: bytes + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., arrowBytes: _Optional[bytes] = ...) -> None: ... + +class StreamInfoParams(_message.Message): + __slots__ = ("uuid", "omitVersion", "omitDescriptor", "role") UUID_FIELD_NUMBER: _ClassVar[int] - fromMajor: int - resolution: int - toMajor: int + OMITVERSION_FIELD_NUMBER: _ClassVar[int] + OMITDESCRIPTOR_FIELD_NUMBER: _ClassVar[int] + ROLE_FIELD_NUMBER: _ClassVar[int] uuid: bytes - def __init__(self, uuid: _Optional[bytes] = ..., fromMajor: _Optional[int] = ..., toMajor: _Optional[int] = ..., resolution: _Optional[int] = ...) -> None: ... + omitVersion: bool + omitDescriptor: bool + role: Role + def __init__(self, uuid: _Optional[bytes] = ..., omitVersion: bool = ..., omitDescriptor: bool = ..., role: _Optional[_Union[Role, _Mapping]] = ...) -> None: ... -class ChangesResponse(_message.Message): - __slots__ = ["ranges", "stat", "versionMajor", "versionMinor"] - RANGES_FIELD_NUMBER: _ClassVar[int] +class StreamInfoResponse(_message.Message): + __slots__ = ("stat", "versionMajor", "versionMinor", "descriptor") STAT_FIELD_NUMBER: _ClassVar[int] VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] - ranges: _containers.RepeatedCompositeFieldContainer[ChangedRange] + DESCRIPTOR_FIELD_NUMBER: _ClassVar[int] stat: Status versionMajor: int versionMinor: int - def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., ranges: _Optional[_Iterable[_Union[ChangedRange, _Mapping]]] = ...) -> None: ... + descriptor: StreamDescriptor + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., descriptor: _Optional[_Union[StreamDescriptor, _Mapping]] = ...) -> None: ... -class CreateParams(_message.Message): - __slots__ = ["annotations", "collection", "tags", "uuid"] - ANNOTATIONS_FIELD_NUMBER: _ClassVar[int] +class StreamDescriptor(_message.Message): + __slots__ = ("uuid", "collection", "tags", "annotations", "propertyVersion") + UUID_FIELD_NUMBER: _ClassVar[int] COLLECTION_FIELD_NUMBER: _ClassVar[int] TAGS_FIELD_NUMBER: _ClassVar[int] - UUID_FIELD_NUMBER: _ClassVar[int] - annotations: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + ANNOTATIONS_FIELD_NUMBER: _ClassVar[int] + PROPERTYVERSION_FIELD_NUMBER: _ClassVar[int] + uuid: bytes collection: str tags: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + annotations: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + propertyVersion: int + def __init__(self, uuid: _Optional[bytes] = ..., collection: _Optional[str] = ..., tags: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., annotations: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., propertyVersion: _Optional[int] = ...) -> None: ... + +class SetStreamAnnotationsParams(_message.Message): + __slots__ = ("uuid", "expectedPropertyVersion", "changes", "removals") + UUID_FIELD_NUMBER: _ClassVar[int] + EXPECTEDPROPERTYVERSION_FIELD_NUMBER: _ClassVar[int] + CHANGES_FIELD_NUMBER: _ClassVar[int] + REMOVALS_FIELD_NUMBER: _ClassVar[int] uuid: bytes - def __init__(self, uuid: _Optional[bytes] = ..., collection: _Optional[str] = ..., tags: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., annotations: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ...) -> None: ... + expectedPropertyVersion: int + changes: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + removals: _containers.RepeatedScalarFieldContainer[str] + def __init__(self, uuid: _Optional[bytes] = ..., expectedPropertyVersion: _Optional[int] = ..., changes: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., removals: _Optional[_Iterable[str]] = ...) -> None: ... -class CreateResponse(_message.Message): - __slots__ = ["stat"] +class SetStreamAnnotationsResponse(_message.Message): + __slots__ = ("stat",) STAT_FIELD_NUMBER: _ClassVar[int] stat: Status def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ...) -> None: ... -class DeleteParams(_message.Message): - __slots__ = ["end", "start", "uuid"] - END_FIELD_NUMBER: _ClassVar[int] - START_FIELD_NUMBER: _ClassVar[int] +class SetStreamTagsParams(_message.Message): + __slots__ = ("uuid", "expectedPropertyVersion", "tags", "collection", "remove") UUID_FIELD_NUMBER: _ClassVar[int] - end: int - start: int + EXPECTEDPROPERTYVERSION_FIELD_NUMBER: _ClassVar[int] + TAGS_FIELD_NUMBER: _ClassVar[int] + COLLECTION_FIELD_NUMBER: _ClassVar[int] + REMOVE_FIELD_NUMBER: _ClassVar[int] uuid: bytes - def __init__(self, uuid: _Optional[bytes] = ..., start: _Optional[int] = ..., end: _Optional[int] = ...) -> None: ... + expectedPropertyVersion: int + tags: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + collection: str + remove: _containers.RepeatedScalarFieldContainer[str] + def __init__(self, uuid: _Optional[bytes] = ..., expectedPropertyVersion: _Optional[int] = ..., tags: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., collection: _Optional[str] = ..., remove: _Optional[_Iterable[str]] = ...) -> None: ... -class DeleteResponse(_message.Message): - __slots__ = ["stat", "versionMajor", "versionMinor"] +class SetStreamTagsResponse(_message.Message): + __slots__ = ("stat",) STAT_FIELD_NUMBER: _ClassVar[int] - VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] - VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] stat: Status - versionMajor: int - versionMinor: int - def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ...) -> None: ... + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ...) -> None: ... -class FaultInjectParams(_message.Message): - __slots__ = ["params", "type"] - PARAMS_FIELD_NUMBER: _ClassVar[int] - TYPE_FIELD_NUMBER: _ClassVar[int] - params: bytes - type: int - def __init__(self, type: _Optional[int] = ..., params: _Optional[bytes] = ...) -> None: ... +class CreateParams(_message.Message): + __slots__ = ("uuid", "collection", "tags", "annotations") + UUID_FIELD_NUMBER: _ClassVar[int] + COLLECTION_FIELD_NUMBER: _ClassVar[int] + TAGS_FIELD_NUMBER: _ClassVar[int] + ANNOTATIONS_FIELD_NUMBER: _ClassVar[int] + uuid: bytes + collection: str + tags: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + annotations: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + def __init__(self, uuid: _Optional[bytes] = ..., collection: _Optional[str] = ..., tags: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., annotations: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ...) -> None: ... -class FaultInjectResponse(_message.Message): - __slots__ = ["rv", "stat"] - RV_FIELD_NUMBER: _ClassVar[int] +class CreateResponse(_message.Message): + __slots__ = ("stat",) STAT_FIELD_NUMBER: _ClassVar[int] - rv: bytes stat: Status - def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., rv: _Optional[bytes] = ...) -> None: ... + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ...) -> None: ... -class FlushParams(_message.Message): - __slots__ = ["uuid"] - UUID_FIELD_NUMBER: _ClassVar[int] - uuid: bytes - def __init__(self, uuid: _Optional[bytes] = ...) -> None: ... +class MetadataUsageParams(_message.Message): + __slots__ = ("prefix", "role") + PREFIX_FIELD_NUMBER: _ClassVar[int] + ROLE_FIELD_NUMBER: _ClassVar[int] + prefix: str + role: Role + def __init__(self, prefix: _Optional[str] = ..., role: _Optional[_Union[Role, _Mapping]] = ...) -> None: ... -class FlushResponse(_message.Message): - __slots__ = ["stat", "versionMajor", "versionMinor"] +class MetadataUsageResponse(_message.Message): + __slots__ = ("stat", "tags", "annotations") STAT_FIELD_NUMBER: _ClassVar[int] - VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] - VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + TAGS_FIELD_NUMBER: _ClassVar[int] + ANNOTATIONS_FIELD_NUMBER: _ClassVar[int] stat: Status - versionMajor: int - versionMinor: int - def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ...) -> None: ... - -class GenerateCSVParams(_message.Message): - __slots__ = ["depth", "endTime", "includeVersions", "queryType", "startTime", "streams", "windowSize"] - class QueryType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): - __slots__ = [] - ALIGNED_WINDOWS_QUERY: GenerateCSVParams.QueryType - DEPTH_FIELD_NUMBER: _ClassVar[int] - ENDTIME_FIELD_NUMBER: _ClassVar[int] - INCLUDEVERSIONS_FIELD_NUMBER: _ClassVar[int] - QUERYTYPE_FIELD_NUMBER: _ClassVar[int] - RAW_QUERY: GenerateCSVParams.QueryType - STARTTIME_FIELD_NUMBER: _ClassVar[int] - STREAMS_FIELD_NUMBER: _ClassVar[int] - WINDOWSIZE_FIELD_NUMBER: _ClassVar[int] - WINDOWS_QUERY: GenerateCSVParams.QueryType - depth: int - endTime: int - includeVersions: bool - queryType: GenerateCSVParams.QueryType - startTime: int - streams: _containers.RepeatedCompositeFieldContainer[StreamCSVConfig] - windowSize: int - def __init__(self, queryType: _Optional[_Union[GenerateCSVParams.QueryType, str]] = ..., startTime: _Optional[int] = ..., endTime: _Optional[int] = ..., windowSize: _Optional[int] = ..., depth: _Optional[int] = ..., includeVersions: bool = ..., streams: _Optional[_Iterable[_Union[StreamCSVConfig, _Mapping]]] = ...) -> None: ... - -class GenerateCSVResponse(_message.Message): - __slots__ = ["isHeader", "row", "stat"] - ISHEADER_FIELD_NUMBER: _ClassVar[int] - ROW_FIELD_NUMBER: _ClassVar[int] - STAT_FIELD_NUMBER: _ClassVar[int] - isHeader: bool - row: _containers.RepeatedScalarFieldContainer[str] - stat: Status - def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., isHeader: bool = ..., row: _Optional[_Iterable[str]] = ...) -> None: ... - -class GetCompactionConfigParams(_message.Message): - __slots__ = ["uuid"] - UUID_FIELD_NUMBER: _ClassVar[int] - uuid: bytes - def __init__(self, uuid: _Optional[bytes] = ...) -> None: ... - -class GetCompactionConfigResponse(_message.Message): - __slots__ = ["CompactedVersion", "LatestMajorVersion", "reducedResolutionRanges", "stat", "unused0"] - COMPACTEDVERSION_FIELD_NUMBER: _ClassVar[int] - CompactedVersion: int - LATESTMAJORVERSION_FIELD_NUMBER: _ClassVar[int] - LatestMajorVersion: int - REDUCEDRESOLUTIONRANGES_FIELD_NUMBER: _ClassVar[int] - STAT_FIELD_NUMBER: _ClassVar[int] - UNUSED0_FIELD_NUMBER: _ClassVar[int] - reducedResolutionRanges: _containers.RepeatedCompositeFieldContainer[ReducedResolutionRange] - stat: Status - unused0: int - def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., LatestMajorVersion: _Optional[int] = ..., CompactedVersion: _Optional[int] = ..., reducedResolutionRanges: _Optional[_Iterable[_Union[ReducedResolutionRange, _Mapping]]] = ..., unused0: _Optional[int] = ...) -> None: ... - -class InfoParams(_message.Message): - __slots__ = [] - def __init__(self) -> None: ... - -class InfoResponse(_message.Message): - __slots__ = ["build", "majorVersion", "mash", "minorVersion", "proxy", "stat"] - BUILD_FIELD_NUMBER: _ClassVar[int] - MAJORVERSION_FIELD_NUMBER: _ClassVar[int] - MASH_FIELD_NUMBER: _ClassVar[int] - MINORVERSION_FIELD_NUMBER: _ClassVar[int] - PROXY_FIELD_NUMBER: _ClassVar[int] - STAT_FIELD_NUMBER: _ClassVar[int] - build: str - majorVersion: int - mash: Mash - minorVersion: int - proxy: ProxyInfo - stat: Status - def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., mash: _Optional[_Union[Mash, _Mapping]] = ..., majorVersion: _Optional[int] = ..., minorVersion: _Optional[int] = ..., build: _Optional[str] = ..., proxy: _Optional[_Union[ProxyInfo, _Mapping]] = ...) -> None: ... - -class InsertParams(_message.Message): - __slots__ = ["merge_policy", "rounding", "sync", "uuid", "values"] - MERGE_POLICY_FIELD_NUMBER: _ClassVar[int] - ROUNDING_FIELD_NUMBER: _ClassVar[int] - SYNC_FIELD_NUMBER: _ClassVar[int] - UUID_FIELD_NUMBER: _ClassVar[int] - VALUES_FIELD_NUMBER: _ClassVar[int] - merge_policy: MergePolicy - rounding: RoundSpec - sync: bool - uuid: bytes - values: _containers.RepeatedCompositeFieldContainer[RawPoint] - def __init__(self, uuid: _Optional[bytes] = ..., sync: bool = ..., merge_policy: _Optional[_Union[MergePolicy, str]] = ..., rounding: _Optional[_Union[RoundSpec, _Mapping]] = ..., values: _Optional[_Iterable[_Union[RawPoint, _Mapping]]] = ...) -> None: ... - -class InsertResponse(_message.Message): - __slots__ = ["stat", "versionMajor", "versionMinor"] - STAT_FIELD_NUMBER: _ClassVar[int] - VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] - VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] - stat: Status - versionMajor: int - versionMinor: int - def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ...) -> None: ... + tags: _containers.RepeatedCompositeFieldContainer[KeyCount] + annotations: _containers.RepeatedCompositeFieldContainer[KeyCount] + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., tags: _Optional[_Iterable[_Union[KeyCount, _Mapping]]] = ..., annotations: _Optional[_Iterable[_Union[KeyCount, _Mapping]]] = ...) -> None: ... class KeyCount(_message.Message): - __slots__ = ["count", "key"] - COUNT_FIELD_NUMBER: _ClassVar[int] + __slots__ = ("key", "count") KEY_FIELD_NUMBER: _ClassVar[int] - count: int + COUNT_FIELD_NUMBER: _ClassVar[int] key: str + count: int def __init__(self, key: _Optional[str] = ..., count: _Optional[int] = ...) -> None: ... -class KeyOptValue(_message.Message): - __slots__ = ["key", "val"] - KEY_FIELD_NUMBER: _ClassVar[int] - VAL_FIELD_NUMBER: _ClassVar[int] - key: str - val: OptValue - def __init__(self, key: _Optional[str] = ..., val: _Optional[_Union[OptValue, _Mapping]] = ...) -> None: ... - -class KeyValue(_message.Message): - __slots__ = ["key", "value"] - KEY_FIELD_NUMBER: _ClassVar[int] - VALUE_FIELD_NUMBER: _ClassVar[int] - key: str - value: str - def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ... - class ListCollectionsParams(_message.Message): - __slots__ = ["prefix", "role"] + __slots__ = ("prefix", "role") PREFIX_FIELD_NUMBER: _ClassVar[int] ROLE_FIELD_NUMBER: _ClassVar[int] prefix: str @@ -340,381 +313,456 @@ class ListCollectionsParams(_message.Message): def __init__(self, prefix: _Optional[str] = ..., role: _Optional[_Union[Role, _Mapping]] = ...) -> None: ... class ListCollectionsResponse(_message.Message): - __slots__ = ["collections", "stat"] - COLLECTIONS_FIELD_NUMBER: _ClassVar[int] + __slots__ = ("stat", "collections") STAT_FIELD_NUMBER: _ClassVar[int] - collections: _containers.RepeatedScalarFieldContainer[str] + COLLECTIONS_FIELD_NUMBER: _ClassVar[int] stat: Status + collections: _containers.RepeatedScalarFieldContainer[str] def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., collections: _Optional[_Iterable[str]] = ...) -> None: ... class LookupStreamsParams(_message.Message): - __slots__ = ["annotations", "collection", "isCollectionPrefix", "role", "tags"] - ANNOTATIONS_FIELD_NUMBER: _ClassVar[int] + __slots__ = ("collection", "isCollectionPrefix", "tags", "annotations", "role") COLLECTION_FIELD_NUMBER: _ClassVar[int] ISCOLLECTIONPREFIX_FIELD_NUMBER: _ClassVar[int] - ROLE_FIELD_NUMBER: _ClassVar[int] TAGS_FIELD_NUMBER: _ClassVar[int] - annotations: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + ANNOTATIONS_FIELD_NUMBER: _ClassVar[int] + ROLE_FIELD_NUMBER: _ClassVar[int] collection: str isCollectionPrefix: bool - role: Role tags: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + annotations: _containers.RepeatedCompositeFieldContainer[KeyOptValue] + role: Role def __init__(self, collection: _Optional[str] = ..., isCollectionPrefix: bool = ..., tags: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., annotations: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., role: _Optional[_Union[Role, _Mapping]] = ...) -> None: ... class LookupStreamsResponse(_message.Message): - __slots__ = ["results", "stat"] - RESULTS_FIELD_NUMBER: _ClassVar[int] + __slots__ = ("stat", "results") STAT_FIELD_NUMBER: _ClassVar[int] - results: _containers.RepeatedCompositeFieldContainer[StreamDescriptor] + RESULTS_FIELD_NUMBER: _ClassVar[int] stat: Status + results: _containers.RepeatedCompositeFieldContainer[StreamDescriptor] def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., results: _Optional[_Iterable[_Union[StreamDescriptor, _Mapping]]] = ...) -> None: ... -class Mash(_message.Message): - __slots__ = ["healthy", "leader", "leaderRevision", "members", "revision", "totalWeight", "unmapped"] - HEALTHY_FIELD_NUMBER: _ClassVar[int] - LEADERREVISION_FIELD_NUMBER: _ClassVar[int] - LEADER_FIELD_NUMBER: _ClassVar[int] - MEMBERS_FIELD_NUMBER: _ClassVar[int] - REVISION_FIELD_NUMBER: _ClassVar[int] - TOTALWEIGHT_FIELD_NUMBER: _ClassVar[int] - UNMAPPED_FIELD_NUMBER: _ClassVar[int] - healthy: bool - leader: str - leaderRevision: int - members: _containers.RepeatedCompositeFieldContainer[Member] - revision: int - totalWeight: int - unmapped: float - def __init__(self, revision: _Optional[int] = ..., leader: _Optional[str] = ..., leaderRevision: _Optional[int] = ..., totalWeight: _Optional[int] = ..., healthy: bool = ..., unmapped: _Optional[float] = ..., members: _Optional[_Iterable[_Union[Member, _Mapping]]] = ...) -> None: ... - -class Member(_message.Message): - __slots__ = ["enabled", "end", "grpcEndpoints", "hash", "httpEndpoints", "nodename", "readPreference", "start", "up", "weight"] - ENABLED_FIELD_NUMBER: _ClassVar[int] - END_FIELD_NUMBER: _ClassVar[int] - GRPCENDPOINTS_FIELD_NUMBER: _ClassVar[int] - HASH_FIELD_NUMBER: _ClassVar[int] - HTTPENDPOINTS_FIELD_NUMBER: _ClassVar[int] - IN_FIELD_NUMBER: _ClassVar[int] - NODENAME_FIELD_NUMBER: _ClassVar[int] - READPREFERENCE_FIELD_NUMBER: _ClassVar[int] - START_FIELD_NUMBER: _ClassVar[int] - UP_FIELD_NUMBER: _ClassVar[int] - WEIGHT_FIELD_NUMBER: _ClassVar[int] - enabled: bool - end: int - grpcEndpoints: str - hash: int - httpEndpoints: str - nodename: str - readPreference: float - start: int - up: bool - weight: int - def __init__(self, hash: _Optional[int] = ..., nodename: _Optional[str] = ..., up: bool = ..., enabled: bool = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., weight: _Optional[int] = ..., readPreference: _Optional[float] = ..., httpEndpoints: _Optional[str] = ..., grpcEndpoints: _Optional[str] = ..., **kwargs) -> None: ... - -class MetadataUsageParams(_message.Message): - __slots__ = ["prefix", "role"] - PREFIX_FIELD_NUMBER: _ClassVar[int] - ROLE_FIELD_NUMBER: _ClassVar[int] - prefix: str - role: Role - def __init__(self, prefix: _Optional[str] = ..., role: _Optional[_Union[Role, _Mapping]] = ...) -> None: ... - -class MetadataUsageResponse(_message.Message): - __slots__ = ["annotations", "stat", "tags"] - ANNOTATIONS_FIELD_NUMBER: _ClassVar[int] - STAT_FIELD_NUMBER: _ClassVar[int] - TAGS_FIELD_NUMBER: _ClassVar[int] - annotations: _containers.RepeatedCompositeFieldContainer[KeyCount] - stat: Status - tags: _containers.RepeatedCompositeFieldContainer[KeyCount] - def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., tags: _Optional[_Iterable[_Union[KeyCount, _Mapping]]] = ..., annotations: _Optional[_Iterable[_Union[KeyCount, _Mapping]]] = ...) -> None: ... - class NearestParams(_message.Message): - __slots__ = ["backward", "time", "uuid", "versionMajor"] - BACKWARD_FIELD_NUMBER: _ClassVar[int] - TIME_FIELD_NUMBER: _ClassVar[int] + __slots__ = ("uuid", "time", "versionMajor", "backward") UUID_FIELD_NUMBER: _ClassVar[int] + TIME_FIELD_NUMBER: _ClassVar[int] VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] - backward: bool - time: int + BACKWARD_FIELD_NUMBER: _ClassVar[int] uuid: bytes + time: int versionMajor: int + backward: bool def __init__(self, uuid: _Optional[bytes] = ..., time: _Optional[int] = ..., versionMajor: _Optional[int] = ..., backward: bool = ...) -> None: ... class NearestResponse(_message.Message): - __slots__ = ["stat", "value", "versionMajor", "versionMinor"] + __slots__ = ("stat", "versionMajor", "versionMinor", "value") STAT_FIELD_NUMBER: _ClassVar[int] - VALUE_FIELD_NUMBER: _ClassVar[int] VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + VALUE_FIELD_NUMBER: _ClassVar[int] stat: Status - value: RawPoint versionMajor: int versionMinor: int + value: RawPoint def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., value: _Optional[_Union[RawPoint, _Mapping]] = ...) -> None: ... -class ObliterateParams(_message.Message): - __slots__ = ["uuid"] - UUID_FIELD_NUMBER: _ClassVar[int] - uuid: bytes - def __init__(self, uuid: _Optional[bytes] = ...) -> None: ... - -class ObliterateResponse(_message.Message): - __slots__ = ["stat"] - STAT_FIELD_NUMBER: _ClassVar[int] - stat: Status - def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ...) -> None: ... - -class OptValue(_message.Message): - __slots__ = ["value"] - VALUE_FIELD_NUMBER: _ClassVar[int] - value: str - def __init__(self, value: _Optional[str] = ...) -> None: ... - -class ProxyInfo(_message.Message): - __slots__ = ["proxyEndpoints"] - PROXYENDPOINTS_FIELD_NUMBER: _ClassVar[int] - proxyEndpoints: _containers.RepeatedScalarFieldContainer[str] - def __init__(self, proxyEndpoints: _Optional[_Iterable[str]] = ...) -> None: ... - -class RawPoint(_message.Message): - __slots__ = ["time", "value"] - TIME_FIELD_NUMBER: _ClassVar[int] - VALUE_FIELD_NUMBER: _ClassVar[int] - time: int - value: float - def __init__(self, time: _Optional[int] = ..., value: _Optional[float] = ...) -> None: ... - -class RawPointVec(_message.Message): - __slots__ = ["time", "value"] - TIME_FIELD_NUMBER: _ClassVar[int] - VALUE_FIELD_NUMBER: _ClassVar[int] - time: int - value: _containers.RepeatedScalarFieldContainer[float] - def __init__(self, time: _Optional[int] = ..., value: _Optional[_Iterable[float]] = ...) -> None: ... - -class RawValuesParams(_message.Message): - __slots__ = ["end", "start", "uuid", "versionMajor"] - END_FIELD_NUMBER: _ClassVar[int] - START_FIELD_NUMBER: _ClassVar[int] +class ChangesParams(_message.Message): + __slots__ = ("uuid", "fromMajor", "toMajor", "resolution") UUID_FIELD_NUMBER: _ClassVar[int] - VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] - end: int - start: int + FROMMAJOR_FIELD_NUMBER: _ClassVar[int] + TOMAJOR_FIELD_NUMBER: _ClassVar[int] + RESOLUTION_FIELD_NUMBER: _ClassVar[int] uuid: bytes - versionMajor: int - def __init__(self, uuid: _Optional[bytes] = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., versionMajor: _Optional[int] = ...) -> None: ... + fromMajor: int + toMajor: int + resolution: int + def __init__(self, uuid: _Optional[bytes] = ..., fromMajor: _Optional[int] = ..., toMajor: _Optional[int] = ..., resolution: _Optional[int] = ...) -> None: ... -class RawValuesResponse(_message.Message): - __slots__ = ["stat", "values", "versionMajor", "versionMinor"] +class ChangesResponse(_message.Message): + __slots__ = ("stat", "versionMajor", "versionMinor", "ranges") STAT_FIELD_NUMBER: _ClassVar[int] - VALUES_FIELD_NUMBER: _ClassVar[int] VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + RANGES_FIELD_NUMBER: _ClassVar[int] stat: Status - values: _containers.RepeatedCompositeFieldContainer[RawPoint] versionMajor: int versionMinor: int - def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., values: _Optional[_Iterable[_Union[RawPoint, _Mapping]]] = ...) -> None: ... - -class ReducedResolutionRange(_message.Message): - __slots__ = ["End", "Resolution", "Start"] - END_FIELD_NUMBER: _ClassVar[int] - End: int - RESOLUTION_FIELD_NUMBER: _ClassVar[int] - Resolution: int - START_FIELD_NUMBER: _ClassVar[int] - Start: int - def __init__(self, Start: _Optional[int] = ..., End: _Optional[int] = ..., Resolution: _Optional[int] = ...) -> None: ... - -class Role(_message.Message): - __slots__ = ["name"] - NAME_FIELD_NUMBER: _ClassVar[int] - name: str - def __init__(self, name: _Optional[str] = ...) -> None: ... + ranges: _containers.RepeatedCompositeFieldContainer[ChangedRange] + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., ranges: _Optional[_Iterable[_Union[ChangedRange, _Mapping]]] = ...) -> None: ... class RoundSpec(_message.Message): - __slots__ = ["bits"] + __slots__ = ("bits",) BITS_FIELD_NUMBER: _ClassVar[int] bits: int def __init__(self, bits: _Optional[int] = ...) -> None: ... -class SQLQueryParams(_message.Message): - __slots__ = ["params", "query", "role"] - PARAMS_FIELD_NUMBER: _ClassVar[int] - QUERY_FIELD_NUMBER: _ClassVar[int] - ROLE_FIELD_NUMBER: _ClassVar[int] - params: _containers.RepeatedScalarFieldContainer[str] - query: str - role: Role - def __init__(self, query: _Optional[str] = ..., params: _Optional[_Iterable[str]] = ..., role: _Optional[_Union[Role, _Mapping]] = ...) -> None: ... +class InsertParams(_message.Message): + __slots__ = ("uuid", "sync", "merge_policy", "rounding", "values") + UUID_FIELD_NUMBER: _ClassVar[int] + SYNC_FIELD_NUMBER: _ClassVar[int] + MERGE_POLICY_FIELD_NUMBER: _ClassVar[int] + ROUNDING_FIELD_NUMBER: _ClassVar[int] + VALUES_FIELD_NUMBER: _ClassVar[int] + uuid: bytes + sync: bool + merge_policy: MergePolicy + rounding: RoundSpec + values: _containers.RepeatedCompositeFieldContainer[RawPoint] + def __init__(self, uuid: _Optional[bytes] = ..., sync: bool = ..., merge_policy: _Optional[_Union[MergePolicy, str]] = ..., rounding: _Optional[_Union[RoundSpec, _Mapping]] = ..., values: _Optional[_Iterable[_Union[RawPoint, _Mapping]]] = ...) -> None: ... -class SQLQueryResponse(_message.Message): - __slots__ = ["SQLQueryRow", "stat"] - SQLQUERYROW_FIELD_NUMBER: _ClassVar[int] - SQLQueryRow: _containers.RepeatedScalarFieldContainer[bytes] +class ArrowInsertParams(_message.Message): + __slots__ = ("uuid", "sync", "merge_policy", "rounding", "arrowBytes") + UUID_FIELD_NUMBER: _ClassVar[int] + SYNC_FIELD_NUMBER: _ClassVar[int] + MERGE_POLICY_FIELD_NUMBER: _ClassVar[int] + ROUNDING_FIELD_NUMBER: _ClassVar[int] + ARROWBYTES_FIELD_NUMBER: _ClassVar[int] + uuid: bytes + sync: bool + merge_policy: MergePolicy + rounding: RoundSpec + arrowBytes: bytes + def __init__(self, uuid: _Optional[bytes] = ..., sync: bool = ..., merge_policy: _Optional[_Union[MergePolicy, str]] = ..., rounding: _Optional[_Union[RoundSpec, _Mapping]] = ..., arrowBytes: _Optional[bytes] = ...) -> None: ... + +class InsertResponse(_message.Message): + __slots__ = ("stat", "versionMajor", "versionMinor") STAT_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] stat: Status - def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., SQLQueryRow: _Optional[_Iterable[bytes]] = ...) -> None: ... + versionMajor: int + versionMinor: int + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ...) -> None: ... -class SetCompactionConfigParams(_message.Message): - __slots__ = ["CompactedVersion", "reducedResolutionRanges", "unused0", "uuid"] - COMPACTEDVERSION_FIELD_NUMBER: _ClassVar[int] - CompactedVersion: int - REDUCEDRESOLUTIONRANGES_FIELD_NUMBER: _ClassVar[int] - UNUSED0_FIELD_NUMBER: _ClassVar[int] +class DeleteParams(_message.Message): + __slots__ = ("uuid", "start", "end") UUID_FIELD_NUMBER: _ClassVar[int] - reducedResolutionRanges: _containers.RepeatedCompositeFieldContainer[ReducedResolutionRange] - unused0: int + START_FIELD_NUMBER: _ClassVar[int] + END_FIELD_NUMBER: _ClassVar[int] uuid: bytes - def __init__(self, uuid: _Optional[bytes] = ..., CompactedVersion: _Optional[int] = ..., reducedResolutionRanges: _Optional[_Iterable[_Union[ReducedResolutionRange, _Mapping]]] = ..., unused0: _Optional[int] = ...) -> None: ... + start: int + end: int + def __init__(self, uuid: _Optional[bytes] = ..., start: _Optional[int] = ..., end: _Optional[int] = ...) -> None: ... -class SetCompactionConfigResponse(_message.Message): - __slots__ = ["stat"] +class DeleteResponse(_message.Message): + __slots__ = ("stat", "versionMajor", "versionMinor") STAT_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] stat: Status - def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ...) -> None: ... + versionMajor: int + versionMinor: int + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ...) -> None: ... -class SetStreamAnnotationsParams(_message.Message): - __slots__ = ["changes", "expectedPropertyVersion", "removals", "uuid"] - CHANGES_FIELD_NUMBER: _ClassVar[int] - EXPECTEDPROPERTYVERSION_FIELD_NUMBER: _ClassVar[int] - REMOVALS_FIELD_NUMBER: _ClassVar[int] +class InfoParams(_message.Message): + __slots__ = () + def __init__(self) -> None: ... + +class InfoResponse(_message.Message): + __slots__ = ("stat", "mash", "majorVersion", "minorVersion", "build", "proxy") + STAT_FIELD_NUMBER: _ClassVar[int] + MASH_FIELD_NUMBER: _ClassVar[int] + MAJORVERSION_FIELD_NUMBER: _ClassVar[int] + MINORVERSION_FIELD_NUMBER: _ClassVar[int] + BUILD_FIELD_NUMBER: _ClassVar[int] + PROXY_FIELD_NUMBER: _ClassVar[int] + stat: Status + mash: Mash + majorVersion: int + minorVersion: int + build: str + proxy: ProxyInfo + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., mash: _Optional[_Union[Mash, _Mapping]] = ..., majorVersion: _Optional[int] = ..., minorVersion: _Optional[int] = ..., build: _Optional[str] = ..., proxy: _Optional[_Union[ProxyInfo, _Mapping]] = ...) -> None: ... + +class ProxyInfo(_message.Message): + __slots__ = ("proxyEndpoints",) + PROXYENDPOINTS_FIELD_NUMBER: _ClassVar[int] + proxyEndpoints: _containers.RepeatedScalarFieldContainer[str] + def __init__(self, proxyEndpoints: _Optional[_Iterable[str]] = ...) -> None: ... + +class FaultInjectParams(_message.Message): + __slots__ = ("type", "params") + TYPE_FIELD_NUMBER: _ClassVar[int] + PARAMS_FIELD_NUMBER: _ClassVar[int] + type: int + params: bytes + def __init__(self, type: _Optional[int] = ..., params: _Optional[bytes] = ...) -> None: ... + +class FaultInjectResponse(_message.Message): + __slots__ = ("stat", "rv") + STAT_FIELD_NUMBER: _ClassVar[int] + RV_FIELD_NUMBER: _ClassVar[int] + stat: Status + rv: bytes + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., rv: _Optional[bytes] = ...) -> None: ... + +class FlushParams(_message.Message): + __slots__ = ("uuid",) UUID_FIELD_NUMBER: _ClassVar[int] - changes: _containers.RepeatedCompositeFieldContainer[KeyOptValue] - expectedPropertyVersion: int - removals: _containers.RepeatedScalarFieldContainer[str] uuid: bytes - def __init__(self, uuid: _Optional[bytes] = ..., expectedPropertyVersion: _Optional[int] = ..., changes: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., removals: _Optional[_Iterable[str]] = ...) -> None: ... + def __init__(self, uuid: _Optional[bytes] = ...) -> None: ... -class SetStreamAnnotationsResponse(_message.Message): - __slots__ = ["stat"] +class FlushResponse(_message.Message): + __slots__ = ("stat", "versionMajor", "versionMinor") STAT_FIELD_NUMBER: _ClassVar[int] + VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] + VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] stat: Status - def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ...) -> None: ... + versionMajor: int + versionMinor: int + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ...) -> None: ... -class SetStreamTagsParams(_message.Message): - __slots__ = ["collection", "expectedPropertyVersion", "remove", "tags", "uuid"] - COLLECTION_FIELD_NUMBER: _ClassVar[int] - EXPECTEDPROPERTYVERSION_FIELD_NUMBER: _ClassVar[int] - REMOVE_FIELD_NUMBER: _ClassVar[int] - TAGS_FIELD_NUMBER: _ClassVar[int] +class ObliterateParams(_message.Message): + __slots__ = ("uuid",) UUID_FIELD_NUMBER: _ClassVar[int] - collection: str - expectedPropertyVersion: int - remove: _containers.RepeatedScalarFieldContainer[str] - tags: _containers.RepeatedCompositeFieldContainer[KeyOptValue] uuid: bytes - def __init__(self, uuid: _Optional[bytes] = ..., expectedPropertyVersion: _Optional[int] = ..., tags: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., collection: _Optional[str] = ..., remove: _Optional[_Iterable[str]] = ...) -> None: ... + def __init__(self, uuid: _Optional[bytes] = ...) -> None: ... -class SetStreamTagsResponse(_message.Message): - __slots__ = ["stat"] +class ObliterateResponse(_message.Message): + __slots__ = ("stat",) STAT_FIELD_NUMBER: _ClassVar[int] stat: Status def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ...) -> None: ... +class RawPoint(_message.Message): + __slots__ = ("time", "value") + TIME_FIELD_NUMBER: _ClassVar[int] + VALUE_FIELD_NUMBER: _ClassVar[int] + time: int + value: float + def __init__(self, time: _Optional[int] = ..., value: _Optional[float] = ...) -> None: ... + class StatPoint(_message.Message): - __slots__ = ["count", "max", "mean", "min", "stddev", "time"] - COUNT_FIELD_NUMBER: _ClassVar[int] - MAX_FIELD_NUMBER: _ClassVar[int] - MEAN_FIELD_NUMBER: _ClassVar[int] + __slots__ = ("time", "min", "mean", "max", "count", "stddev") + TIME_FIELD_NUMBER: _ClassVar[int] MIN_FIELD_NUMBER: _ClassVar[int] + MEAN_FIELD_NUMBER: _ClassVar[int] + MAX_FIELD_NUMBER: _ClassVar[int] + COUNT_FIELD_NUMBER: _ClassVar[int] STDDEV_FIELD_NUMBER: _ClassVar[int] - TIME_FIELD_NUMBER: _ClassVar[int] - count: int - max: float - mean: float + time: int min: float + mean: float + max: float + count: int stddev: float - time: int def __init__(self, time: _Optional[int] = ..., min: _Optional[float] = ..., mean: _Optional[float] = ..., max: _Optional[float] = ..., count: _Optional[int] = ..., stddev: _Optional[float] = ...) -> None: ... +class ChangedRange(_message.Message): + __slots__ = ("start", "end") + START_FIELD_NUMBER: _ClassVar[int] + END_FIELD_NUMBER: _ClassVar[int] + start: int + end: int + def __init__(self, start: _Optional[int] = ..., end: _Optional[int] = ...) -> None: ... + class Status(_message.Message): - __slots__ = ["code", "mash", "msg"] + __slots__ = ("code", "msg", "mash") CODE_FIELD_NUMBER: _ClassVar[int] - MASH_FIELD_NUMBER: _ClassVar[int] MSG_FIELD_NUMBER: _ClassVar[int] + MASH_FIELD_NUMBER: _ClassVar[int] code: int - mash: Mash msg: str + mash: Mash def __init__(self, code: _Optional[int] = ..., msg: _Optional[str] = ..., mash: _Optional[_Union[Mash, _Mapping]] = ...) -> None: ... +class Mash(_message.Message): + __slots__ = ("revision", "leader", "leaderRevision", "totalWeight", "healthy", "unmapped", "members") + REVISION_FIELD_NUMBER: _ClassVar[int] + LEADER_FIELD_NUMBER: _ClassVar[int] + LEADERREVISION_FIELD_NUMBER: _ClassVar[int] + TOTALWEIGHT_FIELD_NUMBER: _ClassVar[int] + HEALTHY_FIELD_NUMBER: _ClassVar[int] + UNMAPPED_FIELD_NUMBER: _ClassVar[int] + MEMBERS_FIELD_NUMBER: _ClassVar[int] + revision: int + leader: str + leaderRevision: int + totalWeight: int + healthy: bool + unmapped: float + members: _containers.RepeatedCompositeFieldContainer[Member] + def __init__(self, revision: _Optional[int] = ..., leader: _Optional[str] = ..., leaderRevision: _Optional[int] = ..., totalWeight: _Optional[int] = ..., healthy: bool = ..., unmapped: _Optional[float] = ..., members: _Optional[_Iterable[_Union[Member, _Mapping]]] = ...) -> None: ... + +class Member(_message.Message): + __slots__ = ("hash", "nodename", "up", "enabled", "start", "end", "weight", "readPreference", "httpEndpoints", "grpcEndpoints") + HASH_FIELD_NUMBER: _ClassVar[int] + NODENAME_FIELD_NUMBER: _ClassVar[int] + UP_FIELD_NUMBER: _ClassVar[int] + IN_FIELD_NUMBER: _ClassVar[int] + ENABLED_FIELD_NUMBER: _ClassVar[int] + START_FIELD_NUMBER: _ClassVar[int] + END_FIELD_NUMBER: _ClassVar[int] + WEIGHT_FIELD_NUMBER: _ClassVar[int] + READPREFERENCE_FIELD_NUMBER: _ClassVar[int] + HTTPENDPOINTS_FIELD_NUMBER: _ClassVar[int] + GRPCENDPOINTS_FIELD_NUMBER: _ClassVar[int] + hash: int + nodename: str + up: bool + enabled: bool + start: int + end: int + weight: int + readPreference: float + httpEndpoints: str + grpcEndpoints: str + def __init__(self, hash: _Optional[int] = ..., nodename: _Optional[str] = ..., up: bool = ..., enabled: bool = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., weight: _Optional[int] = ..., readPreference: _Optional[float] = ..., httpEndpoints: _Optional[str] = ..., grpcEndpoints: _Optional[str] = ..., **kwargs) -> None: ... + +class KeyOptValue(_message.Message): + __slots__ = ("key", "val") + KEY_FIELD_NUMBER: _ClassVar[int] + VAL_FIELD_NUMBER: _ClassVar[int] + key: str + val: OptValue + def __init__(self, key: _Optional[str] = ..., val: _Optional[_Union[OptValue, _Mapping]] = ...) -> None: ... + +class OptValue(_message.Message): + __slots__ = ("value",) + VALUE_FIELD_NUMBER: _ClassVar[int] + value: str + def __init__(self, value: _Optional[str] = ...) -> None: ... + +class KeyValue(_message.Message): + __slots__ = ("key", "value") + KEY_FIELD_NUMBER: _ClassVar[int] + VALUE_FIELD_NUMBER: _ClassVar[int] + key: str + value: str + def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ... + class StreamCSVConfig(_message.Message): - __slots__ = ["label", "uuid", "version"] + __slots__ = ("version", "label", "uuid") + VERSION_FIELD_NUMBER: _ClassVar[int] LABEL_FIELD_NUMBER: _ClassVar[int] UUID_FIELD_NUMBER: _ClassVar[int] - VERSION_FIELD_NUMBER: _ClassVar[int] + version: int label: str uuid: bytes - version: int def __init__(self, version: _Optional[int] = ..., label: _Optional[str] = ..., uuid: _Optional[bytes] = ...) -> None: ... -class StreamDescriptor(_message.Message): - __slots__ = ["annotations", "collection", "propertyVersion", "tags", "uuid"] - ANNOTATIONS_FIELD_NUMBER: _ClassVar[int] - COLLECTION_FIELD_NUMBER: _ClassVar[int] - PROPERTYVERSION_FIELD_NUMBER: _ClassVar[int] - TAGS_FIELD_NUMBER: _ClassVar[int] - UUID_FIELD_NUMBER: _ClassVar[int] - annotations: _containers.RepeatedCompositeFieldContainer[KeyOptValue] - collection: str - propertyVersion: int - tags: _containers.RepeatedCompositeFieldContainer[KeyOptValue] - uuid: bytes - def __init__(self, uuid: _Optional[bytes] = ..., collection: _Optional[str] = ..., tags: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., annotations: _Optional[_Iterable[_Union[KeyOptValue, _Mapping]]] = ..., propertyVersion: _Optional[int] = ...) -> None: ... +class GenerateCSVParams(_message.Message): + __slots__ = ("queryType", "startTime", "endTime", "windowSize", "depth", "includeVersions", "streams") + class QueryType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): + __slots__ = () + ALIGNED_WINDOWS_QUERY: _ClassVar[GenerateCSVParams.QueryType] + WINDOWS_QUERY: _ClassVar[GenerateCSVParams.QueryType] + RAW_QUERY: _ClassVar[GenerateCSVParams.QueryType] + ALIGNED_WINDOWS_QUERY: GenerateCSVParams.QueryType + WINDOWS_QUERY: GenerateCSVParams.QueryType + RAW_QUERY: GenerateCSVParams.QueryType + QUERYTYPE_FIELD_NUMBER: _ClassVar[int] + STARTTIME_FIELD_NUMBER: _ClassVar[int] + ENDTIME_FIELD_NUMBER: _ClassVar[int] + WINDOWSIZE_FIELD_NUMBER: _ClassVar[int] + DEPTH_FIELD_NUMBER: _ClassVar[int] + INCLUDEVERSIONS_FIELD_NUMBER: _ClassVar[int] + STREAMS_FIELD_NUMBER: _ClassVar[int] + queryType: GenerateCSVParams.QueryType + startTime: int + endTime: int + windowSize: int + depth: int + includeVersions: bool + streams: _containers.RepeatedCompositeFieldContainer[StreamCSVConfig] + def __init__(self, queryType: _Optional[_Union[GenerateCSVParams.QueryType, str]] = ..., startTime: _Optional[int] = ..., endTime: _Optional[int] = ..., windowSize: _Optional[int] = ..., depth: _Optional[int] = ..., includeVersions: bool = ..., streams: _Optional[_Iterable[_Union[StreamCSVConfig, _Mapping]]] = ...) -> None: ... -class StreamInfoParams(_message.Message): - __slots__ = ["omitDescriptor", "omitVersion", "role", "uuid"] - OMITDESCRIPTOR_FIELD_NUMBER: _ClassVar[int] - OMITVERSION_FIELD_NUMBER: _ClassVar[int] +class GenerateCSVResponse(_message.Message): + __slots__ = ("stat", "isHeader", "row") + STAT_FIELD_NUMBER: _ClassVar[int] + ISHEADER_FIELD_NUMBER: _ClassVar[int] + ROW_FIELD_NUMBER: _ClassVar[int] + stat: Status + isHeader: bool + row: _containers.RepeatedScalarFieldContainer[str] + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., isHeader: bool = ..., row: _Optional[_Iterable[str]] = ...) -> None: ... + +class SQLQueryParams(_message.Message): + __slots__ = ("query", "params", "role") + QUERY_FIELD_NUMBER: _ClassVar[int] + PARAMS_FIELD_NUMBER: _ClassVar[int] ROLE_FIELD_NUMBER: _ClassVar[int] - UUID_FIELD_NUMBER: _ClassVar[int] - omitDescriptor: bool - omitVersion: bool + query: str + params: _containers.RepeatedScalarFieldContainer[str] role: Role + def __init__(self, query: _Optional[str] = ..., params: _Optional[_Iterable[str]] = ..., role: _Optional[_Union[Role, _Mapping]] = ...) -> None: ... + +class SQLQueryResponse(_message.Message): + __slots__ = ("stat", "SQLQueryRow") + STAT_FIELD_NUMBER: _ClassVar[int] + SQLQUERYROW_FIELD_NUMBER: _ClassVar[int] + stat: Status + SQLQueryRow: _containers.RepeatedScalarFieldContainer[bytes] + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., SQLQueryRow: _Optional[_Iterable[bytes]] = ...) -> None: ... + +class Role(_message.Message): + __slots__ = ("name",) + NAME_FIELD_NUMBER: _ClassVar[int] + name: str + def __init__(self, name: _Optional[str] = ...) -> None: ... + +class SetCompactionConfigParams(_message.Message): + __slots__ = ("uuid", "CompactedVersion", "reducedResolutionRanges", "unused0") + UUID_FIELD_NUMBER: _ClassVar[int] + COMPACTEDVERSION_FIELD_NUMBER: _ClassVar[int] + REDUCEDRESOLUTIONRANGES_FIELD_NUMBER: _ClassVar[int] + UNUSED0_FIELD_NUMBER: _ClassVar[int] uuid: bytes - def __init__(self, uuid: _Optional[bytes] = ..., omitVersion: bool = ..., omitDescriptor: bool = ..., role: _Optional[_Union[Role, _Mapping]] = ...) -> None: ... + CompactedVersion: int + reducedResolutionRanges: _containers.RepeatedCompositeFieldContainer[ReducedResolutionRange] + unused0: int + def __init__(self, uuid: _Optional[bytes] = ..., CompactedVersion: _Optional[int] = ..., reducedResolutionRanges: _Optional[_Iterable[_Union[ReducedResolutionRange, _Mapping]]] = ..., unused0: _Optional[int] = ...) -> None: ... -class StreamInfoResponse(_message.Message): - __slots__ = ["descriptor", "stat", "versionMajor", "versionMinor"] - DESCRIPTOR_FIELD_NUMBER: _ClassVar[int] +class SetCompactionConfigResponse(_message.Message): + __slots__ = ("stat",) STAT_FIELD_NUMBER: _ClassVar[int] - VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] - VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] - descriptor: StreamDescriptor stat: Status - versionMajor: int - versionMinor: int - def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., descriptor: _Optional[_Union[StreamDescriptor, _Mapping]] = ...) -> None: ... + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ...) -> None: ... -class WindowsParams(_message.Message): - __slots__ = ["depth", "end", "start", "uuid", "versionMajor", "width"] - DEPTH_FIELD_NUMBER: _ClassVar[int] - END_FIELD_NUMBER: _ClassVar[int] - START_FIELD_NUMBER: _ClassVar[int] +class GetCompactionConfigParams(_message.Message): + __slots__ = ("uuid",) UUID_FIELD_NUMBER: _ClassVar[int] - VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] - WIDTH_FIELD_NUMBER: _ClassVar[int] - depth: int - end: int - start: int uuid: bytes - versionMajor: int - width: int - def __init__(self, uuid: _Optional[bytes] = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., versionMajor: _Optional[int] = ..., width: _Optional[int] = ..., depth: _Optional[int] = ...) -> None: ... + def __init__(self, uuid: _Optional[bytes] = ...) -> None: ... -class WindowsResponse(_message.Message): - __slots__ = ["stat", "values", "versionMajor", "versionMinor"] +class GetCompactionConfigResponse(_message.Message): + __slots__ = ("stat", "LatestMajorVersion", "CompactedVersion", "reducedResolutionRanges", "unused0") STAT_FIELD_NUMBER: _ClassVar[int] - VALUES_FIELD_NUMBER: _ClassVar[int] - VERSIONMAJOR_FIELD_NUMBER: _ClassVar[int] - VERSIONMINOR_FIELD_NUMBER: _ClassVar[int] + LATESTMAJORVERSION_FIELD_NUMBER: _ClassVar[int] + COMPACTEDVERSION_FIELD_NUMBER: _ClassVar[int] + REDUCEDRESOLUTIONRANGES_FIELD_NUMBER: _ClassVar[int] + UNUSED0_FIELD_NUMBER: _ClassVar[int] stat: Status - values: _containers.RepeatedCompositeFieldContainer[StatPoint] - versionMajor: int - versionMinor: int - def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., versionMajor: _Optional[int] = ..., versionMinor: _Optional[int] = ..., values: _Optional[_Iterable[_Union[StatPoint, _Mapping]]] = ...) -> None: ... + LatestMajorVersion: int + CompactedVersion: int + reducedResolutionRanges: _containers.RepeatedCompositeFieldContainer[ReducedResolutionRange] + unused0: int + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., LatestMajorVersion: _Optional[int] = ..., CompactedVersion: _Optional[int] = ..., reducedResolutionRanges: _Optional[_Iterable[_Union[ReducedResolutionRange, _Mapping]]] = ..., unused0: _Optional[int] = ...) -> None: ... -class MergePolicy(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): - __slots__ = [] +class ReducedResolutionRange(_message.Message): + __slots__ = ("Start", "End", "Resolution") + START_FIELD_NUMBER: _ClassVar[int] + END_FIELD_NUMBER: _ClassVar[int] + RESOLUTION_FIELD_NUMBER: _ClassVar[int] + Start: int + End: int + Resolution: int + def __init__(self, Start: _Optional[int] = ..., End: _Optional[int] = ..., Resolution: _Optional[int] = ...) -> None: ... + +class SubscriptionUpdate(_message.Message): + __slots__ = ("op", "uuid") + OP_FIELD_NUMBER: _ClassVar[int] + UUID_FIELD_NUMBER: _ClassVar[int] + op: SubscriptionUpdateOp + uuid: _containers.RepeatedScalarFieldContainer[bytes] + def __init__(self, op: _Optional[_Union[SubscriptionUpdateOp, str]] = ..., uuid: _Optional[_Iterable[bytes]] = ...) -> None: ... + +class SubscriptionResp(_message.Message): + __slots__ = ("stat", "uuid", "arrowBytes") + STAT_FIELD_NUMBER: _ClassVar[int] + UUID_FIELD_NUMBER: _ClassVar[int] + ARROWBYTES_FIELD_NUMBER: _ClassVar[int] + stat: Status + uuid: bytes + arrowBytes: bytes + def __init__(self, stat: _Optional[_Union[Status, _Mapping]] = ..., uuid: _Optional[bytes] = ..., arrowBytes: _Optional[bytes] = ...) -> None: ... diff --git a/btrdb/grpcinterface/btrdb_pb2_grpc.py b/btrdb/grpcinterface/btrdb_pb2_grpc.py index 52a72bd..d0db693 100644 --- a/btrdb/grpcinterface/btrdb_pb2_grpc.py +++ b/btrdb/grpcinterface/btrdb_pb2_grpc.py @@ -21,7 +21,7 @@ def __init__(self, channel): ) self.ArrowRawValues = channel.unary_stream( '/v5api.BTrDB/ArrowRawValues', - request_serializer=btrdb__pb2.RawValuesParams.SerializeToString, + request_serializer=btrdb__pb2.ArrowRawValuesParams.SerializeToString, response_deserializer=btrdb__pb2.ArrowRawValuesResponse.FromString, ) self.ArrowMultiValues = channel.unary_stream( @@ -139,6 +139,11 @@ def __init__(self, channel): request_serializer=btrdb__pb2.SQLQueryParams.SerializeToString, response_deserializer=btrdb__pb2.SQLQueryResponse.FromString, ) + self.Subscribe = channel.stream_stream( + '/v5api.BTrDB/Subscribe', + request_serializer=btrdb__pb2.SubscriptionUpdate.SerializeToString, + response_deserializer=btrdb__pb2.SubscriptionResp.FromString, + ) class BTrDBServicer(object): @@ -289,6 +294,12 @@ def GenerateCSV(self, request, context): raise NotImplementedError('Method not implemented!') def SQLQuery(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Subscribe(self, request_iterator, context): """rpc SetCompactionConfig(SetCompactionConfigParams) returns (SetCompactionConfigResponse); rpc GetCompactionConfig(GetCompactionConfigParams) returns (GetCompactionConfigResponse); """ @@ -306,7 +317,7 @@ def add_BTrDBServicer_to_server(servicer, server): ), 'ArrowRawValues': grpc.unary_stream_rpc_method_handler( servicer.ArrowRawValues, - request_deserializer=btrdb__pb2.RawValuesParams.FromString, + request_deserializer=btrdb__pb2.ArrowRawValuesParams.FromString, response_serializer=btrdb__pb2.ArrowRawValuesResponse.SerializeToString, ), 'ArrowMultiValues': grpc.unary_stream_rpc_method_handler( @@ -424,6 +435,11 @@ def add_BTrDBServicer_to_server(servicer, server): request_deserializer=btrdb__pb2.SQLQueryParams.FromString, response_serializer=btrdb__pb2.SQLQueryResponse.SerializeToString, ), + 'Subscribe': grpc.stream_stream_rpc_method_handler( + servicer.Subscribe, + request_deserializer=btrdb__pb2.SubscriptionUpdate.FromString, + response_serializer=btrdb__pb2.SubscriptionResp.SerializeToString, + ), } generic_handler = grpc.method_handlers_generic_handler( 'v5api.BTrDB', rpc_method_handlers) @@ -463,7 +479,7 @@ def ArrowRawValues(request, timeout=None, metadata=None): return grpc.experimental.unary_stream(request, target, '/v5api.BTrDB/ArrowRawValues', - btrdb__pb2.RawValuesParams.SerializeToString, + btrdb__pb2.ArrowRawValuesParams.SerializeToString, btrdb__pb2.ArrowRawValuesResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @@ -858,3 +874,20 @@ def SQLQuery(request, btrdb__pb2.SQLQueryResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Subscribe(request_iterator, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.stream_stream(request_iterator, target, '/v5api.BTrDB/Subscribe', + btrdb__pb2.SubscriptionUpdate.SerializeToString, + btrdb__pb2.SubscriptionResp.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/btrdb/stream.py b/btrdb/stream.py index 7343def..e9d4ac5 100644 --- a/btrdb/stream.py +++ b/btrdb/stream.py @@ -17,14 +17,24 @@ import json import logging import re +import uuid import uuid as uuidlib import warnings from collections import deque from collections.abc import Sequence from copy import deepcopy -from typing import List +from typing import TYPE_CHECKING, Dict, List -import pyarrow as pa +import pyarrow +import pyarrow.compute as pc + +try: + import pyarrow as pa +except ImportError: + pa = None + +if TYPE_CHECKING: + import pyarrow as pa from btrdb.exceptions import ( BTrDBError, @@ -52,6 +62,9 @@ MINIMUM_TIME = -(16 << 56) MAXIMUM_TIME = (48 << 56) - 1 + +_ARROW_IMPORT_MSG = """Package pyarrow required, please pip install.""" + try: RE_PATTERN = re._pattern_type except Exception: @@ -140,6 +153,18 @@ def exists(self): ------- bool Indicates whether stream is extant in the BTrDB server. + + Examples + -------- + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream = conn.stream_from_uuid("...") + >>> stream.uuid + UUID('...') + >>> stream.exists() + True + + """ if self._known_to_exist: @@ -165,15 +190,7 @@ def count( Compute the total number of points in the stream Counts the number of points in the specified window and version. By - default returns the latest total count of points in the stream. This - helper method sums the counts of all StatPoints returned by - ``aligned_windows``. Because of this, note that the start and end - timestamps may be adjusted if they are not powers of 2. For smaller - windows of time, you may also need to adjust the pointwidth to ensure - that the count granularity is captured appropriately. - - Alternatively you can set the precise argument to True which will - give an exact count to the nanosecond but may be slower to execute. + default, returns the latest total count of points in the stream. Parameters ---------- @@ -202,6 +219,32 @@ def count( ------- int The total number of points in the stream for the specified window. + + + .. note:: + + This helper method sums the counts of all StatPoints returned by + ``aligned_windows``. Because of this, note that the start and end + timestamps may be adjusted if they are not powers of 2. For smaller + windows of time, you may also need to adjust the ``pointwidth`` to ensure + that the count granularity is captured appropriately. + + Alternatively you can set the ``precise`` argument to ``True`` which will + give an exact count to the nanosecond but may be slower to execute. + + + + Examples + -------- + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream = conn.stream_from_uuid("...") + >>> stream.count() + 1234 + >>> stream.count(start=1500000000000000000, end=1603680000000000000, pointwidth=55) + 567 + >>> stream.count(start=1500000000000000000, end=1603680000000000000, precise=True) + 789 """ if not precise: @@ -220,16 +263,26 @@ def count( @property def btrdb(self): """ - Returns the stream's BTrDB object. + Returns the stream's BTrDB object. - Parameters - ---------- - None + Parameters + ---------- + None - Returns - ------- - BTrDB - The BTrDB database object. + Returns + ------- + BTrDB + The BTrDB database object. + + Examples + -------- + + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream = conn.stream_from_uuid("...") + >>> btrdb_obj = stream.btrdb + >>> btrdb_obj + """ return self._btrdb @@ -265,6 +318,14 @@ def name(self): str The name of the stream. + Examples + -------- + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream = conn.stream_from_uuid("...") + >>> stream.name + 'foo' + """ return self.tags()["name"] @@ -280,6 +341,14 @@ def unit(self): str The unit for values of the stream. + Examples + -------- + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream = conn.stream_from_uuid("...") + >>> stream.unit + 'volts' + """ return self.tags()["unit"] @@ -298,6 +367,14 @@ def collection(self): str the collection of the stream + Examples + -------- + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream = conn.stream_from_uuid("...") + >>> stream.collection + 'foo/bar' + """ if self._collection is not None: return self._collection @@ -341,6 +418,20 @@ def earliest( The first data point in the stream and the version of the stream the value was retrieved at (tuple(RawPoint, int)). + + Examples + -------- + Get the earliest point for a stream using ``version`` ``0``. + + >>> stream.earliest(version=0) + (, 1234567) + + Extract just the ``RawPoint`` data. + + >>> pt, _ = stream.earliest(version=0) + >>> print(pt.time, pt.value) + 1547241923338098176 123.7 + """ start = MINIMUM_TIME return self.nearest(start, version=version, backward=False) @@ -381,6 +472,18 @@ def latest( The last data point in the stream and the version of the stream the value was retrieved at (tuple(RawPoint, int)). + Examples + -------- + Get the latest point for a stream using ``version`` ``0``. + + >>> stream.latest(version=0) + (, 1234567) + + Extract just the ``RawPoint`` data. + + >>> pt, _ = stream.latest(version=0) + >>> print(pt.time, pt.value) + 1547241923338098176 123.7 """ start = MAXIMUM_TIME return self.nearest(start, version=version, backward=True) @@ -397,7 +500,7 @@ def current( """ Returns the point that is closest to the current timestamp, e.g. the latest point in the stream up until now. Note that no future values will be returned. - Returns None if errors during lookup or there are no values before now. + Returns None if errors occur during lookup or there are no values before now. Parameters ---------- @@ -508,6 +611,25 @@ def annotations( A tuple containing a dictionary of annotations and an integer representing the version of the metadata (tuple(dict, int)). + + .. note:: + + This ``version`` is not the same as the ``stream.version``. + + Examples + -------- + Accessing a streams annotations. + + >>> stream.annotations() + ({"foo":"bar", "baz":"bazaar"}, 231) + + Extract the version and metadata separately. + + >>> annotations, metadata_version = stream.annotations() + >>> annotations + {"foo":"bar", "baz":"bazaar"} + >>> metadata_version + 231 """ if refresh or self._annotations is None: self.refresh_metadata() @@ -525,9 +647,11 @@ def version( """ Returns the current data version of the stream. - Version returns the current data version of the stream. This is not - cached, it queries each time. Take care that you do not intorduce races - in your code by assuming this function will always return the same vaue + .. warning:: + + Version returns the current data version of the stream. This is not + cached, it queries each time. Take care that you do not introduce races + in your code by assuming this function will always return the same value. Parameters ---------- @@ -577,6 +701,18 @@ def insert(self, data, merge="never"): ------- int The version of the stream after inserting new points. + + + Examples + -------- + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream = conn.stream_from_uuid("...") + >>> data = [(1500000000000000000, 1.0), (1500000000100000000, 2.0)] + >>> stream.insert(data) + 1234 + >>> stream.insert(data, merge="replace") + 1235 """ version = 0 i = 0 @@ -613,12 +749,44 @@ def arrow_insert(self, data: pa.Table, merge: str = "never") -> int: int The version of the stream after inserting new points. - Notes - ----- - This method is available for commercial customers with arrow-enabled servers. + + .. note:: + + This method is available for commercial customers with arrow-enabled servers. + + Examples + -------- + Assuming we have a sequence of ``times`` and ``values`` where ``times`` are in nanoseconds. + Insert the data as a pyarrow table, and if there are duplicate timestamps already in the database, + replace with the new ones in ``payload``. + + >>> conn = btrdb.connect() + >>> import pyarrow as pa + >>> for t, v in zip(times, vals): + ... print(t,v) + 1500000000000000000 1.0 + 1500000000100000000 2.0 + 1500000000200000000 3.0 + 1500000000300000000 4.0 + 1500000000400000000 5.0 + 1500000000500000000 6.0 + 1500000000600000000 7.0 + 1500000000700000000 8.0 + 1500000000800000000 9.0 + 1500000000900000000 10.0 + >>> schema = pa.schema( + ... [ + ... pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False), + ... pa.field("value", pa.float64(), nullable=False), + ... ] + ... ) + >>> payload = pa.Table.from_arrays([times, vals], schema=schema) + >>> version = stream.arrow_insert(payload, merge="replace") """ if not self._btrdb._ARROW_ENABLED: raise NotImplementedError(_arrow_not_impl_str.format("arrow_insert")) + if pa is None: + raise ImportError(_ARROW_IMPORT_MSG) chunksize = INSERT_BATCH_SIZE assert isinstance(data, pa.Table) tmp_table = data.rename_columns(["time", "value"]) @@ -652,6 +820,9 @@ def arrow_insert(self, data: pa.Table, merge: str = "never") -> int: return max(version) def _update_tags_collection(self, tags, collection): + """ + :meta private: + """ tags = self.tags() if tags is None else tags collection = self.collection if collection is None else collection if collection is None: @@ -667,6 +838,9 @@ def _update_tags_collection(self, tags, collection): ) def _update_annotations(self, annotations, encoder, replace): + """ + :meta private: + """ # make a copy of the annotations to prevent accidental mutable object mutation serialized = deepcopy(annotations) if encoder is not None: @@ -712,14 +886,10 @@ def update( None, they will remain unchanged in the database. To delete either tags or annotations, you must specify exactly which - keys and values you want set for the field and set `replace=True`. For - example: + keys and values you want set for the field and set ``replace=True``. - >>> annotations, _ = stream.anotations() - >>> del annotations["key_to_delete"] - >>> stream.update(annotations=annotations, replace=True) - This ensures that all of the keys and values for the annotations are + This ensures that all the keys and values for the annotations are preserved except for the key to be deleted. Parameters @@ -735,11 +905,11 @@ def update( Specify a new collection for the stream. If None, the collection will remain unchanged. encoder : json.JSONEncoder or None - JSON encoder class to use for annotation serialization. Set to None + JSON encoder class to use for annotation serialization. Set to ``None`` to prevent JSON encoding of the annotations. replace : bool, default: False Replace all annotations or tags with the specified dictionaries - instead of performing the normal upsert operation. Specifying True + instead of performing the normal upsert operation. Specifying ``True`` is the only way to remove annotation keys. auto_retry: bool, default: False Whether to retry this request in the event of an error @@ -758,6 +928,18 @@ def update( int The version of the metadata (separate from the version of the data) also known as the "property version". + + + Examples + -------- + >>> annotations, _ = stream.anotations() + >>> del annotations["key_to_delete"] + >>> stream.update(annotations=annotations, replace=True) + 12345 + >>> annotations, _ = stream.annotations() + >>> "key_to_delete" in annotations + False + """ if tags is None and annotations is None and collection is None: raise BTRDBValueError( @@ -794,21 +976,25 @@ def delete( retry_backoff=4, ): """ - "Delete" all points between [`start`, `end`) + "Delete" all points between [``start``, ``end``) + + "Delete" all points between ``start`` (inclusive) and ``end`` (exclusive), + both in nanoseconds. + + .. note:: - "Delete" all points between `start` (inclusive) and `end` (exclusive), - both in nanoseconds. As BTrDB has persistent multiversioning, the - deleted points will still exist as part of an older version of the - stream. + As ``BTrDB`` has persistent multiversioning, the + deleted points will still exist as part of an older version of the + stream. Parameters ---------- start : int or datetime like object The start time in nanoseconds for the range to be deleted. (see - :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) + :func:``btrdb.utils.timez.to_nanoseconds`` for valid input types) end : int or datetime like object The end time in nanoseconds for the range to be deleted. (see - :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) + :func:``btrdb.utils.timez.to_nanoseconds`` for valid input types) auto_retry: bool, default: False Whether to retry this request in the event of an error retries: int, default: 5 @@ -825,6 +1011,19 @@ def delete( ------- int The version of the new stream created + + + Examples + -------- + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream = conn.stream_from_uuid("...") + >>> start = 1500000000000000000 + >>> end = 1500000001000000000 + >>> stream.delete(start, end) + 1234 + >>> stream.count(start=start, end=end) + 0 """ return self._btrdb.ep.deleteRange( self._uuid, to_nanoseconds(start), to_nanoseconds(end) @@ -877,13 +1076,13 @@ def values( version (list(tuple(RawPoint,int))). - Notes - ----- - Note that the raw data points are the original values at the sensor's - native sampling rate (assuming the time series represents measurements - from a sensor). This is the lowest level of data with the finest time - granularity. In the tree data structure of BTrDB, this data is stored in - the vector nodes. + .. note:: + + Note that the raw data points are the original values at the sensor's + native sampling rate (assuming the time series represents measurements + from a sensor). This is the lowest level of data with the finest time + granularity. In the tree data structure of BTrDB, this data is stored in + the vector nodes. """ materialized = [] @@ -905,6 +1104,7 @@ def arrow_values( retries=5, retry_delay=3, retry_backoff=4, + schema=None, ) -> pa.Table: """Read raw values from BTrDB between time [a, b) in nanoseconds. @@ -920,6 +1120,9 @@ def arrow_values( :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) version: int, default: 0 The version of the stream to be queried + schema: pyarrow.Schema + Optional arrow schema the server will cast the returned data to before sending it over + the network. You can use this to change the timestamp format, column names or data sizes. auto_retry: bool, default: False Whether to retry this request in the event of an error retries: int, default: 5 @@ -939,34 +1142,42 @@ def arrow_values( A pyarrow table of the raw values with time and value columns. - Notes - ----- - Note that the raw data points are the original values at the sensor's - native sampling rate (assuming the time series represents measurements - from a sensor). This is the lowest level of data with the finest time - granularity. In the tree data structure of BTrDB, this data is stored in - the vector nodes. + .. note:: + + Note that the raw data points are the original values at the sensor's + native sampling rate (assuming the time series represents measurements + from a sensor). This is the lowest level of data with the finest time + granularity. In the tree data structure of BTrDB, this data is stored in + the vector nodes. + + + .. note:: + + This method is available for commercial customers with arrow-enabled servers. + - This method is available for commercial customers with arrow-enabled servers. """ if not self._btrdb._ARROW_ENABLED: raise NotImplementedError(_arrow_not_impl_str.format("arrow_values")) + if pa is None: + raise ImportError(_ARROW_IMPORT_MSG) start = to_nanoseconds(start) end = to_nanoseconds(end) arrow_and_versions = self._btrdb.ep.arrowRawValues( - uu=self.uuid, start=start, end=end, version=version + uu=self.uuid, start=start, end=end, version=version, schema=schema ) tables = list(arrow_and_versions) if len(tables) > 0: tabs, ver = zip(*tables) return pa.concat_tables(tabs) else: - schema = pa.schema( - [ - pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False), - pa.field("value", pa.float64(), nullable=False), - ] - ) + if schema is None: + schema = pa.schema( + [ + pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False), + pa.field("value", pa.float64(), nullable=False), + ] + ) return pa.Table.from_arrays([pa.array([]), pa.array([])], schema=schema) @retry @@ -985,16 +1196,18 @@ def aligned_windows( Read statistical aggregates of windows of data from BTrDB. Query BTrDB for aggregates (or roll ups or windows) of the time series - with `version` between time `start` (inclusive) and `end` (exclusive) in + with `version` between time ``start`` (inclusive) and ``end`` (exclusive) in nanoseconds. Each point returned is a statistical aggregate of all the - raw data within a window of width 2**`pointwidth` nanoseconds. These + raw data within a window of width ``2**pointwidth`` nanoseconds. These statistical aggregates currently include the mean, minimum, and maximum of the data and the count of data points composing the window. - Note that `start` is inclusive, but `end` is exclusive. That is, results + .. note:: + + ``start`` is inclusive, but ``end`` is exclusive. That is, results will be returned for all windows that start in the interval [start, end). If end < start+2^pointwidth you will not get any results. If start and - end are not powers of two, the bottom pointwidth bits will be cleared. + end are not powers of two, the bottom ``pointwidth`` bits will be cleared. Each window will contain statistical summaries of the window. Statistical points with count == 0 will be omitted. @@ -1029,10 +1242,12 @@ def aligned_windows( containing data tuples. Each data tuple contains a StatPoint and the stream version. - Notes - ----- - As the window-width is a power-of-two, it aligns with BTrDB internal - tree data structure and is faster to execute than `windows()`. + + .. note:: + + As the window-width is a power-of-two, it aligns with ``BTrDB`` internal + tree data structure and is faster to execute than ``windows()``. + """ materialized = [] start = to_nanoseconds(start) @@ -1051,7 +1266,6 @@ def arrow_aligned_windows( start: int, end: int, pointwidth: int, - sort_time: bool = False, version: int = 0, auto_retry=False, retries=5, @@ -1067,25 +1281,26 @@ def arrow_aligned_windows( statistical aggregates currently include the mean, minimum, and maximum of the data and the count of data points composing the window. - Note that `start` is inclusive, but `end` is exclusive. That is, results - will be returned for all windows that start in the interval [start, end). - If end < start+2^pointwidth you will not get any results. If start and - end are not powers of two, the bottom pointwidth bits will be cleared. - Each window will contain statistical summaries of the window. - Statistical points with count == 0 will be omitted. + + .. note:: + + ``start`` is inclusive, but ``end`` is exclusive. That is, results + will be returned for all windows that start in the interval [start, end). + If end < start+2^pointwidth you will not get any results. If start and + end are not powers of two, the bottom ``pointwidth`` bits will be cleared. + Each window will contain statistical summaries of the window. + Statistical points with ``count == 0`` will be omitted. Parameters ---------- start : int or datetime like object, required The start time in nanoseconds for the range to be queried. (see - :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) + :func:``btrdb.utils.timez.to_nanoseconds`` for valid input types) end : int or datetime like object, required The end time in nanoseconds for the range to be queried. (see - :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) + :func:``btrdb.utils.timez.to_nanoseconds`` for valid input types) pointwidth : int, required Specify the number of ns between data points (2**pointwidth) - sort_time : bool, default: False - Should the table be sorted on the 'time' column? version : int, default: 0 Version of the stream to query auto_retry: bool, default: False @@ -1105,17 +1320,22 @@ def arrow_aligned_windows( pyarrow.Table Returns a pyarrow table containing the windows of data. - Notes - ----- - As the window-width is a power-of-two, it aligns with BTrDB internal - tree data structure and is faster to execute than `windows()`. - This method is available for commercial customers with arrow-enabled servers. + .. note:: + + As the window-width is a power-of-two, it aligns with BTrDB internal + tree data structure and is faster to execute than `windows()`. + + .. note:: + + This method is available for commercial customers with arrow-enabled servers. """ if not self._btrdb._ARROW_ENABLED: raise NotImplementedError( _arrow_not_impl_str.format("arrow_aligned_windows") ) + if pa is None: + raise ImportError(_ARROW_IMPORT_MSG) if IS_DEBUG: logger.debug(f"For stream - {self.uuid} - {self.name}") @@ -1128,10 +1348,9 @@ def arrow_aligned_windows( ) if len(tables) > 0: tabs, ver = zip(*tables) - if sort_time: - return pa.concat_tables(tabs).sort_by("time") - else: - return pa.concat_tables(tabs) + # assume that time column is the first column returned + time_col = tabs[0].column_names[0] + return pa.concat_tables(tabs).sort_by(time_col) else: schema = pa.schema( [ @@ -1193,21 +1412,21 @@ def windows( containing data tuples. Each data tuple contains a StatPoint and the stream version (tuple(tuple(StatPoint, int), ...)). - Notes - ----- - Windows returns arbitrary precision windows from BTrDB. It is slower - than AlignedWindows, but still significantly faster than RawValues. Each - returned window will be `width` nanoseconds long. `start` is inclusive, - but `end` is exclusive (e.g if end < start+width you will get no - results). That is, results will be returned for all windows that start - at a time less than the end timestamp. If (`end` - `start`) is not a - multiple of width, then end will be decreased to the greatest value less - than end such that (end - start) is a multiple of `width` (i.e., we set - end = start + width * floordiv(end - start, width). The `depth` - parameter previously available has been deprecated. The only valid value - for depth is now 0. - This method is available for commercial customers with arrow-enabled servers. + .. note:: + + ``windows`` returns arbitrary precision windows from BTrDB. It is slower + than ``aligned_windows``, but can be significantly faster than raw value queries (``values``). Each + returned window will be ``width`` nanoseconds long. ``start`` is inclusive, + but ``end`` is exclusive (e.g if ``end < start+width`` you will get no + results). That is, results will be returned for all windows that start + at a time less than the end timestamp. If (``end`` - ``start``) is not a + multiple of ``width``, then ``end`` will be decreased to the greatest value less + than ``end`` such that (``end`` - ``start``) is a multiple of ``width`` (i.e., we set + ``end = start + width * floordiv(end - start, width)``. The ``depth`` + parameter previously available has been deprecated. The only valid value + for ``depth`` is now ``0``. + """ materialized = [] start = to_nanoseconds(start) @@ -1225,7 +1444,6 @@ def arrow_windows( start: int, end: int, width: int, - sort_time: bool = False, version: int = 0, auto_retry=False, retries=5, @@ -1244,8 +1462,6 @@ def arrow_windows( :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) width : int, required The number of nanoseconds in each window. - sort_time : bool, default: False - Should the table be sorted on the 'time' column. version : int, default=0, optional The version of the stream to query. auto_retry: bool, default: False @@ -1265,23 +1481,29 @@ def arrow_windows( pyarrow.Table Returns a pyarrow Table containing windows of data. - Notes - ----- - Windows returns arbitrary precision windows from BTrDB. It is slower - than AlignedWindows, but still significantly faster than RawValues. Each - returned window will be `width` nanoseconds long. `start` is inclusive, - but `end` is exclusive (e.g if end < start+width you will get no - results). That is, results will be returned for all windows that start - at a time less than the end timestamp. If (`end` - `start`) is not a - multiple of width, then end will be decreased to the greatest value less - than end such that (end - start) is a multiple of `width` (i.e., we set - end = start + width * floordiv(end - start, width). The `depth` - parameter previously available has been deprecated. The only valid value - for depth is now 0. - This method is available for commercial customers with arrow-enabled servers. + + .. note:: + + ``windows`` returns arbitrary precision windows from ``BTrDB``. It is slower + than ``aligned_windows``, but still significantly faster than RawValues. Each + returned window will be ``width`` nanoseconds long. ``start`` is inclusive, + but ``end`` is exclusive (e.g if end < start+width you will get no + results). That is, results will be returned for all windows that start + at a time less than the ``end`` timestamp. If (``end`` - ``start``) is not a + multiple of ``width``, then ``end`` will be decreased to the greatest value less + than ``end`` such that (``end`` - ``start``) is a multiple of `width` (i.e., we set + ``end = start + width * floordiv(end - start, width)``. The ``depth`` + parameter previously available has been deprecated. The only valid value + for ``depth`` is now 0. + + .. note:: + + This method is available for commercial customers with arrow-enabled servers. """ if not self._btrdb._ARROW_ENABLED: raise NotImplementedError(_arrow_not_impl_str.format("arrow_windows")) + if pa is None: + raise ImportError(_ARROW_IMPORT_MSG) start = to_nanoseconds(start) end = to_nanoseconds(end) tables = list( @@ -1296,10 +1518,8 @@ def arrow_windows( ) if len(tables) > 0: tabs, ver = zip(*tables) - if sort_time: - return pa.concat_tables(tabs).sort_by("time") - else: - return pa.concat_tables(tabs) + time_col = tabs[0].column_names[0] + return pa.concat_tables(tabs).sort_by(time_col) else: schema = pa.schema( [ @@ -1327,17 +1547,17 @@ def nearest( """ Finds the closest point in the stream to a specified time. - Return the point nearest to the specified `time` in nanoseconds since - Epoch in the stream with `version` while specifying whether to search - forward or backward in time. If `backward` is false, the returned point - will be >= `time`. If backward is true, the returned point will be < - `time`. The version of the stream used to satisfy the query is returned. + Return the point nearest to the specified ``time`` in nanoseconds since + Epoch in the stream with ``version`` while specifying whether to search + forward or backward in time. If ``backward`` is ``false``, the returned point + will be >= ``time``. If ``backward`` is ``true``, the returned point will be < + ``time``. The ``version`` of the ``stream`` used to satisfy the query is returned. Parameters ---------- time : int or datetime like object The time (in nanoseconds since Epoch) to search near (see - :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) + :func:``btrdb.utils.timez.to_nanoseconds`` for valid input types) version : int Version of the stream to use in search backward : boolean @@ -1480,6 +1700,9 @@ def allow_window(self): return not bool(self.pointwidth or (self.width and self.depth == 0)) def _latest_versions(self): + """ + :meta private: + """ uuid_ver_tups = self._btrdb._executor.map( lambda s: (s.uuid, s.version()), self._streams ) @@ -1489,7 +1712,7 @@ def pin_versions(self, versions=None): """ Saves the stream versions that future materializations should use. If no pin is requested then the first materialization will automatically - pin the return versions. Versions can also be supplied through a dict + pin the return versions. Versions can also be supplied through a ``dict`` object with key:UUID, value:stream.version(). Parameters @@ -1502,6 +1725,10 @@ def pin_versions(self, versions=None): StreamSet Returns self + Examples + -------- + >>> version_map = {s.uuid: 0 for s in streamset} + >>> pinned_streamset = streamset.pin_versions(versions=version_map) """ if versions is not None: if not isinstance(versions, dict): @@ -1530,6 +1757,18 @@ def versions(self): dict A dict containing the stream UUID and version ints as key/values + + Examples + -------- + A pinned vs non-pinned streamset + + >>> streamset = btrdb.stream.StreamSet([stream1, stream2]) + >>> version_map = {s.uuid: 0 for s streamset} + >>> pinned_streamset = streamset.pin_versions(versions=version_map) + >>> pinned_streamset.versions() + {UUID('fa42f64a-a851-408f-aa7e-88a85b3d295c'): 0, UUID('18e5527a-ed13-424d-bb97-3e06a763609e'): 0} + >>> streamset.versions() + {UUID('fa42f64a-a851-408f-aa7e-88a85b3d295c'): 34532, UUID('18e5527a-ed13-424d-bb97-3e06a763609e'): 12345} """ return ( self._pinned_versions if self._pinned_versions else self._latest_versions() @@ -1544,10 +1783,6 @@ def count(self, precise: bool = False): all points in the streams. The count is modified by start and end filters or by pinning versions. - Note that this helper method sums the counts of all StatPoints returned - by ``aligned_windows``. Because of this the start and end timestamps - may be adjusted if they are not powers of 2. - Parameters ---------- precise : bool, default = False @@ -1557,6 +1792,27 @@ def count(self, precise: bool = False): ------- int The total number of points in all streams for the specified filters. + + .. note:: + + Note that this helper method sums the counts of all StatPoints returned + by ``aligned_windows``. Because of this the start and end timestamps + may be adjusted if they are not powers of 2. + + Examples + -------- + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream1 = conn.stream_from_uuid("...") + >>> stream2 = conn.stream_from_uuid("...") + >>> streamset = btrdb.stream.StreamSet([stream1, stream2]) + >>> streamset.count() + 2345 + >>> filtered_streamset = streamset.filter(start=1500000000000000000, end=1500000001000000000) + >>> filtered_streamset.count(precise=True) + 734 + >>> streamset.filter(start=1500000000000000000, end=1500000001000000000).count(precise=True) + 734 """ params = self._params_from_filters() start = params.get("start", MINIMUM_TIME) @@ -1588,6 +1844,15 @@ def earliest(self): tuple The earliest points of data found among all streams + Examples + -------- + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream1 = conn.stream_from_uuid("...") + >>> stream2 = conn.stream_from_uuid("...") + >>> streamset = btrdb.stream.StreamSet([stream1, stream2]) + >>> streamset.earliest() + (, ) """ earliest = [] params = self._params_from_filters() @@ -1599,8 +1864,11 @@ def earliest(self): lambda s: s.nearest(start, version=versions.get(s.uuid, 0), backward=False), self._streams, ) - for point, _ in earliest_points_gen: - earliest.append(point) + for point in earliest_points_gen: + if point is not None: + earliest.append(point[0]) + else: + earliest.append(None) return tuple(earliest) @@ -1617,6 +1885,15 @@ def latest(self): tuple The latest points of data found among all streams + Examples + -------- + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream1 = conn.stream_from_uuid("...") + >>> stream2 = conn.stream_from_uuid("...") + >>> streamset = btrdb.stream.StreamSet([stream1, stream2]) + >>> streamset.earliest() + (, ) """ latest = [] params = self._params_from_filters() @@ -1626,8 +1903,11 @@ def latest(self): lambda s: s.nearest(start, version=versions.get(s.uuid, 0), backward=True), self._streams, ) - for point, _ in latest_points_gen: - latest.append(point) + for point in latest_points_gen: + if point is not None: + latest.append(point[0]) + else: + latest.append(None) return tuple(latest) @@ -1641,6 +1921,16 @@ def current(self): ------- tuple The latest points of data found among all streams + + Examples + -------- + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream1 = conn.stream_from_uuid("...") + >>> stream2 = conn.stream_from_uuid("...") + >>> streamset = btrdb.stream.StreamSet([stream1, stream2]) + >>> streamset.current() + (, ) """ latest = [] params = self._params_from_filters() @@ -1654,12 +1944,15 @@ def current(self): ) versions = self.versions() - latest_points_gen = self._btrdb._executor.map( + current_points_gen = self._btrdb._executor.map( lambda s: (s.nearest(now, version=versions.get(s.uuid, 0), backward=True)), self._streams, ) - for point in latest_points_gen: - latest.append(point) + for point in current_points_gen: + if point is not None: + latest.append(point[0]) + else: + latest.append(None) return tuple(latest) @@ -1673,6 +1966,7 @@ def filter( tags=None, annotations=None, sampling_frequency=None, + schema=None, ): """ Provides a new StreamSet instance containing stored query parameters and @@ -1710,25 +2004,61 @@ def filter( key/value pairs for filtering streams based on annotations sampling_frequency : float The sampling frequency of the data streams in Hz, set this if you want timesnapped values. + schema: pyarrow.Schema + Optional arrow schema the server will cast the returned data to before sending it over + the network. You can use this to change the timestamp format, column names or data sizes. Returns ------- StreamSet a new instance cloned from the original with filters applied - Notes - ----- - If you set `sampling_frequency` to a non-zero value, the stream data returned will be aligned to a - grid of timestamps based on the period of the sampling frequency. For example, a sampling rate of 30hz will - have a sampling period of 1/30hz -> ~33_333_333 ns per sample. Leave sampling_frequency as None, or set to 0 to - prevent time alignment. You should **not** use aligned data for frequency-based analysis. + + .. note:: + + If you set ``sampling_frequency`` to a non-zero value, the stream data returned will be aligned to a + grid of timestamps based on the period of the sampling frequency. For example, a sampling rate of 30hz will + have a sampling period of 1/30hz -> ~33_333_333 ns per sample. Leave ``sampling_frequency`` as ``None``, or set to ``0`` to + prevent time alignment. You should **not** use aligned data for frequency-based analysis. + + + Examples + -------- + create a streamset and apply a few filters + + >>> streamset = btrdb.stream.StreamSet(list_of_streams) + >>> print(f"Total streams: {len(streamset)}") + Total streams: 89 + + >>> streamset.filter(units="Volts") + >>> print(f"Total streams: {len(streamset)}") + Total streams: 89 + + >>> filtered_streamset = streamset.filter(units="Volts") + >>> print(f"Total streams: {len(filtered_streamset)}") + Total streams: 23 + + >>> multiple_filters_streamset = (streamset.filter(unit="Volts") + >>> .filter(name="Sensor 1") + >>> .filter(annotations={"phase":"A"}) + >>> ) + >>> print(f"Total streams: {len(multiple_filters_streamset)}") + Total streams: 1 """ obj = self.clone() - if start is not None or end is not None or sampling_frequency is not None: + if ( + start is not None + or end is not None + or sampling_frequency is not None + or schema is not None + ): obj.filters.append( StreamFilter( - start=start, end=end, sampling_frequency=sampling_frequency + start=start, + end=end, + sampling_frequency=sampling_frequency, + schema=schema, ) ) @@ -1839,19 +2169,35 @@ def windows(self, width, depth=0): Returns self - Notes - ----- - Windows returns arbitrary precision windows from BTrDB. It is slower - than aligned_windows, but still significantly faster than values. Each - returned window will be width nanoseconds long. start is inclusive, but - end is exclusive (e.g. if end < start+width you will get no results). - That is, results will be returned for all windows that start at a time - less than the end timestamp. If (end - start) is not a multiple of - width, then end will be decreased to the greatest value less than end - such that (end - start) is a multiple of width (i.e., we set end = start - + width * floordiv(end - start, width)). The `depth` parameter previously - available has been deprecated. The only valid value for depth is now 0. + .. note:: + + ``windows`` returns arbitrary precision windows from BTrDB. It is slower + than ``aligned_windows``, but can be significantly faster than values. + Each returned window will be ``width`` nanoseconds long. + ``start`` is inclusive, but ``end`` is exclusive ( ``[start, end) ) + (e.g. if end < start+width you will get no results). + That is, results will be returned for all windows that start at a time + less than the end timestamp. If (``end`` - ``start``) is not a multiple of + ``width``, then ``end`` will be decreased to the greatest value less than ``end`` + such that (end - start) is a multiple of width (i.e., we set end = start + + width * floordiv(end - start, width)). The ``depth`` parameter previously + available has been deprecated. The only valid value for ``depth`` is now ``0``. + Examples + -------- + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream1 = conn.stream_from_uuid("...") + >>> stream2 = conn.stream_from_uuid("...") + >>> streamset = btrdb.stream.StreamSet([stream1, stream2]) + >>> streamset.windows(width=1000000000) + + >>> streamset.windows(width=1000000000, depth=0) + + >>> streamset.aligned_windows(pointwidth=30) + Traceback (most recent call last): + ... + btrdb.exceptions.InvalidOperation: A window operation is already requested """ if not self.allow_window: raise InvalidOperation("A window operation is already requested") @@ -1880,18 +2226,32 @@ def aligned_windows(self, pointwidth): StreamSet Returns self - Notes - ----- - `aligned_windows` reads power-of-two aligned windows from BTrDB. It is - faster than Windows(). Each returned window will be 2^pointwidth - nanoseconds long, starting at start. Note that start is inclusive, but - end is exclusive. That is, results will be returned for all windows that - start in the interval [start, end). If end < start+2^pointwidth you will - not get any results. If start and end are not powers of two, the bottom - pointwidth bits will be cleared. Each window will contain statistical - summaries of the window. Statistical points with count == 0 will be - omitted. + .. note:: + + ``aligned_windows`` reads power-of-two aligned windows from BTrDB. It is + faster than ``windows()``. Each returned window will be 2^``pointwidth`` + nanoseconds long, beginning at ``start``. Note that start is inclusive, but + end is exclusive. That is, results will be returned for all windows that + start in the interval ``[start, end)``. If ``end`` < ``start``+2^``pointwidth`` you will + not get any results. If ``start`` and ``end`` are not powers of two, the bottom + ``pointwidth`` bits will be cleared. Each window will contain statistical + summaries of the window. Statistical points with ``count`` == 0 will be + omitted. + + Examples + -------- + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream1 = conn.stream_from_uuid("...") + >>> stream2 = conn.stream_from_uuid("...") + >>> streamset = btrdb.stream.StreamSet([stream1, stream2]) + >>> streamset.aligned_windows(pointwidth=30) + + >>> streamset.windows(width=1000000000) + Traceback (most recent call last): + ... + btrdb.exceptions.InvalidOperation: A window operation is already requested """ if not self.allow_window: raise InvalidOperation("A window operation is already requested") @@ -1908,6 +2268,8 @@ def _streamset_data(self, as_iterators=False): ---------- as_iterators : bool Returns each single stream's data as an iterator. Defaults to False. + + :meta private: """ params = self._params_from_filters() # sampling freq not supported for non-arrow streamset ops @@ -1966,6 +2328,20 @@ def rows(self): A list of tuples containing a RawPoint (or StatPoint) and the stream version (list(tuple(RawPoint, int))). + Examples + -------- + >>> for row in streams.rows(): + >>> print(row) + (None, RawPoint(1500000000000000000, 1.0), RawPoint(1500000000000000000, 1.0), RawPoint(1500000000000000000, 1.0)) + (RawPoint(1500000000100000000, 2.0), None, RawPoint(1500000000100000000, 2.0), RawPoint(1500000000100000000, 2.0)) + (None, RawPoint(1500000000200000000, 3.0), None, RawPoint(1500000000200000000, 3.0)) + (RawPoint(1500000000300000000, 4.0), None, RawPoint(1500000000300000000, 4.0), RawPoint(1500000000300000000, 4.0)) + (None, RawPoint(1500000000400000000, 5.0), RawPoint(1500000000400000000, 5.0), RawPoint(1500000000400000000, 5.0)) + (RawPoint(1500000000500000000, 6.0), None, None, RawPoint(1500000000500000000, 6.0)) + (None, RawPoint(1500000000600000000, 7.0), RawPoint(1500000000600000000, 7.0), RawPoint(1500000000600000000, 7.0)) + (RawPoint(1500000000700000000, 8.0), None, RawPoint(1500000000700000000, 8.0), RawPoint(1500000000700000000, 8.0)) + (None, RawPoint(1500000000800000000, 9.0), RawPoint(1500000000800000000, 9.0), RawPoint(1500000000800000000, 9.0)) + (RawPoint(1500000000900000000, 10.0), None, RawPoint(1500000000900000000, 10.0), RawPoint(1500000000900000000, 10.0)) """ result = [] streamset_data = self._streamset_data(as_iterators=True) @@ -2015,14 +2391,34 @@ def insert(self, data_map: dict, merge: str = "never") -> dict: - 'retain': if two points have the same timestamp, the old one is kept - 'replace': if two points have the same timestamp, the new one is kept - Notes - ----- - You MUST convert your datetimes into utc+0 yourself. BTrDB expects utc+0 datetimes. Returns ------- dict[uuid, int] The versions of the stream after inserting new points. + + + .. note:: + + You MUST convert your datetimes into UTC+0 **yourself**. BTrDB expects UTC+0 datetimes. + + + Examples + -------- + >>> import pandas as pd + >>> import btrdb + >>> conn = btrdb.connect() + >>> stream1 = conn.stream_from_uuid("...") + >>> stream2 = conn.stream_from_uuid("...") + >>> streamset = btrdb.stream.StreamSet([stream1, stream2]) + >>> data_map = { + ... stream1.uuid: pd.DataFrame({'time': [1500000000000000000, 1500000000100000000], 'value': [1.0, 2.0]}), + ... stream2.uuid: pd.DataFrame({'time': [1500000000000000000, 1500000000100000000], 'value': [3.0, 4.0]}) + ... } + >>> streamset.insert(data_map) + {UUID('...'): 1234, UUID('...'): 5678} + >>> streamset.insert(data_map, merge='replace') + {UUID('...'): 1235, UUID('...'): 5679} """ filtered_data_map = {s.uuid: data_map[s.uuid] for s in self._streams} for key, dat in filtered_data_map.items(): @@ -2083,6 +2479,9 @@ def arrow_insert(self, data_map: dict, merge: str = "never") -> dict: return versions def _params_from_filters(self): + """ + :meta private: + """ params = {} for filter in self.filters: if filter.start is not None: @@ -2091,11 +2490,15 @@ def _params_from_filters(self): params["end"] = filter.end if filter.sampling_frequency is not None: params["sampling_frequency"] = filter.sampling_frequency + if filter.schema is not None: + params["schema"] = filter.schema return params def values_iter(self): """ Must return context object which would then close server cursor on __exit__ + + :meta private: """ raise NotImplementedError() @@ -2116,10 +2519,12 @@ def arrow_values( This data will be sorted by the 'time' column. - Notes - ----- - This method is available for commercial customers with arrow-enabled servers. + .. note:: + + This method is available for commercial customers with arrow-enabled servers. """ + if pa is None: + raise ImportError(_ARROW_IMPORT_MSG) params = self._params_from_filters() versions = self._pinned_versions if versions is None: @@ -2129,6 +2534,10 @@ def arrow_values( _ = params.pop("sampling_frequency", None) if self.pointwidth is not None: + if params.pop("schema", None) is not None: + raise NotImplementedError( + "aligned windows queries do not yet support an arrow schema" + ) # create list of stream.aligned_windows data params.update({"pointwidth": self.pointwidth}) _ = params.pop("sampling_frequency", None) @@ -2144,27 +2553,18 @@ def arrow_values( ) stream_uus = [str(s.uuid) for s in self._streams] data = list(aligned_windows_gen) - tablex = data.pop(0) - uu = stream_uus.pop(0) - tab_columns = [ - c if c == "time" else uu + "/" + c for c in tablex.column_names - ] - tablex = tablex.rename_columns(tab_columns) - if data: - for tab, uu in zip(data, stream_uus): - tab_columns = [ - c if c == "time" else uu + "/" + c for c in tab.column_names - ] - tab = tab.rename_columns(tab_columns) - tablex = tablex.join(tab, "time", join_type="full outer") - data = tablex - else: - data = tablex - data = data.sort_by("time") + table_joined = _merge_pyarrow_tables( + {uu: tab for uu, tab in zip(stream_uus, data)} + ) + data = table_joined elif self.width is not None and self.depth is not None: # create list of stream.windows data (the windows method should # prevent the possibility that only one of these is None) + if params.pop("schema", None) is not None: + raise NotImplementedError( + "windows queries do not yet support an arrow schema" + ) _ = params.pop("sampling_frequency", None) params.update({"width": self.width}) windows_gen = self._btrdb._executor.map( @@ -2175,23 +2575,10 @@ def arrow_values( ) stream_uus = [str(s.uuid) for s in self._streams] data = list(windows_gen) - tablex = data.pop(0) - uu = stream_uus.pop(0) - tab_columns = [ - c if c == "time" else uu + "/" + c for c in tablex.column_names - ] - tablex = tablex.rename_columns(tab_columns) - if data: - for tab, uu in zip(data, stream_uus): - tab_columns = [ - c if c == "time" else uu + "/" + c for c in tab.column_names - ] - tab = tab.rename_columns(tab_columns) - tablex = tablex.join(tab, "time", join_type="full outer") - data = tablex - else: - data = tablex - data = data.sort_by("time") + table_joined = _merge_pyarrow_tables( + {uu: tab for uu, tab in zip(stream_uus, data)} + ) + data = table_joined else: sampling_freq = params.pop("sampling_frequency", 0) period_ns = 0 @@ -2204,17 +2591,20 @@ def arrow_values( if len(table) > 0: data = pa.concat_tables(table) else: - schema = pa.schema( - [pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False)] - + [ - pa.field(str(s.uuid), pa.float64(), nullable=False) - for s in self._streams - ], - ) + schema = params.pop("schema", None) + if schema is None: + schema = pa.schema( + [pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False)] + + [ + pa.field(str(s.uuid), pa.float64(), nullable=False) + for s in self._streams + ], + ) data = pa.Table.from_arrays( [pa.array([]) for i in range(1 + len(self._streams))], schema=schema ) - return data + time_col = data.column_names[0] + return data.sort_by(time_col) def __repr__(self): token = "stream" if len(self) == 1 else "streams" @@ -2270,10 +2660,15 @@ class StreamFilter(object): """ def __init__( - self, start: int = None, end: int = None, sampling_frequency: int = None + self, + start: int = None, + end: int = None, + sampling_frequency: int = None, + schema=None, ): self.start = to_nanoseconds(start) if start else None self.end = to_nanoseconds(end) if end else None + self.schema = schema self.sampling_frequency = sampling_frequency if sampling_frequency else None if self.start is None and self.end is None: @@ -2300,3 +2695,77 @@ def _coalesce_table_deque(tables: deque): t2, "time", join_type="full outer", right_suffix=f"_{idx}" ) return main_table + + +def _extract_unique_times(stream_map: Dict[uuid.UUID, pa.Table]) -> pa.Array: + """Extracts and returns unique 'time' values from all tables.""" + time_arrays = [ + table.column("time").combine_chunks() + for table in stream_map.values() + if table.num_rows > 0 + ] + if not time_arrays: # Check if the list is empty + return pa.array([], type=pa.timestamp("ns")) + + all_times = pa.concat_arrays(time_arrays) + return pc.unique(all_times).sort() + + +def _build_combined_schema( + stream_map: Dict[uuid.UUID, pa.Table], unique_times: pa.Array +) -> pa.Schema: + """Constructs a combined schema for the merged table, ensuring unique column names.""" + combined_schema = [("time", unique_times.type)] + # Use a list comprehension to flatten the loop into a single iterable over all columns + combined_schema += [ + pa.field( + f"{uu}/{col_name}", + table.column(col_name).type if table.num_rows > 0 else pa.null(), + ) + for uu, table in stream_map.items() + for col_name in table.column_names + if col_name != "time" + ] + + return pa.schema(combined_schema) + + +def _merge_pyarrow_tables(stream_map: Dict[uuid.UUID, pa.Table]) -> pa.Table: + """Merges PyArrow tables based on 'time' values into a single table.""" + unique_times = _extract_unique_times(stream_map) + combined_schema = _build_combined_schema(stream_map, unique_times) + + none_data = [None] * len(unique_times) + preallocated_data = { + field.name: pa.array(none_data, type=field.type) for field in combined_schema + } + preallocated_data["time"] = unique_times + + for uu, table in stream_map.items(): + if table.num_rows > 0: + time_indices = pc.index_in( + preallocated_data["time"], + value_set=table.column("time"), + skip_nulls=True, + ) + for col_name in table.column_names: + if col_name == "time": + continue + combined_col_name = f"{str(uu)}/{col_name}" + preallocated_data[combined_col_name] = pc.take( + table.column(col_name), indices=time_indices + ) + else: + # For empty tables, ensure their columns are represented with all nulls + for col_name in combined_schema.names: + if ( + col_name.startswith(f"{str(uu)}/") + and col_name not in preallocated_data + ): + field_type = combined_schema.field(col_name).type + preallocated_data[col_name] = pa.array( + [None] * preallocated_data["time"].length(), type=field_type + ) + + arrays = [preallocated_data[col] for col in combined_schema.names] + return pa.Table.from_arrays(arrays=arrays, schema=combined_schema) diff --git a/btrdb/transformers.py b/btrdb/transformers.py index 9bcf7b5..dcdee4f 100644 --- a/btrdb/transformers.py +++ b/btrdb/transformers.py @@ -18,11 +18,34 @@ import contextlib import csv from collections import OrderedDict -from typing import Sequence -from warnings import warn - -import pandas as pd -import pyarrow +from typing import TYPE_CHECKING + +try: + import pyarrow as pa +except ImportError: + pa = None +try: + import polars as pl +except ImportError: + pl = None +try: + import pandas as pd +except ImportError: + pd = None +try: + import numpy as np +except ImportError: + np = None + +if TYPE_CHECKING: + import numpy as np + import pandas as pd + import polars as pl + import pyarrow as pa + +_IMPORT_ERR_MSG = ( + """Package(s) expected, but not found. Please pip install the following: {}""" +) ########################################################################## ## Helper Functions @@ -71,11 +94,14 @@ def to_series(streamset, datetime64_index=True, agg="mean", name_callable=None): Specify a callable that can be used to determine the series name given a Stream object. + + .. note:: + + This method does **not** use the ``arrow`` -accelerated endpoints for faster and more efficient data retrieval. + """ - try: - import pandas as pd - except ImportError: - raise ImportError("Please install Pandas to use this transformation function.") + if pd is None: + raise ImportError(_IMPORT_ERR_MSG.format("pandas")) # TODO: allow this at some future point if agg == "all": @@ -97,13 +123,13 @@ def to_series(streamset, datetime64_index=True, agg="mean", name_callable=None): values.append(getattr(point, agg)) if datetime64_index: - times = pd.Index(times, dtype="datetime64[ns]") + times = pd.Index(times, dtype="datetime64[ns]", name="time") result.append(pd.Series(data=values, index=times, name=stream_names[idx])) return result -def arrow_to_series(streamset, agg="mean", name_callable=None): +def arrow_to_series(streamset, agg=None, name_callable=None): """ Returns a list of Pandas Series objects indexed by time @@ -118,14 +144,89 @@ def arrow_to_series(streamset, agg="mean", name_callable=None): Specify a callable that can be used to determine the series name given a Stream object. - Notes - ----- - This method is available for commercial customers with arrow-enabled servers. + + Returns + ------- + List[pandas.Series] + + + .. note:: + + This method is available for commercial customers with arrow-enabled servers. + + + .. note:: + + If you are not performing a ``window`` or ``aligned_window`` query, the ``agg`` parameter will be ignored. + + Examples + -------- + Return a list of series of raw data per stream. + + >>> conn = btrdb.connect() + >>> s1 = conn.stream_from_uuid('c9fd8735-5ec5-4141-9a51-d23e1b2dfa42') + >>> s2 = conn.stream_from_uuid('9173fa70-87ab-4ac8-ac08-4fd63b910cae' + >>> streamset = btrdb.stream.StreamSet([s1,s2]) + >>> streamset.filter(start=1500000000000000000, end=1500000000900000001).arrow_to_series(agg=None) + [time + 2017-07-14 02:40:00+00:00 1.0 + 2017-07-14 02:40:00.100000+00:00 2.0 + 2017-07-14 02:40:00.200000+00:00 3.0 + 2017-07-14 02:40:00.300000+00:00 4.0 + 2017-07-14 02:40:00.400000+00:00 5.0 + 2017-07-14 02:40:00.500000+00:00 6.0 + 2017-07-14 02:40:00.600000+00:00 7.0 + 2017-07-14 02:40:00.700000+00:00 8.0 + 2017-07-14 02:40:00.800000+00:00 9.0 + 2017-07-14 02:40:00.900000+00:00 10.0 + Name: new/stream/collection/foo, dtype: double[pyarrow], + time + 2017-07-14 02:40:00+00:00 1.0 + 2017-07-14 02:40:00.100000+00:00 2.0 + 2017-07-14 02:40:00.200000+00:00 3.0 + 2017-07-14 02:40:00.300000+00:00 4.0 + 2017-07-14 02:40:00.400000+00:00 5.0 + 2017-07-14 02:40:00.500000+00:00 6.0 + 2017-07-14 02:40:00.600000+00:00 7.0 + 2017-07-14 02:40:00.700000+00:00 8.0 + 2017-07-14 02:40:00.800000+00:00 9.0 + 2017-07-14 02:40:00.900000+00:00 10.0 + Name: new/stream/bar, dtype: double[pyarrow]] + + + A window query of 0.5seconds long. + + >>> streamset.filter(start=1500000000000000000, end=1500000000900000001) + ... .windows(width=int(0.5 * 10**9)) + ... .arrow_to_series(agg=["mean", "count"]) + [time + 2017-07-14 02:40:00+00:00 2.5 + 2017-07-14 02:40:00.400000+00:00 6.5 + Name: new/stream/collection/foo/mean, dtype: double[pyarrow], + ... + time + 2017-07-14 02:40:00+00:00 4 + 2017-07-14 02:40:00.400000+00:00 4 + Name: new/stream/collection/foo/count, dtype: uint64[pyarrow], + ... + time + 2017-07-14 02:40:00+00:00 2.5 + 2017-07-14 02:40:00.400000+00:00 6.5 + Name: new/stream/bar/mean, dtype: double[pyarrow], + ... + time + 2017-07-14 02:40:00+00:00 4 + 2017-07-14 02:40:00.400000+00:00 4 + Name: new/stream/bar/count, dtype: uint64[pyarrow]] + + """ if not streamset._btrdb._ARROW_ENABLED: raise NotImplementedError( "arrow_to_series requires an arrow-enabled BTrDB server." ) + if pa is None or pd is None: + raise ImportError(_IMPORT_ERR_MSG.format(",".join(["pyarrow", "pandas"]))) if agg is None: agg = ["mean"] if not isinstance(agg, list): @@ -138,18 +239,13 @@ def arrow_to_series(streamset, agg="mean", name_callable=None): return [arrow_df[col] for col in arrow_df] -def arrow_to_dataframe( - streamset, columns=None, agg=None, name_callable=None -) -> pd.DataFrame: +def arrow_to_dataframe(streamset, agg=None, name_callable=None) -> pd.DataFrame: """ Returns a Pandas DataFrame object indexed by time and using the values of a stream for each column. Parameters ---------- - columns: sequence - column names to use for DataFrame. Deprecated and not compatible with name_callable. - agg : List[str], default: ["mean"] Specify the StatPoint fields (e.g. aggregating function) to create the dataframe from. Must be one or more of "min", "mean", "max", "count", "stddev", or "all". This @@ -159,29 +255,94 @@ def arrow_to_dataframe( Specify a callable that can be used to determine the series name given a Stream object. - Notes - ----- - This method is available for commercial customers with arrow-enabled servers. + + .. note:: + + This method is available for commercial customers with arrow-enabled servers. + + + Examples + -------- + + >>> conn = btrdb.connect() + >>> s1 = conn.stream_from_uuid('c9fd8735-5ec5-4141-9a51-d23e1b2dfa42') + >>> s2 = conn.stream_from_uuid('9173fa70-87ab-4ac8-ac08-4fd63b910cae' + >>> streamset = btrdb.stream.StreamSet([s1,s2]) + >>> streamset.filter(start=1500000000000000000, end=1500000000900000001).arrow_to_dataframe() + new/stream/collection/foo new/stream/bar + time + 2017-07-14 02:40:00+00:00 1.0 1.0 + 2017-07-14 02:40:00.100000+00:00 2.0 2.0 + 2017-07-14 02:40:00.200000+00:00 3.0 3.0 + 2017-07-14 02:40:00.300000+00:00 4.0 4.0 + 2017-07-14 02:40:00.400000+00:00 5.0 5.0 + 2017-07-14 02:40:00.500000+00:00 6.0 6.0 + 2017-07-14 02:40:00.600000+00:00 7.0 7.0 + 2017-07-14 02:40:00.700000+00:00 8.0 8.0 + 2017-07-14 02:40:00.800000+00:00 9.0 9.0 + 2017-07-14 02:40:00.900000+00:00 10.0 10.0 + + + Use the stream uuids as their column names instead, using a lambda function. + + >>> streamset.filter(start=1500000000000000000, end=1500000000900000001) + ... .arrow_to_dataframe( + ... name_callable=lambda s: str(s.uuid) + ... ) + c9fd8735-5ec5-4141-9a51-d23e1b2dfa42 9173fa70-87ab-4ac8-ac08-4fd63b910cae + time + 2017-07-14 02:40:00+00:00 1.0 1.0 + 2017-07-14 02:40:00.100000+00:00 2.0 2.0 + 2017-07-14 02:40:00.200000+00:00 3.0 3.0 + 2017-07-14 02:40:00.300000+00:00 4.0 4.0 + 2017-07-14 02:40:00.400000+00:00 5.0 5.0 + 2017-07-14 02:40:00.500000+00:00 6.0 6.0 + 2017-07-14 02:40:00.600000+00:00 7.0 7.0 + 2017-07-14 02:40:00.700000+00:00 8.0 8.0 + 2017-07-14 02:40:00.800000+00:00 9.0 9.0 + 2017-07-14 02:40:00.900000+00:00 10.0 10.0 + + + A window query, with a window width of 0.4 seconds, and only showing the ``mean`` statpoint. + + >>> streamset.filter(start=1500000000000000000, end=1500000000900000001) + ... .windows(width=int(0.4*10**9)) + ... .arrow_to_dataframe(agg=["mean"]) + new/stream/collection/foo/mean new/stream/bar/mean + time + 2017-07-14 02:40:00+00:00 2.5 2.5 + 2017-07-14 02:40:00.400000+00:00 6.5 6.5 + + + A window query, with a window width of 0.4 seconds, and only showing the ``mean`` and ``count`` statpoints. + + >>> streamset.filter(start=1500000000000000000, end=1500000000900000001) + ... .windows(width=int(0.4*10**9)) + ... .arrow_to_dataframe(agg=["mean", "count"]) + new/stream/collection/foo/mean new/stream/collection/foo/count new/stream/bar/mean new/stream/bar/count + time + 2017-07-14 02:40:00+00:00 2.5 4 2.5 4 + 2017-07-14 02:40:00.400000+00:00 6.5 4 6.5 4 """ + + def _rename(col_name, col_names_map): + if col_name == "time": + return col_name + col_name_parts = col_name.split("/") + if len(col_name_parts) == 1: + uuid = col_name_parts[0] + new_col_name = col_names_map[uuid] + elif len(col_name_parts) == 2: + uuid, _ = col_name_parts + new_col_name = col_name.replace(uuid, col_names_map[uuid]) + return new_col_name + if not streamset._btrdb._ARROW_ENABLED: raise NotImplementedError( "arrow_to_dataframe requires an arrow-enabled BTrDB server." ) - - try: - import pandas as pd - import pyarrow as pa - except ImportError as err: - raise ImportError( - f"Please install Pandas and pyarrow to use this transformation function. ErrorMessage: {err}" - ) - # deprecation warning added in v5.8 - if columns: - warn( - "the columns argument is deprecated and will be removed in a future release", - DeprecationWarning, - stacklevel=2, - ) + if pa is None or pd is None: + raise ImportError(_IMPORT_ERR_MSG.format(",".join(["pyarrow", "pandas"]))) if agg is None: agg = ["mean"] @@ -201,42 +362,40 @@ def arrow_to_dataframe( if not callable(name_callable): name_callable = lambda s: s.collection + "/" + s.name # format is: uuid/stat_type - tmp_table = streamset.arrow_values() + tmp = streamset.arrow_values() + # assume time col is the first column + time_col = tmp.column_names[0] + + tmp_df = tmp.to_pandas( + date_as_object=False, + types_mapper=pd.ArrowDtype, + split_blocks=True, + self_destruct=True, + ).set_index(time_col) + tmp_df.index.name = time_col col_names = _stream_names(streamset, name_callable) col_names_map = {str(s.uuid): c for s, c in zip(streamset, col_names)} - updated_table_columns = [] - for old_col in tmp_table.column_names: - if old_col == "time": - updated_table_columns.append("time") - else: - for uu, new_name in col_names_map.items(): - if uu in old_col: - updated_table_columns.append(old_col.replace(uu, new_name)) - else: - continue - tmp_table = tmp_table.rename_columns(updated_table_columns) + tmp_df.rename( + columns=lambda col_name: _rename(col_name, col_names_map), inplace=True + ) if not streamset.allow_window: usable_cols = [] - for column_str in tmp_table.column_names: + for column_str in tmp_df.columns: for agg_name in agg: if agg_name in column_str: usable_cols.append(column_str) - tmp = tmp_table.select(["time", *usable_cols]) - else: - tmp = tmp_table - return tmp.to_pandas(date_as_object=False, types_mapper=pd.ArrowDtype) + tmp_df = tmp_df.loc[:, usable_cols] + + return tmp_df -def to_dataframe(streamset, columns=None, agg="mean", name_callable=None): +def to_dataframe(streamset, agg="mean", name_callable=None): """ Returns a Pandas DataFrame object indexed by time and using the values of a stream for each column. Parameters ---------- - columns: sequence - column names to use for DataFrame. Deprecated and not compatible with name_callable. - agg : str, default: "mean" Specify the StatPoint field (e.g. aggregating function) to create the Series from. Must be one of "min", "mean", "max", "count", "stddev", or "all". This @@ -247,19 +406,13 @@ def to_dataframe(streamset, columns=None, agg="mean", name_callable=None): Stream object. This is not compatible with agg == "all" at this time + .. note:: + + This method does **not** use the ``arrow`` -accelerated endpoints for faster and more efficient data retrieval. + """ - try: - import pandas as pd - except ImportError: - raise ImportError("Please install Pandas to use this transformation function.") - - # deprecation warning added in v5.8 - if columns: - warn( - "the columns argument is deprecated and will be removed in a future release", - DeprecationWarning, - stacklevel=2, - ) + if pd is None: + raise ImportError(_IMPORT_ERR_MSG.format("pandas")) # TODO: allow this at some future point if agg == "all" and name_callable is not None: @@ -279,6 +432,7 @@ def to_dataframe(streamset, columns=None, agg="mean", name_callable=None): if not df.empty: df = df.set_index("time") + df.index.name = "time" if agg == "all" and not streamset.allow_window: stream_names = [ @@ -288,7 +442,7 @@ def to_dataframe(streamset, columns=None, agg="mean", name_callable=None): ] df.columns = pd.MultiIndex.from_tuples(stream_names) else: - df.columns = columns if columns else _stream_names(streamset, name_callable) + df.columns = _stream_names(streamset, name_callable) return df @@ -309,18 +463,20 @@ def arrow_to_polars(streamset, agg=None, name_callable=None): Specify a callable that can be used to determine the series name given a Stream object. - Notes - ----- - This method is available for commercial customers with arrow-enabled servers. + + .. note:: + + This method is available for commercial customers with arrow-enabled servers. + """ if not streamset._btrdb._ARROW_ENABLED: raise NotImplementedError( "arrow_to_polars requires an arrow-enabled BTrDB server." ) - try: - import polars as pl - except ImportError: - raise ImportError("Please install polars to use this transformation function.") + if pa is None or pd is None or pl is None: + raise ImportError( + _IMPORT_ERR_MSG.format(",".join(["pyarrow", "pandas", "polars"])) + ) if agg is None: agg = ["mean"] if not isinstance(agg, list): @@ -336,14 +492,18 @@ def arrow_to_polars(streamset, agg=None, name_callable=None): def arrow_to_arrow_table(streamset): """Return a pyarrow table of data. - Notes - ----- - This method is available for commercial customers with arrow-enabled servers. + + .. note:: + + This method is available for commercial customers with arrow-enabled servers. + """ if not streamset._btrdb._ARROW_ENABLED: raise NotImplementedError( "arrow_to_arrow_table requires an arrow-enabled BTrDB server." ) + if pa is None: + raise ImportError(_IMPORT_ERR_MSG.format("pyarrow")) return streamset.arrow_values() @@ -362,11 +522,16 @@ def to_polars(streamset, agg="mean", name_callable=None): name_callable : lambda, default: lambda s: s.collection + "/" + s.name Specify a callable that can be used to determine the series name given a Stream object. This is not compatible with agg == "all" at this time + + + + .. note:: + + This method does **not** use the ``arrow`` -accelerated endpoints for faster and more efficient data retrieval. + """ - try: - import polars as pl - except ImportError: - raise ImportError("Please install polars to use this transformation function.") + if pl is None or pd is None: + raise ImportError(_IMPORT_ERR_MSG.format(",".join(["polars", "pandas"]))) # TODO: allow this at some future point if agg == "all" and name_callable is not None: @@ -392,7 +557,7 @@ def to_polars(streamset, agg="mean", name_callable=None): else: df = df.set_index("time") - df.index = pd.DatetimeIndex(df.index, tz="UTC") + df.index = pd.DatetimeIndex(df.index, tz="UTC", name="time") if agg == "all" and streamset.allow_window: stream_names = [ [s.collection, s.name, prop] @@ -403,7 +568,7 @@ def to_polars(streamset, agg="mean", name_callable=None): else: df.columns = _stream_names(streamset, name_callable) - return pl.from_pandas(df.reset_index(), nan_to_null=False) + return pl.from_pandas(df, nan_to_null=False, include_index=True) def to_array(streamset, agg="mean"): @@ -418,11 +583,14 @@ def to_array(streamset, agg="mean"): arrays. Must be one of "min", "mean", "max", "count", or "stddev". This argument is ignored if RawPoint values are passed into the function. + + .. note:: + + This method does **not** use the ``arrow`` -accelerated endpoints for faster and more efficient data retrieval. + """ - try: - import numpy as np - except ImportError: - raise ImportError("Please install Numpy to use this transformation function.") + if np is None: + raise ImportError(_IMPORT_ERR_MSG.format("numpy")) # TODO: allow this at some future point if agg == "all": @@ -450,16 +618,25 @@ def arrow_to_numpy(streamset, agg=None): arrays. Must be one or more of "min", "mean", "max", "count", or "stddev". This argument is ignored if RawPoint values are passed into the function. - Notes - ----- - This method first converts to a pandas data frame then to a numpy array. - This method is available for commercial customers with arrow-enabled servers. + .. note:: + + This method first converts to a pandas data frame then to a numpy array. + + + .. note:: + + This method is available for commercial customers with arrow-enabled servers. + """ if not streamset._btrdb._ARROW_ENABLED: raise NotImplementedError( "arrow_to_numpy requires an arrow-enabled BTrDB server." ) + if np is None or pa is None or pd is None: + raise ImportError( + _IMPORT_ERR_MSG.format(",".join(["numpy", "pyarrow", "pandas"])) + ) arrow_df = arrow_to_dataframe(streamset=streamset, agg=agg, name_callable=None) return arrow_df.values @@ -480,6 +657,11 @@ def to_dict(streamset, agg="mean", name_callable=None): Specify a callable that can be used to determine the series name given a Stream object. + + .. note:: + + This method does **not** use the ``arrow`` -accelerated endpoints for faster and more efficient data retrieval. + """ if not callable(name_callable): name_callable = lambda s: s.collection + "/" + s.name @@ -524,14 +706,18 @@ def arrow_to_dict(streamset, agg=None, name_callable=None): Specify a callable that can be used to determine the series name given a Stream object. - Notes - ----- - This method is available for commercial customers with arrow-enabled servers. + + .. note:: + + This method is available for commercial customers with arrow-enabled servers. + """ if not streamset._btrdb._ARROW_ENABLED: raise NotImplementedError( "arrow_to_dict requires an arrow-enabled BTrDB server." ) + if pa is None or pd is None: + raise ImportError(_IMPORT_ERR_MSG.format(",".join(["pyarrow", "pandas"]))) if agg is None: agg = ["mean"] if not isinstance(agg, list): @@ -571,6 +757,12 @@ def to_csv( name_callable : lambda, default: lambda s: s.collection + "/" + s.name Specify a callable that can be used to determine the series name given a Stream object. + + + .. note:: + + This method does **not** use the ``arrow`` -accelerated endpoints for faster and more efficient data retrieval. + """ # TODO: allow this at some future point @@ -620,6 +812,11 @@ def to_table(streamset, agg="mean", name_callable=None): Specify a callable that can be used to determine the column name given a Stream object. + + .. note:: + + This method does **not** use the ``arrow`` -accelerated endpoints for faster and more efficient data retrieval. + """ try: from tabulate import tabulate diff --git a/btrdb/utils/general.py b/btrdb/utils/general.py index 6344fe6..d438fce 100644 --- a/btrdb/utils/general.py +++ b/btrdb/utils/general.py @@ -12,12 +12,14 @@ """ General utilities for btrdb bindings """ +from datetime import timedelta + +from btrdb.utils.timez import to_nanoseconds + ########################################################################## ## Functions ########################################################################## - - def unpack_stream_descriptor(desc): """ Returns dicts for tags and annotations found in supplied stream @@ -74,6 +76,50 @@ def from_nanoseconds(cls, nsec): break return cls(pos) + def for_aligned_windows(self, start, end): + """ + Returns the aligned_windows's first and last timestamps, as well as the number of windows, + based on a given pointwidth set. + + Parameters + ---------- + start : int or datetime like object + The start time in nanoseconds for the range to be queried. (see + :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) + + end : int or datetime like object + The end time in nanoseconds for the range to be queried. (see + :func:`btrdb.utils.timez.to_nanoseconds` for valid input types) + + + Returns + ------- + aligned_start: int + First timestamp would be returned by `aligned_windows` that is inclusive of specified start + timestamp. + aligned_end: int + Last timestamp would be returned by `aligned_windows` that is inclusive of specified end + timestamp. + n_windows: int + The number of windows would be returned by `aligned_windows`. + + Examples + -------- + Querying `aligned_windows` of pointwidth of 30, spaning 1 day (~24 hours). + + >>> start, end = "2016-03-01", "2016-03-02" + >>> pointwidth(30).for_aligned_windows(start, end) + (1456790399996657664, 1456876798632525824, 80466) + # output timestamp's `strftime` is: + # ['2016-02-29 23:59:59.996657664', '2016-03-01 23:59:58.632525824'] + + """ + start, end = to_nanoseconds(start), to_nanoseconds(end) + aligned_start = start - (start % self.nanoseconds) + n_windows = (end - aligned_start) // self.nanoseconds + aligned_end = aligned_start + (self.nanoseconds * (n_windows - 1)) + return aligned_start, aligned_end, n_windows + def __init__(self, p): self._pointwidth = int(p) @@ -123,6 +169,12 @@ def decr(self): def incr(self): return pointwidth(self + 1) + def to_timedelta(self): + """ + Returns the timedelta of the pointwidth. + """ + return timedelta(microseconds=self.microseconds) + def __int__(self): return self._pointwidth diff --git a/btrdb/utils/timez.py b/btrdb/utils/timez.py index 32cec9e..3c0428a 100644 --- a/btrdb/utils/timez.py +++ b/btrdb/utils/timez.py @@ -55,7 +55,7 @@ def currently_as_ns(): def ns_to_datetime(ns): """ - Converts nanoseconds to a datetime object (UTC) + Converts nanoseconds to a naive datetime object (UTC+0) Parameters ---------- @@ -178,7 +178,7 @@ def ns_delta( days=0, hours=0, minutes=0, seconds=0, milliseconds=0, microseconds=0, nanoseconds=0 ): """ - Similar to `timedelta`, ns_delta represents a span of time but as + Similar to ``timedelta``, ``ns_delta`` represents a span of time but as the total number of nanoseconds. Parameters @@ -202,6 +202,14 @@ def ns_delta( ------- amount of time in nanoseconds : int + + Examples + -------- + 1 minute time delta should be 60 billion nanoseconds + + >>> deltaT = ns_delta(minutes=1) + >>> deltaT == 1 * 60 * 10**9 # 1 minute * 60 seconds * 1billion nanoseconds / second + True """ MICROSECOND = 1000 MILLISECOND = MICROSECOND * 1000 diff --git a/btrdb/version.py b/btrdb/version.py index d657ab4..7bf90da 100644 --- a/btrdb/version.py +++ b/btrdb/version.py @@ -15,7 +15,7 @@ ## Module Info ########################################################################## -__version_info__ = {"major": 5, "minor": 31, "micro": 0, "releaselevel": "final"} +__version_info__ = {"major": 5, "minor": 32, "micro": 0, "releaselevel": "final"} ########################################################################## ## Helper Functions diff --git a/docs/requirements.txt b/docs/requirements.txt index 49e71d8..2e6e1f1 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,8 @@ alabaster>=0.7.12 Sphinx>=1.7 +ipython sphinx-rtd-theme +sphinx-copybutton +sphinx-design numpydoc pydata-sphinx-theme diff --git a/docs/source/api/streams.rst b/docs/source/api/streams.rst index 7137d36..7707bbc 100644 --- a/docs/source/api/streams.rst +++ b/docs/source/api/streams.rst @@ -1,6 +1,7 @@ btrdb.stream ============== + .. _StreamGeneralDocs: .. automodule:: btrdb.stream @@ -8,5 +9,7 @@ btrdb.stream :members: .. _StreamSet API: -.. autoclass:: StreamSetBase +.. autoclass:: StreamSet + :show-inheritance: + :inherited-members: :members: diff --git a/docs/source/concepts.rst b/docs/source/concepts.rst index 4685556..2392034 100644 --- a/docs/source/concepts.rst +++ b/docs/source/concepts.rst @@ -11,7 +11,7 @@ of their behavior which will allow you to use them effectively. .. note:: - Data requests are fully materialized at this time. A future release will include the option to process data using generators to save on memory usage. + Data requests are fully materialized at this time. A future release will include the option to process data using generators to save on memory usage. BTrDB Server @@ -38,48 +38,46 @@ value within the stream. .. code-block:: python - # view time and value of a single point in the stream + >>> # view time and value of a single point in the stream + >>> point.time + 1547241923338098176 - point.time - >>> 1547241923338098176 - - point.value - >>> 120.5 + >>> point.value + 120.5 StatPoint ^^^^^^^^^^^^ The StatPoint provides statistics about multiple points and gives -aggregation values such as `min`, `max`, `mean`, and `stddev` (standard deviation). +aggregation values such as :code:`min`, :code:`max`, :code:`mean`, :code:`count` and :code:`stddev` (standard deviation). This is most useful when you don't need to touch every individual value such as when you only need the count of the values over a range of time. These statistical queries execute in time proportional to the number of results, not the number of underlying points (i.e logarithmic time) and so you can attain valuable data in a fraction of the time when compared with retrieving -all of the individual values. Due to the internal data structures, BTrDB does +all of the individual values. Due to the internal data structures, :code:`BTrDB` does not need to read the underlying points to return these statistics! .. code-block:: python - # view aggregate values for points in a stream - - point.time - >>> 1547241923338098176 + >>> # view aggregate values for points in a stream + >>> point.time + 1547241923338098176 - point.min - >>> 42.1 + >>>point.min + 42.1 - point.mean - >>> 78.477 + >>> point.mean + 78.477 - point.max - >>> 122.4 + >>> point.max + 122.4 - point.count - >>> 18600 + >>> point.count + 18600 - point.stddev - >>> 3.4 + >>> point.stddev + 3.4 Tabular Data @@ -91,52 +89,55 @@ Refer to the :ref:`arrow enabled queries page ` and the :ref:`API do Streams ------------ -Streams represent a single series of time/value pairs. As such, the database +:code:`Stream` s represent a single series of time/value pairs. As such, the database can hold an almost unlimited amount of individual streams. Each stream has a :code:`collection` which is similar to a "path" or grouping for multiple streams. Each steam will also have a :code:`name` as well as a :code:`uuid` which is guaranteed to be unique across streams. -BTrDB data is versioned such that changes to a given stream (time series) will +:code:`BTrDB` data is versioned such that changes to a given stream (time series) will result in a new version for the stream. In this manner, you can pin your interactions to a specific version ensuring the values do not change over the course of your -interactions. If you want to work with the most recent version/data then -specify a version of zero (the default). +interactions. + +.. note:: + + If you want to work with the most recent version/data then specify a version of :code:`0` (the default). Each stream has a number of attributes and methods available and these are documented -within the API section of this publication. But the most common interactions -by users are to access the UUID, tags, annotations, version, and underlying data. +within the :ref:`API Reference ` section of this publication. But the most common interactions +by users are to access the :code:`UUID`, :code:`tags`, :code:`annotations`, :code:`version`, and underlying data. -Each stream uses a UUID as its unique identifier which can also be used when querying -for streams. Metadata is provided by tags and annotations which are both provided -as dictionaries of data. Tags are used internally and have very specific keys -while annotations are more free-form and can be used by you to store your own +Each stream uses a :code:`UUID` as its unique identifier which can also be used when querying +for streams. Metadata is provided by :code:`tags` and :code:`annotations` which are both provided +as dictionaries of data. :code:`tags` are used internally and have very specific keys +while :code:`annotations` are more free-form and can be used by you to store your own metadata. .. code-block:: python - # retrieve stream's UUID - stream.uuid - >>> UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + >>> # retrieve stream's UUID + >>> stream.uuid + UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") - # retrieve stream's current version - stream.version() - >>> 244 + >>> # retrieve stream's current version + >>> stream.version() + 244 - # retrieve stream tags - stream.tags() - >>> {'name': 'L1MAG', 'unit': 'volts', 'ingress': ''} + >>> # retrieve stream tags + >>> stream.tags() + {'name': 'L1MAG', 'unit': 'volts', 'ingress': ''} - # retrieve stream annotations - stream.annotations() - >>> {'poc': 'Salvatore McFesterson', 'region': 'northwest', 'state': 'WA'} + >>> # retrieve stream annotations + >>> stream.annotations() + ({'poc': 'Salvatore McFesterson', 'region': 'northwest', 'state': 'WA'}, 23) - # loop through points in the stream - for point, _ in stream.values(end=1547241923338098176, version=133): - print(point) - >>> RawPoint(1500000000100000000, 2.4) - >>> RawPoint(1500000000200000000, 2.8) - >>> RawPoint(1500000000300000000, 3.6) + >>> # loop through points in the stream + >>> for point, _ in stream.values(end=1547241923338098176, version=133): + >>> print(point) + RawPoint(1500000000100000000, 2.4) + RawPoint(1500000000200000000, 2.8) + RawPoint(1500000000300000000, 3.6) ... @@ -144,8 +145,8 @@ StreamSets ------------ Often you will want to query and work with multiple streams instead of just an -individual stream - StreamSets allow you to do this effectively. It is a light -wrapper around a list of Stream objects with convenience methods provided to +individual stream - :code:`StreamSets` allow you to do this effectively. It is a light +wrapper around a list of :code:`Stream` objects with convenience methods provided to help you work with multiple streams of data. As an example, you can filter the stream data with a single method call and then @@ -153,17 +154,45 @@ easily transform the data into other data types such as a pandas DataFrame or to disk as a CSV file. See the examples below for a quick sample and then visit our API docs to see the full list of features provided to you. +.. note:: + + :code:`StreamSet` methods that filter and operate on the :code:`StreamSet` object (like :code:`StreamSet.filter` ) return new copies of the :code:`StreamSet` itself rather than modifying in place. Similar to how most :code:`pandas.DataFrame` methods return a new :code:`DataFrame` object. This lets you compose multiple functions in a single call, which can improve readability, but can be tricky if you are not expecting this behavior. + +Lets explore a common use-case, filtering a streamset. + +.. code-block:: python + + >>> # create a streamset and apply a few filters + >>> streamset = btrdb.stream.StreamSet(list_of_streams) + >>> print(f"Total streams: {len(streamset)}") + Total streams: 89 + + >>> streamset.filter(units="Volts") + >>> print(f"Total streams: {len(streamset)}") + Total streams: 89 + + >>> filtered_streamset = streamset.filter(units="Volts") + >>> print(f"Total streams: {len(filtered_streamset)}") + Total streams: 23 + + >>> multiple_filters_streamset = (streamset.filter(unit="Volts") + >>> .filter(name="Sensor 1") + >>> .filter(annotations={"phase":"A"}) + >>> ) + >>> print(f"Total streams: {len(multiple_filters_streamset)}") + Total streams: 1 + .. code-block:: python - # establish database connection and query for streams by UUID - db = connect() - uuid_list = ["0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a", ...] - streams = db.streams(*uuid_list) + >>> # establish database connection and query for streams by UUID + >>> db = connect() + >>> uuid_list = ["0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a", ...] + >>> streams = db.streams(*uuid_list) - streams.filter(start=1500000000000000000).to_csv("data.csv") + >>> streams.filter(start=1500000000000000000).to_csv("data.csv") - streams.filter(start=1500000000000000000).to_dataframe() - >> time NW/stream0 NW/stream1 + >>> streams.filter(start=1500000000000000000).to_dataframe() + time NW/stream0 NW/stream1 0 1500000000000000000 NaN 1.0 1 1500000000100000000 2.0 NaN 2 1500000000200000000 NaN 3.0 diff --git a/docs/source/conf.py b/docs/source/conf.py index 1cc1ebf..5db17d4 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -27,7 +27,7 @@ # -- Project information ----------------------------------------------------- project = "btrdb" -copyright = "2023, Ping Things, Inc." +copyright = "2024, Ping Things, Inc." author = "PingThingsIO" # The short X.Y version @@ -48,10 +48,18 @@ extensions = [ "sphinx.ext.autodoc", "sphinx.ext.napoleon", + "sphinx.ext.autosummary", + # 'sphinx.ext.inheritance_diagram', + # 'sphinx.ext.intersphinx', + "sphinx.ext.ifconfig", + # 'IPython.sphinxext.ipython_console_highlighting', + # 'IPython.sphinxext.ipython_directive', "sphinx.ext.todo", "sphinx.ext.githubpages", - "sphinx.ext.intersphinx", + # "sphinx.ext.intersphinx", "numpydoc", + "sphinx_copybutton", + "sphinx_design", ] # Add any paths that contain templates here, relative to this directory. @@ -96,16 +104,19 @@ # documentation. # html_theme_options = { - "show_powered_by": False, - "github_user": "PingThingsIO", - "github_repo": "btrdb-python", - "travis_button": False, - "github_banner": False, - "show_related": False, - "note_bg": "#FFF59C", - "description": "A midweight library to converse with the BTrDB database.", - "extra_nav_links": {"btrdb": "http://btrdb-python.readthedocs.io"}, - "show_relbars": True, + # "show_powered_by": False, + # "github_user": "PingThingsIO", + # "github_repo": "btrdb-python", + # "travis_button": False, + # "github_banner": False, + # "show_related": False, + # "note_bg": "#FFF59C", + # "description": "A midweight library to converse with the BTrDB database.", + # "extra_nav_links": {"btrdb": "http://btrdb-python.readthedocs.io"}, + # "show_relbars": True, + "show_toc_level": 2, + "navigation_depth": 4, # Adjust the depth of the sidebar TOC + "show_nav_level": 2, # Initially shown levels of the TOC } # Add any paths that contain custom static files (such as style sheets) here, diff --git a/docs/source/explained.rst b/docs/source/explained.rst index 68bdfe7..d40eea7 100644 --- a/docs/source/explained.rst +++ b/docs/source/explained.rst @@ -5,7 +5,7 @@ BTrDB Explained **A next-gen timeseries database for dense, streaming telemetry.** -**Problem**: Existing timeseries databases are poorly equipped for a new generation of ultra-fafst sensor telemetry. Specifically, millions of high-precision power meters are to be deployed through the power grid to help analyze and prevent blackouts. Thus, new software must be built to facilitate the storage and analysis of its data. +**Problem**: Existing timeseries databases are poorly equipped for a new generation of ultra-fast sensor telemetry. Specifically, millions of high-precision power meters are to be deployed through the power grid to help analyze and prevent blackouts. Thus, new software must be built to facilitate the storage and analysis of its data. **Baseline**: We need 1.4M inserts/second and 5x that in reads if we are to support 1000 `micro-synchrophasors`_ per server node. No timeseries database can do this. diff --git a/docs/source/index.rst b/docs/source/index.rst index 3f1076b..5320d6f 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -16,15 +16,6 @@ Welcome to btrdb docs! .. image:: https://img.shields.io/pypi/v/btrdb.svg :target: https://pypi.python.org/project/btrdb/ -.. note:: - - Starting with the 5.0 release, btrdb-python will be Python 3 only! This decision - was not made lightly but is necessary to keep compatibility with underlying - packages. - - In addition, this software is only compatible with version 5.x of the BTrDB - server. To communicate with a 4.x server, please install an earlier version - of this software. Welcome to btrdb-python's documentation. We provide Python access to the Berkeley Tree Database (BTrBD) along with some select convenience methods. If you are @@ -59,6 +50,9 @@ your appetite. for point, _ in stream.values(start, end): print(point.time, point.value) + # return the data as an arrow table instead + data = stream.arrow_values(start, end) + User Guide ---------- diff --git a/docs/source/installing.rst b/docs/source/installing.rst index 71cab8e..99a873b 100644 --- a/docs/source/installing.rst +++ b/docs/source/installing.rst @@ -1,13 +1,13 @@ Installing ======================== -The btrdb package has only a few requirements and is relatively easy to install. +The :code:`btrdb` package has only a few requirements and is relatively easy to install. A number of installation options are available as detailed below. Installing with pip ------------------- -We recommend using pip to install btrdb-python on all platforms: +We recommend using :code:`pip` to install :code:`btrdb-python` on all platforms: .. code-block:: bash @@ -24,7 +24,7 @@ We recommend installing the :code:`data` extra dependencies (the second option i $ pip install "btrdb[all]>=5.30.2" # btrdb with testing, data science and all other optional packages -To get a specific version of btrdb-python supply the version number. The major +To get a specific version of :code:`btrdb-python` supply the version number. The major version of this library is tied to the major version of the BTrDB database as in the 4.X bindings are best used to speak to a 4.X BTrDB database, the 5.X bindings for 5.X platform.. @@ -43,4 +43,4 @@ To upgrade using pip: Installing with Anaconda ------------------------ -We recommend installing using ``pip``. +We recommend installing using :code:`pip`. diff --git a/docs/source/quick-start.rst b/docs/source/quick-start.rst index 8901177..7e70ce8 100644 --- a/docs/source/quick-start.rst +++ b/docs/source/quick-start.rst @@ -9,22 +9,22 @@ Connecting to a server is easy with the supplied :code:`connect` function from t .. code-block:: python - import btrdb - - # connect without credentials - conn = btrdb.connect("192.168.1.101:4410") - - # connect using TLS - conn = btrdb.connect("192.168.1.101:4411") - - # connect with API key - conn = btrdb.connect("192.168.1.101:4411", apikey="123456789123456789") + >>> import btrdb + >>> # connect with API key + >>> conn = btrdb.connect("192.168.1.101:4411", apikey="123456789123456789") + >>> conn + Get Platform Information ^^^^^^^^^^^^^^^^^^^^^^^^ .. code-block:: python - conn.info() + >>> conn.info() + {'majorVersion': ..., + 'minorVersion': ..., + 'build': ..., + 'proxy': {...}} + Refer to :ref:`the connection API documentation page. ` @@ -32,8 +32,8 @@ Refer to :ref:`the connection API documentation page. ` Retrieving a Stream ---------------------- -In order to interact with data, you'll need to obtain or create a :code:`Stream` object. A -number of options are available to get existing streams. +In order to interact with data, you'll need to obtain or create a :code:`Stream` object. +A number of options are available to get existing streams. Find streams by collection ^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -43,9 +43,9 @@ collection you can use the :code:`streams_in_collection` method. .. code-block:: python - streams = conn.streams_in_collection("USEAST_NOC1/90807") - for stream in streams: - print(stream.uuid, stream.name) + >>> streams = conn.streams_in_collection("USEAST_NOC1/90807") + >>> for stream in streams: + >>> print(stream.uuid, stream.name) Find stream by UUID ^^^^^^^^^^^^^^^^^^^^^ @@ -55,7 +55,7 @@ would like to retrieve. For convenience, this method accepts instances of either .. code-block:: python - stream = conn.stream_from_uuid("07d28a44-4991-492d-b9c5-2d8cec5aa6d4") + >>> stream = conn.stream_from_uuid("07d28a44-4991-492d-b9c5-2d8cec5aa6d4") @@ -69,42 +69,65 @@ if needed. .. code-block:: python - start = datetime(2018,1,1,12,30, tzinfo=timezone.utc) - start = start.timestamp() * 1e9 - end = start + (3600 * 1e9) + >>> start = datetime(2018,1,1,12,30, tzinfo=timezone.utc) + >>> start = start.timestamp() * 1e9 + >>> end = start + (3600 * 1e9) - for point, _ in stream.values(start, end): - print(point.time, point.value) + >>> for point, _ in stream.values(start, end): + >>> print(point.time, point.value) Some convenience functions are available to make it easier to deal with converting to nanoseconds. .. code-block:: python - from btrdb.utils.timez import to_nanoseconds, currently_as_ns + >>> from btrdb.utils.timez import to_nanoseconds, currently_as_ns - start = to_nanoseconds(datetime(2018,1,1, tzinfo=timezone.utc)) - end = currently_as_ns() + >>> start = to_nanoseconds(datetime(2018,1,1, tzinfo=timezone.utc)) + >>> end = currently_as_ns() - for point, _ in stream.values(start, end): - print(point.time, point.value) + >>> for point, _ in stream.values(start, end): + >>> print(point.time, point.value) You can also view windows of data at arbitrary levels of detail. One such windowing feature is shown below. .. code-block:: python - # query for windows of data 10,000 nanoseconds wide using a depth of zero - # which is accurate to the nanosecond. - params = { - "start": 1500000000000000000, - "end": 1500000000010000000, - "width": 2000000, - "depth": 0, - } - for window in stream.windows(**params): - for point, version in window: - print(point, version) + >>> # query for windows of data 10,000 nanoseconds wide using a depth of zero + >>> # which is accurate to the nanosecond. + >>> params = { + ... "start": 1500000000000000000, + ... "end": 1500000000010000000, + ... "width": 2000000, + ... "depth": 0, + ... } + >>> for window in stream.windows(**params): + >>> for point, version in window: + >>> print(point, version) + + +Return data as :code:`arrow` tables +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Instead of returning data a :code:`RawPoint` at a time, which can be more computationally intensive, there is now the ability to return the data in a tabular format from the start, which can drastically save on run time as well as facilitate interoperability with many more data-science driven tools. +`Apache Arrow is a language agnostic columnar data schema `_ that has become a defacto standard for in-memory data analytics. +All data retrieval methods in :code:`BTrDB` now have corresponding :code:`arrow-` prepended methods that natively return :code:`pyarrow` data tables. + +.. code-block:: python + + >>> s.arrow_values(start=1500000000000000000, end=1500000002000000001).to_pandas() + time value + 0 2017-07-14 02:40:00+00:00 1.0 + 1 2017-07-14 02:40:00.100000+00:00 2.0 + 2 2017-07-14 02:40:00.200000+00:00 3.0 + 3 2017-07-14 02:40:00.300000+00:00 4.0 + 4 2017-07-14 02:40:00.400000+00:00 5.0 + 5 2017-07-14 02:40:00.500000+00:00 6.0 + 6 2017-07-14 02:40:00.600000+00:00 7.0 + 7 2017-07-14 02:40:00.700000+00:00 8.0 + 8 2017-07-14 02:40:00.800000+00:00 9.0 + 9 2017-07-14 02:40:00.900000+00:00 10.0 + Using StreamSets -------------------- @@ -118,33 +141,45 @@ level of the individual :code:`Stream` object. Aside from being useful to see concurrent data across streams, you can also easily transform the data to other data structures or even serialize the data to disk in one operation. -Some quick examples are shown below but please review the API docs for the full +Some quick examples are shown below but please review the :ref:`API docs ` for the full list of features. + +.. note:: + + In the following examples, notice that the end time is **not** inclusive of the data that is present at :code:`end` . :code:`start` is **inclusive** while :code:`end` is **exclusive**. This is the case for **all** :code:`BTrDB` data query operations. + + .. math:: + [start, end) + + .. code-block:: python - streams = db.streams(*uuid_list) - - # serialize data to disk as CSV - streams.filter(start=1500000000000000000).to_csv("data.csv") - - # convert data to a pandas DataFrame - streams.filter(start=1500000000000000000).to_dataframe() - >> time NW/stream0 NW/stream1 - 0 1500000000000000000 NaN 1.0 - 1 1500000000100000000 2.0 NaN - 2 1500000000200000000 NaN 3.0 - 3 1500000000300000000 4.0 NaN - 4 1500000000400000000 NaN 5.0 - 5 1500000000500000000 6.0 NaN - 6 1500000000600000000 NaN 7.0 - 7 1500000000700000000 8.0 NaN - 8 1500000000800000000 NaN 9.0 - 9 1500000000900000000 10.0 NaN - - # materialize the streams' data - streams.filter(start=1500000000000000000).values() - >> [[RawPoint(1500000000100000000, 2.0), + >>> streams = db.streams(*uuid_list) + + >>> # serialize data to disk as CSV + >>> streams.filter(start=1500000000000000000, end=1500000000900000000).to_csv("data.csv") + + >>> # convert data to a pandas DataFrame + >>> streams.filter(start=1500000000000000000, end=1500000000900000000).to_dataframe() + nw/stream0 nw/stream1 + time + 1500000000000000000 nan 1.0 + 1500000000100000000 2.0 nan + 1500000000200000000 nan 3.0 + 1500000000300000000 4.0 nan + 1500000000400000000 nan 5.0 + 1500000000500000000 6.0 nan + 1500000000600000000 nan 7.0 + 1500000000700000000 8.0 nan + 1500000000800000000 nan 9.0 + + + + + >>> # materialize the streams' data + >>> streams.filter(start=1500000000000000000, end=1500000000900000000).values() + [[RawPoint(1500000000100000000, 2.0), RawPoint(1500000000300000000, 4.0), RawPoint(1500000000500000000, 6.0), RawPoint(1500000000700000000, 8.0), @@ -153,3 +188,48 @@ list of features. RawPoint(1500000000200000000, 3.0), RawPoint(1500000000400000000, 5.0), ... + + + + + + +Return data as :code:`arrow` tables +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:code:`StreamSets` are also able to return :code:`arrow` tables for the group of streams they represent. +This is especially convenient and is usually **much** faster than using the traditional :code:`RawPoint` -based data representation. +We recommend using the :code:`arrow` functions whenever possible. + +.. code-block:: python + + >>> # convert data to a pandas DataFrame, using pyarrow + >>> streams.filter(start=1500000000000000000, end=1500000000900000000) + ... .arrow_to_dataframe() + NW/stream0 NW/stream1 + time + 2017-07-14 02:40:00+00:00 NaN 1.0 + 2017-07-14 02:40:00.100000+00:00 2.0 NaN + 2017-07-14 02:40:00.200000+00:00 NaN 3.0 + 2017-07-14 02:40:00.300000+00:00 4.0 NaN + 2017-07-14 02:40:00.400000+00:00 NaN 5.0 + 2017-07-14 02:40:00.500000+00:00 6.0 NaN + 2017-07-14 02:40:00.600000+00:00 NaN 7.0 + 2017-07-14 02:40:00.700000+00:00 8.0 NaN + 2017-07-14 02:40:00.800000+00:00 NaN 9.0 + + + >>> # materialize the streams' data as an arrow table + >>> streams.filter(start=1500000000000000000, end=1500000000900000000).arrow_values() + pyarrow.Table + time: timestamp[ns, tz=UTC] not null + b29204f4-6c13-4ec7-a149-88e2ff950a72: double not null + 99a0d0b0-e24f-4875-b7d8-eae0036f2149: double not null + ---- + time: [ + ... [2017-07-14 02:40:00.000000000Z,2017-07-14 02:40:00.100000000Z, + ... 2017-07-14 02:40:00.200000000Z,2017-07-14 02:40:00.300000000Z, + ... 2017-07-14 02:40:00.400000000Z,2017-07-14 02:40:00.500000000Z, + ... 2017-07-14 02:40:00.600000000Z,2017-07-14 02:40:00.700000000Z, + ... 2017-07-14 02:40:00.800000000Z]] + b29204f4-6c13-4ec7-a149-88e2ff950a72: [[nan,2,nan,4,nan,6,nan,8,nan]] + 99a0d0b0-e24f-4875-b7d8-eae0036f2149: [[1,nan,3,nan,5,nan,7,nan,9]] diff --git a/docs/source/working/arrow-enabled-queries.rst b/docs/source/working/arrow-enabled-queries.rst index 8a40a5e..fa531c9 100644 --- a/docs/source/working/arrow-enabled-queries.rst +++ b/docs/source/working/arrow-enabled-queries.rst @@ -24,9 +24,9 @@ To learn more about these methods, please refer to the :ref:`arrow_ prefixed met True Multistream Support ^^^^^^^^^^^^^^^^^^^^^^^^ -Until now, there has not been a true multistream query support, our previous api and with the new edits, emulates multistream support with :code:`StreamSet`s and using multithreading. +Until now, there has not been a true multistream query support, our previous api and with the new edits, emulates multistream support with :code:`StreamSet` s and using multithreading. However, this will still only scale to an amount of streams based on the amount of threads that the python threadpool logic can support. -Due to this, raw data queries for :code:`StreamSet`s using our arrow api :code:`StreamSet.filter(start=X, end=Y,).arrow_values()` will now perform true multistream queries. +Due to this, raw data queries for :code:`StreamSet` s using our arrow api :code:`StreamSet.filter(start=X, end=Y,).arrow_values()` will now perform true multistream queries. The platform, instead of the python client, will now quickly grab all the stream data for all streams in your streamset, and then package that back to the python client in an :code:`arrow` table! This leads to data fetch speedups on the order of 10-50x based on the amount and kind of streams. diff --git a/docs/source/working/stream-manage-metadata.rst b/docs/source/working/stream-manage-metadata.rst index 75b7fd6..7d90655 100644 --- a/docs/source/working/stream-manage-metadata.rst +++ b/docs/source/working/stream-manage-metadata.rst @@ -102,7 +102,7 @@ that the data has already been changed by another user or process. annotations=annotations ) -If you would like to remove any keys from your annotations you must use the `replace=True` keyword argument. This will ensure that the annotations dictionary you provide completely replaces the existing values rather than perform an UPSERT operation. The example below shows how you could remove an existing key from the annotations dictionary. +If you would like to remove any keys from your annotations you must use the :code:`replace=True` keyword argument. This will ensure that the annotations dictionary you provide completely replaces the existing values rather than perform an UPSERT operation. The example below shows how you could remove an existing key from the annotations dictionary. .. code-block:: python diff --git a/docs/source/working/stream-query-manage.rst b/docs/source/working/stream-query-manage.rst index d0f83c6..9476381 100644 --- a/docs/source/working/stream-query-manage.rst +++ b/docs/source/working/stream-query-manage.rst @@ -75,15 +75,15 @@ detail. Querying Metadata ----------------- Finally, you can query for metadata using standard SQL although at the moment, only the -`streams` table is available. SQL queries can be submitted using the `query` -method which accepts both a `stmt` and `params` argument. The `stmt` should -contain the SQL you'd like executed with parameter placeholders such as `$1` or -`$2` as shown below. +:code:`streams` table is available. SQL queries can be submitted using the :code:`query` +method which accepts both a :code:`stmt` and :code:`params` argument. The :code:`stmt` should +contain the SQL you'd like executed with parameter placeholders such as :code:`$1` or +:code:`$2` as shown below. .. code-block:: python conn = btrdb.connect() - stmt = "select uuid from streams where name = $1 or name = $2" + stmt = "SELECT uuid FROM streams WHERE name = $1 OR name = $2" params = ["Boston_1", "Boston_2"] for row in conn.query(stmt, params): diff --git a/docs/source/working/stream-view-data.rst b/docs/source/working/stream-view-data.rst index 915e75e..b9c41c7 100644 --- a/docs/source/working/stream-view-data.rst +++ b/docs/source/working/stream-view-data.rst @@ -45,7 +45,7 @@ you query for all the data using the :code:`Stream.values` method due to the mem consumption implied). Each of these three methods returns a tuple containing a RawPoint and the data version number. The exact timestamp can be obtained from the RawPoint. Keep in mind that all of these methods accept a :code:`version` argument so that -you can ask for the earliest, latest, or nearest point from a previous version of the stream. +you can ask for the :code:`earliest`, :code:`latest`, or :code:`nearest` point from a previous version of the stream. .. code-block:: python @@ -86,7 +86,7 @@ aggregate of all the raw data within a window of width 2^pointwidth nanoseconds. Note that :code:`start` is inclusive, but :code:`end` is exclusive. That is, results -will be returned for all windows that start in the interval [start, end). +will be returned for all windows that start in the interval :math:`[start, end)`. If end < start+2^pointwidth you will not get any results. If start and end are not powers of two, the bottom pointwidth bits will be cleared. Each window will contain statistical summaries of the window. Statistical points @@ -150,37 +150,37 @@ of sample depths are provided below. +-------+-------------+---------------------------+-----------------+ As usual when querying data from BTrDB, the :code:`start` time is inclusive -while the :code:`end` time is exclusive. Note that if your last window spans +while the :code:`end` time is exclusive. **Note**: that if your last window spans across the end time then it will not be included in the results. .. code-block:: python - start = 1500000000000000000 - end = 1500000001000000000 - - # view underlying data for comparison - for point, _ in stream.values(start=start, end=end): - print(point) - >> RawPoint(1500000000000000000, 1.0) - >> RawPoint(1500000000100000000, 2.0) - >> RawPoint(1500000000200000000, 3.0) - >> RawPoint(1500000000300000000, 4.0) - >> RawPoint(1500000000400000000, 5.0) - >> RawPoint(1500000000500000000, 6.0) - >> RawPoint(1500000000600000000, 7.0) - >> RawPoint(1500000000700000000, 8.0) - >> RawPoint(1500000000800000000, 9.0) - >> RawPoint(1500000000900000000, 10.0) - - # each window spans 300 milleseconds - width = 300000000 - - # request a precision of roughly 1 millesecond - depth = 20 - - # view windowed data - for point, _ in stream.windows(start=start, end=end, - width=width, depth=depth): - >> StatPoint(1500000000000000000, 1.0, 2.0, 3.0, 3, 0.816496580927726) - >> StatPoint(1500000000300000000, 4.0, 5.0, 6.0, 3, 0.816496580927726) - >> StatPoint(1500000000600000000, 7.0, 8.0, 9.0, 3, 0.816496580927726) + >>> start = 1500000000000000000 + >>> end = 1500000001000000000 + + >>> # view underlying data for comparison + >>> for point, _ in stream.values(start=start, end=end): + >>> print(point) + RawPoint(1500000000000000000, 1.0) + RawPoint(1500000000100000000, 2.0) + RawPoint(1500000000200000000, 3.0) + RawPoint(1500000000300000000, 4.0) + RawPoint(1500000000400000000, 5.0) + RawPoint(1500000000500000000, 6.0) + RawPoint(1500000000600000000, 7.0) + RawPoint(1500000000700000000, 8.0) + RawPoint(1500000000800000000, 9.0) + RawPoint(1500000000900000000, 10.0) + + >>> # each window spans 300 milliseconds + >>> width = 300000000 + + >>> # request a precision of roughly 1 millisecond + >>> depth = 20 + + >>> # view windowed data + >>> for point, _ in stream.windows(start=start, end=end, + ... width=width, depth=depth): + StatPoint(1500000000000000000, 1.0, 2.0, 3.0, 3, 0.816496580927726) + StatPoint(1500000000300000000, 4.0, 5.0, 6.0, 3, 0.816496580927726) + StatPoint(1500000000600000000, 7.0, 8.0, 9.0, 3, 0.816496580927726) diff --git a/docs/source/working/streamsets.rst b/docs/source/working/streamsets.rst index 8a9d807..da6c1d0 100644 --- a/docs/source/working/streamsets.rst +++ b/docs/source/working/streamsets.rst @@ -38,8 +38,8 @@ Filtering To apply query parameters to your request, you should use the :code:`filter` method to supply a :code:`start` or :code:`end` argument. -Keep in mind that :code:`filter` will return a new object so you can keep -multiple filtered StreamSets in memory while you explore your data. The +Keep in mind that :code:`filter` will **return a new object so you can keep +multiple filtered StreamSets in memory while you explore your data**. The :code:`filter` method may be called multiple times but only the final values will be used when it is time to fulfill the request by the server. @@ -83,8 +83,8 @@ re.search to choose the streams to include. Retrieving Data ---------------- -There are two options available when you are ready to process the data from the -server. Both options are fully materialized but are organized in different ways +There are three options available when you are ready to process the data from the +server. All options are fully materialized but are organized in different ways according to what is more convenient for you. StreamSet.values() @@ -131,6 +131,10 @@ consists of 4 streams. >> RawPoint(1500000000900000000, 10.0)]] +.. attention:: + + + StreamSet.rows() ^^^^^^^^^^^^^^^^^^ By contrast, the :code:`rows` method aligns data by time rather than by stream. diff --git a/pyproject.toml b/pyproject.toml index 80f656d..a57516f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,12 @@ [project] name = "btrdb" -version = "5.31.0" +version = "5.32.0" authors = [ {name="PingThingsIO", email="support@pingthings.io"}, ] +maintainers = [ + {name="PingThingsIO", email="support@pingthings.io"}, +] description = "Bindings to interact with the Berkeley Tree Database using gRPC." readme = "README.md" license = {file="LICENSE.txt"} @@ -33,10 +36,6 @@ dependencies = [ "pytz", "pyyaml", "certifi", - "pyarrow", - "polars", - "numpy", - "pandas>=2.0", ] [project.optional-dependencies] @@ -65,8 +64,7 @@ all = [ ] [project.urls] -"Homepage" = "https://btrdb.io" -"Docs" = "https://btrdb.readthedocs.io" +"Docs" = "https://btrdb-python.readthedocs.io/" "Repository" = "https://github.com/pingthingsio/btrdb-python.git" [build-system] diff --git a/release.sh b/release.sh index 31f849e..b63f28c 100755 --- a/release.sh +++ b/release.sh @@ -35,14 +35,12 @@ fi echo "Setting version to v$1.$2.$3" -VERSION_CODE="__version_info__ = { 'major': $1, 'minor': $2, 'micro': $3, 'releaselevel': 'final'}" +VERSION_CODE="__version_info__ = {\"major\": $1, \"minor\": $2, \"micro\": $3, \"releaselevel\": \"final\"}" sed -i.bak "s/^__version_info__.*$/${VERSION_CODE}/g" btrdb/version.py sed -i.bak "s/^version.*$/version\ = \"$1.$2.$3\"/g" pyproject.toml git add btrdb/version.py git add pyproject.toml git commit -m "Release v$1.$2.$3" -git tag v$1.$2.$3 -git push origin v$1.$2.$3 -git push +echo "Now make a PR and merge it into main, then make tag :)" diff --git a/setup.cfg b/setup.cfg index 3f445b3..8d1166f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] -description-file = DESCRIPTION.md -license_file = LICENSE.txt +description_file = DESCRIPTION.md +license_files = LICENSE.txt [aliases] test=pytest diff --git a/setup.py b/setup.py index a5b82d1..df6ad76 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ REPOSITORY = "https://github.com/PingThingsIO/btrdb-python" PACKAGE = "btrdb" URL = "http://btrdb.io/" -DOCS_URL = "https://btrdb.readthedocs.io/en/latest/" +DOCS_URL = "https://btrdb-python.readthedocs.io/" ## Define the keywords KEYWORDS = ("btrdb", "berkeley", "timeseries", "database", "bindings" "gRPC") @@ -133,7 +133,6 @@ def get_description_type(path=PKG_DESCRIBE): "license": LICENSE, "author": AUTHOR, "author_email": EMAIL, - "url": URL, "maintainer": MAINTAINER, "maintainer_email": EMAIL, "project_urls": { diff --git a/tests/btrdb/test_base.py b/tests/btrdb/test_base.py index ac55c07..0fddc90 100644 --- a/tests/btrdb/test_base.py +++ b/tests/btrdb/test_base.py @@ -16,11 +16,11 @@ ########################################################################## import os -from unittest.mock import patch +from unittest.mock import MagicMock, Mock, patch import pytest -from btrdb import BTRDB_API_KEY, BTRDB_ENDPOINTS, __version__, connect +from btrdb import BTRDB_API_KEY, BTRDB_ENDPOINTS, Endpoint, __version__, connect from btrdb.exceptions import ConnectionError ########################################################################## @@ -81,21 +81,34 @@ def test_uses_env_over_profile( connect() mock_connect.assert_called_once_with(endpoints="c", apikey="d") + # @patch("btrdb.utils.credentials.credentials_by_env") + @patch("btrdb.utils.credentials.credentials_by_env") @patch("btrdb.utils.credentials.credentials_by_profile") @patch("btrdb.Connection") - def test_connect_with_env(self, mock_conn, mock_credentials_by_profile): + def test_connect_with_env( + self, mock_conn, mock_credentials_by_profile, mock_credentials_by_env + ): """ Assert connect uses ENV variables """ - mock_credentials_by_profile.return_value = {} - address = "127.0.0.1:4410" - os.environ[BTRDB_ENDPOINTS] = address + with patch( + "btrdb.endpoint.Endpoint.info", return_value=Mock(Endpoint) + ) as mock_ep_info: + mock_ep_info.return_value = MagicMock() + mock_credentials_by_profile.return_value = {} + address = "127.0.0.1:4410" + mock_credentials_by_env.return_value = {"endpoints": address} + os.environ[BTRDB_ENDPOINTS] = address - btrdb = connect() - mock_conn.assert_called_once_with(address, apikey=None) - mock_conn.reset_mock() + btrdb = connect() + mock_conn.assert_called_once_with(address, apikey=None) + mock_conn.reset_mock() - apikey = "abcd" - os.environ[BTRDB_API_KEY] = apikey - btrdb = connect() - mock_conn.assert_called_once_with(address, apikey=apikey) + apikey = "abcd" + os.environ[BTRDB_API_KEY] = apikey + mock_credentials_by_env.return_value = { + "endpoints": address, + "apikey": apikey, + } + btrdb = connect() + mock_conn.assert_called_once_with(address, apikey=apikey) diff --git a/tests/btrdb/test_conn.py b/tests/btrdb/test_conn.py index ed00aa9..3720ef2 100644 --- a/tests/btrdb/test_conn.py +++ b/tests/btrdb/test_conn.py @@ -16,7 +16,7 @@ ########################################################################## import uuid as uuidlib -from unittest.mock import Mock, PropertyMock, call, patch +from unittest.mock import MagicMock, Mock, PropertyMock, call, patch import pytest @@ -41,7 +41,7 @@ def stream1(): type(stream).name = PropertyMock(return_value="gala") stream.tags = Mock(return_value={"name": "gala", "unit": "volts"}) stream.annotations = Mock(return_value=({"owner": "ABC", "color": "red"}, 11)) - stream._btrdb = Mock() + stream._btrdb = MagicMock() return stream @@ -55,7 +55,7 @@ def stream2(): type(stream).name = PropertyMock(return_value="blood") stream.tags = Mock(return_value={"name": "blood", "unit": "amps"}) stream.annotations = Mock(return_value=({"owner": "ABC", "color": "orange"}, 22)) - stream._btrdb = Mock() + stream._btrdb = MagicMock() return stream @@ -69,7 +69,7 @@ def stream3(): type(stream).name = PropertyMock(return_value="yellow") stream.tags = Mock(return_value={"name": "yellow", "unit": "watts"}) stream.annotations = Mock(return_value=({"owner": "ABC", "color": "yellow"}, 33)) - stream._btrdb = Mock() + stream._btrdb = MagicMock() return stream @@ -98,110 +98,120 @@ class TestBTrDB(object): ########################################################################## ## .streams tests ########################################################################## + # TODO: remove this when removing future warnings from `streams_in_collection` + pytestmark = pytest.mark.filterwarnings("ignore::FutureWarning") def test_streams_raises_err_if_version_not_list(self): """ Assert streams raises TypeError if versions is not list """ - db = BTrDB(None) - with pytest.raises(TypeError) as exc: - db.streams("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a", versions="2,2") + with patch("btrdb.endpoint.Endpoint", return_value=MagicMock()) as ep: + db = BTrDB(ep) + with pytest.raises(TypeError) as exc: + db.streams("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a", versions="2,2") - assert "versions argument must be of type list" in str(exc) + assert "versions argument must be of type list" in str(exc) def test_streams_raises_err_if_version_argument_mismatch(self): """ Assert streams raises ValueError if len(identifiers) doesnt match length of versions """ - db = BTrDB(None) - with pytest.raises(ValueError) as exc: - db.streams("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a", versions=[2, 2]) + with patch("btrdb.endpoint.Endpoint", return_value=MagicMock()) as ep: + db = BTrDB(ep) + with pytest.raises(ValueError) as exc: + db.streams("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a", versions=[2, 2]) - assert "versions does not match identifiers" in str(exc) + assert "versions does not match identifiers" in str(exc) def test_streams_stores_versions(self): """ Assert streams correctly stores supplied version info """ - db = BTrDB(None) - uuid1 = uuidlib.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") - uuid2 = uuidlib.UUID("17dbe387-89ea-42b6-864b-f505cdb483f5") - versions = [22, 44] - expected = dict(zip([uuid1, uuid2], versions)) + with patch("btrdb.endpoint.Endpoint", return_value=MagicMock()) as ep: + db = BTrDB(ep) + uuid1 = uuidlib.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + uuid2 = uuidlib.UUID("17dbe387-89ea-42b6-864b-f505cdb483f5") + versions = [22, 44] + expected = dict(zip([uuid1, uuid2], versions)) - streams = db.streams(uuid1, uuid2, versions=versions) - assert streams._pinned_versions == expected + streams = db.streams(uuid1, uuid2, versions=versions) + assert streams._pinned_versions == expected @patch("btrdb.conn.BTrDB.stream_from_uuid") def test_streams_recognizes_uuid(self, mock_func): """ Assert streams recognizes uuid strings """ - db = BTrDB(None) - uuid1 = uuidlib.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") - mock_func.return_value = Stream(db, uuid1) - db.streams(uuid1) + with patch("btrdb.endpoint.Endpoint", return_value=MagicMock()) as ep: + db = BTrDB(ep) + uuid1 = uuidlib.UUID("0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + mock_func.return_value = Stream(db, uuid1) + db.streams(uuid1) - mock_func.assert_called_once() - assert mock_func.call_args[0][0] == uuid1 + mock_func.assert_called_once() + assert mock_func.call_args[0][0] == uuid1 @patch("btrdb.conn.BTrDB.stream_from_uuid") def test_streams_recognizes_uuid_string(self, mock_func): """ Assert streams recognizes uuid strings """ - db = BTrDB(None) - uuid1 = "0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a" - mock_func.return_value = Stream(db, uuid1) - db.streams(uuid1) + with patch("btrdb.endpoint.Endpoint", return_value=MagicMock()) as ep: + db = BTrDB(ep) + uuid1 = "0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a" + mock_func.return_value = Stream(db, uuid1) + db.streams(uuid1) - mock_func.assert_called_once() - assert mock_func.call_args[0][0] == uuid1 + mock_func.assert_called_once() + assert mock_func.call_args[0][0] == uuid1 @patch("btrdb.conn.BTrDB.streams_in_collection") def test_streams_handles_path(self, mock_func): """ Assert streams calls streams_in_collection for collection/name paths """ - db = BTrDB(None) - ident = "zoo/animal/dog" - mock_func.return_value = [ - Stream(db, "0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a"), - ] - db.streams(ident, "0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") - - mock_func.assert_called_once() - assert mock_func.call_args[0][0] == "zoo/animal" - assert mock_func.call_args[1] == { - "is_collection_prefix": False, - "tags": {"name": "dog"}, - } + with patch("btrdb.endpoint.Endpoint", return_value=MagicMock()) as ep: + db = BTrDB(ep) + ident = "zoo/animal/dog" + mock_func.return_value = [ + Stream(db, "0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a"), + ] + db.streams(ident, "0d22a53b-e2ef-4e0a-ab89-b2d48fb2592a") + + mock_func.assert_called_once() + assert mock_func.call_args[0][0] == "zoo/animal" + assert mock_func.call_args[1] == { + "is_collection_prefix": False, + "tags": {"name": "dog"}, + } @patch("btrdb.conn.BTrDB.streams_in_collection") def test_streams_raises_err(self, mock_func): """ Assert streams raises StreamNotFoundError """ - db = BTrDB(None) - ident = "zoo/animal/dog" - - mock_func.return_value = [] - with pytest.raises(StreamNotFoundError) as exc: + with patch("btrdb.endpoint.Endpoint", return_value=MagicMock()) as ep: + db = BTrDB(ep) + ident = "zoo/animal/dog" + + mock_func.return_value = [] + with pytest.raises(StreamNotFoundError) as exc: + db.streams(ident) + + # check that does not raise if one returned + mock_func.return_value = [ + Stream(db, ident), + ] db.streams(ident) - # check that does not raise if one returned - mock_func.return_value = [ - Stream(db, ident), - ] - db.streams(ident) - def test_streams_raises_valueerror(self): """ Assert streams raises ValueError if not uuid, uuid str, or path """ - db = BTrDB(None) - with pytest.raises(ValueError) as exc: - db.streams(11) + with patch("btrdb.endpoint.Endpoint", return_value=MagicMock()) as ep: + db = BTrDB(ep) + with pytest.raises(ValueError) as exc: + db.streams(11) ########################################################################## ## other tests diff --git a/tests/btrdb/test_transformers.py b/tests/btrdb/test_transformers.py index 833de69..c7ce719 100644 --- a/tests/btrdb/test_transformers.py +++ b/tests/btrdb/test_transformers.py @@ -601,7 +601,7 @@ def test_to_series(self, streamset): def test_to_series_name_lambda(self, streamset): """ - assert to_dateframe uses name lambda + assert to_series uses name lambda """ result = streamset.to_series(name_callable=lambda s: s.name) assert [s.name for s in result] == ["stream0", "stream1", "stream2", "stream3"] @@ -691,23 +691,16 @@ def test_to_dataframe(self, streamset): df.set_index("time", inplace=True) assert to_dataframe(streamset).equals(df) - def test_to_dataframe_column_issues_warning(self, statpoint_streamset): + def test_to_dataframe_column_issues_error(self, statpoint_streamset): """ - assert to_dateframe with column argument issues warning + assert to_dateframe with column argument issues error """ columns = ["test/cats", "test/dogs", "test/horses", "test/fishes"] - with pytest.deprecated_call(): + with pytest.raises(TypeError) as unexpected_key_err: statpoint_streamset.to_dataframe(columns=columns) - - def test_to_dataframe_column(self, statpoint_streamset): - """ - assert to_dateframe with column argument actually renames columns - """ - columns = ["test/cats", "test/dogs", "test/horses", "test/fishes"] - with pytest.deprecated_call(): - df = statpoint_streamset.to_dataframe(columns=columns) - - assert df.columns.tolist() == columns + assert "got an unexpected keyword argument 'columns'" in str( + unexpected_key_err.value + ) def test_to_dataframe_multindex(self, statpoint_streamset): """ diff --git a/tests/btrdb/utils/test_general.py b/tests/btrdb/utils/test_general.py index a13cf3c..0edb86e 100644 --- a/tests/btrdb/utils/test_general.py +++ b/tests/btrdb/utils/test_general.py @@ -56,6 +56,36 @@ def test_from_nanoseconds(self, nsec, expected): """ assert pointwidth.from_nanoseconds(nsec) == expected + @pytest.mark.parametrize( + "pw_time_range, expected", + [ + ( + (52, 1560127027000000000, 1702702800000000000), + (1558245471070191616, 1697857059518676992, 32), + ), + ( + (48, 1560127027000000000, 1702702800000000000), + (1559934320930455552, 1702360659146047488, 507), + ), + ( + (44, 1456876800000000000, 1464652800000000000), + (1456861702896222208, 1464619856941809664, 442), + ), + ( + (38, 1456876800000000000, 1464652800000000000), + (1456876546303197184, 1464652292534829056, 28289), + ), + ( + (32, 1456876800000000000, 1464652800000000000), + (1456876799706267648, 1464652795046002688, 1810491), + ), + ], + ) + def test_for_aligned_windows(self, pw_time_range, expected): + pw, start, end = pw_time_range + result = pointwidth(pw).for_aligned_windows(start, end) + assert result == expected + def test_time_conversions(self): """ Test standard pointwidth time conversions @@ -114,3 +144,19 @@ def test_incr(self): Test incrementing a pointwidth """ assert pointwidth(23).incr() == 24 + + @pytest.mark.parametrize( + "pw, expected", + [ + (54, timedelta(days=208, seconds=43198, microseconds=509482)), + (51, timedelta(days=26, seconds=5399, microseconds=813685)), + (49, timedelta(days=6, seconds=44549, microseconds=953421)), + (46, timedelta(seconds=70368, microseconds=744178)), + (43, timedelta(seconds=8796, microseconds=93022)), + (39, timedelta(seconds=549, microseconds=755814)), + (34, timedelta(seconds=17, microseconds=179869)), + (30, timedelta(seconds=1, microseconds=73742)), + ], + ) + def test_to_timedelta(self, pw, expected): + assert pointwidth(pw).to_timedelta() == expected diff --git a/tests/btrdb_integration/test_conn.py b/tests/btrdb_integration/test_conn.py index 5d30c4f..bc6c9ac 100644 --- a/tests/btrdb_integration/test_conn.py +++ b/tests/btrdb_integration/test_conn.py @@ -1,36 +1,57 @@ import logging +import os from uuid import uuid4 as new_uuid import pytest import btrdb - - -def test_connection_info(conn): - info = conn.info() - logging.info(f"connection info: {info}") - - -def test_create_stream(conn, tmp_collection): - uuid = new_uuid() - stream = conn.create(uuid, tmp_collection, tags={"name": "s"}) - assert stream.uuid == uuid - assert stream.name == "s" - - -def test_query(conn, tmp_collection): - conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) - conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) - uuids = conn.query( - "select name from streams where collection = $1 order by name;", - [tmp_collection], +from btrdb.exceptions import BTrDBError + + +@pytest.mark.skipif( + os.getenv("BTRDB_INTEGRATION_TEST_PROFILE") is None, + reason="Need an integration test server to run integration tests.", +) +class TestConnection: + def test_connection_info(self, conn): + info = conn.info() + logging.info(f"connection info: {info}") + + def test_incorrect_connect( + self, + ): + err_msg = r"""Could not connect to the database, error message: <_InactiveRpcError of RPC that terminated with:\n\tstatus = StatusCode.UNAUTHENTICATED\n\tdetails = "invalid api key"\n""" + with pytest.raises(BTrDBError, match=err_msg): + conn = btrdb.connect(conn_str="127.0.0.1:4410", apikey="BOGUS_KEY") + + @pytest.mark.xfail( + reason="Should return BTrDBError, but returns GRPCError instead, FIXME" ) - assert len(uuids) == 2 - assert uuids[0]["name"] == "s1" - assert uuids[1]["name"] == "s2" - - -def test_list_collections(conn, tmp_collection): - assert tmp_collection not in conn.list_collections() - stream = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) - assert tmp_collection in conn.list_collections() + def test_from_uuid(self, conn): + # todo, investigate this, the stat code returned is 0, which is why its not being returned as a btrdb error + uu = new_uuid() + stream = conn.stream_from_uuid(uu) + with pytest.raises(btrdb.exceptions.BTrDBError): + print(stream) + + def test_create_stream(self, conn, tmp_collection): + uuid = new_uuid() + stream = conn.create(uuid, tmp_collection, tags={"name": "s"}) + assert stream.uuid == uuid + assert stream.name == "s" + + def test_query(self, conn, tmp_collection): + conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + uuids = conn.query( + "select name from streams where collection = $1 order by name;", + [tmp_collection], + ) + assert len(uuids) == 2 + assert uuids[0]["name"] == "s1" + assert uuids[1]["name"] == "s2" + + def test_list_collections(self, conn, tmp_collection): + assert tmp_collection not in conn.list_collections() + stream = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) + assert tmp_collection in conn.list_collections() diff --git a/tests/btrdb_integration/test_stream.py b/tests/btrdb_integration/test_stream.py index e65c71d..5fba2a1 100644 --- a/tests/btrdb_integration/test_stream.py +++ b/tests/btrdb_integration/test_stream.py @@ -61,6 +61,24 @@ def test_arrow_insert_and_values( assert data == fetched_data +def test_arrow_values_template_schema(conn, tmp_collection): + s = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) + t = currently_as_ns() + times = [100, 200, 300, 400] + values = [1.0, 2.0, 3.0, 4.0] + s.insert(list(zip(times, values))) + schema = pa.schema( + [ + pa.field("t", pa.int64(), nullable=False), + pa.field("v", pa.float32(), nullable=False), + ] + ) + expected = pa.Table.from_arrays([pa.array(times), pa.array(values)], schema=schema) + fetched_data = s.arrow_values(start=times[0], end=times[-1] + 1, schema=schema) + assert expected == fetched_data + assert expected.schema.equals(fetched_data.schema) + + @pytest.mark.parametrize( "merge_policy,duplicates_expected", [("never", True), ("equal", True), ("retain", False), ("replace", False)], @@ -103,7 +121,7 @@ def test_arrow_values_table_schema( assert single_stream_values_arrow_schema.equals(fetched_data.schema) -def test_arrow_values_table_schema( +def test_arrow_values_table_schema_2( conn, tmp_collection, single_stream_values_arrow_schema ): s = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) @@ -330,19 +348,21 @@ def test_arrow_empty_values_schema(conn, tmp_collection): assert schema.equals(data.schema) -@pytest.mark.xfail def test_stream_annotation_update(conn, tmp_collection): - # XXX marked as expected failure until someone has time to investigate. s = conn.create( - new_uuid(), tmp_collection, tags={"name": "s"}, annotations={"foo": "bar"} + new_uuid(), + tmp_collection + "foo/", + tags={"name": "s"}, + annotations={"foo": "bar"}, ) annotations1, version1 = s.annotations() - assert version1 == 0 + assert version1 == 1 assert annotations1["foo"] == "bar" s.update(annotations={"foo": "baz"}) annotations2, version2 = s.annotations() assert version2 > version1 assert annotations2["foo"] == "baz" s.update(annotations={}, replace=True) - annotations3, _ = s.annotations() + annotations3, version3 = s.annotations() assert len(annotations3) == 0 + assert version3 > version2 diff --git a/tests/btrdb_integration/test_streamset.py b/tests/btrdb_integration/test_streamset.py index d173818..5a0fbec 100644 --- a/tests/btrdb_integration/test_streamset.py +++ b/tests/btrdb_integration/test_streamset.py @@ -73,6 +73,36 @@ def test_streamset_arrow_values(conn, tmp_collection): assert expected_schema.equals(values.schema) +def test_streamset_template_schema(conn, tmp_collection): + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + t1 = [100, 105, 110, 115, 120] + t2 = [101, 106, 110, 114, 119] + d1 = [0.0, 1.0, 2.0, 3.0, 4.0] + d2 = [5.0, 6.0, 7.0, 8.0, 9.0] + s1.insert(list(zip(t1, d1))) + s2.insert(list(zip(t2, d2))) + schema = pa.schema( + [ + pa.field("t", pa.timestamp("ns", tz="UTC"), nullable=False), + pa.field("a", pa.float64(), nullable=False), + pa.field("b", pa.float32(), nullable=False), + ] + ) + ss = btrdb.stream.StreamSet([s1, s2]).filter(start=100, end=121, schema=schema) + expected_times = [100, 101, 105, 106, 110, 114, 115, 119, 120] + expected_col1 = [0.0, np.NaN, 1.0, np.NaN, 2.0, np.NaN, 3.0, np.NaN, 4.0] + expected_col2 = [np.NaN, 5.0, np.NaN, 6.0, 7.0, 8.0, np.NaN, 9.0, np.NaN] + values = ss.arrow_values() + times = [t.value for t in values["t"]] + col1 = [np.NaN if isnan(v.as_py()) else v.as_py() for v in values["a"]] + col2 = [np.NaN if isnan(v.as_py()) else v.as_py() for v in values["b"]] + assert times == expected_times + assert col1 == expected_col1 + assert col2 == expected_col2 + assert schema.equals(values.schema) + + @pytest.mark.parametrize( "name_callable", [(None), (lambda s: str(s.uuid)), (lambda s: s.name + "/" + s.collection)], @@ -96,7 +126,6 @@ def test_streamset_arrow_windows_vs_windows(conn, tmp_collection, name_callable) .windows(width=btrdb.utils.timez.ns_delta(nanoseconds=10)) ) values_arrow = ss.arrow_to_dataframe(name_callable=name_callable) - values_arrow.set_index("time", inplace=True) values_arrow.index = pd.DatetimeIndex(values_arrow.index) values_prev = ss.to_dataframe(name_callable=name_callable).convert_dtypes( dtype_backend="pyarrow" @@ -128,7 +157,6 @@ def test_streamset_arrow_windows_vs_windows_agg_all(conn, tmp_collection): .windows(width=btrdb.utils.timez.ns_delta(nanoseconds=10)) ) values_arrow = ss.arrow_to_dataframe(name_callable=None, agg=["all"]) - values_arrow.set_index("time", inplace=True) values_arrow.index = pd.DatetimeIndex(values_arrow.index) values_prev = ss.to_dataframe(name_callable=None, agg="all") values_prev = values_prev.apply(lambda x: x.astype(str(x.dtype) + "[pyarrow]")) @@ -181,10 +209,48 @@ def test_streamset_arrow_aligned_windows_vs_aligned_windows( ss = ( btrdb.stream.StreamSet([s1, s2, s3]) .filter(start=100, end=121) - .windows(width=btrdb.utils.general.pointwidth.from_nanoseconds(10)) + .aligned_windows(pointwidth=btrdb.utils.general.pointwidth.from_nanoseconds(10)) + ) + values_arrow = ss.arrow_to_dataframe(name_callable=name_callable) + values_arrow.index = pd.DatetimeIndex(values_arrow.index) + values_prev = ss.to_dataframe( + name_callable=name_callable + ) # .convert_dtypes(dtype_backend='pyarrow') + values_prev = values_prev.apply(lambda x: x.astype(str(x.dtype) + "[pyarrow]")) + values_prev = values_prev.apply( + lambda x: x.astype("uint64[pyarrow]") if "count" in x.name else x + ) + values_prev.index = pd.DatetimeIndex(values_prev.index, tz="UTC") + col_map = {old_col: old_col + "/mean" for old_col in values_prev.columns} + values_prev = values_prev.rename(columns=col_map) + assert values_arrow.equals(values_prev) + + +@pytest.mark.parametrize( + "name_callable", + [(None), (lambda s: str(s.uuid)), (lambda s: s.name + "/" + s.collection)], + ids=["empty", "uu_as_str", "name_collection"], +) +def test_streamset_arrow_aligned_windows_join_logic( + conn, tmp_collection, name_callable +): + s1 = conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) + s2 = conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) + s3 = conn.create(new_uuid(), tmp_collection, tags={"name": "s3"}) + t1 = [100, 105, 110, 115, 120] + t2 = [101, 106, 110, 132, 140] + d1 = [0.0, 1.0, 2.0, 3.0, 4.0] + d2 = [5.0, 6.0, 7.0, 8.0, 9.0] + d3 = [1.0, 9.0, 44.0, 8.0, 9.0] + s1.insert(list(zip(t1, d1))) + s2.insert(list(zip(t2, d2))) + s3.insert(list(zip(t2, d3))) + ss = ( + btrdb.stream.StreamSet([s1, s2, s3]) + .filter(start=100, end=141) + .aligned_windows(pointwidth=btrdb.utils.general.pointwidth.from_nanoseconds(8)) ) values_arrow = ss.arrow_to_dataframe(name_callable=name_callable) - values_arrow.set_index("time", inplace=True) values_arrow.index = pd.DatetimeIndex(values_arrow.index) values_prev = ss.to_dataframe( name_callable=name_callable @@ -247,7 +313,6 @@ def test_arrow_streamset_to_dataframe(conn, tmp_collection): s2.insert(list(zip(t2, d2))) ss = btrdb.stream.StreamSet([s1, s2]).filter(start=100, end=121) values = ss.arrow_to_dataframe() - values.set_index("time", inplace=True) expected_times = [100, 101, 105, 106, 110, 114, 115, 119, 120] expected_times = [ pa.scalar(v, type=pa.timestamp("ns", tz="UTC")).as_py() for v in expected_times @@ -270,11 +335,11 @@ def test_arrow_streamset_to_dataframe(conn, tmp_collection): pa.field(tmp_collection + "/s2", type=pa.float64(), nullable=False), ] ) - expected_table = pa.Table.from_pydict(expected_dat, schema=schema) + expected_table = pa.Table.from_pydict(mapping=expected_dat, schema=schema) expected_df = expected_table.to_pandas( - timestamp_as_object=False, types_mapper=pd.ArrowDtype - ) - expected_df.set_index("time", inplace=True) + timestamp_as_object=False, + types_mapper=pd.ArrowDtype, + ).set_index("time") expected_df.index = pd.DatetimeIndex(expected_df.index, tz="UTC") np_test.assert_array_equal( values.values.astype(float), expected_df.values.astype(float) @@ -303,9 +368,9 @@ def test_arrow_streamset_to_polars(conn, tmp_collection): tmp_collection + "/s2": expected_col2, } expected_df = pd.DataFrame( - expected_dat, index=pd.DatetimeIndex(expected_times) - ).reset_index(names="time") - expected_df_pl = pl.from_pandas(expected_df, nan_to_null=False) + expected_dat, index=pd.DatetimeIndex(expected_times, name="time") + ) + expected_df_pl = pl.from_pandas(expected_df, nan_to_null=False, include_index=True) pl_test.assert_frame_equal(values, expected_df_pl) @@ -337,9 +402,9 @@ def test_streamset_arrow_polars_vs_old_to_polars(conn, tmp_collection, name_call tmp_collection + "/s2": expected_col2, } expected_df = pd.DataFrame( - expected_dat, index=pd.DatetimeIndex(expected_times, tz="UTC") - ).reset_index(names="time") - expected_df_pl = pl.from_pandas(expected_df, nan_to_null=False) + expected_dat, index=pd.DatetimeIndex(expected_times, tz="UTC", name="time") + ) + expected_df_pl = pl.from_pandas(expected_df, nan_to_null=False, include_index=True) pl_test.assert_frame_equal(values_arrow, expected_df_pl) pl_test.assert_frame_equal(values_non_arrow, expected_df_pl) pl_test.assert_frame_equal(values_non_arrow, values_arrow) @@ -377,7 +442,7 @@ def test_streamset_windows_arrow_polars_vs_old_to_polars( } new_names["time"] = "time" values_non_arrow_pl = values_non_arrow_pl.rename(mapping=new_names) - assert values_arrow_pl.frame_equal(values_non_arrow_pl) + assert values_arrow_pl.equals(values_non_arrow_pl) def test_streamset_windows_aggregates_filter(conn, tmp_collection): @@ -398,7 +463,6 @@ def test_streamset_windows_aggregates_filter(conn, tmp_collection): .windows(width=btrdb.utils.timez.ns_delta(nanoseconds=10)) ) values_arrow_df = ss.arrow_to_dataframe(agg=["mean", "stddev"]) - values_arrow_df.set_index("time", inplace=True) values_arrow_df.index = pd.DatetimeIndex(values_arrow_df.index) values_non_arrow_df = ss.to_dataframe(agg="all") values_non_arrow_df.index = pd.DatetimeIndex(values_non_arrow_df.index, tz="UTC") @@ -487,7 +551,6 @@ def test_timesnap_with_different_sampling_frequencies(freq, conn, tmp_collection df = stset.filter( start=start, end=stop, sampling_frequency=freq ).arrow_to_dataframe() - df.set_index("time", inplace=True) total_points = df.shape[0] * df.shape[1] total_raw_pts = len(v1) * len(stset) expected_frac_of_pts = 1 if freq is None else freq / data_insert_freq From 44019370406577e8b44824ad3aa1dbf6eb492455 Mon Sep 17 00:00:00 2001 From: Justin Gilmer <16173348+justinGilmer@users.noreply.github.com> Date: Tue, 26 Mar 2024 16:58:28 -0400 Subject: [PATCH 112/131] Update readthedocs requirements file for building html. --- docs/requirements.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/requirements.txt b/docs/requirements.txt index 2e6e1f1..6d1cc57 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -6,3 +6,9 @@ sphinx-copybutton sphinx-design numpydoc pydata-sphinx-theme +pyarrow +pandas +polars +grpcio +grpcio-tools +pyyaml From a943d2e6021821a0cc0c55755d397aee1c5e9bd0 Mon Sep 17 00:00:00 2001 From: Justin Gilmer <16173348+justinGilmer@users.noreply.github.com> Date: Mon, 8 Apr 2024 16:29:57 -0400 Subject: [PATCH 113/131] Update changelog (#85) --- CHANGELOG.md | 31 +++++++++++++++-------------- docs/source/changelog.rst | 41 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d0c080..78b4425 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,39 +1,40 @@ # Changelog -## 5.32.0 - YYYY-MM-DD +## 5.32.0 ### What's Changed -- Initial docstring overhaul and a new test for better documentation and test coverage. by @JustinGilmer in #82 -- Test new join functionality for improved data loading for windowed queries. by @JustinGilmer in #80 -- Improve `arrow_to_dataframe` function for handling large amounts of columns, enhancing performance and usability. by @Jefflinf in #73 -- Expand testing to include Python 3.11, ensuring compatibility and stability. by @JustinGilmer in #74 -- Update exception handling to better support `RpcErrors`, improving error management and debugging. by @JustinGilmer in #72 -- Introduce an option for specifying the schema of the returned data, allowing for more flexibility in data handling. by @TODO in #51 -- Remove non-required dependencies and migrate to 'data' optional dependency for a lighter package and easier installation. by @JustinGilmer in #71 -- New method to get first and last timestamps from `aligned_windows`, enhancing data analysis capabilities. by @Jefflinf in #70 -- Add `to_timedelta` method for `pointwidth` class, providing more options for time-based data manipulation. by @Jefflinf in #69 +- Initial docstring overhaul and a new test for better documentation and test coverage. by @JustinGilmer in [#82](https://github.com/pingthingsio/btrdb-python/pull/82) +- Test new join logic for improved data loading for windowed queries. by @JustinGilmer in [#80](https://github.com/pingthingsio/btrdb-python/pull/80) +- Improve `arrow_to_dataframe` function for handling large amounts of columns, enhancing performance and usability. by @Jefflinf in [#73](https://github.com/pingthingsio/btrdb-python/pull/73) +- Expand testing to include Python 3.11, ensuring compatibility and stability. by @JustinGilmer in [#74](https://github.com/pingthingsio/btrdb-python/pull/74) +- Update exception handling to better support `RpcErrors`, improving error management and debugging. by @JustinGilmer in [#72](https://github.com/pingthingsio/btrdb-python/pull/72) +- Introduce an option for specifying the schema of the returned raw data, allowing for more flexibility in data handling. by @andrewchambers in [#51](https://github.com/pingthingsio/btrdb-python/pull/51) +- Remove non-required dependencies and migrate to 'data' optional dependency for a lighter package and easier installation. by @JustinGilmer in [#71](https://github.com/pingthingsio/btrdb-python/pull/71) +- New method to get first and last timestamps from `aligned_windows`, enhancing data analysis capabilities. by @Jefflinf in [#70](https://github.com/pingthingsio/btrdb-python/pull/70) +- Add `to_timedelta` method for `pointwidth` class, providing more options for time-based data manipulation. by @Jefflinf in [#69](https://github.com/pingthingsio/btrdb-python/pull/69) ### Fixed -- Fix `NoneType` error for `earliest/latest` for empty streams, ensuring reliability and error handling. by @Jefflinf in #64 -- Correct integration tests where the time column is not automatically set as the index, improving test accuracy and reliability. by @JustinGilmer in #56 +- Fix `NoneType` error for `earliest/latest` for empty streams, ensuring reliability and error handling. by @Jefflinf in [#64](https://github.com/pingthingsio/btrdb-python/pull/64) +- Correct integration tests where the time column is not automatically set as the index, improving test accuracy and reliability. by @JustinGilmer in [#56](https://github.com/pingthingsio/btrdb-python/pull/56) ### Deprecated -- FutureWarning for `streams_in_collection` to return `StreamSet` in the future, preparing users for upcoming API changes. by @Jefflinf in #60 +- FutureWarning for `streams_in_collection` to return `StreamSet` in the future, preparing users for upcoming API changes. by @Jefflinf in [#60](https://github.com/pingthingsio/btrdb-python/pull/60) **Full Changelog**: [GitHub compare view](https://github.com/PingThingsIO/btrdb-python/compare/v5.31.0...v5.32.0) ## 5.31.0 ## What's Changed -* Release v5.30.2 by @youngale-pingthings in https://github.com/PingThingsIO/btrdb-python/pull/42 * Have release script update pyproject.toml file by @youngale-pingthings in https://github.com/PingThingsIO/btrdb-python/pull/48 * Provide option to sort the arrow tables by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/47 -* [sc-25841] Remove 4MB limit for gRPC message payloads by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/49 +* Remove 4MB limit for gRPC message payloads by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/49 * Update documentation for arrow methods by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/50 * Update from staging by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/54 * Sort tables by time by default for any `pyarrow` tables. by @justinGilmer in * Fix deprecation warnings for pip installations. by @jleifnf in +**Full Changelog**: [GitHub compare view](https://github.com/PingThingsIO/btrdb-python/compare/v5.30.2...v5.31.0) + ## 5.30.2 ### What's Changed * Update readthedocs to new yaml for testing. by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/40 diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index 8a6881d..59ff417 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -2,6 +2,47 @@ Changelog ========= +5.32.0 +------ + +What's Changed +^^^^^^^^^^^^^^ +- Initial docstring overhaul and a new test for better documentation and test coverage. by @JustinGilmer in `#82 `_ +- Test new join logic for improved data loading for windowed queries. by @JustinGilmer in `#80 `_ +- Improve :code:`arrow_to_dataframe` function for handling large amounts of columns, enhancing performance and usability. by @Jefflinf in `#73 `_ +- Expand testing to include Python 3.11, ensuring compatibility and stability. by @JustinGilmer in `#74 `_ +- Update exception handling to better support :code:`RpcErrors`, improving error management and debugging. by @JustinGilmer in `#72 `_ +- Introduce an option for specifying the schema of the returned raw data, allowing for more flexibility in data handling. by @andrewchambers in `#51 `_ +- Remove non-required dependencies and migrate to 'data' optional dependency for a lighter package and easier installation. by @JustinGilmer in `#71 `_ +- New method to get first and last timestamps from ``aligned_windows``, enhancing data analysis capabilities. by @Jefflinf in `#70 `_ +- Add ``to_timedelta`` method for ``pointwidth`` class, providing more options for time-based data manipulation. by @Jefflinf in `#69 `_ + +Fixed +^^^^^ +- Fix ``NoneType`` error for ``earliest/latest`` for empty streams, ensuring reliability and error handling. by @Jefflinf in `#64 `_ +- Correct integration tests where the time column is not automatically set as the index, improving test accuracy and reliability. by @JustinGilmer in `#56 `_ + +Deprecated +^^^^^^^^^^ +- FutureWarning for ``streams_in_collection`` to return ``StreamSet`` in the future, preparing users for upcoming API changes. by @Jefflinf in `#60 `_ + +**Full Changelog**: `GitHub compare view `_ + + +5.31.0 +------ +What's Changed +^^^^^^^^^^^^^^ +* Have release script update pyproject.toml file by @youngale-pingthings in https://github.com/PingThingsIO/btrdb-python/pull/48 +* Provide option to sort the arrow tables by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/47 +* Remove 4MB limit for gRPC message payloads by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/49 +* Update documentation for arrow methods by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/50 +* Update from staging by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/54 +* Sort tables by time by default for any ``pyarrow`` tables. by @justinGilmer in +* Fix deprecation warnings for pip installations. by @jleifnf in + +**Full Changelog**: `GitHub compare view `_ + 5.30.2 ------ What’s Changed From 77c33561643e2bbdc2e21f55d4e480bddde06cfa Mon Sep 17 00:00:00 2001 From: Justin Gilmer <16173348+justinGilmer@users.noreply.github.com> Date: Mon, 8 Apr 2024 16:53:23 -0400 Subject: [PATCH 114/131] Update changelog (#85) (#86) --- CHANGELOG.md | 31 +++++++++++++++-------------- docs/source/changelog.rst | 41 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d0c080..78b4425 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,39 +1,40 @@ # Changelog -## 5.32.0 - YYYY-MM-DD +## 5.32.0 ### What's Changed -- Initial docstring overhaul and a new test for better documentation and test coverage. by @JustinGilmer in #82 -- Test new join functionality for improved data loading for windowed queries. by @JustinGilmer in #80 -- Improve `arrow_to_dataframe` function for handling large amounts of columns, enhancing performance and usability. by @Jefflinf in #73 -- Expand testing to include Python 3.11, ensuring compatibility and stability. by @JustinGilmer in #74 -- Update exception handling to better support `RpcErrors`, improving error management and debugging. by @JustinGilmer in #72 -- Introduce an option for specifying the schema of the returned data, allowing for more flexibility in data handling. by @TODO in #51 -- Remove non-required dependencies and migrate to 'data' optional dependency for a lighter package and easier installation. by @JustinGilmer in #71 -- New method to get first and last timestamps from `aligned_windows`, enhancing data analysis capabilities. by @Jefflinf in #70 -- Add `to_timedelta` method for `pointwidth` class, providing more options for time-based data manipulation. by @Jefflinf in #69 +- Initial docstring overhaul and a new test for better documentation and test coverage. by @JustinGilmer in [#82](https://github.com/pingthingsio/btrdb-python/pull/82) +- Test new join logic for improved data loading for windowed queries. by @JustinGilmer in [#80](https://github.com/pingthingsio/btrdb-python/pull/80) +- Improve `arrow_to_dataframe` function for handling large amounts of columns, enhancing performance and usability. by @Jefflinf in [#73](https://github.com/pingthingsio/btrdb-python/pull/73) +- Expand testing to include Python 3.11, ensuring compatibility and stability. by @JustinGilmer in [#74](https://github.com/pingthingsio/btrdb-python/pull/74) +- Update exception handling to better support `RpcErrors`, improving error management and debugging. by @JustinGilmer in [#72](https://github.com/pingthingsio/btrdb-python/pull/72) +- Introduce an option for specifying the schema of the returned raw data, allowing for more flexibility in data handling. by @andrewchambers in [#51](https://github.com/pingthingsio/btrdb-python/pull/51) +- Remove non-required dependencies and migrate to 'data' optional dependency for a lighter package and easier installation. by @JustinGilmer in [#71](https://github.com/pingthingsio/btrdb-python/pull/71) +- New method to get first and last timestamps from `aligned_windows`, enhancing data analysis capabilities. by @Jefflinf in [#70](https://github.com/pingthingsio/btrdb-python/pull/70) +- Add `to_timedelta` method for `pointwidth` class, providing more options for time-based data manipulation. by @Jefflinf in [#69](https://github.com/pingthingsio/btrdb-python/pull/69) ### Fixed -- Fix `NoneType` error for `earliest/latest` for empty streams, ensuring reliability and error handling. by @Jefflinf in #64 -- Correct integration tests where the time column is not automatically set as the index, improving test accuracy and reliability. by @JustinGilmer in #56 +- Fix `NoneType` error for `earliest/latest` for empty streams, ensuring reliability and error handling. by @Jefflinf in [#64](https://github.com/pingthingsio/btrdb-python/pull/64) +- Correct integration tests where the time column is not automatically set as the index, improving test accuracy and reliability. by @JustinGilmer in [#56](https://github.com/pingthingsio/btrdb-python/pull/56) ### Deprecated -- FutureWarning for `streams_in_collection` to return `StreamSet` in the future, preparing users for upcoming API changes. by @Jefflinf in #60 +- FutureWarning for `streams_in_collection` to return `StreamSet` in the future, preparing users for upcoming API changes. by @Jefflinf in [#60](https://github.com/pingthingsio/btrdb-python/pull/60) **Full Changelog**: [GitHub compare view](https://github.com/PingThingsIO/btrdb-python/compare/v5.31.0...v5.32.0) ## 5.31.0 ## What's Changed -* Release v5.30.2 by @youngale-pingthings in https://github.com/PingThingsIO/btrdb-python/pull/42 * Have release script update pyproject.toml file by @youngale-pingthings in https://github.com/PingThingsIO/btrdb-python/pull/48 * Provide option to sort the arrow tables by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/47 -* [sc-25841] Remove 4MB limit for gRPC message payloads by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/49 +* Remove 4MB limit for gRPC message payloads by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/49 * Update documentation for arrow methods by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/50 * Update from staging by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/54 * Sort tables by time by default for any `pyarrow` tables. by @justinGilmer in * Fix deprecation warnings for pip installations. by @jleifnf in +**Full Changelog**: [GitHub compare view](https://github.com/PingThingsIO/btrdb-python/compare/v5.30.2...v5.31.0) + ## 5.30.2 ### What's Changed * Update readthedocs to new yaml for testing. by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/40 diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index 8a6881d..59ff417 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -2,6 +2,47 @@ Changelog ========= +5.32.0 +------ + +What's Changed +^^^^^^^^^^^^^^ +- Initial docstring overhaul and a new test for better documentation and test coverage. by @JustinGilmer in `#82 `_ +- Test new join logic for improved data loading for windowed queries. by @JustinGilmer in `#80 `_ +- Improve :code:`arrow_to_dataframe` function for handling large amounts of columns, enhancing performance and usability. by @Jefflinf in `#73 `_ +- Expand testing to include Python 3.11, ensuring compatibility and stability. by @JustinGilmer in `#74 `_ +- Update exception handling to better support :code:`RpcErrors`, improving error management and debugging. by @JustinGilmer in `#72 `_ +- Introduce an option for specifying the schema of the returned raw data, allowing for more flexibility in data handling. by @andrewchambers in `#51 `_ +- Remove non-required dependencies and migrate to 'data' optional dependency for a lighter package and easier installation. by @JustinGilmer in `#71 `_ +- New method to get first and last timestamps from ``aligned_windows``, enhancing data analysis capabilities. by @Jefflinf in `#70 `_ +- Add ``to_timedelta`` method for ``pointwidth`` class, providing more options for time-based data manipulation. by @Jefflinf in `#69 `_ + +Fixed +^^^^^ +- Fix ``NoneType`` error for ``earliest/latest`` for empty streams, ensuring reliability and error handling. by @Jefflinf in `#64 `_ +- Correct integration tests where the time column is not automatically set as the index, improving test accuracy and reliability. by @JustinGilmer in `#56 `_ + +Deprecated +^^^^^^^^^^ +- FutureWarning for ``streams_in_collection`` to return ``StreamSet`` in the future, preparing users for upcoming API changes. by @Jefflinf in `#60 `_ + +**Full Changelog**: `GitHub compare view `_ + + +5.31.0 +------ +What's Changed +^^^^^^^^^^^^^^ +* Have release script update pyproject.toml file by @youngale-pingthings in https://github.com/PingThingsIO/btrdb-python/pull/48 +* Provide option to sort the arrow tables by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/47 +* Remove 4MB limit for gRPC message payloads by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/49 +* Update documentation for arrow methods by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/50 +* Update from staging by @justinGilmer in https://github.com/PingThingsIO/btrdb-python/pull/54 +* Sort tables by time by default for any ``pyarrow`` tables. by @justinGilmer in +* Fix deprecation warnings for pip installations. by @jleifnf in + +**Full Changelog**: `GitHub compare view `_ + 5.30.2 ------ What’s Changed From 896921c1b7deb392e88f2a848d8d2b5dd2294fef Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 16 Apr 2024 17:41:20 +0000 Subject: [PATCH 115/131] Bump idna from 3.4 to 3.7 (#87) --- requirements-all-locked.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-all-locked.txt b/requirements-all-locked.txt index 5cf5c57..f4c9ff8 100644 --- a/requirements-all-locked.txt +++ b/requirements-all-locked.txt @@ -44,7 +44,7 @@ grpcio==1.54.3 grpcio-tools==1.54.2 # via # btrdb (pyproject.toml) -idna==3.4 +idna==3.7 # via requests iniconfig==2.0.0 # via pytest From 519734dafac0485ab02e4a8e6e54ea25e47f02d7 Mon Sep 17 00:00:00 2001 From: Justin Gilmer <16173348+justinGilmer@users.noreply.github.com> Date: Fri, 19 Apr 2024 14:40:31 -0400 Subject: [PATCH 116/131] Remove locked requirements-all.txt file, is not necessary. (#88) --- requirements-all-locked.txt | 128 ------------------------------------ 1 file changed, 128 deletions(-) delete mode 100644 requirements-all-locked.txt diff --git a/requirements-all-locked.txt b/requirements-all-locked.txt deleted file mode 100644 index 5cf5c57..0000000 --- a/requirements-all-locked.txt +++ /dev/null @@ -1,128 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: -# -# pip-compile --extra=all --output-file=requirements-all.txt --resolver=backtracking pyproject.toml -# -aiosignal==1.3.1 - # via ray -attrs==23.1.0 - # via - # jsonschema - # ray -certifi==2023.7.22 - # via - # btrdb (pyproject.toml) - # requests -charset-normalizer==3.1.0 - # via requests -click==8.1.3 - # via ray -coverage[toml]==7.2.7 - # via pytest-cov -distlib==0.3.6 - # via virtualenv -exceptiongroup==1.1.1 - # via pytest -filelock==3.12.0 - # via - # ray - # virtualenv -flake8==6.0.0 - # via pytest-flake8 -freezegun==1.2.2 - # via btrdb -frozenlist==1.3.3 - # via - # aiosignal - # ray -grpcio==1.54.3 - # via - # btrdb (pyproject.toml) - # grpcio-tools - # ray -grpcio-tools==1.54.2 - # via - # btrdb (pyproject.toml) -idna==3.4 - # via requests -iniconfig==2.0.0 - # via pytest -jsonschema==4.17.3 - # via ray -mccabe==0.7.0 - # via flake8 -msgpack==1.0.5 - # via ray -numpy==1.24.3 - # via - # btrdb - # pandas - # pyarrow - # ray -packaging==23.1 - # via - # pytest - # ray -pandas==2.0.2 - # via btrdb -platformdirs==3.5.1 - # via virtualenv -pluggy==1.0.0 - # via pytest -polars==0.18.0 - # via btrdb -protobuf==4.23.2 - # via - # grpcio-tools - # ray -pyarrow==14.0.1 - # via btrdb -pycodestyle==2.10.0 - # via flake8 -pyflakes==3.0.1 - # via flake8 -pyrsistent==0.19.3 - # via jsonschema -pytest==7.3.1 - # via - # btrdb - # pytest-cov - # pytest-flake8 -pytest-cov==4.1.0 - # via btrdb -pytest-flake8==1.1.1 - # via btrdb -python-dateutil==2.8.2 - # via - # freezegun - # pandas -pytz==2023.3 - # via - # btrdb (pyproject.toml) - # pandas -pyyaml==6.0 - # via - # btrdb (pyproject.toml) - # ray -ray==2.2.0 - # via btrdb -requests==2.31.0 - # via ray -six==1.16.0 - # via python-dateutil -tabulate==0.9.0 - # via btrdb -tomli==2.0.1 - # via - # coverage - # pytest -tzdata==2023.3 - # via pandas -urllib3==2.0.7 - # via requests -virtualenv==20.23.0 - # via ray - -# The following packages are considered to be unsafe in a requirements file: -# setuptools From 4468f69280e053934899b7e35b87de0542359753 Mon Sep 17 00:00:00 2001 From: Justin Gilmer <16173348+justinGilmer@users.noreply.github.com> Date: Mon, 17 Jun 2024 18:08:05 -0400 Subject: [PATCH 117/131] Chore/update deps ensure tempsgs testing (#90) Numpy 2.0 has some breaking changes we are currently using, and the tests for a bogus api key was not generic enough to support tempsgs profiles. We also dropped the 3.7 support and made a todo to support 3.12 --- .github/workflows/release.yaml | 2 +- tests/btrdb_integration/test_conn.py | 6 +++- tests/btrdb_integration/test_streamset.py | 36 +++++++++++------------ 3 files changed, 24 insertions(+), 20 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 15dc856..5186bd8 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -13,7 +13,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: [3.7, 3.8, 3.9, "3.10", "3.11"] + python-version: [3.8, 3.9, "3.10", "3.11"] # TODO 3.12 os: [ubuntu-latest, macos-latest, windows-latest] steps: diff --git a/tests/btrdb_integration/test_conn.py b/tests/btrdb_integration/test_conn.py index bc6c9ac..4744f6b 100644 --- a/tests/btrdb_integration/test_conn.py +++ b/tests/btrdb_integration/test_conn.py @@ -6,6 +6,7 @@ import btrdb from btrdb.exceptions import BTrDBError +from btrdb.utils.credentials import credentials_by_profile @pytest.mark.skipif( @@ -20,9 +21,12 @@ def test_connection_info(self, conn): def test_incorrect_connect( self, ): + env_name = os.getenv("BTRDB_INTEGRATION_TEST_PROFILE") + creds = credentials_by_profile(env_name) + conn_str = creds["endpoints"] err_msg = r"""Could not connect to the database, error message: <_InactiveRpcError of RPC that terminated with:\n\tstatus = StatusCode.UNAUTHENTICATED\n\tdetails = "invalid api key"\n""" with pytest.raises(BTrDBError, match=err_msg): - conn = btrdb.connect(conn_str="127.0.0.1:4410", apikey="BOGUS_KEY") + conn = btrdb.connect(conn_str=conn_str, apikey="BOGUS_KEY") @pytest.mark.xfail( reason="Should return BTrDBError, but returns GRPCError instead, FIXME" diff --git a/tests/btrdb_integration/test_streamset.py b/tests/btrdb_integration/test_streamset.py index 5a0fbec..0029700 100644 --- a/tests/btrdb_integration/test_streamset.py +++ b/tests/btrdb_integration/test_streamset.py @@ -54,8 +54,8 @@ def test_streamset_arrow_values(conn, tmp_collection): s2.insert(list(zip(t2, d2))) ss = btrdb.stream.StreamSet([s1, s2]).filter(start=100, end=121) expected_times = [100, 101, 105, 106, 110, 114, 115, 119, 120] - expected_col1 = [0.0, np.NaN, 1.0, np.NaN, 2.0, np.NaN, 3.0, np.NaN, 4.0] - expected_col2 = [np.NaN, 5.0, np.NaN, 6.0, 7.0, 8.0, np.NaN, 9.0, np.NaN] + expected_col1 = [0.0, np.nan, 1.0, np.nan, 2.0, np.nan, 3.0, np.nan, 4.0] + expected_col2 = [np.nan, 5.0, np.nan, 6.0, 7.0, 8.0, np.nan, 9.0, np.nan] expected_schema = pa.schema( [ pa.field("time", pa.timestamp("ns", tz="UTC"), nullable=False), @@ -65,8 +65,8 @@ def test_streamset_arrow_values(conn, tmp_collection): ) values = ss.arrow_values() times = [t.value for t in values["time"]] - col1 = [np.NaN if isnan(v.as_py()) else v.as_py() for v in values[str(s1.uuid)]] - col2 = [np.NaN if isnan(v.as_py()) else v.as_py() for v in values[str(s2.uuid)]] + col1 = [np.nan if isnan(v.as_py()) else v.as_py() for v in values[str(s1.uuid)]] + col2 = [np.nan if isnan(v.as_py()) else v.as_py() for v in values[str(s2.uuid)]] assert times == expected_times assert col1 == expected_col1 assert col2 == expected_col2 @@ -91,12 +91,12 @@ def test_streamset_template_schema(conn, tmp_collection): ) ss = btrdb.stream.StreamSet([s1, s2]).filter(start=100, end=121, schema=schema) expected_times = [100, 101, 105, 106, 110, 114, 115, 119, 120] - expected_col1 = [0.0, np.NaN, 1.0, np.NaN, 2.0, np.NaN, 3.0, np.NaN, 4.0] - expected_col2 = [np.NaN, 5.0, np.NaN, 6.0, 7.0, 8.0, np.NaN, 9.0, np.NaN] + expected_col1 = [0.0, np.nan, 1.0, np.nan, 2.0, np.nan, 3.0, np.nan, 4.0] + expected_col2 = [np.nan, 5.0, np.nan, 6.0, 7.0, 8.0, np.nan, 9.0, np.nan] values = ss.arrow_values() times = [t.value for t in values["t"]] - col1 = [np.NaN if isnan(v.as_py()) else v.as_py() for v in values["a"]] - col2 = [np.NaN if isnan(v.as_py()) else v.as_py() for v in values["b"]] + col1 = [np.nan if isnan(v.as_py()) else v.as_py() for v in values["a"]] + col2 = [np.nan if isnan(v.as_py()) else v.as_py() for v in values["b"]] assert times == expected_times assert col1 == expected_col1 assert col2 == expected_col2 @@ -292,8 +292,8 @@ def test_streamset_to_dataframe(conn, tmp_collection): ss = btrdb.stream.StreamSet([s1, s2]).filter(start=100, end=121) values = ss.to_dataframe() expected_times = [100, 101, 105, 106, 110, 114, 115, 119, 120] - expected_col1 = [0.0, np.NaN, 1.0, np.NaN, 2.0, np.NaN, 3.0, np.NaN, 4.0] - expected_col2 = [np.NaN, 5.0, np.NaN, 6.0, 7.0, 8.0, np.NaN, 9.0, np.NaN] + expected_col1 = [0.0, np.nan, 1.0, np.nan, 2.0, np.nan, 3.0, np.nan, 4.0] + expected_col2 = [np.nan, 5.0, np.nan, 6.0, 7.0, 8.0, np.nan, 9.0, np.nan] expected_dat = { tmp_collection + "/s1": expected_col1, tmp_collection + "/s2": expected_col2, @@ -318,10 +318,10 @@ def test_arrow_streamset_to_dataframe(conn, tmp_collection): pa.scalar(v, type=pa.timestamp("ns", tz="UTC")).as_py() for v in expected_times ] expected_col1 = pa.array( - [0.0, np.NaN, 1.0, np.NaN, 2.0, np.NaN, 3.0, np.NaN, 4.0], mask=[False] * 9 + [0.0, np.nan, 1.0, np.nan, 2.0, np.nan, 3.0, np.nan, 4.0], mask=[False] * 9 ) expected_col2 = pa.array( - [np.NaN, 5.0, np.NaN, 6.0, 7.0, 8.0, np.NaN, 9.0, np.NaN], mask=[False] * 9 + [np.nan, 5.0, np.nan, 6.0, 7.0, 8.0, np.nan, 9.0, np.nan], mask=[False] * 9 ) expected_dat = { "time": expected_times, @@ -361,8 +361,8 @@ def test_arrow_streamset_to_polars(conn, tmp_collection): expected_times = [ pa.scalar(v, type=pa.timestamp("ns", tz="UTC")).as_py() for v in expected_times ] - expected_col1 = [0.0, np.NaN, 1.0, np.NaN, 2.0, np.NaN, 3.0, np.NaN, 4.0] - expected_col2 = [np.NaN, 5.0, np.NaN, 6.0, 7.0, 8.0, np.NaN, 9.0, np.NaN] + expected_col1 = [0.0, np.nan, 1.0, np.nan, 2.0, np.nan, 3.0, np.nan, 4.0] + expected_col2 = [np.nan, 5.0, np.nan, 6.0, 7.0, 8.0, np.nan, 9.0, np.nan] expected_dat = { tmp_collection + "/s1": expected_col1, tmp_collection + "/s2": expected_col2, @@ -395,8 +395,8 @@ def test_streamset_arrow_polars_vs_old_to_polars(conn, tmp_collection, name_call expected_times = [ pa.scalar(v, type=pa.timestamp("ns", tz="UTC")).as_py() for v in expected_times ] - expected_col1 = [0.0, np.NaN, 1.0, np.NaN, 2.0, np.NaN, 3.0, np.NaN, 4.0] - expected_col2 = [np.NaN, 5.0, np.NaN, 6.0, 7.0, 8.0, np.NaN, 9.0, np.NaN] + expected_col1 = [0.0, np.nan, 1.0, np.nan, 2.0, np.nan, 3.0, np.nan, 4.0] + expected_col2 = [np.nan, 5.0, np.nan, 6.0, 7.0, 8.0, np.nan, 9.0, np.nan] expected_dat = { tmp_collection + "/s1": expected_col1, tmp_collection + "/s2": expected_col2, @@ -509,8 +509,8 @@ def test_timesnap_backward_extends_range(conn, tmp_collection): values = ss.arrow_values() assert [1 * sec, 2 * sec] == [t.value for t in values["time"]] assert [0.5, 2.0] == [v.as_py() for v in values[str(s1.uuid)]] - assert [np.NaN, 2.0] == [ - np.NaN if isnan(v.as_py()) else v.as_py() for v in values[str(s2.uuid)] + assert [np.nan, 2.0] == [ + np.nan if isnan(v.as_py()) else v.as_py() for v in values[str(s2.uuid)] ] assert [1.0, 2.0] == [v.as_py() for v in values[str(s3.uuid)]] From 5866d93bff97fd376fc97bda94c979765de4186c Mon Sep 17 00:00:00 2001 From: Alexander Young Date: Mon, 17 Jun 2024 17:11:50 -0500 Subject: [PATCH 118/131] Release v5.33.0 --- btrdb/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/btrdb/version.py b/btrdb/version.py index 7bf90da..38d4c2f 100644 --- a/btrdb/version.py +++ b/btrdb/version.py @@ -15,7 +15,7 @@ ## Module Info ########################################################################## -__version_info__ = {"major": 5, "minor": 32, "micro": 0, "releaselevel": "final"} +__version_info__ = {"major": 5, "minor": 33, "micro": 0, "releaselevel": "final"} ########################################################################## ## Helper Functions diff --git a/pyproject.toml b/pyproject.toml index a57516f..b3aee86 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "btrdb" -version = "5.32.0" +version = "5.33.0" authors = [ {name="PingThingsIO", email="support@pingthings.io"}, ] From 3722e5f875a743f3d2b3554f0a347a514c956c9e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 Mar 2025 18:15:05 +0000 Subject: [PATCH 119/131] Bump tj-actions/changed-files from 43 to 46 in /.github/workflows (#92) --- .github/workflows/pre-commit.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml index 98e35c2..2717add 100644 --- a/.github/workflows/pre-commit.yaml +++ b/.github/workflows/pre-commit.yaml @@ -30,7 +30,7 @@ jobs: pip install pre-commit - name: Get changed files id: changed-files - uses: tj-actions/changed-files@v43 + uses: tj-actions/changed-files@v46 with: token: ${{ secrets.GITHUB_TOKEN }} - name: Run pre-commit From 980f74e266eb1b1d94aa35439192b8f0ec50ebf8 Mon Sep 17 00:00:00 2001 From: Alexander Young Date: Wed, 20 Aug 2025 17:37:24 -0500 Subject: [PATCH 120/131] Updated old api for new metadata changes (#93) Co-authored-by: Jonathon Ferrell Co-authored-by: Justin Gilmer <16173348+justinGilmer@users.noreply.github.com> --- .github/workflows/pre-commit.yaml | 2 +- btrdb/conn.py | 12 ++++++++++++ pyproject.toml | 2 +- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml index 2717add..cf5dc26 100644 --- a/.github/workflows/pre-commit.yaml +++ b/.github/workflows/pre-commit.yaml @@ -34,6 +34,6 @@ jobs: with: token: ${{ secrets.GITHUB_TOKEN }} - name: Run pre-commit - uses: pre-commit/action@v2.0.3 + uses: pre-commit/action@v3.0.1 with: extra_args: --files ${{ steps.changed-files.outputs.all_changed_files }} diff --git a/btrdb/conn.py b/btrdb/conn.py index 0e9b5fc..f6f75d1 100644 --- a/btrdb/conn.py +++ b/btrdb/conn.py @@ -15,6 +15,7 @@ ## Imports ########################################################################## +import importlib.metadata import json import logging import os @@ -65,6 +66,11 @@ def __init__(self, addrportstr, apikey=None): The ``btrdb.connect`` method is a helper function to make connecting to the platform easier usually that will be sufficient for most users. """ + warn( + "This API is deprecated in favor of the pingthings_api, refer to your hub landing page for further documentation.", + DeprecationWarning, + stacklevel=2, + ) addrport = addrportstr.split(":", 2) # 100MB size limit ~ 2500 streams for 5000 points with each point being 64bit # 500MB size limit ~ 13K streams for 5000 points @@ -126,6 +132,12 @@ def __init__(self, apikey, client_call_details): if client_call_details.metadata is not None: metadata = list(client_call_details.metadata) metadata.append(("authorization", "Bearer " + apikey)) + metadata.append( + ( + "x-api-client", + "btrdbpy-" + importlib.metadata.version("btrdb"), + ) + ) self.method = client_call_details.method self.timeout = client_call_details.timeout self.credentials = client_call_details.credentials diff --git a/pyproject.toml b/pyproject.toml index b3aee86..d2efe1d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ maintainers = [ description = "Bindings to interact with the Berkeley Tree Database using gRPC." readme = "README.md" license = {file="LICENSE.txt"} -requires-python = ">=3.7,<=3.10" +requires-python = ">=3.7,<=3.12" classifiers = [ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', From 75c992aa04b65fd40877dadd68b36a054ba7fb80 Mon Sep 17 00:00:00 2001 From: SiverJohn Date: Wed, 20 Aug 2025 19:24:25 -0500 Subject: [PATCH 121/131] Release v5.34.0 Co-authored-by: Justin Gilmer <16173348+justinGilmer@users.noreply.github.com> --- btrdb/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/btrdb/version.py b/btrdb/version.py index 38d4c2f..45d10db 100644 --- a/btrdb/version.py +++ b/btrdb/version.py @@ -15,7 +15,7 @@ ## Module Info ########################################################################## -__version_info__ = {"major": 5, "minor": 33, "micro": 0, "releaselevel": "final"} +__version_info__ = {"major": 5, "minor": 34, "micro": 0, "releaselevel": "final"} ########################################################################## ## Helper Functions diff --git a/pyproject.toml b/pyproject.toml index d2efe1d..8ae72c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "btrdb" -version = "5.33.0" +version = "5.34.0" authors = [ {name="PingThingsIO", email="support@pingthings.io"}, ] From cb1abc9f82105c5b43954200fa93ee97f5db82c6 Mon Sep 17 00:00:00 2001 From: SiverJohn Date: Thu, 21 Aug 2025 12:19:13 -0500 Subject: [PATCH 122/131] Minor Bump To Fix Logging (#95) --- btrdb/conn.py | 7 ++++++- btrdb/version.py | 4 +++- pyproject.toml | 4 +++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/btrdb/conn.py b/btrdb/conn.py index f6f75d1..72f7061 100644 --- a/btrdb/conn.py +++ b/btrdb/conn.py @@ -132,10 +132,15 @@ def __init__(self, apikey, client_call_details): if client_call_details.metadata is not None: metadata = list(client_call_details.metadata) metadata.append(("authorization", "Bearer " + apikey)) + version = "unknown" + try: + version = importlib.metadata.version("btrdb") + except: + pass metadata.append( ( "x-api-client", - "btrdbpy-" + importlib.metadata.version("btrdb"), + "btrdbpy-" + version, ) ) self.method = client_call_details.method diff --git a/btrdb/version.py b/btrdb/version.py index 45d10db..635d037 100644 --- a/btrdb/version.py +++ b/btrdb/version.py @@ -15,7 +15,9 @@ ## Module Info ########################################################################## -__version_info__ = {"major": 5, "minor": 34, "micro": 0, "releaselevel": "final"} + +__version_info__ = {"major": 5, "minor": 34, "micro": 1, "releaselevel": "final"} + ########################################################################## ## Helper Functions diff --git a/pyproject.toml b/pyproject.toml index 8ae72c1..f479eb3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "btrdb" -version = "5.34.0" +version = "5.34.1" authors = [ {name="PingThingsIO", email="support@pingthings.io"}, ] @@ -24,6 +24,8 @@ classifiers = [ 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', 'Topic :: Database', 'Topic :: Software Development :: Libraries :: Python Modules', ] From e3b4c7317d84b0348d47e6c27eb3a9d5d6a58ca9 Mon Sep 17 00:00:00 2001 From: Jonathon Ferrell Date: Wed, 27 Aug 2025 11:56:34 -0500 Subject: [PATCH 123/131] Added a FutureWarning to show deprecation which can't be ignored --- btrdb/conn.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/btrdb/conn.py b/btrdb/conn.py index 72f7061..ca8b9b6 100644 --- a/btrdb/conn.py +++ b/btrdb/conn.py @@ -24,6 +24,7 @@ from concurrent.futures import ThreadPoolExecutor from typing import List, Tuple, Union from warnings import warn +import warnings import certifi import grpc @@ -66,10 +67,11 @@ def __init__(self, addrportstr, apikey=None): The ``btrdb.connect`` method is a helper function to make connecting to the platform easier usually that will be sufficient for most users. """ + #4 Is a magic number to make sure the error propagates where btrdb.connect is called. warn( "This API is deprecated in favor of the pingthings_api, refer to your hub landing page for further documentation.", - DeprecationWarning, - stacklevel=2, + FutureWarning, + stacklevel=4 ) addrport = addrportstr.split(":", 2) # 100MB size limit ~ 2500 streams for 5000 points with each point being 64bit From 4cf9811ce74bf49aa0f1aec41778e54a5d8deecd Mon Sep 17 00:00:00 2001 From: Jonathon Ferrell Date: Wed, 27 Aug 2025 12:09:51 -0500 Subject: [PATCH 124/131] Cleaned up unused library and bare except --- btrdb/conn.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/btrdb/conn.py b/btrdb/conn.py index ca8b9b6..4d26400 100644 --- a/btrdb/conn.py +++ b/btrdb/conn.py @@ -24,7 +24,6 @@ from concurrent.futures import ThreadPoolExecutor from typing import List, Tuple, Union from warnings import warn -import warnings import certifi import grpc @@ -67,11 +66,11 @@ def __init__(self, addrportstr, apikey=None): The ``btrdb.connect`` method is a helper function to make connecting to the platform easier usually that will be sufficient for most users. """ - #4 Is a magic number to make sure the error propagates where btrdb.connect is called. + # 4 Is a magic number to make sure the error propagates where btrdb.connect is called. warn( "This API is deprecated in favor of the pingthings_api, refer to your hub landing page for further documentation.", FutureWarning, - stacklevel=4 + stacklevel=4, ) addrport = addrportstr.split(":", 2) # 100MB size limit ~ 2500 streams for 5000 points with each point being 64bit @@ -137,7 +136,7 @@ def __init__(self, apikey, client_call_details): version = "unknown" try: version = importlib.metadata.version("btrdb") - except: + except Exception: pass metadata.append( ( From 154c981dfd3d5eb718c802d051840af852763370 Mon Sep 17 00:00:00 2001 From: Jonathon Ferrell Date: Fri, 12 Sep 2025 17:32:32 -0500 Subject: [PATCH 125/131] Fixes redundant tag sending and updates the test to be correct with the new behavior --- btrdb/stream.py | 2 +- tests/btrdb/test_stream.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/btrdb/stream.py b/btrdb/stream.py index e9d4ac5..e1cd2d7 100644 --- a/btrdb/stream.py +++ b/btrdb/stream.py @@ -823,7 +823,7 @@ def _update_tags_collection(self, tags, collection): """ :meta private: """ - tags = self.tags() if tags is None else tags + tags = {} if tags is None else tags collection = self.collection if collection is None else collection if collection is None: raise BTRDBValueError( diff --git a/tests/btrdb/test_stream.py b/tests/btrdb/test_stream.py index 4fede74..66198ef 100644 --- a/tests/btrdb/test_stream.py +++ b/tests/btrdb/test_stream.py @@ -278,7 +278,7 @@ def test_update_collection(self): stream.update(collection=collection) stream._btrdb.ep.setStreamTags.assert_called_once_with( - uu=uu, expected=42, tags=stream.tags(), collection=collection + uu=uu, expected=None, tags={}, collection=collection ) stream._btrdb.ep.setStreamAnnotations.assert_not_called() From 0a1fc678d6c4fee14b43c5524ef570945f49bfbe Mon Sep 17 00:00:00 2001 From: Justin Gilmer <16173348+justinGilmer@users.noreply.github.com> Date: Mon, 15 Sep 2025 17:47:57 -0400 Subject: [PATCH 126/131] Update the mocked test to properly mock the expected property version. --- tests/btrdb/test_stream.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/btrdb/test_stream.py b/tests/btrdb/test_stream.py index 66198ef..5887afa 100644 --- a/tests/btrdb/test_stream.py +++ b/tests/btrdb/test_stream.py @@ -272,15 +272,14 @@ def test_update_collection(self): endpoint = Mock(Endpoint) endpoint.streamInfo = Mock(return_value=("koala", 42, {}, {}, None)) endpoint.info = Mock(return_value={"majorVersion": 5, "minorVersion": 30}) - stream = Stream(btrdb=BTrDB(endpoint), uuid=uu) + stream = Stream(btrdb=BTrDB(endpoint), uuid=uu, property_version=42) collection = "giraffe" stream.update(collection=collection) stream._btrdb.ep.setStreamTags.assert_called_once_with( - uu=uu, expected=None, tags={}, collection=collection + uu=uu, expected=42, tags={}, collection=collection ) - stream._btrdb.ep.setStreamAnnotations.assert_not_called() def test_update_annotations(self): """ From 98d20189129a8df9bc747cd46554f087cec80e63 Mon Sep 17 00:00:00 2001 From: Jonathon Ferrell Date: Tue, 16 Sep 2025 12:29:53 -0500 Subject: [PATCH 127/131] Release v5.32.2 --- btrdb/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/btrdb/version.py b/btrdb/version.py index 635d037..0129c25 100644 --- a/btrdb/version.py +++ b/btrdb/version.py @@ -16,7 +16,7 @@ ########################################################################## -__version_info__ = {"major": 5, "minor": 34, "micro": 1, "releaselevel": "final"} +__version_info__ = {"major": 5, "minor": 32, "micro": 2, "releaselevel": "final"} ########################################################################## diff --git a/pyproject.toml b/pyproject.toml index f479eb3..22801e7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "btrdb" -version = "5.34.1" +version = "5.32.2" authors = [ {name="PingThingsIO", email="support@pingthings.io"}, ] From 7ec12d9345ad226be2a2229dc3fa1ab7c066ea36 Mon Sep 17 00:00:00 2001 From: Jonathon Ferrell Date: Tue, 16 Sep 2025 12:31:48 -0500 Subject: [PATCH 128/131] Release v5.34.2 --- btrdb/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/btrdb/version.py b/btrdb/version.py index 0129c25..6727e46 100644 --- a/btrdb/version.py +++ b/btrdb/version.py @@ -16,7 +16,7 @@ ########################################################################## -__version_info__ = {"major": 5, "minor": 32, "micro": 2, "releaselevel": "final"} +__version_info__ = {"major": 5, "minor": 34, "micro": 2, "releaselevel": "final"} ########################################################################## diff --git a/pyproject.toml b/pyproject.toml index 22801e7..238dae6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "btrdb" -version = "5.32.2" +version = "5.34.2" authors = [ {name="PingThingsIO", email="support@pingthings.io"}, ] From a09c0730da1a3a1df5afc2a0440014e6a4e16985 Mon Sep 17 00:00:00 2001 From: recontech404 Date: Tue, 30 Dec 2025 15:05:02 -0600 Subject: [PATCH 129/131] remove broken test for deprecated endpoint. --- tests/btrdb_integration/test_conn.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/tests/btrdb_integration/test_conn.py b/tests/btrdb_integration/test_conn.py index 4744f6b..dc8fd13 100644 --- a/tests/btrdb_integration/test_conn.py +++ b/tests/btrdb_integration/test_conn.py @@ -44,17 +44,6 @@ def test_create_stream(self, conn, tmp_collection): assert stream.uuid == uuid assert stream.name == "s" - def test_query(self, conn, tmp_collection): - conn.create(new_uuid(), tmp_collection, tags={"name": "s1"}) - conn.create(new_uuid(), tmp_collection, tags={"name": "s2"}) - uuids = conn.query( - "select name from streams where collection = $1 order by name;", - [tmp_collection], - ) - assert len(uuids) == 2 - assert uuids[0]["name"] == "s1" - assert uuids[1]["name"] == "s2" - def test_list_collections(self, conn, tmp_collection): assert tmp_collection not in conn.list_collections() stream = conn.create(new_uuid(), tmp_collection, tags={"name": "s"}) From a638de10fa4127a8aefec85cac14501ca62d0226 Mon Sep 17 00:00:00 2001 From: SiverJohn Date: Mon, 15 Jun 2026 13:18:21 -0500 Subject: [PATCH 130/131] Pinned pytest to version 8 to keep tests running for the time being (#101) --- tests/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/requirements.txt b/tests/requirements.txt index 4a71eb1..a58a885 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -34,7 +34,7 @@ protobuf # ray pyarrow # via btrdb -pytest +pytest <9 # via # btrdb # pytest-cov From 682b4974e81d52e3a69596f3828d6c2f16ea6503 Mon Sep 17 00:00:00 2001 From: Casey Primozic Date: Tue, 23 Jun 2026 10:12:12 -0500 Subject: [PATCH 131/131] Tweak error messages in E2E test to match updats in SGS (#100) --- tests/btrdb_integration/test_conn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/btrdb_integration/test_conn.py b/tests/btrdb_integration/test_conn.py index dc8fd13..dc6f47a 100644 --- a/tests/btrdb_integration/test_conn.py +++ b/tests/btrdb_integration/test_conn.py @@ -24,7 +24,7 @@ def test_incorrect_connect( env_name = os.getenv("BTRDB_INTEGRATION_TEST_PROFILE") creds = credentials_by_profile(env_name) conn_str = creds["endpoints"] - err_msg = r"""Could not connect to the database, error message: <_InactiveRpcError of RPC that terminated with:\n\tstatus = StatusCode.UNAUTHENTICATED\n\tdetails = "invalid api key"\n""" + err_msg = r"""Could not connect to the database, error message: <_InactiveRpcError of RPC that terminated with:\n\tstatus = StatusCode.UNAUTHENTICATED\n\tdetails = "could not validate api key:""" with pytest.raises(BTrDBError, match=err_msg): conn = btrdb.connect(conn_str=conn_str, apikey="BOGUS_KEY")