From b27127fd09629568d51570b721e515331a65f6e6 Mon Sep 17 00:00:00 2001 From: lingyongxu Date: Wed, 7 Jun 2017 17:42:16 +0800 Subject: [PATCH 001/238] Drop py34 target in tox.ini We support py35 now.so it is no need to keep the supoort for py34. Change-Id: Ie76e897bea3c184410e2b151fbe978d93bc21624 --- setup.cfg | 1 - tox.ini | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 4af3151e..2b15aef0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -17,7 +17,6 @@ classifier = Programming Language :: Python :: 2 Programming Language :: Python :: 2.7 Programming Language :: Python :: 3 - Programming Language :: Python :: 3.4 Programming Language :: Python :: 3.5 [global] diff --git a/tox.ini b/tox.ini index df01bf8a..c1c69ae9 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,py34,py35,pypy,pep8 +envlist = py27,py35,pypy,pep8 minversion = 2.0 skipsdist = True From 01f5a9f3af3a1630297a92dff0f998b4f0e97a49 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Tue, 13 Jun 2017 10:46:10 -0700 Subject: [PATCH 002/238] Support pdb in tests better Not really "better" so much as "at all" - the thing we do with the capture stderr *everywhere* is probably brilliant - but absolutely not strictly necessary for every MockHttpTest TestCase and comes with the annoying overhead of trying to get into a debugger causes tests to hang inexplicably and you can't even do debug prints in tests!? Now if you add SWIFTCLIENT_DEBUG=1 to your nose -vsx command you can not only jump into debugger, but if you're "in the know" you could even get some stderr print debugging going on! If you're not "in the know" when you try to pdb.set_trace() the tests will blow-up for you because we monkeypatch pdb when not in SWIFTCLIENT_DEBUG mode, you're welcome. Change-Id: I21298bfd39fe386b5ea19e3a6f4408d8a0459c92 --- tests/unit/utils.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/unit/utils.py b/tests/unit/utils.py index c05146ec..f7e48d30 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -20,6 +20,7 @@ import unittest import mock import six +import os from six.moves import reload_module from six.moves.urllib.parse import urlparse, ParseResult from swiftclient import client as c @@ -206,7 +207,19 @@ def setUp(self): # won't cover the references to sys.stdout/sys.stderr in # swiftclient.multithreading self.capture_output = CaptureOutput() - self.capture_output.__enter__() + if 'SWIFTCLIENT_DEBUG' not in os.environ: + self.capture_output.__enter__() + self.addCleanup(self.capture_output.__exit__) + + # since we're going to steal all stderr output globally; we should + # give the developer an escape hatch or risk scorn + def blowup_but_with_the_helpful(*args, **kwargs): + raise Exception( + "You tried to enter a debugger while stderr is " + "patched, you need to set SWIFTCLIENT_DEBUG=1 " + "and try again") + import pdb + pdb.set_trace = blowup_but_with_the_helpful def fake_http_connection(*args, **kwargs): self.validateMockedRequestsConsumed() @@ -384,7 +397,6 @@ def tearDown(self): # un-hygienic mocking on the swiftclient.client module; which may lead # to some unfortunate test order dependency bugs by way of the broken # window theory if any other modules are similarly patched - self.capture_output.__exit__() reload_module(c) From 484d7ee9b21396d066604e8e876ffb3d6ed6d359 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Thu, 6 Jul 2017 12:43:11 -0700 Subject: [PATCH 003/238] Allow --meta on upload Previously, the --meta option was only allowed on post or copy subcommands. Change-Id: I87bf0338c34b5e89aa946505bee68dbeb37d784c Closes-Bug: #1616238 --- swiftclient/service.py | 2 ++ swiftclient/shell.py | 11 +++++++++-- tests/unit/test_service.py | 35 +++++++++-------------------------- tests/unit/test_shell.py | 6 ++++-- 4 files changed, 24 insertions(+), 30 deletions(-) diff --git a/swiftclient/service.py b/swiftclient/service.py index b9b843eb..8301ae93 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -1912,6 +1912,8 @@ def _upload_object_job(self, conn, container, source, obj, options, return res # Merge the command line header options to the put_headers + put_headers.update(split_headers( + options['meta'], 'X-Object-Meta-')) put_headers.update(split_headers(options['header'], '')) # Don't do segment job if object is not big enough, and never do diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 841ed6e5..58b9f54d 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -881,8 +881,8 @@ def st_copy(parser, args, output_manager): st_upload_options = '''[--changed] [--skip-identical] [--segment-size ] [--segment-container ] [--leave-segments] [--object-threads ] [--segment-threads ] - [--header
] [--use-slo] [--ignore-checksum] - [--object-name ] + [--meta ] [--header
] [--use-slo] + [--ignore-checksum] [--object-name ] [] [...] ''' @@ -916,6 +916,9 @@ def st_copy(parser, args, output_manager): --segment-threads Number of threads to use for uploading object segments. Default is 10. + -m, --meta + Sets a meta data item. This option may be repeated. + Example: -m Color:Blue -m Size:Large -H, --header Adds a customized request header. This option may be repeated. Example: -H "content-type:text/plain" @@ -966,6 +969,10 @@ def st_upload(parser, args, output_manager): '--segment-threads', type=int, default=10, help='Number of threads to use for uploading object segments. ' 'Its value must be a positive integer. Default is 10.') + parser.add_argument( + '-m', '--meta', action='append', dest='meta', default=[], + help='Sets a meta data item. This option may be repeated. ' + 'Example: -m Color:Blue -m Size:Large') parser.add_argument( '-H', '--header', action='append', dest='header', default=[], help='Set request headers with the syntax header:value. ' diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index b759e6ba..2a477fed 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -1146,14 +1146,9 @@ def test_upload_object_job_file_with_unicode_path(self): container='test_c', source=f.name, obj='テスト/dummy.dat', - options={'changed': False, - 'skip_identical': False, - 'leave_segments': True, - 'header': '', - 'segment_size': 10, - 'segment_container': None, - 'use_slo': False, - 'checksum': True}) + options=dict(s._options, + segment_size=10, + leave_segments=True)) mtime = r['headers']['x-object-meta-mtime'] self.assertEqual(expected_mtime, mtime) @@ -1350,12 +1345,8 @@ def _consuming_conn(*a, **kw): container='test_c', source=f.name, obj='test_o', - options={'changed': False, - 'skip_identical': False, - 'leave_segments': True, - 'header': '', - 'segment_size': 0, - 'checksum': True}) + options=dict(s._options, + leave_segments=True)) mtime = r['headers']['x-object-meta-mtime'] self.assertEqual(expected_mtime, mtime) @@ -1405,12 +1396,8 @@ def test_upload_object_job_stream(self, time_mock): container='test_c', source=f, obj='test_o', - options={'changed': False, - 'skip_identical': False, - 'leave_segments': True, - 'header': '', - 'segment_size': 0, - 'checksum': True}) + options=dict(s._options, + leave_segments=True)) mtime = float(r['headers']['x-object-meta-mtime']) self.assertEqual(mtime, expected_mtime) @@ -1452,12 +1439,8 @@ def _consuming_conn(*a, **kw): container='test_c', source=f.name, obj='test_o', - options={'changed': False, - 'skip_identical': False, - 'leave_segments': True, - 'header': '', - 'segment_size': 0, - 'checksum': True}) + options=dict(s._options, + leave_segments=True)) self.assertIs(r['success'], False) self.assertIn('md5 mismatch', str(r.get('error'))) diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index db96df75..3f87c6d9 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -623,7 +623,8 @@ def test_upload(self, connection, walk): connection.return_value.put_object.return_value = EMPTY_ETAG connection.return_value.attempts = 0 argv = ["", "upload", "container", self.tmpfile, - "-H", "X-Storage-Policy:one"] + "-H", "X-Storage-Policy:one", + "--meta", "Color:Blue"] swiftclient.shell.main(argv) connection.return_value.put_container.assert_called_once_with( 'container', @@ -636,7 +637,8 @@ def test_upload(self, connection, walk): mock.ANY, content_length=0, headers={'x-object-meta-mtime': mock.ANY, - 'X-Storage-Policy': 'one'}, + 'X-Storage-Policy': 'one', + 'X-Object-Meta-Color': 'Blue'}, response_dict={}) # upload to pseudo-folder (via param) From ae5fd46e8772e50a0ab2ac89216188b1cdc83d8e Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Fri, 25 Aug 2017 12:13:12 -0700 Subject: [PATCH 004/238] Stop mutating header dicts Change-Id: Ia1638c216eff9db6fbe416bc0570c27cfdcfe730 --- swiftclient/client.py | 65 ++++++++++++++++++---------------- tests/unit/test_swiftclient.py | 15 ++++++-- 2 files changed, 47 insertions(+), 33 deletions(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index 80bc4a3a..95e89b8f 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -302,7 +302,7 @@ def __init__(self, resp, connection, container, obj, self.obj = obj self.query_string = query_string self.response_dict = response_dict - self.headers = headers if headers is not None else {} + self.headers = dict(headers) if headers is not None else {} self.bytes_read = 0 def read(self, length=None): @@ -834,13 +834,15 @@ def post_account(url, token, headers, http_conn=None, response_dict=None, path = parsed.path if query_string: path += '?' + query_string - headers['X-Auth-Token'] = token + req_headers = {'X-Auth-Token': token} if service_token: - headers['X-Service-Token'] = service_token - conn.request(method, path, data, headers) + req_headers['X-Service-Token'] = service_token + if headers: + req_headers.update(headers) + conn.request(method, path, data, req_headers) resp = conn.getresponse() body = resp.read() - http_log((url, method,), {'headers': headers}, resp, body) + http_log((url, method,), {'headers': req_headers}, resp, body) store_response(resp, response_dict) @@ -882,12 +884,6 @@ def get_container(url, token, container, marker=None, limit=None, """ if not http_conn: http_conn = http_connection(url) - if headers: - headers = dict(headers) - else: - headers = {} - headers['X-Auth-Token'] = token - headers['Accept-Encoding'] = 'gzip' if full_listing: rv = get_container(url, token, container, marker, limit, prefix, delimiter, end_marker, path, http_conn, @@ -922,17 +918,20 @@ def get_container(url, token, container, marker=None, limit=None, qs += '&path=%s' % quote(path) if query_string: qs += '&%s' % query_string.lstrip('?') + req_headers = {'X-Auth-Token': token, 'Accept-Encoding': 'gzip'} if service_token: - headers['X-Service-Token'] = service_token + req_headers['X-Service-Token'] = service_token + if headers: + req_headers.update(headers) method = 'GET' - conn.request(method, '%s?%s' % (cont_path, qs), '', headers) + conn.request(method, '%s?%s' % (cont_path, qs), '', req_headers) resp = conn.getresponse() body = resp.read() http_log(('%(url)s%(cont_path)s?%(qs)s' % {'url': url.replace(parsed.path, ''), 'cont_path': cont_path, 'qs': qs}, method,), - {'headers': headers}, resp, body) + {'headers': req_headers}, resp, body) if resp.status < 200 or resp.status >= 300: raise ClientException.from_response(resp, 'Container GET failed', body) @@ -1005,23 +1004,23 @@ def put_container(url, token, container, headers=None, http_conn=None, parsed, conn = http_connection(url) path = '%s/%s' % (parsed.path, quote(container)) method = 'PUT' - if not headers: - headers = {} - headers['X-Auth-Token'] = token + req_headers = {'X-Auth-Token': token} if service_token: - headers['X-Service-Token'] = service_token - if 'content-length' not in (k.lower() for k in headers): - headers['Content-Length'] = '0' + req_headers['X-Service-Token'] = service_token + if headers: + req_headers.update(headers) + if 'content-length' not in (k.lower() for k in req_headers): + req_headers['Content-Length'] = '0' if query_string: path += '?' + query_string.lstrip('?') - conn.request(method, path, '', headers) + conn.request(method, path, '', req_headers) resp = conn.getresponse() body = resp.read() store_response(resp, response_dict) http_log(('%s%s' % (url.replace(parsed.path, ''), path), method,), - {'headers': headers}, resp, body) + {'headers': req_headers}, resp, body) if resp.status < 200 or resp.status >= 300: raise ClientException.from_response(resp, 'Container PUT failed', body) @@ -1048,16 +1047,18 @@ def post_container(url, token, container, headers, http_conn=None, parsed, conn = http_connection(url) path = '%s/%s' % (parsed.path, quote(container)) method = 'POST' - headers['X-Auth-Token'] = token + req_headers = {'X-Auth-Token': token} if service_token: - headers['X-Service-Token'] = service_token + req_headers['X-Service-Token'] = service_token + if headers: + req_headers.update(headers) if 'content-length' not in (k.lower() for k in headers): - headers['Content-Length'] = '0' - conn.request(method, path, '', headers) + req_headers['Content-Length'] = '0' + conn.request(method, path, '', req_headers) resp = conn.getresponse() body = resp.read() http_log(('%s%s' % (url.replace(parsed.path, ''), path), method,), - {'headers': headers}, resp, body) + {'headers': req_headers}, resp, body) store_response(resp, response_dict) @@ -1352,14 +1353,16 @@ def post_object(url, token, container, name, headers, http_conn=None, else: parsed, conn = http_connection(url) path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) - headers['X-Auth-Token'] = token + req_headers = {'X-Auth-Token': token} if service_token: - headers['X-Service-Token'] = service_token - conn.request('POST', path, '', headers) + req_headers['X-Service-Token'] = service_token + if headers: + req_headers.update(headers) + conn.request('POST', path, '', req_headers) resp = conn.getresponse() body = resp.read() http_log(('%s%s' % (url.replace(parsed.path, ''), path), 'POST',), - {'headers': headers}, resp, body) + {'headers': req_headers}, resp, body) store_response(resp, response_dict) diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index 5384ac7f..852c2d39 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -680,9 +680,10 @@ def test_ok(self): c.http_connection = self.fake_http_connection(200, headers={ 'X-Account-Meta-Color': 'blue', }, body='foo') + headers = {'x-account-meta-shape': 'square'} resp_headers, body = c.post_account( 'http://www.tests.com/path/to/account', 'asdf', - {'x-account-meta-shape': 'square'}, query_string='bar=baz', + headers, query_string='bar=baz', data='some data') self.assertEqual('blue', resp_headers.get('x-account-meta-color')) self.assertEqual('foo', body) @@ -691,6 +692,8 @@ def test_ok(self): 'some data', {'x-auth-token': 'asdf', 'x-account-meta-shape': 'square'}) ]) + # Check that we didn't mutate the request ehader dict + self.assertEqual(headers, {'x-account-meta-shape': 'square'}) def test_server_error(self): body = 'c' * 65 @@ -1434,6 +1437,11 @@ def test_ok(self): 'X-Object-Meta-Test': 'mymeta', 'X-Delete-At': delete_at}), ]) + # Check that the request header dict didn't get mutated + self.assertEqual(args[-1], { + 'X-Object-Meta-Test': 'mymeta', + 'X-Delete-At': delete_at, + }) def test_unicode_ok(self): conn = c.http_connection(u'http://www.test.com/') @@ -2996,10 +3004,11 @@ def test_service_token_head_container(self): self.assertEqual(conn.attempts, 1) def test_service_token_post_container(self): + headers = {'X-Container-Meta-Color': 'blue'} with mock.patch('swiftclient.client.http_connection', self.fake_http_connection(201)): conn = self.get_connection() - conn.post_container('container1', {}) + conn.post_container('container1', headers) self.assertEqual(1, len(self.request_log), self.request_log) for actual in self.iter_request_log(): self.assertEqual('POST', actual['method']) @@ -3009,6 +3018,8 @@ def test_service_token_post_container(self): self.assertEqual('http://storage_url.com/container1', actual['full_path']) self.assertEqual(conn.attempts, 1) + # Check that we didn't mutate the request header dict + self.assertEqual(headers, {'X-Container-Meta-Color': 'blue'}) def test_service_token_put_container(self): with mock.patch('swiftclient.client.http_connection', From f48f421a86a9086c50211d4c50d639c5b4a46c7c Mon Sep 17 00:00:00 2001 From: Andreas Jaeger Date: Thu, 16 Nov 2017 20:49:19 +0100 Subject: [PATCH 005/238] Remove setting of version/release from releasenotes Release notes are version independent, so remove version/release values. We've found that projects now require the service package to be installed in order to build release notes, and this is entirely due to the current convention of pulling in the version information. Release notes should not need installation in order to build, so this unnecessary version setting needs to be removed. This is needed for new release notes publishing, see I56909152975f731a9d2c21b2825b972195e48ee8 and the discussion starting at http://lists.openstack.org/pipermail/openstack-dev/2017-November/124480.html . Change-Id: I623fe918c1e4ddafa93efc91ed550a365cec1cf0 --- releasenotes/source/conf.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/releasenotes/source/conf.py b/releasenotes/source/conf.py index 214bcef6..b27aa963 100644 --- a/releasenotes/source/conf.py +++ b/releasenotes/source/conf.py @@ -34,8 +34,6 @@ import datetime -from swiftclient import __version__ - # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. @@ -70,14 +68,11 @@ project = u'Swift Client Release Notes' copyright = u'%d, OpenStack Foundation' % datetime.datetime.now().year -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# +# Release notes are version independent. # The short X.Y version. -version = __version__.rsplit('.', 1)[0] +version = '' # The full version, including alpha/beta/rc tags. -release = __version__ +release = '' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. From ae2dfaec367dabadb4c74a5d635d9633247465f1 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 22 Nov 2017 07:20:00 -0600 Subject: [PATCH 006/238] Update tox_install.sh to align for sphinx jobs The updates to the sphinx docs jobs in support of the updates to the PTI wound up exposing an unintended interface. There are two flavors of the tox_install.sh file out there, and we basically need to collapse them into one flavor. Update the tox_install.sh script to match the constraints-as-first-argument form. Change-Id: I7cb4b44952713752435e1faf0f63bf0d37e7dda6 --- tools/tox_install.sh | 41 ++++++++++++++++++++--------------------- tox.ini | 3 +-- 2 files changed, 21 insertions(+), 23 deletions(-) diff --git a/tools/tox_install.sh b/tools/tox_install.sh index 15aa9dec..43468e45 100755 --- a/tools/tox_install.sh +++ b/tools/tox_install.sh @@ -4,28 +4,27 @@ # with installing the client from source. We should remove the version pin in # the constraints file before applying it for from-source installation. +CONSTRAINTS_FILE=$1 +shift 1 + set -e -if [[ -z "$CONSTRAINTS_FILE" ]]; then - echo 'WARNING: expected $CONSTRAINTS_FILE to be set' >&2 - PIP_FLAGS=(-U) -else - # NOTE(tonyb): Place this in the tox enviroment's log dir so it will get - # published to logs.openstack.org for easy debugging. - localfile="$VIRTUAL_ENV/log/upper-constraints.txt" - - if [[ "$CONSTRAINTS_FILE" != http* ]]; then - CONSTRAINTS_FILE="file://$CONSTRAINTS_FILE" - fi - curl "$CONSTRAINTS_FILE" --insecure --progress-bar --output "$localfile" - - pip install -c"$localfile" openstack-requirements - - # This is the main purpose of the script: Allow local installation of - # the current repo. It is listed in constraints file and thus any - # install will be constrained and we need to unconstrain it. - edit-constraints "$localfile" -- "$CLIENT_NAME" - PIP_FLAGS=(-c"$localfile" -U) +# NOTE(tonyb): Place this in the tox enviroment's log dir so it will get +# published to logs.openstack.org for easy debugging. +localfile="$VIRTUAL_ENV/log/upper-constraints.txt" + +if [[ $CONSTRAINTS_FILE != http* ]]; then + CONSTRAINTS_FILE=file://$CONSTRAINTS_FILE fi +# NOTE(tonyb): need to add curl to bindep.txt if the project supports bindep +curl $CONSTRAINTS_FILE --insecure --progress-bar --output $localfile + +pip install -c$localfile openstack-requirements + +# This is the main purpose of the script: Allow local installation of +# the current repo. It is listed in constraints file and thus any +# install will be constrained and we need to unconstrain it. +edit-constraints $localfile -- $CLIENT_NAME -pip install "${PIP_FLAGS[@]}" "$@" +pip install -c$localfile -U $* +exit $? diff --git a/tox.ini b/tox.ini index 1be9a39c..303ce660 100644 --- a/tox.ini +++ b/tox.ini @@ -5,13 +5,12 @@ skipsdist = True [testenv] usedevelop = True -install_command = {toxinidir}/tools/tox_install.sh {opts} {packages} +install_command = {toxinidir}/tools/tox_install.sh {env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages} setenv = LANG=en_US.utf8 VIRTUAL_ENV={envdir} BRANCH_NAME=master CLIENT_NAME=python-swiftclient - CONSTRAINTS_FILE={env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt From cb2778659e8d69d8741ca3167314862f8555a989 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Tue, 28 Nov 2017 11:02:45 -0800 Subject: [PATCH 007/238] Make tox runnable in a directory with spaces I noticed a disturbing lack of quote-wrapping in change I7cb4b44952713752435e1faf0f63bf0d37e7dda6 but as I poked at it, I realized that trouble runs rampant. This seems to clean it all up, though I haven't tested *every* environment we define. Change-Id: I1454eb113e5bd9125d39f2e57e2ed96f6ddc42fc --- tools/tox_install.sh | 14 +++++++------- tox.ini | 5 +++-- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/tools/tox_install.sh b/tools/tox_install.sh index 43468e45..f3a83e94 100755 --- a/tools/tox_install.sh +++ b/tools/tox_install.sh @@ -4,7 +4,7 @@ # with installing the client from source. We should remove the version pin in # the constraints file before applying it for from-source installation. -CONSTRAINTS_FILE=$1 +CONSTRAINTS_FILE="$1" shift 1 set -e @@ -13,18 +13,18 @@ set -e # published to logs.openstack.org for easy debugging. localfile="$VIRTUAL_ENV/log/upper-constraints.txt" -if [[ $CONSTRAINTS_FILE != http* ]]; then - CONSTRAINTS_FILE=file://$CONSTRAINTS_FILE +if [[ "$CONSTRAINTS_FILE" != http* ]]; then + CONSTRAINTS_FILE="file://$CONSTRAINTS_FILE" fi # NOTE(tonyb): need to add curl to bindep.txt if the project supports bindep -curl $CONSTRAINTS_FILE --insecure --progress-bar --output $localfile +curl "$CONSTRAINTS_FILE" --insecure --progress-bar --output "$localfile" -pip install -c$localfile openstack-requirements +python -m pip install -c"$localfile" openstack-requirements # This is the main purpose of the script: Allow local installation of # the current repo. It is listed in constraints file and thus any # install will be constrained and we need to unconstrain it. -edit-constraints $localfile -- $CLIENT_NAME +python "$(which edit-constraints)" "$localfile" -- $CLIENT_NAME -pip install -c$localfile -U $* +python -m pip install -c"$localfile" -U "$@" exit $? diff --git a/tox.ini b/tox.ini index 303ce660..e7372413 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,8 @@ skipsdist = True [testenv] usedevelop = True -install_command = {toxinidir}/tools/tox_install.sh {env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages} +install_command = "{toxinidir}/tools/tox_install.sh" "{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt}" {opts} {packages} +list_dependencies_command = python -m pip freeze setenv = LANG=en_US.utf8 VIRTUAL_ENV={envdir} @@ -24,7 +25,7 @@ passenv = SWIFT_* *_proxy [testenv:pep8] commands = - flake8 swiftclient tests + python -m flake8 swiftclient tests [testenv:venv] commands = {posargs} From a9b8f0a0d191873ac88b0c70166a2b889096fa69 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Wed, 17 Jan 2018 10:09:47 -0800 Subject: [PATCH 008/238] Revert "Add Constraints support" Per http://lists.openstack.org/pipermail/openstack-dev/2017-December/125348.html > For many projects, tox_install.sh is not needed at all Let's see if that holds for python-swiftclient! This reverts commit f2f278fcbec3ad52a1726bb5a3f775d13bcc99dc. Change-Id: I0462c50ec71d87bac226f83a0d0942871ef5a0e7 --- bindep.txt | 1 - tools/tox_install.sh | 30 ------------------------------ tox.ini | 10 +++------- 3 files changed, 3 insertions(+), 38 deletions(-) delete mode 100755 tools/tox_install.sh diff --git a/bindep.txt b/bindep.txt index cc3a7700..27f27363 100644 --- a/bindep.txt +++ b/bindep.txt @@ -1,7 +1,6 @@ # This is a cross-platform list tracking distribution packages needed by tests; # see http://docs.openstack.org/infra/bindep/ for additional information. -curl pypy [test !platform:fedora] pypy-dev [test platform:dpkg] pypy-devel [test platform:rpm !platform:fedora] diff --git a/tools/tox_install.sh b/tools/tox_install.sh deleted file mode 100755 index f3a83e94..00000000 --- a/tools/tox_install.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash - -# Client constraint file contains this client version pin that is in conflict -# with installing the client from source. We should remove the version pin in -# the constraints file before applying it for from-source installation. - -CONSTRAINTS_FILE="$1" -shift 1 - -set -e - -# NOTE(tonyb): Place this in the tox enviroment's log dir so it will get -# published to logs.openstack.org for easy debugging. -localfile="$VIRTUAL_ENV/log/upper-constraints.txt" - -if [[ "$CONSTRAINTS_FILE" != http* ]]; then - CONSTRAINTS_FILE="file://$CONSTRAINTS_FILE" -fi -# NOTE(tonyb): need to add curl to bindep.txt if the project supports bindep -curl "$CONSTRAINTS_FILE" --insecure --progress-bar --output "$localfile" - -python -m pip install -c"$localfile" openstack-requirements - -# This is the main purpose of the script: Allow local installation of -# the current repo. It is listed in constraints file and thus any -# install will be constrained and we need to unconstrain it. -python "$(which edit-constraints)" "$localfile" -- $CLIENT_NAME - -python -m pip install -c"$localfile" -U "$@" -exit $? diff --git a/tox.ini b/tox.ini index e7372413..541df654 100644 --- a/tox.ini +++ b/tox.ini @@ -5,13 +5,11 @@ skipsdist = True [testenv] usedevelop = True -install_command = "{toxinidir}/tools/tox_install.sh" "{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt}" {opts} {packages} +install_command = python -m pip install -U {opts} {packages} list_dependencies_command = python -m pip freeze setenv = LANG=en_US.utf8 VIRTUAL_ENV={envdir} - BRANCH_NAME=master - CLIENT_NAME=python-swiftclient deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt @@ -31,13 +29,11 @@ commands = commands = {posargs} [testenv:cover] -commands = python setup.py testr --coverage +commands = python setup.py testr --coverage coverage report [testenv:func] -setenv = - {[testenv]setenv} - OS_TEST_PATH=tests.functional +setenv = OS_TEST_PATH=tests.functional whitelist_externals = coverage rm From 2faea932870956583f83226886d33304ee1eee46 Mon Sep 17 00:00:00 2001 From: Timur Alperovich Date: Wed, 28 Jun 2017 12:02:21 -0700 Subject: [PATCH 009/238] Allow for object uploads > 5GB from stdin. When uploading from standard input, swiftclient should turn the upload into an SLO in the case of large objects. This patch picks the threshold as 10MB (and uses that as the default segment size). The consumers can also supply the --segment-size option to alter that threshold and the SLO segment size. The patch does buffer one segment in memory (which is why 10MB default was chosen). (test is updated) Change-Id: Ib13e0b687bc85930c29fe9f151cf96bc53b2e594 --- swiftclient/service.py | 244 +++++++++++++++++++++++++++++++++---- swiftclient/shell.py | 8 ++ tests/unit/test_service.py | 214 ++++++++++++++++++++++++++++++++ tests/unit/utils.py | 21 ++++ 4 files changed, 462 insertions(+), 25 deletions(-) diff --git a/swiftclient/service.py b/swiftclient/service.py index 7b5ecd44..ed5e9e98 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -1502,7 +1502,8 @@ def upload(self, container, objects, options=None): if hasattr(s, 'read'): # We've got a file like object to upload to o file_future = self.thread_manager.object_uu_pool.submit( - self._upload_object_job, container, s, o, object_options + self._upload_object_job, container, s, o, object_options, + results_queue=rq ) details['file'] = s details['object'] = o @@ -1784,6 +1785,132 @@ def _upload_segment_job(conn, path, container, segment_name, segment_start, if fp is not None: fp.close() + @staticmethod + def _put_object(conn, container, name, content, headers=None, md5=None): + """ + Upload object into a given container and verify the resulting ETag, if + the md5 optional parameter is passed. + + :param conn: The Swift connection to use for uploads. + :param container: The container to put the object into. + :param name: The name of the object. + :param content: Object content. + :param headers: Headers (optional) to associate with the object. + :param md5: MD5 sum of the content. If passed in, will be used to + verify the returned ETag. + + :returns: A dictionary as the response from calling put_object. + The keys are: + - status + - reason + - headers + On error, the dictionary contains the following keys: + - success (with value False) + - error - the encountered exception (object) + - error_timestamp + - response_dict - results from the put_object call, as + documented above + - attempts - number of attempts made + """ + if headers is None: + headers = {} + else: + headers = dict(headers) + if md5 is not None: + headers['etag'] = md5 + results = {} + try: + etag = conn.put_object( + container, name, content, content_length=len(content), + headers=headers, response_dict=results) + if md5 is not None and etag != md5: + raise SwiftError('Upload verification failed for {0}: md5 ' + 'mismatch {1} != {2}'.format(name, md5, etag)) + results['success'] = True + except Exception as err: + traceback, err_time = report_traceback() + logger.exception(err) + return { + 'success': False, + 'error': err, + 'error_timestamp': err_time, + 'response_dict': results, + 'attempts': conn.attempts, + 'traceback': traceback + } + return results + + @staticmethod + def _upload_stream_segment(conn, container, object_name, + segment_container, segment_name, + segment_size, segment_index, + headers, fd): + """ + Upload a segment from a stream, buffering it in memory first. The + resulting object is placed either as a segment in the segment + container, or if it is smaller than a single segment, as the given + object name. + + :param conn: Swift Connection to use. + :param container: Container in which the object would be placed. + :param object_name: Name of the final object (used in case the stream + is smaller than the segment_size) + :param segment_container: Container to hold the object segments. + :param segment_name: The name of the segment. + :param segment_size: Minimum segment size. + :param segment_index: The segment index. + :param headers: Headers to attach to the segment/object. + :param fd: File-like handle for the content. Must implement read(). + + :returns: Dictionary, containing the following keys: + - complete -- whether the stream is exhausted + - segment_size - the actual size of the segment (may be + smaller than the passed in segment_size) + - segment_location - path to the segment + - segment_index - index of the segment + - segment_etag - the ETag for the segment + """ + buf = [] + dgst = md5() + bytes_read = 0 + while bytes_read < segment_size: + data = fd.read(segment_size - bytes_read) + if not data: + break + bytes_read += len(data) + dgst.update(data) + buf.append(data) + buf = b''.join(buf) + segment_hash = dgst.hexdigest() + + if not buf and segment_index > 0: + # Happens if the segment size aligns with the object size + return {'complete': True, + 'segment_size': 0, + 'segment_index': None, + 'segment_etag': None, + 'segment_location': None, + 'success': True} + + if segment_index == 0 and len(buf) < segment_size: + ret = SwiftService._put_object( + conn, container, object_name, buf, headers, segment_hash) + ret['segment_location'] = '/%s/%s' % (container, object_name) + else: + ret = SwiftService._put_object( + conn, segment_container, segment_name, buf, headers, + segment_hash) + ret['segment_location'] = '/%s/%s' % ( + segment_container, segment_name) + + ret.update( + dict(complete=len(buf) < segment_size, + segment_size=len(buf), + segment_index=segment_index, + segment_etag=segment_hash, + for_object=object_name)) + return ret + def _get_chunk_data(self, conn, container, obj, headers, manifest=None): chunks = [] if 'x-object-manifest' in headers: @@ -1833,6 +1960,47 @@ def _is_identical(self, chunk_data, path): # Each chunk is verified; check that we're at the end of the file return not fp.read(1) + @staticmethod + def _upload_slo_manifest(conn, segment_results, container, obj, headers): + """ + Upload an SLO manifest, given the results of uploading each segment, to + the specified container. + + :param segment_results: List of response_dict structures, as populated + by _upload_segment_job. Specifically, each + entry must container the following keys: + - segment_location + - segment_etag + - segment_size + - segment_index + :param container: The container to put the manifest into. + :param obj: The name of the manifest object to use. + :param headers: Optional set of headers to attach to the manifest. + """ + if headers is None: + headers = {} + segment_results.sort(key=lambda di: di['segment_index']) + for seg in segment_results: + seg_loc = seg['segment_location'].lstrip('/') + if isinstance(seg_loc, text_type): + seg_loc = seg_loc.encode('utf-8') + + manifest_data = json.dumps([ + { + 'path': d['segment_location'], + 'etag': d['segment_etag'], + 'size_bytes': d['segment_size'] + } for d in segment_results + ]) + + response = {} + conn.put_object( + container, obj, manifest_data, + headers=headers, + query_string='multipart-manifest=put', + response_dict=response) + return response + def _upload_object_job(self, conn, container, source, obj, options, results_queue=None): if obj.startswith('./') or obj.startswith('.\\'): @@ -1990,29 +2158,11 @@ def _upload_object_job(self, conn, container, source, obj, options, res['segment_results'] = segment_results if options['use_slo']: - segment_results.sort(key=lambda di: di['segment_index']) - for seg in segment_results: - seg_loc = seg['segment_location'].lstrip('/') - if isinstance(seg_loc, text_type): - seg_loc = seg_loc.encode('utf-8') - new_slo_manifest_paths.add(seg_loc) - - manifest_data = json.dumps([ - { - 'path': d['segment_location'], - 'etag': d['segment_etag'], - 'size_bytes': d['segment_size'] - } for d in segment_results - ]) - - mr = {} - conn.put_object( - container, obj, manifest_data, - headers=put_headers, - query_string='multipart-manifest=put', - response_dict=mr - ) - res['manifest_response_dict'] = mr + response = self._upload_slo_manifest( + conn, segment_results, container, obj, put_headers) + res['manifest_response_dict'] = response + new_slo_manifest_paths = { + seg['segment_location'] for seg in segment_results} else: new_object_manifest = '%s/%s/%s/%s/%s/' % ( quote(seg_container.encode('utf8')), @@ -2030,6 +2180,51 @@ def _upload_object_job(self, conn, container, source, obj, options, response_dict=mr ) res['manifest_response_dict'] = mr + elif options['use_slo'] and segment_size and not path: + segment = 0 + results = [] + while True: + segment_name = '%s/slo/%s/%s/%08d' % ( + obj, put_headers['x-object-meta-mtime'], + segment_size, segment + ) + seg_container = container + '_segments' + if options['segment_container']: + seg_container = options['segment_container'] + ret = self._upload_stream_segment( + conn, container, obj, + seg_container, + segment_name, + segment_size, + segment, + put_headers, + stream + ) + if not ret['success']: + return ret + if (ret['complete'] and segment == 0) or\ + ret['segment_size'] > 0: + results.append(ret) + if results_queue is not None: + # Don't insert the 0-sized segments or objects + # themselves + if ret['segment_location'] != '/%s/%s' % ( + container, obj) and ret['segment_size'] > 0: + results_queue.put(ret) + if ret['complete']: + break + segment += 1 + if results[0]['segment_location'] != '/%s/%s' % ( + container, obj): + response = self._upload_slo_manifest( + conn, results, container, obj, put_headers) + res['manifest_response_dict'] = response + new_slo_manifest_paths = { + r['segment_location'] for r in results} + res['large_object'] = True + else: + res['response_dict'] = ret + res['large_object'] = False else: res['large_object'] = False obr = {} @@ -2063,7 +2258,6 @@ def _upload_object_job(self, conn, container, source, obj, options, finally: if fp is not None: fp.close() - if old_manifest or old_slo_manifest_paths: drs = [] delobjsmap = {} diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 43fcf475..d02c709f 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -947,6 +947,8 @@ def st_copy(parser, args, output_manager): def st_upload(parser, args, output_manager): + DEFAULT_STDIN_SEGMENT = 10 * 1024 * 1024 + parser.add_argument( '-c', '--changed', action='store_true', dest='changed', default=False, help='Only upload files that have changed since ' @@ -1060,6 +1062,12 @@ def st_upload(parser, args, output_manager): st_upload_help) return + if from_stdin: + if not options['use_slo']: + options['use_slo'] = True + if not options['segment_size']: + options['segment_size'] = DEFAULT_STDIN_SEGMENT + options['object_uu_threads'] = options['object_threads'] with SwiftService(options=options) as swift: try: diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index 5ccc0813..12fbaa00 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -36,6 +36,8 @@ SwiftService, SwiftError, SwiftUploadObject ) +from tests.unit import utils as test_utils + clean_os_environ = {} environ_prefixes = ('ST_', 'OS_') @@ -1088,6 +1090,83 @@ def test_upload_with_relative_path(self, *args, **kwargs): self.assertEqual(upload_obj_resp['path'], obj['path']) self.assertTrue(mock_open.return_value.closed) + @mock.patch('swiftclient.service.Connection') + def test_upload_stream(self, mock_conn): + service = SwiftService({}) + + stream = test_utils.FakeStream(2048) + segment_etag = md5(b'A' * 1024).hexdigest() + + mock_conn.return_value.head_object.side_effect = \ + ClientException('Not Found', http_status=404) + mock_conn.return_value.put_object.return_value = \ + segment_etag + options = {'use_slo': True, 'segment_size': 1024} + resp_iter = service.upload( + 'container', + [SwiftUploadObject(stream, object_name='streamed')], + options) + responses = [x for x in resp_iter] + for resp in responses: + self.assertFalse('error' in resp) + self.assertTrue(resp['success']) + self.assertEqual(5, len(responses)) + container_resp, segment_container_resp = responses[0:2] + segment_response = responses[2:4] + upload_obj_resp = responses[-1] + self.assertEqual(container_resp['action'], + 'create_container') + self.assertEqual(upload_obj_resp['action'], + 'upload_object') + self.assertEqual(upload_obj_resp['object'], + 'streamed') + self.assertTrue(upload_obj_resp['path'] is None) + self.assertTrue(upload_obj_resp['large_object']) + self.assertIn('manifest_response_dict', upload_obj_resp) + self.assertEqual(upload_obj_resp['manifest_response_dict'], {}) + for i, resp in enumerate(segment_response): + self.assertEqual(i, resp['segment_index']) + self.assertEqual(1024, resp['segment_size']) + self.assertEqual('d47b127bc2de2d687ddc82dac354c415', + resp['segment_etag']) + self.assertTrue(resp['segment_location'].endswith( + '/0000000%d' % i)) + self.assertTrue(resp['segment_location'].startswith( + '/container_segments/streamed')) + + @mock.patch('swiftclient.service.Connection') + def test_upload_stream_fits_in_one_segment(self, mock_conn): + service = SwiftService({}) + + stream = test_utils.FakeStream(2048) + whole_etag = md5(b'A' * 2048).hexdigest() + + mock_conn.return_value.head_object.side_effect = \ + ClientException('Not Found', http_status=404) + mock_conn.return_value.put_object.return_value = \ + whole_etag + options = {'use_slo': True, 'segment_size': 10240} + resp_iter = service.upload( + 'container', + [SwiftUploadObject(stream, object_name='streamed')], + options) + responses = [x for x in resp_iter] + for resp in responses: + self.assertNotIn('error', resp) + self.assertTrue(resp['success']) + self.assertEqual(3, len(responses)) + container_resp, segment_container_resp = responses[0:2] + upload_obj_resp = responses[-1] + self.assertEqual(container_resp['action'], + 'create_container') + self.assertEqual(upload_obj_resp['action'], + 'upload_object') + self.assertEqual(upload_obj_resp['object'], + 'streamed') + self.assertTrue(upload_obj_resp['path'] is None) + self.assertFalse(upload_obj_resp['large_object']) + self.assertNotIn('manifest_response_dict', upload_obj_resp) + class TestServiceUpload(_TestServiceBase): @@ -1226,6 +1305,141 @@ def _consuming_conn(*a, **kw): self.assertIsInstance(contents, utils.LengthWrapper) self.assertEqual(len(contents), 10) + def test_upload_stream_segment(self): + common_params = { + 'segment_container': 'segments', + 'segment_name': 'test_stream_2', + 'container': 'test_stream', + 'object': 'stream_object', + } + tests = [ + {'test_params': { + 'segment_size': 1024, + 'segment_index': 2, + 'content_size': 1024}, + 'put_object_args': { + 'container': 'segments', + 'object': 'test_stream_2'}, + 'expected': { + 'complete': False, + 'segment_etag': md5(b'A' * 1024).hexdigest()}}, + {'test_params': { + 'segment_size': 2048, + 'segment_index': 0, + 'content_size': 512}, + 'put_object_args': { + 'container': 'test_stream', + 'object': 'stream_object'}, + 'expected': { + 'complete': True, + 'segment_etag': md5(b'A' * 512).hexdigest()}}, + # 0-sized segment should not be uploaded + {'test_params': { + 'segment_size': 1024, + 'segment_index': 1, + 'content_size': 0}, + 'put_object_args': {}, + 'expected': { + 'complete': True}}, + # 0-sized objects should be uploaded + {'test_params': { + 'segment_size': 1024, + 'segment_index': 0, + 'content_size': 0}, + 'put_object_args': { + 'container': 'test_stream', + 'object': 'stream_object'}, + 'expected': { + 'complete': True, + 'segment_etag': md5(b'').hexdigest()}}, + # Test boundary conditions + {'test_params': { + 'segment_size': 1024, + 'segment_index': 1, + 'content_size': 1023}, + 'put_object_args': { + 'container': 'segments', + 'object': 'test_stream_2'}, + 'expected': { + 'complete': True, + 'segment_etag': md5(b'A' * 1023).hexdigest()}}, + {'test_params': { + 'segment_size': 2048, + 'segment_index': 0, + 'content_size': 2047}, + 'put_object_args': { + 'container': 'test_stream', + 'object': 'stream_object'}, + 'expected': { + 'complete': True, + 'segment_etag': md5(b'A' * 2047).hexdigest()}}, + {'test_params': { + 'segment_size': 1024, + 'segment_index': 2, + 'content_size': 1025}, + 'put_object_args': { + 'container': 'segments', + 'object': 'test_stream_2'}, + 'expected': { + 'complete': False, + 'segment_etag': md5(b'A' * 1024).hexdigest()}}, + ] + + for test_args in tests: + params = test_args['test_params'] + stream = test_utils.FakeStream(params['content_size']) + segment_size = params['segment_size'] + segment_index = params['segment_index'] + + def _fake_put_object(*args, **kwargs): + contents = args[2] + # Consume and compute md5 + return md5(contents).hexdigest() + + mock_conn = mock.Mock() + mock_conn.put_object.side_effect = _fake_put_object + + s = SwiftService() + resp = s._upload_stream_segment( + conn=mock_conn, + container=common_params['container'], + object_name=common_params['object'], + segment_container=common_params['segment_container'], + segment_name=common_params['segment_name'], + segment_size=segment_size, + segment_index=segment_index, + headers={}, + fd=stream) + expected_args = test_args['expected'] + put_args = test_args['put_object_args'] + expected_response = { + 'segment_size': min(len(stream), segment_size), + 'complete': expected_args['complete'], + 'success': True, + } + if len(stream) or segment_index == 0: + segment_location = '/%s/%s' % (put_args['container'], + put_args['object']) + expected_response.update( + {'segment_index': segment_index, + 'segment_location': segment_location, + 'segment_etag': expected_args['segment_etag'], + 'for_object': common_params['object']}) + mock_conn.put_object.assert_called_once_with( + put_args['container'], + put_args['object'], + mock.ANY, + content_length=min(len(stream), segment_size), + headers={'etag': expected_args['segment_etag']}, + response_dict=mock.ANY) + else: + self.assertEqual([], mock_conn.put_object.mock_calls) + expected_response.update( + {'segment_index': None, + 'segment_location': None, + 'segment_etag': None}) + self.assertEqual(expected_response, resp) + def test_etag_mismatch_with_ignore_checksum(self): def _consuming_conn(*a, **kw): contents = a[2] diff --git a/tests/unit/utils.py b/tests/unit/utils.py index c05146ec..2def73f5 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -548,3 +548,24 @@ def _fake_import_keystone_client(auth_version): return fake_import, fake_import return _fake_import_keystone_client + + +class FakeStream(object): + def __init__(self, size): + self.bytes_read = 0 + self.size = size + + def read(self, size=-1): + if self.bytes_read == self.size: + return b'' + + if size == -1 or size + self.bytes_read > self.size: + remaining = self.size - self.bytes_read + self.bytes_read = self.size + return b'A' * remaining + + self.bytes_read += size + return b'A' * size + + def __len__(self): + return self.size From b91651eba09ed43903c55f24e3a1a52aefeea75f Mon Sep 17 00:00:00 2001 From: John Dickinson Date: Tue, 23 Jan 2018 11:15:26 +1100 Subject: [PATCH 010/238] authors/changelog updates for 3.5.0 release Change-Id: I70b79c0fd6e9adbfdcc799459dc52063c7402be2 --- AUTHORS | 1 + ChangeLog | 19 +++++++++++++++++++ .../notes/350_notes-ad0ae19704b2eb88.yaml | 18 ++++++++++++++++++ 3 files changed, 38 insertions(+) create mode 100644 releasenotes/notes/350_notes-ad0ae19704b2eb88.yaml diff --git a/AUTHORS b/AUTHORS index bf69dba5..388d8700 100644 --- a/AUTHORS +++ b/AUTHORS @@ -42,6 +42,7 @@ Florent Flament (florent.flament-ext@cloudwatt.com) Greg Holt (gholt@rackspace.com) Greg Lange (greglange@gmail.com) groqez (groqez@yopmail.net) +Hangdong Zhang (hdzhang@fiberhome.com) Hemanth Makkapati (hemanth.makkapati@mailtrust.com) hgangwx (hgangwx@cn.ibm.com) Hirokazu Sakata (h.sakata@staff.east.ntt.co.jp) diff --git a/ChangeLog b/ChangeLog index 967d7e22..efa7e8a8 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,22 @@ +3.5.0 +----- + +* Allow for object uploads > 5GB from stdin. + + When uploading from standard input, swiftclient will turn the upload + into an SLO in the case of large objects. By default, input larger + than 10MB will be uploaded as an SLO with 10MB segment sizes. Users + can also supply the ``--segment-size`` option to alter that + threshold and the SLO segment size. One segment is buffered in + memory (which is why 10MB default was chosen). + +* The ``--meta`` option can now be set on the upload command. + +* Updated PyPy test dependency references to be more accurate + on different distros. + +* Various other minor bug fixes and improvements. + 3.4.0 ----- diff --git a/releasenotes/notes/350_notes-ad0ae19704b2eb88.yaml b/releasenotes/notes/350_notes-ad0ae19704b2eb88.yaml new file mode 100644 index 00000000..2e6b4eac --- /dev/null +++ b/releasenotes/notes/350_notes-ad0ae19704b2eb88.yaml @@ -0,0 +1,18 @@ +--- +features: + - | + Allow for object uploads > 5GB from stdin. + + When uploading from standard input, swiftclient will turn the upload + into an SLO in the case of large objects. By default, input larger + than 10MB will be uploaded as an SLO with 10MB segment sizes. Users + can also supply the ``--segment-size`` option to alter that + threshold and the SLO segment size. One segment is buffered in + memory (which is why 10MB default was chosen). + + - | + The ``--meta`` option can now be set on the upload command. + + - | + Updated PyPy test dependency references to be more accurate + on different distros. From 2901e1e9ef1932b0b3874e608422bbfdfbd1448a Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Fri, 26 Jan 2018 14:14:18 -0800 Subject: [PATCH 011/238] Treat 404 as success when deleting segments Change-Id: I76be70ddb289bd4f1054a684a247279ab16ca34a --- swiftclient/service.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/swiftclient/service.py b/swiftclient/service.py index ed5e9e98..607c1aea 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -2471,17 +2471,18 @@ def _per_item_delete(self, container, objects, options, rdict, rq): def _delete_segment(conn, container, obj, results_queue=None): results_dict = {} try: - conn.delete_object(container, obj, response_dict=results_dict) res = {'success': True} + conn.delete_object(container, obj, response_dict=results_dict) except Exception as err: - traceback, err_time = report_traceback() - logger.exception(err) - res = { - 'success': False, - 'error': err, - 'traceback': traceback, - 'error_timestamp': err_time - } + if not isinstance(err, ClientException) or err.http_status != 404: + traceback, err_time = report_traceback() + logger.exception(err) + res = { + 'success': False, + 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time + } res.update({ 'action': 'delete_segment', From 65bfbb00bf31c1cca0b37bb6a3db33f38e7146d5 Mon Sep 17 00:00:00 2001 From: OpenStack Release Bot Date: Mon, 29 Jan 2018 16:54:44 +0000 Subject: [PATCH 012/238] Update reno for stable/queens Change-Id: I7f7b21e3dd0d1ef71c159e2b72f9ad9f963b0773 --- releasenotes/source/index.rst | 1 + releasenotes/source/queens.rst | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 releasenotes/source/queens.rst diff --git a/releasenotes/source/index.rst b/releasenotes/source/index.rst index 6e3419df..a5240ea2 100644 --- a/releasenotes/source/index.rst +++ b/releasenotes/source/index.rst @@ -6,6 +6,7 @@ :maxdepth: 1 current + queens pike ocata newton diff --git a/releasenotes/source/queens.rst b/releasenotes/source/queens.rst new file mode 100644 index 00000000..36ac6160 --- /dev/null +++ b/releasenotes/source/queens.rst @@ -0,0 +1,6 @@ +=================================== + Queens Series Release Notes +=================================== + +.. release-notes:: + :branch: stable/queens From 8bf86accca4d3a742c552cac2e523a13b6293fb6 Mon Sep 17 00:00:00 2001 From: shangxiaobj Date: Mon, 22 Jan 2018 18:58:54 -0800 Subject: [PATCH 013/238] Update the old http doc links Update the old http doc links to the https ones according to the official OpenStack website. Change-Id: Ibf9ecbccb743d2b9a678a1ca69f0b3adc9106a12 --- CONTRIBUTING.rst | 4 ++-- README.rst | 8 ++++---- bindep.txt | 2 +- doc/source/cli/index.rst | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index ec32d855..0bde9680 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -1,13 +1,13 @@ If you would like to contribute to the development of OpenStack, you must follow the steps in this page: - http://docs.openstack.org/infra/manual/developers.html + https://docs.openstack.org/infra/manual/developers.html Once those steps have been completed, changes to OpenStack should be submitted for review via the Gerrit tool, following the workflow documented at: - http://docs.openstack.org/infra/manual/developers.html#development-workflow. + https://docs.openstack.org/infra/manual/developers.html#development-workflow Gerrit is the review system used in the OpenStack projects. We're sorry, but we won't be able to respond to pull requests submitted through diff --git a/README.rst b/README.rst index 38104392..cd6a7aca 100644 --- a/README.rst +++ b/README.rst @@ -24,13 +24,13 @@ This is a python client for the Swift API. There's a Python API (the Development takes place via the usual OpenStack processes as outlined in the `OpenStack wiki`__. -__ http://docs.openstack.org/infra/manual/developers.html +__ https://docs.openstack.org/infra/manual/developers.html This code is based on the original client previously included with `OpenStack's Swift`__ The python-swiftclient is licensed under the Apache License like the rest of OpenStack. -__ http://github.com/openstack/swift +__ https://github.com/openstack/swift * Free software: Apache license * `PyPI`_ - package installation @@ -48,8 +48,8 @@ __ http://github.com/openstack/swift .. _Blueprints: https://blueprints.launchpad.net/python-swiftclient .. _Bugs: https://bugs.launchpad.net/python-swiftclient .. _Source: https://git.openstack.org/cgit/openstack/python-swiftclient -.. _How to Contribute: http://docs.openstack.org/infra/manual/developers.html -.. _Specs: http://specs.openstack.org/openstack/swift-specs/ +.. _How to Contribute: https://docs.openstack.org/infra/manual/developers.html +.. _Specs: https://specs.openstack.org/openstack/swift-specs/ .. contents:: Contents: diff --git a/bindep.txt b/bindep.txt index 27f27363..17c0cd56 100644 --- a/bindep.txt +++ b/bindep.txt @@ -1,5 +1,5 @@ # This is a cross-platform list tracking distribution packages needed by tests; -# see http://docs.openstack.org/infra/bindep/ for additional information. +# see https://docs.openstack.org/infra/bindep/ for additional information. pypy [test !platform:fedora] pypy-dev [test platform:dpkg] diff --git a/doc/source/cli/index.rst b/doc/source/cli/index.rst index bec1f5e5..4aa3bfcd 100644 --- a/doc/source/cli/index.rst +++ b/doc/source/cli/index.rst @@ -313,7 +313,7 @@ The ``-p `` or ``--prefix `` is an option that will only list items beginning with that prefix. The ``-d `` or ``--delimiter `` is an option (for container listings only) that will roll up items with the given delimiter (see `OpenStack Swift -general documentation ` for +general documentation ` for what this means). The ``-l`` and ``--lh`` options provide more detail, similar to ``ls -l`` @@ -457,7 +457,7 @@ and is used to define the user metadata items to set in the form ``Name:Value``. You can repeat this option. For example: ``post -m Color:Blue -m Size:Large`` For more information about ACL formats see the documentation: -`ACLs `_. +`ACLs `_. **Positional arguments:** From 097f4b26d9e68fb818c078cd0a0d30658042fff7 Mon Sep 17 00:00:00 2001 From: Kota Tsuyuzaki Date: Fri, 23 Feb 2018 18:02:40 +0900 Subject: [PATCH 014/238] Add missing value in command line docs Because it should take a value of either realm or full url. Change-Id: I1fe30825ef1620e256c9fd3057da6808b03d7200 --- doc/source/cli/index.rst | 2 +- swiftclient/shell.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/cli/index.rst b/doc/source/cli/index.rst index 4aa3bfcd..b57c25ef 100644 --- a/doc/source/cli/index.rst +++ b/doc/source/cli/index.rst @@ -442,7 +442,7 @@ swift post .. code-block:: console - Usage: swift post [--read-acl ] [--write-acl ] [--sync-to] + Usage: swift post [--read-acl ] [--write-acl ] [--sync-to ] [--sync-key ] [--meta ] [--header
] [ []] diff --git a/swiftclient/shell.py b/swiftclient/shell.py index d02c709f..6ccc16df 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -687,7 +687,7 @@ def st_stat(parser, args, output_manager): output_manager.error(e.value) -st_post_options = '''[--read-acl ] [--write-acl ] [--sync-to] +st_post_options = '''[--read-acl ] [--write-acl ] [--sync-to ] [--sync-key ] [--meta ] [--header
] [ []] From cec0f0e4ed6d04515441390c01de096d2afb60dd Mon Sep 17 00:00:00 2001 From: wangqi Date: Mon, 5 Mar 2018 13:23:36 +0000 Subject: [PATCH 015/238] Update links in README Change the outdated links to the latest links in README Change-Id: Ic0adc686592265f2be2acb55f0520c35d1717f76 --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index cd6a7aca..46c0b0ac 100644 --- a/README.rst +++ b/README.rst @@ -2,8 +2,8 @@ Team and repository tags ======================== -.. image:: https://governance.openstack.org/badges/python-swiftclient.svg - :target: https://governance.openstack.org/reference/tags/index.html +.. image:: https://governance.openstack.org/tc/badges/python-swiftclient.svg + :target: https://governance.openstack.org/tc/reference/tags/index.html .. Change things from this point on From a36c3cfda1c243273fcd11b9e123aca877869244 Mon Sep 17 00:00:00 2001 From: Timur Alperovich Date: Mon, 5 Mar 2018 17:33:22 -0800 Subject: [PATCH 016/238] Add a query_string option to head_object(). Submitting a path parameter with a HEAD request on an object can be useful if one is trying to find out information about an SLO/DLO without retrieving the manifest. Change-Id: I39efd098e72bd31de271ac51d4d75381929c9638 --- swiftclient/client.py | 9 ++++++--- tests/unit/test_swiftclient.py | 15 +++++++++++++-- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index 7db75f00..60abbd81 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -1188,7 +1188,7 @@ def get_object(url, token, container, name, http_conn=None, def head_object(url, token, container, name, http_conn=None, - service_token=None, headers=None): + service_token=None, headers=None, query_string=None): """ Get object info @@ -1209,6 +1209,8 @@ def head_object(url, token, container, name, http_conn=None, else: parsed, conn = http_connection(url) path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) + if query_string: + path += '?' + query_string if headers: headers = dict(headers) else: @@ -1785,9 +1787,10 @@ def delete_container(self, container, response_dict=None, query_string=query_string, headers=headers) - def head_object(self, container, obj, headers=None): + def head_object(self, container, obj, headers=None, query_string=None): """Wrapper for :func:`head_object`""" - return self._retry(None, head_object, container, obj, headers=headers) + return self._retry(None, head_object, container, obj, headers=headers, + query_string=query_string) def get_object(self, container, obj, resp_chunk_size=None, query_string=None, response_dict=None, headers=None): diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index 3de5f027..7b8628bb 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -1180,6 +1180,16 @@ def test_request_headers(self): }), ]) + def test_query_string(self): + c.http_connection = self.fake_http_connection(204) + conn = c.http_connection('http://www.test.com') + query_string = 'foo=bar' + c.head_object('url_is_irrelevant', 'token', 'container', 'key', + http_conn=conn, query_string=query_string) + self.assertRequests([ + ('HEAD', '/container/key?foo=bar', '', {'x-auth-token': 'token'}) + ]) + class TestPutObject(MockHttpTest): @@ -2459,16 +2469,17 @@ def test_head_container(self): def test_head_object(self): headers = {'X-Favourite-Pet': 'Aardvark'} + query_string = 'foo=bar' with mock.patch('swiftclient.client.http_connection', self.fake_http_connection(200)): with mock.patch('swiftclient.client.get_auth', lambda *a, **k: ('http://url:8080/v1/a', 'token')): conn = c.Connection() conn.head_object('c1', 'o1', - headers=headers) + headers=headers, query_string=query_string) self.assertEqual(1, len(self.request_log), self.request_log) self.assertRequests([ - ('HEAD', '/v1/a/c1/o1', '', { + ('HEAD', '/v1/a/c1/o1?foo=bar', '', { 'x-auth-token': 'token', 'X-Favourite-Pet': 'Aardvark', }), From e65070964c7b1e04119c87e5f344d39358780d18 Mon Sep 17 00:00:00 2001 From: Kota Tsuyuzaki Date: Mon, 12 Mar 2018 17:54:17 +0900 Subject: [PATCH 017/238] Add force auth retry mode in swiftclient This patch attemps to add an option to force get_auth call while retrying an operation even if it gets errors other than 401 Unauthorized. Why we need this: The main reason why we need this is current python-swiftclient requests could never get succeeded under certion situation using third party proxies/load balancers between the client and swift-proxy server. I think, it would be general situation of the use case. Specifically describing nginx case, the nginx can close the socket from the client when the response code from swift is not 2xx series. In default, nginx can wait the buffers from the client for a while (default 30s)[1] but after the time past, nginx will close the socket immediately. Unfortunately, if python-swiftclient has still been sending the data into the socket, python-swiftclient will get socket error (EPIPE, BrokenPipe). From the swiftclient perspective, this is absolutely not an auth error, so current python-swiftclient will continue to retry without re-auth. However, if the root cause is sort of 401 (i.e. nginx got 401 unauthorized from the swift-proxy because of token expiration), swiftclient will loop 401 -> EPIPE -> 401... until it consume the max retry times. In particlar, less time to live of the token and multipart object upload with large segments could not get succeeded as below: Connection Model: python-swiftclient -> nginx -> swift-proxy -> swift-backend Case: Try to create slo with large segments and the auth token expired with 1 hour 1. client create a connection to nginx with successful response from swift-proxy and its auth 2. client continue to put large segment objects (e.g. 1~5GB for each and the total would 20~30GB, i.e. 20~30 segments) 3. after some of segments uploaded, 1 hour past but client is still trying to send remaining segment objects. 4. nginx got 401 from swift-proxy for a request and wait that the connection is closed from the client but timeout past because the python-swiftclient is still sending much data into the socket before reading the 401 response. 5. client got socket error because nginx closed the connection during sending the buffer. 6. client retries a new connection to nginx without re-auth... 7. finally python-swiftclient failed with socket error (Broken Pipe) In operational perspective, setting longer timeout for lingering close would be an option but it's not complete solution because any other proxy/LB may not support the options. If we actually do THE RIGHT THING in python-swiftclient, we should send expects: 100-continue header and handle the first response to re-auth correctly. HOWEVER, the current python's httplib and requests module used by python-swiftclient doesn't support expects: 100-continue header [2] and the thread proposed a fix [3] is not super active. And we know the reason we depends on the library is to fix a security issue that existed in older python-swiftclient [4] so that we should touch around it super carefully. In the reality, as the hot fix, this patch try to mitigate the unfortunate situation described above WITHOUT 100-continue fix, just users can force to re-auth when any errors occurred during the retries that can be accepted in the upstream. 1: http://nginx.org/en/docs/http/ngx_http_core_module.html#lingering_close 2: https://github.com/requests/requests/issues/713 3: https://bugs.python.org/issue1346874 4: https://review.openstack.org/#/c/69187/ Change-Id: I3470b56e3f9cf9cdb8c2fc2a94b2c551927a3440 --- swiftclient/client.py | 9 ++++- swiftclient/service.py | 4 +- swiftclient/shell.py | 6 +++ tests/unit/test_swiftclient.py | 67 ++++++++++++++++++++++++++++++++++ 4 files changed, 84 insertions(+), 2 deletions(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index 60abbd81..ab0fde80 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -1542,7 +1542,7 @@ def __init__(self, authurl=None, user=None, key=None, retries=5, os_options=None, auth_version="1", cacert=None, insecure=False, cert=None, cert_key=None, ssl_compression=True, retry_on_ratelimit=False, - timeout=None, session=None): + timeout=None, session=None, force_auth_retry=False): """ :param authurl: authentication URL :param user: user name to authenticate as @@ -1578,6 +1578,8 @@ def __init__(self, authurl=None, user=None, key=None, retries=5, after a backoff. :param timeout: The connect timeout for the HTTP connection. :param session: A keystoneauth session object. + :param force_auth_retry: reset auth info even if client got unexpected + error except 401 Unauthorized. """ self.session = session self.authurl = authurl @@ -1610,6 +1612,7 @@ def __init__(self, authurl=None, user=None, key=None, retries=5, self.auth_end_time = 0 self.retry_on_ratelimit = retry_on_ratelimit self.timeout = timeout + self.force_auth_retry = force_auth_retry def close(self): if (self.http_conn and isinstance(self.http_conn, tuple) @@ -1724,6 +1727,10 @@ def _retry(self, reset_func, func, *args, **kwargs): pass else: raise + + if self.force_auth_retry: + self.url = self.token = self.service_token = None + sleep(backoff) backoff = min(backoff * 2, self.max_backoff) if reset_func: diff --git a/swiftclient/service.py b/swiftclient/service.py index ed5e9e98..0679fec4 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -144,6 +144,7 @@ def _build_default_global_options(): "user": environ.get('ST_USER'), "key": environ.get('ST_KEY'), "retries": 5, + "force_auth_retry": False, "os_username": environ.get('OS_USERNAME'), "os_user_id": environ.get('OS_USER_ID'), "os_user_domain_name": environ.get('OS_USER_DOMAIN_NAME'), @@ -261,7 +262,8 @@ def get_conn(options): insecure=options['insecure'], cert=options['os_cert'], cert_key=options['os_key'], - ssl_compression=options['ssl_compression']) + ssl_compression=options['ssl_compression'], + force_auth_retry=options['force_auth_retry']) def mkdirs(path): diff --git a/swiftclient/shell.py b/swiftclient/shell.py index d02c709f..6010b5d4 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -1501,6 +1501,7 @@ def main(arguments=None): [--os-cert ] [--os-key ] [--no-ssl-compression] + [--force-auth-retry] [--help] [] Command-line interface to the OpenStack Swift API. @@ -1610,6 +1611,11 @@ def main(arguments=None): help='This option is deprecated and not used anymore. ' 'SSL compression should be disabled by default ' 'by the system SSL library.') + parser.add_argument('--force-auth-retry', + action='store_true', dest='force_auth_retry', + default=False, + help='Force a re-auth attempt on ' + 'any error other than 401 unauthorized') os_grp = parser.add_argument_group("OpenStack authentication options") os_grp.add_argument('--os-username', diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index 7b8628bb..52153dca 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -26,11 +26,13 @@ from hashlib import md5 from six import binary_type from six.moves.urllib.parse import urlparse +from requests.exceptions import RequestException from .utils import (MockHttpTest, fake_get_auth_keystone, StubResponse, FakeKeystone, _make_fake_import_keystone_client) from swiftclient.utils import EMPTY_ETAG +from swiftclient.exceptions import ClientException from swiftclient import client as c import swiftclient.utils import swiftclient @@ -1978,6 +1980,71 @@ def quick_sleep(*args): self.assertIn('Account HEAD failed', str(exc_context.exception)) self.assertEqual(conn.attempts, 1) + def test_retry_with_socket_error(self): + def quick_sleep(*args): + pass + c.sleep = quick_sleep + conn = c.Connection('http://www.test.com', 'asdf', 'asdf') + with mock.patch('swiftclient.client.http_connection') as \ + fake_http_connection, \ + mock.patch('swiftclient.client.get_auth_1_0') as mock_auth: + mock_auth.return_value = ('http://mock.com', 'mock_token') + fake_http_connection.side_effect = socket.error + self.assertRaises(socket.error, conn.head_account) + self.assertEqual(mock_auth.call_count, 1) + self.assertEqual(conn.attempts, conn.retries + 1) + + def test_retry_with_force_auth_retry_exceptions(self): + def quick_sleep(*args): + pass + + def do_test(exception): + c.sleep = quick_sleep + conn = c.Connection( + 'http://www.test.com', 'asdf', 'asdf', + force_auth_retry=True) + with mock.patch('swiftclient.client.http_connection') as \ + fake_http_connection, \ + mock.patch('swiftclient.client.get_auth_1_0') as mock_auth: + mock_auth.return_value = ('http://mock.com', 'mock_token') + fake_http_connection.side_effect = exception + self.assertRaises(exception, conn.head_account) + self.assertEqual(mock_auth.call_count, conn.retries + 1) + self.assertEqual(conn.attempts, conn.retries + 1) + + do_test(socket.error) + do_test(RequestException) + + def test_retry_with_force_auth_retry_client_exceptions(self): + def quick_sleep(*args): + pass + + def do_test(http_status, count): + + def mock_http_connection(*args, **kwargs): + raise ClientException('fake', http_status=http_status) + + c.sleep = quick_sleep + conn = c.Connection( + 'http://www.test.com', 'asdf', 'asdf', + force_auth_retry=True) + with mock.patch('swiftclient.client.http_connection') as \ + fake_http_connection, \ + mock.patch('swiftclient.client.get_auth_1_0') as mock_auth: + mock_auth.return_value = ('http://mock.com', 'mock_token') + fake_http_connection.side_effect = mock_http_connection + self.assertRaises(ClientException, conn.head_account) + self.assertEqual(mock_auth.call_count, count) + self.assertEqual(conn.attempts, count) + + # sanity, in case of 401, the auth will be called only twice because of + # retried_auth mechanism + do_test(401, 2) + # others will be tried until retry limits + do_test(408, 6) + do_test(500, 6) + do_test(503, 6) + def test_resp_read_on_server_error(self): conn = c.Connection('http://www.test.com', 'asdf', 'asdf', retries=0) From 071926d19b7305830920964434e993bbc1c41b18 Mon Sep 17 00:00:00 2001 From: Thiago da Silva Date: Sat, 17 Mar 2018 13:00:05 -0400 Subject: [PATCH 018/238] show option per line ading multiple options on the same line makes it easy to miss when quickly scanning the options. Change-Id: I8e324fca48cd05d9e381d5106135542274c2ff7f Signed-off-by: Thiago da Silva --- swiftclient/shell.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/swiftclient/shell.py b/swiftclient/shell.py index d02c709f..31b27eaf 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -1483,7 +1483,8 @@ def main(arguments=None): --os-identity-api-version ] [--user ] [--key ] [--retries ] - [--os-username ] [--os-password ] + [--os-username ] + [--os-password ] [--os-user-id ] [--os-user-domain-id ] [--os-user-domain-name ] @@ -1493,11 +1494,14 @@ def main(arguments=None): [--os-project-name ] [--os-project-domain-id ] [--os-project-domain-name ] - [--os-auth-url ] [--os-auth-token ] - [--os-storage-url ] [--os-region-name ] + [--os-auth-url ] + [--os-auth-token ] + [--os-storage-url ] + [--os-region-name ] [--os-service-type ] [--os-endpoint-type ] - [--os-cacert ] [--insecure] + [--os-cacert ] + [--insecure] [--os-cert ] [--os-key ] [--no-ssl-compression] From 5f23f9a70277314308432d73d2a2f2d78a844a97 Mon Sep 17 00:00:00 2001 From: Nguyen Hai Date: Fri, 16 Mar 2018 00:12:39 +0900 Subject: [PATCH 019/238] Remove py34 from envlist in tox.ini py35 is enough. Change-Id: Iebd7a6741dd60ed2fb11d1758bfec8e03e30a086 --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 541df654..5b4d3a43 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,py34,py35,pypy,pep8 +envlist = py27,py35,pypy,pep8 minversion = 2.0 skipsdist = True From 046e04a7a43e3712241b2391a3864d843c7651e8 Mon Sep 17 00:00:00 2001 From: Kota Tsuyuzaki Date: Thu, 12 Apr 2018 17:37:01 +0900 Subject: [PATCH 020/238] Remove trailing white space in tox.ini Change-Id: I706b69b7230390234ab255682478e8f69261cafe --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 541df654..2a27bfdf 100644 --- a/tox.ini +++ b/tox.ini @@ -29,7 +29,7 @@ commands = commands = {posargs} [testenv:cover] -commands = python setup.py testr --coverage +commands = python setup.py testr --coverage coverage report [testenv:func] From 5ed910cd32b0148b086ba4102a082aadb5e283bc Mon Sep 17 00:00:00 2001 From: Tovin Seven Date: Fri, 20 Apr 2018 17:25:34 +0700 Subject: [PATCH 021/238] Trivial: Update pypi url to new url Pypi url changed from [1] to [2] [1] https://pypi.python.org/pypi/ [2] https://pypi.org/project/ Change-Id: Ia406e9c8be6ba672b96e8584ef26f92348c8328b --- README.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 46c0b0ac..bc06f544 100644 --- a/README.rst +++ b/README.rst @@ -11,11 +11,11 @@ Python bindings to the OpenStack Object Storage API =================================================== .. image:: https://img.shields.io/pypi/v/python-swiftclient.svg - :target: https://pypi.python.org/pypi/python-swiftclient/ + :target: https://pypi.org/project/python-swiftclient/ :alt: Latest Version .. image:: https://img.shields.io/pypi/dm/python-swiftclient.svg - :target: https://pypi.python.org/pypi/python-swiftclient/ + :target: https://pypi.org/project/python-swiftclient/ :alt: Downloads This is a python client for the Swift API. There's a Python API (the @@ -42,7 +42,7 @@ __ https://github.com/openstack/swift * `Specs`_ * `How to Contribute`_ -.. _PyPI: https://pypi.python.org/pypi/python-swiftclient +.. _PyPI: https://pypi.org/project/python-swiftclient .. _Online Documentation: https://docs.openstack.org/python-swiftclient/latest/ .. _Launchpad project: https://launchpad.net/python-swiftclient .. _Blueprints: https://blueprints.launchpad.net/python-swiftclient From b5ed14f90a8214700a9be2139d9e7aba82baaa99 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 22 Mar 2018 18:04:06 -0400 Subject: [PATCH 022/238] add lower-constraints job Create a tox environment for running the unit tests against the lower bounds of the dependencies. Create a lower-constraints.txt to be used to enforce the lower bounds in those tests. Add openstack-tox-lower-constraints job to the zuul configuration. Update the dependencies needed to make the unit tests pass while constrained to the lower bounds. See http://lists.openstack.org/pipermail/openstack-dev/2018-March/128352.html for more details. Co-Authored-By: Nguyen Hai Change-Id: I2a8f465c8b08370517cbec857933b08fca94ca38 Depends-On: https://review.openstack.org/555034 Signed-off-by: Doug Hellmann --- .zuul.yaml | 7 +++++++ lower-constraints.txt | 46 +++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 9 ++++++--- setup.cfg | 2 +- setup.py | 14 +++++++++---- test-requirements.txt | 13 ++++++------ tox.ini | 7 +++++++ 7 files changed, 84 insertions(+), 14 deletions(-) create mode 100644 .zuul.yaml create mode 100644 lower-constraints.txt diff --git a/.zuul.yaml b/.zuul.yaml new file mode 100644 index 00000000..67a39c42 --- /dev/null +++ b/.zuul.yaml @@ -0,0 +1,7 @@ +- project: + check: + jobs: + - openstack-tox-lower-constraints + gate: + jobs: + - openstack-tox-lower-constraints diff --git a/lower-constraints.txt b/lower-constraints.txt new file mode 100644 index 00000000..6488b28f --- /dev/null +++ b/lower-constraints.txt @@ -0,0 +1,46 @@ +alabaster==0.7.10 +Babel==2.3.4 +certifi==2018.1.18 +chardet==3.0.4 +coverage==4.0 +docutils==0.11 +dulwich==0.15.0 +extras==1.0.0 +fixtures==3.0.0 +flake8==2.2.4 +futures==3.0.0 +hacking==0.10.0 +idna==2.6 +imagesize==0.7.1 +iso8601==0.1.8 +Jinja2==2.10 +keystoneauth1==3.4.0 +linecache2==1.0.0 +MarkupSafe==1.0 +mccabe==0.2.1 +mock==1.2.0 +netaddr==0.7.10 +openstackdocstheme==1.18.1 +oslo.config==1.2.0 +oslosphinx==4.7.0 +pbr==2.0.0 +pep8==1.5.7 +PrettyTable==0.7 +pyflakes==0.8.1 +Pygments==2.2.0 +python-keystoneclient==3.8.0 +python-mimeparse==1.6.0 +python-subunit==1.0.0 +pytz==2013.6 +PyYAML==3.12 +reno==2.5.0 +requests==2.14.2 +six==1.10.0 +snowballstemmer==1.2.1 +sphinx==1.6.2 +sphinxcontrib-websupport==1.0.1 +testrepository==0.0.18 +testtools==2.2.0 +traceback2==1.4.0 +unittest2==1.1.0 +urllib3==1.22 diff --git a/requirements.txt b/requirements.txt index 6d31e09b..6b527918 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,6 @@ -futures>=3.0;python_version=='2.7' or python_version=='2.6' # BSD -requests>=1.1 -six>=1.5.2 +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. +futures>=3.0.0;python_version=='2.7' or python_version=='2.6' # BSD +requests>=2.14.2 # Apache-2.0 +six>=1.10.0 # MIT diff --git a/setup.cfg b/setup.cfg index e4963dba..f20bbcaf 100644 --- a/setup.cfg +++ b/setup.cfg @@ -34,7 +34,7 @@ data_files = [extras] keystone = - python-keystoneclient>=0.7.0 + python-keystoneclient>=3.8.0 # Apache-2.0 [entry_points] console_scripts = diff --git a/setup.py b/setup.py index 16a18f6e..518f1d34 100644 --- a/setup.py +++ b/setup.py @@ -17,10 +17,16 @@ # THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT import setuptools, sys -if sys.version_info < (2, 7): - sys.exit('Sorry, Python < 2.7 is not supported for' - ' python-swiftclient>=3.0') +import setuptools + +# In python < 2.7.4, a lazy loading of package `pbr` will break +# setuptools if some other modules registered functions in `atexit`. +# solution from: http://bugs.python.org/issue15881#msg170215 +try: + import multiprocessing # noqa +except ImportError: + pass setuptools.setup( - setup_requires=['pbr'], + setup_requires=['pbr>=2.0.0'], pbr=True) diff --git a/test-requirements.txt b/test-requirements.txt index a9a0c7fb..634851e7 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,9 +1,10 @@ -hacking>=0.10.0,<0.11 +hacking<0.11,>=0.10.0 -coverage>=3.6 -mock>=1.2 +coverage!=4.4,>=4.0 # Apache-2.0 +keystoneauth1>=3.4.0 # Apache-2.0 +mock>=1.2.0 # BSD oslosphinx>=4.7.0 # Apache-2.0 -sphinx>=1.1.2,<1.2 +sphinx!=1.6.6,!=1.6.7,>=1.6.2 # BSD testrepository>=0.0.18 -reno>=1.8.0,!=2.3.1 # Apache-2.0 -openstackdocstheme>=1.16.0 # Apache-2.0 +reno>=2.5.0 # Apache-2.0 +openstackdocstheme>=1.18.1 # Apache-2.0 diff --git a/tox.ini b/tox.ini index 015913c5..660248b4 100644 --- a/tox.ini +++ b/tox.ini @@ -70,3 +70,10 @@ commands = bindep test [testenv:releasenotes] commands = sphinx-build -a -W -E -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html + +[testenv:lower-constraints] +basepython = python3 +deps = + -c{toxinidir}/lower-constraints.txt + -r{toxinidir}/test-requirements.txt + .[keystone] From 7a13754eebb15a3c7fa507ad1d58e3c4460e302c Mon Sep 17 00:00:00 2001 From: Pete Zaitcev Date: Fri, 4 May 2018 13:31:03 -0500 Subject: [PATCH 023/238] Use a valid default for auth_version The valid set of values for auth_version does not include values starting with the 'v'. In this particular function, the auth_version variable is only used for comparisons with v3. So, the code worked correctly. However, let's clean this up in order to reduce review confusion and defuse possible future landmine in case of code changes. Change-Id: I671016d7992a1922b786b4eb8876b3fbb2532e15 --- swiftclient/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index ab0fde80..e92e3ca7 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -565,7 +565,7 @@ def get_auth_keystone(auth_url, user, key, os_options, **kwargs): # Legacy default if not set if auth_version is None: - auth_version = 'v2.0' + auth_version = '2' ksclient, exceptions = _import_keystone_client(auth_version) try: From 1971ef880ff225379d4a91f00f89f323a1605eeb Mon Sep 17 00:00:00 2001 From: Erik Olof Gunnar Andersson Date: Wed, 16 May 2018 16:03:57 -0700 Subject: [PATCH 024/238] Make swiftclient respect region_name when using sessions Change-Id: I94aca6f1120c34616562be7345f0e5aa51a69499 --- swiftclient/client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index ab0fde80..59617446 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -644,8 +644,10 @@ def get_auth(auth_url, user, key, **kwargs): if session: service_type = os_options.get('service_type', 'object-store') interface = os_options.get('endpoint_type', 'public') + region_name = os_options.get('region_name') storage_url = session.get_endpoint(service_type=service_type, - interface=interface) + interface=interface, + region_name=region_name) token = session.get_token() elif auth_version in AUTH_VERSIONS_V1: storage_url, token = get_auth_1_0(auth_url, From 78b839007294b7c9e7fd0402801775283a4045bc Mon Sep 17 00:00:00 2001 From: Nguyen Hai Date: Fri, 16 Mar 2018 00:28:01 +0900 Subject: [PATCH 025/238] Switch from oslosphinx to openstackdocstheme openstackdocstheme is a theme and extension support for Sphinx documentation that is published to docs.openstack.org and developer.openstack.org. Change-Id: I37d1d50fb88b35e72b017d5dfbf148c35ac7e323 --- doc/source/conf.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 3505f13c..f56b643f 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -31,8 +31,11 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.todo', - 'sphinx.ext.coverage', 'oslosphinx'] +extensions = ['sphinx.ext.autodoc', + 'sphinx.ext.doctest', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'openstackdocstheme'] autoclass_content = 'both' autodoc_default_flags = ['members', 'undoc-members', 'show-inheritance'] @@ -104,7 +107,7 @@ # The theme to use for HTML and HTML Help pages. Major themes that come with # Sphinx are currently 'default' and 'sphinxdoc'. -#html_theme = 'nature' +html_theme = 'openstackdocs' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the From 2312182241d36c716e624a23acd51f2b0252e4aa Mon Sep 17 00:00:00 2001 From: Chen Date: Thu, 7 Jun 2018 22:38:41 +0800 Subject: [PATCH 026/238] Remove PyPI downloads According to official site, https://packaging.python.org/guides/analyzing-pypi-package-downloads/ PyPI package download statistics is no longer maintained and thus should be removed. Change-Id: I05e7d48d191bdaaf029f0ad1373a9c7c8b22e81e --- README.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.rst b/README.rst index bc06f544..f41e5b79 100644 --- a/README.rst +++ b/README.rst @@ -14,10 +14,6 @@ Python bindings to the OpenStack Object Storage API :target: https://pypi.org/project/python-swiftclient/ :alt: Latest Version -.. image:: https://img.shields.io/pypi/dm/python-swiftclient.svg - :target: https://pypi.org/project/python-swiftclient/ - :alt: Downloads - This is a python client for the Swift API. There's a Python API (the ``swiftclient`` module), and a command-line script (``swift``). From 33ad9fd4cc0ade9f0800a2815ee0ef514ae8f264 Mon Sep 17 00:00:00 2001 From: Alistair Coles Date: Mon, 11 Jun 2018 13:19:05 +0100 Subject: [PATCH 027/238] Add option for user to enter password Add the --prompt option for the CLI which will cause the user to be prompted to enter a password. Any password otherwise specified by --key, --os-password or an environment variable will be ignored. The swift client will exit with a warning if the password cannot be entered without its value being echoed. Closes-Bug: #1357562 Change-Id: I513647eed460007617f129691069c6fb1bfe62d7 --- doc/source/cli/index.rst | 4 +++ swiftclient/shell.py | 37 ++++++++++++++++++++++++++ tests/unit/test_shell.py | 57 +++++++++++++++++++++++++++++++++++++--- 3 files changed, 95 insertions(+), 3 deletions(-) diff --git a/doc/source/cli/index.rst b/doc/source/cli/index.rst index 4aa3bfcd..88fafa1d 100644 --- a/doc/source/cli/index.rst +++ b/doc/source/cli/index.rst @@ -139,6 +139,10 @@ swift optional arguments compression should be disabled by default by the system SSL library. +``--prompt`` + Prompt user to enter a password which overrides any password supplied via + ``--key``, ``--os-password`` or environment variables. + Authentication ~~~~~~~~~~~~~~ diff --git a/swiftclient/shell.py b/swiftclient/shell.py index c15d7cfc..e91a16ff 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -17,11 +17,13 @@ from __future__ import print_function, unicode_literals import argparse +import getpass import io import json import logging import signal import socket +import warnings from os import environ, walk, _exit as os_exit from os.path import isfile, isdir, join @@ -1410,6 +1412,30 @@ def _get_default_metavar_for_positional(self, action): return action.dest +def prompt_for_password(): + """ + Prompt the user for a password. + + :raise SystemExit: if a password cannot be entered without it being echoed + to the terminal. + :return: the entered password. + """ + with warnings.catch_warnings(): + warnings.filterwarnings('error', category=getpass.GetPassWarning, + append=True) + try: + # temporarily set signal handling back to default to avoid user + # Ctrl-c leaving terminal in weird state + signal.signal(signal.SIGINT, signal.SIG_DFL) + return getpass.getpass() + except EOFError: + return None + except getpass.GetPassWarning: + exit('Input stream incompatible with --prompt option') + finally: + signal.signal(signal.SIGINT, immediate_exit) + + def parse_args(parser, args, enforce_requires=True): options, args = parser.parse_known_args(args or ['-h']) options = vars(options) @@ -1435,6 +1461,10 @@ def parse_args(parser, args, enforce_requires=True): if args and args[0] == 'tempurl': return options, args + # do this before process_options sets default auth version + if enforce_requires and options['prompt']: + options['key'] = options['os_password'] = prompt_for_password() + # Massage auth version; build out os_options subdict process_options(options) @@ -1506,6 +1536,7 @@ def main(arguments=None): [--os-key ] [--no-ssl-compression] [--force-auth-retry] + [--prompt] [--help] [] Command-line interface to the OpenStack Swift API. @@ -1620,6 +1651,12 @@ def main(arguments=None): default=False, help='Force a re-auth attempt on ' 'any error other than 401 unauthorized') + parser.add_argument('--prompt', + action='store_true', dest='prompt', + default=False, + help='Prompt user to enter a password which overrides ' + 'any password supplied via --key, --os-password ' + 'or environment variables.') os_grp = parser.add_argument_group("OpenStack authentication options") os_grp.add_argument('--os-username', diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index 110fb01f..3db48a48 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -13,8 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. from __future__ import unicode_literals + from genericpath import getmtime +import getpass import hashlib import json import logging @@ -2283,17 +2285,66 @@ def test_insufficient_args_v3(self): os_opts = {"password": "secret", "auth_url": "http://example.com:5000/v3"} args = _make_args("stat", opts, os_opts) - self.assertRaises(SystemExit, swiftclient.shell.main, args) + with self.assertRaises(SystemExit) as cm: + swiftclient.shell.main(args) + self.assertIn( + 'Auth version 3 requires either OS_USERNAME or OS_USER_ID', + str(cm.exception)) os_opts = {"username": "user", "auth_url": "http://example.com:5000/v3"} args = _make_args("stat", opts, os_opts) - self.assertRaises(SystemExit, swiftclient.shell.main, args) + with self.assertRaises(SystemExit) as cm: + swiftclient.shell.main(args) + self.assertIn('Auth version 3 requires OS_PASSWORD', str(cm.exception)) os_opts = {"username": "user", "password": "secret"} args = _make_args("stat", opts, os_opts) - self.assertRaises(SystemExit, swiftclient.shell.main, args) + with self.assertRaises(SystemExit) as cm: + swiftclient.shell.main(args) + self.assertIn('Auth version 3 requires OS_AUTH_URL', str(cm.exception)) + + def test_password_prompt(self): + def do_test(opts, os_opts, auth_version): + args = _make_args("stat", opts, os_opts) + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch('swiftclient.shell.st_stat', fake_command): + with mock.patch('getpass.getpass', + return_value='input_pwd') as mock_getpass: + swiftclient.shell.main(args) + mock_getpass.assert_called_once_with() + self.assertEqual('input_pwd', result[0]['key']) + self.assertEqual('input_pwd', result[0]['os_password']) + + # ctrl-D + with self.assertRaises(SystemExit) as cm: + with mock.patch('swiftclient.shell.st_stat', fake_command): + with mock.patch('getpass.getpass', + side_effect=EOFError) as mock_getpass: + swiftclient.shell.main(args) + mock_getpass.assert_called_once_with() + self.assertIn( + 'Auth version %s requires' % auth_version, str(cm.exception)) + + # force getpass to think it needs to use raw input + with self.assertRaises(SystemExit) as cm: + with mock.patch('getpass.getpass', getpass.fallback_getpass): + swiftclient.shell.main(args) + self.assertIn( + 'Input stream incompatible', str(cm.exception)) + + opts = {"prompt": None, "user": "bob", "key": "secret", + "auth": "http://example.com:8080/auth/v1.0"} + do_test(opts, {}, '1.0') + os_opts = {"username": "user", + "password": "secret", + "auth_url": "http://example.com:5000/v3"} + opts = {"auth_version": "2.0", "prompt": None} + do_test(opts, os_opts, '2.0') + opts = {"auth_version": "3", "prompt": None} + do_test(opts, os_opts, '3') def test_no_tenant_name_or_id_v2(self): os_opts = {"password": "secret", From 11908250549fed549169519d411394296fb4a57b Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Tue, 13 Jun 2017 11:14:52 -0700 Subject: [PATCH 028/238] Make OS_AUTH_URL work in DevStack by default An earlier change added support for versionless authurls, but the huristic to detect them didn't work for some configurations I've encountered. Now we use a little bit tighter pattern matching and support auth_url values with more than one path component. Change-Id: I5a99c7b4e957ee7c8a5b5470477db49ab2ddba4b Related-Change-Id: If7ecb67776cb77828f93ad8278cc5040015216b7 --- swiftclient/client.py | 7 ++++- tests/unit/test_swiftclient.py | 47 ++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index 45188083..cc2a124b 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -17,6 +17,7 @@ OpenStack Swift client library used internally """ import socket +import re import requests import logging import warnings @@ -38,6 +39,7 @@ # Default is 100, increase to 256 http_client._MAXHEADERS = 256 +VERSIONFUL_AUTH_PATH = re.compile('v[2-3](?:\.0)?$') AUTH_VERSIONS_V1 = ('1.0', '1', 1) AUTH_VERSIONS_V2 = ('2.0', '2', 2) AUTH_VERSIONS_V3 = ('3.0', '3', 3) @@ -555,7 +557,10 @@ def get_auth_keystone(auth_url, user, key, os_options, **kwargs): # Add the version suffix in case of versionless Keystone endpoints. If # auth_version is also unset it is likely that it is v3 - if len(urlparse(auth_url).path) <= 1: + if not VERSIONFUL_AUTH_PATH.match( + urlparse(auth_url).path.rstrip('/').rsplit('/', 1)[-1]): + # Normalize auth_url to end in a slash because urljoin + auth_url = auth_url.rstrip('/') + '/' if auth_version and auth_version in AUTH_VERSIONS_V2: auth_url = urljoin(auth_url, "v2.0") else: diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index 52153dca..009a026d 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -597,6 +597,53 @@ def test_get_auth_keystone_versionless_auth_version_set(self): self.assertEqual('http://auth_url/v2.0', fake_ks.calls[0].get('auth_url')) + def test_get_auth_keystone_versionful(self): + fake_ks = FakeKeystone(endpoint='http://some_url', token='secret') + + with mock.patch('swiftclient.client._import_keystone_client', + _make_fake_import_keystone_client(fake_ks)): + c.get_auth_keystone('http://auth_url/v3', 'user', 'key', + {}, auth_version='3') + self.assertEqual(1, len(fake_ks.calls)) + self.assertEqual('http://auth_url/v3', + fake_ks.calls[0].get('auth_url')) + + def test_get_auth_keystone_devstack_versionful(self): + fake_ks = FakeKeystone( + endpoint='http://storage.example.com/v1/AUTH_user', token='secret') + with mock.patch('swiftclient.client._import_keystone_client', + _make_fake_import_keystone_client(fake_ks)): + c.get_auth_keystone('https://192.168.8.8/identity/v3', + 'user', 'key', {}, auth_version='3') + self.assertEqual(1, len(fake_ks.calls)) + self.assertEqual('https://192.168.8.8/identity/v3', + fake_ks.calls[0].get('auth_url')) + + def test_get_auth_keystone_devstack_versionless(self): + fake_ks = FakeKeystone( + endpoint='http://storage.example.com/v1/AUTH_user', token='secret') + with mock.patch('swiftclient.client._import_keystone_client', + _make_fake_import_keystone_client(fake_ks)): + c.get_auth_keystone('https://192.168.8.8/identity', + 'user', 'key', {}, auth_version='3') + self.assertEqual(1, len(fake_ks.calls)) + self.assertEqual('https://192.168.8.8/identity/v3', + fake_ks.calls[0].get('auth_url')) + + def test_auth_keystone_url_some_junk_nonsense(self): + fake_ks = FakeKeystone( + endpoint='http://storage.example.com/v1/AUTH_user', + token='secret') + with mock.patch('swiftclient.client._import_keystone_client', + _make_fake_import_keystone_client(fake_ks)): + c.get_auth_keystone('http://blah.example.com/v2moo', + 'user', 'key', {}, auth_version='3') + self.assertEqual(1, len(fake_ks.calls)) + # v2 looks sorta version-y, but it's not an exact match, so this is + # probably about just as bad as anything else we might guess at + self.assertEqual('http://blah.example.com/v2moo/v3', + fake_ks.calls[0].get('auth_url')) + def test_auth_with_session(self): mock_session = mock.MagicMock() mock_session.get_endpoint.return_value = 'http://storagehost/v1/acct' From 85bb28eab4fab97e834ebfd2ef29bb3f4fc9bf5f Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Fri, 22 Jun 2018 16:23:57 -0700 Subject: [PATCH 029/238] Remove some pointless code Change-Id: I3163834c330c5ea44c1096e83127588c88f0d761 --- tests/functional/test_swiftclient.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/functional/test_swiftclient.py b/tests/functional/test_swiftclient.py index d60ae063..0380d961 100644 --- a/tests/functional/test_swiftclient.py +++ b/tests/functional/test_swiftclient.py @@ -485,9 +485,6 @@ def _get_connection(self): self.auth_url, username, password, auth_version=self.auth_version, os_options=os_options) - def setUp(self): - super(TestUsingKeystone, self).setUp() - class TestUsingKeystoneV3(TestFunctional): """ @@ -514,6 +511,3 @@ def _get_connection(self): return swiftclient.Connection(self.auth_url, username, password, auth_version=self.auth_version, os_options=os_options) - - def setUp(self): - super(TestUsingKeystoneV3, self).setUp() From c1e896c8612dfcf91235d3a67d0c17595777764c Mon Sep 17 00:00:00 2001 From: "wu.chunyang" Date: Thu, 28 Jun 2018 14:07:09 +0800 Subject: [PATCH 030/238] Add release note link in README Change-Id: Ie282f4a731e333b5d21490a8fb59e1c7e8640821 --- README.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index f41e5b79..b105094a 100644 --- a/README.rst +++ b/README.rst @@ -37,6 +37,7 @@ __ https://github.com/openstack/swift * `Source`_ * `Specs`_ * `How to Contribute`_ +* `Release Notes`_ .. _PyPI: https://pypi.org/project/python-swiftclient .. _Online Documentation: https://docs.openstack.org/python-swiftclient/latest/ @@ -46,7 +47,7 @@ __ https://github.com/openstack/swift .. _Source: https://git.openstack.org/cgit/openstack/python-swiftclient .. _How to Contribute: https://docs.openstack.org/infra/manual/developers.html .. _Specs: https://specs.openstack.org/openstack/swift-specs/ - +.. _Release Notes: https://docs.openstack.org/releasenotes/python-swiftclient .. contents:: Contents: :local: From 47fb18c41b4851ba6071f0215e96e222b8ccef29 Mon Sep 17 00:00:00 2001 From: mmcardle Date: Tue, 10 Jul 2018 14:45:32 +0100 Subject: [PATCH 031/238] Add ability to generate a temporary URL with an IP range restriction Change-Id: I4734599886e4f4a563162390d0ff3bb1ef639db4 --- swiftclient/shell.py | 11 ++++++++- swiftclient/utils.py | 25 ++++++++++++++++++--- tests/unit/test_shell.py | 30 ++++++++++++++++++++----- tests/unit/test_utils.py | 48 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 105 insertions(+), 9 deletions(-) diff --git a/swiftclient/shell.py b/swiftclient/shell.py index e91a16ff..74a47b70 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -1325,6 +1325,8 @@ def st_auth(parser, args, thread_manager): generated. --iso8601 If present, the generated temporary URL will contain an ISO 8601 UTC timestamp instead of a Unix timestamp. + --ip-range If present, the temporary URL will be restricted to the + given ip or ip range. '''.strip('\n') @@ -1348,6 +1350,12 @@ def st_tempurl(parser, args, thread_manager): help=("If present, the temporary URL will contain an ISO 8601 UTC " "timestamp instead of a Unix timestamp."), ) + parser.add_argument( + '--ip-range', action='store', + default=None, + help=("If present, the temporary URL will be restricted to the " + "given ip or ip range."), + ) (options, args) = parse_args(parser, args) args = args[1:] @@ -1367,7 +1375,8 @@ def st_tempurl(parser, args, thread_manager): path = generate_temp_url(parsed.path, timestamp, key, method, absolute=options['absolute_expiry'], iso8601=options['iso8601'], - prefix=options['prefix_based']) + prefix=options['prefix_based'], + ip_range=options['ip_range']) except ValueError as err: thread_manager.error(err) return diff --git a/swiftclient/utils.py b/swiftclient/utils.py index 8afcde97..5c17c613 100644 --- a/swiftclient/utils.py +++ b/swiftclient/utils.py @@ -69,7 +69,7 @@ def prt_bytes(num_bytes, human_flag): def generate_temp_url(path, seconds, key, method, absolute=False, - prefix=False, iso8601=False): + prefix=False, iso8601=False, ip_range=None): """Generates a temporary URL that gives unauthenticated access to the Swift object. @@ -92,6 +92,8 @@ def generate_temp_url(path, seconds, key, method, absolute=False, :param prefix: if True then a prefix-based temporary URL will be generated. :param iso8601: if True, a URL containing an ISO 8601 UTC timestamp instead of a UNIX timestamp will be created. + :param ip_range: if a valid ip range, restricts the temporary URL to the + range of ips. :raises ValueError: if timestamp or path is not in valid format. :return: the path portion of a temporary URL """ @@ -155,8 +157,21 @@ def generate_temp_url(path, seconds, key, method, absolute=False, expiration = int(time.time() + timestamp) else: expiration = timestamp - hmac_body = u'\n'.join([method.upper(), str(expiration), - ('prefix:' if prefix else '') + path_for_body]) + + hmac_parts = [method.upper(), str(expiration), + ('prefix:' if prefix else '') + path_for_body] + + if ip_range: + if isinstance(ip_range, six.binary_type): + try: + ip_range = ip_range.decode('utf-8') + except UnicodeDecodeError: + raise ValueError( + 'ip_range must be representable as UTF-8' + ) + hmac_parts.insert(0, "ip=%s" % ip_range) + + hmac_body = u'\n'.join(hmac_parts) # Encode to UTF-8 for py3 compatibility if not isinstance(key, six.binary_type): @@ -169,6 +184,10 @@ def generate_temp_url(path, seconds, key, method, absolute=False, temp_url = u'{path}?temp_url_sig={sig}&temp_url_expires={exp}'.format( path=path_for_body, sig=sig, exp=expiration) + + if ip_range: + temp_url += u'&temp_url_ip_range={}'.format(ip_range) + if prefix: temp_url += u'&temp_url_prefix={}'.format(parts[4]) # Have return type match path from caller diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index 3db48a48..8c995e5d 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -1667,7 +1667,7 @@ def test_temp_url(self, temp_url): swiftclient.shell.main(argv) temp_url.assert_called_with( '/v1/AUTH_account/c/o', "60", 'secret_key', 'GET', absolute=False, - iso8601=False, prefix=False) + iso8601=False, prefix=False, ip_range=None) @mock.patch('swiftclient.shell.generate_temp_url', return_value='') def test_temp_url_prefix_based(self, temp_url): @@ -1676,7 +1676,7 @@ def test_temp_url_prefix_based(self, temp_url): swiftclient.shell.main(argv) temp_url.assert_called_with( '/v1/AUTH_account/c/', "60", 'secret_key', 'GET', absolute=False, - iso8601=False, prefix=True) + iso8601=False, prefix=True, ip_range=None) @mock.patch('swiftclient.shell.generate_temp_url', return_value='') def test_temp_url_iso8601_in(self, temp_url): @@ -1688,7 +1688,7 @@ def test_temp_url_iso8601_in(self, temp_url): swiftclient.shell.main(argv) temp_url.assert_called_with( '/v1/AUTH_account/c/', d, 'secret_key', 'GET', absolute=False, - iso8601=False, prefix=False) + iso8601=False, prefix=False, ip_range=None) @mock.patch('swiftclient.shell.generate_temp_url', return_value='') def test_temp_url_iso8601_out(self, temp_url): @@ -1697,7 +1697,7 @@ def test_temp_url_iso8601_out(self, temp_url): swiftclient.shell.main(argv) temp_url.assert_called_with( '/v1/AUTH_account/c/', "60", 'secret_key', 'GET', absolute=False, - iso8601=True, prefix=False) + iso8601=True, prefix=False, ip_range=None) @mock.patch('swiftclient.shell.generate_temp_url', return_value='') def test_absolute_expiry_temp_url(self, temp_url): @@ -1706,7 +1706,16 @@ def test_absolute_expiry_temp_url(self, temp_url): swiftclient.shell.main(argv) temp_url.assert_called_with( '/v1/AUTH_account/c/o', "60", 'secret_key', 'GET', absolute=True, - iso8601=False, prefix=False) + iso8601=False, prefix=False, ip_range=None) + + @mock.patch('swiftclient.shell.generate_temp_url', return_value='') + def test_temp_url_with_ip_range(self, temp_url): + argv = ["", "tempurl", "GET", "60", "/v1/AUTH_account/c/o", + "secret_key", "--ip-range", "1.2.3.4"] + swiftclient.shell.main(argv) + temp_url.assert_called_with( + '/v1/AUTH_account/c/o', "60", 'secret_key', 'GET', absolute=False, + iso8601=False, prefix=False, ip_range='1.2.3.4') def test_temp_url_output(self): argv = ["", "tempurl", "GET", "60", "/v1/a/c/o", @@ -1769,6 +1778,17 @@ def test_temp_url_output(self): swiftclient.shell.main(argv) self.assertEqual(expected, output.out) + argv = ["", "tempurl", "GET", "60", "/v1/a/c/o", + "secret_key", "--absolute", "--ip-range", "1.2.3.4"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + sig = "6a6ec8efa4be53904ecba8d055d841e24a937c98" + expected = ( + "/v1/a/c/o?temp_url_sig=%s&temp_url_expires=60" + "&temp_url_ip_range=1.2.3.4\n" % sig + ) + self.assertEqual(expected, output.out) + def test_temp_url_error_output(self): expected = 'path must be full path to an object e.g. /v1/a/c/o\n' for bad_path in ('/v1/a/c', 'v1/a/c/o', '/v1/a/c/', '/v1/a//o', diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index adead005..e54b90c7 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -151,6 +151,54 @@ def test_generate_temp_url(self, time_mock, hmac_mock): ]) self.assertIsInstance(url, type(self.url)) + @mock.patch('hmac.HMAC') + @mock.patch('time.time', return_value=1400000000) + def test_generate_temp_url_ip_range(self, time_mock, hmac_mock): + hmac_mock().hexdigest.return_value = 'temp_url_signature' + ip_ranges = [ + '1.2.3.4', '1.2.3.4/24', '2001:db8::', + b'1.2.3.4', b'1.2.3.4/24', b'2001:db8::', + ] + path = '/v1/AUTH_account/c/o/' + expected_url = path + ('?temp_url_sig=temp_url_signature' + '&temp_url_expires=1400003600' + '&temp_url_ip_range=') + for ip_range in ip_ranges: + hmac_mock.reset_mock() + url = u.generate_temp_url(path, self.seconds, + self.key, self.method, + ip_range=ip_range) + key = self.key + if not isinstance(key, six.binary_type): + key = key.encode('utf-8') + + if isinstance(ip_range, six.binary_type): + ip_range_expected_url = ( + expected_url + ip_range.decode('utf-8') + ) + expected_body = '\n'.join([ + 'ip=' + ip_range.decode('utf-8'), + self.method, + '1400003600', + path, + ]).encode('utf-8') + else: + ip_range_expected_url = expected_url + ip_range + expected_body = '\n'.join([ + 'ip=' + ip_range, + self.method, + '1400003600', + path, + ]).encode('utf-8') + + self.assertEqual(url, ip_range_expected_url) + + self.assertEqual(hmac_mock.mock_calls, [ + mock.call(key, expected_body, sha1), + mock.call().hexdigest(), + ]) + self.assertIsInstance(url, type(path)) + @mock.patch('hmac.HMAC') def test_generate_temp_url_iso8601_argument(self, hmac_mock): hmac_mock().hexdigest.return_value = 'temp_url_signature' From da362a653e9c70cb6ae17a7c3764887b4fd3fcf2 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Wed, 16 May 2018 17:33:40 +0000 Subject: [PATCH 032/238] Back out some version bumps I'm giving up on trying to back out all of the test-requirements up-revs, but let's try to stay compatibile with old requests/six. As part of that, only disable some requests warnings on new-enough requests. Note that we should now be compatible with distro packages back to Ubuntu 16.04 and CentOS 6. Our six is still too new for Trusty, but hey, there's less than a year left on that anyway, right? Change-Id: Iccb23638393616f9ec3da660dd5e39ea4ea94220 Related-Change: I2a8f465c8b08370517cbec857933b08fca94ca38 --- lower-constraints.txt | 6 +++--- requirements.txt | 7 ++----- setup.cfg | 2 +- setup.py | 14 ++++---------- swiftclient/shell.py | 11 ++++++++--- tests/unit/test_shell.py | 35 ++++++++++++++++++++++++++--------- 6 files changed, 44 insertions(+), 31 deletions(-) diff --git a/lower-constraints.txt b/lower-constraints.txt index 6488b28f..9aae7927 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -28,14 +28,14 @@ pep8==1.5.7 PrettyTable==0.7 pyflakes==0.8.1 Pygments==2.2.0 -python-keystoneclient==3.8.0 +python-keystoneclient==0.7.0 python-mimeparse==1.6.0 python-subunit==1.0.0 pytz==2013.6 PyYAML==3.12 reno==2.5.0 -requests==2.14.2 -six==1.10.0 +requests==1.1.0 +six==1.9.0 snowballstemmer==1.2.1 sphinx==1.6.2 sphinxcontrib-websupport==1.0.1 diff --git a/requirements.txt b/requirements.txt index 6b527918..1c2ce33d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,3 @@ -# The order of packages is significant, because pip processes them in the order -# of appearance. Changing the order has an impact on the overall integration -# process, which may cause wedges in the gate later. futures>=3.0.0;python_version=='2.7' or python_version=='2.6' # BSD -requests>=2.14.2 # Apache-2.0 -six>=1.10.0 # MIT +requests>=1.1.0 +six>=1.9.0 diff --git a/setup.cfg b/setup.cfg index f20bbcaf..e4963dba 100644 --- a/setup.cfg +++ b/setup.cfg @@ -34,7 +34,7 @@ data_files = [extras] keystone = - python-keystoneclient>=3.8.0 # Apache-2.0 + python-keystoneclient>=0.7.0 [entry_points] console_scripts = diff --git a/setup.py b/setup.py index 518f1d34..16a18f6e 100644 --- a/setup.py +++ b/setup.py @@ -17,16 +17,10 @@ # THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT import setuptools, sys -import setuptools - -# In python < 2.7.4, a lazy loading of package `pbr` will break -# setuptools if some other modules registered functions in `atexit`. -# solution from: http://bugs.python.org/issue15881#msg170215 -try: - import multiprocessing # noqa -except ImportError: - pass +if sys.version_info < (2, 7): + sys.exit('Sorry, Python < 2.7 is not supported for' + ' python-swiftclient>=3.0') setuptools.setup( - setup_requires=['pbr>=2.0.0'], + setup_requires=['pbr'], pbr=True) diff --git a/swiftclient/shell.py b/swiftclient/shell.py index e91a16ff..3f66d5f6 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -1819,9 +1819,14 @@ def main(arguments=None): parser.usage = globals()['st_%s_help' % args[0]] if options['insecure']: import requests - from requests.packages.urllib3.exceptions import \ - InsecureRequestWarning - requests.packages.urllib3.disable_warnings(InsecureRequestWarning) + try: + from requests.packages.urllib3.exceptions import \ + InsecureRequestWarning + except ImportError: + pass + else: + requests.packages.urllib3.disable_warnings( + InsecureRequestWarning) try: globals()['st_%s' % args[0]](parser, argv[1:], output) except ClientException as err: diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index 3db48a48..54238591 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -14,8 +14,8 @@ # limitations under the License. from __future__ import unicode_literals +import contextlib from genericpath import getmtime - import getpass import hashlib import json @@ -27,7 +27,6 @@ import textwrap from time import localtime, mktime, strftime, strptime -from requests.packages.urllib3.exceptions import InsecureRequestWarning import six import sys @@ -44,6 +43,10 @@ EMPTY_ETAG, EXPIRES_ISO8601_FORMAT, SHORT_EXPIRES_ISO8601_FORMAT, TIME_ERRMSG) +try: + from requests.packages.urllib3.exceptions import InsecureRequestWarning +except ImportError: + InsecureRequestWarning = None if six.PY2: BUILTIN_OPEN = '__builtin__.open' @@ -114,6 +117,20 @@ def _make_cmd(cmd, opts, os_opts, use_env=False, flags=None, cmd_args=None): return args, env +@contextlib.contextmanager +def patch_disable_warnings(): + if InsecureRequestWarning is None: + # If InsecureRequestWarning isn't available, disbale_warnings won't + # be either; they both came in with + # https://github.com/requests/requests/commit/811ee4e and left again + # in https://github.com/requests/requests/commit/8e17600 + yield None + else: + with mock.patch('requests.packages.urllib3.disable_warnings') \ + as patched: + yield patched + + @mock.patch.dict(os.environ, mocked_os_environ) class TestShell(unittest.TestCase): def setUp(self): @@ -2509,8 +2526,7 @@ def _test_options_passed_to_keystone(self, cmd, opts, os_opts, _make_fake_import_keystone_client(fake_ks)), \ mock.patch('swiftclient.client.http_connection', fake_conn), \ mock.patch.dict(os.environ, env, clear=True), \ - mock.patch('requests.packages.urllib3.disable_warnings') as \ - mock_disable_warnings: + patch_disable_warnings() as mock_disable_warnings: try: swiftclient.shell.main(args) except SystemExit as e: @@ -2518,11 +2534,12 @@ def _test_options_passed_to_keystone(self, cmd, opts, os_opts, except SwiftError as err: self.fail('Unexpected SwiftError: %s' % err) - if 'insecure' in flags: - self.assertEqual([mock.call(InsecureRequestWarning)], - mock_disable_warnings.mock_calls) - else: - self.assertEqual([], mock_disable_warnings.mock_calls) + if InsecureRequestWarning is not None: + if 'insecure' in flags: + self.assertEqual([mock.call(InsecureRequestWarning)], + mock_disable_warnings.mock_calls) + else: + self.assertEqual([], mock_disable_warnings.mock_calls) if no_auth: # check that keystone client was not used and terminate tests From 45ed21c6c433e2f5979df2820424bf5b44c478db Mon Sep 17 00:00:00 2001 From: Matthew Oliver Date: Fri, 29 Jun 2018 11:07:00 +1000 Subject: [PATCH 033/238] Add bash_completion to swiftclient This patch basically follows the bash completion model that other OpenStack clients use. It creates a new command to swiftclient called `bash_completion`. The `bash_completion` command by default will print all base flags and exsiting commands. If you pass it a command, it'll print out all base flags and any flags that command accepts. So as you type out your swift command and auto-complete, only the current available flags are offered to you. This is used by the swift.bash_completion script to allow swift commands to be bash completed. To make it work, place the swift.bash_completion file into /etc/bash_completion.d and source it: cp tools/swift.bash_completion /etc/bash_completion.d/swift source /etc/bash_completion.d/swift Because swiftclient itself is creating this flag/command output it should automatically add anything we add to the swiftclient CLI. Change-Id: I5609a19018269762b4640403daae5827bb9ad724 --- swiftclient/shell.py | 312 ++++++++++++++++++++++++------------ tools/swift.bash_completion | 32 ++++ 2 files changed, 245 insertions(+), 99 deletions(-) create mode 100644 tools/swift.bash_completion diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 74a47b70..ff5b2be5 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -51,7 +51,7 @@ BASENAME = 'swift' commands = ('delete', 'download', 'list', 'post', 'copy', 'stat', 'upload', - 'capabilities', 'info', 'tempurl', 'auth') + 'capabilities', 'info', 'tempurl', 'auth', 'bash_completion') def immediate_exit(signum, frame): @@ -90,7 +90,7 @@ def immediate_exit(signum, frame): '''.strip("\n") -def st_delete(parser, args, output_manager): +def st_delete(parser, args, output_manager, return_parser=False): parser.add_argument( '-a', '--all', action='store_true', dest='yes_all', default=False, help='Delete all containers and objects.') @@ -114,6 +114,11 @@ def st_delete(parser, args, output_manager): '--container-threads', type=int, default=10, help='Number of threads to use for deleting containers. ' 'Its value must be a positive integer. Default is 10.') + + # We return the parser to build up the bash_completion + if return_parser: + return parser + (options, args) = parse_args(parser, args) args = args[1:] if (not args and not options['yes_all']) or (args and options['yes_all']): @@ -281,7 +286,7 @@ def st_delete(parser, args, output_manager): '''.strip("\n") -def st_download(parser, args, output_manager): +def st_download(parser, args, output_manager, return_parser=False): parser.add_argument( '-a', '--all', action='store_true', dest='yes_all', default=False, help='Indicates that you really want to download ' @@ -344,6 +349,11 @@ def st_download(parser, args, output_manager): 'to store the access and modified timestamp for the downloaded file. ' 'With this option, the header is ignored and the timestamps are ' 'created freshly.') + + # We return the parser to build up the bash_completion + if return_parser: + return parser + (options, args) = parse_args(parser, args) args = args[1:] if options['out_file'] == '-': @@ -494,7 +504,7 @@ def st_download(parser, args, output_manager): '''.strip('\n') -def st_list(parser, args, output_manager): +def st_list(parser, args, output_manager, return_parser=False): def _print_stats(options, stats, human): total_count = total_bytes = 0 @@ -571,6 +581,11 @@ def _print_stats(options, stats, human): '-H', '--header', action='append', dest='header', default=[], help='Adds a custom request header to use for listing.') + + # We return the parser to build up the bash_completion + if return_parser: + return parser + options, args = parse_args(parser, args) args = args[1:] if options['delimiter'] and not args: @@ -629,7 +644,7 @@ def _print_stats(options, stats, human): '''.strip('\n') -def st_stat(parser, args, output_manager): +def st_stat(parser, args, output_manager, return_parser=False): parser.add_argument( '--lh', dest='human', action='store_true', default=False, help='Report sizes in human readable format similar to ls -lh.') @@ -638,6 +653,10 @@ def st_stat(parser, args, output_manager): default=[], help='Adds a custom request header to use for stat.') + # We return the parser to build up the bash_completion + if return_parser: + return parser + options, args = parse_args(parser, args) args = args[1:] @@ -725,7 +744,7 @@ def st_stat(parser, args, output_manager): '''.strip('\n') -def st_post(parser, args, output_manager): +def st_post(parser, args, output_manager, return_parser=False): parser.add_argument( '-r', '--read-acl', dest='read_acl', help='Read ACL for containers. ' 'Quick summary of ACL syntax: .r:*, .r:-.example.com, ' @@ -750,6 +769,11 @@ def st_post(parser, args, output_manager): 'This option may be repeated. ' 'Example: -H "content-type:text/plain" ' '-H "Content-Length: 4000"') + + # We return the parser to build up the bash_completion + if return_parser: + return parser + (options, args) = parse_args(parser, args) args = args[1:] if (options['read_acl'] or options['write_acl'] or options['sync_to'] or @@ -822,7 +846,7 @@ def st_post(parser, args, output_manager): '''.strip('\n') -def st_copy(parser, args, output_manager): +def st_copy(parser, args, output_manager, return_parser=False): parser.add_argument( '-d', '--destination', help='The container and name of the ' 'destination object') @@ -839,6 +863,11 @@ def st_copy(parser, args, output_manager): 'This option may be repeated. ' 'Example: -H "content-type:text/plain" ' '-H "Content-Length: 4000"') + + # We return the parser to build up the bash_completion + if return_parser: + return parser + (options, args) = parse_args(parser, args) args = args[1:] @@ -948,7 +977,7 @@ def st_copy(parser, args, output_manager): '''.strip('\n') -def st_upload(parser, args, output_manager): +def st_upload(parser, args, output_manager, return_parser=False): DEFAULT_STDIN_SEGMENT = 10 * 1024 * 1024 parser.add_argument( @@ -1006,6 +1035,11 @@ def st_upload(parser, args, output_manager): parser.add_argument( '--ignore-checksum', dest='checksum', default=True, action='store_false', help='Turn off checksum validation for uploads.') + + # We return the parser to build up the bash_completion + if return_parser: + return parser + options, args = parse_args(parser, args) args = args[1:] if len(args) < 2: @@ -1185,7 +1219,7 @@ def st_upload(parser, args, output_manager): st_info_help = st_capabilities_help -def st_capabilities(parser, args, output_manager): +def st_capabilities(parser, args, output_manager, return_parser=False): def _print_compo_cap(name, capabilities): for feature, options in sorted(capabilities.items(), key=lambda x: x[0]): @@ -1198,6 +1232,11 @@ def _print_compo_cap(name, capabilities): parser.add_argument('--json', action='store_true', help='print capability information in json') + + # We return the parser to build up the bash_completion + if return_parser: + return parser + (options, args) = parse_args(parser, args) if args and len(args) > 2: output_manager.error('Usage: %s capabilities %s\n%s', @@ -1246,7 +1285,12 @@ def _print_compo_cap(name, capabilities): '''.strip('\n') -def st_auth(parser, args, thread_manager): +def st_auth(parser, args, thread_manager, return_parser=False): + + # We return the parser to build up the bash_completion + if return_parser: + return parser + (options, args) = parse_args(parser, args) if options['verbose'] > 1: if options['auth_version'] in ('1', '1.0'): @@ -1330,7 +1374,7 @@ def st_auth(parser, args, thread_manager): '''.strip('\n') -def st_tempurl(parser, args, thread_manager): +def st_tempurl(parser, args, thread_manager, return_parser=False): parser.add_argument( '--absolute', action='store_true', dest='absolute_expiry', default=False, @@ -1357,6 +1401,10 @@ def st_tempurl(parser, args, thread_manager): "given ip or ip range."), ) + # We return the parser to build up the bash_completion + if return_parser: + return parser + (options, args) = parse_args(parser, args) args = args[1:] if len(args) < 4: @@ -1388,6 +1436,65 @@ def st_tempurl(parser, args, thread_manager): thread_manager.print_msg(url) +st_bash_completion_help = '''Retrieve command specific flags used by bash_completion. + +Optional positional arguments: + Swift client command to filter the flags by. +'''.strip('\n') + + +st_bash_completion_options = '''[command] +''' + + +def st_bash_completion(parser, args, thread_manager, return_parser=False): + if return_parser: + return parser + + global commands + com = args[1] if len(args) > 1 else None + + if com: + if com in commands: + fn_commands = ["st_%s" % com] + else: + print("") + return + else: + fn_commands = [fn for fn in globals().keys() + if fn.startswith('st_') and not fn.endswith('_options') + and not fn.endswith('_help')] + + subparsers = parser.add_subparsers() + subcommands = {} + if not com: + subcommands['base'] = parser + for command in fn_commands: + cmd = command[3:] + if com: + subparser = subparsers.add_parser( + cmd, help=globals()['%s_help' % command]) + add_default_args(subparser) + subparser = globals()[command]( + subparser, args, thread_manager, True) + subcommands[cmd] = subparser + else: + subcommands[cmd] = None + + cmds = set() + opts = set() + for sc_str, sc in list(subcommands.items()): + cmds.add(sc_str) + if sc: + for option in sc._optionals._option_string_actions: + opts.add(option) + + for cmd_to_remove in (com, 'bash_completion', 'base'): + if cmd_to_remove in cmds: + cmds.remove(cmd_to_remove) + print(' '.join(cmds | opts)) + + class HelpFormatter(argparse.HelpFormatter): def _format_action_invocation(self, action): if not action.option_strings: @@ -1508,94 +1615,7 @@ def parse_args(parser, args, enforce_requires=True): return options, args -def main(arguments=None): - argv = sys_argv if arguments is None else arguments - - argv = [a if isinstance(a, text_type) else a.decode('utf-8') for a in argv] - - version = client_version - parser = argparse.ArgumentParser( - add_help=False, formatter_class=HelpFormatter, usage=''' -%(prog)s [--version] [--help] [--os-help] [--snet] [--verbose] - [--debug] [--info] [--quiet] [--auth ] - [--auth-version | - --os-identity-api-version ] - [--user ] - [--key ] [--retries ] - [--os-username ] - [--os-password ] - [--os-user-id ] - [--os-user-domain-id ] - [--os-user-domain-name ] - [--os-tenant-id ] - [--os-tenant-name ] - [--os-project-id ] - [--os-project-name ] - [--os-project-domain-id ] - [--os-project-domain-name ] - [--os-auth-url ] - [--os-auth-token ] - [--os-storage-url ] - [--os-region-name ] - [--os-service-type ] - [--os-endpoint-type ] - [--os-cacert ] - [--insecure] - [--os-cert ] - [--os-key ] - [--no-ssl-compression] - [--force-auth-retry] - [--prompt] - [--help] [] - -Command-line interface to the OpenStack Swift API. - -Positional arguments: - - delete Delete a container or objects within a container. - download Download objects from containers. - list Lists the containers for the account or the objects - for a container. - post Updates meta information for the account, container, - or object; creates containers if not present. - copy Copies object, optionally adds meta - stat Displays information for the account, container, - or object. - upload Uploads files or directories to the given container. - capabilities List cluster capabilities. - tempurl Create a temporary URL. - auth Display auth related environment variables. - -Examples: - %(prog)s download --help - - %(prog)s -A https://api.example.com/v1.0 \\ - -U user -K api_key stat -v - - %(prog)s --os-auth-url https://api.example.com/v2.0 \\ - --os-tenant-name tenant \\ - --os-username user --os-password password list - - %(prog)s --os-auth-url https://api.example.com/v3 --auth-version 3\\ - --os-project-name project1 --os-project-domain-name domain1 \\ - --os-username user --os-user-domain-name domain1 \\ - --os-password password list - - %(prog)s --os-auth-url https://api.example.com/v3 --auth-version 3\\ - --os-project-id 0123456789abcdef0123456789abcdef \\ - --os-user-id abcdef0123456789abcdef0123456789 \\ - --os-password password list - - %(prog)s --os-auth-token 6ee5eb33efad4e45ab46806eac010566 \\ - --os-storage-url https://10.1.5.2:8080/v1/AUTH_ced809b6a4baea7aeab61a \\ - list - - %(prog)s list --lh -'''.strip('\n')) - parser.add_argument('--version', action='version', - version='python-swiftclient %s' % version) - parser.add_argument('-h', '--help', action='store_true') - +def add_default_args(parser): default_auth_version = '1.0' for k in ('ST_AUTH_VERSION', 'OS_AUTH_VERSION', 'OS_IDENTITY_API_VERSION'): try: @@ -1808,6 +1828,100 @@ def main(arguments=None): default=environ.get('OS_KEY'), help='Specify a client certificate key file (for ' 'client auth). Defaults to env[OS_KEY].') + + +def main(arguments=None): + argv = sys_argv if arguments is None else arguments + + argv = [a if isinstance(a, text_type) else a.decode('utf-8') for a in argv] + + parser = argparse.ArgumentParser( + add_help=False, formatter_class=HelpFormatter, usage=''' +%(prog)s [--version] [--help] [--os-help] [--snet] [--verbose] + [--debug] [--info] [--quiet] [--auth ] + [--auth-version | + --os-identity-api-version ] + [--user ] + [--key ] [--retries ] + [--os-username ] + [--os-password ] + [--os-user-id ] + [--os-user-domain-id ] + [--os-user-domain-name ] + [--os-tenant-id ] + [--os-tenant-name ] + [--os-project-id ] + [--os-project-name ] + [--os-project-domain-id ] + [--os-project-domain-name ] + [--os-auth-url ] + [--os-auth-token ] + [--os-storage-url ] + [--os-region-name ] + [--os-service-type ] + [--os-endpoint-type ] + [--os-cacert ] + [--insecure] + [--os-cert ] + [--os-key ] + [--no-ssl-compression] + [--force-auth-retry] + [--help] [] + +Command-line interface to the OpenStack Swift API. + +Positional arguments: + + delete Delete a container or objects within a container. + download Download objects from containers. + list Lists the containers for the account or the objects + for a container. + post Updates meta information for the account, container, + or object; creates containers if not present. + copy Copies object, optionally adds meta + stat Displays information for the account, container, + or object. + upload Uploads files or directories to the given container. + capabilities List cluster capabilities. + tempurl Create a temporary URL. + auth Display auth related environment variables. + bash_completion Outputs option and flag cli data ready for + bash_completion. + +Examples: + %(prog)s download --help + + %(prog)s -A https://api.example.com/v1.0 \\ + -U user -K api_key stat -v + + %(prog)s --os-auth-url https://api.example.com/v2.0 \\ + --os-tenant-name tenant \\ + --os-username user --os-password password list + + %(prog)s --os-auth-url https://api.example.com/v3 --auth-version 3\\ + --os-project-name project1 --os-project-domain-name domain1 \\ + --os-username user --os-user-domain-name domain1 \\ + --os-password password list + + %(prog)s --os-auth-url https://api.example.com/v3 --auth-version 3\\ + --os-project-id 0123456789abcdef0123456789abcdef \\ + --os-user-id abcdef0123456789abcdef0123456789 \\ + --os-password password list + + %(prog)s --os-auth-token 6ee5eb33efad4e45ab46806eac010566 \\ + --os-storage-url https://10.1.5.2:8080/v1/AUTH_ced809b6a4baea7aeab61a \\ + list + + %(prog)s list --lh +'''.strip('\n')) + + version = client_version + parser.add_argument('--version', action='version', + version='python-swiftclient %s' % version) + parser.add_argument('-h', '--help', action='store_true') + + add_default_args(parser) + options, args = parse_args(parser, argv[1:], enforce_requires=False) if options['help'] or options['os_help']: diff --git a/tools/swift.bash_completion b/tools/swift.bash_completion new file mode 100644 index 00000000..2f98a6b5 --- /dev/null +++ b/tools/swift.bash_completion @@ -0,0 +1,32 @@ +declare -a _swift_opts # lazy init + +_swift_get_current_opt() +{ + local opt + for opt in ${_swift_opts[@]} ; do + if [[ $(echo ${COMP_WORDS[*]} |grep -c " $opt\$") > 0 ]] || [[ $(echo ${COMP_WORDS[*]} |grep -c " $opt ") > 0 ]] ; then + echo $opt + return 0 + fi + done + echo "" + return 0 +} + +_swift() +{ + local opt cur prev sflags + COMPREPLY=() + cur="${COMP_WORDS[COMP_CWORD]}" + prev="${COMP_WORDS[COMP_CWORD-1]}" + + if [ "x$_swift_opts" == "x" ] ; then + _swift_opts=(`swift bash_completion "$sbc" | sed -e "s/-[-A-Za-z0-9_]*//g" -e "s/ */ /g"`) + fi + + opt="$(_swift_get_current_opt)" + COMPREPLY=($(compgen -W "$(swift bash_completion $opt)" -- ${cur})) + + return 0 +} +complete -F _swift swift From 02b08aaa10b3655aee95e6069b5116db0524a268 Mon Sep 17 00:00:00 2001 From: Timur Alperovich Date: Mon, 22 Jan 2018 18:22:04 -0800 Subject: [PATCH 034/238] Add close() to _RetryBody. Allows clients to give up on reading the rest of the server response, if they so choose. Change-Id: Iccc95b1b5e7d066470966ee0c62a3beb260846e5 --- swiftclient/client.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/swiftclient/client.py b/swiftclient/client.py index 8cbdf452..8d28ffb6 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -273,6 +273,9 @@ def next(self): def __next__(self): return self.next() + def close(self): + self.resp.close() + class _RetryBody(_ObjectBody): """ From f4a2b16c2cc65410765abdff7a45532305a4548f Mon Sep 17 00:00:00 2001 From: Timur Alperovich Date: Tue, 17 Apr 2018 14:36:57 -0700 Subject: [PATCH 035/238] Properly handle unicode headers. Fix unicode handling in Python 3 and Python 2. There are currently two failure modes. In python 2, swiftclient fails to log in debug mode if the account name has a non-ASCII character. This is because the account name will appear in the storage URL, which we attempt to pass to the logger as a byte string (whereas it should be a unicode string). This patch changes the behavior to convert the path strings into unicode by calling the parse_header_string() function. The second failure mode is with Python 3, where http_lib returns headers that are latin-1 encoded, but swiftclient expects UTF-8. The patch automatically converts headers from latin-1 (iso-8859-1) to UTF-8, so that we can properly handle non-ASCII headers in responses. Change-Id: Ifa7f3d5af71bde8127129f1f8603772d80d063c1 --- swiftclient/client.py | 29 +++++++++++++--- tests/unit/test_swiftclient.py | 61 ++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 5 deletions(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index 8cbdf452..a518d321 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -151,7 +151,7 @@ def http_log(args, kwargs, resp, body): elif element in ('GET', 'POST', 'PUT'): string_parts.append(' -X %s' % element) else: - string_parts.append(' %s' % element) + string_parts.append(' %s' % parse_header_string(element)) if 'headers' in kwargs: headers = scrub_headers(kwargs['headers']) for element in headers: @@ -455,11 +455,23 @@ def getresponse(self): self.resp.status = self.resp.status_code old_getheader = self.resp.raw.getheader + def _decode_header(string): + if string is None or six.PY2: + return string + return string.encode('iso-8859-1').decode('utf-8') + + def _encode_header(string): + if string is None or six.PY2: + return string + return string.encode('utf-8').decode('iso-8859-1') + def getheaders(): - return self.resp.headers.items() + return [(_decode_header(k), _decode_header(v)) + for k, v in self.resp.headers.items()] def getheader(k, v=None): - return old_getheader(k.lower(), v) + return _decode_header(old_getheader( + _encode_header(k.lower()), _encode_header(v))) def releasing_read(*args, **kwargs): chunk = self.resp.raw.read(*args, **kwargs) @@ -513,8 +525,11 @@ def get_auth_1_0(url, user, key, snet, **kwargs): netloc = parsed[1] parsed[1] = 'snet-' + netloc url = urlunparse(parsed) - return url, resp.getheader('x-storage-token', - resp.getheader('x-auth-token')) + + auth_token = resp.getheader('x-auth-token') + if auth_token is not None: + auth_token = parse_header_string(auth_token) + return url, resp.getheader('x-storage-token', auth_token) def get_keystoneclient_2_0(auth_url, user, key, os_options, **kwargs): @@ -694,10 +709,14 @@ def get_auth(auth_url, user, key, **kwargs): raise ClientException('Unknown auth_version %s specified and no ' 'session found.' % auth_version) + if token is not None: + token = parse_header_string(token) # Override storage url, if necessary if os_options.get('object_storage_url'): return os_options['object_storage_url'], token else: + if storage_url is not None: + return parse_header_string(storage_url), token return storage_url, token diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index b6d68568..3303372d 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -1896,6 +1896,57 @@ def test_response_connection_released(self): self.assertFalse(resp.read()) self.assertTrue(resp.closed) + @unittest.skipIf(six.PY3, 'python2 specific test') + def test_response_python2_headers(self): + '''Test utf-8 headers in Python 2. + ''' + _, conn = c.http_connection(u'http://www.test.com/') + conn.resp = MockHttpResponse( + status=200, + headers={ + '\xd8\xaa-unicode': '\xd8\xaa-value', + 'empty-header': '' + } + ) + + resp = conn.getresponse() + self.assertEqual( + '\xd8\xaa-value', resp.getheader('\xd8\xaa-unicode')) + self.assertEqual( + '\xd8\xaa-value', resp.getheader('\xd8\xaa-UNICODE')) + self.assertEqual('', resp.getheader('empty-header')) + self.assertEqual( + dict([('\xd8\xaa-unicode', '\xd8\xaa-value'), + ('empty-header', ''), + ('etag', '"%s"' % EMPTY_ETAG)]), + dict(resp.getheaders())) + + @unittest.skipIf(six.PY2, 'python3 specific test') + def test_response_python3_headers(self): + '''Test latin1-encoded headers in Python 3. + ''' + _, conn = c.http_connection(u'http://www.test.com/') + conn.resp = MockHttpResponse( + status=200, + headers={ + b'\xd8\xaa-unicode'.decode('iso-8859-1'): + b'\xd8\xaa-value'.decode('iso-8859-1'), + 'empty-header': '' + } + ) + + resp = conn.getresponse() + self.assertEqual( + '\u062a-value', resp.getheader('\u062a-unicode')) + self.assertEqual( + '\u062a-value', resp.getheader('\u062a-UNICODE')) + self.assertEqual('', resp.getheader('empty-header')) + self.assertEqual( + dict([('\u062a-unicode', '\u062a-value'), + ('empty-header', ''), + ('etag', ('"%s"' % EMPTY_ETAG))]), + dict(resp.getheaders())) + class TestConnection(MockHttpTest): @@ -2839,6 +2890,16 @@ def test_show_token(self): self.assertIn('X-Storage-Token', output) self.assertIn(unicode_token_value, output) + @mock.patch('swiftclient.client.logger.debug') + def test_unicode_path(self, mock_log): + path = u'http://swift/v1/AUTH_account-\u062a'.encode('utf-8') + c.http_log(['GET', path], {}, + MockHttpResponse(status=200, headers=[]), '') + request_log_line = mock_log.mock_calls[0] + self.assertEqual('REQ: %s', request_log_line[1][0]) + self.assertEqual(u'curl -i -X GET %s' % path.decode('utf-8'), + request_log_line[1][1]) + class TestCloseConnection(MockHttpTest): From 172a09a4019dc637e525d14aef76f10e812385dd Mon Sep 17 00:00:00 2001 From: John Dickinson Date: Thu, 19 Jul 2018 10:54:48 -0700 Subject: [PATCH 036/238] authors/changelog update for 3.6.0 Change-Id: I471d3d56d98915804aaf848f7ff98d91f586d572 --- .mailmap | 3 +- AUTHORS | 10 ++++- ChangeLog | 33 +++++++++++++++ .../notes/360_notes-1ec385df13a3a735.yaml | 40 +++++++++++++++++++ 4 files changed, 84 insertions(+), 2 deletions(-) create mode 100644 releasenotes/notes/360_notes-1ec385df13a3a735.yaml diff --git a/.mailmap b/.mailmap index 9e53d387..ecbcad1f 100644 --- a/.mailmap +++ b/.mailmap @@ -58,7 +58,8 @@ Madhuri Kumari madhuri Hua Zhang Yummy Bian -Alistair Coles +Alistair Coles +Alistair Coles Tong Li Paul Luse Yuan Zhou diff --git a/AUTHORS b/AUTHORS index 388d8700..abc21e98 100644 --- a/AUTHORS +++ b/AUTHORS @@ -2,7 +2,7 @@ Alessandro Pilotti (ap@pilotti.it) Alex Gaynor (alex.gaynor@gmail.com) Alexandra Settle (alexandra.settle@rackspace.com) Alexis Lee (lxsli@hpe.com) -Alistair Coles (alistair.coles@hpe.com) +Alistair Coles (alistairncoles@gmail.com) Andreas Jaeger (aj@suse.de) Andrew Welleck (awellec@us.ibm.com) Andy McCrae (andy.mccrae@gmail.com) @@ -12,6 +12,7 @@ Ben McCann (ben@benmccann.com) Cedric Brandily (zzelle@gmail.com) Chaozhe.Chen (chaozhe.chen@easystack.cn) Charles Hsu (charles0126@gmail.com) +Chen (dstbtgagt@foxmail.com) Cheng Li (shcli@cn.ibm.com) Chmouel Boudjnah (chmouel@enovance.com) Chris Buccella (chris.buccella@antallagon.com) @@ -35,6 +36,7 @@ Dirk Mueller (dirk@dmllr.de) Donagh McCabe (donagh.mccabe@hpe.com) Doug Hellmann (doug@doughellmann.com) EdLeafe (ed@leafe.com) +Erik Olof Gunnar Andersson (eandersson@blizzard.com) Fabien Boucher (fabien.boucher@enovance.com) Feng Liu (mefengliu23@gmail.com) Flavio Percoco (flaper87@gmail.com) @@ -71,6 +73,7 @@ Kota Tsuyuzaki (tsuyuzaki.kota@lab.ntt.co.jp) Kun Huang (gareth@unitedstack.com) Leah Klearman (lklrmn@gmail.com) Li Riqiang (lrqrun@gmail.com) +lingyongxu (lyxu@fiberhome.com) liuyamin (liuyamin@fiberhome.com) Luis de Bethencourt (luis@debethencourt.com) M V P Nitesh (m.nitesh@nectechnologies.in) @@ -83,10 +86,12 @@ Matthew Oliver (matt@oliver.net.au) Matthieu Huin (mhu@enovance.com) Mike Widman (mwidman@endurancewindpower.com) Min Min Ren (rminmin@cn.ibm.com) +mmcardle (mark.mcardle@sohonet.com) Mohit Motiani (mohit.motiani@intel.com) Monty Taylor (mordred@inaugust.com) Nandini Tata (nandini.tata@intel.com) Nelson Marcos (nelsonmarcos@gmail.com) +Nguyen Hai (nguyentrihai93@gmail.com) Nguyen Hung Phuong (phuongnh@vn.fujitsu.com) Nick Craig-Wood (nick@craig-wood.com) Ondrej Novy (ondrej.novy@firma.seznam.cz) @@ -110,6 +115,7 @@ Sean Dague (sean@dague.net) Sergey Gotliv (sgotliv@redhat.com) Sergio Cazzolato (sergio.j.cazzolato@intel.com) Shane Wang (shane.wang@intel.com) +shangxiaobj (shangxiaobj@inspur.com) Shashi Kant (shashi.kant@nectechnologies.in) Shashirekha Gundur (shashirekha.j.gundur@intel.com) shu-mutou (shu-mutou@rf.jp.nec.com) @@ -129,11 +135,13 @@ Tim Burke (tim.burke@gmail.com) Timur Alperovich (timuralp@swiftstack.com) Tong Li (litong01@us.ibm.com) Tony Breeds (tony@bakeyournoodle.com) +Tovin Seven (vinhnt@vn.fujitsu.com) Tristan Cacqueray (tristan.cacqueray@enovance.com) Vasyl Khomenko (vasiliyk@yahoo-inc.com) venkatamahesh (venkatamaheshkotha@gmail.com) Victor Stinner (victor.stinner@enovance.com) Vitaly Gridnev (vgridnev@mirantis.com) +wangqi (wang.qi@99cloud.net) wangxiyuan (wangxiyuan@huawei.com) Wu Wenxiang (wu.wenxiang@99cloud.net) YangLei (yanglyy@cn.ibm.com) diff --git a/ChangeLog b/ChangeLog index efa7e8a8..a37a6db0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,36 @@ +3.6.0 +----- + +* Add the `--prompt` option for the CLI which will cause the user to be + prompted to enter a password. Any password otherwise specified by + `--key`, `--os-password` or an environment variable will be ignored. + +* Added bash completion support to the `swift` CLI. Enable this by sourcing + the included `tools/swift.bash_completion` file. Make it permanent by + including this file in the system's `/etc/bash_completion.d` directory. + +* Add ability to generate a temporary URL with an IP range restriction. + TempURLs with IP restrictions are supported in Swift 2.19.0 or later. + +* The client.py SDK now supports a `query_string` option on the + `head_object()` method. This is useful for finding information on + SLO/DLO manifests without fetching the entire manifest. + +* The client.py SDK now respects `region_name` when using sessions. + +* Added a `.close()` method to an object response, allowing clients to give + up on reading the rest of the response body, if they so choose. + +* Fixed a bug where using `--debug` in the CLI with unicode account names + would cause a client crash. + +* Make OS_AUTH_URL work in DevStack (for testing) by default. + +* Dropped Python 3.4 testing. + +* Various other minor bug fixes and improvements. + + 3.5.0 ----- diff --git a/releasenotes/notes/360_notes-1ec385df13a3a735.yaml b/releasenotes/notes/360_notes-1ec385df13a3a735.yaml new file mode 100644 index 00000000..8d82b063 --- /dev/null +++ b/releasenotes/notes/360_notes-1ec385df13a3a735.yaml @@ -0,0 +1,40 @@ +--- +features: + - | + Add the ``--prompt`` option for the CLI which will cause the user to be + prompted to enter a password. Any password otherwise specified by + ``--key`` , ``--os-password`` or an environment variable will be ignored. + + - | + Added bash completion support to the ``swift`` CLI. Enable this by sourcing + the included ``tools/swift.bash_completion`` file. Make it permanent by + including this file in the system's ``/etc/bash_completion.d`` directory. + + - | + Add ability to generate a temporary URL with an IP range restriction. + TempURLs with IP restrictions are supported are Swift 2.19.0 or later. + + - | + The client.py SDK now supports a ``query_string`` option on the + ``head_object()`` method. This is useful for finding information on + SLO/DLO manifests without fetching the entire manifest. + + - | + The client.py SDK now respects ``region_name`` when using sessions. + + - | + Added a ``.close()`` method to an object response, allowing clients to give + up on reading the rest of the response body, if they so choose. + + - | + Fixed a bug where using ``--debug`` in the CLI with unicode account names + would cause a client crash. + + - | + Make OS_AUTH_URL work in DevStack (for testing) by default. + + - | + Dropped Python 3.4 testing. + + - | + Various other minor bug fixes and improvements. From e28b12a1f2156b2e8a790290ae6da08adde3875f Mon Sep 17 00:00:00 2001 From: Timur Alperovich Date: Tue, 24 Jul 2018 11:53:25 -0700 Subject: [PATCH 037/238] Remove unnecessary calls to parse_header_string(). Since we define the getheader() method on the response from HTTPConnection, we don't have to call parse_header_string, as the values will already be converted properly. Change-Id: Ia81e8674b828b3ff1f014454126b469e41adfc23 --- swiftclient/client.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index 410a0012..71801c62 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -529,10 +529,8 @@ def get_auth_1_0(url, user, key, snet, **kwargs): parsed[1] = 'snet-' + netloc url = urlunparse(parsed) - auth_token = resp.getheader('x-auth-token') - if auth_token is not None: - auth_token = parse_header_string(auth_token) - return url, resp.getheader('x-storage-token', auth_token) + token = resp.getheader('x-storage-token', resp.getheader('x-auth-token')) + return url, token def get_keystoneclient_2_0(auth_url, user, key, os_options, **kwargs): @@ -712,14 +710,10 @@ def get_auth(auth_url, user, key, **kwargs): raise ClientException('Unknown auth_version %s specified and no ' 'session found.' % auth_version) - if token is not None: - token = parse_header_string(token) # Override storage url, if necessary if os_options.get('object_storage_url'): return os_options['object_storage_url'], token else: - if storage_url is not None: - return parse_header_string(storage_url), token return storage_url, token From 79e00ea0d62d9c6e9a53ba8d80fcf27b7e2c79c8 Mon Sep 17 00:00:00 2001 From: zhubx007 Date: Fri, 10 Aug 2018 17:42:19 +0800 Subject: [PATCH 038/238] Add .idea into .gitignore Generated by IDE PyCharm Change-Id: Ifc99b34aae581221ae4b8d2533adfc21e91cd291 --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index f2699820..16da5895 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ cover/ coverage.xml doc/build doc/source/api/ +.idea From bd6a12c7518f40f80100396147c143caf133215c Mon Sep 17 00:00:00 2001 From: Nguyen Hai Date: Fri, 24 Aug 2018 16:37:43 +0900 Subject: [PATCH 039/238] import zuul job settings from project-config This is a mechanically generated patch to complete step 1 of moving the zuul job settings out of project-config and into each project repository. Because there will be a separate patch on each branch, the branch specifiers for branch-specific jobs have been removed. Because this patch is generated by a script, there may be some cosmetic changes to the layout of the YAML file(s) as the contents are normalized. See the python3-first goal document for details: https://governance.openstack.org/tc/goals/stein/python3-first.html Change-Id: I59f4cbc0a21b8be3a1cae28a64f90d5adcf6be24 Story: #2002586 Task: #24337 --- .zuul.yaml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.zuul.yaml b/.zuul.yaml index 67a39c42..414e3c05 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -1,7 +1,28 @@ - project: + templates: + - openstack-python-jobs + - openstack-python35-jobs + - publish-openstack-sphinx-docs + - check-requirements + - openstack-pypy-jobs-nonvoting + - lib-forward-testing + - release-notes-jobs check: jobs: - openstack-tox-lower-constraints + - legacy-swift-dsvm-functional + - legacy-swift-dsvm-functional-identity-v3-only: + voting: false + - legacy-swiftclient-dsvm-functional + - legacy-swiftclient-dsvm-functional-identity-v3-only: + voting: false + - openstack-tox-py36: + voting: false gate: jobs: - openstack-tox-lower-constraints + - legacy-swift-dsvm-functional + - legacy-swiftclient-dsvm-functional + post: + jobs: + - openstack-tox-cover From 420be0c99e4161c56947413a15a39575d4e3724e Mon Sep 17 00:00:00 2001 From: Nguyen Hai Date: Fri, 24 Aug 2018 16:38:24 +0900 Subject: [PATCH 040/238] switch documentation job to new PTI This is a mechanically generated patch to switch the documentation jobs to use the new PTI versions of the jobs as part of the python3-first goal. See the python3-first goal document for details: https://governance.openstack.org/tc/goals/stein/python3-first.html Change-Id: I510e70f2222006df661c6a3d9e26af57b68be835 Story: #2002586 Task: #24337 --- .zuul.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index 414e3c05..d4ef79ac 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -2,11 +2,11 @@ templates: - openstack-python-jobs - openstack-python35-jobs - - publish-openstack-sphinx-docs + - publish-openstack-docs-pti - check-requirements - openstack-pypy-jobs-nonvoting - lib-forward-testing - - release-notes-jobs + - release-notes-jobs-python3 check: jobs: - openstack-tox-lower-constraints From 5aee0732ffc045c725ad3af601ebef6ddd3a63ce Mon Sep 17 00:00:00 2001 From: Nguyen Hai Date: Fri, 24 Aug 2018 16:38:25 +0900 Subject: [PATCH 041/238] add python 3.6 unit test job This is a mechanically generated patch to add a unit test job running under Python 3.6 as part of the python3-first goal. See the python3-first goal document for details: https://governance.openstack.org/tc/goals/stein/python3-first.html Change-Id: I6fd051fd0b01a308d16734c5b12e11a12a38c3be Story: #2002586 Task: #24337 --- .zuul.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.zuul.yaml b/.zuul.yaml index d4ef79ac..affe46cf 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -2,6 +2,7 @@ templates: - openstack-python-jobs - openstack-python35-jobs + - openstack-python36-jobs - publish-openstack-docs-pti - check-requirements - openstack-pypy-jobs-nonvoting From edbc5d8e21af689ffac09e9a1a434c09364cbcdb Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Fri, 31 Aug 2018 16:37:41 +0000 Subject: [PATCH 042/238] Make py36 unit test job voting Change-Id: I42cd4e19bba89c9dd4d7d20c75ee59217b9ea75d --- .zuul.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index affe46cf..7541e014 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -17,8 +17,6 @@ - legacy-swiftclient-dsvm-functional - legacy-swiftclient-dsvm-functional-identity-v3-only: voting: false - - openstack-tox-py36: - voting: false gate: jobs: - openstack-tox-lower-constraints From 37ee6459cd2bb0197637f6d38e454bfe59192637 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 6 Jun 2018 17:58:19 -0400 Subject: [PATCH 043/238] fix tox python3 overrides We want to default to running all tox environments under python 3, so set the basepython value in each environment. We do not want to specify a minor version number, because we do not want to have to update the file every time we upgrade python. We do not want to set the override once in testenv, because that breaks the more specific versions used in default environments like py35 and py36. Change-Id: I86d24104033b490a35178fc504d88c1e4a566628 Signed-off-by: Doug Hellmann --- tox.ini | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tox.ini b/tox.ini index 660248b4..ec7d3442 100644 --- a/tox.ini +++ b/tox.ini @@ -22,17 +22,21 @@ whitelist_externals = sh passenv = SWIFT_* *_proxy [testenv:pep8] +basepython = python3 commands = python -m flake8 swiftclient tests [testenv:venv] +basepython = python3 commands = {posargs} [testenv:cover] +basepython = python3 commands = python setup.py testr --coverage coverage report [testenv:func] +basepython = python3 setenv = OS_TEST_PATH=tests.functional whitelist_externals = coverage @@ -43,6 +47,7 @@ commands = rm -f .coverage [testenv:docs] +basepython = python3 commands= python setup.py build_sphinx @@ -60,6 +65,7 @@ show-source = True exclude = .venv,.tox,dist,doc,*egg [testenv:bindep] +basepython = python3 # Do not install any requirements. We want this to be fast and work even if # system dependencies are missing, since it's used to tell you what system # dependencies are missing! This also means that bindep must be installed @@ -69,6 +75,7 @@ deps = bindep commands = bindep test [testenv:releasenotes] +basepython = python3 commands = sphinx-build -a -W -E -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html [testenv:lower-constraints] From 70e20b62e6719c97267aa0e5a80dae2e31dfba76 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Mon, 13 Aug 2018 22:02:38 +0000 Subject: [PATCH 044/238] Use Swift's in-tree DSVM test While we're at it, make a new job that inherits from it to bring the legacy-swiftclient-dsvm-functional testing in-tree, too. For naming, follow naming policy and remove "dsvm" from names. Remove legacy jobs, they are not needed anymore. Change-Id: I919c0b77ac4888350194f55a9c12e0742845024f Depends-On: https://review.openstack.org/589270 --- .zuul.yaml | 38 ++++++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index 7541e014..6c37f8c1 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -1,3 +1,29 @@ +- job: + name: swiftclient-swift-functional + parent: swift-dsvm-functional + description: | + Run swift's functional tests with python-swiftclient + installed from source instead as package from PyPI. + # Ensure that we install python-swiftclient from git and + # do not install from pypi. This is needed since the parent + # job sets zuul_work_dir to the swift directory and uses tox + # for installation. + required-projects: + - git.openstack.org/openstack/python-swiftclient + +- job: + name: swiftclient-functional + parent: swift-dsvm-functional + description: | + Run functional tests of python-swiftclient with + python-swiftclient installed from source instead as package from + PyPI. + required-projects: + - git.openstack.org/openstack/python-swiftclient + vars: + # Override value from parent job to use swiftclient tests + zuul_work_dir: "{{ zuul.projects['git.openstack.org/openstack/python-swiftclient'].src_dir }}" + - project: templates: - openstack-python-jobs @@ -10,18 +36,14 @@ - release-notes-jobs-python3 check: jobs: + - swiftclient-swift-functional + - swiftclient-functional - openstack-tox-lower-constraints - - legacy-swift-dsvm-functional - - legacy-swift-dsvm-functional-identity-v3-only: - voting: false - - legacy-swiftclient-dsvm-functional - - legacy-swiftclient-dsvm-functional-identity-v3-only: - voting: false gate: jobs: + - swiftclient-swift-functional + - swiftclient-functional - openstack-tox-lower-constraints - - legacy-swift-dsvm-functional - - legacy-swiftclient-dsvm-functional post: jobs: - openstack-tox-cover From d1e1f8d8d6a8890c71eea8a3c2a488af30a8147b Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Fri, 7 Sep 2018 16:50:10 -0700 Subject: [PATCH 045/238] Stop lazy importing keystoneclient There were two basic problems: - We'd try to import on every attempt at getting auth, even when we already know keystoneclient isn't available. - Sometimes devs would hit some crazy import race involving (some combination of?) greenthreads and OS threads. So let's just try the imports *once*, at import time, and have None sentinels if it fails. Try both versions separately to decouple failures; this should let us support a wider range of keystoneclient versions. Change-Id: I2367310aac74f1b7c5ea0cb1a822a491e4ba8e68 --- swiftclient/client.py | 53 +++++++++++++++------------- tests/unit/test_shell.py | 63 ++++++++++++++++------------------ tests/unit/test_swiftclient.py | 26 +++++--------- tests/unit/utils.py | 8 ----- 4 files changed, 68 insertions(+), 82 deletions(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index 71801c62..d843aecc 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -60,6 +60,19 @@ def emit(self, record): def createLock(self): self.lock = None +ksexceptions = ksclient_v2 = ksclient_v3 = None +try: + from keystoneclient import exceptions as ksexceptions + # prevent keystoneclient warning us that it has no log handlers + logging.getLogger('keystoneclient').addHandler(NullHandler()) + from keystoneclient.v2_0 import client as ksclient_v2 +except ImportError: + pass +try: + from keystoneclient.v3 import client as ksclient_v3 +except ImportError: + pass + # requests version 1.2.3 try to encode headers in ascii, preventing # utf-8 encoded header to be 'prepared' if StrictVersion(requests.__version__) < StrictVersion('2.0.0'): @@ -540,25 +553,6 @@ def get_keystoneclient_2_0(auth_url, user, key, os_options, **kwargs): return get_auth_keystone(auth_url, user, key, os_options, **kwargs) -def _import_keystone_client(auth_version): - # the attempted imports are encapsulated in this function to allow - # mocking for tests - try: - if auth_version in AUTH_VERSIONS_V3: - from keystoneclient.v3 import client as ksclient - else: - from keystoneclient.v2_0 import client as ksclient - from keystoneclient import exceptions - # prevent keystoneclient warning us that it has no log handlers - logging.getLogger('keystoneclient').addHandler(NullHandler()) - return ksclient, exceptions - except ImportError: - raise ClientException(''' -Auth versions 2.0 and 3 require python-keystoneclient, install it or use Auth -version 1.0 which requires ST_AUTH, ST_USER, and ST_KEY environment -variables to be set or overridden with -A, -U, or -K.''') - - def get_auth_keystone(auth_url, user, key, os_options, **kwargs): """ Authenticate against a keystone server. @@ -587,7 +581,20 @@ def get_auth_keystone(auth_url, user, key, os_options, **kwargs): # Legacy default if not set if auth_version is None: auth_version = '2' - ksclient, exceptions = _import_keystone_client(auth_version) + + ksclient = None + if auth_version in AUTH_VERSIONS_V3: + if ksclient_v3 is not None: + ksclient = ksclient_v3 + else: + if ksclient_v2 is not None: + ksclient = ksclient_v2 + + if ksclient is None: + raise ClientException(''' +Auth versions 2.0 and 3 require python-keystoneclient, install it or use Auth +version 1.0 which requires ST_AUTH, ST_USER, and ST_KEY environment +variables to be set or overridden with -A, -U, or -K.''') try: _ksclient = ksclient.Client( @@ -608,13 +615,13 @@ def get_auth_keystone(auth_url, user, key, os_options, **kwargs): cert=kwargs.get('cert'), key=kwargs.get('cert_key'), auth_url=auth_url, insecure=insecure, timeout=timeout) - except exceptions.Unauthorized: + except ksexceptions.Unauthorized: msg = 'Unauthorized. Check username, password and tenant name/id.' if auth_version in AUTH_VERSIONS_V3: msg = ('Unauthorized. Check username/id, password, ' 'tenant name/id and user/tenant domain name/id.') raise ClientException(msg) - except exceptions.AuthorizationFailure as err: + except ksexceptions.AuthorizationFailure as err: raise ClientException('Authorization Failure. %s' % err) service_type = os_options.get('service_type') or 'object-store' endpoint_type = os_options.get('endpoint_type') or 'publicURL' @@ -627,7 +634,7 @@ def get_auth_keystone(auth_url, user, key, os_options, **kwargs): service_type=service_type, endpoint_type=endpoint_type, **filter_kwargs) - except exceptions.EndpointNotFound: + except ksexceptions.EndpointNotFound: raise ClientException('Endpoint for %s not found - ' 'have you specified a region?' % service_type) return endpoint, _ksclient.auth_token diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index 91496b84..9ef46859 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -37,7 +37,7 @@ from os.path import basename, dirname from .utils import ( - CaptureOutput, fake_get_auth_keystone, _make_fake_import_keystone_client, + CaptureOutput, fake_get_auth_keystone, FakeKeystone, StubResponse, MockHttpTest) from swiftclient.utils import ( EMPTY_ETAG, EXPIRES_ISO8601_FORMAT, @@ -2534,7 +2534,17 @@ def _test_options_passed_to_keystone(self, cmd, opts, os_opts, cmd_args=cmd_args) ks_endpoint = 'http://example.com:8080/v1/AUTH_acc' ks_token = 'fake_auth_token' + # check correct auth version gets used + key = 'auth-version' fake_ks = FakeKeystone(endpoint=ks_endpoint, token=ks_token) + if no_auth: + fake_ks2 = fake_ks3 = None + elif opts.get(key, self.defaults.get(key)) == '2.0': + fake_ks2 = fake_ks + fake_ks3 = None + else: + fake_ks2 = None + fake_ks3 = fake_ks # fake_conn will check that storage_url and auth_token are as expected endpoint = os_opts.get('storage-url', ks_endpoint) token = os_opts.get('auth-token', ks_token) @@ -2542,8 +2552,8 @@ def _test_options_passed_to_keystone(self, cmd, opts, os_opts, storage_url=endpoint, auth_token=token) - with mock.patch('swiftclient.client._import_keystone_client', - _make_fake_import_keystone_client(fake_ks)), \ + with mock.patch('swiftclient.client.ksclient_v2', fake_ks2), \ + mock.patch('swiftclient.client.ksclient_v3', fake_ks3), \ mock.patch('swiftclient.client.http_connection', fake_conn), \ mock.patch.dict(os.environ, env, clear=True), \ patch_disable_warnings() as mock_disable_warnings: @@ -2562,16 +2572,11 @@ def _test_options_passed_to_keystone(self, cmd, opts, os_opts, self.assertEqual([], mock_disable_warnings.mock_calls) if no_auth: - # check that keystone client was not used and terminate tests - self.assertIsNone(getattr(fake_ks, 'auth_version')) - self.assertEqual(len(fake_ks.calls), 0) + # We patched out both keystoneclient versions to be None; + # they *can't* have been used and if we tried to, we would + # have raised ClientExceptions return - # check correct auth version was passed to _import_keystone_client - key = 'auth-version' - expected = opts.get(key, self.defaults.get(key)) - self.assertEqual(expected, fake_ks.auth_version) - # check args passed to keystone Client __init__ self.assertEqual(len(fake_ks.calls), 1) actual_args = fake_ks.calls[0] @@ -2942,9 +2947,9 @@ def setUp(self): self.account = 'AUTH_alice' # keystone returns endpoint for another account - fake_ks = FakeKeystone(endpoint='http://example.com:8080/v1/AUTH_bob', - token='bob_token') - self.fake_ks_import = _make_fake_import_keystone_client(fake_ks) + self.fake_ks = FakeKeystone( + endpoint='http://example.com:8080/v1/AUTH_bob', + token='bob_token') self.cont = 'c1' self.cont_path = '/v1/%s/%s' % (self.account, self.cont) @@ -3023,8 +3028,7 @@ def test_upload_with_read_write_access(self): args, env = self._make_cmd('upload', cmd_args=[self.cont, self.obj, '--leave-segments']) - with mock.patch('swiftclient.client._import_keystone_client', - self.fake_ks_import): + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): with mock.patch('swiftclient.client.http_connection', fake_conn): with mock.patch.dict(os.environ, env): with CaptureOutput() as out: @@ -3046,8 +3050,7 @@ def test_upload_with_write_only_access(self): on_request=req_handler) args, env = self._make_cmd('upload', cmd_args=[self.cont, self.obj, '--leave-segments']) - with mock.patch('swiftclient.client._import_keystone_client', - self.fake_ks_import): + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): with mock.patch('swiftclient.client.http_connection', fake_conn): with mock.patch.dict(os.environ, env): with CaptureOutput() as out: @@ -3073,8 +3076,7 @@ def test_segment_upload_with_write_only_access(self): '--segment-size=10', '--segment-container=%s' % self.cont]) - with mock.patch('swiftclient.client._import_keystone_client', - self.fake_ks_import): + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): with mock.patch('swiftclient.client.http_connection', fake_conn): with mock.patch.dict(os.environ, env): with CaptureOutput() as out: @@ -3112,8 +3114,7 @@ def test_segment_upload_with_write_only_access_segments_container(self): cmd_args=[self.cont, self.obj, '--leave-segments', '--segment-size=10']) - with mock.patch('swiftclient.client._import_keystone_client', - self.fake_ks_import): + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): with mock.patch('swiftclient.client.http_connection', fake_conn): with mock.patch.dict(os.environ, env): with CaptureOutput() as out: @@ -3149,8 +3150,7 @@ def test_upload_with_no_access(self): args, env = self._make_cmd('upload', cmd_args=[self.cont, self.obj, '--leave-segments']) - with mock.patch('swiftclient.client._import_keystone_client', - self.fake_ks_import): + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): with mock.patch('swiftclient.client.http_connection', fake_conn): with mock.patch.dict(os.environ, env): with CaptureOutput() as out: @@ -3207,8 +3207,7 @@ def test_download_with_read_write_access(self): args, env = self._make_cmd('download', cmd_args=[self.cont, self.obj.lstrip('/'), '--no-download']) - with mock.patch('swiftclient.client._import_keystone_client', - self.fake_ks_import): + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): with mock.patch('swiftclient.client.http_connection', fake_conn): with mock.patch.dict(os.environ, env): with CaptureOutput() as out: @@ -3229,8 +3228,7 @@ def test_download_with_read_only_access(self): args, env = self._make_cmd('download', cmd_args=[self.cont, self.obj.lstrip('/'), '--no-download']) - with mock.patch('swiftclient.client._import_keystone_client', - self.fake_ks_import): + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): with mock.patch('swiftclient.client.http_connection', fake_conn): with mock.patch.dict(os.environ, env): with CaptureOutput() as out: @@ -3248,8 +3246,7 @@ def test_download_with_no_access(self): args, env = self._make_cmd('download', cmd_args=[self.cont, self.obj.lstrip('/'), '--no-download']) - with mock.patch('swiftclient.client._import_keystone_client', - self.fake_ks_import): + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): with mock.patch('swiftclient.client.http_connection', fake_conn): with mock.patch.dict(os.environ, env): with CaptureOutput() as out: @@ -3273,8 +3270,7 @@ def test_list_with_read_access(self): fake_conn = self.fake_http_connection(resp, on_request=req_handler) args, env = self._make_cmd('download', cmd_args=[self.cont]) - with mock.patch('swiftclient.client._import_keystone_client', - self.fake_ks_import): + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): with mock.patch('swiftclient.client.http_connection', fake_conn): with mock.patch.dict(os.environ, env): with CaptureOutput() as out: @@ -3291,8 +3287,7 @@ def test_list_with_no_access(self): fake_conn = self.fake_http_connection(403) args, env = self._make_cmd('download', cmd_args=[self.cont]) - with mock.patch('swiftclient.client._import_keystone_client', - self.fake_ks_import): + with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): with mock.patch('swiftclient.client.http_connection', fake_conn): with mock.patch.dict(os.environ, env): with CaptureOutput() as out: diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index 3303372d..f1147748 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -29,7 +29,7 @@ from requests.exceptions import RequestException from .utils import (MockHttpTest, fake_get_auth_keystone, StubResponse, - FakeKeystone, _make_fake_import_keystone_client) + FakeKeystone) from swiftclient.utils import EMPTY_ETAG from swiftclient.exceptions import ClientException @@ -322,8 +322,7 @@ def test_auth_v2_timeout(self): # TestConnection.test_timeout_passed_down but is required to check that # get_auth does the right thing when it is not passed a timeout arg fake_ks = FakeKeystone(endpoint='http://some_url', token='secret') - with mock.patch('swiftclient.client._import_keystone_client', - _make_fake_import_keystone_client(fake_ks)): + with mock.patch('swiftclient.client.ksclient_v2', fake_ks): c.get_auth('http://www.test.com', 'asdf', 'asdf', os_options=dict(tenant_name='tenant'), auth_version="2.0", timeout=42.0) @@ -580,8 +579,7 @@ def test_get_keystone_client_2_0(self): def test_get_auth_keystone_versionless(self): fake_ks = FakeKeystone(endpoint='http://some_url', token='secret') - with mock.patch('swiftclient.client._import_keystone_client', - _make_fake_import_keystone_client(fake_ks)): + with mock.patch('swiftclient.client.ksclient_v3', fake_ks): c.get_auth_keystone('http://authurl', 'user', 'key', {}) self.assertEqual(1, len(fake_ks.calls)) self.assertEqual('http://authurl/v3', fake_ks.calls[0].get('auth_url')) @@ -589,8 +587,7 @@ def test_get_auth_keystone_versionless(self): def test_get_auth_keystone_versionless_auth_version_set(self): fake_ks = FakeKeystone(endpoint='http://some_url', token='secret') - with mock.patch('swiftclient.client._import_keystone_client', - _make_fake_import_keystone_client(fake_ks)): + with mock.patch('swiftclient.client.ksclient_v2', fake_ks): c.get_auth_keystone('http://auth_url', 'user', 'key', {}, auth_version='2.0') self.assertEqual(1, len(fake_ks.calls)) @@ -600,8 +597,7 @@ def test_get_auth_keystone_versionless_auth_version_set(self): def test_get_auth_keystone_versionful(self): fake_ks = FakeKeystone(endpoint='http://some_url', token='secret') - with mock.patch('swiftclient.client._import_keystone_client', - _make_fake_import_keystone_client(fake_ks)): + with mock.patch('swiftclient.client.ksclient_v3', fake_ks): c.get_auth_keystone('http://auth_url/v3', 'user', 'key', {}, auth_version='3') self.assertEqual(1, len(fake_ks.calls)) @@ -611,8 +607,7 @@ def test_get_auth_keystone_versionful(self): def test_get_auth_keystone_devstack_versionful(self): fake_ks = FakeKeystone( endpoint='http://storage.example.com/v1/AUTH_user', token='secret') - with mock.patch('swiftclient.client._import_keystone_client', - _make_fake_import_keystone_client(fake_ks)): + with mock.patch('swiftclient.client.ksclient_v3', fake_ks): c.get_auth_keystone('https://192.168.8.8/identity/v3', 'user', 'key', {}, auth_version='3') self.assertEqual(1, len(fake_ks.calls)) @@ -622,8 +617,7 @@ def test_get_auth_keystone_devstack_versionful(self): def test_get_auth_keystone_devstack_versionless(self): fake_ks = FakeKeystone( endpoint='http://storage.example.com/v1/AUTH_user', token='secret') - with mock.patch('swiftclient.client._import_keystone_client', - _make_fake_import_keystone_client(fake_ks)): + with mock.patch('swiftclient.client.ksclient_v3', fake_ks): c.get_auth_keystone('https://192.168.8.8/identity', 'user', 'key', {}, auth_version='3') self.assertEqual(1, len(fake_ks.calls)) @@ -634,8 +628,7 @@ def test_auth_keystone_url_some_junk_nonsense(self): fake_ks = FakeKeystone( endpoint='http://storage.example.com/v1/AUTH_user', token='secret') - with mock.patch('swiftclient.client._import_keystone_client', - _make_fake_import_keystone_client(fake_ks)): + with mock.patch('swiftclient.client.ksclient_v3', fake_ks): c.get_auth_keystone('http://blah.example.com/v2moo', 'user', 'key', {}, auth_version='3') self.assertEqual(1, len(fake_ks.calls)) @@ -2456,8 +2449,7 @@ def shim_connection(*a, **kw): 'http://auth.example.com', 'user', 'password', timeout=33.0, os_options=os_options, auth_version=2.0) fake_ks = FakeKeystone(endpoint='http://some_url', token='secret') - with mock.patch('swiftclient.client._import_keystone_client', - _make_fake_import_keystone_client(fake_ks)): + with mock.patch('swiftclient.client.ksclient_v2', fake_ks): with mock.patch.multiple('swiftclient.client', http_connection=shim_connection, sleep=mock.DEFAULT): diff --git a/tests/unit/utils.py b/tests/unit/utils.py index 2def73f5..aab3b59c 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -542,14 +542,6 @@ class EndpointNotFound(Exception): pass -def _make_fake_import_keystone_client(fake_import): - def _fake_import_keystone_client(auth_version): - fake_import.auth_version = auth_version - return fake_import, fake_import - - return _fake_import_keystone_client - - class FakeStream(object): def __init__(self, size): self.bytes_read = 0 From a6baf00e245275634175d827552ffde7d655ad4d Mon Sep 17 00:00:00 2001 From: John Dickinson Date: Wed, 26 Sep 2018 18:29:18 -0700 Subject: [PATCH 046/238] py2 functional testing Change-Id: I24ff8fb28969a0b074313bc9491b299afac3b49c --- .zuul.yaml | 10 ++++++++++ tox.ini | 6 ++++++ 2 files changed, 16 insertions(+) diff --git a/.zuul.yaml b/.zuul.yaml index 6c37f8c1..9eb1752f 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -24,6 +24,14 @@ # Override value from parent job to use swiftclient tests zuul_work_dir: "{{ zuul.projects['git.openstack.org/openstack/python-swiftclient'].src_dir }}" +- job: + name: swiftclient-functional-py2 + parent: swiftclient-functional + description: | + Run functional tests of python-swiftclient under Python 2 + vars: + tox_envlist: py2func + - project: templates: - openstack-python-jobs @@ -38,11 +46,13 @@ jobs: - swiftclient-swift-functional - swiftclient-functional + - swiftclient-functional-py2 - openstack-tox-lower-constraints gate: jobs: - swiftclient-swift-functional - swiftclient-functional + - swiftclient-functional-py2 - openstack-tox-lower-constraints post: jobs: diff --git a/tox.ini b/tox.ini index ec7d3442..ed8d772e 100644 --- a/tox.ini +++ b/tox.ini @@ -46,6 +46,12 @@ commands = coverage report -m rm -f .coverage +[testenv:py2func] +basepython=python2 +setenv = {[testenv:func]setenv} +whitelist_externals = {[testenv:func]whitelist_externals} +commands = {[testenv:func]commands} + [testenv:docs] basepython = python3 commands= From 9acdfe0b460048420551bb84fb3cf41fb1e4a67e Mon Sep 17 00:00:00 2001 From: Vu Cong Tuan Date: Wed, 11 Jul 2018 14:33:41 +0700 Subject: [PATCH 047/238] Switch to stestr According to Openstack summit session [1], stestr is maintained project to which all Openstack projects should migrate. Let's switch to stestr as other projects have already moved to it. [1] https://etherpad.openstack.org/p/YVR-python-pti Change-Id: Ic098f8560599554e0b6bb16ae326d4d30a8a5504 --- .gitignore | 1 + .stestr.conf | 4 ++++ .testr.conf | 4 ---- lower-constraints.txt | 2 +- test-requirements.txt | 2 +- tox.ini | 23 +++++++++++++++++------ 6 files changed, 24 insertions(+), 12 deletions(-) create mode 100644 .stestr.conf delete mode 100644 .testr.conf diff --git a/.gitignore b/.gitignore index 16da5895..af50ddda 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ dist/ .DS_Store *.log .testrepository +.stestr/ subunit.log build swiftclient/versioninfo diff --git a/.stestr.conf b/.stestr.conf new file mode 100644 index 00000000..5228f209 --- /dev/null +++ b/.stestr.conf @@ -0,0 +1,4 @@ +[DEFAULT] +test_path=${OS_TEST_PATH:-./tests/unit} +top_dir=./ + diff --git a/.testr.conf b/.testr.conf deleted file mode 100644 index f3fca908..00000000 --- a/.testr.conf +++ /dev/null @@ -1,4 +0,0 @@ -[DEFAULT] -test_command=${PYTHON:-python} -m subunit.run discover -t ./ ${OS_TEST_PATH:-./tests/unit} $LISTOPT $IDOPTION -test_id_option=--load-list $IDFILE -test_list_option=--list diff --git a/lower-constraints.txt b/lower-constraints.txt index 9aae7927..fefb90ac 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -39,7 +39,7 @@ six==1.9.0 snowballstemmer==1.2.1 sphinx==1.6.2 sphinxcontrib-websupport==1.0.1 -testrepository==0.0.18 +stestr==2.0.0 testtools==2.2.0 traceback2==1.4.0 unittest2==1.1.0 diff --git a/test-requirements.txt b/test-requirements.txt index 634851e7..9e6b84a9 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -5,6 +5,6 @@ keystoneauth1>=3.4.0 # Apache-2.0 mock>=1.2.0 # BSD oslosphinx>=4.7.0 # Apache-2.0 sphinx!=1.6.6,!=1.6.7,>=1.6.2 # BSD -testrepository>=0.0.18 +stestr>=2.0.0 # Apache-2.0 reno>=2.5.0 # Apache-2.0 openstackdocstheme>=1.18.1 # Apache-2.0 diff --git a/tox.ini b/tox.ini index ed8d772e..26f8767e 100644 --- a/tox.ini +++ b/tox.ini @@ -16,8 +16,8 @@ deps = -r{toxinidir}/requirements.txt .[keystone] commands = sh -c '(find . -not \( -type d -name .?\* -prune \) \ \( -type d -name "__pycache__" -or -type f -name "*.py[co]" \) \ - -print0; find . -name "*.dbm*" -print0) | xargs -0 rm -rf' - python setup.py testr --testr-args="{posargs}" + -print0) | xargs -0 rm -rf' + stestr run {posargs} whitelist_externals = sh passenv = SWIFT_* *_proxy @@ -32,17 +32,28 @@ commands = {posargs} [testenv:cover] basepython = python3 -commands = python setup.py testr --coverage - coverage report +setenv = + PYTHON=coverage run --source swiftclient --parallel-mode +commands = + stestr run + coverage combine + coverage html -d cover + coverage xml -o cover/coverage.xml + coverage report [testenv:func] basepython = python3 -setenv = OS_TEST_PATH=tests.functional +setenv = + OS_TEST_PATH=tests.functional + PYTHON=coverage run --source swiftclient --parallel-mode whitelist_externals = coverage rm commands = - python setup.py testr --coverage --testr-args="--concurrency=1" + stestr run --concurrency=1 + coverage combine + coverage html -d cover + coverage xml -o cover/coverage.xml coverage report -m rm -f .coverage From 5e988c5cded09135f7e130704f169a1353730703 Mon Sep 17 00:00:00 2001 From: Nguyen Hai Truong Date: Tue, 6 Nov 2018 23:17:30 +0700 Subject: [PATCH 048/238] Add python 3.6 unit test job This is a mechanically generated patch to add a unit test job running under Python 3.6 as part of the python3-first goal. See the python3-first goal document for details: https://governance.openstack.org/tc/goals/stein/python3-first.html Change-Id: Iae4acab507e45a379c8af129912e13621a2a553b --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 26f8767e..53fb0a75 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,py35,pypy,pep8 +envlist = py36,py35,py27,pypy,pep8 minversion = 2.0 skipsdist = True From 411ef48e5bca1ed66a2e4dd7ecd8695e2bf6c94e Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Fri, 22 Jun 2018 16:49:03 -0700 Subject: [PATCH 049/238] Stop leaking quite so many connections While investigating the failures when you move func tests to py3, I noticed a whole bunch of ResourceWarning: unclosed noise. This should fix it. While we're at it, make get_capabilities less stupid. Change-Id: I3913e9334090b04a78143e0b70f621aad30fc642 Related-Change: I86d24104033b490a35178fc504d88c1e4a566628 --- swiftclient/client.py | 28 ++++++++++++++++------------ tests/functional/test_swiftclient.py | 1 + tests/unit/test_swiftclient.py | 8 +++++++- tests/unit/utils.py | 11 +++++++++-- 4 files changed, 33 insertions(+), 15 deletions(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index d843aecc..049b4afc 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -402,6 +402,7 @@ def __init__(self, url, proxy=None, cacert=None, insecure=False, self.request_session = requests.Session() # Don't use requests's default headers self.request_session.headers = None + self.resp = None if self.parsed_url.scheme not in ('http', 'https'): raise ClientException('Unsupported scheme "%s" in url "%s"' % (self.parsed_url.scheme, url)) @@ -506,6 +507,11 @@ def releasing_read(*args, **kwargs): return self.resp + def close(self): + if self.resp: + self.resp.close() + self.request_session.close() + def http_connection(*arg, **kwarg): """:returns: tuple of (parsed url, connection object)""" @@ -527,6 +533,8 @@ def get_auth_1_0(url, user, key, snet, **kwargs): conn.request(method, parsed.path, '', headers) resp = conn.getresponse() body = resp.read() + resp.close() + conn.close() http_log((url, method,), headers, resp, body) url = resp.getheader('x-storage-url') @@ -1651,11 +1659,8 @@ def close(self): if (self.http_conn and isinstance(self.http_conn, tuple) and len(self.http_conn) > 1): conn = self.http_conn[1] - if hasattr(conn, 'close') and callable(conn.close): - # XXX: Our HTTPConnection object has no close, should be - # trying to close the requests.Session here? - conn.close() - self.http_conn = None + conn.close() + self.http_conn = None def get_auth(self): self.url, self.token = get_auth(self.authurl, self.user, self.key, @@ -1715,10 +1720,10 @@ def _retry(self, reset_func, func, *args, **kwargs): try: if not self.url or not self.token: self.url, self.token = self.get_auth() - self.http_conn = None + self.close() if self.service_auth and not self.service_token: self.url, self.service_token = self.get_service_auth() - self.http_conn = None + self.close() self.auth_end_time = time() if not self.http_conn: self.http_conn = self.http_connection() @@ -1908,8 +1913,7 @@ def get_capabilities(self, url=None): url = url or self.url if not url: url, _ = self.get_auth() - scheme = urlparse(url).scheme - netloc = urlparse(url).netloc - url = scheme + '://' + netloc + '/info' - http_conn = self.http_connection(url) - return get_capabilities(http_conn) + parsed = urlparse(urljoin(url, '/info')) + if not self.http_conn: + self.http_conn = self.http_connection(url) + return get_capabilities((parsed, self.http_conn[1])) diff --git a/tests/functional/test_swiftclient.py b/tests/functional/test_swiftclient.py index 0380d961..1d76a8d1 100644 --- a/tests/functional/test_swiftclient.py +++ b/tests/functional/test_swiftclient.py @@ -108,6 +108,7 @@ def tearDown(self): self.conn.delete_container(container) except swiftclient.ClientException: pass + self.conn.close() def _check_account_headers(self, headers): headers_to_check = [ diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index f1147748..62875a5d 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -2532,6 +2532,9 @@ def getheaders(self): def read(self, *args, **kwargs): return '' + def close(self): + pass + def local_http_connection(url, proxy=None, cacert=None, insecure=False, cert=None, cert_key=None, ssl_compression=True, timeout=None): @@ -2901,6 +2904,9 @@ def test_close_none(self): self.assertIsNone(conn.http_conn) conn.close() self.assertIsNone(conn.http_conn) + # Can re-close + conn.close() + self.assertIsNone(conn.http_conn) def test_close_ok(self): url = 'http://www.test.com' @@ -2911,7 +2917,7 @@ def test_close_ok(self): self.assertEqual(len(conn.http_conn), 2) http_conn_obj = conn.http_conn[1] self.assertIsInstance(http_conn_obj, c.HTTPConnection) - self.assertFalse(hasattr(http_conn_obj, 'close')) + self.assertTrue(hasattr(http_conn_obj, 'close')) conn.close() diff --git a/tests/unit/utils.py b/tests/unit/utils.py index aab3b59c..8081501f 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -78,6 +78,10 @@ def __init__(self, status=200, body='', headers=None): self.body = body self.headers = headers or {} + def __repr__(self): + return '%s(%r, %r, %r)' % (self.__class__.__name__, self.status, + self.body, self.headers) + def fake_http_connect(*code_iter, **kwargs): """ @@ -102,7 +106,6 @@ def __init__(self, status, etag=None, body='', timestamp='1', self.etag = etag self.content = self.body = body self.timestamp = timestamp - self._is_closed = True self.headers = headers or {} self.request = None @@ -162,6 +165,9 @@ def send(self, amt=None): def getheader(self, name, default=None): return dict(self.getheaders()).get(name.lower(), default) + def close(self): + pass + timestamps_iter = iter(kwargs.get('timestamps') or ['1'] * len(code_iter)) etag_iter = iter(kwargs.get('etags') or [None] * len(code_iter)) x = kwargs.get('missing_container', [False] * len(code_iter)) @@ -228,7 +234,8 @@ def wrapper(url, proxy=None, cacert=None, insecure=False, parsed, _conn = _orig_http_connection(url, proxy=proxy) class RequestsWrapper(object): - pass + def close(self): + pass conn = RequestsWrapper() def request(method, path, *args, **kwargs): From 0197a0f9ca406a7314650d9c46b300ece3eba6b6 Mon Sep 17 00:00:00 2001 From: OpenStack Release Bot Date: Wed, 25 Jul 2018 16:51:52 +0000 Subject: [PATCH 050/238] Update reno for stable/rocky Change-Id: I840f4363dfdb3b485dbaf768c71fbcc5227c330f --- releasenotes/source/index.rst | 1 + releasenotes/source/rocky.rst | 6 ++++++ tox.ini | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 releasenotes/source/rocky.rst diff --git a/releasenotes/source/index.rst b/releasenotes/source/index.rst index a5240ea2..92da0e8f 100644 --- a/releasenotes/source/index.rst +++ b/releasenotes/source/index.rst @@ -6,6 +6,7 @@ :maxdepth: 1 current + rocky queens pike ocata diff --git a/releasenotes/source/rocky.rst b/releasenotes/source/rocky.rst new file mode 100644 index 00000000..40dd517b --- /dev/null +++ b/releasenotes/source/rocky.rst @@ -0,0 +1,6 @@ +=================================== + Rocky Series Release Notes +=================================== + +.. release-notes:: + :branch: stable/rocky diff --git a/tox.ini b/tox.ini index 660248b4..46d24f9f 100644 --- a/tox.ini +++ b/tox.ini @@ -8,7 +8,7 @@ usedevelop = True install_command = python -m pip install -U {opts} {packages} list_dependencies_command = python -m pip freeze setenv = - LANG=en_US.utf8 + LANG=en_US.utf-8 VIRTUAL_ENV={envdir} deps = -r{toxinidir}/requirements.txt From fc128672f63fe663950e13da105e084321937fe5 Mon Sep 17 00:00:00 2001 From: Thiago da Silva Date: Sat, 24 Nov 2018 08:11:26 -0500 Subject: [PATCH 051/238] update .functests to run stestr Updated .functests script to run similar to how it's defined in tox.ini Change-Id: I17df28d8cbe0e10e48b26c2f9737308552ea88db --- .functests | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.functests b/.functests index 16c9e5de..d199ec84 100755 --- a/.functests +++ b/.functests @@ -2,9 +2,13 @@ set -e export OS_TEST_PATH='tests.functional' -python setup.py testr --coverage --testr-args="--concurrency=1" +export PYTHON='coverage run --source swiftclient --parallel-mode' +stestr run --concurrency=1 RET=$? +coverage combine +coverage html -d cover +coverage xml -o cover/coverage.xml coverage report -m rm -f .coverage exit $RET From 8a4228146899534f906f8e10ec9040ab3f1ee970 Mon Sep 17 00:00:00 2001 From: qingszhao Date: Fri, 30 Nov 2018 07:31:51 +0000 Subject: [PATCH 052/238] Add Python 3.6 classifier to setup.cfg Change-Id: If3b2cdcd009136286d68fe07b14e06261b3069a0 --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index ddb64a07..531458fa 100644 --- a/setup.cfg +++ b/setup.cfg @@ -18,6 +18,7 @@ classifier = Programming Language :: Python :: 2.7 Programming Language :: Python :: 3 Programming Language :: Python :: 3.5 + Programming Language :: Python :: 3.6 [global] setup-hooks = From edfeae372312b3370dc12deea8cd8028ecba6bd6 Mon Sep 17 00:00:00 2001 From: Timur Alperovich Date: Fri, 23 Nov 2018 22:47:15 -0800 Subject: [PATCH 053/238] Add delimiter to get_account(). Exposes the delimiter parameter, which the Swift API supports for container listings. Change-Id: Id8dfce01a9b64de9d1222aab9a4a682ce9e0f2b7 --- swiftclient/client.py | 19 ++++++++++++------- tests/functional/test_swiftclient.py | 12 ++++++++++++ tests/unit/test_swiftclient.py | 12 ++++++++++++ 3 files changed, 36 insertions(+), 7 deletions(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index 049b4afc..c9efc79b 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -757,7 +757,7 @@ def store_response(resp, response_dict): def get_account(url, token, marker=None, limit=None, prefix=None, end_marker=None, http_conn=None, full_listing=False, - service_token=None, headers=None): + service_token=None, headers=None, delimiter=None): """ Get a listing of containers for the account. @@ -773,6 +773,7 @@ def get_account(url, token, marker=None, limit=None, prefix=None, of 10000 listings :param service_token: service auth token :param headers: additional headers to include in the request + :param delimiter: delimiter query :returns: a tuple of (response headers, a list of containers) The response headers will be a dict and all header names will be lowercase. :raises ClientException: HTTP GET request failed @@ -786,14 +787,14 @@ def get_account(url, token, marker=None, limit=None, prefix=None, if not http_conn: http_conn = http_connection(url) if full_listing: - rv = get_account(url, token, marker, limit, prefix, - end_marker, http_conn, headers=req_headers) + rv = get_account(url, token, marker, limit, prefix, end_marker, + http_conn, headers=req_headers, delimiter=delimiter) listing = rv[1] while listing: marker = listing[-1]['name'] listing = get_account(url, token, marker, limit, prefix, - end_marker, http_conn, - headers=req_headers)[1] + end_marker, http_conn, headers=req_headers, + delimiter=delimiter)[1] if listing: rv[1].extend(listing) return rv @@ -805,6 +806,8 @@ def get_account(url, token, marker=None, limit=None, prefix=None, qs += '&limit=%d' % limit if prefix: qs += '&prefix=%s' % quote(prefix) + if delimiter: + qs += '&delimiter=%s' % quote(delimiter) if end_marker: qs += '&end_marker=%s' % quote(end_marker) full_path = '%s?%s' % (parsed.path, qs) @@ -1779,14 +1782,16 @@ def head_account(self, headers=None): return self._retry(None, head_account, headers=headers) def get_account(self, marker=None, limit=None, prefix=None, - end_marker=None, full_listing=False, headers=None): + end_marker=None, full_listing=False, headers=None, + delimiter=None): """Wrapper for :func:`get_account`""" # TODO(unknown): With full_listing=True this will restart the entire # listing with each retry. Need to make a better version that just # retries where it left off. return self._retry(None, get_account, marker=marker, limit=limit, prefix=prefix, end_marker=end_marker, - full_listing=full_listing, headers=headers) + full_listing=full_listing, headers=headers, + delimiter=delimiter) def post_account(self, headers, response_dict=None, query_string=None, data=None): diff --git a/tests/functional/test_swiftclient.py b/tests/functional/test_swiftclient.py index 1d76a8d1..b4f275b2 100644 --- a/tests/functional/test_swiftclient.py +++ b/tests/functional/test_swiftclient.py @@ -154,6 +154,18 @@ def test_list_account(self): self.assertTrue(len(containers) >= 1) self.assertEqual(self.containername_2, containers[0].get('name')) + # Test prefix + _, containers = self.conn.get_account(prefix='dne') + self.assertEqual(0, len(containers)) + + # Test delimiter + _, containers = self.conn.get_account( + prefix=self.containername, delimiter='_') + self.assertEqual(2, len(containers)) + self.assertEqual(self.containername, containers[0].get('name')) + self.assertTrue( + self.containername_2.startswith(containers[1].get('subdir'))) + def _check_container_headers(self, headers): self.assertTrue(headers.get('content-length')) self.assertTrue(headers.get('x-container-object-count')) diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index 62875a5d..2d45deb8 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -704,6 +704,18 @@ def test_param_end_marker(self): 'x-auth-token': 'asdf'}), ]) + def test_param_delimiter(self): + c.http_connection = self.fake_http_connection( + 204, + query_string="format=json&delimiter=-") + c.get_account('http://www.test.com/v1/acct', 'asdf', + delimiter='-') + self.assertRequests([ + ('GET', '/v1/acct?format=json&delimiter=-', '', { + 'accept-encoding': 'gzip', + 'x-auth-token': 'asdf'}), + ]) + class TestHeadAccount(MockHttpTest): From 0e6c63dfd49fcc18108cc248df0c1731ae0f1197 Mon Sep 17 00:00:00 2001 From: sunjia Date: Mon, 3 Dec 2018 22:01:15 -0500 Subject: [PATCH 054/238] Change openstack-dev to openstack-discuss Mailinglists have been updated. Openstack-discuss replaces openstack-dev. Change-Id: I3193f2d12f75c36b59881a51b605d25274b335e0 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 531458fa..cdf10b48 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,7 +4,7 @@ summary = OpenStack Object Storage API Client Library description-file = README.rst author = OpenStack -author-email = openstack-dev@lists.openstack.org +author-email = openstack-discuss@lists.openstack.org home-page = https://docs.openstack.org/python-swiftclient/latest/ classifier = Environment :: OpenStack From 9da26369125c91632964abccf00fa8288d91315c Mon Sep 17 00:00:00 2001 From: Andreas Jaeger Date: Thu, 20 Dec 2018 21:52:45 +0100 Subject: [PATCH 055/238] Use template for lower-constraints Small cleanups: * Use openstack-lower-constraints-jobs template, remove individual jobs. * Sort list of templates Change-Id: Idb31ca14478641cba6f896af35fa766d7bdb9e1e Needed-By: https://review.openstack.org/623229 --- .zuul.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index 9eb1752f..56c9faaf 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -34,26 +34,25 @@ - project: templates: + - check-requirements + - lib-forward-testing + - openstack-lower-constraints-jobs + - openstack-pypy-jobs-nonvoting - openstack-python-jobs - openstack-python35-jobs - openstack-python36-jobs - publish-openstack-docs-pti - - check-requirements - - openstack-pypy-jobs-nonvoting - - lib-forward-testing - release-notes-jobs-python3 check: jobs: - swiftclient-swift-functional - swiftclient-functional - swiftclient-functional-py2 - - openstack-tox-lower-constraints gate: jobs: - swiftclient-swift-functional - swiftclient-functional - swiftclient-functional-py2 - - openstack-tox-lower-constraints post: jobs: - openstack-tox-cover From 2ff36fde575fa6987ee5954f526ad6c9460633a5 Mon Sep 17 00:00:00 2001 From: ZhijunWei Date: Fri, 28 Dec 2018 23:04:37 +0800 Subject: [PATCH 056/238] Update hacking version 1. update hacking version to latest 2. fix pep8 failed Change-Id: Ifc3bfeff4038c93d8c8cf2c9d7814c3003e73504 --- swiftclient/client.py | 14 +++++++------- swiftclient/service.py | 26 +++++++++++++------------- swiftclient/shell.py | 5 +++-- test-requirements.txt | 2 +- tests/unit/test_shell.py | 12 ++++++------ tox.ini | 2 +- 6 files changed, 31 insertions(+), 30 deletions(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index c9efc79b..6a2b43b2 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -248,8 +248,8 @@ def encode_meta_headers(headers): value = encode_utf8(value) header = header.lower() - if (isinstance(header, six.string_types) - and header.startswith(USER_METADATA_TYPE)): + if (isinstance(header, six.string_types) and + header.startswith(USER_METADATA_TYPE)): header = encode_utf8(header) ret[header] = value @@ -706,9 +706,9 @@ def get_auth(auth_url, user, key, **kwargs): if kwargs.get('tenant_name'): os_options['tenant_name'] = kwargs['tenant_name'] - if not (os_options.get('tenant_name') or os_options.get('tenant_id') - or os_options.get('project_name') - or os_options.get('project_id')): + if not (os_options.get('tenant_name') or os_options.get('tenant_id') or + os_options.get('project_name') or + os_options.get('project_id')): if auth_version in AUTH_VERSIONS_V2: raise ClientException('No tenant specified') raise ClientException('No project name or project id specified.') @@ -1659,8 +1659,8 @@ def __init__(self, authurl=None, user=None, key=None, retries=5, self.force_auth_retry = force_auth_retry def close(self): - if (self.http_conn and isinstance(self.http_conn, tuple) - and len(self.http_conn) > 1): + if (self.http_conn and isinstance(self.http_conn, tuple) and + len(self.http_conn) > 1): conn = self.http_conn[1] conn.close() self.http_conn = None diff --git a/swiftclient/service.py b/swiftclient/service.py index eedad46e..71c36ec6 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -424,8 +424,8 @@ def _check_contents(self): '{1} != {2}'.format( self._path, etag, self._expected_md5)) - if (self._content_length is not None - and self._actual_read != self._content_length): + if (self._content_length is not None and + self._actual_read != self._content_length): raise SwiftError('Error downloading {0}: read_length != ' 'content_length, {1:d} != {2:d}'.format( self._path, self._actual_read, @@ -1244,8 +1244,8 @@ def _download_object_job(self, conn, container, obj, options): bytes_read = obj_body.bytes_read() if fp is not None: fp.close() - if ('x-object-meta-mtime' in headers and not no_file - and not options['ignore_mtime']): + if ('x-object-meta-mtime' in headers and not no_file and + not options['ignore_mtime']): try: mtime = float(headers['x-object-meta-mtime']) except ValueError: @@ -2036,8 +2036,8 @@ def _upload_object_job(self, conn, container, source, obj, options, new_slo_manifest_paths = set() segment_size = int(0 if options['segment_size'] is None else options['segment_size']) - if (options['changed'] or options['skip_identical'] - or not options['leave_segments']): + if (options['changed'] or options['skip_identical'] or + not options['leave_segments']): try: headers = conn.head_object(container, obj) is_slo = config_true_value( @@ -2058,9 +2058,9 @@ def _upload_object_job(self, conn, container, source, obj, options, cl = int(headers.get('content-length')) mt = headers.get('x-object-meta-mtime') - if (path is not None and options['changed'] - and cl == getsize(path) - and mt == put_headers['x-object-meta-mtime']): + if (path is not None and options['changed'] and + cl == getsize(path) and + mt == put_headers['x-object-meta-mtime']): res.update({ 'success': True, 'status': 'skipped-changed' @@ -2095,8 +2095,8 @@ def _upload_object_job(self, conn, container, source, obj, options, # a segment job if we're reading from a stream - we may fail if we # go over the single object limit, but this gives us a nice way # to create objects from memory - if (path is not None and segment_size - and (getsize(path) > segment_size)): + if (path is not None and segment_size and + (getsize(path) > segment_size)): res['large_object'] = True seg_container = container + '_segments' if options['segment_container']: @@ -2425,8 +2425,8 @@ def delete(self, container=None, objects=None, options=None): # Cancel the remaining container deletes, but yield # any pending results - if (not cancelled and options['fail_fast'] - and not res['success']): + if (not cancelled and options['fail_fast'] and + not res['success']): cancelled = True def _bulk_delete_page_size(self, objects): diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 9ca28b1e..9ea5e952 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -1462,8 +1462,9 @@ def st_bash_completion(parser, args, thread_manager, return_parser=False): return else: fn_commands = [fn for fn in globals().keys() - if fn.startswith('st_') and not fn.endswith('_options') - and not fn.endswith('_help')] + if fn.startswith('st_') and + not fn.endswith('_options') and + not fn.endswith('_help')] subparsers = parser.add_subparsers() subcommands = {} diff --git a/test-requirements.txt b/test-requirements.txt index 9e6b84a9..13cf1e93 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,4 +1,4 @@ -hacking<0.11,>=0.10.0 +hacking>=1.1.0,<1.2.0 # Apache-2.0 coverage!=4.4,>=4.0 # Apache-2.0 keystoneauth1>=3.4.0 # Apache-2.0 diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index 9ef46859..f5d2f15b 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -1996,8 +1996,8 @@ def _remove_swift_env_vars(self): self._environ_vars = {} keys = list(os.environ.keys()) for k in keys: - if (k in ('ST_KEY', 'ST_USER', 'ST_AUTH') - or k.startswith('OS_')): + if (k in ('ST_KEY', 'ST_USER', 'ST_AUTH') or + k.startswith('OS_')): self._environ_vars[k] = os.environ.pop(k) def _replace_swift_env_vars(self): @@ -2979,12 +2979,12 @@ def on_request(method, path, *args, **kwargs): Modify response code to 200 if cross account permissions match. """ status = 403 - if (path.startswith('/v1/%s/%s' % (self.account, self.cont)) - and read_ok and method in ('GET', 'HEAD')): + if (path.startswith('/v1/%s/%s' % (self.account, self.cont)) and + read_ok and method in ('GET', 'HEAD')): status = 200 elif (path.startswith('/v1/%s/%s%s' - % (self.account, self.cont, self.obj)) - and write_ok and method in ('PUT', 'POST', 'DELETE')): + % (self.account, self.cont, self.obj)) and + write_ok and method in ('PUT', 'POST', 'DELETE')): status = 200 return status return on_request diff --git a/tox.ini b/tox.ini index bd20632a..fb639d00 100644 --- a/tox.ini +++ b/tox.ini @@ -77,7 +77,7 @@ commands= # H403: multi line docstrings should end on a new line # H404: multi line docstring should start without a leading new line # H405: multi line docstring summary not separated with an empty line -ignore = H101,H301,H306,H401,H403,H404,H405 +ignore = E731,H101,H301,H306,H401,H403,H404,H405 show-source = True exclude = .venv,.tox,dist,doc,*egg From 0ee7c8272e0de2d8c44fd98306d54fc290d74f38 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Thu, 21 Feb 2019 09:26:10 -0800 Subject: [PATCH 057/238] Make proper functions instead of assigning lambdas Change-Id: I89255f6923c649c7b9d3d36e96c09f8bc4f51a3c --- swiftclient/client.py | 4 +++- swiftclient/service.py | 4 +++- tox.ini | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index 6a2b43b2..44066891 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -1885,7 +1885,9 @@ def _default_reset(*args, **kwargs): reset = getattr(contents, 'reset', None) if tell and seek: orig_pos = tell() - reset_func = lambda *a, **k: seek(orig_pos) + + def reset_func(*a, **kw): + seek(orig_pos) elif reset: reset_func = reset return self._retry(reset_func, put_object, container, obj, contents, diff --git a/swiftclient/service.py b/swiftclient/service.py index 71c36ec6..8f3648ee 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -451,7 +451,9 @@ def __init__(self, options=None): **_default_local_options ) process_options(self._options) - create_connection = lambda: get_conn(self._options) + + def create_connection(): + return get_conn(self._options) self.thread_manager = MultiThreadingManager( create_connection, segment_threads=self._options['segment_threads'], diff --git a/tox.ini b/tox.ini index fb639d00..bd20632a 100644 --- a/tox.ini +++ b/tox.ini @@ -77,7 +77,7 @@ commands= # H403: multi line docstrings should end on a new line # H404: multi line docstring should start without a leading new line # H405: multi line docstring summary not separated with an empty line -ignore = E731,H101,H301,H306,H401,H403,H404,H405 +ignore = H101,H301,H306,H401,H403,H404,H405 show-source = True exclude = .venv,.tox,dist,doc,*egg From fd6e76029dc725ad48ae4e250f1c70a58740cad7 Mon Sep 17 00:00:00 2001 From: wangzhenyu Date: Tue, 27 Jun 2017 18:15:51 +0800 Subject: [PATCH 058/238] Enable some off-by-default checks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Some of the available checks are disabled by default, like: [H106] Don’t put vim configuration in source files [H203] Use assertIs(Not)None to check for None Change-Id: I36a6997fdb806b4d0a9d064107cc1451c766c987 --- tox.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tox.ini b/tox.ini index bd20632a..49de93dc 100644 --- a/tox.ini +++ b/tox.ini @@ -78,6 +78,9 @@ commands= # H404: multi line docstring should start without a leading new line # H405: multi line docstring summary not separated with an empty line ignore = H101,H301,H306,H401,H403,H404,H405 +# H106: Don’t put vim configuration in source files +# H203: Use assertIs(Not)None to check for None +enable-extensions=H106,H203 show-source = True exclude = .venv,.tox,dist,doc,*egg From d32deaadddeaf5e5364acfb3a7f727594d1bc581 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Wed, 27 Feb 2019 11:08:28 -0800 Subject: [PATCH 059/238] Add py37 check/gate jobs; add py37 to default tox env list Change-Id: Ifbc6dc731df20b4bba905a110e71ea5c9cc52c0f --- .zuul.yaml | 1 + setup.cfg | 1 + tox.ini | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.zuul.yaml b/.zuul.yaml index 56c9faaf..f9c22665 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -41,6 +41,7 @@ - openstack-python-jobs - openstack-python35-jobs - openstack-python36-jobs + - openstack-python37-jobs - publish-openstack-docs-pti - release-notes-jobs-python3 check: diff --git a/setup.cfg b/setup.cfg index cdf10b48..d3b13a6b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,6 +19,7 @@ classifier = Programming Language :: Python :: 3 Programming Language :: Python :: 3.5 Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 [global] setup-hooks = diff --git a/tox.ini b/tox.ini index 22ef5266..613e3f86 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py36,py35,py27,pypy,pep8 +envlist = py37,py36,py35,py27,pypy,pep8 minversion = 2.0 skipsdist = True From 344711771db6f3bbeaf85cbb1aa787b89d8d08e3 Mon Sep 17 00:00:00 2001 From: John Dickinson Date: Tue, 26 Feb 2019 16:51:49 -0800 Subject: [PATCH 060/238] authors/changelog updates for release Change-Id: Ic14916c314043155a5ec3c5b29331862c6aded43 --- .mailmap | 2 ++ AUTHORS | 9 ++++++++- ChangeLog | 12 ++++++++++++ releasenotes/notes/361_notes-59e020e68bcdd709.yaml | 12 ++++++++++++ 4 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/361_notes-59e020e68bcdd709.yaml diff --git a/.mailmap b/.mailmap index ecbcad1f..bc092300 100644 --- a/.mailmap +++ b/.mailmap @@ -95,3 +95,5 @@ Andreas Jaeger Shashi Kant Nandini Tata Flavio Percoco +Timur Alperovich +Thiago da Silva diff --git a/AUTHORS b/AUTHORS index abc21e98..1fcf65d4 100644 --- a/AUTHORS +++ b/AUTHORS @@ -92,6 +92,7 @@ Monty Taylor (mordred@inaugust.com) Nandini Tata (nandini.tata@intel.com) Nelson Marcos (nelsonmarcos@gmail.com) Nguyen Hai (nguyentrihai93@gmail.com) +Nguyen Hai Truong (truongnh@vn.fujitsu.com) Nguyen Hung Phuong (phuongnh@vn.fujitsu.com) Nick Craig-Wood (nick@craig-wood.com) Ondrej Novy (ondrej.novy@firma.seznam.cz) @@ -103,6 +104,7 @@ Peter Lisak (peter.lisak@firma.seznam.cz) Petr Kovar (pkovar@redhat.com) Pradeep Kumar Singh (pradeep.singh@nectechnologies.in) Pratik Mallya (pratik.mallya@gmail.com) +qingszhao (zhao.daqing@99cloud.net) Qiu Yu (qiuyu@ebaysf.com) Ray Chen (oldsharp@163.com) ricolin (rico.l@inwinstack.com) @@ -124,11 +126,12 @@ Stanislaw Pitucha (stanislaw.pitucha@hpe.com) Steve Martinelli (stevemar@ca.ibm.com) Steven Hardy (shardy@redhat.com) Stuart McLaren (stuart.mclaren@hpe.com) +sunjia (sunjia@inspur.com) Sushil Kumar (sushil.kumar2@globallogic.com) tanlin (lin.tan@intel.com) Taurus Cheung (Taurus.Cheung@harmonicinc.com) TheSriram (sriram@klusterkloud.com) -Thiago da Silva (thiago@redhat.com) +Thiago da Silva (thiagodasilva@gmail.com) Thomas Goirand (thomas@goirand.fr) Tihomir Trifonov (t.trifonov@gmail.com) Tim Burke (tim.burke@gmail.com) @@ -141,9 +144,11 @@ Vasyl Khomenko (vasiliyk@yahoo-inc.com) venkatamahesh (venkatamaheshkotha@gmail.com) Victor Stinner (victor.stinner@enovance.com) Vitaly Gridnev (vgridnev@mirantis.com) +Vu Cong Tuan (tuanvc@vn.fujitsu.com) wangqi (wang.qi@99cloud.net) wangxiyuan (wangxiyuan@huawei.com) Wu Wenxiang (wu.wenxiang@99cloud.net) +wu.chunyang (wu.chunyang@99cloud.net) YangLei (yanglyy@cn.ibm.com) yangxurong (yangxurong@huawei.com) You Yamagata (bi.yamagata@gmail.com) @@ -157,3 +162,5 @@ zhang-jinnan (ben.os@99cloud.net) zhangyanxian (zhangyanxianmail@163.com) zheng yin (yin.zheng@easystack.cn) Zhenguo Niu (zhenguo@unitedstack.com) +ZhijunWei (wzj334965317@outlook.com) +zhubx007 (zhu.boxiang@99cloud.net) diff --git a/ChangeLog b/ChangeLog index a37a6db0..26b46eaa 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,15 @@ +3.6.1 +----- + +* Added the delimiter keyword parameter to `get_account()` to match the + functionality of `get_container()`. + +* Fixed an issue in the client module where socket connections weren't + closed properly before being dereferenced. + +* Various other minor bug fixes and improvements. + + 3.6.0 ----- diff --git a/releasenotes/notes/361_notes-59e020e68bcdd709.yaml b/releasenotes/notes/361_notes-59e020e68bcdd709.yaml new file mode 100644 index 00000000..f6b48927 --- /dev/null +++ b/releasenotes/notes/361_notes-59e020e68bcdd709.yaml @@ -0,0 +1,12 @@ +--- +fixes: + - | + Added the delimiter keyword parameter to ``get_account()`` to match the + functionality of ``get_container()``. + + - | + Fixed an issue in the client module where socket connections weren't + closed properly before being dereferenced. + + - | + Various other minor bug fixes and improvements. From 991a6cebb900f9ece5279f1c3c2bf33bcbb74086 Mon Sep 17 00:00:00 2001 From: Thiago da Silva Date: Mon, 4 Mar 2019 11:55:46 -0500 Subject: [PATCH 061/238] Update release to 3.7.0 Due to Openstack Release policies, the next release needs to be a minor release, thus bumping to 3.7.0. Change-Id: If52d48908cfd47c5b94265ebd9ab8e3289c7b19c --- ChangeLog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 26b46eaa..6f6bf8bb 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,4 @@ -3.6.1 +3.7.0 ----- * Added the delimiter keyword parameter to `get_account()` to match the From 5333f3e98ab446e24faf9e421d83b8b04f3fccba Mon Sep 17 00:00:00 2001 From: Dirk Mueller Date: Sat, 23 Mar 2019 23:19:45 +0100 Subject: [PATCH 062/238] Remove oslosphinx usage The client actually uses the newer openstackdocstheme. Change-Id: If78d5fba58cf9e611253259fcdff2191ad3b8709 --- lower-constraints.txt | 1 - test-requirements.txt | 1 - 2 files changed, 2 deletions(-) diff --git a/lower-constraints.txt b/lower-constraints.txt index fefb90ac..ab45e39d 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -22,7 +22,6 @@ mock==1.2.0 netaddr==0.7.10 openstackdocstheme==1.18.1 oslo.config==1.2.0 -oslosphinx==4.7.0 pbr==2.0.0 pep8==1.5.7 PrettyTable==0.7 diff --git a/test-requirements.txt b/test-requirements.txt index 13cf1e93..d8222142 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -3,7 +3,6 @@ hacking>=1.1.0,<1.2.0 # Apache-2.0 coverage!=4.4,>=4.0 # Apache-2.0 keystoneauth1>=3.4.0 # Apache-2.0 mock>=1.2.0 # BSD -oslosphinx>=4.7.0 # Apache-2.0 sphinx!=1.6.6,!=1.6.7,>=1.6.2 # BSD stestr>=2.0.0 # Apache-2.0 reno>=2.5.0 # Apache-2.0 From 7563d9cb5642c85a31708fd863e2737f9d6c9419 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Mon, 25 Mar 2019 09:35:47 -0700 Subject: [PATCH 063/238] docs: Clean up formatting Change-Id: I0bcaf15c54dd3b3c590a543569699fe8ec5b0c7c --- doc/source/cli/index.rst | 40 +++++++---------- doc/source/index.rst | 20 ++++----- doc/source/introduction.rst | 90 ++++++++++++++++++------------------- doc/source/service-api.rst | 54 +++++++++++----------- 4 files changed, 99 insertions(+), 105 deletions(-) diff --git a/doc/source/cli/index.rst b/doc/source/cli/index.rst index 88fafa1d..d6841d55 100644 --- a/doc/source/cli/index.rst +++ b/doc/source/cli/index.rst @@ -245,13 +245,10 @@ storage URL options shown below: --os-storage-url https://10.1.5.2:8080/v1/AUTH_ced809b6a4baea7aeab61a \ list -.. We need the backslash below in order to indent the note -\ +.. note:: - .. note:: - - Leftover environment variables are a common source of confusion when - authorization fails. + Leftover environment variables are a common source of confusion when + authorization fails. CLI commands ~~~~~~~~~~~~ @@ -739,15 +736,15 @@ is passed, the Unix timestamp when the temporary URL will expire. But beyond that, ``time`` can also be specified as an ISO 8601 timestamp in one of following formats: - i) Complete date: YYYY-MM-DD (eg 1997-07-16) +i) Complete date: YYYY-MM-DD (e.g. 1997-07-16) - ii) Complete date plus hours, minutes and seconds: - YYYY-MM-DDThh:mm:ss - (eg 1997-07-16T19:20:30) +ii) Complete date plus hours, minutes and seconds: + YYYY-MM-DDThh:mm:ss + (e.g. 1997-07-16T19:20:30) - iii) Complete date plus hours, minutes and seconds with UTC designator: - YYYY-MM-DDThh:mm:ssZ - (eg 1997-07-16T19:20:30Z) +iii) Complete date plus hours, minutes and seconds with UTC designator: + YYYY-MM-DDThh:mm:ssZ + (e.g. 1997-07-16T19:20:30Z) Please be aware that if you don't provide the UTC designator (i.e., Z) the timestamp is generated using your local timezone. If only a date is @@ -881,17 +878,14 @@ Download an object from a container: testSwift.txt [auth 0.028s, headers 0.045s, total 0.045s, 0.002 MB/s] -.. We need the backslash below in order to indent the note -\ - - .. note:: +.. note:: - To upload an object to a container, your current working directory must be - where the file is located or you must provide the complete path to the file. - In other words, the --object-name is an option that will upload - file and name object to or upload directory and use as - object prefix. In the case that you provide the complete path of the file, - that complete path will be the name of the uploaded object. + To upload an object to a container, your current working directory must be + where the file is located or you must provide the complete path to the file. + In other words, the --object-name is an option that will upload + file and name object to or upload directory and use as + object prefix. In the case that you provide the complete path of the file, + that complete path will be the name of the uploaded object. For example: diff --git a/doc/source/index.rst b/doc/source/index.rst index 3c2cb1eb..ab05c6bd 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -39,17 +39,17 @@ Indices and tables License ~~~~~~~ - Copyright 2013 OpenStack, LLC. +Copyright 2013 OpenStack, LLC. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 +* http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/doc/source/introduction.rst b/doc/source/introduction.rst index 926b1b90..d6f98195 100644 --- a/doc/source/introduction.rst +++ b/doc/source/introduction.rst @@ -16,41 +16,41 @@ Use Cases Alongside the command line tool, the ``python-swiftclient`` includes two levels of API: - * A low level client API that provides simple Python wrappers around the - various authentication mechanisms and the individual HTTP requests. - * A high level service API that provides methods for performing common - operations in parallel on a thread pool. +* A low level client API that provides simple Python wrappers around the + various authentication mechanisms and the individual HTTP requests. +* A high level service API that provides methods for performing common + operations in parallel on a thread pool. Example use cases: - * Uploading and retrieving data - Use the command line tool if you are simply uploading and downloading - files and directories to and from your filesystem. The command line tool - can be integrated into a shell script to automate tasks. - - * Integrating into an automated Python workflow - Use the ``SwiftService`` API to perform operations offered by the CLI - if your use case requires integration with a Python-based workflow. - This method offers greater control and flexibility over individual object - operations, such as the metadata set on each object. The ``SwiftService`` - class provides methods to perform multiple sets of operations against a - swift object store using a configurable shared thread pool. A single - instance of the ``SwiftService`` class can be shared between multiple - threads in your own code. - - * Developing an application in Python to access a swift object store - Use the ``SwiftService`` API to develop Python applications that use - swift to store and retrieve objects. A ``SwiftService`` instance provides - a configurable thread pool for performing all operations supported by the - CLI. - - * Fine-grained control over threading or the requests being performed - Use the ``Connection`` API if your use case requires fine grained control - over advanced features or you wish to use your own existing threading - model. Examples of advanced features requiring the use of the - ``Connection`` API include creating an SLO manifest that references - already existing objects, or fine grained control over the query strings - supplied with each HTTP request. +* Uploading and retrieving data + Use the command line tool if you are simply uploading and downloading + files and directories to and from your filesystem. The command line tool + can be integrated into a shell script to automate tasks. + +* Integrating into an automated Python workflow + Use the ``SwiftService`` API to perform operations offered by the CLI + if your use case requires integration with a Python-based workflow. + This method offers greater control and flexibility over individual object + operations, such as the metadata set on each object. The ``SwiftService`` + class provides methods to perform multiple sets of operations against a + swift object store using a configurable shared thread pool. A single + instance of the ``SwiftService`` class can be shared between multiple + threads in your own code. + +* Developing an application in Python to access a swift object store + Use the ``SwiftService`` API to develop Python applications that use + swift to store and retrieve objects. A ``SwiftService`` instance provides + a configurable thread pool for performing all operations supported by the + CLI. + +* Fine-grained control over threading or the requests being performed + Use the ``Connection`` API if your use case requires fine grained control + over advanced features or you wish to use your own existing threading + model. Examples of advanced features requiring the use of the + ``Connection`` API include creating an SLO manifest that references + already existing objects, or fine grained control over the query strings + supplied with each HTTP request. Important considerations ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -66,19 +66,19 @@ the proper use case, and not treat the storage like a traditional filesystem. There are two main restrictions to bear in mind when designing an application that uses an object store: - * You cannot rename objects. Due to fact that the name of an object is one - of the factors that determines where the object and its replicas are stored, - renaming would require multiple copies of the data to be moved between - physical storage devices. If you want to rename an object you must upload - to the new location, or make a server side copy request to the new location, - and then delete the original. - - * You cannot modify objects. Objects are stored in multiple locations and - are checked for integrity based on the MD5 sum calculated during - upload. In order to modify the contents of an object, the entire desired - contents must be re-uploaded. In certain special cases it is possible to - work around this restriction using large objects, but no general - file-like access is available to modify a stored object. +* You cannot rename objects. Due to fact that the name of an object is one + of the factors that determines where the object and its replicas are stored, + renaming would require multiple copies of the data to be moved between + physical storage devices. If you want to rename an object you must upload + to the new location, or make a server side copy request to the new location, + and then delete the original. + +* You cannot modify objects. Objects are stored in multiple locations and + are checked for integrity based on the MD5 sum calculated during + upload. In order to modify the contents of an object, the entire desired + contents must be re-uploaded. In certain special cases it is possible to + work around this restriction using large objects, but no general + file-like access is available to modify a stored object. Objects cannot be locked ------------------------ diff --git a/doc/source/service-api.rst b/doc/source/service-api.rst index 8efd43a0..cc4dcc26 100644 --- a/doc/source/service-api.rst +++ b/doc/source/service-api.rst @@ -26,10 +26,10 @@ the auth version based on the combination of options specified, but supplying options from multiple different auth versions can cause unexpected behaviour. - .. note:: +.. note:: - Leftover environment variables are a common source of confusion when - authorization fails. + Leftover environment variables are a common source of confusion when + authorization fails. Keystone V3 ~~~~~~~~~~~ @@ -109,17 +109,17 @@ in this dictionary are described below, along with their defaults: Options ~~~~~~~ - ``retries``: ``5`` +``retries``: ``5`` The number of times that the library should attempt to retry HTTP actions before giving up and reporting a failure. - ``container_threads``: ``10`` +``container_threads``: ``10`` - ``object_dd_threads``: ``10`` +``object_dd_threads``: ``10`` - ``object_uu_threads``: ``10`` +``object_uu_threads``: ``10`` - ``segment_threads``: ``10`` +``segment_threads``: ``10`` The above options determine the size of the available thread pools for performing swift operations. Container operations (such as listing a container) operate in the container threads, and a similar pattern @@ -131,86 +131,86 @@ Options ``uu`` and ``dd``. This stands for "upload/update" and "download/delete", and the corresponding actions will be run on separate threads pools. - ``segment_size``: ``None`` +``segment_size``: ``None`` If specified, this option enables uploading of large objects. Should the object being uploaded be larger than 5G in size, this option is mandatory otherwise the upload will fail. This option should be specified as a size in bytes. - ``use_slo``: ``False`` +``use_slo``: ``False`` Used in combination with the above option, ``use_slo`` will upload large objects as static rather than dynamic. Only static large objects provide error checking for the downloaded object, so we recommend this option. - ``segment_container``: ``None`` +``segment_container``: ``None`` Allows the user to select the container into which large object segments will be uploaded. We do not recommend changing this value as it could make locating orphaned segments more difficult in the case of errors. - ``leave_segments``: ``False`` +``leave_segments``: ``False`` Setting this option to true means that when deleting or overwriting a large object, its segments will be left in the object store and must be cleaned up manually. This option can be useful when sharing large object segments between multiple objects in more advanced scenarios, but must be treated with care, as it could lead to ever increasing storage usage. - ``changed``: ``None`` +``changed``: ``None`` This option affects uploads and simply means that those objects which already exist in the object store will not be overwritten if the ``mtime`` and size of the source is the same as the existing object. - ``skip_identical``: ``False`` +``skip_identical``: ``False`` A slightly more thorough case of the above, but rather than ``mtime`` and size uses an object's ``MD5 sum``. - ``yes_all``: ``False`` +``yes_all``: ``False`` This options affects only download and delete, and in each case must be specified in order to download/delete the entire contents of an account. This option has no effect on any other calls. - ``no_download``: ``False`` +``no_download``: ``False`` This option only affects download and means that all operations proceed as normal with the exception that no data is written to disk. - ``header``: ``[]`` +``header``: ``[]`` Used with upload and post operations to set headers on objects. Headers are specified as colon separated strings, e.g. "content-type:text/plain". - ``meta``: ``[]`` +``meta``: ``[]`` Used to set metadata on an object similarly to headers. .. note:: Setting metadata is a destructive operation, so when updating one of many metadata values all desired metadata for an object must be re-applied. - ``long``: ``False`` +``long``: ``False`` Affects only list operations, and results in more metrics being made available in the results at the expense of lower performance. - ``fail_fast``: ``False`` +``fail_fast``: ``False`` Applies to delete and upload operations, and attempts to abort queued tasks in the event of errors. - ``prefix``: ``None`` +``prefix``: ``None`` Affects list operations; only objects with the given prefix will be returned/affected. It is not advisable to set at the service level, as those operations that call list to discover objects on which they should operate will also be affected. - ``delimiter``: ``None`` +``delimiter``: ``None`` Affects list operations, and means that listings only contain results up to the first instance of the delimiter in the object name. This is useful for working with objects containing '/' in their names to simulate folder structures. - ``dir_marker``: ``False`` +``dir_marker``: ``False`` Affects uploads, and allows empty 'pseudofolder' objects to be created when the source of an upload is ``None``. - ``checksum``: ``True`` +``checksum``: ``True`` Affects uploads and downloads. If set check md5 sum for the transfer. - ``shuffle``: ``False`` +``shuffle``: ``False`` When downloading objects, the default behaviour of the CLI is to shuffle lists of objects in order to spread the load on storage drives when multiple clients are downloading the same files to multiple locations (e.g. in the @@ -220,12 +220,12 @@ Options are downloaded in lexically-sorted order. Setting this option to ``True`` gives the same shuffling behaviour as the CLI. - ``destination``: ``None`` +``destination``: ``None`` When copying objects, this specifies the destination where the object will be copied to. The default of None means copy will be the same as source. - ``fresh_metadata``: ``None`` +``fresh_metadata``: ``None`` When copying objects, this specifies that the object metadata on the source will *not* be applied to the destination object - the destination object will have a new fresh set of metadata that includes From 7e9717c04c570d6cab9890265ec67fea1d937596 Mon Sep 17 00:00:00 2001 From: OpenStack Release Bot Date: Mon, 18 Mar 2019 14:55:40 +0000 Subject: [PATCH 064/238] Update master for stable/stein Add file to the reno documentation build to show release notes for stable/stein. Use pbr instruction to increment the minor version number automatically so that master versions are higher than the versions on stable/stein. Change-Id: I05c44e97e766aa4130b72f6d8d1a6a111ccafd12 Sem-Ver: feature --- releasenotes/source/index.rst | 1 + releasenotes/source/stein.rst | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 releasenotes/source/stein.rst diff --git a/releasenotes/source/index.rst b/releasenotes/source/index.rst index 92da0e8f..27f675ee 100644 --- a/releasenotes/source/index.rst +++ b/releasenotes/source/index.rst @@ -6,6 +6,7 @@ :maxdepth: 1 current + stein rocky queens pike diff --git a/releasenotes/source/stein.rst b/releasenotes/source/stein.rst new file mode 100644 index 00000000..efaceb66 --- /dev/null +++ b/releasenotes/source/stein.rst @@ -0,0 +1,6 @@ +=================================== + Stein Series Release Notes +=================================== + +.. release-notes:: + :branch: stable/stein From af4bac31fadf1fb15271f49791f16581219d74a2 Mon Sep 17 00:00:00 2001 From: OpenDev Sysadmins Date: Fri, 19 Apr 2019 19:32:30 +0000 Subject: [PATCH 065/238] OpenDev Migration Patch This commit was bulk generated and pushed by the OpenDev sysadmins as a part of the Git hosting and code review systems migration detailed in these mailing list posts: http://lists.openstack.org/pipermail/openstack-discuss/2019-March/003603.html http://lists.openstack.org/pipermail/openstack-discuss/2019-April/004920.html Attempts have been made to correct repository namespaces and hostnames based on simple pattern matching, but it's possible some were updated incorrectly or missed entirely. Please reach out to us via the contact information listed at https://opendev.org/ with any questions you may have. --- .gitreview | 2 +- .zuul.yaml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitreview b/.gitreview index 0387a741..6cf44857 100644 --- a/.gitreview +++ b/.gitreview @@ -1,4 +1,4 @@ [gerrit] -host=review.openstack.org +host=review.opendev.org port=29418 project=openstack/python-swiftclient.git diff --git a/.zuul.yaml b/.zuul.yaml index f9c22665..ac1d9668 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -9,7 +9,7 @@ # job sets zuul_work_dir to the swift directory and uses tox # for installation. required-projects: - - git.openstack.org/openstack/python-swiftclient + - opendev.org/openstack/python-swiftclient - job: name: swiftclient-functional @@ -19,10 +19,10 @@ python-swiftclient installed from source instead as package from PyPI. required-projects: - - git.openstack.org/openstack/python-swiftclient + - opendev.org/openstack/python-swiftclient vars: # Override value from parent job to use swiftclient tests - zuul_work_dir: "{{ zuul.projects['git.openstack.org/openstack/python-swiftclient'].src_dir }}" + zuul_work_dir: "{{ zuul.projects['opendev.org/openstack/python-swiftclient'].src_dir }}" - job: name: swiftclient-functional-py2 From 7103da3467c555e6ab44fc9de5036540fba15c95 Mon Sep 17 00:00:00 2001 From: jacky06 Date: Tue, 23 Apr 2019 13:44:51 +0800 Subject: [PATCH 066/238] Replace git.openstack.org URLs with opendev.org URLs Change-Id: I0991c93fbf5b015b68dd94f3fe805ec705014f06 --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index b105094a..5243dd6b 100644 --- a/README.rst +++ b/README.rst @@ -44,7 +44,7 @@ __ https://github.com/openstack/swift .. _Launchpad project: https://launchpad.net/python-swiftclient .. _Blueprints: https://blueprints.launchpad.net/python-swiftclient .. _Bugs: https://bugs.launchpad.net/python-swiftclient -.. _Source: https://git.openstack.org/cgit/openstack/python-swiftclient +.. _Source: https://opendev.org/openstack/python-swiftclient .. _How to Contribute: https://docs.openstack.org/infra/manual/developers.html .. _Specs: https://specs.openstack.org/openstack/swift-specs/ .. _Release Notes: https://docs.openstack.org/releasenotes/python-swiftclient From 113eacf3b80f61d366b3e95b558b40f82ff728a4 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Tue, 25 Jun 2019 15:43:29 -0700 Subject: [PATCH 067/238] Isolate docs requirements ...since modern sphinx won't install on py27. While we're at it, clean up some warnings and treat warnings as errors. Also, fix up how we parse test configs so we can run func tests. Related-Change: Id3c2ed87230c5918c18e2c01d086df8157f036b1 Change-Id: I3718f69610545b0dbcb0a2ab45b400da3a45682c --- doc/requirements.txt | 5 ++++ doc/source/_static/.gitignore | 0 swiftclient/multithreading.py | 8 ++++++ swiftclient/utils.py | 2 +- test-requirements.txt | 3 -- tests/functional/test_swiftclient.py | 42 ++++++++++++++++++---------- tox.ini | 6 +++- 7 files changed, 47 insertions(+), 19 deletions(-) create mode 100644 doc/requirements.txt create mode 100644 doc/source/_static/.gitignore diff --git a/doc/requirements.txt b/doc/requirements.txt new file mode 100644 index 00000000..d8f432ee --- /dev/null +++ b/doc/requirements.txt @@ -0,0 +1,5 @@ +keystoneauth1>=3.4.0 # Apache-2.0 +sphinx!=1.6.6,!=1.6.7,<2.0.0,>=1.6.2;python_version=='2.7' # BSD +sphinx!=1.6.6,!=1.6.7,!=2.1.0,>=1.6.2;python_version>='3.4' # BSD +reno>=2.5.0 # Apache-2.0 +openstackdocstheme>=1.18.1 # Apache-2.0 diff --git a/doc/source/_static/.gitignore b/doc/source/_static/.gitignore new file mode 100644 index 00000000..e69de29b diff --git a/swiftclient/multithreading.py b/swiftclient/multithreading.py index 5e03ed79..fcf0ed95 100644 --- a/swiftclient/multithreading.py +++ b/swiftclient/multithreading.py @@ -175,6 +175,14 @@ def __init__(self, create_connection, max_workers): super(ConnectionThreadPoolExecutor, self).__init__(max_workers) def submit(self, fn, *args, **kwargs): + """ + Schedules the callable, `fn`, to be executed + + :param fn: the callable to be invoked + :param args: the positional arguments for the callable + :param kwargs: the keyword arguments for the callable + :returns: a Future object representing the execution of the callable + """ def conn_fn(): priority = None conn = None diff --git a/swiftclient/utils.py b/swiftclient/utils.py index 5c17c613..87a43902 100644 --- a/swiftclient/utils.py +++ b/swiftclient/utils.py @@ -74,7 +74,7 @@ def generate_temp_url(path, seconds, key, method, absolute=False, Swift object. :param path: The full path to the Swift object or prefix if - a prefix-based temporary URL should be generated. Example: + a prefix-based temporary URL should be generated. Example: /v1/AUTH_account/c/o or /v1/AUTH_account/c/prefix. :param seconds: time in seconds or ISO 8601 timestamp. If absolute is False and this is the string representation of an diff --git a/test-requirements.txt b/test-requirements.txt index d8222142..b3ca5f89 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -3,7 +3,4 @@ hacking>=1.1.0,<1.2.0 # Apache-2.0 coverage!=4.4,>=4.0 # Apache-2.0 keystoneauth1>=3.4.0 # Apache-2.0 mock>=1.2.0 # BSD -sphinx!=1.6.6,!=1.6.7,>=1.6.2 # BSD stestr>=2.0.0 # Apache-2.0 -reno>=2.5.0 # Apache-2.0 -openstackdocstheme>=1.18.1 # Apache-2.0 diff --git a/tests/functional/test_swiftclient.py b/tests/functional/test_swiftclient.py index b4f275b2..bae30444 100644 --- a/tests/functional/test_swiftclient.py +++ b/tests/functional/test_swiftclient.py @@ -46,11 +46,34 @@ def _get_config(self): config.read(config_file) self.config = config if config.has_section('func_test'): - auth_host = config.get('func_test', 'auth_host') - auth_port = config.getint('func_test', 'auth_port') - auth_ssl = config.getboolean('func_test', 'auth_ssl') - auth_prefix = config.get('func_test', 'auth_prefix') - self.auth_version = config.get('func_test', 'auth_version') + if config.has_option('func_test', 'auth_uri'): + self.auth_url = config.get('func_test', 'auth_uri') + try: + self.auth_version = config.get('func_test', 'auth_version') + except configparser.NoOptionError: + last_piece = self.auth_url.rstrip('/').rsplit('/', 1)[1] + if last_piece.endswith('.0'): + last_piece = last_piece[:-2] + if last_piece in ('1', '2', '3'): + self.auth_version = last_piece + else: + raise + else: + auth_host = config.get('func_test', 'auth_host') + auth_port = config.getint('func_test', 'auth_port') + auth_ssl = config.getboolean('func_test', 'auth_ssl') + auth_prefix = config.get('func_test', 'auth_prefix') + self.auth_version = config.get('func_test', 'auth_version') + self.auth_url = "" + if auth_ssl: + self.auth_url += "https://" + else: + self.auth_url += "http://" + self.auth_url += "%s:%s%s" % ( + auth_host, auth_port, auth_prefix) + if self.auth_version == "1": + self.auth_url += 'v1.0' + try: self.account_username = config.get('func_test', 'account_username') @@ -59,15 +82,6 @@ def _get_config(self): username = config.get('func_test', 'username') self.account_username = "%s:%s" % (account, username) self.password = config.get('func_test', 'password') - self.auth_url = "" - if auth_ssl: - self.auth_url += "https://" - else: - self.auth_url += "http://" - self.auth_url += "%s:%s%s" % (auth_host, auth_port, auth_prefix) - if self.auth_version == "1": - self.auth_url += 'v1.0' - else: self.skip_tests = True diff --git a/tox.ini b/tox.ini index 46354918..84a54197 100644 --- a/tox.ini +++ b/tox.ini @@ -65,8 +65,10 @@ commands = {[testenv:func]commands} [testenv:docs] basepython = python3 +usedevelop = False +deps = -r{toxinidir}/doc/requirements.txt commands= - python setup.py build_sphinx + python setup.py build_sphinx -W [flake8] # it's not a bug that we aren't using all of hacking, ignore: @@ -96,6 +98,8 @@ commands = bindep test [testenv:releasenotes] basepython = python3 +usedevelop = False +deps = -r{toxinidir}/doc/requirements.txt commands = sphinx-build -a -W -E -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html [testenv:lower-constraints] From 3b21157a844be5b71fba2216486c3ef412e7ae1a Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Tue, 25 Jun 2019 15:25:53 -0700 Subject: [PATCH 068/238] Clean up warnings from newer flake8 Change-Id: I18a6327b3acdd4db5ae80097080c043f7c20c353 --- swiftclient/client.py | 2 +- swiftclient/service.py | 1 + swiftclient/shell.py | 1 + tests/unit/test_shell.py | 4 ++-- tox.ini | 3 ++- 5 files changed, 7 insertions(+), 4 deletions(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index 44066891..f0711822 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -39,7 +39,7 @@ # Default is 100, increase to 256 http_client._MAXHEADERS = 256 -VERSIONFUL_AUTH_PATH = re.compile('v[2-3](?:\.0)?$') +VERSIONFUL_AUTH_PATH = re.compile(r'v[2-3](?:\.0)?$') AUTH_VERSIONS_V1 = ('1.0', '1', 1) AUTH_VERSIONS_V2 = ('2.0', '2', 2) AUTH_VERSIONS_V3 = ('3.0', '3', 3) diff --git a/swiftclient/service.py b/swiftclient/service.py index 8f3648ee..2663ace4 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -173,6 +173,7 @@ def _build_default_global_options(): 'container_threads': 10 } + _default_global_options = _build_default_global_options() _default_local_options = { diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 9ea5e952..0459533d 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -58,6 +58,7 @@ def immediate_exit(signum, frame): stderr.write(" Aborted\n") os_exit(2) + st_delete_options = '''[--all] [--leave-segments] [--object-threads ] [--container-threads ] diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index f5d2f15b..7c8faa29 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -1822,7 +1822,7 @@ def test_temp_url_error_output(self): argv = ["", "tempurl", "GET", "60", '/v1/a/c', "secret_key", "--absolute", '--prefix-based'] with CaptureOutput(suppress_systemexit=True) as output: - swiftclient.shell.main(argv) + swiftclient.shell.main(argv) self.assertEqual(expected, output.err, 'Expected %r but got %r for path %r' % (expected, output.err, '/v1/a/c')) @@ -1832,7 +1832,7 @@ def test_temp_url_error_output(self): argv = ["", "tempurl", "GET", bad_time, '/v1/a/c/o', "secret_key", "--absolute"] with CaptureOutput(suppress_systemexit=True) as output: - swiftclient.shell.main(argv) + swiftclient.shell.main(argv) self.assertEqual(expected, output.err, 'Expected %r but got %r for time %r' % (expected, output.err, bad_time)) diff --git a/tox.ini b/tox.ini index 84a54197..e029efd9 100644 --- a/tox.ini +++ b/tox.ini @@ -79,7 +79,8 @@ commands= # H403: multi line docstrings should end on a new line # H404: multi line docstring should start without a leading new line # H405: multi line docstring summary not separated with an empty line -ignore = H101,H301,H306,H401,H403,H404,H405 +# W504: line break after binary operator +ignore = H101,H301,H306,H401,H403,H404,H405,W504 # H106: Don’t put vim configuration in source files # H203: Use assertIs(Not)None to check for None enable-extensions=H106,H203 From 9021a58c240e156f54ffafdc4609868f348d3ebc Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Wed, 10 Apr 2019 16:22:27 -0700 Subject: [PATCH 069/238] Fix SLO re-upload Previously, if you uploaded a file as an SLO then re-uploaded it with the same segment size and mtime, the second upload would go delete the segments it just (re)uploaded. This was due to us tracking old_slo_manifest_paths and new_slo_manifest_paths in different formats; one would have a leading slash while the other would not. Now, normalize to the stripped-slash version so we stop deleting segments we just uploaded. Change-Id: Ibcbed3df4febe81cdf13855656e2daaca8d521b4 --- swiftclient/service.py | 30 +++++++++++++-------------- swiftclient/utils.py | 8 ++++++++ tests/unit/test_shell.py | 44 ++++++++++++++++++++++++++++++++++++++-- 3 files changed, 64 insertions(+), 18 deletions(-) diff --git a/swiftclient/service.py b/swiftclient/service.py index 8f3648ee..3ea7f614 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -17,6 +17,7 @@ import os +from collections import defaultdict from concurrent.futures import as_completed, CancelledError, TimeoutError from copy import deepcopy from errno import EEXIST, ENOENT @@ -44,7 +45,7 @@ from swiftclient.utils import ( config_true_value, ReadableToIterable, LengthWrapper, EMPTY_ETAG, parse_api_response, report_traceback, n_groups, split_request_headers, - n_at_a_time + n_at_a_time, normalize_manifest_path ) from swiftclient.exceptions import ClientException from swiftclient.multithreading import MultiThreadingManager @@ -2071,11 +2072,9 @@ def _upload_object_job(self, conn, container, source, obj, options, if not options['leave_segments']: old_manifest = headers.get('x-object-manifest') if is_slo: - for old_seg in chunk_data: - seg_path = old_seg['name'].lstrip('/') - if isinstance(seg_path, text_type): - seg_path = seg_path.encode('utf-8') - old_slo_manifest_paths.append(seg_path) + old_slo_manifest_paths.extend( + normalize_manifest_path(old_seg['name']) + for old_seg in chunk_data) except ClientException as err: if err.http_status != 404: traceback, err_time = report_traceback() @@ -2165,8 +2164,9 @@ def _upload_object_job(self, conn, container, source, obj, options, response = self._upload_slo_manifest( conn, segment_results, container, obj, put_headers) res['manifest_response_dict'] = response - new_slo_manifest_paths = { - seg['segment_location'] for seg in segment_results} + new_slo_manifest_paths.update( + normalize_manifest_path(new_seg['segment_location']) + for new_seg in segment_results) else: new_object_manifest = '%s/%s/%s/%s/%s/' % ( quote(seg_container.encode('utf8')), @@ -2223,8 +2223,9 @@ def _upload_object_job(self, conn, container, source, obj, options, response = self._upload_slo_manifest( conn, results, container, obj, put_headers) res['manifest_response_dict'] = response - new_slo_manifest_paths = { - r['segment_location'] for r in results} + new_slo_manifest_paths.update( + normalize_manifest_path(new_seg['segment_location']) + for new_seg in results) res['large_object'] = True else: res['response_dict'] = ret @@ -2264,11 +2265,10 @@ def _upload_object_job(self, conn, container, source, obj, options, fp.close() if old_manifest or old_slo_manifest_paths: drs = [] - delobjsmap = {} + delobjsmap = defaultdict(list) if old_manifest: scontainer, sprefix = old_manifest.split('/', 1) sprefix = sprefix.rstrip('/') + '/' - delobjsmap[scontainer] = [] for part in self.list(scontainer, {'prefix': sprefix}): if not part["success"]: raise part["error"] @@ -2280,10 +2280,8 @@ def _upload_object_job(self, conn, container, source, obj, options, if seg_to_delete in new_slo_manifest_paths: continue scont, sobj = \ - seg_to_delete.split(b'/', 1) - delobjs_cont = delobjsmap.get(scont, []) - delobjs_cont.append(sobj) - delobjsmap[scont] = delobjs_cont + seg_to_delete.split('/', 1) + delobjsmap[scont].append(sobj) del_segs = [] for dscont, dsobjs in delobjsmap.items(): diff --git a/swiftclient/utils.py b/swiftclient/utils.py index 87a43902..2b208b9f 100644 --- a/swiftclient/utils.py +++ b/swiftclient/utils.py @@ -395,3 +395,11 @@ def n_at_a_time(seq, n): def n_groups(seq, n): items_per_group = ((len(seq) - 1) // n) + 1 return n_at_a_time(seq, items_per_group) + + +def normalize_manifest_path(path): + if six.PY2 and isinstance(path, six.text_type): + path = path.encode('utf-8') + if path.startswith('/'): + return path[1:] + return path diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index f5d2f15b..88e2c478 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -799,11 +799,11 @@ def test_upload_delete_slo_segments(self, connection): response_dict={}) expected_delete_calls = [ mock.call( - b'container1', b'old_seg1', + 'container1', 'old_seg1', response_dict={} ), mock.call( - b'container2', b'old_seg2', + 'container2', 'old_seg2', response_dict={} ) ] @@ -834,6 +834,46 @@ def test_upload_leave_slo_segments(self, connection): response_dict={}) self.assertFalse(connection.return_value.delete_object.mock_calls) + @mock.patch('swiftclient.service.Connection') + def test_reupload_leaves_slo_segments(self, connection): + with open(self.tmpfile, "wb") as fh: + fh.write(b'12345678901234567890') + mtime = '{:.6f}'.format(os.path.getmtime(self.tmpfile)) + expected_segments = [ + 'container_segments/{}/slo/{}/20/10/{:08d}'.format( + self.tmpfile[1:], mtime, i) + for i in range(2) + ] + + # Test re-upload overwriting a manifest doesn't remove + # segments it just wrote + connection.return_value.head_container.return_value = { + 'x-storage-policy': 'one'} + connection.return_value.attempts = 0 + argv = ["", "upload", "container", self.tmpfile, + "--use-slo", "-S", "10"] + connection.return_value.head_object.side_effect = [ + {'x-static-large-object': 'true', # For the upload call + 'content-length': '20'}] + connection.return_value.get_object.return_value = ( + {}, + # we've already *got* the expected manifest! + json.dumps([ + {'name': seg} for seg in expected_segments + ]).encode('ascii') + ) + connection.return_value.put_object.return_value = ( + 'd41d8cd98f00b204e9800998ecf8427e') + swiftclient.shell.main(argv) + connection.return_value.put_object.assert_called_with( + 'container', + self.tmpfile[1:], # drop leading / + mock.ANY, + headers={'x-object-meta-mtime': mtime}, + query_string='multipart-manifest=put', + response_dict={}) + self.assertFalse(connection.return_value.delete_object.mock_calls) + @mock.patch('swiftclient.service.Connection') def test_upload_delete_dlo_segments(self, connection): # Upload delete existing segments From 591c3e23804519f405a6645232c2246b173b5e75 Mon Sep 17 00:00:00 2001 From: pengyuesheng Date: Wed, 3 Jul 2019 15:01:30 +0800 Subject: [PATCH 070/238] Bump the openstackdocstheme extension to 1.20 Some options are now automatically configured by the version 1.20: - project - html_last_updated_fmt - latex_engine - latex_elements - version - release. Change-Id: I0573c7feaea991e6b33bdee6dc358c9206a9bfd8 --- doc/requirements.txt | 2 +- doc/source/conf.py | 9 --------- lower-constraints.txt | 2 +- releasenotes/source/conf.py | 12 ------------ 4 files changed, 2 insertions(+), 23 deletions(-) diff --git a/doc/requirements.txt b/doc/requirements.txt index d8f432ee..6cdad2ab 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -2,4 +2,4 @@ keystoneauth1>=3.4.0 # Apache-2.0 sphinx!=1.6.6,!=1.6.7,<2.0.0,>=1.6.2;python_version=='2.7' # BSD sphinx!=1.6.6,!=1.6.7,!=2.1.0,>=1.6.2;python_version>='3.4' # BSD reno>=2.5.0 # Apache-2.0 -openstackdocstheme>=1.18.1 # Apache-2.0 +openstackdocstheme>=1.20.0 # Apache-2.0 diff --git a/doc/source/conf.py b/doc/source/conf.py index f56b643f..85dd81ef 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -53,17 +53,8 @@ master_doc = 'index' # General information about the project. -project = u'Swiftclient' copyright = u'2013-2016 OpenStack, LLC.' -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -import swiftclient.version -release = swiftclient.version.version_string -version = swiftclient.version.version_string - # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # language = None diff --git a/lower-constraints.txt b/lower-constraints.txt index ab45e39d..ae619488 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -20,7 +20,7 @@ MarkupSafe==1.0 mccabe==0.2.1 mock==1.2.0 netaddr==0.7.10 -openstackdocstheme==1.18.1 +openstackdocstheme==1.20.0 oslo.config==1.2.0 pbr==2.0.0 pep8==1.5.7 diff --git a/releasenotes/source/conf.py b/releasenotes/source/conf.py index b27aa963..c71f41d4 100644 --- a/releasenotes/source/conf.py +++ b/releasenotes/source/conf.py @@ -65,15 +65,8 @@ master_doc = 'index' # General information about the project. -project = u'Swift Client Release Notes' copyright = u'%d, OpenStack Foundation' % datetime.datetime.now().year -# Release notes are version independent. -# The short X.Y version. -version = '' -# The full version, including alpha/beta/rc tags. -release = '' - # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # @@ -173,11 +166,6 @@ # # html_extra_path = [] -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -# html_last_updated_fmt = '%b %d, %Y' -html_last_updated_fmt = '%Y-%m-%d %H:%M' - # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. # From efe3d084ded690a68b21947c6d0a96a5f74144c8 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Fri, 5 Jul 2019 12:02:26 -0400 Subject: [PATCH 071/238] Add Python 3 Train unit tests This is a mechanically generated patch to ensure unit testing is in place for all of the Tested Runtimes for Train. See the Train python3-updates goal document for details: https://governance.openstack.org/tc/goals/train/python3-updates.html Change-Id: I764b9765484e1d8217d56796d984f910aa5f9c5a Story: #2005924 Task: #34249 --- .zuul.yaml | 4 +--- setup.cfg | 1 - tox.ini | 2 +- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index ac1d9668..4a878bae 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -39,9 +39,7 @@ - openstack-lower-constraints-jobs - openstack-pypy-jobs-nonvoting - openstack-python-jobs - - openstack-python35-jobs - - openstack-python36-jobs - - openstack-python37-jobs + - openstack-python3-train-jobs - publish-openstack-docs-pti - release-notes-jobs-python3 check: diff --git a/setup.cfg b/setup.cfg index d3b13a6b..5653bf30 100644 --- a/setup.cfg +++ b/setup.cfg @@ -17,7 +17,6 @@ classifier = Programming Language :: Python :: 2 Programming Language :: Python :: 2.7 Programming Language :: Python :: 3 - Programming Language :: Python :: 3.5 Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.7 diff --git a/tox.ini b/tox.ini index e029efd9..a2b63b3f 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py37,py36,py35,py27,pypy,pep8 +envlist = py27,py37,pypy,pep8 minversion = 2.0 skipsdist = True From 936631eac617c83bec1f1c44b1adcaa51d36329c Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Tue, 2 Jul 2019 11:23:22 -0500 Subject: [PATCH 072/238] Optionally display listings in raw json Symlinks have recently added some new keys to container listings. It's very convenient to be able to see and reason about the extra information in container listings. Allowing raw json output is similar with what the client already does for the info command, and it's forward compatible with any listing enhancements added by future middleware development. Change-Id: I88fb38529342ac4e4198aeccd2f10c69c7396704 --- swiftclient/shell.py | 19 ++++++++++++++++++- swiftclient/utils.py | 22 ++++++++++++++++++++++ tests/unit/test_shell.py | 20 ++++++++++++++++++++ tests/unit/test_utils.py | 39 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 99 insertions(+), 1 deletion(-) diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 0459533d..2be85aff 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -33,7 +33,8 @@ from time import gmtime, strftime from swiftclient import RequestException -from swiftclient.utils import config_true_value, generate_temp_url, prt_bytes +from swiftclient.utils import config_true_value, generate_temp_url, \ + prt_bytes, JSONableIterable from swiftclient.multithreading import OutputManager from swiftclient.exceptions import ClientException from swiftclient import __version__ as client_version @@ -578,6 +579,8 @@ def _print_stats(options, stats, human): help='Roll up items with the given delimiter. For containers ' 'only. See OpenStack Swift API documentation for ' 'what this means.') + parser.add_argument('-j', '--json', action='store_true', + help='print listing information in json') parser.add_argument( '-H', '--header', action='append', dest='header', default=[], @@ -616,6 +619,20 @@ def _print_stats(options, stats, human): else: stats_parts_gen = swift.list(container=container) + if options.get('json', False): + def listing(stats_parts_gen=stats_parts_gen): + for stats in stats_parts_gen: + if stats["success"]: + for item in stats['listing']: + yield item + else: + raise stats["error"] + + json.dump( + JSONableIterable(listing()), output_manager.print_stream, + sort_keys=True, indent=2) + output_manager.print_msg('') + return for stats in stats_parts_gen: if stats["success"]: _print_stats(options, stats, human) diff --git a/swiftclient/utils.py b/swiftclient/utils.py index 2b208b9f..9e43237c 100644 --- a/swiftclient/utils.py +++ b/swiftclient/utils.py @@ -403,3 +403,25 @@ def normalize_manifest_path(path): if path.startswith('/'): return path[1:] return path + + +class JSONableIterable(list): + def __init__(self, iterable): + self._iterable = iter(iterable) + try: + self._peeked = next(self._iterable) + self._has_items = True + except StopIteration: + self._peeked = None + self._has_items = False + + def __bool__(self): + return self._has_items + + __nonzero__ = __bool__ + + def __iter__(self): + if self._has_items: + yield self._peeked + for item in self._iterable: + yield item diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index f729c250..d9ddb3ee 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -297,6 +297,26 @@ def test_stat_object_with_headers(self, connection): mock.call('container', 'object', headers={'Skip-Middleware': 'Test'})]) + @mock.patch('swiftclient.service.Connection') + def test_list_json(self, connection): + connection.return_value.get_account.side_effect = [ + [None, [{'name': 'container'}]], + [None, [{'name': u'\u263A', 'some-custom-key': 'and value'}]], + [None, []], + ] + + argv = ["", "list", "--json"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + calls = [mock.call(marker='', prefix=None, headers={}), + mock.call(marker='container', prefix=None, headers={})] + connection.return_value.get_account.assert_has_calls(calls) + + listing = [{'name': 'container'}, + {'name': u'\u263A', 'some-custom-key': 'and value'}] + expected = json.dumps(listing, sort_keys=True, indent=2) + '\n' + self.assertEqual(output.out, expected) + @mock.patch('swiftclient.service.Connection') def test_list_account(self, connection): # Test account listing diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index e54b90c7..97abc444 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -14,6 +14,7 @@ # limitations under the License. import gzip +import json import unittest import mock import six @@ -638,3 +639,41 @@ def test_gzipped_body(self): {'content-encoding': 'gzip'}, buf.getvalue()) self.assertEqual({'test': u'\u2603'}, result) + + +class JSONTracker(object): + def __init__(self, data): + self.data = data + self.calls = [] + + def __iter__(self): + for item in self.data: + self.calls.append(('read', item)) + yield item + + def write(self, s): + self.calls.append(('write', s)) + + +class TestJSONableIterable(unittest.TestCase): + def test_json_dump_iterencodes(self): + t = JSONTracker([1, 'fish', 2, 'fish']) + json.dump(u.JSONableIterable(t), t) + self.assertEqual(t.calls, [ + ('read', 1), + ('write', '[1'), + ('read', 'fish'), + ('write', ', "fish"'), + ('read', 2), + ('write', ', 2'), + ('read', 'fish'), + ('write', ', "fish"'), + ('write', ']'), + ]) + + def test_json_dump_empty_iter(self): + t = JSONTracker([]) + json.dump(u.JSONableIterable(t), t) + self.assertEqual(t.calls, [ + ('write', '[]'), + ]) From 47d5f44c3dbb5b9172a6ac7e894bf2b940ad5707 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Fri, 28 Jun 2019 16:43:32 -0700 Subject: [PATCH 073/238] Authors/changelog for 3.8.0 Change-Id: I5de701c6282ffb4a3776009aeb16531f29162306 --- AUTHORS | 3 +++ ChangeLog | 12 ++++++++++++ .../notes/3_8_0_release-bd867fbdb8c895d3.yaml | 9 +++++++++ 3 files changed, 24 insertions(+) create mode 100644 releasenotes/notes/3_8_0_release-bd867fbdb8c895d3.yaml diff --git a/AUTHORS b/AUTHORS index 1fcf65d4..16dfcf9c 100644 --- a/AUTHORS +++ b/AUTHORS @@ -52,6 +52,7 @@ Hiroshi Miura (miurahr@nttdata.co.jp) howardlee (lihongweibj@inspur.com) Hu Bing (hubingsh@cn.ibm.com) Ian Cordasco (ian.cordasco@rackspace.com) +jacky06 (zhang.min@99cloud.net) Jaivish Kothari (jaivish.kothari@nectechnologies.in) Jakub Krajcovic (jakub.krajcovic@gmail.com) James Nzomo (james@tdt.rocks) @@ -99,6 +100,7 @@ Ondrej Novy (ondrej.novy@firma.seznam.cz) Pallavi (pallavi.s@nectechnologies.in) Paul Belanger (pabelanger@redhat.com) Paulo Ewerton (pauloewerton@lsd.ufcg.edu.br) +pengyuesheng (pengyuesheng@gohighsec.com) Pete Zaitcev (zaitcev@kotori.zaitcev.us) Peter Lisak (peter.lisak@firma.seznam.cz) Petr Kovar (pkovar@redhat.com) @@ -147,6 +149,7 @@ Vitaly Gridnev (vgridnev@mirantis.com) Vu Cong Tuan (tuanvc@vn.fujitsu.com) wangqi (wang.qi@99cloud.net) wangxiyuan (wangxiyuan@huawei.com) +wangzhenyu (wangzy@fiberhome.com) Wu Wenxiang (wu.wenxiang@99cloud.net) wu.chunyang (wu.chunyang@99cloud.net) YangLei (yanglyy@cn.ibm.com) diff --git a/ChangeLog b/ChangeLog index 6f6bf8bb..253a3cc1 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,15 @@ +3.8.0 +----- + +* Added a new `--json` option to `swift list`. + +* Fixed an issue introduced in 3.5.0 where re-uploading an SLO with + the same size, mtime, and segment size would delete all of the + just-uploaded segments. + +* Various other minor bug fixes and improvements. + + 3.7.0 ----- diff --git a/releasenotes/notes/3_8_0_release-bd867fbdb8c895d3.yaml b/releasenotes/notes/3_8_0_release-bd867fbdb8c895d3.yaml new file mode 100644 index 00000000..85ae2c0c --- /dev/null +++ b/releasenotes/notes/3_8_0_release-bd867fbdb8c895d3.yaml @@ -0,0 +1,9 @@ +--- +features: + - | + Added a new ``--json`` option to ``swift list``. +fixes: + - | + Fixed an issue introduced in 3.5.0 where re-uploading an SLO with + the same size, mtime, and segment size would delete all of the + just-uploaded segments. From 5bd66947fc3d8987d4b24d5119a346031004229e Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Thu, 25 Jul 2019 14:21:27 -0700 Subject: [PATCH 074/238] Drag forward prettytable in lower-constraints Apparently version 0.7 got unpublished recently. Change-Id: I8669130f8477a577781e17c6d428aacff53cab92 --- lower-constraints.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lower-constraints.txt b/lower-constraints.txt index ae619488..88d28650 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -24,7 +24,7 @@ openstackdocstheme==1.20.0 oslo.config==1.2.0 pbr==2.0.0 pep8==1.5.7 -PrettyTable==0.7 +PrettyTable==0.7.1 pyflakes==0.8.1 Pygments==2.2.0 python-keystoneclient==0.7.0 From 7175069b3e95c38070bb4373019f78c87ab103d0 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Thu, 25 Jul 2019 11:21:29 -0700 Subject: [PATCH 075/238] Fix up requests so we can send non-RFC-compliant headers on py3 Change-Id: I3dac826c1f208569c5f40431f59a2045e5744415 --- swiftclient/client.py | 6 ++++-- tests/functional/test_swiftclient.py | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index f0711822..4be2e2d6 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -74,8 +74,10 @@ def createLock(self): pass # requests version 1.2.3 try to encode headers in ascii, preventing -# utf-8 encoded header to be 'prepared' -if StrictVersion(requests.__version__) < StrictVersion('2.0.0'): +# utf-8 encoded header to be 'prepared'. This also affects all +# (or at least most) versions of requests on py3 +if StrictVersion(requests.__version__) < StrictVersion('2.0.0') \ + or not six.PY2: from requests.structures import CaseInsensitiveDict def prepare_unicode_headers(self, headers): diff --git a/tests/functional/test_swiftclient.py b/tests/functional/test_swiftclient.py index bae30444..9a74c63f 100644 --- a/tests/functional/test_swiftclient.py +++ b/tests/functional/test_swiftclient.py @@ -18,6 +18,7 @@ import time from io import BytesIO +import six from six.moves import configparser import swiftclient @@ -446,6 +447,22 @@ def test_post_object(self): self.assertEqual('45.67', headers.get('x-object-meta-float')) self.assertEqual('False', headers.get('x-object-meta-bool')) + def test_post_object_unicode_header_name(self): + self.conn.post_object(self.containername, + self.objectname, + {u'x-object-meta-\U0001f44d': u'\U0001f44d'}) + + # Note that we can't actually read this header back on py3; see + # https://bugs.python.org/issue37093 + # We'll have to settle for just testing that the POST doesn't blow up + # with a UnicodeDecodeError + if six.PY2: + headers = self.conn.head_object( + self.containername, self.objectname) + self.assertIn(u'x-object-meta-\U0001f44d', headers) + self.assertEqual(u'\U0001f44d', + headers.get(u'x-object-meta-\U0001f44d')) + def test_copy_object(self): self.conn.put_object( self.containername, self.objectname, self.test_data) From 78753987468cb6b04d0b4e06b432e22f5a7189bd Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Fri, 26 Jul 2019 22:56:08 -0700 Subject: [PATCH 076/238] Delete/overwrite symlinks better Previously, when deleting a symlink that points to an xLO, we'd clean up the xLO's segments then delete the symlink, leaving the xLO itself busted. Similar trouble would come from overwriting a symlink pointing to an xLO. Check for a Content-Location in the HEAD response and leave such segments. Co-Authored-By: Clay Gerrard Change-Id: I45b210cf380a68bd88187c91fa2d63a8b2bb709b --- swiftclient/service.py | 6 ++++-- tests/unit/test_service.py | 15 ++++++++------- tests/unit/test_shell.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 9 deletions(-) diff --git a/swiftclient/service.py b/swiftclient/service.py index 06de091e..5292dc5a 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -2070,7 +2070,8 @@ def _upload_object_job(self, conn, container, source, obj, options, 'status': 'skipped-changed' }) return res - if not options['leave_segments']: + if not options['leave_segments'] and not headers.get( + 'content-location'): old_manifest = headers.get('x-object-manifest') if is_slo: old_slo_manifest_paths.extend( @@ -2515,7 +2516,8 @@ def _delete_object(self, conn, container, obj, options, if not options['leave_segments']: try: headers = conn.head_object(container, obj, - headers=_headers) + headers=_headers, + query_string='symlink=get') old_manifest = headers.get('x-object-manifest') if config_true_value(headers.get('x-static-large-object')): query_string = 'multipart-manifest=delete' diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index 12fbaa00..b7603522 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -312,8 +312,8 @@ def test_delete_object(self): s = SwiftService() r = s._delete_object(mock_conn, 'test_c', 'test_o', self.opts, mock_q) - mock_conn.head_object.assert_called_once_with('test_c', 'test_o', - headers={}) + mock_conn.head_object.assert_called_once_with( + 'test_c', 'test_o', query_string='symlink=get', headers={}) mock_conn.delete_object.assert_called_once_with( 'test_c', 'test_o', query_string=None, response_dict={}, headers={} @@ -335,7 +335,8 @@ def test_delete_object_with_headers(self): r = s._delete_object(mock_conn, 'test_c', 'test_o', opt_c, mock_q) mock_conn.head_object.assert_called_once_with( - 'test_c', 'test_o', headers={'Skip-Middleware': 'Test'}) + 'test_c', 'test_o', headers={'Skip-Middleware': 'Test'}, + query_string='symlink=get') mock_conn.delete_object.assert_called_once_with( 'test_c', 'test_o', query_string=None, response_dict={}, headers={'Skip-Middleware': 'Test'} @@ -362,8 +363,8 @@ def test_delete_object_exception(self): r = s._delete_object(mock_conn, 'test_c', 'test_o', self.opts, mock_q) after = time.time() - mock_conn.head_object.assert_called_once_with('test_c', 'test_o', - headers={}) + mock_conn.head_object.assert_called_once_with( + 'test_c', 'test_o', query_string='symlink=get', headers={}) mock_conn.delete_object.assert_called_once_with( 'test_c', 'test_o', query_string=None, response_dict={}, headers={} @@ -389,8 +390,8 @@ def test_delete_object_slo_support(self): s = SwiftService() r = s._delete_object(mock_conn, 'test_c', 'test_o', self.opts, mock_q) - mock_conn.head_object.assert_called_once_with('test_c', 'test_o', - headers={}) + mock_conn.head_object.assert_called_once_with( + 'test_c', 'test_o', query_string='symlink=get', headers={}) mock_conn.delete_object.assert_called_once_with( 'test_c', 'test_o', query_string='multipart-manifest=delete', diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index d9ddb3ee..c9722815 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -832,6 +832,35 @@ def test_upload_delete_slo_segments(self, connection): sorted(connection.return_value.delete_object.mock_calls) ) + @mock.patch('swiftclient.service.Connection') + def test_upload_over_symlink_to_slo(self, connection): + # Upload delete existing segments + connection.return_value.head_container.return_value = { + 'x-storage-policy': 'one'} + connection.return_value.attempts = 0 + connection.return_value.head_object.side_effect = [ + {'x-static-large-object': 'true', + 'content-location': '/v1/a/c/manifest', + 'content-length': '2'}, + ] + connection.return_value.get_object.return_value = ( + {'content-location': '/v1/a/c/manifest'}, + b'[{"name": "container1/old_seg1"},' + b' {"name": "container2/old_seg2"}]' + ) + connection.return_value.put_object.return_value = EMPTY_ETAG + connection.return_value.delete_object.return_value = None + argv = ["", "upload", "container", self.tmpfile] + swiftclient.shell.main(argv) + connection.return_value.put_object.assert_called_with( + 'container', + self.tmpfile.lstrip('/'), + mock.ANY, + content_length=0, + headers={'x-object-meta-mtime': mock.ANY}, + response_dict={}) + self.assertEqual([], connection.return_value.delete_object.mock_calls) + @mock.patch('swiftclient.service.Connection') def test_upload_leave_slo_segments(self, connection): # Test upload overwriting a manifest respects --leave-segments From a0f0aedb41ba790266cca678fc3bd91696888835 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Wed, 4 Sep 2019 11:21:04 -0700 Subject: [PATCH 077/238] docs: Fix warning treated as error Change-Id: I669533334419e94ca925e859f2b0d5d2afe9f7f1 --- swiftclient/multithreading.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/swiftclient/multithreading.py b/swiftclient/multithreading.py index fcf0ed95..f128790e 100644 --- a/swiftclient/multithreading.py +++ b/swiftclient/multithreading.py @@ -168,6 +168,12 @@ class ConnectionThreadPoolExecutor(ThreadPoolExecutor): We will only create as many connections as are required concurrently. """ def __init__(self, create_connection, max_workers): + """ + Initializes a new ThreadPoolExecutor instance. + + :param create_connection: callable to use to create new connections + :param max_workers: the maximum number of threads that can be used + """ self._connections = PriorityQueue() self._create_connection = create_connection for p in range(0, max_workers): From 14095c109fe35bdbbaf7703328edf8bfe759700b Mon Sep 17 00:00:00 2001 From: Matthew Oliver Date: Wed, 4 Sep 2019 14:39:40 +1000 Subject: [PATCH 078/238] PDF Documentation Build tox target This patch adds a `pdf-docs` tox target that will build PDF versions of our docs. As per the Train community goal: https://governance.openstack.org/tc/goals/selected/train/pdf-doc-generation.html Story: 2006122 Task: 35514 Change-Id: I7e0ee410ac603774e4b03f859ac3aa20e5afc9b8 --- doc/source/conf.py | 4 +++- tox.ini | 9 +++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 85dd81ef..a8ad3ad7 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -181,7 +181,7 @@ # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]) latex_documents = [ - ('index', 'SwiftClient.tex', u'SwiftClient Documentation', + ('index', 'doc-python-swiftclient.tex', u'SwiftClient Documentation', u'OpenStack, LLC.', 'manual'), ] @@ -201,3 +201,5 @@ # If false, no module index is generated. # latex_use_modindex = True + +latex_use_xindy = False diff --git a/tox.ini b/tox.ini index a2b63b3f..7dd864b3 100644 --- a/tox.ini +++ b/tox.ini @@ -109,3 +109,12 @@ deps = -c{toxinidir}/lower-constraints.txt -r{toxinidir}/test-requirements.txt .[keystone] + +[testenv:pdf-docs] +basepython = python3 +deps = {[testenv:docs]deps} +whitelist_externals = + make +commands = + sphinx-build -W -b latex doc/source doc/build/pdf + make -C doc/build/pdf From 72b90fed4c62cf75497a7d66714f5fb0aa91f18b Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Thu, 12 Sep 2019 16:28:40 -0700 Subject: [PATCH 079/238] Authors/changelog for 3.8.1 Change-Id: I4f178c30723e0da6ba1ec8c8c171137ada631496 --- AUTHORS | 1 + ChangeLog | 15 +++++++++++++++ .../notes/3_8_1_release-cb5648c3ae69bde1.yaml | 14 ++++++++++++++ 3 files changed, 30 insertions(+) create mode 100644 releasenotes/notes/3_8_1_release-cb5648c3ae69bde1.yaml diff --git a/AUTHORS b/AUTHORS index 16dfcf9c..da8e44bb 100644 --- a/AUTHORS +++ b/AUTHORS @@ -24,6 +24,7 @@ Clark Boylan (clark.boylan@gmail.com) Claudiu Belu (cbelu@cloudbasesolutions.com) Clay Gerrard (clay.gerrard@gmail.com) Clint Byrum (clint@fewbar.com) +Corey Bryant (corey.bryant@canonical.com) Dan Prince (dprince@redhat.com) Daniel Wakefield (daniel.wakefield@hp.com) Darrell Bishop (darrell@swiftstack.com) diff --git a/ChangeLog b/ChangeLog index 253a3cc1..501491bf 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,18 @@ +3.8.1 +----- + +* Deleting or overwriting a symlink to an SLO or DLO will no longer attempt + to clean up the large object's segments. + +* Fixed an issue sending non-ASCII metadata keys on Python 3. + Note that receiving such metadata on py3 is still broken; + see https://bugs.python.org/issue37093 + +* Documentation can now be rendered as a PDF. + +* Dropped Python 3.5 testing. + + 3.8.0 ----- diff --git a/releasenotes/notes/3_8_1_release-cb5648c3ae69bde1.yaml b/releasenotes/notes/3_8_1_release-cb5648c3ae69bde1.yaml new file mode 100644 index 00000000..7985d373 --- /dev/null +++ b/releasenotes/notes/3_8_1_release-cb5648c3ae69bde1.yaml @@ -0,0 +1,14 @@ +--- +fixes: + - | + Deleting or overwriting a symlink to an SLO or DLO will no longer attempt + to clean up the large object's segments. + - | + Fixed an issue sending non-ASCII metadata keys on Python 3. + Note that *receiving* such metadata on py3 is `still broken + `__. +other: + - | + Documentation can now be rendered as a PDF. + - | + Dropped Python 3.5 testing. From 606951c736cde79418e9ebcd790c4af3686a2170 Mon Sep 17 00:00:00 2001 From: OpenStack Release Bot Date: Fri, 13 Sep 2019 21:34:49 +0000 Subject: [PATCH 080/238] Update master for stable/train Add file to the reno documentation build to show release notes for stable/train. Use pbr instruction to increment the minor version number automatically so that master versions are higher than the versions on stable/train. Change-Id: I8831476757575fd54fc07154450c6d545fbe6463 Sem-Ver: feature --- releasenotes/source/index.rst | 1 + releasenotes/source/train.rst | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 releasenotes/source/train.rst diff --git a/releasenotes/source/index.rst b/releasenotes/source/index.rst index 27f675ee..662c6f6d 100644 --- a/releasenotes/source/index.rst +++ b/releasenotes/source/index.rst @@ -6,6 +6,7 @@ :maxdepth: 1 current + train stein rocky queens diff --git a/releasenotes/source/train.rst b/releasenotes/source/train.rst new file mode 100644 index 00000000..58390039 --- /dev/null +++ b/releasenotes/source/train.rst @@ -0,0 +1,6 @@ +========================== +Train Series Release Notes +========================== + +.. release-notes:: + :branch: stable/train From 9527d0497f19487114e158de6387fb5483cf3182 Mon Sep 17 00:00:00 2001 From: kangyufei Date: Tue, 22 Oct 2019 14:33:13 +0800 Subject: [PATCH 081/238] Switch to Ussuri jobs Change-Id: Ibf51928ce55e4a96f0d674c693559b2bf9256f11 --- .zuul.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.zuul.yaml b/.zuul.yaml index 4a878bae..4f288d79 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -39,7 +39,7 @@ - openstack-lower-constraints-jobs - openstack-pypy-jobs-nonvoting - openstack-python-jobs - - openstack-python3-train-jobs + - openstack-python3-ussuri-jobs - publish-openstack-docs-pti - release-notes-jobs-python3 check: From 1eda8f9f3eac55953b3d54e32754e2ec312ab348 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Wed, 9 Oct 2019 21:05:47 -0700 Subject: [PATCH 082/238] Rename "tests" directory to be "test" like in the swift repo In addition to being less confusing for devs, this lets us actually run tempauth tests in swiftclient dsvm jobs. The job definition (over in the swift repo) specifies test/sample.conf, which does not exist in this repo. As a result, those tests would skip with SKIPPING FUNCTIONAL TESTS DUE TO NO CONFIG Change-Id: I558dbf9a657d442e6e19468e543bbec855129eeb --- .functests | 2 +- .stestr.conf | 2 +- .unittests | 2 +- {tests => test}/__init__.py | 0 {tests => test}/functional/__init__.py | 0 {tests => test}/functional/test_swiftclient.py | 0 {tests => test}/sample.conf | 0 {tests => test}/unit/__init__.py | 0 {tests => test}/unit/test_authv1.py | 0 {tests => test}/unit/test_command_helpers.py | 0 {tests => test}/unit/test_multithreading.py | 0 {tests => test}/unit/test_service.py | 8 ++++---- {tests => test}/unit/test_shell.py | 0 {tests => test}/unit/test_swiftclient.py | 0 {tests => test}/unit/test_utils.py | 0 {tests => test}/unit/utils.py | 0 tox.ini | 4 ++-- 17 files changed, 9 insertions(+), 9 deletions(-) rename {tests => test}/__init__.py (100%) rename {tests => test}/functional/__init__.py (100%) rename {tests => test}/functional/test_swiftclient.py (100%) rename {tests => test}/sample.conf (100%) rename {tests => test}/unit/__init__.py (100%) rename {tests => test}/unit/test_authv1.py (100%) rename {tests => test}/unit/test_command_helpers.py (100%) rename {tests => test}/unit/test_multithreading.py (100%) rename {tests => test}/unit/test_service.py (99%) rename {tests => test}/unit/test_shell.py (100%) rename {tests => test}/unit/test_swiftclient.py (100%) rename {tests => test}/unit/test_utils.py (100%) rename {tests => test}/unit/utils.py (100%) diff --git a/.functests b/.functests index d199ec84..288c9e61 100755 --- a/.functests +++ b/.functests @@ -1,7 +1,7 @@ #!/bin/bash set -e -export OS_TEST_PATH='tests.functional' +export OS_TEST_PATH='test.functional' export PYTHON='coverage run --source swiftclient --parallel-mode' stestr run --concurrency=1 diff --git a/.stestr.conf b/.stestr.conf index 5228f209..90f5b813 100644 --- a/.stestr.conf +++ b/.stestr.conf @@ -1,4 +1,4 @@ [DEFAULT] -test_path=${OS_TEST_PATH:-./tests/unit} +test_path=${OS_TEST_PATH:-./test/unit} top_dir=./ diff --git a/.unittests b/.unittests index 5d935b1a..3a7ffe9b 100755 --- a/.unittests +++ b/.unittests @@ -1,7 +1,7 @@ #!/bin/bash set -e -python setup.py testr --coverage --testr-args="tests.unit" +python setup.py testr --coverage --testr-args="test.unit" RET=$? coverage report -m rm -f .coverage diff --git a/tests/__init__.py b/test/__init__.py similarity index 100% rename from tests/__init__.py rename to test/__init__.py diff --git a/tests/functional/__init__.py b/test/functional/__init__.py similarity index 100% rename from tests/functional/__init__.py rename to test/functional/__init__.py diff --git a/tests/functional/test_swiftclient.py b/test/functional/test_swiftclient.py similarity index 100% rename from tests/functional/test_swiftclient.py rename to test/functional/test_swiftclient.py diff --git a/tests/sample.conf b/test/sample.conf similarity index 100% rename from tests/sample.conf rename to test/sample.conf diff --git a/tests/unit/__init__.py b/test/unit/__init__.py similarity index 100% rename from tests/unit/__init__.py rename to test/unit/__init__.py diff --git a/tests/unit/test_authv1.py b/test/unit/test_authv1.py similarity index 100% rename from tests/unit/test_authv1.py rename to test/unit/test_authv1.py diff --git a/tests/unit/test_command_helpers.py b/test/unit/test_command_helpers.py similarity index 100% rename from tests/unit/test_command_helpers.py rename to test/unit/test_command_helpers.py diff --git a/tests/unit/test_multithreading.py b/test/unit/test_multithreading.py similarity index 100% rename from tests/unit/test_multithreading.py rename to test/unit/test_multithreading.py diff --git a/tests/unit/test_service.py b/test/unit/test_service.py similarity index 99% rename from tests/unit/test_service.py rename to test/unit/test_service.py index b7603522..ed3a2d6e 100644 --- a/tests/unit/test_service.py +++ b/test/unit/test_service.py @@ -36,7 +36,7 @@ SwiftService, SwiftError, SwiftUploadObject ) -from tests.unit import utils as test_utils +from test.unit import utils as test_utils clean_os_environ = {} @@ -1060,11 +1060,11 @@ def test_upload_with_bad_segment_size(self): @mock.patch('swiftclient.service.getsize', return_value=4) def test_upload_with_relative_path(self, *args, **kwargs): service = SwiftService({}) - objects = [{'path': "./test", + objects = [{'path': "./testobj", 'strt_indx': 2}, - {'path': os.path.join(os.getcwd(), "test"), + {'path': os.path.join(os.getcwd(), "testobj"), 'strt_indx': 1}, - {'path': ".\\test", + {'path': ".\\testobj", 'strt_indx': 2}] for obj in objects: with mock.patch('swiftclient.service.Connection') as mock_conn, \ diff --git a/tests/unit/test_shell.py b/test/unit/test_shell.py similarity index 100% rename from tests/unit/test_shell.py rename to test/unit/test_shell.py diff --git a/tests/unit/test_swiftclient.py b/test/unit/test_swiftclient.py similarity index 100% rename from tests/unit/test_swiftclient.py rename to test/unit/test_swiftclient.py diff --git a/tests/unit/test_utils.py b/test/unit/test_utils.py similarity index 100% rename from tests/unit/test_utils.py rename to test/unit/test_utils.py diff --git a/tests/unit/utils.py b/test/unit/utils.py similarity index 100% rename from tests/unit/utils.py rename to test/unit/utils.py diff --git a/tox.ini b/tox.ini index 7dd864b3..002d24c5 100644 --- a/tox.ini +++ b/tox.ini @@ -24,7 +24,7 @@ passenv = SWIFT_* *_proxy [testenv:pep8] basepython = python3 commands = - python -m flake8 swiftclient tests + python -m flake8 swiftclient test [testenv:venv] basepython = python3 @@ -44,7 +44,7 @@ commands = [testenv:func] basepython = python3 setenv = - OS_TEST_PATH=tests.functional + OS_TEST_PATH=test.functional PYTHON=coverage run --source swiftclient --parallel-mode whitelist_externals = coverage From c4bef14fc1975f2e115a6ec8560e674e8aa5b1bf Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Wed, 9 Oct 2019 16:59:23 -0700 Subject: [PATCH 083/238] v1auth: support endpoint_data_for() api ...so we can be used with openstacksdk. Also, add a few functests that use openstacksdk. Change-Id: Ie6987f5de48914ec8932254cde79a973a0264877 --- lower-constraints.txt | 1 + swiftclient/authv1.py | 12 +++- test-requirements.txt | 1 + test/functional/__init__.py | 93 ++++++++++++++++++++++++++++ test/functional/test_openstacksdk.py | 92 +++++++++++++++++++++++++++ test/functional/test_swiftclient.py | 85 +++++++------------------ 6 files changed, 221 insertions(+), 63 deletions(-) create mode 100644 test/functional/test_openstacksdk.py diff --git a/lower-constraints.txt b/lower-constraints.txt index 88d28650..ead02791 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -21,6 +21,7 @@ mccabe==0.2.1 mock==1.2.0 netaddr==0.7.10 openstackdocstheme==1.20.0 +openstacksdk==0.11.0 oslo.config==1.2.0 pbr==2.0.0 pep8==1.5.7 diff --git a/swiftclient/authv1.py b/swiftclient/authv1.py index 55469acf..d70acac3 100644 --- a/swiftclient/authv1.py +++ b/swiftclient/authv1.py @@ -45,6 +45,7 @@ # Note that while we import keystoneauth1 here, we *don't* need to add it to # requirements.txt -- this entire module only makes sense (and should only be # loaded) if keystoneauth is already installed. +from keystoneauth1 import discover from keystoneauth1 import plugin from keystoneauth1 import exceptions from keystoneauth1 import loading @@ -110,11 +111,20 @@ def catalog(self): ] def url_for(self, **kwargs): + return self.endpoint_data_for(**kwargs).url + + def endpoint_data_for(self, **kwargs): kwargs.setdefault('interface', 'public') kwargs.setdefault('service_type', None) if kwargs['service_type'] == 'object-store': - return self.storage_url + return discover.EndpointData( + service_type='object-store', + service_name='swift', + interface=kwargs['interface'], + region_name='default', + catalog_url=self.storage_url, + ) # Although our "catalog" includes an identity entry, nothing that uses # url_for() (including `openstack endpoint list`) will know what to do diff --git a/test-requirements.txt b/test-requirements.txt index b3ca5f89..13732533 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -4,3 +4,4 @@ coverage!=4.4,>=4.0 # Apache-2.0 keystoneauth1>=3.4.0 # Apache-2.0 mock>=1.2.0 # BSD stestr>=2.0.0 # Apache-2.0 +openstacksdk>=0.11.0 # Apache-2.0 diff --git a/test/functional/__init__.py b/test/functional/__init__.py index e69de29b..248875a6 100644 --- a/test/functional/__init__.py +++ b/test/functional/__init__.py @@ -0,0 +1,93 @@ +# Copyright (c) 2014 Christian Schwede +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from six.moves import configparser + +TEST_CONFIG = None + + +def _load_config(force_reload=False): + global TEST_CONFIG + if not force_reload and TEST_CONFIG is not None: + return TEST_CONFIG + + config_file = os.environ.get('SWIFT_TEST_CONFIG_FILE', + '/etc/swift/test.conf') + parser = configparser.ConfigParser({'auth_version': '1'}) + parser.read(config_file) + conf = {} + if parser.has_section('func_test'): + if parser.has_option('func_test', 'auth_uri'): + conf['auth_url'] = parser.get('func_test', 'auth_uri') + try: + conf['auth_version'] = parser.get('func_test', 'auth_version') + except configparser.NoOptionError: + last_piece = conf['auth_url'].rstrip('/').rsplit('/', 1)[1] + if last_piece.endswith('.0'): + last_piece = last_piece[:-2] + if last_piece in ('1', '2', '3'): + conf['auth_version'] = last_piece + else: + raise + else: + auth_host = parser.get('func_test', 'auth_host') + auth_port = parser.getint('func_test', 'auth_port') + auth_ssl = parser.getboolean('func_test', 'auth_ssl') + auth_prefix = parser.get('func_test', 'auth_prefix') + conf['auth_version'] = parser.get('func_test', 'auth_version') + if auth_ssl: + auth_url = "https://" + else: + auth_url = "http://" + auth_url += "%s:%s%s" % (auth_host, auth_port, auth_prefix) + if conf['auth_version'] == "1": + auth_url += 'v1.0' + conf['auth_url'] = auth_url + + try: + conf['account_username'] = parser.get('func_test', + 'account_username') + except configparser.NoOptionError: + conf['account'] = parser.get('func_test', 'account') + conf['username'] = parser.get('func_test', 'username') + conf['account_username'] = "%s:%s" % (conf['account'], + conf['username']) + else: + # Still try to get separate account/usernames for keystone tests + try: + conf['account'] = parser.get('func_test', 'account') + conf['username'] = parser.get('func_test', 'username') + except configparser.NoOptionError: + pass + + conf['password'] = parser.get('func_test', 'password') + + # For keystone v3 + try: + conf['account4'] = parser.get('func_test', 'account4') + conf['username4'] = parser.get('func_test', 'username4') + conf['domain4'] = parser.get('func_test', 'domain4') + conf['password4'] = parser.get('func_test', 'password4') + except configparser.NoOptionError: + pass + + TEST_CONFIG = conf + + +try: + _load_config() +except configparser.NoOptionError: + TEST_CONFIG = None # sentinel used in test setup diff --git a/test/functional/test_openstacksdk.py b/test/functional/test_openstacksdk.py new file mode 100644 index 00000000..cee7f4e9 --- /dev/null +++ b/test/functional/test_openstacksdk.py @@ -0,0 +1,92 @@ +# Copyright (c) 2019 Tim Burke +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import uuid + +import openstack + +from . import TEST_CONFIG + +PREFIX = 'test-swiftclient-' + + +class TestOpenStackSDK(unittest.TestCase): + @classmethod + def setUpClass(cls): + # NB: Only runs for v1 auth, to exercise our keystoneauth plugin + cls.skip_tests = (TEST_CONFIG is None or + TEST_CONFIG['auth_version'] != '1') + if not cls.skip_tests: + cls.conn = openstack.connect( + auth_type='v1password', + auth_url=TEST_CONFIG['auth_url'], + username=TEST_CONFIG['account_username'], + password=TEST_CONFIG['password'], + ) + cls.object_store = cls.conn.object_store + + def setUp(self): + if self.skip_tests: + raise unittest.SkipTest('SKIPPING V1-AUTH TESTS') + + def tearDown(self): + if self.skip_tests: + return + for c in self.object_store.containers(): + if c.name.startswith(PREFIX): + for o in self.object_store.objects(c.name): + self.object_store.delete_object( + o.name, container=c.name) + self.object_store.delete_container(c.name) + + def test_containers(self): + meta = self.object_store.get_account_metadata() + count_before = meta.account_container_count + containers = sorted(PREFIX + str(uuid.uuid4()) + for _ in range(10)) + for c in containers: + self.object_store.create_container(c) + self.assertEqual([ + c.name for c in self.object_store.containers() + if c.name.startswith(PREFIX) + ], containers) + meta = self.object_store.get_account_metadata() + self.assertEqual(count_before + len(containers), + meta.account_container_count) + + def test_objects(self): + container = PREFIX + str(uuid.uuid4()) + self.object_store.create_container(container) + objects = sorted(str(uuid.uuid4()) for _ in range(10)) + for o in objects: + self.object_store.create_object(container, o, data=b'x') + self.assertEqual([ + o.name for o in self.object_store.objects(container) + ], objects) + meta = self.object_store.get_container_metadata(container) + self.assertEqual(len(objects), meta.object_count) + + def test_object_metadata(self): + container = PREFIX + str(uuid.uuid4()) + self.object_store.create_container(container) + obj = str(uuid.uuid4()) + obj_meta = {str(uuid.uuid4()): str(uuid.uuid4()) for _ in range(10)} + # NB: as of 0.36.0, create_object() doesn't play well with passing + # both data and metadata, so we do a PUT then POST + self.object_store.create_object(container, obj, data=b'x') + self.object_store.set_object_metadata(obj, container, **obj_meta) + meta = self.object_store.get_object_metadata(obj, container) + self.assertEqual(obj_meta, meta.metadata) diff --git a/test/functional/test_swiftclient.py b/test/functional/test_swiftclient.py index 9a74c63f..54c514de 100644 --- a/test/functional/test_swiftclient.py +++ b/test/functional/test_swiftclient.py @@ -13,23 +13,23 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import unittest import time from io import BytesIO import six -from six.moves import configparser import swiftclient +from . import TEST_CONFIG class TestFunctional(unittest.TestCase): def __init__(self, *args, **kwargs): super(TestFunctional, self).__init__(*args, **kwargs) - self.skip_tests = False - self._get_config() + self.skip_tests = (TEST_CONFIG is None) + if not self.skip_tests: + self._get_config() self.test_data = b'42' * 10 self.etag = '2704306ec982238d85d4b235c925d58e' @@ -41,50 +41,10 @@ def __init__(self, *args, **kwargs): self.objectname_2 = self.objectname + '_second' def _get_config(self): - config_file = os.environ.get('SWIFT_TEST_CONFIG_FILE', - '/etc/swift/test.conf') - config = configparser.ConfigParser({'auth_version': '1'}) - config.read(config_file) - self.config = config - if config.has_section('func_test'): - if config.has_option('func_test', 'auth_uri'): - self.auth_url = config.get('func_test', 'auth_uri') - try: - self.auth_version = config.get('func_test', 'auth_version') - except configparser.NoOptionError: - last_piece = self.auth_url.rstrip('/').rsplit('/', 1)[1] - if last_piece.endswith('.0'): - last_piece = last_piece[:-2] - if last_piece in ('1', '2', '3'): - self.auth_version = last_piece - else: - raise - else: - auth_host = config.get('func_test', 'auth_host') - auth_port = config.getint('func_test', 'auth_port') - auth_ssl = config.getboolean('func_test', 'auth_ssl') - auth_prefix = config.get('func_test', 'auth_prefix') - self.auth_version = config.get('func_test', 'auth_version') - self.auth_url = "" - if auth_ssl: - self.auth_url += "https://" - else: - self.auth_url += "http://" - self.auth_url += "%s:%s%s" % ( - auth_host, auth_port, auth_prefix) - if self.auth_version == "1": - self.auth_url += 'v1.0' - - try: - self.account_username = config.get('func_test', - 'account_username') - except configparser.NoOptionError: - account = config.get('func_test', 'account') - username = config.get('func_test', 'username') - self.account_username = "%s:%s" % (account, username) - self.password = config.get('func_test', 'password') - else: - self.skip_tests = True + self.auth_url = TEST_CONFIG['auth_url'] + self.auth_version = TEST_CONFIG['auth_version'] + self.account_username = TEST_CONFIG['account_username'] + self.password = TEST_CONFIG['password'] def _get_connection(self): """ @@ -514,20 +474,20 @@ class TestUsingKeystone(TestFunctional): """ def _get_connection(self): - account = username = password = None + account = username = None if self.auth_version not in ('2', '3'): self.skipTest('SKIPPING KEYSTONE-SPECIFIC FUNCTIONAL TESTS') try: - account = self.config.get('func_test', 'account') - username = self.config.get('func_test', 'username') - password = self.config.get('func_test', 'password') - except Exception: + account = TEST_CONFIG['account'] + username = TEST_CONFIG['username'] + except KeyError: self.skipTest('SKIPPING KEYSTONE-SPECIFIC FUNCTIONAL TESTS' + ' - NO CONFIG') - os_options = {'tenant_name': account} + return swiftclient.Connection( - self.auth_url, username, password, auth_version=self.auth_version, - os_options=os_options) + self.auth_url, username, self.password, + auth_version=self.auth_version, + os_options={'tenant_name': account}) class TestUsingKeystoneV3(TestFunctional): @@ -539,13 +499,14 @@ def _get_connection(self): account = username = password = project_domain = user_domain = None if self.auth_version != '3': self.skipTest('SKIPPING KEYSTONE-V3-SPECIFIC FUNCTIONAL TESTS') + try: - account = self.config.get('func_test', 'account4') - username = self.config.get('func_test', 'username4') - user_domain = self.config.get('func_test', 'domain4') - project_domain = self.config.get('func_test', 'domain4') - password = self.config.get('func_test', 'password4') - except Exception: + account = TEST_CONFIG['account4'] + username = TEST_CONFIG['username4'] + user_domain = TEST_CONFIG['domain4'] + project_domain = TEST_CONFIG['domain4'] + password = TEST_CONFIG['password4'] + except KeyError: self.skipTest('SKIPPING KEYSTONE-V3-SPECIFIC FUNCTIONAL TESTS' + ' - NO CONFIG') From 709ab385c6eb3d3c7b313bc48c959e9ace606ae5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Blaisot?= Date: Fri, 15 Nov 2019 22:37:38 +0100 Subject: [PATCH 084/238] Fix printed object names on successful bulk-delete Replace the 1 always concatenated to printed object names for each successfully deleted object in bulk-delete with an optional [after x attempts] if x > 1 Change-Id: If4af9141fe4f3436a4e9e0e2dfc24c6ec7292996 Closes-Bug: 1852808 --- swiftclient/shell.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/swiftclient/shell.py b/swiftclient/shell.py index cc4f325a..5e23bc46 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -169,7 +169,8 @@ def st_delete(parser, args, output_manager, return_parser=False): for r in del_iter: c = r.get('container', '') o = r.get('object', '') - a = r.get('attempts') + a = (' [after {0} attempts]'.format(r.get('attempts')) + if r.get('attempts') > 1 else '') if r['action'] == 'bulk_delete': if r['success']: @@ -202,9 +203,6 @@ def st_delete(parser, args, output_manager, return_parser=False): else: if r['success']: if options['verbose']: - a = (' [after {0} attempts]'.format(a) - if a > 1 else '') - if r['action'] == 'delete_object': if options['yes_all']: p = '{0}/{1}'.format(c, o) From e83cd32e2af26ecb9ac9520ac2958f186ba1888c Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Fri, 15 Nov 2019 22:08:51 +0000 Subject: [PATCH 085/238] Add test for bulk-delete-attempt-counter fix Change-Id: Ifdeefeb4a5a3fc6895bd6cda695684de02f8c602 Related-Change: If4af9141fe4f3436a4e9e0e2dfc24c6ec7292996 Related-Bug: #1852808 --- swiftclient/shell.py | 2 +- test/unit/test_shell.py | 22 +++++++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 5e23bc46..d18fc9e7 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -170,7 +170,7 @@ def st_delete(parser, args, output_manager, return_parser=False): c = r.get('container', '') o = r.get('object', '') a = (' [after {0} attempts]'.format(r.get('attempts')) - if r.get('attempts') > 1 else '') + if r.get('attempts', 1) > 1 else '') if r['action'] == 'bulk_delete': if r['success']: diff --git a/test/unit/test_shell.py b/test/unit/test_shell.py index c9722815..1fa0db40 100644 --- a/test/unit/test_shell.py +++ b/test/unit/test_shell.py @@ -1420,12 +1420,32 @@ def test_delete_bulk_object(self, connection): b'{"Number Not Found": 0, "Response Status": "200 OK", ' b'"Errors": [], "Number Deleted": 1, "Response Body": ""}') connection.return_value.attempts = 0 - swiftclient.shell.main(argv) + with CaptureOutput() as out: + swiftclient.shell.main(argv) + connection.return_value.post_account.assert_called_with( + query_string='bulk-delete', data=b'/container/object\n', + headers={'Content-Type': 'text/plain', + 'Accept': 'application/json'}, + response_dict={}) + self.assertEqual('object\n', out.out) + + @mock.patch.object(swiftclient.service.SwiftService, + '_bulk_delete_page_size', lambda *a: 10) + @mock.patch('swiftclient.service.Connection') + def test_delete_bulk_object_with_retry(self, connection): + argv = ["", "delete", "container", "object"] + connection.return_value.post_account.return_value = {}, ( + b'{"Number Not Found": 0, "Response Status": "200 OK", ' + b'"Errors": [], "Number Deleted": 1, "Response Body": ""}') + connection.return_value.attempts = 3 + with CaptureOutput() as out: + swiftclient.shell.main(argv) connection.return_value.post_account.assert_called_with( query_string='bulk-delete', data=b'/container/object\n', headers={'Content-Type': 'text/plain', 'Accept': 'application/json'}, response_dict={}) + self.assertEqual('object [after 3 attempts]\n', out.out) def test_delete_verbose_output(self): del_obj_res = {'success': True, 'response_dict': {}, 'attempts': 2, From 1f26c5736949e1c3b57c024a315e33fc419f126e Mon Sep 17 00:00:00 2001 From: Alex Schultz Date: Fri, 2 Aug 2019 08:20:56 -0600 Subject: [PATCH 086/238] Cleanup session on delete If an external http connection was not passed into the client, we create one with a requests.Session() on our own. Once this is used, it may still have an open socket when the connection is closed. We need to handle the closing of the requests.Session() ourselves if we created one. If you do not close it, a ResourceWarning may be reported about the socket that is left open. Change-Id: I200ad0cdc8b7999c3f5521b9a822122bd18714bf Closes-Bug: #1838775 --- swiftclient/client.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/swiftclient/client.py b/swiftclient/client.py index 4be2e2d6..448bb468 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -438,6 +438,15 @@ def __init__(self, url, proxy=None, cacert=None, insecure=False, if timeout: self.requests_args['timeout'] = timeout + def __del__(self): + """Cleanup resources other than memory""" + if self.request_session: + # The session we create must be closed to free up file descriptors + try: + self.request_session.close() + finally: + self.request_session = None + def _request(self, *arg, **kwarg): """Final wrapper before requests call, to be patched in tests""" return self.request_session.request(*arg, **kwarg) From 13970ac5fa2361a97fd2565bde9f08e23262d6fd Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Mon, 20 Jan 2020 21:20:12 -0800 Subject: [PATCH 087/238] packaging: Properly flag universal wheel I'm not sure we've *ever* done this properly; see the PyPA docs: https://packaging.python.org/guides/distributing-packages-using-setuptools/#universal-wheels Change-Id: I8bb9e05f386076aa652b3955f0abf757d229afed --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 5653bf30..bcb4d223 100644 --- a/setup.cfg +++ b/setup.cfg @@ -51,7 +51,7 @@ all_files = 1 [upload_sphinx] upload-dir = doc/build/html -[wheel] +[bdist_wheel] universal = 1 [pbr] From 0bbc2cdbec2ffa6482c40fc45af22140f48d2670 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Wed, 22 Jan 2020 08:31:23 -0800 Subject: [PATCH 088/238] Make py38 job voting Depends-On: https://review.opendev.org/#/c/693401/ Change-Id: Ifb4e466eda0c45b49c16e63b0c77023f65b039b8 --- .zuul.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.zuul.yaml b/.zuul.yaml index 4f288d79..edf0d28e 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -47,11 +47,15 @@ - swiftclient-swift-functional - swiftclient-functional - swiftclient-functional-py2 + - openstack-tox-py38: + voting: true gate: jobs: - swiftclient-swift-functional - swiftclient-functional - swiftclient-functional-py2 + - openstack-tox-py38: + voting: true post: jobs: - openstack-tox-cover From 259b98f69dd0b9929ee9e04ccb37183d2666d227 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Wed, 12 Feb 2020 17:34:25 -0800 Subject: [PATCH 089/238] Authors/changelog for 3.9.0 Change-Id: I661503e0d6bb7934f7e7a28b094264a2ee73a419 --- AUTHORS | 3 +++ ChangeLog | 11 +++++++++++ .../notes/3_9_0_release-3c293d277f14ec22.yaml | 12 ++++++++++++ 3 files changed, 26 insertions(+) create mode 100644 releasenotes/notes/3_9_0_release-3c293d277f14ec22.yaml diff --git a/AUTHORS b/AUTHORS index da8e44bb..165e9d07 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,5 +1,6 @@ Alessandro Pilotti (ap@pilotti.it) Alex Gaynor (alex.gaynor@gmail.com) +Alex Schultz (aschultz@redhat.com) Alexandra Settle (alexandra.settle@rackspace.com) Alexis Lee (lxsli@hpe.com) Alistair Coles (alistairncoles@gmail.com) @@ -70,6 +71,7 @@ Josh Gachnang (josh@pcsforeducation.com) Juan J. Martinez (juan@memset.com) Jude Job (judeopenstack@gmail.com) Julien Danjou (julien@danjou.info) +kangyufei (kangyf@inspur.com) Kazufumi Noto (noto.kazufumi@gmail.com) Kota Tsuyuzaki (tsuyuzaki.kota@lab.ntt.co.jp) Kun Huang (gareth@unitedstack.com) @@ -117,6 +119,7 @@ SaiKiran (saikiranveeravarapu@gmail.com) Sam Morrison (sorrison@gmail.com) Samuel Merritt (sam@swiftstack.com) Sean Dague (sean@dague.net) +Sébastien Blaisot (sebastien@blaisot.org) Sergey Gotliv (sgotliv@redhat.com) Sergio Cazzolato (sergio.j.cazzolato@intel.com) Shane Wang (shane.wang@intel.com) diff --git a/ChangeLog b/ChangeLog index 501491bf..cecee2da 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,14 @@ +3.9.0 +----- + +* Now tested under Python 3.8. + +* Better clean up connections when using the low-level client.py API. + +* Fixed a display issue when `swift delete` made multiple attempts to bulk + delete objects. + + 3.8.1 ----- diff --git a/releasenotes/notes/3_9_0_release-3c293d277f14ec22.yaml b/releasenotes/notes/3_9_0_release-3c293d277f14ec22.yaml new file mode 100644 index 00000000..6286b616 --- /dev/null +++ b/releasenotes/notes/3_9_0_release-3c293d277f14ec22.yaml @@ -0,0 +1,12 @@ +--- +features: + - | + Now tested under Python 3.8. + +fixes: + - | + Better clean up connections when using the low-level ``client.py`` API. + + - | + Fixed a display issue when ``swift delete`` made multiple attempts to + bulk delete objects. From 02e8f4f228c006927fe87f8a350c281b9cfccd98 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Tue, 7 Apr 2020 22:44:50 -0700 Subject: [PATCH 090/238] Blacklist stestr 3.0.0 It claims py2 support, but that's a lie. Also, switch our tempest job to the py3 variant, now that glance (at least) is py3-only and tempest-full is broken. Change-Id: Ic30fe82ff72fe4d138ec4823d36f2a1cc56f1ac7 --- .zuul.yaml | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index edf0d28e..f0bd82c5 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -35,7 +35,7 @@ - project: templates: - check-requirements - - lib-forward-testing + - lib-forward-testing-python3 - openstack-lower-constraints-jobs - openstack-pypy-jobs-nonvoting - openstack-python-jobs diff --git a/test-requirements.txt b/test-requirements.txt index 13732533..5dba1a60 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -3,5 +3,5 @@ hacking>=1.1.0,<1.2.0 # Apache-2.0 coverage!=4.4,>=4.0 # Apache-2.0 keystoneauth1>=3.4.0 # Apache-2.0 mock>=1.2.0 # BSD -stestr>=2.0.0 # Apache-2.0 +stestr>=2.0.0,!=3.0.0 # Apache-2.0 openstacksdk>=0.11.0 # Apache-2.0 From 78edffa46c591fdc53f253b343e1ea144e24089d Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Tue, 29 Oct 2019 09:59:03 -0500 Subject: [PATCH 091/238] object versioning features * add --versions to list * add --versions to delete * add --version-id to stat * add --version-id to delete * add --version-id to download Change-Id: I89802064921778fee7efe57c7d60c976cdde3a27 --- swiftclient/client.py | 24 ++- swiftclient/command_helpers.py | 6 +- swiftclient/service.py | 122 +++++++++++++--- swiftclient/shell.py | 39 ++++- test/unit/test_service.py | 259 ++++++++++++++++++++++++++++++--- test/unit/test_shell.py | 220 ++++++++++++++++++++++++++-- 6 files changed, 611 insertions(+), 59 deletions(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index 448bb468..449b6cd3 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -921,7 +921,7 @@ def post_account(url, token, headers, http_conn=None, response_dict=None, def get_container(url, token, container, marker=None, limit=None, prefix=None, delimiter=None, end_marker=None, - path=None, http_conn=None, + version_marker=None, path=None, http_conn=None, full_listing=False, service_token=None, headers=None, query_string=None): """ @@ -935,6 +935,7 @@ def get_container(url, token, container, marker=None, limit=None, :param prefix: prefix query :param delimiter: string to delimit the queries on :param end_marker: marker query + :param version_marker: version marker query :param path: path query (equivalent: "delimiter=/" and "prefix=path/") :param http_conn: a tuple of (parsed url, HTTPConnection object), (If None, it will create the conn object) @@ -951,17 +952,20 @@ def get_container(url, token, container, marker=None, limit=None, http_conn = http_connection(url) if full_listing: rv = get_container(url, token, container, marker, limit, prefix, - delimiter, end_marker, path, http_conn, - service_token=service_token, headers=headers) + delimiter, end_marker, version_marker, path=path, + http_conn=http_conn, service_token=service_token, + headers=headers) listing = rv[1] while listing: if not delimiter: marker = listing[-1]['name'] else: marker = listing[-1].get('name', listing[-1].get('subdir')) + version_marker = listing[-1].get('version_id') listing = get_container(url, token, container, marker, limit, - prefix, delimiter, end_marker, path, - http_conn, service_token=service_token, + prefix, delimiter, end_marker, + version_marker, path, http_conn, + service_token=service_token, headers=headers)[1] if listing: rv[1].extend(listing) @@ -979,6 +983,8 @@ def get_container(url, token, container, marker=None, limit=None, qs += '&delimiter=%s' % quote(delimiter) if end_marker: qs += '&end_marker=%s' % quote(end_marker) + if version_marker: + qs += '&version_marker=%s' % quote(version_marker) if path: qs += '&path=%s' % quote(path) if query_string: @@ -1816,15 +1822,17 @@ def head_container(self, container, headers=None): return self._retry(None, head_container, container, headers=headers) def get_container(self, container, marker=None, limit=None, prefix=None, - delimiter=None, end_marker=None, path=None, - full_listing=False, headers=None, query_string=None): + delimiter=None, end_marker=None, version_marker=None, + path=None, full_listing=False, headers=None, + query_string=None): """Wrapper for :func:`get_container`""" # TODO(unknown): With full_listing=True this will restart the entire # listing with each retry. Need to make a better version that just # retries where it left off. return self._retry(None, get_container, container, marker=marker, limit=limit, prefix=prefix, delimiter=delimiter, - end_marker=end_marker, path=path, + end_marker=end_marker, + version_marker=version_marker, path=path, full_listing=full_listing, headers=headers, query_string=query_string) diff --git a/swiftclient/command_helpers.py b/swiftclient/command_helpers.py index 49ccad1d..f37040f8 100644 --- a/swiftclient/command_helpers.py +++ b/swiftclient/command_helpers.py @@ -143,7 +143,11 @@ def print_container_stats(items, headers, output_manager): def stat_object(conn, options, container, obj): req_headers = split_request_headers(options.get('header', [])) - headers = conn.head_object(container, obj, headers=req_headers) + query_string = None + if options.get('version_id') is not None: + query_string = 'version-id=%s' % options['version_id'] + headers = conn.head_object(container, obj, headers=req_headers, + query_string=query_string) items = [] if options['verbose'] > 1: path = '%s/%s/%s' % (conn.url, container, obj) diff --git a/swiftclient/service.py b/swiftclient/service.py index 5292dc5a..fb334fde 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -86,6 +86,9 @@ def __str__(self): value += " segment:%s" % self.segment return value + def __repr__(self): + return str(self) + def process_options(options): # tolerate sloppy auth_version @@ -186,6 +189,7 @@ def _build_default_global_options(): 'leave_segments': False, 'changed': None, 'skip_identical': False, + 'version_id': None, 'yes_all': False, 'read_acl': None, 'write_acl': None, @@ -200,6 +204,7 @@ def _build_default_global_options(): 'meta': [], 'prefix': None, 'delimiter': None, + 'versions': False, 'fail_fast': False, 'human': False, 'dir_marker': False, @@ -336,6 +341,20 @@ def __init__(self, object_name, options=None): self.options = options +class SwiftDeleteObject(object): + """ + Class for specifying an object delete, allowing the headers/metadata to be + specified separately for each individual object. + """ + def __init__(self, object_name, options=None): + if not (isinstance(object_name, string_types) and object_name): + raise SwiftError( + "Object names must be specified as non-empty strings" + ) + self.object_name = object_name + self.options = options + + class SwiftCopyObject(object): """ Class for specifying an object copy, @@ -489,6 +508,7 @@ def stat(self, container=None, objects=None, options=None): { 'human': False, + 'version_id': None, 'header': [] } @@ -871,6 +891,7 @@ def list(self, container=None, options=None): 'long': False, 'prefix': None, 'delimiter': None, + 'versions': False, 'header': [] } @@ -967,13 +988,19 @@ def _list_account_job(conn, options, result_queue): @staticmethod def _list_container_job(conn, container, options, result_queue): marker = options.get('marker', '') + version_marker = options.get('version_marker', '') error = None req_headers = split_headers(options.get('header', [])) + if options.get('versions', False): + query_string = 'versions=true' + else: + query_string = None try: while True: _, items = conn.get_container( - container, marker=marker, prefix=options['prefix'], - delimiter=options['delimiter'], headers=req_headers + container, marker=marker, version_marker=version_marker, + prefix=options['prefix'], delimiter=options['delimiter'], + headers=req_headers, query_string=query_string ) if not items: @@ -991,6 +1018,7 @@ def _list_container_job(conn, container, options, result_queue): result_queue.put(res) marker = items[-1].get('name', items[-1].get('subdir')) + version_marker = items[-1].get('version_id', '') except ClientException as err: traceback, err_time = report_traceback() logger.exception(err) @@ -1016,6 +1044,7 @@ def _list_container_job(conn, container, options, result_queue): 'prefix': options['prefix'], 'success': False, 'marker': marker, + 'version_marker': version_marker, 'error': error[0], 'traceback': error[1], 'error_timestamp': error[2] @@ -1042,6 +1071,7 @@ def download(self, container=None, objects=None, options=None): 'no_download': False, 'header': [], 'skip_identical': False, + 'version_id': None, 'out_directory': None, 'checksum': True, 'out_file': None, @@ -1151,6 +1181,9 @@ def _download_object_job(self, conn, container, obj, options): get_args = {'resp_chunk_size': DISK_BUFFER, 'headers': req_headers, 'response_dict': results_dict} + if options.get('version_id') is not None: + get_args['query_string'] = ( + 'version-id=%s' % options['version_id']) if options['skip_identical']: # Assume the file is a large object; if we're wrong, the query # string is ignored and the If-None-Match header will trigger @@ -2337,14 +2370,28 @@ def delete(self, container=None, objects=None, options=None): of objects. :param container: The container to delete or delete from. - :param objects: The list of objects to delete. + :param objects: A list of object names (strings) or SwiftDeleteObject + instances containing an object name, and an + options dict (can be None) to override the options for + that individual delete operation:: + + [ + 'object_name', + SwiftDeleteObject('object_name', + options={...}), + ... + ] + + The options dict is described below. :param options: A dictionary containing options to override the global options specified during the service object creation:: { 'yes_all': False, 'leave_segments': False, + 'version_id': None, 'prefix': None, + 'versions': False, 'header': [], } @@ -2364,23 +2411,28 @@ def delete(self, container=None, objects=None, options=None): if container is not None: if objects is not None: + delete_objects = self._make_delete_objects(objects) if options['prefix']: - objects = [obj for obj in objects - if obj.startswith(options['prefix'])] + delete_objects = [ + obj for obj in delete_objects + if obj.object_name.startswith(options['prefix'])] rq = Queue() obj_dels = {} - bulk_page_size = self._bulk_delete_page_size(objects) + bulk_page_size = self._bulk_delete_page_size(delete_objects) if bulk_page_size > 1: - page_at_a_time = n_at_a_time(objects, bulk_page_size) + page_at_a_time = n_at_a_time(delete_objects, + bulk_page_size) for page_slice in page_at_a_time: for obj_slice in n_groups( page_slice, self._options['object_dd_threads']): - self._bulk_delete(container, obj_slice, options, + object_names = [ + obj.object_name for obj in obj_slice] + self._bulk_delete(container, object_names, options, obj_dels) else: - self._per_item_delete(container, objects, options, + self._per_item_delete(container, delete_objects, options, obj_dels, rq) # Start a thread to watch for delete results @@ -2445,6 +2497,11 @@ def _bulk_delete_page_size(self, objects): # Not many objects; may as well delete one-by-one return 1 + if any(obj.options for obj in objects + if isinstance(obj, SwiftDeleteObject)): + # we can't do per option deletes for bulk + return 1 + try: cap_result = self.capabilities() if not cap_result['success']: @@ -2463,9 +2520,11 @@ def _bulk_delete_page_size(self, objects): return 1 def _per_item_delete(self, container, objects, options, rdict, rq): - for obj in objects: + for delete_obj in objects: + obj = delete_obj.object_name + obj_options = dict(options, **delete_obj.options or {}) obj_del = self.thread_manager.object_dd_pool.submit( - self._delete_object, container, obj, options, + self._delete_object, container, obj, obj_options, results_queue=rq ) obj_details = {'container': container, 'object': obj} @@ -2500,6 +2559,24 @@ def _delete_segment(conn, container, obj, results_queue=None): results_queue.put(res) return res + @staticmethod + def _make_delete_objects(objects): + delete_objects = [] + + for o in objects: + if isinstance(o, string_types): + obj = SwiftDeleteObject(o) + delete_objects.append(obj) + elif isinstance(o, SwiftDeleteObject): + delete_objects.append(o) + else: + raise SwiftError( + "The delete operation takes only strings or " + "SwiftDeleteObjects as input", + obj=o) + + return delete_objects + def _delete_object(self, conn, container, obj, options, results_queue=None): _headers = {} @@ -2511,7 +2588,7 @@ def _delete_object(self, conn, container, obj, options, } try: old_manifest = None - query_string = None + query_params = {} if not options['leave_segments']: try: @@ -2520,11 +2597,15 @@ def _delete_object(self, conn, container, obj, options, query_string='symlink=get') old_manifest = headers.get('x-object-manifest') if config_true_value(headers.get('x-static-large-object')): - query_string = 'multipart-manifest=delete' + query_params['multipart-manifest'] = 'delete' except ClientException as err: if err.http_status != 404: raise + if options.get('version_id') is not None: + query_params['version-id'] = options['version_id'] + query_string = '&'.join('%s=%s' % (k, v) for (k, v) + in sorted(query_params.items())) results_dict = {} conn.delete_object(container, obj, headers=_headers, @@ -2611,12 +2692,17 @@ def _delete_container(self, container, options): try: for part in self.list(container=container, options=options): if not part["success"]: - raise part["error"] - + delete_objects = [] + for item in part['listing']: + delete_opts = {} + if options.get('versions', False) and 'version_id' in item: + delete_opts['version_id'] = item['version_id'] + delete_obj = SwiftDeleteObject(item['name'], delete_opts) + delete_objects.append(delete_obj) for res in self.delete( container=container, - objects=[o['name'] for o in part['listing']], + objects=delete_objects, options=options): yield res if options['prefix']: @@ -2679,7 +2765,9 @@ def _bulkdelete(conn, container, objects, options): 'No content received on account POST. ' 'Is the bulk operations middleware enabled?')}) except Exception as e: - res.update({'success': False, 'error': e}) + traceback, err_time = report_traceback() + logger.exception(e) + res.update({'success': False, 'error': e, 'traceback': traceback}) res.update({ 'action': 'bulk_delete', diff --git a/swiftclient/shell.py b/swiftclient/shell.py index d18fc9e7..03a8fa64 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -65,7 +65,8 @@ def immediate_exit(signum, frame): [--container-threads ] [--header ] [--prefix ] - [ [] [...]] + [--versions] + [ [] [--version-id ] [...]] ''' st_delete_help = ''' @@ -78,6 +79,7 @@ def immediate_exit(signum, frame): Optional arguments: -a, --all Delete all containers and objects. + --versions Delete all versions --leave-segments Do not delete segments of manifest objects. -H, --header Adds a custom request header to use for deleting @@ -89,6 +91,8 @@ def immediate_exit(signum, frame): Number of threads to use for deleting containers. Default is 10. --prefix Only delete objects beginning with . + --version-id + Delete specific version of a versioned object. '''.strip("\n") @@ -96,9 +100,14 @@ def st_delete(parser, args, output_manager, return_parser=False): parser.add_argument( '-a', '--all', action='store_true', dest='yes_all', default=False, help='Delete all containers and objects.') + parser.add_argument('--versions', action='store_true', + help='delete all versions') parser.add_argument( '-p', '--prefix', dest='prefix', help='Only delete items beginning with .') + parser.add_argument( + '--version-id', action='store', default=None, + help='Delete a specific version of a versioned object') parser.add_argument( '-H', '--header', action='append', dest='header', default=[], @@ -128,6 +137,10 @@ def st_delete(parser, args, output_manager, return_parser=False): BASENAME, st_delete_options, st_delete_help) return + if options['versions'] and len(args) >= 2: + exit('--versions option not allowed for object deletes') + if options['version_id'] and len(args) < 2: + exit('--version-id option only allowed for object deletes') if options['object_threads'] <= 0: output_manager.error( @@ -227,6 +240,7 @@ def st_delete(parser, args, output_manager, return_parser=False): [--object-threads ] [--ignore-checksum] [--container-threads ] [--no-download] [--skip-identical] [--remove-prefix] + [--version-id ] [--header ] [--no-shuffle] [ [] [...]] ''' @@ -271,6 +285,8 @@ def st_delete(parser, args, output_manager, return_parser=False): Example: --header "content-type:text/plain" --skip-identical Skip downloading files that are identical on both sides. + --version-id + Download specific version of a versioned object. --ignore-checksum Turn off checksum validation for downloads. --no-shuffle By default, when downloading a complete account or container, download order is randomised in order to @@ -332,6 +348,9 @@ def st_download(parser, args, output_manager, return_parser=False): '--skip-identical', action='store_true', dest='skip_identical', default=False, help='Skip downloading files that are identical on ' 'both sides.') + parser.add_argument( + '--version-id', action='store', default=None, + help='Download a specific version of a versioned object') parser.add_argument( '--ignore-checksum', action='store_false', dest='checksum', default=True, help='Turn off checksum validation for downloads.') @@ -372,6 +391,8 @@ def st_download(parser, args, output_manager, return_parser=False): output_manager.error('Usage: %s download %s\n%s', BASENAME, st_download_options, st_download_help) return + if options['version_id'] and len(args) < 2: + exit('--version-id option only allowed for object downloads') if options['object_threads'] <= 0: output_manager.error( @@ -479,7 +500,7 @@ def st_download(parser, args, output_manager, return_parser=False): st_list_options = '''[--long] [--lh] [--totals] [--prefix ] [--delimiter ] [--header ] - [] + [--versions] [] ''' st_list_help = ''' @@ -499,6 +520,8 @@ def st_download(parser, args, output_manager, return_parser=False): Roll up items with the given delimiter. For containers only. See OpenStack Swift API documentation for what this means. + -j, --json Display listing information in json + --versions Display listing information for all versions -H, --header Adds a custom request header to use for listing. '''.strip('\n') @@ -579,6 +602,8 @@ def _print_stats(options, stats, human): 'what this means.') parser.add_argument('-j', '--json', action='store_true', help='print listing information in json') + parser.add_argument('--versions', action='store_true', + help='display all versions') parser.add_argument( '-H', '--header', action='append', dest='header', default=[], @@ -592,6 +617,8 @@ def _print_stats(options, stats, human): args = args[1:] if options['delimiter'] and not args: exit('-d option only allowed for container listings') + if options['versions'] and not args: + exit('--versions option only allowed for container listings') human = options.pop('human') if human: @@ -642,6 +669,7 @@ def listing(stats_parts_gen=stats_parts_gen): st_stat_options = '''[--lh] [--header ] + [--version-id ] [ []] ''' @@ -655,6 +683,8 @@ def listing(stats_parts_gen=stats_parts_gen): Optional arguments: --lh Report sizes in human readable format similar to ls -lh. + --version-id + Report stat of specific version of a versioned object. -H, --header Adds a custom request header to use for stat. '''.strip('\n') @@ -664,6 +694,9 @@ def st_stat(parser, args, output_manager, return_parser=False): parser.add_argument( '--lh', dest='human', action='store_true', default=False, help='Report sizes in human readable format similar to ls -lh.') + parser.add_argument( + '--version-id', action='store', default=None, + help='Report stat of a specific version of a versioned object') parser.add_argument( '-H', '--header', action='append', dest='header', default=[], @@ -675,6 +708,8 @@ def st_stat(parser, args, output_manager, return_parser=False): options, args = parse_args(parser, args) args = args[1:] + if options['version_id'] and len(args) < 2: + exit('--version-id option only allowed for object stats') with SwiftService(options=options) as swift: try: diff --git a/test/unit/test_service.py b/test/unit/test_service.py index ed3a2d6e..e86a4ff1 100644 --- a/test/unit/test_service.py +++ b/test/unit/test_service.py @@ -21,6 +21,7 @@ import tempfile import unittest import time +import json from concurrent.futures import Future from hashlib import md5 @@ -33,7 +34,7 @@ import swiftclient.utils as utils from swiftclient.client import Connection, ClientException from swiftclient.service import ( - SwiftService, SwiftError, SwiftUploadObject + SwiftService, SwiftError, SwiftUploadObject, SwiftDeleteObject ) from test.unit import utils as test_utils @@ -315,11 +316,39 @@ def test_delete_object(self): mock_conn.head_object.assert_called_once_with( 'test_c', 'test_o', query_string='symlink=get', headers={}) mock_conn.delete_object.assert_called_once_with( - 'test_c', 'test_o', query_string=None, response_dict={}, + 'test_c', 'test_o', query_string='', response_dict={}, headers={} ) self.assertEqual(expected_r, r) + @mock.patch('swiftclient.service.Connection') + def test_delete_object_version(self, mock_connection_class): + mock_conn = mock_connection_class.return_value + mock_conn.url = 'http://saio/v1/AUTH_test' + mock_conn.attempts = 0 + mock_conn.head_object.return_value = {} + mock_conn.delete_object.return_value = {} + expected = { + 'action': 'delete_object', + 'attempts': 0, + 'container': 'c', + 'object': 'o', + 'response_dict': {}, + 'success': True} + with SwiftService() as swift: + delete_results = swift.delete( + container='c', objects='o', options={ + 'version_id': '234567.8'}) + for delete_result in delete_results: + self.assertEqual(delete_result, expected) + self.assertEqual(mock_conn.mock_calls, [ + mock.call.head_object('c', 'o', headers={}, + query_string='symlink=get'), + mock.call.delete_object('c', 'o', headers={}, + query_string='version-id=234567.8', + response_dict={}), + ]) + def test_delete_object_with_headers(self): mock_q = Queue() mock_conn = self._get_mock_connection() @@ -338,7 +367,7 @@ def test_delete_object_with_headers(self): 'test_c', 'test_o', headers={'Skip-Middleware': 'Test'}, query_string='symlink=get') mock_conn.delete_object.assert_called_once_with( - 'test_c', 'test_o', query_string=None, response_dict={}, + 'test_c', 'test_o', query_string='', response_dict={}, headers={'Skip-Middleware': 'Test'} ) self.assertEqual(expected_r, r) @@ -366,7 +395,7 @@ def test_delete_object_exception(self): mock_conn.head_object.assert_called_once_with( 'test_c', 'test_o', query_string='symlink=get', headers={}) mock_conn.delete_object.assert_called_once_with( - 'test_c', 'test_o', query_string=None, response_dict={}, + 'test_c', 'test_o', query_string='', response_dict={}, headers={} ) self.assertEqual(expected_r, r) @@ -431,7 +460,7 @@ def get_mock_list_conn(options): self.assertEqual(expected_r, r) expected = [ - mock.call('test_c', 'test_o', query_string=None, response_dict={}, + mock.call('test_c', 'test_o', query_string='', response_dict={}, headers={}), mock.call('manifest_c', 'test_seg_1', response_dict={}), mock.call('manifest_c', 'test_seg_2', response_dict={})] @@ -529,6 +558,63 @@ def test_bulk_delete_page_size(self): if errors: self.fail('_bulk_delete_page_size() failed\n' + '\n'.join(errors)) + @mock.patch('swiftclient.service.Connection') + def test_bulk_delete(self, mock_connection_class): + mock_conn = mock_connection_class.return_value + mock_conn.attempts = 0 + mock_conn.get_capabilities.return_value = { + 'bulk_delete': {}} + stub_headers = {} + stub_resp = [] + mock_conn.post_account.return_value = ( + stub_headers, json.dumps(stub_resp).encode('utf8')) + obj_list = ['x%02d' % i for i in range(100)] + expected = [{ + 'action': u'bulk_delete', + 'attempts': 0, + 'container': 'c', + 'objects': list(objs), + 'response_dict': {}, + 'result': [], + 'success': True, + } for objs in zip(*[iter(obj_list)] * 10)] + found_result = [] + with SwiftService(options={'object_dd_threads': 10}) as swift: + delete_results = swift.delete(container='c', objects=obj_list) + for delete_result in delete_results: + found_result.append(delete_result) + self.assertEqual(sorted(found_result, key=lambda r: r['objects'][0]), + expected) + + @mock.patch('swiftclient.service.Connection') + def test_bulk_delete_versions(self, mock_connection_class): + mock_conn = mock_connection_class.return_value + mock_conn.attempts = 0 + mock_conn.get_capabilities.return_value = { + 'bulk_delete': {}} + mock_conn.head_object.return_value = {} + stub_headers = {} + stub_resp = [] + mock_conn.post_account.return_value = ( + stub_headers, json.dumps(stub_resp)) + obj_list = [SwiftDeleteObject('x%02d' % i, options={'version_id': i}) + for i in range(100)] + expected = [{ + 'action': u'delete_object', + 'attempts': 0, + 'container': 'c', + 'object': obj.object_name, + 'response_dict': {}, + 'success': True, + } for obj in obj_list] + found_result = [] + with SwiftService(options={'object_dd_threads': 10}) as swift: + delete_results = swift.delete(container='c', objects=obj_list) + for delete_result in delete_results: + found_result.append(delete_result) + self.assertEqual(sorted(found_result, key=lambda r: r['object']), + expected) + class TestSwiftError(unittest.TestCase): @@ -938,9 +1024,11 @@ def test_list_container_with_headers(self): self.assertIsNone(self._get_queue(mock_q)) self.assertEqual(mock_conn.get_container.mock_calls, [ mock.call('test_c', headers={'Skip-Middleware': 'Test'}, - delimiter='', marker='', prefix=None), + delimiter='', marker='', prefix=None, + query_string=None, version_marker=''), mock.call('test_c', headers={'Skip-Middleware': 'Test'}, - delimiter='', marker='test_o', prefix=None)]) + delimiter='', marker='test_o', prefix=None, + query_string=None, version_marker='')]) def test_list_container_exception(self): mock_q = Queue() @@ -952,6 +1040,7 @@ def test_list_container_exception(self): 'success': False, 'error': self.exc, 'marker': '', + 'version_marker': '', 'error_timestamp': mock.ANY, 'traceback': mock.ANY }) @@ -961,11 +1050,61 @@ def test_list_container_exception(self): ) mock_conn.get_container.assert_called_once_with( - 'test_c', marker='', delimiter='', prefix=None, headers={} + 'test_c', marker='', delimiter='', prefix=None, headers={}, + query_string=None, version_marker='', ) self.assertEqual(expected_r, self._get_queue(mock_q)) self.assertIsNone(self._get_queue(mock_q)) + @mock.patch('swiftclient.service.Connection') + def test_list_container_versions(self, mock_connection_class): + mock_conn = mock_connection_class.return_value + mock_conn.url = 'http://saio/v1/AUTH_test' + resp_headers = {} + items = [{ + "bytes": 9, + "content_type": "application/octet-stream", + "hash": "e55cedc11adb39c404b7365f7d6291fa", + "is_latest": True, + "last_modified": "2019-11-08T05:00:15.115360", + "name": "test", + "version_id": "1573189215.11536" + }, { + "bytes": 8, + "content_type": "application/octet-stream", + "hash": "70c1db56f301c9e337b0099bd4174b28", + "is_latest": False, + "last_modified": "2019-11-08T05:00:14.730240", + "name": "test", + "version_id": "1573184903.06720" + }] + mock_conn.get_container.side_effect = [ + (resp_headers, items), + (resp_headers, []), + ] + expected = { + 'action': 'list_container_part', + 'container': 'c', + 'listing': items, + 'marker': '', + 'prefix': None, + 'success': True, + } + with SwiftService() as swift: + list_result_gen = swift.list(container='c', options={ + 'versions': True}) + self.maxDiff = None + for result in list_result_gen: + self.assertEqual(result, expected) + self.assertEqual(mock_conn.get_container.mock_calls, [ + mock.call('c', delimiter=None, headers={}, marker='', + prefix=None, query_string='versions=true', + version_marker=''), + mock.call('c', delimiter=None, headers={}, marker='test', + prefix=None, query_string='versions=true', + version_marker='1573184903.06720'), + ]) + @mock.patch('swiftclient.service.get_conn') def test_list_queue_size(self, mock_get_conn): mock_conn = self._get_mock_connection() @@ -1042,6 +1181,67 @@ def test_list_queue_size(self, mock_get_conn): self.assertEqual(observed_listing, expected_listing) +class TestServiceStat(_TestServiceBase): + + maxDiff = None + + @mock.patch('swiftclient.service.Connection') + def test_stat_object(self, mock_connection_class): + mock_conn = mock_connection_class.return_value + mock_conn.url = 'http://saio/v1/AUTH_test' + mock_conn.head_object.return_value = {} + expected = { + 'action': 'stat_object', + 'container': 'c', + 'object': 'o', + 'headers': {}, + 'items': [('Account', 'AUTH_test'), + ('Container', 'c'), + ('Object', 'o'), + ('Content Type', None), + ('Content Length', '0'), + ('Last Modified', None), + ('ETag', None), + ('Manifest', None)], + 'success': True} + with SwiftService() as swift: + stat_results = swift.stat(container='c', objects='o') + for stat_result in stat_results: + self.assertEqual(stat_result, expected) + self.assertEqual(mock_conn.head_object.mock_calls, [ + mock.call('c', 'o', headers={}, query_string=None), + ]) + + @mock.patch('swiftclient.service.Connection') + def test_stat_versioned_object(self, mock_connection_class): + mock_conn = mock_connection_class.return_value + mock_conn.url = 'http://saio/v1/AUTH_test' + mock_conn.head_object.return_value = {} + expected = { + 'action': 'stat_object', + 'container': 'c', + 'object': 'o', + 'headers': {}, + 'items': [('Account', 'AUTH_test'), + ('Container', 'c'), + ('Object', 'o'), + ('Content Type', None), + ('Content Length', '0'), + ('Last Modified', None), + ('ETag', None), + ('Manifest', None)], + 'success': True} + with SwiftService() as swift: + stat_results = swift.stat(container='c', objects='o', options={ + 'version_id': '234567.8'}) + for stat_result in stat_results: + self.assertEqual(stat_result, expected) + self.assertEqual(mock_conn.head_object.mock_calls, [ + mock.call('c', 'o', headers={}, + query_string='version-id=234567.8'), + ]) + + class TestService(unittest.TestCase): def test_upload_with_bad_segment_size(self): @@ -1791,13 +1991,14 @@ def test_upload_object_job_identical_dlo(self): mock_conn.head_object.assert_called_with('test_c', 'test_o') expected = [ mock.call('test_c_segments', prefix='test_o/prefix', - marker='', delimiter=None, headers={}), + marker='', delimiter=None, headers={}, + query_string=None, version_marker=''), mock.call('test_c_segments', prefix='test_o/prefix', marker="test_o/prefix/01", delimiter=None, - headers={}), + headers={}, query_string=None, version_marker=''), mock.call('test_c_segments', prefix='test_o/prefix', marker="test_o/prefix/02", delimiter=None, - headers={}), + headers={}, query_string=None, version_marker=''), ] mock_conn.get_container.assert_has_calls(expected) @@ -2332,6 +2533,29 @@ def test_download(self): self.assertEqual(resp['object'], 'test') self.assertEqual(resp['path'], 'test') + def test_download_version_id(self): + self.opts['version_id'] = '23456.7' + with mock.patch('swiftclient.service.Connection') as mock_conn: + header = {'content-length': self.obj_len, + 'etag': self.obj_etag} + mock_conn.get_object.return_value = header, self._readbody() + + resp = SwiftService()._download_object_job(mock_conn, + 'c', + 'test', + self.opts) + + self.assertIsNone(resp.get('error')) + self.assertIs(True, resp['success']) + self.assertEqual(resp['action'], 'download_object') + self.assertEqual(resp['object'], 'test') + self.assertEqual(resp['path'], 'test') + self.assertEqual(mock_conn.get_object.mock_calls, [ + mock.call( + 'c', 'test', headers={}, query_string='version-id=23456.7', + resp_chunk_size=65536, response_dict={}), + ]) + @mock.patch('swiftclient.service.interruptable_as_completed') @mock.patch('swiftclient.service.SwiftService._download_container') @mock.patch('swiftclient.service.SwiftService._download_object_job') @@ -2545,17 +2769,17 @@ def test_download_object_job_skip_identical_dlo(self): delimiter=None, prefix='test_o/prefix', marker='', - headers={}), + headers={}, query_string=None, version_marker=''), mock.call('test_c_segments', delimiter=None, prefix='test_o/prefix', marker='test_o/prefix/2', - headers={}), + headers={}, query_string=None, version_marker=''), mock.call('test_c_segments', delimiter=None, prefix='test_o/prefix', marker='test_o/prefix/3', - headers={})]) + headers={}, query_string=None, version_marker='')]) def test_download_object_job_skip_identical_nested_slo(self): with tempfile.NamedTemporaryFile() as f: @@ -2682,6 +2906,7 @@ def test_download_object_job_skip_identical_diff_dlo(self): obj='test_o', options=options) + self.maxDiff = None self.assertEqual(r, expected_r) self.assertEqual(mock_conn.get_container.mock_calls, [ @@ -2689,17 +2914,17 @@ def test_download_object_job_skip_identical_diff_dlo(self): delimiter=None, prefix='test_o/prefix', marker='', - headers={}), + headers={}, query_string=None, version_marker=''), mock.call('test_c_segments', delimiter=None, prefix='test_o/prefix', marker='test_o/prefix/2', - headers={}), + headers={}, query_string=None, version_marker=''), mock.call('test_c_segments', delimiter=None, prefix='test_o/prefix', marker='test_o/prefix/3', - headers={})]) + headers={}, query_string=None, version_marker='')]) self.assertEqual(mock_conn.get_object.mock_calls, [ mock.call('test_c', 'test_o', diff --git a/test/unit/test_shell.py b/test/unit/test_shell.py index 1fa0db40..b94cdcff 100644 --- a/test/unit/test_shell.py +++ b/test/unit/test_shell.py @@ -241,6 +241,30 @@ def test_stat_container_with_headers(self, connection): self.assertEqual(connection.return_value.head_container.mock_calls, [ mock.call('container', headers={'Skip-Middleware': 'Test'})]) + @mock.patch('swiftclient.service.Connection') + def test_stat_version_id(self, connection): + argv = ["", "stat", "--version-id", "1"] + with self.assertRaises(SystemExit) as caught: + swiftclient.shell.main(argv) + self.assertEqual(str(caught.exception), + "--version-id option only allowed for " + "object stats") + + argv = ["", "stat", "--version-id", "1", "container"] + with self.assertRaises(SystemExit) as caught: + swiftclient.shell.main(argv) + self.assertEqual(str(caught.exception), + "--version-id option only allowed for " + "object stats") + + argv = ["", "stat", "--version-id", "1", "container", "object"] + connection.return_value.head_object.return_value = {} + with CaptureOutput(): + swiftclient.shell.main(argv) + self.assertEqual([mock.call('container', 'object', headers={}, + query_string='version-id=1')], + connection.return_value.head_object.mock_calls) + @mock.patch('swiftclient.service.Connection') def test_stat_object(self, connection): return_headers = { @@ -295,7 +319,45 @@ def test_stat_object_with_headers(self, connection): ' Manifest: manifest\n') self.assertEqual(connection.return_value.head_object.mock_calls, [ mock.call('container', 'object', - headers={'Skip-Middleware': 'Test'})]) + headers={'Skip-Middleware': 'Test'}, + query_string=None)]) + + def test_list_account_with_delimiter(self): + argv = ["", "list", "--delimiter", "foo"] + with self.assertRaises(SystemExit) as caught: + swiftclient.shell.main(argv) + self.assertEqual(str(caught.exception), + "-d option only allowed for " + "container listings") + + @mock.patch('swiftclient.service.Connection') + def test_list_container_with_versions(self, connection): + connection.return_value.get_container.side_effect = [ + [None, [ + {'name': 'foo', 'version_id': '2'}, + {'name': 'foo', 'version_id': '1'}, + ]], + [None, []], + ] + argv = ["", "list", "container", "--versions"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + calls = [mock.call('container', delimiter=None, headers={}, marker='', + prefix=None, query_string='versions=true', + version_marker=''), + mock.call('container', delimiter=None, headers={}, + marker='foo', prefix=None, + query_string='versions=true', version_marker='1')] + connection.return_value.get_container.assert_has_calls(calls) + self.assertEqual(output.out, 'foo\nfoo\n') + + def test_list_account_with_versions(self): + argv = ["", "list", "--versions"] + with self.assertRaises(SystemExit) as caught: + swiftclient.shell.main(argv) + self.assertEqual(str(caught.exception), + "--versions option only allowed for " + "container listings") @mock.patch('swiftclient.service.Connection') def test_list_json(self, connection): @@ -431,9 +493,11 @@ def test_list_container(self, connection): swiftclient.shell.main(argv) calls = [ mock.call('container', marker='', - delimiter=None, prefix=None, headers={}), + delimiter=None, prefix=None, headers={}, + query_string=None, version_marker=''), mock.call('container', marker='object_a', - delimiter=None, prefix=None, headers={})] + delimiter=None, prefix=None, headers={}, + query_string=None, version_marker='')] connection.return_value.get_container.assert_has_calls(calls) self.assertEqual(output.out, 'object_a\n') @@ -450,9 +514,11 @@ def test_list_container(self, connection): swiftclient.shell.main(argv) calls = [ mock.call('container', marker='', - delimiter=None, prefix=None, headers={}), + delimiter=None, prefix=None, headers={}, + query_string=None, version_marker=''), mock.call('container', marker='object_a', - delimiter=None, prefix=None, headers={})] + delimiter=None, prefix=None, headers={}, + query_string=None, version_marker='')] connection.return_value.get_container.assert_has_calls(calls) self.assertEqual(output.out, @@ -472,14 +538,44 @@ def test_list_container_with_headers(self, connection): calls = [ mock.call('container', marker='', delimiter=None, prefix=None, - headers={'Skip-Middleware': 'Test'}), + headers={'Skip-Middleware': 'Test'}, + query_string=None, version_marker=''), mock.call('container', marker='object_a', delimiter=None, prefix=None, - headers={'Skip-Middleware': 'Test'})] + headers={'Skip-Middleware': 'Test'}, + query_string=None, version_marker='')] connection.return_value.get_container.assert_has_calls(calls) self.assertEqual(output.out, 'object_a\n') + @mock.patch('swiftclient.service.Connection') + def test_download_version_id(self, connection): + argv = ["", "download", "--yes-all", "--version-id", "5"] + with self.assertRaises(SystemExit) as caught: + swiftclient.shell.main(argv) + self.assertEqual(str(caught.exception), + "--version-id option only allowed for " + "object downloads") + + argv = ["", "download", "--version-id", "2", "container"] + with self.assertRaises(SystemExit) as caught: + swiftclient.shell.main(argv) + self.assertEqual(str(caught.exception), + "--version-id option only allowed for " + "object downloads") + + argv = ["", "download", "--version-id", "1", "container", "object"] + connection.return_value.head_object.return_value = {} + connection.return_value.get_object.return_value = {}, '' + connection.return_value.attempts = 0 + with CaptureOutput(): + swiftclient.shell.main(argv) + self.assertEqual([mock.call('container', 'object', headers={}, + query_string='version-id=1', + resp_chunk_size=65536, + response_dict={})], + connection.return_value.get_object.mock_calls) + @mock.patch('swiftclient.service.makedirs') @mock.patch('swiftclient.service.Connection') def test_download(self, connection, makedirs): @@ -1085,6 +1181,33 @@ def check_good(argv): check_good(["--object-threads", "1"]) check_good(["--container-threads", "1"]) + @mock.patch('swiftclient.service.Connection') + def test_delete_version_id(self, connection): + argv = ["", "delete", "--yes-all", "--version-id", "3"] + with self.assertRaises(SystemExit) as caught: + swiftclient.shell.main(argv) + self.assertEqual(str(caught.exception), + "--version-id option only allowed for " + "object deletes") + + argv = ["", "delete", "--version-id", "1", "container"] + with self.assertRaises(SystemExit) as caught: + swiftclient.shell.main(argv) + self.assertEqual(str(caught.exception), + "--version-id option only allowed for " + "object deletes") + + argv = ["", "delete", "--version-id", "1", "container", "object"] + connection.return_value.head_object.return_value = {} + connection.return_value.delete_object.return_value = None + connection.return_value.attempts = 0 + with CaptureOutput(): + swiftclient.shell.main(argv) + self.assertEqual([mock.call('container', 'object', headers={}, + query_string='version-id=1', + response_dict={})], + connection.return_value.delete_object.mock_calls) + @mock.patch.object(swiftclient.service.SwiftService, '_bulk_delete_page_size', lambda *a: 1) @mock.patch('swiftclient.service.Connection') @@ -1094,10 +1217,12 @@ def test_delete_account(self, connection): [None, [{'name': 'empty_container'}]], [None, []], ] + # N.B: missing --versions flag, version-id gets ignored + # only latest object is deleted connection.return_value.get_container.side_effect = [ [None, [{'name': 'object'}, {'name': 'obj\xe9ct2'}]], [None, []], - [None, [{'name': 'object'}]], + [None, [{'name': 'object', 'version_id': 1}]], [None, []], [None, []], ] @@ -1107,11 +1232,48 @@ def test_delete_account(self, connection): connection.return_value.delete_object.return_value = None swiftclient.shell.main(argv) connection.return_value.delete_object.assert_has_calls([ - mock.call('container', 'object', query_string=None, + mock.call('container', 'object', query_string='', response_dict={}, headers={}), - mock.call('container', 'obj\xe9ct2', query_string=None, + mock.call('container', 'obj\xe9ct2', query_string='', response_dict={}, headers={}), - mock.call('container2', 'object', query_string=None, + mock.call('container2', 'object', query_string='', + response_dict={}, headers={})], any_order=True) + self.assertEqual(3, connection.return_value.delete_object.call_count, + 'Expected 3 calls but found\n%r' + % connection.return_value.delete_object.mock_calls) + self.assertEqual( + connection.return_value.delete_container.mock_calls, [ + mock.call('container', response_dict={}, headers={}), + mock.call('container2', response_dict={}, headers={}), + mock.call('empty_container', response_dict={}, headers={})]) + + @mock.patch.object(swiftclient.service.SwiftService, + '_bulk_delete_page_size', lambda *a: 1) + @mock.patch('swiftclient.service.Connection') + def test_delete_account_versions(self, connection): + connection.return_value.get_account.side_effect = [ + [None, [{'name': 'container'}, {'name': 'container2'}]], + [None, [{'name': 'empty_container'}]], + [None, []], + ] + connection.return_value.get_container.side_effect = [ + [None, [{'name': 'object'}, {'name': 'obj\xe9ct2'}]], + [None, []], + [None, [{'name': 'obj', 'version_id': 1}]], + [None, []], + [None, []], + ] + connection.return_value.attempts = 0 + argv = ["", "delete", "--all", "--versions"] + connection.return_value.head_object.return_value = {} + connection.return_value.delete_object.return_value = None + swiftclient.shell.main(argv) + connection.return_value.delete_object.assert_has_calls([ + mock.call('container', 'object', query_string='', + response_dict={}, headers={}), + mock.call('container', 'obj\xe9ct2', query_string='', + response_dict={}, headers={}), + mock.call('container2', 'obj', query_string='version-id=1', response_dict={}, headers={})], any_order=True) self.assertEqual(3, connection.return_value.delete_object.call_count, 'Expected 3 calls but found\n%r' @@ -1323,9 +1485,39 @@ def test_delete_container(self, connection): connection.return_value.delete_container.assert_called_with( 'container', response_dict={}, headers={}) connection.return_value.delete_object.assert_called_with( - 'container', 'object', query_string=None, response_dict={}, + 'container', 'object', query_string='', response_dict={}, headers={}) + @mock.patch.object(swiftclient.service.SwiftService, + '_bulk_delete_page_size', lambda *a: 1) + @mock.patch('swiftclient.service.Connection') + def test_delete_container_versions(self, connection): + argv = ["", "delete", "--versions", "container", "obj"] + with self.assertRaises(SystemExit) as caught: + swiftclient.shell.main(argv) + self.assertEqual(str(caught.exception), + "--versions option not allowed for object deletes") + + connection.return_value.get_container.side_effect = [ + [None, [{'name': 'object', 'version_id': 2}, + {'name': 'object', 'version_id': 1}]], + [None, []], + ] + connection.return_value.attempts = 0 + argv = ["", "delete", "--versions", "container"] + connection.return_value.head_object.return_value = {} + swiftclient.shell.main(argv) + connection.return_value.delete_container.assert_called_with( + 'container', response_dict={}, headers={}) + expected_calls = [ + mock.call('container', 'object', query_string='version-id=2', + response_dict={}, headers={}), + mock.call('container', 'object', query_string='version-id=1', + response_dict={}, headers={})] + + self.assertEqual(connection.return_value.delete_object.mock_calls, + expected_calls) + @mock.patch.object(swiftclient.service.SwiftService, '_bulk_delete_page_size', lambda *a: 1) @mock.patch('swiftclient.service.Connection') @@ -1342,7 +1534,7 @@ def test_delete_container_headers(self, connection): 'container', response_dict={}, headers={'Skip-Middleware': 'Test'}) connection.return_value.delete_object.assert_called_with( - 'container', 'object', query_string=None, response_dict={}, + 'container', 'object', query_string='', response_dict={}, headers={'Skip-Middleware': 'Test'}) @mock.patch.object(swiftclient.service.SwiftService, @@ -1408,7 +1600,7 @@ def test_delete_per_object(self, connection): connection.return_value.attempts = 0 swiftclient.shell.main(argv) connection.return_value.delete_object.assert_called_with( - 'container', 'object', query_string=None, response_dict={}, + 'container', 'object', query_string='', response_dict={}, headers={}) @mock.patch.object(swiftclient.service.SwiftService, From 9b0da49c0b337585e24825de2ad670a0798179ac Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Fri, 10 Apr 2020 17:16:11 -0700 Subject: [PATCH 092/238] Improve `list --versions` output Have `--versions` imply `--long` and add a new column for version_id. Also, have version-aware listings show all versions as "null" on old Swifts that don't support object versioning (or when object versioning is not enabled). Change-Id: I0e009bce2471d1c140ac9b83700591cb355fee3f --- swiftclient/shell.py | 15 ++++++++---- test/unit/test_shell.py | 53 +++++++++++++++++++++++++++++++++++++---- 2 files changed, 59 insertions(+), 9 deletions(-) diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 03a8fa64..1b34c084 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -534,7 +534,7 @@ def _print_stats(options, stats, human): container = stats.get("container", None) for item in stats["listing"]: item_name = item.get('name') - if not options['long'] and not human: + if not options['long'] and not human and not options['versions']: output_manager.print_msg(item.get('name', item.get('subdir'))) else: if not container: # listing containers @@ -566,9 +566,16 @@ def _print_stats(options, stats, human): date = xtime = '' item_name = subdir if not options['totals']: - output_manager.print_msg( - "%s %10s %8s %24s %s", - byte_str, date, xtime, content_type, item_name) + if options['versions']: + output_manager.print_msg( + "%s %10s %8s %16s %24s %s", + byte_str, date, xtime, + item.get('version_id', 'null'), + content_type, item_name) + else: + output_manager.print_msg( + "%s %10s %8s %24s %s", + byte_str, date, xtime, content_type, item_name) total_bytes += item_bytes # report totals diff --git a/test/unit/test_shell.py b/test/unit/test_shell.py index b94cdcff..a63d16b2 100644 --- a/test/unit/test_shell.py +++ b/test/unit/test_shell.py @@ -334,8 +334,15 @@ def test_list_account_with_delimiter(self): def test_list_container_with_versions(self, connection): connection.return_value.get_container.side_effect = [ [None, [ - {'name': 'foo', 'version_id': '2'}, - {'name': 'foo', 'version_id': '1'}, + {'name': 'foo', 'version_id': '2', + 'content_type': 'text/plain', + 'last_modified': '123T456', 'bytes': 78}, + {'name': 'foo', 'version_id': '1', + 'content_type': 'text/rtf', + 'last_modified': '123T456', 'bytes': 90}, + {'name': 'bar', 'version_id': 'null', + 'content_type': 'text/plain', + 'last_modified': '123T456', 'bytes': 123}, ]], [None, []], ] @@ -346,10 +353,46 @@ def test_list_container_with_versions(self, connection): prefix=None, query_string='versions=true', version_marker=''), mock.call('container', delimiter=None, headers={}, - marker='foo', prefix=None, - query_string='versions=true', version_marker='1')] + marker='bar', prefix=None, + query_string='versions=true', + version_marker='null')] connection.return_value.get_container.assert_has_calls(calls) - self.assertEqual(output.out, 'foo\nfoo\n') + self.assertEqual([line.split() for line in output.out.split('\n')], [ + ['78', '123', '456', '2', 'text/plain', 'foo'], + ['90', '123', '456', '1', 'text/rtf', 'foo'], + ['123', '123', '456', 'null', 'text/plain', 'bar'], + [], + ]) + + @mock.patch('swiftclient.service.Connection') + def test_list_container_with_versions_old_swift(self, connection): + # Versions of swift that don't support object-versioning won't + # include verison_id keys in listings. We want to present that + # as though the container is unversioned. + connection.return_value.get_container.side_effect = [ + [None, [ + {'name': 'foo', 'content_type': 'text/plain', + 'last_modified': '123T456', 'bytes': 78}, + {'name': 'bar', 'content_type': 'text/plain', + 'last_modified': '123T456', 'bytes': 123}, + ]], + [None, []], + ] + argv = ["", "list", "container", "--versions"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + calls = [mock.call('container', delimiter=None, headers={}, marker='', + prefix=None, query_string='versions=true', + version_marker=''), + mock.call('container', delimiter=None, headers={}, + marker='bar', prefix=None, + query_string='versions=true', version_marker='')] + connection.return_value.get_container.assert_has_calls(calls) + self.assertEqual([line.split() for line in output.out.split('\n')], [ + ['78', '123', '456', 'null', 'text/plain', 'foo'], + ['123', '123', '456', 'null', 'text/plain', 'bar'], + [], + ]) def test_list_account_with_versions(self): argv = ["", "list", "--versions"] From f9f2090a025afc5e11cc889b5a31d159f78aaaa2 Mon Sep 17 00:00:00 2001 From: Andreas Jaeger Date: Sun, 12 Apr 2020 20:41:23 +0200 Subject: [PATCH 093/238] Drop pypy testing The pypy job is always failing, drop it. Change-Id: Ibc80d23846b364bfcd82043430ef71ad4b6e271b --- .zuul.yaml | 1 - tox.ini | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index f0bd82c5..eabae2cb 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -37,7 +37,6 @@ - check-requirements - lib-forward-testing-python3 - openstack-lower-constraints-jobs - - openstack-pypy-jobs-nonvoting - openstack-python-jobs - openstack-python3-ussuri-jobs - publish-openstack-docs-pti diff --git a/tox.ini b/tox.ini index 002d24c5..2b4f6e36 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,py37,pypy,pep8 +envlist = py27,py37,pep8 minversion = 2.0 skipsdist = True From a15eaec82667aa00b1056c802aaa3e470471755e Mon Sep 17 00:00:00 2001 From: OpenStack Release Bot Date: Mon, 13 Apr 2020 15:30:38 +0000 Subject: [PATCH 094/238] Update master for stable/ussuri Add file to the reno documentation build to show release notes for stable/ussuri. Use pbr instruction to increment the minor version number automatically so that master versions are higher than the versions on stable/ussuri. Change-Id: I75dc03a9d29c65b8d104b8d3a95915094cba6320 Sem-Ver: feature --- releasenotes/source/index.rst | 1 + releasenotes/source/ussuri.rst | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 releasenotes/source/ussuri.rst diff --git a/releasenotes/source/index.rst b/releasenotes/source/index.rst index 662c6f6d..d46593b0 100644 --- a/releasenotes/source/index.rst +++ b/releasenotes/source/index.rst @@ -6,6 +6,7 @@ :maxdepth: 1 current + ussuri train stein rocky diff --git a/releasenotes/source/ussuri.rst b/releasenotes/source/ussuri.rst new file mode 100644 index 00000000..e21e50e0 --- /dev/null +++ b/releasenotes/source/ussuri.rst @@ -0,0 +1,6 @@ +=========================== +Ussuri Series Release Notes +=========================== + +.. release-notes:: + :branch: stable/ussuri From 89ae9d77cbb25493d86dd766613eecb6f6120114 Mon Sep 17 00:00:00 2001 From: OpenStack Release Bot Date: Mon, 13 Apr 2020 15:30:41 +0000 Subject: [PATCH 095/238] Add Python3 victoria unit tests This is an automatically generated patch to ensure unit testing is in place for all the of the tested runtimes for victoria. See also the PTI in governance [1]. [1]: https://governance.openstack.org/tc/reference/project-testing-interface.html Change-Id: I2fa8505451caadbe895cc3262c0bf1470795968b --- .zuul.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.zuul.yaml b/.zuul.yaml index f0bd82c5..9cd52c16 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -39,7 +39,7 @@ - openstack-lower-constraints-jobs - openstack-pypy-jobs-nonvoting - openstack-python-jobs - - openstack-python3-ussuri-jobs + - openstack-python3-victoria-jobs - publish-openstack-docs-pti - release-notes-jobs-python3 check: From 2eba2dd42a7fa9663b59486f25707c16ef3731d7 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Mon, 13 Apr 2020 22:28:53 -0700 Subject: [PATCH 096/238] Add cacert test config option Change-Id: I3936f862e0fef176cad34f277598b136a40de1eb --- test/functional/__init__.py | 5 +++++ test/functional/test_openstacksdk.py | 1 + test/functional/test_swiftclient.py | 6 ++++-- test/sample.conf | 5 +++++ 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/test/functional/__init__.py b/test/functional/__init__.py index 248875a6..f0fea3be 100644 --- a/test/functional/__init__.py +++ b/test/functional/__init__.py @@ -57,6 +57,11 @@ def _load_config(force_reload=False): auth_url += 'v1.0' conf['auth_url'] = auth_url + try: + conf['cacert'] = parser.get('func_test', 'cacert') + except configparser.NoOptionError: + conf['cacert'] = None + try: conf['account_username'] = parser.get('func_test', 'account_username') diff --git a/test/functional/test_openstacksdk.py b/test/functional/test_openstacksdk.py index cee7f4e9..e51773ae 100644 --- a/test/functional/test_openstacksdk.py +++ b/test/functional/test_openstacksdk.py @@ -35,6 +35,7 @@ def setUpClass(cls): auth_url=TEST_CONFIG['auth_url'], username=TEST_CONFIG['account_username'], password=TEST_CONFIG['password'], + cacert=TEST_CONFIG['cacert'], ) cls.object_store = cls.conn.object_store diff --git a/test/functional/test_swiftclient.py b/test/functional/test_swiftclient.py index 54c514de..aaade879 100644 --- a/test/functional/test_swiftclient.py +++ b/test/functional/test_swiftclient.py @@ -42,6 +42,7 @@ def __init__(self, *args, **kwargs): def _get_config(self): self.auth_url = TEST_CONFIG['auth_url'] + self.cacert = TEST_CONFIG['cacert'] self.auth_version = TEST_CONFIG['auth_version'] self.account_username = TEST_CONFIG['account_username'] self.password = TEST_CONFIG['password'] @@ -52,7 +53,7 @@ def _get_connection(self): """ return swiftclient.Connection( self.auth_url, self.account_username, self.password, - auth_version=self.auth_version) + auth_version=self.auth_version, cacert=self.cacert) def setUp(self): super(TestFunctional, self).setUp() @@ -486,7 +487,7 @@ def _get_connection(self): return swiftclient.Connection( self.auth_url, username, self.password, - auth_version=self.auth_version, + auth_version=self.auth_version, cacert=self.cacert, os_options={'tenant_name': account}) @@ -515,4 +516,5 @@ def _get_connection(self): 'user_domain_name': user_domain} return swiftclient.Connection(self.auth_url, username, password, auth_version=self.auth_version, + cacert=self.cacert, os_options=os_options) diff --git a/test/sample.conf b/test/sample.conf index 95c1a478..2b19de4f 100644 --- a/test/sample.conf +++ b/test/sample.conf @@ -12,6 +12,11 @@ auth_prefix = /auth/ #auth_ssl = no #auth_prefix = /v2.0/ +# You may want to run tests against endpoints that use development certs +# without installing the CA system-wide. Use this to trust an extra set +# of certificates. +#cacert = /path/to/trusted-ca.crt + # Primary functional test account (needs admin access to the account). # By default the tests use a swiftclient.client.Connection instance with user # attribute set to 'account:username' based on the options 'account' and From 06c5c30fa462d07ef6f63935542e280cf2fd02b4 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Mon, 13 Apr 2020 22:45:25 -0700 Subject: [PATCH 097/238] Change recommended test config to use auth_uri ...instead of piecing it together from auth_host, auth_port, auth_ssl, auth_prefix, and (sort of, sometimes) auth_version. Change-Id: Ie9c36e778d6a03f905899074d7136b767812ea11 --- test/sample.conf | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/test/sample.conf b/test/sample.conf index 95c1a478..49bca90a 100644 --- a/test/sample.conf +++ b/test/sample.conf @@ -1,16 +1,10 @@ [func_test] # sample config -auth_host = 127.0.0.1 -auth_port = 8080 -auth_ssl = no -auth_prefix = /auth/ +auth_uri = http://127.0.0.1:8080/auth/v1.0/ ## sample config for Swift with Keystone v2 API # For keystone v3 change auth_version to 3 and auth_prefix to /v3/ #auth_version = 2 -#auth_host = localhost -#auth_port = 5000 -#auth_ssl = no -#auth_prefix = /v2.0/ +#auth_uri = http://localhost:5000/v2.0/ # Primary functional test account (needs admin access to the account). # By default the tests use a swiftclient.client.Connection instance with user From 02b637cdca6963e8dcab5170422347df99606f92 Mon Sep 17 00:00:00 2001 From: Charles Hsu Date: Wed, 18 Dec 2019 00:32:36 +0800 Subject: [PATCH 098/238] Support v3 application credentials auth. Use keystoneauth1 application credential plugin and session to fetch a token and endpoint catalog url. $ swift --os-auth-url http://172.16.1.2:5000/v3 --auth-version 3\ --os-application-credential-id THE_ID \ --os-application-credential-secret THE_SECRET \ --os-auth-type v3applicationcredential auth Change-Id: I9190e5e7e24b6a741970fa0d0ac792deccf73d25 Closes-Bug: 1843901 Closes-Bug: 1856635 --- swiftclient/client.py | 59 +++++++++++++++++++++++++++++------ swiftclient/service.py | 13 ++++++++ swiftclient/shell.py | 54 +++++++++++++++++++++++++++++--- test/unit/test_shell.py | 46 +++++++++++++++++++++++++++ test/unit/test_swiftclient.py | 57 +++++++++++++++++++++++++++++++++ 5 files changed, 215 insertions(+), 14 deletions(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index 449b6cd3..ee85a144 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -70,6 +70,9 @@ def createLock(self): pass try: from keystoneclient.v3 import client as ksclient_v3 + from keystoneauth1.identity import v3 + from keystoneauth1 import session + from keystoneauth1 import exceptions as ksauthexceptions except ImportError: pass @@ -615,6 +618,46 @@ def get_auth_keystone(auth_url, user, key, os_options, **kwargs): version 1.0 which requires ST_AUTH, ST_USER, and ST_KEY environment variables to be set or overridden with -A, -U, or -K.''') + filter_kwargs = {} + service_type = os_options.get('service_type') or 'object-store' + endpoint_type = os_options.get('endpoint_type') or 'publicURL' + if os_options.get('region_name'): + filter_kwargs['attr'] = 'region' + filter_kwargs['filter_value'] = os_options['region_name'] + + if os_options.get('auth_type') == 'v3applicationcredential': + try: + v3 + except NameError: + raise ClientException('Auth v3applicationcredential requires ' + 'python-keystoneclient>=2.0.0') + + try: + auth = v3.ApplicationCredential( + auth_url=auth_url, + application_credential_secret=os_options.get( + 'application_credential_secret'), + application_credential_id=os_options.get( + 'application_credential_id')) + sses = session.Session(auth=auth) + token = sses.get_token() + except ksauthexceptions.Unauthorized: + msg = 'Unauthorized. Check application credential id and secret.' + raise ClientException(msg) + except ksauthexceptions.AuthorizationFailure as err: + raise ClientException('Authorization Failure. %s' % err) + + try: + endpoint = sses.get_endpoint_data(service_type=service_type, + endpoint_type=endpoint_type, + **filter_kwargs) + + return endpoint.catalog_url, token + except ksauthexceptions.EndpointNotFound: + raise ClientException( + 'Endpoint for %s not found - ' + 'have you specified a region?' % service_type) + try: _ksclient = ksclient.Client( username=user, @@ -642,13 +685,8 @@ def get_auth_keystone(auth_url, user, key, os_options, **kwargs): raise ClientException(msg) except ksexceptions.AuthorizationFailure as err: raise ClientException('Authorization Failure. %s' % err) - service_type = os_options.get('service_type') or 'object-store' - endpoint_type = os_options.get('endpoint_type') or 'publicURL' + try: - filter_kwargs = {} - if os_options.get('region_name'): - filter_kwargs['attr'] = 'region' - filter_kwargs['filter_value'] = os_options['region_name'] endpoint = _ksclient.service_catalog.url_for( service_type=service_type, endpoint_type=endpoint_type, @@ -717,9 +755,12 @@ def get_auth(auth_url, user, key, **kwargs): if kwargs.get('tenant_name'): os_options['tenant_name'] = kwargs['tenant_name'] - if not (os_options.get('tenant_name') or os_options.get('tenant_id') or - os_options.get('project_name') or - os_options.get('project_id')): + if os_options.get('auth_type') == 'v3applicationcredential': + pass + elif not (os_options.get('tenant_name') or + os_options.get('tenant_id') or + os_options.get('project_name') or + os_options.get('project_id')): if auth_version in AUTH_VERSIONS_V2: raise ClientException('No tenant specified') raise ClientException('No project name or project id specified.') diff --git a/swiftclient/service.py b/swiftclient/service.py index fb334fde..b89399fe 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -110,6 +110,9 @@ def process_options(options): else: options['auth_version'] = '2.0' + if options.get('os_auth_type', None) == 'v3applicationcredential': + options['auth_version'] == '3' + # Use new-style args if old ones not present if not options['auth'] and options['os_auth_url']: options['auth'] = options['os_auth_url'] @@ -134,6 +137,11 @@ def process_options(options): 'auth_token': options['os_auth_token'], 'object_storage_url': options['os_storage_url'], 'region_name': options['os_region_name'], + 'auth_type': options['os_auth_type'], + 'application_credential_id': + options['os_application_credential_id'], + 'application_credential_secret': + options['os_application_credential_secret'], } @@ -162,6 +170,11 @@ def _build_default_global_options(): "os_project_domain_id": environ.get('OS_PROJECT_DOMAIN_ID'), "os_auth_url": environ.get('OS_AUTH_URL'), "os_auth_token": environ.get('OS_AUTH_TOKEN'), + "os_auth_type": environ.get('OS_AUTH_TYPE'), + "os_application_credential_id": + environ.get('OS_APPLICATION_CREDENTIAL_ID'), + "os_application_credential_secret": + environ.get('OS_APPLICATION_CREDENTIAL_SECRET'), "os_storage_url": environ.get('OS_STORAGE_URL'), "os_region_name": environ.get('OS_REGION_NAME'), "os_service_type": environ.get('OS_SERVICE_TYPE'), diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 1b34c084..0fef755f 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -1651,16 +1651,27 @@ def parse_args(parser, args, enforce_requires=True): return options, args if enforce_requires: - if options['auth_version'] == '3': + if options['os_auth_type'] == 'v3applicationcredential': + if not (options['os_application_credential_id'] and + options['os_application_credential_secret']): + exit('Auth version 3 (application credential) requires ' + 'OS_APPLICATION_CREDENTIAL_ID and ' + 'OS_APPLICATION_CREDENTIAL_SECRET to be set or ' + 'overridden with --os-application-credential-id and ' + '--os-application-credential-secret respectively.') + elif options['os_auth_type']: + exit('Only "v3applicationcredential" is supported for ' + '--os-auth-type') + elif options['auth_version'] == '3': if not options['auth']: - exit('Auth version 3 requires OS_AUTH_URL to be set or ' + + exit('Auth version 3 requires OS_AUTH_URL to be set or ' 'overridden with --os-auth-url') if not (options['user'] or options['os_user_id']): - exit('Auth version 3 requires either OS_USERNAME or ' + - 'OS_USER_ID to be set or overridden with ' + + exit('Auth version 3 requires either OS_USERNAME or ' + 'OS_USER_ID to be set or overridden with ' '--os-username or --os-user-id respectively.') if not options['key']: - exit('Auth version 3 requires OS_PASSWORD to be set or ' + + exit('Auth version 3 requires OS_PASSWORD to be set or ' 'overridden with --os-password') elif not (options['auth'] and options['user'] and options['key']): exit(''' @@ -1831,6 +1842,29 @@ def add_default_args(parser): 'env[OS_AUTH_URL].') os_grp.add_argument('--os_auth_url', help=argparse.SUPPRESS) + os_grp.add_argument('--os-auth-type', + metavar='', + default=environ.get('OS_AUTH_TYPE'), + help='OpenStack auth type for v3. Defaults to ' + 'env[OS_AUTH_TYPE].') + os_grp.add_argument('--os_auth_type', + help=argparse.SUPPRESS) + os_grp.add_argument('--os-application-credential-id', + metavar='', + default=environ.get('OS_APPLICATION_CREDENTIAL_ID'), + help='OpenStack appplication credential id. ' + 'Defaults to env[OS_APPLICATION_CREDENTIAL_ID].') + os_grp.add_argument('--os_application_credential_id', + help=argparse.SUPPRESS) + os_grp.add_argument('--os-application-credential-secret', + metavar='', + default=environ.get( + 'OS_APPLICATION_CREDENTIAL_SECRET'), + help='OpenStack appplication credential secret. ' + 'Defaults to ' + 'env[OS_APPLICATION_CREDENTIAL_SECRET].') + os_grp.add_argument('--os_application_credential_secret', + help=argparse.SUPPRESS) os_grp.add_argument('--os-auth-token', metavar='', default=environ.get('OS_AUTH_TOKEN'), @@ -1915,6 +1949,11 @@ def main(arguments=None): [--os-project-domain-name ] [--os-auth-url ] [--os-auth-token ] + [--os-auth-type ] + [--os-application-credential-id + ] + [--os-application-credential-secret + ] [--os-storage-url ] [--os-region-name ] [--os-service-type ] @@ -1967,6 +2006,11 @@ def main(arguments=None): --os-user-id abcdef0123456789abcdef0123456789 \\ --os-password password list + %(prog)s --os-auth-url https://api.example.com/v3 --auth-version 3\\ + --os-application-credential-id d78683c92f0e4f9b9b02a2e208039412 \\ + --os-application-credential-secret APPLICTION_CREDENTIAL_SECRET \\ + --os-auth-type v3applicationcredential list + %(prog)s --os-auth-token 6ee5eb33efad4e45ab46806eac010566 \\ --os-storage-url https://10.1.5.2:8080/v1/AUTH_ced809b6a4baea7aeab61a \\ list diff --git a/test/unit/test_shell.py b/test/unit/test_shell.py index a63d16b2..3c082184 100644 --- a/test/unit/test_shell.py +++ b/test/unit/test_shell.py @@ -2395,6 +2395,8 @@ def _verify_opts(self, actual_opts, expected_opts, expected_os_opts=None, 'object_storage_url', 'project_domain_id', 'user_id', 'user_domain_id', 'tenant_id', 'service_type', 'project_id', 'auth_token', + 'auth_type', 'application_credential_id', + 'application_credential_secret', 'project_domain_name'] for key in expected_os_opts_keys: self.assertIn(key, actual_os_opts_dict) @@ -2686,6 +2688,50 @@ def test_insufficient_args_v3(self): swiftclient.shell.main(args) self.assertIn('Auth version 3 requires OS_AUTH_URL', str(cm.exception)) + def test_command_args_v3applicationcredential(self): + result = [None, None] + fake_command = self._make_fake_command(result) + opts = {"auth_version": "3"} + os_opts = { + "auth_type": "v3applicationcredential", + "application_credential_id": "proejct_id", + "application_credential_secret": "secret", + "auth_url": "http://example.com:5000/v3"} + + args = _make_args("stat", opts, os_opts) + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + self.assertEqual(['stat'], result[1]) + with mock.patch('swiftclient.shell.st_stat', fake_command): + args = args + ["container_name"] + swiftclient.shell.main(args) + self.assertEqual(["stat", "container_name"], result[1]) + + def test_insufficient_args_v3applicationcredential(self): + opts = {"auth_version": "3"} + os_opts = { + "auth_type": "v3applicationcredential", + "application_credential_secret": "secret", + "auth_url": "http://example.com:5000/v3"} + + args = _make_args("stat", opts, os_opts) + with self.assertRaises(SystemExit) as cm: + swiftclient.shell.main(args) + self.assertIn('Auth version 3 (application credential) requires', + str(cm.exception)) + + os_opts = { + "auth_type": "v3password", + "application_credential_id": "proejct_id", + "application_credential_secret": "secret", + "auth_url": "http://example.com:5000/v3"} + + args = _make_args("stat", opts, os_opts) + with self.assertRaises(SystemExit) as cm: + swiftclient.shell.main(args) + self.assertIn('Only "v3applicationcredential" is supported for', + str(cm.exception)) + def test_password_prompt(self): def do_test(opts, os_opts, auth_version): args = _make_args("stat", opts, os_opts) diff --git a/test/unit/test_swiftclient.py b/test/unit/test_swiftclient.py index 2d45deb8..7354bdc8 100644 --- a/test/unit/test_swiftclient.py +++ b/test/unit/test_swiftclient.py @@ -562,6 +562,63 @@ def test_auth_v3_with_tenant_name(self): self.assertTrue(url.startswith("http")) self.assertTrue(token) + def test_auth_v3applicationcredential(self): + from keystoneauth1 import exceptions as ksauthexceptions + + os_options = { + "auth_type": "v3applicationcredential", + "application_credential_id": "proejct_id", + "application_credential_secret": "secret"} + + class FakeEndpointData(object): + catalog_url = 'http://swift.cluster/v1/KEY_project_id' + + class FakeKeystoneuth1v3Session(object): + + def __init__(self, auth): + self.auth = auth + self.token = 'token' + + def get_token(self): + if self.auth.auth_url == 'http://keystone:5000/v3': + return self.token + elif self.auth.auth_url == 'http://keystone:9000/v3': + raise ksauthexceptions.AuthorizationFailure + else: + raise ksauthexceptions.Unauthorized + + def get_endpoint_data(self, service_type, endpoint_type, **kwargs): + return FakeEndpointData() + + mock_sess = FakeKeystoneuth1v3Session + with mock.patch('keystoneauth1.session.Session', mock_sess): + url, token = c.get_auth('http://keystone:5000', '', '', + os_options=os_options, + auth_version="3") + + self.assertTrue(url.startswith("http")) + self.assertEqual(url, 'http://swift.cluster/v1/KEY_project_id') + self.assertEqual(token, 'token') + + with mock.patch('keystoneauth1.session.Session', mock_sess): + with self.assertRaises(c.ClientException) as exc_mgr: + url, token = c.get_auth('http://keystone:9000', '', '', + os_options=os_options, + auth_version="3") + + body = 'Unauthorized. Check application credential id and secret.' + body = 'Authorization Failure. Cannot authorize API client.' + self.assertEqual(exc_mgr.exception.__str__()[-89:], body) + + with mock.patch('keystoneauth1.session.Session', mock_sess): + with self.assertRaises(c.ClientException) as exc_mgr: + url, token = c.get_auth('http://keystone:5000', '', '', + os_options=os_options, + auth_version="2") + + body = 'Unauthorized. Check application credential id and secret.' + self.assertEqual(exc_mgr.exception.__str__()[-89:], body) + def test_get_keystone_client_2_0(self): # check the correct auth version is passed to get_auth_keystone os_options = {'tenant_name': 'asdf'} From bb3888e73e8bcbdc2340309a85a5f57a0c5bccf0 Mon Sep 17 00:00:00 2001 From: Andreas Jaeger Date: Sun, 19 Apr 2020 09:54:27 +0200 Subject: [PATCH 099/238] Update docs building Update docs building: * Switch to sphinx-build * Update requirements for Sphinx and openstackdocstheme for python 3 * Remove unneeded doc and translation sections from setup.cfg * Remove install_command, it's unneeded, the default is fine. Change-Id: Ib9fe754b700bceb164ba0f596cbcc6d864ccbadc --- doc/requirements.txt | 4 ++-- lower-constraints.txt | 4 ++-- setup.cfg | 12 ------------ tox.ini | 3 +-- 4 files changed, 5 insertions(+), 18 deletions(-) diff --git a/doc/requirements.txt b/doc/requirements.txt index 6cdad2ab..3ee9fc2a 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,5 +1,5 @@ keystoneauth1>=3.4.0 # Apache-2.0 sphinx!=1.6.6,!=1.6.7,<2.0.0,>=1.6.2;python_version=='2.7' # BSD -sphinx!=1.6.6,!=1.6.7,!=2.1.0,>=1.6.2;python_version>='3.4' # BSD +sphinx>=2.0.0,!=2.1.0;python_version>='3.4' # BSD reno>=2.5.0 # Apache-2.0 -openstackdocstheme>=1.20.0 # Apache-2.0 +openstackdocstheme>=1.31.2 # Apache-2.0 diff --git a/lower-constraints.txt b/lower-constraints.txt index ead02791..1e4ffb90 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -20,7 +20,7 @@ MarkupSafe==1.0 mccabe==0.2.1 mock==1.2.0 netaddr==0.7.10 -openstackdocstheme==1.20.0 +openstackdocstheme==2.0.0 openstacksdk==0.11.0 oslo.config==1.2.0 pbr==2.0.0 @@ -37,7 +37,7 @@ reno==2.5.0 requests==1.1.0 six==1.9.0 snowballstemmer==1.2.1 -sphinx==1.6.2 +sphinx==2.0.0 sphinxcontrib-websupport==1.0.1 stestr==2.0.0 testtools==2.2.0 diff --git a/setup.cfg b/setup.cfg index bcb4d223..3ce8e637 100644 --- a/setup.cfg +++ b/setup.cfg @@ -20,10 +20,6 @@ classifier = Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.7 -[global] -setup-hooks = - pbr.hooks.setup_hook - [files] packages = swiftclient @@ -43,14 +39,6 @@ console_scripts = keystoneauth1.plugin = v1password = swiftclient.authv1:PasswordLoader -[build_sphinx] -source-dir = doc/source -build-dir = doc/build -all_files = 1 - -[upload_sphinx] -upload-dir = doc/build/html - [bdist_wheel] universal = 1 diff --git a/tox.ini b/tox.ini index 2b4f6e36..8a50bb16 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,6 @@ skipsdist = True [testenv] usedevelop = True -install_command = python -m pip install -U {opts} {packages} list_dependencies_command = python -m pip freeze setenv = LANG=en_US.utf-8 @@ -68,7 +67,7 @@ basepython = python3 usedevelop = False deps = -r{toxinidir}/doc/requirements.txt commands= - python setup.py build_sphinx -W + sphinx-build -W -b html doc/source doc/build/html -W [flake8] # it's not a bug that we aren't using all of hacking, ignore: From 77993f242be5a2824cc9524836bea6fa6f77620a Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Mon, 20 Apr 2020 10:10:46 -0700 Subject: [PATCH 100/238] Add py38 classifier Change-Id: I28bf4aeb12c1f3833d2c6501b49d184b54d36093 --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 3ce8e637..95801a88 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,6 +19,7 @@ classifier = Programming Language :: Python :: 3 Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 [files] packages = From 947c09f30c6b603e3f4da060bc913407b158a0ca Mon Sep 17 00:00:00 2001 From: Ivan Kolodyazhny Date: Thu, 23 Apr 2020 19:01:47 +0300 Subject: [PATCH 101/238] Fixed capability discovery endpoint hardcode It fixes get_capabilities() method to process correctly endpoints like: 'https://:/v1', 'https://:/swift/v1'. Co-Authored-By: Daniel Cech Change-Id: Ib4037d0b49da1bce959947100629370805f510d5 Closes-bug: #1712358 --- swiftclient/client.py | 17 +++++++++++++++-- test/unit/test_swiftclient.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index 449b6cd3..3c3abc0b 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -45,6 +45,8 @@ AUTH_VERSIONS_V3 = ('3.0', '3', 3) USER_METADATA_TYPE = tuple('x-%s-meta-' % type_ for type_ in ('container', 'account', 'object')) +URI_PATTERN_INFO = re.compile(r'/info') +URI_PATTERN_VERSION = re.compile(r'\/v\d+\.?\d*(\/.*)?') try: from logging import NullHandler @@ -1935,11 +1937,22 @@ def delete_object(self, container, obj, query_string=None, response_dict=response_dict, headers=headers) - def get_capabilities(self, url=None): + def _map_url(self, url): url = url or self.url if not url: url, _ = self.get_auth() - parsed = urlparse(urljoin(url, '/info')) + scheme, netloc, path, params, query, fragment = urlparse(url) + if URI_PATTERN_VERSION.search(path): + path = URI_PATTERN_VERSION.sub('/info', path) + elif not URI_PATTERN_INFO.search(path): + if path.endswith('/'): + path += 'info' + else: + path += '/info' + return urlunparse((scheme, netloc, path, params, query, fragment)) + + def get_capabilities(self, url=None): + parsed = urlparse(self._map_url(url)) if not self.http_conn: self.http_conn = self.http_connection(url) return get_capabilities((parsed, self.http_conn[1])) diff --git a/test/unit/test_swiftclient.py b/test/unit/test_swiftclient.py index 2d45deb8..e3d0742d 100644 --- a/test/unit/test_swiftclient.py +++ b/test/unit/test_swiftclient.py @@ -2035,6 +2035,38 @@ def test_storage_url_override(self): self.assertEqual(request['headers']['x-auth-token'], 'tToken') + def test_url_mapping(self): + conn = c.Connection() + uri_versions = { + 'http://storage.test.com': + 'http://storage.test.com/info', + 'http://storage.test.com/': + 'http://storage.test.com/info', + 'http://storage.test.com/v1': + 'http://storage.test.com/info', + 'http://storage.test.com/v1/': + 'http://storage.test.com/info', + 'http://storage.test.com/swift': + 'http://storage.test.com/swift/info', + 'http://storage.test.com/swift/': + 'http://storage.test.com/swift/info', + 'http://storage.test.com/v1.0': + 'http://storage.test.com/info', + 'http://storage.test.com/swift/v1.0': + 'http://storage.test.com/swift/info', + 'http://storage.test.com/v111': + 'http://storage.test.com/info', + 'http://storage.test.com/v111/test': + 'http://storage.test.com/info', + 'http://storage.test.com/v1/test': + 'http://storage.test.com/info', + 'http://storage.test.com/swift/v1.0/test': + 'http://storage.test.com/swift/info', + 'http://storage.test.com/v1.0/test': + 'http://storage.test.com/info'} + for uri_k, uri_v in uri_versions.items(): + self.assertEqual(conn._map_url(uri_k), uri_v) + def test_get_capabilities(self): conn = c.Connection() with mock.patch('swiftclient.client.get_capabilities') as get_cap: From e44ca6d8af33d5f84b8c2c6c19c40b6c681c68fe Mon Sep 17 00:00:00 2001 From: Sean McGinnis Date: Fri, 24 Apr 2020 10:25:58 -0500 Subject: [PATCH 102/238] Bump default tox env from py37 to py38 Python 3.8 is now our highest level supported python runtime. This updates the default tox target environments to swap out py37 for py38 to make sure local development testing is covering this version. This does not impact zuul jobs in any way, nor prevent local tests against py37. It just changes the default if none is explicitly provided. Change-Id: I7bc7d7a49de746cc8fdb58a44619bc5ce66b7003 Signed-off-by: Sean McGinnis --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 8a50bb16..41314364 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,py37,pep8 +envlist = py27,py38,pep8 minversion = 2.0 skipsdist = True From 6e000d11e45e6c6c4d7c236d3c42338d685b95ae Mon Sep 17 00:00:00 2001 From: Matthew Oliver Date: Wed, 13 May 2020 15:36:22 +1000 Subject: [PATCH 103/238] Ussuri contrib docs community goal This patch standardizes the CONTRIBUTING.rst file and adds the required doc/source/contributor/contributing.rst The contibuting.txt points to the Swift contributor documentation. Change-Id: Ia6c105698dd0269479536645270d12a7c1061bc7 --- CONTRIBUTING.rst | 25 +++++++++++++------------ doc/source/contributor/contributing.rst | 14 ++++++++++++++ doc/source/index.rst | 1 + 3 files changed, 28 insertions(+), 12 deletions(-) create mode 100644 doc/source/contributor/contributing.rst diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 0bde9680..493a6c59 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -1,18 +1,19 @@ -If you would like to contribute to the development of OpenStack, you -must follow the steps in this page: +The source repository for this project can be found at: - https://docs.openstack.org/infra/manual/developers.html + https://opendev.org/openstack/python-swiftclient -Once those steps have been completed, changes to OpenStack should be -submitted for review via the Gerrit tool, following the workflow -documented at: +Pull requests submitted through GitHub are not monitored. - https://docs.openstack.org/infra/manual/developers.html#development-workflow +To start contributing to OpenStack, follow the steps in the contribution guide +to set up and use Gerrit: -Gerrit is the review system used in the OpenStack projects. We're sorry, -but we won't be able to respond to pull requests submitted through -GitHub. + https://docs.openstack.org/contributors/code-and-documentation/quick-start.html -Bugs should be filed on Launchpad, not Github: +Bugs should be filed on Launchpad: - https://bugs.launchpad.net/python-swiftclient + https://bugs.launchpad.net/python-swiftclient + +For more specific information about contributing to this repository, see the +swiftclient contributor guide: + + https://docs.openstack.org/python-swiftclient/latest/contributor/contributing.html diff --git a/doc/source/contributor/contributing.rst b/doc/source/contributor/contributing.rst new file mode 100644 index 00000000..b9f1e918 --- /dev/null +++ b/doc/source/contributor/contributing.rst @@ -0,0 +1,14 @@ +============================ +So You Want to Contribute... +============================ + +For general information on contributing to OpenStack, please check out the +`contributor guide `_ to get started. +It covers all the basics that are common to all OpenStack projects: the +accounts you need, the basics of interacting with our Gerrit review system, how +we communicate as a community, etc. + +The python-swiftclient is maintained by the OpenStack Swift project. +To understand our development process and how you can contribute to it, please +look at the Swift project's general contributor's page: +http://docs.openstack.org/swift/latest/contributor/contributing.html \ No newline at end of file diff --git a/doc/source/index.rst b/doc/source/index.rst index ab05c6bd..ae309721 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -16,6 +16,7 @@ Developer Documentation .. toctree:: :maxdepth: 2 + contributor/contributing cli/index service-api client-api From 9b6d1c7e88df409fdf272dd0a97e5f65c5fadd50 Mon Sep 17 00:00:00 2001 From: fuzihao Date: Wed, 20 May 2020 10:25:25 +0800 Subject: [PATCH 104/238] Fix pygments style New theme of docs (Victoria+) respects pygments_style. Since we starts using Victoria reqs while being on Ussuri, this patch ensures proper rendering both in Ussuri and Victoria. Change-Id: Iad418798277b9d7a1190e42e9079080a3d2707f3 --- doc/source/conf.py | 2 +- releasenotes/source/conf.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index a8ad3ad7..5f88f5ac 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -88,7 +88,7 @@ # show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = 'native' # A list of ignored prefixes for module index sorting. # modindex_common_prefix = [] diff --git a/releasenotes/source/conf.py b/releasenotes/source/conf.py index c71f41d4..9bdbbe40 100644 --- a/releasenotes/source/conf.py +++ b/releasenotes/source/conf.py @@ -108,7 +108,7 @@ # show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = 'native' # A list of ignored prefixes for module index sorting. # modindex_common_prefix = [] From 257a7185a8d5fdc11d91058f1735fa4273719aa9 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Wed, 13 May 2020 10:30:30 -0700 Subject: [PATCH 105/238] Application credential support follow-up Following the recent v3applicationcredentials patch, if you have your environment variables set up to work with python-openstackclient using swiftclient's v1password plugin, swiftclient won't work: $ env | egrep '^(OS|ST)_' ST_KEY=testing ST_USER=test:tester OS_AUTH_URL=http://saio/auth/v1.0 ST_AUTH=http://saio/auth/v1.0 OS_USERNAME=test:tester OS_AUTH_TYPE=v1password OS_PASSWORD=testing $ openstack object store account show +------------+----------------------------+ | Field | Value | +------------+----------------------------+ | Account | AUTH_test | | Bytes | 0 | | Containers | 11 | | Objects | 0 | +------------+----------------------------+ $ swift stat Only "v3applicationcredential" is supported for --os-auth-type We don't really want to allow (and mostly ignore) arbitrary OS_AUTH_TYPE values, though -- there are a whole bunch of plugins we don't remotely support. But it seems OK to allow any of the password plugins; while we won't actually use them (currently), we provide roughly equivalent functionality. Handful of other drive-bys: * Use a None sentinel to determine whether keystoneauth1 is installed instead of trying to catch a NameError. * Clarify error state when keystoneauth1 is not installed. * Fix a typo: "sses" -> "sess". Change-Id: Id7ea9c3ea8278ae86a04d057a472a8f8a87b8eae Related-Change: I9190e5e7e24b6a741970fa0d0ac792deccf73d25 --- swiftclient/client.py | 29 +++++++++++++++++------------ swiftclient/shell.py | 10 ++++++---- test/unit/test_shell.py | 2 +- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index 67440dd9..0aba6291 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -62,7 +62,7 @@ def emit(self, record): def createLock(self): self.lock = None -ksexceptions = ksclient_v2 = ksclient_v3 = None +ksexceptions = ksclient_v2 = ksclient_v3 = ksa_v3 = None try: from keystoneclient import exceptions as ksexceptions # prevent keystoneclient warning us that it has no log handlers @@ -72,8 +72,8 @@ def createLock(self): pass try: from keystoneclient.v3 import client as ksclient_v3 - from keystoneauth1.identity import v3 - from keystoneauth1 import session + from keystoneauth1.identity import v3 as ksa_v3 + from keystoneauth1 import session as ksa_session from keystoneauth1 import exceptions as ksauthexceptions except ImportError: pass @@ -627,22 +627,27 @@ def get_auth_keystone(auth_url, user, key, os_options, **kwargs): filter_kwargs['attr'] = 'region' filter_kwargs['filter_value'] = os_options['region_name'] - if os_options.get('auth_type') == 'v3applicationcredential': - try: - v3 - except NameError: + if os_options.get('auth_type') and os_options['auth_type'] not in ( + 'password', 'v2password', 'v3password', + 'v3applicationcredential'): + raise ClientException( + 'Swiftclient currently only supports v3applicationcredential ' + 'for auth_type') + elif os_options.get('auth_type') == 'v3applicationcredential': + if ksa_v3 is None: raise ClientException('Auth v3applicationcredential requires ' - 'python-keystoneclient>=2.0.0') + 'keystoneauth1 package; consider upgrading ' + 'to python-keystoneclient>=2.0.0') try: - auth = v3.ApplicationCredential( + auth = ksa_v3.ApplicationCredential( auth_url=auth_url, application_credential_secret=os_options.get( 'application_credential_secret'), application_credential_id=os_options.get( 'application_credential_id')) - sses = session.Session(auth=auth) - token = sses.get_token() + sess = ksa_session.Session(auth=auth) + token = sess.get_token() except ksauthexceptions.Unauthorized: msg = 'Unauthorized. Check application credential id and secret.' raise ClientException(msg) @@ -650,7 +655,7 @@ def get_auth_keystone(auth_url, user, key, os_options, **kwargs): raise ClientException('Authorization Failure. %s' % err) try: - endpoint = sses.get_endpoint_data(service_type=service_type, + endpoint = sess.get_endpoint_data(service_type=service_type, endpoint_type=endpoint_type, **filter_kwargs) diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 0fef755f..b129d637 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -1651,7 +1651,12 @@ def parse_args(parser, args, enforce_requires=True): return options, args if enforce_requires: - if options['os_auth_type'] == 'v3applicationcredential': + if options['os_auth_type'] and options['os_auth_type'] not in ( + 'password', 'v1password', 'v2password', 'v3password', + 'v3applicationcredential'): + exit('Only "v3applicationcredential" is supported for ' + '--os-auth-type') + elif options['os_auth_type'] == 'v3applicationcredential': if not (options['os_application_credential_id'] and options['os_application_credential_secret']): exit('Auth version 3 (application credential) requires ' @@ -1659,9 +1664,6 @@ def parse_args(parser, args, enforce_requires=True): 'OS_APPLICATION_CREDENTIAL_SECRET to be set or ' 'overridden with --os-application-credential-id and ' '--os-application-credential-secret respectively.') - elif options['os_auth_type']: - exit('Only "v3applicationcredential" is supported for ' - '--os-auth-type') elif options['auth_version'] == '3': if not options['auth']: exit('Auth version 3 requires OS_AUTH_URL to be set or ' diff --git a/test/unit/test_shell.py b/test/unit/test_shell.py index 3c082184..f94e5e23 100644 --- a/test/unit/test_shell.py +++ b/test/unit/test_shell.py @@ -2721,7 +2721,7 @@ def test_insufficient_args_v3applicationcredential(self): str(cm.exception)) os_opts = { - "auth_type": "v3password", + "auth_type": "v3oidcpassword", "application_credential_id": "proejct_id", "application_credential_secret": "secret", "auth_url": "http://example.com:5000/v3"} From d1f894c124df668b9669d64ce4970c55e9c404fd Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Tue, 2 Jun 2020 13:34:49 -0700 Subject: [PATCH 106/238] Remove references to swift-specs and blueprints Those have been dead for a long while. Change-Id: I21306a40479ad319c0ef48aa8bfb88f261011390 --- README.rst | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/README.rst b/README.rst index 5243dd6b..c9ae3d42 100644 --- a/README.rst +++ b/README.rst @@ -23,7 +23,7 @@ in the `OpenStack wiki`__. __ https://docs.openstack.org/infra/manual/developers.html This code is based on the original client previously included with -`OpenStack's Swift`__ The python-swiftclient is licensed under the +`OpenStack's Swift`__. The python-swiftclient is licensed under the Apache License like the rest of OpenStack. __ https://github.com/openstack/swift @@ -32,21 +32,17 @@ __ https://github.com/openstack/swift * `PyPI`_ - package installation * `Online Documentation`_ * `Launchpad project`_ - release management -* `Blueprints`_ - feature specifications * `Bugs`_ - issue tracking * `Source`_ -* `Specs`_ * `How to Contribute`_ * `Release Notes`_ .. _PyPI: https://pypi.org/project/python-swiftclient .. _Online Documentation: https://docs.openstack.org/python-swiftclient/latest/ .. _Launchpad project: https://launchpad.net/python-swiftclient -.. _Blueprints: https://blueprints.launchpad.net/python-swiftclient .. _Bugs: https://bugs.launchpad.net/python-swiftclient .. _Source: https://opendev.org/openstack/python-swiftclient .. _How to Contribute: https://docs.openstack.org/infra/manual/developers.html -.. _Specs: https://specs.openstack.org/openstack/swift-specs/ .. _Release Notes: https://docs.openstack.org/releasenotes/python-swiftclient .. contents:: Contents: From 22d1f3a39a9b23a62f443c2bd8ab9639c1b5a669 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Wed, 17 Jun 2020 21:05:03 -0700 Subject: [PATCH 107/238] Clean up some warnings Change-Id: Iae149533d04c7b173c4ef88fb775f5fe13c16466 --- swiftclient/utils.py | 7 ++++-- test/unit/test_utils.py | 52 ++++++++++++++++++++--------------------- 2 files changed, 31 insertions(+), 28 deletions(-) diff --git a/swiftclient/utils.py b/swiftclient/utils.py index 9e43237c..656acad4 100644 --- a/swiftclient/utils.py +++ b/swiftclient/utils.py @@ -14,7 +14,10 @@ # limitations under the License. """Miscellaneous utility functions for use with Swift.""" from calendar import timegm -import collections +try: + from collections.abc import Mapping +except ImportError: + from collections import Mapping import gzip import hashlib import hmac @@ -218,7 +221,7 @@ def parse_api_response(headers, body): def split_request_headers(options, prefix=''): headers = {} - if isinstance(options, collections.Mapping): + if isinstance(options, Mapping): options = options.items() for item in options: if isinstance(item, six.string_types): diff --git a/test/unit/test_utils.py b/test/unit/test_utils.py index 97abc444..cbee82bf 100644 --- a/test/unit/test_utils.py +++ b/test/unit/test_utils.py @@ -521,15 +521,15 @@ def test_tempfile(self): with tempfile.NamedTemporaryFile(mode='wb') as f: f.write(b'a' * 100) f.flush() - contents = open(f.name, 'rb') - data = u.LengthWrapper(contents, 42, True) - s = b'a' * 42 - read_data = b''.join(iter(data.read, '')) + with open(f.name, 'rb') as contents: + data = u.LengthWrapper(contents, 42, True) + s = b'a' * 42 + read_data = b''.join(iter(data.read, '')) - self.assertEqual(42, len(data)) - self.assertEqual(42, len(read_data)) - self.assertEqual(s, read_data) - self.assertEqual(md5(s).hexdigest(), data.get_md5sum()) + self.assertEqual(42, len(data)) + self.assertEqual(42, len(read_data)) + self.assertEqual(s, read_data) + self.assertEqual(md5(s).hexdigest(), data.get_md5sum()) def test_segmented_file(self): with tempfile.NamedTemporaryFile(mode='wb') as f: @@ -539,24 +539,24 @@ def test_segmented_file(self): f.write((c * segment_length).encode()) f.flush() for i, c in enumerate(segments): - contents = open(f.name, 'rb') - contents.seek(i * segment_length) - data = u.LengthWrapper(contents, segment_length, True) - read_data = b''.join(iter(data.read, '')) - s = (c * segment_length).encode() - - self.assertEqual(segment_length, len(data)) - self.assertEqual(segment_length, len(read_data)) - self.assertEqual(s, read_data) - self.assertEqual(md5(s).hexdigest(), data.get_md5sum()) - - data.reset() - self.assertEqual(md5().hexdigest(), data.get_md5sum()) - read_data = b''.join(iter(data.read, '')) - self.assertEqual(segment_length, len(data)) - self.assertEqual(segment_length, len(read_data)) - self.assertEqual(s, read_data) - self.assertEqual(md5(s).hexdigest(), data.get_md5sum()) + with open(f.name, 'rb') as contents: + contents.seek(i * segment_length) + data = u.LengthWrapper(contents, segment_length, True) + read_data = b''.join(iter(data.read, '')) + s = (c * segment_length).encode() + + self.assertEqual(segment_length, len(data)) + self.assertEqual(segment_length, len(read_data)) + self.assertEqual(s, read_data) + self.assertEqual(md5(s).hexdigest(), data.get_md5sum()) + + data.reset() + self.assertEqual(md5().hexdigest(), data.get_md5sum()) + read_data = b''.join(iter(data.read, '')) + self.assertEqual(segment_length, len(data)) + self.assertEqual(segment_length, len(read_data)) + self.assertEqual(s, read_data) + self.assertEqual(md5(s).hexdigest(), data.get_md5sum()) class TestGroupers(unittest.TestCase): From 842086d27fbb8ae8b5b723a0167d33ca2050feea Mon Sep 17 00:00:00 2001 From: Meuh Date: Tue, 28 Jul 2020 16:16:19 +0200 Subject: [PATCH 108/238] Add max_backoff and starting_backoff for get_conn in swift service Change-Id: I45f5d3009e0e2015c7366384ee826113fc27c70b --- swiftclient/service.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/swiftclient/service.py b/swiftclient/service.py index b89399fe..cd96a5b9 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -274,7 +274,7 @@ def get_conn(options): return Connection(options['auth'], options['user'], options['key'], - options['retries'], + retries=options['retries'], auth_version=options['auth_version'], os_options=options['os_options'], snet=options['snet'], @@ -283,7 +283,9 @@ def get_conn(options): cert=options['os_cert'], cert_key=options['os_key'], ssl_compression=options['ssl_compression'], - force_auth_retry=options['force_auth_retry']) + force_auth_retry=options['force_auth_retry'], + starting_backoff=options.get('starting_backoff', 1), + max_backoff=options.get('max_backoff', 64)) def mkdirs(path): From 89c8d9b853aa02e0a90682d04cec2cd42f740e80 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Tue, 28 Jul 2020 11:27:33 -0700 Subject: [PATCH 109/238] Speed up test_lazy_connections It doesn't really need to sleep a full second. Change-Id: Ida80f0c5a983edb33a93662badb6aa1a25f9a27c --- test/unit/test_multithreading.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/test_multithreading.py b/test/unit/test_multithreading.py index 8944d48e..e9732cd8 100644 --- a/test/unit/test_multithreading.py +++ b/test/unit/test_multithreading.py @@ -37,7 +37,7 @@ def _func(self, conn, item, *args, **kwargs): self.got_args_kwargs.put((args, kwargs)) if item == 'sleep': - sleep(1) + sleep(.1) if item == 'go boom': raise Exception('I went boom!') From 0c70d164ba52d76a6dbbbe8765d15fb969fc07ff Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Wed, 17 Jun 2020 15:44:22 -0700 Subject: [PATCH 110/238] (Mostly) revert "Cleanup session on delete" This reverts commit 1f26c5736949e1c3b57c024a315e33fc419f126e for py2. Apparently the existence of the __del__ method on Python 2 prevents us from cleaning up all file descriptors. Change-Id: Id6cff5dd7b9faf9c4240c0cb26b74d05ed37da5b Closes-Bug: #1873435 Related-Bug: #1838775 --- swiftclient/client.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index 0aba6291..5c63b606 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -443,14 +443,16 @@ def __init__(self, url, proxy=None, cacert=None, insecure=False, if timeout: self.requests_args['timeout'] = timeout - def __del__(self): - """Cleanup resources other than memory""" - if self.request_session: - # The session we create must be closed to free up file descriptors - try: - self.request_session.close() - finally: - self.request_session = None + if not six.PY2: + def __del__(self): + """Cleanup resources other than memory""" + if self.request_session: + # The session we create must be closed to free up + # file descriptors + try: + self.request_session.close() + finally: + self.request_session = None def _request(self, *arg, **kwarg): """Final wrapper before requests call, to be patched in tests""" From 0f6713ed5b8cdcf5cbc0850dea224b41d90e63f4 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Fri, 14 Aug 2020 10:41:15 -0700 Subject: [PATCH 111/238] Include transaction ID in ClientException.__str__ It's fairly annoying getting a traceback in swift's probe tests then only having a URL and status code to go searching for in logs. Leave the shell.py output untouched, though, since we output the transaction ID on a new line anyway. Change-Id: Idb849848ec08b6c04812b088467c9a687c2a7e27 --- swiftclient/exceptions.py | 12 +++++++++++- swiftclient/shell.py | 3 ++- test/unit/test_swiftclient.py | 17 +++++++++++++++++ 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/swiftclient/exceptions.py b/swiftclient/exceptions.py index da70379e..a9b993ce 100644 --- a/swiftclient/exceptions.py +++ b/swiftclient/exceptions.py @@ -35,6 +35,13 @@ def __init__(self, msg, http_scheme='', http_host='', http_port='', self.http_response_content = http_response_content self.http_response_headers = http_response_headers + self.transaction_id = None + if self.http_response_headers: + for header in ('X-Trans-Id', 'X-Openstack-Request-Id'): + if header in self.http_response_headers: + self.transaction_id = self.http_response_headers[header] + break + @classmethod def from_response(cls, resp, msg=None, body=None): msg = msg or '%s %s' % (resp.status_code, resp.reason) @@ -78,4 +85,7 @@ def __str__(self): else: b += ' [first 60 chars of response] %s' \ % self.http_response_content[:60] - return b and '%s: %s' % (a, b) or a + c = '' + if self.transaction_id: + c = ' (txn: %s)' % self.transaction_id + return b and '%s: %s%s' % (a, b, c) or (a + c) diff --git a/swiftclient/shell.py b/swiftclient/shell.py index b129d637..dbcd437b 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -2058,8 +2058,9 @@ def main(arguments=None): try: globals()['st_%s' % args[0]](parser, argv[1:], output) except ClientException as err: + trans_id = err.transaction_id + err.transaction_id = None # clear it so we aren't overly noisy output.error(str(err)) - trans_id = (err.http_response_headers or {}).get('X-Trans-Id') if trans_id: output.error("Failed Transaction ID: %s", parse_header_string(trans_id)) diff --git a/test/unit/test_swiftclient.py b/test/unit/test_swiftclient.py index dfd79c77..2644e337 100644 --- a/test/unit/test_swiftclient.py +++ b/test/unit/test_swiftclient.py @@ -84,6 +84,23 @@ def test_attrs(self): self.assertIs(True, hasattr(exc, key)) self.assertEqual(getattr(exc, key), value) + def test_transaction_id_from_headers(self): + exc = c.ClientException('test') + self.assertIsNone(exc.transaction_id) + + exc = c.ClientException('test', http_response_headers={}) + self.assertIsNone(exc.transaction_id) + + exc = c.ClientException('test', http_response_headers={ + 'X-Trans-Id': 'some-id'}) + self.assertEqual(exc.transaction_id, 'some-id') + self.assertIn('(txn: some-id)', str(exc)) + + exc = c.ClientException('test', http_response_headers={ + 'X-Openstack-Request-Id': 'some-other-id'}) + self.assertEqual(exc.transaction_id, 'some-other-id') + self.assertIn('(txn: some-other-id)', str(exc)) + class MockHttpResponse(object): def __init__(self, status=0, headers=None, verify=False): From 5cb906148709ccc74ca96463fd69a5cae2381edb Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Tue, 8 Sep 2020 12:07:48 -0700 Subject: [PATCH 112/238] Clean up some requirements * Drop the py26 marker for futures; we don't support 2.6 anymore. * Split hacking version used based on python version. * Clean up sphinx split -- 2.0+ aren't available to install on py2, anyway. Depends-On: https://review.opendev.org/#/c/752340/ Depends-On: https://review.opendev.org/#/c/752736/ Change-Id: I5a6ba8e65c23ada7297f6684dcbdd886591d0af5 --- doc/requirements.txt | 3 +-- requirements.txt | 2 +- test-requirements.txt | 3 ++- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/requirements.txt b/doc/requirements.txt index 3ee9fc2a..5700e3b1 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,5 +1,4 @@ keystoneauth1>=3.4.0 # Apache-2.0 -sphinx!=1.6.6,!=1.6.7,<2.0.0,>=1.6.2;python_version=='2.7' # BSD -sphinx>=2.0.0,!=2.1.0;python_version>='3.4' # BSD +sphinx>=1.6.2,!=1.6.6,!=1.6.7,!=2.1.0,!=3.0.0 # BSD reno>=2.5.0 # Apache-2.0 openstackdocstheme>=1.31.2 # Apache-2.0 diff --git a/requirements.txt b/requirements.txt index 1c2ce33d..4757239b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -futures>=3.0.0;python_version=='2.7' or python_version=='2.6' # BSD +futures>=3.0.0;python_version=='2.7' # BSD requests>=1.1.0 six>=1.9.0 diff --git a/test-requirements.txt b/test-requirements.txt index 5dba1a60..c2fb2c6e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,4 +1,5 @@ -hacking>=1.1.0,<1.2.0 # Apache-2.0 +hacking>=1.1.0,<1.2.0;python_version<'3.0' # Apache-2.0 +hacking>=3.2.0,<3.3.0;python_version>='3.0' # Apache-2.0 coverage!=4.4,>=4.0 # Apache-2.0 keystoneauth1>=3.4.0 # Apache-2.0 From e367b6270c6d544312817ab31723aae54f4c234a Mon Sep 17 00:00:00 2001 From: Andreas Jaeger Date: Tue, 2 Jun 2020 19:01:07 +0200 Subject: [PATCH 113/238] Switch to newer openstackdocstheme and reno versions Switch to openstackdocstheme 2.2.1 and reno 3.1.0 versions. Using these versions will allow especially: * Linking from HTML to PDF document * Allow parallel building of documents * Fix some rendering problems Update Sphinx version as well. Set openstackdocs_pdf_link to link to PDF file. Note that the link to the published document only works on docs.openstack.org where the PDF file is placed in the top-level html directory. The site-preview places the PDF in a pdf directory. Disable openstackdocs_auto_name to use 'project' variable as name. Change pygments_style to 'native' since old theme version always used 'native' and the theme now respects the setting and using 'sphinx' can lead to some strange rendering. Remove docs requirements from lower-constraints, they are not needed during install or test but only for docs building. openstackdocstheme renames some variables, so follow the renames before the next release removes them. A couple of variables are also not needed anymore, remove them. See also http://lists.openstack.org/pipermail/openstack-discuss/2020-May/014971.html Change-Id: I4418f86c3066353d43758118865baf66d1741c79 --- doc/requirements.txt | 4 ++-- doc/source/conf.py | 10 ++++++---- lower-constraints.txt | 4 ---- releasenotes/source/conf.py | 6 +++--- 4 files changed, 11 insertions(+), 13 deletions(-) diff --git a/doc/requirements.txt b/doc/requirements.txt index 5700e3b1..6894ce18 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,4 +1,4 @@ keystoneauth1>=3.4.0 # Apache-2.0 sphinx>=1.6.2,!=1.6.6,!=1.6.7,!=2.1.0,!=3.0.0 # BSD -reno>=2.5.0 # Apache-2.0 -openstackdocstheme>=1.31.2 # Apache-2.0 +reno>=3.1.0 # Apache-2.0 +openstackdocstheme>=2.2.1 # Apache-2.0 diff --git a/doc/source/conf.py b/doc/source/conf.py index 5f88f5ac..83816050 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -55,6 +55,12 @@ # General information about the project. copyright = u'2013-2016 OpenStack, LLC.' +# -- Options for openstackdocstheme ------------------------------------------- +openstackdocs_repo_name = 'openstack/python-swiftclient' +openstackdocs_bug_project = 'python-swiftclient' +openstackdocs_bug_tag = '' +openstackdocs_pdf_link = True + # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # language = None @@ -131,10 +137,6 @@ # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -# html_last_updated_fmt = '%b %d, %Y' - # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. # html_use_smartypants = True diff --git a/lower-constraints.txt b/lower-constraints.txt index 1e4ffb90..28a10600 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -20,7 +20,6 @@ MarkupSafe==1.0 mccabe==0.2.1 mock==1.2.0 netaddr==0.7.10 -openstackdocstheme==2.0.0 openstacksdk==0.11.0 oslo.config==1.2.0 pbr==2.0.0 @@ -33,12 +32,9 @@ python-mimeparse==1.6.0 python-subunit==1.0.0 pytz==2013.6 PyYAML==3.12 -reno==2.5.0 requests==1.1.0 six==1.9.0 snowballstemmer==1.2.1 -sphinx==2.0.0 -sphinxcontrib-websupport==1.0.1 stestr==2.0.0 testtools==2.2.0 traceback2==1.4.0 diff --git a/releasenotes/source/conf.py b/releasenotes/source/conf.py index 9bdbbe40..a050a546 100644 --- a/releasenotes/source/conf.py +++ b/releasenotes/source/conf.py @@ -339,6 +339,6 @@ locale_dirs = ['locale/'] # -- Options for openstackdocstheme ------------------------------------------- -repository_name = 'openstack/python-swiftclient' -bug_project = 'python-swiftclient' -bug_tag = '' +openstackdocs_repo_name = 'openstack/python-swiftclient' +openstackdocs_bug_project = 'python-swiftclient' +openstackdocs_bug_tag = '' From 9f69908f759daafc99a162b136b24570a2822ee2 Mon Sep 17 00:00:00 2001 From: Ghanshyam Mann Date: Fri, 18 Sep 2020 16:01:28 -0500 Subject: [PATCH 114/238] [goal] Migrate testing to ubuntu focal As per victoria cycle testing runtime and community goal[1] we need to migrate upstream CI/CD to Ubuntu Focal(20.04). - Keep py2 functional job run on Bionic node Story: #2007865 Task: #40221 Change-Id: I578abeb1552d73a2e5c5a24ba7afab975508ea0c --- .zuul.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.zuul.yaml b/.zuul.yaml index 87ac54b8..334eefc8 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -27,6 +27,7 @@ - job: name: swiftclient-functional-py2 parent: swiftclient-functional + nodeset: openstack-single-node-bionic description: | Run functional tests of python-swiftclient under Python 2 vars: From 16b377c62304c5a4f9f02cd9985534e60a04725b Mon Sep 17 00:00:00 2001 From: OpenStack Release Bot Date: Fri, 11 Sep 2020 18:40:16 +0000 Subject: [PATCH 115/238] Update master for stable/victoria Add file to the reno documentation build to show release notes for stable/victoria. Use pbr instruction to increment the minor version number automatically so that master versions are higher than the versions on stable/victoria. Change-Id: Ic1dc50f7e9d0b38f04a114eb22de25466ba83e56 Sem-Ver: feature --- releasenotes/source/index.rst | 1 + releasenotes/source/victoria.rst | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 releasenotes/source/victoria.rst diff --git a/releasenotes/source/index.rst b/releasenotes/source/index.rst index d46593b0..a63715f1 100644 --- a/releasenotes/source/index.rst +++ b/releasenotes/source/index.rst @@ -6,6 +6,7 @@ :maxdepth: 1 current + victoria ussuri train stein diff --git a/releasenotes/source/victoria.rst b/releasenotes/source/victoria.rst new file mode 100644 index 00000000..4efc7b6f --- /dev/null +++ b/releasenotes/source/victoria.rst @@ -0,0 +1,6 @@ +============================= +Victoria Series Release Notes +============================= + +.. release-notes:: + :branch: stable/victoria From 5728bf4a502ba11689ec9a600670539898a8bf9c Mon Sep 17 00:00:00 2001 From: OpenStack Release Bot Date: Fri, 11 Sep 2020 18:40:19 +0000 Subject: [PATCH 116/238] Add Python3 wallaby unit tests This is an automatically generated patch to ensure unit testing is in place for all the of the tested runtimes for wallaby. See also the PTI in governance [1]. [1]: https://governance.openstack.org/tc/reference/project-testing-interface.html Change-Id: I9a2e832e8165232c15731cfb97d401c84abf95c0 --- .zuul.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.zuul.yaml b/.zuul.yaml index 87ac54b8..29f58ae0 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -38,7 +38,7 @@ - lib-forward-testing-python3 - openstack-lower-constraints-jobs - openstack-python-jobs - - openstack-python3-victoria-jobs + - openstack-python3-wallaby-jobs - publish-openstack-docs-pti - release-notes-jobs-python3 check: From c73792c2e55e4fbdff78a192a991e064a317e840 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Mon, 21 Sep 2020 16:40:05 -0700 Subject: [PATCH 117/238] tests: Make test_delete_container_versions less flakey Hammering that test in a tight loop, I'd often see failures due to ordering issues. Make the delete single-threaded to avoid that. Change-Id: Iff45be32a7c3f258214cce78001fd33ad0a39b8c --- test/unit/test_shell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/test_shell.py b/test/unit/test_shell.py index f94e5e23..46ba52ca 100644 --- a/test/unit/test_shell.py +++ b/test/unit/test_shell.py @@ -1547,7 +1547,7 @@ def test_delete_container_versions(self, connection): [None, []], ] connection.return_value.attempts = 0 - argv = ["", "delete", "--versions", "container"] + argv = ["", "delete", "--versions", "container", "--object-threads=1"] connection.return_value.head_object.return_value = {} swiftclient.shell.main(argv) connection.return_value.delete_container.assert_called_with( From 97aa3e65412ee241fce7721927b0b003daf51ed4 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Sat, 18 Apr 2020 22:41:55 -0700 Subject: [PATCH 118/238] Close connections created when calling module-level functions Co-Authored-By: Clay Gerrard Change-Id: Id62e63afc6f2ffa32eb8640787c78559481050f9 Related-Change: I200ad0cdc8b7999c3f5521b9a822122bd18714bf Related-Bug: #1873435 Closes-Bug: #1838775 --- swiftclient/client.py | 87 ++++++++++++++++++++++++++++------- test/unit/test_swiftclient.py | 1 + test/unit/utils.py | 6 ++- 3 files changed, 75 insertions(+), 19 deletions(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index 5c63b606..544247a6 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -268,7 +268,7 @@ class _ObjectBody(object): Readable and iterable object body response wrapper. """ - def __init__(self, resp, chunk_size): + def __init__(self, resp, chunk_size, conn_to_close): """ Wrap the underlying response @@ -277,9 +277,13 @@ def __init__(self, resp, chunk_size): """ self.resp = resp self.chunk_size = chunk_size + self.conn_to_close = conn_to_close def read(self, length=None): - return self.resp.read(length) + buf = self.resp.read(length) + if length != 0 and not buf: + self.close() + return buf def __iter__(self): return self @@ -295,6 +299,8 @@ def __next__(self): def close(self): self.resp.close() + if self.conn_to_close: + self.conn_to_close.close() class _RetryBody(_ObjectBody): @@ -320,7 +326,7 @@ def __init__(self, resp, connection, container, obj, :param headers: an optional dictionary with additional headers to include in the request """ - super(_RetryBody, self).__init__(resp, resp_chunk_size) + super(_RetryBody, self).__init__(resp, resp_chunk_size, None) self.expected_length = int(self.resp.getheader('Content-Length')) self.conn = connection self.container = container @@ -443,17 +449,6 @@ def __init__(self, url, proxy=None, cacert=None, insecure=False, if timeout: self.requests_args['timeout'] = timeout - if not six.PY2: - def __del__(self): - """Cleanup resources other than memory""" - if self.request_session: - # The session we create must be closed to free up - # file descriptors - try: - self.request_session.close() - finally: - self.request_session = None - def _request(self, *arg, **kwarg): """Final wrapper before requests call, to be patched in tests""" return self.request_session.request(*arg, **kwarg) @@ -515,7 +510,7 @@ def releasing_read(*args, **kwargs): # urllib3's connection pool. This will reduce the number of # log messages seen in bug #1341777. This does not actually # close a socket. It will also prevent people from being - # mislead as to the cause of a bug as in bug #1424732. + # misled as to the cause of a bug as in bug #1424732. self.resp.close() return chunk @@ -845,8 +840,10 @@ def get_account(url, token, marker=None, limit=None, prefix=None, if headers: req_headers.update(headers) + close_conn = False if not http_conn: http_conn = http_connection(url) + close_conn = True if full_listing: rv = get_account(url, token, marker, limit, prefix, end_marker, http_conn, headers=req_headers, delimiter=delimiter) @@ -876,6 +873,8 @@ def get_account(url, token, marker=None, limit=None, prefix=None, conn.request(method, full_path, '', req_headers) resp = conn.getresponse() body = resp.read() + if close_conn: + conn.close() http_log(("%s?%s" % (url, qs), method,), {'headers': req_headers}, resp, body) @@ -902,10 +901,12 @@ def head_account(url, token, http_conn=None, headers=None, be lowercase) :raises ClientException: HTTP HEAD request failed """ + close_conn = False if http_conn: parsed, conn = http_conn else: parsed, conn = http_connection(url) + close_conn = True method = "HEAD" req_headers = {'X-Auth-Token': token} if service_token: @@ -916,6 +917,8 @@ def head_account(url, token, http_conn=None, headers=None, conn.request(method, parsed.path, '', req_headers) resp = conn.getresponse() body = resp.read() + if close_conn: + conn.close() http_log((url, method,), {'headers': req_headers}, resp, body) if resp.status < 200 or resp.status >= 300: raise ClientException.from_response(resp, 'Account HEAD failed', body) @@ -941,10 +944,12 @@ def post_account(url, token, headers, http_conn=None, response_dict=None, :raises ClientException: HTTP POST request failed :returns: resp_headers, body """ + close_conn = False if http_conn: parsed, conn = http_conn else: parsed, conn = http_connection(url) + close_conn = True method = 'POST' path = parsed.path if query_string: @@ -957,6 +962,8 @@ def post_account(url, token, headers, http_conn=None, response_dict=None, conn.request(method, path, data, req_headers) resp = conn.getresponse() body = resp.read() + if close_conn: + conn.close() http_log((url, method,), {'headers': req_headers}, resp, body) store_response(resp, response_dict) @@ -998,8 +1005,10 @@ def get_container(url, token, container, marker=None, limit=None, headers will be a dict and all header names will be lowercase. :raises ClientException: HTTP GET request failed """ + close_conn = False if not http_conn: http_conn = http_connection(url) + close_conn = True if full_listing: rv = get_container(url, token, container, marker, limit, prefix, delimiter, end_marker, version_marker, path=path, @@ -1048,6 +1057,8 @@ def get_container(url, token, container, marker=None, limit=None, conn.request(method, '%s?%s' % (cont_path, qs), '', req_headers) resp = conn.getresponse() body = resp.read() + if close_conn: + conn.close() http_log(('%(url)s%(cont_path)s?%(qs)s' % {'url': url.replace(parsed.path, ''), 'cont_path': cont_path, @@ -1078,10 +1089,12 @@ def head_container(url, token, container, http_conn=None, headers=None, be lowercase) :raises ClientException: HTTP HEAD request failed """ + close_conn = False if http_conn: parsed, conn = http_conn else: parsed, conn = http_connection(url) + close_conn = True path = '%s/%s' % (parsed.path, quote(container)) method = 'HEAD' req_headers = {'X-Auth-Token': token} @@ -1092,6 +1105,8 @@ def head_container(url, token, container, http_conn=None, headers=None, conn.request(method, path, '', req_headers) resp = conn.getresponse() body = resp.read() + if close_conn: + conn.close() http_log(('%s%s' % (url.replace(parsed.path, ''), path), method,), {'headers': req_headers}, resp, body) @@ -1119,10 +1134,12 @@ def put_container(url, token, container, headers=None, http_conn=None, :param query_string: if set will be appended with '?' to generated path :raises ClientException: HTTP PUT request failed """ + close_conn = False if http_conn: parsed, conn = http_conn else: parsed, conn = http_connection(url) + close_conn = True path = '%s/%s' % (parsed.path, quote(container)) method = 'PUT' req_headers = {'X-Auth-Token': token} @@ -1137,6 +1154,8 @@ def put_container(url, token, container, headers=None, http_conn=None, conn.request(method, path, '', req_headers) resp = conn.getresponse() body = resp.read() + if close_conn: + conn.close() store_response(resp, response_dict) @@ -1162,10 +1181,12 @@ def post_container(url, token, container, headers, http_conn=None, :param service_token: service auth token :raises ClientException: HTTP POST request failed """ + close_conn = False if http_conn: parsed, conn = http_conn else: parsed, conn = http_connection(url) + close_conn = True path = '%s/%s' % (parsed.path, quote(container)) method = 'POST' req_headers = {'X-Auth-Token': token} @@ -1178,6 +1199,8 @@ def post_container(url, token, container, headers, http_conn=None, conn.request(method, path, '', req_headers) resp = conn.getresponse() body = resp.read() + if close_conn: + conn.close() http_log(('%s%s' % (url.replace(parsed.path, ''), path), method,), {'headers': req_headers}, resp, body) @@ -1206,10 +1229,12 @@ def delete_container(url, token, container, http_conn=None, :param headers: additional headers to include in the request :raises ClientException: HTTP DELETE request failed """ + close_conn = False if http_conn: parsed, conn = http_conn else: parsed, conn = http_connection(url) + close_conn = True path = '%s/%s' % (parsed.path, quote(container)) if headers: headers = dict(headers) @@ -1225,6 +1250,8 @@ def delete_container(url, token, container, http_conn=None, conn.request(method, path, '', headers) resp = conn.getresponse() body = resp.read() + if close_conn: + conn.close() http_log(('%s%s' % (url.replace(parsed.path, ''), path), method,), {'headers': headers}, resp, body) @@ -1246,7 +1273,8 @@ def get_object(url, token, container, name, http_conn=None, :param container: container name that the object is in :param name: object name to get :param http_conn: a tuple of (parsed url, HTTPConnection object), - (If None, it will create the conn object) + (If None, it will create the conn object and close it + after all content is read) :param resp_chunk_size: if defined, chunk size of data to read. NOTE: If you specify a resp_chunk_size you must fully read the object's contents before making another @@ -1261,10 +1289,12 @@ def get_object(url, token, container, name, http_conn=None, headers will be a dict and all header names will be lowercase. :raises ClientException: HTTP GET request failed """ + close_conn = False if http_conn: parsed, conn = http_conn else: parsed, conn = http_connection(url) + close_conn = True path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) if query_string: path += '?' + query_string @@ -1287,9 +1317,12 @@ def get_object(url, token, container, name, http_conn=None, {'headers': headers}, resp, body) raise ClientException.from_response(resp, 'Object GET failed', body) if resp_chunk_size: - object_body = _ObjectBody(resp, resp_chunk_size) + object_body = _ObjectBody(resp, resp_chunk_size, + conn_to_close=conn if close_conn else None) else: object_body = resp.read() + if close_conn: + conn.close() http_log(('%s%s' % (url.replace(parsed.path, ''), path), method,), {'headers': headers}, resp, None) @@ -1313,10 +1346,12 @@ def head_object(url, token, container, name, http_conn=None, be lowercase) :raises ClientException: HTTP HEAD request failed """ + close_conn = False if http_conn: parsed, conn = http_conn else: parsed, conn = http_connection(url) + close_conn = True path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) if query_string: path += '?' + query_string @@ -1331,6 +1366,8 @@ def head_object(url, token, container, name, http_conn=None, conn.request(method, path, '', headers) resp = conn.getresponse() body = resp.read() + if close_conn: + conn.close() http_log(('%s%s' % (url.replace(parsed.path, ''), path), method,), {'headers': headers}, resp, body) if resp.status < 200 or resp.status >= 300: @@ -1380,10 +1417,12 @@ def put_object(url, token=None, container=None, name=None, contents=None, :returns: etag :raises ClientException: HTTP PUT request failed """ + close_conn = False if http_conn: parsed, conn = http_conn else: parsed, conn = http_connection(url, proxy=proxy) + close_conn = True path = parsed.path if container: path = '%s/%s' % (path.rstrip('/'), quote(container)) @@ -1442,6 +1481,8 @@ def put_object(url, token=None, container=None, name=None, contents=None, resp = conn.getresponse() body = resp.read() + if close_conn: + conn.close() http_log(('%s%s' % (url.replace(parsed.path, ''), path), 'PUT',), {'headers': headers}, resp, body) @@ -1471,10 +1512,12 @@ def post_object(url, token, container, name, headers, http_conn=None, :param service_token: service auth token :raises ClientException: HTTP POST request failed """ + close_conn = False if http_conn: parsed, conn = http_conn else: parsed, conn = http_connection(url) + close_conn = True path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) req_headers = {'X-Auth-Token': token} if service_token: @@ -1484,6 +1527,8 @@ def post_object(url, token, container, name, headers, http_conn=None, conn.request('POST', path, '', req_headers) resp = conn.getresponse() body = resp.read() + if close_conn: + conn.close() http_log(('%s%s' % (url.replace(parsed.path, ''), path), 'POST',), {'headers': req_headers}, resp, body) @@ -1516,10 +1561,12 @@ def copy_object(url, token, container, name, destination=None, :param service_token: service auth token :raises ClientException: HTTP COPY request failed """ + close_conn = False if http_conn: parsed, conn = http_conn else: parsed, conn = http_connection(url) + close_conn = True path = parsed.path container = quote(container) @@ -1548,6 +1595,8 @@ def copy_object(url, token, container, name, destination=None, conn.request('COPY', path, '', headers) resp = conn.getresponse() body = resp.read() + if close_conn: + conn.close() http_log(('%s%s' % (url.replace(parsed.path, ''), path), 'COPY',), {'headers': headers}, resp, body) @@ -1580,10 +1629,12 @@ def delete_object(url, token=None, container=None, name=None, http_conn=None, :param service_token: service auth token :raises ClientException: HTTP DELETE request failed """ + close_conn = False if http_conn: parsed, conn = http_conn else: parsed, conn = http_connection(url, proxy=proxy) + close_conn = True path = parsed.path if container: path = '%s/%s' % (path.rstrip('/'), quote(container)) @@ -1602,6 +1653,8 @@ def delete_object(url, token=None, container=None, name=None, http_conn=None, conn.request('DELETE', path, '', headers) resp = conn.getresponse() body = resp.read() + if close_conn: + conn.close() http_log(('%s%s' % (url.replace(parsed.path, ''), path), 'DELETE',), {'headers': headers}, resp, body) diff --git a/test/unit/test_swiftclient.py b/test/unit/test_swiftclient.py index dfd79c77..bfeb61b9 100644 --- a/test/unit/test_swiftclient.py +++ b/test/unit/test_swiftclient.py @@ -785,6 +785,7 @@ def test_ok(self): self.assertRequests([ ('HEAD', 'http://www.tests.com', '', {'x-auth-token': 'asdf'}) ]) + self.assertTrue(self.request_log[-1][-1]._closed) def test_server_error(self): body = 'c' * 65 diff --git a/test/unit/utils.py b/test/unit/utils.py index 025a2342..3190e9d2 100644 --- a/test/unit/utils.py +++ b/test/unit/utils.py @@ -109,6 +109,7 @@ def __init__(self, status, etag=None, body='', timestamp='1', self.timestamp = timestamp self.headers = headers or {} self.request = None + self._closed = False def getresponse(self): if kwargs.get('raise_exc'): @@ -167,7 +168,7 @@ def getheader(self, name, default=None): return dict(self.getheaders()).get(name.lower(), default) def close(self): - pass + self._closed = True timestamps_iter = iter(kwargs.get('timestamps') or ['1'] * len(code_iter)) etag_iter = iter(kwargs.get('etags') or [None] * len(code_iter)) @@ -248,7 +249,8 @@ def wrapper(url, proxy=None, cacert=None, insecure=False, class RequestsWrapper(object): def close(self): - pass + if hasattr(self, 'resp'): + self.resp.close() conn = RequestsWrapper() def request(method, path, *args, **kwargs): From de5c74069e9fe0e7ae6b696bbd2900a7c853e960 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Thu, 15 Oct 2020 10:13:21 -0700 Subject: [PATCH 119/238] Remove some py38 job cruft For a time, we wanted to flag it as being voting while it still wasn't voting for most of OpenStack. It's not needed now, though. Change-Id: Idccb731a0814335fc4d314eed5caa1f336212b22 --- .zuul.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index 480c1a81..86654ba1 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -47,15 +47,11 @@ - swiftclient-swift-functional - swiftclient-functional - swiftclient-functional-py2 - - openstack-tox-py38: - voting: true gate: jobs: - swiftclient-swift-functional - swiftclient-functional - swiftclient-functional-py2 - - openstack-tox-py38: - voting: true post: jobs: - openstack-tox-cover From 74c50dee2de1fdda0d58cc295a70c7ff7d1700e6 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Fri, 3 Apr 2020 09:34:38 -0700 Subject: [PATCH 120/238] Have `delete --all` imply `--versions` for the CLI Change-Id: Id5a6d4cef3d4ed76c897a099a62a4ba3ed8f8dab --- swiftclient/shell.py | 6 ++++-- test/unit/test_shell.py | 5 ++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/swiftclient/shell.py b/swiftclient/shell.py index dbcd437b..7c6f0f74 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -78,8 +78,8 @@ def immediate_exit(signum, frame): for multiple objects. Optional arguments: - -a, --all Delete all containers and objects. - --versions Delete all versions + -a, --all Delete all containers and objects. Implies --versions. + --versions Delete all versions. --leave-segments Do not delete segments of manifest objects. -H, --header Adds a custom request header to use for deleting @@ -132,6 +132,8 @@ def st_delete(parser, args, output_manager, return_parser=False): (options, args) = parse_args(parser, args) args = args[1:] + if options['yes_all']: + options['versions'] = True if (not args and not options['yes_all']) or (args and options['yes_all']): output_manager.error('Usage: %s delete %s\n%s', BASENAME, st_delete_options, diff --git a/test/unit/test_shell.py b/test/unit/test_shell.py index 46ba52ca..8c525d9a 100644 --- a/test/unit/test_shell.py +++ b/test/unit/test_shell.py @@ -1260,8 +1260,7 @@ def test_delete_account(self, connection): [None, [{'name': 'empty_container'}]], [None, []], ] - # N.B: missing --versions flag, version-id gets ignored - # only latest object is deleted + # N.B: --all implies --versions, clear it all out connection.return_value.get_container.side_effect = [ [None, [{'name': 'object'}, {'name': 'obj\xe9ct2'}]], [None, []], @@ -1279,7 +1278,7 @@ def test_delete_account(self, connection): response_dict={}, headers={}), mock.call('container', 'obj\xe9ct2', query_string='', response_dict={}, headers={}), - mock.call('container2', 'object', query_string='', + mock.call('container2', 'object', query_string='version-id=1', response_dict={}, headers={})], any_order=True) self.assertEqual(3, connection.return_value.delete_object.call_count, 'Expected 3 calls but found\n%r' From a5aebc3b9ae39d65ba6c7a744637796700fbad4e Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Mon, 19 Oct 2020 11:05:49 -0700 Subject: [PATCH 121/238] Make py39 voting Also, add trove classifier for py39. Depends-On: https://review.opendev.org/#/c/758813/ Change-Id: I8d33b4dd6af990b09141acd52d36e44e9c871b3b --- .zuul.yaml | 4 ++++ setup.cfg | 1 + 2 files changed, 5 insertions(+) diff --git a/.zuul.yaml b/.zuul.yaml index 86654ba1..3eac7537 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -47,11 +47,15 @@ - swiftclient-swift-functional - swiftclient-functional - swiftclient-functional-py2 + - openstack-tox-py39: + voting: true gate: jobs: - swiftclient-swift-functional - swiftclient-functional - swiftclient-functional-py2 + - openstack-tox-py39: + voting: true post: jobs: - openstack-tox-cover diff --git a/setup.cfg b/setup.cfg index 95801a88..a9a59405 100644 --- a/setup.cfg +++ b/setup.cfg @@ -20,6 +20,7 @@ classifier = Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 [files] packages = From e6876361f079e9ccd526958e005258bdc68273e9 Mon Sep 17 00:00:00 2001 From: zhangboye Date: Sun, 3 Jan 2021 16:44:04 +0800 Subject: [PATCH 122/238] remove unicode from code Change-Id: I791cc993aef832b30c08fdb5bdd7165a074d263f --- doc/source/conf.py | 6 +++--- releasenotes/source/conf.py | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 83816050..2673d6c5 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -53,7 +53,7 @@ master_doc = 'index' # General information about the project. -copyright = u'2013-2016 OpenStack, LLC.' +copyright = '2013-2016 OpenStack, LLC.' # -- Options for openstackdocstheme ------------------------------------------- openstackdocs_repo_name = 'openstack/python-swiftclient' @@ -183,8 +183,8 @@ # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]) latex_documents = [ - ('index', 'doc-python-swiftclient.tex', u'SwiftClient Documentation', - u'OpenStack, LLC.', 'manual'), + ('index', 'doc-python-swiftclient.tex', 'SwiftClient Documentation', + 'OpenStack, LLC.', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of diff --git a/releasenotes/source/conf.py b/releasenotes/source/conf.py index a050a546..0945c81e 100644 --- a/releasenotes/source/conf.py +++ b/releasenotes/source/conf.py @@ -65,7 +65,7 @@ master_doc = 'index' # General information about the project. -copyright = u'%d, OpenStack Foundation' % datetime.datetime.now().year +copyright = '%d, OpenStack Foundation' % datetime.datetime.now().year # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -138,7 +138,7 @@ # The name for this set of Sphinx documents. # " v documentation" by default. # -# html_title = u'swift v2.10.0' +# html_title = 'swift v2.10.0' # A shorter title for the navigation bar. Default is the same as html_title. # @@ -258,8 +258,8 @@ # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). # latex_documents = [ -# (master_doc, 'swift.tex', u'swift Documentation', -# u'swift', 'manual'), +# (master_doc, 'swift.tex', 'swift Documentation', +# 'swift', 'manual'), # ] # The name of an image file (relative to this directory) to place at the top of @@ -300,7 +300,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). # man_pages = [ -# (master_doc, 'swift', u'swift Documentation', +# (master_doc, 'swift', 'swift Documentation', # [author], 1) # ] @@ -315,7 +315,7 @@ # (source start file, target name, title, author, # dir menu entry, description, category) # texinfo_documents = [ -# (master_doc, 'swift', u'swift Documentation', +# (master_doc, 'swift', 'swift Documentation', # author, 'swift', 'One line description of project.', # 'Miscellaneous'), # ] From 06b36ae0e2ee6054162bfb1d9f3cdda39eacbad5 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Mon, 22 Feb 2021 12:21:43 -0800 Subject: [PATCH 123/238] Drop lower-constraints testing The OpenStack community consensus seems to be that it's not worth the hassle of fixing. *Maybe* we can revisit this if we ever drop py2 support? Reasonable spot to start on ML threads: http://lists.openstack.org/pipermail/openstack-discuss/2021-January/019672.html Change-Id: Ibf0c891782afe014cc453b713a94c187340d172e Depends-On: https://review.opendev.org/c/openstack/requirements/+/777025 --- .zuul.yaml | 3 ++- lower-constraints.txt | 42 ------------------------------------------ tox.ini | 7 ------- 3 files changed, 2 insertions(+), 50 deletions(-) delete mode 100644 lower-constraints.txt diff --git a/.zuul.yaml b/.zuul.yaml index 3eac7537..fb27594f 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -23,6 +23,8 @@ vars: # Override value from parent job to use swiftclient tests zuul_work_dir: "{{ zuul.projects['opendev.org/openstack/python-swiftclient'].src_dir }}" + # swift can use different tox env names + tox_envlist: func - job: name: swiftclient-functional-py2 @@ -37,7 +39,6 @@ templates: - check-requirements - lib-forward-testing-python3 - - openstack-lower-constraints-jobs - openstack-python-jobs - openstack-python3-wallaby-jobs - publish-openstack-docs-pti diff --git a/lower-constraints.txt b/lower-constraints.txt deleted file mode 100644 index 28a10600..00000000 --- a/lower-constraints.txt +++ /dev/null @@ -1,42 +0,0 @@ -alabaster==0.7.10 -Babel==2.3.4 -certifi==2018.1.18 -chardet==3.0.4 -coverage==4.0 -docutils==0.11 -dulwich==0.15.0 -extras==1.0.0 -fixtures==3.0.0 -flake8==2.2.4 -futures==3.0.0 -hacking==0.10.0 -idna==2.6 -imagesize==0.7.1 -iso8601==0.1.8 -Jinja2==2.10 -keystoneauth1==3.4.0 -linecache2==1.0.0 -MarkupSafe==1.0 -mccabe==0.2.1 -mock==1.2.0 -netaddr==0.7.10 -openstacksdk==0.11.0 -oslo.config==1.2.0 -pbr==2.0.0 -pep8==1.5.7 -PrettyTable==0.7.1 -pyflakes==0.8.1 -Pygments==2.2.0 -python-keystoneclient==0.7.0 -python-mimeparse==1.6.0 -python-subunit==1.0.0 -pytz==2013.6 -PyYAML==3.12 -requests==1.1.0 -six==1.9.0 -snowballstemmer==1.2.1 -stestr==2.0.0 -testtools==2.2.0 -traceback2==1.4.0 -unittest2==1.1.0 -urllib3==1.22 diff --git a/tox.ini b/tox.ini index 41314364..0837a063 100644 --- a/tox.ini +++ b/tox.ini @@ -102,13 +102,6 @@ usedevelop = False deps = -r{toxinidir}/doc/requirements.txt commands = sphinx-build -a -W -E -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html -[testenv:lower-constraints] -basepython = python3 -deps = - -c{toxinidir}/lower-constraints.txt - -r{toxinidir}/test-requirements.txt - .[keystone] - [testenv:pdf-docs] basepython = python3 deps = {[testenv:docs]deps} From c8b48dee58ef25b3ff86b47f4194a197d1bc4cdd Mon Sep 17 00:00:00 2001 From: OpenStack Release Bot Date: Thu, 18 Mar 2021 11:11:36 +0000 Subject: [PATCH 124/238] Update master for stable/wallaby Add file to the reno documentation build to show release notes for stable/wallaby. Use pbr instruction to increment the minor version number automatically so that master versions are higher than the versions on stable/wallaby. Sem-Ver: feature Change-Id: Idcc50c36c1fac81ed214bcbaf6a97307e3db7ee7 --- releasenotes/source/index.rst | 1 + releasenotes/source/wallaby.rst | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 releasenotes/source/wallaby.rst diff --git a/releasenotes/source/index.rst b/releasenotes/source/index.rst index a63715f1..52c38140 100644 --- a/releasenotes/source/index.rst +++ b/releasenotes/source/index.rst @@ -6,6 +6,7 @@ :maxdepth: 1 current + wallaby victoria ussuri train diff --git a/releasenotes/source/wallaby.rst b/releasenotes/source/wallaby.rst new file mode 100644 index 00000000..d77b5659 --- /dev/null +++ b/releasenotes/source/wallaby.rst @@ -0,0 +1,6 @@ +============================ +Wallaby Series Release Notes +============================ + +.. release-notes:: + :branch: stable/wallaby From 6966fbea57e9fefbd944ed080db384b6c2a9a390 Mon Sep 17 00:00:00 2001 From: OpenStack Release Bot Date: Thu, 18 Mar 2021 11:11:42 +0000 Subject: [PATCH 125/238] Add Python3 xena unit tests This is an automatically generated patch to ensure unit testing is in place for all the of the tested runtimes for xena. See also the PTI in governance [1]. [1]: https://governance.openstack.org/tc/reference/project-testing-interface.html Change-Id: Ic272b2583f9e0ff5f1b542ce9596af9d3bf8edef --- .zuul.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.zuul.yaml b/.zuul.yaml index fb27594f..7badb0ec 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -40,7 +40,7 @@ - check-requirements - lib-forward-testing-python3 - openstack-python-jobs - - openstack-python3-wallaby-jobs + - openstack-python3-xena-jobs - publish-openstack-docs-pti - release-notes-jobs-python3 check: From 998bb8b578019df3a7b229eac806287f3fd32069 Mon Sep 17 00:00:00 2001 From: zhangboye Date: Tue, 20 Apr 2021 15:11:00 +0800 Subject: [PATCH 126/238] Use py3 as the default runtime for tox Moving on py3 as the default runtime for tox to avoid to update this at each new cycle. Change-Id: I4a54455e7e8b1b4de2f9656d13f65ef090da68eb --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 0837a063..10e7b8d4 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,py38,pep8 +envlist = py27,py3,pep8 minversion = 2.0 skipsdist = True From 58f0700ad93c4cfaf3f924fa3ca1c405470d93d6 Mon Sep 17 00:00:00 2001 From: jonasdlindner Date: Thu, 29 Apr 2021 17:39:28 +0200 Subject: [PATCH 127/238] Fix Typo in shell.py Change-Id: I2615e0d6b54d8cc020c24d1b5b4064e038f0934b --- swiftclient/shell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 7c6f0f74..43950e6c 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -2012,7 +2012,7 @@ def main(arguments=None): %(prog)s --os-auth-url https://api.example.com/v3 --auth-version 3\\ --os-application-credential-id d78683c92f0e4f9b9b02a2e208039412 \\ - --os-application-credential-secret APPLICTION_CREDENTIAL_SECRET \\ + --os-application-credential-secret APPLICATION_CREDENTIAL_SECRET \\ --os-auth-type v3applicationcredential list %(prog)s --os-auth-token 6ee5eb33efad4e45ab46806eac010566 \\ From 2beddc2f30c601d1cbbb15cb8c2d69b0776df093 Mon Sep 17 00:00:00 2001 From: yangyawei Date: Mon, 3 May 2021 10:51:06 +0800 Subject: [PATCH 128/238] setup.cfg: Replace dashes with underscores Setuptools v54.1.0 introduces a warning that the use of dash-separated options in 'setup.cfg' will not be supported in a future version [1]. Get ahead of the issue by replacing the dashes with underscores. Without this, we see 'UserWarning' messages like the following on new enough versions of setuptools: UserWarning: Usage of dash-separated 'description-file' will not be supported in future versions. Please use the underscore name 'description_file' instead [1] https://github.com/pypa/setuptools/commit/a2e9ae4cb Change-Id: Ief2c7a217914dc5cacdffc6959ed0585fc6a1225 --- setup.cfg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index a9a59405..c780c388 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,11 +1,11 @@ [metadata] name = python-swiftclient summary = OpenStack Object Storage API Client Library -description-file = +description_file = README.rst author = OpenStack -author-email = openstack-discuss@lists.openstack.org -home-page = https://docs.openstack.org/python-swiftclient/latest/ +author_email = openstack-discuss@lists.openstack.org +home_page = https://docs.openstack.org/python-swiftclient/latest/ classifier = Environment :: OpenStack Intended Audience :: Information Technology From 99b5b81217abdfc4b7a8c388242aaf50c64f1f5b Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Mon, 10 May 2021 22:51:22 -0700 Subject: [PATCH 129/238] Allow unit tests to be run via pytest You can run all tests by just running $ pytest Or just unit tests with $ pytest test/unit/ Or one specific test with $ pytest test/unit/test_swiftclient.py::TestConnection::test_reauth Change-Id: I1dfa239f9ee9ea85663b5c1f22631a97f87b4dfc --- test/unit/test_shell.py | 52 ++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/test/unit/test_shell.py b/test/unit/test_shell.py index 8c525d9a..ef5d1669 100644 --- a/test/unit/test_shell.py +++ b/test/unit/test_shell.py @@ -28,7 +28,6 @@ from time import localtime, mktime, strftime, strptime import six -import sys import swiftclient from swiftclient.service import SwiftError @@ -1155,9 +1154,12 @@ def test_upload_segments_to_same_container(self, connection): 'x-object-meta-mtime': mock.ANY}, response_dict={}) + @mock.patch('swiftclient.shell.stdin') @mock.patch('swiftclient.shell.io.open') @mock.patch('swiftclient.service.SwiftService.upload') - def test_upload_from_stdin(self, upload_mock, io_open_mock): + def test_upload_from_stdin(self, upload_mock, io_open_mock, stdin_mock): + stdin_mock.fileno.return_value = 123 + def fake_open(fd, mode): mock_io = mock.Mock() mock_io.fileno.return_value = fd @@ -1173,8 +1175,8 @@ def fake_open(fd, mode): # element. This is because the upload method takes a container and a # list of SwiftUploadObjects. swift_upload_obj = upload_mock.mock_calls[0][1][1][0] - self.assertEqual(sys.stdin.fileno(), swift_upload_obj.source.fileno()) - io_open_mock.assert_called_once_with(sys.stdin.fileno(), mode='rb') + self.assertEqual(123, swift_upload_obj.source.fileno()) + io_open_mock.assert_called_once_with(123, mode='rb') @mock.patch('swiftclient.service.SwiftService.upload') def test_upload_from_stdin_no_name(self, upload_mock): @@ -3015,24 +3017,30 @@ def _test_options(self, opts, os_opts, flags=None, no_auth=False): no_auth=no_auth) def test_all_args_passed_to_keystone(self): - # check that all possible command line args are passed to keystone - opts = {'auth-version': '3'} - os_opts = dict(self.all_os_opts) - os_opts.update(self.catalog_opts) - self._test_options(opts, os_opts, flags=self.flags) - - opts = {'auth-version': '2.0'} - self._test_options(opts, os_opts, flags=self.flags) - - opts = {} - self.defaults['auth-version'] = '3' - self._test_options(opts, os_opts, flags=self.flags) - - for o in ('user-domain-name', 'user-domain-id', - 'project-domain-name', 'project-domain-id'): - os_opts.pop(o) - self.defaults['auth-version'] = '2.0' - self._test_options(opts, os_opts, flags=self.flags) + rootLogger = logging.getLogger() + orig_lvl = rootLogger.getEffectiveLevel() + try: + rootLogger.setLevel(logging.DEBUG) + # check that all possible command line args are passed to keystone + opts = {'auth-version': '3'} + os_opts = dict(self.all_os_opts) + os_opts.update(self.catalog_opts) + self._test_options(opts, os_opts, flags=self.flags) + + opts = {'auth-version': '2.0'} + self._test_options(opts, os_opts, flags=self.flags) + + opts = {} + self.defaults['auth-version'] = '3' + self._test_options(opts, os_opts, flags=self.flags) + + for o in ('user-domain-name', 'user-domain-id', + 'project-domain-name', 'project-domain-id'): + os_opts.pop(o) + self.defaults['auth-version'] = '2.0' + self._test_options(opts, os_opts, flags=self.flags) + finally: + rootLogger.setLevel(orig_lvl) def test_catalog_options_and_flags_not_required_v3(self): # check that all possible command line args are passed to keystone From 6d8138ebcf3cc7cc3c50cf8c06b58d5369d5c395 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Tue, 11 May 2021 13:42:47 -0700 Subject: [PATCH 130/238] Allow functional tests to pass with etag_quoter enabled by default Change-Id: I861b5e0a172f0ea0a5b1fe8389cd70da8d4b5d5d --- test/functional/test_swiftclient.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/functional/test_swiftclient.py b/test/functional/test_swiftclient.py index aaade879..8d001d0f 100644 --- a/test/functional/test_swiftclient.py +++ b/test/functional/test_swiftclient.py @@ -207,7 +207,7 @@ def test_upload_object(self): hdrs = self.conn.head_object(self.containername, self.objectname) self.assertEqual(str(len(self.test_data)), hdrs.get('content-length')) - self.assertEqual(self.etag, hdrs.get('etag')) + self.assertEqual(self.etag, hdrs.get('etag').strip('"')) self.assertEqual('application/octet-stream', hdrs.get('content-type')) @@ -218,7 +218,7 @@ def test_upload_object(self): hdrs = self.conn.head_object(self.containername, self.objectname) self.assertEqual(str(len(self.test_data)), hdrs.get('content-length')) - self.assertEqual(self.etag, hdrs.get('etag')) + self.assertEqual(self.etag, hdrs.get('etag').strip('"')) self.assertEqual('text/plain', hdrs.get('content-type')) @@ -229,7 +229,7 @@ def test_upload_object(self): hdrs = self.conn.head_object(self.containername, self.objectname) self.assertEqual(str(len(self.test_data)), hdrs.get('content-length')) - self.assertEqual(self.etag, hdrs.get('etag')) + self.assertEqual(self.etag, hdrs.get('etag').strip('"')) self.assertEqual('text/plain', hdrs.get('content-type')) @@ -241,7 +241,7 @@ def test_upload_object(self): hdrs = self.conn.head_object(self.containername, self.objectname) self.assertEqual(str(len(self.test_data)), hdrs.get('content-length')) - self.assertEqual(self.etag, hdrs.get('etag')) + self.assertEqual(self.etag, hdrs.get('etag').strip('"')) self.assertEqual('image/jpeg', hdrs.get('content-type')) @@ -252,7 +252,7 @@ def test_upload_object(self): hdrs = self.conn.head_object(self.containername, self.objectname) self.assertEqual(str(len(self.test_data)), hdrs.get('content-length')) - self.assertEqual(self.etag, hdrs.get('etag')) + self.assertEqual(self.etag, hdrs.get('etag').strip('"')) self.assertEqual('application/octet-stream', hdrs.get('content-type')) # Content from File-like object @@ -262,7 +262,7 @@ def test_upload_object(self): hdrs = self.conn.head_object(self.containername, self.objectname) self.assertEqual(str(len(self.test_data)), hdrs.get('content-length')) - self.assertEqual(self.etag, hdrs.get('etag')) + self.assertEqual(self.etag, hdrs.get('etag').strip('"')) self.assertEqual('application/octet-stream', hdrs.get('content-type')) # Content from File-like object, but read in chunks @@ -274,7 +274,7 @@ def test_upload_object(self): hdrs = self.conn.head_object(self.containername, self.objectname) self.assertEqual(str(len(self.test_data)), hdrs.get('content-length')) - self.assertEqual(self.etag, hdrs.get('etag')) + self.assertEqual(self.etag, hdrs.get('etag').strip('"')) self.assertEqual('application/octet-stream', hdrs.get('content-type')) # Wrong etag arg, should raise an exception From abda44a87dd2bdee23edd69d78df7259017162d9 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Tue, 25 May 2021 11:01:01 -0700 Subject: [PATCH 131/238] Use upper-constraints for docs jobs Looks like there are some issues with too-new-Sphinx and the PDF docs builds. Also, get devstack installing on bionic again for our py2 func test job. Change-Id: I633398054694fe6ba1e0de50278f274daf69fefd --- .zuul.yaml | 4 ++++ tox.ini | 6 ++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index 7badb0ec..a389849a 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -33,6 +33,10 @@ description: | Run functional tests of python-swiftclient under Python 2 vars: + devstack_localrc: + # devstack dropped support for bionic, but we want it for easier py2 support. + # Set this so we install anyway. + FORCE: "yes" tox_envlist: py2func - project: diff --git a/tox.ini b/tox.ini index 10e7b8d4..287a244c 100644 --- a/tox.ini +++ b/tox.ini @@ -65,7 +65,8 @@ commands = {[testenv:func]commands} [testenv:docs] basepython = python3 usedevelop = False -deps = -r{toxinidir}/doc/requirements.txt +deps = -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} + -r{toxinidir}/doc/requirements.txt commands= sphinx-build -W -b html doc/source doc/build/html -W @@ -99,7 +100,8 @@ commands = bindep test [testenv:releasenotes] basepython = python3 usedevelop = False -deps = -r{toxinidir}/doc/requirements.txt +deps = -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} + -r{toxinidir}/doc/requirements.txt commands = sphinx-build -a -W -E -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html [testenv:pdf-docs] From b846071d2fbc22e0fc2f471d8f2e0bc8c912ff1f Mon Sep 17 00:00:00 2001 From: "wu.shiming" Date: Tue, 6 Jul 2021 16:10:09 +0800 Subject: [PATCH 132/238] Changed minversion in tox to 3.18.0 The patch bumps min version of tox to 3.18.0 in order to replace tox's whitelist_externals by allowlist_externals option: https://github.com/tox-dev/tox/blob/master/docs/changelog.rst#v3180-2020-07-23 Change-Id: I244d98bb3fc7cb75624b598f4d26f784159f5428 --- tox.ini | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tox.ini b/tox.ini index 287a244c..fbc58c78 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] envlist = py27,py3,pep8 -minversion = 2.0 +minversion = 3.18.0 skipsdist = True [testenv] @@ -17,7 +17,7 @@ commands = sh -c '(find . -not \( -type d -name .?\* -prune \) \ \( -type d -name "__pycache__" -or -type f -name "*.py[co]" \) \ -print0) | xargs -0 rm -rf' stestr run {posargs} -whitelist_externals = sh +allowlist_externals = sh passenv = SWIFT_* *_proxy [testenv:pep8] @@ -45,7 +45,7 @@ basepython = python3 setenv = OS_TEST_PATH=test.functional PYTHON=coverage run --source swiftclient --parallel-mode -whitelist_externals = +allowlist_externals = coverage rm commands = @@ -59,7 +59,7 @@ commands = [testenv:py2func] basepython=python2 setenv = {[testenv:func]setenv} -whitelist_externals = {[testenv:func]whitelist_externals} +allowlist_externals = {[testenv:func]allowlist_externals} commands = {[testenv:func]commands} [testenv:docs] @@ -107,7 +107,7 @@ commands = sphinx-build -a -W -E -d releasenotes/build/doctrees -b html releasen [testenv:pdf-docs] basepython = python3 deps = {[testenv:docs]deps} -whitelist_externals = +allowlist_externals = make commands = sphinx-build -W -b latex doc/source doc/build/pdf From afa2c4642a467862a34dea48dc2bd4c9203c6a2a Mon Sep 17 00:00:00 2001 From: jinyuanliu Date: Wed, 15 Sep 2021 01:54:42 -0400 Subject: [PATCH 133/238] Clean up extra spaces Although these errors are not important, they affect the code specification. Change-Id: Ifab29d9c803a78ee994b99b4b410893864b90908 --- run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_tests.sh b/run_tests.sh index 39ce1915..6991cd1d 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -18,7 +18,7 @@ command -v tox > /dev/null 2>&1 if [ $? -ne 0 ]; then echo 'This script requires "tox" to run.' echo 'You can install it with "pip install tox".' - exit 1; + exit 1; fi just_pep8=0 From 218676a9905f1fb4d96214831d98c1b2e3924071 Mon Sep 17 00:00:00 2001 From: OpenStack Release Bot Date: Fri, 10 Sep 2021 15:16:54 +0000 Subject: [PATCH 134/238] Update master for stable/xena Add file to the reno documentation build to show release notes for stable/xena. Use pbr instruction to increment the minor version number automatically so that master versions are higher than the versions on stable/xena. Sem-Ver: feature Change-Id: I79a7fbda1ea9ae034d009c54be4b401809183c42 --- releasenotes/source/index.rst | 1 + releasenotes/source/xena.rst | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 releasenotes/source/xena.rst diff --git a/releasenotes/source/index.rst b/releasenotes/source/index.rst index 52c38140..fb60ee00 100644 --- a/releasenotes/source/index.rst +++ b/releasenotes/source/index.rst @@ -6,6 +6,7 @@ :maxdepth: 1 current + xena wallaby victoria ussuri diff --git a/releasenotes/source/xena.rst b/releasenotes/source/xena.rst new file mode 100644 index 00000000..1be85be3 --- /dev/null +++ b/releasenotes/source/xena.rst @@ -0,0 +1,6 @@ +========================= +Xena Series Release Notes +========================= + +.. release-notes:: + :branch: stable/xena From bcf19d47a3e040f3af1c63ebbfd81f372e784cb3 Mon Sep 17 00:00:00 2001 From: OpenStack Release Bot Date: Fri, 10 Sep 2021 15:16:57 +0000 Subject: [PATCH 135/238] Add Python3 yoga unit tests This is an automatically generated patch to ensure unit testing is in place for all the of the tested runtimes for yoga. See also the PTI in governance [1]. [1]: https://governance.openstack.org/tc/reference/project-testing-interface.html Change-Id: Icf192aedeadf7fbed6e0f217da5b348172dc4478 --- .zuul.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.zuul.yaml b/.zuul.yaml index a389849a..e26260a8 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -44,7 +44,7 @@ - check-requirements - lib-forward-testing-python3 - openstack-python-jobs - - openstack-python3-xena-jobs + - openstack-python3-yoga-jobs - publish-openstack-docs-pti - release-notes-jobs-python3 check: From 553e34ebfe891fe234abd722ceeba0809775a175 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Mon, 20 Sep 2021 12:33:16 -0700 Subject: [PATCH 136/238] Improve formatting for billions of objects Change-Id: If8aa08c4c8c8ad6ca2c861602baf1eefa8642a8a --- swiftclient/shell.py | 4 ++-- test/unit/test_shell.py | 16 +++++++++------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 43950e6c..cf90ffcf 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -552,7 +552,7 @@ def _print_stats(options, stats, human): datestamp = '????-??-?? ??:??:??' if not options['totals']: output_manager.print_msg( - "%5s %s %s %s", count, byte_str, + "%12s %s %s %s", count, byte_str, datestamp, item_name) else: # list container contents subdir = item.get('subdir') @@ -584,7 +584,7 @@ def _print_stats(options, stats, human): if options['long'] or human: if not container: output_manager.print_msg( - "%5s %s", prt_bytes(total_count, True), + "%12s %s", prt_bytes(total_count, True), prt_bytes(total_bytes, human)) else: output_manager.print_msg( diff --git a/test/unit/test_shell.py b/test/unit/test_shell.py index 8c525d9a..84dd681e 100644 --- a/test/unit/test_shell.py +++ b/test/unit/test_shell.py @@ -475,9 +475,10 @@ def test_list_account_long(self, connection): mock.call(marker='container', prefix=None, headers={})] connection.return_value.get_account.assert_has_calls(calls) - self.assertEqual(output.out, - ' 0 0 1970-01-01 00:00:01 container\n' - ' 0 0\n') + self.assertEqual( + output.out, + ' 0 0 1970-01-01 00:00:01 container\n' + ' 0 0\n') # Now test again, this time without returning metadata connection.return_value.head_container.return_value = {} @@ -495,9 +496,10 @@ def test_list_account_long(self, connection): mock.call(marker='container', prefix=None, headers={})] connection.return_value.get_account.assert_has_calls(calls) - self.assertEqual(output.out, - ' 0 0 ????-??-?? ??:??:?? container\n' - ' 0 0\n') + self.assertEqual( + output.out, + ' 0 0 ????-??-?? ??:??:?? container\n' + ' 0 0\n') def test_list_account_totals_error(self): # No --lh provided: expect info message about incorrect --totals use @@ -523,7 +525,7 @@ def test_list_account_totals(self, connection): swiftclient.shell.main(argv) calls = [mock.call(marker='', prefix=None, headers={})] connection.return_value.get_account.assert_has_calls(calls) - self.assertEqual(output.out, ' 6 3\n') + self.assertEqual(output.out, ' 6 3\n') @mock.patch('swiftclient.service.Connection') def test_list_container(self, connection): From 5129b33505691f5c4a3e33f2878b157cbd5b4a62 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Mon, 20 Sep 2021 12:35:37 -0700 Subject: [PATCH 137/238] Only log the traceback for non-404s Change-Id: I08ba4a3120e99b444b13f1ca6f5493529868df26 --- swiftclient/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swiftclient/service.py b/swiftclient/service.py index cd96a5b9..8e2c7b00 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -1036,8 +1036,8 @@ def _list_container_job(conn, container, options, result_queue): version_marker = items[-1].get('version_id', '') except ClientException as err: traceback, err_time = report_traceback() - logger.exception(err) if err.http_status != 404: + logger.exception(err) error = (err, traceback, err_time) else: error = ( From ad3e8e49d072d8137c85c451db427d07e1301799 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Thu, 8 Apr 2021 17:16:05 -0700 Subject: [PATCH 138/238] Include storage policy when listing account with --long Change-Id: Ibc2f9445b5a8e80cfb73d0706e20a7e4c62eec4a --- swiftclient/shell.py | 5 +++-- test/unit/test_shell.py | 20 ++++++++++++-------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/swiftclient/shell.py b/swiftclient/shell.py index cf90ffcf..18782720 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -550,10 +550,11 @@ def _print_stats(options, stats, human): datestamp = strftime('%Y-%m-%d %H:%M:%S', utc) except TypeError: datestamp = '????-??-?? ??:??:??' + storage_policy = meta.get('x-storage-policy', '???') if not options['totals']: output_manager.print_msg( - "%12s %s %s %s", count, byte_str, - datestamp, item_name) + "%12s %s %s %-15s %s", count, byte_str, + datestamp, storage_policy, item_name) else: # list container contents subdir = item.get('subdir') content_type = item.get('content_type') diff --git a/test/unit/test_shell.py b/test/unit/test_shell.py index 84dd681e..0cf2258d 100644 --- a/test/unit/test_shell.py +++ b/test/unit/test_shell.py @@ -467,6 +467,10 @@ def test_list_account_long(self, connection): [None, [{'name': 'container', 'bytes': 0, 'count': 0}]], [None, []], ] + connection.return_value.head_container.return_value = { + 'x-timestamp': '1617393213.49752', + 'x-storage-policy': 'some-policy', + } argv = ["", "list", "--lh"] with CaptureOutput() as output: @@ -475,10 +479,10 @@ def test_list_account_long(self, connection): mock.call(marker='container', prefix=None, headers={})] connection.return_value.get_account.assert_has_calls(calls) - self.assertEqual( - output.out, - ' 0 0 1970-01-01 00:00:01 container\n' - ' 0 0\n') + self.assertEqual( + output.out, + ' 0 0 2021-04-02 19:53:33 some-policy container\n' + ' 0 0\n') # Now test again, this time without returning metadata connection.return_value.head_container.return_value = {} @@ -496,10 +500,10 @@ def test_list_account_long(self, connection): mock.call(marker='container', prefix=None, headers={})] connection.return_value.get_account.assert_has_calls(calls) - self.assertEqual( - output.out, - ' 0 0 ????-??-?? ??:??:?? container\n' - ' 0 0\n') + self.assertEqual( + output.out, + ' 0 0 ????-??-?? ??:??:?? ??? container\n' + ' 0 0\n') def test_list_account_totals_error(self): # No --lh provided: expect info message about incorrect --totals use From 373fa26ce1620d096918bd34ea4983b8ca96898c Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Tue, 21 Sep 2021 16:02:47 -0700 Subject: [PATCH 139/238] Correctly aggregate totals for >10k items Previously, we would write out totals for every page of listings, like $ swift list sync --prefix=09-21 --total -l 80000000000 80000000000 80000000000 58096000000 Now, roll those all into a single total: $ swift list sync --prefix=09-21 --total -l 298096000000 Change-Id: Icc265636815220e33e8c9eec0a3ab80e9f899038 --- swiftclient/shell.py | 32 +++++++++++++++++--------------- test/unit/test_shell.py | 13 +++++++++---- 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/swiftclient/shell.py b/swiftclient/shell.py index cf90ffcf..fed0ef95 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -531,8 +531,7 @@ def st_download(parser, args, output_manager, return_parser=False): def st_list(parser, args, output_manager, return_parser=False): - def _print_stats(options, stats, human): - total_count = total_bytes = 0 + def _print_stats(options, stats, human, totals): container = stats.get("container", None) for item in stats["listing"]: item_name = item.get('name') @@ -543,7 +542,7 @@ def _print_stats(options, stats, human): item_bytes = item.get('bytes') byte_str = prt_bytes(item_bytes, human) count = item.get('count') - total_count += count + totals['count'] += count try: meta = item.get('meta') utc = gmtime(float(meta.get('x-timestamp'))) @@ -578,17 +577,7 @@ def _print_stats(options, stats, human): output_manager.print_msg( "%s %10s %8s %24s %s", byte_str, date, xtime, content_type, item_name) - total_bytes += item_bytes - - # report totals - if options['long'] or human: - if not container: - output_manager.print_msg( - "%12s %s", prt_bytes(total_count, True), - prt_bytes(total_bytes, human)) - else: - output_manager.print_msg( - prt_bytes(total_bytes, human)) + totals['bytes'] += item_bytes parser.add_argument( '-l', '--long', dest='long', action='store_true', default=False, @@ -642,6 +631,7 @@ def _print_stats(options, stats, human): try: if not args: stats_parts_gen = swift.list() + container = None else: container = args[0] args = args[1:] @@ -667,12 +657,24 @@ def listing(stats_parts_gen=stats_parts_gen): sort_keys=True, indent=2) output_manager.print_msg('') return + + totals = {'count': 0, 'bytes': 0} for stats in stats_parts_gen: if stats["success"]: - _print_stats(options, stats, human) + _print_stats(options, stats, human, totals) else: raise stats["error"] + # report totals + if options['long'] or human: + if container is None: + output_manager.print_msg( + "%12s %s", prt_bytes(totals['count'], True), + prt_bytes(totals['bytes'], human)) + else: + output_manager.print_msg( + prt_bytes(totals['bytes'], human)) + except SwiftError as e: output_manager.error(e.value) diff --git a/test/unit/test_shell.py b/test/unit/test_shell.py index 84dd681e..84793ae5 100644 --- a/test/unit/test_shell.py +++ b/test/unit/test_shell.py @@ -547,9 +547,12 @@ def test_list_container(self, connection): self.assertEqual(output.out, 'object_a\n') - # Test container listing with --long + # Test container listing with --long and multiple pages connection.return_value.get_container.side_effect = [ - [None, [{'name': 'object_a', 'bytes': 0, + [None, [{'name': 'object_a', 'bytes': 3, + 'content_type': 'type/content', + 'last_modified': '123T456'}]], + [None, [{'name': 'object_b', 'bytes': 5, 'content_type': 'type/content', 'last_modified': '123T456'}]], [None, []], @@ -567,9 +570,11 @@ def test_list_container(self, connection): connection.return_value.get_container.assert_has_calls(calls) self.assertEqual(output.out, - ' 0 123 456' + ' 3 123 456' ' type/content object_a\n' - ' 0\n') + ' 5 123 456' + ' type/content object_b\n' + ' 8\n') @mock.patch('swiftclient.service.Connection') def test_list_container_with_headers(self, connection): From f1858d89e0e1889664ced654755c508f47a0c1f3 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Tue, 11 Jan 2022 16:05:39 -0800 Subject: [PATCH 140/238] Add option to skip container PUT during upload Currently, a user with read/write access to a container (but without access to creat new containers) recieves a warning every time they upload. Now, allow them to avoid the extra request and warning by specifying --skip-container-put on the command line. This is also useful when testing: developers can HEAD a container to ensure it's in memcache, shut down all container servers, then upload and creaate a bunch of async pendings. Previously, the 503 on container PUT would prevent the object upload from even being attempted. Closes-Bug: 1317956 Related-Bug: 1204558 Change-Id: I3d9129a0b6b65c6c6187ae6af003b221afceef47 Related-Change: If1f8a02ee7459ea2158ffa6e958f67d299ec529e --- swiftclient/service.py | 95 ++++++++++++++++++++++------------------- swiftclient/shell.py | 13 ++++-- test/unit/test_shell.py | 42 ++++++++++++++++++ 3 files changed, 102 insertions(+), 48 deletions(-) diff --git a/swiftclient/service.py b/swiftclient/service.py index 8e2c7b00..685b7482 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -202,6 +202,7 @@ def _build_default_global_options(): 'leave_segments': False, 'changed': None, 'skip_identical': False, + 'skip_container_put': False, 'version_id': None, 'yes_all': False, 'read_acl': None, @@ -1462,6 +1463,7 @@ def upload(self, container, objects, options=None): 'leave_segments': False, 'changed': None, 'skip_identical': False, + 'skip_container_put': False, 'fail_fast': False, 'dir_marker': False # Only for None sources } @@ -1487,54 +1489,57 @@ def upload(self, container, objects, options=None): # the object name. (same as passing --object-name). container, _sep, pseudo_folder = container.partition('/') - # Try to create the container, just in case it doesn't exist. If this - # fails, it might just be because the user doesn't have container PUT - # permissions, so we'll ignore any error. If there's really a problem, - # it'll surface on the first object PUT. - policy_header = {} - _header = split_headers(options["header"]) - if POLICY in _header: - policy_header[POLICY] = \ - _header[POLICY] - create_containers = [ - self.thread_manager.container_pool.submit( - self._create_container_job, container, headers=policy_header) - ] + if not options['skip_container_put']: + # Try to create the container, just in case it doesn't exist. If + # this fails, it might just be because the user doesn't have + # container PUT permissions, so we'll ignore any error. If there's + # really a problem, it'll surface on the first object PUT. + policy_header = {} + _header = split_headers(options["header"]) + if POLICY in _header: + policy_header[POLICY] = \ + _header[POLICY] + create_containers = [ + self.thread_manager.container_pool.submit( + self._create_container_job, container, + headers=policy_header) + ] - # wait for first container job to complete before possibly attempting - # segment container job because segment container job may attempt - # to HEAD the first container - for r in interruptable_as_completed(create_containers): - res = r.result() - yield res + # wait for first container job to complete before possibly + # attempting segment container job because segment container job + # may attempt to HEAD the first container + for r in interruptable_as_completed(create_containers): + res = r.result() + yield res - if segment_size: - seg_container = container + '_segments' - if options['segment_container']: - seg_container = options['segment_container'] - if seg_container != container: - if not policy_header: - # Since no storage policy was specified on the command - # line, rather than just letting swift pick the default - # storage policy, we'll try to create the segments - # container with the same policy as the upload container - create_containers = [ - self.thread_manager.container_pool.submit( - self._create_container_job, seg_container, - policy_source=container - ) - ] - else: - create_containers = [ - self.thread_manager.container_pool.submit( - self._create_container_job, seg_container, - headers=policy_header - ) - ] + if segment_size: + seg_container = container + '_segments' + if options['segment_container']: + seg_container = options['segment_container'] + if seg_container != container: + if not policy_header: + # Since no storage policy was specified on the command + # line, rather than just letting swift pick the default + # storage policy, we'll try to create the segments + # container with the same policy as the upload + # container + create_containers = [ + self.thread_manager.container_pool.submit( + self._create_container_job, seg_container, + policy_source=container + ) + ] + else: + create_containers = [ + self.thread_manager.container_pool.submit( + self._create_container_job, seg_container, + headers=policy_header + ) + ] - for r in interruptable_as_completed(create_containers): - res = r.result() - yield res + for r in interruptable_as_completed(create_containers): + res = r.result() + yield res # We maintain a results queue here and a separate thread to monitor # the futures because we want to get results back from potential diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 6da9d667..76473fd6 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -985,8 +985,9 @@ def st_copy(parser, args, output_manager, return_parser=False): st_upload_options = '''[--changed] [--skip-identical] [--segment-size ] [--segment-container ] [--leave-segments] [--object-threads ] [--segment-threads ] - [--meta ] [--header
] [--use-slo] - [--ignore-checksum] [--object-name ] + [--meta ] [--header
] + [--use-slo] [--ignore-checksum] [--skip-container-put] + [--object-name ] [] [...] ''' @@ -1032,11 +1033,13 @@ def st_copy(parser, args, output_manager, return_parser=False): --use-slo When used in conjunction with --segment-size it will create a Static Large Object instead of the default Dynamic Large Object. + --ignore-checksum Turn off checksum validation for uploads. + --skip-container-put Assume all necessary containers already exist; don't + automatically try to create them. --object-name Upload file and name object to or upload dir and use as object prefix instead of folder name. - --ignore-checksum Turn off checksum validation for uploads. '''.strip('\n') @@ -1051,6 +1054,10 @@ def st_upload(parser, args, output_manager, return_parser=False): '--skip-identical', action='store_true', dest='skip_identical', default=False, help='Skip uploading files that are identical on ' 'both sides.') + parser.add_argument( + '--skip-container-put', action='store_true', dest='skip_container_put', + default=False, help='Assume all necessary containers already exist; ' + "don't automatically try to create them.") parser.add_argument( '-S', '--segment-size', dest='segment_size', help='Upload files ' 'in segments no larger than (in Bytes) and then create a ' diff --git a/test/unit/test_shell.py b/test/unit/test_shell.py index 295c918a..2331eaa0 100644 --- a/test/unit/test_shell.py +++ b/test/unit/test_shell.py @@ -912,6 +912,48 @@ def test_upload(self, connection, walk): query_string='multipart-manifest=put', response_dict=mock.ANY) + @mock.patch('swiftclient.shell.walk') + @mock.patch('swiftclient.service.Connection') + def test_upload_skip_container_put(self, connection, walk): + connection.return_value.head_object.return_value = { + 'content-length': '0'} + connection.return_value.put_object.return_value = EMPTY_ETAG + connection.return_value.attempts = 0 + argv = ["", "upload", "container", "--skip-container-put", + self.tmpfile, "-H", "X-Storage-Policy:one", + "--meta", "Color:Blue"] + swiftclient.shell.main(argv) + connection.return_value.put_container.assert_not_called() + + connection.return_value.put_object.assert_called_with( + 'container', + self.tmpfile.lstrip('/'), + mock.ANY, + content_length=0, + headers={'x-object-meta-mtime': mock.ANY, + 'X-Storage-Policy': 'one', + 'X-Object-Meta-Color': 'Blue'}, + response_dict={}) + + # Upload in segments + connection.return_value.head_container.return_value = { + 'x-storage-policy': 'one'} + argv = ["", "upload", "container", "--skip-container-put", + self.tmpfile, "-S", "10"] + with open(self.tmpfile, "wb") as fh: + fh.write(b'12345678901234567890') + swiftclient.shell.main(argv) + # Both base and segments container are assumed to exist already + connection.return_value.put_container.assert_not_called() + connection.return_value.put_object.assert_called_with( + 'container', + self.tmpfile.lstrip('/'), + '', + content_length=0, + headers={'x-object-manifest': mock.ANY, + 'x-object-meta-mtime': mock.ANY}, + response_dict={}) + @mock.patch('swiftclient.service.SwiftService.upload') def test_upload_object_with_account_readonly(self, upload): argv = ["", "upload", "container", self.tmpfile] From 4989d94663b59eac34b7cd5dfaa7e673fc73e862 Mon Sep 17 00:00:00 2001 From: PAPAMICA Date: Wed, 12 Jan 2022 09:57:05 +0100 Subject: [PATCH 141/238] Fix copy.py example. Change-Id: Id2a58a085f58b0bf17eda636593bb482d614c245 --- examples/copy.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/copy.py b/examples/copy.py index e928db4e..808cbd53 100644 --- a/examples/copy.py +++ b/examples/copy.py @@ -9,17 +9,17 @@ with SwiftService() as swift: try: - obj = SwiftCopyObject("c", {"Destination": "/cont/d"}) + obj = SwiftCopyObject("c", {"destination": "/cont/d"}) for i in swift.copy( "cont", ["a", "b", obj], - {"meta": ["foo:bar"], "Destination": "/cc"}): + {"meta": ["foo:bar"], "destination": "/cc"}): if i["success"]: if i["action"] == "copy_object": print( "object %s copied from /%s/%s" % (i["destination"], i["container"], i["object"]) ) - if i["action"] == "create_container": + elif i["action"] == "create_container": print( "container %s created" % i["container"] ) From c2d4fc70540eadd665097945b1376d0ddf88c2b0 Mon Sep 17 00:00:00 2001 From: OpenStack Release Bot Date: Fri, 4 Mar 2022 17:17:25 +0000 Subject: [PATCH 142/238] Update master for stable/yoga Add file to the reno documentation build to show release notes for stable/yoga. Use pbr instruction to increment the minor version number automatically so that master versions are higher than the versions on stable/yoga. Sem-Ver: feature Change-Id: I55d45e6f076974a3b44ab8b26adbccef26373f85 --- releasenotes/source/index.rst | 1 + releasenotes/source/yoga.rst | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 releasenotes/source/yoga.rst diff --git a/releasenotes/source/index.rst b/releasenotes/source/index.rst index fb60ee00..f1da9a60 100644 --- a/releasenotes/source/index.rst +++ b/releasenotes/source/index.rst @@ -6,6 +6,7 @@ :maxdepth: 1 current + yoga xena wallaby victoria diff --git a/releasenotes/source/yoga.rst b/releasenotes/source/yoga.rst new file mode 100644 index 00000000..7cd5e908 --- /dev/null +++ b/releasenotes/source/yoga.rst @@ -0,0 +1,6 @@ +========================= +Yoga Series Release Notes +========================= + +.. release-notes:: + :branch: stable/yoga From 00f5b892273a0f88c368b71404006c53bb6ac749 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Wed, 9 Mar 2022 16:50:33 -0800 Subject: [PATCH 143/238] CI: Drop swiftclient-swift-functional-py2 job It got busted somewhere between 2022-01-15 and 2022-02-11; looks like needing to override devstack to install on bionic finally caught up with us. ;-) FWIW, it fails down in stack.sh while trying to pip install grpcio: commands.CommandError: We expect a missing `_needs_stub` attribute from older versions of setuptools. Consider upgrading setuptools. Change-Id: Iff9094e9dd7a3d0bcdb0dee3b08fa5b61c9186d0 --- .zuul.yaml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index e26260a8..bbbdf109 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -26,19 +26,6 @@ # swift can use different tox env names tox_envlist: func -- job: - name: swiftclient-functional-py2 - parent: swiftclient-functional - nodeset: openstack-single-node-bionic - description: | - Run functional tests of python-swiftclient under Python 2 - vars: - devstack_localrc: - # devstack dropped support for bionic, but we want it for easier py2 support. - # Set this so we install anyway. - FORCE: "yes" - tox_envlist: py2func - - project: templates: - check-requirements @@ -51,14 +38,12 @@ jobs: - swiftclient-swift-functional - swiftclient-functional - - swiftclient-functional-py2 - openstack-tox-py39: voting: true gate: jobs: - swiftclient-swift-functional - swiftclient-functional - - swiftclient-functional-py2 - openstack-tox-py39: voting: true post: From 22a05b2039d0178b52fa3546de8ab265df112636 Mon Sep 17 00:00:00 2001 From: OpenStack Release Bot Date: Fri, 4 Mar 2022 17:17:27 +0000 Subject: [PATCH 144/238] Add Python3 zed unit tests This is an automatically generated patch to ensure unit testing is in place for all the of the tested runtimes for zed. See also the PTI in governance [1]. [1]: https://governance.openstack.org/tc/reference/project-testing-interface.html Change-Id: Ia832ebda47bb798876edf3c013e9b9a583405c11 --- .zuul.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.zuul.yaml b/.zuul.yaml index bbbdf109..92889dcd 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -31,7 +31,7 @@ - check-requirements - lib-forward-testing-python3 - openstack-python-jobs - - openstack-python3-yoga-jobs + - openstack-python3-zed-jobs - publish-openstack-docs-pti - release-notes-jobs-python3 check: From 2636965f38a7788bbee19fc90088384834670b10 Mon Sep 17 00:00:00 2001 From: Stephen Finucane Date: Thu, 17 Feb 2022 11:21:40 +0000 Subject: [PATCH 145/238] Drop support for Python 2 There's a lot of cleanup possible, but this is a start. Signed-off-by: Stephen Finucane Change-Id: Ia1176b7fd5434d52070d482a37abfbb98800cdb3 --- .zuul.yaml | 1 - requirements.txt | 2 +- run_tests.sh | 4 ++-- setup.cfg | 8 +++----- setup.py | 9 ++------- swiftclient/client.py | 18 ++---------------- test/unit/test_swiftclient.py | 28 +--------------------------- tox.ini | 16 +--------------- 8 files changed, 12 insertions(+), 74 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index 92889dcd..d38f37fa 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -30,7 +30,6 @@ templates: - check-requirements - lib-forward-testing-python3 - - openstack-python-jobs - openstack-python3-zed-jobs - publish-openstack-docs-pti - release-notes-jobs-python3 diff --git a/requirements.txt b/requirements.txt index 4757239b..b7c92400 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -futures>=3.0.0;python_version=='2.7' # BSD + requests>=1.1.0 six>=1.9.0 diff --git a/run_tests.sh b/run_tests.sh index 6991cd1d..d1fc50ad 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -9,7 +9,7 @@ function usage { echo "" echo "This script is deprecated and currently retained for compatibility." echo 'You can run the full test suite for multiple environments by running "tox".' - echo 'You can run tests for only python 2.7 by running "tox -e py27", or run only' + echo 'You can run tests for only python 3.9 by running "tox -e py39", or run only' echo 'the pep8 tests with "tox -e pep8".' exit } @@ -39,7 +39,7 @@ if [ $just_pep8 -eq 1 ]; then exit fi -tox -e py27 $toxargs 2>&1 | tee run_tests.err.log || exit +tox -e py39 $toxargs 2>&1 | tee run_tests.err.log || exit if [ ${PIPESTATUS[0]} -ne 0 ]; then exit ${PIPESTATUS[0]} fi diff --git a/setup.cfg b/setup.cfg index c780c388..e429b12f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,9 +3,11 @@ name = python-swiftclient summary = OpenStack Object Storage API Client Library description_file = README.rst +license = Apache License, Version 2.0 author = OpenStack author_email = openstack-discuss@lists.openstack.org home_page = https://docs.openstack.org/python-swiftclient/latest/ +python_requires = >=3.6 classifier = Environment :: OpenStack Intended Audience :: Information Technology @@ -14,13 +16,12 @@ classifier = Operating System :: POSIX :: Linux Operating System :: Microsoft :: Windows Programming Language :: Python - Programming Language :: Python :: 2 - Programming Language :: Python :: 2.7 Programming Language :: Python :: 3 Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3 :: Only [files] packages = @@ -41,9 +42,6 @@ console_scripts = keystoneauth1.plugin = v1password = swiftclient.authv1:PasswordLoader -[bdist_wheel] -universal = 1 - [pbr] skip_authors = True skip_changelog = True diff --git a/setup.py b/setup.py index 16a18f6e..22cfdce8 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2013 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,12 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT -import setuptools, sys - -if sys.version_info < (2, 7): - sys.exit('Sorry, Python < 2.7 is not supported for' - ' python-swiftclient>=3.0') +import setuptools setuptools.setup( setup_requires=['pbr'], diff --git a/swiftclient/client.py b/swiftclient/client.py index 544247a6..cc5478a8 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -48,25 +48,11 @@ URI_PATTERN_INFO = re.compile(r'/info') URI_PATTERN_VERSION = re.compile(r'\/v\d+\.?\d*(\/.*)?') -try: - from logging import NullHandler -except ImportError: - # Added in Python 2.7 - class NullHandler(logging.Handler): - def handle(self, record): - pass - - def emit(self, record): - pass - - def createLock(self): - self.lock = None - ksexceptions = ksclient_v2 = ksclient_v3 = ksa_v3 = None try: from keystoneclient import exceptions as ksexceptions # prevent keystoneclient warning us that it has no log handlers - logging.getLogger('keystoneclient').addHandler(NullHandler()) + logging.getLogger('keystoneclient').addHandler(logging.NullHandler()) from keystoneclient.v2_0 import client as ksclient_v2 except ImportError: pass @@ -93,7 +79,7 @@ def prepare_unicode_headers(self, headers): requests.models.PreparedRequest.prepare_headers = prepare_unicode_headers logger = logging.getLogger("swiftclient") -logger.addHandler(NullHandler()) +logger.addHandler(logging.NullHandler()) #: Default behaviour is to redact header values known to contain secrets, #: such as ``X-Auth-Key`` and ``X-Auth-Token``. Up to the first 16 chars diff --git a/test/unit/test_swiftclient.py b/test/unit/test_swiftclient.py index ea5f502f..f6dc258f 100644 --- a/test/unit/test_swiftclient.py +++ b/test/unit/test_swiftclient.py @@ -1976,34 +1976,8 @@ def test_response_connection_released(self): self.assertFalse(resp.read()) self.assertTrue(resp.closed) - @unittest.skipIf(six.PY3, 'python2 specific test') - def test_response_python2_headers(self): - '''Test utf-8 headers in Python 2. - ''' - _, conn = c.http_connection(u'http://www.test.com/') - conn.resp = MockHttpResponse( - status=200, - headers={ - '\xd8\xaa-unicode': '\xd8\xaa-value', - 'empty-header': '' - } - ) - - resp = conn.getresponse() - self.assertEqual( - '\xd8\xaa-value', resp.getheader('\xd8\xaa-unicode')) - self.assertEqual( - '\xd8\xaa-value', resp.getheader('\xd8\xaa-UNICODE')) - self.assertEqual('', resp.getheader('empty-header')) - self.assertEqual( - dict([('\xd8\xaa-unicode', '\xd8\xaa-value'), - ('empty-header', ''), - ('etag', '"%s"' % EMPTY_ETAG)]), - dict(resp.getheaders())) - - @unittest.skipIf(six.PY2, 'python3 specific test') def test_response_python3_headers(self): - '''Test latin1-encoded headers in Python 3. + '''Test latin1-encoded headers. ''' _, conn = c.http_connection(u'http://www.test.com/') conn.resp = MockHttpResponse( diff --git a/tox.ini b/tox.ini index fbc58c78..e1e679d2 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,py3,pep8 +envlist = py3,pep8 minversion = 3.18.0 skipsdist = True @@ -21,16 +21,13 @@ allowlist_externals = sh passenv = SWIFT_* *_proxy [testenv:pep8] -basepython = python3 commands = python -m flake8 swiftclient test [testenv:venv] -basepython = python3 commands = {posargs} [testenv:cover] -basepython = python3 setenv = PYTHON=coverage run --source swiftclient --parallel-mode commands = @@ -41,7 +38,6 @@ commands = coverage report [testenv:func] -basepython = python3 setenv = OS_TEST_PATH=test.functional PYTHON=coverage run --source swiftclient --parallel-mode @@ -56,14 +52,7 @@ commands = coverage report -m rm -f .coverage -[testenv:py2func] -basepython=python2 -setenv = {[testenv:func]setenv} -allowlist_externals = {[testenv:func]allowlist_externals} -commands = {[testenv:func]commands} - [testenv:docs] -basepython = python3 usedevelop = False deps = -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} -r{toxinidir}/doc/requirements.txt @@ -88,7 +77,6 @@ show-source = True exclude = .venv,.tox,dist,doc,*egg [testenv:bindep] -basepython = python3 # Do not install any requirements. We want this to be fast and work even if # system dependencies are missing, since it's used to tell you what system # dependencies are missing! This also means that bindep must be installed @@ -98,14 +86,12 @@ deps = bindep commands = bindep test [testenv:releasenotes] -basepython = python3 usedevelop = False deps = -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} -r{toxinidir}/doc/requirements.txt commands = sphinx-build -a -W -E -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html [testenv:pdf-docs] -basepython = python3 deps = {[testenv:docs]deps} allowlist_externals = make From c09621eb4269e9eaa2c5f386976f356fb0701236 Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Thu, 10 Feb 2022 18:37:09 +0200 Subject: [PATCH 146/238] Don't patch Requests globally on import This also upgrades the Requests dependency to 2.4+ (released in 2014) to avoid having to do version comparisons altogether. Refs https://bugs.launchpad.net/python-swiftclient/+bug/1904551 Signed-off-by: Aarni Koskela Change-Id: I58399f6c526b0b78462f31739c43076314ba9e76 --- requirements.txt | 2 +- swiftclient/client.py | 24 ++------------ swiftclient/requests_compat.py | 57 ++++++++++++++++++++++++++++++++++ test/unit/test_swiftclient.py | 20 ++---------- 4 files changed, 62 insertions(+), 41 deletions(-) create mode 100644 swiftclient/requests_compat.py diff --git a/requirements.txt b/requirements.txt index b7c92400..c802baa3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -requests>=1.1.0 +requests>=2.4.0 six>=1.9.0 diff --git a/swiftclient/client.py b/swiftclient/client.py index cc5478a8..ddcf63af 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -18,11 +18,9 @@ """ import socket import re -import requests import logging import warnings -from distutils.version import StrictVersion from requests.exceptions import RequestException, SSLError from six.moves import http_client from six.moves.urllib.parse import quote as _quote, unquote @@ -32,6 +30,7 @@ from swiftclient import version as swiftclient_version from swiftclient.exceptions import ClientException +from swiftclient.requests_compat import SwiftClientRequestsSession from swiftclient.utils import ( iter_wrapper, LengthWrapper, ReadableToIterable, parse_api_response, get_body) @@ -64,20 +63,6 @@ except ImportError: pass -# requests version 1.2.3 try to encode headers in ascii, preventing -# utf-8 encoded header to be 'prepared'. This also affects all -# (or at least most) versions of requests on py3 -if StrictVersion(requests.__version__) < StrictVersion('2.0.0') \ - or not six.PY2: - from requests.structures import CaseInsensitiveDict - - def prepare_unicode_headers(self, headers): - if headers: - self.headers = CaseInsensitiveDict(headers) - else: - self.headers = CaseInsensitiveDict() - requests.models.PreparedRequest.prepare_headers = prepare_unicode_headers - logger = logging.getLogger("swiftclient") logger.addHandler(logging.NullHandler()) @@ -398,7 +383,7 @@ def __init__(self, url, proxy=None, cacert=None, insecure=False, self.host = self.parsed_url.netloc self.port = self.parsed_url.port self.requests_args = {} - self.request_session = requests.Session() + self.request_session = SwiftClientRequestsSession() # Don't use requests's default headers self.request_session.headers = None self.resp = None @@ -1434,11 +1419,6 @@ def put_object(url, token=None, container=None, name=None, contents=None, content_length = int(v) if content_type is not None: headers['Content-Type'] = content_type - elif 'Content-Type' not in headers: - if StrictVersion(requests.__version__) < StrictVersion('2.4.0'): - # python-requests sets application/x-www-form-urlencoded otherwise - # if using python3. - headers['Content-Type'] = '' if not contents: headers['Content-Length'] = '0' diff --git a/swiftclient/requests_compat.py b/swiftclient/requests_compat.py new file mode 100644 index 00000000..c2371b74 --- /dev/null +++ b/swiftclient/requests_compat.py @@ -0,0 +1,57 @@ +# Copyright (c) 2010-2022 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import requests +from requests.sessions import merge_setting, merge_hooks +from requests.structures import CaseInsensitiveDict + + +class SwiftClientPreparedRequest(requests.PreparedRequest): + def prepare_headers(self, headers): + try: + return super().prepare_headers(headers) + except UnicodeError: + # If we got an unicode error from the superclass's prepare_headers, + # we had a non-spec-compliant non-ASCII header + # (e.g. an UTF-8 encoded Swift object metadata header). + # In that case, we just pass it through and hope nothing + # bad will happen from not following the HTTP spec. + self.headers = CaseInsensitiveDict(headers or {}) + + +class SwiftClientRequestsSession(requests.Session): + + def prepare_request(self, request): + # Close to the superclass's implementation, + # but no cookies or .netrc authentication overrides here. + p = SwiftClientPreparedRequest() + headers = merge_setting( + request.headers, + self.headers, + dict_class=CaseInsensitiveDict, + ) + p.prepare( + method=request.method.upper(), + url=request.url, + files=request.files, + data=request.data, + json=request.json, + headers=headers, + params=merge_setting(request.params, self.params), + auth=merge_setting(request.auth, self.auth), + cookies=None, + hooks=merge_hooks(request.hooks, self.hooks), + ) + return p diff --git a/test/unit/test_swiftclient.py b/test/unit/test_swiftclient.py index f6dc258f..6bdd6ad6 100644 --- a/test/unit/test_swiftclient.py +++ b/test/unit/test_swiftclient.py @@ -1325,7 +1325,6 @@ def test_query_string(self): class TestPutObject(MockHttpTest): - @mock.patch('swiftclient.requests.__version__', '2.2.0') def test_ok(self): c.http_connection = self.fake_http_connection(200) args = ('http://www.test.com', 'TOKEN', 'container', 'obj', 'body', 4) @@ -1336,7 +1335,6 @@ def test_ok(self): ('PUT', '/container/obj', 'body', { 'x-auth-token': 'TOKEN', 'content-length': '4', - 'content-type': '' }), ]) @@ -1383,7 +1381,6 @@ def test_chunk_warning(self): self.assertEqual(len(w), 1) self.assertTrue(issubclass(w[-1].category, UserWarning)) - @mock.patch('swiftclient.requests.__version__', '2.2.0') def test_server_error(self): body = 'c' * 60 headers = {'foo': 'bar'} @@ -1398,8 +1395,7 @@ def test_server_error(self): self.assertEqual(e.http_status, 500) self.assertRequests([ ('PUT', '/asdf/asdf', 'asdf', { - 'x-auth-token': 'asdf', - 'content-type': ''}), + 'x-auth-token': 'asdf'}), ]) def test_query_string(self): @@ -1540,19 +1536,7 @@ def test_params(self): self.assertEqual(request_header['etag'], b'1234-5678') self.assertEqual(request_header['content-type'], b'text/plain') - @mock.patch('swiftclient.requests.__version__', '2.2.0') - def test_no_content_type_old_requests(self): - conn = c.http_connection(u'http://www.test.com/') - resp = MockHttpResponse(status=200) - conn[1].getresponse = resp.fake_response - conn[1]._request = resp._fake_request - - c.put_object(url='http://www.test.com', http_conn=conn) - request_header = resp.requests_params['headers'] - self.assertEqual(request_header['content-type'], b'') - - @mock.patch('swiftclient.requests.__version__', '2.4.0') - def test_no_content_type_new_requests(self): + def test_no_content_type_requests(self): conn = c.http_connection(u'http://www.test.com/') resp = MockHttpResponse(status=200) conn[1].getresponse = resp.fake_response From 3d10744c55d8cfddcd486a3e8521333b9cf606e1 Mon Sep 17 00:00:00 2001 From: Stephen Finucane Date: Thu, 17 Feb 2022 11:38:11 +0000 Subject: [PATCH 147/238] Remove __future__ imports These aren't needed in modern Python 3 versions. Signed-off-by: Stephen Finucane Change-Id: I5e81d6fb2e2cb8e4bfae4ed746da002f44e871c4 --- swiftclient/multithreading.py | 2 -- swiftclient/service.py | 3 +-- swiftclient/shell.py | 2 -- test/unit/test_service.py | 2 +- test/unit/test_shell.py | 1 - 5 files changed, 2 insertions(+), 8 deletions(-) diff --git a/swiftclient/multithreading.py b/swiftclient/multithreading.py index f128790e..665ba63b 100644 --- a/swiftclient/multithreading.py +++ b/swiftclient/multithreading.py @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import six import sys diff --git a/swiftclient/service.py b/swiftclient/service.py index 8e2c7b00..6b8f73dc 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -12,9 +12,8 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import unicode_literals -import logging +import logging import os from collections import defaultdict diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 6da9d667..36d07575 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -14,8 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function, unicode_literals - import argparse import getpass import io diff --git a/test/unit/test_service.py b/test/unit/test_service.py index e86a4ff1..7f90eb49 100644 --- a/test/unit/test_service.py +++ b/test/unit/test_service.py @@ -13,7 +13,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import unicode_literals + import contextlib import mock import os diff --git a/test/unit/test_shell.py b/test/unit/test_shell.py index 295c918a..8d3b1635 100644 --- a/test/unit/test_shell.py +++ b/test/unit/test_shell.py @@ -12,7 +12,6 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import unicode_literals import contextlib from genericpath import getmtime From 4983b909831b72b5361aadf573cadd3afaaf8976 Mon Sep 17 00:00:00 2001 From: Stephen Finucane Date: Thu, 17 Feb 2022 11:39:46 +0000 Subject: [PATCH 148/238] Remove coding comments Everything is unicode in Python 3. Signed-off-by: Stephen Finucane Change-Id: I6a076dc67c461f265ed99878e3959e1992a88189 --- doc/source/conf.py | 2 -- releasenotes/source/conf.py | 1 - swiftclient/__init__.py | 1 - test/unit/test_service.py | 1 - 4 files changed, 5 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 2673d6c5..1c5fc692 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- -# # Swiftclient documentation build configuration file, created by # sphinx-quickstart on Tue Apr 17 02:17:37 2012. # diff --git a/releasenotes/source/conf.py b/releasenotes/source/conf.py index 0945c81e..a1385e55 100644 --- a/releasenotes/source/conf.py +++ b/releasenotes/source/conf.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/swiftclient/__init__.py b/swiftclient/__init__.py index dc192afe..38750d10 100644 --- a/swiftclient/__init__.py +++ b/swiftclient/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2012 Rackspace # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/test/unit/test_service.py b/test/unit/test_service.py index 7f90eb49..8c38bd35 100644 --- a/test/unit/test_service.py +++ b/test/unit/test_service.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2014 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); From fa137a5bf1f2a86cc15ebc4d973f245e1543105d Mon Sep 17 00:00:00 2001 From: Stephen Finucane Date: Thu, 17 Feb 2022 11:37:13 +0000 Subject: [PATCH 149/238] Remove six This mostly affects tests. Nothing too complicated Signed-off-by: Stephen Finucane Change-Id: Iabc78f651e1d48db35638280722f8019798eccd6 --- requirements.txt | 2 - swiftclient/authv1.py | 2 +- swiftclient/client.py | 59 ++++++++++------------------- swiftclient/exceptions.py | 2 +- swiftclient/multithreading.py | 13 ++----- swiftclient/service.py | 30 +++++++-------- swiftclient/shell.py | 9 +---- swiftclient/utils.py | 24 +++++------- test/functional/__init__.py | 2 +- test/functional/test_swiftclient.py | 8 ---- test/unit/test_command_helpers.py | 2 +- test/unit/test_multithreading.py | 10 ++--- test/unit/test_service.py | 23 +++++------ test/unit/test_shell.py | 24 +++++------- test/unit/test_swiftclient.py | 27 +++++++------ test/unit/test_utils.py | 28 +++++++------- test/unit/utils.py | 26 ++++++------- 17 files changed, 116 insertions(+), 175 deletions(-) diff --git a/requirements.txt b/requirements.txt index c802baa3..94cc57fe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1 @@ - requests>=2.4.0 -six>=1.9.0 diff --git a/swiftclient/authv1.py b/swiftclient/authv1.py index d70acac3..705dcb40 100644 --- a/swiftclient/authv1.py +++ b/swiftclient/authv1.py @@ -40,7 +40,7 @@ import json import time -from six.moves.urllib.parse import urljoin +from urllib.parse import urljoin # Note that while we import keystoneauth1 here, we *don't* need to add it to # requirements.txt -- this entire module only makes sense (and should only be diff --git a/swiftclient/client.py b/swiftclient/client.py index ddcf63af..df5344df 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -22,11 +22,10 @@ import warnings from requests.exceptions import RequestException, SSLError -from six.moves import http_client -from six.moves.urllib.parse import quote as _quote, unquote -from six.moves.urllib.parse import urljoin, urlparse, urlunparse +import http.client as http_client +from urllib.parse import quote as _quote, unquote +from urllib.parse import urljoin, urlparse, urlunparse from time import sleep, time -import six from swiftclient import version as swiftclient_version from swiftclient.exceptions import ClientException @@ -165,34 +164,20 @@ def http_log(args, kwargs, resp, body): def parse_header_string(data): - if not isinstance(data, (six.text_type, six.binary_type)): + if not isinstance(data, (str, bytes)): data = str(data) - if six.PY2: - if isinstance(data, six.text_type): - # Under Python2 requests only returns binary_type, but if we get - # some stray text_type input, this should prevent unquote from - # interpreting %-encoded data as raw code-points. - data = data.encode('utf8') + if isinstance(data, bytes): + # Under Python3 requests only returns text_type and tosses (!) the + # rest of the headers. If that ever changes, this should be a sane + # approach. try: - unquoted = unquote(data).decode('utf8') + data = data.decode('ascii') except UnicodeDecodeError: - try: - return data.decode('utf8') - except UnicodeDecodeError: - return quote(data).decode('utf8') - else: - if isinstance(data, six.binary_type): - # Under Python3 requests only returns text_type and tosses (!) the - # rest of the headers. If that ever changes, this should be a sane - # approach. - try: - data = data.decode('ascii') - except UnicodeDecodeError: - data = quote(data) - try: - unquoted = unquote(data, errors='strict') - except UnicodeDecodeError: - return data + data = quote(data) + try: + unquoted = unquote(data, errors='strict') + except UnicodeDecodeError: + return data return unquoted @@ -201,20 +186,18 @@ def quote(value, safe='/'): Patched version of urllib.quote that encodes utf8 strings before quoting. On Python 3, call directly urllib.parse.quote(). """ - if six.PY3: - return _quote(value, safe=safe) - return _quote(encode_utf8(value), safe) + return _quote(value, safe=safe) def encode_utf8(value): - if type(value) in six.integer_types + (float, bool): + if type(value) in (int, float, bool): # As of requests 2.11.0, headers must be byte- or unicode-strings. # Convert some known-good types as a convenience for developers. # Note that we *don't* convert subclasses, as they may have overriddden # __str__ or __repr__. # See https://github.com/kennethreitz/requests/pull/3366 for more info value = str(value) - if isinstance(value, six.text_type): + if isinstance(value, str): value = value.encode('utf8') return value @@ -226,7 +209,7 @@ def encode_meta_headers(headers): value = encode_utf8(value) header = header.lower() - if (isinstance(header, six.string_types) and + if (isinstance(header, str) and header.startswith(USER_METADATA_TYPE)): header = encode_utf8(header) @@ -457,12 +440,12 @@ def getresponse(self): old_getheader = self.resp.raw.getheader def _decode_header(string): - if string is None or six.PY2: + if string is None: return string return string.encode('iso-8859-1').decode('utf-8') def _encode_header(string): - if string is None or six.PY2: + if string is None: return string return string.encode('utf-8').decode('iso-8859-1') @@ -1441,7 +1424,7 @@ def put_object(url, token=None, container=None, name=None, contents=None, warnings.warn(warn_msg, stacklevel=2) # Match requests's is_stream test if hasattr(contents, '__iter__') and not isinstance(contents, ( - six.text_type, six.binary_type, list, tuple, dict)): + str, bytes, list, tuple, dict)): contents = iter_wrapper(contents) conn.request('PUT', path, contents, headers) diff --git a/swiftclient/exceptions.py b/swiftclient/exceptions.py index a9b993ce..f0d1b5db 100644 --- a/swiftclient/exceptions.py +++ b/swiftclient/exceptions.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from six.moves import urllib +import urllib class ClientException(Exception): diff --git a/swiftclient/multithreading.py b/swiftclient/multithreading.py index 665ba63b..cf72360e 100644 --- a/swiftclient/multithreading.py +++ b/swiftclient/multithreading.py @@ -13,11 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import six import sys from concurrent.futures import ThreadPoolExecutor -from six.moves.queue import PriorityQueue +from queue import PriorityQueue class OutputManager(object): @@ -70,12 +69,8 @@ def print_raw(self, data): self.print_pool.submit(self._write, data, self.print_stream) def _write(self, data, stream): - if six.PY3: - stream.buffer.write(data) - stream.flush() - if six.PY2: - stream.write(data) - stream.flush() + stream.buffer.write(data) + stream.flush() def print_msg(self, msg, *fmt_args): if fmt_args: @@ -100,8 +95,6 @@ def get_error_count(self): def _print(self, item, stream=None): if stream is None: stream = self.print_stream - if six.PY2 and isinstance(item, six.text_type): - item = item.encode('utf8') print(item, file=stream) def _print_error(self, item, count=1): diff --git a/swiftclient/service.py b/swiftclient/service.py index 6b8f73dc..289e29e8 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -21,6 +21,7 @@ from copy import deepcopy from errno import EEXIST, ENOENT from hashlib import md5 +from io import StringIO from os import environ, makedirs, stat, utime from os.path import ( basename, dirname, getmtime, getsize, isdir, join, sep as os_path_sep @@ -29,10 +30,9 @@ from random import shuffle from time import time from threading import Thread -from six import Iterator, StringIO, string_types, text_type -from six.moves.queue import Queue -from six.moves.queue import Empty as QueueEmpty -from six.moves.urllib.parse import quote +from queue import Queue +from queue import Empty as QueueEmpty +from urllib.parse import quote import json @@ -54,7 +54,7 @@ logger = logging.getLogger("swiftclient.service") -class ResultsIterator(Iterator): +class ResultsIterator: def __init__(self, futures): self.futures = interruptable_as_completed(futures) @@ -321,10 +321,10 @@ class SwiftUploadObject(object): options to be specified separately for each individual object. """ def __init__(self, source, object_name=None, options=None): - if isinstance(source, string_types): + if isinstance(source, str): self.object_name = object_name or source elif source is None or hasattr(source, 'read'): - if not object_name or not isinstance(object_name, string_types): + if not object_name or not isinstance(object_name, str): raise SwiftError('Object names must be specified as ' 'strings for uploads from None or file ' 'like objects.') @@ -347,7 +347,7 @@ class SwiftPostObject(object): specified separately for each individual object. """ def __init__(self, object_name, options=None): - if not (isinstance(object_name, string_types) and object_name): + if not (isinstance(object_name, str) and object_name): raise SwiftError( "Object names must be specified as non-empty strings" ) @@ -361,7 +361,7 @@ class SwiftDeleteObject(object): specified separately for each individual object. """ def __init__(self, object_name, options=None): - if not (isinstance(object_name, string_types) and object_name): + if not (isinstance(object_name, str) and object_name): raise SwiftError( "Object names must be specified as non-empty strings" ) @@ -377,7 +377,7 @@ class SwiftCopyObject(object): destination and fresh_metadata should be set in options """ def __init__(self, object_name, options=None): - if not (isinstance(object_name, string_types) and object_name): + if not (isinstance(object_name, str) and object_name): raise SwiftError( "Object names must be specified as non-empty strings" ) @@ -835,7 +835,7 @@ def _make_post_objects(objects): post_objects = [] for o in objects: - if isinstance(o, string_types): + if isinstance(o, str): obj = SwiftPostObject(o) post_objects.append(obj) elif isinstance(o, SwiftPostObject): @@ -1637,7 +1637,7 @@ def _make_upload_objects(objects, pseudo_folder=''): upload_objects = [] for o in objects: - if isinstance(o, string_types): + if isinstance(o, str): obj = SwiftUploadObject(o, urljoin(pseudo_folder, o.lstrip('/'))) upload_objects.append(obj) @@ -2035,7 +2035,7 @@ def _upload_slo_manifest(conn, segment_results, container, obj, headers): segment_results.sort(key=lambda di: di['segment_index']) for seg in segment_results: seg_loc = seg['segment_location'].lstrip('/') - if isinstance(seg_loc, text_type): + if isinstance(seg_loc, str): seg_loc = seg_loc.encode('utf-8') manifest_data = json.dumps([ @@ -2578,7 +2578,7 @@ def _make_delete_objects(objects): delete_objects = [] for o in objects: - if isinstance(o, string_types): + if isinstance(o, str): obj = SwiftDeleteObject(o) delete_objects.append(obj) elif isinstance(o, SwiftDeleteObject): @@ -2933,7 +2933,7 @@ def _make_copy_objects(objects, options): copy_objects = [] for o in objects: - if isinstance(o, string_types): + if isinstance(o, str): obj = SwiftCopyObject(o, options) copy_objects.append(obj) elif isinstance(o, SwiftCopyObject): diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 36d07575..a16de884 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -25,8 +25,7 @@ from os import environ, walk, _exit as os_exit from os.path import isfile, isdir, join -from six import text_type, PY2 -from six.moves.urllib.parse import unquote, urlparse +from urllib.parse import unquote, urlparse from sys import argv as sys_argv, exit, stderr, stdin from time import gmtime, strftime @@ -191,10 +190,6 @@ def st_delete(parser, args, output_manager, return_parser=False): for o, err in r.get('result', {}).get('Errors', []): # o will be of the form quote("//") o = unquote(o) - if PY2: - # In PY3, unquote(unicode) uses utf-8 like we - # want, but PY2 uses latin-1 - o = o.encode('latin-1').decode('utf-8') output_manager.error('Error Deleting: {0}: {1}' .format(o[1:], err)) try: @@ -1931,7 +1926,7 @@ def add_default_args(parser): def main(arguments=None): argv = sys_argv if arguments is None else arguments - argv = [a if isinstance(a, text_type) else a.decode('utf-8') for a in argv] + argv = [a if isinstance(a, str) else a.decode('utf-8') for a in argv] parser = argparse.ArgumentParser( add_help=False, formatter_class=HelpFormatter, usage=''' diff --git a/swiftclient/utils.py b/swiftclient/utils.py index 656acad4..03e5e7b2 100644 --- a/swiftclient/utils.py +++ b/swiftclient/utils.py @@ -13,17 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. """Miscellaneous utility functions for use with Swift.""" + from calendar import timegm -try: - from collections.abc import Mapping -except ImportError: - from collections import Mapping +from collections.abc import Mapping import gzip import hashlib import hmac +import io import json import logging -import six import time import traceback @@ -42,7 +40,7 @@ def config_true_value(value): This function comes from swift.common.utils.config_true_value() """ return value is True or \ - (isinstance(value, six.string_types) and value.lower() in TRUE_VALUES) + (isinstance(value, str) and value.lower() in TRUE_VALUES) def prt_bytes(num_bytes, human_flag): @@ -134,7 +132,7 @@ def generate_temp_url(path, seconds, key, method, absolute=False, except ValueError: raise ValueError(TIME_ERRMSG) - if isinstance(path, six.binary_type): + if isinstance(path, bytes): try: path_for_body = path.decode('utf-8') except UnicodeDecodeError: @@ -165,7 +163,7 @@ def generate_temp_url(path, seconds, key, method, absolute=False, ('prefix:' if prefix else '') + path_for_body] if ip_range: - if isinstance(ip_range, six.binary_type): + if isinstance(ip_range, bytes): try: ip_range = ip_range.decode('utf-8') except UnicodeDecodeError: @@ -177,7 +175,7 @@ def generate_temp_url(path, seconds, key, method, absolute=False, hmac_body = u'\n'.join(hmac_parts) # Encode to UTF-8 for py3 compatibility - if not isinstance(key, six.binary_type): + if not isinstance(key, bytes): key = key.encode('utf-8') sig = hmac.new(key, hmac_body.encode('utf-8'), hashlib.sha1).hexdigest() @@ -194,7 +192,7 @@ def generate_temp_url(path, seconds, key, method, absolute=False, if prefix: temp_url += u'&temp_url_prefix={}'.format(parts[4]) # Have return type match path from caller - if isinstance(path, six.binary_type): + if isinstance(path, bytes): return temp_url.encode('utf-8') else: return temp_url @@ -202,7 +200,7 @@ def generate_temp_url(path, seconds, key, method, absolute=False, def get_body(headers, body): if headers.get('content-encoding') == 'gzip': - with gzip.GzipFile(fileobj=six.BytesIO(body), mode='r') as gz: + with gzip.GzipFile(fileobj=io.BytesIO(body), mode='r') as gz: nbody = gz.read() return nbody return body @@ -224,7 +222,7 @@ def split_request_headers(options, prefix=''): if isinstance(options, Mapping): options = options.items() for item in options: - if isinstance(item, six.string_types): + if isinstance(item, str): if ':' not in item: raise ValueError( "Metadata parameter %s must contain a ':'.\n" @@ -401,8 +399,6 @@ def n_groups(seq, n): def normalize_manifest_path(path): - if six.PY2 and isinstance(path, six.text_type): - path = path.encode('utf-8') if path.startswith('/'): return path[1:] return path diff --git a/test/functional/__init__.py b/test/functional/__init__.py index f0fea3be..249dafe0 100644 --- a/test/functional/__init__.py +++ b/test/functional/__init__.py @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import configparser import os -from six.moves import configparser TEST_CONFIG = None diff --git a/test/functional/test_swiftclient.py b/test/functional/test_swiftclient.py index 8d001d0f..5fc8df75 100644 --- a/test/functional/test_swiftclient.py +++ b/test/functional/test_swiftclient.py @@ -17,8 +17,6 @@ import time from io import BytesIO -import six - import swiftclient from . import TEST_CONFIG @@ -417,12 +415,6 @@ def test_post_object_unicode_header_name(self): # https://bugs.python.org/issue37093 # We'll have to settle for just testing that the POST doesn't blow up # with a UnicodeDecodeError - if six.PY2: - headers = self.conn.head_object( - self.containername, self.objectname) - self.assertIn(u'x-object-meta-\U0001f44d', headers) - self.assertEqual(u'\U0001f44d', - headers.get(u'x-object-meta-\U0001f44d')) def test_copy_object(self): self.conn.put_object( diff --git a/test/unit/test_command_helpers.py b/test/unit/test_command_helpers.py index 24684ae2..1cb3bb12 100644 --- a/test/unit/test_command_helpers.py +++ b/test/unit/test_command_helpers.py @@ -14,7 +14,7 @@ # limitations under the License. import mock -from six import StringIO +from io import StringIO import unittest from swiftclient import command_helpers as h diff --git a/test/unit/test_multithreading.py b/test/unit/test_multithreading.py index e9732cd8..ee9e7cc5 100644 --- a/test/unit/test_multithreading.py +++ b/test/unit/test_multithreading.py @@ -12,13 +12,13 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. + +from queue import Queue, Empty import sys import unittest import threading -import six from concurrent.futures import as_completed -from six.moves.queue import Queue, Empty from time import sleep from swiftclient import multithreading as mt @@ -216,11 +216,7 @@ def test_printers(self): # The threads should have been cleaned up self.assertEqual(starting_thread_count, threading.active_count()) - if six.PY3: - over_the = "over the '\u062a\u062a'\n" - else: - over_the = "over the u'\\u062a\\u062a'\n" - # We write to the CaptureStream so no decoding is performed + over_the = "over the '\u062a\u062a'\n" self.assertEqual(''.join([ 'one-argument\n', 'one fish, 88 fish\n', diff --git a/test/unit/test_service.py b/test/unit/test_service.py index 8c38bd35..9f3a2a2f 100644 --- a/test/unit/test_service.py +++ b/test/unit/test_service.py @@ -13,20 +13,21 @@ # See the License for the specific language governing permissions and # limitations under the License. +import builtins import contextlib +import io import mock import os -import six import tempfile import unittest import time import json +from io import BytesIO from concurrent.futures import Future from hashlib import md5 from mock import Mock, PropertyMock -from six.moves.queue import Queue, Empty as QueueEmptyError -from six import BytesIO +from queue import Queue, Empty as QueueEmptyError from time import sleep import swiftclient @@ -46,12 +47,6 @@ clean_os_environ[key] = '' -if six.PY2: - import __builtin__ as builtins -else: - import builtins - - class TestSwiftPostObject(unittest.TestCase): def setUp(self): @@ -1268,7 +1263,7 @@ def test_upload_with_relative_path(self, *args, **kwargs): for obj in objects: with mock.patch('swiftclient.service.Connection') as mock_conn, \ mock.patch.object(builtins, 'open') as mock_open: - mock_open.return_value = six.StringIO('asdf') + mock_open.return_value = io.StringIO('asdf') mock_conn.return_value.head_object.side_effect = \ ClientException('Not Found', http_status=404) mock_conn.return_value.put_object.return_value =\ @@ -2318,7 +2313,7 @@ def _make_result(): def test_download_object_job(self): mock_conn = self._get_mock_connection() - objcontent = six.BytesIO(b'objcontent') + objcontent = io.BytesIO(b'objcontent') mock_conn.get_object.side_effect = [ ({'content-type': 'text/plain', 'etag': '2cbbfe139a744d6abbe695e17f3c1991'}, @@ -2360,7 +2355,7 @@ def test_download_object_job(self): def test_download_object_job_with_mtime(self): mock_conn = self._get_mock_connection() - objcontent = six.BytesIO(b'objcontent') + objcontent = io.BytesIO(b'objcontent') mock_conn.get_object.side_effect = [ ({'content-type': 'text/plain', 'etag': '2cbbfe139a744d6abbe695e17f3c1991', @@ -2406,7 +2401,7 @@ def test_download_object_job_with_mtime(self): def test_download_object_job_bad_mtime(self): mock_conn = self._get_mock_connection() - objcontent = six.BytesIO(b'objcontent') + objcontent = io.BytesIO(b'objcontent') mock_conn.get_object.side_effect = [ ({'content-type': 'text/plain', 'etag': '2cbbfe139a744d6abbe695e17f3c1991', @@ -2451,7 +2446,7 @@ def test_download_object_job_bad_mtime(self): def test_download_object_job_ignore_mtime(self): mock_conn = self._get_mock_connection() - objcontent = six.BytesIO(b'objcontent') + objcontent = io.BytesIO(b'objcontent') mock_conn.get_object.side_effect = [ ({'content-type': 'text/plain', 'etag': '2cbbfe139a744d6abbe695e17f3c1991', diff --git a/test/unit/test_shell.py b/test/unit/test_shell.py index 8d3b1635..adf37767 100644 --- a/test/unit/test_shell.py +++ b/test/unit/test_shell.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import io import contextlib from genericpath import getmtime import getpass @@ -26,8 +27,6 @@ import textwrap from time import localtime, mktime, strftime, strptime -import six - import swiftclient from swiftclient.service import SwiftError import swiftclient.shell @@ -46,10 +45,7 @@ except ImportError: InsecureRequestWarning = None -if six.PY2: - BUILTIN_OPEN = '__builtin__.open' -else: - BUILTIN_OPEN = 'builtins.open' +BUILTIN_OPEN = 'builtins.open' mocked_os_environ = { 'ST_AUTH': 'http://localhost:8080/auth/v1.0', @@ -631,7 +627,7 @@ def test_download_version_id(self, connection): @mock.patch('swiftclient.service.makedirs') @mock.patch('swiftclient.service.Connection') def test_download(self, connection, makedirs): - objcontent = six.BytesIO(b'objcontent') + objcontent = io.BytesIO(b'objcontent') connection.return_value.get_object.side_effect = [ ({'content-type': 'text/plain', 'etag': '2cbbfe139a744d6abbe695e17f3c1991'}, @@ -666,7 +662,7 @@ def test_download(self, connection, makedirs): makedirs.reset_mock() # Test downloading single object - objcontent = six.BytesIO(b'objcontent') + objcontent = io.BytesIO(b'objcontent') connection.return_value.get_object.side_effect = [ ({'content-type': 'text/plain', 'etag': '2cbbfe139a744d6abbe695e17f3c1991'}, @@ -682,7 +678,7 @@ def test_download(self, connection, makedirs): self.assertEqual([], makedirs.mock_calls) # Test downloading without md5 checks - objcontent = six.BytesIO(b'objcontent') + objcontent = io.BytesIO(b'objcontent') connection.return_value.get_object.side_effect = [ ({'content-type': 'text/plain', 'etag': '2cbbfe139a744d6abbe695e17f3c1991'}, @@ -700,7 +696,7 @@ def test_download(self, connection, makedirs): self.assertEqual([], makedirs.mock_calls) # Test downloading single object to stdout - objcontent = six.BytesIO(b'objcontent') + objcontent = io.BytesIO(b'objcontent') connection.return_value.get_object.side_effect = [ ({'content-type': 'text/plain', 'etag': '2cbbfe139a744d6abbe695e17f3c1991'}, @@ -3246,7 +3242,7 @@ def test_auth(self): } mock_resp = self.fake_http_connection(200, headers=headers) with mock.patch('swiftclient.client.http_connection', new=mock_resp): - stdout = six.StringIO() + stdout = io.StringIO() with mock.patch('sys.stdout', new=stdout): argv = [ '', @@ -3265,7 +3261,7 @@ def test_auth(self): def test_auth_verbose(self): with mock.patch('swiftclient.client.http_connection') as mock_conn: - stdout = six.StringIO() + stdout = io.StringIO() with mock.patch('sys.stdout', new=stdout): argv = [ '', @@ -3289,7 +3285,7 @@ def test_auth_v2(self): os_options = {'tenant_name': 'demo'} with mock.patch('swiftclient.client.get_auth_keystone', new=fake_get_auth_keystone(os_options)): - stdout = six.StringIO() + stdout = io.StringIO() with mock.patch('sys.stdout', new=stdout): argv = [ '', @@ -3310,7 +3306,7 @@ def test_auth_v2(self): def test_auth_verbose_v2(self): with mock.patch('swiftclient.client.get_auth_keystone') \ as mock_keystone: - stdout = six.StringIO() + stdout = io.StringIO() with mock.patch('sys.stdout', new=stdout): argv = [ '', diff --git a/test/unit/test_swiftclient.py b/test/unit/test_swiftclient.py index 6bdd6ad6..55c18fab 100644 --- a/test/unit/test_swiftclient.py +++ b/test/unit/test_swiftclient.py @@ -17,15 +17,14 @@ import json import logging import mock -import six +import io import socket import string import unittest import warnings import tempfile from hashlib import md5 -from six import binary_type -from six.moves.urllib.parse import urlparse +from urllib.parse import urlparse from requests.exceptions import RequestException from .utils import (MockHttpTest, fake_get_auth_keystone, StubResponse, @@ -214,12 +213,12 @@ def test_encode_meta_headers(self): self.assertEqual(len(headers), len(r)) # ensure non meta headers are not encoded - self.assertIs(type(r.get('abc')), binary_type) + self.assertIs(type(r.get('abc')), bytes) del r['abc'] for k, v in r.items(): - self.assertIs(type(k), binary_type) - self.assertIs(type(v), binary_type) + self.assertIs(type(k), bytes) + self.assertIs(type(v), bytes) self.assertIn(v, (b'123', b'12.3', b'True')) def test_set_user_agent_default(self): @@ -1329,7 +1328,7 @@ def test_ok(self): c.http_connection = self.fake_http_connection(200) args = ('http://www.test.com', 'TOKEN', 'container', 'obj', 'body', 4) value = c.put_object(*args) - self.assertIsInstance(value, six.string_types) + self.assertIsInstance(value, str) self.assertEqual(value, EMPTY_ETAG) self.assertRequests([ ('PUT', '/container/obj', 'body', { @@ -1340,7 +1339,7 @@ def test_ok(self): def test_unicode_ok(self): conn = c.http_connection(u'http://www.test.com/') - mock_file = six.StringIO(u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91') + mock_file = io.StringIO(u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91') args = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', @@ -1354,7 +1353,7 @@ def test_unicode_ok(self): conn[1].getresponse = resp.fake_response conn[1]._request = resp._fake_request value = c.put_object(*args, headers=headers, http_conn=conn) - self.assertIsInstance(value, six.string_types) + self.assertIsInstance(value, str) # Test for RFC-2616 encoded symbols self.assertIn(("a-b", b".x:yz mn:fg:lp"), resp.buffer) @@ -1364,7 +1363,7 @@ def test_unicode_ok(self): def test_chunk_warning(self): conn = c.http_connection('http://www.test.com/') - mock_file = six.StringIO('asdf') + mock_file = io.StringIO('asdf') args = ('asdf', 'asdf', 'asdf', 'asdf', mock_file) resp = MockHttpResponse() conn[1].getresponse = resp.fake_response @@ -1960,7 +1959,7 @@ def test_response_connection_released(self): self.assertFalse(resp.read()) self.assertTrue(resp.closed) - def test_response_python3_headers(self): + def test_response_headers(self): '''Test latin1-encoded headers. ''' _, conn = c.http_connection(u'http://www.test.com/') @@ -2547,7 +2546,7 @@ def test_reset_stream(self): class LocalContents(object): def __init__(self, tell_value=0): - self.data = six.BytesIO(string.ascii_letters.encode() * 10) + self.data = io.BytesIO(string.ascii_letters.encode() * 10) self.data.seek(tell_value) self.reads = [] self.seeks = [] @@ -2845,7 +2844,7 @@ def test_put_ok(self): c.http_connection = self.fake_http_connection(200) args = ('http://www.test.com', 'asdf', 'asdf', 'asdf', 'asdf') value = c.put_object(*args) - self.assertIsInstance(value, six.string_types) + self.assertIsInstance(value, str) def test_head_error(self): c.http_connection = self.fake_http_connection(500) @@ -2859,7 +2858,7 @@ def test_get_error(self): self.assertEqual(exc_context.exception.http_status, 404) def test_content_encoding_gzip_body_is_logged_decoded(self): - buf = six.BytesIO() + buf = io.BytesIO() gz = gzip.GzipFile(fileobj=buf, mode='w') data = {"test": u"\u2603"} decoded_body = json.dumps(data).encode('utf-8') diff --git a/test/unit/test_utils.py b/test/unit/test_utils.py index cbee82bf..02d19f3c 100644 --- a/test/unit/test_utils.py +++ b/test/unit/test_utils.py @@ -14,10 +14,10 @@ # limitations under the License. import gzip +import io import json import unittest import mock -import six import tempfile from time import gmtime, localtime, mktime, strftime, strptime from hashlib import md5, sha1 @@ -142,7 +142,7 @@ def test_generate_temp_url(self, time_mock, hmac_mock): url = u.generate_temp_url(self.url, self.seconds, self.key, self.method) key = self.key - if not isinstance(key, six.binary_type): + if not isinstance(key, bytes): key = key.encode('utf-8') self.assertEqual(url, self.expected_url) self.assertEqual(hmac_mock.mock_calls, [ @@ -170,10 +170,10 @@ def test_generate_temp_url_ip_range(self, time_mock, hmac_mock): self.key, self.method, ip_range=ip_range) key = self.key - if not isinstance(key, six.binary_type): + if not isinstance(key, bytes): key = key.encode('utf-8') - if isinstance(ip_range, six.binary_type): + if isinstance(ip_range, bytes): ip_range_expected_url = ( expected_url + ip_range.decode('utf-8') ) @@ -215,7 +215,7 @@ def test_generate_temp_url_iso8601_argument(self, hmac_mock): lt = localtime() expires = strftime(u.EXPIRES_ISO8601_FORMAT[:-1], lt) - if not isinstance(self.expected_url, six.string_types): + if not isinstance(self.expected_url, str): expected_url = self.expected_url.replace( b'1400003600', bytes(str(int(mktime(lt))), encoding='ascii')) else: @@ -228,7 +228,7 @@ def test_generate_temp_url_iso8601_argument(self, hmac_mock): expires = strftime(u.SHORT_EXPIRES_ISO8601_FORMAT, lt) lt = strptime(expires, u.SHORT_EXPIRES_ISO8601_FORMAT) - if not isinstance(self.expected_url, six.string_types): + if not isinstance(self.expected_url, str): expected_url = self.expected_url.replace( b'1400003600', bytes(str(int(mktime(lt))), encoding='ascii')) else: @@ -246,11 +246,11 @@ def test_generate_temp_url_iso8601_output(self, time_mock, hmac_mock): self.key, self.method, iso8601=True) key = self.key - if not isinstance(key, six.binary_type): + if not isinstance(key, bytes): key = key.encode('utf-8') expires = strftime(u.EXPIRES_ISO8601_FORMAT, gmtime(1400003600)) - if not isinstance(self.url, six.string_types): + if not isinstance(self.url, str): self.assertTrue(url.endswith(bytes(expires, 'utf-8'))) else: self.assertTrue(url.endswith(expires)) @@ -280,7 +280,7 @@ def test_generate_temp_url_prefix(self, time_mock, hmac_mock): url = u.generate_temp_url(path, self.seconds, self.key, self.method, prefix=True) key = self.key - if not isinstance(key, six.binary_type): + if not isinstance(key, bytes): key = key.encode('utf-8') self.assertEqual(url, expected_url) self.assertEqual(hmac_mock.mock_calls, [ @@ -299,7 +299,7 @@ def test_generate_temp_url_invalid_path(self): @mock.patch('hmac.HMAC.hexdigest', return_value="temp_url_signature") def test_generate_absolute_expiry_temp_url(self, hmac_mock): - if isinstance(self.expected_url, six.binary_type): + if isinstance(self.expected_url, bytes): expected_url = self.expected_url.replace( b'1400003600', b'2146636800') else: @@ -486,7 +486,7 @@ def test_unicode(self): class TestLengthWrapper(unittest.TestCase): def test_stringio(self): - contents = six.StringIO(u'a' * 50 + u'b' * 50) + contents = io.StringIO(u'a' * 50 + u'b' * 50) contents.seek(22) data = u.LengthWrapper(contents, 42, True) s = u'a' * 28 + u'b' * 14 @@ -506,7 +506,7 @@ def test_stringio(self): self.assertEqual(md5(s.encode()).hexdigest(), data.get_md5sum()) def test_bytesio(self): - contents = six.BytesIO(b'a' * 50 + b'b' * 50) + contents = io.BytesIO(b'a' * 50 + b'b' * 50) contents.seek(22) data = u.LengthWrapper(contents, 42, True) s = b'a' * 28 + b'b' * 14 @@ -613,7 +613,7 @@ def test_latin_1(self): self.assertEqual({u't\xe9st': u'\xff'}, result) def test_gzipped_utf8(self): - buf = six.BytesIO() + buf = io.BytesIO() gz = gzip.GzipFile(fileobj=buf, mode='w') gz.write(u'{"test": "\u2603"}'.encode('utf8')) gz.close() @@ -631,7 +631,7 @@ def test_not_gzipped(self): self.assertEqual({'test': u'\u2603'}, result) def test_gzipped_body(self): - buf = six.BytesIO() + buf = io.BytesIO() gz = gzip.GzipFile(fileobj=buf, mode='w') gz.write(u'{"test": "\u2603"}'.encode('utf8')) gz.close() diff --git a/test/unit/utils.py b/test/unit/utils.py index 3190e9d2..0dc0b3c9 100644 --- a/test/unit/utils.py +++ b/test/unit/utils.py @@ -12,17 +12,19 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. + import functools +import io +import importlib +import os import sys -from requests import RequestException -from requests.structures import CaseInsensitiveDict from time import sleep import unittest + +from requests import RequestException +from requests.structures import CaseInsensitiveDict import mock -import six -import os -from six.moves import reload_module -from six.moves.urllib.parse import urlparse, ParseResult +from urllib.parse import urlparse, ParseResult from swiftclient import client as c from swiftclient import shell as s from swiftclient.utils import EMPTY_ETAG @@ -406,7 +408,7 @@ def tearDown(self): # un-hygienic mocking on the swiftclient.client module; which may lead # to some unfortunate test order dependency bugs by way of the broken # window theory if any other modules are similarly patched - reload_module(c) + importlib.reload(c) class CaptureStreamPrinter(object): @@ -421,24 +423,20 @@ def write(self, data): # No encoding, just convert the raw bytes into a str for testing # The below call also validates that we have a byte string. self._captured_stream.write( - data if isinstance(data, six.binary_type) else data.encode('utf8')) + data if isinstance(data, bytes) else data.encode('utf8')) class CaptureStream(object): def __init__(self, stream): self.stream = stream - self._buffer = six.BytesIO() + self._buffer = io.BytesIO() self._capture = CaptureStreamPrinter(self._buffer) self.streams = [self._capture] @property def buffer(self): - if six.PY3: - return self._buffer - else: - raise AttributeError( - 'Output stream has no attribute "buffer" in Python2') + return self._buffer def flush(self): pass From 61ce5ac8244206c5e469e24459e365a5b0767dd5 Mon Sep 17 00:00:00 2001 From: Stephen Finucane Date: Mon, 21 Mar 2022 18:19:50 +0000 Subject: [PATCH 150/238] Remove unnecessary object subclassing All classes subclass from object by default in Python 3. Signed-off-by: Stephen Finucane Change-Id: I5a1ad57bcc092861ce969759b06a07c880ad3d35 --- swiftclient/authv1.py | 4 ++-- swiftclient/client.py | 6 +++--- swiftclient/multithreading.py | 4 ++-- swiftclient/service.py | 12 ++++++------ swiftclient/utils.py | 6 +++--- test/unit/test_authv1.py | 4 ++-- test/unit/test_swiftclient.py | 12 ++++++------ test/unit/test_utils.py | 2 +- 8 files changed, 25 insertions(+), 25 deletions(-) diff --git a/swiftclient/authv1.py b/swiftclient/authv1.py index 705dcb40..84bc38ae 100644 --- a/swiftclient/authv1.py +++ b/swiftclient/authv1.py @@ -68,7 +68,7 @@ def dst(self, dt): del _UTC -class ServiceCatalogV1(object): +class ServiceCatalogV1: def __init__(self, auth_url, storage_url, account): self.auth_url = auth_url self._storage_url = storage_url @@ -148,7 +148,7 @@ def endpoint_data_for(self, **kwargs): raise exceptions.EndpointNotFound(msg) -class AccessInfoV1(object): +class AccessInfoV1: """An object for encapsulating a raw v1 auth token.""" def __init__(self, auth_url, storage_url, account, username, auth_token, diff --git a/swiftclient/client.py b/swiftclient/client.py index df5344df..a9130ab5 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -217,7 +217,7 @@ def encode_meta_headers(headers): return ret -class _ObjectBody(object): +class _ObjectBody: """ Readable and iterable object body response wrapper. """ @@ -331,7 +331,7 @@ def read(self, length=None): return buf -class HTTPConnection(object): +class HTTPConnection: def __init__(self, url, proxy=None, cacert=None, insecure=False, cert=None, cert_key=None, ssl_compression=False, default_user_agent=None, timeout=None): @@ -1634,7 +1634,7 @@ def get_capabilities(http_conn): return parse_api_response(resp_headers, body) -class Connection(object): +class Connection: """ Convenience class to make requests that will also retry the request diff --git a/swiftclient/multithreading.py b/swiftclient/multithreading.py index cf72360e..eaccec68 100644 --- a/swiftclient/multithreading.py +++ b/swiftclient/multithreading.py @@ -19,7 +19,7 @@ from queue import PriorityQueue -class OutputManager(object): +class OutputManager: """ One object to manage and provide helper functions for output. @@ -108,7 +108,7 @@ def warning(self, msg, *fmt_args): self.error_print_pool.submit(self._print_error, msg, count=0) -class MultiThreadingManager(object): +class MultiThreadingManager: """ One object to manage context for multi-threading. This should make bin/swift less error-prone and allow us to test this code. diff --git a/swiftclient/service.py b/swiftclient/service.py index 289e29e8..4a7b1205 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -315,7 +315,7 @@ def split_headers(options, prefix=''): return headers -class SwiftUploadObject(object): +class SwiftUploadObject: """ Class for specifying an object upload, allowing the object source, name and options to be specified separately for each individual object. @@ -341,7 +341,7 @@ def __init__(self, source, object_name=None, options=None): self.source = source -class SwiftPostObject(object): +class SwiftPostObject: """ Class for specifying an object post, allowing the headers/metadata to be specified separately for each individual object. @@ -355,7 +355,7 @@ def __init__(self, object_name, options=None): self.options = options -class SwiftDeleteObject(object): +class SwiftDeleteObject: """ Class for specifying an object delete, allowing the headers/metadata to be specified separately for each individual object. @@ -369,7 +369,7 @@ def __init__(self, object_name, options=None): self.options = options -class SwiftCopyObject(object): +class SwiftCopyObject: """ Class for specifying an object copy, allowing the destination/headers/metadata/fresh_metadata to be specified @@ -405,7 +405,7 @@ def __init__(self, object_name, options=None): ) -class _SwiftReader(object): +class _SwiftReader: """ Class for downloading objects from swift and raising appropriate errors on failures caused by either invalid md5sum or size of the @@ -470,7 +470,7 @@ def bytes_read(self): return self._actual_read -class SwiftService(object): +class SwiftService: """ Service for performing swift operations """ diff --git a/swiftclient/utils.py b/swiftclient/utils.py index 03e5e7b2..e99ed37f 100644 --- a/swiftclient/utils.py +++ b/swiftclient/utils.py @@ -254,7 +254,7 @@ def report_traceback(): return None, None -class NoopMD5(object): +class NoopMD5: def __init__(self, *a, **kw): pass @@ -265,7 +265,7 @@ def hexdigest(self, *a, **kw): return '' -class ReadableToIterable(object): +class ReadableToIterable: """ Wrap a filelike object and act as an iterator. @@ -314,7 +314,7 @@ def __iter__(self): return self -class LengthWrapper(object): +class LengthWrapper: """ Wrap a filelike object with a maximum length. diff --git a/test/unit/test_authv1.py b/test/unit/test_authv1.py index 2ddf24b3..c16227fb 100644 --- a/test/unit/test_authv1.py +++ b/test/unit/test_authv1.py @@ -22,7 +22,7 @@ from swiftclient import authv1 -class TestDataNoAccount(object): +class TestDataNoAccount: options = dict( auth_url='http://saio:8080/auth/v1.0', username='test:tester', @@ -32,7 +32,7 @@ class TestDataNoAccount(object): token = 'token' -class TestDataWithAccount(object): +class TestDataWithAccount: options = dict( auth_url='http://saio:8080/auth/v1.0', username='test2:tester2', diff --git a/test/unit/test_swiftclient.py b/test/unit/test_swiftclient.py index 55c18fab..673f65fc 100644 --- a/test/unit/test_swiftclient.py +++ b/test/unit/test_swiftclient.py @@ -101,7 +101,7 @@ def test_transaction_id_from_headers(self): self.assertIn('(txn: some-other-id)', str(exc)) -class MockHttpResponse(object): +class MockHttpResponse: def __init__(self, status=0, headers=None, verify=False): self.status = status self.status_code = status @@ -115,7 +115,7 @@ def __init__(self, status=0, headers=None, verify=False): self.headers.update(headers) self.closed = False - class Raw(object): + class Raw: def __init__(self, headers): self.headers = headers @@ -586,10 +586,10 @@ def test_auth_v3applicationcredential(self): "application_credential_id": "proejct_id", "application_credential_secret": "secret"} - class FakeEndpointData(object): + class FakeEndpointData: catalog_url = 'http://swift.cluster/v1/KEY_project_id' - class FakeKeystoneuth1v3Session(object): + class FakeKeystoneuth1v3Session: def __init__(self, auth): self.auth = auth @@ -2543,7 +2543,7 @@ def shim_connection(*a, **kw): def test_reset_stream(self): - class LocalContents(object): + class LocalContents: def __init__(self, tell_value=0): self.data = io.BytesIO(string.ascii_letters.encode() * 10) @@ -2565,7 +2565,7 @@ def read(self, size=-1): self.reads.append((size, read_data)) return read_data - class LocalConnection(object): + class LocalConnection: def __init__(self, parsed_url=None): self.reason = "" diff --git a/test/unit/test_utils.py b/test/unit/test_utils.py index 02d19f3c..007b91e8 100644 --- a/test/unit/test_utils.py +++ b/test/unit/test_utils.py @@ -641,7 +641,7 @@ def test_gzipped_body(self): self.assertEqual({'test': u'\u2603'}, result) -class JSONTracker(object): +class JSONTracker: def __init__(self, data): self.data = data self.calls = [] From 20d837a27665632cc5b164b96671290b04c48a58 Mon Sep 17 00:00:00 2001 From: Stephen Finucane Date: Mon, 21 Mar 2022 18:17:41 +0000 Subject: [PATCH 151/238] Remove unnecessary unicode prefixes All strings are unicode by default in Python 3. No need to mark them as such. Signed-off-by: Stephen Finucane Change-Id: I68fb60ef271abfddebcc9d2137424f5db2a17e92 --- swiftclient/utils.py | 8 +- test/functional/test_swiftclient.py | 2 +- test/unit/test_multithreading.py | 14 +-- test/unit/test_service.py | 4 +- test/unit/test_shell.py | 4 +- test/unit/test_swiftclient.py | 136 ++++++++++++++-------------- test/unit/test_utils.py | 70 +++++++------- 7 files changed, 119 insertions(+), 119 deletions(-) diff --git a/swiftclient/utils.py b/swiftclient/utils.py index e99ed37f..860d8bf4 100644 --- a/swiftclient/utils.py +++ b/swiftclient/utils.py @@ -172,7 +172,7 @@ def generate_temp_url(path, seconds, key, method, absolute=False, ) hmac_parts.insert(0, "ip=%s" % ip_range) - hmac_body = u'\n'.join(hmac_parts) + hmac_body = '\n'.join(hmac_parts) # Encode to UTF-8 for py3 compatibility if not isinstance(key, bytes): @@ -183,14 +183,14 @@ def generate_temp_url(path, seconds, key, method, absolute=False, expiration = time.strftime( EXPIRES_ISO8601_FORMAT, time.gmtime(expiration)) - temp_url = u'{path}?temp_url_sig={sig}&temp_url_expires={exp}'.format( + temp_url = '{path}?temp_url_sig={sig}&temp_url_expires={exp}'.format( path=path_for_body, sig=sig, exp=expiration) if ip_range: - temp_url += u'&temp_url_ip_range={}'.format(ip_range) + temp_url += '&temp_url_ip_range={}'.format(ip_range) if prefix: - temp_url += u'&temp_url_prefix={}'.format(parts[4]) + temp_url += '&temp_url_prefix={}'.format(parts[4]) # Have return type match path from caller if isinstance(path, bytes): return temp_url.encode('utf-8') diff --git a/test/functional/test_swiftclient.py b/test/functional/test_swiftclient.py index 5fc8df75..91e31af0 100644 --- a/test/functional/test_swiftclient.py +++ b/test/functional/test_swiftclient.py @@ -409,7 +409,7 @@ def test_post_object(self): def test_post_object_unicode_header_name(self): self.conn.post_object(self.containername, self.objectname, - {u'x-object-meta-\U0001f44d': u'\U0001f44d'}) + {'x-object-meta-\U0001f44d': '\U0001f44d'}) # Note that we can't actually read this header back on py3; see # https://bugs.python.org/issue37093 diff --git a/test/unit/test_multithreading.py b/test/unit/test_multithreading.py index ee9e7cc5..d51bfb7c 100644 --- a/test/unit/test_multithreading.py +++ b/test/unit/test_multithreading.py @@ -192,18 +192,18 @@ def test_printers(self): thread_manager.print_msg('one-argument') thread_manager.print_msg('one %s, %d fish', 'fish', 88) thread_manager.error('I have %d problems, but a %s is not one', - 99, u'\u062A\u062A') + 99, '\u062A\u062A') thread_manager.print_msg('some\n%s\nover the %r', 'where', - u'\u062A\u062A') + '\u062A\u062A') thread_manager.error('one-error-argument') thread_manager.error('Sometimes\n%.1f%% just\ndoes not\nwork!', 3.14159) thread_manager.print_raw( - u'some raw bytes: \u062A\u062A'.encode('utf-8')) + 'some raw bytes: \u062A\u062A'.encode('utf-8')) thread_manager.print_items([ ('key', 'value'), - ('object', u'O\u0308bject'), + ('object', 'O\u0308bject'), ]) thread_manager.print_raw(b'\xffugly\xffraw') @@ -222,13 +222,13 @@ def test_printers(self): 'one fish, 88 fish\n', 'some\n', 'where\n', over_the, - u'some raw bytes: \u062a\u062a', + 'some raw bytes: \u062a\u062a', ' key: value\n', - u' object: O\u0308bject\n' + ' object: O\u0308bject\n' ]).encode('utf8') + b'\xffugly\xffraw', out_stream.getvalue()) self.assertEqual(''.join([ - u'I have 99 problems, but a \u062A\u062A is not one\n', + 'I have 99 problems, but a \u062A\u062A is not one\n', 'one-error-argument\n', 'Sometimes\n', '3.1% just\n', 'does not\n', 'work!\n' ]), err_stream.getvalue().decode('utf8')) diff --git a/test/unit/test_service.py b/test/unit/test_service.py index 9f3a2a2f..0ad53782 100644 --- a/test/unit/test_service.py +++ b/test/unit/test_service.py @@ -564,7 +564,7 @@ def test_bulk_delete(self, mock_connection_class): stub_headers, json.dumps(stub_resp).encode('utf8')) obj_list = ['x%02d' % i for i in range(100)] expected = [{ - 'action': u'bulk_delete', + 'action': 'bulk_delete', 'attempts': 0, 'container': 'c', 'objects': list(objs), @@ -594,7 +594,7 @@ def test_bulk_delete_versions(self, mock_connection_class): obj_list = [SwiftDeleteObject('x%02d' % i, options={'version_id': i}) for i in range(100)] expected = [{ - 'action': u'delete_object', + 'action': 'delete_object', 'attempts': 0, 'container': 'c', 'object': obj.object_name, diff --git a/test/unit/test_shell.py b/test/unit/test_shell.py index adf37767..eb70a923 100644 --- a/test/unit/test_shell.py +++ b/test/unit/test_shell.py @@ -400,7 +400,7 @@ def test_list_account_with_versions(self): def test_list_json(self, connection): connection.return_value.get_account.side_effect = [ [None, [{'name': 'container'}]], - [None, [{'name': u'\u263A', 'some-custom-key': 'and value'}]], + [None, [{'name': '\u263A', 'some-custom-key': 'and value'}]], [None, []], ] @@ -412,7 +412,7 @@ def test_list_json(self, connection): connection.return_value.get_account.assert_has_calls(calls) listing = [{'name': 'container'}, - {'name': u'\u263A', 'some-custom-key': 'and value'}] + {'name': '\u263A', 'some-custom-key': 'and value'}] expected = json.dumps(listing, sort_keys=True, indent=2) + '\n' self.assertEqual(output.out, expected) diff --git a/test/unit/test_swiftclient.py b/test/unit/test_swiftclient.py index 673f65fc..3b591662 100644 --- a/test/unit/test_swiftclient.py +++ b/test/unit/test_swiftclient.py @@ -163,34 +163,34 @@ def test_quote(self): self.assertEqual('bytes%FF', c.quote(value)) value = 'native string' self.assertEqual('native%20string', c.quote(value)) - value = u'unicode string' + value = 'unicode string' self.assertEqual('unicode%20string', c.quote(value)) - value = u'unicode:\xe9\u20ac' + value = 'unicode:\xe9\u20ac' self.assertEqual('unicode%3A%C3%A9%E2%82%AC', c.quote(value)) def test_parse_header_string(self): value = b'bytes' - self.assertEqual(u'bytes', c.parse_header_string(value)) - value = u'unicode:\xe9\u20ac' - self.assertEqual(u'unicode:\xe9\u20ac', c.parse_header_string(value)) + self.assertEqual('bytes', c.parse_header_string(value)) + value = 'unicode:\xe9\u20ac' + self.assertEqual('unicode:\xe9\u20ac', c.parse_header_string(value)) value = 'native%20string' - self.assertEqual(u'native string', c.parse_header_string(value)) + self.assertEqual('native string', c.parse_header_string(value)) value = b'encoded%20bytes%E2%82%AC' - self.assertEqual(u'encoded bytes\u20ac', c.parse_header_string(value)) + self.assertEqual('encoded bytes\u20ac', c.parse_header_string(value)) value = 'encoded%20unicode%E2%82%AC' - self.assertEqual(u'encoded unicode\u20ac', + self.assertEqual('encoded unicode\u20ac', c.parse_header_string(value)) value = b'bad%20bytes%ff%E2%82%AC' - self.assertEqual(u'bad%20bytes%ff%E2%82%AC', + self.assertEqual('bad%20bytes%ff%E2%82%AC', c.parse_header_string(value)) - value = u'bad%20unicode%ff\u20ac' - self.assertEqual(u'bad%20unicode%ff\u20ac', + value = 'bad%20unicode%ff\u20ac' + self.assertEqual('bad%20unicode%ff\u20ac', c.parse_header_string(value)) value = b'really%20bad\xffbytes' - self.assertEqual(u'really%2520bad%FFbytes', + self.assertEqual('really%2520bad%FFbytes', c.parse_header_string(value)) def test_http_connection(self): @@ -205,9 +205,9 @@ def test_http_connection(self): def test_encode_meta_headers(self): headers = {'abc': '123', - u'x-container-meta-\u0394': 123, - u'x-account-meta-\u0394': 12.3, - u'x-object-meta-\u0394': True} + 'x-container-meta-\u0394': 123, + 'x-account-meta-\u0394': 12.3, + 'x-object-meta-\u0394': True} r = swiftclient.encode_meta_headers(headers) @@ -1111,9 +1111,9 @@ def test_response_headers(self): conn = c.http_connection('http://www.test.com') headers, data = c.get_object('url_is_irrelevant', 'TOKEN', 'container', 'object', http_conn=conn) - self.assertEqual(u't\xe9st', headers.get('x-utf-8-header', '')) - self.assertEqual(u'%ff', headers.get('x-non-utf-8-header', '')) - self.assertEqual(u'%FF', headers.get('x-binary-header', '')) + self.assertEqual('t\xe9st', headers.get('x-utf-8-header', '')) + self.assertEqual('%ff', headers.get('x-non-utf-8-header', '')) + self.assertEqual('%FF', headers.get('x-binary-header', '')) def test_chunk_size_read_method(self): conn = c.Connection('http://auth.url/', 'some_user', 'some_key') @@ -1338,14 +1338,14 @@ def test_ok(self): ]) def test_unicode_ok(self): - conn = c.http_connection(u'http://www.test.com/') - mock_file = io.StringIO(u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91') - args = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', - u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', - u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', - u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', + conn = c.http_connection('http://www.test.com/') + mock_file = io.StringIO('\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91') + args = ('\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', + '\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', + '\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', + '\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', mock_file) - text = u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' + text = '\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' headers = {'X-Header1': text, 'X-2': '1', 'X-3': "{'a': 'b'}", 'a-b': '.x:yz mn:fg:lp'} @@ -1410,7 +1410,7 @@ def test_query_string(self): def test_raw_upload(self): # Raw upload happens when content_length is passed to put_object - conn = c.http_connection(u'http://www.test.com/') + conn = c.http_connection('http://www.test.com/') resp = MockHttpResponse(status=200) conn[1].getresponse = resp.fake_response conn[1]._request = resp._fake_request @@ -1432,7 +1432,7 @@ def test_raw_upload(self): def test_chunk_upload(self): # Chunked upload happens when no content_length is passed to put_object - conn = c.http_connection(u'http://www.test.com/') + conn = c.http_connection('http://www.test.com/') resp = MockHttpResponse(status=200) conn[1].getresponse = resp.fake_response conn[1]._request = resp._fake_request @@ -1457,7 +1457,7 @@ def test_iter_upload(self): def data(): for chunk in ('foo', '', 'bar'): yield chunk - conn = c.http_connection(u'http://www.test.com/') + conn = c.http_connection('http://www.test.com/') resp = MockHttpResponse(status=200) conn[1].getresponse = resp.fake_response conn[1]._request = resp._fake_request @@ -1524,7 +1524,7 @@ def test_md5_match(self): self.assertEqual(etag, contents.get_md5sum()) def test_params(self): - conn = c.http_connection(u'http://www.test.com/') + conn = c.http_connection('http://www.test.com/') resp = MockHttpResponse(status=200) conn[1].getresponse = resp.fake_response conn[1]._request = resp._fake_request @@ -1535,8 +1535,8 @@ def test_params(self): self.assertEqual(request_header['etag'], b'1234-5678') self.assertEqual(request_header['content-type'], b'text/plain') - def test_no_content_type_requests(self): - conn = c.http_connection(u'http://www.test.com/') + def test_no_content_type(self): + conn = c.http_connection('http://www.test.com/') resp = MockHttpResponse(status=200) conn[1].getresponse = resp.fake_response conn[1]._request = resp._fake_request @@ -1546,7 +1546,7 @@ def test_no_content_type_requests(self): self.assertNotIn('content-type', request_header) def test_content_type_in_headers(self): - conn = c.http_connection(u'http://www.test.com/') + conn = c.http_connection('http://www.test.com/') resp = MockHttpResponse(status=200) conn[1].getresponse = resp.fake_response conn[1]._request = resp._fake_request @@ -1586,12 +1586,12 @@ def test_ok(self): }) def test_unicode_ok(self): - conn = c.http_connection(u'http://www.test.com/') - args = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', - u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', - u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', - u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91') - text = u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' + conn = c.http_connection('http://www.test.com/') + args = ('\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', + '\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', + '\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91', + '\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91') + text = '\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' headers = {'X-Header1': text, b'X-Header2': 'value', 'X-2': '1', 'X-3': "{'a': 'b'}", 'a-b': '.x:yz mn:kl:qr', @@ -1890,66 +1890,66 @@ def test_conn_get_capabilities_with_os_options(self): class TestHTTPConnection(MockHttpTest): def test_bad_url_scheme(self): - url = u'www.test.com' + url = 'www.test.com' with self.assertRaises(c.ClientException) as exc_context: c.http_connection(url) exc = exc_context.exception - expected = u'Unsupported scheme "" in url "www.test.com"' + expected = 'Unsupported scheme "" in url "www.test.com"' self.assertEqual(expected, str(exc)) - url = u'://www.test.com' + url = '://www.test.com' with self.assertRaises(c.ClientException) as exc_context: c.http_connection(url) exc = exc_context.exception - expected = u'Unsupported scheme "" in url "://www.test.com"' + expected = 'Unsupported scheme "" in url "://www.test.com"' self.assertEqual(expected, str(exc)) - url = u'blah://www.test.com' + url = 'blah://www.test.com' with self.assertRaises(c.ClientException) as exc_context: c.http_connection(url) exc = exc_context.exception - expected = u'Unsupported scheme "blah" in url "blah://www.test.com"' + expected = 'Unsupported scheme "blah" in url "blah://www.test.com"' self.assertEqual(expected, str(exc)) def test_ok_url_scheme(self): for scheme in ('http', 'https', 'HTTP', 'HTTPS'): - url = u'%s://www.test.com' % scheme + url = '%s://www.test.com' % scheme parsed_url, conn = c.http_connection(url) self.assertEqual(scheme.lower(), parsed_url.scheme) - self.assertEqual(u'%s://www.test.com' % scheme, conn.url) + self.assertEqual('%s://www.test.com' % scheme, conn.url) def test_ok_proxy(self): - conn = c.http_connection(u'http://www.test.com/', + conn = c.http_connection('http://www.test.com/', proxy='http://localhost:8080') self.assertEqual(conn[1].requests_args['proxies']['http'], 'http://localhost:8080') def test_bad_proxy(self): try: - c.http_connection(u'http://www.test.com/', proxy='localhost:8080') + c.http_connection('http://www.test.com/', proxy='localhost:8080') except c.ClientException as e: self.assertEqual(e.msg, "Proxy's missing scheme") def test_cacert(self): - conn = c.http_connection(u'http://www.test.com/', + conn = c.http_connection('http://www.test.com/', cacert='/dev/urandom') self.assertEqual(conn[1].requests_args['verify'], '/dev/urandom') def test_insecure(self): - conn = c.http_connection(u'http://www.test.com/', insecure=True) + conn = c.http_connection('http://www.test.com/', insecure=True) self.assertEqual(conn[1].requests_args['verify'], False) def test_cert(self): - conn = c.http_connection(u'http://www.test.com/', cert='minnie') + conn = c.http_connection('http://www.test.com/', cert='minnie') self.assertEqual(conn[1].requests_args['cert'], 'minnie') def test_cert_key(self): conn = c.http_connection( - u'http://www.test.com/', cert='minnie', cert_key='mickey') + 'http://www.test.com/', cert='minnie', cert_key='mickey') self.assertEqual(conn[1].requests_args['cert'], ('minnie', 'mickey')) def test_response_connection_released(self): - _parsed_url, conn = c.http_connection(u'http://www.test.com/') + _parsed_url, conn = c.http_connection('http://www.test.com/') conn.resp = MockHttpResponse() conn.resp.raw = mock.Mock() conn.resp.raw.read.side_effect = ["Chunk", ""] @@ -1962,7 +1962,7 @@ def test_response_connection_released(self): def test_response_headers(self): '''Test latin1-encoded headers. ''' - _, conn = c.http_connection(u'http://www.test.com/') + _, conn = c.http_connection('http://www.test.com/') conn.resp = MockHttpResponse( status=200, headers={ @@ -2860,7 +2860,7 @@ def test_get_error(self): def test_content_encoding_gzip_body_is_logged_decoded(self): buf = io.BytesIO() gz = gzip.GzipFile(fileobj=buf, mode='w') - data = {"test": u"\u2603"} + data = {"test": "\u2603"} decoded_body = json.dumps(data).encode('utf-8') gz.write(decoded_body) gz.close() @@ -2877,7 +2877,7 @@ def test_content_encoding_gzip_body_is_logged_decoded(self): self.assertEqual(exc_context.exception.http_status, 500) # it will log the decoded body self.assertEqual([ - mock.call('REQ: %s', u'curl -i http://www.test.com/asdf/asdf ' + mock.call('REQ: %s', 'curl -i http://www.test.com/asdf/asdf ' '-X GET -H "X-Auth-Token: ..."'), mock.call('RESP STATUS: %s %s', 500, 'Fake'), mock.call('RESP HEADERS: %s', {'content-encoding': 'gzip'}), @@ -2888,9 +2888,9 @@ def test_redact_token(self): with mock.patch('swiftclient.client.logger.debug') as mock_log: token_value = 'tkee96b40a8ca44fc5ad72ec5a7c90d9b' token_encoded = token_value.encode('utf8') - unicode_token_value = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' - u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' - u'\u5929\u7a7a\u4e2d\u7684\u4e4c') + unicode_token_value = ('\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' + '\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' + '\u5929\u7a7a\u4e2d\u7684\u4e4c') unicode_token_encoded = unicode_token_value.encode('utf8') set_cookie_value = 'X-Auth-Token=%s' % token_value set_cookie_encoded = set_cookie_value.encode('utf8') @@ -2914,8 +2914,8 @@ def test_redact_token(self): out = [] for _, args, kwargs in mock_log.mock_calls: for arg in args: - out.append(u'%s' % arg) - output = u''.join(out) + out.append('%s' % arg) + output = ''.join(out) self.assertIn('X-Auth-Token', output) self.assertIn(token_value[:16] + '...', output) self.assertIn('X-Storage-Token', output) @@ -2930,9 +2930,9 @@ def test_show_token(self): with mock.patch('swiftclient.client.logger.debug') as mock_log: token_value = 'tkee96b40a8ca44fc5ad72ec5a7c90d9b' token_encoded = token_value.encode('utf8') - unicode_token_value = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' - u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' - u'\u5929\u7a7a\u4e2d\u7684\u4e4c') + unicode_token_value = ('\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' + '\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' + '\u5929\u7a7a\u4e2d\u7684\u4e4c') c.logger_settings['redact_sensitive_headers'] = False unicode_token_encoded = unicode_token_value.encode('utf8') c.http_log( @@ -2954,8 +2954,8 @@ def test_show_token(self): out = [] for _, args, kwargs in mock_log.mock_calls: for arg in args: - out.append(u'%s' % arg) - output = u''.join(out) + out.append('%s' % arg) + output = ''.join(out) self.assertIn('X-Auth-Token', output) self.assertIn(token_value, output) self.assertIn('X-Storage-Token', output) @@ -2963,12 +2963,12 @@ def test_show_token(self): @mock.patch('swiftclient.client.logger.debug') def test_unicode_path(self, mock_log): - path = u'http://swift/v1/AUTH_account-\u062a'.encode('utf-8') + path = 'http://swift/v1/AUTH_account-\u062a'.encode('utf-8') c.http_log(['GET', path], {}, MockHttpResponse(status=200, headers=[]), '') request_log_line = mock_log.mock_calls[0] self.assertEqual('REQ: %s', request_log_line[1][0]) - self.assertEqual(u'curl -i -X GET %s' % path.decode('utf-8'), + self.assertEqual('curl -i -X GET %s' % path.decode('utf-8'), request_log_line[1][1]) diff --git a/test/unit/test_utils.py b/test/unit/test_utils.py index 007b91e8..38331546 100644 --- a/test/unit/test_utils.py +++ b/test/unit/test_utils.py @@ -304,7 +304,7 @@ def test_generate_absolute_expiry_temp_url(self, hmac_mock): b'1400003600', b'2146636800') else: expected_url = self.expected_url.replace( - u'1400003600', u'2146636800') + '1400003600', '2146636800') url = u.generate_temp_url(self.url, 2146636800, self.key, self.method, absolute=True) self.assertEqual(url, expected_url) @@ -372,32 +372,32 @@ def test_generate_temp_url_bad_path(self): class TestTempURLUnicodePathAndKey(TestTempURL): - url = u'/v1/\u00e4/c/\u00f3' - key = u'k\u00e9y' - expected_url = (u'%s?temp_url_sig=temp_url_signature' - u'&temp_url_expires=1400003600') % url - expected_body = u'\n'.join([ - u'GET', - u'1400003600', + url = '/v1/\u00e4/c/\u00f3' + key = 'k\u00e9y' + expected_url = ('%s?temp_url_sig=temp_url_signature' + '&temp_url_expires=1400003600') % url + expected_body = '\n'.join([ + 'GET', + '1400003600', url, ]).encode('utf-8') class TestTempURLUnicodePathBytesKey(TestTempURL): - url = u'/v1/\u00e4/c/\u00f3' - key = u'k\u00e9y'.encode('utf-8') - expected_url = (u'%s?temp_url_sig=temp_url_signature' - u'&temp_url_expires=1400003600') % url + url = '/v1/\u00e4/c/\u00f3' + key = 'k\u00e9y'.encode('utf-8') + expected_url = ('%s?temp_url_sig=temp_url_signature' + '&temp_url_expires=1400003600') % url expected_body = '\n'.join([ - u'GET', - u'1400003600', + 'GET', + '1400003600', url, ]).encode('utf-8') class TestTempURLBytesPathUnicodeKey(TestTempURL): - url = u'/v1/\u00e4/c/\u00f3'.encode('utf-8') - key = u'k\u00e9y' + url = '/v1/\u00e4/c/\u00f3'.encode('utf-8') + key = 'k\u00e9y' expected_url = url + (b'?temp_url_sig=temp_url_signature' b'&temp_url_expires=1400003600') expected_body = b'\n'.join([ @@ -408,8 +408,8 @@ class TestTempURLBytesPathUnicodeKey(TestTempURL): class TestTempURLBytesPathAndKey(TestTempURL): - url = u'/v1/\u00e4/c/\u00f3'.encode('utf-8') - key = u'k\u00e9y'.encode('utf-8') + url = '/v1/\u00e4/c/\u00f3'.encode('utf-8') + key = 'k\u00e9y'.encode('utf-8') expected_url = url + (b'?temp_url_sig=temp_url_signature' b'&temp_url_expires=1400003600') expected_body = b'\n'.join([ @@ -420,7 +420,7 @@ class TestTempURLBytesPathAndKey(TestTempURL): class TestTempURLBytesPathAndNonUtf8Key(TestTempURL): - url = u'/v1/\u00e4/c/\u00f3'.encode('utf-8') + url = '/v1/\u00e4/c/\u00f3'.encode('utf-8') key = b'k\xffy' expected_url = url + (b'?temp_url_sig=temp_url_signature' b'&temp_url_expires=1400003600') @@ -463,7 +463,7 @@ def test_md5_creation(self): def test_unicode(self): # Check no errors are raised if unicode data is feed in. - unicode_data = u'abc' + unicode_data = 'abc' actual_md5sum = md5(unicode_data.encode()).hexdigest() chunk_size = 2 @@ -486,11 +486,11 @@ def test_unicode(self): class TestLengthWrapper(unittest.TestCase): def test_stringio(self): - contents = io.StringIO(u'a' * 50 + u'b' * 50) + contents = io.StringIO('a' * 50 + 'b' * 50) contents.seek(22) data = u.LengthWrapper(contents, 42, True) - s = u'a' * 28 + u'b' * 14 - read_data = u''.join(iter(data.read, '')) + s = 'a' * 28 + 'b' * 14 + read_data = ''.join(iter(data.read, '')) self.assertEqual(42, len(data)) self.assertEqual(42, len(read_data)) @@ -500,7 +500,7 @@ def test_stringio(self): data.reset() self.assertEqual(md5().hexdigest(), data.get_md5sum()) - read_data = u''.join(iter(data.read, '')) + read_data = ''.join(iter(data.read, '')) self.assertEqual(42, len(read_data)) self.assertEqual(s, read_data) self.assertEqual(md5(s.encode()).hexdigest(), data.get_md5sum()) @@ -591,12 +591,12 @@ class TestApiResponeParser(unittest.TestCase): def test_utf8_default(self): result = u.parse_api_response( - {}, u'{"test": "\u2603"}'.encode('utf8')) - self.assertEqual({'test': u'\u2603'}, result) + {}, '{"test": "\u2603"}'.encode('utf8')) + self.assertEqual({'test': '\u2603'}, result) result = u.parse_api_response( - {}, u'{"test": "\\u2603"}'.encode('utf8')) - self.assertEqual({'test': u'\u2603'}, result) + {}, '{"test": "\\u2603"}'.encode('utf8')) + self.assertEqual({'test': '\u2603'}, result) def test_bad_json(self): self.assertRaises(ValueError, u.parse_api_response, @@ -610,35 +610,35 @@ def test_latin_1(self): result = u.parse_api_response( {'content-type': 'application/json; charset=iso8859-1'}, b'{"t\xe9st": "\xff"}') - self.assertEqual({u't\xe9st': u'\xff'}, result) + self.assertEqual({'t\xe9st': '\xff'}, result) def test_gzipped_utf8(self): buf = io.BytesIO() gz = gzip.GzipFile(fileobj=buf, mode='w') - gz.write(u'{"test": "\u2603"}'.encode('utf8')) + gz.write('{"test": "\u2603"}'.encode('utf8')) gz.close() result = u.parse_api_response( {'content-encoding': 'gzip'}, buf.getvalue()) - self.assertEqual({'test': u'\u2603'}, result) + self.assertEqual({'test': '\u2603'}, result) class TestGetBody(unittest.TestCase): def test_not_gzipped(self): result = u.parse_api_response( - {}, u'{"test": "\\u2603"}'.encode('utf8')) - self.assertEqual({'test': u'\u2603'}, result) + {}, '{"test": "\\u2603"}'.encode('utf8')) + self.assertEqual({'test': '\u2603'}, result) def test_gzipped_body(self): buf = io.BytesIO() gz = gzip.GzipFile(fileobj=buf, mode='w') - gz.write(u'{"test": "\u2603"}'.encode('utf8')) + gz.write('{"test": "\u2603"}'.encode('utf8')) gz.close() result = u.parse_api_response( {'content-encoding': 'gzip'}, buf.getvalue()) - self.assertEqual({'test': u'\u2603'}, result) + self.assertEqual({'test': '\u2603'}, result) class JSONTracker: From 95f68cd673afe07aef9d14904de1c974fc684ef2 Mon Sep 17 00:00:00 2001 From: Pavel Abalikhin Date: Thu, 21 Apr 2022 17:08:17 +0300 Subject: [PATCH 152/238] Add timeout for Swift service Connection class has timeout parameter but SwiftService and shell don't use it. That can lead to an endless wait when network is unreachable. Change-Id: Iafa42fc2f8b56feefa2bc8ea6a1b8845717d3bab --- swiftclient/service.py | 1 + swiftclient/shell.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/swiftclient/service.py b/swiftclient/service.py index 4a7b1205..9d9fc594 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -273,6 +273,7 @@ def get_conn(options): return Connection(options['auth'], options['user'], options['key'], + timeout=options.get('timeout'), retries=options['retries'], auth_version=options['auth_version'], os_options=options['os_options'], diff --git a/swiftclient/shell.py b/swiftclient/shell.py index a16de884..df7c5115 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -1728,6 +1728,9 @@ def add_default_args(parser): parser.add_argument('-K', '--key', dest='key', default=environ.get('ST_KEY'), help='Key for obtaining an auth token.') + parser.add_argument('-T', '--timeout', type=int, dest='timeout', + default=None, + help='Timeout in seconds to wait for response.') parser.add_argument('-R', '--retries', type=int, default=5, dest='retries', help='The number of times to retry a failed ' 'connection.') From 20c97e83d3af855e5e238dfa4b10b7bf29533d57 Mon Sep 17 00:00:00 2001 From: Steve Kowalik Date: Tue, 24 May 2022 11:56:35 +1000 Subject: [PATCH 153/238] Remove use of mock Since Python 3.4, the unittest module has provided mock, negating the need for the external dependancy. Switch to using unittest.mock. Change-Id: Idec3aaed2fddd1ece3ed86ee0bcc48f7616d56fa --- test-requirements.txt | 1 - test/unit/test_authv1.py | 2 +- test/unit/test_command_helpers.py | 2 +- test/unit/test_service.py | 67 +++++++++++++++---------------- test/unit/test_shell.py | 2 +- test/unit/test_swiftclient.py | 2 +- test/unit/test_utils.py | 2 +- test/unit/utils.py | 2 +- 8 files changed, 39 insertions(+), 41 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index c2fb2c6e..b0633eb9 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -3,6 +3,5 @@ hacking>=3.2.0,<3.3.0;python_version>='3.0' # Apache-2.0 coverage!=4.4,>=4.0 # Apache-2.0 keystoneauth1>=3.4.0 # Apache-2.0 -mock>=1.2.0 # BSD stestr>=2.0.0,!=3.0.0 # Apache-2.0 openstacksdk>=0.11.0 # Apache-2.0 diff --git a/test/unit/test_authv1.py b/test/unit/test_authv1.py index c16227fb..b4de7e0f 100644 --- a/test/unit/test_authv1.py +++ b/test/unit/test_authv1.py @@ -14,8 +14,8 @@ import datetime import json -import mock import unittest +from unittest import mock from keystoneauth1 import plugin from keystoneauth1 import loading from keystoneauth1 import exceptions diff --git a/test/unit/test_command_helpers.py b/test/unit/test_command_helpers.py index 1cb3bb12..3e51aa90 100644 --- a/test/unit/test_command_helpers.py +++ b/test/unit/test_command_helpers.py @@ -13,9 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import mock from io import StringIO import unittest +from unittest import mock from swiftclient import command_helpers as h from swiftclient.multithreading import OutputManager diff --git a/test/unit/test_service.py b/test/unit/test_service.py index 0ad53782..1176a1f8 100644 --- a/test/unit/test_service.py +++ b/test/unit/test_service.py @@ -16,17 +16,16 @@ import builtins import contextlib import io -import mock import os import tempfile import unittest import time import json from io import BytesIO +from unittest import mock from concurrent.futures import Future from hashlib import md5 -from mock import Mock, PropertyMock from queue import Queue, Empty as QueueEmptyError from time import sleep @@ -215,9 +214,9 @@ def _consume(sr): class _TestServiceBase(unittest.TestCase): def _get_mock_connection(self, attempts=2): - m = Mock(spec=Connection) - type(m).attempts = PropertyMock(return_value=attempts) - type(m).auth_end_time = PropertyMock(return_value=4) + m = mock.Mock(spec=Connection) + type(m).attempts = mock.PropertyMock(return_value=attempts) + type(m).auth_end_time = mock.PropertyMock(return_value=4) return m def _get_queue(self, q): @@ -272,7 +271,7 @@ def test_delete_segment(self): def test_delete_segment_exception(self): mock_q = Queue() mock_conn = self._get_mock_connection() - mock_conn.delete_object = Mock(side_effect=self.exc) + mock_conn.delete_object = mock.Mock(side_effect=self.exc) expected_r = self._get_expected({ 'action': 'delete_segment', 'object': 'test_s', @@ -298,7 +297,7 @@ def test_delete_segment_exception(self): def test_delete_object(self): mock_q = Queue() mock_conn = self._get_mock_connection() - mock_conn.head_object = Mock(return_value={}) + mock_conn.head_object = mock.Mock(return_value={}) expected_r = self._get_expected({ 'action': 'delete_object', 'success': True @@ -346,7 +345,7 @@ def test_delete_object_version(self, mock_connection_class): def test_delete_object_with_headers(self): mock_q = Queue() mock_conn = self._get_mock_connection() - mock_conn.head_object = Mock(return_value={}) + mock_conn.head_object = mock.Mock(return_value={}) expected_r = self._get_expected({ 'action': 'delete_object', 'success': True @@ -369,7 +368,7 @@ def test_delete_object_with_headers(self): def test_delete_object_exception(self): mock_q = Queue() mock_conn = self._get_mock_connection() - mock_conn.delete_object = Mock(side_effect=self.exc) + mock_conn.delete_object = mock.Mock(side_effect=self.exc) expected_r = self._get_expected({ 'action': 'delete_object', 'success': False, @@ -402,7 +401,7 @@ def test_delete_object_slo_support(self): # additional query string to cause the right delete server side mock_q = Queue() mock_conn = self._get_mock_connection() - mock_conn.head_object = Mock( + mock_conn.head_object = mock.Mock( return_value={'x-static-large-object': True} ) expected_r = self._get_expected({ @@ -435,10 +434,10 @@ def test_delete_object_dlo_support(self): # A DLO object is determined in _delete_object by heading the object # and checking for the existence of a x-object-manifest header. # Mock that here. - mock_conn.head_object = Mock( + mock_conn.head_object = mock.Mock( return_value={'x-object-manifest': 'manifest_c/manifest_p'} ) - mock_conn.get_container = Mock( + mock_conn.get_container = mock.Mock( side_effect=[(None, [{'name': 'test_seg_1'}, {'name': 'test_seg_2'}]), (None, {})] @@ -495,7 +494,7 @@ def test_delete_empty_container_with_headers(self): def test_delete_empty_container_exception(self): mock_conn = self._get_mock_connection() - mock_conn.delete_container = Mock(side_effect=self.exc) + mock_conn.delete_container = mock.Mock(side_effect=self.exc) expected_r = self._get_expected({ 'action': 'delete_container', 'success': False, @@ -825,7 +824,7 @@ def test_list_account(self): (None, [{'name': 'test_c'}]), (None, []) ] - mock_conn.get_account = Mock(side_effect=get_account_returns) + mock_conn.get_account = mock.Mock(side_effect=get_account_returns) expected_r = self._get_expected({ 'action': 'list_account_part', @@ -841,12 +840,12 @@ def test_list_account(self): self.assertIsNone(self._get_queue(mock_q)) long_opts = dict(self.opts, **{'long': True}) - mock_conn.head_container = Mock(return_value={'test_m': '1'}) + mock_conn.head_container = mock.Mock(return_value={'test_m': '1'}) get_account_returns = [ (None, [{'name': 'test_c'}]), (None, []) ] - mock_conn.get_account = Mock(side_effect=get_account_returns) + mock_conn.get_account = mock.Mock(side_effect=get_account_returns) expected_r_long = self._get_expected({ 'action': 'list_account_part', @@ -868,7 +867,7 @@ def test_list_account_with_headers(self): (None, [{'name': 'test_c'}]), (None, []) ] - mock_conn.get_account = Mock(side_effect=get_account_returns) + mock_conn.get_account = mock.Mock(side_effect=get_account_returns) expected_r = self._get_expected({ 'action': 'list_account_part', @@ -892,7 +891,7 @@ def test_list_account_with_headers(self): def test_list_account_exception(self): mock_q = Queue() mock_conn = self._get_mock_connection() - mock_conn.get_account = Mock(side_effect=self.exc) + mock_conn.get_account = mock.Mock(side_effect=self.exc) expected_r = self._get_expected({ 'action': 'list_account_part', 'success': False, @@ -918,7 +917,7 @@ def test_list_container(self): (None, [{'name': 'test_o'}]), (None, []) ] - mock_conn.get_container = Mock(side_effect=get_container_returns) + mock_conn.get_container = mock.Mock(side_effect=get_container_returns) expected_r = self._get_expected({ 'action': 'list_container_part', @@ -935,12 +934,12 @@ def test_list_container(self): self.assertIsNone(self._get_queue(mock_q)) long_opts = dict(self.opts, **{'long': True}) - mock_conn.head_container = Mock(return_value={'test_m': '1'}) + mock_conn.head_container = mock.Mock(return_value={'test_m': '1'}) get_container_returns = [ (None, [{'name': 'test_o'}]), (None, []) ] - mock_conn.get_container = Mock(side_effect=get_container_returns) + mock_conn.get_container = mock.Mock(side_effect=get_container_returns) expected_r_long = self._get_expected({ 'action': 'list_container_part', @@ -964,7 +963,7 @@ def test_list_container_marker(self): (None, [{'name': 'b'}, {'name': 'c'}]), (None, []) ] - mock_get_cont = Mock(side_effect=get_container_returns) + mock_get_cont = mock.Mock(side_effect=get_container_returns) mock_conn.get_container = mock_get_cont expected_r = self._get_expected({ @@ -998,7 +997,7 @@ def test_list_container_with_headers(self): (None, [{'name': 'test_o'}]), (None, []) ] - mock_conn.get_container = Mock(side_effect=get_container_returns) + mock_conn.get_container = mock.Mock(side_effect=get_container_returns) expected_r = self._get_expected({ 'action': 'list_container_part', @@ -1027,7 +1026,7 @@ def test_list_container_with_headers(self): def test_list_container_exception(self): mock_q = Queue() mock_conn = self._get_mock_connection() - mock_conn.get_container = Mock(side_effect=self.exc) + mock_conn.get_container = mock.Mock(side_effect=self.exc) expected_r = self._get_expected({ 'action': 'list_container_part', 'container': 'test_c', @@ -1120,7 +1119,7 @@ def test_list_queue_size(self, mock_get_conn): (None, [{'name': 'container14'}]), (None, []) ] - mock_conn.get_account = Mock(side_effect=get_account_returns) + mock_conn.get_account = mock.Mock(side_effect=get_account_returns) mock_get_conn.return_value = mock_conn s = SwiftService(options=self.opts) @@ -2218,7 +2217,7 @@ def fake_sub_page(*args): sub_page.side_effect = fake_sub_page - r = Mock(spec=Future) + r = mock.Mock(spec=Future) r.result.return_value = self._get_expected({ 'success': True, 'start_time': 1, @@ -2257,7 +2256,7 @@ def __str__(self): return repr(self.value) def _make_result(): - r = Mock(spec=Future) + r = mock.Mock(spec=Future) r.result.return_value = self._get_expected({ 'success': True, 'start_time': 1, @@ -2329,7 +2328,7 @@ def test_download_object_job(self): }) with mock.patch.object(builtins, 'open') as mock_open: - written_content = Mock() + written_content = mock.Mock() mock_open.return_value = written_content s = SwiftService() _opts = self.opts.copy() @@ -2373,7 +2372,7 @@ def test_download_object_job_with_mtime(self): with mock.patch.object(builtins, 'open') as mock_open, \ mock.patch('swiftclient.service.utime') as mock_utime: - written_content = Mock() + written_content = mock.Mock() mock_open.return_value = written_content s = SwiftService() _opts = self.opts.copy() @@ -2419,7 +2418,7 @@ def test_download_object_job_bad_mtime(self): with mock.patch.object(builtins, 'open') as mock_open, \ mock.patch('swiftclient.service.utime') as mock_utime: - written_content = Mock() + written_content = mock.Mock() mock_open.return_value = written_content s = SwiftService() _opts = self.opts.copy() @@ -2464,7 +2463,7 @@ def test_download_object_job_ignore_mtime(self): with mock.patch.object(builtins, 'open') as mock_open, \ mock.patch('swiftclient.service.utime') as mock_utime: - written_content = Mock() + written_content = mock.Mock() mock_open.return_value = written_content s = SwiftService() _opts = self.opts.copy() @@ -2492,7 +2491,7 @@ def test_download_object_job_ignore_mtime(self): def test_download_object_job_exception(self): mock_conn = self._get_mock_connection() - mock_conn.get_object = Mock(side_effect=self.exc) + mock_conn.get_object = mock.Mock(side_effect=self.exc) expected_r = self._get_expected({ 'success': False, 'error': self.exc, @@ -3026,7 +3025,7 @@ def test_object_post(self, res_iter, thread_manager): Check post method translates strings and objects to _post_object_job calls correctly """ - tm_instance = Mock() + tm_instance = mock.Mock() thread_manager.return_value = tm_instance self.opts.update({'meta': ["meta1:test1"], "header": ["hdr1:test1"]}) @@ -3071,7 +3070,7 @@ def test_object_copy(self, inter_compl, thread_manager): Check copy method translates strings and objects to _copy_object_job calls correctly """ - tm_instance = Mock() + tm_instance = mock.Mock() thread_manager.return_value = tm_instance self.opts.update({'meta': ["meta1:test1"], "header": ["hdr1:test1"]}) diff --git a/test/unit/test_shell.py b/test/unit/test_shell.py index eb70a923..94168b5d 100644 --- a/test/unit/test_shell.py +++ b/test/unit/test_shell.py @@ -20,10 +20,10 @@ import hashlib import json import logging -import mock import os import tempfile import unittest +from unittest import mock import textwrap from time import localtime, mktime, strftime, strptime diff --git a/test/unit/test_swiftclient.py b/test/unit/test_swiftclient.py index 3b591662..ad2af50c 100644 --- a/test/unit/test_swiftclient.py +++ b/test/unit/test_swiftclient.py @@ -16,11 +16,11 @@ import gzip import json import logging -import mock import io import socket import string import unittest +from unittest import mock import warnings import tempfile from hashlib import md5 diff --git a/test/unit/test_utils.py b/test/unit/test_utils.py index 38331546..33fd4157 100644 --- a/test/unit/test_utils.py +++ b/test/unit/test_utils.py @@ -17,7 +17,7 @@ import io import json import unittest -import mock +from unittest import mock import tempfile from time import gmtime, localtime, mktime, strftime, strptime from hashlib import md5, sha1 diff --git a/test/unit/utils.py b/test/unit/utils.py index 0dc0b3c9..87d32107 100644 --- a/test/unit/utils.py +++ b/test/unit/utils.py @@ -20,10 +20,10 @@ import sys from time import sleep import unittest +from unittest import mock from requests import RequestException from requests.structures import CaseInsensitiveDict -import mock from urllib.parse import urlparse, ParseResult from swiftclient import client as c from swiftclient import shell as s From 5d451fb92031989ea9982fc29140175014448ea2 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Tue, 22 Mar 2022 10:14:19 -0700 Subject: [PATCH 154/238] More cleanup following py2 removal * Drop py2-only hacking pin from test-requirements. * Remove quote() helper; urllib.parse.quote() works fine. * Remove some useless code. Change-Id: I9ffc923f58f1d11538f83ff26f7beb53cdf134c3 --- swiftclient/client.py | 10 +--------- swiftclient/service.py | 5 ----- swiftclient/shell.py | 2 -- test-requirements.txt | 7 +++---- test/unit/test_multithreading.py | 3 +-- test/unit/test_shell.py | 4 ++-- 6 files changed, 7 insertions(+), 24 deletions(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index a9130ab5..168bfeda 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -23,7 +23,7 @@ from requests.exceptions import RequestException, SSLError import http.client as http_client -from urllib.parse import quote as _quote, unquote +from urllib.parse import quote, unquote from urllib.parse import urljoin, urlparse, urlunparse from time import sleep, time @@ -181,14 +181,6 @@ def parse_header_string(data): return unquoted -def quote(value, safe='/'): - """ - Patched version of urllib.quote that encodes utf8 strings before quoting. - On Python 3, call directly urllib.parse.quote(). - """ - return _quote(value, safe=safe) - - def encode_utf8(value): if type(value) in (int, float, bool): # As of requests 2.11.0, headers must be byte- or unicode-strings. diff --git a/swiftclient/service.py b/swiftclient/service.py index 9d9fc594..ed0f40a7 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -2034,11 +2034,6 @@ def _upload_slo_manifest(conn, segment_results, container, obj, headers): if headers is None: headers = {} segment_results.sort(key=lambda di: di['segment_index']) - for seg in segment_results: - seg_loc = seg['segment_location'].lstrip('/') - if isinstance(seg_loc, str): - seg_loc = seg_loc.encode('utf-8') - manifest_data = json.dumps([ { 'path': d['segment_location'], diff --git a/swiftclient/shell.py b/swiftclient/shell.py index df7c5115..445d4cb8 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -1929,8 +1929,6 @@ def add_default_args(parser): def main(arguments=None): argv = sys_argv if arguments is None else arguments - argv = [a if isinstance(a, str) else a.decode('utf-8') for a in argv] - parser = argparse.ArgumentParser( add_help=False, formatter_class=HelpFormatter, usage=''' %(prog)s [--version] [--help] [--os-help] [--snet] [--verbose] diff --git a/test-requirements.txt b/test-requirements.txt index b0633eb9..a4b64ee7 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,7 +1,6 @@ -hacking>=1.1.0,<1.2.0;python_version<'3.0' # Apache-2.0 -hacking>=3.2.0,<3.3.0;python_version>='3.0' # Apache-2.0 +hacking>=3.2.0,<3.3.0 # Apache-2.0 -coverage!=4.4,>=4.0 # Apache-2.0 +coverage!=4.4,>=4.0 # Apache-2.0 keystoneauth1>=3.4.0 # Apache-2.0 stestr>=2.0.0,!=3.0.0 # Apache-2.0 -openstacksdk>=0.11.0 # Apache-2.0 +openstacksdk>=0.11.0 # Apache-2.0 diff --git a/test/unit/test_multithreading.py b/test/unit/test_multithreading.py index d51bfb7c..8237d829 100644 --- a/test/unit/test_multithreading.py +++ b/test/unit/test_multithreading.py @@ -216,12 +216,11 @@ def test_printers(self): # The threads should have been cleaned up self.assertEqual(starting_thread_count, threading.active_count()) - over_the = "over the '\u062a\u062a'\n" self.assertEqual(''.join([ 'one-argument\n', 'one fish, 88 fish\n', 'some\n', 'where\n', - over_the, + "over the '\u062a\u062a'\n", 'some raw bytes: \u062a\u062a', ' key: value\n', ' object: O\u0308bject\n' diff --git a/test/unit/test_shell.py b/test/unit/test_shell.py index 94168b5d..105dddb1 100644 --- a/test/unit/test_shell.py +++ b/test/unit/test_shell.py @@ -1622,7 +1622,7 @@ def test_delete_verbose_output_utf8(self): with mock.patch('swiftclient.shell.SwiftService.delete') as mock_func: with CaptureOutput() as out: mock_func.return_value = [res] - swiftclient.shell.main(base_argv + [container.encode('utf-8')]) + swiftclient.shell.main(base_argv + [container]) mock_func.assert_called_once_with(container=container) self.assertTrue(out.out.find( @@ -1635,7 +1635,7 @@ def test_delete_verbose_output_utf8(self): with mock.patch('swiftclient.shell.SwiftService.delete') as mock_func: with CaptureOutput() as out: mock_func.return_value = [res] - swiftclient.shell.main(base_argv + [container.encode('utf-8')]) + swiftclient.shell.main(base_argv + [container]) mock_func.assert_called_once_with(container=container) self.assertTrue(out.out.find( From 1dc635a32c30d4e6c823b8367247f4554365c456 Mon Sep 17 00:00:00 2001 From: Takashi Kajinami Date: Tue, 7 Jun 2022 00:35:53 +0900 Subject: [PATCH 155/238] doc: Comment out language option ... because explicit language=None causes the below warning since Sphinx 5.0.0. Invalid configuration value found: 'language = None'. Update your configuration to a valid language code. Falling back to 'en' (English). Change-Id: I842fd6c1eb5c0e14d85f8eec6078d735fbd506b8 --- releasenotes/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/releasenotes/source/conf.py b/releasenotes/source/conf.py index a1385e55..0d202563 100644 --- a/releasenotes/source/conf.py +++ b/releasenotes/source/conf.py @@ -71,7 +71,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +# language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: From 9eee29d2e46e774eb08acb76c3317a58856f3f71 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Wed, 8 Jun 2022 09:30:17 -0700 Subject: [PATCH 156/238] tempurl: Support sha256 and sha512 signatures Up the default to sha256 since * the proxy has supported (and defaulted to allowing) it for four years now, and * Rackspace has supported it for even longer. Include a note in the --help about older clusters likely requiring sha1. Change-Id: Ibac2bb7e2e4c9946c7384f0aab8e43d0d79ba645 Related-Change: Ia9dd1a91cc3c9c946f5f029cdefc9e66bcf01046 Related-Bug: #1733634 Closes-Bug: #1977867 --- swiftclient/shell.py | 11 ++++- swiftclient/utils.py | 22 +++++++-- test/unit/test_shell.py | 63 ++++++++++++++++++------ test/unit/test_utils.py | 103 ++++++++++++++++++++++++++++------------ 4 files changed, 151 insertions(+), 48 deletions(-) diff --git a/swiftclient/shell.py b/swiftclient/shell.py index df7c5115..b4721fe4 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -1427,6 +1427,8 @@ def st_auth(parser, args, thread_manager, return_parser=False): ISO 8601 UTC timestamp instead of a Unix timestamp. --ip-range If present, the temporary URL will be restricted to the given ip or ip range. + --digest The digest algorithm to use. Defaults to sha256, but + older clusters may only support sha1. '''.strip('\n') @@ -1456,6 +1458,12 @@ def st_tempurl(parser, args, thread_manager, return_parser=False): help=("If present, the temporary URL will be restricted to the " "given ip or ip range."), ) + parser.add_argument( + '--digest', choices=('sha1', 'sha256', 'sha512'), + default='sha256', + help=("The digest algorithm to use. Defaults to sha256, but " + "older clusters may only support sha1."), + ) # We return the parser to build up the bash_completion if return_parser: @@ -1480,7 +1488,8 @@ def st_tempurl(parser, args, thread_manager, return_parser=False): absolute=options['absolute_expiry'], iso8601=options['iso8601'], prefix=options['prefix_based'], - ip_range=options['ip_range']) + ip_range=options['ip_range'], + digest=options['digest']) except ValueError as err: thread_manager.error(err) return diff --git a/swiftclient/utils.py b/swiftclient/utils.py index 860d8bf4..c865d273 100644 --- a/swiftclient/utils.py +++ b/swiftclient/utils.py @@ -14,6 +14,7 @@ # limitations under the License. """Miscellaneous utility functions for use with Swift.""" +import base64 from calendar import timegm from collections.abc import Mapping import gzip @@ -70,7 +71,8 @@ def prt_bytes(num_bytes, human_flag): def generate_temp_url(path, seconds, key, method, absolute=False, - prefix=False, iso8601=False, ip_range=None): + prefix=False, iso8601=False, ip_range=None, + digest='sha256'): """Generates a temporary URL that gives unauthenticated access to the Swift object. @@ -95,7 +97,11 @@ def generate_temp_url(path, seconds, key, method, absolute=False, instead of a UNIX timestamp will be created. :param ip_range: if a valid ip range, restricts the temporary URL to the range of ips. - :raises ValueError: if timestamp or path is not in valid format. + :param digest: digest algorithm to use. Must be one of ``sha1``, + ``sha256``, or ``sha512``. + :raises ValueError: if timestamp or path is not in valid format, + or if digest is not one of ``sha1``, ``sha256``, or + ``sha512``. :return: the path portion of a temporary URL """ try: @@ -140,6 +146,11 @@ def generate_temp_url(path, seconds, key, method, absolute=False, else: path_for_body = path + if isinstance(digest, str) and digest in ('sha1', 'sha256', 'sha512'): + digest = getattr(hashlib, digest) + if digest not in (hashlib.sha1, hashlib.sha256, hashlib.sha512): + raise ValueError('digest must be one of sha1, sha256, or sha512') + parts = path_for_body.split('/', 4) if len(parts) != 5 or parts[0] or not all(parts[1:(4 if prefix else 5)]): if prefix: @@ -177,7 +188,12 @@ def generate_temp_url(path, seconds, key, method, absolute=False, # Encode to UTF-8 for py3 compatibility if not isinstance(key, bytes): key = key.encode('utf-8') - sig = hmac.new(key, hmac_body.encode('utf-8'), hashlib.sha1).hexdigest() + mac = hmac.new(key, hmac_body.encode('utf-8'), digest) + if digest == hashlib.sha512: + sig = 'sha512:' + base64.urlsafe_b64encode( + mac.digest()).decode('ascii').strip('=') + else: + sig = mac.hexdigest() if iso8601: expiration = time.strftime( diff --git a/test/unit/test_shell.py b/test/unit/test_shell.py index 94168b5d..db0d66c3 100644 --- a/test/unit/test_shell.py +++ b/test/unit/test_shell.py @@ -2035,7 +2035,7 @@ def test_temp_url(self, temp_url): swiftclient.shell.main(argv) temp_url.assert_called_with( '/v1/AUTH_account/c/o', "60", 'secret_key', 'GET', absolute=False, - iso8601=False, prefix=False, ip_range=None) + iso8601=False, prefix=False, ip_range=None, digest='sha256') @mock.patch('swiftclient.shell.generate_temp_url', return_value='') def test_temp_url_prefix_based(self, temp_url): @@ -2044,7 +2044,7 @@ def test_temp_url_prefix_based(self, temp_url): swiftclient.shell.main(argv) temp_url.assert_called_with( '/v1/AUTH_account/c/', "60", 'secret_key', 'GET', absolute=False, - iso8601=False, prefix=True, ip_range=None) + iso8601=False, prefix=True, ip_range=None, digest='sha256') @mock.patch('swiftclient.shell.generate_temp_url', return_value='') def test_temp_url_iso8601_in(self, temp_url): @@ -2056,7 +2056,7 @@ def test_temp_url_iso8601_in(self, temp_url): swiftclient.shell.main(argv) temp_url.assert_called_with( '/v1/AUTH_account/c/', d, 'secret_key', 'GET', absolute=False, - iso8601=False, prefix=False, ip_range=None) + iso8601=False, prefix=False, ip_range=None, digest='sha256') @mock.patch('swiftclient.shell.generate_temp_url', return_value='') def test_temp_url_iso8601_out(self, temp_url): @@ -2065,7 +2065,7 @@ def test_temp_url_iso8601_out(self, temp_url): swiftclient.shell.main(argv) temp_url.assert_called_with( '/v1/AUTH_account/c/', "60", 'secret_key', 'GET', absolute=False, - iso8601=True, prefix=False, ip_range=None) + iso8601=True, prefix=False, ip_range=None, digest='sha256') @mock.patch('swiftclient.shell.generate_temp_url', return_value='') def test_absolute_expiry_temp_url(self, temp_url): @@ -2074,7 +2074,7 @@ def test_absolute_expiry_temp_url(self, temp_url): swiftclient.shell.main(argv) temp_url.assert_called_with( '/v1/AUTH_account/c/o', "60", 'secret_key', 'GET', absolute=True, - iso8601=False, prefix=False, ip_range=None) + iso8601=False, prefix=False, ip_range=None, digest='sha256') @mock.patch('swiftclient.shell.generate_temp_url', return_value='') def test_temp_url_with_ip_range(self, temp_url): @@ -2083,11 +2083,11 @@ def test_temp_url_with_ip_range(self, temp_url): swiftclient.shell.main(argv) temp_url.assert_called_with( '/v1/AUTH_account/c/o', "60", 'secret_key', 'GET', absolute=False, - iso8601=False, prefix=False, ip_range='1.2.3.4') + iso8601=False, prefix=False, ip_range='1.2.3.4', digest='sha256') def test_temp_url_output(self): argv = ["", "tempurl", "GET", "60", "/v1/a/c/o", - "secret_key", "--absolute"] + "secret_key", "--absolute", "--digest", "sha1"] with CaptureOutput(suppress_systemexit=True) as output: swiftclient.shell.main(argv) sig = "63bc77a473a1c2ce956548cacf916f292eb9eac3" @@ -2095,14 +2095,14 @@ def test_temp_url_output(self): self.assertEqual(expected, output.out) argv = ["", "tempurl", "GET", "60", "http://saio:8080/v1/a/c/o", - "secret_key", "--absolute"] + "secret_key", "--absolute", "--digest", "sha1"] with CaptureOutput(suppress_systemexit=True) as output: swiftclient.shell.main(argv) expected = "http://saio:8080%s" % expected self.assertEqual(expected, output.out) argv = ["", "tempurl", "GET", "60", "/v1/a/c/", - "secret_key", "--absolute", "--prefix"] + "secret_key", "--absolute", "--prefix", "--digest", "sha1"] with CaptureOutput(suppress_systemexit=True) as output: swiftclient.shell.main(argv) sig = '00008c4be1573ba74fc2ab9bce02e3a93d04b349' @@ -2111,7 +2111,8 @@ def test_temp_url_output(self): self.assertEqual(expected, output.out) argv = ["", "tempurl", "GET", "60", "/v1/a/c/", - "secret_key", "--absolute", "--prefix", '--iso8601'] + "secret_key", "--absolute", "--prefix", '--iso8601', + "--digest", "sha1"] with CaptureOutput(suppress_systemexit=True) as output: swiftclient.shell.main(argv) sig = '00008c4be1573ba74fc2ab9bce02e3a93d04b349' @@ -2124,7 +2125,7 @@ def test_temp_url_output(self): strftime(EXPIRES_ISO8601_FORMAT[:-1], localtime(60))) for d in dates: argv = ["", "tempurl", "GET", d, "/v1/a/c/o", - "secret_key"] + "secret_key", "--digest", "sha1"] with CaptureOutput(suppress_systemexit=True) as output: swiftclient.shell.main(argv) sig = "63bc77a473a1c2ce956548cacf916f292eb9eac3" @@ -2135,19 +2136,20 @@ def test_temp_url_output(self): mktime(strptime('2005-05-01', SHORT_EXPIRES_ISO8601_FORMAT)))) argv = ["", "tempurl", "GET", ts, "/v1/a/c/", - "secret_key", "--absolute"] + "secret_key", "--absolute", "--digest", "sha1"] with CaptureOutput(suppress_systemexit=True) as output: swiftclient.shell.main(argv) expected = output.out argv = ["", "tempurl", "GET", '2005-05-01', "/v1/a/c/", - "secret_key", "--absolute"] + "secret_key", "--absolute", "--digest", "sha1"] with CaptureOutput(suppress_systemexit=True) as output: swiftclient.shell.main(argv) self.assertEqual(expected, output.out) argv = ["", "tempurl", "GET", "60", "/v1/a/c/o", - "secret_key", "--absolute", "--ip-range", "1.2.3.4"] + "secret_key", "--absolute", "--ip-range", "1.2.3.4", + "--digest", "sha1"] with CaptureOutput(suppress_systemexit=True) as output: swiftclient.shell.main(argv) sig = "6a6ec8efa4be53904ecba8d055d841e24a937c98" @@ -2157,6 +2159,39 @@ def test_temp_url_output(self): ) self.assertEqual(expected, output.out) + def test_temp_url_digests_output(self): + argv = ["", "tempurl", "GET", "60", "/v1/a/c/o", + "secret_key", "--absolute"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + s = "db04994a589b1a2538bff694f0a4f57c7a397617ac2cb49f924d222bbe2b3e01" + expected = "/v1/a/c/o?temp_url_sig=%s&temp_url_expires=60\n" % s + self.assertEqual(expected, output.out) + + argv = ["", "tempurl", "GET", "60", "/v1/a/c/o", + "secret_key", "--absolute", "--digest", "sha256"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + # same signature/expectation + self.assertEqual(expected, output.out) + + argv = ["", "tempurl", "GET", "60", "/v1/a/c/o", + "secret_key", "--absolute", "--digest", "sha1"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + sig = "63bc77a473a1c2ce956548cacf916f292eb9eac3" + expected = "/v1/a/c/o?temp_url_sig=%s&temp_url_expires=60\n" % sig + self.assertEqual(expected, output.out) + + argv = ["", "tempurl", "GET", "60", "/v1/a/c/o", + "secret_key", "--absolute", "--digest", "sha512"] + with CaptureOutput(suppress_systemexit=True) as output: + swiftclient.shell.main(argv) + sig = ("sha512:nMXwEAHu3jzlCZi4wWO1juEq4DikFlX8a729PLJVvUp" + "vg0GpgkJnX5uCG1x-v2KfTrmRtLOcT7KBK2RXLW1uKw") + expected = "/v1/a/c/o?temp_url_sig=%s&temp_url_expires=60\n" % sig + self.assertEqual(expected, output.out) + def test_temp_url_error_output(self): expected = 'path must be full path to an object e.g. /v1/a/c/o\n' for bad_path in ('/v1/a/c', 'v1/a/c/o', '/v1/a/c/', '/v1/a//o', diff --git a/test/unit/test_utils.py b/test/unit/test_utils.py index 33fd4157..129208d5 100644 --- a/test/unit/test_utils.py +++ b/test/unit/test_utils.py @@ -20,7 +20,7 @@ from unittest import mock import tempfile from time import gmtime, localtime, mktime, strftime, strptime -from hashlib import md5, sha1 +import hashlib from swiftclient import utils as u @@ -127,17 +127,65 @@ class TestTempURL(unittest.TestCase): seconds = 3600 key = 'correcthorsebatterystaple' method = 'GET' - expected_url = url + ('?temp_url_sig=temp_url_signature' - '&temp_url_expires=1400003600') expected_body = '\n'.join([ method, '1400003600', url, ]).encode('utf-8') + @property + def expected_url(self): + if isinstance(self.url, bytes): + return self.url + (b'?temp_url_sig=temp_url_signature' + b'&temp_url_expires=1400003600') + return self.url + (u'?temp_url_sig=temp_url_signature' + u'&temp_url_expires=1400003600') + + @property + def expected_sha512_url(self): + if isinstance(self.url, bytes): + return self.url + (b'?temp_url_sig=sha512:dGVtcF91cmxfc2lnbmF0dXJl' + b'&temp_url_expires=1400003600') + return self.url + (u'?temp_url_sig=sha512:dGVtcF91cmxfc2lnbmF0dXJl' + u'&temp_url_expires=1400003600') + + @mock.patch('hmac.HMAC') + @mock.patch('time.time', return_value=1400000000) + def test_generate_sha1_temp_url(self, time_mock, hmac_mock): + hmac_mock().hexdigest.return_value = 'temp_url_signature' + url = u.generate_temp_url(self.url, self.seconds, + self.key, self.method, digest='sha1') + key = self.key + if not isinstance(key, bytes): + key = key.encode('utf-8') + self.assertEqual(url, self.expected_url) + self.assertEqual(hmac_mock.mock_calls, [ + mock.call(), + mock.call(key, self.expected_body, hashlib.sha1), + mock.call().hexdigest(), + ]) + self.assertIsInstance(url, type(self.url)) + + @mock.patch('hmac.HMAC') + @mock.patch('time.time', return_value=1400000000) + def test_generate_sha512_temp_url(self, time_mock, hmac_mock): + hmac_mock().digest.return_value = b'temp_url_signature' + url = u.generate_temp_url(self.url, self.seconds, + self.key, self.method, digest=hashlib.sha512) + key = self.key + if not isinstance(key, bytes): + key = key.encode('utf-8') + self.assertEqual(url, self.expected_sha512_url) + self.assertEqual(hmac_mock.mock_calls, [ + mock.call(), + mock.call(key, self.expected_body, hashlib.sha512), + mock.call().digest(), + ]) + self.assertIsInstance(url, type(self.url)) + @mock.patch('hmac.HMAC') @mock.patch('time.time', return_value=1400000000) - def test_generate_temp_url(self, time_mock, hmac_mock): + def test_generate_sha256_temp_url_by_default(self, time_mock, hmac_mock): hmac_mock().hexdigest.return_value = 'temp_url_signature' url = u.generate_temp_url(self.url, self.seconds, self.key, self.method) @@ -147,7 +195,7 @@ def test_generate_temp_url(self, time_mock, hmac_mock): self.assertEqual(url, self.expected_url) self.assertEqual(hmac_mock.mock_calls, [ mock.call(), - mock.call(key, self.expected_body, sha1), + mock.call(key, self.expected_body, hashlib.sha256), mock.call().hexdigest(), ]) self.assertIsInstance(url, type(self.url)) @@ -195,7 +243,7 @@ def test_generate_temp_url_ip_range(self, time_mock, hmac_mock): self.assertEqual(url, ip_range_expected_url) self.assertEqual(hmac_mock.mock_calls, [ - mock.call(key, expected_body, sha1), + mock.call(key, expected_body, hashlib.sha256), mock.call().hexdigest(), ]) self.assertIsInstance(url, type(path)) @@ -256,7 +304,7 @@ def test_generate_temp_url_iso8601_output(self, time_mock, hmac_mock): self.assertTrue(url.endswith(expires)) self.assertEqual(hmac_mock.mock_calls, [ mock.call(), - mock.call(key, self.expected_body, sha1), + mock.call(key, self.expected_body, hashlib.sha256), mock.call().hexdigest(), ]) self.assertIsInstance(url, type(self.url)) @@ -284,7 +332,7 @@ def test_generate_temp_url_prefix(self, time_mock, hmac_mock): key = key.encode('utf-8') self.assertEqual(url, expected_url) self.assertEqual(hmac_mock.mock_calls, [ - mock.call(key, expected_body, sha1), + mock.call(key, expected_body, hashlib.sha256), mock.call().hexdigest(), ]) @@ -374,8 +422,6 @@ def test_generate_temp_url_bad_path(self): class TestTempURLUnicodePathAndKey(TestTempURL): url = '/v1/\u00e4/c/\u00f3' key = 'k\u00e9y' - expected_url = ('%s?temp_url_sig=temp_url_signature' - '&temp_url_expires=1400003600') % url expected_body = '\n'.join([ 'GET', '1400003600', @@ -386,8 +432,6 @@ class TestTempURLUnicodePathAndKey(TestTempURL): class TestTempURLUnicodePathBytesKey(TestTempURL): url = '/v1/\u00e4/c/\u00f3' key = 'k\u00e9y'.encode('utf-8') - expected_url = ('%s?temp_url_sig=temp_url_signature' - '&temp_url_expires=1400003600') % url expected_body = '\n'.join([ 'GET', '1400003600', @@ -398,8 +442,6 @@ class TestTempURLUnicodePathBytesKey(TestTempURL): class TestTempURLBytesPathUnicodeKey(TestTempURL): url = '/v1/\u00e4/c/\u00f3'.encode('utf-8') key = 'k\u00e9y' - expected_url = url + (b'?temp_url_sig=temp_url_signature' - b'&temp_url_expires=1400003600') expected_body = b'\n'.join([ b'GET', b'1400003600', @@ -410,8 +452,6 @@ class TestTempURLBytesPathUnicodeKey(TestTempURL): class TestTempURLBytesPathAndKey(TestTempURL): url = '/v1/\u00e4/c/\u00f3'.encode('utf-8') key = 'k\u00e9y'.encode('utf-8') - expected_url = url + (b'?temp_url_sig=temp_url_signature' - b'&temp_url_expires=1400003600') expected_body = b'\n'.join([ b'GET', b'1400003600', @@ -422,8 +462,6 @@ class TestTempURLBytesPathAndKey(TestTempURL): class TestTempURLBytesPathAndNonUtf8Key(TestTempURL): url = '/v1/\u00e4/c/\u00f3'.encode('utf-8') key = b'k\xffy' - expected_url = url + (b'?temp_url_sig=temp_url_signature' - b'&temp_url_expires=1400003600') expected_body = b'\n'.join([ b'GET', b'1400003600', @@ -436,7 +474,7 @@ class TestReadableToIterable(unittest.TestCase): def test_iter(self): chunk_size = 4 write_data = tuple(x.encode() for x in ('a', 'b', 'c', 'd')) - actual_md5sum = md5() + actual_md5sum = hashlib.md5() with tempfile.TemporaryFile() as f: for x in write_data: @@ -454,8 +492,8 @@ def test_iter(self): def test_md5_creation(self): # Check creation with a real and noop md5 class data = u.ReadableToIterable(None, None, md5=True) - self.assertEqual(md5().hexdigest(), data.get_md5sum()) - self.assertIs(type(md5()), type(data.md5sum)) + self.assertEqual(hashlib.md5().hexdigest(), data.get_md5sum()) + self.assertIs(type(hashlib.md5()), type(data.md5sum)) data = u.ReadableToIterable(None, None, md5=False) self.assertEqual('', data.get_md5sum()) @@ -464,7 +502,7 @@ def test_md5_creation(self): def test_unicode(self): # Check no errors are raised if unicode data is feed in. unicode_data = 'abc' - actual_md5sum = md5(unicode_data.encode()).hexdigest() + actual_md5sum = hashlib.md5(unicode_data.encode()).hexdigest() chunk_size = 2 with tempfile.TemporaryFile(mode='w+') as f: @@ -495,15 +533,17 @@ def test_stringio(self): self.assertEqual(42, len(data)) self.assertEqual(42, len(read_data)) self.assertEqual(s, read_data) - self.assertEqual(md5(s.encode()).hexdigest(), data.get_md5sum()) + self.assertEqual(hashlib.md5(s.encode()).hexdigest(), + data.get_md5sum()) data.reset() - self.assertEqual(md5().hexdigest(), data.get_md5sum()) + self.assertEqual(hashlib.md5().hexdigest(), data.get_md5sum()) read_data = ''.join(iter(data.read, '')) self.assertEqual(42, len(read_data)) self.assertEqual(s, read_data) - self.assertEqual(md5(s.encode()).hexdigest(), data.get_md5sum()) + self.assertEqual(hashlib.md5(s.encode()).hexdigest(), + data.get_md5sum()) def test_bytesio(self): contents = io.BytesIO(b'a' * 50 + b'b' * 50) @@ -515,7 +555,7 @@ def test_bytesio(self): self.assertEqual(42, len(data)) self.assertEqual(42, len(read_data)) self.assertEqual(s, read_data) - self.assertEqual(md5(s).hexdigest(), data.get_md5sum()) + self.assertEqual(hashlib.md5(s).hexdigest(), data.get_md5sum()) def test_tempfile(self): with tempfile.NamedTemporaryFile(mode='wb') as f: @@ -529,7 +569,7 @@ def test_tempfile(self): self.assertEqual(42, len(data)) self.assertEqual(42, len(read_data)) self.assertEqual(s, read_data) - self.assertEqual(md5(s).hexdigest(), data.get_md5sum()) + self.assertEqual(hashlib.md5(s).hexdigest(), data.get_md5sum()) def test_segmented_file(self): with tempfile.NamedTemporaryFile(mode='wb') as f: @@ -548,15 +588,18 @@ def test_segmented_file(self): self.assertEqual(segment_length, len(data)) self.assertEqual(segment_length, len(read_data)) self.assertEqual(s, read_data) - self.assertEqual(md5(s).hexdigest(), data.get_md5sum()) + self.assertEqual(hashlib.md5(s).hexdigest(), + data.get_md5sum()) data.reset() - self.assertEqual(md5().hexdigest(), data.get_md5sum()) + self.assertEqual(hashlib.md5().hexdigest(), + data.get_md5sum()) read_data = b''.join(iter(data.read, '')) self.assertEqual(segment_length, len(data)) self.assertEqual(segment_length, len(read_data)) self.assertEqual(s, read_data) - self.assertEqual(md5(s).hexdigest(), data.get_md5sum()) + self.assertEqual(hashlib.md5(s).hexdigest(), + data.get_md5sum()) class TestGroupers(unittest.TestCase): From 0bd2ab5cb0020bc15f36a2614aecdc28feecbea0 Mon Sep 17 00:00:00 2001 From: Takashi Natsume Date: Thu, 18 Aug 2022 22:24:00 +0900 Subject: [PATCH 157/238] Fix misuse of assertTrue Replace assertTrue with assertEqual. Change-Id: Ia3524bc5b3b01c0039bede6bb172535eb85bac08 Closes-Bug: 1986948 Signed-off-by: Takashi Natsume --- test/functional/test_swiftclient.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/functional/test_swiftclient.py b/test/functional/test_swiftclient.py index 91e31af0..a5c1211c 100644 --- a/test/functional/test_swiftclient.py +++ b/test/functional/test_swiftclient.py @@ -332,7 +332,7 @@ def test_download_object_retry_chunked(self): resp_chunk_size=resp_chunk_size) data = next(body) self.assertEqual(self.test_data[:resp_chunk_size], data) - self.assertTrue(1, self.conn.attempts) + self.assertEqual(1, self.conn.attempts) for chunk in body.resp: # Flush remaining data from underlying response # (simulate a dropped connection) @@ -369,13 +369,13 @@ def test_download_object_non_chunked(self): hdrs, body = self.conn.get_object(self.containername, self.objectname) data = body self.assertEqual(self.test_data, data) - self.assertTrue(1, self.conn.attempts) + self.assertEqual(1, self.conn.attempts) hdrs, body = self.conn.get_object(self.containername, self.objectname, resp_chunk_size=0) data = body self.assertEqual(self.test_data, data) - self.assertTrue(1, self.conn.attempts) + self.assertEqual(1, self.conn.attempts) def test_post_account(self): self.conn.post_account({'x-account-meta-data': 'Something'}) From a1d2f31131d79d7c551dbac4fc1e9c4d177d2df5 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Wed, 17 Aug 2022 16:58:36 -0700 Subject: [PATCH 158/238] Enable retry_on_ratelimit by default UpgradeImpact ============= The Connection class now enables retry_on_ratelimit by default. If you need to return to the old behavior, explicitly pass retry_on_ratelimit=False as a keyword arg. The SwiftService class will now enables the retry_on_ratelimit option by default. If you need to return to the old behavior, explicitly set it to false in your options dict. Change-Id: I3221fda84f0b8031c50128aa600e2c19deb5b102 --- swiftclient/client.py | 8 ++++---- swiftclient/service.py | 3 +++ test/unit/test_swiftclient.py | 3 ++- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index a9130ab5..8415db22 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -1654,7 +1654,7 @@ def __init__(self, authurl=None, user=None, key=None, retries=5, starting_backoff=1, max_backoff=64, tenant_name=None, os_options=None, auth_version="1", cacert=None, insecure=False, cert=None, cert_key=None, - ssl_compression=True, retry_on_ratelimit=False, + ssl_compression=True, retry_on_ratelimit=True, timeout=None, session=None, force_auth_retry=False): """ :param authurl: authentication URL @@ -1686,9 +1686,9 @@ def __init__(self, authurl=None, user=None, key=None, retries=5, will be made. This may provide a performance increase for https upload/download operations. :param retry_on_ratelimit: by default, a ratelimited connection will - raise an exception to the caller. Setting - this parameter to True will cause a retry - after a backoff. + retry after a backoff. Setting this + parameter to False will cause an exception + to be raised to the caller. :param timeout: The connect timeout for the HTTP connection. :param session: A keystoneauth session object. :param force_auth_retry: reset auth info even if client got unexpected diff --git a/swiftclient/service.py b/swiftclient/service.py index 9d9fc594..545ea47f 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -155,6 +155,7 @@ def _build_default_global_options(): "user": environ.get('ST_USER'), "key": environ.get('ST_KEY'), "retries": 5, + "retry_on_ratelimit": True, "force_auth_retry": False, "os_username": environ.get('OS_USERNAME'), "os_user_id": environ.get('OS_USER_ID'), @@ -270,10 +271,12 @@ def get_conn(options): """ Return a connection building it from the options. """ + options = dict(_default_global_options, **options) return Connection(options['auth'], options['user'], options['key'], timeout=options.get('timeout'), + retry_on_ratelimit=options['retry_on_ratelimit'], retries=options['retries'], auth_version=options['auth_version'], os_options=options['os_options'], diff --git a/test/unit/test_swiftclient.py b/test/unit/test_swiftclient.py index ad2af50c..436245d8 100644 --- a/test/unit/test_swiftclient.py +++ b/test/unit/test_swiftclient.py @@ -2149,7 +2149,8 @@ def quick_sleep(*args): c.http_connection = self.fake_http_connection( 200, 498, headers=auth_resp_headers) - conn = c.Connection('http://www.test.com/auth/v1.0', 'asdf', 'asdf') + conn = c.Connection('http://www.test.com/auth/v1.0', 'asdf', 'asdf', + retry_on_ratelimit=False) with self.assertRaises(c.ClientException) as exc_context: conn.head_account() self.assertIn('Account HEAD failed', str(exc_context.exception)) From 653cbcb686fc34cc28f9c7889e8746e77b95371a Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Wed, 10 Aug 2022 12:38:54 -0500 Subject: [PATCH 159/238] Expand retry handling on ratelimit response We have seen middlewares that return ratelimit responses as 498 or 429, so tolerate either. Closes-Bug: #1879572 Change-Id: I027222157f6c2ad7882a0508302c9de097baae4c --- swiftclient/client.py | 2 +- test/unit/test_swiftclient.py | 54 +++++++++++++++++++---------------- 2 files changed, 31 insertions(+), 25 deletions(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index 8415db22..16bbba82 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -1833,7 +1833,7 @@ def _retry(self, reset_func, func, *args, **kwargs): self.http_conn = None elif 500 <= err.http_status <= 599: pass - elif self.retry_on_ratelimit and err.http_status == 498: + elif self.retry_on_ratelimit and err.http_status in (498, 429): pass else: raise diff --git a/test/unit/test_swiftclient.py b/test/unit/test_swiftclient.py index 436245d8..ae3e76fd 100644 --- a/test/unit/test_swiftclient.py +++ b/test/unit/test_swiftclient.py @@ -2130,31 +2130,37 @@ def quick_sleep(*args): pass c.sleep = quick_sleep - # test retries - conn = c.Connection('http://www.test.com/auth/v1.0', 'asdf', 'asdf', - retry_on_ratelimit=True) - code_iter = [200] + [498] * (conn.retries + 1) - auth_resp_headers = { - 'x-auth-token': 'asdf', - 'x-storage-url': 'http://storage/v1/test', - } - c.http_connection = self.fake_http_connection( - *code_iter, headers=auth_resp_headers) - with self.assertRaises(c.ClientException) as exc_context: - conn.head_account() - self.assertIn('Account HEAD failed', str(exc_context.exception)) - self.assertEqual(conn.attempts, conn.retries + 1) + def test_status_code(code): + # test retries + conn = c.Connection('http://www.test.com/auth/v1.0', + 'asdf', 'asdf', retry_on_ratelimit=True) + code_iter = [200] + [code] * (conn.retries + 1) + auth_resp_headers = { + 'x-auth-token': 'asdf', + 'x-storage-url': 'http://storage/v1/test', + } + c.http_connection = self.fake_http_connection( + *code_iter, headers=auth_resp_headers) + with self.assertRaises(c.ClientException) as exc_context: + conn.head_account() + self.assertIn('Account HEAD failed', str(exc_context.exception)) + self.assertEqual(code, exc_context.exception.http_status) + self.assertEqual(conn.attempts, conn.retries + 1) - # test default no-retry - c.http_connection = self.fake_http_connection( - 200, 498, - headers=auth_resp_headers) - conn = c.Connection('http://www.test.com/auth/v1.0', 'asdf', 'asdf', - retry_on_ratelimit=False) - with self.assertRaises(c.ClientException) as exc_context: - conn.head_account() - self.assertIn('Account HEAD failed', str(exc_context.exception)) - self.assertEqual(conn.attempts, 1) + # test default no-retry + c.http_connection = self.fake_http_connection( + 200, code, + headers=auth_resp_headers) + conn = c.Connection('http://www.test.com/auth/v1.0', + 'asdf', 'asdf', retry_on_ratelimit=False) + with self.assertRaises(c.ClientException) as exc_context: + conn.head_account() + self.assertIn('Account HEAD failed', str(exc_context.exception)) + self.assertEqual(code, exc_context.exception.http_status) + self.assertEqual(conn.attempts, 1) + + test_status_code(498) + test_status_code(429) def test_retry_with_socket_error(self): def quick_sleep(*args): From defbb4a8f390c7de73ac6a90fc1ab5009e8105ee Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Thu, 15 Oct 2020 14:05:28 -0700 Subject: [PATCH 160/238] Allow tempurl times to have units Specifically, let users add a suffix for seconds, minutes, hours, or days. Change-Id: Ibbe7e5aa8aa8e54935da76109c2ea13fb83bc7ab --- swiftclient/shell.py | 18 +++++---- swiftclient/utils.py | 89 +++++++++++++++++++++++++---------------- test/unit/test_shell.py | 8 ++++ test/unit/test_utils.py | 39 ++++++++++++++++++ 4 files changed, 112 insertions(+), 42 deletions(-) diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 5bcff7fc..ed39b819 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -1381,14 +1381,16 @@ def st_auth(parser, args, thread_manager, return_parser=False): An HTTP method to allow for this temporary URL. Usually 'GET' or 'PUT'.