From d931ec1381b6160c06036c6038f247c8619392ef Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Thu, 5 Mar 2015 11:58:26 -0800 Subject: [PATCH 001/454] Remove all DLO segments on upload of replacement Previously, only the first container-listing's worth of segments was deleted, which would leave behind orphaned segments when the object was very large with small segments or the server's container_listing_limit was small. In addition, process DLO and SLO deletions on the segment thread pool, rather than the object thread pool. Change-Id: I1587375261a6237fa55a9cb96bda8dae918cc795 Related-Bug: #1418007 --- swiftclient/service.py | 32 ++++++++++++++++++++------------ tests/unit/test_shell.py | 16 +++++++++------- 2 files changed, 29 insertions(+), 19 deletions(-) diff --git a/swiftclient/service.py b/swiftclient/service.py index f24d4300..7a104673 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -1728,19 +1728,19 @@ def _upload_object_job(self, conn, container, source, obj, options, if old_manifest or old_slo_manifest_paths: drs = [] + delobjsmap = {} if old_manifest: scontainer, sprefix = old_manifest.split('/', 1) scontainer = unquote(scontainer) sprefix = unquote(sprefix).rstrip('/') + '/' - delobjs = [] - for delobj in conn.get_container(scontainer, - prefix=sprefix)[1]: - delobjs.append(delobj['name']) - for dr in self.delete(container=scontainer, - objects=delobjs): - drs.append(dr) + delobjsmap[scontainer] = [] + for part in self.list(scontainer, {'prefix': sprefix}): + if not part["success"]: + raise part["error"] + delobjsmap[scontainer].extend( + seg['name'] for seg in part['listing']) + if old_slo_manifest_paths: - delobjsmap = {} for seg_to_delete in old_slo_manifest_paths: if seg_to_delete in new_slo_manifest_paths: continue @@ -1749,10 +1749,18 @@ def _upload_object_job(self, conn, container, source, obj, options, delobjs_cont = delobjsmap.get(scont, []) delobjs_cont.append(sobj) delobjsmap[scont] = delobjs_cont - for (dscont, dsobjs) in delobjsmap.items(): - for dr in self.delete(container=dscont, - objects=dsobjs): - drs.append(dr) + + del_segs = [] + for dscont, dsobjs in delobjsmap.items(): + for dsobj in dsobjs: + del_seg = self.thread_manager.segment_pool.submit( + self._delete_segment, dscont, dsobj, + results_queue=results_queue + ) + del_segs.append(del_seg) + + for del_seg in interruptable_as_completed(del_segs): + drs.append(del_seg.result()) res['segment_delete_results'] = drs # return dict for printing diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index 5f2d91cb..4c4a26cd 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -489,11 +489,11 @@ def test_upload_delete_slo_segments(self, connection): expected_delete_calls = [ mock.call( b'container1', b'old_seg1', - query_string=None, response_dict={} + response_dict={} ), mock.call( b'container2', b'old_seg2', - query_string=None, response_dict={} + response_dict={} ) ] self.assertEqual( @@ -538,9 +538,11 @@ def test_upload_delete_dlo_segments(self, connection): ] connection.return_value.get_container.side_effect = [ [None, [{'name': 'prefix_a', 'bytes': 0, - 'last_modified': '123T456'}, - {'name': 'prefix_b', 'bytes': 0, - 'last_modified': '123T456'}]] + 'last_modified': '123T456'}]], + # Have multiple pages worth of DLO segments + [None, [{'name': 'prefix_b', 'bytes': 0, + 'last_modified': '123T456'}]], + [None, []] ] connection.return_value.put_object.return_value = ( 'd41d8cd98f00b204e9800998ecf8427e') @@ -555,11 +557,11 @@ def test_upload_delete_dlo_segments(self, connection): expected_delete_calls = [ mock.call( 'container1', 'prefix_a', - query_string=None, response_dict={} + response_dict={} ), mock.call( 'container1', 'prefix_b', - query_string=None, response_dict={} + response_dict={} ) ] self.assertEqual( From 71a18351939b273ef16b032587ec9bf836004f15 Mon Sep 17 00:00:00 2001 From: Alistair Coles Date: Tue, 2 Jun 2015 10:13:48 +0100 Subject: [PATCH 002/454] Document missing functional test config option Adds doc for account_username option to sample test.conf. This option was added in [1]. [1] change id Ic484e9a0c186c9283c4012c6a2fa77b96b8edf8a Change-Id: Ic86b274e9d954822da521360981f796d61efaad9 --- tests/sample.conf | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/sample.conf b/tests/sample.conf index 3b9b03d5..ebbdc17c 100644 --- a/tests/sample.conf +++ b/tests/sample.conf @@ -12,7 +12,13 @@ auth_prefix = /auth/ #auth_ssl = no #auth_prefix = /v2.0/ -# Primary functional test account (needs admin access to the account) +# Primary functional test account (needs admin access to the account). +# By default the tests use a swiftclient.client.Connection instance with user +# attribute set to 'account:username' based on the options 'account' and +# 'username' specified below. This can be overridden for auth systems that +# expect a different form of user attribute by setting the option +# 'account_username'. +# account_username = test_tester account = test username = tester password = testing From 259b434ae25e894e66f6561c5bbc3f34603f8fab Mon Sep 17 00:00:00 2001 From: Alistair Coles Date: Wed, 3 Jun 2015 17:39:15 +0100 Subject: [PATCH 003/454] Add passenv to tox.ini to make functests run with tempauth Since tox version 2.0.0 env vars are not passed to the test env, which means that the SWIFT_TEST_CONFIG_FILE var is not passed in to tox -e func env. That means that both times tox -e func runs it is using keystone auth, and never using tempauth. Related-Bug: 1455102 Co-Authored-By: Christian Schwede Change-Id: I23dcdbcde0bf8adc9429eb2d294a2c778005d136 --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 94a98200..10377ccd 100644 --- a/tox.ini +++ b/tox.ini @@ -11,6 +11,7 @@ setenv = VIRTUAL_ENV={envdir} deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt commands = python setup.py testr --testr-args="{posargs}" +passenv = SWIFT_* *_proxy [testenv:pep8] commands = From f0aad4c364cb280c29033f7ac777a3a1b2e3bec0 Mon Sep 17 00:00:00 2001 From: Alistair Coles Date: Tue, 23 Sep 2014 17:52:51 +0100 Subject: [PATCH 004/454] Run functional tests using keystone auth options Makes the existing functional tests run using three auth modes: tempauth (v1), keystone v2 and keystone v3. The latter uses an account in a non-default domain (which exists in devstack setup). This should help avoid regressions in handling different auth options. Change-Id: Ifee6a4fa418242892bf73eda5e2cad7b803b1bee --- tests/functional/test_swiftclient.py | 69 ++++++++++++++++++++++++++-- tests/sample.conf | 8 ++++ 2 files changed, 73 insertions(+), 4 deletions(-) diff --git a/tests/functional/test_swiftclient.py b/tests/functional/test_swiftclient.py index 4b57f1d9..f9965c55 100644 --- a/tests/functional/test_swiftclient.py +++ b/tests/functional/test_swiftclient.py @@ -45,6 +45,7 @@ def _get_config(self): '/etc/swift/test.conf') config = configparser.SafeConfigParser({'auth_version': '1'}) config.read(config_file) + self.config = config if config.has_section('func_test'): auth_host = config.get('func_test', 'auth_host') auth_port = config.getint('func_test', 'auth_port') @@ -71,15 +72,20 @@ def _get_config(self): else: self.skip_tests = True + def _get_connection(self): + """ + Subclasses may override to use different connection setup + """ + return swiftclient.Connection( + self.auth_url, self.account_username, self.password, + auth_version=self.auth_version) + def setUp(self): super(TestFunctional, self).setUp() if self.skip_tests: self.skipTest('SKIPPING FUNCTIONAL TESTS DUE TO NO CONFIG') - self.conn = swiftclient.Connection( - self.auth_url, self.account_username, self.password, - auth_version=self.auth_version) - + self.conn = self._get_connection() self.conn.put_container(self.containername) self.conn.put_container(self.containername_2) self.conn.put_object( @@ -286,3 +292,58 @@ def test_post_object(self): def test_get_capabilities(self): resp = self.conn.get_capabilities() self.assertTrue(resp.get('swift')) + + +class TestUsingKeystone(TestFunctional): + """ + Repeat tests using os_options parameter to Connection. + """ + + def _get_connection(self): + account = username = password = None + if self.auth_version not in ('2', '3'): + self.skipTest('SKIPPING KEYSTONE-SPECIFIC FUNCTIONAL TESTS') + try: + account = self.config.get('func_test', 'account') + username = self.config.get('func_test', 'username') + password = self.config.get('func_test', 'password') + except Exception: + self.skipTest('SKIPPING KEYSTONE-SPECIFIC FUNCTIONAL TESTS' + + ' - NO CONFIG') + os_options = {'tenant_name': account} + return swiftclient.Connection( + self.auth_url, username, password, auth_version=self.auth_version, + os_options=os_options) + + def setUp(self): + super(TestUsingKeystone, self).setUp() + + +class TestUsingKeystoneV3(TestFunctional): + """ + Repeat tests using a keystone user with domain specified. + """ + + def _get_connection(self): + account = username = password = project_domain = user_domain = None + if self.auth_version != '3': + self.skipTest('SKIPPING KEYSTONE-V3-SPECIFIC FUNCTIONAL TESTS') + try: + account = self.config.get('func_test', 'account4') + username = self.config.get('func_test', 'username4') + user_domain = self.config.get('func_test', 'domain4') + project_domain = self.config.get('func_test', 'domain4') + password = self.config.get('func_test', 'password4') + except Exception: + self.skipTest('SKIPPING KEYSTONE-V3-SPECIFIC FUNCTIONAL TESTS' + + ' - NO CONFIG') + + os_options = {'project_name': account, + 'project_domain_name': project_domain, + 'user_domain_name': user_domain} + return swiftclient.Connection(self.auth_url, username, password, + auth_version=self.auth_version, + os_options=os_options) + + def setUp(self): + super(TestUsingKeystoneV3, self).setUp() diff --git a/tests/sample.conf b/tests/sample.conf index 3b9b03d5..3a6bc8c0 100644 --- a/tests/sample.conf +++ b/tests/sample.conf @@ -16,3 +16,11 @@ auth_prefix = /auth/ account = test username = tester password = testing + +# Another user is required for keystone v3 specific tests. +# Account must be in a non-default domain. +# (Suffix '4' is used to be consistent with swift functional test config). +#account4 = test4 +#username4 = tester4 +#password4 = testing4 +#domain4 = test-domain From 7f2ee7322b3f16fdd2c848be07c67107f4e065dd Mon Sep 17 00:00:00 2001 From: Christian Schwede Date: Wed, 25 Feb 2015 10:58:27 +0000 Subject: [PATCH 005/454] Add connection release test This patch adds a small test to ensure a connection is released after all chunks have been consumed. It's a follow up to commit 8756591b and added to ensure there will be no regression in the future (this test fails also with that patch not applied). Change-Id: I6a6fcd26879eb2070f418c8770a395ff6c30aa51 --- tests/unit/test_swiftclient.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index ae460999..1cfe2044 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -75,6 +75,7 @@ def __init__(self, status=0, headers=None, verify=False): self.headers = {'etag': '"%s"' % EMPTY_ETAG} if headers: self.headers.update(headers) + self.closed = False class Raw(object): def __init__(self, headers): @@ -92,7 +93,7 @@ def read(self): return "" def close(self): - pass + self.closed = True def getheader(self, name, default): return self.headers.get(name, default) @@ -1145,6 +1146,17 @@ def test_insecure(self): conn = c.http_connection(u'http://www.test.com/', insecure=True) self.assertEqual(conn[1].requests_args['verify'], False) + def test_response_connection_released(self): + _parsed_url, conn = c.http_connection(u'http://www.test.com/') + conn.resp = MockHttpResponse() + conn.resp.raw = mock.Mock() + conn.resp.raw.read.side_effect = ["Chunk", ""] + resp = conn.getresponse() + self.assertFalse(resp.closed) + self.assertEqual("Chunk", resp.read()) + self.assertFalse(resp.read()) + self.assertTrue(resp.closed) + class TestConnection(MockHttpTest): From e48f487335e970f8b61a7f891118639203a746b8 Mon Sep 17 00:00:00 2001 From: Andreas Jaeger Date: Mon, 8 Jun 2015 20:20:21 +0200 Subject: [PATCH 006/454] Fix inconsistent usage of "Positional argument" All help texts uses "Positional argument" with the exception of tempurl. Update tempurl to use this as well so that the formatting tools work fine and can show this nicely on http://docs.openstack.org/cli-reference/content/swiftclient_commands.html#swiftclient_subcommand_tempurl like it's done for other options. Change-Id: Ib5502c23b236986bea5a4d4a63a46fca411a8494 Closes-Bug: #1463081 --- swiftclient/shell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 430efd29..29b34573 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -912,7 +912,7 @@ def _print_compo_cap(name, capabilities): st_tempurl_help = ''' Generates a temporary URL for a Swift object. -Positions arguments: +Positional arguments: [method] An HTTP method to allow for this temporary URL. Usually 'GET' or 'PUT'. [seconds] The amount of time in seconds the temporary URL will From 17feec709c02612c285ebddaf712876b619aa5ed Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Tue, 8 Apr 2014 21:14:13 -0700 Subject: [PATCH 007/454] Add some bash helpers for auth stuff Change-Id: If61ac9a050e7a115f37dbf4e74b904ac5dfd2052 --- swiftclient/shell.py | 54 +++++++++++++++++++-- tests/unit/test_shell.py | 102 +++++++++++++++++++++++++++++++++++++++ tests/unit/utils.py | 6 ++- 3 files changed, 156 insertions(+), 6 deletions(-) diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 430efd29..31147f13 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -31,14 +31,19 @@ from swiftclient.multithreading import OutputManager from swiftclient.exceptions import ClientException from swiftclient import __version__ as client_version -from swiftclient.service import SwiftService, SwiftError, SwiftUploadObject +from swiftclient.service import SwiftService, SwiftError, \ + SwiftUploadObject, get_conn from swiftclient.command_helpers import print_account_stats, \ print_container_stats, print_object_stats +try: + from shlex import quote as sh_quote +except ImportError: + from pipes import quote as sh_quote BASENAME = 'swift' -commands = ('delete', 'download', 'list', 'post', - 'stat', 'upload', 'capabilities', 'info', 'tempurl') +commands = ('delete', 'download', 'list', 'post', 'stat', 'upload', + 'capabilities', 'info', 'tempurl', 'auth') def immediate_exit(signum, frame): @@ -905,6 +910,46 @@ def _print_compo_cap(name, capabilities): st_info = st_capabilities +st_auth_help = ''' +Display auth related authentication variables in shell friendly format. + + Commands to run to export storage url and auth token into + OS_STORAGE_URL and OS_AUTH_TOKEN: + + swift auth + + Commands to append to a runcom file (e.g. ~/.bashrc, /etc/profile) for + automatic authentication: + + swift auth -v -U test:tester -K testing \ + -A http://localhost:8080/auth/v1.0 + +'''.strip('\n') + + +def st_auth(parser, args, thread_manager): + (options, args) = parse_args(parser, args) + _opts = vars(options) + if options.verbose > 1: + if options.auth_version in ('1', '1.0'): + print('export ST_AUTH=%s' % sh_quote(options.auth)) + print('export ST_USER=%s' % sh_quote(options.user)) + print('export ST_KEY=%s' % sh_quote(options.key)) + else: + print('export OS_IDENTITY_API_VERSION=%s' % sh_quote( + options.auth_version)) + print('export OS_AUTH_VERSION=%s' % sh_quote(options.auth_version)) + print('export OS_AUTH_URL=%s' % sh_quote(options.auth)) + for k, v in sorted(_opts.items()): + if v and k.startswith('os_') and \ + k not in ('os_auth_url', 'os_options'): + print('export %s=%s' % (k.upper(), sh_quote(v))) + else: + conn = get_conn(_opts) + url, token = conn.get_auth() + print('export OS_STORAGE_URL=%s' % sh_quote(url)) + print('export OS_AUTH_TOKEN=%s' % sh_quote(token)) + st_tempurl_options = ' ' @@ -1073,7 +1118,8 @@ def main(arguments=None): or object. upload Uploads files or directories to the given container. capabilities List cluster capabilities. - tempurl Create a temporary URL + tempurl Create a temporary URL. + auth Display auth related environment variables. Examples: %%prog download --help diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index 98cef855..56c96f86 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -19,8 +19,10 @@ import os import tempfile import unittest +import textwrap from testtools import ExpectedException + import six import swiftclient @@ -46,6 +48,11 @@ 'ST_USER': 'test:tester', 'ST_KEY': 'testing' } +clean_os_environ = {} +environ_prefixes = ('ST_', 'OS_') +for key in os.environ: + if any(key.startswith(m) for m in environ_prefixes): + clean_os_environ[key] = '' clean_os_environ = {} environ_prefixes = ('ST_', 'OS_') @@ -1529,6 +1536,101 @@ def test_os_pre_authed_request(self): }), ]) + def test_auth(self): + headers = { + 'x-auth-token': 'AUTH_tk5b6b12', + 'x-storage-url': 'https://swift.storage.example.com/v1/AUTH_test', + } + mock_resp = self.fake_http_connection(200, headers=headers) + with mock.patch('swiftclient.client.http_connection', new=mock_resp): + stdout = six.StringIO() + with mock.patch('sys.stdout', new=stdout): + argv = [ + '', + 'auth', + '--auth', 'https://swift.storage.example.com/auth/v1.0', + '--user', 'test:tester', '--key', 'testing', + ] + swiftclient.shell.main(argv) + + expected = """ + export OS_STORAGE_URL=https://swift.storage.example.com/v1/AUTH_test + export OS_AUTH_TOKEN=AUTH_tk5b6b12 + """ + self.assertEquals(textwrap.dedent(expected).lstrip(), + stdout.getvalue()) + + def test_auth_verbose(self): + with mock.patch('swiftclient.client.http_connection') as mock_conn: + stdout = six.StringIO() + with mock.patch('sys.stdout', new=stdout): + argv = [ + '', + 'auth', + '--auth', 'https://swift.storage.example.com/auth/v1.0', + '--user', 'test:tester', '--key', 'te$tin&', + '--verbose', + ] + swiftclient.shell.main(argv) + + expected = """ + export ST_AUTH=https://swift.storage.example.com/auth/v1.0 + export ST_USER=test:tester + export ST_KEY='te$tin&' + """ + self.assertEquals(textwrap.dedent(expected).lstrip(), + stdout.getvalue()) + self.assertEqual([], mock_conn.mock_calls) + + def test_auth_v2(self): + os_options = {'tenant_name': 'demo'} + with mock.patch('swiftclient.client.get_auth_keystone', + new=fake_get_auth_keystone(os_options)): + stdout = six.StringIO() + with mock.patch('sys.stdout', new=stdout): + argv = [ + '', + 'auth', '-V2', + '--auth', 'https://keystone.example.com/v2.0/', + '--os-tenant-name', 'demo', + '--os-username', 'demo', '--os-password', 'admin', + ] + swiftclient.shell.main(argv) + + expected = """ + export OS_STORAGE_URL=http://url/ + export OS_AUTH_TOKEN=token + """ + self.assertEquals(textwrap.dedent(expected).lstrip(), + stdout.getvalue()) + + def test_auth_verbose_v2(self): + with mock.patch('swiftclient.client.get_auth_keystone') \ + as mock_keystone: + stdout = six.StringIO() + with mock.patch('sys.stdout', new=stdout): + argv = [ + '', + 'auth', '-V2', + '--auth', 'https://keystone.example.com/v2.0/', + '--os-tenant-name', 'demo', + '--os-username', 'demo', '--os-password', '$eKr3t', + '--verbose', + ] + swiftclient.shell.main(argv) + + expected = """ + export OS_IDENTITY_API_VERSION=2.0 + export OS_AUTH_VERSION=2.0 + export OS_AUTH_URL=https://keystone.example.com/v2.0/ + export OS_PASSWORD='$eKr3t' + export OS_TENANT_NAME=demo + export OS_USERNAME=demo + """ + self.assertEquals(textwrap.dedent(expected).lstrip(), + stdout.getvalue()) + self.assertEqual([], mock_keystone.mock_calls) + class TestCrossAccountObjectAccess(TestBase, MockHttpTest): """ diff --git a/tests/unit/utils.py b/tests/unit/utils.py index 955296ef..0a45437f 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -38,8 +38,10 @@ def fake_get_auth_keystone(auth_url, if exc: raise exc('test') # TODO: some way to require auth_url, user and key? - if expected_os_options and actual_os_options != expected_os_options: - return "", None + if expected_os_options: + for key, value in actual_os_options.items(): + if value and value != expected_os_options.get(key): + return "", None if 'required_kwargs' in kwargs: for k, v in kwargs['required_kwargs'].items(): if v != actual_kwargs.get(k): From 794b125e76e8ab4798d777ec85c559fb0aa1e5d1 Mon Sep 17 00:00:00 2001 From: Pradeep Kumar Singh Date: Sun, 24 May 2015 21:16:54 +0530 Subject: [PATCH 008/454] SwiftClient object upload beginning with / or "./" Currently SwiftClient populate response dictionary before removing "./" or "/" at begining of object name. This patch fixes that by changing that order. Closes-bug: #1412425 Change-Id: I80222754caba5d42a468f4677ac539e46682dd31 --- swiftclient/service.py | 8 +++---- tests/unit/test_service.py | 45 ++++++++++++++++++++++++++++++++++---- tests/unit/test_shell.py | 8 +++---- 3 files changed, 48 insertions(+), 13 deletions(-) diff --git a/swiftclient/service.py b/swiftclient/service.py index c533297e..5a24dac4 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -1552,6 +1552,10 @@ def _is_identical(self, chunk_data, path): def _upload_object_job(self, conn, container, source, obj, options, results_queue=None): + if obj.startswith('./') or obj.startswith('.\\'): + obj = obj[2:] + if obj.startswith('/'): + obj = obj[1:] res = { 'action': 'upload_object', 'container': container, @@ -1564,10 +1568,6 @@ def _upload_object_job(self, conn, container, source, obj, options, path = source res['path'] = path try: - if obj.startswith('./') or obj.startswith('.\\'): - obj = obj[2:] - if obj.startswith('/'): - obj = obj[1:] if path is not None: put_headers = {'x-object-meta-mtime': "%f" % getmtime(path)} else: diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index 0e8aff10..68d2e123 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -21,12 +21,16 @@ from mock import Mock, PropertyMock from six.moves.queue import Queue, Empty as QueueEmptyError from six import BytesIO - import swiftclient import swiftclient.utils as utils -from swiftclient.client import Connection -from swiftclient.service import SwiftService, SwiftError - +from swiftclient.client import Connection, ClientException +from swiftclient.service import SwiftService, SwiftError,\ + SwiftUploadObject +import six +if six.PY2: + import __builtin__ as builtins +else: + import builtins clean_os_environ = {} environ_prefixes = ('ST_', 'OS_') @@ -551,6 +555,39 @@ def test_upload_with_bad_segment_size(self): self.assertEqual('Segment size should be an integer value', exc.value) + @mock.patch('swiftclient.service.stat') + @mock.patch('swiftclient.service.getmtime', return_value=1.0) + @mock.patch('swiftclient.service.getsize', return_value=4) + @mock.patch.object(builtins, 'open', return_value=six.StringIO('asdf')) + def test_upload_with_relative_path(self, *args, **kwargs): + service = SwiftService({}) + objects = [{'path': "./test", + 'strt_indx': 2}, + {'path': os.path.join(os.getcwd(), "test"), + 'strt_indx': 1}, + {'path': ".\\test", + 'strt_indx': 2}] + for obj in objects: + with mock.patch('swiftclient.service.Connection') as mock_conn: + mock_conn.return_value.head_object.side_effect = \ + ClientException('Not Found', http_status=404) + mock_conn.return_value.put_object.return_value =\ + 'd41d8cd98f00b204e9800998ecf8427e' + resp_iter = service.upload( + 'c', [SwiftUploadObject(obj['path'])]) + responses = [x for x in resp_iter] + for resp in responses: + self.assertTrue(resp['success']) + self.assertEqual(2, len(responses)) + create_container_resp, upload_obj_resp = responses + self.assertEqual(create_container_resp['action'], + 'create_container') + self.assertEqual(upload_obj_resp['action'], + 'upload_object') + self.assertEqual(upload_obj_resp['object'], + obj['path'][obj['strt_indx']:]) + self.assertEqual(upload_obj_resp['path'], obj['path']) + class TestServiceUpload(testtools.TestCase): diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index b89df96f..fd41069e 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -1608,7 +1608,7 @@ def test_upload_with_read_write_access(self): self.assertRequests([('PUT', self.cont_path), ('PUT', self.obj_path)]) - self.assertEqual(self.obj, out.strip()) + self.assertEqual(self.obj[1:], out.strip()) expected_err = 'Warning: failed to create container %r: 403 Fake' \ % self.cont self.assertEqual(expected_err, out.err.strip()) @@ -1617,7 +1617,6 @@ def test_upload_with_write_only_access(self): req_handler = self._fake_cross_account_auth(False, True) fake_conn = self.fake_http_connection(403, 403, on_request=req_handler) - args, env = self._make_cmd('upload', cmd_args=[self.cont, self.obj, '--leave-segments']) with mock.patch('swiftclient.client._import_keystone_client', @@ -1629,10 +1628,9 @@ def test_upload_with_write_only_access(self): swiftclient.shell.main(args) except SystemExit as e: self.fail('Unexpected SystemExit: %s' % e) - self.assertRequests([('PUT', self.cont_path), ('PUT', self.obj_path)]) - self.assertEqual(self.obj, out.strip()) + self.assertEqual(self.obj[1:], out.strip()) expected_err = 'Warning: failed to create container %r: 403 Fake' \ % self.cont self.assertEqual(expected_err, out.err.strip()) @@ -1667,7 +1665,7 @@ def test_segment_upload_with_write_only_access(self): self.assert_request(('PUT', segment_path_0)) self.assert_request(('PUT', segment_path_1)) self.assert_request(('PUT', self.obj_path)) - self.assertTrue(self.obj in out.out) + self.assertTrue(self.obj[1:] in out.out) expected_err = 'Warning: failed to create container %r: 403 Fake' \ % self.cont self.assertEqual(expected_err, out.err.strip()) From 7c716997a8ede3f98741709b0594340df9849a76 Mon Sep 17 00:00:00 2001 From: Clint Byrum Date: Thu, 18 Jun 2015 09:04:33 -0700 Subject: [PATCH 009/454] Fix docstring typo for SwiftService.upload The option from argparse in shell is 'header' and that is what is used in the code. Change-Id: I5c889192ef7c46c299dc0ec7cbc7c4d027dae6d5 --- swiftclient/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swiftclient/service.py b/swiftclient/service.py index 90daf5ad..2760a08d 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -1202,7 +1202,7 @@ def upload(self, container, objects, options=None): { 'meta': [], - 'headers': [], + 'header': [], 'segment_size': None, 'use_slo': False, 'segment_container': None, From e596489020438c7f3b747fc0efe93c158d45c4a5 Mon Sep 17 00:00:00 2001 From: Pradeep Kumar Singh Date: Tue, 23 Jun 2015 10:48:50 +0900 Subject: [PATCH 010/454] Added check for negative segment-size Closes-Bug: #1453135 Change-Id: Ia9c2b27d998e6ac1889cc74c12e456c06ecd84d9 --- swiftclient/shell.py | 3 +++ tests/unit/test_shell.py | 23 +++++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 29b34573..4438e9d0 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -763,6 +763,9 @@ def st_upload(parser, args, output_manager): return options.segment_size = str((1024 ** size_mod) * multiplier) + if int(options.segment_size) <= 0: + output_manager.error("segment-size should be positive") + return _opts = vars(options) _opts['object_uu_threads'] = options.object_threads diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index 98cef855..4ab9ad73 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -897,6 +897,29 @@ def _check_expected(x, expected): swiftclient.shell.main(argv) self.assertEquals(output.err, "Invalid segment size\n") + def test_negative_upload_segment_size(self): + with CaptureOutput() as output: + with ExpectedException(SystemExit): + argv = ["", "upload", "-S", "-40", "container", "object"] + swiftclient.shell.main(argv) + self.assertEquals(output.err, "segment-size should be positive\n") + output.clear() + with ExpectedException(SystemExit): + argv = ["", "upload", "-S", "-40K", "container", "object"] + swiftclient.shell.main(argv) + self.assertEquals(output.err, "segment-size should be positive\n") + output.clear() + with ExpectedException(SystemExit): + argv = ["", "upload", "-S", "-40M", "container", "object"] + swiftclient.shell.main(argv) + self.assertEquals(output.err, "segment-size should be positive\n") + output.clear() + with ExpectedException(SystemExit): + argv = ["", "upload", "-S", "-40G", "container", "object"] + swiftclient.shell.main(argv) + self.assertEquals(output.err, "segment-size should be positive\n") + output.clear() + class TestSubcommandHelp(unittest.TestCase): From ef467ddee494a9c0154c31fa6aae781d89ed6abe Mon Sep 17 00:00:00 2001 From: janonymous Date: Sun, 28 Jun 2015 07:34:18 +0530 Subject: [PATCH 011/454] Python 3: Replacing unicode with six.text_type for py3 compatibility The "unicode" type was renamed to "str" in Python 3. Use six.text_type to make swiftclient compatible with Python 3. For more information about changes needed for py3 compatibility, see: https://wiki.openstack.org/wiki/Python3 Change-Id: Ic65607a69935652a1874340928f626fbcc35c014 --- swiftclient/multithreading.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swiftclient/multithreading.py b/swiftclient/multithreading.py index c53d9870..2778face 100644 --- a/swiftclient/multithreading.py +++ b/swiftclient/multithreading.py @@ -102,7 +102,7 @@ def get_error_count(self): def _print(self, item, stream=None): if stream is None: stream = self.print_stream - if six.PY2 and isinstance(item, unicode): + if six.PY2 and isinstance(item, six.text_type): item = item.encode('utf8') print(item, file=stream) From 91855bd912c75d4b6b86a3245a44099d8d03c676 Mon Sep 17 00:00:00 2001 From: YangLei Date: Tue, 30 Jun 2015 15:05:40 +0800 Subject: [PATCH 012/454] Correct the help message of swift tempurl correct the help message of swift tempurl use <> instead of [] in Positional arguments. Change-Id: Ib60ce97cef03e0423082c497604525eba2300fa9 --- swiftclient/shell.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 29b34573..af9ff259 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -913,13 +913,13 @@ def _print_compo_cap(name, capabilities): Generates a temporary URL for a Swift object. Positional arguments: - [method] An HTTP method to allow for this temporary URL. + An HTTP method to allow for this temporary URL. Usually 'GET' or 'PUT'. - [seconds] The amount of time in seconds the temporary URL will + The amount of time in seconds the temporary URL will be valid for. - [path] The full path to the Swift object. Example: + The full path to the Swift object. Example: /v1/AUTH_account/c/o. - [key] The secret temporary URL key set on the Swift cluster. + The secret temporary URL key set on the Swift cluster. To set a key, run \'swift post -m "Temp-URL-Key:b3968d0207b54ece87cccc06515a89d4"\' '''.strip('\n') From 70afe207cf1022e120c49622b3e6a9740cf2eb05 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Tue, 30 Jun 2015 16:17:03 -0700 Subject: [PATCH 013/454] Bump hacking in test-requirements The gate-swiftclient-dsvm-functional job takes updates from openstack/requirements and tries to run functional tests against them. However, the global requirements recently added environment markers (like "futures>=3.0;python_version=='2.7' or python_version=='2.6'") which require pbr>=1.2. pbr was getting capped at 1.0 by hacking<0.9, so bump it to get newer pbr. Change-Id: I10af36602d23db78c2863d84c0012835b310bbdd --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 54279207..3c804117 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,4 +1,4 @@ -hacking>=0.8.0,<0.9 +hacking>=0.10.0,<0.11 coverage>=3.6 discover From cf0b6c03df5b014e8c21ffcb0fe92f58e11c7ae3 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Sun, 17 May 2015 23:51:37 -0700 Subject: [PATCH 014/454] Properly test raw writes in Python 3 Previously we were trying to test writing bytes in Python 3 using only native (unicode) string objects. That doesn't test what we thought we were testing. Change-Id: I10a0a38143d7f7d850ab9a7005ad87f5d314c375 --- tests/unit/test_multithreading.py | 27 ++++++++++----------------- tests/unit/utils.py | 25 ++++++++++++------------- 2 files changed, 22 insertions(+), 30 deletions(-) diff --git a/tests/unit/test_multithreading.py b/tests/unit/test_multithreading.py index 65977934..42abbbf8 100644 --- a/tests/unit/test_multithreading.py +++ b/tests/unit/test_multithreading.py @@ -12,7 +12,6 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. - import sys import testtools import threading @@ -206,10 +205,12 @@ def test_printers(self): u'some raw bytes: \u062A\u062A'.encode('utf-8')) thread_manager.print_items([ - ('key', u'value'), - ('object', 'O\xcc\x88bject') + ('key', 'value'), + ('object', u'O\u0308bject'), ]) + thread_manager.print_raw(b'\xffugly\xffraw') + # Now we have a thread for error printing and a thread for # normal print messages self.assertEqual(starting_thread_count + 2, @@ -220,31 +221,23 @@ def test_printers(self): if six.PY3: over_the = "over the '\u062a\u062a'\n" - # The CaptureStreamBuffer just encodes all bytes written to it by - # mapping chr over the byte string to produce a str. - raw_bytes = ''.join( - map(chr, u'some raw bytes: \u062A\u062A'.encode('utf-8')) - ) else: over_the = "over the u'\\u062a\\u062a'\n" # We write to the CaptureStream so no decoding is performed - raw_bytes = 'some raw bytes: \xd8\xaa\xd8\xaa' self.assertEqual(''.join([ 'one-argument\n', 'one fish, 88 fish\n', 'some\n', 'where\n', - over_the, raw_bytes, + over_the, + u'some raw bytes: \u062a\u062a', ' key: value\n', - ' object: O\xcc\x88bject\n' - ]), out_stream.getvalue()) + u' object: O\u0308bject\n' + ]).encode('utf8') + b'\xffugly\xffraw', out_stream.getvalue()) - first_item = u'I have 99 problems, but a \u062A\u062A is not one\n' - if six.PY2: - first_item = first_item.encode('utf8') self.assertEqual(''.join([ - first_item, + u'I have 99 problems, but a \u062A\u062A is not one\n', 'one-error-argument\n', 'Sometimes\n', '3.1% just\n', 'does not\n', 'work!\n' - ]), err_stream.getvalue()) + ]), err_stream.getvalue().decode('utf8')) self.assertEqual(3, thread_manager.error_count) diff --git a/tests/unit/utils.py b/tests/unit/utils.py index 955296ef..aced1735 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -381,28 +381,27 @@ def tearDown(self): reload_module(c) -class CaptureStreamBuffer(object): +class CaptureStreamPrinter(object): """ - CaptureStreamBuffer is used for testing raw byte writing for PY3. Anything - written here is decoded as utf-8 and written to the parent CaptureStream + CaptureStreamPrinter is used for testing unicode writing for PY3. Anything + written here is encoded as utf-8 and written to the parent CaptureStream """ def __init__(self, captured_stream): self._captured_stream = captured_stream - def write(self, bytes_data): + def write(self, data): # No encoding, just convert the raw bytes into a str for testing # The below call also validates that we have a byte string. self._captured_stream.write( - ''.join(map(chr, bytes_data)) - ) + data if isinstance(data, six.binary_type) else data.encode('utf8')) class CaptureStream(object): def __init__(self, stream): self.stream = stream - self._capture = six.StringIO() - self._buffer = CaptureStreamBuffer(self) + self._buffer = six.BytesIO() + self._capture = CaptureStreamPrinter(self._buffer) self.streams = [self._capture] @property @@ -425,11 +424,11 @@ def writelines(self, *args, **kwargs): stream.writelines(*args, **kwargs) def getvalue(self): - return self._capture.getvalue() + return self._buffer.getvalue() def clear(self): - self._capture.truncate(0) - self._capture.seek(0) + self._buffer.truncate(0) + self._buffer.seek(0) class CaptureOutput(object): @@ -467,11 +466,11 @@ def __exit__(self, *args, **kwargs): @property def out(self): - return self._out.getvalue() + return self._out.getvalue().decode('utf8') @property def err(self): - return self._err.getvalue() + return self._err.getvalue().decode('utf8') def clear(self): self._out.clear() From bb252130ac2f332110172b8e2094dc629c8a896b Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Mon, 18 May 2015 10:14:09 -0700 Subject: [PATCH 015/454] Always decode command-line arguments as UTF-8 There was always an implicit assumption that they were UTF-8 before, and by converting them to unicode we close another hole allowing raw bytes to appear in user-facing messages. Closes-Bug: #1431866 Change-Id: If2e41d9a592c3ad02818e9c6f0959fd4b73cd0e0 --- swiftclient/shell.py | 15 ++++---- tests/unit/test_shell.py | 75 +++++++++++++++++++++++++++++----------- 2 files changed, 63 insertions(+), 27 deletions(-) diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 4438e9d0..0663e4f9 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function +from __future__ import print_function, unicode_literals import logging import signal @@ -23,6 +23,7 @@ from optparse import OptionParser, OptionGroup, SUPPRESS_HELP from os import environ, walk, _exit as os_exit from os.path import isfile, isdir, join +from six import text_type from sys import argv as sys_argv, exit, stderr from time import gmtime, strftime @@ -106,7 +107,7 @@ def st_delete(parser, args, output_manager): if '/' in container: output_manager.error( 'WARNING: / in container name; you ' - 'might have meant %r instead of %r.' % ( + "might have meant '%s' instead of '%s'." % ( container.replace('/', ' ', 1), container) ) return @@ -249,7 +250,7 @@ def st_download(parser, args, output_manager): if '/' in container: output_manager.error( 'WARNING: / in container name; you ' - 'might have meant %r instead of %r.' % ( + "might have meant '%s' instead of '%s'." % ( container.replace('/', ' ', 1), container) ) return @@ -500,7 +501,7 @@ def st_stat(parser, args, output_manager): if '/' in container: output_manager.error( 'WARNING: / in container name; you might have ' - 'meant %r instead of %r.' % + "meant '%s' instead of '%s'." % (container.replace('/', ' ', 1), container)) return args = args[1:] @@ -604,7 +605,7 @@ def st_post(parser, args, output_manager): if '/' in container: output_manager.error( 'WARNING: / in container name; you might have ' - 'meant %r instead of %r.' % + "meant '%s' instead of '%s'." % (args[0].replace('/', ' ', 1), args[0])) return args = args[1:] @@ -844,7 +845,7 @@ def st_upload(parser, args, output_manager): msg = ': %s' % error output_manager.warning( 'Warning: failed to create container ' - '%r%s', container, msg + "'%s'%s", container, msg ) else: output_manager.error("%s" % error) @@ -1037,6 +1038,8 @@ def main(arguments=None): else: argv = sys_argv + argv = [a if isinstance(a, text_type) else a.decode('utf-8') for a in argv] + version = client_version parser = OptionParser(version='python-swiftclient %s' % version, usage=''' diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index 117b66be..9f979908 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -12,6 +12,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import unicode_literals from genericpath import getmtime import hashlib @@ -650,6 +651,38 @@ def test_delete_container(self, connection): connection.return_value.delete_object.assert_called_with( 'container', 'object', query_string=None, response_dict={}) + def test_delete_verbose_output_utf8(self): + container = 't\u00e9st_c' + base_argv = ['', '--verbose', 'delete'] + + # simulate container having an object with utf-8 code points in name, + # just returning the object delete result + res = {'success': True, 'response_dict': {}, 'attempts': 2, + 'container': container, 'action': 'delete_object', + 'object': 'obj_t\u00east_o'} + + with mock.patch('swiftclient.shell.SwiftService.delete') as mock_func: + with CaptureOutput() as out: + mock_func.return_value = [res] + swiftclient.shell.main(base_argv + [container.encode('utf-8')]) + + mock_func.assert_called_once_with(container=container) + self.assertTrue(out.out.find( + 'obj_t\u00east_o [after 2 attempts]') >= 0, out) + + # simulate empty container + res = {'success': True, 'response_dict': {}, 'attempts': 2, + 'container': container, 'action': 'delete_container'} + + with mock.patch('swiftclient.shell.SwiftService.delete') as mock_func: + with CaptureOutput() as out: + mock_func.return_value = [res] + swiftclient.shell.main(base_argv + [container.encode('utf-8')]) + + mock_func.assert_called_once_with(container=container) + self.assertTrue(out.out.find( + 't\u00e9st_c [after 2 attempts]') >= 0, out) + @mock.patch('swiftclient.service.Connection') def test_delete_object(self, connection): argv = ["", "delete", "container", "object"] @@ -661,8 +694,8 @@ def test_delete_object(self, connection): def test_delete_verbose_output(self): del_obj_res = {'success': True, 'response_dict': {}, 'attempts': 2, - 'container': 'test_c', 'action': 'delete_object', - 'object': 'test_o'} + 'container': 't\xe9st_c', 'action': 'delete_object', + 'object': 't\xe9st_o'} del_seg_res = del_obj_res.copy() del_seg_res.update({'action': 'delete_segment'}) @@ -670,7 +703,7 @@ def test_delete_verbose_output(self): del_con_res = del_obj_res.copy() del_con_res.update({'action': 'delete_container', 'object': None}) - test_exc = Exception('test_exc') + test_exc = Exception('t\xe9st_exc') error_res = del_obj_res.copy() error_res.update({'success': False, 'error': test_exc, 'object': None}) @@ -680,39 +713,39 @@ def test_delete_verbose_output(self): with mock.patch('swiftclient.shell.SwiftService.delete', mock_delete): with CaptureOutput() as out: mock_delete.return_value = [del_obj_res] - swiftclient.shell.main(base_argv + ['test_c', 'test_o']) + swiftclient.shell.main(base_argv + ['t\xe9st_c', 't\xe9st_o']) - mock_delete.assert_called_once_with(container='test_c', - objects=['test_o']) + mock_delete.assert_called_once_with(container='t\xe9st_c', + objects=['t\xe9st_o']) self.assertTrue(out.out.find( - 'test_o [after 2 attempts]') >= 0) + 't\xe9st_o [after 2 attempts]') >= 0) with CaptureOutput() as out: mock_delete.return_value = [del_seg_res] - swiftclient.shell.main(base_argv + ['test_c', 'test_o']) + swiftclient.shell.main(base_argv + ['t\xe9st_c', 't\xe9st_o']) - mock_delete.assert_called_with(container='test_c', - objects=['test_o']) + mock_delete.assert_called_with(container='t\xe9st_c', + objects=['t\xe9st_o']) self.assertTrue(out.out.find( - 'test_c/test_o [after 2 attempts]') >= 0) + 't\xe9st_c/t\xe9st_o [after 2 attempts]') >= 0) with CaptureOutput() as out: mock_delete.return_value = [del_con_res] - swiftclient.shell.main(base_argv + ['test_c']) + swiftclient.shell.main(base_argv + ['t\xe9st_c']) - mock_delete.assert_called_with(container='test_c') + mock_delete.assert_called_with(container='t\xe9st_c') self.assertTrue(out.out.find( - 'test_c [after 2 attempts]') >= 0) + 't\xe9st_c [after 2 attempts]') >= 0) with CaptureOutput() as out: mock_delete.return_value = [error_res] self.assertRaises(SystemExit, swiftclient.shell.main, - base_argv + ['test_c']) + base_argv + ['t\xe9st_c']) - mock_delete.assert_called_with(container='test_c') + mock_delete.assert_called_with(container='t\xe9st_c') self.assertTrue(out.err.find( - 'Error Deleting: test_c: test_exc') >= 0) + 'Error Deleting: t\xe9st_c: t\xe9st_exc') >= 0) @mock.patch('swiftclient.service.Connection') def test_post_account(self, connection): @@ -1636,7 +1669,7 @@ def test_upload_with_read_write_access(self): self.assertRequests([('PUT', self.cont_path), ('PUT', self.obj_path)]) self.assertEqual(self.obj[1:], out.strip()) - expected_err = 'Warning: failed to create container %r: 403 Fake' \ + expected_err = "Warning: failed to create container '%s': 403 Fake" \ % self.cont self.assertEqual(expected_err, out.err.strip()) @@ -1658,7 +1691,7 @@ def test_upload_with_write_only_access(self): self.assertRequests([('PUT', self.cont_path), ('PUT', self.obj_path)]) self.assertEqual(self.obj[1:], out.strip()) - expected_err = 'Warning: failed to create container %r: 403 Fake' \ + expected_err = "Warning: failed to create container '%s': 403 Fake" \ % self.cont self.assertEqual(expected_err, out.err.strip()) @@ -1693,7 +1726,7 @@ def test_segment_upload_with_write_only_access(self): self.assert_request(('PUT', segment_path_1)) self.assert_request(('PUT', self.obj_path)) self.assertTrue(self.obj[1:] in out.out) - expected_err = 'Warning: failed to create container %r: 403 Fake' \ + expected_err = "Warning: failed to create container '%s': 403 Fake" \ % self.cont self.assertEqual(expected_err, out.err.strip()) @@ -1782,7 +1815,7 @@ def test_download_with_no_access(self): self.assertRequests([('GET', self.obj_path)]) path = '%s%s' % (self.cont, self.obj) - expected_err = 'Error downloading object %r' % path + expected_err = "Error downloading object '%s'" % path self.assertTrue(out.err.startswith(expected_err)) self.assertEqual('', out) From d5d312774445b5a0445ac6a4cd77091fdc0da90e Mon Sep 17 00:00:00 2001 From: Charles Hsu Date: Mon, 2 Mar 2015 17:54:05 +0800 Subject: [PATCH 016/454] Add ability to download objects to particular folder. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch adds "--output-dir" and "--remove-prefix" options to the "download" command and unit tests for it. Example: $ swift list example --prefix swift2.2 swift2.2/bin/swift-object-auditor swift2.2/bin/swift-object-expirer swift2.2/bin/swift-object-info swift2.2/bin/swift-object-replicator swift2.2/bin/swift-object-server swift2.2/bin/swift-object-updater When given "--output-dir ", client downloads objects to . $ swift download example --prefix swift2.2 \ --output-dir new/swift/dir The folder structure: . └── new └── swift └── dir └── swift2.2 └── bin ├── swift-object-auditor ├── swift-object-expirer ├── swift-object-info ├── swift-object-replicator ├── swift-object-server └── swift-object-updater When given "--remove-prefix", client downloads objects without . $ swift download example --prefix swift2.2 \ --remove-prefix \ --output-dir swift The folder structure: . └── swift └── bin ├── swift-object-auditor ├── swift-object-expirer ├── swift-object-info ├── swift-object-replicator ├── swift-object-server └── swift-object-updater Co-Authored-By: Clay Gerrard Change-Id: I7463fe2941cc94f9a50a4756a97c2ccdf946294d Implements: blueprint swiftclient-download-pseudo-folder-to-specific-target --- swiftclient/service.py | 13 +++- swiftclient/shell.py | 26 ++++++- tests/unit/test_service.py | 135 ++++++++++++++++++++++++++++++++++--- 3 files changed, 161 insertions(+), 13 deletions(-) diff --git a/swiftclient/service.py b/swiftclient/service.py index 90daf5ad..ebbc54d7 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -12,6 +12,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +import os from concurrent.futures import as_completed, CancelledError, TimeoutError from copy import deepcopy from errno import EEXIST, ENOENT @@ -162,6 +163,8 @@ def _build_default_global_options(): 'read_acl': None, 'write_acl': None, 'out_file': None, + 'out_directory': None, + 'remove_prefix': False, 'no_download': False, 'long': False, 'totals': False, @@ -889,7 +892,9 @@ def download(self, container=None, objects=None, options=None): 'no_download': False, 'header': [], 'skip_identical': False, - 'out_file': None + 'out_directory': None, + 'out_file': None, + 'remove_prefix': False, } :returns: A generator for returning the results of the download @@ -986,6 +991,12 @@ def _download_object_job(self, conn, container, obj, options): options['skip_identical'] = (options['skip_identical'] and out_file != '-') + if options['prefix'] and options['remove_prefix']: + path = path[len(options['prefix']):].lstrip('/') + + if options['out_directory']: + path = os.path.join(options['out_directory'], path) + if options['skip_identical']: filename = out_file if out_file else path try: diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 430efd29..8d87edca 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -146,9 +146,11 @@ def st_delete(parser, args, output_manager): st_download_options = '''[--all] [--marker] [--prefix ] - [--output ] [--object-threads ] + [--output ] [--output-dir ] + [--object-threads ] [--container-threads ] [--no-download] - [--skip-identical] + [--skip-identical] [--remove-prefix] + ''' st_download_help = ''' @@ -167,9 +169,15 @@ def st_delete(parser, args, output_manager): --marker Marker to use when starting a container or account download. --prefix Only download items beginning with + --remove-prefix An optional flag for --prefix , use this + option to download items without --output For a single file download, stream the output to . Specifying "-" as will redirect to stdout. + --output-dir + An optional directory to which to store objects. + By default, all objects are recreated in the current + directory. --object-threads Number of threads to use for downloading objects. Default is 10. @@ -203,6 +211,14 @@ def st_download(parser, args, output_manager): '-o', '--output', dest='out_file', help='For a single ' 'download, stream the output to . ' 'Specifying "-" as will redirect to stdout.') + parser.add_option( + '-D', '--output-dir', dest='out_directory', + help='An optional directory to which to store objects. ' + 'By default, all objects are recreated in the current directory.') + parser.add_option( + '-r', '--remove-prefix', action='store_true', dest='remove_prefix', + default=False, help='An optional flag for --prefix , ' + 'use this option to download items without .') parser.add_option( '', '--object-threads', type=int, default=10, help='Number of threads to use for downloading objects. ' @@ -233,6 +249,12 @@ def st_download(parser, args, output_manager): if options.out_file and len(args) != 2: exit('-o option only allowed for single file downloads') + if not options.prefix: + options.remove_prefix = False + + if options.out_directory and len(args) == 2: + exit('Please use -o option for single file downloads and renames') + if (not args and not options.yes_all) or (args and options.yes_all): output_manager.error('Usage: %s download %s\n%s', BASENAME, st_download_options, st_download_help) diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index 74a6ce32..3a1a8acc 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -992,6 +992,17 @@ def test_upload_object_job_identical_dlo(self): class TestServiceDownload(testtools.TestCase): + def setUp(self): + super(TestServiceDownload, self).setUp() + self.opts = swiftclient.service._default_local_options.copy() + self.opts['no_download'] = True + self.obj_content = b'c' * 10 + self.obj_etag = md5(self.obj_content).hexdigest() + self.obj_len = len(self.obj_content) + + def _readbody(self): + yield self.obj_content + def _assertDictEqual(self, a, b, m=None): # assertDictEqual is not available in py2.6 so use a shallow check # instead @@ -1008,6 +1019,103 @@ def _assertDictEqual(self, a, b, m=None): self.assertIn(k, b, m) self.assertEqual(b[k], v, m) + def test_download(self): + service = SwiftService() + with mock.patch('swiftclient.service.Connection') as mock_conn: + header = {'content-length': self.obj_len, + 'etag': self.obj_etag} + mock_conn.get_object.return_value = header, self._readbody() + + resp = service._download_object_job(mock_conn, + 'c', + 'test', + self.opts) + + self.assertTrue(resp['success']) + self.assertEqual(resp['action'], 'download_object') + self.assertEqual(resp['object'], 'test') + self.assertEqual(resp['path'], 'test') + + def test_download_with_output_dir(self): + service = SwiftService() + with mock.patch('swiftclient.service.Connection') as mock_conn: + header = {'content-length': self.obj_len, + 'etag': self.obj_etag} + mock_conn.get_object.return_value = header, self._readbody() + + options = self.opts.copy() + options['out_directory'] = 'temp_dir' + resp = service._download_object_job(mock_conn, + 'c', + 'example/test', + options) + + self.assertTrue(resp['success']) + self.assertEqual(resp['action'], 'download_object') + self.assertEqual(resp['object'], 'example/test') + self.assertEqual(resp['path'], 'temp_dir/example/test') + + def test_download_with_remove_prefix(self): + service = SwiftService() + with mock.patch('swiftclient.service.Connection') as mock_conn: + header = {'content-length': self.obj_len, + 'etag': self.obj_etag} + mock_conn.get_object.return_value = header, self._readbody() + + options = self.opts.copy() + options['prefix'] = 'example/' + options['remove_prefix'] = True + resp = service._download_object_job(mock_conn, + 'c', + 'example/test', + options) + + self.assertTrue(resp['success']) + self.assertEqual(resp['action'], 'download_object') + self.assertEqual(resp['object'], 'example/test') + self.assertEqual(resp['path'], 'test') + + def test_download_with_remove_prefix_and_remove_slashes(self): + service = SwiftService() + with mock.patch('swiftclient.service.Connection') as mock_conn: + header = {'content-length': self.obj_len, + 'etag': self.obj_etag} + mock_conn.get_object.return_value = header, self._readbody() + + options = self.opts.copy() + options['prefix'] = 'example' + options['remove_prefix'] = True + resp = service._download_object_job(mock_conn, + 'c', + 'example/test', + options) + + self.assertTrue(resp['success']) + self.assertEqual(resp['action'], 'download_object') + self.assertEqual(resp['object'], 'example/test') + self.assertEqual(resp['path'], 'test') + + def test_download_with_output_dir_and_remove_prefix(self): + service = SwiftService() + with mock.patch('swiftclient.service.Connection') as mock_conn: + header = {'content-length': self.obj_len, + 'etag': self.obj_etag} + mock_conn.get_object.return_value = header, self._readbody() + + options = self.opts.copy() + options['prefix'] = 'example' + options['out_directory'] = 'new/dir' + options['remove_prefix'] = True + resp = service._download_object_job(mock_conn, + 'c', + 'example/test', + options) + + self.assertTrue(resp['success']) + self.assertEqual(resp['action'], 'download_object') + self.assertEqual(resp['object'], 'example/test') + self.assertEqual(resp['path'], 'new/dir/test') + def test_download_object_job_skip_identical(self): with tempfile.NamedTemporaryFile() as f: f.write(b'a' * 30) @@ -1040,6 +1148,9 @@ def fake_get(*args, **kwargs): container='test_c', obj='test_o', options={'out_file': f.name, + 'out_directory': None, + 'prefix': None, + 'remove_prefix': False, 'header': {}, 'yes_all': False, 'skip_identical': True}) @@ -1092,6 +1203,9 @@ def test_download_object_job_skip_identical_dlo(self): container='test_c', obj='test_o', options={'out_file': f.name, + 'out_directory': None, + 'prefix': None, + 'remove_prefix': False, 'header': {}, 'yes_all': False, 'skip_identical': True}) @@ -1170,6 +1284,9 @@ def test_download_object_job_skip_identical_nested_slo(self): container='test_c', obj='test_o', options={'out_file': f.name, + 'out_directory': None, + 'prefix': None, + 'remove_prefix': False, 'header': {}, 'yes_all': False, 'skip_identical': True}) @@ -1231,6 +1348,9 @@ def test_download_object_job_skip_identical_diff_dlo(self): 'auth_end_time': mock_conn.auth_end_time, } + options = self.opts.copy() + options['out_file'] = f.name + options['skip_identical'] = True s = SwiftService() with mock.patch('swiftclient.service.time', side_effect=range(3)): with mock.patch('swiftclient.service.get_conn', @@ -1239,11 +1359,7 @@ def test_download_object_job_skip_identical_diff_dlo(self): conn=mock_conn, container='test_c', obj='test_o', - options={'out_file': f.name, - 'header': {}, - 'no_download': True, - 'yes_all': False, - 'skip_identical': True}) + options=options) self._assertDictEqual(r, expected_r) @@ -1323,6 +1439,9 @@ def test_download_object_job_skip_identical_diff_nested_slo(self): 'auth_end_time': mock_conn.auth_end_time, } + options = self.opts.copy() + options['out_file'] = f.name + options['skip_identical'] = True s = SwiftService() with mock.patch('swiftclient.service.time', side_effect=range(3)): with mock.patch('swiftclient.service.get_conn', @@ -1331,11 +1450,7 @@ def test_download_object_job_skip_identical_diff_nested_slo(self): conn=mock_conn, container='test_c', obj='test_o', - options={'out_file': f.name, - 'header': {}, - 'no_download': True, - 'yes_all': False, - 'skip_identical': True}) + options=options) self._assertDictEqual(r, expected_r) self.assertEqual(mock_conn.get_object.mock_calls, [ From 87c0a839888123c8a1b5d0ec061e244bfc3e044c Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Fri, 10 Jul 2015 08:10:30 -0700 Subject: [PATCH 017/454] Update mock requirements The last version supporting python 2.6 was 1.0.1. Change-Id: Ib7d51157f9654d240cdd67a0a1e6fcb14a70c84a --- test-requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index 3c804117..909cb043 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,7 +2,8 @@ hacking>=0.10.0,<0.11 coverage>=3.6 discover -mock>=1.0 +mock>=1.0;python_version!='2.6' +mock==1.0.1;python_version=='2.6' oslosphinx python-keystoneclient>=0.7.0 sphinx>=1.1.2,<1.2 From ca70dd9e158cc18faad165246c93c4d6a6aa6628 Mon Sep 17 00:00:00 2001 From: "Zack M. Davis" Date: Mon, 22 Jun 2015 17:45:30 -0700 Subject: [PATCH 018/454] add tempurl command to swift.1 man page Change-Id: Ifccc7f6dc049ca0ac2c53c00b1704cff4d1a770f Closes-Bug: #1450606 --- doc/manpages/swift.1 | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/manpages/swift.1 b/doc/manpages/swift.1 index a4704323..446ade49 100644 --- a/doc/manpages/swift.1 +++ b/doc/manpages/swift.1 @@ -104,6 +104,14 @@ is not provided the storage-url retrieved after authentication is used as proxy-url. .RE +\fBtempurl\fR method seconds path key +.RS 4 +Generates a temporary URL allowing unauthenticated access to the Swift object at +the given path, using the given HTTP method, for the given number of seconds, +using the given TempURL key. \fBExample\fR: tempurl GET 86400 +/v1/AUTH_foo/bar_container/quux.md my_secret_tempurl_key +.RE + .SH OPTIONS .PD 0 .IP "--version Show program's version number and exit" From 7442f0dcc842bb000a1baba6d1975bf5de599280 Mon Sep 17 00:00:00 2001 From: Hiroshi Miura Date: Fri, 17 Jul 2015 16:03:39 +0900 Subject: [PATCH 019/454] swiftclient: add short options to help message - add usage strings for short option such as '-a' for '--all' This add all short options to usage text. - add missing --header usage help for download command - some cometic changes Closes-bug: #1475511 Change-Id: Ibfecac8764669540fa025787548133a50fa50b10 Signed-off-by: Hiroshi Miura --- swiftclient/shell.py | 77 +++++++++++++++++++++++++------------------- 1 file changed, 43 insertions(+), 34 deletions(-) diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 3cc73145..865c6059 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -65,7 +65,7 @@ def immediate_exit(signum, frame): for multiple objects. Optional arguments: - --all Delete all containers and objects. + -a, --all Delete all containers and objects. --leave-segments Do not delete segments of manifest objects. --object-threads Number of threads to use for deleting objects. @@ -88,10 +88,10 @@ def st_delete(parser, args, output_manager): '', '--object-threads', type=int, default=10, help='Number of threads to use for deleting objects. ' 'Default is 10.') - parser.add_option('', '--container-threads', type=int, - default=10, help='Number of threads to use for ' - 'deleting containers. ' - 'Default is 10.') + parser.add_option( + '', '--container-threads', type=int, + default=10, help='Number of threads to use for deleting containers. ' + 'Default is 10.') (options, args) = parse_args(parser, args) args = args[1:] if (not args and not options.yes_all) or (args and options.yes_all): @@ -155,6 +155,7 @@ def st_delete(parser, args, output_manager): [--object-threads ] [--container-threads ] [--no-download] [--skip-identical] [--remove-prefix] + [--header ] ''' @@ -169,17 +170,18 @@ def st_delete(parser, args, output_manager): objects from the container. Optional arguments: - --all Indicates that you really want to download + -a, --all Indicates that you really want to download everything in the account. - --marker Marker to use when starting a container or account + -m, --marker Marker to use when starting a container or account download. - --prefix Only download items beginning with - --remove-prefix An optional flag for --prefix , use this + -p, --prefix Only download items beginning with + -r, --remove-prefix An optional flag for --prefix , use this option to download items without - --output For a single file download, stream the output to + -o, --output + For a single file download, stream the output to . Specifying "-" as will redirect to stdout. - --output-dir + -D, --output-dir An optional directory to which to store objects. By default, all objects are recreated in the current directory. @@ -191,9 +193,9 @@ def st_delete(parser, args, output_manager): Default is 10. --no-download Perform download(s), but don't actually write anything to disk. - --header + -H, --header Adds a customized request header to the query, like - "Range" or "If-Match". This argument is repeatable. + "Range" or "If-Match". This option may be repeated. Example --header "content-type:text/plain" --skip-identical Skip downloading files that are identical on both sides. @@ -240,7 +242,7 @@ def st_download(parser, args, output_manager): '-H', '--header', action='append', dest='header', default=[], help='Adds a customized request header to the query, like "Range" or ' - '"If-Match". This argument is repeatable. ' + '"If-Match". This option may be repeated. ' 'Example: --header "content-type:text/plain"') parser.add_option( '--skip-identical', action='store_true', dest='skip_identical', @@ -365,12 +367,12 @@ def st_download(parser, args, output_manager): [container] Name of container to list object in. Optional arguments: - --long Long listing format, similar to ls -l. + -l, --long Long listing format, similar to ls -l. --lh Report sizes in human readable format similar to ls -lh. - --totals Used with -l or --lh, only report totals. - --prefix Only list items beginning with the prefix. - --delimiter Roll up items with the given delimiter. For containers + -t, --totals Used with -l or --lh, only report totals. + -p, --prefix Only list items beginning with the prefix. + -d, --delimiter Roll up items with the given delimiter. For containers only. See OpenStack Swift API documentation for what this means. '''.strip('\n') @@ -576,17 +578,22 @@ def st_stat(parser, args, output_manager): [object] Name of object to post. Optional arguments: - --read-acl Read ACL for containers. Quick summary of ACL syntax: + -r, --read-acl Read ACL for containers. Quick summary of ACL syntax: .r:*, .r:-.example.com, .r:www.example.com, account1, account2:user2 - --write-acl Write ACL for containers. Quick summary of ACL syntax: + -w, --write-acl Write ACL for containers. Quick summary of ACL syntax: account1 account2:user2 - --sync-to Sync To for containers, for multi-cluster replication. - --sync-key Sync Key for containers, for multi-cluster replication. - --meta Sets a meta data item. This option may be repeated. + -t, --sync-to + Sync To for containers, for multi-cluster replication. + -k, --sync-key + Sync Key for containers, for multi-cluster replication. + -m, --meta + Sets a meta data item. This option may be repeated. Example: -m Color:Blue -m Size:Large - --header
Set request headers. This option may be repeated. - Example -H "content-type:text/plain" + -H, --header + Adds a customized request header. + This option may be repeated. Example + -H "content-type:text/plain" -H "Content-Length: 4000" '''.strip('\n') @@ -611,7 +618,8 @@ def st_post(parser, args, output_manager): 'Example: -m Color:Blue -m Size:Large') parser.add_option( '-H', '--header', action='append', dest='header', - default=[], help='Set request headers. This option may be repeated. ' + default=[], help='Adds a customized request header. ' + 'This option may be repeated. ' 'Example: -H "content-type:text/plain" ' '-H "Content-Length: 4000"') (options, args) = parse_args(parser, args) @@ -664,8 +672,7 @@ def st_post(parser, args, output_manager): ''' -st_upload_help = ''' -Uploads specified files and directories to the given container. +st_upload_help = ''' Uploads specified files and directories to the given container. Positional arguments: Name of container to upload to. @@ -673,10 +680,11 @@ def st_post(parser, args, output_manager): times for multiple uploads. Optional arguments: - --changed Only upload files that have changed since the last + -c, --changed Only upload files that have changed since the last upload. --skip-identical Skip uploading files that are identical on both sides. - --segment-size Upload files in segments no larger than (in + -S, --segment-size + Upload files in segments no larger than (in Bytes) and then create a "manifest" file that will download all the segments as if it were the original file. @@ -693,9 +701,10 @@ def st_post(parser, args, output_manager): --segment-threads Number of threads to use for uploading object segments. Default is 10. - --header
Set request headers with the syntax header:value. - This option may be repeated. - Example -H "content-type:text/plain". + -H, --header + Adds a customized request header. This option may be + repeated. Example -H "content-type:text/plain" + -H "Content-Length: 4000". --use-slo When used in conjunction with --segment-size it will create a Static Large Object instead of the default Dynamic Large Object. @@ -1127,7 +1136,7 @@ def main(arguments=None): [--os-endpoint-type ] [--os-cacert ] [--insecure] [--no-ssl-compression] - [--help] + [--help] [] Command-line interface to the OpenStack Swift API. From a8c4df98eee43b419d6dd30e80c838d9f2efd025 Mon Sep 17 00:00:00 2001 From: Joel Wright Date: Tue, 14 Oct 2014 16:54:41 +0100 Subject: [PATCH 020/454] Reduce memory usage for download/delete and add --no-shuffle option to st_download The current code builds a full object listing before performing either a multiple download or delete operation (and also shuffles this complete list in the case of a download). This patch removes the creation of the full object list and adds the ability to turn off shuffle for files when downloading. Also added is a limit on the number of list results that can be queued by a single call to service.list without consuming results (reduces memory overhead for large listings). Some tests added for service.py download and list. Change-Id: Ie737cbb7f8b1fa8a79bbb88914730b05aa7f2906 --- swiftclient/service.py | 129 ++++++---- swiftclient/shell.py | 18 ++ tests/unit/test_service.py | 492 +++++++++++++++++++++++++++++++++---- tests/unit/test_shell.py | 56 +++++ 4 files changed, 595 insertions(+), 100 deletions(-) diff --git a/swiftclient/service.py b/swiftclient/service.py index 7c557691..4f331c4e 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -176,7 +176,8 @@ def _build_default_global_options(): 'fail_fast': False, 'human': False, 'dir_marker': False, - 'checksum': True + 'checksum': True, + 'shuffle': False } POLICY = 'X-Storage-Policy' @@ -752,7 +753,7 @@ def list(self, container=None, options=None): else: options = self._options - rq = Queue() + rq = Queue(maxsize=10) # Just stop list running away consuming memory if container is None: listing_future = self.thread_manager.container_pool.submit( @@ -895,6 +896,7 @@ def download(self, container=None, objects=None, options=None): 'out_directory': None, 'out_file': None, 'remove_prefix': False, + 'shuffle' : False } :returns: A generator for returning the results of the download @@ -916,39 +918,20 @@ def download(self, container=None, objects=None, options=None): try: options_copy = deepcopy(options) options_copy["long"] = False - containers = [] + for part in self.list(options=options_copy): if part["success"]: - containers.extend([ - i['name'] for i in part["listing"] - ]) - else: - raise part["error"] + containers = [i['name'] for i in part["listing"]] - shuffle(containers) - - o_downs = [] - for con in containers: - objs = [] - for part in self.list( - container=con, options=options_copy): - if part["success"]: - objs.extend([ - i['name'] for i in part["listing"] - ]) - else: - raise part["error"] - shuffle(objs) - - o_downs.extend( - self.thread_manager.object_dd_pool.submit( - self._download_object_job, con, obj, - options_copy - ) for obj in objs - ) + if options['shuffle']: + shuffle(containers) - for o_down in interruptable_as_completed(o_downs): - yield o_down.result() + for con in containers: + for res in self._download_container( + con, options_copy): + yield res + else: + raise part["error"] # If we see a 404 here, the listing of the account failed except ClientException as err: @@ -1153,14 +1136,17 @@ def _download_object_job(self, conn, container, obj, options): } return res - def _download_container(self, container, options): + def _submit_page_downloads(self, container, page_generator, options): try: - objects = [] - for part in self.list(container=container, options=options): - if part["success"]: - objects.extend([o["name"] for o in part["listing"]]) - else: - raise part["error"] + list_page = next(page_generator) + except StopIteration: + return None + + if list_page["success"]: + objects = [o["name"] for o in list_page["listing"]] + + if options["shuffle"]: + shuffle(objects) o_downs = [ self.thread_manager.object_dd_pool.submit( @@ -1168,14 +1154,60 @@ def _download_container(self, container, options): ) for obj in objects ] - for o_down in interruptable_as_completed(o_downs): - yield o_down.result() + return o_downs + else: + raise list_page["error"] + def _download_container(self, container, options): + _page_generator = self.list(container=container, options=options) + try: + next_page_downs = self._submit_page_downloads( + container, _page_generator, options + ) except ClientException as err: if err.http_status != 404: raise - raise SwiftError('Container %r not found' % container, - container=container) + raise SwiftError( + 'Container %r not found' % container, container=container + ) + + error = None + while next_page_downs: + page_downs = next_page_downs + next_page_downs = None + + # Start downloading the next page of list results when + # we have completed 80% of the previous page + next_page_triggered = False + next_page_trigger_point = 0.8 * len(page_downs) + + page_results_yielded = 0 + for o_down in interruptable_as_completed(page_downs): + yield o_down.result() + + # Do we need to start the next set of downloads yet? + if not next_page_triggered: + page_results_yielded += 1 + if page_results_yielded >= next_page_trigger_point: + try: + next_page_downs = self._submit_page_downloads( + container, _page_generator, options + ) + except ClientException as err: + # Allow the current page to finish downloading + error = err + except Exception: + # Something unexpected went wrong - cancel + # remaining downloads + for _d in page_downs: + _d.cancel() + raise + finally: + # Stop counting and testing + next_page_triggered = True + + if error: + raise error # Upload related methods # @@ -2080,17 +2112,18 @@ def _delete_empty_container(conn, container): def _delete_container(self, container, options): try: - objs = [] for part in self.list(container=container): if part["success"]: - objs.extend([o['name'] for o in part['listing']]) + objs = [o['name'] for o in part['listing']] + + o_dels = self.delete( + container=container, objects=objs, options=options + ) + for res in o_dels: + yield res else: raise part["error"] - for res in self.delete( - container=container, objects=objs, options=options): - yield res - con_del = self.thread_manager.container_pool.submit( self._delete_empty_container, container ) diff --git a/swiftclient/shell.py b/swiftclient/shell.py index a77ea07c..35d7c505 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -198,6 +198,14 @@ def st_delete(parser, args, output_manager): Example --header "content-type:text/plain" --skip-identical Skip downloading files that are identical on both sides. + --no-shuffle By default, when downloading a complete account or + container, download order is randomised in order to + to reduce the load on individual drives when multiple + clients are executed simultaneously to download the + same set of objects (e.g. a nightly automated download + script to multiple servers). Enable this option to + submit download jobs to the thread pool in the order + they are listed in the object store. '''.strip("\n") @@ -247,6 +255,14 @@ def st_download(parser, args, output_manager): '--skip-identical', action='store_true', dest='skip_identical', default=False, help='Skip downloading files that are identical on ' 'both sides.') + parser.add_option( + '--no-shuffle', action='store_false', dest='shuffle', + default=True, help='By default, download order is randomised in order ' + 'to reduce the load on individual drives when multiple clients are ' + 'executed simultaneously to download the same set of objects (e.g. a ' + 'nightly automated download script to multiple servers). Enable this ' + 'option to submit download jobs to the thread pool in the order they ' + 'are listed in the object store.') (options, args) = parse_args(parser, args) args = args[1:] if options.out_file == '-': @@ -353,6 +369,8 @@ def st_download(parser, args, output_manager): except SwiftError as e: output_manager.error(e.value) + except Exception as e: + output_manager.error(e) st_list_options = '''[--long] [--lh] [--totals] [--prefix ] diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index 339aca1f..db47263f 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -14,23 +14,25 @@ # limitations under the License. import mock import os +import six import tempfile import testtools import time + +from concurrent.futures import Future from hashlib import md5 from mock import Mock, PropertyMock from six.moves.queue import Queue, Empty as QueueEmptyError from six import BytesIO +from time import sleep + import swiftclient import swiftclient.utils as utils from swiftclient.client import Connection, ClientException -from swiftclient.service import SwiftService, SwiftError,\ - SwiftUploadObject -import six -if six.PY2: - import __builtin__ as builtins -else: - import builtins +from swiftclient.service import ( + SwiftService, SwiftError, SwiftUploadObject +) + clean_os_environ = {} environ_prefixes = ('ST_', 'OS_') @@ -39,6 +41,12 @@ clean_os_environ[key] = '' +if six.PY2: + import __builtin__ as builtins +else: + import builtins + + class TestSwiftPostObject(testtools.TestCase): def setUp(self): @@ -142,25 +150,24 @@ def _consume(sr): '97ac82a5b825239e782d0339e2d7b910') -class TestServiceDelete(testtools.TestCase): - def setUp(self): - super(TestServiceDelete, self).setUp() - self.opts = {'leave_segments': False, 'yes_all': False} - self.exc = Exception('test_exc') - # Base response to be copied and updated to matched the expected - # response for each test - self.expected = { - 'action': None, # Should be string in the form delete_XX - 'container': 'test_c', - 'object': 'test_o', - 'attempts': 2, - 'response_dict': {}, - 'success': None # Should be a bool - } +class _TestServiceBase(testtools.TestCase): + def _assertDictEqual(self, a, b, m=None): + # assertDictEqual is not available in py2.6 so use a shallow check + # instead + if hasattr(self, 'assertDictEqual'): + self.assertDictEqual(a, b, m) + else: + self.assertTrue(isinstance(a, dict)) + self.assertTrue(isinstance(b, dict)) + self.assertEqual(len(a), len(b), m) + for k, v in a.items(): + self.assertTrue(k in b, m) + self.assertEqual(b[k], v, m) def _get_mock_connection(self, attempts=2): m = Mock(spec=Connection) type(m).attempts = PropertyMock(return_value=attempts) + type(m).auth_end_time = PropertyMock(return_value=4) return m def _get_queue(self, q): @@ -178,18 +185,22 @@ def _get_expected(self, update=None): return expected - def _assertDictEqual(self, a, b, m=None): - # assertDictEqual is not available in py2.6 so use a shallow check - # instead - if hasattr(self, 'assertDictEqual'): - self.assertDictEqual(a, b, m) - else: - self.assertTrue(isinstance(a, dict)) - self.assertTrue(isinstance(b, dict)) - self.assertEqual(len(a), len(b), m) - for k, v in a.items(): - self.assertTrue(k in b, m) - self.assertEqual(b[k], v, m) + +class TestServiceDelete(_TestServiceBase): + def setUp(self): + super(TestServiceDelete, self).setUp() + self.opts = {'leave_segments': False, 'yes_all': False} + self.exc = Exception('test_exc') + # Base response to be copied and updated to matched the expected + # response for each test + self.expected = { + 'action': None, # Should be string in the form delete_XX + 'container': 'test_c', + 'object': 'test_o', + 'attempts': 2, + 'response_dict': {}, + 'success': None # Should be a bool + } def test_delete_segment(self): mock_q = Queue() @@ -542,6 +553,226 @@ def test_create_with_invalid_source(self): self.assertRaises(SwiftError, self.suo, []) +class TestServiceList(_TestServiceBase): + def setUp(self): + super(TestServiceList, self).setUp() + self.opts = {'prefix': None, 'long': False, 'delimiter': ''} + self.exc = Exception('test_exc') + # Base response to be copied and updated to matched the expected + # response for each test + self.expected = { + 'action': None, # Should be list_X_part (account or container) + 'container': None, # Should be a string when listing a container + 'prefix': None, + 'success': None # Should be a bool + } + + def test_list_account(self): + mock_q = Queue() + mock_conn = self._get_mock_connection() + get_account_returns = [ + (None, [{'name': 'test_c'}]), + (None, []) + ] + mock_conn.get_account = Mock(side_effect=get_account_returns) + + expected_r = self._get_expected({ + 'action': 'list_account_part', + 'success': True, + 'listing': [{'name': 'test_c'}], + 'marker': '' + }) + + SwiftService._list_account_job( + mock_conn, self.opts, mock_q + ) + self._assertDictEqual(expected_r, self._get_queue(mock_q)) + self.assertIsNone(self._get_queue(mock_q)) + + long_opts = dict(self.opts, **{'long': True}) + mock_conn.head_container = Mock(return_value={'test_m': '1'}) + get_account_returns = [ + (None, [{'name': 'test_c'}]), + (None, []) + ] + mock_conn.get_account = Mock(side_effect=get_account_returns) + + expected_r_long = self._get_expected({ + 'action': 'list_account_part', + 'success': True, + 'listing': [{'name': 'test_c', 'meta': {'test_m': '1'}}], + 'marker': '', + }) + + SwiftService._list_account_job( + mock_conn, long_opts, mock_q + ) + self._assertDictEqual(expected_r_long, self._get_queue(mock_q)) + self.assertIsNone(self._get_queue(mock_q)) + + def test_list_account_exception(self): + mock_q = Queue() + mock_conn = self._get_mock_connection() + mock_conn.get_account = Mock(side_effect=self.exc) + expected_r = self._get_expected({ + 'action': 'list_account_part', + 'success': False, + 'error': self.exc, + 'marker': '' + }) + + SwiftService._list_account_job( + mock_conn, self.opts, mock_q) + + mock_conn.get_account.assert_called_once_with( + marker='', prefix=None + ) + self._assertDictEqual(expected_r, self._get_queue(mock_q)) + self.assertIsNone(self._get_queue(mock_q)) + + def test_list_container(self): + mock_q = Queue() + mock_conn = self._get_mock_connection() + get_container_returns = [ + (None, [{'name': 'test_o'}]), + (None, []) + ] + mock_conn.get_container = Mock(side_effect=get_container_returns) + + expected_r = self._get_expected({ + 'action': 'list_container_part', + 'container': 'test_c', + 'success': True, + 'listing': [{'name': 'test_o'}], + 'marker': '' + }) + + SwiftService._list_container_job( + mock_conn, 'test_c', self.opts, mock_q + ) + self._assertDictEqual(expected_r, self._get_queue(mock_q)) + self.assertIsNone(self._get_queue(mock_q)) + + long_opts = dict(self.opts, **{'long': True}) + mock_conn.head_container = Mock(return_value={'test_m': '1'}) + get_container_returns = [ + (None, [{'name': 'test_o'}]), + (None, []) + ] + mock_conn.get_container = Mock(side_effect=get_container_returns) + + expected_r_long = self._get_expected({ + 'action': 'list_container_part', + 'container': 'test_c', + 'success': True, + 'listing': [{'name': 'test_o'}], + 'marker': '' + }) + + SwiftService._list_container_job( + mock_conn, 'test_c', long_opts, mock_q + ) + self._assertDictEqual(expected_r_long, self._get_queue(mock_q)) + self.assertIsNone(self._get_queue(mock_q)) + + def test_list_container_exception(self): + mock_q = Queue() + mock_conn = self._get_mock_connection() + mock_conn.get_container = Mock(side_effect=self.exc) + expected_r = self._get_expected({ + 'action': 'list_container_part', + 'container': 'test_c', + 'success': False, + 'error': self.exc, + 'marker': '' + }) + + SwiftService._list_container_job( + mock_conn, 'test_c', self.opts, mock_q + ) + + mock_conn.get_container.assert_called_once_with( + 'test_c', marker='', delimiter='', prefix=None + ) + self._assertDictEqual(expected_r, self._get_queue(mock_q)) + self.assertIsNone(self._get_queue(mock_q)) + + @mock.patch('swiftclient.service.get_conn') + def test_list_queue_size(self, mock_get_conn): + mock_conn = self._get_mock_connection() + # Return more results than should fit in the results queue + get_account_returns = [ + (None, [{'name': 'container1'}]), + (None, [{'name': 'container2'}]), + (None, [{'name': 'container3'}]), + (None, [{'name': 'container4'}]), + (None, [{'name': 'container5'}]), + (None, [{'name': 'container6'}]), + (None, [{'name': 'container7'}]), + (None, [{'name': 'container8'}]), + (None, [{'name': 'container9'}]), + (None, [{'name': 'container10'}]), + (None, [{'name': 'container11'}]), + (None, [{'name': 'container12'}]), + (None, [{'name': 'container13'}]), + (None, [{'name': 'container14'}]), + (None, []) + ] + mock_conn.get_account = Mock(side_effect=get_account_returns) + mock_get_conn.return_value = mock_conn + + s = SwiftService(options=self.opts) + lg = s.list() + + # Start the generator + first_list_part = next(lg) + + # Wait for the number of calls to get_account to reach our expected + # value, then let it run some more to make sure the value remains + # stable + count = mock_conn.get_account.call_count + stable = 0 + while mock_conn.get_account.call_count != count or stable < 5: + if mock_conn.get_account.call_count == count: + stable += 1 + else: + count = mock_conn.get_account.call_count + stable = 0 + # The test requires a small sleep to allow other threads to + # execute - in this mocked environment we assume that if the call + # count to get_account has not changed in 0.25s then no more calls + # will be made. + sleep(0.05) + + stable_get_account_call_count = mock_conn.get_account.call_count + + # Collect all remaining results from the generator + list_results = [first_list_part] + list(lg) + + # Make sure the stable call count is correct - this should be 12 calls + # to get_account; + # 1 for first_list_part + # 10 for the values on the queue + # 1 for the value blocking whilst trying to place onto the queue + self.assertEqual(12, stable_get_account_call_count) + + # Make sure all the containers were listed and placed onto the queue + self.assertEqual(15, mock_conn.get_account.call_count) + + # Check the results were all returned + observed_listing = [] + for lir in list_results: + observed_listing.append( + [li['name'] for li in lir['listing']] + ) + expected_listing = [] + for gar in get_account_returns[:-1]: # The empty list is not returned + expected_listing.append( + [li['name'] for li in gar[1]] + ) + self.assertEqual(observed_listing, expected_listing) + + class TestService(testtools.TestCase): def test_upload_with_bad_segment_size(self): @@ -589,23 +820,7 @@ def test_upload_with_relative_path(self, *args, **kwargs): self.assertEqual(upload_obj_resp['path'], obj['path']) -class TestServiceUpload(testtools.TestCase): - - def _assertDictEqual(self, a, b, m=None): - # assertDictEqual is not available in py2.6 so use a shallow check - # instead - if not m: - m = '{0} != {1}'.format(a, b) - - if hasattr(self, 'assertDictEqual'): - self.assertDictEqual(a, b, m) - else: - self.assertIsInstance(a, dict, m) - self.assertIsInstance(b, dict, m) - self.assertEqual(len(a), len(b), m) - for k, v in a.items(): - self.assertIn(k, b, m) - self.assertEqual(b[k], v, m) +class TestServiceUpload(_TestServiceBase): def test_upload_segment_job(self): with tempfile.NamedTemporaryFile() as f: @@ -1027,7 +1242,7 @@ def test_upload_object_job_identical_dlo(self): mock_conn.get_container.assert_has_calls(expected) -class TestServiceDownload(testtools.TestCase): +class TestServiceDownload(_TestServiceBase): def setUp(self): super(TestServiceDownload, self).setUp() @@ -1036,6 +1251,19 @@ def setUp(self): self.obj_content = b'c' * 10 self.obj_etag = md5(self.obj_content).hexdigest() self.obj_len = len(self.obj_content) + self.exc = Exception('test_exc') + # Base response to be copied and updated to matched the expected + # response for each test + self.expected = { + 'action': 'download_object', # Should always be download_object + 'container': 'test_c', + 'object': 'test_o', + 'attempts': 2, + 'response_dict': {}, + 'path': 'test_o', + 'pseudodir': False, + 'success': None # Should be a bool + } def _readbody(self): yield self.obj_content @@ -1056,6 +1284,166 @@ def _assertDictEqual(self, a, b, m=None): self.assertIn(k, b, m) self.assertEqual(b[k], v, m) + @mock.patch('swiftclient.service.SwiftService.list') + @mock.patch('swiftclient.service.SwiftService._submit_page_downloads') + @mock.patch('swiftclient.service.interruptable_as_completed') + def test_download_container_job(self, as_comp, sub_page, service_list): + """ + Check that paged downloads work correctly + """ + as_comp.side_effect = [ + + ] + sub_page.side_effect = [ + range(0, 10), range(0, 10), [] # simulate multiple result pages + ] + r = Mock(spec=Future) + r.result.return_value = self._get_expected({ + 'success': True, + 'start_time': 1, + 'finish_time': 2, + 'headers_receipt': 3, + 'auth_end_time': 4, + 'read_length': len(b'objcontent'), + }) + as_comp.side_effect = [ + [r for _ in range(0, 10)], + [r for _ in range(0, 10)] + ] + + s = SwiftService() + down_gen = s._download_container('test_c', self.opts) + results = list(down_gen) + self.assertEqual(20, len(results)) + + @mock.patch('swiftclient.service.SwiftService.list') + @mock.patch('swiftclient.service.SwiftService._submit_page_downloads') + @mock.patch('swiftclient.service.interruptable_as_completed') + def test_download_container_job_error( + self, as_comp, sub_page, service_list): + """ + Check that paged downloads work correctly + """ + class BoomError(Exception): + def __init__(self, value): + self.value = value + + def __str__(self): + return repr(self.value) + + def _make_result(): + r = Mock(spec=Future) + r.result.return_value = self._get_expected({ + 'success': True, + 'start_time': 1, + 'finish_time': 2, + 'headers_receipt': 3, + 'auth_end_time': 4, + 'read_length': len(b'objcontent'), + }) + return r + + as_comp.side_effect = [ + + ] + # We need Futures here because the error will cause a call to .cancel() + sub_page_effects = [ + [_make_result() for _ in range(0, 10)], + BoomError('Go Boom') + ] + sub_page.side_effect = sub_page_effects + # ...but we must also mock the returns to as_completed + as_comp.side_effect = [ + [_make_result() for _ in range(0, 10)] + ] + + s = SwiftService() + self.assertRaises( + BoomError, + lambda: list(s._download_container('test_c', self.opts)) + ) + # This was an unknown error, so make sure we attempt to cancel futures + for spe in sub_page_effects[0]: + spe.cancel.assert_called_once_with() + + # Now test ClientException + sub_page_effects = [ + [_make_result() for _ in range(0, 10)], + ClientException('Go Boom') + ] + sub_page.side_effect = sub_page_effects + as_comp.side_effect = [ + [_make_result() for _ in range(0, 10)], + [_make_result() for _ in range(0, 10)] + ] + self.assertRaises( + ClientException, + lambda: list(s._download_container('test_c', self.opts)) + ) + # This was a ClientException, so make sure we don't cancel futures + for spe in sub_page_effects[0]: + self.assertFalse(spe.cancel.called) + + def test_download_object_job(self): + mock_conn = self._get_mock_connection() + objcontent = six.BytesIO(b'objcontent') + mock_conn.get_object.side_effect = [ + ({'content-type': 'text/plain', + 'etag': '2cbbfe139a744d6abbe695e17f3c1991'}, + objcontent) + ] + expected_r = self._get_expected({ + 'success': True, + 'start_time': 1, + 'finish_time': 2, + 'headers_receipt': 3, + 'auth_end_time': 4, + 'read_length': len(b'objcontent'), + }) + + with mock.patch.object(builtins, 'open') as mock_open: + written_content = Mock() + mock_open.return_value = written_content + s = SwiftService() + _opts = self.opts.copy() + _opts['no_download'] = False + actual_r = s._download_object_job( + mock_conn, 'test_c', 'test_o', _opts) + actual_r = dict( # Need to override the times we got from the call + actual_r, + **{ + 'start_time': 1, + 'finish_time': 2, + 'headers_receipt': 3 + } + ) + mock_open.assert_called_once_with('test_o', 'wb') + written_content.write.assert_called_once_with(b'objcontent') + + mock_conn.get_object.assert_called_once_with( + 'test_c', 'test_o', resp_chunk_size=65536, headers={}, + response_dict={} + ) + self._assertDictEqual(expected_r, actual_r) + + def test_download_object_job_exception(self): + mock_conn = self._get_mock_connection() + mock_conn.get_object = Mock(side_effect=self.exc) + expected_r = self._get_expected({ + 'success': False, + 'error': self.exc + }) + + s = SwiftService() + actual_r = s._download_object_job( + mock_conn, 'test_c', 'test_o', self.opts) + + mock_conn.get_object.assert_called_once_with( + 'test_c', 'test_o', resp_chunk_size=65536, headers={}, + response_dict={} + ) + self._assertDictEqual(expected_r, actual_r) + def test_download(self): service = SwiftService() with mock.patch('swiftclient.service.Connection') as mock_conn: diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index 8384f7a7..5f9d4977 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -377,6 +377,62 @@ def test_download(self, connection, makedirs): swiftclient.shell.main(argv) self.assertEqual('objcontent', output.out) + @mock.patch('swiftclient.service.shuffle') + @mock.patch('swiftclient.service.Connection') + def test_download_shuffle(self, connection, mock_shuffle): + # Test that the container and object lists are shuffled + mock_shuffle.side_effect = lambda l: l + connection.return_value.get_object.return_value = [ + {'content-type': 'text/plain', + 'etag': EMPTY_ETAG}, + ''] + + connection.return_value.get_container.side_effect = [ + (None, [{'name': 'object'}]), + (None, [{'name': 'pseudo/'}]), + (None, []), + ] + connection.return_value.auth_end_time = 0 + connection.return_value.attempts = 0 + connection.return_value.get_account.side_effect = [ + (None, [{'name': 'container'}]), + (None, []) + ] + + with mock.patch(BUILTIN_OPEN) as mock_open: + argv = ["", "download", "--all"] + swiftclient.shell.main(argv) + self.assertEqual(3, mock_shuffle.call_count) + mock_shuffle.assert_any_call(['container']) + mock_shuffle.assert_any_call(['object']) + mock_shuffle.assert_any_call(['pseudo/']) + mock_open.assert_called_once_with('container/object', 'wb') + + # Test that the container and object lists are not shuffled + mock_shuffle.reset_mock() + connection.return_value.get_object.return_value = [ + {'content-type': 'text/plain', + 'etag': 'd41d8cd98f00b204e9800998ecf8427e'}, + ''] + + connection.return_value.get_container.side_effect = [ + (None, [{'name': 'object'}]), + (None, [{'name': 'pseudo/'}]), + (None, []), + ] + connection.return_value.auth_end_time = 0 + connection.return_value.attempts = 0 + connection.return_value.get_account.side_effect = [ + (None, [{'name': 'container'}]), + (None, []) + ] + + with mock.patch(BUILTIN_OPEN) as mock_open: + argv = ["", "download", "--all", "--no-shuffle"] + swiftclient.shell.main(argv) + self.assertEqual(0, mock_shuffle.call_count) + mock_open.assert_called_once_with('container/object', 'wb') + @mock.patch('swiftclient.service.Connection') def test_download_no_content_type(self, connection): connection.return_value.get_object.return_value = [ From 3cd1faa7afdfc829ee218f86b9bc9d914c0d0b75 Mon Sep 17 00:00:00 2001 From: "Zack M. Davis" Date: Fri, 24 Jul 2015 10:57:29 -0700 Subject: [PATCH 021/454] make Connection.get_auth set url and token attributes on self When a Connection is first __init__ialized (without providing a preauthurl or preauthtoken), the url and token attributes are None; they get set (to be reused on future requests) after one of the wrapper methods internally using _retry (head_account, get_container, put_object, and similar friends) is called. However, this had not been the case for get_auth, much to the momentary confusion and disappointment of programmers using swiftclient who expected to be able to get the token or storage URL off the Connection object after calling get_auth (perhaps in order to make an unusual kind of request that swiftclient doesn't already have a function for). This commit makes get_auth set the url and token attributes as one might expect. Change-Id: I0d9593eb5b072c8e3fa84f7d5a4c948c0bc6037a --- swiftclient/client.py | 15 ++++++++------- tests/unit/test_swiftclient.py | 11 +++++++++++ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index 0ddf4e8b..74e60c0a 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -1258,13 +1258,14 @@ def close(self): self.http_conn = None def get_auth(self): - return get_auth(self.authurl, self.user, self.key, - snet=self.snet, - auth_version=self.auth_version, - os_options=self.os_options, - cacert=self.cacert, - insecure=self.insecure, - timeout=self.timeout) + self.url, self.token = get_auth(self.authurl, self.user, self.key, + snet=self.snet, + auth_version=self.auth_version, + os_options=self.os_options, + cacert=self.cacert, + insecure=self.insecure, + timeout=self.timeout) + return self.url, self.token def http_connection(self, url=None): return http_connection(url if url else self.url, diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index 1cfe2044..73c0e944 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -1481,6 +1481,17 @@ def test_os_preauth_url_trumps_auth_url(self): ('HEAD', '/v1/AUTH_pre_url', '', {'x-auth-token': 'post_token'}), ]) + def test_get_auth_sets_url_and_token(self): + with mock.patch('swiftclient.client.get_auth') as mock_get_auth: + mock_get_auth.return_value = ( + "https://storage.url/v1/AUTH_storage_acct", "AUTH_token" + ) + conn = c.Connection("https://auth.url/auth/v2.0", "user", "passkey", + tenant_name="tenant") + conn.get_auth() + self.assertEqual("https://storage.url/v1/AUTH_storage_acct", conn.url) + self.assertEqual("AUTH_token", conn.token) + def test_timeout_passed_down(self): # We want to avoid mocking http_connection(), and most especially # avoid passing it down in argument. However, we cannot simply From 1c644d8dc12352b4b1f4ebb1700f9dcf029cbb4c Mon Sep 17 00:00:00 2001 From: Ondrej Novy Date: Thu, 30 Jul 2015 09:24:49 +0200 Subject: [PATCH 022/454] Test auth params together with --help option. Change-Id: I2691739cc14bb8d384cc7ef0f8a3e73d1b898f88 --- tests/unit/test_shell.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index 8384f7a7..17ac995c 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -1271,9 +1271,19 @@ def test_help(self): # --help returns condensed help message, overrides --os-help opts = {"help": ""} os_opts = {"help": ""} - # "password": "secret", - # "username": "user", - # "auth_url": "http://example.com:5000/v3"} + args = _make_args("", opts, os_opts) + with CaptureOutput() as out: + self.assertRaises(SystemExit, swiftclient.shell.main, args) + self.assertTrue(out.find('[--key ]') > 0) + self.assertEqual(-1, out.find('--os-username=')) + + # --os-password, --os-username and --os-auth_url should be ignored + # because --help overrides it + opts = {"help": ""} + os_opts = {"help": "", + "password": "secret", + "username": "user", + "auth_url": "http://example.com:5000/v3"} args = _make_args("", opts, os_opts) with CaptureOutput() as out: self.assertRaises(SystemExit, swiftclient.shell.main, args) From 847f135f9759ba9605175371d3c85eeb9038a1a7 Mon Sep 17 00:00:00 2001 From: Ondrej Novy Date: Sun, 26 Jul 2015 19:45:58 +0200 Subject: [PATCH 023/454] Block comment PEP8 fix. Change-Id: I999425902a5c82f7af129a8a1b6998d80edb225a --- tests/unit/test_shell.py | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index 17ac995c..ca14045f 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -1290,7 +1290,7 @@ def test_help(self): self.assertTrue(out.find('[--key ]') > 0) self.assertEqual(-1, out.find('--os-username=')) - ## --os-help return os options help + # --os-help return os options help opts = {} args = _make_args("", opts, os_opts) with CaptureOutput() as out: diff --git a/tox.ini b/tox.ini index 10377ccd..e5b207c6 100644 --- a/tox.ini +++ b/tox.ini @@ -50,4 +50,4 @@ commands= ignore = H select = H102, H103, H201, H501, H903 show-source = True -exclude = .venv,.tox,dist,doc,test,*egg +exclude = .venv,.tox,dist,doc,*egg From a056f1b3742812a2c861a0d01678dfed3b0087e4 Mon Sep 17 00:00:00 2001 From: Hiroshi Miura Date: Mon, 3 Aug 2015 12:23:04 +0900 Subject: [PATCH 024/454] fix old style class definition(H238) Change-Id: Ib5be06fa544f5eb3061c6a3077a3b9986382ecfe Signed-off-by: Hiroshi Miura --- tests/unit/utils.py | 2 +- tox.ini | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/unit/utils.py b/tests/unit/utils.py index 4f7c8ec4..ac9aefdb 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -503,7 +503,7 @@ def __init__(self, endpoint, token): self.endpoint = endpoint self.token = token - class _Client(): + class _Client(object): def __init__(self, endpoint, token, **kwargs): self.auth_token = token self.endpoint = endpoint diff --git a/tox.ini b/tox.ini index e5b207c6..1008f5db 100644 --- a/tox.ini +++ b/tox.ini @@ -45,9 +45,10 @@ commands= # H102 -> apache2 license exists # H103 -> license is apache # H201 -> no bare excepts +# H238 -> old style classes are deprecated and not available in python3 # H501 -> don't use locals() for str formatting # H903 -> \n not \r\n ignore = H -select = H102, H103, H201, H501, H903 +select = H102, H103, H201, H238, H501, H903 show-source = True exclude = .venv,.tox,dist,doc,*egg From be0f1aad8a5ed6f8d61016902c713db609ad834c Mon Sep 17 00:00:00 2001 From: Hiroshi Miura Date: Mon, 3 Aug 2015 12:20:44 +0900 Subject: [PATCH 025/454] change deprecated assertEquals to assertEqual fix against H234: assertEquals() logs a DeprecationWarning in Python3.x. use assertEqual() instead. Closes-bug: #1480776 Change-Id: Iffda6bb5f2616d4af4567eeea37bb26531e34371 Signed-off-by: Hiroshi Miura --- tests/unit/test_multithreading.py | 6 ++-- tests/unit/test_shell.py | 46 +++++++++++++++---------------- tests/unit/test_swiftclient.py | 28 +++++++++---------- tests/unit/test_utils.py | 20 +++++++------- tox.ini | 3 +- 5 files changed, 52 insertions(+), 51 deletions(-) diff --git a/tests/unit/test_multithreading.py b/tests/unit/test_multithreading.py index 42abbbf8..2c45b47e 100644 --- a/tests/unit/test_multithreading.py +++ b/tests/unit/test_multithreading.py @@ -88,7 +88,7 @@ def test_submit_good_connection(self): f.result() except Exception as e: went_boom = True - self.assertEquals('I went boom!', str(e)) + self.assertEqual('I went boom!', str(e)) self.assertTrue(went_boom) # Has the connection been returned to the pool? @@ -112,7 +112,7 @@ def test_submit_bad_connection(self): f.result() except Exception as e: connection_failed = True - self.assertEquals('This is a failed connection', str(e)) + self.assertEqual('This is a failed connection', str(e)) self.assertTrue(connection_failed) # Make sure we don't lock up on failed connections @@ -122,7 +122,7 @@ def test_submit_bad_connection(self): f.result() except Exception as e: connection_failed = True - self.assertEquals('This is a failed connection', str(e)) + self.assertEqual('This is a failed connection', str(e)) self.assertTrue(connection_failed) def test_lazy_connections(self): diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index ca14045f..4ac2f5bb 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -138,7 +138,7 @@ def test_stat_account(self, connection): with CaptureOutput() as output: swiftclient.shell.main(argv) - self.assertEquals(output.out, + self.assertEqual(output.out, ' Account: AUTH_account\n' 'Containers: 1\n' ' Objects: 2\n' @@ -160,7 +160,7 @@ def test_stat_container(self, connection): with CaptureOutput() as output: swiftclient.shell.main(argv) - self.assertEquals(output.out, + self.assertEqual(output.out, ' Account: AUTH_account\n' 'Container: container\n' ' Objects: 1\n' @@ -186,7 +186,7 @@ def test_stat_object(self, connection): with CaptureOutput() as output: swiftclient.shell.main(argv) - self.assertEquals(output.out, + self.assertEqual(output.out, ' Account: AUTH_account\n' ' Container: container\n' ' Object: object\n' @@ -212,7 +212,7 @@ def test_list_account(self, connection): mock.call(marker='container', prefix=None)] connection.return_value.get_account.assert_has_calls(calls) - self.assertEquals(output.out, 'container\n') + self.assertEqual(output.out, 'container\n') @mock.patch('swiftclient.service.Connection') def test_list_account_long(self, connection): @@ -229,7 +229,7 @@ def test_list_account_long(self, connection): mock.call(marker='container', prefix=None)] connection.return_value.get_account.assert_has_calls(calls) - self.assertEquals(output.out, + self.assertEqual(output.out, ' 0 0 1970-01-01 00:00:01 container\n' ' 0 0\n') @@ -249,7 +249,7 @@ def test_list_account_long(self, connection): mock.call(marker='container', prefix=None)] connection.return_value.get_account.assert_has_calls(calls) - self.assertEquals(output.out, + self.assertEqual(output.out, ' 0 0 ????-??-?? ??:??:?? container\n' ' 0 0\n') @@ -294,7 +294,7 @@ def test_list_container(self, connection): delimiter=None, prefix=None)] connection.return_value.get_container.assert_has_calls(calls) - self.assertEquals(output.out, 'object_a\n') + self.assertEqual(output.out, 'object_a\n') # Test container listing with --long connection.return_value.get_container.side_effect = [ @@ -311,7 +311,7 @@ def test_list_container(self, connection): delimiter=None, prefix=None)] connection.return_value.get_container.assert_has_calls(calls) - self.assertEquals(output.out, + self.assertEqual(output.out, ' 0 123 456 object_a\n' ' 0\n') @@ -771,7 +771,7 @@ def test_post_account_bad_auth(self, connection): with ExpectedException(SystemExit): swiftclient.shell.main(argv) - self.assertEquals(output.err, 'bad auth\n') + self.assertEqual(output.err, 'bad auth\n') @mock.patch('swiftclient.service.Connection') def test_post_account_not_found(self, connection): @@ -783,7 +783,7 @@ def test_post_account_not_found(self, connection): with ExpectedException(SystemExit): swiftclient.shell.main(argv) - self.assertEquals(output.err, 'Account not found\n') + self.assertEqual(output.err, 'Account not found\n') @mock.patch('swiftclient.service.Connection') def test_post_container(self, connection): @@ -802,7 +802,7 @@ def test_post_container_bad_auth(self, connection): with ExpectedException(SystemExit): swiftclient.shell.main(argv) - self.assertEquals(output.err, 'bad auth\n') + self.assertEqual(output.err, 'bad auth\n') @mock.patch('swiftclient.service.Connection') def test_post_container_not_found_causes_put(self, connection): @@ -860,7 +860,7 @@ def test_post_object_bad_auth(self, connection): with ExpectedException(SystemExit): swiftclient.shell.main(argv) - self.assertEquals(output.err, 'bad auth\n') + self.assertEqual(output.err, 'bad auth\n') def test_post_object_too_many_args(self): argv = ["", "post", "container", "object", "bad_arg"] @@ -925,41 +925,41 @@ def _check_expected(x, expected): # Test invalid states argv = ["", "upload", "-S", "1234X", "container", "object"] swiftclient.shell.main(argv) - self.assertEquals(output.err, "Invalid segment size\n") + self.assertEqual(output.err, "Invalid segment size\n") output.clear() with ExpectedException(SystemExit): argv = ["", "upload", "-S", "K1234", "container", "object"] swiftclient.shell.main(argv) - self.assertEquals(output.err, "Invalid segment size\n") + self.assertEqual(output.err, "Invalid segment size\n") output.clear() with ExpectedException(SystemExit): argv = ["", "upload", "-S", "K", "container", "object"] swiftclient.shell.main(argv) - self.assertEquals(output.err, "Invalid segment size\n") + self.assertEqual(output.err, "Invalid segment size\n") def test_negative_upload_segment_size(self): with CaptureOutput() as output: with ExpectedException(SystemExit): argv = ["", "upload", "-S", "-40", "container", "object"] swiftclient.shell.main(argv) - self.assertEquals(output.err, "segment-size should be positive\n") + self.assertEqual(output.err, "segment-size should be positive\n") output.clear() with ExpectedException(SystemExit): argv = ["", "upload", "-S", "-40K", "container", "object"] swiftclient.shell.main(argv) - self.assertEquals(output.err, "segment-size should be positive\n") + self.assertEqual(output.err, "segment-size should be positive\n") output.clear() with ExpectedException(SystemExit): argv = ["", "upload", "-S", "-40M", "container", "object"] swiftclient.shell.main(argv) - self.assertEquals(output.err, "segment-size should be positive\n") + self.assertEqual(output.err, "segment-size should be positive\n") output.clear() with ExpectedException(SystemExit): argv = ["", "upload", "-S", "-40G", "container", "object"] swiftclient.shell.main(argv) - self.assertEquals(output.err, "segment-size should be positive\n") + self.assertEqual(output.err, "segment-size should be positive\n") output.clear() @@ -1625,7 +1625,7 @@ def test_auth(self): export OS_STORAGE_URL=https://swift.storage.example.com/v1/AUTH_test export OS_AUTH_TOKEN=AUTH_tk5b6b12 """ - self.assertEquals(textwrap.dedent(expected).lstrip(), + self.assertEqual(textwrap.dedent(expected).lstrip(), stdout.getvalue()) def test_auth_verbose(self): @@ -1646,7 +1646,7 @@ def test_auth_verbose(self): export ST_USER=test:tester export ST_KEY='te$tin&' """ - self.assertEquals(textwrap.dedent(expected).lstrip(), + self.assertEqual(textwrap.dedent(expected).lstrip(), stdout.getvalue()) self.assertEqual([], mock_conn.mock_calls) @@ -1669,7 +1669,7 @@ def test_auth_v2(self): export OS_STORAGE_URL=http://url/ export OS_AUTH_TOKEN=token """ - self.assertEquals(textwrap.dedent(expected).lstrip(), + self.assertEqual(textwrap.dedent(expected).lstrip(), stdout.getvalue()) def test_auth_verbose_v2(self): @@ -1695,7 +1695,7 @@ def test_auth_verbose_v2(self): export OS_TENANT_NAME=demo export OS_USERNAME=demo """ - self.assertEquals(textwrap.dedent(expected).lstrip(), + self.assertEqual(textwrap.dedent(expected).lstrip(), stdout.getvalue()) self.assertEqual([], mock_keystone.mock_calls) diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index 1cfe2044..bd5281f4 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -693,9 +693,9 @@ def test_chunk_size_read_method(self): c.http_connection = self.fake_http_connection(200, body='abcde') __, resp = conn.get_object('asdf', 'asdf', resp_chunk_size=3) self.assertTrue(hasattr(resp, 'read')) - self.assertEquals(resp.read(3), 'abc') - self.assertEquals(resp.read(None), 'de') - self.assertEquals(resp.read(), '') + self.assertEqual(resp.read(3), 'abc') + self.assertEqual(resp.read(None), 'de') + self.assertEqual(resp.read(), '') def test_chunk_size_iter(self): conn = c.Connection('http://auth.url/', 'some_user', 'some_key') @@ -704,8 +704,8 @@ def test_chunk_size_iter(self): c.http_connection = self.fake_http_connection(200, body='abcde') __, resp = conn.get_object('asdf', 'asdf', resp_chunk_size=3) self.assertTrue(hasattr(resp, 'next')) - self.assertEquals(next(resp), 'abc') - self.assertEquals(next(resp), 'de') + self.assertEqual(next(resp), 'abc') + self.assertEqual(next(resp), 'de') self.assertRaises(StopIteration, next, resp) def test_chunk_size_read_and_iter(self): @@ -715,11 +715,11 @@ def test_chunk_size_read_and_iter(self): c.http_connection = self.fake_http_connection(200, body='abcdef') __, resp = conn.get_object('asdf', 'asdf', resp_chunk_size=2) self.assertTrue(hasattr(resp, 'read')) - self.assertEquals(resp.read(3), 'abc') - self.assertEquals(next(resp), 'de') - self.assertEquals(resp.read(), 'f') + self.assertEqual(resp.read(3), 'abc') + self.assertEqual(next(resp), 'de') + self.assertEqual(resp.read(), 'f') self.assertRaises(StopIteration, next, resp) - self.assertEquals(resp.read(), '') + self.assertEqual(resp.read(), '') class TestHeadObject(MockHttpTest): @@ -871,9 +871,9 @@ def test_md5_mismatch(self): contents=contents, chunk_size=chunk_size) - self.assertNotEquals(etag, contents.get_md5sum()) - self.assertEquals(etag, 'badresponseetag') - self.assertEquals(raw_data_md5, contents.get_md5sum()) + self.assertNotEqual(etag, contents.get_md5sum()) + self.assertEqual(etag, 'badresponseetag') + self.assertEqual(raw_data_md5, contents.get_md5sum()) def test_md5_match(self): conn = c.http_connection('http://www.test.com') @@ -896,8 +896,8 @@ def test_md5_match(self): contents=contents, chunk_size=chunk_size) - self.assertEquals(raw_data_md5, contents.get_md5sum()) - self.assertEquals(etag, contents.get_md5sum()) + self.assertEqual(raw_data_md5, contents.get_md5sum()) + self.assertEqual(etag, contents.get_md5sum()) def test_params(self): conn = c.http_connection(u'http://www.test.com/') diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index d82d2b8e..ca3531e1 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -176,19 +176,19 @@ def test_iter(self): data = u.ReadableToIterable(f, chunk_size, True) for i, data_chunk in enumerate(data): - self.assertEquals(chunk_size, len(data_chunk)) - self.assertEquals(data_chunk, write_data[i] * chunk_size) + self.assertEqual(chunk_size, len(data_chunk)) + self.assertEqual(data_chunk, write_data[i] * chunk_size) - self.assertEquals(actual_md5sum.hexdigest(), data.get_md5sum()) + self.assertEqual(actual_md5sum.hexdigest(), data.get_md5sum()) def test_md5_creation(self): # Check creation with a real and noop md5 class data = u.ReadableToIterable(None, None, md5=True) - self.assertEquals(md5().hexdigest(), data.get_md5sum()) + self.assertEqual(md5().hexdigest(), data.get_md5sum()) self.assertTrue(isinstance(data.md5sum, type(md5()))) data = u.ReadableToIterable(None, None, md5=False) - self.assertEquals('', data.get_md5sum()) + self.assertEqual('', data.get_md5sum()) self.assertTrue(isinstance(data.md5sum, type(u.NoopMD5()))) def test_unicode(self): @@ -203,14 +203,14 @@ def test_unicode(self): data = u.ReadableToIterable(f, chunk_size, True) x = next(data) - self.assertEquals(2, len(x)) - self.assertEquals(unicode_data[:2], x) + self.assertEqual(2, len(x)) + self.assertEqual(unicode_data[:2], x) x = next(data) - self.assertEquals(1, len(x)) - self.assertEquals(unicode_data[2:], x) + self.assertEqual(1, len(x)) + self.assertEqual(unicode_data[2:], x) - self.assertEquals(actual_md5sum, data.get_md5sum()) + self.assertEqual(actual_md5sum, data.get_md5sum()) class TestLengthWrapper(testtools.TestCase): diff --git a/tox.ini b/tox.ini index 1008f5db..7aaec00d 100644 --- a/tox.ini +++ b/tox.ini @@ -45,10 +45,11 @@ commands= # H102 -> apache2 license exists # H103 -> license is apache # H201 -> no bare excepts +# H234 -> assertEquals is deprecated, use assertEqual # H238 -> old style classes are deprecated and not available in python3 # H501 -> don't use locals() for str formatting # H903 -> \n not \r\n ignore = H -select = H102, H103, H201, H238, H501, H903 +select = H102, H103, H201, H234, H238, H501, H903 show-source = True exclude = .venv,.tox,dist,doc,*egg From 7c7f46a33d30a68b4e070fc5d1515f8a668f3b04 Mon Sep 17 00:00:00 2001 From: Christian Schwede Date: Thu, 23 Jul 2015 16:32:50 +0000 Subject: [PATCH 026/454] Update mock to get away from env markers Closes-Bug: 1476585 Bug has been reproduced on a fresh installed SAIO using https://github.com/swiftstack/vagrant-swift-all-in-one. Change-Id: I0300319baf7e2d8c27d1c19957894396505caeb8 --- test-requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 909cb043..7f7e405f 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,8 +2,7 @@ hacking>=0.10.0,<0.11 coverage>=3.6 discover -mock>=1.0;python_version!='2.6' -mock==1.0.1;python_version=='2.6' +mock>=1.2 oslosphinx python-keystoneclient>=0.7.0 sphinx>=1.1.2,<1.2 From 38a82e903514c242810e677f12f5eebdb775ef0b Mon Sep 17 00:00:00 2001 From: Hiroshi Miura Date: Fri, 17 Jul 2015 16:16:07 +0900 Subject: [PATCH 027/454] flake8 ignores same hacks as swift - blacklisted flake8 hacking - fix against E122 continuation line missing indentation or outdented Closes-bug: #1475516 Change-Id: I708d0a3466a1f85c84e478873e142821ce0774cb Signed-off-by: Hiroshi Miura --- swiftclient/shell.py | 8 ++--- tests/unit/test_shell.py | 60 +++++++++++++++++----------------- tests/unit/test_swiftclient.py | 4 +-- tox.ini | 26 +++++++++------ 4 files changed, 52 insertions(+), 46 deletions(-) diff --git a/swiftclient/shell.py b/swiftclient/shell.py index f2388fcf..f0b0fdd5 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -112,8 +112,8 @@ def st_delete(parser, args, output_manager): if '/' in container: output_manager.error( 'WARNING: / in container name; you ' - "might have meant '%s' instead of '%s'." % ( - container.replace('/', ' ', 1), container) + "might have meant '%s' instead of '%s'." % + (container.replace('/', ' ', 1), container) ) return objects = args[1:] @@ -279,8 +279,8 @@ def st_download(parser, args, output_manager): if '/' in container: output_manager.error( 'WARNING: / in container name; you ' - "might have meant '%s' instead of '%s'." % ( - container.replace('/', ' ', 1), container) + "might have meant '%s' instead of '%s'." % + (container.replace('/', ' ', 1), container) ) return objects = args[1:] diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index 4ac2f5bb..46b1f31b 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -139,10 +139,10 @@ def test_stat_account(self, connection): swiftclient.shell.main(argv) self.assertEqual(output.out, - ' Account: AUTH_account\n' - 'Containers: 1\n' - ' Objects: 2\n' - ' Bytes: 3\n') + ' Account: AUTH_account\n' + 'Containers: 1\n' + ' Objects: 2\n' + ' Bytes: 3\n') @mock.patch('swiftclient.service.Connection') def test_stat_container(self, connection): @@ -161,14 +161,14 @@ def test_stat_container(self, connection): swiftclient.shell.main(argv) self.assertEqual(output.out, - ' Account: AUTH_account\n' - 'Container: container\n' - ' Objects: 1\n' - ' Bytes: 2\n' - ' Read ACL: test2:tester2\n' - 'Write ACL: test3:tester3\n' - ' Sync To: other\n' - ' Sync Key: secret\n') + ' Account: AUTH_account\n' + 'Container: container\n' + ' Objects: 1\n' + ' Bytes: 2\n' + ' Read ACL: test2:tester2\n' + 'Write ACL: test3:tester3\n' + ' Sync To: other\n' + ' Sync Key: secret\n') @mock.patch('swiftclient.service.Connection') def test_stat_object(self, connection): @@ -187,14 +187,14 @@ def test_stat_object(self, connection): swiftclient.shell.main(argv) self.assertEqual(output.out, - ' Account: AUTH_account\n' - ' Container: container\n' - ' Object: object\n' - ' Content Type: text/plain\n' - 'Content Length: 42\n' - ' Last Modified: yesterday\n' - ' ETag: md5\n' - ' Manifest: manifest\n') + ' Account: AUTH_account\n' + ' Container: container\n' + ' Object: object\n' + ' Content Type: text/plain\n' + 'Content Length: 42\n' + ' Last Modified: yesterday\n' + ' ETag: md5\n' + ' Manifest: manifest\n') @mock.patch('swiftclient.service.Connection') def test_list_account(self, connection): @@ -230,8 +230,8 @@ def test_list_account_long(self, connection): connection.return_value.get_account.assert_has_calls(calls) self.assertEqual(output.out, - ' 0 0 1970-01-01 00:00:01 container\n' - ' 0 0\n') + ' 0 0 1970-01-01 00:00:01 container\n' + ' 0 0\n') # Now test again, this time without returning metadata connection.return_value.head_container.return_value = {} @@ -250,8 +250,8 @@ def test_list_account_long(self, connection): connection.return_value.get_account.assert_has_calls(calls) self.assertEqual(output.out, - ' 0 0 ????-??-?? ??:??:?? container\n' - ' 0 0\n') + ' 0 0 ????-??-?? ??:??:?? container\n' + ' 0 0\n') def test_list_account_totals_error(self): # No --lh provided: expect info message about incorrect --totals use @@ -312,8 +312,8 @@ def test_list_container(self, connection): connection.return_value.get_container.assert_has_calls(calls) self.assertEqual(output.out, - ' 0 123 456 object_a\n' - ' 0\n') + ' 0 123 456 object_a\n' + ' 0\n') @mock.patch('swiftclient.service.makedirs') @mock.patch('swiftclient.service.Connection') @@ -1626,7 +1626,7 @@ def test_auth(self): export OS_AUTH_TOKEN=AUTH_tk5b6b12 """ self.assertEqual(textwrap.dedent(expected).lstrip(), - stdout.getvalue()) + stdout.getvalue()) def test_auth_verbose(self): with mock.patch('swiftclient.client.http_connection') as mock_conn: @@ -1647,7 +1647,7 @@ def test_auth_verbose(self): export ST_KEY='te$tin&' """ self.assertEqual(textwrap.dedent(expected).lstrip(), - stdout.getvalue()) + stdout.getvalue()) self.assertEqual([], mock_conn.mock_calls) def test_auth_v2(self): @@ -1670,7 +1670,7 @@ def test_auth_v2(self): export OS_AUTH_TOKEN=token """ self.assertEqual(textwrap.dedent(expected).lstrip(), - stdout.getvalue()) + stdout.getvalue()) def test_auth_verbose_v2(self): with mock.patch('swiftclient.client.get_auth_keystone') \ @@ -1696,7 +1696,7 @@ def test_auth_verbose_v2(self): export OS_USERNAME=demo """ self.assertEqual(textwrap.dedent(expected).lstrip(), - stdout.getvalue()) + stdout.getvalue()) self.assertEqual([], mock_keystone.mock_calls) diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index cefae089..c84d7d71 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -1486,8 +1486,8 @@ def test_get_auth_sets_url_and_token(self): mock_get_auth.return_value = ( "https://storage.url/v1/AUTH_storage_acct", "AUTH_token" ) - conn = c.Connection("https://auth.url/auth/v2.0", "user", "passkey", - tenant_name="tenant") + conn = c.Connection("https://auth.url/auth/v2.0", + "user", "passkey", tenant_name="tenant") conn.get_auth() self.assertEqual("https://storage.url/v1/AUTH_storage_acct", conn.url) self.assertEqual("AUTH_token", conn.token) diff --git a/tox.ini b/tox.ini index 7aaec00d..207d3c33 100644 --- a/tox.ini +++ b/tox.ini @@ -41,15 +41,21 @@ commands= python setup.py build_sphinx [flake8] -# it's not a bug that we aren't using all of hacking -# H102 -> apache2 license exists -# H103 -> license is apache -# H201 -> no bare excepts -# H234 -> assertEquals is deprecated, use assertEqual -# H238 -> old style classes are deprecated and not available in python3 -# H501 -> don't use locals() for str formatting -# H903 -> \n not \r\n -ignore = H -select = H102, H103, H201, H234, H238, H501, H903 +# it's not a bug that we aren't using all of hacking, ignore: +# H101: Use TODO(NAME) +# H202: assertRaises Exception too broad +# H232: Python 3.x incompatible octal 000001234 should be written as 0o1234 +# H233: Python 3.x incompatible use of print operator +# H235: assert_ is deprecated, use assertTrue +# H301: one import per line +# H306: imports not in alphabetical order (time, os) +# H401: docstring should not start with a space +# H403: multi line docstrings should end on a new line +# H404: multi line docstring should start without a leading new line +# H405: multi line docstring summary not separated with an empty line +# H501: Do not use self.__dict__ for string formatting +# H702: Formatting operation should be outside of localization method call +# H703: Multiple positional placeholders +ignore = H101,H202,H232,H233,H235,H301,H306,H401,H403,H404,H405,H501,H702,H703 show-source = True exclude = .venv,.tox,dist,doc,*egg From 91d82aff25d1c7887046db81a91aed263d15ab20 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Wed, 19 Aug 2015 11:43:08 -0700 Subject: [PATCH 028/454] Drop flake8 ignores for already-passing tests Change-Id: I5fb349b2f7808a3f97d95fc7db6b5cf5842a9f7c --- tox.ini | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/tox.ini b/tox.ini index 207d3c33..8a70619b 100644 --- a/tox.ini +++ b/tox.ini @@ -43,19 +43,12 @@ commands= [flake8] # it's not a bug that we aren't using all of hacking, ignore: # H101: Use TODO(NAME) -# H202: assertRaises Exception too broad -# H232: Python 3.x incompatible octal 000001234 should be written as 0o1234 -# H233: Python 3.x incompatible use of print operator -# H235: assert_ is deprecated, use assertTrue # H301: one import per line # H306: imports not in alphabetical order (time, os) # H401: docstring should not start with a space # H403: multi line docstrings should end on a new line # H404: multi line docstring should start without a leading new line # H405: multi line docstring summary not separated with an empty line -# H501: Do not use self.__dict__ for string formatting -# H702: Formatting operation should be outside of localization method call -# H703: Multiple positional placeholders -ignore = H101,H202,H232,H233,H235,H301,H306,H401,H403,H404,H405,H501,H702,H703 +ignore = H101,H301,H306,H401,H403,H404,H405 show-source = True exclude = .venv,.tox,dist,doc,*egg From 1789c2654d604f8f5befc39638d2e8860dadc6ef Mon Sep 17 00:00:00 2001 From: Stuart McLaren Date: Wed, 13 May 2015 09:48:41 +0000 Subject: [PATCH 029/454] Add minimal working service token support. Add client changes to allow accessing alternative reseller_prefixes via a service token. ie client changes for this server side spec: https://review.openstack.org/#/c/105228 We assume that the service storage url has been passed in as a preauthurl. We rely on get_auth preserving this url. Change-Id: I1cfda178f0b6c8add46cfebd6bf38440caae2036 --- swiftclient/client.py | 122 +++++++++++-- tests/unit/test_swiftclient.py | 307 ++++++++++++++++++++++++++++++++- 2 files changed, 409 insertions(+), 20 deletions(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index 74e60c0a..4819c124 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -489,7 +489,8 @@ def store_response(resp, response_dict): def get_account(url, token, marker=None, limit=None, prefix=None, - end_marker=None, http_conn=None, full_listing=False): + end_marker=None, http_conn=None, full_listing=False, + service_token=None): """ Get a listing of containers for the account. @@ -503,6 +504,7 @@ def get_account(url, token, marker=None, limit=None, prefix=None, conn object) :param full_listing: if True, return a full listing, else returns a max of 10000 listings + :param service_token: service auth token :returns: a tuple of (response headers, a list of containers) The response headers will be a dict and all header names will be lowercase. :raises ClientException: HTTP GET request failed @@ -532,6 +534,8 @@ def get_account(url, token, marker=None, limit=None, prefix=None, qs += '&end_marker=%s' % quote(end_marker) full_path = '%s?%s' % (parsed.path, qs) headers = {'X-Auth-Token': token} + if service_token: + headers['X-Service-Token'] = service_token method = 'GET' conn.request(method, full_path, '', headers) resp = conn.getresponse() @@ -552,7 +556,7 @@ def get_account(url, token, marker=None, limit=None, prefix=None, return resp_headers, parse_api_response(resp_headers, body) -def head_account(url, token, http_conn=None): +def head_account(url, token, http_conn=None, service_token=None): """ Get account stats. @@ -560,6 +564,7 @@ def head_account(url, token, http_conn=None): :param token: auth token :param http_conn: HTTP connection object (If None, it will create the conn object) + :param service_token: service auth token :returns: a dict containing the response's headers (all header names will be lowercase) :raises ClientException: HTTP HEAD request failed @@ -570,6 +575,8 @@ def head_account(url, token, http_conn=None): parsed, conn = http_connection(url) method = "HEAD" headers = {'X-Auth-Token': token} + if service_token: + headers['X-Service-Token'] = service_token conn.request(method, parsed.path, '', headers) resp = conn.getresponse() body = resp.read() @@ -585,7 +592,8 @@ def head_account(url, token, http_conn=None): return resp_headers -def post_account(url, token, headers, http_conn=None, response_dict=None): +def post_account(url, token, headers, http_conn=None, response_dict=None, + service_token=None): """ Update an account's metadata. @@ -596,6 +604,7 @@ def post_account(url, token, headers, http_conn=None, response_dict=None): conn object) :param response_dict: an optional dictionary into which to place the response - status, reason and headers + :param service_token: service auth token :raises ClientException: HTTP POST request failed """ if http_conn: @@ -604,6 +613,8 @@ def post_account(url, token, headers, http_conn=None, response_dict=None): parsed, conn = http_connection(url) method = 'POST' headers['X-Auth-Token'] = token + if service_token: + headers['X-Service-Token'] = service_token conn.request(method, parsed.path, '', headers) resp = conn.getresponse() body = resp.read() @@ -624,7 +635,7 @@ def post_account(url, token, headers, http_conn=None, response_dict=None): def get_container(url, token, container, marker=None, limit=None, prefix=None, delimiter=None, end_marker=None, path=None, http_conn=None, - full_listing=False): + full_listing=False, service_token=None): """ Get a listing of objects for the container. @@ -641,6 +652,7 @@ def get_container(url, token, container, marker=None, limit=None, conn object) :param full_listing: if True, return a full listing, else returns a max of 10000 listings + :param service_token: service auth token :returns: a tuple of (response headers, a list of objects) The response headers will be a dict and all header names will be lowercase. :raises ClientException: HTTP GET request failed @@ -649,7 +661,8 @@ def get_container(url, token, container, marker=None, limit=None, http_conn = http_connection(url) if full_listing: rv = get_container(url, token, container, marker, limit, prefix, - delimiter, end_marker, path, http_conn) + delimiter, end_marker, path, http_conn, + service_token) listing = rv[1] while listing: if not delimiter: @@ -658,7 +671,7 @@ def get_container(url, token, container, marker=None, limit=None, marker = listing[-1].get('name', listing[-1].get('subdir')) listing = get_container(url, token, container, marker, limit, prefix, delimiter, end_marker, path, - http_conn)[1] + http_conn, service_token)[1] if listing: rv[1].extend(listing) return rv @@ -678,6 +691,8 @@ def get_container(url, token, container, marker=None, limit=None, if path: qs += '&path=%s' % quote(path) headers = {'X-Auth-Token': token} + if service_token: + headers['X-Service-Token'] = service_token method = 'GET' conn.request(method, '%s?%s' % (cont_path, qs), '', headers) resp = conn.getresponse() @@ -702,7 +717,8 @@ def get_container(url, token, container, marker=None, limit=None, return resp_headers, parse_api_response(resp_headers, body) -def head_container(url, token, container, http_conn=None, headers=None): +def head_container(url, token, container, http_conn=None, headers=None, + service_token=None): """ Get container stats. @@ -711,6 +727,7 @@ def head_container(url, token, container, http_conn=None, headers=None): :param container: container name to get stats for :param http_conn: HTTP connection object (If None, it will create the conn object) + :param service_token: service auth token :returns: a dict containing the response's headers (all header names will be lowercase) :raises ClientException: HTTP HEAD request failed @@ -722,6 +739,8 @@ def head_container(url, token, container, http_conn=None, headers=None): path = '%s/%s' % (parsed.path, quote(container)) method = 'HEAD' req_headers = {'X-Auth-Token': token} + if service_token: + req_headers['X-Service-Token'] = service_token if headers: req_headers.update(headers) conn.request(method, path, '', req_headers) @@ -743,7 +762,7 @@ def head_container(url, token, container, http_conn=None, headers=None): def put_container(url, token, container, headers=None, http_conn=None, - response_dict=None): + response_dict=None, service_token=None): """ Create a container @@ -755,6 +774,7 @@ def put_container(url, token, container, headers=None, http_conn=None, conn object) :param response_dict: an optional dictionary into which to place the response - status, reason and headers + :param service_token: service auth token :raises ClientException: HTTP PUT request failed """ if http_conn: @@ -766,6 +786,8 @@ def put_container(url, token, container, headers=None, http_conn=None, if not headers: headers = {} headers['X-Auth-Token'] = token + if service_token: + headers['X-Service-Token'] = service_token if 'content-length' not in (k.lower() for k in headers): headers['Content-Length'] = '0' conn.request(method, path, '', headers) @@ -785,7 +807,7 @@ def put_container(url, token, container, headers=None, http_conn=None, def post_container(url, token, container, headers, http_conn=None, - response_dict=None): + response_dict=None, service_token=None): """ Update a container's metadata. @@ -797,6 +819,7 @@ def post_container(url, token, container, headers, http_conn=None, conn object) :param response_dict: an optional dictionary into which to place the response - status, reason and headers + :param service_token: service auth token :raises ClientException: HTTP POST request failed """ if http_conn: @@ -806,6 +829,8 @@ def post_container(url, token, container, headers, http_conn=None, path = '%s/%s' % (parsed.path, quote(container)) method = 'POST' headers['X-Auth-Token'] = token + if service_token: + headers['X-Service-Token'] = service_token if 'content-length' not in (k.lower() for k in headers): headers['Content-Length'] = '0' conn.request(method, path, '', headers) @@ -825,7 +850,7 @@ def post_container(url, token, container, headers, http_conn=None, def delete_container(url, token, container, http_conn=None, - response_dict=None): + response_dict=None, service_token=None): """ Delete a container @@ -836,6 +861,7 @@ def delete_container(url, token, container, http_conn=None, conn object) :param response_dict: an optional dictionary into which to place the response - status, reason and headers + :param service_token: service auth token :raises ClientException: HTTP DELETE request failed """ if http_conn: @@ -844,6 +870,8 @@ def delete_container(url, token, container, http_conn=None, parsed, conn = http_connection(url) path = '%s/%s' % (parsed.path, quote(container)) headers = {'X-Auth-Token': token} + if service_token: + headers['X-Service-Token'] = service_token method = 'DELETE' conn.request(method, path, '', headers) resp = conn.getresponse() @@ -863,7 +891,7 @@ def delete_container(url, token, container, http_conn=None, def get_object(url, token, container, name, http_conn=None, resp_chunk_size=None, query_string=None, - response_dict=None, headers=None): + response_dict=None, headers=None, service_token=None): """ Get an object @@ -882,6 +910,7 @@ def get_object(url, token, container, name, http_conn=None, the response - status, reason and headers :param headers: an optional dictionary with additional headers to include in the request + :param service_token: service auth token :returns: a tuple of (response headers, the object's contents) The response headers will be a dict and all header names will be lowercase. :raises ClientException: HTTP GET request failed @@ -896,6 +925,8 @@ def get_object(url, token, container, name, http_conn=None, method = 'GET' headers = headers.copy() if headers else {} headers['X-Auth-Token'] = token + if service_token: + headers['X-Service-Token'] = service_token conn.request(method, path, '', headers) resp = conn.getresponse() @@ -923,7 +954,8 @@ def get_object(url, token, container, name, http_conn=None, return parsed_response['headers'], object_body -def head_object(url, token, container, name, http_conn=None): +def head_object(url, token, container, name, http_conn=None, + service_token=None): """ Get object info @@ -933,6 +965,7 @@ def head_object(url, token, container, name, http_conn=None): :param name: object name to get info for :param http_conn: HTTP connection object (If None, it will create the conn object) + :param service_token: service auth token :returns: a dict containing the response's headers (all header names will be lowercase) :raises ClientException: HTTP HEAD request failed @@ -944,6 +977,8 @@ def head_object(url, token, container, name, http_conn=None): path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) method = 'HEAD' headers = {'X-Auth-Token': token} + if service_token: + headers['X-Service-Token'] = service_token conn.request(method, path, '', headers) resp = conn.getresponse() body = resp.read() @@ -963,7 +998,7 @@ def head_object(url, token, container, name, http_conn=None): def put_object(url, token=None, container=None, name=None, contents=None, content_length=None, etag=None, chunk_size=None, content_type=None, headers=None, http_conn=None, proxy=None, - query_string=None, response_dict=None): + query_string=None, response_dict=None, service_token=None): """ Put an object @@ -994,6 +1029,7 @@ def put_object(url, token=None, container=None, name=None, contents=None, :param query_string: if set will be appended with '?' to generated path :param response_dict: an optional dictionary into which to place the response - status, reason and headers + :param service_token: service auth token :returns: etag :raises ClientException: HTTP PUT request failed """ @@ -1014,6 +1050,8 @@ def put_object(url, token=None, container=None, name=None, contents=None, headers = {} if token: headers['X-Auth-Token'] = token + if service_token: + headers['X-Service-Token'] = service_token if etag: headers['ETag'] = etag.strip('"') if content_length is not None: @@ -1067,7 +1105,7 @@ def put_object(url, token=None, container=None, name=None, contents=None, def post_object(url, token, container, name, headers, http_conn=None, - response_dict=None): + response_dict=None, service_token=None): """ Update object metadata @@ -1080,6 +1118,7 @@ def post_object(url, token, container, name, headers, http_conn=None, conn object) :param response_dict: an optional dictionary into which to place the response - status, reason and headers + :param service_token: service auth token :raises ClientException: HTTP POST request failed """ if http_conn: @@ -1088,6 +1127,8 @@ def post_object(url, token, container, name, headers, http_conn=None, parsed, conn = http_connection(url) path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) headers['X-Auth-Token'] = token + if service_token: + headers['X-Service-Token'] = service_token conn.request('POST', path, '', headers) resp = conn.getresponse() body = resp.read() @@ -1105,7 +1146,7 @@ def post_object(url, token, container, name, headers, http_conn=None, def delete_object(url, token=None, container=None, name=None, http_conn=None, headers=None, proxy=None, query_string=None, - response_dict=None): + response_dict=None, service_token=None): """ Delete object @@ -1123,6 +1164,7 @@ def delete_object(url, token=None, container=None, name=None, http_conn=None, :param query_string: if set will be appended with '?' to generated path :param response_dict: an optional dictionary into which to place the response - status, reason and headers + :param service_token: service auth token :raises ClientException: HTTP DELETE request failed """ if http_conn: @@ -1142,6 +1184,8 @@ def delete_object(url, token=None, container=None, name=None, http_conn=None, headers = {} if token: headers['X-Auth-Token'] = token + if service_token: + headers['X-Service-Token'] = service_token conn.request('DELETE', path, '', headers) resp = conn.getresponse() body = resp.read() @@ -1184,7 +1228,19 @@ def get_capabilities(http_conn): class Connection(object): - """Convenience class to make requests that will also retry the request""" + + """ + Convenience class to make requests that will also retry the request + + Requests will have an X-Auth-Token header whose value is either + the preauthtoken or a token obtained from the auth service using + the user credentials provided as args to the constructor. If + os_options includes a service_username then requests will also have + an X-Service-Token header whose value is a token obtained from the + auth service using the service credentials. In this case the request + url will be set to the storage_url obtained from the auth service + for the service user, unless this is overridden by a preauthurl. + """ def __init__(self, authurl=None, user=None, key=None, retries=5, preauthurl=None, preauthtoken=None, snet=False, @@ -1209,7 +1265,8 @@ def __init__(self, authurl=None, user=None, key=None, retries=5, to an auth 2.0 system. :param os_options: The OpenStack options which can have tenant_id, auth_token, service_type, endpoint_type, - tenant_name, object_storage_url, region_name + tenant_name, object_storage_url, region_name, + service_username, service_project_name, service_key :param insecure: Allow to access servers without checking SSL certs. The server's certificate will not be verified. :param ssl_compression: Whether to enable compression at the SSL layer. @@ -1240,6 +1297,11 @@ def __init__(self, authurl=None, user=None, key=None, retries=5, self.os_options['object_storage_url'] = preauthurl self.url = preauthurl or self.os_options.get('object_storage_url') self.token = preauthtoken or self.os_options.get('auth_token') + if self.os_options.get('service_username', None): + self.service_auth = True + else: + self.service_auth = False + self.service_token = None self.cacert = cacert self.insecure = insecure self.ssl_compression = ssl_compression @@ -1267,6 +1329,24 @@ def get_auth(self): timeout=self.timeout) return self.url, self.token + def get_service_auth(self): + opts = self.os_options + service_options = {} + service_options['tenant_name'] = opts.get('service_project_name', None) + service_options['region_name'] = opts.get('region_name', None) + service_options['object_storage_url'] = opts.get('object_storage_url', + None) + service_user = opts.get('service_username', None) + service_key = opts.get('service_key', None) + return get_auth(self.authurl, service_user, + service_key, + snet=self.snet, + auth_version=self.auth_version, + os_options=service_options, + cacert=self.cacert, + insecure=self.insecure, + timeout=self.timeout) + def http_connection(self, url=None): return http_connection(url if url else self.url, cacert=self.cacert, @@ -1294,13 +1374,17 @@ def _retry(self, reset_func, func, *args, **kwargs): if not self.url or not self.token: self.url, self.token = self.get_auth() self.http_conn = None + if self.service_auth and not self.service_token: + self.url, self.service_token = self.get_service_auth() + self.http_conn = None self.auth_end_time = time() if not self.http_conn: self.http_conn = self.http_connection() kwargs['http_conn'] = self.http_conn if caller_response_dict is not None: kwargs['response_dict'] = {} - rv = func(self.url, self.token, *args, **kwargs) + rv = func(self.url, self.token, *args, + service_token=self.service_token, **kwargs) self._add_response_dict(caller_response_dict, kwargs) return rv except SSLError: @@ -1317,7 +1401,7 @@ def _retry(self, reset_func, func, *args, **kwargs): logger.exception(err) raise if err.http_status == 401: - self.url = self.token = None + self.url = self.token = self.service_token = None if retried_auth or not all((self.authurl, self.user, self.key)): diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index cefae089..a65e712f 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -1333,7 +1333,7 @@ def get_auth(*args, **kwargs): # represenative of the unit under test. The real get_auth # method will always return the os_option dict's # object_storage_url which will be overridden by the - # preauthurl paramater to Connection if it is provided. + # preauthurl parameter to Connection if it is provided. return 'http://www.new.com', 'new' def swap_sleep(*args): @@ -1806,3 +1806,308 @@ def test_close_ok(self): self.assertIsInstance(http_conn_obj, c.HTTPConnection) self.assertFalse(hasattr(http_conn_obj, 'close')) conn.close() + + +class TestServiceToken(MockHttpTest): + + def setUp(self): + super(TestServiceToken, self).setUp() + self.os_options = { + 'object_storage_url': 'http://storage_url.com', + 'service_username': 'service_username', + 'service_project_name': 'service_project_name', + 'service_key': 'service_key'} + + def get_connection(self): + conn = c.Connection('http://www.test.com', 'asdf', 'asdf', + os_options=self.os_options) + + self.assertTrue(isinstance(conn, c.Connection)) + conn.get_auth = self.get_auth + conn.get_service_auth = self.get_service_auth + + self.assertEqual(conn.attempts, 0) + self.assertEqual(conn.service_token, None) + + self.assertTrue(isinstance(conn, c.Connection)) + return conn + + def get_auth(self): + # The real get_auth function will always return the os_option + # dict's object_storage_url which will be overridden by the + # preauthurl paramater to Connection if it is provided. + return self.os_options.get('object_storage_url'), 'token' + + def get_service_auth(self): + # The real get_auth function will always return the os_option + # dict's object_storage_url which will be overridden by the + # preauthurl parameter to Connection if it is provided. + return self.os_options.get('object_storage_url'), 'stoken' + + def test_service_token_reauth(self): + get_auth_call_list = [] + + def get_auth(url, user, key, **kwargs): + # The real get_auth function will always return the os_option + # dict's object_storage_url which will be overridden by the + # preauthurl parameter to Connection if it is provided. + args = {'url': url, 'user': user, 'key': key, 'kwargs': kwargs} + get_auth_call_list.append(args) + return_dict = {'asdf': 'new', 'service_username': 'newserv'} + storage_url = kwargs['os_options'].get('object_storage_url') + return storage_url, return_dict[user] + + def swap_sleep(*args): + self.swap_sleep_called = True + c.get_auth = get_auth + + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(401, 200)): + with mock.patch('swiftclient.client.sleep', swap_sleep): + self.swap_sleep_called = False + + conn = c.Connection('http://www.test.com', 'asdf', 'asdf', + preauthurl='http://www.old.com', + preauthtoken='old', + os_options=self.os_options) + + self.assertEqual(conn.attempts, 0) + self.assertEqual(conn.url, 'http://www.old.com') + self.assertEqual(conn.token, 'old') + + conn.head_account() + + self.assertTrue(self.swap_sleep_called) + self.assertEqual(conn.attempts, 2) + # The original 'preauth' storage URL *must* be preserved + self.assertEqual(conn.url, 'http://www.old.com') + self.assertEqual(conn.token, 'new') + self.assertEqual(conn.service_token, 'newserv') + + # Check get_auth was called with expected args + auth_args = get_auth_call_list[0] + auth_kwargs = get_auth_call_list[0]['kwargs'] + self.assertEqual('asdf', auth_args['user']) + self.assertEqual('asdf', auth_args['key']) + self.assertEqual('service_key', + auth_kwargs['os_options']['service_key']) + self.assertEqual('service_username', + auth_kwargs['os_options']['service_username']) + self.assertEqual('service_project_name', + auth_kwargs['os_options']['service_project_name']) + + auth_args = get_auth_call_list[1] + auth_kwargs = get_auth_call_list[1]['kwargs'] + self.assertEqual('service_username', auth_args['user']) + self.assertEqual('service_key', auth_args['key']) + self.assertEqual('service_project_name', + auth_kwargs['os_options']['tenant_name']) + + def test_service_token_get_account(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200)): + with mock.patch('swiftclient.client.parse_api_response'): + conn = self.get_connection() + conn.get_account() + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('GET', actual['method']) + actual_hdrs = actual['headers'] + self.assertTrue('X-Service-Token' in actual_hdrs) + self.assertEqual('stoken', actual_hdrs['X-Service-Token']) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/?format=json', + actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_head_account(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200)): + conn = self.get_connection() + conn.head_account() + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('HEAD', actual['method']) + actual_hdrs = actual['headers'] + self.assertTrue('X-Service-Token' in actual_hdrs) + self.assertEqual('stoken', actual_hdrs['X-Service-Token']) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com', actual['full_path']) + + self.assertEqual(conn.attempts, 1) + + def test_service_token_post_account(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(201)): + conn = self.get_connection() + conn.post_account(headers={}) + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('POST', actual['method']) + actual_hdrs = actual['headers'] + self.assertTrue('X-Service-Token' in actual_hdrs) + self.assertEqual('stoken', actual_hdrs['X-Service-Token']) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com', actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_delete_container(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(204)): + conn = self.get_connection() + conn.delete_container('container1') + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('DELETE', actual['method']) + actual_hdrs = actual['headers'] + self.assertTrue('X-Service-Token' in actual_hdrs) + self.assertEqual('stoken', actual_hdrs['X-Service-Token']) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/container1', + actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_get_container(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200)): + with mock.patch('swiftclient.client.parse_api_response'): + conn = self.get_connection() + conn.get_container('container1') + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('GET', actual['method']) + actual_hdrs = actual['headers'] + self.assertTrue('X-Service-Token' in actual_hdrs) + self.assertEqual('stoken', actual_hdrs['X-Service-Token']) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/container1?format=json', + actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_head_container(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200)): + conn = self.get_connection() + conn.head_container('container1') + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('HEAD', actual['method']) + actual_hdrs = actual['headers'] + self.assertTrue('X-Service-Token' in actual_hdrs) + self.assertEqual('stoken', actual_hdrs['X-Service-Token']) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/container1', + actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_post_container(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(201)): + conn = self.get_connection() + conn.post_container('container1', {}) + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('POST', actual['method']) + actual_hdrs = actual['headers'] + self.assertTrue('X-Service-Token' in actual_hdrs) + self.assertEqual('stoken', actual_hdrs['X-Service-Token']) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/container1', + actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_put_container(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200)): + conn = self.get_connection() + conn.put_container('container1') + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('PUT', actual['method']) + actual_hdrs = actual['headers'] + self.assertTrue('X-Service-Token' in actual_hdrs) + self.assertEqual('stoken', actual_hdrs['X-Service-Token']) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/container1', + actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_get_object(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200)): + conn = self.get_connection() + conn.get_object('container1', 'obj1') + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('GET', actual['method']) + actual_hdrs = actual['headers'] + self.assertTrue('X-Service-Token' in actual_hdrs) + self.assertEqual('stoken', actual_hdrs['X-Service-Token']) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/container1/obj1', + actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_head_object(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200)): + conn = self.get_connection() + conn.head_object('container1', 'obj1') + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('HEAD', actual['method']) + actual_hdrs = actual['headers'] + self.assertTrue('X-Service-Token' in actual_hdrs) + self.assertEqual('stoken', actual_hdrs['X-Service-Token']) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/container1/obj1', + actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_put_object(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200)): + conn = self.get_connection() + conn.put_object('container1', 'obj1', 'a_string') + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('PUT', actual['method']) + actual_hdrs = actual['headers'] + self.assertTrue('X-Service-Token' in actual_hdrs) + self.assertEqual('stoken', actual_hdrs['X-Service-Token']) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/container1/obj1', + actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_post_object(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(202)): + conn = self.get_connection() + conn.post_object('container1', 'obj1', {}) + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('POST', actual['method']) + actual_hdrs = actual['headers'] + self.assertTrue('X-Service-Token' in actual_hdrs) + self.assertEqual('stoken', actual_hdrs['X-Service-Token']) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/container1/obj1', + actual['full_path']) + self.assertEqual(conn.attempts, 1) + + def test_service_token_delete_object(self): + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(202)): + conn = self.get_connection() + conn.delete_object('container1', 'obj1', 'a_string') + self.assertEqual(1, len(self.request_log), self.request_log) + for actual in self.iter_request_log(): + self.assertEqual('DELETE', actual['method']) + actual_hdrs = actual['headers'] + self.assertTrue('X-Service-Token' in actual_hdrs) + self.assertEqual('stoken', actual_hdrs['X-Service-Token']) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual('http://storage_url.com/container1/obj1?a_string', + actual['full_path']) + self.assertEqual(conn.attempts, 1) From 4b310083dfebe8c54e599fe319f801cca87f8dd6 Mon Sep 17 00:00:00 2001 From: Alistair Coles Date: Mon, 24 Aug 2015 12:34:45 +0100 Subject: [PATCH 030/454] Stop Connection class modifying os_options parameter When a caller passes an os_options dict to the Connection class constructor, the constructor may modify the os_options dict, which can surprise the caller if they re-use the os_options dict. Specifically the os_options tenant_name and object_storage_url may be modified, and the changed values would then leak through to a subsequent Connection constructed using the same os_options dict. This fix simply constructs a new dict from the supplied os_options. The patch also adds a test that covers this and also verifies that a preauth_url passed as a keyword arg to Connection() will take precedence over any object_storage_url in an os_options parameter. Closes-Bug: 1488070 Change-Id: Ic6b5cf3ac68c505de155619f2610be9529e15432 --- swiftclient/client.py | 2 +- tests/unit/test_swiftclient.py | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index 4819c124..a2d7adab 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -1290,7 +1290,7 @@ def __init__(self, authurl=None, user=None, key=None, retries=5, self.starting_backoff = starting_backoff self.max_backoff = max_backoff self.auth_version = auth_version - self.os_options = os_options or {} + self.os_options = dict(os_options or {}) if tenant_name: self.os_options['tenant_name'] = tenant_name if preauthurl: diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index 97ae467a..23b31388 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -1481,6 +1481,33 @@ def test_os_preauth_url_trumps_auth_url(self): ('HEAD', '/v1/AUTH_pre_url', '', {'x-auth-token': 'post_token'}), ]) + def test_preauth_url_trumps_os_preauth_url(self): + storage_url = 'http://storage.example.com/v1/AUTH_pre_url' + os_storage_url = 'http://storage.example.com/v1/AUTH_os_pre_url' + os_preauth_options = { + 'tenant_name': 'demo', + 'object_storage_url': os_storage_url, + } + orig_os_preauth_options = dict(os_preauth_options) + conn = c.Connection('http://auth.example.com', 'user', 'password', + os_options=os_preauth_options, auth_version=2, + preauthurl=storage_url, tenant_name='not_demo') + fake_keystone = fake_get_auth_keystone( + storage_url='http://storage.example.com/v1/AUTH_post_url', + token='post_token') + fake_conn = self.fake_http_connection(200) + with mock.patch.multiple('swiftclient.client', + get_auth_keystone=fake_keystone, + http_connection=fake_conn, + sleep=mock.DEFAULT): + conn.head_account() + self.assertRequests([ + ('HEAD', '/v1/AUTH_pre_url', '', {'x-auth-token': 'post_token'}), + ]) + + # check that Connection has not modified our os_options + self.assertEqual(orig_os_preauth_options, os_preauth_options) + def test_get_auth_sets_url_and_token(self): with mock.patch('swiftclient.client.get_auth') as mock_get_auth: mock_get_auth.return_value = ( From 4b627327c9cca5d0ddd038cd5b42466945ae1657 Mon Sep 17 00:00:00 2001 From: Charles Hsu Date: Tue, 18 Aug 2015 19:22:24 +0800 Subject: [PATCH 031/454] Increase httplib._MAXHEADERS to 256. By default Swift increase the number of max metadata count to 90 and extra header count to 32. That mean we can put 90 metadata to Account/Container/Object by default, when user put 90 metadata to a Account, the Account header count is close or more than 100. The swift client unable to access Account and get an error likes, ('Connection aborted.', HTTPException('got more than 100 headers',)) So the default _MAXHEADERS(100) won't enough. Change-Id: I5ffc4eb5d3e1ebc3dbdd7dc69376919ae3e1c5a8 --- swiftclient/client.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index 74e60c0a..25c21336 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -16,7 +16,6 @@ """ OpenStack Swift client library used internally """ - import socket import requests import logging @@ -24,6 +23,7 @@ from distutils.version import StrictVersion from requests.exceptions import RequestException, SSLError +from six.moves import http_client from six.moves.urllib.parse import quote as _quote from six.moves.urllib.parse import urlparse, urlunparse from time import sleep, time @@ -34,6 +34,9 @@ from swiftclient.utils import ( LengthWrapper, ReadableToIterable, parse_api_response) +# Defautl is 100, increase to 256 +http_client._MAXHEADERS = 256 + AUTH_VERSIONS_V1 = ('1.0', '1', 1) AUTH_VERSIONS_V2 = ('2.0', '2', 2) AUTH_VERSIONS_V3 = ('3.0', '3', 3) From 3c0289844f73b9e685b8443c90b0ca80e0b18f28 Mon Sep 17 00:00:00 2001 From: Joel Wright Date: Sun, 5 Apr 2015 16:48:34 +0100 Subject: [PATCH 032/454] Log and report trace on service operation fails This patch adds exception logging to the swift service API. Each operation that results in failure of any operation will now log the exception as well as report a timestamp and full stack trace in the results returned by the service API calls. Change-Id: I7336b28354e7740ea7d048bdf355e3c1a1b4436c --- swiftclient/service.py | 237 +++++++++++++++++++++++++++++-------- swiftclient/utils.py | 18 ++- tests/unit/test_service.py | 47 ++++++-- 3 files changed, 241 insertions(+), 61 deletions(-) diff --git a/swiftclient/service.py b/swiftclient/service.py index 4f331c4e..c013b902 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -12,7 +12,9 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +import logging import os + from concurrent.futures import as_completed, CancelledError, TimeoutError from copy import deepcopy from errno import EEXIST, ENOENT @@ -39,12 +41,15 @@ ) from swiftclient.utils import ( config_true_value, ReadableToIterable, LengthWrapper, EMPTY_ETAG, - parse_api_response + parse_api_response, report_traceback ) from swiftclient.exceptions import ClientException from swiftclient.multithreading import MultiThreadingManager +logger = logging.getLogger("swiftclient.service") + + class ResultsIterator(Iterator): def __init__(self, futures): self.futures = interruptable_as_completed(futures) @@ -435,16 +440,24 @@ def stat(self, container=None, objects=None, options=None): return res except ClientException as err: if err.http_status != 404: + traceback, err_time = report_traceback() + logger.exception(err) res.update({ 'success': False, - 'error': err + 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time }) return res raise SwiftError('Account not found', exc=err) except Exception as err: + traceback, err_time = report_traceback() + logger.exception(err) res.update({ 'success': False, - 'error': err + 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time }) return res else: @@ -467,17 +480,25 @@ def stat(self, container=None, objects=None, options=None): return res except ClientException as err: if err.http_status != 404: + traceback, err_time = report_traceback() + logger.exception(err) res.update({ 'success': False, - 'error': err + 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time }) return res raise SwiftError('Container %r not found' % container, container=container, exc=err) except Exception as err: + traceback, err_time = report_traceback() + logger.exception(err) res.update({ 'success': False, - 'error': err + 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time }) return res else: @@ -506,9 +527,13 @@ def _stat_object(conn, container, obj, options): }) return res except Exception as err: + traceback, err_time = report_traceback() + logger.exception(err) res.update({ 'success': False, - 'error': err + 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time }) return res @@ -581,18 +606,26 @@ def post(self, container=None, objects=None, options=None): get_future_result(post) except ClientException as err: if err.http_status != 404: + traceback, err_time = report_traceback() + logger.exception(err) res.update({ 'success': False, 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time, 'response_dict': response_dict }) return res - raise SwiftError('Account not found') + raise SwiftError('Account not found', exc=err) except Exception as err: + traceback, err_time = report_traceback() + logger.exception(err) res.update({ 'success': False, 'error': err, - 'response_dict': response_dict + 'response_dict': response_dict, + 'traceback': traceback, + 'error_timestamp': err_time }) return res if not objects: @@ -619,23 +652,31 @@ def post(self, container=None, objects=None, options=None): get_future_result(post) except ClientException as err: if err.http_status != 404: + traceback, err_time = report_traceback() + logger.exception(err) res.update({ 'action': 'post_container', 'success': False, 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time, 'response_dict': response_dict }) return res raise SwiftError( "Container '%s' not found" % container, - container=container + container=container, exc=err ) except Exception as err: + traceback, err_time = report_traceback() + logger.exception(err) res.update({ 'action': 'post_container', 'success': False, 'error': err, - 'response_dict': response_dict + 'response_dict': response_dict, + 'traceback': traceback, + 'error_timestamp': err_time }) return res else: @@ -720,9 +761,13 @@ def _post_object_job(conn, container, obj, headers, result): conn.post_object( container, obj, headers=headers, response_dict=result) except Exception as err: + traceback, err_time = report_traceback() + logger.exception(err) res.update({ 'success': False, - 'error': err + 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time }) return res @@ -775,7 +820,6 @@ def list(self, container=None, options=None): @staticmethod def _list_account_job(conn, options, result_queue): marker = '' - success = True error = None try: while True: @@ -804,23 +848,30 @@ def _list_account_job(conn, options, result_queue): marker = items[-1].get('name', items[-1].get('subdir')) except ClientException as err: - success = False + traceback, err_time = report_traceback() + logger.exception(err) if err.http_status != 404: - error = err + error = (err, traceback, err_time) else: - error = SwiftError('Account not found') + error = ( + SwiftError('Account not found', exc=err), + traceback, err_time + ) except Exception as err: - success = False - error = err + traceback, err_time = report_traceback() + logger.exception(err) + error = (err, traceback, err_time) res = { 'action': 'list_account_part', 'container': None, 'prefix': options['prefix'], - 'success': success, + 'success': False, 'marker': marker, - 'error': error, + 'error': error[0], + 'traceback': error[1], + 'error_timestamp': error[2] } result_queue.put(res) result_queue.put(None) @@ -828,7 +879,6 @@ def _list_account_job(conn, options, result_queue): @staticmethod def _list_container_job(conn, container, options, result_queue): marker = '' - success = True error = None try: while True: @@ -853,23 +903,33 @@ def _list_container_job(conn, container, options, result_queue): marker = items[-1].get('name', items[-1].get('subdir')) except ClientException as err: - success = False + traceback, err_time = report_traceback() + logger.exception(err) if err.http_status != 404: - error = err + error = (err, traceback, err_time) else: - error = SwiftError('Container %r not found' % container, - container=container) + error = ( + SwiftError( + 'Container %r not found' % container, + container=container, exc=err + ), + traceback, + err_time + ) except Exception as err: - success = False - error = err + traceback, err_time = report_traceback() + logger.exception(err) + error = (err, traceback, err_time) res = { 'action': 'list_container_part', 'container': container, 'prefix': options['prefix'], - 'success': success, + 'success': False, 'marker': marker, - 'error': error, + 'error': error[0], + 'traceback': error[1], + 'error_timestamp': error[2] } result_queue.put(res) result_queue.put(None) @@ -937,7 +997,7 @@ def download(self, container=None, objects=None, options=None): except ClientException as err: if err.http_status != 404: raise - raise SwiftError('Account not found') + raise SwiftError('Account not found', exc=err) elif not objects: if '/' in container: @@ -1123,12 +1183,16 @@ def _download_object_job(self, conn, container, obj, options): return res except Exception as err: + traceback, err_time = report_traceback() + logger.exception(err) res = { 'action': 'download_object', 'container': container, 'object': obj, 'success': False, 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time, 'response_dict': results_dict, 'path': path, 'pseudodir': pseudodir, @@ -1168,7 +1232,8 @@ def _download_container(self, container, options): if err.http_status != 404: raise raise SwiftError( - 'Container %r not found' % container, container=container + 'Container %r not found' % container, + container=container, exc=err ) error = None @@ -1195,6 +1260,7 @@ def _download_container(self, container, options): ) except ClientException as err: # Allow the current page to finish downloading + logger.exception(err) error = err except Exception: # Something unexpected went wrong - cancel @@ -1368,12 +1434,16 @@ def upload(self, container, objects, options=None): file_jobs[file_future] = details except OSError as err: # Avoid tying up threads with jobs that will fail + traceback, err_time = report_traceback() + logger.exception(err) res = { 'action': 'upload_object', 'container': container, 'object': o, 'success': False, 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time, 'path': s } rq.put(res) @@ -1472,9 +1542,13 @@ def _create_container_job( 'response_dict': create_response }) except Exception as err: + traceback, err_time = report_traceback() + logger.exception(err) res.update({ 'success': False, 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time, 'response_dict': create_response }) return res @@ -1513,9 +1587,14 @@ def _create_dir_marker_job(conn, container, obj, options, path=None): return res except ClientException as err: if err.http_status != 404: + traceback, err_time = report_traceback() + logger.exception(err) res.update({ 'success': False, - 'error': err}) + 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time + }) return res try: conn.put_object(container, obj, '', content_length=0, @@ -1527,9 +1606,13 @@ def _create_dir_marker_job(conn, container, obj, options, path=None): 'response_dict': results_dict}) return res except Exception as err: + traceback, err_time = report_traceback() + logger.exception(err) res.update({ 'success': False, 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time, 'response_dict': results_dict}) return res @@ -1582,9 +1665,13 @@ def _upload_segment_job(conn, path, container, segment_name, segment_start, return res except Exception as err: + traceback, err_time = report_traceback() + logger.exception(err) res.update({ 'success': False, 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time, 'response_dict': results_dict, 'attempts': conn.attempts }) @@ -1713,9 +1800,13 @@ def _upload_object_job(self, conn, container, source, obj, options, old_slo_manifest_paths.append(seg_path) except ClientException as err: if err.http_status != 404: + traceback, err_time = report_traceback() + logger.exception(err) res.update({ 'success': False, - 'error': err + 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time }) return res @@ -1770,9 +1861,11 @@ def _upload_object_job(self, conn, container, source, obj, options, if not r['success']: errors = True segment_results.append(r) - except Exception as e: + except Exception as err: + traceback, err_time = report_traceback() + logger.exception(err) errors = True - exceptions.append(e) + exceptions.append((err, traceback, err_time)) if errors: err = ClientException( 'Aborting manifest creation ' @@ -1901,16 +1994,26 @@ def _upload_object_job(self, conn, container, source, obj, options, return res except OSError as err: + traceback, err_time = report_traceback() + logger.exception(err) if err.errno == ENOENT: - err = SwiftError('Local file %r not found' % path) + error = SwiftError('Local file %r not found' % path, exc=err) + else: + error = err res.update({ 'success': False, - 'error': err + 'error': error, + 'traceback': traceback, + 'error_timestamp': err_time }) except Exception as err: + traceback, err_time = report_traceback() + logger.exception(err) res.update({ 'success': False, - 'error': err + 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time }) return res @@ -2009,8 +2112,15 @@ def _delete_segment(conn, container, obj, results_queue=None): try: conn.delete_object(container, obj, response_dict=results_dict) res = {'success': True} - except Exception as e: - res = {'success': False, 'error': e} + except Exception as err: + traceback, err_time = report_traceback() + logger.exception(err) + res = { + 'success': False, + 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time + } res.update({ 'action': 'delete_segment', @@ -2026,12 +2136,12 @@ def _delete_segment(conn, container, obj, results_queue=None): def _delete_object(self, conn, container, obj, options, results_queue=None): + res = { + 'action': 'delete_object', + 'container': container, + 'object': obj + } try: - res = { - 'action': 'delete_object', - 'container': container, - 'object': obj - } old_manifest = None query_string = None @@ -2086,8 +2196,14 @@ def _delete_object(self, conn, container, obj, options, }) except Exception as err: - res['success'] = False - res['error'] = err + traceback, err_time = report_traceback() + logger.exception(err) + res.update({ + 'success': False, + 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time + }) return res return res @@ -2098,8 +2214,15 @@ def _delete_empty_container(conn, container): try: conn.delete_container(container, response_dict=results_dict) res = {'success': True} - except Exception as e: - res = {'success': False, 'error': e} + except Exception as err: + traceback, err_time = report_traceback() + logger.exception(err) + res = { + 'success': False, + 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time + } res.update({ 'action': 'delete_container', @@ -2130,12 +2253,16 @@ def _delete_container(self, container, options): con_del_res = get_future_result(con_del) except Exception as err: + traceback, err_time = report_traceback() + logger.exception(err) con_del_res = { 'action': 'delete_container', 'container': container, 'object': None, 'success': False, - 'error': err + 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time } yield con_del_res @@ -2173,7 +2300,7 @@ def capabilities(self, url=None): except ClientException as err: if err.http_status != 404: raise err - raise SwiftError('Account not found') + raise SwiftError('Account not found', exc=err) return res @@ -2208,10 +2335,16 @@ def _watch_futures(futures, result_queue): res['status'] = 'cancelled' result_queue.put(res) except Exception as err: + traceback, err_time = report_traceback() + logger.exception(err) details = futures[f] res = details - res['success'] = False - res['error'] = err + res.update({ + 'success': False, + 'error': err, + 'traceback': traceback, + 'error_timestamp': err_time + }) result_queue.put(res) result_queue.put(None) diff --git a/swiftclient/utils.py b/swiftclient/utils.py index 8edfcfa6..6ff62594 100644 --- a/swiftclient/utils.py +++ b/swiftclient/utils.py @@ -17,9 +17,9 @@ import hmac import json import logging -import time - import six +import time +import traceback TRUE_VALUES = set(('true', '1', 'yes', 'on', 't', 'y')) EMPTY_ETAG = 'd41d8cd98f00b204e9800998ecf8427e' @@ -119,6 +119,20 @@ def parse_api_response(headers, body): return json.loads(body.decode(charset)) +def report_traceback(): + """ + Reports a timestamp and full traceback for a given exception. + + :return: Full traceback and timestamp. + """ + try: + formatted_lines = traceback.format_exc() + now = time.time() + return formatted_lines, now + except AttributeError: + return None, None + + class NoopMD5(object): def __init__(self, *a, **kw): pass diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index db47263f..715a7b7f 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -227,16 +227,23 @@ def test_delete_segment_exception(self): 'action': 'delete_segment', 'object': 'test_s', 'success': False, - 'error': self.exc + 'error': self.exc, + 'traceback': mock.ANY, + 'error_timestamp': mock.ANY }) + before = time.time() r = SwiftService._delete_segment(mock_conn, 'test_c', 'test_s', mock_q) + after = time.time() mock_conn.delete_object.assert_called_once_with( 'test_c', 'test_s', response_dict={} ) self._assertDictEqual(expected_r, r) self._assertDictEqual(expected_r, self._get_queue(mock_q)) + self.assertGreaterEqual(r['error_timestamp'], before) + self.assertLessEqual(r['error_timestamp'], after) + self.assertTrue('Traceback' in r['traceback']) def test_delete_object(self): mock_q = Queue() @@ -263,20 +270,27 @@ def test_delete_object_exception(self): expected_r = self._get_expected({ 'action': 'delete_object', 'success': False, - 'error': self.exc + 'error': self.exc, + 'traceback': mock.ANY, + 'error_timestamp': mock.ANY }) # _delete_object doesnt populate attempts or response dict if it hits # an error. This may not be the correct behaviour. del expected_r['response_dict'], expected_r['attempts'] + before = time.time() s = SwiftService() r = s._delete_object(mock_conn, 'test_c', 'test_o', self.opts, mock_q) + after = time.time() mock_conn.head_object.assert_called_once_with('test_c', 'test_o') mock_conn.delete_object.assert_called_once_with( 'test_c', 'test_o', query_string=None, response_dict={} ) self._assertDictEqual(expected_r, r) + self.assertGreaterEqual(r['error_timestamp'], before) + self.assertLessEqual(r['error_timestamp'], after) + self.assertTrue('Traceback' in r['traceback']) def test_delete_object_slo_support(self): # If SLO headers are present the delete call should include an @@ -353,23 +367,30 @@ def test_delete_empty_container(self): ) self._assertDictEqual(expected_r, r) - def test_delete_empty_container_excpetion(self): + def test_delete_empty_container_exception(self): mock_conn = self._get_mock_connection() mock_conn.delete_container = Mock(side_effect=self.exc) expected_r = self._get_expected({ 'action': 'delete_container', 'success': False, 'object': None, - 'error': self.exc + 'error': self.exc, + 'traceback': mock.ANY, + 'error_timestamp': mock.ANY }) + before = time.time() s = SwiftService() r = s._delete_empty_container(mock_conn, 'test_c') + after = time.time() mock_conn.delete_container.assert_called_once_with( 'test_c', response_dict={} ) self._assertDictEqual(expected_r, r) + self.assertGreaterEqual(r['error_timestamp'], before) + self.assertLessEqual(r['error_timestamp'], after) + self.assertTrue('Traceback' in r['traceback']) class TestSwiftError(testtools.TestCase): @@ -618,7 +639,9 @@ def test_list_account_exception(self): 'action': 'list_account_part', 'success': False, 'error': self.exc, - 'marker': '' + 'marker': '', + 'traceback': mock.ANY, + 'error_timestamp': mock.ANY }) SwiftService._list_account_job( @@ -684,7 +707,9 @@ def test_list_container_exception(self): 'container': 'test_c', 'success': False, 'error': self.exc, - 'marker': '' + 'marker': '', + 'error_timestamp': mock.ANY, + 'traceback': mock.ANY }) SwiftService._list_container_job( @@ -1431,7 +1456,9 @@ def test_download_object_job_exception(self): mock_conn.get_object = Mock(side_effect=self.exc) expected_r = self._get_expected({ 'success': False, - 'error': self.exc + 'error': self.exc, + 'error_timestamp': mock.ANY, + 'traceback': mock.ANY }) s = SwiftService() @@ -1566,6 +1593,8 @@ def fake_get(*args, **kwargs): 'path': 'test_o', 'pseudodir': False, 'attempts': 2, + 'traceback': mock.ANY, + 'error_timestamp': mock.ANY } s = SwiftService() @@ -1619,6 +1648,8 @@ def test_download_object_job_skip_identical_dlo(self): 'path': 'test_o', 'pseudodir': False, 'attempts': 2, + 'traceback': mock.ANY, + 'error_timestamp': mock.ANY } s = SwiftService() @@ -1700,6 +1731,8 @@ def test_download_object_job_skip_identical_nested_slo(self): 'path': 'test_o', 'pseudodir': False, 'attempts': 2, + 'traceback': mock.ANY, + 'error_timestamp': mock.ANY } s = SwiftService() From d5eb818228d98bf7477658b7309750afe1b0423a Mon Sep 17 00:00:00 2001 From: Alistair Coles Date: Wed, 26 Aug 2015 15:41:05 +0100 Subject: [PATCH 033/454] Cleanup and improve tests for download Some improvements to the tests for staggered download that were added in [1]. [1] Ie737cbb7f8b1fa8a79bbb88914730b05aa7f2906 Change-Id: Ib999bc7bd198c1d9c217c57501f751e854d4c6ad --- tests/unit/test_service.py | 69 +++++++++++++++++++++++--------------- tests/unit/test_shell.py | 6 ---- 2 files changed, 42 insertions(+), 33 deletions(-) diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index db47263f..d47c382c 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -154,14 +154,19 @@ class _TestServiceBase(testtools.TestCase): def _assertDictEqual(self, a, b, m=None): # assertDictEqual is not available in py2.6 so use a shallow check # instead + if not m: + m = '{0} != {1}'.format(a, b) + if hasattr(self, 'assertDictEqual'): self.assertDictEqual(a, b, m) else: - self.assertTrue(isinstance(a, dict)) - self.assertTrue(isinstance(b, dict)) + self.assertTrue(isinstance(a, dict), + 'First argument is not a dictionary') + self.assertTrue(isinstance(b, dict), + 'Second argument is not a dictionary') self.assertEqual(len(a), len(b), m) for k, v in a.items(): - self.assertTrue(k in b, m) + self.assertIn(k, b, m) self.assertEqual(b[k], v, m) def _get_mock_connection(self, attempts=2): @@ -1268,22 +1273,6 @@ def setUp(self): def _readbody(self): yield self.obj_content - def _assertDictEqual(self, a, b, m=None): - # assertDictEqual is not available in py2.6 so use a shallow check - # instead - if not m: - m = '{0} != {1}'.format(a, b) - - if hasattr(self, 'assertDictEqual'): - self.assertDictEqual(a, b, m) - else: - self.assertTrue(isinstance(a, dict), m) - self.assertTrue(isinstance(b, dict), m) - self.assertEqual(len(a), len(b), m) - for k, v in a.items(): - self.assertIn(k, b, m) - self.assertEqual(b[k], v, m) - @mock.patch('swiftclient.service.SwiftService.list') @mock.patch('swiftclient.service.SwiftService._submit_page_downloads') @mock.patch('swiftclient.service.interruptable_as_completed') @@ -1291,12 +1280,32 @@ def test_download_container_job(self, as_comp, sub_page, service_list): """ Check that paged downloads work correctly """ - as_comp.side_effect = [ + obj_count = [0] + + def make_counting_generator(object_to_yield, total_count): + # maintain a counter of objects yielded + count = [0] + + def counting_generator(): + while count[0] < 10: + yield object_to_yield + count[0] += 1 + total_count[0] += 1 + return counting_generator() + + obj_count_on_sub_page_call = [] + sub_page_call_count = [0] + + def fake_sub_page(*args): + # keep a record of obj_count when this function is called + obj_count_on_sub_page_call.append(obj_count[0]) + sub_page_call_count[0] += 1 + if sub_page_call_count[0] < 3: + return range(0, 10) + return None + + sub_page.side_effect = fake_sub_page - ] - sub_page.side_effect = [ - range(0, 10), range(0, 10), [] # simulate multiple result pages - ] r = Mock(spec=Future) r.result.return_value = self._get_expected({ 'success': True, @@ -1306,15 +1315,19 @@ def test_download_container_job(self, as_comp, sub_page, service_list): 'auth_end_time': 4, 'read_length': len(b'objcontent'), }) + as_comp.side_effect = [ - [r for _ in range(0, 10)], - [r for _ in range(0, 10)] + make_counting_generator(r, obj_count), + make_counting_generator(r, obj_count) ] s = SwiftService() down_gen = s._download_container('test_c', self.opts) results = list(down_gen) self.assertEqual(20, len(results)) + self.assertEqual(2, as_comp.call_count) + self.assertEqual(3, sub_page_call_count[0]) + self.assertEqual([0, 7, 17], obj_count_on_sub_page_call) @mock.patch('swiftclient.service.SwiftService.list') @mock.patch('swiftclient.service.SwiftService._submit_page_downloads') @@ -1365,6 +1378,7 @@ def _make_result(): # This was an unknown error, so make sure we attempt to cancel futures for spe in sub_page_effects[0]: spe.cancel.assert_called_once_with() + self.assertEqual(1, as_comp.call_count) # Now test ClientException sub_page_effects = [ @@ -1372,9 +1386,9 @@ def _make_result(): ClientException('Go Boom') ] sub_page.side_effect = sub_page_effects + as_comp.reset_mock() as_comp.side_effect = [ [_make_result() for _ in range(0, 10)], - [_make_result() for _ in range(0, 10)] ] self.assertRaises( ClientException, @@ -1383,6 +1397,7 @@ def _make_result(): # This was a ClientException, so make sure we don't cancel futures for spe in sub_page_effects[0]: self.assertFalse(spe.cancel.called) + self.assertEqual(1, as_comp.call_count) def test_download_object_job(self): mock_conn = self._get_mock_connection() diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index 7dce03bd..12ceadbb 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -410,18 +410,12 @@ def test_download_shuffle(self, connection, mock_shuffle): # Test that the container and object lists are not shuffled mock_shuffle.reset_mock() - connection.return_value.get_object.return_value = [ - {'content-type': 'text/plain', - 'etag': 'd41d8cd98f00b204e9800998ecf8427e'}, - ''] connection.return_value.get_container.side_effect = [ (None, [{'name': 'object'}]), (None, [{'name': 'pseudo/'}]), (None, []), ] - connection.return_value.auth_end_time = 0 - connection.return_value.attempts = 0 connection.return_value.get_account.side_effect = [ (None, [{'name': 'container'}]), (None, []) From ce569f46517e10f2ce0d27e9ee0a922ad1d84e2f Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Mon, 18 May 2015 08:05:02 -0700 Subject: [PATCH 034/454] Centralize header parsing All response headers are now exposed as unicode objects. Any url-encoding is interpretted as UTF-8; if that causes decoding to fail, the url-encoded form is returned. As a result, deleting DLOs with unicode characters will no longer raise UnicodeEncodeErrors under Python 2. Related-Bug: #1431866 Change-Id: Idb111c5bf3ac1f5ccfa724b3f4ede8f37d5bfac4 --- swiftclient/client.py | 70 ++++++++++++++++++++++------------ swiftclient/service.py | 8 ++-- tests/unit/test_swiftclient.py | 37 ++++++++++++++++++ tests/unit/utils.py | 4 +- 4 files changed, 88 insertions(+), 31 deletions(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index 8466cc5b..c0af72a2 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -24,7 +24,7 @@ from distutils.version import StrictVersion from requests.exceptions import RequestException, SSLError from six.moves import http_client -from six.moves.urllib.parse import quote as _quote +from six.moves.urllib.parse import quote as _quote, unquote from six.moves.urllib.parse import urlparse, urlunparse from time import sleep, time import six @@ -103,6 +103,36 @@ def http_log(args, kwargs, resp, body): log_method("RESP BODY: %s", body) +def parse_header_string(data): + if six.PY2: + if isinstance(data, six.text_type): + # Under Python2 requests only returns binary_type, but if we get + # some stray text_type input, this should prevent unquote from + # interpretting %-encoded data as raw code-points. + data = data.encode('utf8') + try: + unquoted = unquote(data).decode('utf8') + except UnicodeDecodeError: + try: + return data.decode('utf8') + except UnicodeDecodeError: + return quote(data).decode('utf8') + else: + if isinstance(data, six.binary_type): + # Under Python3 requests only returns text_type and tosses (!) the + # rest of the headers. If that ever changes, this should be a sane + # approach. + try: + data = data.decode('ascii') + except UnicodeDecodeError: + data = quote(data) + try: + unquoted = unquote(data, errors='strict') + except UnicodeDecodeError: + return data + return unquoted + + def quote(value, safe='/'): """ Patched version of urllib.quote that encodes utf8 strings before quoting. @@ -472,6 +502,14 @@ def get_auth(auth_url, user, key, **kwargs): return storage_url, token +def resp_header_dict(resp): + resp_headers = {} + for header, value in resp.getheaders(): + header = parse_header_string(header).lower() + resp_headers[header] = parse_header_string(value) + return resp_headers + + def store_response(resp, response_dict): """ store information about an operation into a dict @@ -482,13 +520,9 @@ def store_response(resp, response_dict): status, reason and a dict of lower-cased headers """ if response_dict is not None: - resp_headers = {} - for header, value in resp.getheaders(): - resp_headers[header.lower()] = value - response_dict['status'] = resp.status response_dict['reason'] = resp.reason - response_dict['headers'] = resp_headers + response_dict['headers'] = resp_header_dict(resp) def get_account(url, token, marker=None, limit=None, prefix=None, @@ -545,9 +579,7 @@ def get_account(url, token, marker=None, limit=None, prefix=None, body = resp.read() http_log(("%s?%s" % (url, qs), method,), {'headers': headers}, resp, body) - resp_headers = {} - for header, value in resp.getheaders(): - resp_headers[header.lower()] = value + resp_headers = resp_header_dict(resp) if resp.status < 200 or resp.status >= 300: raise ClientException('Account GET failed', http_scheme=parsed.scheme, http_host=conn.host, http_path=parsed.path, @@ -589,9 +621,7 @@ def head_account(url, token, http_conn=None, service_token=None): http_host=conn.host, http_path=parsed.path, http_status=resp.status, http_reason=resp.reason, http_response_content=body) - resp_headers = {} - for header, value in resp.getheaders(): - resp_headers[header.lower()] = value + resp_headers = resp_header_dict(resp) return resp_headers @@ -712,9 +742,7 @@ def get_container(url, token, container, marker=None, limit=None, http_path=cont_path, http_query=qs, http_status=resp.status, http_reason=resp.reason, http_response_content=body) - resp_headers = {} - for header, value in resp.getheaders(): - resp_headers[header.lower()] = value + resp_headers = resp_header_dict(resp) if resp.status == 204: return resp_headers, [] return resp_headers, parse_api_response(resp_headers, body) @@ -758,9 +786,7 @@ def head_container(url, token, container, http_conn=None, headers=None, http_path=path, http_status=resp.status, http_reason=resp.reason, http_response_content=body) - resp_headers = {} - for header, value in resp.getheaders(): - resp_headers[header.lower()] = value + resp_headers = resp_header_dict(resp) return resp_headers @@ -992,9 +1018,7 @@ def head_object(url, token, container, name, http_conn=None, http_host=conn.host, http_path=path, http_status=resp.status, http_reason=resp.reason, http_response_content=body) - resp_headers = {} - for header, value in resp.getheaders(): - resp_headers[header.lower()] = value + resp_headers = resp_header_dict(resp) return resp_headers @@ -1224,9 +1248,7 @@ def get_capabilities(http_conn): http_host=conn.host, http_path=parsed.path, http_status=resp.status, http_reason=resp.reason, http_response_content=body) - resp_headers = {} - for header, value in resp.getheaders(): - resp_headers[header.lower()] = value + resp_headers = resp_header_dict(resp) return parse_api_response(resp_headers, body) diff --git a/swiftclient/service.py b/swiftclient/service.py index 4f331c4e..129ca2f9 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -27,7 +27,7 @@ from six import StringIO, text_type from six.moves.queue import Queue from six.moves.queue import Empty as QueueEmpty -from six.moves.urllib.parse import quote, unquote +from six.moves.urllib.parse import quote from six import Iterator, string_types import json @@ -1861,8 +1861,7 @@ def _upload_object_job(self, conn, container, source, obj, options, delobjsmap = {} if old_manifest: scontainer, sprefix = old_manifest.split('/', 1) - scontainer = unquote(scontainer) - sprefix = unquote(sprefix).rstrip('/') + '/' + sprefix = sprefix.rstrip('/') + '/' delobjsmap[scontainer] = [] for part in self.list(scontainer, {'prefix': sprefix}): if not part["success"]: @@ -2054,8 +2053,7 @@ def _delete_object(self, conn, container, obj, options, dlo_segments_deleted = True segment_pool = self.thread_manager.segment_pool s_container, s_prefix = old_manifest.split('/', 1) - s_container = unquote(s_container) - s_prefix = unquote(s_prefix).rstrip('/') + '/' + s_prefix = s_prefix.rstrip('/') + '/' del_segs = [] for part in self.list( diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index 23b31388..28556295 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -130,6 +130,31 @@ def test_quote(self): value = u'unicode:\xe9\u20ac' self.assertEqual('unicode%3A%C3%A9%E2%82%AC', c.quote(value)) + def test_parse_header_string(self): + value = b'bytes' + self.assertEqual(u'bytes', c.parse_header_string(value)) + value = u'unicode:\xe9\u20ac' + self.assertEqual(u'unicode:\xe9\u20ac', c.parse_header_string(value)) + value = 'native%20string' + self.assertEqual(u'native string', c.parse_header_string(value)) + + value = b'encoded%20bytes%E2%82%AC' + self.assertEqual(u'encoded bytes\u20ac', c.parse_header_string(value)) + value = 'encoded%20unicode%E2%82%AC' + self.assertEqual(u'encoded unicode\u20ac', + c.parse_header_string(value)) + + value = b'bad%20bytes%ff%E2%82%AC' + self.assertEqual(u'bad%20bytes%ff%E2%82%AC', + c.parse_header_string(value)) + value = u'bad%20unicode%ff\u20ac' + self.assertEqual(u'bad%20unicode%ff\u20ac', + c.parse_header_string(value)) + + value = b'really%20bad\xffbytes' + self.assertEqual(u'really%2520bad%FFbytes', + c.parse_header_string(value)) + def test_http_connection(self): url = 'http://www.test.com' _junk, conn = c.http_connection(url) @@ -686,6 +711,18 @@ def test_request_headers(self): }), ]) + def test_response_headers(self): + c.http_connection = self.fake_http_connection( + 200, headers={'X-Utf-8-Header': b't%c3%a9st', + 'X-Non-Utf-8-Header': b'%ff', + 'X-Binary-Header': b'\xff'}) + conn = c.http_connection('http://www.test.com') + headers, data = c.get_object('url_is_irrelevant', 'TOKEN', + 'container', 'object', http_conn=conn) + self.assertEqual(u't\xe9st', headers.get('x-utf-8-header', '')) + self.assertEqual(u'%ff', headers.get('x-non-utf-8-header', '')) + self.assertEqual(u'%FF', headers.get('x-binary-header', '')) + def test_chunk_size_read_method(self): conn = c.Connection('http://auth.url/', 'some_user', 'some_key') with mock.patch('swiftclient.client.get_auth_1_0') as mock_get_auth: diff --git a/tests/unit/utils.py b/tests/unit/utils.py index ac9aefdb..a9759eb3 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -124,7 +124,7 @@ def getexpect(self): def getheaders(self): if self.headers: return self.headers.items() - headers = {'content-length': len(self.body), + headers = {'content-length': str(len(self.body)), 'content-type': 'x-application/test', 'x-timestamp': self.timestamp, 'last-modified': self.timestamp, @@ -132,7 +132,7 @@ def getheaders(self): 'etag': self.etag or '"%s"' % EMPTY_ETAG, 'x-works': 'yes', - 'x-account-container-count': 12345} + 'x-account-container-count': '12345'} if not self.timestamp: del headers['x-timestamp'] try: From 9ef4c97de2fef9e25292c8f346b6802129b68cec Mon Sep 17 00:00:00 2001 From: John Dickinson Date: Tue, 14 Jul 2015 20:50:16 -0700 Subject: [PATCH 035/454] do hand-curated authors/changelog files Change-Id: I1a264c9ce1d137b18a6dc62623a282e4a2fe839c --- .gitignore | 2 - .mailmap | 29 +++- AUTHORS | 103 +++++++++++++ ChangeLog | 418 +++++++++++++++++++++++++++++++++++++++++++++++++++++ setup.cfg | 4 + 5 files changed, 553 insertions(+), 3 deletions(-) create mode 100644 AUTHORS create mode 100644 ChangeLog diff --git a/.gitignore b/.gitignore index 2064ff3a..f2699820 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,3 @@ -AUTHORS -ChangeLog *.sw? dist/ .tox diff --git a/.mailmap b/.mailmap index 1f2d5673..4a6368f2 100644 --- a/.mailmap +++ b/.mailmap @@ -10,6 +10,7 @@ Michael Barton Michael Barton Mike Barton Clay Gerrard Clay Gerrard +Clay Gerrard Clay Gerrard clayg David Goetz David Goetz @@ -50,4 +51,30 @@ Tom Fifield Tom Fifield Sascha Peilicke Sascha Peilicke Zhenguo Niu Peter Portante -Christian Schwede +Christian Schwede +Christian Schwede +Constantine Peresypkin +Madhuri Kumari madhuri +Morgan Fainberg +Hua Zhang +Yummy Bian +Alistair Coles +Tong Li +Paul Luse +Yuan Zhou +Jola Mirecka +Ning Zhang +Mauro Stettler +Pawel Palucki +Guang Yee +Jing Liuqing +Lorcan Browne +Eohyung Lee +Harshit Chitalia +Richard Hawkins +Sarvesh Ranjan +Minwoo Bae Minwoo B +Jaivish Kothari +Michael Matur +Kazuhiro Miyahara +Alexandra Settle diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 00000000..65a82cb3 --- /dev/null +++ b/AUTHORS @@ -0,0 +1,103 @@ +Christian Berendt (berendt@b1-systems.de) +Luis de Bethencourt (luis@debethencourt.com) +Darrell Bishop (darrell@swiftstack.com) +Fabien Boucher (fabien.boucher@enovance.com) +Chmouel Boudjnah (chmouel@enovance.com) +Clark Boylan (clark.boylan@gmail.com) +Chris Buccella (chris.buccella@antallagon.com) +Tim Burke (tim.burke@gmail.com) +Clint Byrum (clint@fewbar.com) +Tristan Cacqueray (tristan.cacqueray@enovance.com) +Sergio Cazzolato (sergio.j.cazzolato@intel.com) +Mahati Chamarthy (mahati.chamarthy@gmail.com) +Ray Chen (oldsharp@163.com) +Taurus Cheung (Taurus.Cheung@harmonicinc.com) +Alistair Coles (alistair.coles@hp.com) +Ian Cordasco (ian.cordasco@rackspace.com) +Nick Craig-Wood (nick@craig-wood.com) +Sean Dague (sean@dague.net) +Zack M. Davis (zdavis@swiftstack.com) +John Dickinson (me@not.mn) +EdLeafe (ed@leafe.com) +Sahid Orentino Ferdjaoui (sahid.ferdjaoui@cloudwatt.com) +Flaper Fesp (flaper87@gmail.com) +Florent Flament (florent.flament-ext@cloudwatt.com) +Josh Gachnang (josh@pcsforeducation.com) +Alex Gaynor (alex.gaynor@gmail.com) +Martin Geisler (martin@geisler.net) +Anne Gentle (anne@openstack.org) +Clay Gerrard (clay.gerrard@gmail.com) +David Goetz (david.goetz@rackspace.com) +Thomas Goirand (thomas@goirand.fr) +Davide Guerri (davide.guerri@hp.com) +Romain Hardouin (romain_hardouin@yahoo.fr) +Steven Hardy (shardy@redhat.com) +Doug Hellmann (doug.hellmann@dreamhost.com) +Greg Holt (gholt@rackspace.com) +Charles Hsu (charles0126@gmail.com) +Kun Huang (gareth@unitedstack.com) +Matthieu Huin (mhu@enovance.com) +Andreas Jaeger (aj@suse.de) +OpenStack Jenkins (jenkins@openstack.org) +Vasyl Khomenko (vasiliyk@yahoo-inc.com) +Leah Klearman (lklrmn@gmail.com) +Jaivish Kothari (jaivish.kothari@nectechnologies.in) +Jakub Krajcovic (jakub.krajcovic@gmail.com) +David Kranz (david.kranz@qrclab.com) +Sushil Kumar (sushil.kumar2@globallogic.com) +Greg Lange (greglange@gmail.com) +Alexis Lee (alexisl@hp.com) +Tong Li (litong01@us.ibm.com) +Feng Liu (mefengliu23@gmail.com) +Jing Liuqing (jing.liuqing@99cloud.net) +Hemanth Makkapati (hemanth.makkapati@mailtrust.com) +Steve Martinelli (stevemar@ca.ibm.com) +Juan J. Martinez (juan@memset.com) +Donagh McCabe (donagh.mccabe@hp.com) +Ben McCann (ben@benmccann.com) +Andy McCrae (andy.mccrae@gmail.com) +Stuart McLaren (stuart.mclaren@hp.com) +Samuel Merritt (sam@swiftstack.com) +Jola Mirecka (jola.mirecka@hp.com) +Hiroshi Miura (miurahr@nttdata.co.jp) +Sam Morrison (sorrison@gmail.com) +Dirk Mueller (dirk@dmllr.de) +Zhenguo Niu (zhenguo@unitedstack.com) +Ondrej Novy (ondrej.novy@firma.seznam.cz) +Alessandro Pilotti (apilotti@cloudbasesolutions.com) +Alessandro Pilotti (ap@pilotti.it) +Stanislaw Pitucha (stanislaw.pitucha@hp.com) +Dan Prince (dprince@redhat.com) +Li Riqiang (lrqrun@gmail.com) +Hirokazu Sakata (h.sakata@staff.east.ntt.co.jp) +Christian Schwede (cschwede@redhat.com) +Mark Seger (Mark.Seger@hp.com) +Mark Seger (mark.seger@hp.com) +Chuck Short (chuck.short@canonical.com) +David Shrewsbury (shrewsbury.dave@gmail.com) +Pradeep Kumar Singh (pradeep.singh@nectechnologies.in) +Jeremy Stanley (fungi@yuggoth.org) +Victor Stinner (victor.stinner@enovance.com) +Jiří Suchomel (jsuchome@suse.cz) +YUZAWA Takahiko (yuzawataka@intellilink.co.jp) +Monty Taylor (mordred@inaugust.com) +TheSriram (sriram@klusterkloud.com) +Tihomir Trifonov (t.trifonov@gmail.com) +Dean Troyer (dtroyer@gmail.com) +Stanislav Vitkovskiy (stas.vitkovsky@gmail.com) +Daniel Wakefield (daniel.wakefield@hp.com) +Shane Wang (shane.wang@intel.com) +Mark Washenberger (mark.washenberger@rackspace.com) +Wu Wenxiang (wu.wenxiang@99cloud.net) +Mike Widman (mwidman@endurancewindpower.com) +Joel Wright (joel.wright@sohonet.com) +You Yamagata (bi.yamagata@gmail.com) +YangLei (yanglyy@cn.ibm.com) +Pete Zaitcev (zaitcev@kotori.zaitcev.us) +Jian Zhang (jian.zhang@intel.com) +Yuan Zhou (yuan.zhou@intel.com) +groqez (groqez@yopmail.net) +tanlin (lin.tan@intel.com) +yangxurong (yangxurong@huawei.com) +yuxcer (yuxcer@126.com) +zhang-jinnan (ben.os@99cloud.net) diff --git a/ChangeLog b/ChangeLog new file mode 100644 index 00000000..83d2865e --- /dev/null +++ b/ChangeLog @@ -0,0 +1,418 @@ +2.5.1 +----- + +* Several CLI options have learned short options. The usage strings have + been updated to reflect this. + +* CLI arguments are now always decoded as UTF-8. + +* Stop Connection class modifying os_options parameter. + +* Reduce memory usage for download/delete. + +* Added --no-shuffle option to the CLI download command. + +* The swift service API now logs and reports the traceback + on failed operations. + +* Increase httplib._MAXHEADERS to 256. + +* Add minimal working service token support to client.py. + +* Various other minor bug fixes and improvements. + + +2.5.0 +----- + +* The CLI learned an "auth" subcommand which returns bash environment + snippets for auth credentials. + +* The CLI --version option is now more explicit by calling itself + "python-swiftclient" rather than the name of the binary. + +* Now validates the checksum of each chunk of a large object as it is + uploaded. + +* Fixes uploading an object with a relative path. + +* Added the ability to download objects to a particular folder. + +* Now correctly removes all old segments of an object when replacing a + Dynamic Large Object (DLO). + +* The --skip-identical option now works properly when downloading + large objects. + +* The client.get_object() response learned a .read([length]) method. + +* Fixed an issue where an intermediate caching/proxy service could cause + object content to be improperly decoded. + +* Added a timeout parameter to HTTPConnection objects for socket-level + read timeouts. + +* Removed a dependency on simplejson. + +* Various other minor bug fixes and improvements. + +2.4.0 +----- + +* Mention --segment-size option after 413 response +* Add improvements to MD5 validation +* Unindent a chunk of st_list +* Release connection after consuming the content +* Verify MD5 of uploaded objects +* Fix crash with -l, -d /, and pseudo folders +* add functional tox target +* Fix crash when stat'ing objects with non-ascii names +* Add help message for " --help" +* Fix missing ca-certificate parameter to get_auth +* Fix deleting SLO segments on overwrite +* This patch fixes downloading files to stdout +* Fix environment sanitization for TestServiceUtils +* Fix cross account upload using --os-storage-url +* Change tests to use CaptureOutput class +* Print info message about incorrect --totals usage when neither -l nor --lh is provided. Added test coverage for --totals +* Make preauth params work +* Fix misplaced check for None in SwiftUploadObject +* Fix misnamed dictionary key +* Change tests to use new CaptureOutput class +* Workflow documentation is now in infra-manual +* Show warning when auth_version >= 2 and keystoneclient is missing +* Capture test output better +* Suppress 'No handlers...' message from keystoneclient logger +* Add unit tests for _encode_meta_headers +* Fix misnamed variable in SwiftReader +* Check that content_type header exists before using +* Adds user friendly message when --segment-size is a non-integer +* Make swift post output an error message when failing +* Replaces Stacktraces with useful error messages +* Fix KeyError raised from client Connection +* Fix race in shell when testing for errors to raise SysExit +* Fix race between container create jobs during upload +* Fix the info command with --insecure +* Allow segment size to be specified in a human readable way +* Use skipTest from testtools instead of inherited Exception +* Add tests for account listing using --lh switch +* Do not crash with "swift list --lh" for Ceph RadosGW + +2.3.1 +----- + +* Remove a debugging print statement +* Fix unit tests failing when OS_ env vars are set +* Fix bug with some OS options not being passed to client +* Add per policy container count to account stat output +* Stop creating extraneous directories + +2.3.0 +----- + +* Work toward Python 3.4 support and testing +* Add importable SwiftService incorporating shell.py logic +* Adds console script entry point +* Do not create an empty directory 'pseudo/' +* fixed unit tests when env vars are set +* Fix crash when downloading a pseudo-directory +* Clean up raw policy stats in account stat +* Update theme for docs +* Add a tox job for generating docs +* Add keystone v3 auth support + +2.2.0 +----- + +* Fix context sensitive help for info and tempurl +* Allow to specify storage policy when uploading objects +* Adding Swift Temporary URL support +* Add CONTRIBUTING.md +* Add context sensitive help +* Relax requirement for tenant_name in get_auth() +* replace string format arguments with function parameters +* Removed now unnecesary workaround for PyPy +* Use Emacs-friendly coding line +* Remove extra double quote from docstring +* Fix wrong assertions in unit tests +* fixed several pep8 issues + +2.1.0 +----- + +* Fix Python3 bugs +* Remove testtools.main() call from tests +* Move test_shell.py under tests/unit/ +* Mark swiftclient as being a universal wheel +* change assert_ to assertTrue +* change assertEquals to assertEqual +* Provide a link to the documentation to the README +* fixed typos found by RETF rules +* Fix running the unittests under py3 +* Add "." for help strings +* Declare that we support Python 3 +* Make the function tests Python3-import friendly +* Only encode metadata for user customed headers +* Add functional tests for python-swiftclient +* Removed a duplicate word in a dostring +* Mock auth_end_time in test_shell.test_download +* Don't utf8 encode urls +* Fixed several shell tests on Python3 +* Fix up StringIO use in tests for py3 +* Updated test_shell for Python3 +* Fix test_raw_upload test +* Remove validate_headers +* Use quote/unquote from six module for py3 +* Makes use of requests.Session +* Fix test_multithreading on Python 3 +* Add tests for bin/swift +* Fix swiftclient.client.quote() for Python 3 +* Add requests related unit-tests +* Update help message to specify unit of --segment-size option +* Python 3: fix tests on HTTP headers +* Updated from global requirements +* Use the standard library's copy of mock when it's available +* Replaced print statements with print function +* Removed usage of tuple unpacking in parameters +* don't use mutable defaults in kwargs +* set user-agent header +* Python 3: Get compatible types from six +* Python 3: Fix module names in import +* Python 3: Add six dependency +* Replace dict.iteritems() with dict.items() +* Python 3: Replace iter.next() with six.next(iter) +* Make bin/swift testable part 2 +* Make bin/swift testable part 1 +* Python 3: Fix tests using temporary text files +* Python 3: cast map() result to list +* Fix temporary pypy gate issue with setuptools +* Decode HTTP responses, fixes bug #1282861 +* Copy Swift's .mailmap to swiftclient repo +* Improve help strings +* TCP port is appended two time in ClientException +* add "info" as an alias to "capabilities" +* Use six.StringIO instead of StringIO.StringIO + +2.0.3 +----- + +* Help string format persistent +* Make the help strings constant +* Add LengthWrapper in put_object to honor content_length param +* Updated from global requirements +* Remove useless statement +* swift.1 manpage fix for groff warnings + +2.0.2 +----- + +* Remove multipart/form-data file upload + +2.0.1 +----- + +* Fix --insecure option on auth +* Only run flake8 on swiftclient code + +2.0 +--- + + +1.9.0 +----- + +* Remove extraneous vim configuration comments +* Rename Openstack to OpenStack +* Port to python-requests +* Add option to skip downloading/uploading identical files +* Remove tox locale overrides +* Fix swiftclient help +* Fix misspellings in python swiftclient +* changed things because reasons +* Add missing backslash +* match hacking rules in swift +* Updated from global requirements +* Install manpage in share/man/man1 instead of man/man1 +* assertEquals is deprecated, use assertEqual +* Add capabilities option +* Install swiftclient manpage +* Replace xrange in for loop with range +* Add --object-name +* retry on ratelimit +* Fix help of some optional arguments +* Updates tox.ini to use new features +* Fix Sphinx version issue +* Enable usage of proxies defined in environment (http(s)_proxy) +* Don't crash when header is value of None +* Fix download bandwidth for swift command +* Updates .gitignore +* Allow custom headers when using swift download (CLI) +* Replaced two references to Cloud Files with Swift +* Fix a typo in help text: "downlad" +* Add close to swiftclient.client.Connection +* enhance swiftclient logging +* Clarify main help for post subcommand +* Fixes python-swiftclient debugging message + +1.8.0 +----- + +* Make pbr only a build-time dependency +* Add verbose output to all stat commands +* assertEquals is deprecated, use assertEqual (H602) +* Skip sniffing and reseting if retry is disabled +* user defined headers added to swift post queries + +1.7.0 +----- + +* Sync with global requirements +* fix bug with replace old *LOs +* Extend usage message for `swift download` + +1.6.0 +----- + +* Added support for running the tests under PyPy with tox +* Remove redundant unit suffix +* Reformat help outputs +* Add a NullHandler when setting up library logging +* Assignment to reserved built-in symbol "file" +* Added headers argument support to get_object() +* Move multi-threading code to a library +* fix(gitignore) : Ignore *.egg files +* python3: Start of adding basic python3 support +* Added log statements in swift client +* Update docstring for swiftclient.Connection.__init__ +* Refuse carriage return in header value +* Adds max-backoff for retries in Connection +* Allow setting # of retries in the binary + +1.5.0 +----- + +* Note '-V 2' is necessary for auth 2.0 +* Allow storage url override for both auth vers +* Add *.swp into .gitignore +* Add -p option to download command +* add -t for totals to list command and --lh to stat +* add optional 'response_dict' parameters to many calls into which they'll return a dictionary of the response status, reason and headers +* Fixes re-auth flow with expired tokens +* Remove explicit distribute depend +* Add -l and --lh switches to swift 'list' command +* Changed the call to set_tunnel to work in python 2.6 or python 2.7 since its name changed between versions +* Add option to disable SSL compression +* python3: Introduce py33 to tox.ini +* Rename requires files to standard names +* remove busy-wait so that swift client won't use up all CPU cycles +* log get_auth request url instead of x-storage-url +* Update the man page +* Add .coveragerc file to show correct code coverage +* do not warn about etag for slo +* Eradicate eventlet and fix bug lp:959221 +* Add end_marker and path query parameters +* Switch to pbr for setup +* Switch to flake8 +* Improve Python 3.x compatibility +* Confirm we have auth creds before clearing preauth + +1.4.0 +----- + +* Improve auth option help +* Static large object support +* Fixed pep8 errors in test directory +* Allow user to specify headers at the command line +* Enhance put_object to inform when chunk is ignored +* Allow v2 to use storage_url/storage_token directly +* Add client man page swift.1 +* Allow to specify segment container +* Added "/" check when list containers +* Print useful message when keystoneclient is not installed +* Fix reporting version + +1.3.0 +----- + +* Use testr instead of nose +* Update to latest oslo version/setup +* Add generated files to .gitignore +* Add env[SWIFTCLIENT_INSECURE] +* Fix debug feature and add --debug to swift +* Use testtools as base class for test cases +* Add --os-cacert +* Add --insecure option to fix bug #1077869 +* Don't segment objects smaller than --segment-size +* Don't add trailing slash to auth URL +* Adding segment size as another x-object-manifest component +* Stop loss of precision when writing 'x-object-meta-mtime' +* Remove unused json_request +* fixed inconsistencies in parameter descriptions +* tell nose to explicity test the 'tests' directory +* Fixes setup compatibility issue on Windows +* Force utf-8 encode of HTTPConnection params +* swiftclient Connection : default optional arguments to None +* Add OpenStack trove classifier for PyPI +* Resolves issue with empty os_options for swift-bench & swift-dispersion-report +* Catch authorization failures +* Do not use dictionaries as default parameters + +1.2.0 +----- + +* Add region_name support +* Allow endpoint type to be specified +* PEP8 cleanup +* PEP8 issues fixed +* Add ability to download without writing to disk +* Fix PEP8 issues +* Change '_' to '-' in options +* Fix swiftclient 400 error when OS_AUTH_URL is set +* Add nosehtmloutput as a test dependency +* Shuffle download order (of containers and objects) +* Add timing stats to verbose download output +* Ensure Content-Length header when PUT/POST a container +* Make python-keystoneclient optional +* Fix container delete throughput and 409 retries +* Consume version info from pkg_resources +* Use keystoneclient for authentication +* Removes the title "Swift Web" from landing page + +1.1.1 +----- + +* Now url encodes/decodes x-object-manifest values +* Configurable concurrency for swift client +* Allow specify tenant:user in user +* Make swift exit on ctrl-c +* Add post-tag versioning +* Don't suppress openstack auth options +* Make swift not hang on error +* Fix pep8 errors w/pep8==1.3 +* Add missing test/tools files to the tarball +* Add build_sphinx options +* Make CLI exit nonzero on error +* Add doc and version in swiftclient.__init__.py +* Raise ClientException for invalid auth version +* Version bump after pypi release + +1.1.0 +----- + +* Removed now-unused .cache.bundle references +* Added setup.cfg for verbose test output +* Add run_tests.sh script here +* Adding fake_http_connect to test.utils +* Add openstack project infrastructure +* Add logging +* Defined version to 1.0 +* Add CHANGELOG LICENSE and MANIFEST.in +* Delete old test_client and add a gitignore +* Rename client to swiftclient +* Fix links +* Import script from swift to run unittests +* Add test_client from original swift repository +* Add AUTHORS file +* Make sure we get a header StorageURL with 1.0 +* Allow specify the tenant in user +* First commit diff --git a/setup.cfg b/setup.cfg index 32e05f17..3d97de06 100644 --- a/setup.cfg +++ b/setup.cfg @@ -46,3 +46,7 @@ upload-dir = doc/build/html [wheel] universal = 1 + +[pbr] +skip_authors = True +skip_changelog = True From 52d39bebc11979fa1be5090ff75466710638e561 Mon Sep 17 00:00:00 2001 From: "Zack M. Davis" Date: Fri, 4 Sep 2015 14:57:30 -0700 Subject: [PATCH 036/454] absolute expiry option for tempURL generation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `tempurl` subcommand's second positional argument is called `seconds` and has heretofore interpreted as the number of seconds for which the tempURL should be valid, counting from the moment of running the command. This is indeed a common, if not the most common, use-case. But some users, occasionally, might want to generate a tempURL that expires at some particular ("absolute") time, rather than a particular amount of time relative to the moment of happening to run the command. (One might make an analogy to the way in which Swift's expiring object support supports an `X-Delete-At` header in addition to `X-Delete-After`—and it's the former that must be regarded as ontologically prior.) Thus, this commit adds an `--absolute` optional argument to the `tempurl` subcommand; if present, the `seconds` argument will be interpreted as a Unix timestamp of when the tempURL should be expire, rather than a duration for which the tempURL should be valid starting from "now". Change-Id: If9ded96f2799800958d5063127f3de812f50ef06 --- doc/manpages/swift.1 | 12 +++++++----- swiftclient/shell.py | 20 +++++++++++++++++--- swiftclient/utils.py | 9 ++++++--- tests/unit/test_shell.py | 16 +++++++++++----- tests/unit/test_utils.py | 15 +++++++++++---- 5 files changed, 52 insertions(+), 20 deletions(-) diff --git a/doc/manpages/swift.1 b/doc/manpages/swift.1 index 446ade49..4cfc23fd 100644 --- a/doc/manpages/swift.1 +++ b/doc/manpages/swift.1 @@ -104,12 +104,14 @@ is not provided the storage-url retrieved after authentication is used as proxy-url. .RE -\fBtempurl\fR method seconds path key +\fBtempurl\fR \fImethod\fR \fIseconds\fR \fIpath\fR \fIkey\fR [\fI--absolute\fR] .RS 4 -Generates a temporary URL allowing unauthenticated access to the Swift object at -the given path, using the given HTTP method, for the given number of seconds, -using the given TempURL key. \fBExample\fR: tempurl GET 86400 -/v1/AUTH_foo/bar_container/quux.md my_secret_tempurl_key +Generates a temporary URL allowing unauthenticated access to the Swift object +at the given path, using the given HTTP method, for the given number of +seconds, using the given TempURL key. If optional --absolute argument is +provided, seconds is instead interpreted as a Unix timestamp at which the URL +should expire. \fBExample\fR: tempurl GET $(date -d "Jan 1 2016" +%s) +/v1/AUTH_foo/bar_container/quux.md my_secret_tempurl_key --absolute .RE .SH OPTIONS diff --git a/swiftclient/shell.py b/swiftclient/shell.py index d908d667..652980f3 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -1013,17 +1013,30 @@ def st_auth(parser, args, thread_manager): Positional arguments: An HTTP method to allow for this temporary URL. Usually 'GET' or 'PUT'. - The amount of time in seconds the temporary URL will - be valid for. + The amount of time in seconds the temporary URL will be + valid for; or, if --absolute is passed, the Unix + timestamp when the temporary URL will expire. The full path to the Swift object. Example: /v1/AUTH_account/c/o. The secret temporary URL key set on the Swift cluster. To set a key, run \'swift post -m "Temp-URL-Key:b3968d0207b54ece87cccc06515a89d4"\' + +Optional arguments: + --absolute Interpet the positional argument as a Unix + timestamp rather than a number of seconds in the + future. '''.strip('\n') def st_tempurl(parser, args, thread_manager): + parser.add_option( + '--absolute', action='store_true', + dest='absolute_expiry', default=False, + help=("If present, seconds argument will be interpreted as a Unix " + "timestamp representing when the tempURL should expire, rather " + "than an offset from the current time") + ) (options, args) = parse_args(parser, args) args = args[1:] if len(args) < 4: @@ -1040,7 +1053,8 @@ def st_tempurl(parser, args, thread_manager): thread_manager.print_msg('WARNING: Non default HTTP method %s for ' 'tempurl specified, possibly an error' % method.upper()) - url = generate_temp_url(path, seconds, key, method) + url = generate_temp_url(path, seconds, key, method, + absolute=options.absolute_expiry) thread_manager.print_msg(url) diff --git a/swiftclient/utils.py b/swiftclient/utils.py index 6ff62594..8316a8f8 100644 --- a/swiftclient/utils.py +++ b/swiftclient/utils.py @@ -65,8 +65,8 @@ def prt_bytes(bytes, human_flag): return bytes -def generate_temp_url(path, seconds, key, method): - """ Generates a temporary URL that gives unauthenticated access to the +def generate_temp_url(path, seconds, key, method, absolute=False): + """Generates a temporary URL that gives unauthenticated access to the Swift object. :param path: The full path to the Swift object. Example: @@ -85,7 +85,10 @@ def generate_temp_url(path, seconds, key, method): if seconds < 0: raise ValueError('seconds must be a positive integer') try: - expiration = int(time.time() + seconds) + if not absolute: + expiration = int(time.time() + seconds) + else: + expiration = int(seconds) except TypeError: raise TypeError('seconds must be an integer') diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index 12ceadbb..e2b87d0f 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -922,15 +922,21 @@ def test_post_object_too_many_args(self): self.assertTrue(output.err != '') self.assertTrue(output.err.startswith('Usage')) - @mock.patch('swiftclient.shell.generate_temp_url') + @mock.patch('swiftclient.shell.generate_temp_url', return_value='') def test_temp_url(self, temp_url): argv = ["", "tempurl", "GET", "60", "/v1/AUTH_account/c/o", - "secret_key" - ] - temp_url.return_value = "" + "secret_key"] + swiftclient.shell.main(argv) + temp_url.assert_called_with( + '/v1/AUTH_account/c/o', 60, 'secret_key', 'GET', absolute=False) + + @mock.patch('swiftclient.shell.generate_temp_url', return_value='') + def test_absolute_expiry_temp_url(self, temp_url): + argv = ["", "tempurl", "GET", "60", "/v1/AUTH_account/c/o", + "secret_key", "--absolute"] swiftclient.shell.main(argv) temp_url.assert_called_with( - '/v1/AUTH_account/c/o', 60, 'secret_key', 'GET') + '/v1/AUTH_account/c/o', 60, 'secret_key', 'GET', absolute=True) @mock.patch('swiftclient.service.Connection') def test_capabilities(self, connection): diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index ca3531e1..7d7f6b67 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -132,11 +132,9 @@ def setUp(self): self.key = 'correcthorsebatterystaple' self.method = 'GET' - @mock.patch('hmac.HMAC.hexdigest') - @mock.patch('time.time') + @mock.patch('hmac.HMAC.hexdigest', return_value='temp_url_signature') + @mock.patch('time.time', return_value=1400000000) def test_generate_temp_url(self, time_mock, hmac_mock): - time_mock.return_value = 1400000000 - hmac_mock.return_value = 'temp_url_signature' expected_url = ( '/v1/AUTH_account/c/o?' 'temp_url_sig=temp_url_signature&' @@ -145,6 +143,15 @@ def test_generate_temp_url(self, time_mock, hmac_mock): self.method) self.assertEqual(url, expected_url) + @mock.patch('hmac.HMAC.hexdigest', return_value="temp_url_signature") + def test_generate_absolute_expiry_temp_url(self, hmac_mock): + expected_url = ('/v1/AUTH_account/c/o?' + 'temp_url_sig=temp_url_signature&' + 'temp_url_expires=2146636800') + url = u.generate_temp_url(self.url, 2146636800, self.key, self.method, + absolute=True) + self.assertEqual(url, expected_url) + def test_generate_temp_url_bad_seconds(self): self.assertRaises(TypeError, u.generate_temp_url, From 5ae4b42392dfabe366531d2545629947aad76bba Mon Sep 17 00:00:00 2001 From: "Zack M. Davis" Date: Fri, 4 Sep 2015 15:45:48 -0700 Subject: [PATCH 037/454] make ClientException.http_status default to None rather than 0 The extant default of zero is a bit counterintuitive; insufficiently-careful programmers using swiftclient in their application might, without carefully reading the source or documentation, write buggy code based on the assumption that the `http_status` attribute is absent or defaults to None if ClientException is raised for reasons other than to indicate an unsuccessful HTTP request. (However improbable this scenario may seem, the present author can sadly attest to it having actually happened at least once.) Just changing the default would break some tests on Python 3, due to the `500 <= err.http_status <= 599` comparison in Connection's _retry method (NoneType and int are not orderable in the Python 3.x series); thus, the case where http_status is None is explicitly folded into a code branch that logs and reraises (whereas previously it would have fallen through to an `else` branch where it would be logged and reraised just the same). While we're here, we might as well make ClientException's __init__ use super() (although admittedly the kinds of multiple-inheritance scenarios in which `super` truly shines seem unlikely to occur here). Change-Id: I8c02bfb4a0ef059e781be5e08fcde13fb1be5b88 --- swiftclient/client.py | 2 +- swiftclient/exceptions.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index 8466cc5b..e0d35a8e 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -1400,7 +1400,7 @@ def _retry(self, reset_func, func, *args, **kwargs): self.http_conn = None except ClientException as err: self._add_response_dict(caller_response_dict, kwargs) - if self.attempts > self.retries: + if self.attempts > self.retries or err.http_status is None: logger.exception(err) raise if err.http_status == 401: diff --git a/swiftclient/exceptions.py b/swiftclient/exceptions.py index 9a776727..370a8d0f 100644 --- a/swiftclient/exceptions.py +++ b/swiftclient/exceptions.py @@ -17,9 +17,9 @@ class ClientException(Exception): def __init__(self, msg, http_scheme='', http_host='', http_port='', - http_path='', http_query='', http_status=0, http_reason='', + http_path='', http_query='', http_status=None, http_reason='', http_device='', http_response_content=''): - Exception.__init__(self, msg) + super(ClientException, self).__init__(msg) self.msg = msg self.http_scheme = http_scheme self.http_host = http_host From ee8c1bab9873c0bf87a12ebebc4a12c233586687 Mon Sep 17 00:00:00 2001 From: Charles Hsu Date: Mon, 17 Aug 2015 17:06:44 +0800 Subject: [PATCH 038/454] Convert http response(byte string) to string in python3. Avoid a TypeError exception in python3. Change-Id: I4039e3f2a88b5f681288b5ca8dd5c63c13b7764f Closes-bug: #1457012 --- swiftclient/shell.py | 3 ++- tests/unit/test_shell.py | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/swiftclient/shell.py b/swiftclient/shell.py index f2388fcf..ed026b8d 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -875,7 +875,8 @@ def st_upload(parser, args, output_manager): if error.http_response_content: if msg: msg += ': ' - msg += error.http_response_content[:60] + msg += (error.http_response_content + .decode('utf8')[:60]) msg = ': %s' % msg else: msg = ': %s' % error diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index 4ac2f5bb..34e32066 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -467,6 +467,27 @@ def test_upload(self, connection, walk): 'x-object-meta-mtime': mock.ANY}, response_dict={}) + @mock.patch('swiftclient.service.SwiftService.upload') + def test_upload_object_with_account_readonly(self, upload): + argv = ["", "upload", "container", self.tmpfile] + upload.return_value = [ + {"success": False, + "headers": {}, + "action": 'create_container', + "error": swiftclient.ClientException( + 'Container PUT failed', + http_status=403, + http_reason='Forbidden', + http_response_content=b'

Forbidden

') + }] + + with CaptureOutput() as output: + swiftclient.shell.main(argv) + self.assertTrue(output.err != '') + warning_msg = "Warning: failed to create container 'container': " \ + "403 Forbidden" + self.assertTrue(output.err.startswith(warning_msg)) + @mock.patch('swiftclient.service.Connection') def test_upload_delete_slo_segments(self, connection): # Upload delete existing segments From 7cd2a01cda4ac592dee465d4acf8a0b246354328 Mon Sep 17 00:00:00 2001 From: John Dickinson Date: Mon, 7 Sep 2015 08:20:14 -0700 Subject: [PATCH 039/454] updated changelog for 2.6.0 release Instead of a 2.5.1 release, add in the absolute tempurl option to the release and bump it to 2.6.0 Change-Id: Ie8335a737aac3211a240c25d88501f8f5dbccbea --- ChangeLog | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/ChangeLog b/ChangeLog index 83d2865e..1602ba4b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,23 +1,28 @@ -2.5.1 +2.6.0 ----- * Several CLI options have learned short options. The usage strings have been updated to reflect this. +* Added --no-shuffle option to the CLI download command. + +* Added --absolute option for CLI TempURL generation and the corresponding + parameter to utils.generate_temp_url(). This allows for an exact, specific + time to be used for the TempURL expiry time. + * CLI arguments are now always decoded as UTF-8. * Stop Connection class modifying os_options parameter. * Reduce memory usage for download/delete. -* Added --no-shuffle option to the CLI download command. - * The swift service API now logs and reports the traceback on failed operations. -* Increase httplib._MAXHEADERS to 256. +* Increase httplib._MAXHEADERS to 256 to work around header limits in recent + Python releases. -* Add minimal working service token support to client.py. +* Added minimal working service token support to client.py. * Various other minor bug fixes and improvements. From 1841bd6010e91859a4fe97afa4da980b8daa1e03 Mon Sep 17 00:00:00 2001 From: Alistair Coles Date: Tue, 8 Sep 2015 10:22:32 +0100 Subject: [PATCH 040/454] Initialise delete_object mock before it is called Attempt to fix the linked bug by initialising the mock instance for Connection.delete_object before calling the SwiftService upload method, so that the delete_object mock already exists before the delete_segments jobs that run in multiple threads call it. Otherwise there is a risk that the delete_segment job threads could race while creating either the delete_object mock or the delete_object.return_value mock, resulting in each thread getting a different instance. That would explain the intermittent test failures reported in the bug. Change-Id: Ia82697c093529076b0bbcc6bccd577afdf0839e1 Partial-Bug: #1480223 --- tests/unit/test_shell.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index e2b87d0f..66bf3284 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -538,6 +538,9 @@ def test_upload_delete_slo_segments(self, connection): b' {"name": "container2/old_seg2"}]' ) connection.return_value.put_object.return_value = EMPTY_ETAG + # create the delete_object child mock here in attempt to fix + # https://bugs.launchpad.net/python-swiftclient/+bug/1480223 + connection.return_value.delete_object.return_value = None swiftclient.shell.main(argv) connection.return_value.put_object.assert_called_with( 'container', @@ -604,9 +607,11 @@ def test_upload_delete_dlo_segments(self, connection): 'last_modified': '123T456'}]], [None, []] ] - connection.return_value.put_object.return_value = ( - 'd41d8cd98f00b204e9800998ecf8427e') + connection.return_value.put_object.return_value = EMPTY_ETAG swiftclient.shell.main(argv) + # create the delete_object child mock here in attempt to fix + # https://bugs.launchpad.net/python-swiftclient/+bug/1480223 + connection.return_value.delete_object.return_value = None connection.return_value.put_object.assert_called_with( 'container', self.tmpfile.lstrip('/'), From a2c84e0c937c49557f905b018eddac9ba82ea2dd Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Tue, 8 Sep 2015 14:37:05 +0000 Subject: [PATCH 041/454] Add links for release notes tool The automated release note tool expects to find links to the bug tracker, documentation, and source using a specific regex. This change adds the links using the expected format so they are found and included in the release announcements. Change-Id: I5fa24f60c7d705593996194b865ffb2c47870808 --- README.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.rst b/README.rst index a755e250..c4fb8df1 100644 --- a/README.rst +++ b/README.rst @@ -4,21 +4,21 @@ Python bindings to the OpenStack Object Storage API This is a python client for the Swift API. There's a Python API (the ``swiftclient`` module), and a command-line script (``swift``). -You can find the `documentation online`__. - -__ http://docs.openstack.org/developer/python-swiftclient/ - Development takes place via the usual OpenStack processes as outlined -in the `OpenStack wiki`__. The master repository is on GitHub__. +in the `OpenStack wiki`__. __ http://docs.openstack.org/infra/manual/developers.html -__ http://github.com/openstack/python-swiftclient This code is based on original the client previously included with -`OpenStack's swift`__ The python-swiftclient is licensed under the +`OpenStack's Swift`__ The python-swiftclient is licensed under the Apache License like the rest of OpenStack. __ http://github.com/openstack/swift +* Free software: Apache license +* Documentation: http://docs.openstack.org/developer/python-swiftclient/ +* Source: http://git.openstack.org/cgit/openstack/python-swiftclient/ +* Bugs: http://bugs.launchpad.net/python-swiftclient + .. contents:: Contents: :local: From 6b3638ecec3049023420dce16f06549585e2fb8b Mon Sep 17 00:00:00 2001 From: "Zack M. Davis" Date: Thu, 10 Sep 2015 14:37:32 -0700 Subject: [PATCH 042/454] enable autodocumentation for utils module; docstring fixes This commit adds the utils module to those for which Sphinx automatically generates documentation from docstrings. (Many of the functions here may be of little interest to users, but `generate_temp_url`, at least, definitely deserves to be in the documentation; in this way, this commit can be seen as a spiritual companion to ca70dd9e.) Also, a few markup errors and perceived infelicities in existing docstrings are amended. Change-Id: I8c66a23cb359d7dd9302a16459fad9825fedb690 --- doc/source/swiftclient.rst | 5 +++++ swiftclient/client.py | 2 +- swiftclient/service.py | 8 ++++---- swiftclient/utils.py | 18 +++++++++--------- 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/doc/source/swiftclient.rst b/doc/source/swiftclient.rst index 0a074713..e96afba5 100644 --- a/doc/source/swiftclient.rst +++ b/doc/source/swiftclient.rst @@ -24,3 +24,8 @@ swiftclient.multithreading ========================== .. automodule:: swiftclient.multithreading + +swiftclient.utils +================= + +.. automodule:: swiftclient.utils diff --git a/swiftclient/client.py b/swiftclient/client.py index 8466cc5b..18449149 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -1011,7 +1011,7 @@ def put_object(url, token=None, container=None, name=None, contents=None, container name is expected to be part of the url :param name: object name to put; if None, the object name is expected to be part of the url - :param contents: a string, a file like object or an iterable + :param contents: a string, a file-like object or an iterable to read object data from; if None, a zero-byte put will be done :param content_length: value to send as content-length header; also limits diff --git a/swiftclient/service.py b/swiftclient/service.py index c013b902..5d16381d 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -1298,10 +1298,10 @@ def upload(self, container, objects, options=None): The SwiftUploadObject source may be one of: - file - A file like object (with a read method) - path - A string containing the path to a local file - or directory - None - Indicates that we want an empty object + * A file-like object (with a read method) + * A string containing the path to a local + file or directory + * None, to indicate that we want an empty object :param options: A dictionary containing options to override the global options specified during the service object creation. diff --git a/swiftclient/utils.py b/swiftclient/utils.py index 8316a8f8..05e7f500 100644 --- a/swiftclient/utils.py +++ b/swiftclient/utils.py @@ -70,14 +70,14 @@ def generate_temp_url(path, seconds, key, method, absolute=False): Swift object. :param path: The full path to the Swift object. Example: - /v1/AUTH_account/c/o. + /v1/AUTH_account/c/o. :param seconds: The amount of time in seconds the temporary URL will - be valid for. - :param key: The secret temporary URL key set on the Swift cluster. - To set a key, run 'swift post -m - "Temp-URL-Key:b3968d0207b54ece87cccc06515a89d4"' - :param method: A HTTP method, typically either GET or PUT, to allow for - this temporary URL. + be valid for. + :param key: The secret temporary URL key set on the Swift + cluster. To set a key, run 'swift post -m + "Temp-URL-Key: "' + :param method: A HTTP method, typically either GET or PUT, to allow + for this temporary URL. :raises: ValueError if seconds is not a positive integer :raises: TypeError if seconds is not an integer :return: the path portion of a temporary URL @@ -152,7 +152,7 @@ class ReadableToIterable(object): Wrap a filelike object and act as an iterator. It is recommended to use this class only on files opened in binary mode. - Due to the Unicode changes in python 3 files are now opened using an + Due to the Unicode changes in Python 3, files are now opened using an encoding not suitable for use with the md5 class and because of this hit the exception on every call to next. This could cause problems, especially with large files and small chunk sizes. @@ -200,7 +200,7 @@ class LengthWrapper(object): """ Wrap a filelike object with a maximum length. - Fix for https://github.com/kennethreitz/requests/issues/1648 + Fix for https://github.com/kennethreitz/requests/issues/1648. It is recommended to use this class only on files opened in binary mode. """ def __init__(self, readable, length, md5=False): From f65641c44da8f16194aae1a95b58150bb09167ae Mon Sep 17 00:00:00 2001 From: Qiu Yu Date: Sat, 12 Sep 2015 03:07:36 +0800 Subject: [PATCH 043/454] Suppress iso8601 logging from --debug output This change silences logging from iso8601 when --debug option set Change-Id: Ib8b8423012d43ef78d7138609fa98f40d46e7d4b Closes-bug: #1324470 --- swiftclient/shell.py | 1 + 1 file changed, 1 insertion(+) diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 652980f3..21c4b78b 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -1414,6 +1414,7 @@ def main(arguments=None): logging.getLogger("swiftclient") if options.debug: logging.basicConfig(level=logging.DEBUG) + logging.getLogger('iso8601').setLevel(logging.WARNING) elif options.info: logging.basicConfig(level=logging.INFO) From 7cb99d3157f24e81737463302c937f1c251b7084 Mon Sep 17 00:00:00 2001 From: Mahati Date: Tue, 15 Sep 2015 14:34:53 +0530 Subject: [PATCH 044/454] Add headers parameter Headers parameter is required when passing client key for encryption. It is missing for get_container and head_object. Change-Id: I35c3b266b3c733f6b1629de4c683ea7d40128032 --- swiftclient/client.py | 25 +++++++++++++++++-------- tests/unit/test_swiftclient.py | 27 +++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index e0d35a8e..1913ea36 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -638,7 +638,7 @@ def post_account(url, token, headers, http_conn=None, response_dict=None, def get_container(url, token, container, marker=None, limit=None, prefix=None, delimiter=None, end_marker=None, path=None, http_conn=None, - full_listing=False, service_token=None): + full_listing=False, service_token=None, headers=None): """ Get a listing of objects for the container. @@ -662,10 +662,15 @@ def get_container(url, token, container, marker=None, limit=None, """ if not http_conn: http_conn = http_connection(url) + if headers: + headers = dict(headers) + else: + headers = {} + headers['X-Auth-Token'] = token if full_listing: rv = get_container(url, token, container, marker, limit, prefix, delimiter, end_marker, path, http_conn, - service_token) + service_token, headers=headers) listing = rv[1] while listing: if not delimiter: @@ -674,7 +679,8 @@ def get_container(url, token, container, marker=None, limit=None, marker = listing[-1].get('name', listing[-1].get('subdir')) listing = get_container(url, token, container, marker, limit, prefix, delimiter, end_marker, path, - http_conn, service_token)[1] + http_conn, service_token, + headers=headers)[1] if listing: rv[1].extend(listing) return rv @@ -693,7 +699,6 @@ def get_container(url, token, container, marker=None, limit=None, qs += '&end_marker=%s' % quote(end_marker) if path: qs += '&path=%s' % quote(path) - headers = {'X-Auth-Token': token} if service_token: headers['X-Service-Token'] = service_token method = 'GET' @@ -958,7 +963,7 @@ def get_object(url, token, container, name, http_conn=None, def head_object(url, token, container, name, http_conn=None, - service_token=None): + service_token=None, headers=None): """ Get object info @@ -978,8 +983,12 @@ def head_object(url, token, container, name, http_conn=None, else: parsed, conn = http_connection(url) path = '%s/%s/%s' % (parsed.path, quote(container), quote(name)) + if headers: + headers = dict(headers) + else: + headers = {} + headers['X-Auth-Token'] = token method = 'HEAD' - headers = {'X-Auth-Token': token} if service_token: headers['X-Service-Token'] = service_token conn.request(method, path, '', headers) @@ -1450,7 +1459,7 @@ def head_container(self, container): def get_container(self, container, marker=None, limit=None, prefix=None, delimiter=None, end_marker=None, path=None, - full_listing=False): + full_listing=False, headers=None): """Wrapper for :func:`get_container`""" # TODO(unknown): With full_listing=True this will restart the entire # listing with each retry. Need to make a better version that just @@ -1458,7 +1467,7 @@ def get_container(self, container, marker=None, limit=None, prefix=None, return self._retry(None, get_container, container, marker=marker, limit=limit, prefix=prefix, delimiter=delimiter, end_marker=end_marker, path=path, - full_listing=full_listing) + full_listing=full_listing, headers=headers) def put_container(self, container, headers=None, response_dict=None): """Wrapper for :func:`put_container`""" diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index 23b31388..410fd6f5 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -600,6 +600,20 @@ def test_param_path(self): c.get_container('http://www.test.com', 'asdf', 'asdf', path='asdf') + def test_request_headers(self): + c.http_connection = self.fake_http_connection( + 204, query_string="format=json") + conn = c.http_connection('http://www.test.com') + headers = {'x-client-key': 'client key'} + c.get_container('url_is_irrelevant', 'TOKEN', 'container', + http_conn=conn, headers=headers) + self.assertRequests([ + ('GET', '/container?format=json', '', { + 'x-auth-token': 'TOKEN', + 'x-client-key': 'client key', + }), + ]) + class TestHeadContainer(MockHttpTest): @@ -729,6 +743,19 @@ def test_server_error(self): self.assertRaises(c.ClientException, c.head_object, 'http://www.test.com', 'asdf', 'asdf', 'asdf') + def test_request_headers(self): + c.http_connection = self.fake_http_connection(204) + conn = c.http_connection('http://www.test.com') + headers = {'x-client-key': 'client key'} + c.head_object('url_is_irrelevant', 'TOKEN', 'container', + 'asdf', http_conn=conn, headers=headers) + self.assertRequests([ + ('HEAD', '/container/asdf', '', { + 'x-auth-token': 'TOKEN', + 'x-client-key': 'client key', + }), + ]) + class TestPutObject(MockHttpTest): From ccc3aa49ee654286c169afce558455cfb9f2d4ba Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Mon, 21 Sep 2015 14:54:35 +0000 Subject: [PATCH 045/454] Change ignore-errors to ignore_errors Needed for coverage 4.0 Change-Id: I2789541f67d6adbdc8454183753832488fe2bbe2 --- .coveragerc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.coveragerc b/.coveragerc index 2bf44866..2f4084ac 100644 --- a/.coveragerc +++ b/.coveragerc @@ -4,4 +4,4 @@ source = swiftclient omit = swiftclient/openstack/common/* [report] -ignore-errors = True +ignore_errors = True From 305cd6253fc4144e3afece6f39aaaabf68ecb160 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Tue, 22 Sep 2015 12:14:18 -0700 Subject: [PATCH 046/454] Actually make assertions when testing get_account and get_container Change-Id: Ibb1301b00d1bc99ec089ead02f944aa94972120a --- tests/unit/test_swiftclient.py | 103 +++++++++++++++++++++++++-------- tests/unit/utils.py | 16 ++++- 2 files changed, 95 insertions(+), 24 deletions(-) diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index 410fd6f5..3272285f 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -499,32 +499,53 @@ class TestGetAccount(MockHttpTest): def test_no_content(self): c.http_connection = self.fake_http_connection(204) - value = c.get_account('http://www.test.com', 'asdf')[1] + value = c.get_account('http://www.test.com/v1/acct', 'asdf')[1] self.assertEqual(value, []) + self.assertRequests([ + ('GET', '/v1/acct?format=json', '', { + 'x-auth-token': 'asdf'}), + ]) def test_param_marker(self): c.http_connection = self.fake_http_connection( 204, query_string="format=json&marker=marker") - c.get_account('http://www.test.com', 'asdf', marker='marker') + c.get_account('http://www.test.com/v1/acct', 'asdf', marker='marker') + self.assertRequests([ + ('GET', '/v1/acct?format=json&marker=marker', '', { + 'x-auth-token': 'asdf'}), + ]) def test_param_limit(self): c.http_connection = self.fake_http_connection( 204, query_string="format=json&limit=10") - c.get_account('http://www.test.com', 'asdf', limit=10) + c.get_account('http://www.test.com/v1/acct', 'asdf', limit=10) + self.assertRequests([ + ('GET', '/v1/acct?format=json&limit=10', '', { + 'x-auth-token': 'asdf'}), + ]) def test_param_prefix(self): c.http_connection = self.fake_http_connection( 204, query_string="format=json&prefix=asdf/") - c.get_account('http://www.test.com', 'asdf', prefix='asdf/') + c.get_account('http://www.test.com/v1/acct', 'asdf', prefix='asdf/') + self.assertRequests([ + ('GET', '/v1/acct?format=json&prefix=asdf/', '', { + 'x-auth-token': 'asdf'}), + ]) def test_param_end_marker(self): c.http_connection = self.fake_http_connection( 204, query_string="format=json&end_marker=end_marker") - c.get_account('http://www.test.com', 'asdf', end_marker='end_marker') + c.get_account('http://www.test.com/v1/acct', 'asdf', + end_marker='end_marker') + self.assertRequests([ + ('GET', '/v1/acct?format=json&end_marker=end_marker', '', { + 'x-auth-token': 'asdf'}), + ]) class TestHeadAccount(MockHttpTest): @@ -559,46 +580,79 @@ class TestGetContainer(MockHttpTest): def test_no_content(self): c.http_connection = self.fake_http_connection(204) - value = c.get_container('http://www.test.com', 'asdf', 'asdf')[1] + value = c.get_container('http://www.test.com/v1/acct', 'token', + 'container')[1] self.assertEqual(value, []) + self.assertRequests([ + ('GET', '/v1/acct/container?format=json', '', { + 'x-auth-token': 'token'}), + ]) def test_param_marker(self): c.http_connection = self.fake_http_connection( 204, query_string="format=json&marker=marker") - c.get_container('http://www.test.com', 'asdf', 'asdf', marker='marker') + c.get_container('http://www.test.com/v1/acct', 'token', 'container', + marker='marker') + self.assertRequests([ + ('GET', '/v1/acct/container?format=json&marker=marker', '', { + 'x-auth-token': 'token'}), + ]) def test_param_limit(self): c.http_connection = self.fake_http_connection( 204, query_string="format=json&limit=10") - c.get_container('http://www.test.com', 'asdf', 'asdf', limit=10) + c.get_container('http://www.test.com/v1/acct', 'token', 'container', + limit=10) + self.assertRequests([ + ('GET', '/v1/acct/container?format=json&limit=10', '', { + 'x-auth-token': 'token'}), + ]) def test_param_prefix(self): c.http_connection = self.fake_http_connection( 204, query_string="format=json&prefix=asdf/") - c.get_container('http://www.test.com', 'asdf', 'asdf', prefix='asdf/') + c.get_container('http://www.test.com/v1/acct', 'token', 'container', + prefix='asdf/') + self.assertRequests([ + ('GET', '/v1/acct/container?format=json&prefix=asdf/', '', { + 'x-auth-token': 'token'}), + ]) def test_param_delimiter(self): c.http_connection = self.fake_http_connection( 204, query_string="format=json&delimiter=/") - c.get_container('http://www.test.com', 'asdf', 'asdf', delimiter='/') + c.get_container('http://www.test.com/v1/acct', 'token', 'container', + delimiter='/') + self.assertRequests([ + ('GET', '/v1/acct/container?format=json&delimiter=/', '', { + 'x-auth-token': 'token'}), + ]) def test_param_end_marker(self): c.http_connection = self.fake_http_connection( 204, query_string="format=json&end_marker=end_marker") - c.get_container('http://www.test.com', 'asdf', 'asdf', + c.get_container('http://www.test.com/v1/acct', 'token', 'container', end_marker='end_marker') + self.assertRequests([ + ('GET', '/v1/acct/container?format=json&end_marker=end_marker', + '', {'x-auth-token': 'token'}), + ]) def test_param_path(self): c.http_connection = self.fake_http_connection( 204, query_string="format=json&path=asdf") - c.get_container('http://www.test.com', 'asdf', 'asdf', + c.get_container('http://www.test.com/v1/acct', 'token', 'container', path='asdf') + self.assertRequests([ + ('GET', '/v1/acct/container?format=json&path=asdf', '', { + 'x-auth-token': 'token'}), + ]) def test_request_headers(self): c.http_connection = self.fake_http_connection( @@ -656,7 +710,9 @@ def test_server_error(self): 'http://www.test.com', 'token', 'container') self.assertEqual(e.http_response_content, body) self.assertRequests([ - ('PUT', '/container', '', {'x-auth-token': 'token'}), + ('PUT', '/container', '', { + 'x-auth-token': 'token', + 'content-length': '0'}), ]) @@ -680,12 +736,10 @@ def test_query_string(self): query_string="hello=20") c.get_object('http://www.test.com', 'asdf', 'asdf', 'asdf', query_string="hello=20") - for req in self.iter_request_log(): - self.assertEqual(req['method'], 'GET') - self.assertEqual(req['parsed_path'].path, '/asdf/asdf') - self.assertEqual(req['parsed_path'].query, 'hello=20') - self.assertEqual(req['body'], '') - self.assertEqual(req['headers']['x-auth-token'], 'asdf') + self.assertRequests([ + ('GET', '/asdf/asdf?hello=20', '', { + 'x-auth-token': 'asdf'}), + ]) def test_request_headers(self): c.http_connection = self.fake_http_connection(200) @@ -816,7 +870,8 @@ def test_server_error(self): self.assertEqual(e.http_response_content, body) self.assertEqual(e.http_status, 500) self.assertRequests([ - ('PUT', '/asdf/asdf', 'asdf', {'x-auth-token': 'asdf'}), + ('PUT', '/asdf/asdf', 'asdf', { + 'x-auth-token': 'asdf', 'content-type': ''}), ]) def test_query_string(self): @@ -1022,7 +1077,7 @@ def test_ok(self): http_conn = conn('http://www.test.com/info') info = c.get_capabilities(http_conn) self.assertRequests([ - ('GET', '/info'), + ('GET', '/info', '', {}), ]) self.assertEqual(info, {}) self.assertTrue(http_conn[1].resp.has_been_read) @@ -1049,8 +1104,10 @@ def test_conn_get_capabilities_with_auth(self): info = conn.get_capabilities() self.assertEqual(info, stub_info) self.assertRequests([ - ('GET', '/auth/v1.0'), - ('GET', 'http://storage.example.com/info'), + ('GET', '/auth/v1.0', '', { + 'x-auth-user': 'user', + 'x-auth-key': 'key'}), + ('GET', 'http://storage.example.com/info', '', {}), ]) def test_conn_get_capabilities_with_os_auth(self): diff --git a/tests/unit/utils.py b/tests/unit/utils.py index ac9aefdb..20272aa2 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -325,7 +325,7 @@ def assert_request_equal(self, expected, real_request): self.orig_assertEqual(body, real_request['body'], err_msg) if len(expected) > 3: - headers = expected[3] + headers = CaseInsensitiveDict(expected[3]) for key, value in headers.items(): real_request['key'] = key real_request['expected_value'] = value @@ -336,16 +336,30 @@ def assert_request_equal(self, expected, real_request): 'for %(method)s %(path)s %(headers)r' % real_request) self.orig_assertEqual(value, real_request['value'], err_msg) + real_request['extra_headers'] = dict( + (key, value) for key, value in real_request['headers'].items() + if key not in headers) + if real_request['extra_headers']: + self.fail('Received unexpected headers for %(method)s ' + '%(path)s, got %(extra_headers)r' % real_request) def assertRequests(self, expected_requests): """ Make sure some requests were made like you expected, provide a list of expected requests, typically in the form of [(method, path), ...] + or [(method, path, body, headers), ...] """ real_requests = self.iter_request_log() for expected in expected_requests: real_request = next(real_requests) self.assert_request_equal(expected, real_request) + try: + real_request = next(real_requests) + except StopIteration: + pass + else: + self.fail('At least one extra request received: %r' % + real_request) def assert_request(self, expected_request): """ From 43b2c6bfe5140f32a37638985bd4cb7b73988160 Mon Sep 17 00:00:00 2001 From: Alistair Coles Date: Wed, 23 Sep 2015 09:55:10 +0100 Subject: [PATCH 047/454] Make more assertions in client unit tests Some tests rely on the fake connection checking expected request parameters, but that assumes that the fake ocnnection is even called, which is not being checked. Add more explicit assertions that requests are in fact made. Change-Id: Id1c48235d7d97fd1b0feec6c19ed59a87bebdf89 --- tests/unit/test_swiftclient.py | 45 +++++++++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 6 deletions(-) diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index 3272285f..111e0771 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -700,8 +700,13 @@ class TestPutContainer(MockHttpTest): def test_ok(self): c.http_connection = self.fake_http_connection(200) - value = c.put_container('http://www.test.com', 'asdf', 'asdf') + value = c.put_container('http://www.test.com', 'token', 'container') self.assertEqual(value, None) + self.assertRequests([ + ('PUT', '/container', '', { + 'x-auth-token': 'token', + 'content-length': '0'}), + ]) def test_server_error(self): body = 'c' * 60 @@ -720,8 +725,12 @@ class TestDeleteContainer(MockHttpTest): def test_ok(self): c.http_connection = self.fake_http_connection(200) - value = c.delete_container('http://www.test.com', 'asdf', 'asdf') + value = c.delete_container('http://www.test.com', 'token', 'container') self.assertEqual(value, None) + self.assertRequests([ + ('DELETE', '/container', '', { + 'x-auth-token': 'token'}), + ]) class TestGetObject(MockHttpTest): @@ -815,9 +824,17 @@ class TestPutObject(MockHttpTest): def test_ok(self): c.http_connection = self.fake_http_connection(200) - args = ('http://www.test.com', 'asdf', 'asdf', 'asdf', 'asdf') + args = ('http://www.test.com', 'TOKEN', 'container', 'obj', 'body', 4) value = c.put_object(*args) self.assertTrue(isinstance(value, six.string_types)) + self.assertEqual(value, EMPTY_ETAG) + self.assertRequests([ + ('PUT', '/container/obj', 'body', { + 'x-auth-token': 'TOKEN', + 'content-length': '4', + 'content-type': '' + }), + ]) def test_unicode_ok(self): conn = c.http_connection(u'http://www.test.com/') @@ -1008,8 +1025,14 @@ class TestPostObject(MockHttpTest): def test_ok(self): c.http_connection = self.fake_http_connection(200) - args = ('http://www.test.com', 'asdf', 'asdf', 'asdf', {}) + args = ('http://www.test.com', 'token', 'container', 'obj', + {'X-Object-Meta-Test': 'mymeta'}) c.post_object(*args) + self.assertRequests([ + ('POST', '/container/obj', '', { + 'x-auth-token': 'token', + 'X-Object-Meta-Test': 'mymeta'}), + ]) def test_unicode_ok(self): conn = c.http_connection(u'http://www.test.com/') @@ -1056,7 +1079,12 @@ class TestDeleteObject(MockHttpTest): def test_ok(self): c.http_connection = self.fake_http_connection(200) - c.delete_object('http://www.test.com', 'asdf', 'asdf', 'asdf') + c.delete_object('http://www.test.com', 'token', 'container', 'obj') + self.assertRequests([ + ('DELETE', 'http://www.test.com/container/obj', '', { + 'x-auth-token': 'token', + }), + ]) def test_server_error(self): c.http_connection = self.fake_http_connection(500) @@ -1066,8 +1094,13 @@ def test_server_error(self): def test_query_string(self): c.http_connection = self.fake_http_connection(200, query_string="hello=20") - c.delete_object('http://www.test.com', 'asdf', 'asdf', 'asdf', + c.delete_object('http://www.test.com', 'token', 'container', 'obj', query_string="hello=20") + self.assertRequests([ + ('DELETE', 'http://www.test.com/container/obj?hello=20', '', { + 'x-auth-token': 'token', + }), + ]) class TestGetCapabilities(MockHttpTest): From 328d6a8d457b8be6fc4b3dfdbb396a44f0b8710b Mon Sep 17 00:00:00 2001 From: Alistair Coles Date: Tue, 22 Sep 2015 11:09:44 +0100 Subject: [PATCH 048/454] Add tests and param definitions for headers parameter Cleanups for change I35c3b266b3c733f6b1629de4c683ea7d40128032 Add missing param definitions to client get_container and head_object docstrings. For consistency, add headers parameter to the Connection class head_object and head_container wrapper methods. Add tests to verify that the headers parameter of Connection get_container, head_container and head_object methods is passed to the module functions. Change-Id: Ib40d5b626b2793840727c58cffbf725bea55651f --- swiftclient/client.py | 11 ++++--- tests/unit/test_swiftclient.py | 53 ++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 4 deletions(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index 1913ea36..2bbd978f 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -656,6 +656,7 @@ def get_container(url, token, container, marker=None, limit=None, :param full_listing: if True, return a full listing, else returns a max of 10000 listings :param service_token: service auth token + :param headers: additional headers to include in the request :returns: a tuple of (response headers, a list of objects) The response headers will be a dict and all header names will be lowercase. :raises ClientException: HTTP GET request failed @@ -735,6 +736,7 @@ def head_container(url, token, container, http_conn=None, headers=None, :param container: container name to get stats for :param http_conn: HTTP connection object (If None, it will create the conn object) + :param headers: additional headers to include in the request :param service_token: service auth token :returns: a dict containing the response's headers (all header names will be lowercase) @@ -974,6 +976,7 @@ def head_object(url, token, container, name, http_conn=None, :param http_conn: HTTP connection object (If None, it will create the conn object) :param service_token: service auth token + :param headers: additional headers to include in the request :returns: a dict containing the response's headers (all header names will be lowercase) :raises ClientException: HTTP HEAD request failed @@ -1453,9 +1456,9 @@ def post_account(self, headers, response_dict=None): return self._retry(None, post_account, headers, response_dict=response_dict) - def head_container(self, container): + def head_container(self, container, headers=None): """Wrapper for :func:`head_container`""" - return self._retry(None, head_container, container) + return self._retry(None, head_container, container, headers=headers) def get_container(self, container, marker=None, limit=None, prefix=None, delimiter=None, end_marker=None, path=None, @@ -1484,9 +1487,9 @@ def delete_container(self, container, response_dict=None): return self._retry(None, delete_container, container, response_dict=response_dict) - def head_object(self, container, obj): + def head_object(self, container, obj, headers=None): """Wrapper for :func:`head_object`""" - return self._retry(None, head_object, container, obj) + return self._retry(None, head_object, container, obj, headers=headers) def get_object(self, container, obj, resp_chunk_size=None, query_string=None, response_dict=None, headers=None): diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index 410fd6f5..01e393c7 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -1708,6 +1708,59 @@ def local_http_connection(url, proxy=None, cacert=None, finally: c.http_connection = orig_conn + def test_get_container(self): + headers = {'X-Favourite-Pet': 'Aardvark'} + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200, body=b'{}')): + with mock.patch('swiftclient.client.get_auth', + lambda *a, **k: ('http://url:8080/v1/a', 'token')): + conn = c.Connection() + conn.get_container('c1', prefix='p', limit=5, + headers=headers) + self.assertEqual(1, len(self.request_log), self.request_log) + self.assertRequests([ + ('GET', '/v1/a/c1?format=json&limit=5&prefix=p', '', { + 'x-auth-token': 'token', + 'X-Favourite-Pet': 'Aardvark', + }), + ]) + self.assertEqual(conn.attempts, 1) + + def test_head_container(self): + headers = {'X-Favourite-Pet': 'Aardvark'} + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200, body=b'{}')): + with mock.patch('swiftclient.client.get_auth', + lambda *a, **k: ('http://url:8080/v1/a', 'token')): + conn = c.Connection() + conn.head_container('c1', headers=headers) + self.assertEqual(1, len(self.request_log), self.request_log) + self.assertRequests([ + ('HEAD', '/v1/a/c1', '', { + 'x-auth-token': 'token', + 'X-Favourite-Pet': 'Aardvark', + }), + ]) + self.assertEqual(conn.attempts, 1) + + def test_head_object(self): + headers = {'X-Favourite-Pet': 'Aardvark'} + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200)): + with mock.patch('swiftclient.client.get_auth', + lambda *a, **k: ('http://url:8080/v1/a', 'token')): + conn = c.Connection() + conn.head_object('c1', 'o1', + headers=headers) + self.assertEqual(1, len(self.request_log), self.request_log) + self.assertRequests([ + ('HEAD', '/v1/a/c1/o1', '', { + 'x-auth-token': 'token', + 'X-Favourite-Pet': 'Aardvark', + }), + ]) + self.assertEqual(conn.attempts, 1) + class TestResponseDict(MockHttpTest): """ From 1b0567b6c7634fad64ff37fa0000893febfc9cc5 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Thu, 1 Oct 2015 12:31:12 -0700 Subject: [PATCH 049/454] Add py35 to default tox environments Change-Id: Ib10eab87b791da561b82c9522ba2686d24966c2d --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 8a70619b..a4670156 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py26,py27,py33,py34,pypy,pep8 +envlist = py26,py27,py33,py34,py35,pypy,pep8 minversion = 1.6 skipsdist = True From 9fed7ed5e1f6dd3e589a35e3ee4abecb676f2188 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Wed, 9 Sep 2015 17:41:21 -0700 Subject: [PATCH 050/454] Miscellaneous (mostly test) cleanup * Always use testtools.TestCase, since we're relying on testtools * Always use mock (as opposed to unittest.mock) since we're relying on mock * Add note about when a missing logging handler was added * Stop %-formatting the giant usage string that doesn't actually need any formatting * Prefer assertIs, assertIn, assertIsInstance over assertTrue * Use else-self.fail instead of sentinel values * Check resp.get('error') is None before checking resp['success'] is True, so test failures actually tell you something useful * Tighten some isinstance assertions * Import MockHttpTest from correct location * Only populate clean_os_environ once * Use setUp for setup, not __init__ * Replace assertIn(key, dict) and assertEqual(foo, dict[key]) with assertEqual(foo, dict.get(key)) when key is a literal and foo is not None * Use mock.patch.object instead of manually patching for tests * Use six.binary_type instead of type(''.encode('utf-8')) * Stop shadowing builtin bytes * Reclaim some margin * Stop checking the return-type of encode_utf8; we already know it's bytes Change-Id: I2138ea553378ce88810b7353147c8645a8f8c90e --- swiftclient/client.py | 7 +- swiftclient/shell.py | 20 ++--- swiftclient/utils.py | 38 ++++----- tests/unit/test_command_helpers.py | 6 +- tests/unit/test_multithreading.py | 15 ++-- tests/unit/test_service.py | 60 +++++++------- tests/unit/test_shell.py | 42 ++++------ tests/unit/test_swiftclient.py | 121 +++++++++++------------------ tests/unit/test_utils.py | 20 ++--- tests/unit/utils.py | 5 +- 10 files changed, 138 insertions(+), 196 deletions(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index 1913ea36..f2d3098c 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -46,6 +46,7 @@ try: from logging import NullHandler except ImportError: + # Added in Python 2.7 class NullHandler(logging.Handler): def handle(self, record): pass @@ -110,11 +111,7 @@ def quote(value, safe='/'): """ if six.PY3: return _quote(value, safe=safe) - value = encode_utf8(value) - if isinstance(value, bytes): - return _quote(value, safe) - else: - return value + return _quote(encode_utf8(value), safe) def encode_utf8(value): diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 652980f3..2d2a7a1f 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -51,7 +51,7 @@ def immediate_exit(signum, frame): stderr.write(" Aborted\n") os_exit(2) -st_delete_options = '''[-all] [--leave-segments] +st_delete_options = '''[--all] [--leave-segments] [--object-threads ] [--container-threads ] [object] @@ -1151,7 +1151,7 @@ def main(arguments=None): version = client_version parser = OptionParser(version='python-swiftclient %s' % version, usage=''' -usage: %%prog [--version] [--help] [--os-help] [--snet] [--verbose] +usage: %prog [--version] [--help] [--os-help] [--snet] [--verbose] [--debug] [--info] [--quiet] [--auth ] [--auth-version ] [--user ] [--key ] [--retries ] @@ -1191,29 +1191,29 @@ def main(arguments=None): auth Display auth related environment variables. Examples: - %%prog download --help + %prog download --help - %%prog -A https://auth.api.rackspacecloud.com/v1.0 -U user -K api_key stat -v + %prog -A https://auth.api.rackspacecloud.com/v1.0 -U user -K api_key stat -v - %%prog --os-auth-url https://api.example.com/v2.0 --os-tenant-name tenant \\ + %prog --os-auth-url https://api.example.com/v2.0 --os-tenant-name tenant \\ --os-username user --os-password password list - %%prog --os-auth-url https://api.example.com/v3 --auth-version 3\\ + %prog --os-auth-url https://api.example.com/v3 --auth-version 3\\ --os-project-name project1 --os-project-domain-name domain1 \\ --os-username user --os-user-domain-name domain1 \\ --os-password password list - %%prog --os-auth-url https://api.example.com/v3 --auth-version 3\\ + %prog --os-auth-url https://api.example.com/v3 --auth-version 3\\ --os-project-id 0123456789abcdef0123456789abcdef \\ --os-user-id abcdef0123456789abcdef0123456789 \\ --os-password password list - %%prog --os-auth-token 6ee5eb33efad4e45ab46806eac010566 \\ + %prog --os-auth-token 6ee5eb33efad4e45ab46806eac010566 \\ --os-storage-url https://10.1.5.2:8080/v1/AUTH_ced809b6a4baea7aeab61a \\ list - %%prog list --lh -'''.strip('\n') % globals()) + %prog list --lh +'''.strip('\n')) parser.add_option('--os-help', action='store_true', dest='os_help', help='Show OpenStack authentication options.') parser.add_option('--os_help', action='store_true', help=SUPPRESS_HELP) diff --git a/swiftclient/utils.py b/swiftclient/utils.py index 8316a8f8..8ef0403b 100644 --- a/swiftclient/utils.py +++ b/swiftclient/utils.py @@ -35,34 +35,30 @@ def config_true_value(value): (isinstance(value, six.string_types) and value.lower() in TRUE_VALUES) -def prt_bytes(bytes, human_flag): +def prt_bytes(num_bytes, human_flag): """ convert a number > 1024 to printable format, either in 4 char -h format as with ls -lh or return as 12 char right justified string """ - if human_flag: - suffix = '' - mods = list('KMGTPEZY') - temp = float(bytes) - if temp > 0: - while temp > 1023: - try: - suffix = mods.pop(0) - except IndexError: - break - temp /= 1024.0 - if suffix != '': - if temp >= 10: - bytes = '%3d%s' % (temp, suffix) - else: - bytes = '%.1f%s' % (temp, suffix) - if suffix == '': # must be < 1024 - bytes = '%4s' % bytes + if not human_flag: + return '%12s' % num_bytes + + num = float(num_bytes) + suffixes = [None] + list('KMGTPEZY') + for suffix in suffixes[:-1]: + if num <= 1023: + break + num /= 1024.0 else: - bytes = '%12s' % bytes + suffix = suffixes[-1] - return bytes + if not suffix: # num_bytes must be < 1024 + return '%4s' % num_bytes + elif num >= 10: + return '%3d%s' % (num, suffix) + else: + return '%.1f%s' % (num, suffix) def generate_temp_url(path, seconds, key, method, absolute=False): diff --git a/tests/unit/test_command_helpers.py b/tests/unit/test_command_helpers.py index 67e9ac28..d9d7efa6 100644 --- a/tests/unit/test_command_helpers.py +++ b/tests/unit/test_command_helpers.py @@ -13,11 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -try: - from unittest import mock -except ImportError: - import mock - +import mock from six import StringIO import testtools diff --git a/tests/unit/test_multithreading.py b/tests/unit/test_multithreading.py index 2c45b47e..76758b69 100644 --- a/tests/unit/test_multithreading.py +++ b/tests/unit/test_multithreading.py @@ -82,14 +82,13 @@ def test_submit_good_connection(self): ) # Now a job that fails - went_boom = False try: f = pool.submit(self._func, "go boom") f.result() except Exception as e: - went_boom = True self.assertEqual('I went boom!', str(e)) - self.assertTrue(went_boom) + else: + self.fail('I never went boom!') # Has the connection been returned to the pool? f = pool.submit(self._func, "succeed") @@ -106,24 +105,22 @@ def test_submit_bad_connection(self): ctpe = mt.ConnectionThreadPoolExecutor(self._create_conn_fail, 1) with ctpe as pool: # Now a connection that fails - connection_failed = False try: f = pool.submit(self._func, "succeed") f.result() except Exception as e: - connection_failed = True self.assertEqual('This is a failed connection', str(e)) - self.assertTrue(connection_failed) + else: + self.fail('The connection did not fail') # Make sure we don't lock up on failed connections - connection_failed = False try: f = pool.submit(self._func, "go boom") f.result() except Exception as e: - connection_failed = True self.assertEqual('This is a failed connection', str(e)) - self.assertTrue(connection_failed) + else: + self.fail('The connection did not fail') def test_lazy_connections(self): ctpe = mt.ConnectionThreadPoolExecutor(self._create_conn, 10) diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index 976d346e..8eea4c3f 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -83,7 +83,7 @@ def test_create(self): self.assertEqual(sr._expected_etag, None) self.assertNotEqual(sr._actual_md5, None) - self.assertTrue(isinstance(sr._actual_md5, self.md5_type)) + self.assertIs(type(sr._actual_md5), self.md5_type) def test_create_with_large_object_headers(self): # md5 should not be initialized if large object headers are present @@ -110,7 +110,7 @@ def test_create_with_content_length(self): self.assertEqual(sr._expected_etag, None) self.assertNotEqual(sr._actual_md5, None) - self.assertTrue(isinstance(sr._actual_md5, self.md5_type)) + self.assertIs(type(sr._actual_md5), self.md5_type) # Check Contentlength raises error if it isnt an integer self.assertRaises(SwiftError, self.sr, 'path', 'body', @@ -160,10 +160,10 @@ def _assertDictEqual(self, a, b, m=None): if hasattr(self, 'assertDictEqual'): self.assertDictEqual(a, b, m) else: - self.assertTrue(isinstance(a, dict), - 'First argument is not a dictionary') - self.assertTrue(isinstance(b, dict), - 'Second argument is not a dictionary') + self.assertIsInstance(a, dict, + 'First argument is not a dictionary') + self.assertIsInstance(b, dict, + 'Second argument is not a dictionary') self.assertEqual(len(a), len(b), m) for k, v in a.items(): self.assertIn(k, b, m) @@ -248,7 +248,7 @@ def test_delete_segment_exception(self): self._assertDictEqual(expected_r, self._get_queue(mock_q)) self.assertGreaterEqual(r['error_timestamp'], before) self.assertLessEqual(r['error_timestamp'], after) - self.assertTrue('Traceback' in r['traceback']) + self.assertIn('Traceback', r['traceback']) def test_delete_object(self): mock_q = Queue() @@ -295,7 +295,7 @@ def test_delete_object_exception(self): self._assertDictEqual(expected_r, r) self.assertGreaterEqual(r['error_timestamp'], before) self.assertLessEqual(r['error_timestamp'], after) - self.assertTrue('Traceback' in r['traceback']) + self.assertIn('Traceback', r['traceback']) def test_delete_object_slo_support(self): # If SLO headers are present the delete call should include an @@ -395,14 +395,14 @@ def test_delete_empty_container_exception(self): self._assertDictEqual(expected_r, r) self.assertGreaterEqual(r['error_timestamp'], before) self.assertLessEqual(r['error_timestamp'], after) - self.assertTrue('Traceback' in r['traceback']) + self.assertIn('Traceback', r['traceback']) class TestSwiftError(testtools.TestCase): def test_is_exception(self): se = SwiftError(5) - self.assertTrue(isinstance(se, Exception)) + self.assertIsInstance(se, Exception) def test_empty_swifterror_creation(self): se = SwiftError(5) @@ -444,7 +444,7 @@ def test_process_options_defaults(self): swiftclient.service.process_options(opt_c) - self.assertTrue('os_options' in opt_c) + self.assertIn('os_options', opt_c) del opt_c['os_options'] self.assertEqual(opt_c['auth_version'], '2.0') opt_c['auth_version'] = '1.0' @@ -838,7 +838,8 @@ def test_upload_with_relative_path(self, *args, **kwargs): 'c', [SwiftUploadObject(obj['path'])]) responses = [x for x in resp_iter] for resp in responses: - self.assertTrue(resp['success']) + self.assertIsNone(resp.get('error')) + self.assertIs(True, resp['success']) self.assertEqual(2, len(responses)) create_container_resp, upload_obj_resp = responses self.assertEqual(create_container_resp['action'], @@ -934,7 +935,7 @@ def _consuming_conn(*a, **kw): options={'segment_container': None, 'checksum': False}) - self.assertNotIn('error', r) + self.assertIsNone(r.get('error')) self.assertEqual(mock_conn.put_object.call_count, 1) mock_conn.put_object.assert_called_with('test_c_segments', 'test_s_1', @@ -973,8 +974,7 @@ def _consuming_conn(*a, **kw): options={'segment_container': None, 'checksum': True}) - self.assertIn('error', r) - self.assertIn('md5 mismatch', str(r['error'])) + self.assertIn('md5 mismatch', str(r.get('error'))) self.assertEqual(mock_conn.put_object.call_count, 1) mock_conn.put_object.assert_called_with('test_c_segments', @@ -1128,9 +1128,8 @@ def _consuming_conn(*a, **kw): 'segment_size': 0, 'checksum': True}) - self.assertEqual(r['success'], False) - self.assertIn('error', r) - self.assertIn('md5 mismatch', str(r['error'])) + self.assertIs(r['success'], False) + self.assertIn('md5 mismatch', str(r.get('error'))) self.assertEqual(mock_conn.put_object.call_count, 1) expected_headers = {'x-object-meta-mtime': mock.ANY} @@ -1165,9 +1164,9 @@ def test_upload_object_job_identical_etag(self): 'header': '', 'segment_size': 0}) - self.assertTrue(r['success']) - self.assertIn('status', r) - self.assertEqual(r['status'], 'skipped-identical') + self.assertIsNone(r.get('error')) + self.assertIs(True, r['success']) + self.assertEqual(r.get('status'), 'skipped-identical') self.assertEqual(mock_conn.put_object.call_count, 0) self.assertEqual(mock_conn.head_object.call_count, 1) mock_conn.head_object.assert_called_with('test_c', 'test_o') @@ -1208,7 +1207,7 @@ def test_upload_object_job_identical_slo_with_nesting(self): 'segment_size': 10}) self.assertIsNone(r.get('error')) - self.assertTrue(r['success']) + self.assertIs(True, r['success']) self.assertEqual('skipped-identical', r.get('status')) self.assertEqual(0, mock_conn.put_object.call_count) self.assertEqual([mock.call('test_c', 'test_o')], @@ -1255,7 +1254,7 @@ def test_upload_object_job_identical_dlo(self): 'segment_size': 10}) self.assertIsNone(r.get('error')) - self.assertTrue(r['success']) + self.assertIs(True, r['success']) self.assertEqual('skipped-identical', r.get('status')) self.assertEqual(0, mock_conn.put_object.call_count) self.assertEqual(1, mock_conn.head_object.call_count) @@ -1498,7 +1497,8 @@ def test_download(self): 'test', self.opts) - self.assertTrue(resp['success']) + self.assertIsNone(resp.get('error')) + self.assertIs(True, resp['success']) self.assertEqual(resp['action'], 'download_object') self.assertEqual(resp['object'], 'test') self.assertEqual(resp['path'], 'test') @@ -1517,7 +1517,8 @@ def test_download_with_output_dir(self): 'example/test', options) - self.assertTrue(resp['success']) + self.assertIsNone(resp.get('error')) + self.assertIs(True, resp['success']) self.assertEqual(resp['action'], 'download_object') self.assertEqual(resp['object'], 'example/test') self.assertEqual(resp['path'], 'temp_dir/example/test') @@ -1537,7 +1538,8 @@ def test_download_with_remove_prefix(self): 'example/test', options) - self.assertTrue(resp['success']) + self.assertIsNone(resp.get('error')) + self.assertIs(True, resp['success']) self.assertEqual(resp['action'], 'download_object') self.assertEqual(resp['object'], 'example/test') self.assertEqual(resp['path'], 'test') @@ -1557,7 +1559,8 @@ def test_download_with_remove_prefix_and_remove_slashes(self): 'example/test', options) - self.assertTrue(resp['success']) + self.assertIsNone(resp.get('error')) + self.assertIs(True, resp['success']) self.assertEqual(resp['action'], 'download_object') self.assertEqual(resp['object'], 'example/test') self.assertEqual(resp['path'], 'test') @@ -1578,7 +1581,8 @@ def test_download_with_output_dir_and_remove_prefix(self): 'example/test', options) - self.assertTrue(resp['success']) + self.assertIsNone(resp.get('error')) + self.assertIs(True, resp['success']) self.assertEqual(resp['action'], 'download_object') self.assertEqual(resp['object'], 'example/test') self.assertEqual(resp['path'], 'new/dir/test') diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index 66bf3284..835d1d44 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -19,7 +19,7 @@ import mock import os import tempfile -import unittest +import testtools import textwrap from testtools import ExpectedException @@ -32,10 +32,9 @@ import swiftclient.utils from os.path import basename, dirname -from tests.unit.test_swiftclient import MockHttpTest -from tests.unit.utils import ( +from .utils import ( CaptureOutput, fake_get_auth_keystone, _make_fake_import_keystone_client, - FakeKeystone, StubResponse) + FakeKeystone, StubResponse, MockHttpTest) from swiftclient.utils import EMPTY_ETAG @@ -55,12 +54,6 @@ if any(key.startswith(m) for m in environ_prefixes): clean_os_environ[key] = '' -clean_os_environ = {} -environ_prefixes = ('ST_', 'OS_') -for key in os.environ: - if any(key.startswith(m) for m in environ_prefixes): - clean_os_environ[key] = '' - def _make_args(cmd, opts, os_opts, separator='-', flags=None, cmd_args=None): """ @@ -112,9 +105,9 @@ def _make_cmd(cmd, opts, os_opts, use_env=False, flags=None, cmd_args=None): @mock.patch.dict(os.environ, mocked_os_environ) -class TestShell(unittest.TestCase): - def __init__(self, *args, **kwargs): - super(TestShell, self).__init__(*args, **kwargs) +class TestShell(testtools.TestCase): + def setUp(self): + super(TestShell, self).setUp() tmpfile = tempfile.NamedTemporaryFile(delete=False) self.tmpfile = tmpfile.name @@ -123,6 +116,7 @@ def tearDown(self): os.remove(self.tmpfile) except OSError: pass + super(TestShell, self).tearDown() @mock.patch('swiftclient.service.Connection') def test_stat_account(self, connection): @@ -1024,12 +1018,12 @@ def test_negative_upload_segment_size(self): output.clear() -class TestSubcommandHelp(unittest.TestCase): +class TestSubcommandHelp(testtools.TestCase): def test_subcommand_help(self): for command in swiftclient.shell.commands: help_var = 'st_%s_help' % command - self.assertTrue(help_var in vars(swiftclient.shell)) + self.assertTrue(hasattr(swiftclient.shell, help_var)) with CaptureOutput() as out: argv = ['', command, '--help'] self.assertRaises(SystemExit, swiftclient.shell.main, argv) @@ -1044,7 +1038,7 @@ def test_no_help(self): self.assertEqual(out.strip('\n'), expected) -class TestBase(unittest.TestCase): +class TestBase(testtools.TestCase): """ Provide some common methods to subclasses """ @@ -1106,7 +1100,7 @@ def _verify_opts(self, actual_opts, opts, os_opts={}, os_opts_dict={}): 'service_type', 'project_id', 'auth_token', 'project_domain_name'] for key in expected_os_opts_keys: - self.assertTrue(key in actual_os_opts_dict) + self.assertIn(key, actual_os_opts_dict) cli_key = key if key == 'object_storage_url': # exceptions to the pattern... @@ -1119,7 +1113,7 @@ def _verify_opts(self, actual_opts, opts, os_opts={}, os_opts_dict={}): self.assertEqual(expect, actual, 'Expected %s for %s, got %s' % (expect, key, actual)) for key in actual_os_opts_dict: - self.assertTrue(key in expected_os_opts_keys) + self.assertIn(key, expected_os_opts_keys) # check that equivalent keys have equal values equivalents = [('os_username', 'user'), @@ -1446,9 +1440,7 @@ def _test_options_passed_to_keystone(self, cmd, opts, os_opts, for key in self.all_os_opts.keys(): expected = os_opts.get(key, self.defaults.get(key)) key = key.replace('-', '_') - self.assertTrue(key in actual_args, - 'Expected key %s not found in args %s' - % (key, actual_args)) + self.assertIn(key, actual_args) self.assertEqual(expected, actual_args[key], 'Expected %s for key %s, found %s' % (expected, key, actual_args[key])) @@ -1464,16 +1456,12 @@ def _test_options_passed_to_keystone(self, cmd, opts, os_opts, key = key.replace('-', '_') if key == 'region_name': key = 'filter_value' - self.assertTrue(key in actual_args, - 'Expected key %s not found in args %s' - % (key, actual_args)) + self.assertIn(key, actual_args) self.assertEqual(expected, actual_args[key], 'Expected %s for key %s, found %s' % (expected, key, actual_args[key])) key, v = 'attr', 'region' - self.assertTrue(key in actual_args, - 'Expected key %s not found in args %s' - % (key, actual_args)) + self.assertIn(key, actual_args) self.assertEqual(v, actual_args[key], 'Expected %s for key %s, found %s' % (v, key, actual_args[key])) diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index 111e0771..53fcccb7 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -14,18 +14,14 @@ # limitations under the License. import logging - -try: - from unittest import mock -except ImportError: - import mock - +import mock import six import socket import testtools import warnings import tempfile from hashlib import md5 +from six import binary_type from six.moves.urllib.parse import urlparse from .utils import (MockHttpTest, fake_get_auth_keystone, StubResponse, @@ -44,7 +40,7 @@ def test_is_exception(self): def test_format(self): exc = c.ClientException('something failed') - self.assertTrue('something failed' in str(exc)) + self.assertIn('something failed', str(exc)) test_kwargs = ( 'scheme', 'host', @@ -60,7 +56,7 @@ def test_format(self): 'http_%s' % value: value, } exc = c.ClientException('test', **kwargs) - self.assertTrue(value in str(exc)) + self.assertIn(value, str(exc)) class MockHttpResponse(object): @@ -133,10 +129,10 @@ def test_quote(self): def test_http_connection(self): url = 'http://www.test.com' _junk, conn = c.http_connection(url) - self.assertTrue(isinstance(conn, c.HTTPConnection)) + self.assertIs(type(conn), c.HTTPConnection) url = 'https://www.test.com' _junk, conn = c.http_connection(url) - self.assertTrue(isinstance(conn, c.HTTPConnection)) + self.assertIs(type(conn), c.HTTPConnection) url = 'ftp://www.test.com' self.assertRaises(c.ClientException, c.http_connection, url) @@ -146,18 +142,16 @@ def test_encode_meta_headers(self): u'x-account-meta-\u0394': '123', u'x-object-meta-\u0394': '123'} - encoded_str_type = type(''.encode()) r = swiftclient.encode_meta_headers(headers) self.assertEqual(len(headers), len(r)) # ensure non meta headers are not encoded - self.assertTrue('abc' in r) - self.assertTrue(isinstance(r['abc'], encoded_str_type)) + self.assertIs(type(r.get('abc')), binary_type) del r['abc'] for k, v in r.items(): - self.assertTrue(isinstance(k, encoded_str_type)) - self.assertTrue(isinstance(v, encoded_str_type)) + self.assertIs(type(k), binary_type) + self.assertIs(type(v), binary_type) def test_set_user_agent_default(self): _junk, conn = c.http_connection('http://www.example.com') @@ -826,7 +820,7 @@ def test_ok(self): c.http_connection = self.fake_http_connection(200) args = ('http://www.test.com', 'TOKEN', 'container', 'obj', 'body', 4) value = c.put_object(*args) - self.assertTrue(isinstance(value, six.string_types)) + self.assertIsInstance(value, six.string_types) self.assertEqual(value, EMPTY_ETAG) self.assertRequests([ ('PUT', '/container/obj', 'body', { @@ -852,7 +846,7 @@ def test_unicode_ok(self): conn[1].getresponse = resp.fake_response conn[1]._request = resp._fake_request value = c.put_object(*args, headers=headers, http_conn=conn) - self.assertTrue(isinstance(value, six.string_types)) + self.assertIsInstance(value, six.string_types) # Test for RFC-2616 encoded symbols self.assertIn(("a-b", b".x:yz mn:fg:lp"), resp.buffer) @@ -921,8 +915,7 @@ def test_raw_upload(self): contents=mock_file, **kwarg) req_data = resp.requests_params['data'] - self.assertTrue(isinstance(req_data, - swiftclient.utils.LengthWrapper)) + self.assertIs(type(req_data), swiftclient.utils.LengthWrapper) self.assertEqual(raw_data_len, len(req_data.read())) def test_chunk_upload(self): @@ -1691,7 +1684,6 @@ def shim_connection(*a, **kw): # check timeout is passed to keystone client self.assertEqual(1, len(fake_ks.calls)) - self.assertTrue('timeout' in fake_ks.calls[0]) self.assertEqual(33.0, fake_ks.calls[0].get('timeout')) # check timeout passed to HEAD for account self.assertEqual(timeouts, [33.0]) @@ -1761,9 +1753,7 @@ def local_http_connection(url, proxy=None, cacert=None, parsed = urlparse(url) return parsed, LocalConnection() - orig_conn = c.http_connection - try: - c.http_connection = local_http_connection + with mock.patch.object(c, 'http_connection', local_http_connection): conn = c.Connection('http://www.example.com', 'asdf', 'asdf', retries=1, starting_backoff=.0001) @@ -1795,8 +1785,6 @@ def local_http_connection(url, proxy=None, cacert=None, self.assertEqual(contents.seeks, []) self.assertEqual(str(exc), "put_object('c', 'o', ...) failure " "and no ability to reset contents for reupload.") - finally: - c.http_connection = orig_conn class TestResponseDict(MockHttpTest): @@ -1841,10 +1829,8 @@ def test_response_dict_with_request_error(self): *call[1:], response_dict=resp_dict) - self.assertTrue('test' in resp_dict) - self.assertEqual('should be untouched', resp_dict['test']) - self.assertTrue('response_dicts' in resp_dict) - self.assertEqual([{}], resp_dict['response_dicts']) + self.assertEqual('should be untouched', resp_dict.get('test')) + self.assertEqual([{}], resp_dict.get('response_dicts')) def test_response_dict(self): # test response_dict is populated and @@ -1858,15 +1844,13 @@ def test_response_dict(self): conn = c.Connection('http://127.0.0.1:8080', 'user', 'key') getattr(conn, call[0])(*call[1:], response_dict=resp_dict) - for key in ('test', 'status', 'headers', 'reason', - 'response_dicts'): - self.assertTrue(key in resp_dict) - self.assertEqual('should be untouched', resp_dict.pop('test')) - self.assertEqual('Fake', resp_dict['reason']) - self.assertEqual(200, resp_dict['status']) - self.assertTrue('x-works' in resp_dict['headers']) - self.assertEqual('yes', resp_dict['headers']['x-works']) - children = resp_dict.pop('response_dicts') + self.assertEqual('should be untouched', + resp_dict.pop('test', None)) + self.assertEqual('Fake', resp_dict.get('reason')) + self.assertEqual(200, resp_dict.get('status')) + self.assertIn('headers', resp_dict) + self.assertEqual('yes', resp_dict['headers'].get('x-works')) + children = resp_dict.pop('response_dicts', []) self.assertEqual(1, len(children)) self.assertEqual(resp_dict, children[0]) @@ -1883,15 +1867,13 @@ def test_response_dict_with_existing(self): conn = c.Connection('http://127.0.0.1:8080', 'user', 'key') getattr(conn, call[0])(*call[1:], response_dict=resp_dict) - for key in ('test', 'status', 'headers', 'reason', - 'response_dicts'): - self.assertTrue(key in resp_dict) - self.assertEqual('should be untouched', resp_dict.pop('test')) - self.assertEqual('Fake', resp_dict['reason']) - self.assertEqual(200, resp_dict['status']) - self.assertTrue('x-works' in resp_dict['headers']) - self.assertEqual('yes', resp_dict['headers']['x-works']) - children = resp_dict.pop('response_dicts') + self.assertEqual('should be untouched', + resp_dict.pop('test', None)) + self.assertEqual('Fake', resp_dict.get('reason')) + self.assertEqual(200, resp_dict.get('status')) + self.assertIn('headers', resp_dict) + self.assertEqual('yes', resp_dict['headers'].get('x-works')) + children = resp_dict.pop('response_dicts', []) self.assertEqual(2, len(children)) self.assertEqual({'existing': 'response dict'}, children[0]) self.assertEqual(resp_dict, children[1]) @@ -1916,7 +1898,7 @@ def test_put_ok(self): c.http_connection = self.fake_http_connection(200) args = ('http://www.test.com', 'asdf', 'asdf', 'asdf', 'asdf') value = c.put_object(*args) - self.assertTrue(isinstance(value, six.string_types)) + self.assertIsInstance(value, six.string_types) def test_head_error(self): c.http_connection = self.fake_http_connection(500) @@ -1966,14 +1948,14 @@ def get_connection(self): conn = c.Connection('http://www.test.com', 'asdf', 'asdf', os_options=self.os_options) - self.assertTrue(isinstance(conn, c.Connection)) + self.assertIs(type(conn), c.Connection) conn.get_auth = self.get_auth conn.get_service_auth = self.get_service_auth self.assertEqual(conn.attempts, 0) self.assertEqual(conn.service_token, None) - self.assertTrue(isinstance(conn, c.Connection)) + self.assertIs(type(conn), c.Connection) return conn def get_auth(self): @@ -2057,8 +2039,7 @@ def test_service_token_get_account(self): for actual in self.iter_request_log(): self.assertEqual('GET', actual['method']) actual_hdrs = actual['headers'] - self.assertTrue('X-Service-Token' in actual_hdrs) - self.assertEqual('stoken', actual_hdrs['X-Service-Token']) + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) self.assertEqual('token', actual_hdrs['X-Auth-Token']) self.assertEqual('http://storage_url.com/?format=json', actual['full_path']) @@ -2073,8 +2054,7 @@ def test_service_token_head_account(self): for actual in self.iter_request_log(): self.assertEqual('HEAD', actual['method']) actual_hdrs = actual['headers'] - self.assertTrue('X-Service-Token' in actual_hdrs) - self.assertEqual('stoken', actual_hdrs['X-Service-Token']) + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) self.assertEqual('token', actual_hdrs['X-Auth-Token']) self.assertEqual('http://storage_url.com', actual['full_path']) @@ -2089,8 +2069,7 @@ def test_service_token_post_account(self): for actual in self.iter_request_log(): self.assertEqual('POST', actual['method']) actual_hdrs = actual['headers'] - self.assertTrue('X-Service-Token' in actual_hdrs) - self.assertEqual('stoken', actual_hdrs['X-Service-Token']) + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) self.assertEqual('token', actual_hdrs['X-Auth-Token']) self.assertEqual('http://storage_url.com', actual['full_path']) self.assertEqual(conn.attempts, 1) @@ -2104,8 +2083,7 @@ def test_service_token_delete_container(self): for actual in self.iter_request_log(): self.assertEqual('DELETE', actual['method']) actual_hdrs = actual['headers'] - self.assertTrue('X-Service-Token' in actual_hdrs) - self.assertEqual('stoken', actual_hdrs['X-Service-Token']) + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) self.assertEqual('token', actual_hdrs['X-Auth-Token']) self.assertEqual('http://storage_url.com/container1', actual['full_path']) @@ -2121,8 +2099,7 @@ def test_service_token_get_container(self): for actual in self.iter_request_log(): self.assertEqual('GET', actual['method']) actual_hdrs = actual['headers'] - self.assertTrue('X-Service-Token' in actual_hdrs) - self.assertEqual('stoken', actual_hdrs['X-Service-Token']) + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) self.assertEqual('token', actual_hdrs['X-Auth-Token']) self.assertEqual('http://storage_url.com/container1?format=json', actual['full_path']) @@ -2137,8 +2114,7 @@ def test_service_token_head_container(self): for actual in self.iter_request_log(): self.assertEqual('HEAD', actual['method']) actual_hdrs = actual['headers'] - self.assertTrue('X-Service-Token' in actual_hdrs) - self.assertEqual('stoken', actual_hdrs['X-Service-Token']) + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) self.assertEqual('token', actual_hdrs['X-Auth-Token']) self.assertEqual('http://storage_url.com/container1', actual['full_path']) @@ -2153,8 +2129,7 @@ def test_service_token_post_container(self): for actual in self.iter_request_log(): self.assertEqual('POST', actual['method']) actual_hdrs = actual['headers'] - self.assertTrue('X-Service-Token' in actual_hdrs) - self.assertEqual('stoken', actual_hdrs['X-Service-Token']) + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) self.assertEqual('token', actual_hdrs['X-Auth-Token']) self.assertEqual('http://storage_url.com/container1', actual['full_path']) @@ -2169,8 +2144,7 @@ def test_service_token_put_container(self): for actual in self.iter_request_log(): self.assertEqual('PUT', actual['method']) actual_hdrs = actual['headers'] - self.assertTrue('X-Service-Token' in actual_hdrs) - self.assertEqual('stoken', actual_hdrs['X-Service-Token']) + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) self.assertEqual('token', actual_hdrs['X-Auth-Token']) self.assertEqual('http://storage_url.com/container1', actual['full_path']) @@ -2185,8 +2159,7 @@ def test_service_token_get_object(self): for actual in self.iter_request_log(): self.assertEqual('GET', actual['method']) actual_hdrs = actual['headers'] - self.assertTrue('X-Service-Token' in actual_hdrs) - self.assertEqual('stoken', actual_hdrs['X-Service-Token']) + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) self.assertEqual('token', actual_hdrs['X-Auth-Token']) self.assertEqual('http://storage_url.com/container1/obj1', actual['full_path']) @@ -2201,8 +2174,7 @@ def test_service_token_head_object(self): for actual in self.iter_request_log(): self.assertEqual('HEAD', actual['method']) actual_hdrs = actual['headers'] - self.assertTrue('X-Service-Token' in actual_hdrs) - self.assertEqual('stoken', actual_hdrs['X-Service-Token']) + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) self.assertEqual('token', actual_hdrs['X-Auth-Token']) self.assertEqual('http://storage_url.com/container1/obj1', actual['full_path']) @@ -2217,8 +2189,7 @@ def test_service_token_put_object(self): for actual in self.iter_request_log(): self.assertEqual('PUT', actual['method']) actual_hdrs = actual['headers'] - self.assertTrue('X-Service-Token' in actual_hdrs) - self.assertEqual('stoken', actual_hdrs['X-Service-Token']) + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) self.assertEqual('token', actual_hdrs['X-Auth-Token']) self.assertEqual('http://storage_url.com/container1/obj1', actual['full_path']) @@ -2233,8 +2204,7 @@ def test_service_token_post_object(self): for actual in self.iter_request_log(): self.assertEqual('POST', actual['method']) actual_hdrs = actual['headers'] - self.assertTrue('X-Service-Token' in actual_hdrs) - self.assertEqual('stoken', actual_hdrs['X-Service-Token']) + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) self.assertEqual('token', actual_hdrs['X-Auth-Token']) self.assertEqual('http://storage_url.com/container1/obj1', actual['full_path']) @@ -2249,8 +2219,7 @@ def test_service_token_delete_object(self): for actual in self.iter_request_log(): self.assertEqual('DELETE', actual['method']) actual_hdrs = actual['headers'] - self.assertTrue('X-Service-Token' in actual_hdrs) - self.assertEqual('stoken', actual_hdrs['X-Service-Token']) + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) self.assertEqual('token', actual_hdrs['X-Auth-Token']) self.assertEqual('http://storage_url.com/container1/obj1?a_string', actual['full_path']) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 7d7f6b67..4faac6d8 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -28,17 +28,13 @@ def test_TRUE_VALUES(self): for v in u.TRUE_VALUES: self.assertEqual(v, v.lower()) + @mock.patch.object(u, 'TRUE_VALUES', 'hello world'.split()) def test_config_true_value(self): - orig_trues = u.TRUE_VALUES - try: - u.TRUE_VALUES = 'hello world'.split() - for val in 'hello world HELLO WORLD'.split(): - self.assertTrue(u.config_true_value(val) is True) - self.assertTrue(u.config_true_value(True) is True) - self.assertTrue(u.config_true_value('foo') is False) - self.assertTrue(u.config_true_value(False) is False) - finally: - u.TRUE_VALUES = orig_trues + for val in 'hello world HELLO WORLD'.split(): + self.assertIs(u.config_true_value(val), True) + self.assertIs(u.config_true_value(True), True) + self.assertIs(u.config_true_value('foo'), False) + self.assertIs(u.config_true_value(False), False) class TestPrtBytes(testtools.TestCase): @@ -192,11 +188,11 @@ def test_md5_creation(self): # Check creation with a real and noop md5 class data = u.ReadableToIterable(None, None, md5=True) self.assertEqual(md5().hexdigest(), data.get_md5sum()) - self.assertTrue(isinstance(data.md5sum, type(md5()))) + self.assertIs(type(data.md5sum), type(md5())) data = u.ReadableToIterable(None, None, md5=False) self.assertEqual('', data.get_md5sum()) - self.assertTrue(isinstance(data.md5sum, type(u.NoopMD5()))) + self.assertIs(type(data.md5sum), u.NoopMD5) def test_unicode(self): # Check no errors are raised if unicode data is feed in. diff --git a/tests/unit/utils.py b/tests/unit/utils.py index 20272aa2..63b9c06f 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -263,9 +263,8 @@ def read(*args, **kwargs): conn.resp.status = status if auth_token: headers = args[1] - self.assertTrue('X-Auth-Token' in headers) - actual_token = headers.get('X-Auth-Token') - self.assertEqual(auth_token, actual_token) + self.assertEqual(auth_token, + headers.get('X-Auth-Token')) if query_string: self.assertTrue(url.endswith('?' + query_string)) if url.endswith('invalid_cert') and not insecure: From df1f4f3e3932a9653b7b1731e121c51c7fdf31e1 Mon Sep 17 00:00:00 2001 From: "Lisak, Peter" Date: Tue, 6 Oct 2015 10:08:02 +0200 Subject: [PATCH 051/454] swiftclient content-type header According to help `swift upload -h` you can add a customized request header 'Content-Type'. But actually it is ignored (cleared and default is used) if subcommand is upload. Subcommand post works as expected in help. Bug fix: Use 'Content-Type' from the customized request headers also if uploading. Change-Id: If0d1354b6214b909527341078fe1769aa6587457 --- swiftclient/client.py | 10 +++++--- tests/functional/test_swiftclient.py | 34 ++++++++++++++++++++++++++++ tests/unit/test_swiftclient.py | 18 +++++++++++++++ 3 files changed, 59 insertions(+), 3 deletions(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index f2d3098c..cad115f6 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -1028,8 +1028,11 @@ def put_object(url, token=None, container=None, name=None, contents=None, :param chunk_size: chunk size of data to write; it defaults to 65536; used only if the contents object has a 'read' method, e.g. file-like objects, ignored otherwise - :param content_type: value to send as content-type header; if None, an - empty string value will be sent + + :param content_type: value to send as content-type header, overriding any + value included in the headers param; if None and no + value is found in the headers param, an empty string + value will be sent :param headers: additional headers to include in the request, if any :param http_conn: HTTP connection object (If None, it will create the conn object) @@ -1071,7 +1074,8 @@ def put_object(url, token=None, container=None, name=None, contents=None, content_length = int(v) if content_type is not None: headers['Content-Type'] = content_type - else: # python-requests sets application/x-www-form-urlencoded otherwise + elif 'Content-Type' not in headers: + # python-requests sets application/x-www-form-urlencoded otherwise headers['Content-Type'] = '' if not contents: headers['Content-Length'] = '0' diff --git a/tests/functional/test_swiftclient.py b/tests/functional/test_swiftclient.py index 2be280da..35a5ea7e 100644 --- a/tests/functional/test_swiftclient.py +++ b/tests/functional/test_swiftclient.py @@ -217,6 +217,40 @@ def test_upload_object(self): self.assertEqual('application/octet-stream', hdrs.get('content-type')) + # Same but with content_type + self.conn.put_object( + self.containername, self.objectname, + content_type='text/plain', contents=self.test_data) + hdrs = self.conn.head_object(self.containername, self.objectname) + self.assertEqual(str(len(self.test_data)), + hdrs.get('content-length')) + self.assertEqual(self.etag, hdrs.get('etag')) + self.assertEqual('text/plain', + hdrs.get('content-type')) + + # Same but with content-type in headers + self.conn.put_object( + self.containername, self.objectname, + headers={'Content-Type': 'text/plain'}, contents=self.test_data) + hdrs = self.conn.head_object(self.containername, self.objectname) + self.assertEqual(str(len(self.test_data)), + hdrs.get('content-length')) + self.assertEqual(self.etag, hdrs.get('etag')) + self.assertEqual('text/plain', + hdrs.get('content-type')) + + # content_type rewrites content-type in headers + self.conn.put_object( + self.containername, self.objectname, + content_type='image/jpeg', + headers={'Content-Type': 'text/plain'}, contents=self.test_data) + hdrs = self.conn.head_object(self.containername, self.objectname) + self.assertEqual(str(len(self.test_data)), + hdrs.get('content-length')) + self.assertEqual(self.etag, hdrs.get('etag')) + self.assertEqual('image/jpeg', + hdrs.get('content-type')) + # Same but with content-length self.conn.put_object( self.containername, self.objectname, diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index 53fcccb7..68af46ab 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -1013,6 +1013,24 @@ def test_no_content_type(self): request_header = resp.requests_params['headers'] self.assertEqual(request_header['content-type'], b'') + def test_content_type_in_headers(self): + conn = c.http_connection(u'http://www.test.com/') + resp = MockHttpResponse(status=200) + conn[1].getresponse = resp.fake_response + conn[1]._request = resp._fake_request + + # title-case header + hdrs = {'Content-Type': 'text/Plain'} + c.put_object(url='http://www.test.com', http_conn=conn, headers=hdrs) + request_header = resp.requests_params['headers'] + self.assertEqual(request_header['content-type'], b'text/Plain') + + # method param overrides headers + c.put_object(url='http://www.test.com', http_conn=conn, headers=hdrs, + content_type='image/jpeg') + request_header = resp.requests_params['headers'] + self.assertEqual(request_header['content-type'], b'image/jpeg') + class TestPostObject(MockHttpTest): From 3e1a457db04697a9ccb4054eeb9f34953dcfca2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20Lis=C3=A1k?= Date: Tue, 3 Nov 2015 16:11:32 +0100 Subject: [PATCH 052/454] Add content-type in list of container content Change-Id: Ie0787d5ffbee0a7d2429cb285fa6ecdf722e4ae1 --- swiftclient/shell.py | 4 +++- tests/unit/test_shell.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 2d2a7a1f..15ca5706 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -424,6 +424,7 @@ def _print_stats(options, stats): datestamp, item_name) else: # list container contents subdir = item.get('subdir') + content_type = item.get('content_type') if subdir is None: item_bytes = item.get('bytes') byte_str = prt_bytes(item_bytes, options.human) @@ -436,7 +437,8 @@ def _print_stats(options, stats): item_name = subdir if not options.totals: output_manager.print_msg( - "%s %10s %8s %s", byte_str, date, xtime, item_name) + "%s %10s %8s %24s %s", + byte_str, date, xtime, content_type, item_name) total_bytes += item_bytes # report totals diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index 835d1d44..cac4da20 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -293,6 +293,7 @@ def test_list_container(self, connection): # Test container listing with --long connection.return_value.get_container.side_effect = [ [None, [{'name': 'object_a', 'bytes': 0, + 'content_type': 'type/content', 'last_modified': '123T456'}]], [None, []], ] @@ -306,7 +307,8 @@ def test_list_container(self, connection): connection.return_value.get_container.assert_has_calls(calls) self.assertEqual(output.out, - ' 0 123 456 object_a\n' + ' 0 123 456' + ' type/content object_a\n' ' 0\n') @mock.patch('swiftclient.service.makedirs') From 3b1f4fda721addc9f3c2c273091f76148ebae9d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20Lis=C3=A1k?= Date: Fri, 13 Nov 2015 17:20:20 +0100 Subject: [PATCH 053/454] Unification of manpages and docstrings * adding missing options * unification of help Change-Id: I2365e66433b63de8fd4da205611d9c1bf3bb6730 --- doc/manpages/swift.1 | 42 +++++++++++++++++++++++++++--------------- swiftclient/shell.py | 21 ++++++++++++--------- 2 files changed, 39 insertions(+), 24 deletions(-) diff --git a/doc/manpages/swift.1 b/doc/manpages/swift.1 index 4cfc23fd..8672a11d 100644 --- a/doc/manpages/swift.1 +++ b/doc/manpages/swift.1 @@ -36,25 +36,25 @@ several types of operations. .SH COMMANDS .PP -\fBstat\fR [\fIcontainer\fR] [\fIobject\fR] +\fBstat\fR [\fIcommand-options\fR] [\fIcontainer\fR] [\fIobject\fR] .RS 4 Displays information for the account, container, or object depending on the args given (if any). In verbose mode, the Storage URL and the authentication token are displayed -as well. +as well. Option \-\-lh reports sizes in human readable format similar to ls \-lh. .RE \fBlist\fR [\fIcommand-options\fR] [\fIcontainer\fR] .RS 4 Lists the containers for the account or the objects for a container. -The \-p or \-\-prefix is an option that will only list items beginning -with that prefix. The \-d or \-\-delimiter is option (for container listings only) -that will roll up items with the given delimiter (see OpenStack Swift general -documentation for what this means). +The \-p or \-\-prefix is an option that will only list items beginning +with that prefix. The \-d or \-\-delimiter is option +(for container listings only) that will roll up items with the given +delimiter (see OpenStack Swift general documentation for what this means). -The \-l and \-\-lh options provide more detail, similar to ls \-l and ls \-lh, the latter +The \-l or \-\-long and \-\-lh options provide more detail, similar to ls \-l and ls \-lh, the latter providing sizes in human readable format (eg 3K, 12M, etc). These latter 2 switches use more overhead to get those details, which is directly proportional to the number -of container or objects being listed. +of container or objects being listed. With the \-t or \-\-total option they only report totals. .RE \fBupload\fR [\fIcommand-options\fR] container file_or_directory [\fIfile_or_directory\fR] [...] @@ -64,7 +64,7 @@ remaining args. The \-c or \-\-changed is an option that will only upload files that have changed since the last upload. The \-\-object-name is an option that will upload file and name object to or upload dir and use as object prefix. The \-S or \-\-segment\-size -and \-\-leave\-segments are options as well (see \-\-help for more). +and \-\-leave\-segments and others are options as well (see swift upload \-\-help for more). .RE \fBpost\fR [\fIcommand-options\fR] [\fIcontainer\fR] [\fIobject\fR] @@ -75,6 +75,7 @@ automatically; but this is not true for accounts and objects. Containers also allow the \-r (or \-\-read\-acl) and \-w (or \-\-write\-acl) options. The \-m or \-\-meta option is allowed on all and used to define the user meta data items to set in the form Name:Value. This option can be repeated. +For more details and options see swift post \-\-help. \fBExample\fR: post \-m Color:Blue \-m Size:Large .RE @@ -83,9 +84,10 @@ items to set in the form Name:Value. This option can be repeated. Downloads everything in the account (with \-\-all), or everything in a container, or a list of objects depending on the args given. For a single object download, you may use the \-o [\-\-output] option to -redirect the output to a specific file or if "-" then just redirect to stdout. +redirect the output to a specific file or if "-" then just redirect to stdout or +with \-\-no-download actually not to write anything to disk. You can specify optional headers with the repeatable cURL-like option -\-H [\-\-header]. +\-H [\-\-header]. For more details and options see swift download \-\-help. .RE \fBdelete\fR [\fIcommand-options\fR] [\fIcontainer\fR] [\fIobject\fR] [\fIobject\fR] [...] @@ -93,6 +95,7 @@ You can specify optional headers with the repeatable cURL-like option Deletes everything in the account (with \-\-all), or everything in a container, or a list of objects depending on the args given. Segments of manifest objects will be deleted as well, unless you specify the \-\-leave\-segments option. +For more details and options see swift delete \-\-help. .RE \fBcapabilities\fR [\fIproxy-url\fR] @@ -104,7 +107,7 @@ is not provided the storage-url retrieved after authentication is used as proxy-url. .RE -\fBtempurl\fR \fImethod\fR \fIseconds\fR \fIpath\fR \fIkey\fR [\fI--absolute\fR] +\fBtempurl\fR [\fIcommand-option\fR] \fImethod\fR \fIseconds\fR \fIpath\fR \fIkey\fR .RS 4 Generates a temporary URL allowing unauthenticated access to the Swift object at the given path, using the given HTTP method, for the given number of @@ -114,20 +117,29 @@ should expire. \fBExample\fR: tempurl GET $(date -d "Jan 1 2016" +%s) /v1/AUTH_foo/bar_container/quux.md my_secret_tempurl_key --absolute .RE +\fBauth\fR +.RS 4 +Display auth related authentication variables in shell friendly format. +For examples see swift auth \-\-help. +.RE + .SH OPTIONS .PD 0 .IP "--version Show program's version number and exit" -.IP "-h, --help Show this (or any subcommand) help message and exit" +.IP "-h, --help Show this (or any subcommand if after command) help message and exit" .IP "-s, --snet Use SERVICENET internal network" .IP "-v, --verbose Print more info" .IP "-q, --quiet Suppress status output" .IP "-A AUTH, --auth=AUTH URL for obtaining an auth token " .IP "-U USER, --user=USER User name for obtaining an auth token" -.IP "-V 1|2 Authentication protocol version" +.IP "-V 1|2, --auth-version=VERSION Authentication protocol version" .IP "-K KEY, --key=KEY Key for obtaining an auth token" .IP "--os-storage-url=URL Use this instead of URL returned from auth" - +.IP "--os-help Show all OpenStack authentication options" .PD +.RS 4 +For more options see swift \-\-help and swift \-\-os-help. +.RE .SH EXAMPLE diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 15ca5706..0b8af5cb 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -54,15 +54,15 @@ def immediate_exit(signum, frame): st_delete_options = '''[--all] [--leave-segments] [--object-threads ] [--container-threads ] - [object] + [] [] [...] ''' st_delete_help = ''' Delete a container or objects within a container. Positional arguments: - Name of container to delete from. - [object] Name of object to delete. Specify multiple times + [] Name of container to delete from. + [] Name of object to delete. Specify multiple times for multiple objects. Optional arguments: @@ -156,7 +156,7 @@ def st_delete(parser, args, output_manager): [--object-threads ] [--container-threads ] [--no-download] [--skip-identical] [--remove-prefix] - [--header ] + [--header ] [--no-shuffle] ''' @@ -376,7 +376,7 @@ def st_download(parser, args, output_manager): st_list_options = '''[--long] [--lh] [--totals] [--prefix ] - [--delimiter ] + [--delimiter ] [container] ''' st_list_help = ''' @@ -390,8 +390,10 @@ def st_download(parser, args, output_manager): --lh Report sizes in human readable format similar to ls -lh. -t, --totals Used with -l or --lh, only report totals. - -p, --prefix Only list items beginning with the prefix. - -d, --delimiter Roll up items with the given delimiter. For containers + -p , --prefix + Only list items beginning with the prefix. + -d , --delimiter + Roll up items with the given delimiter. For containers only. See OpenStack Swift API documentation for what this means. '''.strip('\n') @@ -690,7 +692,7 @@ def st_post(parser, args, output_manager): [--object-threads ] [--segment-threads ] [--header
] [--use-slo] [--ignore-checksum] [--object-name ] - + [] [...] ''' st_upload_help = ''' Uploads specified files and directories to the given container. @@ -1006,7 +1008,8 @@ def st_auth(parser, args, thread_manager): print('export OS_AUTH_TOKEN=%s' % sh_quote(token)) -st_tempurl_options = ' ' +st_tempurl_options = '''[--absolute] + ''' st_tempurl_help = ''' From 7f304337a084a9e55ba376375699a963f4e68bd2 Mon Sep 17 00:00:00 2001 From: ricolin Date: Thu, 15 Oct 2015 17:58:03 +0800 Subject: [PATCH 054/454] improve readme contents Add more information in README.rst Change-Id: I9fb9bea648fb1d1e26b0db4b590f72ffc85b7a33 --- README.rst | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index c4fb8df1..3677406e 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,14 @@ Python bindings to the OpenStack Object Storage API =================================================== +.. image:: https://img.shields.io/pypi/v/python-swiftclient.svg + :target: https://pypi.python.org/pypi/python-swiftclient/ + :alt: Latest Version + +.. image:: https://img.shields.io/pypi/dm/python-swiftclient.svg + :target: https://pypi.python.org/pypi/python-swiftclient/ + :alt: Downloads + This is a python client for the Swift API. There's a Python API (the ``swiftclient`` module), and a command-line script (``swift``). @@ -16,9 +24,24 @@ Apache License like the rest of OpenStack. __ http://github.com/openstack/swift * Free software: Apache license -* Documentation: http://docs.openstack.org/developer/python-swiftclient/ -* Source: http://git.openstack.org/cgit/openstack/python-swiftclient/ -* Bugs: http://bugs.launchpad.net/python-swiftclient +* `PyPI`_ - package installation +* `Online Documentation`_ +* `Launchpad project`_ - release management +* `Blueprints`_ - feature specifications +* `Bugs`_ - issue tracking +* `Source`_ +* `Specs`_ +* `How to Contribute`_ + +.. _PyPI: https://pypi.python.org/pypi/python-swiftclient +.. _Online Documentation: http://docs.openstack.org/developer/python-swiftclient +.. _Launchpad project: https://launchpad.net/python-swiftclient +.. _Blueprints: https://blueprints.launchpad.net/python-swiftclient +.. _Bugs: https://bugs.launchpad.net/python-swiftclient +.. _Source: https://git.openstack.org/cgit/openstack/python-swiftclient +.. _How to Contribute: http://docs.openstack.org/infra/manual/developers.html +.. _Specs: http://specs.openstack.org/openstack/swift-specs/ + .. contents:: Contents: :local: From 562f386e931ca0e3720a566646267074f3a44b45 Mon Sep 17 00:00:00 2001 From: Alistair Coles Date: Tue, 24 Nov 2015 16:28:33 +0000 Subject: [PATCH 055/454] Update mailmap Change-Id: I0531928b531694008520298bea7d37b73216787e --- .mailmap | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.mailmap b/.mailmap index 4a6368f2..55553712 100644 --- a/.mailmap +++ b/.mailmap @@ -58,7 +58,7 @@ Madhuri Kumari madhuri Hua Zhang Yummy Bian -Alistair Coles +Alistair Coles Tong Li Paul Luse Yuan Zhou @@ -78,3 +78,8 @@ Jaivish Kothari Kazuhiro Miyahara Alexandra Settle +Mark Seger +Donagh McCabe +Stuart McLaren +Alexis Lee +Stanislaw Pitucha \ No newline at end of file From a3a78be87b88beca83a8cc0c96e209ab8e1a4189 Mon Sep 17 00:00:00 2001 From: Joel Wright Date: Mon, 8 Jun 2015 15:19:11 +0100 Subject: [PATCH 056/454] New API documentation for python-swiftclient New documentation for python-swiftclient that introduces the APIs available and gives some opinionated advice about when to use the shell, the client API and the service API. Change-Id: I19020f041fab2e72469979f712ffe3951c431d24 --- doc/source/apis.rst | 719 +++++++++++++++++++++++++++++++++++++ doc/source/index.rst | 39 +- swiftclient/client.py | 1 - swiftclient/service.py | 3 +- tests/unit/test_service.py | 1 + 5 files changed, 747 insertions(+), 16 deletions(-) create mode 100644 doc/source/apis.rst diff --git a/doc/source/apis.rst b/doc/source/apis.rst new file mode 100644 index 00000000..1a8e8f7d --- /dev/null +++ b/doc/source/apis.rst @@ -0,0 +1,719 @@ +============ +Introduction +============ + +The python-swiftclient includes two levels of API; a low level client API that +provides simple python wrappers around the various authentication mechanisms +and the individual HTTP requests, and a high level service API that provides +methods for performing common operations in parallel on a thread pool. + +This document aims to provide guidance for choosing between these APIs and +examples of usage for the service API. + +------------------------ +Important Considerations +------------------------ + +This section covers some important considerations, helpful hints, and things +to avoid when integrating an object store into your workflow. + +An Object Store is not a filesystem +----------------------------------- + +It cannot be stressed enough that your usage of the object store should reflect +the proper use case, and not treat the storage like a filesystem. There are 2 +main restrictions to bear in mind here when designing your use of the object +store: + + * Objects cannot be renamed due to the way in which objects are stored and + references by the object store. This usually requires multiple copies of + the data to be moved between physical storage devices. + As a result, a move operation is not provided. If the user wants to move an + object they must re-upload to the new location and delete the + original. + * Objects cannot be modified. Objects are stored in multiple locations and are + checked for integrity based on the ``MD5 sum`` calculated during upload. + Object creation is a 1-shot event, and in order to modify the contents of an + object the entire new contents must be re-uploaded. In certain special cases + it is possible to work around this restriction using large objects, but no + general file-like access is available to modify a stored object. + +------------------------------ +The swiftclient.Connection API +------------------------------ + +A low level API that provides methods for authentication and methods that +correspond to the individual REST API calls described in the swift +documentation. + +For usage details see the client docs: :mod:`swiftclient.client`. + +-------------------------------- +The swiftclient.SwiftService API +-------------------------------- + +A higher level API aimed at allowing developers an easy way to perform multiple +operations asynchronously using a configurable thread pool. Docs for each +service method call can be found here: :mod:`swiftclient.service`. + +Configuration +------------- + +When you create an instance of a ``SwiftService``, you can override a collection +of default options to suit your use case. Typically, the defaults are sensible to +get us started, but depending on your needs you might want to tweak them to +improve performance (options affecting large objects and thread counts can +significantly alter performance in the right situation). + +Service level defaults and some extra options can also be overridden on a +per-operation (or even in some cases per-object) basis, and you will call out +which options affect which operations later in the document. + +The configuration of the service API is performed using an options dictionary +passed to the ``SwiftService`` during initialisation. The options available +in this dictionary are described below, along with their defaults: + +Options +~~~~~~~ + + ``retries``: ``5`` + The number of times that the library should attempt to retry HTTP + actions before giving up and reporting a failure. + + ``container_threads``: ``10`` + + ``object_dd_threads``: ``10`` + + ``object_uu_threads``: ``10`` + + ``segment_threads``: ``10`` + The above options determine the size of the available thread pools for + performing swift operations. Container operations (such as listing a + container) operate in the container threads, and a similar pattern + applies to object and segment threads. + + .. note:: + + Object threads are separated into two separate thread pools: + ``uu`` and ``dd``. This stands for "upload/update" and "download/delete", + and the corresponding actions will be run on separate threads pools. + + ``segment_size``: ``None`` + If specified, this option enables uploading of large objects. Should the + object being uploaded be larger than 5G in size, this option is + mandatory otherwise the upload will fail. This option should be + specified as a size in bytes. + + ``use_slo``: ``False`` + Used in combination with the above option, ``use_slo`` will upload large + objects as static rather than dynamic. Only static large objects provide + error checking for the downloaded object, so we recommend this option. + + ``segment_container``: ``None`` + Allows the user to select the container into which large object segments + will be uploaded. We do not recommend changing this value as it could make + locating orphaned segments more difficult in the case of errors. + + ``leave_segments``: ``False`` + Setting this option to true means that when deleting or overwriting a large + object, its segments will be left in the object store and must be cleaned + up manually. This option can be useful when sharing large object segments + between multiple objects in more advanced scenarios, but must be treated + with care, as it could lead to ever increasing storage usage. + + ``changed``: ``None`` + This option affects uploads and simply means that those objects which + already exist in the object store will not be overwritten if the ``mtime`` + and size of the source is the same as the existing object. + + ``skip_identical``: ``False`` + A slightly more thorough case of the above, but rather than ``mtime`` and size + uses an object's ``MD5 sum``. + + ``yes_all``: ``False`` + This options affects only download and delete, and in each case must be + specified in order to download/delete the entire contents of an account. + This option has no effect on any other calls. + + ``no_download``: ``False`` + This option only affects download and means that all operations proceed as + normal with the exception that no data is written to disk. + + ``header``: ``[]`` + Used with upload and post operations to set headers on objects. Headers + are specified as colon separated strings, e.g. "content-type:text/plain". + + ``meta``: ``[]`` + Used to set metadata on an object similarly to headers. + + .. note:: + Setting metadata is a destructive operation, so when updating one + of many metadata values all desired metadata for an object must be re-applied. + + ``long``: ``False`` + Affects only list operations, and results in more metrics being made + available in the results at the expense of lower performance. + + ``fail_fast``: ``False`` + Applies to delete and upload operations, and attempts to abort queued + tasks in the event of errors. + + ``prefix``: ``None`` + Affects list operations; only objects with the given prefix will be + returned/affected. It is not advisable to set at the service level, as + those operations that call list to discover objects on which they should + operate will also be affected. + + ``delimiter``: ``None`` + Affects list operations, and means that listings only contain results up + to the first instance of the delimiter in the object name. This is useful + for working with objects containing '/' in their names to simulate folder + structures. + + ``dir_marker``: ``False`` + Affects uploads, and allows empty 'pseudofolder' objects to be created + when the source of an upload is ``None``. + + ``shuffle``: ``False`` + When downloading objects, the default behaviour of the CLI is to shuffle + lists of objects in order to spread the load on storage drives when multiple + clients are downloading the same files to multiple locations (e.g. in the + event of distributing an update). When using the ``SwiftService`` directly, + object downloads are scheduled in the same order as they appear in the container + listing. When combined with a single download thread this means that objects + are downloaded in lexically-sorted order. Setting this option to ``True`` + gives the same shuffling behaviour as the CLI. + +Other available options can be found in ``swiftclient/service.py`` in the +source code for ``python-swiftclient``. Each ``SwiftService`` method also allows +for an optional dictionary to override those specified at init time, and the +appropriate docstrings show which options modify each method's behaviour. + +Authentication +-------------- + +This section covers the various options for authenticating with a swift +object store. The combinations of options required for each authentication +version are detailed below. + +Version 1.0 Auth +~~~~~~~~~~~~~~~~ + + ``auth_version``: ``environ.get('ST_AUTH_VERSION')`` + + ``auth``: ``environ.get('ST_AUTH')`` + + ``user``: ``environ.get('ST_USER')`` + + ``key``: ``environ.get('ST_KEY')`` + + +Version 2.0 & 3.0 Auth +~~~~~~~~~~~~~~~~~~~~~~ + + ``auth_version``: ``environ.get('ST_AUTH_VERSION')`` + + ``os_username``: ``environ.get('OS_USERNAME')`` + + ``os_password``: ``environ.get('OS_PASSWORD')`` + + ``os_tenant_name``: ``environ.get('OS_TENANT_NAME')`` + + ``os_auth_url``: ``environ.get('OS_AUTH_URL')`` + +As is evident from the default values, if these options are not set explicitly +in the options dictionary, then they will default to the values of the given +environment variables. The ``SwiftService`` authentication automatically selects +the auth version based on the combination of options specified, but +having options from different auth versions can cause unexpected behaviour. + + .. note:: + + Leftover environment variables are a common source of confusion when + authorization fails. + +Operation Return Values +----------------------- + +Each operation provided by the service API may raise a ``SwiftError`` or +``ClientException`` for any call that fails completely (or a call which +performs only one operation at an account or container level). In the case of a +successful call an operation returns one of the following: + +* A dictionary detailing the results of a single operation. +* An iterator that produces result dictionaries (for calls that perform + multiple sub-operations). + +A result dictionary can indicate either the success or failure of an individual +operation (detailed in the ``success`` key), and will either contain the +successful result, or an ``error`` key detailing the error encountered +(usually an instance of Exception). + +An example result dictionary is given below: + +.. code-block:: python + + result = { + 'action': 'download_object', + 'success': True, + 'container': container, + 'object': obj, + 'path': path, + 'start_time': start_time, + 'finish_time': finish_time, + 'headers_receipt': headers_receipt, + 'auth_end_time': conn.auth_end_time, + 'read_length': bytes_read, + 'attempts': conn.attempts + } + +All the possible ``action`` values are detailed below: + +.. code-block:: python + + [ + 'stat_account', + 'stat_container', + 'stat_object', + 'post_account', + 'post_container', + 'post_object', + 'list_part', # list yields zero or more 'list_part' results + 'download_object', + 'create_container', # from upload + 'create_dir_marker', # from upload + 'upload_object', + 'upload_segment', + 'delete_container', + 'delete_object', + 'delete_segment', # from delete_object operations + 'capabilities', + ] + +Stat +---- + +Stat can be called against an account, a container, or a list of objects to +get account stats, container stats or information about the given objects. In +the first two cases a dictionary is returned containing the results of the +operation, and in the case of a list of object names being supplied, an +iterator over the results generated for each object is returned. + +Information returned includes the amount of data used by the given +object/container/account and any headers or metadata set (this includes +user set data as well as content-type and modification times). + +See :mod:`swiftclient.service.SwiftService.stat` for docs generated from the +method docstring. + +Valid calls for this method are as follows: + + * ``stat([options])``: Returns stats for the configured account. + * ``stat(, [options])``: Returns stats for the given container. + * ``stat(, , [options])``: Returns stats for each + of the given objects in the the given container (through the returned + iterator). + +Results from stat are dictionaries indicating the success or failure of each +operation. In the case of a successful stat against an account or container, +the method returns immediately with one of the following results: + +.. code-block:: python + + { + 'action': 'stat_account', + 'success': True, + 'items': items, + 'headers': headers + } + +.. code-block:: python + + { + 'action': 'stat_container', + 'container': , + 'success': True, + 'items': items, + 'headers': headers + } + +In the case of stat called against a list of objects, the method returns a +generator that returns the results of individual object stat operations as they +are performed on the thread pool: + +.. code-block:: python + + { + 'action': 'stat_object', + 'object': , + 'container': , + 'success': True, + 'items': items, + 'headers': headers + } + +In the case of a failure the dictionary returned will indicate that the +operation was not successful, and will include the keys below: + +.. code-block:: python + + { + 'action': <'stat_object'|'stat_container'|'stat_account'>, + 'object': <'object_name'>, # Only for stat with objects list + 'container': , # Only for stat with objects list or container + 'success': False, + 'error': , + 'traceback': , + 'error_timestamp': + } + +Example +~~~~~~~ + +The code below demonstrates the use of ``stat`` to retrieve the headers for a +given list of objects in a container using 20 threads. The code creates a +mapping from object name to headers. + +.. code-block:: python + + import logging + + from swiftclient.service import SwiftService + + logger = logging.getLogger() + _opts = {'object_dd_threads': 20} + with SwiftService(options=_opts) as swift: + container = 'container1' + objects = [ 'object_%s' % n for n in range(0,100) ] + header_data = {} + stats_it = swift.stat(container=container, objects=objects) + for stat_res in stats_it: + if stat_res['success']: + header_data[stat_res['object']] = stat_res['headers'] + else: + logger.error( + 'Failed to retrieve stats for %s' % stat_res['object'] + ) + +List +---- + +List can be called against an account or a container to retrieve the containers +or objects contained within them. Each call returns an iterator that returns +pages of results (by default, up to 10000 results in each page). + +See :mod:`swiftclient.service.SwiftService.list` for docs generated from the +method docstring. + +If the given container or account does not exist, the list method will raise +a ``SwiftError``, but for all other success/failures a dictionary is returned. +Each successfully listed page returns a dictionary as described below: + +.. code-block:: python + + { + 'action': <'list_account_part'|'list_container_part'>, + 'container': , # Only for listing a container + 'prefix': , # The prefix of returned objects/containers + 'success': True, + 'listing': [Item], # A list of results + # (only in the event of success) + 'marker': # The last item name in the list + # (only in the event of success) + } + +Where an item contains the following keys: + +.. code-block:: python + + { + 'name': , + 'bytes': 10485760, + 'last_modified': '2014-12-11T12:02:38.774540', + 'hash': 'fb938269cbeabe4c234e1127bbd3b74a', + 'content_type': 'application/octet-stream', + 'meta': # Full metadata listing from stat'ing each object + # this key only exists if 'long' is specified in options + } + +Any failure listing an account or container that exists will return a failure +dictionary as described below: + +.. code-block:: python + + { + 'action': <'list_account_part'|'list_container_part'>,, + 'container': container, # Only for listing a container + 'prefix': options['prefix'], + 'success': success, + 'marker': marker, + 'error': error, + 'traceback': , + 'error_timestamp': + } + +Example +~~~~~~~ + +The code below demonstrates the use of ``list`` to list all items in a +container that are over 10MiB in size: + +.. code-block:: python + + container = 'example_container' + minimum_size = 10*1024**2 + with SwiftService() as swift: + try: + stats_parts_gen = swift.list(container=container) + for stats in stats_parts_gen: + if stats["success"]: + for item in stats["listing"]: + i_size = int(item["bytes"]) + if i_size > minimum_size: + i_name = item["name"] + i_etag = item["hash"] + print( + "%s [size: %s] [etag: %s]" % + (i_name, i_size, i_etag) + ) + else: + raise stats["error"] + except SwiftError as e: + output_manager.error(e.value) + +Post +---- + +Post can be called against an account, container or list of objects in order to +update the metadata attached to the given items. Each element of the object list +may be a plain string of the object name, or a ``SwiftPostObject`` that +allows finer control over the options applied to each of the individual post +operations. In the first two cases a single dictionary is returned containing the +results of the operation, and in the case of a list of objects being supplied, +an iterator over the results generated for each object post is returned. If the +given container or account does not exist, the ``post`` method will raise a +``SwiftError``. + +When a string is given for the object name, the options + +Successful metadata update results are dictionaries as described below: + +.. code-block:: python + + { + 'action': <'post_account'|<'post_container'>|'post_object'>, + 'success': True, + 'container': , + 'object': , + 'headers': {}, + 'response_dict': + } + +.. note:: + Updating user metadata keys will not only add any specified keys, but + will also remove user metadata that has previously been set. This means + that each time user metadata is updated, the complete set of desired + key-value pairs must be specified. + +Example +~~~~~~~ + +.. Do we want to hide this section until it is complete? + +TBD + +Download +-------- + +.. Do we want to hide this section until it is complete? + +TBD + +Example +~~~~~~~ + +.. Do we want to hide this section until it is complete? + +TBD + +Upload +------ + +Upload is always called against an account and container and with a list of +objects to upload. Each element of the object list may be a plain string +detailing the path of the object to upload, or a ``SwiftUploadObject`` that +allows finer control over some aspects of the individual operations. + +When a simple string is supplied to specify a file to upload, the name of the +object uploaded is the full path of the specified file and the options used for +the upload are those supplied to the call to ``upload``. + +Constructing a ``SwiftUploadObject`` allows the user to supply an object name +for the uploaded file, and modify the options used by ``upload`` at the +granularity of invidivual files. + +If the given container or account does not exist, the ``upload`` method will +raise a ``SwiftError``, otherwise an iterator over the results generated for +each object upload is returned. + +See :mod:`swiftclient.service.SwiftService.upload` for docs generated from the +method docstring. + +For each successfully uploaded object (or object segment), the results returned +by the iterator will be a dictionary as described below: + +.. code-block:: python + + { + 'action': 'upload_object', + 'container': , + 'object': , + 'success': True, + 'status': <'uploaded'|'skipped-identical'|'skipped-changed'>, + 'attempts': , + 'response_dict': + } + + { + 'action': 'upload_segment', + 'for_container': , + 'for_object': , + 'segment_index': , + 'segment_size': , + 'segment_location': + 'segment_etag': , + 'log_line': + 'success': True, + 'response_dict': , + 'attempts': + } + +Any failure uploading an object will return a failure dictionary as described +below: + +.. code-block:: python + + { + 'action': 'upload_object', + 'container': , + 'object': , + 'success': False, + 'attempts': , + 'error': , + 'traceback': , + 'error_timestamp': , + 'response_dict': + } + + { + 'action': 'upload_segment', + 'for_container': , + 'for_object': , + 'segment_index': , + 'segment_size': , + 'segment_location': , + 'log_line': , + 'success': False, + 'error': , + 'traceback': , + 'error_timestamp': , + 'response_dict': , + 'attempts': + } + +Example +~~~~~~~ + +The code below demonstrates the use of ``upload`` to upload all files and +folders in ``/tmp``, and renaming each object by replacing ``/tmp`` in the +object or directory marker names with ``temporary-objects``: + +.. code-block:: python + + _opts['object_uu_threads'] = 20 + with SwiftService(options=_opts) as swift, OutputManager() as out_manager: + try: + # Collect all the files and folders in '/tmp' + objs = [] + dir_markers = [] + dir = '/tmp': + for (_dir, _ds, _fs) in walk(f): + if not (_ds + _fs): + dir_markers.append(_dir) + else: + objs.extend([join(_dir, _f) for _f in _fs]) + + # Now that we've collected all the required files and dir markers + # build the ``SwiftUploadObject``s for the call to upload + objs = [ + SwiftUploadObject( + o, object_name=o.replace( + '/tmp', 'temporary-objects', 1 + ) + ) for o in objs + ] + dir_markers = [ + SwiftUploadObject( + None, object_name=d.replace( + '/tmp', 'temporary-objects', 1 + ), options={'dir_marker': True} + ) for d in dir_markers + ] + + # Schedule uploads on the SwiftService thread pool and iterate + # over the results + for r in swift.upload(container, objs + dir_markers): + if r['success']: + if 'object' in r: + out_manager.print_msg(r['object']) + elif 'for_object' in r: + out_manager.print_msg( + '%s segment %s' % (r['for_object'], + r['segment_index']) + ) + else: + error = r['error'] + if r['action'] == "create_container": + out_manager.warning( + 'Warning: failed to create container ' + "'%s'%s", container, msg + ) + elif r['action'] == "upload_object": + out_manager.error( + "Failed to upload object %s to container %s: %s" % + (container, r['object'], error) + ) + else: + out_manager.error("%s" % error) + + except SwiftError as e: + out_manager.error(e.value) + +Delete +------ + +.. Do we want to hide this section until it is complete? + +TBD + +Example +~~~~~~~ + +.. Do we want to hide this section until it is complete? + +TBD + +Capabilities +------------ + +.. Do we want to hide this section until it is complete? + +TBD + +Example +~~~~~~~ + +.. Do we want to hide this section until it is complete? + +TBD + diff --git a/doc/source/index.rst b/doc/source/index.rst index 55ec112a..3b8535af 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -1,19 +1,13 @@ -SwiftClient Web -*************** +Welcome to the python-swiftclient Docs +************************************** - Copyright 2013 OpenStack, LLC. +Developer Documentation +======================= - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at +.. toctree:: + :maxdepth: 2 - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + apis Code-Generated Documentation ============================ @@ -23,10 +17,27 @@ Code-Generated Documentation swiftclient - Indices and tables ================== * :ref:`genindex` * :ref:`modindex` * :ref:`search` + +License +======= + + Copyright 2013 OpenStack, LLC. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/swiftclient/client.py b/swiftclient/client.py index 2e0cf72a..bd41ec1b 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -452,7 +452,6 @@ def get_auth(auth_url, user, key, **kwargs): auth_version = kwargs.get('auth_version', '1') os_options = kwargs.get('os_options', {}) - storage_url, token = None, None cacert = kwargs.get('cacert', None) insecure = kwargs.get('insecure', False) timeout = kwargs.get('timeout', None) diff --git a/swiftclient/service.py b/swiftclient/service.py index 3f1f7e59..8df13897 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -92,7 +92,7 @@ def process_options(options): # Use new-style args if old ones not present if not options['auth'] and options['os_auth_url']: options['auth'] = options['os_auth_url'] - if not options['user']and options['os_username']: + if not options['user'] and options['os_username']: options['user'] = options['os_username'] if not options['key'] and options['os_password']: options['key'] = options['os_password'] @@ -1628,6 +1628,7 @@ def _upload_segment_job(conn, path, container, segment_name, segment_start, res = { 'action': 'upload_segment', + 'for_container': container, 'for_object': obj_name, 'segment_index': segment_index, 'segment_size': segment_size, diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index 8eea4c3f..6304b82b 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -868,6 +868,7 @@ def test_upload_segment_job(self): type(mock_conn).attempts = mock.PropertyMock(return_value=2) expected_r = { 'action': 'upload_segment', + 'for_container': 'test_c', 'for_object': 'test_o', 'segment_index': 2, 'segment_size': 10, From cffdc9d3579d764500f613b74eaaba14520313bd Mon Sep 17 00:00:00 2001 From: ricolin Date: Wed, 2 Dec 2015 22:53:00 +0800 Subject: [PATCH 057/454] Remove py26 support As of mitaka, the infra team won't have the resources available to reasonably test py26, also the oslo team is dropping py26 support from their libraries. sine we rely on oslo for a lot of our work, and depend on infra for our CI, we should drop py26 support too. Closes-Bug: 1519510 Depends-On: I37116731db11449d0c374a6a83a3a43789a19d5f Change-Id: I776847ce77dfe82880f34d0b7804514e5aed3f8d --- setup.cfg | 1 - tox.ini | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 3d97de06..fdf6da71 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,7 +16,6 @@ classifier = Programming Language :: Python Programming Language :: Python :: 2 Programming Language :: Python :: 2.7 - Programming Language :: Python :: 2.6 Programming Language :: Python :: 3 Programming Language :: Python :: 3.3 diff --git a/tox.ini b/tox.ini index a4670156..e9049246 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py26,py27,py33,py34,py35,pypy,pep8 +envlist = py27,py33,py34,py35,pypy,pep8 minversion = 1.6 skipsdist = True From 2d6e96d2f948c582703ae7a997d21f7167da93e4 Mon Sep 17 00:00:00 2001 From: John Dickinson Date: Thu, 3 Dec 2015 14:05:58 -0800 Subject: [PATCH 058/454] authors and changelog update for 2.6.1 release Change-Id: Icefe2f62c2d2d41c5ee9764c51b0ae4ce1d9b3f3 --- .mailmap | 5 ++++- AUTHORS | 18 ++++++++++-------- ChangeLog | 19 +++++++++++++++++++ 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/.mailmap b/.mailmap index 55553712..c3ae3733 100644 --- a/.mailmap +++ b/.mailmap @@ -82,4 +82,7 @@ Mark Seger Donagh McCabe Stuart McLaren Alexis Lee -Stanislaw Pitucha \ No newline at end of file +Stanislaw Pitucha +Mahati Chamarthy +Peter Lisak +Doug Hellmann diff --git a/AUTHORS b/AUTHORS index 65a82cb3..5644db91 100644 --- a/AUTHORS +++ b/AUTHORS @@ -12,7 +12,7 @@ Sergio Cazzolato (sergio.j.cazzolato@intel.com) Mahati Chamarthy (mahati.chamarthy@gmail.com) Ray Chen (oldsharp@163.com) Taurus Cheung (Taurus.Cheung@harmonicinc.com) -Alistair Coles (alistair.coles@hp.com) +Alistair Coles (alistair.coles@hpe.com) Ian Cordasco (ian.cordasco@rackspace.com) Nick Craig-Wood (nick@craig-wood.com) Sean Dague (sean@dague.net) @@ -32,7 +32,7 @@ Thomas Goirand (thomas@goirand.fr) Davide Guerri (davide.guerri@hp.com) Romain Hardouin (romain_hardouin@yahoo.fr) Steven Hardy (shardy@redhat.com) -Doug Hellmann (doug.hellmann@dreamhost.com) +Doug Hellmann (doug@doughellmann.com) Greg Holt (gholt@rackspace.com) Charles Hsu (charles0126@gmail.com) Kun Huang (gareth@unitedstack.com) @@ -46,17 +46,18 @@ Jakub Krajcovic (jakub.krajcovic@gmail.com) David Kranz (david.kranz@qrclab.com) Sushil Kumar (sushil.kumar2@globallogic.com) Greg Lange (greglange@gmail.com) -Alexis Lee (alexisl@hp.com) +Alexis Lee (lxsli@hpe.com) Tong Li (litong01@us.ibm.com) +Peter Lisak (peter.lisak@firma.seznam.cz) Feng Liu (mefengliu23@gmail.com) Jing Liuqing (jing.liuqing@99cloud.net) Hemanth Makkapati (hemanth.makkapati@mailtrust.com) Steve Martinelli (stevemar@ca.ibm.com) Juan J. Martinez (juan@memset.com) -Donagh McCabe (donagh.mccabe@hp.com) +Donagh McCabe (donagh.mccabe@hpe.com) Ben McCann (ben@benmccann.com) Andy McCrae (andy.mccrae@gmail.com) -Stuart McLaren (stuart.mclaren@hp.com) +Stuart McLaren (stuart.mclaren@hpe.com) Samuel Merritt (sam@swiftstack.com) Jola Mirecka (jola.mirecka@hp.com) Hiroshi Miura (miurahr@nttdata.co.jp) @@ -66,13 +67,13 @@ Zhenguo Niu (zhenguo@unitedstack.com) Ondrej Novy (ondrej.novy@firma.seznam.cz) Alessandro Pilotti (apilotti@cloudbasesolutions.com) Alessandro Pilotti (ap@pilotti.it) -Stanislaw Pitucha (stanislaw.pitucha@hp.com) +Stanislaw Pitucha (stanislaw.pitucha@hpe.com) Dan Prince (dprince@redhat.com) +ricolin (rico.l@inwinstack.com) Li Riqiang (lrqrun@gmail.com) Hirokazu Sakata (h.sakata@staff.east.ntt.co.jp) Christian Schwede (cschwede@redhat.com) -Mark Seger (Mark.Seger@hp.com) -Mark Seger (mark.seger@hp.com) +Mark Seger (mark.seger@hpe.com) Chuck Short (chuck.short@canonical.com) David Shrewsbury (shrewsbury.dave@gmail.com) Pradeep Kumar Singh (pradeep.singh@nectechnologies.in) @@ -92,6 +93,7 @@ Wu Wenxiang (wu.wenxiang@99cloud.net) Mike Widman (mwidman@endurancewindpower.com) Joel Wright (joel.wright@sohonet.com) You Yamagata (bi.yamagata@gmail.com) +Qiu Yu (qiuyu@ebaysf.com) YangLei (yanglyy@cn.ibm.com) Pete Zaitcev (zaitcev@kotori.zaitcev.us) Jian Zhang (jian.zhang@intel.com) diff --git a/ChangeLog b/ChangeLog index 1602ba4b..9a3ce941 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,22 @@ +2.6.1 +----- + +* This is the very last release to support Python 2.6 + +* Added content type to CLI object list long-form output + +* client.get_container() and client.head_object now accept a headers parameter + +* Fixed bug when setting Content-Type on upload from CLI + +* Fixed bug when deleting DLOs with unicode characters + +* Updated man pages and docstrings + +* Suppress iso8601 logging in --debug output + +* Various other minor bug fixes and improvements. + 2.6.0 ----- From 2345ae54f1395783d39d31226a0e7a9ea7ac557a Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Mon, 16 Nov 2015 15:49:30 -0800 Subject: [PATCH 059/454] Stop passing attr to keystoneclient when there's no filter_value It was dropping warnings like "UserWarning: Providing attr without filter_value to get_urls() is deprecated as of the 1.7.0 release and may be removed in the 2.0.0 release. Either both should be provided or neither should be provided." Change-Id: Iead0bcf36b4a46bf465a55a33a21fd7f14f0ac40 --- swiftclient/client.py | 11 +++++++---- tests/unit/test_shell.py | 17 ++++++++++++----- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index 2e0cf72a..bcc1eec9 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -388,7 +388,7 @@ def get_auth_keystone(auth_url, user, key, os_options, **kwargs): insecure = kwargs.get('insecure', False) timeout = kwargs.get('timeout', None) auth_version = kwargs.get('auth_version', '2.0') - debug = logger.isEnabledFor(logging.DEBUG) and True or False + debug = logger.isEnabledFor(logging.DEBUG) ksclient, exceptions = _import_keystone_client(auth_version) @@ -419,11 +419,14 @@ def get_auth_keystone(auth_url, user, key, os_options, **kwargs): service_type = os_options.get('service_type') or 'object-store' endpoint_type = os_options.get('endpoint_type') or 'publicURL' try: + filter_kwargs = {} + if os_options.get('region_name'): + filter_kwargs['attr'] = 'region' + filter_kwargs['filter_value'] = os_options['region_name'] endpoint = _ksclient.service_catalog.url_for( - attr='region', - filter_value=os_options.get('region_name'), service_type=service_type, - endpoint_type=endpoint_type) + endpoint_type=endpoint_type, + **filter_kwargs) except exceptions.EndpointNotFound: raise ClientException('Endpoint for %s not found - ' 'have you specified a region?' % service_type) diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index cac4da20..f962c636 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -1450,6 +1450,7 @@ def _test_options_passed_to_keystone(self, cmd, opts, os_opts, self.assertTrue(flag in actual_args) self.assertTrue(actual_args[flag]) + check_attr = True # check args passed to ServiceCatalog.url_for() method self.assertEqual(len(fake_ks.client.service_catalog.calls), 1) actual_args = fake_ks.client.service_catalog.calls[0] @@ -1458,15 +1459,21 @@ def _test_options_passed_to_keystone(self, cmd, opts, os_opts, key = key.replace('-', '_') if key == 'region_name': key = 'filter_value' + if expected is None: + check_attr = False + self.assertNotIn(key, actual_args) + self.assertNotIn('attr', actual_args) + continue self.assertIn(key, actual_args) self.assertEqual(expected, actual_args[key], 'Expected %s for key %s, found %s' % (expected, key, actual_args[key])) - key, v = 'attr', 'region' - self.assertIn(key, actual_args) - self.assertEqual(v, actual_args[key], - 'Expected %s for key %s, found %s' - % (v, key, actual_args[key])) + if check_attr: + key, v = 'attr', 'region' + self.assertIn(key, actual_args) + self.assertEqual(v, actual_args[key], + 'Expected %s for key %s, found %s' + % (v, key, actual_args[key])) def _test_options(self, opts, os_opts, flags=None, no_auth=False): # repeat test for different commands using env and command line options From bf07a69e0e124d4d19a844a8e35a2d5831f23109 Mon Sep 17 00:00:00 2001 From: John Dickinson Date: Mon, 7 Dec 2015 14:55:27 -0800 Subject: [PATCH 060/454] fix release version Change-Id: I0237adbcbd6249bab12ab1624b78b537511fc971 --- ChangeLog | 5 +++-- setup.cfg | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 9a3ce941..1e7624c7 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,7 +1,8 @@ -2.6.1 +2.7.0 ----- -* This is the very last release to support Python 2.6 +* This is the very last release to support Python 2.6. Any further + development on the 2.7.x release series will only be for security bugfixes. * Added content type to CLI object list long-form output diff --git a/setup.cfg b/setup.cfg index 3d97de06..290c1bad 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,6 +19,8 @@ classifier = Programming Language :: Python :: 2.6 Programming Language :: Python :: 3 Programming Language :: Python :: 3.3 + Programming Language :: Python :: 3.4 + Programming Language :: Python :: 3.5 [global] setup-hooks = From 7c78c7bc2efb63bd17c54c6ed720b54bcaab0703 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Nov=C3=BD?= Date: Fri, 11 Dec 2015 21:06:13 +0100 Subject: [PATCH 061/454] Deprecated tox -downloadcache option removed Caching is enabled by default from pip version 6.0 More info: https://testrun.org/tox/latest/config.html#confval-downloadcache=path https://pip.pypa.io/en/stable/reference/pip_install/#caching Change-Id: I95015c79049633ed97714d6de8dd8f231bd15a03 --- tox.ini | 3 --- 1 file changed, 3 deletions(-) diff --git a/tox.ini b/tox.ini index e9049246..7847e381 100644 --- a/tox.ini +++ b/tox.ini @@ -23,9 +23,6 @@ commands = {posargs} [testenv:cover] commands = python setup.py testr --coverage -[tox:jenkins] -downloadcache = ~/cache/pip - [testenv:func] setenv = OS_TEST_PATH=tests.functional whitelist_externals = From 6bb97044c22b241c6b1e6d3e35df14131ca2547c Mon Sep 17 00:00:00 2001 From: shu-mutou Date: Wed, 2 Dec 2015 15:31:04 +0900 Subject: [PATCH 062/454] Delete python bytecode before every test run Because python creates pyc|pyo files and __pycache__ directories during tox runs, certain changes in the tree, like deletes of files, or switching branches, can create spurious errors. Change-Id: Ibaac514521bab11bbf552e0310d1203230c0d984 Closes-Bug: #1368661 --- tox.ini | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index a4670156..d7babc7c 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,11 @@ setenv = VIRTUAL_ENV={envdir} deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt -commands = python setup.py testr --testr-args="{posargs}" +commands = sh -c 'find . -not \( -type d -name .?\* -prune \) \ + \( -type d -name "__pycache__" -or -type f -name "*.py[co]" \) \ + -print0 | xargs -0 rm -rf' + python setup.py testr --testr-args="{posargs}" +whitelist_externals = sh passenv = SWIFT_* *_proxy [testenv:pep8] From 0103465fcb5d0119459569e7952cc1c580d045e2 Mon Sep 17 00:00:00 2001 From: Stuart McLaren Date: Tue, 15 Dec 2015 16:54:04 +0000 Subject: [PATCH 063/454] Test 'string' behaviour of get_object Add a unit test to test the 'string' like behaviour of get_object when it is called without resp_chunk_size set. Co-Authored-By: Clay Gerrard Change-Id: I496032a76036141d027c30b076c810b34bc6bef0 --- tests/unit/test_swiftclient.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index e39e05f5..317a6b58 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -769,6 +769,12 @@ def test_query_string(self): 'x-auth-token': 'asdf'}), ]) + def test_get_object_as_string(self): + c.http_connection = self.fake_http_connection(200, body='abcde') + __, resp = c.get_object('http://storage.example.com', 'TOKEN', + 'container_name', 'object_name') + self.assertEqual(resp, 'abcde') + def test_request_headers(self): c.http_connection = self.fake_http_connection(200) conn = c.http_connection('http://www.test.com') From ab65eef4ce4096410bdfec9ea7d8780f800321df Mon Sep 17 00:00:00 2001 From: hgangwx Date: Wed, 30 Dec 2015 14:25:22 +0800 Subject: [PATCH 064/454] Wrong usage of "an" Wrong usage of "an" in the messages: "the optional os_options paramater includes an non-empty" "We are allowing to have an tenant_name argument" Should be: "the optional os_options paramater includes a non-empty" "We are allowing to have a tenant_name argument" Totally 2 occurrences in python-swiftclient base code. Change-Id: I2f2f7e07432fedfee5ccb418d9505250b3fed597 --- swiftclient/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index 959ed8f0..925b3966 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -442,7 +442,7 @@ def get_auth(auth_url, user, key, **kwargs): :returns: a tuple, (storage_url, token) - N.B. if the optional os_options paramater includes an non-empty + N.B. if the optional os_options paramater includes a non-empty 'object_storage_url' key it will override the the default storage url returned by the auth service. @@ -472,7 +472,7 @@ def get_auth(auth_url, user, key, **kwargs): if user and not kwargs.get('tenant_name') and ':' in user: os_options['tenant_name'], user = user.split(':') - # We are allowing to have an tenant_name argument in get_auth + # We are allowing to have a tenant_name argument in get_auth # directly without having os_options if kwargs.get('tenant_name'): os_options['tenant_name'] = kwargs['tenant_name'] From 6da38adb8de0e26ee4a6d9353e45a05df12e155f Mon Sep 17 00:00:00 2001 From: SaiKiran Date: Wed, 30 Dec 2015 17:20:35 +0530 Subject: [PATCH 065/454] Replace assertEqual(arg, None) with assertIsNone(arg) In python-swiftclient some test cases using asserEqual(arg, None) instead of assertIsNone(arg).assertIsNone method provides clear error message. Change-Id: I4d673ede0965408344325c9c234c5c4b1ae4146a Closes-Bug: #1527556 --- tests/unit/test_swiftclient.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index 317a6b58..5431ccf2 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -226,8 +226,8 @@ class TestGetAuth(MockHttpTest): def test_ok(self): c.http_connection = self.fake_http_connection(200) url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf') - self.assertEqual(url, None) - self.assertEqual(token, None) + self.assertIsNone(url) + self.assertIsNone(token) def test_invalid_auth(self): self.assertRaises(c.ClientException, c.get_auth, @@ -720,7 +720,7 @@ class TestPutContainer(MockHttpTest): def test_ok(self): c.http_connection = self.fake_http_connection(200) value = c.put_container('http://www.test.com', 'token', 'container') - self.assertEqual(value, None) + self.assertIsNone(value) self.assertRequests([ ('PUT', '/container', '', { 'x-auth-token': 'token', @@ -745,7 +745,7 @@ class TestDeleteContainer(MockHttpTest): def test_ok(self): c.http_connection = self.fake_http_connection(200) value = c.delete_container('http://www.test.com', 'token', 'container') - self.assertEqual(value, None) + self.assertIsNone(value) self.assertRequests([ ('DELETE', '/container', '', { 'x-auth-token': 'token'}), @@ -2031,14 +2031,14 @@ class TestCloseConnection(MockHttpTest): def test_close_none(self): c.http_connection = self.fake_http_connection() conn = c.Connection('http://www.test.com', 'asdf', 'asdf') - self.assertEqual(conn.http_conn, None) + self.assertIsNone(conn.http_conn) conn.close() - self.assertEqual(conn.http_conn, None) + self.assertIsNone(conn.http_conn) def test_close_ok(self): url = 'http://www.test.com' conn = c.Connection(url, 'asdf', 'asdf') - self.assertEqual(conn.http_conn, None) + self.assertIsNone(conn.http_conn) conn.http_conn = c.http_connection(url) self.assertEqual(type(conn.http_conn), tuple) self.assertEqual(len(conn.http_conn), 2) @@ -2067,7 +2067,7 @@ def get_connection(self): conn.get_service_auth = self.get_service_auth self.assertEqual(conn.attempts, 0) - self.assertEqual(conn.service_token, None) + self.assertIsNone(conn.service_token) self.assertIs(type(conn), c.Connection) return conn From 62bfe10f58ce777a1b70549a641528b88f94f246 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Tue, 29 Dec 2015 16:54:05 -0800 Subject: [PATCH 066/454] Fix some typos Change-Id: Iaf7f30a7ae0c2ac76fc5cdcee31ea74c08ce601e --- ChangeLog | 6 +++--- swiftclient/client.py | 8 ++++---- swiftclient/shell.py | 2 +- swiftclient/version.py | 2 +- tests/unit/test_service.py | 10 +++++----- tests/unit/test_swiftclient.py | 4 ++-- tests/unit/utils.py | 2 +- 7 files changed, 17 insertions(+), 17 deletions(-) diff --git a/ChangeLog b/ChangeLog index 1e7624c7..4cbb8788 100644 --- a/ChangeLog +++ b/ChangeLog @@ -156,7 +156,7 @@ * Add context sensitive help * Relax requirement for tenant_name in get_auth() * replace string format arguments with function parameters -* Removed now unnecesary workaround for PyPy +* Removed now unnecessary workaround for PyPy * Use Emacs-friendly coding line * Remove extra double quote from docstring * Fix wrong assertions in unit tests @@ -179,7 +179,7 @@ * Make the function tests Python3-import friendly * Only encode metadata for user customed headers * Add functional tests for python-swiftclient -* Removed a duplicate word in a dostring +* Removed a duplicate word in a docstring * Mock auth_end_time in test_shell.test_download * Don't utf8 encode urls * Fixed several shell tests on Python3 @@ -285,7 +285,7 @@ * Make pbr only a build-time dependency * Add verbose output to all stat commands * assertEquals is deprecated, use assertEqual (H602) -* Skip sniffing and reseting if retry is disabled +* Skip sniffing and resetting if retry is disabled * user defined headers added to swift post queries 1.7.0 diff --git a/swiftclient/client.py b/swiftclient/client.py index 925b3966..34edd110 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -34,7 +34,7 @@ from swiftclient.utils import ( LengthWrapper, ReadableToIterable, parse_api_response) -# Defautl is 100, increase to 256 +# Default is 100, increase to 256 http_client._MAXHEADERS = 256 AUTH_VERSIONS_V1 = ('1.0', '1', 1) @@ -109,7 +109,7 @@ def parse_header_string(data): if isinstance(data, six.text_type): # Under Python2 requests only returns binary_type, but if we get # some stray text_type input, this should prevent unquote from - # interpretting %-encoded data as raw code-points. + # interpreting %-encoded data as raw code-points. data = data.encode('utf8') try: unquoted = unquote(data).decode('utf8') @@ -438,11 +438,11 @@ def get_auth(auth_url, user, key, **kwargs): Get authentication/authorization credentials. :kwarg auth_version: the api version of the supplied auth params - :kwarg os_options: a dict, the openstack idenity service options + :kwarg os_options: a dict, the openstack identity service options :returns: a tuple, (storage_url, token) - N.B. if the optional os_options paramater includes a non-empty + N.B. if the optional os_options parameter includes a non-empty 'object_storage_url' key it will override the the default storage url returned by the auth service. diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 8fc5c0c9..a2e96a4b 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -1028,7 +1028,7 @@ def st_auth(parser, args, thread_manager): "Temp-URL-Key:b3968d0207b54ece87cccc06515a89d4"\' Optional arguments: - --absolute Interpet the positional argument as a Unix + --absolute Interpret the positional argument as a Unix timestamp rather than a number of seconds in the future. '''.strip('\n') diff --git a/swiftclient/version.py b/swiftclient/version.py index 0be287d2..6f01f369 100644 --- a/swiftclient/version.py +++ b/swiftclient/version.py @@ -16,7 +16,7 @@ try: # First, try to get our version out of PKG-INFO. If we're installed, - # this'll let us find our version without pulling in pbr. After all, if + # this will let us find our version without pulling in pbr. After all, if # we're installed on a system, we're not in a Git-managed source tree, so # pbr doesn't really buy us anything. version_string = pkg_resources.get_provider( diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index 6304b82b..68184b28 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -124,7 +124,7 @@ def _consume(sr): sr = self.sr('path', BytesIO(b'body'), {}) _consume(sr) - # Check error is raised if expected etag doesnt match calculated md5. + # Check error is raised if expected etag doesn't match calculated md5. # md5 for a SwiftReader that has done nothing is # d41d8cd98f00b204e9800998ecf8427e i.e md5 of nothing sr = self.sr('path', BytesIO(b'body'), {'etag': 'doesntmatch'}) @@ -134,7 +134,7 @@ def _consume(sr): {'etag': '841a2d689ad86bd1611447453c22c6fc'}) _consume(sr) - # Check error is raised if SwiftReader doesnt read the same length + # Check error is raised if SwiftReader doesn't read the same length # as the content length it is created with sr = self.sr('path', BytesIO(b'body'), {'content-length': 5}) self.assertRaises(SwiftError, _consume, sr) @@ -279,7 +279,7 @@ def test_delete_object_exception(self): 'traceback': mock.ANY, 'error_timestamp': mock.ANY }) - # _delete_object doesnt populate attempts or response dict if it hits + # _delete_object doesn't populate attempts or response dict if it hits # an error. This may not be the correct behaviour. del expected_r['response_dict'], expected_r['attempts'] @@ -454,7 +454,7 @@ def test_process_options_defaults(self): def test_process_options_auth_version(self): # auth_version should be set to 2.0 # if it isnt already set to 3.0 - # and the v1 command line arguments arent present + # and the v1 command line arguments aren't present opt_c = self.opts.copy() # Check v3 isnt changed @@ -473,7 +473,7 @@ def test_process_options_auth_version(self): def test_process_options_new_style_args(self): # checks new style args are copied to old style - # when old style dont exist + # when old style don't exist opt_c = self.opts.copy() opt_c['auth'] = '' diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index 317a6b58..e837288e 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -1501,7 +1501,7 @@ def test_reauth(self): def get_auth(*args, **kwargs): # this mock, and by extension this test are not - # represenative of the unit under test. The real get_auth + # representative of the unit under test. The real get_auth # method will always return the os_option dict's # object_storage_url which will be overridden by the # preauthurl parameter to Connection if it is provided. @@ -2075,7 +2075,7 @@ def get_connection(self): def get_auth(self): # The real get_auth function will always return the os_option # dict's object_storage_url which will be overridden by the - # preauthurl paramater to Connection if it is provided. + # preauthurl parameter to Connection if it is provided. return self.os_options.get('object_storage_url'), 'token' def get_service_auth(self): diff --git a/tests/unit/utils.py b/tests/unit/utils.py index f3483ebe..0f013a8a 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -213,7 +213,7 @@ def setUp(self): self.fake_connect = None self.request_log = [] - # Capture output, since the test-runner stdout/stderr moneky-patching + # Capture output, since the test-runner stdout/stderr monkey-patching # won't cover the references to sys.stdout/sys.stderr in # swiftclient.multithreading self.capture_output = CaptureOutput() From 39b1a31d8a187534f54e32e9aec2cb2bb839a390 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Wed, 30 Dec 2015 11:15:02 -0800 Subject: [PATCH 067/454] Wrap raw iterators to ensure we send entire contents to server Currently, if you attempt to stream an upload from an iterator, as in def data(): yield 'foo' yield '' yield 'bar' conn.put_object('c', 'o', data()) ... requests will faithfully emit a zero-length chunk, ending the transfer. Swift will then close the connection, possibly (if Connection: keep-alive was set) after attempting to parse the next chunk as a new request. Now, Swift will receive all of the bytes from the iterable, and any zero-byte chunks will be ignored. This will be fixed in requests [1], but not until an eventual 3.0.0 release. [1] https://github.com/kennethreitz/requests/pull/2631 Change-Id: I19579ed7a0181ac3f488433e7c1839f7f7a040b8 --- swiftclient/client.py | 6 +++++- swiftclient/utils.py | 9 +++++++++ tests/unit/test_swiftclient.py | 20 ++++++++++++++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index 925b3966..ae723a37 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -32,7 +32,7 @@ from swiftclient import version as swiftclient_version from swiftclient.exceptions import ClientException from swiftclient.utils import ( - LengthWrapper, ReadableToIterable, parse_api_response) + iter_wrapper, LengthWrapper, ReadableToIterable, parse_api_response) # Defautl is 100, increase to 256 http_client._MAXHEADERS = 256 @@ -1126,6 +1126,10 @@ def put_object(url, token=None, container=None, name=None, contents=None, warn_msg = ('%s object has no "read" method, ignoring chunk_size' % type(contents).__name__) warnings.warn(warn_msg, stacklevel=2) + # Match requests's is_stream test + if hasattr(contents, '__iter__') and not isinstance(contents, ( + six.text_type, six.binary_type, list, tuple, dict)): + contents = iter_wrapper(contents) conn.request('PUT', path, contents, headers) resp = conn.getresponse() diff --git a/swiftclient/utils.py b/swiftclient/utils.py index 742ec06e..9d94b6b5 100644 --- a/swiftclient/utils.py +++ b/swiftclient/utils.py @@ -231,3 +231,12 @@ def read(self, *args, **kwargs): self.md5sum.update(chunk.encode()) return chunk + + +def iter_wrapper(iterable): + for chunk in iterable: + if len(chunk) == 0: + # If we emit an empty chunk, requests will go ahead and send it, + # causing the server to close the connection + continue + yield chunk diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index 317a6b58..1909c04f 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -984,6 +984,26 @@ def test_chunk_upload(self): data += chunk self.assertEqual(data, raw_data) + def test_iter_upload(self): + def data(): + for chunk in ('foo', '', 'bar'): + yield chunk + conn = c.http_connection(u'http://www.test.com/') + resp = MockHttpResponse(status=200) + conn[1].getresponse = resp.fake_response + conn[1]._request = resp._fake_request + + c.put_object(url='http://www.test.com', http_conn=conn, + contents=data()) + req_headers = resp.requests_params['headers'] + self.assertNotIn('Content-Length', req_headers) + req_data = resp.requests_params['data'] + self.assertTrue(hasattr(req_data, '__iter__')) + # If we emit an empty chunk, requests will go ahead and send it, + # causing the server to close the connection. So make sure we don't + # do that. + self.assertEqual(['foo', 'bar'], list(req_data)) + def test_md5_mismatch(self): conn = c.http_connection('http://www.test.com') resp = MockHttpResponse(status=200, verify=True, From 109e8f519f103334cc49479b3fb552084eb2929a Mon Sep 17 00:00:00 2001 From: Christian Schwede Date: Wed, 2 Dec 2015 09:49:50 +0000 Subject: [PATCH 068/454] Fix debug and info option parsing The debug and info options need to be set before a subcommand method is called, otherwise they are simply ignored. This is kind of irritating - other options (for example -U, -A, -K) are usable after a positional command. This patch fixes this, and commands like these are no longer ignoring --debug or --info: swift stat --debug swift list container --info Co-Authored-By: Alistair Coles Change-Id: Ib19b05deef7a015881f1eed4a3946025e16bf922 --- swiftclient/shell.py | 15 +++++++-------- tests/unit/test_shell.py | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 8fc5c0c9..010739c1 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -1067,6 +1067,13 @@ def parse_args(parser, args, enforce_requires=True): if not args: args = ['-h'] (options, args) = parser.parse_args(args) + if enforce_requires and (options.debug or options.info): + logging.getLogger("swiftclient") + if options.debug: + logging.basicConfig(level=logging.DEBUG) + logging.getLogger('iso8601').setLevel(logging.WARNING) + elif options.info: + logging.basicConfig(level=logging.INFO) if len(args) > 1 and args[1] == '--help': _help = globals().get('st_%s_help' % args[0], @@ -1415,14 +1422,6 @@ def main(arguments=None): signal.signal(signal.SIGINT, immediate_exit) - if options.debug or options.info: - logging.getLogger("swiftclient") - if options.debug: - logging.basicConfig(level=logging.DEBUG) - logging.getLogger('iso8601').setLevel(logging.WARNING) - elif options.info: - logging.basicConfig(level=logging.INFO) - with OutputManager() as output: parser.usage = globals()['st_%s_help' % args[0]] diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index cac4da20..4dec22c2 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -16,6 +16,7 @@ from genericpath import getmtime import hashlib +import logging import mock import os import tempfile @@ -1040,6 +1041,20 @@ def test_no_help(self): self.assertEqual(out.strip('\n'), expected) +@mock.patch.dict(os.environ, mocked_os_environ) +class TestOptionAfterPosArg(testtools.TestCase): + @mock.patch('logging.basicConfig') + @mock.patch('swiftclient.service.Connection') + def test_option_after_posarg(self, connection, mock_logging): + argv = ["", "stat", "--info"] + swiftclient.shell.main(argv) + mock_logging.assert_called_with(level=logging.INFO) + + argv = ["", "stat", "--debug"] + swiftclient.shell.main(argv) + mock_logging.assert_called_with(level=logging.DEBUG) + + class TestBase(testtools.TestCase): """ Provide some common methods to subclasses From 2c6f367035bd09978d0fbf8c959d03210e421a9b Mon Sep 17 00:00:00 2001 From: James Nzomo Date: Mon, 4 Jan 2016 16:09:29 +0300 Subject: [PATCH 069/454] Fix upload to pseudo-dir passed by arg This fix makes it possible to upload objects to pseudo-folders by passing the upload paths via arg regardless of whether the container or folder path exist or not. Change-Id: I575e58aa12adcf71cdaa70d025a0ea5c63f46903 Closes-Bug: #1478210 Partial-Bug: #1432734 Related-Bug: #1432734 --- swiftclient/service.py | 10 +++++++++- tests/unit/test_shell.py | 20 +++++++++++++++++++- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/swiftclient/service.py b/swiftclient/service.py index 8df13897..3d32fe75 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -1338,6 +1338,12 @@ def upload(self, container, objects, options=None): except ValueError: raise SwiftError('Segment size should be an integer value') + # Incase we have a psudeo-folder path for arg, derive + # the container name from the top path to ensure new folder creation + # and prevent spawning zero-byte objects shadowing pseudo-folders + # by name. + container_name = container.split('/', 1)[0] + # Try to create the container, just in case it doesn't exist. If this # fails, it might just be because the user doesn't have container PUT # permissions, so we'll ignore any error. If there's really a problem, @@ -1349,7 +1355,9 @@ def upload(self, container, objects, options=None): _header[POLICY] create_containers = [ self.thread_manager.container_pool.submit( - self._create_container_job, container, headers=policy_header + self._create_container_job, + container_name, + headers=policy_header ) ] diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index f962c636..662fbcc7 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -463,7 +463,7 @@ def test_upload(self, connection, walk): swiftclient.shell.main(argv) connection.return_value.put_container.assert_called_once_with( 'container', - {'X-Storage-Policy': mock.ANY}, + {'X-Storage-Policy': 'one'}, response_dict={}) connection.return_value.put_object.assert_called_with( @@ -475,6 +475,24 @@ def test_upload(self, connection, walk): 'X-Storage-Policy': 'one'}, response_dict={}) + # upload to pseudo-folder (via param) + argv = ["", "upload", "container/pseudo-folder/nested", self.tmpfile, + "-H", "X-Storage-Policy:one"] + swiftclient.shell.main(argv) + connection.return_value.put_container.assert_called_with( + 'container', + {'X-Storage-Policy': 'one'}, + response_dict={}) + + connection.return_value.put_object.assert_called_with( + 'container/pseudo-folder/nested', + self.tmpfile.lstrip('/'), + mock.ANY, + content_length=0, + headers={'x-object-meta-mtime': mock.ANY, + 'X-Storage-Policy': 'one'}, + response_dict={}) + # Upload whole directory argv = ["", "upload", "container", "/tmp"] _tmpfile = self.tmpfile From ab3460905084aeb0961ad8e1ca280ff6d6b6c9dd Mon Sep 17 00:00:00 2001 From: Alistair Coles Date: Tue, 5 Jan 2016 18:25:19 +0000 Subject: [PATCH 070/454] Add functional test for object PUT with raw iterator Adds a functional test to verify change made in [1] [1] change id I19579ed7a0181ac3f488433e7c1839f7f7a040b8 Change-Id: I45dbf66edab645e6339e67906aee5faa4fb7efbd --- tests/functional/test_swiftclient.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/functional/test_swiftclient.py b/tests/functional/test_swiftclient.py index 35a5ea7e..ef1b9d52 100644 --- a/tests/functional/test_swiftclient.py +++ b/tests/functional/test_swiftclient.py @@ -318,6 +318,21 @@ def test_download_object(self): downloaded_contents += body.read() self.assertEqual(self.test_data, downloaded_contents) + def test_put_object_using_generator(self): + # verify that put using a generator yielding empty strings does not + # cause connection to be closed + def data(): + yield "should" + yield "" + yield " tolerate" + yield "" + yield " empty chunks" + + self.conn.put_object( + self.containername, self.objectname, data()) + hdrs, body = self.conn.get_object(self.containername, self.objectname) + self.assertEqual("should tolerate empty chunks", body) + def test_post_account(self): self.conn.post_account({'x-account-meta-data': 'Something'}) headers = self.conn.head_account() From 21a841a00395c8ffd521457a53ff941a33559704 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Nov=C3=BD?= Date: Tue, 5 Jan 2016 20:24:26 +0100 Subject: [PATCH 071/454] Fixed few misspellings in comments Change-Id: I29d891e2dc900eb93e703f77f4e56f68710a8955 --- tests/unit/test_service.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index 68184b28..003a51f8 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -112,7 +112,7 @@ def test_create_with_content_length(self): self.assertNotEqual(sr._actual_md5, None) self.assertIs(type(sr._actual_md5), self.md5_type) - # Check Contentlength raises error if it isnt an integer + # Check Contentlength raises error if it isn't an integer self.assertRaises(SwiftError, self.sr, 'path', 'body', {'content-length': 'notanint'}) @@ -453,16 +453,16 @@ def test_process_options_defaults(self): def test_process_options_auth_version(self): # auth_version should be set to 2.0 - # if it isnt already set to 3.0 + # if it isn't already set to 3.0 # and the v1 command line arguments aren't present opt_c = self.opts.copy() - # Check v3 isnt changed + # Check v3 isn't changed opt_c['auth_version'] = '3' swiftclient.service.process_options(opt_c) self.assertEqual(opt_c['auth_version'], '3') - # Check v1 isnt changed if user, key and auth are set + # Check v1 isn't changed if user, key and auth are set opt_c = self.opts.copy() opt_c['auth_version'] = '1' opt_c['auth'] = True @@ -862,7 +862,7 @@ def test_upload_segment_job(self): # Mock the connection to return an empty etag. This # skips etag validation which would fail as the LengthWrapper - # isnt read from. + # isn't read from. mock_conn = mock.Mock() mock_conn.put_object.return_value = '' type(mock_conn).attempts = mock.PropertyMock(return_value=2) From 4af623bcf171a63240849b84b9359a4f74471455 Mon Sep 17 00:00:00 2001 From: Stuart McLaren Date: Wed, 4 Mar 2015 14:31:00 +0000 Subject: [PATCH 072/454] Retry download of object body Currently the swift client retries establishing a connection to the server (by default up to 5 times). However, when downloading an object, once the connection has been established and the inital headers have been returned, no attempt is made to retry. So, for example, if 99MB of a 100MB object have been downloaded and the connection is then lost, the download will fail. This patch changes the behaviour to re-establish the connection and fetch the remaining bytes using the 'Range' header to offset. Data retry is not yet supported if the original request is for a subset of the object data (ie uses the 'Range' header), or if resp_chunk_size has not been set. The object's etag is checked using If-Match to make sure the object data hasn't changed since the start of the download. Change-Id: Iab47f10081ff39f6d344dbc2479cbc3bfd1c5b29 --- swiftclient/client.py | 88 ++++++++++++++++++++++++++-- tests/functional/test_swiftclient.py | 52 ++++++++++++++++ tests/unit/test_swiftclient.py | 17 ++++++ 3 files changed, 151 insertions(+), 6 deletions(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index aba986e6..406149f4 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -187,7 +187,7 @@ def __iter__(self): return self def next(self): - buf = self.resp.read(self.chunk_size) + buf = self.read(self.chunk_size) if not buf: raise StopIteration() return buf @@ -196,6 +196,67 @@ def __next__(self): return self.next() +class _RetryBody(_ObjectBody): + """ + Wrapper for object body response which triggers a retry + (from offset) if the connection is dropped after partially + downloading the object. + """ + def __init__(self, resp, expected_length, etag, connection, container, obj, + resp_chunk_size=None, query_string=None, response_dict=None, + headers=None): + """ + Wrap the underlying response + + :param resp: the response to wrap + :param expected_length: the object size in bytes + :param etag: the object's etag + :param connection: Connection class instance + :param container: the name of the container the object is in + :param obj: the name of object we are downloading + :param resp_chunk_size: if defined, chunk size of data to read + :param query_string: if set will be appended with '?' to generated path + :param response_dict: an optional dictionary into which to place + the response - status, reason and headers + :param headers: an optional dictionary with additional headers to + include in the request + """ + super(_RetryBody, self).__init__(resp, resp_chunk_size) + self.expected_length = expected_length + self.expected_etag = etag + self.conn = connection + self.container = container + self.obj = obj + self.query_string = query_string + self.response_dict = response_dict + self.headers = headers if headers is not None else {} + self.bytes_read = 0 + + def read(self, length=None): + buf = None + try: + buf = self.resp.read(length) + self.bytes_read += len(buf) + except (socket.error, RequestException) as e: + if self.conn.attempts > self.conn.retries: + logger.exception(e) + raise + if (not buf and self.bytes_read < self.expected_length and + self.conn.attempts <= self.conn.retries): + self.headers['Range'] = 'bytes=%d-' % self.bytes_read + self.headers['If-Match'] = self.expected_etag + hdrs, body = self.conn._retry(None, get_object, + self.container, self.obj, + resp_chunk_size=self.chunk_size, + query_string=self.query_string, + response_dict=self.response_dict, + headers=self.headers, + attempts=self.conn.attempts) + self.resp = body + buf = self.read(length) + return buf + + class HTTPConnection(object): def __init__(self, url, proxy=None, cacert=None, insecure=False, ssl_compression=False, default_user_agent=None, timeout=None): @@ -1408,10 +1469,10 @@ def _add_response_dict(self, target_dict, kwargs): target_dict.update(response_dict) def _retry(self, reset_func, func, *args, **kwargs): - self.attempts = 0 retried_auth = False backoff = self.starting_backoff caller_response_dict = kwargs.pop('response_dict', None) + self.attempts = kwargs.pop('attempts', 0) while self.attempts <= self.retries: self.attempts += 1 try: @@ -1523,10 +1584,25 @@ def head_object(self, container, obj, headers=None): def get_object(self, container, obj, resp_chunk_size=None, query_string=None, response_dict=None, headers=None): """Wrapper for :func:`get_object`""" - return self._retry(None, get_object, container, obj, - resp_chunk_size=resp_chunk_size, - query_string=query_string, - response_dict=response_dict, headers=headers) + rheaders, body = self._retry(None, get_object, container, obj, + resp_chunk_size=resp_chunk_size, + query_string=query_string, + response_dict=response_dict, + headers=headers) + is_not_range_request = ( + not headers or 'range' not in (k.lower() for k in headers)) + retry_is_possible = ( + is_not_range_request and resp_chunk_size and + self.attempts <= self.retries) + if retry_is_possible: + body = _RetryBody(body.resp, int(rheaders['content-length']), + rheaders['etag'], + self, container, obj, + resp_chunk_size=resp_chunk_size, + query_string=query_string, + response_dict=response_dict, + headers=headers) + return rheaders, body def put_object(self, container, obj, contents, content_length=None, etag=None, chunk_size=None, content_type=None, diff --git a/tests/functional/test_swiftclient.py b/tests/functional/test_swiftclient.py index ef1b9d52..5f9e271f 100644 --- a/tests/functional/test_swiftclient.py +++ b/tests/functional/test_swiftclient.py @@ -333,6 +333,58 @@ def data(): hdrs, body = self.conn.get_object(self.containername, self.objectname) self.assertEqual("should tolerate empty chunks", body) + def test_download_object_retry_chunked(self): + resp_chunk_size = 2 + hdrs, body = self.conn.get_object(self.containername, + self.objectname, + resp_chunk_size=resp_chunk_size) + data = next(body) + self.assertEqual(self.test_data[:resp_chunk_size], data) + self.assertTrue(1, self.conn.attempts) + for chunk in body.resp: + # Flush remaining data from underlying response + # (simulate a dropped connection) + pass + # Trigger the retry + for chunk in body: + data += chunk + self.assertEqual(self.test_data, data) + self.assertEqual(2, self.conn.attempts) + + def test_download_object_retry_chunked_auth_failure(self): + resp_chunk_size = 2 + self.conn.token = 'invalid' + hdrs, body = self.conn.get_object(self.containername, + self.objectname, + resp_chunk_size=resp_chunk_size) + self.assertEqual(2, self.conn.attempts) + for chunk in body.resp: + # Flush remaining data from underlying response + # (simulate a dropped connection) + pass + + self.conn.token = 'invalid' + data = next(body) + self.assertEqual(4, self.conn.attempts) + + for chunk in body: + data += chunk + + self.assertEqual(self.test_data, data) + self.assertEqual(4, self.conn.attempts) + + def test_download_object_non_chunked(self): + hdrs, body = self.conn.get_object(self.containername, self.objectname) + data = body + self.assertEqual(self.test_data, data) + self.assertTrue(1, self.conn.attempts) + + hdrs, body = self.conn.get_object(self.containername, self.objectname, + resp_chunk_size=0) + data = body + self.assertEqual(self.test_data, data) + self.assertTrue(1, self.conn.attempts) + def test_post_account(self): self.conn.post_account({'x-account-meta-data': 'Something'}) headers = self.conn.head_account() diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index 1f6a7707..56d3ff86 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -835,6 +835,23 @@ def test_chunk_size_read_and_iter(self): self.assertRaises(StopIteration, next, resp) self.assertEqual(resp.read(), '') + def test_get_object_with_resp_chunk_size_zero(self): + def get_connection(self): + def get_auth(): + return 'http://auth.test.com', 'token' + + conn = c.Connection('http://www.test.com', 'asdf', 'asdf') + self.assertIs(type(conn), c.Connection) + conn.get_auth = get_auth + self.assertEqual(conn.attempts, 0) + return conn + + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200)): + conn = get_connection(self) + conn.get_object('container1', 'obj1', resp_chunk_size=0) + self.assertEqual(conn.attempts, 1) + class TestHeadObject(MockHttpTest): From d4157ce5b5eeeebb3516092de995cee20025a5c1 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Wed, 23 Sep 2015 10:42:43 -0700 Subject: [PATCH 073/454] Retry file uploads via SwiftService When we introduced LengthWrapper, we neglected to make it resettable. As a result, upload failures result in errors like: put_object(...) failure and no ability to reset contents for reupload. Now, LengthWrappers will be resettable if their _readable has seek/tell. Related-Change: I6c8bc1366dfb591a26d934a30cd21c9e6b9a04ce Change-Id: I21f43f06e8c78b24d1fc081efedf2687942e042f --- swiftclient/client.py | 4 ++- swiftclient/utils.py | 32 +++++++++++++++--- tests/unit/test_swiftclient.py | 62 ++++++++++++++++++++++++++-------- tests/unit/test_utils.py | 26 +++++++++++--- 4 files changed, 101 insertions(+), 23 deletions(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index aba986e6..172f5290 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -1546,10 +1546,12 @@ def _default_reset(*args, **kwargs): if self.retries > 0: tell = getattr(contents, 'tell', None) seek = getattr(contents, 'seek', None) + reset = getattr(contents, 'reset', None) if tell and seek: orig_pos = tell() reset_func = lambda *a, **k: seek(orig_pos) - + elif reset: + reset_func = reset return self._retry(reset_func, put_object, container, obj, contents, content_length=content_length, etag=etag, chunk_size=chunk_size, content_type=content_type, diff --git a/swiftclient/utils.py b/swiftclient/utils.py index 9d94b6b5..ef65bbba 100644 --- a/swiftclient/utils.py +++ b/swiftclient/utils.py @@ -202,27 +202,36 @@ class LengthWrapper(object): def __init__(self, readable, length, md5=False): """ :param readable: The filelike object to read from. - :param length: The maximum amount of content to that can be read from + :param length: The maximum amount of content that can be read from the filelike object before it is simulated to be empty. :param md5: Flag to enable calculating the MD5 of the content as it is read. """ - self.md5sum = hashlib.md5() if md5 else NoopMD5() + self._md5 = md5 + self._reset_md5() self._length = self._remaining = length self._readable = readable + self._can_reset = all(hasattr(readable, attr) + for attr in ('seek', 'tell')) + if self._can_reset: + self._start = readable.tell() def __len__(self): return self._length + def _reset_md5(self): + self.md5sum = hashlib.md5() if self._md5 else NoopMD5() + def get_md5sum(self): return self.md5sum.hexdigest() - def read(self, *args, **kwargs): + def read(self, size=-1): if self._remaining <= 0: return '' - chunk = self._readable.read(*args, **kwargs)[:self._remaining] + to_read = self._remaining if size < 0 else min(size, self._remaining) + chunk = self._readable.read(to_read) self._remaining -= len(chunk) try: @@ -232,6 +241,21 @@ def read(self, *args, **kwargs): return chunk + @property + def reset(self): + if self._can_reset: + return self._reset + raise AttributeError("%r object has no attribute 'reset'" % + type(self).__name__) + + def _reset(self, *args, **kwargs): + if not self._can_reset: + raise TypeError('%r object cannot be reset; needs both seek and ' + 'tell methods' % type(self._readable).__name__) + self._readable.seek(self._start) + self._reset_md5() + self._remaining = self._length + def iter_wrapper(iterable): for chunk in iterable: diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index 60f65c92..03f49a67 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -17,6 +17,7 @@ import mock import six import socket +import string import testtools import warnings import tempfile @@ -1774,23 +1775,24 @@ def test_reset_stream(self): class LocalContents(object): def __init__(self, tell_value=0): - self.already_read = False + self.data = six.BytesIO(string.ascii_letters.encode() * 10) + self.data.seek(tell_value) + self.reads = [] self.seeks = [] - self.tell_value = tell_value + self.tells = [] def tell(self): - return self.tell_value + self.tells.append(self.data.tell()) + return self.tells[-1] - def seek(self, position): - self.seeks.append(position) - self.already_read = False + def seek(self, position, mode=0): + self.seeks.append((position, mode)) + self.data.seek(position, mode) def read(self, size=-1): - if self.already_read: - return '' - else: - self.already_read = True - return 'abcdef' + read_data = self.data.read(size) + self.reads.append((size, read_data)) + return read_data class LocalConnection(object): @@ -1801,7 +1803,7 @@ def __init__(self, parsed_url=None): self.port = parsed_url.netloc def putrequest(self, *args, **kwargs): - self.send() + self.send('PUT', *args, **kwargs) def putheader(self, *args, **kwargs): return @@ -1810,6 +1812,13 @@ def endheaders(self, *args, **kwargs): return def send(self, *args, **kwargs): + data = kwargs.get('data') + if data is not None: + if hasattr(data, 'read'): + data.read() + else: + for datum in data: + pass raise socket.error('oops') def request(self, *args, **kwargs): @@ -1844,7 +1853,12 @@ def local_http_connection(url, proxy=None, cacert=None, conn.put_object('c', 'o', contents) except socket.error as err: exc = err - self.assertEqual(contents.seeks, [0]) + self.assertEqual(contents.tells, [0]) + self.assertEqual(contents.seeks, [(0, 0)]) + # four reads: two in the initial pass, two in the retry + self.assertEqual(4, len(contents.reads)) + self.assertEqual((65536, b''), contents.reads[1]) + self.assertEqual((65536, b''), contents.reads[3]) self.assertEqual(str(exc), 'oops') contents = LocalContents(tell_value=123) @@ -1853,9 +1867,29 @@ def local_http_connection(url, proxy=None, cacert=None, conn.put_object('c', 'o', contents) except socket.error as err: exc = err - self.assertEqual(contents.seeks, [123]) + self.assertEqual(contents.tells, [123]) + self.assertEqual(contents.seeks, [(123, 0)]) + # four reads: two in the initial pass, two in the retry + self.assertEqual(4, len(contents.reads)) + self.assertEqual((65536, b''), contents.reads[1]) + self.assertEqual((65536, b''), contents.reads[3]) self.assertEqual(str(exc), 'oops') + contents = LocalContents(tell_value=123) + wrapped_contents = swiftclient.utils.LengthWrapper( + contents, 6, md5=True) + exc = None + try: + conn.put_object('c', 'o', wrapped_contents) + except socket.error as err: + exc = err + self.assertEqual(contents.tells, [123]) + self.assertEqual(contents.seeks, [(123, 0)]) + self.assertEqual(contents.reads, [(6, b'tuvwxy')] * 2) + self.assertEqual(str(exc), 'oops') + self.assertEqual(md5(b'tuvwxy').hexdigest(), + wrapped_contents.get_md5sum()) + contents = LocalContents() contents.tell = None exc = None diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 4faac6d8..3439f4a2 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -219,9 +219,10 @@ def test_unicode(self): class TestLengthWrapper(testtools.TestCase): def test_stringio(self): - contents = six.StringIO(u'a' * 100) + contents = six.StringIO(u'a' * 50 + u'b' * 50) + contents.seek(22) data = u.LengthWrapper(contents, 42, True) - s = u'a' * 42 + s = u'a' * 28 + u'b' * 14 read_data = u''.join(iter(data.read, '')) self.assertEqual(42, len(data)) @@ -229,10 +230,19 @@ def test_stringio(self): self.assertEqual(s, read_data) self.assertEqual(md5(s.encode()).hexdigest(), data.get_md5sum()) + data.reset() + self.assertEqual(md5().hexdigest(), data.get_md5sum()) + + read_data = u''.join(iter(data.read, '')) + self.assertEqual(42, len(read_data)) + self.assertEqual(s, read_data) + self.assertEqual(md5(s.encode()).hexdigest(), data.get_md5sum()) + def test_bytesio(self): - contents = six.BytesIO(b'a' * 100) + contents = six.BytesIO(b'a' * 50 + b'b' * 50) + contents.seek(22) data = u.LengthWrapper(contents, 42, True) - s = b'a' * 42 + s = b'a' * 28 + b'b' * 14 read_data = b''.join(iter(data.read, '')) self.assertEqual(42, len(data)) @@ -272,3 +282,11 @@ def test_segmented_file(self): self.assertEqual(segment_length, len(read_data)) self.assertEqual(s, read_data) self.assertEqual(md5(s).hexdigest(), data.get_md5sum()) + + data.reset() + self.assertEqual(md5().hexdigest(), data.get_md5sum()) + read_data = b''.join(iter(data.read, '')) + self.assertEqual(segment_length, len(data)) + self.assertEqual(segment_length, len(read_data)) + self.assertEqual(s, read_data) + self.assertEqual(md5(s).hexdigest(), data.get_md5sum()) From 5050027610cb4c8f20c85b7c60c1cc7f2c44121c Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Fri, 8 Jan 2016 11:31:32 -0800 Subject: [PATCH 074/454] _RetryBody doesn't need to take explicit etag/content-length Also, don't try to do int(None) for chunk-encoded responses (like DLOs that are longer than a single container listing). Change-Id: Ibacd75d5ee46135d62388786903c895fda8ed3ba --- swiftclient/client.py | 18 +++++------- tests/unit/test_swiftclient.py | 52 ++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 11 deletions(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index 406149f4..a5c4dc3f 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -202,15 +202,13 @@ class _RetryBody(_ObjectBody): (from offset) if the connection is dropped after partially downloading the object. """ - def __init__(self, resp, expected_length, etag, connection, container, obj, + def __init__(self, resp, connection, container, obj, resp_chunk_size=None, query_string=None, response_dict=None, headers=None): """ Wrap the underlying response :param resp: the response to wrap - :param expected_length: the object size in bytes - :param etag: the object's etag :param connection: Connection class instance :param container: the name of the container the object is in :param obj: the name of object we are downloading @@ -222,8 +220,7 @@ def __init__(self, resp, expected_length, etag, connection, container, obj, include in the request """ super(_RetryBody, self).__init__(resp, resp_chunk_size) - self.expected_length = expected_length - self.expected_etag = etag + self.expected_length = int(self.resp.getheader('Content-Length')) self.conn = connection self.container = container self.obj = obj @@ -244,7 +241,7 @@ def read(self, length=None): if (not buf and self.bytes_read < self.expected_length and self.conn.attempts <= self.conn.retries): self.headers['Range'] = 'bytes=%d-' % self.bytes_read - self.headers['If-Match'] = self.expected_etag + self.headers['If-Match'] = self.resp.getheader('ETag') hdrs, body = self.conn._retry(None, get_object, self.container, self.obj, resp_chunk_size=self.chunk_size, @@ -252,7 +249,7 @@ def read(self, length=None): response_dict=self.response_dict, headers=self.headers, attempts=self.conn.attempts) - self.resp = body + self.resp = body.resp buf = self.read(length) return buf @@ -1593,11 +1590,10 @@ def get_object(self, container, obj, resp_chunk_size=None, not headers or 'range' not in (k.lower() for k in headers)) retry_is_possible = ( is_not_range_request and resp_chunk_size and - self.attempts <= self.retries) + self.attempts <= self.retries and + rheaders.get('transfer-encoding') is None) if retry_is_possible: - body = _RetryBody(body.resp, int(rheaders['content-length']), - rheaders['etag'], - self, container, obj, + body = _RetryBody(body.resp, self, container, obj, resp_chunk_size=resp_chunk_size, query_string=query_string, response_dict=response_dict, diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index 56d3ff86..b2453b0a 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -835,6 +835,58 @@ def test_chunk_size_read_and_iter(self): self.assertRaises(StopIteration, next, resp) self.assertEqual(resp.read(), '') + def test_chunk_size_iter_chunked_no_retry(self): + conn = c.Connection('http://auth.url/', 'some_user', 'some_key') + with mock.patch('swiftclient.client.get_auth_1_0') as mock_get_auth: + mock_get_auth.return_value = ('http://auth.url/', 'tToken') + c.http_connection = self.fake_http_connection( + 200, body='abcdef', headers={'Transfer-Encoding': 'chunked'}) + __, resp = conn.get_object('asdf', 'asdf', resp_chunk_size=2) + self.assertEqual(next(resp), 'ab') + # simulate a dropped connection + resp.resp.read() + self.assertRaises(StopIteration, next, resp) + + def test_chunk_size_iter_retry(self): + conn = c.Connection('http://auth.url/', 'some_user', 'some_key') + with mock.patch('swiftclient.client.get_auth_1_0') as mock_get_auth: + mock_get_auth.return_value = ('http://auth.url', 'tToken') + c.http_connection = self.fake_http_connection( + StubResponse(200, 'abcdef', {'etag': 'some etag', + 'content-length': '6'}), + StubResponse(206, 'cdef', {'etag': 'some etag', + 'content-length': '4'}), + StubResponse(206, 'ef', {'etag': 'some etag', + 'content-length': '2'}), + ) + __, resp = conn.get_object('asdf', 'asdf', resp_chunk_size=2) + self.assertEqual(next(resp), 'ab') + self.assertEqual(1, conn.attempts) + # simulate a dropped connection + resp.resp.read() + self.assertEqual(next(resp), 'cd') + self.assertEqual(2, conn.attempts) + # simulate a dropped connection + resp.resp.read() + self.assertEqual(next(resp), 'ef') + self.assertEqual(3, conn.attempts) + self.assertRaises(StopIteration, next, resp) + self.assertRequests([ + ('GET', '/asdf/asdf', '', { + 'x-auth-token': 'tToken', + }), + ('GET', '/asdf/asdf', '', { + 'range': 'bytes=2-', + 'if-match': 'some etag', + 'x-auth-token': 'tToken', + }), + ('GET', '/asdf/asdf', '', { + 'range': 'bytes=4-', + 'if-match': 'some etag', + 'x-auth-token': 'tToken', + }), + ]) + def test_get_object_with_resp_chunk_size_zero(self): def get_connection(self): def get_auth(): From 7a1e192803b8f4b739c9c3086bbfdc9a9c8d6753 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Thu, 11 Jun 2015 14:33:39 -0700 Subject: [PATCH 075/454] Use bulk-delete middleware when available When issuing `delete` commands that would require three or more individual deletes, check whether the cluster supports bulk deletes and use that if it's available. Additionally, a new option is added to the `delete` command: * --prefix Delete all objects that start with . This is similar to the --prefix option for the `list` command. Example: $ swift delete c --prefix obj_prefix/ ...will delete from container "c" all objects whose name begins with "obj_prefix/", such as "obj_prefix/foo" and "obj_prefix/bar". Change-Id: I6b9504848d6ef562cf4f570bbcd17db4e3da8264 --- doc/manpages/swift.1 | 1 + swiftclient/client.py | 18 +++- swiftclient/service.py | 165 +++++++++++++++++++++++-------- swiftclient/shell.py | 64 +++++++++--- swiftclient/utils.py | 10 ++ tests/unit/test_shell.py | 174 +++++++++++++++++++++++++++++++-- tests/unit/test_swiftclient.py | 37 ++++++- tests/unit/test_utils.py | 28 ++++++ 8 files changed, 432 insertions(+), 65 deletions(-) diff --git a/doc/manpages/swift.1 b/doc/manpages/swift.1 index 8672a11d..b9f99c4d 100644 --- a/doc/manpages/swift.1 +++ b/doc/manpages/swift.1 @@ -93,6 +93,7 @@ You can specify optional headers with the repeatable cURL-like option \fBdelete\fR [\fIcommand-options\fR] [\fIcontainer\fR] [\fIobject\fR] [\fIobject\fR] [...] .RS 4 Deletes everything in the account (with \-\-all), or everything in a container, +or all objects in a container that start with a given string (given by \-\-prefix), or a list of objects depending on the args given. Segments of manifest objects will be deleted as well, unless you specify the \-\-leave\-segments option. For more details and options see swift delete \-\-help. diff --git a/swiftclient/client.py b/swiftclient/client.py index e2f30f52..e1f52bf8 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -686,7 +686,7 @@ def head_account(url, token, http_conn=None, service_token=None): def post_account(url, token, headers, http_conn=None, response_dict=None, - service_token=None): + service_token=None, query_string=None, data=None): """ Update an account's metadata. @@ -698,17 +698,23 @@ def post_account(url, token, headers, http_conn=None, response_dict=None, :param response_dict: an optional dictionary into which to place the response - status, reason and headers :param service_token: service auth token + :param query_string: if set will be appended with '?' to generated path + :param data: an optional message body for the request :raises ClientException: HTTP POST request failed + :returns: resp_headers, body """ if http_conn: parsed, conn = http_conn else: parsed, conn = http_connection(url) method = 'POST' + path = parsed.path + if query_string: + path += '?' + query_string headers['X-Auth-Token'] = token if service_token: headers['X-Service-Token'] = service_token - conn.request(method, parsed.path, '', headers) + conn.request(method, path, data, headers) resp = conn.getresponse() body = resp.read() http_log((url, method,), {'headers': headers}, resp, body) @@ -723,6 +729,10 @@ def post_account(url, token, headers, http_conn=None, response_dict=None, http_status=resp.status, http_reason=resp.reason, http_response_content=body) + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value + return resp_headers, body def get_container(url, token, container, marker=None, limit=None, @@ -1541,9 +1551,11 @@ def get_account(self, marker=None, limit=None, prefix=None, prefix=prefix, end_marker=end_marker, full_listing=full_listing) - def post_account(self, headers, response_dict=None): + def post_account(self, headers, response_dict=None, + query_string=None, data=None): """Wrapper for :func:`post_account`""" return self._retry(None, post_account, headers, + query_string=query_string, data=data, response_dict=response_dict) def head_container(self, container, headers=None): diff --git a/swiftclient/service.py b/swiftclient/service.py index 3d32fe75..09245d32 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -12,7 +12,9 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import unicode_literals import logging + import os from concurrent.futures import as_completed, CancelledError, TimeoutError @@ -41,7 +43,7 @@ ) from swiftclient.utils import ( config_true_value, ReadableToIterable, LengthWrapper, EMPTY_ETAG, - parse_api_response, report_traceback + parse_api_response, report_traceback, n_groups ) from swiftclient.exceptions import ClientException from swiftclient.multithreading import MultiThreadingManager @@ -380,6 +382,7 @@ def __init__(self, options=None): object_uu_threads=self._options['object_uu_threads'], container_threads=self._options['container_threads'] ) + self.capabilities_cache = {} # Each instance should have its own cache def __enter__(self): self.thread_manager.__enter__() @@ -2040,13 +2043,14 @@ def delete(self, container=None, objects=None, options=None): { 'yes_all': False, 'leave_segments': False, + 'prefix': None, } :returns: A generator for returning the results of the delete operations. Each result yielded from the generator is either - a 'delete_container', 'delete_object' or 'delete_segment' - dictionary containing the results of an individual delete - operation. + a 'delete_container', 'delete_object', 'delete_segment', or + 'bulk_delete' dictionary containing the results of an + individual delete operation. :raises: ClientException :raises: SwiftError @@ -2056,19 +2060,24 @@ def delete(self, container=None, objects=None, options=None): else: options = self._options - rq = Queue() if container is not None: if objects is not None: + if options['prefix']: + objects = [obj for obj in objects + if obj.startswith(options['prefix'])] + rq = Queue() obj_dels = {} - for obj in objects: - obj_del = self.thread_manager.object_dd_pool.submit( - self._delete_object, container, obj, options, - results_queue=rq - ) - obj_details = {'container': container, 'object': obj} - obj_dels[obj_del] = obj_details - # Start a thread to watch for upload results + if self._should_bulk_delete(objects): + for obj_slice in n_groups( + objects, self._options['object_dd_threads']): + self._bulk_delete(container, obj_slice, options, + obj_dels) + else: + self._per_item_delete(container, objects, options, + obj_dels, rq) + + # Start a thread to watch for delete results Thread( target=self._watch_futures, args=(obj_dels, rq) ).start() @@ -2091,6 +2100,8 @@ def delete(self, container=None, objects=None, options=None): else: if objects: raise SwiftError('Objects specified without container') + if options['prefix']: + raise SwiftError('Prefix specified without container') if options['yes_all']: cancelled = False containers = [] @@ -2114,6 +2125,33 @@ def delete(self, container=None, objects=None, options=None): and not res['success']): cancelled = True + def _should_bulk_delete(self, objects): + if len(objects) < 2 * self._options['object_dd_threads']: + # Not many objects; may as well delete one-by-one + return False + + try: + cap_result = self.capabilities() + if not cap_result['success']: + # This shouldn't actually happen, but just in case we start + # being more nuanced about our capabilities result... + return False + except ClientException: + # Old swift, presumably; assume no bulk middleware + return False + + swift_info = cap_result['capabilities'] + return 'bulk_delete' in swift_info + + def _per_item_delete(self, container, objects, options, rdict, rq): + for obj in objects: + obj_del = self.thread_manager.object_dd_pool.submit( + self._delete_object, container, obj, options, + results_queue=rq + ) + obj_details = {'container': container, 'object': obj} + rdict[obj_del] = obj_details + @staticmethod def _delete_segment(conn, container, obj, results_queue=None): results_dict = {} @@ -2242,18 +2280,20 @@ def _delete_empty_container(conn, container): def _delete_container(self, container, options): try: - for part in self.list(container=container): - if part["success"]: - objs = [o['name'] for o in part['listing']] + for part in self.list(container=container, options=options): + if not part["success"]: - o_dels = self.delete( - container=container, objects=objs, options=options - ) - for res in o_dels: - yield res - else: raise part["error"] + for res in self.delete( + container=container, + objects=[o['name'] for o in part['listing']], + options=options): + yield res + if options['prefix']: + # We're only deleting a subset of objects within the container + return + con_del = self.thread_manager.container_pool.submit( self._delete_empty_container, container ) @@ -2274,9 +2314,55 @@ def _delete_container(self, container, options): yield con_del_res + # Bulk methods + # + def _bulk_delete(self, container, objects, options, rdict): + if objects: + bulk_del = self.thread_manager.object_dd_pool.submit( + self._bulkdelete, container, objects, options + ) + bulk_details = {'container': container, 'objects': objects} + rdict[bulk_del] = bulk_details + + @staticmethod + def _bulkdelete(conn, container, objects, options): + results_dict = {} + try: + headers = { + 'Accept': 'application/json', + 'Content-Type': 'text/plain', + } + res = {'container': container, 'objects': objects} + objects = [quote(('/%s/%s' % (container, obj)).encode('utf-8')) + for obj in objects] + headers, body = conn.post_account( + headers=headers, + query_string='bulk-delete', + data=b''.join(obj.encode('utf-8') + b'\n' for obj in objects), + response_dict=results_dict) + if body: + res.update({'success': True, + 'result': parse_api_response(headers, body)}) + else: + res.update({ + 'success': False, + 'error': SwiftError( + 'No content received on account POST. ' + 'Is the bulk operations middleware enabled?')}) + except Exception as e: + res.update({'success': False, 'error': e}) + + res.update({ + 'action': 'bulk_delete', + 'attempts': conn.attempts, + 'response_dict': results_dict + }) + + return res + # Capabilities related methods # - def capabilities(self, url=None): + def capabilities(self, url=None, refresh_cache=False): """ List the cluster capabilities. @@ -2285,30 +2371,29 @@ def capabilities(self, url=None): :returns: A dictionary containing the capabilities of the cluster. :raises: ClientException - :raises: SwiftError """ + if not refresh_cache and url in self.capabilities_cache: + return self.capabilities_cache[url] + res = { - 'action': 'capabilities' + 'action': 'capabilities', + 'timestamp': time(), } - try: - cap = self.thread_manager.container_pool.submit( - self._get_capabilities, url - ) - capabilities = get_future_result(cap) + cap = self.thread_manager.container_pool.submit( + self._get_capabilities, url + ) + capabilities = get_future_result(cap) + res.update({ + 'success': True, + 'capabilities': capabilities + }) + if url is not None: res.update({ - 'success': True, - 'capabilities': capabilities + 'url': url }) - if url is not None: - res.update({ - 'url': url - }) - except ClientException as err: - if err.http_status != 404: - raise err - raise SwiftError('Account not found', exc=err) + self.capabilities_cache[url] = res return res @staticmethod diff --git a/swiftclient/shell.py b/swiftclient/shell.py index a2e96a4b..55bd138a 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -23,7 +23,8 @@ from optparse import OptionParser, OptionGroup, SUPPRESS_HELP from os import environ, walk, _exit as os_exit from os.path import isfile, isdir, join -from six import text_type +from six import text_type, PY2 +from six.moves.urllib.parse import unquote from sys import argv as sys_argv, exit, stderr from time import gmtime, strftime @@ -81,6 +82,9 @@ def st_delete(parser, args, output_manager): parser.add_option( '-a', '--all', action='store_true', dest='yes_all', default=False, help='Delete all containers and objects.') + parser.add_option( + '-p', '--prefix', dest='prefix', + help='Only delete items beginning with the .') parser.add_option( '', '--leave-segments', action='store_true', dest='leave_segments', default=False, @@ -128,25 +132,55 @@ def st_delete(parser, args, output_manager): o = r.get('object', '') a = r.get('attempts') - if r['success']: - if options.verbose: - a = ' [after {0} attempts]'.format(a) if a > 1 else '' - - if r['action'] == 'delete_object': + if r['action'] == 'bulk_delete': + if r['success']: + objs = r.get('objects', []) + for o, err in r.get('result', {}).get('Errors', []): + # o will be of the form quote("//") + o = unquote(o) + if PY2: + # In PY3, unquote(unicode) uses utf-8 like we + # want, but PY2 uses latin-1 + o = o.encode('latin-1').decode('utf-8') + output_manager.error('Error Deleting: {0}: {1}' + .format(o[1:], err)) + try: + objs.remove(o[len(c) + 2:]) + except ValueError: + # shouldn't happen, but ignoring it won't hurt + pass + + for o in objs: if options.yes_all: p = '{0}/{1}'.format(c, o) else: p = o - elif r['action'] == 'delete_segment': - p = '{0}/{1}'.format(c, o) - elif r['action'] == 'delete_container': - p = c - - output_manager.print_msg('{0}{1}'.format(p, a)) + output_manager.print_msg('{0}{1}'.format(p, a)) + else: + for o in r.get('objects', []): + output_manager.error('Error Deleting: {0}/{1}: {2}' + .format(c, o, r['error'])) else: - p = '{0}/{1}'.format(c, o) if o else c - output_manager.error('Error Deleting: {0}: {1}' - .format(p, r['error'])) + if r['success']: + if options.verbose: + a = (' [after {0} attempts]'.format(a) + if a > 1 else '') + + if r['action'] == 'delete_object': + if options.yes_all: + p = '{0}/{1}'.format(c, o) + else: + p = o + elif r['action'] == 'delete_segment': + p = '{0}/{1}'.format(c, o) + elif r['action'] == 'delete_container': + p = c + + output_manager.print_msg('{0}{1}'.format(p, a)) + else: + p = '{0}/{1}'.format(c, o) if o else c + output_manager.error('Error Deleting: {0}: {1}' + .format(p, r['error'])) except SwiftError as err: output_manager.error(err.value) diff --git a/swiftclient/utils.py b/swiftclient/utils.py index ef65bbba..0abaed6f 100644 --- a/swiftclient/utils.py +++ b/swiftclient/utils.py @@ -264,3 +264,13 @@ def iter_wrapper(iterable): # causing the server to close the connection continue yield chunk + + +def n_at_a_time(seq, n): + for i in range(0, len(seq), n): + yield seq[i:i + n] + + +def n_groups(seq, n): + items_per_group = ((len(seq) - 1) // n) + 1 + return n_at_a_time(seq, items_per_group) diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index 662fbcc7..13c26634 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -693,26 +693,148 @@ def test_upload_segments_to_same_container(self, connection): 'x-object-meta-mtime': mock.ANY}, response_dict={}) + @mock.patch.object(swiftclient.service.SwiftService, '_should_bulk_delete', + lambda *a: False) @mock.patch('swiftclient.service.Connection') def test_delete_account(self, connection): connection.return_value.get_account.side_effect = [ - [None, [{'name': 'container'}]], + [None, [{'name': 'container'}, {'name': 'container2'}]], + [None, [{'name': 'empty_container'}]], [None, []], ] connection.return_value.get_container.side_effect = [ + [None, [{'name': 'object'}, {'name': 'obj\xe9ct2'}]], + [None, []], [None, [{'name': 'object'}]], [None, []], + [None, []], ] connection.return_value.attempts = 0 argv = ["", "delete", "--all"] connection.return_value.head_object.return_value = {} swiftclient.shell.main(argv) - connection.return_value.delete_container.assert_called_with( - 'container', response_dict={}) - connection.return_value.delete_object.assert_called_with( - 'container', 'object', query_string=None, response_dict={}) + self.assertEqual( + connection.return_value.delete_object.mock_calls, [ + mock.call('container', 'object', query_string=None, + response_dict={}), + mock.call('container', 'obj\xe9ct2', query_string=None, + response_dict={}), + mock.call('container2', 'object', query_string=None, + response_dict={})]) + self.assertEqual( + connection.return_value.delete_container.mock_calls, [ + mock.call('container', response_dict={}), + mock.call('container2', response_dict={}), + mock.call('empty_container', response_dict={})]) + + @mock.patch.object(swiftclient.service.SwiftService, '_should_bulk_delete', + lambda *a: True) + @mock.patch('swiftclient.service.Connection') + def test_delete_bulk_account(self, connection): + connection.return_value.get_account.side_effect = [ + [None, [{'name': 'container'}, {'name': 'container2'}]], + [None, [{'name': 'empty_container'}]], + [None, []], + ] + connection.return_value.get_container.side_effect = [ + [None, [{'name': 'object'}, {'name': 'obj\xe9ct2'}, + {'name': 'object3'}]], + [None, []], + [None, [{'name': 'object'}]], + [None, []], + [None, []], + ] + connection.return_value.attempts = 0 + argv = ["", "delete", "--all", "--object-threads", "2"] + connection.return_value.post_account.return_value = {}, ( + b'{"Number Not Found": 0, "Response Status": "200 OK", ' + b'"Errors": [], "Number Deleted": 1, "Response Body": ""}') + swiftclient.shell.main(argv) + self.assertEqual( + connection.return_value.post_account.mock_calls, [ + mock.call(query_string='bulk-delete', + data=b'/container/object\n/container/obj%C3%A9ct2\n', + headers={'Content-Type': 'text/plain', + 'Accept': 'application/json'}, + response_dict={}), + mock.call(query_string='bulk-delete', + data=b'/container/object3\n', + headers={'Content-Type': 'text/plain', + 'Accept': 'application/json'}, + response_dict={}), + mock.call(query_string='bulk-delete', + data=b'/container2/object\n', + headers={'Content-Type': 'text/plain', + 'Accept': 'application/json'}, + response_dict={})]) + self.assertEqual( + connection.return_value.delete_container.mock_calls, [ + mock.call('container', response_dict={}), + mock.call('container2', response_dict={}), + mock.call('empty_container', response_dict={})]) @mock.patch('swiftclient.service.Connection') + def test_delete_bulk_account_with_capabilities(self, connection): + connection.return_value.get_capabilities.return_value = { + 'bulk_delete': { + 'max_deletes_per_request': 10000, + 'max_failed_deletes': 1000, + }, + } + connection.return_value.get_account.side_effect = [ + [None, [{'name': 'container'}]], + [None, [{'name': 'container2'}]], + [None, [{'name': 'empty_container'}]], + [None, []], + ] + connection.return_value.get_container.side_effect = [ + [None, [{'name': 'object'}, {'name': 'obj\xe9ct2'}, + {'name': 'z_object'}, {'name': 'z_obj\xe9ct2'}]], + [None, []], + [None, [{'name': 'object'}, {'name': 'obj\xe9ct2'}, + {'name': 'z_object'}, {'name': 'z_obj\xe9ct2'}]], + [None, []], + [None, []], + ] + connection.return_value.attempts = 0 + argv = ["", "delete", "--all", "--object-threads", "1"] + connection.return_value.post_account.return_value = {}, ( + b'{"Number Not Found": 0, "Response Status": "200 OK", ' + b'"Errors": [], "Number Deleted": 1, "Response Body": ""}') + swiftclient.shell.main(argv) + self.assertEqual( + connection.return_value.post_account.mock_calls, [ + mock.call(query_string='bulk-delete', + data=b''.join([ + b'/container/object\n', + b'/container/obj%C3%A9ct2\n', + b'/container/z_object\n', + b'/container/z_obj%C3%A9ct2\n' + ]), + headers={'Content-Type': 'text/plain', + 'Accept': 'application/json'}, + response_dict={}), + mock.call(query_string='bulk-delete', + data=b''.join([ + b'/container2/object\n', + b'/container2/obj%C3%A9ct2\n', + b'/container2/z_object\n', + b'/container2/z_obj%C3%A9ct2\n' + ]), + headers={'Content-Type': 'text/plain', + 'Accept': 'application/json'}, + response_dict={})]) + self.assertEqual( + connection.return_value.delete_container.mock_calls, [ + mock.call('container', response_dict={}), + mock.call('container2', response_dict={}), + mock.call('empty_container', response_dict={})]) + self.assertEqual(connection.return_value.get_capabilities.mock_calls, + [mock.call(None)]) # only one /info request + + @mock.patch.object(swiftclient.service.SwiftService, '_should_bulk_delete', + lambda *a: False) + @mock.patch('swiftclient.service.Connection') def test_delete_container(self, connection): connection.return_value.get_container.side_effect = [ [None, [{'name': 'object'}]], @@ -727,6 +849,28 @@ def test_delete_container(self, connection): connection.return_value.delete_object.assert_called_with( 'container', 'object', query_string=None, response_dict={}) + @mock.patch.object(swiftclient.service.SwiftService, '_should_bulk_delete', + lambda *a: True) + @mock.patch('swiftclient.service.Connection') + def test_delete_bulk_container(self, connection): + connection.return_value.get_container.side_effect = [ + [None, [{'name': 'object'}]], + [None, []], + ] + connection.return_value.attempts = 0 + argv = ["", "delete", "container"] + connection.return_value.post_account.return_value = {}, ( + b'{"Number Not Found": 0, "Response Status": "200 OK", ' + b'"Errors": [], "Number Deleted": 1, "Response Body": ""}') + swiftclient.shell.main(argv) + connection.return_value.post_account.assert_called_with( + query_string='bulk-delete', data=b'/container/object\n', + headers={'Content-Type': 'text/plain', + 'Accept': 'application/json'}, + response_dict={}) + connection.return_value.delete_container.assert_called_with( + 'container', response_dict={}) + def test_delete_verbose_output_utf8(self): container = 't\u00e9st_c' base_argv = ['', '--verbose', 'delete'] @@ -759,8 +903,10 @@ def test_delete_verbose_output_utf8(self): self.assertTrue(out.out.find( 't\u00e9st_c [after 2 attempts]') >= 0, out) + @mock.patch.object(swiftclient.service.SwiftService, '_should_bulk_delete', + lambda *a: False) @mock.patch('swiftclient.service.Connection') - def test_delete_object(self, connection): + def test_delete_per_object(self, connection): argv = ["", "delete", "container", "object"] connection.return_value.head_object.return_value = {} connection.return_value.attempts = 0 @@ -768,6 +914,22 @@ def test_delete_object(self, connection): connection.return_value.delete_object.assert_called_with( 'container', 'object', query_string=None, response_dict={}) + @mock.patch.object(swiftclient.service.SwiftService, '_should_bulk_delete', + lambda *a: True) + @mock.patch('swiftclient.service.Connection') + def test_delete_bulk_object(self, connection): + argv = ["", "delete", "container", "object"] + connection.return_value.post_account.return_value = {}, ( + b'{"Number Not Found": 0, "Response Status": "200 OK", ' + b'"Errors": [], "Number Deleted": 1, "Response Body": ""}') + connection.return_value.attempts = 0 + swiftclient.shell.main(argv) + connection.return_value.post_account.assert_called_with( + query_string='bulk-delete', data=b'/container/object\n', + headers={'Content-Type': 'text/plain', + 'Accept': 'application/json'}, + response_dict={}) + def test_delete_verbose_output(self): del_obj_res = {'success': True, 'response_dict': {}, 'attempts': 2, 'container': 't\xe9st_c', 'action': 'delete_object', diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index 050f8b2e..5a6cbfaa 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -596,6 +596,40 @@ def test_server_error(self): self.assertEqual(e.__str__()[-89:], new_body) +class TestPostAccount(MockHttpTest): + + def test_ok(self): + c.http_connection = self.fake_http_connection(200, headers={ + 'X-Account-Meta-Color': 'blue', + }, body='foo') + resp_headers, body = c.post_account( + 'http://www.tests.com/path/to/account', 'asdf', + {'x-account-meta-shape': 'square'}, query_string='bar=baz', + data='some data') + self.assertEqual('blue', resp_headers.get('x-account-meta-color')) + self.assertEqual('foo', body) + self.assertRequests([ + ('POST', 'http://www.tests.com/path/to/account?bar=baz', + 'some data', {'x-auth-token': 'asdf', + 'x-account-meta-shape': 'square'}) + ]) + + def test_server_error(self): + body = 'c' * 65 + c.http_connection = self.fake_http_connection(500, body=body) + e = self.assertRaises(c.ClientException, c.post_account, + 'http://www.tests.com', 'asdf', {}) + self.assertEqual(e.http_response_content, body) + self.assertEqual(e.http_status, 500) + self.assertRequests([ + ('POST', 'http://www.tests.com', None, {'x-auth-token': 'asdf'}) + ]) + # TODO: this is a fairly brittle test of the __repr__ on the + # ClientException which should probably be in a targeted test + new_body = "[first 60 chars of response] " + body[0:60] + self.assertEqual(e.__str__()[-89:], new_body) + + class TestGetContainer(MockHttpTest): def test_no_content(self): @@ -1976,7 +2010,8 @@ class TestResponseDict(MockHttpTest): """ Verify handling of optional response_dict argument. """ - calls = [('post_container', 'c', {}), + calls = [('post_account', {}), + ('post_container', 'c', {}), ('put_container', 'c'), ('delete_container', 'c'), ('post_object', 'c', 'o', {}), diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 3439f4a2..fe50f556 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -290,3 +290,31 @@ def test_segmented_file(self): self.assertEqual(segment_length, len(read_data)) self.assertEqual(s, read_data) self.assertEqual(md5(s).hexdigest(), data.get_md5sum()) + + +class TestGroupers(testtools.TestCase): + def test_n_at_a_time(self): + result = list(u.n_at_a_time(range(100), 9)) + self.assertEqual([9] * 11 + [1], list(map(len, result))) + + result = list(u.n_at_a_time(range(100), 10)) + self.assertEqual([10] * 10, list(map(len, result))) + + result = list(u.n_at_a_time(range(100), 11)) + self.assertEqual([11] * 9 + [1], list(map(len, result))) + + result = list(u.n_at_a_time(range(100), 12)) + self.assertEqual([12] * 8 + [4], list(map(len, result))) + + def test_n_groups(self): + result = list(u.n_groups(range(100), 9)) + self.assertEqual([12] * 8 + [4], list(map(len, result))) + + result = list(u.n_groups(range(100), 10)) + self.assertEqual([10] * 10, list(map(len, result))) + + result = list(u.n_groups(range(100), 11)) + self.assertEqual([10] * 10, list(map(len, result))) + + result = list(u.n_groups(range(100), 12)) + self.assertEqual([9] * 11 + [1], list(map(len, result))) From 61880c6f980cb8e613bdf6cb48a9a61ce7488162 Mon Sep 17 00:00:00 2001 From: Min Min Ren Date: Wed, 2 Dec 2015 08:27:53 +0000 Subject: [PATCH 076/454] Fix the http request headers being overwritten in logging Fix the http request headers in put_object being overwritten in logging Change-Id: Id0d1e36561a61ed1ce30d93c801ec32f058a6fa4 Closes-bug: #1501292 --- swiftclient/client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index 2e0cf72a..490ffc16 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -1128,7 +1128,6 @@ def put_object(url, token=None, container=None, name=None, contents=None, resp = conn.getresponse() body = resp.read() - headers = {'X-Auth-Token': token} http_log(('%s%s' % (url.replace(parsed.path, ''), path), 'PUT',), {'headers': headers}, resp, body) From 47f673ed9f8652a3e19ec12a03196e2fa79ef92a Mon Sep 17 00:00:00 2001 From: Jude Job Date: Sat, 9 Jan 2016 18:49:17 +0530 Subject: [PATCH 077/454] Error with uploading large object includes unicode path This patch include a test case to test unicode path. Change-Id: I7697679f0034ce65b068791d7d5145286d992bd1 Closes-Bug: #1532096 --- swiftclient/service.py | 3 +- tests/unit/test_service.py | 68 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/swiftclient/service.py b/swiftclient/service.py index 8df13897..bba9583f 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -1909,7 +1909,8 @@ def _upload_object_job(self, conn, container, source, obj, options, res['manifest_response_dict'] = mr else: new_object_manifest = '%s/%s/%s/%s/%s/' % ( - quote(seg_container), quote(obj), + quote(seg_container.encode('utf8')), + quote(obj.encode('utf8')), put_headers['x-object-meta-mtime'], full_size, options['segment_size']) if old_manifest and old_manifest.rstrip('/') == \ diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index 003a51f8..33176773 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # Copyright (c) 2014 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,6 +13,7 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import unicode_literals import mock import os import six @@ -853,6 +855,72 @@ def test_upload_with_relative_path(self, *args, **kwargs): class TestServiceUpload(_TestServiceBase): + def test_upload_object_job_file_with_unicode_path(self): + # Uploading a file results in the file object being wrapped in a + # LengthWrapper. This test sets the options in such a way that much + # of _upload_object_job is skipped bringing the critical path down + # to around 60 lines to ease testing. + with tempfile.NamedTemporaryFile() as f: + f.write(b'a' * 30) + f.flush() + expected_r = { + 'action': 'upload_object', + 'attempts': 2, + 'container': 'test_c', + 'headers': {}, + 'large_object': True, + 'object': 'テスト/dummy.dat', + 'manifest_response_dict': {}, + 'segment_results': [{'action': 'upload_segment', + 'success': True}] * 3, + 'status': 'uploaded', + 'success': True, + } + expected_mtime = float(os.path.getmtime(f.name)) + + mock_conn = mock.Mock() + mock_conn.put_object.return_value = '' + type(mock_conn).attempts = mock.PropertyMock(return_value=2) + + s = SwiftService() + with mock.patch.object(s, '_upload_segment_job') as mock_job: + mock_job.return_value = { + 'action': 'upload_segment', + 'success': True} + + r = s._upload_object_job(conn=mock_conn, + container='test_c', + source=f.name, + obj='テスト/dummy.dat', + options={'changed': False, + 'skip_identical': False, + 'leave_segments': True, + 'header': '', + 'segment_size': 10, + 'segment_container': None, + 'use_slo': False, + 'checksum': True}) + + mtime = float(r['headers']['x-object-meta-mtime']) + self.assertAlmostEqual(mtime, expected_mtime, delta=0.5) + del r['headers']['x-object-meta-mtime'] + + self.assertEqual( + 'test_c_segments/%E3%83%86%E3%82%B9%E3%83%88/dummy.dat/' + + '%f/30/10/' % mtime, r['headers']['x-object-manifest']) + del r['headers']['x-object-manifest'] + + self.assertEqual(r['path'], f.name) + del r['path'] + + self._assertDictEqual(r, expected_r) + self.assertEqual(mock_conn.put_object.call_count, 1) + mock_conn.put_object.assert_called_with('test_c', 'テスト/dummy.dat', + '', + content_length=0, + headers={}, + response_dict={}) + def test_upload_segment_job(self): with tempfile.NamedTemporaryFile() as f: f.write(b'a' * 10) From a175689418208e1c72d8db03d6b59893c73b2445 Mon Sep 17 00:00:00 2001 From: Pratik Mallya Date: Sun, 12 Apr 2015 22:33:57 -0500 Subject: [PATCH 078/454] Accept token and tenant_id for authenticating against KS Allow swiftclient to authenticate against keystone using tenant name/id and token only. Without this patch, the password is required, which may not always be available. Authentication against keystone is required to get the service catalog, which includes the endpoints for swift. Change-Id: I4477af445474c5fa97ff864c4942f1330b59e5d6 Closes-Bug: #1476002 --- swiftclient/client.py | 1 + tests/unit/test_swiftclient.py | 7 ++++++- tests/unit/utils.py | 8 ++++---- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index 4819c124..fdd66850 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -366,6 +366,7 @@ def get_auth_keystone(auth_url, user, key, os_options, **kwargs): _ksclient = ksclient.Client( username=user, password=key, + token=os_options.get('auth_token'), tenant_name=os_options.get('tenant_name'), tenant_id=os_options.get('tenant_id'), user_id=os_options.get('user_id'), diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index 97ae467a..6d772cb8 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -1534,9 +1534,10 @@ def shim_connection(*a, **kw): # v2 auth timeouts = [] + os_options = {'tenant_name': 'tenant', 'auth_token': 'meta-token'} conn = c.Connection( 'http://auth.example.com', 'user', 'password', timeout=33.0, - os_options=dict(tenant_name='tenant'), auth_version=2.0) + os_options=os_options, auth_version=2.0) fake_ks = FakeKeystone(endpoint='http://some_url', token='secret') with mock.patch('swiftclient.client._import_keystone_client', _make_fake_import_keystone_client(fake_ks)): @@ -1552,6 +1553,10 @@ def shim_connection(*a, **kw): # check timeout passed to HEAD for account self.assertEqual(timeouts, [33.0]) + # check token passed to keystone client + self.assertIn('token', fake_ks.calls[0]) + self.assertEqual('meta-token', fake_ks.calls[0].get('token')) + def test_reset_stream(self): class LocalContents(object): diff --git a/tests/unit/utils.py b/tests/unit/utils.py index ac9aefdb..6fc68e6b 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -504,8 +504,8 @@ def __init__(self, endpoint, token): self.token = token class _Client(object): - def __init__(self, endpoint, token, **kwargs): - self.auth_token = token + def __init__(self, endpoint, auth_token, **kwargs): + self.auth_token = auth_token self.endpoint = endpoint self.service_catalog = self.ServiceCatalog(endpoint) @@ -520,8 +520,8 @@ def url_for(self, **kwargs): def Client(self, **kwargs): self.calls.append(kwargs) - self.client = self._Client(endpoint=self.endpoint, token=self.token, - **kwargs) + self.client = self._Client( + endpoint=self.endpoint, auth_token=self.token, **kwargs) return self.client class Unauthorized(Exception): From dcdd7152152b59a33d9af8f518eb0e05e4f125fe Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Mon, 18 Jan 2016 18:34:47 -0800 Subject: [PATCH 079/454] Get rid of FakeConn cruft Presumably, this was left over from before the httplib -> requests transition? Change-Id: I7f505514070bf9d8fefda77203bee78f0a5dd71d --- tests/unit/utils.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/tests/unit/utils.py b/tests/unit/utils.py index 0f013a8a..17e07ac4 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -100,27 +100,11 @@ def __init__(self, status, etag=None, body='', timestamp='1', self._is_closed = True self.headers = headers or {} - def connect(self): - self._is_closed = False - - def close(self): - self._is_closed = True - - def isclosed(self): - return self._is_closed - def getresponse(self): if kwargs.get('raise_exc'): raise Exception('test') return self - def getexpect(self): - if self.status == -2: - raise RequestException() - if self.status == -3: - return FakeConn(507) - return FakeConn(100) - def getheaders(self): if self.headers: return self.headers.items() @@ -199,7 +183,6 @@ def connect(*args, **ckwargs): timestamp=timestamp) if fake_conn.status <= 0: raise RequestException() - fake_conn.connect() return fake_conn connect.code_iter = code_iter @@ -250,7 +233,6 @@ def request(method, url, *args, **kwargs): self.request_log.append((parsed, method, url, args, kwargs, conn.resp)) conn.host = conn.resp.host - conn.isclosed = conn.resp.isclosed conn.resp.has_been_read = False _orig_read = conn.resp.read From 38f96641671fff670eaae99b7b6f6fcd902438e2 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Tue, 19 Jan 2016 14:26:13 -0800 Subject: [PATCH 080/454] Prevent test runs from cluttering current directory Previously, the following empty directories would be created: * container * container/pseudo * pseudo Change-Id: I002e2da8d28a873728e0b5c2d33f94f21132d058 --- tests/unit/test_shell.py | 87 ++++++++++++++++++++++++---------------- 1 file changed, 52 insertions(+), 35 deletions(-) diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index 662fbcc7..6e473e38 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -336,15 +336,17 @@ def test_download(self, connection, makedirs): with mock.patch(BUILTIN_OPEN) as mock_open: argv = ["", "download", "container"] swiftclient.shell.main(argv) - calls = [mock.call('container', 'object', - headers={}, resp_chunk_size=65536, - response_dict={}), - mock.call('container', 'pseudo/', - headers={}, resp_chunk_size=65536, - response_dict={})] - connection.return_value.get_object.assert_has_calls( - calls, any_order=True) - mock_open.assert_called_once_with('object', 'wb') + calls = [mock.call('container', 'object', + headers={}, resp_chunk_size=65536, + response_dict={}), + mock.call('container', 'pseudo/', + headers={}, resp_chunk_size=65536, + response_dict={})] + connection.return_value.get_object.assert_has_calls( + calls, any_order=True) + mock_open.assert_called_once_with('object', 'wb') + self.assertEqual([mock.call('pseudo')], makedirs.mock_calls) + makedirs.reset_mock() # Test downloading single object objcontent = six.BytesIO(b'objcontent') @@ -356,10 +358,11 @@ def test_download(self, connection, makedirs): with mock.patch(BUILTIN_OPEN) as mock_open: argv = ["", "download", "container", "object"] swiftclient.shell.main(argv) - connection.return_value.get_object.assert_called_with( - 'container', 'object', headers={}, resp_chunk_size=65536, - response_dict={}) - mock_open.assert_called_with('object', 'wb') + connection.return_value.get_object.assert_called_with( + 'container', 'object', headers={}, resp_chunk_size=65536, + response_dict={}) + mock_open.assert_called_with('object', 'wb') + self.assertEqual([], makedirs.mock_calls) # Test downloading single object to stdout objcontent = six.BytesIO(b'objcontent') @@ -396,13 +399,18 @@ def test_download_shuffle(self, connection, mock_shuffle): ] with mock.patch(BUILTIN_OPEN) as mock_open: - argv = ["", "download", "--all"] - swiftclient.shell.main(argv) - self.assertEqual(3, mock_shuffle.call_count) - mock_shuffle.assert_any_call(['container']) - mock_shuffle.assert_any_call(['object']) - mock_shuffle.assert_any_call(['pseudo/']) - mock_open.assert_called_once_with('container/object', 'wb') + with mock.patch('swiftclient.service.makedirs') as mock_mkdir: + argv = ["", "download", "--all"] + swiftclient.shell.main(argv) + self.assertEqual(3, mock_shuffle.call_count) + mock_shuffle.assert_any_call(['container']) + mock_shuffle.assert_any_call(['object']) + mock_shuffle.assert_any_call(['pseudo/']) + mock_open.assert_called_once_with('container/object', 'wb') + self.assertEqual([ + mock.call('container'), + mock.call('container/pseudo'), + ], mock_mkdir.mock_calls) # Test that the container and object lists are not shuffled mock_shuffle.reset_mock() @@ -418,10 +426,15 @@ def test_download_shuffle(self, connection, mock_shuffle): ] with mock.patch(BUILTIN_OPEN) as mock_open: - argv = ["", "download", "--all", "--no-shuffle"] - swiftclient.shell.main(argv) - self.assertEqual(0, mock_shuffle.call_count) - mock_open.assert_called_once_with('container/object', 'wb') + with mock.patch('swiftclient.service.makedirs') as mock_mkdir: + argv = ["", "download", "--all", "--no-shuffle"] + swiftclient.shell.main(argv) + self.assertEqual(0, mock_shuffle.call_count) + mock_open.assert_called_once_with('container/object', 'wb') + self.assertEqual([ + mock.call('container'), + mock.call('container/pseudo'), + ], mock_mkdir.mock_calls) @mock.patch('swiftclient.service.Connection') def test_download_no_content_type(self, connection): @@ -439,17 +452,21 @@ def test_download_no_content_type(self, connection): connection.return_value.attempts = 0 with mock.patch(BUILTIN_OPEN) as mock_open: - argv = ["", "download", "container"] - swiftclient.shell.main(argv) - calls = [mock.call('container', 'object', - headers={}, resp_chunk_size=65536, - response_dict={}), - mock.call('container', 'pseudo/', - headers={}, resp_chunk_size=65536, - response_dict={})] - connection.return_value.get_object.assert_has_calls( - calls, any_order=True) - mock_open.assert_called_once_with('object', 'wb') + with mock.patch('swiftclient.service.makedirs') as mock_mkdir: + argv = ["", "download", "container"] + swiftclient.shell.main(argv) + calls = [mock.call('container', 'object', + headers={}, resp_chunk_size=65536, + response_dict={}), + mock.call('container', 'pseudo/', + headers={}, resp_chunk_size=65536, + response_dict={})] + connection.return_value.get_object.assert_has_calls( + calls, any_order=True) + mock_open.assert_called_once_with('object', 'wb') + self.assertEqual([ + mock.call('pseudo'), + ], mock_mkdir.mock_calls) @mock.patch('swiftclient.shell.walk') @mock.patch('swiftclient.service.Connection') From 0fe02eb1c006d7d70f638f8012aa370fdf4b6096 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Nov=C3=BD?= Date: Thu, 28 Jan 2016 23:36:07 +0100 Subject: [PATCH 081/454] mock time in unit test It's crashing now (sometimes): _StringException: Traceback (most recent call last): File "/build/python-swiftclient-2.6.0/tests/unit/test_service.py", line 1085, in test_upload_object_job_stream self.assertAlmostEqual(mtime, expected_mtime, delta=0.5) File "/usr/lib/python3/dist-packages/unittest2/case.py", line 883, in assertAlmostEqual raise self.failureException(msg) AssertionError: 1453224313.0 != 1453224312.4944572 within 0.5 delta Change-Id: Ib2eeb13cd07febcb7c8b4e1435b885c4339093e4 --- tests/unit/test_service.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index 33176773..7522e657 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -1114,7 +1114,8 @@ def test_upload_object_job_file(self): self.assertEqual(contents.read(), b'a' * 30) self.assertEqual(contents.get_md5sum(), md5(b'a' * 30).hexdigest()) - def test_upload_object_job_stream(self): + @mock.patch('swiftclient.service.time', return_value=1400000000) + def test_upload_object_job_stream(self, time_mock): # Streams are wrapped as ReadableToIterable with tempfile.TemporaryFile() as f: f.write(b'a' * 30) @@ -1132,7 +1133,7 @@ def test_upload_object_job_stream(self): 'success': True, 'path': None, } - expected_mtime = float(time.time()) + expected_mtime = 1400000000 mock_conn = mock.Mock() mock_conn.put_object.return_value = '' @@ -1151,7 +1152,7 @@ def test_upload_object_job_stream(self): 'checksum': True}) mtime = float(r['headers']['x-object-meta-mtime']) - self.assertAlmostEqual(mtime, expected_mtime, delta=0.5) + self.assertEqual(mtime, expected_mtime) del r['headers']['x-object-meta-mtime'] self._assertDictEqual(r, expected_r) From 337570a03a57b2bceb615c4fe99ccaa18e0220c9 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Fri, 29 Jan 2016 16:50:15 -0800 Subject: [PATCH 082/454] Don't trust X-Object-Meta-Mtime Still use it if we can, but stop throwing ValueErrors if it's garbage. Change-Id: I2cf25b535ad62cfacb7561954a92a4a73d91000a --- swiftclient/service.py | 12 +++-- tests/unit/test_service.py | 91 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 4 deletions(-) diff --git a/swiftclient/service.py b/swiftclient/service.py index 492055dd..20c023b4 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -1162,11 +1162,15 @@ def _download_object_job(self, conn, container, obj, options): if fp is not None: fp.close() if 'x-object-meta-mtime' in headers and not no_file: - mtime = float(headers['x-object-meta-mtime']) - if options['out_file']: - utime(options['out_file'], (mtime, mtime)) + try: + mtime = float(headers['x-object-meta-mtime']) + except ValueError: + pass # no real harm; couldn't trust it anyway else: - utime(path, (mtime, mtime)) + if options['out_file']: + utime(options['out_file'], (mtime, mtime)) + else: + utime(path, (mtime, mtime)) res = { 'action': 'download_object', diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index 7522e657..51baa1f2 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -1535,6 +1535,97 @@ def test_download_object_job(self): ) self._assertDictEqual(expected_r, actual_r) + def test_download_object_job_with_mtime(self): + mock_conn = self._get_mock_connection() + objcontent = six.BytesIO(b'objcontent') + mock_conn.get_object.side_effect = [ + ({'content-type': 'text/plain', + 'etag': '2cbbfe139a744d6abbe695e17f3c1991', + 'x-object-meta-mtime': '1454113727.682512'}, + objcontent) + ] + expected_r = self._get_expected({ + 'success': True, + 'start_time': 1, + 'finish_time': 2, + 'headers_receipt': 3, + 'auth_end_time': 4, + 'read_length': len(b'objcontent'), + }) + + with mock.patch.object(builtins, 'open') as mock_open, \ + mock.patch('swiftclient.service.utime') as mock_utime: + written_content = Mock() + mock_open.return_value = written_content + s = SwiftService() + _opts = self.opts.copy() + _opts['no_download'] = False + actual_r = s._download_object_job( + mock_conn, 'test_c', 'test_o', _opts) + actual_r = dict( # Need to override the times we got from the call + actual_r, + **{ + 'start_time': 1, + 'finish_time': 2, + 'headers_receipt': 3 + } + ) + mock_open.assert_called_once_with('test_o', 'wb') + mock_utime.assert_called_once_with( + 'test_o', (1454113727.682512, 1454113727.682512)) + written_content.write.assert_called_once_with(b'objcontent') + + mock_conn.get_object.assert_called_once_with( + 'test_c', 'test_o', resp_chunk_size=65536, headers={}, + response_dict={} + ) + self._assertDictEqual(expected_r, actual_r) + + def test_download_object_job_bad_mtime(self): + mock_conn = self._get_mock_connection() + objcontent = six.BytesIO(b'objcontent') + mock_conn.get_object.side_effect = [ + ({'content-type': 'text/plain', + 'etag': '2cbbfe139a744d6abbe695e17f3c1991', + 'x-object-meta-mtime': 'foo'}, + objcontent) + ] + expected_r = self._get_expected({ + 'success': True, + 'start_time': 1, + 'finish_time': 2, + 'headers_receipt': 3, + 'auth_end_time': 4, + 'read_length': len(b'objcontent'), + }) + + with mock.patch.object(builtins, 'open') as mock_open, \ + mock.patch('swiftclient.service.utime') as mock_utime: + written_content = Mock() + mock_open.return_value = written_content + s = SwiftService() + _opts = self.opts.copy() + _opts['no_download'] = False + actual_r = s._download_object_job( + mock_conn, 'test_c', 'test_o', _opts) + actual_r = dict( # Need to override the times we got from the call + actual_r, + **{ + 'start_time': 1, + 'finish_time': 2, + 'headers_receipt': 3 + } + ) + mock_open.assert_called_once_with('test_o', 'wb') + self.assertEqual(0, len(mock_utime.mock_calls)) + written_content.write.assert_called_once_with(b'objcontent') + + mock_conn.get_object.assert_called_once_with( + 'test_c', 'test_o', resp_chunk_size=65536, headers={}, + response_dict={} + ) + self._assertDictEqual(expected_r, actual_r) + def test_download_object_job_exception(self): mock_conn = self._get_mock_connection() mock_conn.get_object = Mock(side_effect=self.exc) From 34ae109173f0b5310d6f195056485623a0fd5fff Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Thu, 28 Jan 2016 19:05:13 -0800 Subject: [PATCH 083/454] Tighten up to unittests to expect rounding Change-Id: Ic453fd990ebbf17e0eeabc39b126d2bc14234e23 --- tests/unit/test_service.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index 7522e657..87008196 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -876,7 +876,7 @@ def test_upload_object_job_file_with_unicode_path(self): 'status': 'uploaded', 'success': True, } - expected_mtime = float(os.path.getmtime(f.name)) + expected_mtime = '%f' % os.path.getmtime(f.name) mock_conn = mock.Mock() mock_conn.put_object.return_value = '' @@ -901,13 +901,13 @@ def test_upload_object_job_file_with_unicode_path(self): 'use_slo': False, 'checksum': True}) - mtime = float(r['headers']['x-object-meta-mtime']) - self.assertAlmostEqual(mtime, expected_mtime, delta=0.5) + mtime = r['headers']['x-object-meta-mtime'] + self.assertEqual(expected_mtime, mtime) del r['headers']['x-object-meta-mtime'] self.assertEqual( 'test_c_segments/%E3%83%86%E3%82%B9%E3%83%88/dummy.dat/' + - '%f/30/10/' % mtime, r['headers']['x-object-manifest']) + '%s/30/10/' % mtime, r['headers']['x-object-manifest']) del r['headers']['x-object-manifest'] self.assertEqual(r['path'], f.name) @@ -1073,7 +1073,7 @@ def test_upload_object_job_file(self): 'status': 'uploaded', 'success': True, } - expected_mtime = float(os.path.getmtime(f.name)) + expected_mtime = '%f' % os.path.getmtime(f.name) mock_conn = mock.Mock() mock_conn.put_object.return_value = '' @@ -1091,8 +1091,8 @@ def test_upload_object_job_file(self): 'segment_size': 0, 'checksum': True}) - mtime = float(r['headers']['x-object-meta-mtime']) - self.assertAlmostEqual(mtime, expected_mtime, delta=0.5) + mtime = r['headers']['x-object-meta-mtime'] + self.assertEqual(expected_mtime, mtime) del r['headers']['x-object-meta-mtime'] self.assertEqual(r['path'], f.name) From fa9251a1ee9bfc27664d88c5fe6bbb23e8fa1e51 Mon Sep 17 00:00:00 2001 From: Alistair Coles Date: Mon, 1 Feb 2016 10:07:55 +0000 Subject: [PATCH 084/454] Fix intermittent fail of test_delete_bulk_account The test asserts calls made in specific order, but they are made from threads so may be in different order. Change-Id: I857ad3b909c3b635927fb1a39682d66d20c6fd59 --- tests/unit/test_shell.py | 41 ++++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index 01288c1f..5313d41d 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -772,22 +772,31 @@ def test_delete_bulk_account(self, connection): b'"Errors": [], "Number Deleted": 1, "Response Body": ""}') swiftclient.shell.main(argv) self.assertEqual( - connection.return_value.post_account.mock_calls, [ - mock.call(query_string='bulk-delete', - data=b'/container/object\n/container/obj%C3%A9ct2\n', - headers={'Content-Type': 'text/plain', - 'Accept': 'application/json'}, - response_dict={}), - mock.call(query_string='bulk-delete', - data=b'/container/object3\n', - headers={'Content-Type': 'text/plain', - 'Accept': 'application/json'}, - response_dict={}), - mock.call(query_string='bulk-delete', - data=b'/container2/object\n', - headers={'Content-Type': 'text/plain', - 'Accept': 'application/json'}, - response_dict={})]) + 3, len(connection.return_value.post_account.mock_calls), + 'Expected 3 calls but found\n%r' + % connection.return_value.post_account.mock_calls) + # POSTs for same container are made in parallel so expect any order + for expected in [ + mock.call(query_string='bulk-delete', + data=b'/container/object\n/container/obj%C3%A9ct2\n', + headers={'Content-Type': 'text/plain', + 'Accept': 'application/json'}, + response_dict={}), + mock.call(query_string='bulk-delete', + data=b'/container/object3\n', + headers={'Content-Type': 'text/plain', + 'Accept': 'application/json'}, + response_dict={})]: + self.assertIn(expected, + connection.return_value.post_account.mock_calls[:2]) + # POSTs for different containers are made sequentially so expect order + self.assertEqual( + mock.call(query_string='bulk-delete', + data=b'/container2/object\n', + headers={'Content-Type': 'text/plain', + 'Accept': 'application/json'}, + response_dict={}), + connection.return_value.post_account.mock_calls[2]) self.assertEqual( connection.return_value.delete_container.mock_calls, [ mock.call('container', response_dict={}), From 14a0447491aa1693b8ca5ef36ab69bed5d44a1e6 Mon Sep 17 00:00:00 2001 From: Alistair Coles Date: Fri, 29 Jan 2016 11:16:37 +0000 Subject: [PATCH 085/454] Fix intermittent fail of test_delete_account The test asserts calls made in specific order, but they are made from threads so may be in different order. Change-Id: I1b6e7303fe0e6fb2afc7da3462b891feab90bc17 Closes-Bug: #1539536 --- tests/unit/test_shell.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index 01288c1f..ead4aa06 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -734,14 +734,16 @@ def test_delete_account(self, connection): argv = ["", "delete", "--all"] connection.return_value.head_object.return_value = {} swiftclient.shell.main(argv) - self.assertEqual( - connection.return_value.delete_object.mock_calls, [ - mock.call('container', 'object', query_string=None, - response_dict={}), - mock.call('container', 'obj\xe9ct2', query_string=None, - response_dict={}), - mock.call('container2', 'object', query_string=None, - response_dict={})]) + connection.return_value.delete_object.assert_has_calls([ + mock.call('container', 'object', query_string=None, + response_dict={}), + mock.call('container', 'obj\xe9ct2', query_string=None, + response_dict={}), + mock.call('container2', 'object', query_string=None, + response_dict={})], any_order=True) + self.assertEqual(3, connection.return_value.delete_object.call_count, + 'Expected 3 calls but found\n%r' + % connection.return_value.delete_object.mock_calls) self.assertEqual( connection.return_value.delete_container.mock_calls, [ mock.call('container', response_dict={}), From 5ed02345d36acf87fc4678e587db713004696124 Mon Sep 17 00:00:00 2001 From: James Nzomo Date: Sun, 24 Jan 2016 02:43:07 +0300 Subject: [PATCH 086/454] Fix segmented upload to pseudo-dir via This fix ensures creation and use of the correct default segment container when pseudo-folder paths are passed via arg. Change-Id: I90356b041dc9dfbd55eb341271975621759476b9 Closes-Bug: 1532981 Related-Bug: 1478210 --- swiftclient/service.py | 25 +++++++++++++------------ tests/unit/test_service.py | 9 +++++++++ tests/unit/test_shell.py | 31 +++++++++++++++++++++++++++++-- 3 files changed, 51 insertions(+), 14 deletions(-) diff --git a/swiftclient/service.py b/swiftclient/service.py index 09245d32..e092beca 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -25,6 +25,7 @@ from os.path import ( basename, dirname, getmtime, getsize, isdir, join, sep as os_path_sep ) +from posixpath import join as urljoin from random import shuffle from time import time from threading import Thread @@ -288,6 +289,7 @@ def __init__(self, source, object_name=None, options=None): if not self.object_name: raise SwiftError('Object names must not be empty strings') + self.object_name = self.object_name.lstrip('/') self.options = options self.source = source @@ -1284,7 +1286,8 @@ def upload(self, container, objects, options=None): """ Upload a list of objects to a given container. - :param container: The container to put the uploads into. + :param container: The container (or pseudo-folder path) to put the + uploads into. :param objects: A list of file/directory names (strings) or SwiftUploadObject instances containing a source for the created object, an object name, and an options dict @@ -1342,10 +1345,9 @@ def upload(self, container, objects, options=None): raise SwiftError('Segment size should be an integer value') # Incase we have a psudeo-folder path for arg, derive - # the container name from the top path to ensure new folder creation - # and prevent spawning zero-byte objects shadowing pseudo-folders - # by name. - container_name = container.split('/', 1)[0] + # the container name from the top path and prepend the rest to + # the object name. (same as passing --object-name). + container, _sep, pseudo_folder = container.partition('/') # Try to create the container, just in case it doesn't exist. If this # fails, it might just be because the user doesn't have container PUT @@ -1358,10 +1360,7 @@ def upload(self, container, objects, options=None): _header[POLICY] create_containers = [ self.thread_manager.container_pool.submit( - self._create_container_job, - container_name, - headers=policy_header - ) + self._create_container_job, container, headers=policy_header) ] # wait for first container job to complete before possibly attempting @@ -1405,7 +1404,7 @@ def upload(self, container, objects, options=None): rq = Queue() file_jobs = {} - upload_objects = self._make_upload_objects(objects) + upload_objects = self._make_upload_objects(objects, pseudo_folder) for upload_object in upload_objects: s = upload_object.source o = upload_object.object_name @@ -1496,14 +1495,16 @@ def upload(self, container, objects, options=None): res = get_from_queue(rq) @staticmethod - def _make_upload_objects(objects): + def _make_upload_objects(objects, pseudo_folder=''): upload_objects = [] for o in objects: if isinstance(o, string_types): - obj = SwiftUploadObject(o) + obj = SwiftUploadObject(o, urljoin(pseudo_folder, + o.lstrip('/'))) upload_objects.append(obj) elif isinstance(o, SwiftUploadObject): + o.object_name = urljoin(pseudo_folder, o.object_name) upload_objects.append(o) else: raise SwiftError( diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index 003a51f8..c2a71434 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -1271,6 +1271,15 @@ def test_upload_object_job_identical_dlo(self): ] mock_conn.get_container.assert_has_calls(expected) + def test_make_upload_objects(self): + # String list + filenames = ['/absolute/file/path', 'relative/file/path'] + self.assertEqual( + [o.object_name for o in SwiftService._make_upload_objects( + filenames, 'pseudo/folder/path')], + ['pseudo/folder/path/absolute/file/path', + 'pseudo/folder/path/relative/file/path']) + class TestServiceDownload(_TestServiceBase): diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index 01288c1f..1efc8dc0 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -485,8 +485,8 @@ def test_upload(self, connection, walk): response_dict={}) connection.return_value.put_object.assert_called_with( - 'container/pseudo-folder/nested', - self.tmpfile.lstrip('/'), + 'container', + 'pseudo-folder/nested' + self.tmpfile, mock.ANY, content_length=0, headers={'x-object-meta-mtime': mock.ANY, @@ -531,6 +531,33 @@ def test_upload(self, connection, walk): 'x-object-meta-mtime': mock.ANY}, response_dict={}) + # upload in segments to pseudo-folder (via param) + connection.reset_mock() + connection.return_value.head_container.return_value = { + 'x-storage-policy': 'one'} + argv = ["", "upload", "container/pseudo-folder/nested", + self.tmpfile, "-S", "10", "--use-slo"] + with open(self.tmpfile, "wb") as fh: + fh.write(b'12345678901234567890') + swiftclient.shell.main(argv) + expected_calls = [mock.call('container', + {}, + response_dict={}), + mock.call('container_segments', + {'X-Storage-Policy': 'one'}, + response_dict={})] + connection.return_value.put_container.assert_has_calls(expected_calls) + connection.return_value.put_object.assert_called_with( + 'container', + 'pseudo-folder/nested' + self.tmpfile, + mock.ANY, + headers={ + 'x-object-meta-mtime': mock.ANY, + 'x-static-large-object': 'true' + }, + query_string='multipart-manifest=put', + response_dict={}) + @mock.patch('swiftclient.service.SwiftService.upload') def test_upload_object_with_account_readonly(self, upload): argv = ["", "upload", "container", self.tmpfile] From d6ef83021e2d8ea4ff7efc5acf7e40f7227a8a26 Mon Sep 17 00:00:00 2001 From: Alistair Coles Date: Wed, 3 Feb 2016 13:44:53 +0000 Subject: [PATCH 087/454] Support --os-identity-api-version option Add support for the auth-version to be specified using --os-identity-api-version or OS_IDENTITY_API_VERSION for compatibility with other openstack client command line options. The auth version used will be selected as follows: - if either --auth-version or --os-identity-api-version is set, use that value - otherwise use the value of ST_AUTH_VERSION, if set - otherwise use the value of OS_AUTH_VERSION, if set - otherwise (new behaviour) use the value of OS_IDENTITY_API_VERSION, if set - otherwise default to 1.0 Note that before this change the auth version might have defaulted to 1.0 despite OS_IDENTITY_API_VERSION being set, but with this change OS_IDENTITY_API_VERSION is preferred. Change-Id: Ifba4c4e43560ede3013337b8cdbc77dc2de6e8ff Closes-Bug: #1541273 --- swiftclient/shell.py | 23 ++++++--- tests/unit/test_shell.py | 104 +++++++++++++++++++++++++++++++++++---- 2 files changed, 112 insertions(+), 15 deletions(-) diff --git a/swiftclient/shell.py b/swiftclient/shell.py index e427a893..4cad5741 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -1193,7 +1193,9 @@ def main(arguments=None): usage=''' usage: %prog [--version] [--help] [--os-help] [--snet] [--verbose] [--debug] [--info] [--quiet] [--auth ] - [--auth-version ] [--user ] + [--auth-version | + --os-identity-api-version ] + [--user ] [--key ] [--retries ] [--os-username ] [--os-password ] [--os-user-id ] @@ -1254,6 +1256,15 @@ def main(arguments=None): %prog list --lh '''.strip('\n')) + + default_auth_version = '1.0' + for k in ('ST_AUTH_VERSION', 'OS_AUTH_VERSION', 'OS_IDENTITY_API_VERSION'): + try: + default_auth_version = environ[k] + break + except KeyError: + pass + parser.add_option('--os-help', action='store_true', dest='os_help', help='Show OpenStack authentication options.') parser.add_option('--os_help', action='store_true', help=SUPPRESS_HELP) @@ -1272,14 +1283,14 @@ def main(arguments=None): parser.add_option('-A', '--auth', dest='auth', default=environ.get('ST_AUTH'), help='URL for obtaining an auth token.') - parser.add_option('-V', '--auth-version', + parser.add_option('-V', '--auth-version', '--os-identity-api-version', dest='auth_version', - default=environ.get('ST_AUTH_VERSION', - (environ.get('OS_AUTH_VERSION', - '1.0'))), + default=default_auth_version, type=str, help='Specify a version for authentication. ' - 'Defaults to 1.0.') + 'Defaults to env[ST_AUTH_VERSION], ' + 'env[OS_AUTH_VERSION], env[OS_IDENTITY_API_VERSION]' + ' or 1.0.') parser.add_option('-U', '--user', dest='user', default=environ.get('ST_USER'), help='User name for obtaining an auth token.') diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index 00546f6b..6bb97c58 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -1291,22 +1291,25 @@ def fake_command(parser, args, thread_manager): result[0], result[1] = swiftclient.shell.parse_args(parser, args) return fake_command - def _verify_opts(self, actual_opts, opts, os_opts={}, os_opts_dict={}): + def _verify_opts(self, actual_opts, expected_opts, expected_os_opts=None, + expected_os_opts_dict=None): """ Check parsed options are correct. - :param opts: v1 style options. - :param os_opts: openstack style options. - :param os_opts_dict: openstack options that should be found in the - os_options dict. + :param expected_opts: v1 style options. + :param expected_os_opts: openstack style options. + :param expected_os_opts_dict: openstack options that should be found in + the os_options dict. """ + expected_os_opts = expected_os_opts or {} + expected_os_opts_dict = expected_os_opts_dict or {} # check the expected opts are set - for key, v in opts.items(): + for key, v in expected_opts.items(): actual = getattr(actual_opts, key) self.assertEqual(v, actual, 'Expected %s for key %s, found %s' % (v, key, actual)) - for key, v in os_opts.items(): + for key, v in expected_os_opts.items(): actual = getattr(actual_opts, "os_" + key) self.assertEqual(v, actual, 'Expected %s for key %s, found %s' % (v, key, actual)) @@ -1327,8 +1330,8 @@ def _verify_opts(self, actual_opts, opts, os_opts={}, os_opts_dict={}): if key == 'object_storage_url': # exceptions to the pattern... cli_key = 'storage_url' - if cli_key in os_opts_dict: - expect = os_opts_dict[cli_key] + if cli_key in expected_os_opts_dict: + expect = expected_os_opts_dict[cli_key] else: expect = None actual = actual_os_opts_dict[key] @@ -1386,6 +1389,89 @@ def test_minimum_required_args_v3(self): swiftclient.shell.main(args) self._verify_opts(result[0], opts, os_opts, os_opts_dict) + def test_os_identity_api_version(self): + os_opts = {"password": "secret", + "username": "user", + "auth_url": "http://example.com:5000/v3", + "identity-api-version": "3"} + + # check os_identity_api_version is sufficient in place of auth_version + args = _make_args("stat", {}, os_opts, '-') + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch.dict(os.environ, {}): + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + expected_opts = {'auth_version': '3'} + expected_os_opts = {"password": "secret", + "username": "user", + "auth_url": "http://example.com:5000/v3"} + self._verify_opts(result[0], expected_opts, expected_os_opts, {}) + + # check again using environment variables + args = _make_args("stat", {}, {}) + env = _make_env({}, os_opts) + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch.dict(os.environ, env): + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + self._verify_opts(result[0], expected_opts, expected_os_opts, {}) + + # check that last of auth-version, os-identity-api-version is preferred + args = _make_args("stat", {}, os_opts, '-') + ['--auth-version', '2.0'] + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch.dict(os.environ, {}): + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + expected_opts = {'auth_version': '2.0'} + self._verify_opts(result[0], expected_opts, expected_os_opts, {}) + + # now put auth_version ahead of os-identity-api-version + args = _make_args("stat", {"auth_version": "2.0"}, os_opts, '-') + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch.dict(os.environ, {}): + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + expected_opts = {'auth_version': '3'} + self._verify_opts(result[0], expected_opts, expected_os_opts, {}) + + # check that OS_AUTH_VERSION overrides OS_IDENTITY_API_VERSION + args = _make_args("stat", {}, {}) + env = _make_env({}, os_opts) + env.update({'OS_AUTH_VERSION': '2.0'}) + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch.dict(os.environ, env): + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + expected_opts = {'auth_version': '2.0'} + self._verify_opts(result[0], expected_opts, expected_os_opts, {}) + + # check that ST_AUTH_VERSION overrides OS_IDENTITY_API_VERSION + args = _make_args("stat", {}, {}) + env = _make_env({}, os_opts) + env.update({'ST_AUTH_VERSION': '2.0'}) + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch.dict(os.environ, env): + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + self._verify_opts(result[0], expected_opts, expected_os_opts, {}) + + # check that ST_AUTH_VERSION overrides OS_AUTH_VERSION + args = _make_args("stat", {}, {}) + env = _make_env({}, os_opts) + env.update({'ST_AUTH_VERSION': '2.0', 'OS_AUTH_VERSION': '3'}) + result = [None, None] + fake_command = self._make_fake_command(result) + with mock.patch.dict(os.environ, env): + with mock.patch('swiftclient.shell.st_stat', fake_command): + swiftclient.shell.main(args) + self._verify_opts(result[0], expected_opts, expected_os_opts, {}) + def test_args_v3(self): opts = {"auth_version": "3"} os_opts = {"password": "secret", From 9e6b12e6b2ecebb6334b6f2a7a1b308e4d3c96e8 Mon Sep 17 00:00:00 2001 From: "Chaozhe.Chen" Date: Tue, 9 Feb 2016 23:03:31 +0800 Subject: [PATCH 088/454] Use "# noqa" instead of "#flake8: noqa" "# flake8: noqa" option disables all checks for the whole file. To disable one line we should use "# noqa". Change-Id: I7859eab30563d0eb91c5f055d1b523173b562e54 Closes-bug: #1540254 --- swiftclient/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/swiftclient/__init__.py b/swiftclient/__init__.py index b412f138..dc192afe 100644 --- a/swiftclient/__init__.py +++ b/swiftclient/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2012 Rackspace -# flake8: noqa +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -17,7 +17,7 @@ """ OpenStack Swift Python client binding. """ -from .client import * +from .client import * # noqa # At setup.py time, we haven't installed anything yet, so there # is nothing that is able to set this version property. Squelching From 9a97b51c0c2f71f9e35a25bdafb42fb541af74c1 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Tue, 9 Feb 2016 13:15:22 -0800 Subject: [PATCH 089/454] more tests for pseudo/dir Change-Id: Idab172aefd8e69ca8e4d623918eba1bc1da91a42 --- tests/unit/test_service.py | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index c2a71434..3f6da2be 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -1272,13 +1272,30 @@ def test_upload_object_job_identical_dlo(self): mock_conn.get_container.assert_has_calls(expected) def test_make_upload_objects(self): - # String list - filenames = ['/absolute/file/path', 'relative/file/path'] - self.assertEqual( - [o.object_name for o in SwiftService._make_upload_objects( - filenames, 'pseudo/folder/path')], - ['pseudo/folder/path/absolute/file/path', - 'pseudo/folder/path/relative/file/path']) + check_names_pseudo_to_expected = { + (('/absolute/file/path',), ''): ['absolute/file/path'], + (('relative/file/path',), ''): ['relative/file/path'], + (('/absolute/file/path',), '/absolute/pseudo/dir'): [ + 'absolute/pseudo/dir/absolute/file/path'], + (('/absolute/file/path',), 'relative/pseudo/dir'): [ + 'relative/pseudo/dir/absolute/file/path'], + (('relative/file/path',), '/absolute/pseudo/dir'): [ + 'absolute/pseudo/dir/relative/file/path'], + (('relative/file/path',), 'relative/pseudo/dir'): [ + 'relative/pseudo/dir/relative/file/path'], + } + errors = [] + for (filenames, pseudo_folder), expected in \ + check_names_pseudo_to_expected.items(): + actual = SwiftService._make_upload_objects( + filenames, pseudo_folder=pseudo_folder) + try: + self.assertEqual(expected, [o.object_name for o in actual]) + except AssertionError as e: + msg = 'given (%r, %r) expected %r, got %s' % ( + filenames, pseudo_folder, expected, e) + errors.append(msg) + self.assertFalse(errors, "\nERRORS:\n%s" % '\n'.join(errors)) class TestServiceDownload(_TestServiceBase): From f82d26063a3d0515886ac9f62ba0c60fe125aa54 Mon Sep 17 00:00:00 2001 From: Alexandra Date: Wed, 10 Feb 2016 18:22:22 +1000 Subject: [PATCH 090/454] New python swiftclient doc Updating with new ToC to be worked on at hackathon Change-Id: I55ee83626dd88fcc3e6352b3854b758dd7090590 --- doc/source/cli.rst | 29 ++++++++++++++++++++++++++ doc/source/index.rst | 2 ++ doc/source/sdk.rst | 48 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+) create mode 100644 doc/source/cli.rst create mode 100644 doc/source/sdk.rst diff --git a/doc/source/cli.rst b/doc/source/cli.rst new file mode 100644 index 00000000..9527fbfc --- /dev/null +++ b/doc/source/cli.rst @@ -0,0 +1,29 @@ +=== +CLI +=== + +Top-level commands +~~~~~~~~~~~~~~~~~~ + +.. TODO + + delete + download + list + post + stat + upload + info/capabilities + tempurl + auth + +Prescriptive examples +~~~~~~~~~~~~~~~~~~~~~ + +.. TODO + + A "Hello World" example + uploading an object + creating a tempurl + listing the contents of a container + downloading an object \ No newline at end of file diff --git a/doc/source/index.rst b/doc/source/index.rst index 3b8535af..da16a3ce 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -8,6 +8,8 @@ Developer Documentation :maxdepth: 2 apis + cli + sdk Code-Generated Documentation ============================ diff --git a/doc/source/sdk.rst b/doc/source/sdk.rst new file mode 100644 index 00000000..aa152509 --- /dev/null +++ b/doc/source/sdk.rst @@ -0,0 +1,48 @@ +=== +SDK +=== + +Where to start? +~~~~~~~~~~~~~~~ + +.. TODO + + when to use SwiftService + when to use client.py + +SwiftService classes and methods +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. TODO + + docs for each method (autogen from docstrings?) + +Client classes and methods +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. TODO + + docs for each method (autogen from docstrings?) + +Guidelines for writing an app +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. TODO + + auth + how to use various features + when to use various features + pooling connections + concurrency + retries + +Prescriptive examples +~~~~~~~~~~~~~~~~~~~~~ + +.. TODO + + A "Hello World" example + connecting + uploading an object + uploading a directory + \ No newline at end of file From 30ca247426c0cc9b64e8b12fbe2b68db8ef5517e Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Mon, 18 Jan 2016 14:54:10 -0800 Subject: [PATCH 091/454] Display proper name when failing to create segments container Previously, we displayed the base container's name. Change-Id: I70f1949a44ba61158e31178e4536f229c37aab47 --- swiftclient/shell.py | 2 +- tests/unit/test_shell.py | 48 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/swiftclient/shell.py b/swiftclient/shell.py index e427a893..75150ea1 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -938,7 +938,7 @@ def st_upload(parser, args, output_manager): msg = ': %s' % error output_manager.warning( 'Warning: failed to create container ' - "'%s'%s", container, msg + "'%s'%s", r['container'], msg ) else: output_manager.error("%s" % error) diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index d6e9b1e5..d61593a6 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -581,6 +581,7 @@ def test_upload_object_with_account_readonly(self, upload): upload.return_value = [ {"success": False, "headers": {}, + "container": 'container', "action": 'create_container', "error": swiftclient.ClientException( 'Container PUT failed', @@ -2156,6 +2157,53 @@ def test_segment_upload_with_write_only_access(self): % self.cont self.assertEqual(expected_err, out.err.strip()) + def test_segment_upload_with_write_only_access_segments_container(self): + fake_conn = self.fake_http_connection( + 403, # PUT c1 + # HEAD c1 to get storage policy + StubResponse(200, headers={'X-Storage-Policy': 'foo'}), + 403, # PUT c1_segments + 201, # PUT c1_segments/...00 + 201, # PUT c1_segments/...01 + 201, # PUT c1/... + ) + + args, env = self._make_cmd('upload', + cmd_args=[self.cont, self.obj, + '--leave-segments', + '--segment-size=10']) + with mock.patch('swiftclient.client._import_keystone_client', + self.fake_ks_import): + with mock.patch('swiftclient.client.http_connection', fake_conn): + with mock.patch.dict(os.environ, env): + with CaptureOutput() as out: + try: + swiftclient.shell.main(args) + except SystemExit as e: + self.fail('Unexpected SystemExit: %s' % e) + + segment_time = getmtime(self.obj) + segment_path_0 = '%s_segments%s/%f/20/10/00000000' % ( + self.cont_path, self.obj, segment_time) + segment_path_1 = '%s_segments%s/%f/20/10/00000001' % ( + self.cont_path, self.obj, segment_time) + # Note that the order of segment PUTs cannot be asserted, so test for + # existence in request log individually + self.assert_request(('PUT', self.cont_path)) + self.assert_request(('PUT', self.cont_path + '_segments', '', { + 'X-Auth-Token': 'bob_token', + 'X-Storage-Policy': 'foo', + 'Content-Length': '0', + })) + self.assert_request(('PUT', segment_path_0)) + self.assert_request(('PUT', segment_path_1)) + self.assert_request(('PUT', self.obj_path)) + self.assertTrue(self.obj[1:] in out.out) + expected_err = ("Warning: failed to create container '%s': 403 Fake\n" + "Warning: failed to create container '%s': 403 Fake" + ) % (self.cont, self.cont + '_segments') + self.assertEqual(expected_err, out.err.strip()) + def test_upload_with_no_access(self): fake_conn = self.fake_http_connection(403, 403) From 28558c7e0adf7d73e273c71dcbcfda02f871f345 Mon Sep 17 00:00:00 2001 From: Alistair Coles Date: Thu, 11 Feb 2016 15:02:09 +0000 Subject: [PATCH 092/454] Add test for --debug taking precedence over --info Change-Id: Ibf0903817852edb36389028383a35e0d65f88a26 --- tests/unit/test_shell.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index 4dec22c2..b75336a6 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -1042,7 +1042,7 @@ def test_no_help(self): @mock.patch.dict(os.environ, mocked_os_environ) -class TestOptionAfterPosArg(testtools.TestCase): +class TestDebugAndInfoOptions(testtools.TestCase): @mock.patch('logging.basicConfig') @mock.patch('swiftclient.service.Connection') def test_option_after_posarg(self, connection, mock_logging): @@ -1054,6 +1054,24 @@ def test_option_after_posarg(self, connection, mock_logging): swiftclient.shell.main(argv) mock_logging.assert_called_with(level=logging.DEBUG) + @mock.patch('logging.basicConfig') + @mock.patch('swiftclient.service.Connection') + def test_debug_trumps_info(self, connection, mock_logging): + argv_scenarios = (["", "stat", "--info", "--debug"], + ["", "stat", "--debug", "--info"], + ["", "--info", "stat", "--debug"], + ["", "--debug", "stat", "--info"], + ["", "--info", "--debug", "stat"], + ["", "--debug", "--info", "stat"]) + for argv in argv_scenarios: + mock_logging.reset_mock() + swiftclient.shell.main(argv) + try: + mock_logging.assert_called_once_with(level=logging.DEBUG) + except AssertionError: + self.fail('Unexpected call(s) %r for args %r' + % (mock_logging.call_args_list, argv)) + class TestBase(testtools.TestCase): """ From bed6bbd5efd24234825e266a50ac37d33447d340 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Fri, 4 Dec 2015 11:28:05 -0800 Subject: [PATCH 093/454] Drop testtools from test-requirements.txt My understanding is that it was mainly being used so we could have sane testing on py26. With py26 support being dropped, we no longer need it. Also drop discover from test-requirements.txt, as we don't seem to actually use it. Change-Id: Iee04c42890596d3b483c1473169480a3ae19aac8 Related-Change: I37116731db11449d0c374a6a83a3a43789a19d5f --- test-requirements.txt | 2 - tests/functional/test_swiftclient.py | 4 +- tests/unit/test_command_helpers.py | 4 +- tests/unit/test_multithreading.py | 6 +- tests/unit/test_service.py | 92 +++++++++++----------------- tests/unit/test_shell.py | 37 ++++++----- tests/unit/test_swiftclient.py | 75 +++++++++++++---------- tests/unit/test_utils.py | 14 ++--- tests/unit/utils.py | 3 +- 9 files changed, 114 insertions(+), 123 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 7f7e405f..044f7c3f 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,10 +1,8 @@ hacking>=0.10.0,<0.11 coverage>=3.6 -discover mock>=1.2 oslosphinx python-keystoneclient>=0.7.0 sphinx>=1.1.2,<1.2 testrepository>=0.0.18 -testtools>=0.9.34 diff --git a/tests/functional/test_swiftclient.py b/tests/functional/test_swiftclient.py index 5f9e271f..7a77c071 100644 --- a/tests/functional/test_swiftclient.py +++ b/tests/functional/test_swiftclient.py @@ -14,7 +14,7 @@ # limitations under the License. import os -import testtools +import unittest import time from io import BytesIO @@ -23,7 +23,7 @@ import swiftclient -class TestFunctional(testtools.TestCase): +class TestFunctional(unittest.TestCase): def __init__(self, *args, **kwargs): super(TestFunctional, self).__init__(*args, **kwargs) diff --git a/tests/unit/test_command_helpers.py b/tests/unit/test_command_helpers.py index d9d7efa6..24684ae2 100644 --- a/tests/unit/test_command_helpers.py +++ b/tests/unit/test_command_helpers.py @@ -15,13 +15,13 @@ import mock from six import StringIO -import testtools +import unittest from swiftclient import command_helpers as h from swiftclient.multithreading import OutputManager -class TestStatHelpers(testtools.TestCase): +class TestStatHelpers(unittest.TestCase): def setUp(self): super(TestStatHelpers, self).setUp() diff --git a/tests/unit/test_multithreading.py b/tests/unit/test_multithreading.py index 76758b69..8944d48e 100644 --- a/tests/unit/test_multithreading.py +++ b/tests/unit/test_multithreading.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import sys -import testtools +import unittest import threading import six @@ -25,7 +25,7 @@ from .utils import CaptureStream -class ThreadTestCase(testtools.TestCase): +class ThreadTestCase(unittest.TestCase): def setUp(self): super(ThreadTestCase, self).setUp() self.got_items = Queue() @@ -163,7 +163,7 @@ def test_lazy_connections(self): ) -class TestOutputManager(testtools.TestCase): +class TestOutputManager(unittest.TestCase): def test_instantiation(self): output_manager = mt.OutputManager() diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index 3fbe987a..997d992d 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -18,7 +18,7 @@ import os import six import tempfile -import testtools +import unittest import time from concurrent.futures import Future @@ -49,7 +49,7 @@ import builtins -class TestSwiftPostObject(testtools.TestCase): +class TestSwiftPostObject(unittest.TestCase): def setUp(self): super(TestSwiftPostObject, self).setUp() @@ -69,7 +69,7 @@ def test_create_with_invalid_name(self): self.assertRaises(SwiftError, self.spo, 1) -class TestSwiftReader(testtools.TestCase): +class TestSwiftReader(unittest.TestCase): def setUp(self): super(TestSwiftReader, self).setUp() @@ -152,25 +152,7 @@ def _consume(sr): '97ac82a5b825239e782d0339e2d7b910') -class _TestServiceBase(testtools.TestCase): - def _assertDictEqual(self, a, b, m=None): - # assertDictEqual is not available in py2.6 so use a shallow check - # instead - if not m: - m = '{0} != {1}'.format(a, b) - - if hasattr(self, 'assertDictEqual'): - self.assertDictEqual(a, b, m) - else: - self.assertIsInstance(a, dict, - 'First argument is not a dictionary') - self.assertIsInstance(b, dict, - 'Second argument is not a dictionary') - self.assertEqual(len(a), len(b), m) - for k, v in a.items(): - self.assertIn(k, b, m) - self.assertEqual(b[k], v, m) - +class _TestServiceBase(unittest.TestCase): def _get_mock_connection(self, attempts=2): m = Mock(spec=Connection) type(m).attempts = PropertyMock(return_value=attempts) @@ -223,8 +205,8 @@ def test_delete_segment(self): mock_conn.delete_object.assert_called_once_with( 'test_c', 'test_s', response_dict={} ) - self._assertDictEqual(expected_r, r) - self._assertDictEqual(expected_r, self._get_queue(mock_q)) + self.assertEqual(expected_r, r) + self.assertEqual(expected_r, self._get_queue(mock_q)) def test_delete_segment_exception(self): mock_q = Queue() @@ -246,8 +228,8 @@ def test_delete_segment_exception(self): mock_conn.delete_object.assert_called_once_with( 'test_c', 'test_s', response_dict={} ) - self._assertDictEqual(expected_r, r) - self._assertDictEqual(expected_r, self._get_queue(mock_q)) + self.assertEqual(expected_r, r) + self.assertEqual(expected_r, self._get_queue(mock_q)) self.assertGreaterEqual(r['error_timestamp'], before) self.assertLessEqual(r['error_timestamp'], after) self.assertIn('Traceback', r['traceback']) @@ -268,7 +250,7 @@ def test_delete_object(self): mock_conn.delete_object.assert_called_once_with( 'test_c', 'test_o', query_string=None, response_dict={} ) - self._assertDictEqual(expected_r, r) + self.assertEqual(expected_r, r) def test_delete_object_exception(self): mock_q = Queue() @@ -294,7 +276,7 @@ def test_delete_object_exception(self): mock_conn.delete_object.assert_called_once_with( 'test_c', 'test_o', query_string=None, response_dict={} ) - self._assertDictEqual(expected_r, r) + self.assertEqual(expected_r, r) self.assertGreaterEqual(r['error_timestamp'], before) self.assertLessEqual(r['error_timestamp'], after) self.assertIn('Traceback', r['traceback']) @@ -321,7 +303,7 @@ def test_delete_object_slo_support(self): query_string='multipart-manifest=delete', response_dict={} ) - self._assertDictEqual(expected_r, r) + self.assertEqual(expected_r, r) def test_delete_object_dlo_support(self): mock_q = Queue() @@ -352,7 +334,7 @@ def get_mock_list_conn(options): mock_conn, 'test_c', 'test_o', self.opts, mock_q ) - self._assertDictEqual(expected_r, r) + self.assertEqual(expected_r, r) expected = [ mock.call('test_c', 'test_o', query_string=None, response_dict={}), mock.call('manifest_c', 'test_seg_1', response_dict={}), @@ -372,7 +354,7 @@ def test_delete_empty_container(self): mock_conn.delete_container.assert_called_once_with( 'test_c', response_dict={} ) - self._assertDictEqual(expected_r, r) + self.assertEqual(expected_r, r) def test_delete_empty_container_exception(self): mock_conn = self._get_mock_connection() @@ -394,13 +376,13 @@ def test_delete_empty_container_exception(self): mock_conn.delete_container.assert_called_once_with( 'test_c', response_dict={} ) - self._assertDictEqual(expected_r, r) + self.assertEqual(expected_r, r) self.assertGreaterEqual(r['error_timestamp'], before) self.assertLessEqual(r['error_timestamp'], after) self.assertIn('Traceback', r['traceback']) -class TestSwiftError(testtools.TestCase): +class TestSwiftError(unittest.TestCase): def test_is_exception(self): se = SwiftError(5) @@ -430,7 +412,7 @@ def test_swifterror_creation(self): self.assertEqual(str(se), '5 container:con object:obj segment:seg') -class TestServiceUtils(testtools.TestCase): +class TestServiceUtils(unittest.TestCase): def setUp(self): super(TestServiceUtils, self).setUp() @@ -525,7 +507,7 @@ def test_split_headers_error(self): mock_headers) -class TestSwiftUploadObject(testtools.TestCase): +class TestSwiftUploadObject(unittest.TestCase): def setUp(self): self.suo = swiftclient.service.SwiftUploadObject @@ -614,7 +596,7 @@ def test_list_account(self): SwiftService._list_account_job( mock_conn, self.opts, mock_q ) - self._assertDictEqual(expected_r, self._get_queue(mock_q)) + self.assertEqual(expected_r, self._get_queue(mock_q)) self.assertIsNone(self._get_queue(mock_q)) long_opts = dict(self.opts, **{'long': True}) @@ -635,7 +617,7 @@ def test_list_account(self): SwiftService._list_account_job( mock_conn, long_opts, mock_q ) - self._assertDictEqual(expected_r_long, self._get_queue(mock_q)) + self.assertEqual(expected_r_long, self._get_queue(mock_q)) self.assertIsNone(self._get_queue(mock_q)) def test_list_account_exception(self): @@ -657,7 +639,7 @@ def test_list_account_exception(self): mock_conn.get_account.assert_called_once_with( marker='', prefix=None ) - self._assertDictEqual(expected_r, self._get_queue(mock_q)) + self.assertEqual(expected_r, self._get_queue(mock_q)) self.assertIsNone(self._get_queue(mock_q)) def test_list_container(self): @@ -680,7 +662,7 @@ def test_list_container(self): SwiftService._list_container_job( mock_conn, 'test_c', self.opts, mock_q ) - self._assertDictEqual(expected_r, self._get_queue(mock_q)) + self.assertEqual(expected_r, self._get_queue(mock_q)) self.assertIsNone(self._get_queue(mock_q)) long_opts = dict(self.opts, **{'long': True}) @@ -702,7 +684,7 @@ def test_list_container(self): SwiftService._list_container_job( mock_conn, 'test_c', long_opts, mock_q ) - self._assertDictEqual(expected_r_long, self._get_queue(mock_q)) + self.assertEqual(expected_r_long, self._get_queue(mock_q)) self.assertIsNone(self._get_queue(mock_q)) def test_list_container_exception(self): @@ -726,7 +708,7 @@ def test_list_container_exception(self): mock_conn.get_container.assert_called_once_with( 'test_c', marker='', delimiter='', prefix=None ) - self._assertDictEqual(expected_r, self._get_queue(mock_q)) + self.assertEqual(expected_r, self._get_queue(mock_q)) self.assertIsNone(self._get_queue(mock_q)) @mock.patch('swiftclient.service.get_conn') @@ -805,7 +787,7 @@ def test_list_queue_size(self, mock_get_conn): self.assertEqual(observed_listing, expected_listing) -class TestService(testtools.TestCase): +class TestService(unittest.TestCase): def test_upload_with_bad_segment_size(self): for bad in ('ten', '1234X', '100.3'): @@ -913,7 +895,7 @@ def test_upload_object_job_file_with_unicode_path(self): self.assertEqual(r['path'], f.name) del r['path'] - self._assertDictEqual(r, expected_r) + self.assertEqual(r, expected_r) self.assertEqual(mock_conn.put_object.call_count, 1) mock_conn.put_object.assert_called_with('test_c', 'テスト/dummy.dat', '', @@ -960,7 +942,7 @@ def test_upload_segment_job(self): options={'segment_container': None, 'checksum': True}) - self._assertDictEqual(r, expected_r) + self.assertEqual(r, expected_r) self.assertEqual(mock_conn.put_object.call_count, 1) mock_conn.put_object.assert_called_with('test_c_segments', @@ -1098,7 +1080,7 @@ def test_upload_object_job_file(self): self.assertEqual(r['path'], f.name) del r['path'] - self._assertDictEqual(r, expected_r) + self.assertEqual(r, expected_r) self.assertEqual(mock_conn.put_object.call_count, 1) mock_conn.put_object.assert_called_with('test_c', 'test_o', mock.ANY, @@ -1155,7 +1137,7 @@ def test_upload_object_job_stream(self, time_mock): self.assertEqual(mtime, expected_mtime) del r['headers']['x-object-meta-mtime'] - self._assertDictEqual(r, expected_r) + self.assertEqual(r, expected_r) self.assertEqual(mock_conn.put_object.call_count, 1) mock_conn.put_object.assert_called_with('test_c', 'test_o', mock.ANY, @@ -1559,7 +1541,7 @@ def test_download_object_job(self): 'test_c', 'test_o', resp_chunk_size=65536, headers={}, response_dict={} ) - self._assertDictEqual(expected_r, actual_r) + self.assertEqual(expected_r, actual_r) def test_download_object_job_with_mtime(self): mock_conn = self._get_mock_connection() @@ -1605,7 +1587,7 @@ def test_download_object_job_with_mtime(self): 'test_c', 'test_o', resp_chunk_size=65536, headers={}, response_dict={} ) - self._assertDictEqual(expected_r, actual_r) + self.assertEqual(expected_r, actual_r) def test_download_object_job_bad_mtime(self): mock_conn = self._get_mock_connection() @@ -1650,7 +1632,7 @@ def test_download_object_job_bad_mtime(self): 'test_c', 'test_o', resp_chunk_size=65536, headers={}, response_dict={} ) - self._assertDictEqual(expected_r, actual_r) + self.assertEqual(expected_r, actual_r) def test_download_object_job_exception(self): mock_conn = self._get_mock_connection() @@ -1670,7 +1652,7 @@ def test_download_object_job_exception(self): 'test_c', 'test_o', resp_chunk_size=65536, headers={}, response_dict={} ) - self._assertDictEqual(expected_r, actual_r) + self.assertEqual(expected_r, actual_r) def test_download(self): service = SwiftService() @@ -1814,7 +1796,7 @@ def fake_get(*args, **kwargs): 'header': {}, 'yes_all': False, 'skip_identical': True}) - self._assertDictEqual(r, expected_r) + self.assertEqual(r, expected_r) self.assertEqual(mock_conn.get_object.call_count, 1) mock_conn.get_object.assert_called_with( @@ -1876,7 +1858,7 @@ def test_download_object_job_skip_identical_dlo(self): self.assertEqual("Large object is identical", err.msg) self.assertEqual(304, err.http_status) - self._assertDictEqual(r, expected_r) + self.assertEqual(r, expected_r) self.assertEqual(mock_conn.get_object.call_count, 1) mock_conn.get_object.assert_called_with( @@ -1959,7 +1941,7 @@ def test_download_object_job_skip_identical_nested_slo(self): self.assertEqual("Large object is identical", err.msg) self.assertEqual(304, err.http_status) - self._assertDictEqual(r, expected_r) + self.assertEqual(r, expected_r) self.assertEqual(mock_conn.get_object.mock_calls, [ mock.call('test_c', 'test_o', @@ -2025,7 +2007,7 @@ def test_download_object_job_skip_identical_diff_dlo(self): obj='test_o', options=options) - self._assertDictEqual(r, expected_r) + self.assertEqual(r, expected_r) self.assertEqual(mock_conn.get_container.mock_calls, [ mock.call('test_c_segments', @@ -2116,7 +2098,7 @@ def test_download_object_job_skip_identical_diff_nested_slo(self): obj='test_o', options=options) - self._assertDictEqual(r, expected_r) + self.assertEqual(r, expected_r) self.assertEqual(mock_conn.get_object.mock_calls, [ mock.call('test_c', 'test_o', diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index 4bde190f..ee97824f 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -20,9 +20,8 @@ import mock import os import tempfile -import testtools +import unittest import textwrap -from testtools import ExpectedException import six @@ -106,7 +105,7 @@ def _make_cmd(cmd, opts, os_opts, use_env=False, flags=None, cmd_args=None): @mock.patch.dict(os.environ, mocked_os_environ) -class TestShell(testtools.TestCase): +class TestShell(unittest.TestCase): def setUp(self): super(TestShell, self).setUp() tmpfile = tempfile.NamedTemporaryFile(delete=False) @@ -1076,7 +1075,7 @@ def test_post_account_bad_auth(self, connection): swiftclient.ClientException('bad auth') with CaptureOutput() as output: - with ExpectedException(SystemExit): + with self.assertRaises(SystemExit): swiftclient.shell.main(argv) self.assertEqual(output.err, 'bad auth\n') @@ -1088,7 +1087,7 @@ def test_post_account_not_found(self, connection): swiftclient.ClientException('test', http_status=404) with CaptureOutput() as output: - with ExpectedException(SystemExit): + with self.assertRaises(SystemExit): swiftclient.shell.main(argv) self.assertEqual(output.err, 'Account not found\n') @@ -1107,7 +1106,7 @@ def test_post_container_bad_auth(self, connection): swiftclient.ClientException('bad auth') with CaptureOutput() as output: - with ExpectedException(SystemExit): + with self.assertRaises(SystemExit): swiftclient.shell.main(argv) self.assertEqual(output.err, 'bad auth\n') @@ -1125,7 +1124,7 @@ def test_post_container_with_bad_name(self): argv = ["", "post", "conta/iner"] with CaptureOutput() as output: - with ExpectedException(SystemExit): + with self.assertRaises(SystemExit): swiftclient.shell.main(argv) self.assertTrue(output.err != '') self.assertTrue(output.err.startswith('WARNING: / in')) @@ -1165,7 +1164,7 @@ def test_post_object_bad_auth(self, connection): swiftclient.ClientException("bad auth") with CaptureOutput() as output: - with ExpectedException(SystemExit): + with self.assertRaises(SystemExit): swiftclient.shell.main(argv) self.assertEqual(output.err, 'bad auth\n') @@ -1174,7 +1173,7 @@ def test_post_object_too_many_args(self): argv = ["", "post", "container", "object", "bad_arg"] with CaptureOutput() as output: - with ExpectedException(SystemExit): + with self.assertRaises(SystemExit): swiftclient.shell.main(argv) self.assertTrue(output.err != '') @@ -1235,49 +1234,49 @@ def _check_expected(x, expected): _check_expected(mock_swift, 12345) with CaptureOutput() as output: - with ExpectedException(SystemExit): + with self.assertRaises(SystemExit): # Test invalid states argv = ["", "upload", "-S", "1234X", "container", "object"] swiftclient.shell.main(argv) self.assertEqual(output.err, "Invalid segment size\n") output.clear() - with ExpectedException(SystemExit): + with self.assertRaises(SystemExit): argv = ["", "upload", "-S", "K1234", "container", "object"] swiftclient.shell.main(argv) self.assertEqual(output.err, "Invalid segment size\n") output.clear() - with ExpectedException(SystemExit): + with self.assertRaises(SystemExit): argv = ["", "upload", "-S", "K", "container", "object"] swiftclient.shell.main(argv) self.assertEqual(output.err, "Invalid segment size\n") def test_negative_upload_segment_size(self): with CaptureOutput() as output: - with ExpectedException(SystemExit): + with self.assertRaises(SystemExit): argv = ["", "upload", "-S", "-40", "container", "object"] swiftclient.shell.main(argv) self.assertEqual(output.err, "segment-size should be positive\n") output.clear() - with ExpectedException(SystemExit): + with self.assertRaises(SystemExit): argv = ["", "upload", "-S", "-40K", "container", "object"] swiftclient.shell.main(argv) self.assertEqual(output.err, "segment-size should be positive\n") output.clear() - with ExpectedException(SystemExit): + with self.assertRaises(SystemExit): argv = ["", "upload", "-S", "-40M", "container", "object"] swiftclient.shell.main(argv) self.assertEqual(output.err, "segment-size should be positive\n") output.clear() - with ExpectedException(SystemExit): + with self.assertRaises(SystemExit): argv = ["", "upload", "-S", "-40G", "container", "object"] swiftclient.shell.main(argv) self.assertEqual(output.err, "segment-size should be positive\n") output.clear() -class TestSubcommandHelp(testtools.TestCase): +class TestSubcommandHelp(unittest.TestCase): def test_subcommand_help(self): for command in swiftclient.shell.commands: @@ -1298,7 +1297,7 @@ def test_no_help(self): @mock.patch.dict(os.environ, mocked_os_environ) -class TestDebugAndInfoOptions(testtools.TestCase): +class TestDebugAndInfoOptions(unittest.TestCase): @mock.patch('logging.basicConfig') @mock.patch('swiftclient.service.Connection') def test_option_after_posarg(self, connection, mock_logging): @@ -1329,7 +1328,7 @@ def test_debug_trumps_info(self, connection, mock_logging): % (mock_logging.call_args_list, argv)) -class TestBase(testtools.TestCase): +class TestBase(unittest.TestCase): """ Provide some common methods to subclasses """ diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index 95a46a5b..c378dbdc 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -18,7 +18,7 @@ import six import socket import string -import testtools +import unittest import warnings import tempfile from hashlib import md5 @@ -34,7 +34,7 @@ import swiftclient -class TestClientException(testtools.TestCase): +class TestClientException(unittest.TestCase): def test_is_exception(self): self.assertTrue(issubclass(c.ClientException, Exception)) @@ -251,12 +251,12 @@ def test_auth_v1_insecure(self): self.assertEqual(url, 'storageURL') self.assertEqual(token, 'someauthtoken') - e = self.assertRaises(c.ClientException, c.get_auth, - 'http://www.test.com/invalid_cert', - 'asdf', 'asdf', auth_version='1.0') + with self.assertRaises(c.ClientException) as exc_context: + c.get_auth('http://www.test.com/invalid_cert', + 'asdf', 'asdf', auth_version='1.0') # TODO: this test is really on validating the mock and not the # the full plumbing into the requests's 'verify' option - self.assertIn('invalid_certificate', str(e)) + self.assertIn('invalid_certificate', str(exc_context.exception)) def test_auth_v1_timeout(self): # this test has some overlap with @@ -583,8 +583,9 @@ def test_ok(self): def test_server_error(self): body = 'c' * 65 c.http_connection = self.fake_http_connection(500, body=body) - e = self.assertRaises(c.ClientException, c.head_account, - 'http://www.tests.com', 'asdf') + with self.assertRaises(c.ClientException) as exc_context: + c.head_account('http://www.tests.com', 'asdf') + e = exc_context.exception self.assertEqual(e.http_response_content, body) self.assertEqual(e.http_status, 500) self.assertRequests([ @@ -617,17 +618,17 @@ def test_ok(self): def test_server_error(self): body = 'c' * 65 c.http_connection = self.fake_http_connection(500, body=body) - e = self.assertRaises(c.ClientException, c.post_account, - 'http://www.tests.com', 'asdf', {}) - self.assertEqual(e.http_response_content, body) - self.assertEqual(e.http_status, 500) + with self.assertRaises(c.ClientException) as exc_mgr: + c.post_account('http://www.tests.com', 'asdf', {}) + self.assertEqual(exc_mgr.exception.http_response_content, body) + self.assertEqual(exc_mgr.exception.http_status, 500) self.assertRequests([ ('POST', 'http://www.tests.com', None, {'x-auth-token': 'asdf'}) ]) # TODO: this is a fairly brittle test of the __repr__ on the # ClientException which should probably be in a targeted test new_body = "[first 60 chars of response] " + body[0:60] - self.assertEqual(e.__str__()[-89:], new_body) + self.assertEqual(exc_mgr.exception.__str__()[-89:], new_body) class TestGetContainer(MockHttpTest): @@ -741,8 +742,9 @@ def test_head_ok(self): def test_server_error(self): body = 'c' * 60 c.http_connection = self.fake_http_connection(500, body=body) - e = self.assertRaises(c.ClientException, c.head_container, - 'http://www.test.com', 'asdf', 'container') + with self.assertRaises(c.ClientException) as exc_context: + c.head_container('http://www.test.com', 'asdf', 'container') + e = exc_context.exception self.assertRequests([ ('HEAD', '/container', '', {'x-auth-token': 'asdf'}), ]) @@ -765,9 +767,9 @@ def test_ok(self): def test_server_error(self): body = 'c' * 60 c.http_connection = self.fake_http_connection(500, body=body) - e = self.assertRaises(c.ClientException, c.put_container, - 'http://www.test.com', 'token', 'container') - self.assertEqual(e.http_response_content, body) + with self.assertRaises(c.ClientException) as exc_context: + c.put_container('http://www.test.com', 'token', 'container') + self.assertEqual(exc_context.exception.http_response_content, body) self.assertRequests([ ('PUT', '/container', '', { 'x-auth-token': 'token', @@ -972,7 +974,9 @@ def test_server_error(self): body = 'c' * 60 c.http_connection = self.fake_http_connection(500, body=body) args = ('http://www.test.com', 'asdf', 'asdf', 'asdf', 'asdf') - e = self.assertRaises(c.ClientException, c.put_object, *args) + with self.assertRaises(c.ClientException) as exc_context: + c.put_object(*args) + e = exc_context.exception self.assertEqual(e.http_response_content, body) self.assertEqual(e.http_status, 500) self.assertRequests([ @@ -1192,8 +1196,9 @@ def test_server_error(self): body = 'c' * 60 c.http_connection = self.fake_http_connection(500, body=body) args = ('http://www.test.com', 'token', 'container', 'obj', {}) - e = self.assertRaises(c.ClientException, c.post_object, *args) - self.assertEqual(e.http_response_content, body) + with self.assertRaises(c.ClientException) as exc_context: + c.post_object(*args) + self.assertEqual(exc_context.exception.http_response_content, body) self.assertRequests([ ('POST', 'http://www.test.com/container/obj', '', { 'x-auth-token': 'token', @@ -1347,17 +1352,23 @@ class TestHTTPConnection(MockHttpTest): def test_bad_url_scheme(self): url = u'www.test.com' - exc = self.assertRaises(c.ClientException, c.http_connection, url) + with self.assertRaises(c.ClientException) as exc_context: + c.http_connection(url) + exc = exc_context.exception expected = u'Unsupported scheme "" in url "www.test.com"' self.assertEqual(expected, str(exc)) url = u'://www.test.com' - exc = self.assertRaises(c.ClientException, c.http_connection, url) + with self.assertRaises(c.ClientException) as exc_context: + c.http_connection(url) + exc = exc_context.exception expected = u'Unsupported scheme "" in url "://www.test.com"' self.assertEqual(expected, str(exc)) url = u'blah://www.test.com' - exc = self.assertRaises(c.ClientException, c.http_connection, url) + with self.assertRaises(c.ClientException) as exc_context: + c.http_connection(url) + exc = exc_context.exception expected = u'Unsupported scheme "blah" in url "blah://www.test.com"' self.assertEqual(expected, str(exc)) @@ -1524,8 +1535,9 @@ def quick_sleep(*args): } c.http_connection = self.fake_http_connection( *code_iter, headers=auth_resp_headers) - e = self.assertRaises(c.ClientException, conn.head_account) - self.assertIn('Account HEAD failed', str(e)) + with self.assertRaises(c.ClientException) as exc_context: + conn.head_account() + self.assertIn('Account HEAD failed', str(exc_context.exception)) self.assertEqual(conn.attempts, conn.retries + 1) # test default no-retry @@ -1533,8 +1545,9 @@ def quick_sleep(*args): 200, 498, headers=auth_resp_headers) conn = c.Connection('http://www.test.com/auth/v1.0', 'asdf', 'asdf') - e = self.assertRaises(c.ClientException, conn.head_account) - self.assertIn('Account HEAD failed', str(e)) + with self.assertRaises(c.ClientException) as exc_context: + conn.head_account() + self.assertIn('Account HEAD failed', str(exc_context.exception)) self.assertEqual(conn.attempts, 1) def test_resp_read_on_server_error(self): @@ -2132,9 +2145,9 @@ def test_head_error(self): def test_get_error(self): c.http_connection = self.fake_http_connection(404) - e = self.assertRaises(c.ClientException, c.get_object, - 'http://www.test.com', 'asdf', 'asdf', 'asdf') - self.assertEqual(e.http_status, 404) + with self.assertRaises(c.ClientException) as exc_context: + c.get_object('http://www.test.com', 'asdf', 'asdf', 'asdf') + self.assertEqual(exc_context.exception.http_status, 404) class TestCloseConnection(MockHttpTest): diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index fe50f556..aae466c7 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import testtools +import unittest import mock import six import tempfile @@ -22,7 +22,7 @@ from swiftclient import utils as u -class TestConfigTrueValue(testtools.TestCase): +class TestConfigTrueValue(unittest.TestCase): def test_TRUE_VALUES(self): for v in u.TRUE_VALUES: @@ -37,7 +37,7 @@ def test_config_true_value(self): self.assertIs(u.config_true_value(False), False) -class TestPrtBytes(testtools.TestCase): +class TestPrtBytes(unittest.TestCase): def test_zero_bytes(self): bytes_ = 0 @@ -119,7 +119,7 @@ def test_overflow(self): self.assertEqual('1024Y', u.prt_bytes(bytes_, True).lstrip()) -class TestTempURL(testtools.TestCase): +class TestTempURL(unittest.TestCase): def setUp(self): super(TestTempURL, self).setUp() @@ -164,7 +164,7 @@ def test_generate_temp_url_bad_seconds(self): self.method) -class TestReadableToIterable(testtools.TestCase): +class TestReadableToIterable(unittest.TestCase): def test_iter(self): chunk_size = 4 @@ -216,7 +216,7 @@ def test_unicode(self): self.assertEqual(actual_md5sum, data.get_md5sum()) -class TestLengthWrapper(testtools.TestCase): +class TestLengthWrapper(unittest.TestCase): def test_stringio(self): contents = six.StringIO(u'a' * 50 + u'b' * 50) @@ -292,7 +292,7 @@ def test_segmented_file(self): self.assertEqual(md5(s).hexdigest(), data.get_md5sum()) -class TestGroupers(testtools.TestCase): +class TestGroupers(unittest.TestCase): def test_n_at_a_time(self): result = list(u.n_at_a_time(range(100), 9)) self.assertEqual([9] * 11 + [1], list(map(len, result))) diff --git a/tests/unit/utils.py b/tests/unit/utils.py index f8f5e908..1bfa8da8 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -18,7 +18,6 @@ from requests.structures import CaseInsensitiveDict from time import sleep import unittest -import testtools import mock import six from six.moves import reload_module @@ -189,7 +188,7 @@ def connect(*args, **ckwargs): return connect -class MockHttpTest(testtools.TestCase): +class MockHttpTest(unittest.TestCase): def setUp(self): super(MockHttpTest, self).setUp() From 67f5468ee485ac3480530df1e59f8f7f92071f66 Mon Sep 17 00:00:00 2001 From: Alistair Coles Date: Mon, 22 Feb 2016 15:05:27 +0000 Subject: [PATCH 094/454] Fix wrong args for get_container with full listing In client get_container(), when full_listing is true, the calls back to get_container() pass service_token as a positional arg which maps its value to the full_listing arg. It should use a keyword. Change-Id: Iac2af45df124ff33fcb7fbaf1ba959ef06c96378 Closes-Bug: #1496093 --- swiftclient/client.py | 4 ++-- tests/unit/test_swiftclient.py | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index 58fff7b8..ad3fd70f 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -769,7 +769,7 @@ def get_container(url, token, container, marker=None, limit=None, if full_listing: rv = get_container(url, token, container, marker, limit, prefix, delimiter, end_marker, path, http_conn, - service_token, headers=headers) + service_token=service_token, headers=headers) listing = rv[1] while listing: if not delimiter: @@ -778,7 +778,7 @@ def get_container(url, token, container, marker=None, limit=None, marker = listing[-1].get('name', listing[-1].get('subdir')) listing = get_container(url, token, container, marker, limit, prefix, delimiter, end_marker, path, - http_conn, service_token, + http_conn, service_token=service_token, headers=headers)[1] if listing: rv[1].extend(listing) diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index a2724650..e4876a1d 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -2382,6 +2382,28 @@ def test_service_token_get_container(self): actual['full_path']) self.assertEqual(conn.attempts, 1) + def test_service_token_get_container_full_listing(self): + # verify service token is sent with each request for a full listing + with mock.patch('swiftclient.client.http_connection', + self.fake_http_connection(200, 200)): + with mock.patch('swiftclient.client.parse_api_response') as resp: + resp.side_effect = ([{"name": "obj1"}], []) + conn = self.get_connection() + conn.get_container('container1', full_listing=True) + self.assertEqual(2, len(self.request_log), self.request_log) + expected_urls = iter(( + 'http://storage_url.com/container1?format=json', + 'http://storage_url.com/container1?format=json&marker=obj1' + )) + for actual in self.iter_request_log(): + self.assertEqual('GET', actual['method']) + actual_hdrs = actual['headers'] + self.assertEqual('stoken', actual_hdrs.get('X-Service-Token')) + self.assertEqual('token', actual_hdrs['X-Auth-Token']) + self.assertEqual(next(expected_urls), + actual['full_path']) + self.assertEqual(conn.attempts, 1) + def test_service_token_head_container(self): with mock.patch('swiftclient.client.http_connection', self.fake_http_connection(200)): From 4d44dcf36086add13d3353915c014f095ab99c6d Mon Sep 17 00:00:00 2001 From: Joel Wright Date: Fri, 19 Feb 2016 13:18:15 +0000 Subject: [PATCH 095/454] Do not reveal auth token in swiftclient log messages by default Currently the swiftclient logs sensitive info in headers when logging HTTP requests. This patch hides sensitive info in headers such as 'X-Auth-Token' in a similar way to swift itself (we add a 'reveal_sensitive_prefix' configuration to the client). With this patch, tokens are truncated by removing the specified number of characters, after which '...' is appended to the logged token to indicate that it has been redacted. Co-Authored-By: Li Cheng Co-Authored-By: Zack M. Davis Change-Id: I43dd7254f7281d4db59b286aa2145643c64e1705 Closes-bug: #1516692 --- swiftclient/client.py | 71 ++++++++++++++++++++++++++++--- swiftclient/shell.py | 2 + tests/unit/test_swiftclient.py | 76 +++++++++++++++++++++++++++++++++- 3 files changed, 141 insertions(+), 8 deletions(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index 58fff7b8..9ebdef9c 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -72,6 +72,64 @@ def prepare_unicode_headers(self, headers): logger = logging.getLogger("swiftclient") logger.addHandler(NullHandler()) +#: Default behaviour is to redact tokens, showing only the initial 16 chars. +#: To disable, set the value of 'redact_sensitive_tokens' to False. +#: When token redaction is enabled 'reveal_sensitive_prefix' configures the +#: maximum length of any sensitive token data sent to the logs (if the token +#: is less than 32 chars long then int(len(token)/2) chars will be logged, +logger_settings = { + 'redact_sensitive_tokens': True, + 'reveal_sensitive_prefix': 16 +} +#: A list of sensitive headers to redact in logs. Note that when extending this +#: list, the header names must be added in all lower case. +LOGGER_SENSITIVE_HEADERS = [ + 'x-auth-token', 'x-auth-key', 'x-service-token', 'x-storage-token', + 'x-account-meta-temp-url-key', 'x-account-meta-temp-url-key-2', + 'x-container-meta-temp-url-key', 'x-container-meta-temp-url-key-2', + 'set-cookie' +] + + +def safe_value(name, value): + """ + Only show up to logger_settings['reveal_sensitive_prefix'] characters + from a sensitive header. + + :param name: Header name + :param value: Header value + :return: Safe (header, value) pair + """ + if name.lower() in LOGGER_SENSITIVE_HEADERS: + prefix_length = logger_settings.get('reveal_sensitive_prefix', 16) + prefix_length = int( + min(prefix_length, (len(value) ** 2) / 32, len(value) / 2) + ) + redacted_value = value[0:prefix_length] + return redacted_value + '...' + return value + + +def scrub_headers(headers): + """ + Redact header values that can contain sensitive information that + should not be logged. + + :param headers: Either a dict or an iterable of two-element tuples + :return: Safe dictionary of headers with sensitive information removed + """ + if isinstance(headers, dict): + headers = headers.items() + headers = [ + (parse_header_string(key), parse_header_string(val)) + for (key, val) in headers + ] + if not logger_settings.get('redact_sensitive_tokens', True): + return dict(headers) + if logger_settings.get('reveal_sensitive_prefix', 16) < 0: + logger_settings['reveal_sensitive_prefix'] = 16 + return {key: safe_value(key, val) for (key, val) in headers} + def http_log(args, kwargs, resp, body): if not logger.isEnabledFor(logging.INFO): @@ -87,8 +145,9 @@ def http_log(args, kwargs, resp, body): else: string_parts.append(' %s' % element) if 'headers' in kwargs: - for element in kwargs['headers']: - header = ' -H "%s: %s"' % (element, kwargs['headers'][element]) + headers = scrub_headers(kwargs['headers']) + for element in headers: + header = ' -H "%s: %s"' % (element, headers[element]) string_parts.append(header) # log response as debug if good, or info if error @@ -99,7 +158,7 @@ def http_log(args, kwargs, resp, body): log_method("REQ: %s", "".join(string_parts)) log_method("RESP STATUS: %s %s", resp.status, resp.reason) - log_method("RESP HEADERS: %s", resp.getheaders()) + log_method("RESP HEADERS: %s", scrub_headers(resp.getheaders())) if body: log_method("RESP BODY: %s", body) @@ -386,11 +445,11 @@ def get_auth_1_0(url, user, key, snet, **kwargs): parsed, conn = http_connection(url, cacert=cacert, insecure=insecure, timeout=timeout) method = 'GET' - conn.request(method, parsed.path, '', - {'X-Auth-User': user, 'X-Auth-Key': key}) + headers = {'X-Auth-User': user, 'X-Auth-Key': key} + conn.request(method, parsed.path, '', headers) resp = conn.getresponse() body = resp.read() - http_log((url, method,), {}, resp, body) + http_log((url, method,), headers, resp, body) url = resp.getheader('x-storage-url') # There is a side-effect on current Rackspace 1.0 server where a diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 4b444a92..02f49dde 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -33,6 +33,7 @@ from swiftclient.multithreading import OutputManager from swiftclient.exceptions import ClientException from swiftclient import __version__ as client_version +from swiftclient.client import logger_settings as client_logger_settings from swiftclient.service import SwiftService, SwiftError, \ SwiftUploadObject, get_conn from swiftclient.command_helpers import print_account_stats, \ @@ -1107,6 +1108,7 @@ def parse_args(parser, args, enforce_requires=True): if options.debug: logging.basicConfig(level=logging.DEBUG) logging.getLogger('iso8601').setLevel(logging.WARNING) + client_logger_settings['redact_sensitive_tokens'] = False elif options.info: logging.basicConfig(level=logging.INFO) diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index a2724650..77cf6076 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -987,7 +987,7 @@ def test_unicode_ok(self): mock_file) text = u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' headers = {'X-Header1': text, - 'X-2': 1, 'X-3': {'a': 'b'}, 'a-b': '.x:yz mn:fg:lp'} + 'X-2': '1', 'X-3': "{'a': 'b'}", 'a-b': '.x:yz mn:fg:lp'} resp = MockHttpResponse() conn[1].getresponse = resp.fake_response @@ -1221,7 +1221,7 @@ def test_unicode_ok(self): text = u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' headers = {'X-Header1': text, b'X-Header2': 'value', - 'X-2': '1', 'X-3': {'a': 'b'}, 'a-b': '.x:yz mn:kl:qr', + 'X-2': '1', 'X-3': "{'a': 'b'}", 'a-b': '.x:yz mn:kl:qr', 'X-Object-Meta-Header-not-encoded': text, b'X-Object-Meta-Header-encoded': 'value'} @@ -2188,6 +2188,78 @@ def test_get_error(self): 'http://www.test.com', 'asdf', 'asdf', 'asdf') self.assertEqual(e.http_status, 404) + def test_redact_token(self): + with mock.patch('swiftclient.client.logger.debug') as mock_log: + token_value = 'tkee96b40a8ca44fc5ad72ec5a7c90d9b' + unicode_token_value = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' + u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' + u'\u5929\u7a7a\u4e2d\u7684\u4e4c') + set_cookie_value = 'X-Auth-Token=%s' % token_value + c.http_log( + ['GET'], + {'headers': { + 'X-Auth-Token': token_value, + 'X-Storage-Token': unicode_token_value + }}, + MockHttpResponse( + status=200, + headers={ + 'X-Auth-Token': token_value, + 'X-Storage-Token': unicode_token_value, + 'Etag': b'mock_etag', + 'Set-Cookie': set_cookie_value + } + ), + '' + ) + out = [] + for _, args, kwargs in mock_log.mock_calls: + for arg in args: + out.append(u'%s' % arg) + output = u''.join(out) + self.assertIn('X-Auth-Token', output) + self.assertIn(token_value[:16] + '...', output) + self.assertIn('X-Storage-Token', output) + self.assertIn(unicode_token_value[:8] + '...', output) + self.assertIn('Set-Cookie', output) + self.assertIn(set_cookie_value[:16] + '...', output) + self.assertNotIn(token_value, output) + self.assertNotIn(unicode_token_value, output) + self.assertNotIn(set_cookie_value, output) + + def test_show_token(self): + with mock.patch('swiftclient.client.logger.debug') as mock_log: + token_value = 'tkee96b40a8ca44fc5ad72ec5a7c90d9b' + unicode_token_value = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' + u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' + u'\u5929\u7a7a\u4e2d\u7684\u4e4c') + c.logger_settings['redact_sensitive_tokens'] = False + c.http_log( + ['GET'], + {'headers': { + 'X-Auth-Token': token_value, + 'X-Storage-Token': unicode_token_value + }}, + MockHttpResponse( + status=200, + headers=[ + ('X-Auth-Token', token_value), + ('X-Storage-Token', unicode_token_value), + ('Etag', b'mock_etag') + ] + ), + '' + ) + out = [] + for _, args, kwargs in mock_log.mock_calls: + for arg in args: + out.append(u'%s' % arg) + output = u''.join(out) + self.assertIn('X-Auth-Token', output) + self.assertIn(token_value, output) + self.assertIn('X-Storage-Token', output) + self.assertIn(unicode_token_value, output) + class TestCloseConnection(MockHttpTest): From 011d730c9bc328084378692ed8f4dfb61024009a Mon Sep 17 00:00:00 2001 From: John Dickinson Date: Mon, 22 Feb 2016 10:32:28 -0800 Subject: [PATCH 096/454] Update api docs title to make ToC better Change-Id: Ie8eeb3dd8eddf1868f0fa99911c459ae9f7b0091 --- doc/source/apis.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/source/apis.rst b/doc/source/apis.rst index 1a8e8f7d..6545554b 100644 --- a/doc/source/apis.rst +++ b/doc/source/apis.rst @@ -1,6 +1,6 @@ -============ -Introduction -============ +====================== +python-swiftclient API +====================== The python-swiftclient includes two levels of API; a low level client API that provides simple python wrappers around the various authentication mechanisms From f95de16875b0f78013c5dccec3b50909fb95c914 Mon Sep 17 00:00:00 2001 From: shu-mutou Date: Tue, 15 Dec 2015 16:37:34 +0900 Subject: [PATCH 097/454] Drop py33 support "Python 3.3 support is being dropped since OpenStack Liberty." written in following URL. https://wiki.openstack.org/wiki/Python3 And already the infra team and the oslo team are dropping py33 support from their projects. Since we rely on oslo for a lot of our work, and depend on infra for our CI, we should drop py33 support too. Change-Id: Ia8f2b26e446175b0d892a11952ef3dc11dcdc73c Closes-Bug: #1526170 --- setup.cfg | 1 - tox.ini | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index d745957d..4c48a658 100644 --- a/setup.cfg +++ b/setup.cfg @@ -17,7 +17,6 @@ classifier = Programming Language :: Python :: 2 Programming Language :: Python :: 2.7 Programming Language :: Python :: 3 - Programming Language :: Python :: 3.3 Programming Language :: Python :: 3.4 Programming Language :: Python :: 3.5 diff --git a/tox.ini b/tox.ini index a21ea995..f841b3a9 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,py33,py34,py35,pypy,pep8 +envlist = py27,py34,py35,pypy,pep8 minversion = 1.6 skipsdist = True From 00de6cfa63ba10ceec5bd47544951aafeca7208d Mon Sep 17 00:00:00 2001 From: Min Min Ren Date: Wed, 24 Feb 2016 19:38:30 +0800 Subject: [PATCH 098/454] Check threads number options validation Add checking object-threads, container-threads and segment-threads options validation The values should be a positive integer Change-Id: Iaf98e33dae4b9a7f82e33f7cc2e5a0b293a1c76f Close-Bug: #1546973 --- swiftclient/shell.py | 56 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 50 insertions(+), 6 deletions(-) diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 4b444a92..e0c4a2af 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -92,11 +92,11 @@ def st_delete(parser, args, output_manager): parser.add_option( '', '--object-threads', type=int, default=10, help='Number of threads to use for deleting objects. ' - 'Default is 10.') + 'Its value must be a positive integer. Default is 10.') parser.add_option( '', '--container-threads', type=int, default=10, help='Number of threads to use for deleting containers. ' - 'Default is 10.') + 'Its value must be a positive integer. Default is 10.') (options, args) = parse_args(parser, args) args = args[1:] if (not args and not options.yes_all) or (args and options.yes_all): @@ -105,6 +105,22 @@ def st_delete(parser, args, output_manager): st_delete_help) return + if options.object_threads <= 0: + output_manager.error( + 'ERROR: option --object-threads should be a positive integer.' + '\n\nUsage: %s delete %s\n%s', + BASENAME, st_delete_options, + st_delete_help) + return + + if options.container_threads <= 0: + output_manager.error( + 'ERROR: option --container-threads should be a positive integer.' + '\n\nUsage: %s delete %s\n%s', + BASENAME, st_delete_options, + st_delete_help) + return + _opts = vars(options) _opts['object_dd_threads'] = options.object_threads with SwiftService(options=_opts) as swift: @@ -272,11 +288,11 @@ def st_download(parser, args, output_manager): parser.add_option( '', '--object-threads', type=int, default=10, help='Number of threads to use for downloading objects. ' - 'Default is 10.') + 'Its value must be a positive integer. Default is 10.') parser.add_option( '', '--container-threads', type=int, default=10, help='Number of threads to use for downloading containers. ' - 'Default is 10.') + 'Its value must be a positive integer. Default is 10.') parser.add_option( '', '--no-download', action='store_true', default=False, @@ -318,6 +334,20 @@ def st_download(parser, args, output_manager): st_download_options, st_download_help) return + if options.object_threads <= 0: + output_manager.error( + 'ERROR: option --object-threads should be a positive integer.\n\n' + 'Usage: %s download %s\n%s', BASENAME, + st_download_options, st_download_help) + return + + if options.container_threads <= 0: + output_manager.error( + 'ERROR: option --container-threads should be a positive integer.' + '\n\nUsage: %s download %s\n%s', BASENAME, + st_download_options, st_download_help) + return + _opts = vars(options) _opts['object_dd_threads'] = options.object_threads with SwiftService(options=_opts) as swift: @@ -803,11 +833,11 @@ def st_upload(parser, args, output_manager): parser.add_option( '', '--object-threads', type=int, default=10, help='Number of threads to use for uploading full objects. ' - 'Default is 10.') + 'Its value must be a positive integer. Default is 10.') parser.add_option( '', '--segment-threads', type=int, default=10, help='Number of threads to use for uploading object segments. ' - 'Default is 10.') + 'Its value must be a positive integer. Default is 10.') parser.add_option( '-H', '--header', action='append', dest='header', default=[], help='Set request headers with the syntax header:value. ' @@ -860,6 +890,20 @@ def st_upload(parser, args, output_manager): output_manager.error("segment-size should be positive") return + if options.object_threads <= 0: + output_manager.error( + 'ERROR: option --object-threads should be a positive integer.' + '\n\nUsage: %s upload %s\n%s', BASENAME, st_upload_options, + st_upload_help) + return + + if options.segment_threads <= 0: + output_manager.error( + 'ERROR: option --segment-threads should be a positive integer.' + '\n\nUsage: %s upload %s\n%s', BASENAME, st_upload_options, + st_upload_help) + return + _opts = vars(options) _opts['object_uu_threads'] = options.object_threads with SwiftService(options=_opts) as swift: From adc3177d1f0ba56e711e1fc7d408d99459645bc4 Mon Sep 17 00:00:00 2001 From: Paul Belanger Date: Wed, 24 Feb 2016 14:45:16 -0500 Subject: [PATCH 099/454] Have tox manage LANG environmental var We need to do this because some of the py34 testing that python-swiftclient uses depends on this variable. The reason we don't see the issue in the gate, is because current bare-trusty images have this set on the jenkins shell user. When we move to just using DIBs, the variable won't be setup by default and python3 tests will fail. For more information: https://review.openstack.org/282898 Change-Id: Id9017f31b0543bccac9c07b83237b909e2bd2b0c Signed-off-by: Paul Belanger --- tox.ini | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index a21ea995..cc68d315 100644 --- a/tox.ini +++ b/tox.ini @@ -6,7 +6,9 @@ skipsdist = True [testenv] usedevelop = True install_command = pip install -U {opts} {packages} -setenv = VIRTUAL_ENV={envdir} +setenv = + LANG=en_US.utf8 + VIRTUAL_ENV={envdir} deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt From c3f06417049e17a8d45ee5926c5043cb6c8aa9ef Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Wed, 24 Feb 2016 16:56:55 -0800 Subject: [PATCH 100/454] Follow-up to patch 282363 * Improve some formatting * Be more explicit about how much will be revealed when * Rename redact_sensitive_tokens to redact_sensitive_headers, as it affects more than tokens. Change-Id: I02b375d914e9f0a210d038ecb31188d09a8ffce3 --- swiftclient/client.py | 19 ++++++++++++------- swiftclient/shell.py | 2 +- tests/unit/test_swiftclient.py | 2 +- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index 9ebdef9c..8375fede 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -72,13 +72,18 @@ def prepare_unicode_headers(self, headers): logger = logging.getLogger("swiftclient") logger.addHandler(NullHandler()) -#: Default behaviour is to redact tokens, showing only the initial 16 chars. -#: To disable, set the value of 'redact_sensitive_tokens' to False. -#: When token redaction is enabled 'reveal_sensitive_prefix' configures the -#: maximum length of any sensitive token data sent to the logs (if the token -#: is less than 32 chars long then int(len(token)/2) chars will be logged, +#: Default behaviour is to redact header values known to contain secrets, +#: such as ``X-Auth-Key`` and ``X-Auth-Token``. Up to the first 16 chars +#: may be revealed. +#: +#: To disable, set the value of ``redact_sensitive_headers`` to ``False``. +#: +#: When header redaction is enabled, ``reveal_sensitive_prefix`` configures the +#: maximum length of any sensitive header data sent to the logs. If the header +#: is less than twice this length, only ``int(len(value)/2)`` chars will be +#: logged; if it is less than 15 chars long, even less will be logged. logger_settings = { - 'redact_sensitive_tokens': True, + 'redact_sensitive_headers': True, 'reveal_sensitive_prefix': 16 } #: A list of sensitive headers to redact in logs. Note that when extending this @@ -124,7 +129,7 @@ def scrub_headers(headers): (parse_header_string(key), parse_header_string(val)) for (key, val) in headers ] - if not logger_settings.get('redact_sensitive_tokens', True): + if not logger_settings.get('redact_sensitive_headers', True): return dict(headers) if logger_settings.get('reveal_sensitive_prefix', 16) < 0: logger_settings['reveal_sensitive_prefix'] = 16 diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 02f49dde..15be20ae 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -1108,7 +1108,7 @@ def parse_args(parser, args, enforce_requires=True): if options.debug: logging.basicConfig(level=logging.DEBUG) logging.getLogger('iso8601').setLevel(logging.WARNING) - client_logger_settings['redact_sensitive_tokens'] = False + client_logger_settings['redact_sensitive_headers'] = False elif options.info: logging.basicConfig(level=logging.INFO) diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index 77cf6076..ae144e24 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -2233,7 +2233,7 @@ def test_show_token(self): unicode_token_value = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' u'\u5929\u7a7a\u4e2d\u7684\u4e4c') - c.logger_settings['redact_sensitive_tokens'] = False + c.logger_settings['redact_sensitive_headers'] = False c.http_log( ['GET'], {'headers': { From 85b4d65f3b75b96e537ccc89a7ad60e365a4446a Mon Sep 17 00:00:00 2001 From: Thiago da Silva Date: Thu, 25 Feb 2016 16:06:56 -0500 Subject: [PATCH 101/454] adding .manpages script to swift client this is the same script as in swift core Change-Id: Ib9ea882cf7c1ba1d663254d38c7ac163b55e45da --- .manpages | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100755 .manpages diff --git a/.manpages b/.manpages new file mode 100755 index 00000000..69fcfc74 --- /dev/null +++ b/.manpages @@ -0,0 +1,18 @@ +#!/bin/sh + +RET=0 +for MAN in doc/manpages/* ; do + OUTPUT=$(LC_ALL=en_US.UTF-8 MANROFFSEQ='' MANWIDTH=80 man --warnings -E UTF-8 -l \ + -Tutf8 -Z "$MAN" 2>&1 >/dev/null) + if [ -n "$OUTPUT" ] ; then + RET=1 + echo "$MAN:" + echo "$OUTPUT" + fi +done + +if [ "$RET" -eq "0" ] ; then + echo "All manpages are fine" +fi + +exit "$RET" From 46d817828082105a69d4da53fef2f2fbefc54809 Mon Sep 17 00:00:00 2001 From: Joel Wright Date: Thu, 25 Feb 2016 17:13:35 +0000 Subject: [PATCH 102/454] Fix test for redacting sensitive data in client.http_log() The test should have included utf8 encoded unicode data to test that encoded unicode data stored in headers was parsed correctly. Also fixes the docstring for swiftclient.safe_value() Change-Id: Id0def0b3af7a364f1257cc22f67b71c0cc5d8479 --- swiftclient/client.py | 2 +- tests/unit/test_swiftclient.py | 23 ++++++++++++++--------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index 8375fede..f65d4bb5 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -103,7 +103,7 @@ def safe_value(name, value): :param name: Header name :param value: Header value - :return: Safe (header, value) pair + :return: Safe header value """ if name.lower() in LOGGER_SENSITIVE_HEADERS: prefix_length = logger_settings.get('reveal_sensitive_prefix', 16) diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index ae144e24..331e8041 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -2191,23 +2191,26 @@ def test_get_error(self): def test_redact_token(self): with mock.patch('swiftclient.client.logger.debug') as mock_log: token_value = 'tkee96b40a8ca44fc5ad72ec5a7c90d9b' + token_encoded = token_value.encode('utf8') unicode_token_value = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' u'\u5929\u7a7a\u4e2d\u7684\u4e4c') + unicode_token_encoded = unicode_token_value.encode('utf8') set_cookie_value = 'X-Auth-Token=%s' % token_value + set_cookie_encoded = set_cookie_value.encode('utf8') c.http_log( ['GET'], {'headers': { - 'X-Auth-Token': token_value, - 'X-Storage-Token': unicode_token_value + 'X-Auth-Token': token_encoded, + 'X-Storage-Token': unicode_token_encoded }}, MockHttpResponse( status=200, headers={ - 'X-Auth-Token': token_value, - 'X-Storage-Token': unicode_token_value, + 'X-Auth-Token': token_encoded, + 'X-Storage-Token': unicode_token_encoded, 'Etag': b'mock_etag', - 'Set-Cookie': set_cookie_value + 'Set-Cookie': set_cookie_encoded } ), '' @@ -2230,21 +2233,23 @@ def test_redact_token(self): def test_show_token(self): with mock.patch('swiftclient.client.logger.debug') as mock_log: token_value = 'tkee96b40a8ca44fc5ad72ec5a7c90d9b' + token_encoded = token_value.encode('utf8') unicode_token_value = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91' u'\u5929\u7a7a\u4e2d\u7684\u4e4c') c.logger_settings['redact_sensitive_headers'] = False + unicode_token_encoded = unicode_token_value.encode('utf8') c.http_log( ['GET'], {'headers': { - 'X-Auth-Token': token_value, - 'X-Storage-Token': unicode_token_value + 'X-Auth-Token': token_encoded, + 'X-Storage-Token': unicode_token_encoded }}, MockHttpResponse( status=200, headers=[ - ('X-Auth-Token', token_value), - ('X-Storage-Token', unicode_token_value), + ('X-Auth-Token', token_encoded), + ('X-Storage-Token', unicode_token_encoded), ('Etag', b'mock_etag') ] ), From 62b09844b60f63769b6590183a829205aa0af616 Mon Sep 17 00:00:00 2001 From: John Dickinson Date: Thu, 11 Feb 2016 22:12:05 -0800 Subject: [PATCH 103/454] authors/changelog updates for 2.8 release Change-Id: I335c30fd7d0a0120c87fa60c5c142b8ed95c56bf --- .mailmap | 3 +++ AUTHORS | 13 +++++++++++-- ChangeLog | 43 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/.mailmap b/.mailmap index c3ae3733..840a8ac8 100644 --- a/.mailmap +++ b/.mailmap @@ -86,3 +86,6 @@ Stanislaw Pitucha Mahati Chamarthy Peter Lisak Doug Hellmann +Ondrej Novy +James Nzomo +Alessandro Pilotti diff --git a/AUTHORS b/AUTHORS index 5644db91..e7215de4 100644 --- a/AUTHORS +++ b/AUTHORS @@ -10,11 +10,14 @@ Clint Byrum (clint@fewbar.com) Tristan Cacqueray (tristan.cacqueray@enovance.com) Sergio Cazzolato (sergio.j.cazzolato@intel.com) Mahati Chamarthy (mahati.chamarthy@gmail.com) +Chaozhe.Chen (chaozhe.chen@easystack.cn) Ray Chen (oldsharp@163.com) +Li Cheng (shcli@cn.ibm.com) Taurus Cheung (Taurus.Cheung@harmonicinc.com) Alistair Coles (alistair.coles@hpe.com) Ian Cordasco (ian.cordasco@rackspace.com) Nick Craig-Wood (nick@craig-wood.com) +Thiago da Silva (thiago@redhat.com) Sean Dague (sean@dague.net) Zack M. Davis (zdavis@swiftstack.com) John Dickinson (me@not.mn) @@ -38,7 +41,7 @@ Charles Hsu (charles0126@gmail.com) Kun Huang (gareth@unitedstack.com) Matthieu Huin (mhu@enovance.com) Andreas Jaeger (aj@suse.de) -OpenStack Jenkins (jenkins@openstack.org) +Jude Job (judeopenstack@gmail.com) Vasyl Khomenko (vasiliyk@yahoo-inc.com) Leah Klearman (lklrmn@gmail.com) Jaivish Kothari (jaivish.kothari@nectechnologies.in) @@ -52,6 +55,7 @@ Peter Lisak (peter.lisak@firma.seznam.cz) Feng Liu (mefengliu23@gmail.com) Jing Liuqing (jing.liuqing@99cloud.net) Hemanth Makkapati (hemanth.makkapati@mailtrust.com) +Pratik Mallya (pratik.mallya@gmail.com) Steve Martinelli (stevemar@ca.ibm.com) Juan J. Martinez (juan@memset.com) Donagh McCabe (donagh.mccabe@hpe.com) @@ -59,13 +63,14 @@ Ben McCann (ben@benmccann.com) Andy McCrae (andy.mccrae@gmail.com) Stuart McLaren (stuart.mclaren@hpe.com) Samuel Merritt (sam@swiftstack.com) +Min Min Ren (rminmin@cn.ibm.com) Jola Mirecka (jola.mirecka@hp.com) Hiroshi Miura (miurahr@nttdata.co.jp) Sam Morrison (sorrison@gmail.com) Dirk Mueller (dirk@dmllr.de) Zhenguo Niu (zhenguo@unitedstack.com) Ondrej Novy (ondrej.novy@firma.seznam.cz) -Alessandro Pilotti (apilotti@cloudbasesolutions.com) +James Nzomo (james@tdt.rocks) Alessandro Pilotti (ap@pilotti.it) Stanislaw Pitucha (stanislaw.pitucha@hpe.com) Dan Prince (dprince@redhat.com) @@ -77,6 +82,7 @@ Mark Seger (mark.seger@hpe.com) Chuck Short (chuck.short@canonical.com) David Shrewsbury (shrewsbury.dave@gmail.com) Pradeep Kumar Singh (pradeep.singh@nectechnologies.in) +Alexandra Settle (alexandra.settle@rackspace.com) Jeremy Stanley (fungi@yuggoth.org) Victor Stinner (victor.stinner@enovance.com) Jiří Suchomel (jsuchome@suse.cz) @@ -103,3 +109,6 @@ tanlin (lin.tan@intel.com) yangxurong (yangxurong@huawei.com) yuxcer (yuxcer@126.com) zhang-jinnan (ben.os@99cloud.net) +hgangwx (hgangwx@cn.ibm.com) +shu-mutou (shu-mutou@rf.jp.nec.com) +SaiKiran (saikiranveeravarapu@gmail.com) diff --git a/ChangeLog b/ChangeLog index 4cbb8788..00899eb9 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,46 @@ +2.8.0 +----- + +* Python 2.6 support has been removed. Also, Python 3.3 gate testing has + been removed. Support for Python 3.3 is only best-effort. Currently + supported and tested versions of Python are Python 2.7 and Python 3.4. + +* Do not reveal sensitive headers in swiftclient log messages by default. + This is controlled by the client.logger_settings dictionary. Setting the + `redact_sensitive_headers` key to False prevents the information hiding. If + the value is True (the default), the `reveal_sensitive_prefix` controls + the maximum length of any sensitive header value logged. The default is + 16 to match the default in Swift. + +* Object downloads that fail partway through will now retry with a Range + request to read the rest of the object. + +* Object uploads will be retried if the source supports seek/tell or has a + reset() method. + +* Delete requests will use the cluster's bulk delete feature, if available, + for requests that would require a lot of individual deletes. + +* The delete CLI option now accepts a --prefix option to delete objects that + start with the given prefix (similar to the same-named option for list). + +* Add support for the auth-version to be specified using + --os-identity-api-version or OS_IDENTITY_API_VERSION + for compatibility with other openstack client command + line options. + +* --debug and --info command-line options now work anywhere in the command. + +* Objects can now be uploaded to pseudo-directories with the CLI. + +* Fixed an issue with uploading a large object that includes a unicode path. + +* swiftclient can now auth against Keystone using only a project (tenant) + and a token. This is useful when the client doesn't have access to the + password for a user but otherwise has been granted access. + +* Various other minor bug fixes and improvements. + 2.7.0 ----- From aa0edd00966237163451fc44cda2c593a5215cbe Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Fri, 26 Feb 2016 11:25:10 -0800 Subject: [PATCH 104/454] Force header keys/values to bytes/unicode before coercing to unicode Previously, parse_header_string was only called with data coming out of requests, which would be either bytes or unicode. Now that we're sending it request headers as well (see related change), we need to be more defensive. If the value given is neither bytes nor unicode, convert it to a native string. This will allow developers using the client API to continue sending header dicts like {'X-Delete-After': 2} ...as in Swift's test/probe/test_object_expirer.py Change-Id: Ie57a93274507b184af5cad4260f244359a585f09 Related-Change: I43dd7254f7281d4db59b286aa2145643c64e1705 --- swiftclient/client.py | 2 ++ tests/unit/test_swiftclient.py | 7 +++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index 8375fede..d2f089a2 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -169,6 +169,8 @@ def http_log(args, kwargs, resp, body): def parse_header_string(data): + if not isinstance(data, (six.text_type, six.binary_type)): + data = str(data) if six.PY2: if isinstance(data, six.text_type): # Under Python2 requests only returns binary_type, but if we get diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index ae144e24..2c552be0 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -1203,13 +1203,16 @@ class TestPostObject(MockHttpTest): def test_ok(self): c.http_connection = self.fake_http_connection(200) + delete_at = 2.1 # not str! we don't know what other devs will use! args = ('http://www.test.com', 'token', 'container', 'obj', - {'X-Object-Meta-Test': 'mymeta'}) + {'X-Object-Meta-Test': 'mymeta', + 'X-Delete-At': delete_at}) c.post_object(*args) self.assertRequests([ ('POST', '/container/obj', '', { 'x-auth-token': 'token', - 'X-Object-Meta-Test': 'mymeta'}), + 'X-Object-Meta-Test': 'mymeta', + 'X-Delete-At': delete_at}), ]) def test_unicode_ok(self): From 671d6febb227c86f5c1801c2d7365c576daeb1ad Mon Sep 17 00:00:00 2001 From: Alexandra Date: Mon, 29 Feb 2016 13:50:10 +0000 Subject: [PATCH 105/454] Minor edits to the api page Change-Id: Ie65f73532e53a858ee9ab4b4634dcfcaf8a93c3b --- doc/source/apis.rst | 131 ++++++++++++++++++++++---------------------- 1 file changed, 65 insertions(+), 66 deletions(-) diff --git a/doc/source/apis.rst b/doc/source/apis.rst index 6545554b..935b4a42 100644 --- a/doc/source/apis.rst +++ b/doc/source/apis.rst @@ -2,17 +2,17 @@ python-swiftclient API ====================== -The python-swiftclient includes two levels of API; a low level client API that -provides simple python wrappers around the various authentication mechanisms -and the individual HTTP requests, and a high level service API that provides +The python-swiftclient includes two levels of API. A low level client API that +provides simple python wrappers around the various authentication mechanisms, +the individual HTTP requests, and a high level service API that provides methods for performing common operations in parallel on a thread pool. This document aims to provide guidance for choosing between these APIs and examples of usage for the service API. ------------------------- + Important Considerations ------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~ This section covers some important considerations, helpful hints, and things to avoid when integrating an object store into your workflow. @@ -20,27 +20,30 @@ to avoid when integrating an object store into your workflow. An Object Store is not a filesystem ----------------------------------- -It cannot be stressed enough that your usage of the object store should reflect -the proper use case, and not treat the storage like a filesystem. There are 2 -main restrictions to bear in mind here when designing your use of the object +.. important:: + + It cannot be stressed enough that your usage of the object store should reflect + the use case, and not treat the storage like a filesystem. + +There are 2 main restrictions to bear in mind here when designing your use of the object store: - * Objects cannot be renamed due to the way in which objects are stored and - references by the object store. This usually requires multiple copies of - the data to be moved between physical storage devices. - As a result, a move operation is not provided. If the user wants to move an - object they must re-upload to the new location and delete the - original. - * Objects cannot be modified. Objects are stored in multiple locations and are - checked for integrity based on the ``MD5 sum`` calculated during upload. - Object creation is a 1-shot event, and in order to modify the contents of an - object the entire new contents must be re-uploaded. In certain special cases - it is possible to work around this restriction using large objects, but no - general file-like access is available to modify a stored object. - ------------------------------- +#. Objects cannot be renamed due to the way in which objects are stored and + references by the object store. This usually requires multiple copies of + the data to be moved between physical storage devices. + As a result, a move operation is not provided. If the user wants to move an + object they must re-upload to the new location and delete the + original. +#. Objects cannot be modified. Objects are stored in multiple locations and are + checked for integrity based on the ``MD5 sum`` calculated during upload. + Object creation is a 1-shot event, and in order to modify the contents of an + object the entire new contents must be re-uploaded. In certain special cases + it is possible to work around this restriction using large objects, but no + general file-like access is available to modify a stored object. + + The swiftclient.Connection API ------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ A low level API that provides methods for authentication and methods that correspond to the individual REST API calls described in the swift @@ -48,12 +51,12 @@ documentation. For usage details see the client docs: :mod:`swiftclient.client`. --------------------------------- + The swiftclient.SwiftService API --------------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ A higher level API aimed at allowing developers an easy way to perform multiple -operations asynchronously using a configurable thread pool. Docs for each +operations asynchronously using a configurable thread pool. Documentation for each service method call can be found here: :mod:`swiftclient.service`. Configuration @@ -74,7 +77,7 @@ passed to the ``SwiftService`` during initialisation. The options available in this dictionary are described below, along with their defaults: Options -~~~~~~~ +^^^^^^^ ``retries``: ``5`` The number of times that the library should attempt to retry HTTP @@ -190,14 +193,14 @@ for an optional dictionary to override those specified at init time, and the appropriate docstrings show which options modify each method's behaviour. Authentication --------------- +~~~~~~~~~~~~~~ This section covers the various options for authenticating with a swift object store. The combinations of options required for each authentication version are detailed below. Version 1.0 Auth -~~~~~~~~~~~~~~~~ +---------------- ``auth_version``: ``environ.get('ST_AUTH_VERSION')`` @@ -208,8 +211,8 @@ Version 1.0 Auth ``key``: ``environ.get('ST_KEY')`` -Version 2.0 & 3.0 Auth -~~~~~~~~~~~~~~~~~~~~~~ +Version 2.0 and 3.0 Auth +------------------------ ``auth_version``: ``environ.get('ST_AUTH_VERSION')`` @@ -233,7 +236,7 @@ having options from different auth versions can cause unexpected behaviour. authorization fails. Operation Return Values ------------------------ +~~~~~~~~~~~~~~~~~~~~~~~ Each operation provided by the service API may raise a ``SwiftError`` or ``ClientException`` for any call that fails completely (or a call which @@ -291,7 +294,7 @@ All the possible ``action`` values are detailed below: ] Stat ----- +~~~~ Stat can be called against an account, a container, or a list of objects to get account stats, container stats or information about the given objects. In @@ -368,7 +371,7 @@ operation was not successful, and will include the keys below: } Example -~~~~~~~ +------- The code below demonstrates the use of ``stat`` to retrieve the headers for a given list of objects in a container using 20 threads. The code creates a @@ -396,7 +399,7 @@ mapping from object name to headers. ) List ----- +~~~~ List can be called against an account or a container to retrieve the containers or objects contained within them. Each call returns an iterator that returns @@ -453,7 +456,7 @@ dictionary as described below: } Example -~~~~~~~ +------- The code below demonstrates the use of ``list`` to list all items in a container that are over 10MiB in size: @@ -482,7 +485,7 @@ container that are over 10MiB in size: output_manager.error(e.value) Post ----- +~~~~ Post can be called against an account, container or list of objects in order to update the metadata attached to the given items. Each element of the object list @@ -494,7 +497,7 @@ an iterator over the results generated for each object post is returned. If the given container or account does not exist, the ``post`` method will raise a ``SwiftError``. -When a string is given for the object name, the options +.. When a string is given for the object name, the options Successful metadata update results are dictionaries as described below: @@ -510,34 +513,31 @@ Successful metadata update results are dictionaries as described below: } .. note:: + Updating user metadata keys will not only add any specified keys, but will also remove user metadata that has previously been set. This means that each time user metadata is updated, the complete set of desired key-value pairs must be specified. -Example -~~~~~~~ -.. Do we want to hide this section until it is complete? -TBD +.. Example +.. ------- -Download --------- +.. TBD -.. Do we want to hide this section until it is complete? +.. Download +.. ~~~~~~~~ -TBD +.. TBD -Example -~~~~~~~ +.. Example +.. ------- -.. Do we want to hide this section until it is complete? - -TBD +.. TBD Upload ------- +~~~~~~ Upload is always called against an account and container and with a list of objects to upload. Each element of the object list may be a plain string @@ -622,7 +622,7 @@ below: } Example -~~~~~~~ +------- The code below demonstrates the use of ``upload`` to upload all files and folders in ``/tmp``, and renaming each object by replacing ``/tmp`` in the @@ -689,31 +689,30 @@ object or directory marker names with ``temporary-objects``: except SwiftError as e: out_manager.error(e.value) -Delete ------- - +.. Delete +.. ~~~~~~ .. Do we want to hide this section until it is complete? -TBD +.. TBD -Example -~~~~~~~ +.. Example +.. ------- .. Do we want to hide this section until it is complete? -TBD +.. TBD -Capabilities ------------- +.. Capabilities +.. ~~~~~~~~~~~~ .. Do we want to hide this section until it is complete? -TBD +.. TBD -Example -~~~~~~~ +.. Example +.. ------- .. Do we want to hide this section until it is complete? -TBD +.. TBD From b7d20b8a1899897e560d378b624163cf1ee1d299 Mon Sep 17 00:00:00 2001 From: Hu Bing Date: Fri, 26 Feb 2016 00:20:29 +0800 Subject: [PATCH 106/454] download method shouldn't download all object in python-swiftclient/swiftclient/service.py, there is a method def download(self, container=None, objects=None, options=None): if container is specified but objects not, it download all objects in specified container. if both container and objects are specified, it download all specified objects in the container. when it comes to the case that, objects argument is specified, but it turned out to be empty array [ ], the download method download all the objects under specified container. this may be not reasonable. for example, the objects was not empty when it came from command line, but it's filtered, maybe by --prefix argument. at last, it turned out to be empty array. when calling download method with objects arguments being empty array, we should download nothing instead of all the objects under the specified container. Change-Id: I81aab935533a50b40679c8b3575f298c285233a8 Closes-bug: #1549881 --- swiftclient/service.py | 2 +- tests/unit/test_service.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/swiftclient/service.py b/swiftclient/service.py index 5fa2870b..f253ec86 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -1004,7 +1004,7 @@ def download(self, container=None, objects=None, options=None): raise raise SwiftError('Account not found', exc=err) - elif not objects: + elif objects is None: if '/' in container: raise SwiftError('\'/\' in container name', container=container) diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index 3fbe987a..dcd2b854 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -1690,6 +1690,22 @@ def test_download(self): self.assertEqual(resp['object'], 'test') self.assertEqual(resp['path'], 'test') + @mock.patch('swiftclient.service.interruptable_as_completed') + @mock.patch('swiftclient.service.SwiftService._download_container') + @mock.patch('swiftclient.service.SwiftService._download_object_job') + def test_download_with_objects_empty(self, mock_down_obj, + mock_down_cont, mock_as_comp): + fake_future = Future() + fake_future.set_result(1) + mock_as_comp.return_value = [fake_future] + service = SwiftService() + next(service.download('c', [], self.opts), None) + mock_down_obj.assert_not_called() + mock_down_cont.assert_not_called() + + next(service.download('c', options=self.opts), None) + self.assertEqual(True, mock_down_cont.called) + def test_download_with_output_dir(self): service = SwiftService() with mock.patch('swiftclient.service.Connection') as mock_conn: From e4d0cc7419b27a2dcb0b257d608fe70a7109aab1 Mon Sep 17 00:00:00 2001 From: John Dickinson Date: Wed, 2 Mar 2016 12:00:05 +0000 Subject: [PATCH 107/454] bumped version at request of release team Change-Id: I81eb6575ef5de8ecc75e781f07ad090024589cec --- ChangeLog | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index 00899eb9..f5e0a60c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,8 +1,7 @@ -2.8.0 +3.0.0 ----- -* Python 2.6 support has been removed. Also, Python 3.3 gate testing has - been removed. Support for Python 3.3 is only best-effort. Currently +* Python 2.6 and Python 3.3 support has been removed. Currently supported and tested versions of Python are Python 2.7 and Python 3.4. * Do not reveal sensitive headers in swiftclient log messages by default. From f179c36aea50ab2dbb8a3d2fb43ea032e8c455e2 Mon Sep 17 00:00:00 2001 From: Pete Zaitcev Date: Wed, 2 Mar 2016 08:59:59 -0700 Subject: [PATCH 108/454] Drop *.dbm* before running tests See change I74fb5122e80a223aaa70afaeec7a7c585aa33577 for the previous discussion. But basically, if you run tox -e py27 then tox -e py34, the latter fails with "db type could not be determined", because of stuck .testrepository/times.dbm. This patch fixes it by clearing the *.dbm*. Should be safe as long as periods aren't used commonly in stable file names. Change-Id: I617eca308261f291c510c8cbd432779f1c00b182 --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index cc68d315..eff0e9f9 100644 --- a/tox.ini +++ b/tox.ini @@ -12,9 +12,9 @@ setenv = deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt -commands = sh -c 'find . -not \( -type d -name .?\* -prune \) \ +commands = sh -c '(find . -not \( -type d -name .?\* -prune \) \ \( -type d -name "__pycache__" -or -type f -name "*.py[co]" \) \ - -print0 | xargs -0 rm -rf' + -print0; find . -name "*.dbm*" -print0) | xargs -0 rm -rf' python setup.py testr --testr-args="{posargs}" whitelist_externals = sh passenv = SWIFT_* *_proxy From 9b8ab67a780416508b995adafb07e96ea646d6f8 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Mon, 18 Jan 2016 17:05:28 -0800 Subject: [PATCH 109/454] Include response headers in ClientExceptions Now, client applications can get to things like transaction IDs for failures without needing to turn on all of logging. While we're at it, add a from_response factory method for ClientException. Co-Authored-By: Alexander Corwin Change-Id: Ib46d5f8fc7f36f651f5908bb9d900316fdaebce3 --- swiftclient/client.py | 90 +++++++--------------------------- swiftclient/exceptions.py | 15 +++++- swiftclient/shell.py | 11 ++++- tests/unit/test_shell.py | 33 ++++++++++++- tests/unit/test_swiftclient.py | 88 +++++++++++++++++++++++++++------ tests/unit/utils.py | 21 +++++--- 6 files changed, 158 insertions(+), 100 deletions(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index e5d564d5..4dbbd49c 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -463,9 +463,7 @@ def get_auth_1_0(url, user, key, snet, **kwargs): # bad URL would get you that document page and a 200. We error out # if we don't have a x-storage-url header and if we get a body. if resp.status < 200 or resp.status >= 300 or (body and not url): - raise ClientException('Auth GET failed', http_scheme=parsed.scheme, - http_host=conn.host, http_path=parsed.path, - http_status=resp.status, http_reason=resp.reason) + raise ClientException.from_response(resp, 'Auth GET failed', body) if snet: parsed = list(urlparse(url)) # Second item in the list is the netloc @@ -705,11 +703,7 @@ def get_account(url, token, marker=None, limit=None, prefix=None, resp_headers = resp_header_dict(resp) if resp.status < 200 or resp.status >= 300: - raise ClientException('Account GET failed', http_scheme=parsed.scheme, - http_host=conn.host, http_path=parsed.path, - http_query=qs, http_status=resp.status, - http_reason=resp.reason, - http_response_content=body) + raise ClientException.from_response(resp, 'Account GET failed', body) if resp.status == 204: return resp_headers, [] return resp_headers, parse_api_response(resp_headers, body) @@ -741,10 +735,7 @@ def head_account(url, token, http_conn=None, service_token=None): body = resp.read() http_log((url, method,), {'headers': headers}, resp, body) if resp.status < 200 or resp.status >= 300: - raise ClientException('Account HEAD failed', http_scheme=parsed.scheme, - http_host=conn.host, http_path=parsed.path, - http_status=resp.status, http_reason=resp.reason, - http_response_content=body) + raise ClientException.from_response(resp, 'Account HEAD failed', body) resp_headers = resp_header_dict(resp) return resp_headers @@ -786,13 +777,7 @@ def post_account(url, token, headers, http_conn=None, response_dict=None, store_response(resp, response_dict) if resp.status < 200 or resp.status >= 300: - raise ClientException('Account POST failed', - http_scheme=parsed.scheme, - http_host=conn.host, - http_path=parsed.path, - http_status=resp.status, - http_reason=resp.reason, - http_response_content=body) + raise ClientException.from_response(resp, 'Account POST failed', body) resp_headers = {} for header, value in resp.getheaders(): resp_headers[header.lower()] = value @@ -877,11 +862,7 @@ def get_container(url, token, container, marker=None, limit=None, {'headers': headers}, resp, body) if resp.status < 200 or resp.status >= 300: - raise ClientException('Container GET failed', - http_scheme=parsed.scheme, http_host=conn.host, - http_path=cont_path, http_query=qs, - http_status=resp.status, http_reason=resp.reason, - http_response_content=body) + raise ClientException.from_response(resp, 'Container GET failed', body) resp_headers = resp_header_dict(resp) if resp.status == 204: return resp_headers, [] @@ -922,11 +903,8 @@ def head_container(url, token, container, http_conn=None, headers=None, {'headers': req_headers}, resp, body) if resp.status < 200 or resp.status >= 300: - raise ClientException('Container HEAD failed', - http_scheme=parsed.scheme, http_host=conn.host, - http_path=path, http_status=resp.status, - http_reason=resp.reason, - http_response_content=body) + raise ClientException.from_response( + resp, 'Container HEAD failed', body) resp_headers = resp_header_dict(resp) return resp_headers @@ -969,11 +947,7 @@ def put_container(url, token, container, headers=None, http_conn=None, http_log(('%s%s' % (url.replace(parsed.path, ''), path), method,), {'headers': headers}, resp, body) if resp.status < 200 or resp.status >= 300: - raise ClientException('Container PUT failed', - http_scheme=parsed.scheme, http_host=conn.host, - http_path=path, http_status=resp.status, - http_reason=resp.reason, - http_response_content=body) + raise ClientException.from_response(resp, 'Container PUT failed', body) def post_container(url, token, container, headers, http_conn=None, @@ -1012,11 +986,8 @@ def post_container(url, token, container, headers, http_conn=None, store_response(resp, response_dict) if resp.status < 200 or resp.status >= 300: - raise ClientException('Container POST failed', - http_scheme=parsed.scheme, http_host=conn.host, - http_path=path, http_status=resp.status, - http_reason=resp.reason, - http_response_content=body) + raise ClientException.from_response( + resp, 'Container POST failed', body) def delete_container(url, token, container, http_conn=None, @@ -1052,11 +1023,8 @@ def delete_container(url, token, container, http_conn=None, store_response(resp, response_dict) if resp.status < 200 or resp.status >= 300: - raise ClientException('Container DELETE failed', - http_scheme=parsed.scheme, http_host=conn.host, - http_path=path, http_status=resp.status, - http_reason=resp.reason, - http_response_content=body) + raise ClientException.from_response( + resp, 'Container DELETE failed', body) def get_object(url, token, container, name, http_conn=None, @@ -1109,11 +1077,7 @@ def get_object(url, token, container, name, http_conn=None, body = resp.read() http_log(('%s%s' % (url.replace(parsed.path, ''), path), method,), {'headers': headers}, resp, body) - raise ClientException('Object GET failed', http_scheme=parsed.scheme, - http_host=conn.host, http_path=path, - http_status=resp.status, - http_reason=resp.reason, - http_response_content=body) + raise ClientException.from_response(resp, 'Object GET failed', body) if resp_chunk_size: object_body = _ObjectBody(resp, resp_chunk_size) else: @@ -1160,10 +1124,7 @@ def head_object(url, token, container, name, http_conn=None, http_log(('%s%s' % (url.replace(parsed.path, ''), path), method,), {'headers': headers}, resp, body) if resp.status < 200 or resp.status >= 300: - raise ClientException('Object HEAD failed', http_scheme=parsed.scheme, - http_host=conn.host, http_path=path, - http_status=resp.status, http_reason=resp.reason, - http_response_content=body) + raise ClientException.from_response(resp, 'Object HEAD failed', body) resp_headers = resp_header_dict(resp) return resp_headers @@ -1275,10 +1236,7 @@ def put_object(url, token=None, container=None, name=None, contents=None, store_response(resp, response_dict) if resp.status < 200 or resp.status >= 300: - raise ClientException('Object PUT failed', http_scheme=parsed.scheme, - http_host=conn.host, http_path=path, - http_status=resp.status, http_reason=resp.reason, - http_response_content=body) + raise ClientException.from_response(resp, 'Object PUT failed', body) etag = resp.getheader('etag', '').strip('"') return etag @@ -1318,10 +1276,7 @@ def post_object(url, token, container, name, headers, http_conn=None, store_response(resp, response_dict) if resp.status < 200 or resp.status >= 300: - raise ClientException('Object POST failed', http_scheme=parsed.scheme, - http_host=conn.host, http_path=path, - http_status=resp.status, http_reason=resp.reason, - http_response_content=body) + raise ClientException.from_response(resp, 'Object POST failed', body) def delete_object(url, token=None, container=None, name=None, http_conn=None, @@ -1375,11 +1330,7 @@ def delete_object(url, token=None, container=None, name=None, http_conn=None, store_response(resp, response_dict) if resp.status < 200 or resp.status >= 300: - raise ClientException('Object DELETE failed', - http_scheme=parsed.scheme, http_host=conn.host, - http_path=path, http_status=resp.status, - http_reason=resp.reason, - http_response_content=body) + raise ClientException.from_response(resp, 'Object DELETE failed', body) def get_capabilities(http_conn): @@ -1396,11 +1347,8 @@ def get_capabilities(http_conn): body = resp.read() http_log((parsed.geturl(), 'GET',), {'headers': {}}, resp, body) if resp.status < 200 or resp.status >= 300: - raise ClientException('Capabilities GET failed', - http_scheme=parsed.scheme, - http_host=conn.host, http_path=parsed.path, - http_status=resp.status, http_reason=resp.reason, - http_response_content=body) + raise ClientException.from_response( + resp, 'Capabilities GET failed', body) resp_headers = resp_header_dict(resp) return parse_api_response(resp_headers, body) diff --git a/swiftclient/exceptions.py b/swiftclient/exceptions.py index 370a8d0f..da70379e 100644 --- a/swiftclient/exceptions.py +++ b/swiftclient/exceptions.py @@ -13,12 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +from six.moves import urllib + class ClientException(Exception): def __init__(self, msg, http_scheme='', http_host='', http_port='', http_path='', http_query='', http_status=None, http_reason='', - http_device='', http_response_content=''): + http_device='', http_response_content='', + http_response_headers=None): super(ClientException, self).__init__(msg) self.msg = msg self.http_scheme = http_scheme @@ -30,6 +33,16 @@ def __init__(self, msg, http_scheme='', http_host='', http_port='', self.http_reason = http_reason self.http_device = http_device self.http_response_content = http_response_content + self.http_response_headers = http_response_headers + + @classmethod + def from_response(cls, resp, msg=None, body=None): + msg = msg or '%s %s' % (resp.status_code, resp.reason) + body = body or resp.content + parsed_url = urllib.parse.urlparse(resp.request.url) + return cls(msg, parsed_url.scheme, parsed_url.hostname, + parsed_url.port, parsed_url.path, parsed_url.query, + resp.status_code, resp.reason, '', body, resp.headers) def __str__(self): a = self.msg diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 15be20ae..68b93445 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -33,7 +33,8 @@ from swiftclient.multithreading import OutputManager from swiftclient.exceptions import ClientException from swiftclient import __version__ as client_version -from swiftclient.client import logger_settings as client_logger_settings +from swiftclient.client import logger_settings as client_logger_settings, \ + parse_header_string from swiftclient.service import SwiftService, SwiftError, \ SwiftUploadObject, get_conn from swiftclient.command_helpers import print_account_stats, \ @@ -1475,7 +1476,13 @@ def main(arguments=None): parser.usage = globals()['st_%s_help' % args[0]] try: globals()['st_%s' % args[0]](parser, argv[1:], output) - except (ClientException, RequestException, socket.error) as err: + except ClientException as err: + output.error(str(err)) + trans_id = (err.http_response_headers or {}).get('X-Trans-Id') + if trans_id: + output.error("Failed Transaction ID: %s", + parse_header_string(trans_id)) + except (RequestException, socket.error) as err: output.error(str(err)) if output.get_error_count() > 0: diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index e3ab0bf2..3ea93361 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -1073,13 +1073,42 @@ def test_post_account(self, connection): def test_post_account_bad_auth(self, connection): argv = ["", "post"] connection.return_value.post_account.side_effect = \ - swiftclient.ClientException('bad auth') + swiftclient.ClientException( + 'bad auth', http_response_headers={'X-Trans-Id': 'trans_id'}) with CaptureOutput() as output: with self.assertRaises(SystemExit): swiftclient.shell.main(argv) - self.assertEqual(output.err, 'bad auth\n') + self.assertEqual(output.err, + 'bad auth\nFailed Transaction ID: trans_id\n') + + # do it again with a unicode token + connection.return_value.post_account.side_effect = \ + swiftclient.ClientException( + 'bad auth', http_response_headers={ + 'X-Trans-Id': 'non\u2011utf8'}) + + with CaptureOutput() as output: + with self.assertRaises(SystemExit): + swiftclient.shell.main(argv) + + self.assertEqual(output.err, + 'bad auth\n' + 'Failed Transaction ID: non\u2011utf8\n') + + # do it again with a wonky token + connection.return_value.post_account.side_effect = \ + swiftclient.ClientException( + 'bad auth', http_response_headers={ + 'X-Trans-Id': b'non\xffutf8'}) + + with CaptureOutput() as output: + with self.assertRaises(SystemExit): + swiftclient.shell.main(argv) + + self.assertEqual(output.err, + 'bad auth\nFailed Transaction ID: non%FFutf8\n') @mock.patch('swiftclient.service.Connection') def test_post_account_not_found(self, connection): diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index 65830f58..f3bee3bf 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -51,6 +51,7 @@ def test_format(self): 'status', 'reason', 'device', + 'response_content', ) for value in test_kwargs: kwargs = { @@ -59,6 +60,26 @@ def test_format(self): exc = c.ClientException('test', **kwargs) self.assertIn(value, str(exc)) + def test_attrs(self): + test_kwargs = ( + 'scheme', + 'host', + 'port', + 'path', + 'query', + 'status', + 'reason', + 'device', + 'response_content', + 'response_headers', + ) + for value in test_kwargs: + key = 'http_%s' % value + kwargs = {key: value} + exc = c.ClientException('test', **kwargs) + self.assertIs(True, hasattr(exc, key)) + self.assertEqual(getattr(exc, key), value) + class MockHttpResponse(object): def __init__(self, status=0, headers=None, verify=False): @@ -582,7 +603,9 @@ def test_ok(self): def test_server_error(self): body = 'c' * 65 - c.http_connection = self.fake_http_connection(500, body=body) + headers = {'foo': 'bar'} + c.http_connection = self.fake_http_connection( + StubResponse(500, body, headers)) with self.assertRaises(c.ClientException) as exc_context: c.head_account('http://www.tests.com', 'asdf') e = exc_context.exception @@ -741,7 +764,9 @@ def test_head_ok(self): def test_server_error(self): body = 'c' * 60 - c.http_connection = self.fake_http_connection(500, body=body) + headers = {'foo': 'bar'} + c.http_connection = self.fake_http_connection( + StubResponse(500, body, headers)) with self.assertRaises(c.ClientException) as exc_context: c.head_container('http://www.test.com', 'asdf', 'container') e = exc_context.exception @@ -750,6 +775,7 @@ def test_server_error(self): ]) self.assertEqual(e.http_status, 500) self.assertEqual(e.http_response_content, body) + self.assertEqual(e.http_response_headers, headers) class TestPutContainer(MockHttpTest): @@ -766,10 +792,13 @@ def test_ok(self): def test_server_error(self): body = 'c' * 60 - c.http_connection = self.fake_http_connection(500, body=body) + headers = {'foo': 'bar'} + c.http_connection = self.fake_http_connection( + StubResponse(500, body, headers)) with self.assertRaises(c.ClientException) as exc_context: c.put_container('http://www.test.com', 'token', 'container') self.assertEqual(exc_context.exception.http_response_content, body) + self.assertEqual(exc_context.exception.http_response_headers, headers) self.assertRequests([ ('PUT', '/container', '', { 'x-auth-token': 'token', @@ -792,9 +821,14 @@ def test_ok(self): class TestGetObject(MockHttpTest): def test_server_error(self): - c.http_connection = self.fake_http_connection(500) - self.assertRaises(c.ClientException, c.get_object, - 'http://www.test.com', 'asdf', 'asdf', 'asdf') + body = 'c' * 60 + headers = {'foo': 'bar'} + c.http_connection = self.fake_http_connection( + StubResponse(500, body, headers)) + with self.assertRaises(c.ClientException) as exc_context: + c.get_object('http://www.test.com', 'asdf', 'asdf', 'asdf') + self.assertEqual(exc_context.exception.http_response_content, body) + self.assertEqual(exc_context.exception.http_response_headers, headers) def test_query_string(self): c.http_connection = self.fake_http_connection(200, @@ -945,9 +979,14 @@ def get_auth(): class TestHeadObject(MockHttpTest): def test_server_error(self): - c.http_connection = self.fake_http_connection(500) - self.assertRaises(c.ClientException, c.head_object, - 'http://www.test.com', 'asdf', 'asdf', 'asdf') + body = 'c' * 60 + headers = {'foo': 'bar'} + c.http_connection = self.fake_http_connection( + StubResponse(500, body, headers)) + with self.assertRaises(c.ClientException) as exc_context: + c.head_object('http://www.test.com', 'asdf', 'asdf', 'asdf') + self.assertEqual(exc_context.exception.http_response_content, body) + self.assertEqual(exc_context.exception.http_response_headers, headers) def test_request_headers(self): c.http_connection = self.fake_http_connection(204) @@ -1024,12 +1063,15 @@ def test_chunk_warning(self): def test_server_error(self): body = 'c' * 60 - c.http_connection = self.fake_http_connection(500, body=body) + headers = {'foo': 'bar'} + c.http_connection = self.fake_http_connection( + StubResponse(500, body, headers)) args = ('http://www.test.com', 'asdf', 'asdf', 'asdf', 'asdf') with self.assertRaises(c.ClientException) as exc_context: c.put_object(*args) e = exc_context.exception self.assertEqual(e.http_response_content, body) + self.assertEqual(e.http_response_headers, headers) self.assertEqual(e.http_status, 500) self.assertRequests([ ('PUT', '/asdf/asdf', 'asdf', { @@ -1249,11 +1291,14 @@ def test_unicode_ok(self): def test_server_error(self): body = 'c' * 60 - c.http_connection = self.fake_http_connection(500, body=body) + headers = {'foo': 'bar'} + c.http_connection = self.fake_http_connection( + StubResponse(500, body, headers)) args = ('http://www.test.com', 'token', 'container', 'obj', {}) with self.assertRaises(c.ClientException) as exc_context: c.post_object(*args) self.assertEqual(exc_context.exception.http_response_content, body) + self.assertEqual(exc_context.exception.http_response_headers, headers) self.assertRequests([ ('POST', 'http://www.test.com/container/obj', '', { 'x-auth-token': 'token', @@ -1273,9 +1318,14 @@ def test_ok(self): ]) def test_server_error(self): - c.http_connection = self.fake_http_connection(500) - self.assertRaises(c.ClientException, c.delete_object, - 'http://www.test.com', 'asdf', 'asdf', 'asdf') + body = 'c' * 60 + headers = {'foo': 'bar'} + c.http_connection = self.fake_http_connection( + StubResponse(500, body, headers)) + with self.assertRaises(c.ClientException) as exc_context: + c.delete_object('http://www.test.com', 'asdf', 'asdf', 'asdf') + self.assertEqual(exc_context.exception.http_response_content, body) + self.assertEqual(exc_context.exception.http_response_headers, headers) def test_query_string(self): c.http_connection = self.fake_http_connection(200, @@ -1302,9 +1352,15 @@ def test_ok(self): self.assertTrue(http_conn[1].resp.has_been_read) def test_server_error(self): - conn = self.fake_http_connection(500) + body = 'c' * 60 + headers = {'foo': 'bar'} + conn = self.fake_http_connection( + StubResponse(500, body, headers)) http_conn = conn('http://www.test.com/info') - self.assertRaises(c.ClientException, c.get_capabilities, http_conn) + with self.assertRaises(c.ClientException) as exc_context: + c.get_capabilities(http_conn) + self.assertEqual(exc_context.exception.http_response_content, body) + self.assertEqual(exc_context.exception.http_response_headers, headers) def test_conn_get_capabilities_with_auth(self): auth_headers = { diff --git a/tests/unit/utils.py b/tests/unit/utils.py index 1bfa8da8..3b043bc7 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -87,17 +87,19 @@ class FakeConn(object): def __init__(self, status, etag=None, body='', timestamp='1', headers=None): - self.status = status + self.status_code = self.status = status self.reason = 'Fake' + self.scheme = 'http' self.host = '1.2.3.4' self.port = '1234' self.sent = 0 self.received = 0 self.etag = etag - self.body = body + self.content = self.body = body self.timestamp = timestamp self._is_closed = True self.headers = headers or {} + self.request = None def getresponse(self): if kwargs.get('raise_exc'): @@ -223,15 +225,18 @@ class RequestsWrapper(object): pass conn = RequestsWrapper() - def request(method, url, *args, **kwargs): + def request(method, path, *args, **kwargs): try: conn.resp = self.fake_connect() except StopIteration: self.fail('Unexpected %s request for %s' % ( - method, url)) - self.request_log.append((parsed, method, url, args, + method, path)) + self.request_log.append((parsed, method, path, args, kwargs, conn.resp)) conn.host = conn.resp.host + conn.resp.request = RequestsWrapper() + conn.resp.request.url = '%s://%s%s' % ( + conn.resp.scheme, conn.resp.host, path) conn.resp.has_been_read = False _orig_read = conn.resp.read @@ -240,15 +245,15 @@ def read(*args, **kwargs): return _orig_read(*args, **kwargs) conn.resp.read = read if on_request: - status = on_request(method, url, *args, **kwargs) + status = on_request(method, path, *args, **kwargs) conn.resp.status = status if auth_token: headers = args[1] self.assertEqual(auth_token, headers.get('X-Auth-Token')) if query_string: - self.assertTrue(url.endswith('?' + query_string)) - if url.endswith('invalid_cert') and not insecure: + self.assertTrue(path.endswith('?' + query_string)) + if path.endswith('invalid_cert') and not insecure: from swiftclient import client as c raise c.ClientException("invalid_certificate") if exc: From 965fd4d3fc890175fd6357d940e6e56a9f33b07a Mon Sep 17 00:00:00 2001 From: Min Min Ren Date: Sat, 5 Mar 2016 02:57:08 +0800 Subject: [PATCH 110/454] Initialise delete_object mock before it's called Fix the linked bug. delete_object mock should be before it's called by swiftclient.shell.main function. Related: https://review.openstack.org/221219 Change-Id: I52143a93c129764c02bba05267f3563c824e82cb Partial-Bug: #1480223 --- tests/unit/test_shell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index e3ab0bf2..59ed17d6 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -688,10 +688,10 @@ def test_upload_delete_dlo_segments(self, connection): [None, []] ] connection.return_value.put_object.return_value = EMPTY_ETAG - swiftclient.shell.main(argv) # create the delete_object child mock here in attempt to fix # https://bugs.launchpad.net/python-swiftclient/+bug/1480223 connection.return_value.delete_object.return_value = None + swiftclient.shell.main(argv) connection.return_value.put_object.assert_called_with( 'container', self.tmpfile.lstrip('/'), From 52eab61e7e2c6e30f7b5dc19399d35baee0f1b81 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Mon, 7 Dec 2015 10:35:02 -0800 Subject: [PATCH 111/454] Move python-keystoneclient to "extras" This should make it more clear to users that they may want to install it. Change-Id: I8bb4f3eba1fc6d2b7b23c3bd51663678e755a69e --- setup.cfg | 4 ++++ test-requirements.txt | 1 - tox.ini | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 4c48a658..036ef430 100644 --- a/setup.cfg +++ b/setup.cfg @@ -32,6 +32,10 @@ scripts = data_files = share/man/man1 = doc/manpages/swift.1 +[extras] +keystone = + python-keystoneclient>=0.7.0 + [entry_points] console_scripts = swift = swiftclient.shell:main diff --git a/test-requirements.txt b/test-requirements.txt index 044f7c3f..0a813987 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -3,6 +3,5 @@ hacking>=0.10.0,<0.11 coverage>=3.6 mock>=1.2 oslosphinx -python-keystoneclient>=0.7.0 sphinx>=1.1.2,<1.2 testrepository>=0.0.18 diff --git a/tox.ini b/tox.ini index 717128b9..26cfb112 100644 --- a/tox.ini +++ b/tox.ini @@ -12,6 +12,7 @@ setenv = deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt + .[keystone] commands = sh -c '(find . -not \( -type d -name .?\* -prune \) \ \( -type d -name "__pycache__" -or -type f -name "*.py[co]" \) \ -print0; find . -name "*.dbm*" -print0) | xargs -0 rm -rf' From f5224a696e55510b3c73b8b60b51b1d97e9e9237 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Thu, 17 Mar 2016 12:29:36 -0700 Subject: [PATCH 112/454] Add tests for thread option validation Change-Id: If84714c7ea6be1c95c5898a82db2d4b6c9637242 --- tests/unit/test_shell.py | 95 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index ddb40f11..8430942a 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -760,6 +760,37 @@ def test_upload_segments_to_same_container(self, connection): 'x-object-meta-mtime': mock.ANY}, response_dict={}) + @mock.patch.object(swiftclient.service.SwiftService, '_should_bulk_delete', + lambda *a: False) + @mock.patch('swiftclient.service.Connection') + def test_delete_bad_threads(self, mock_connection): + mock_connection.return_value.get_container.return_value = (None, []) + mock_connection.return_value.attempts = 0 + + def check_bad(argv): + args, env = _make_cmd( + 'delete', {}, {}, cmd_args=['cont'] + argv) + with mock.patch.dict(os.environ, env): + with CaptureOutput() as output: + self.assertRaises(SystemExit, swiftclient.shell.main, args) + self.assertIn( + 'ERROR: option %s should be a positive integer.' % argv[0], + output.err) + + def check_good(argv): + args, env = _make_cmd( + 'delete', {}, {}, cmd_args=['cont'] + argv) + with mock.patch.dict(os.environ, env): + with CaptureOutput() as output: + swiftclient.shell.main(args) + self.assertEqual('', output.err) + check_bad(["--object-threads", "-1"]) + check_bad(["--object-threads", "0"]) + check_bad(["--container-threads", "-1"]) + check_bad(["--container-threads", "0"]) + check_good(["--object-threads", "1"]) + check_good(["--container-threads", "1"]) + @mock.patch.object(swiftclient.service.SwiftService, '_should_bulk_delete', lambda *a: False) @mock.patch('swiftclient.service.Connection') @@ -2195,6 +2226,38 @@ def on_request(method, path, *args, **kwargs): return status return on_request + @mock.patch.object(swiftclient.service.SwiftService, '_should_bulk_delete', + lambda *a: False) + @mock.patch('swiftclient.service.Connection') + def test_upload_bad_threads(self, mock_connection): + mock_connection.return_value.put_object.return_value = EMPTY_ETAG + mock_connection.return_value.attempts = 0 + + def check_bad(argv): + args, env = self._make_cmd( + 'upload', cmd_args=[self.cont, self.obj] + argv) + with mock.patch.dict(os.environ, env): + with CaptureOutput() as output: + self.assertRaises(SystemExit, swiftclient.shell.main, args) + self.assertIn( + 'ERROR: option %s should be a positive integer.' % argv[0], + output.err) + + def check_good(argv): + args, env = self._make_cmd( + 'upload', + cmd_args=[self.cont, self.obj, '--leave-segments'] + argv) + with mock.patch.dict(os.environ, env): + with CaptureOutput() as output: + swiftclient.shell.main(args) + self.assertEqual('', output.err) + check_bad(["--object-threads", "-1"]) + check_bad(["--object-threads", "0"]) + check_bad(["--segment-threads", "-1"]) + check_bad(["--segment-threads", "0"]) + check_good(["--object-threads", "1"]) + check_good(["--segment-threads", "1"]) + def test_upload_with_read_write_access(self): req_handler = self._fake_cross_account_auth(True, True) fake_conn = self.fake_http_connection(403, 403, @@ -2346,6 +2409,38 @@ def test_upload_with_no_access(self): self.assertTrue(expected_err in out.err) self.assertEqual('', out) + @mock.patch.object(swiftclient.service.SwiftService, '_should_bulk_delete', + lambda *a: False) + @mock.patch('swiftclient.service.Connection') + def test_download_bad_threads(self, mock_connection): + mock_connection.return_value.get_object.return_value = [{}, ''] + mock_connection.return_value.attempts = 0 + + def check_bad(argv): + args, env = self._make_cmd( + 'download', cmd_args=[self.cont, self.obj] + argv) + with mock.patch.dict(os.environ, env): + with CaptureOutput() as output: + self.assertRaises(SystemExit, swiftclient.shell.main, args) + self.assertIn( + 'ERROR: option %s should be a positive integer.' % argv[0], + output.err) + + def check_good(argv): + args, env = self._make_cmd( + 'download', + cmd_args=[self.cont, self.obj, '--no-download'] + argv) + with mock.patch.dict(os.environ, env): + with CaptureOutput() as output: + swiftclient.shell.main(args) + self.assertEqual('', output.err) + check_bad(["--object-threads", "-1"]) + check_bad(["--object-threads", "0"]) + check_bad(["--container-threads", "-1"]) + check_bad(["--container-threads", "0"]) + check_good(["--object-threads", "1"]) + check_good(["--container-threads", "1"]) + def test_download_with_read_write_access(self): req_handler = self._fake_cross_account_auth(True, True) fake_conn = self.fake_http_connection(403, on_request=req_handler, From d2bd2f0859cde8f15c70834e892aab06cd65a064 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Thu, 17 Mar 2016 15:32:10 -0700 Subject: [PATCH 113/454] Initialize delete_object mock *before* creating all the threads Previously, we'd occasionally get spurious failures like FAIL: test_delete_account (tests.unit.test_shell.TestShell) ---------------------------------------------------------------------- Traceback (most recent call last): File ".../mock/mock.py", line 1721, in _inner return f(*args, **kw) File ".../mock/mock.py", line 1305, in patched return func(*args, **keywargs) File ".../tests/unit/test_shell.py", line 788, in test_delete_account response_dict={})], any_order=True) File ".../mock/mock.py", line 983, in assert_has_calls ), cause) File ".../six.py", line 718, in raise_from raise value AssertionError: (call(u'container', u'object', query_string=None, response_dict={}),) not all found in call list Related-Bug: #1539536 Related-Bug: #1480223 Change-Id: I810894545ca74d3b2f2dbde2d0388eb69c2ba710 --- tests/unit/test_shell.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index 59ed17d6..7abfc5ae 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -778,6 +778,7 @@ def test_delete_account(self, connection): connection.return_value.attempts = 0 argv = ["", "delete", "--all"] connection.return_value.head_object.return_value = {} + connection.return_value.delete_object.return_value = None swiftclient.shell.main(argv) connection.return_value.delete_object.assert_has_calls([ mock.call('container', 'object', query_string=None, From 67db1646a3ff4759f93b7305500ac3aa4091b65e Mon Sep 17 00:00:00 2001 From: Nguyen Hung Phuong Date: Fri, 18 Mar 2016 16:14:18 +0700 Subject: [PATCH 114/454] Removes redundant "to" This patch removes "to" in swift guide. Change-Id: I4104bd3651947e341e1ed06c76b3f8f66f565c3f --- swiftclient/shell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 4ba71b49..2c5bca82 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -254,7 +254,7 @@ def st_delete(parser, args, output_manager): sides. --no-shuffle By default, when downloading a complete account or container, download order is randomised in order to - to reduce the load on individual drives when multiple + reduce the load on individual drives when multiple clients are executed simultaneously to download the same set of objects (e.g. a nightly automated download script to multiple servers). Enable this option to From 51a8a5a7ae3a065307974791ed1dd43503fb3b5a Mon Sep 17 00:00:00 2001 From: Marek Kaleta Date: Wed, 23 Mar 2016 12:09:49 +0100 Subject: [PATCH 115/454] Fix SwiftPostObject options usage in SwiftService SwiftService().post(cont, [SwiftPostObject(obj, options]) currently ignores options['header'], raises exception when options['headers'] is set and make malformed metadata when options['meta'] is set. Fix tipos in code, add unittest for SwiftService().post Closes-Bug: #1560052 Change-Id: Ie460f753492e9b73836b4adfc7c9c0f2130a8a91 --- swiftclient/service.py | 6 ++--- tests/unit/test_service.py | 46 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/swiftclient/service.py b/swiftclient/service.py index f253ec86..99c833e0 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -569,7 +569,7 @@ def post(self, container=None, objects=None, options=None): { 'meta': [], - 'headers': [], + 'header': [], 'read_acl': None, # For containers only 'write_acl': None, # For containers only 'sync_to': None, # For containers only @@ -700,10 +700,10 @@ def post(self, container=None, objects=None, options=None): if 'meta' in obj_options: headers.update( split_headers( - obj_options['meta'], 'X-Object-Meta' + obj_options['meta'], 'X-Object-Meta-' ) ) - if 'headers' in obj_options: + if 'header' in obj_options: headers.update( split_headers(obj_options['header'], '') ) diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index 418ee85b..a392967d 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -2130,3 +2130,49 @@ def test_download_object_job_skip_identical_diff_nested_slo(self): resp_chunk_size=65536, headers={'If-None-Match': on_disk_md5}, response_dict={})]) + + +class TestServicePost(_TestServiceBase): + + def setUp(self): + super(TestServicePost, self).setUp() + self.opts = swiftclient.service._default_local_options.copy() + + @mock.patch('swiftclient.service.MultiThreadingManager') + @mock.patch('swiftclient.service.ResultsIterator') + def test_object_post(self, res_iter, thread_manager): + """ + Check post method translates strings and objects to _post_object_job + calls correctly + """ + tm_instance = Mock() + thread_manager.return_value = tm_instance + + self.opts.update({'meta': ["meta1:test1"], "header": ["hdr1:test1"]}) + spo = swiftclient.service.SwiftPostObject( + "test_spo", + {'meta': ["meta1:test2"], "header": ["hdr1:test2"]}) + + service = SwiftService() + SwiftService().post('test_c', ['test_o', spo], self.opts) + + calls = [ + mock.call( + service._post_object_job, 'test_c', 'test_o', + { + "X-Object-Meta-Meta1": "test1", + "Hdr1": "test1"}, + {}), + mock.call( + service._post_object_job, 'test_c', 'test_spo', + { + "X-Object-Meta-Meta1": "test2", + "Hdr1": "test2"}, + {}), + ] + tm_instance.object_uu_pool.submit.assert_has_calls(calls) + self.assertEqual( + tm_instance.object_uu_pool.submit.call_count, len(calls)) + + res_iter.assert_called_with( + [tm_instance.object_uu_pool.submit()] * len(calls)) From f86b2d8138f5c366d0f8ab8e8cb2492d9cd07ca4 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Wed, 23 Mar 2016 10:05:37 -0700 Subject: [PATCH 116/454] Clean up some unnecessary variables Change-Id: Iac93ced6344d4a6fee7e6390e891fde765814c03 --- tests/unit/test_service.py | 50 +++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index a392967d..e9310aa7 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -1655,16 +1655,15 @@ def test_download_object_job_exception(self): self.assertEqual(expected_r, actual_r) def test_download(self): - service = SwiftService() with mock.patch('swiftclient.service.Connection') as mock_conn: header = {'content-length': self.obj_len, 'etag': self.obj_etag} mock_conn.get_object.return_value = header, self._readbody() - resp = service._download_object_job(mock_conn, - 'c', - 'test', - self.opts) + resp = SwiftService()._download_object_job(mock_conn, + 'c', + 'test', + self.opts) self.assertIsNone(resp.get('error')) self.assertIs(True, resp['success']) @@ -1689,7 +1688,6 @@ def test_download_with_objects_empty(self, mock_down_obj, self.assertEqual(True, mock_down_cont.called) def test_download_with_output_dir(self): - service = SwiftService() with mock.patch('swiftclient.service.Connection') as mock_conn: header = {'content-length': self.obj_len, 'etag': self.obj_etag} @@ -1697,10 +1695,10 @@ def test_download_with_output_dir(self): options = self.opts.copy() options['out_directory'] = 'temp_dir' - resp = service._download_object_job(mock_conn, - 'c', - 'example/test', - options) + resp = SwiftService()._download_object_job(mock_conn, + 'c', + 'example/test', + options) self.assertIsNone(resp.get('error')) self.assertIs(True, resp['success']) @@ -1709,7 +1707,6 @@ def test_download_with_output_dir(self): self.assertEqual(resp['path'], 'temp_dir/example/test') def test_download_with_remove_prefix(self): - service = SwiftService() with mock.patch('swiftclient.service.Connection') as mock_conn: header = {'content-length': self.obj_len, 'etag': self.obj_etag} @@ -1718,10 +1715,10 @@ def test_download_with_remove_prefix(self): options = self.opts.copy() options['prefix'] = 'example/' options['remove_prefix'] = True - resp = service._download_object_job(mock_conn, - 'c', - 'example/test', - options) + resp = SwiftService()._download_object_job(mock_conn, + 'c', + 'example/test', + options) self.assertIsNone(resp.get('error')) self.assertIs(True, resp['success']) @@ -1730,7 +1727,6 @@ def test_download_with_remove_prefix(self): self.assertEqual(resp['path'], 'test') def test_download_with_remove_prefix_and_remove_slashes(self): - service = SwiftService() with mock.patch('swiftclient.service.Connection') as mock_conn: header = {'content-length': self.obj_len, 'etag': self.obj_etag} @@ -1739,10 +1735,10 @@ def test_download_with_remove_prefix_and_remove_slashes(self): options = self.opts.copy() options['prefix'] = 'example' options['remove_prefix'] = True - resp = service._download_object_job(mock_conn, - 'c', - 'example/test', - options) + resp = SwiftService()._download_object_job(mock_conn, + 'c', + 'example/test', + options) self.assertIsNone(resp.get('error')) self.assertIs(True, resp['success']) @@ -1751,7 +1747,6 @@ def test_download_with_remove_prefix_and_remove_slashes(self): self.assertEqual(resp['path'], 'test') def test_download_with_output_dir_and_remove_prefix(self): - service = SwiftService() with mock.patch('swiftclient.service.Connection') as mock_conn: header = {'content-length': self.obj_len, 'etag': self.obj_etag} @@ -1761,10 +1756,10 @@ def test_download_with_output_dir_and_remove_prefix(self): options['prefix'] = 'example' options['out_directory'] = 'new/dir' options['remove_prefix'] = True - resp = service._download_object_job(mock_conn, - 'c', - 'example/test', - options) + resp = SwiftService()._download_object_job(mock_conn, + 'c', + 'example/test', + options) self.assertIsNone(resp.get('error')) self.assertIs(True, resp['success']) @@ -2153,18 +2148,17 @@ def test_object_post(self, res_iter, thread_manager): "test_spo", {'meta': ["meta1:test2"], "header": ["hdr1:test2"]}) - service = SwiftService() SwiftService().post('test_c', ['test_o', spo], self.opts) calls = [ mock.call( - service._post_object_job, 'test_c', 'test_o', + SwiftService._post_object_job, 'test_c', 'test_o', { "X-Object-Meta-Meta1": "test1", "Hdr1": "test1"}, {}), mock.call( - service._post_object_job, 'test_c', 'test_spo', + SwiftService._post_object_job, 'test_c', 'test_spo', { "X-Object-Meta-Meta1": "test2", "Hdr1": "test2"}, From 17aa6c789e3c28e59be3b92e6fa65edb89077436 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Wed, 2 Mar 2016 16:02:28 +0000 Subject: [PATCH 117/454] Port from optparse to argparse Why now? * argparse was introduced in Python 3.2 and back-ported to Python 2.7. Until we dropped Python 2.6 support, we were stuck on optparse. * keystoneauth.loading.cli provides register_argparse_arguments and load_from_argparse_arguments helper methods. Now that we're moving toward Keystone Session support, argparse seems required. Closes-Bug: 1553030 Change-Id: I5139fb64a8631a3010680090fd04345f95c55c7b --- swiftclient/shell.py | 563 +++++++++++++++++++++------------------ tests/unit/test_shell.py | 33 +-- 2 files changed, 317 insertions(+), 279 deletions(-) diff --git a/swiftclient/shell.py b/swiftclient/shell.py index aa95c113..4b08e720 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -16,11 +16,11 @@ from __future__ import print_function, unicode_literals +import argparse import logging import signal import socket -from optparse import OptionParser, OptionGroup, SUPPRESS_HELP from os import environ, walk, _exit as os_exit from os.path import isfile, isdir, join from six import text_type, PY2 @@ -80,22 +80,22 @@ def immediate_exit(signum, frame): def st_delete(parser, args, output_manager): - parser.add_option( + parser.add_argument( '-a', '--all', action='store_true', dest='yes_all', default=False, help='Delete all containers and objects.') - parser.add_option( + parser.add_argument( '-p', '--prefix', dest='prefix', help='Only delete items beginning with the .') - parser.add_option( - '', '--leave-segments', action='store_true', + parser.add_argument( + '--leave-segments', action='store_true', dest='leave_segments', default=False, help='Do not delete segments of manifest objects.') - parser.add_option( - '', '--object-threads', type=int, + parser.add_argument( + '--object-threads', type=int, default=10, help='Number of threads to use for deleting objects. ' 'Its value must be a positive integer. Default is 10.') - parser.add_option( - '', '--container-threads', type=int, + parser.add_argument( + '--container-threads', type=int, default=10, help='Number of threads to use for deleting containers. ' 'Its value must be a positive integer. Default is 10.') (options, args) = parse_args(parser, args) @@ -263,52 +263,52 @@ def st_delete(parser, args, output_manager): def st_download(parser, args, output_manager): - parser.add_option( + parser.add_argument( '-a', '--all', action='store_true', dest='yes_all', default=False, help='Indicates that you really want to download ' 'everything in the account.') - parser.add_option( + parser.add_argument( '-m', '--marker', dest='marker', default='', help='Marker to use when starting a container or ' 'account download.') - parser.add_option( + parser.add_argument( '-p', '--prefix', dest='prefix', help='Only download items beginning with the .') - parser.add_option( + parser.add_argument( '-o', '--output', dest='out_file', help='For a single ' 'download, stream the output to . ' 'Specifying "-" as will redirect to stdout.') - parser.add_option( + parser.add_argument( '-D', '--output-dir', dest='out_directory', help='An optional directory to which to store objects. ' 'By default, all objects are recreated in the current directory.') - parser.add_option( + parser.add_argument( '-r', '--remove-prefix', action='store_true', dest='remove_prefix', default=False, help='An optional flag for --prefix , ' 'use this option to download items without .') - parser.add_option( - '', '--object-threads', type=int, + parser.add_argument( + '--object-threads', type=int, default=10, help='Number of threads to use for downloading objects. ' 'Its value must be a positive integer. Default is 10.') - parser.add_option( - '', '--container-threads', type=int, default=10, + parser.add_argument( + '--container-threads', type=int, default=10, help='Number of threads to use for downloading containers. ' 'Its value must be a positive integer. Default is 10.') - parser.add_option( - '', '--no-download', action='store_true', + parser.add_argument( + '--no-download', action='store_true', default=False, help="Perform download(s), but don't actually write anything to disk.") - parser.add_option( + parser.add_argument( '-H', '--header', action='append', dest='header', default=[], help='Adds a customized request header to the query, like "Range" or ' '"If-Match". This option may be repeated. ' 'Example: --header "content-type:text/plain"') - parser.add_option( + parser.add_argument( '--skip-identical', action='store_true', dest='skip_identical', default=False, help='Skip downloading files that are identical on ' 'both sides.') - parser.add_option( + parser.add_argument( '--no-shuffle', action='store_false', dest='shuffle', default=True, help='By default, download order is randomised in order ' 'to reduce the load on individual drives when multiple clients are ' @@ -518,26 +518,26 @@ def _print_stats(options, stats): output_manager.print_msg( prt_bytes(total_bytes, options.human)) - parser.add_option( + parser.add_argument( '-l', '--long', dest='long', action='store_true', default=False, help='Long listing format, similar to ls -l.') - parser.add_option( + parser.add_argument( '--lh', dest='human', action='store_true', default=False, help='Report sizes in human readable format, ' "similar to ls -lh.") - parser.add_option( + parser.add_argument( '-t', '--totals', dest='totals', help='used with -l or --lh, only report totals.', action='store_true', default=False) - parser.add_option( + parser.add_argument( '-p', '--prefix', dest='prefix', help='Only list items beginning with the prefix.') - parser.add_option( + parser.add_argument( '-d', '--delimiter', dest='delimiter', help='Roll up items with the given delimiter. For containers ' 'only. See OpenStack Swift API documentation for ' 'what this means.') - (options, args) = parse_args(parser, args) + options, args = parse_args(parser, args) args = args[1:] if options.delimiter and not args: exit('-d option only allowed for container listings') @@ -595,10 +595,10 @@ def _print_stats(options, stats): def st_stat(parser, args, output_manager): - parser.add_option( + parser.add_argument( '--lh', dest='human', action='store_true', default=False, help='Report sizes in human readable format similar to ls -lh.') - (options, args) = parse_args(parser, args) + options, args = parse_args(parser, args) args = args[1:] _opts = vars(options) @@ -686,25 +686,25 @@ def st_stat(parser, args, output_manager): def st_post(parser, args, output_manager): - parser.add_option( + parser.add_argument( '-r', '--read-acl', dest='read_acl', help='Read ACL for containers. ' 'Quick summary of ACL syntax: .r:*, .r:-.example.com, ' '.r:www.example.com, account1, account2:user2') - parser.add_option( + parser.add_argument( '-w', '--write-acl', dest='write_acl', help='Write ACL for ' 'containers. Quick summary of ACL syntax: account1, ' 'account2:user2') - parser.add_option( + parser.add_argument( '-t', '--sync-to', dest='sync_to', help='Sets the ' 'Sync To for containers, for multi-cluster replication.') - parser.add_option( + parser.add_argument( '-k', '--sync-key', dest='sync_key', help='Sets the ' 'Sync Key for containers, for multi-cluster replication.') - parser.add_option( + parser.add_argument( '-m', '--meta', action='append', dest='meta', default=[], help='Sets a meta data item. This option may be repeated. ' 'Example: -m Color:Blue -m Size:Large') - parser.add_option( + parser.add_argument( '-H', '--header', action='append', dest='header', default=[], help='Adds a customized request header. ' 'This option may be repeated. ' @@ -805,58 +805,58 @@ def st_post(parser, args, output_manager): def st_upload(parser, args, output_manager): - parser.add_option( + parser.add_argument( '-c', '--changed', action='store_true', dest='changed', default=False, help='Only upload files that have changed since ' 'the last upload.') - parser.add_option( + parser.add_argument( '--skip-identical', action='store_true', dest='skip_identical', default=False, help='Skip uploading files that are identical on ' 'both sides.') - parser.add_option( + parser.add_argument( '-S', '--segment-size', dest='segment_size', help='Upload files ' 'in segments no larger than (in Bytes) and then create a ' '"manifest" file that will download all the segments as if it were ' 'the original file. Sizes may also be expressed as bytes with the ' 'B suffix, kilobytes with the K suffix, megabytes with the M suffix ' 'or gigabytes with the G suffix.') - parser.add_option( + parser.add_argument( '-C', '--segment-container', dest='segment_container', help='Upload the segments into the specified container. ' 'If not specified, the segments will be uploaded to a ' '_segments container to not pollute the main ' ' listings.') - parser.add_option( - '', '--leave-segments', action='store_true', + parser.add_argument( + '--leave-segments', action='store_true', dest='leave_segments', default=False, help='Indicates that you want ' 'the older segments of manifest objects left alone (in the case of ' 'overwrites).') - parser.add_option( - '', '--object-threads', type=int, default=10, + parser.add_argument( + '--object-threads', type=int, default=10, help='Number of threads to use for uploading full objects. ' 'Its value must be a positive integer. Default is 10.') - parser.add_option( - '', '--segment-threads', type=int, default=10, + parser.add_argument( + '--segment-threads', type=int, default=10, help='Number of threads to use for uploading object segments. ' 'Its value must be a positive integer. Default is 10.') - parser.add_option( + parser.add_argument( '-H', '--header', action='append', dest='header', default=[], help='Set request headers with the syntax header:value. ' ' This option may be repeated. Example -H "content-type:text/plain" ' '-H "Content-Length: 4000"') - parser.add_option( - '', '--use-slo', action='store_true', default=False, + parser.add_argument( + '--use-slo', action='store_true', default=False, help='When used in conjunction with --segment-size, it will ' 'create a Static Large Object instead of the default ' 'Dynamic Large Object.') - parser.add_option( - '', '--object-name', dest='object_name', + parser.add_argument( + '--object-name', dest='object_name', help='Upload file and name object to or upload dir and ' 'use as object prefix instead of folder name.') - parser.add_option( - '', '--ignore-checksum', dest='checksum', default=True, + parser.add_argument( + '--ignore-checksum', dest='checksum', default=True, action='store_false', help='Turn off checksum validation for uploads.') - (options, args) = parse_args(parser, args) + options, args = parse_args(parser, args) args = args[1:] if len(args) < 2: output_manager.error( @@ -1115,7 +1115,7 @@ def st_auth(parser, args, thread_manager): def st_tempurl(parser, args, thread_manager): - parser.add_option( + parser.add_argument( '--absolute', action='store_true', dest='absolute_expiry', default=False, help=("If present, seconds argument will be interpreted as a Unix " @@ -1143,10 +1143,41 @@ def st_tempurl(parser, args, thread_manager): thread_manager.print_msg(url) +class HelpFormatter(argparse.HelpFormatter): + def _format_action_invocation(self, action): + if not action.option_strings: + default = self._get_default_metavar_for_positional(action) + metavar, = self._metavar_formatter(action, default)(1) + return metavar + + else: + parts = [] + + # if the Optional doesn't take a value, format is: + # -s, --long + if action.nargs == 0: + parts.extend(action.option_strings) + + # if the Optional takes a value, format is: + # -s=ARGS, --long=ARGS + else: + default = self._get_default_metavar_for_optional(action) + args_string = self._format_args(action, default) + for option_string in action.option_strings: + parts.append('%s=%s' % (option_string, args_string)) + + return ', '.join(parts) + + # Back-port py3 methods + def _get_default_metavar_for_optional(self, action): + return action.dest.upper() + + def _get_default_metavar_for_positional(self, action): + return action.dest + + def parse_args(parser, args, enforce_requires=True): - if not args: - args = ['-h'] - (options, args) = parser.parse_args(args) + options, args = parser.parse_known_args(args or ['-h']) if enforce_requires and (options.debug or options.info): logging.getLogger("swiftclient") if options.debug: @@ -1156,14 +1187,14 @@ def parse_args(parser, args, enforce_requires=True): elif options.info: logging.basicConfig(level=logging.INFO) - if len(args) > 1 and args[1] == '--help': + if args and options.help: _help = globals().get('st_%s_help' % args[0], "no help for %s" % args[0]) print(_help) exit() # Short circuit for tempurl, which doesn't need auth - if len(args) > 0 and args[0] == 'tempurl': + if args and args[0] == 'tempurl': return options, args if options.auth_version == '3.0': @@ -1206,7 +1237,7 @@ def parse_args(parser, args, enforce_requires=True): if (options.os_options.get('object_storage_url') and options.os_options.get('auth_token') and - (options.auth_version == '2.0' or options.auth_version == '3')): + options.auth_version in ('2.0', '3')): return options, args if enforce_requires: @@ -1234,17 +1265,14 @@ def parse_args(parser, args, enforce_requires=True): def main(arguments=None): - if arguments: - argv = arguments - else: - argv = sys_argv + argv = sys_argv if arguments is None else arguments argv = [a if isinstance(a, text_type) else a.decode('utf-8') for a in argv] version = client_version - parser = OptionParser(version='python-swiftclient %s' % version, - usage=''' -usage: %prog [--version] [--help] [--os-help] [--snet] [--verbose] + parser = argparse.ArgumentParser( + add_help=False, formatter_class=HelpFormatter, usage=''' +%(prog)s [--version] [--help] [--os-help] [--snet] [--verbose] [--debug] [--info] [--quiet] [--auth ] [--auth-version | --os-identity-api-version ] @@ -1286,29 +1314,34 @@ def main(arguments=None): auth Display auth related environment variables. Examples: - %prog download --help + %(prog)s download --help - %prog -A https://auth.api.rackspacecloud.com/v1.0 -U user -K api_key stat -v + %(prog)s -A https://auth.api.rackspacecloud.com/v1.0 \\ + -U user -K api_key stat -v - %prog --os-auth-url https://api.example.com/v2.0 --os-tenant-name tenant \\ + %(prog)s --os-auth-url https://api.example.com/v2.0 \\ + --os-tenant-name tenant \\ --os-username user --os-password password list - %prog --os-auth-url https://api.example.com/v3 --auth-version 3\\ + %(prog)s --os-auth-url https://api.example.com/v3 --auth-version 3\\ --os-project-name project1 --os-project-domain-name domain1 \\ --os-username user --os-user-domain-name domain1 \\ --os-password password list - %prog --os-auth-url https://api.example.com/v3 --auth-version 3\\ + %(prog)s --os-auth-url https://api.example.com/v3 --auth-version 3\\ --os-project-id 0123456789abcdef0123456789abcdef \\ --os-user-id abcdef0123456789abcdef0123456789 \\ --os-password password list - %prog --os-auth-token 6ee5eb33efad4e45ab46806eac010566 \\ + %(prog)s --os-auth-token 6ee5eb33efad4e45ab46806eac010566 \\ --os-storage-url https://10.1.5.2:8080/v1/AUTH_ced809b6a4baea7aeab61a \\ list - %prog list --lh + %(prog)s list --lh '''.strip('\n')) + parser.add_argument('--version', action='version', + version='python-swiftclient %s' % version) + parser.add_argument('-h', '--help', action='store_true') default_auth_version = '1.0' for k in ('ST_AUTH_VERSION', 'OS_AUTH_VERSION', 'OS_IDENTITY_API_VERSION'): @@ -1318,193 +1351,196 @@ def main(arguments=None): except KeyError: pass - parser.add_option('--os-help', action='store_true', dest='os_help', - help='Show OpenStack authentication options.') - parser.add_option('--os_help', action='store_true', help=SUPPRESS_HELP) - parser.add_option('-s', '--snet', action='store_true', dest='snet', - default=False, help='Use SERVICENET internal network.') - parser.add_option('-v', '--verbose', action='count', dest='verbose', - default=1, help='Print more info.') - parser.add_option('--debug', action='store_true', dest='debug', - default=False, help='Show the curl commands and results ' - 'of all http queries regardless of result status.') - parser.add_option('--info', action='store_true', dest='info', - default=False, help='Show the curl commands and results ' - 'of all http queries which return an error.') - parser.add_option('-q', '--quiet', action='store_const', dest='verbose', - const=0, default=1, help='Suppress status output.') - parser.add_option('-A', '--auth', dest='auth', - default=environ.get('ST_AUTH'), - help='URL for obtaining an auth token.') - parser.add_option('-V', '--auth-version', '--os-identity-api-version', - dest='auth_version', - default=default_auth_version, - type=str, - help='Specify a version for authentication. ' - 'Defaults to env[ST_AUTH_VERSION], ' - 'env[OS_AUTH_VERSION], env[OS_IDENTITY_API_VERSION]' - ' or 1.0.') - parser.add_option('-U', '--user', dest='user', - default=environ.get('ST_USER'), - help='User name for obtaining an auth token.') - parser.add_option('-K', '--key', dest='key', - default=environ.get('ST_KEY'), - help='Key for obtaining an auth token.') - parser.add_option('-R', '--retries', type=int, default=5, dest='retries', - help='The number of times to retry a failed connection.') + parser.add_argument('--os-help', action='store_true', dest='os_help', + help='Show OpenStack authentication options.') + parser.add_argument('--os_help', action='store_true', + help=argparse.SUPPRESS) + parser.add_argument('-s', '--snet', action='store_true', dest='snet', + default=False, help='Use SERVICENET internal network.') + parser.add_argument('-v', '--verbose', action='count', dest='verbose', + default=1, help='Print more info.') + parser.add_argument('--debug', action='store_true', dest='debug', + default=False, help='Show the curl commands and ' + 'results of all http queries regardless of result ' + 'status.') + parser.add_argument('--info', action='store_true', dest='info', + default=False, help='Show the curl commands and ' + 'results of all http queries which return an error.') + parser.add_argument('-q', '--quiet', action='store_const', dest='verbose', + const=0, default=1, help='Suppress status output.') + parser.add_argument('-A', '--auth', dest='auth', + default=environ.get('ST_AUTH'), + help='URL for obtaining an auth token.') + parser.add_argument('-V', '--auth-version', '--os-identity-api-version', + dest='auth_version', + default=default_auth_version, + type=str, + help='Specify a version for authentication. ' + 'Defaults to env[ST_AUTH_VERSION], ' + 'env[OS_AUTH_VERSION], ' + 'env[OS_IDENTITY_API_VERSION] or 1.0.') + parser.add_argument('-U', '--user', dest='user', + default=environ.get('ST_USER'), + help='User name for obtaining an auth token.') + parser.add_argument('-K', '--key', dest='key', + default=environ.get('ST_KEY'), + help='Key for obtaining an auth token.') + parser.add_argument('-R', '--retries', type=int, default=5, dest='retries', + help='The number of times to retry a failed ' + 'connection.') default_val = config_true_value(environ.get('SWIFTCLIENT_INSECURE')) - parser.add_option('--insecure', - action="store_true", dest="insecure", - default=default_val, - help='Allow swiftclient to access servers without ' - 'having to verify the SSL certificate. ' - 'Defaults to env[SWIFTCLIENT_INSECURE] ' - '(set to \'true\' to enable).') - parser.add_option('--no-ssl-compression', - action='store_false', dest='ssl_compression', - default=True, - help='This option is deprecated and not used anymore. ' - 'SSL compression should be disabled by default ' - 'by the system SSL library.') - - os_grp = OptionGroup(parser, "OpenStack authentication options") - os_grp.add_option('--os-username', - metavar='', - default=environ.get('OS_USERNAME'), - help='OpenStack username. Defaults to env[OS_USERNAME].') - os_grp.add_option('--os_username', - help=SUPPRESS_HELP) - os_grp.add_option('--os-user-id', - metavar='', - default=environ.get('OS_USER_ID'), - help='OpenStack user ID. ' - 'Defaults to env[OS_USER_ID].') - os_grp.add_option('--os_user_id', - help=SUPPRESS_HELP) - os_grp.add_option('--os-user-domain-id', - metavar='', - default=environ.get('OS_USER_DOMAIN_ID'), - help='OpenStack user domain ID. ' - 'Defaults to env[OS_USER_DOMAIN_ID].') - os_grp.add_option('--os_user_domain_id', - help=SUPPRESS_HELP) - os_grp.add_option('--os-user-domain-name', - metavar='', - default=environ.get('OS_USER_DOMAIN_NAME'), - help='OpenStack user domain name. ' - 'Defaults to env[OS_USER_DOMAIN_NAME].') - os_grp.add_option('--os_user_domain_name', - help=SUPPRESS_HELP) - os_grp.add_option('--os-password', - metavar='', - default=environ.get('OS_PASSWORD'), - help='OpenStack password. Defaults to env[OS_PASSWORD].') - os_grp.add_option('--os_password', - help=SUPPRESS_HELP) - os_grp.add_option('--os-tenant-id', - metavar='', - default=environ.get('OS_TENANT_ID'), - help='OpenStack tenant ID. ' - 'Defaults to env[OS_TENANT_ID].') - os_grp.add_option('--os_tenant_id', - help=SUPPRESS_HELP) - os_grp.add_option('--os-tenant-name', - metavar='', - default=environ.get('OS_TENANT_NAME'), - help='OpenStack tenant name. ' - 'Defaults to env[OS_TENANT_NAME].') - os_grp.add_option('--os_tenant_name', - help=SUPPRESS_HELP) - os_grp.add_option('--os-project-id', - metavar='', - default=environ.get('OS_PROJECT_ID'), - help='OpenStack project ID. ' - 'Defaults to env[OS_PROJECT_ID].') - os_grp.add_option('--os_project_id', - help=SUPPRESS_HELP) - os_grp.add_option('--os-project-name', - metavar='', - default=environ.get('OS_PROJECT_NAME'), - help='OpenStack project name. ' - 'Defaults to env[OS_PROJECT_NAME].') - os_grp.add_option('--os_project_name', - help=SUPPRESS_HELP) - os_grp.add_option('--os-project-domain-id', - metavar='', - default=environ.get('OS_PROJECT_DOMAIN_ID'), - help='OpenStack project domain ID. ' - 'Defaults to env[OS_PROJECT_DOMAIN_ID].') - os_grp.add_option('--os_project_domain_id', - help=SUPPRESS_HELP) - os_grp.add_option('--os-project-domain-name', - metavar='', - default=environ.get('OS_PROJECT_DOMAIN_NAME'), - help='OpenStack project domain name. ' - 'Defaults to env[OS_PROJECT_DOMAIN_NAME].') - os_grp.add_option('--os_project_domain_name', - help=SUPPRESS_HELP) - os_grp.add_option('--os-auth-url', - metavar='', - default=environ.get('OS_AUTH_URL'), - help='OpenStack auth URL. Defaults to env[OS_AUTH_URL].') - os_grp.add_option('--os_auth_url', - help=SUPPRESS_HELP) - os_grp.add_option('--os-auth-token', - metavar='', - default=environ.get('OS_AUTH_TOKEN'), - help='OpenStack token. Defaults to env[OS_AUTH_TOKEN]. ' - 'Used with --os-storage-url to bypass the ' - 'usual username/password authentication.') - os_grp.add_option('--os_auth_token', - help=SUPPRESS_HELP) - os_grp.add_option('--os-storage-url', - metavar='', - default=environ.get('OS_STORAGE_URL'), - help='OpenStack storage URL. ' - 'Defaults to env[OS_STORAGE_URL]. ' - 'Overrides the storage url returned during auth. ' - 'Will bypass authentication when used with ' - '--os-auth-token.') - os_grp.add_option('--os_storage_url', - help=SUPPRESS_HELP) - os_grp.add_option('--os-region-name', - metavar='', - default=environ.get('OS_REGION_NAME'), - help='OpenStack region name. ' - 'Defaults to env[OS_REGION_NAME].') - os_grp.add_option('--os_region_name', - help=SUPPRESS_HELP) - os_grp.add_option('--os-service-type', - metavar='', - default=environ.get('OS_SERVICE_TYPE'), - help='OpenStack Service type. ' - 'Defaults to env[OS_SERVICE_TYPE].') - os_grp.add_option('--os_service_type', - help=SUPPRESS_HELP) - os_grp.add_option('--os-endpoint-type', - metavar='', - default=environ.get('OS_ENDPOINT_TYPE'), - help='OpenStack Endpoint type. ' - 'Defaults to env[OS_ENDPOINT_TYPE].') - os_grp.add_option('--os_endpoint_type', - help=SUPPRESS_HELP) - os_grp.add_option('--os-cacert', - metavar='', - default=environ.get('OS_CACERT'), - help='Specify a CA bundle file to use in verifying a ' - 'TLS (https) server certificate. ' - 'Defaults to env[OS_CACERT].') - parser.disable_interspersed_args() - # call parse_args before adding os options group so that -h, --help will - # print a condensed help message without the os options - (options, args) = parse_args(parser, argv[1:], enforce_requires=False) - parser.add_option_group(os_grp) - if options.os_help: - # if openstack option help has been explicitly requested then force - # help message, now that os_options group has been added to parser - argv = ['-h'] - (options, args) = parse_args(parser, argv[1:], enforce_requires=False) - parser.enable_interspersed_args() + parser.add_argument('--insecure', + action="store_true", dest="insecure", + default=default_val, + help='Allow swiftclient to access servers without ' + 'having to verify the SSL certificate. ' + 'Defaults to env[SWIFTCLIENT_INSECURE] ' + '(set to \'true\' to enable).') + parser.add_argument('--no-ssl-compression', + action='store_false', dest='ssl_compression', + default=True, + help='This option is deprecated and not used anymore. ' + 'SSL compression should be disabled by default ' + 'by the system SSL library.') + + os_grp = parser.add_argument_group("OpenStack authentication options") + os_grp.add_argument('--os-username', + metavar='', + default=environ.get('OS_USERNAME'), + help='OpenStack username. Defaults to ' + 'env[OS_USERNAME].') + os_grp.add_argument('--os_username', + help=argparse.SUPPRESS) + os_grp.add_argument('--os-user-id', + metavar='', + default=environ.get('OS_USER_ID'), + help='OpenStack user ID. ' + 'Defaults to env[OS_USER_ID].') + os_grp.add_argument('--os_user_id', + help=argparse.SUPPRESS) + os_grp.add_argument('--os-user-domain-id', + metavar='', + default=environ.get('OS_USER_DOMAIN_ID'), + help='OpenStack user domain ID. ' + 'Defaults to env[OS_USER_DOMAIN_ID].') + os_grp.add_argument('--os_user_domain_id', + help=argparse.SUPPRESS) + os_grp.add_argument('--os-user-domain-name', + metavar='', + default=environ.get('OS_USER_DOMAIN_NAME'), + help='OpenStack user domain name. ' + 'Defaults to env[OS_USER_DOMAIN_NAME].') + os_grp.add_argument('--os_user_domain_name', + help=argparse.SUPPRESS) + os_grp.add_argument('--os-password', + metavar='', + default=environ.get('OS_PASSWORD'), + help='OpenStack password. Defaults to ' + 'env[OS_PASSWORD].') + os_grp.add_argument('--os_password', + help=argparse.SUPPRESS) + os_grp.add_argument('--os-tenant-id', + metavar='', + default=environ.get('OS_TENANT_ID'), + help='OpenStack tenant ID. ' + 'Defaults to env[OS_TENANT_ID].') + os_grp.add_argument('--os_tenant_id', + help=argparse.SUPPRESS) + os_grp.add_argument('--os-tenant-name', + metavar='', + default=environ.get('OS_TENANT_NAME'), + help='OpenStack tenant name. ' + 'Defaults to env[OS_TENANT_NAME].') + os_grp.add_argument('--os_tenant_name', + help=argparse.SUPPRESS) + os_grp.add_argument('--os-project-id', + metavar='', + default=environ.get('OS_PROJECT_ID'), + help='OpenStack project ID. ' + 'Defaults to env[OS_PROJECT_ID].') + os_grp.add_argument('--os_project_id', + help=argparse.SUPPRESS) + os_grp.add_argument('--os-project-name', + metavar='', + default=environ.get('OS_PROJECT_NAME'), + help='OpenStack project name. ' + 'Defaults to env[OS_PROJECT_NAME].') + os_grp.add_argument('--os_project_name', + help=argparse.SUPPRESS) + os_grp.add_argument('--os-project-domain-id', + metavar='', + default=environ.get('OS_PROJECT_DOMAIN_ID'), + help='OpenStack project domain ID. ' + 'Defaults to env[OS_PROJECT_DOMAIN_ID].') + os_grp.add_argument('--os_project_domain_id', + help=argparse.SUPPRESS) + os_grp.add_argument('--os-project-domain-name', + metavar='', + default=environ.get('OS_PROJECT_DOMAIN_NAME'), + help='OpenStack project domain name. ' + 'Defaults to env[OS_PROJECT_DOMAIN_NAME].') + os_grp.add_argument('--os_project_domain_name', + help=argparse.SUPPRESS) + os_grp.add_argument('--os-auth-url', + metavar='', + default=environ.get('OS_AUTH_URL'), + help='OpenStack auth URL. Defaults to ' + 'env[OS_AUTH_URL].') + os_grp.add_argument('--os_auth_url', + help=argparse.SUPPRESS) + os_grp.add_argument('--os-auth-token', + metavar='', + default=environ.get('OS_AUTH_TOKEN'), + help='OpenStack token. Defaults to ' + 'env[OS_AUTH_TOKEN]. Used with --os-storage-url ' + 'to bypass the usual username/password ' + 'authentication.') + os_grp.add_argument('--os_auth_token', + help=argparse.SUPPRESS) + os_grp.add_argument('--os-storage-url', + metavar='', + default=environ.get('OS_STORAGE_URL'), + help='OpenStack storage URL. ' + 'Defaults to env[OS_STORAGE_URL]. ' + 'Overrides the storage url returned during auth. ' + 'Will bypass authentication when used with ' + '--os-auth-token.') + os_grp.add_argument('--os_storage_url', + help=argparse.SUPPRESS) + os_grp.add_argument('--os-region-name', + metavar='', + default=environ.get('OS_REGION_NAME'), + help='OpenStack region name. ' + 'Defaults to env[OS_REGION_NAME].') + os_grp.add_argument('--os_region_name', + help=argparse.SUPPRESS) + os_grp.add_argument('--os-service-type', + metavar='', + default=environ.get('OS_SERVICE_TYPE'), + help='OpenStack Service type. ' + 'Defaults to env[OS_SERVICE_TYPE].') + os_grp.add_argument('--os_service_type', + help=argparse.SUPPRESS) + os_grp.add_argument('--os-endpoint-type', + metavar='', + default=environ.get('OS_ENDPOINT_TYPE'), + help='OpenStack Endpoint type. ' + 'Defaults to env[OS_ENDPOINT_TYPE].') + os_grp.add_argument('--os_endpoint_type', + help=argparse.SUPPRESS) + os_grp.add_argument('--os-cacert', + metavar='', + default=environ.get('OS_CACERT'), + help='Specify a CA bundle file to use in verifying a ' + 'TLS (https) server certificate. ' + 'Defaults to env[OS_CACERT].') + options, args = parse_args(parser, argv[1:], enforce_requires=False) + + if options.help or options.os_help: + if options.help: + parser._action_groups.pop() + parser.print_help() + exit() if not args or args[0] not in commands: parser.print_usage() @@ -1515,7 +1551,6 @@ def main(arguments=None): signal.signal(signal.SIGINT, immediate_exit) with OutputManager() as output: - parser.usage = globals()['st_%s_help' % args[0]] try: globals()['st_%s' % args[0]](parser, argv[1:], output) diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index 59ed17d6..58c5a7c3 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -62,16 +62,19 @@ def _make_args(cmd, opts, os_opts, separator='-', flags=None, cmd_args=None): args = [""] flags = flags or [] for k, v in opts.items(): - arg = "--" + k.replace("_", "-") - args = args + [arg, v] + args.append("--" + k.replace("_", "-")) + if v is not None: + args.append(v) for k, v in os_opts.items(): - arg = "--os" + separator + k.replace("_", separator) - args = args + [arg, v] + args.append("--os" + separator + k.replace("_", separator)) + if v is not None: + args.append(v) for flag in flags: args.append('--%s' % flag) - args = args + [cmd] + if cmd: + args.append(cmd) if cmd_args: - args = args + cmd_args + args.extend(cmd_args) return args @@ -1261,17 +1264,17 @@ def test_negative_upload_segment_size(self): self.assertEqual(output.err, "segment-size should be positive\n") output.clear() with self.assertRaises(SystemExit): - argv = ["", "upload", "-S", "-40K", "container", "object"] + argv = ["", "upload", "-S=-40K", "container", "object"] swiftclient.shell.main(argv) self.assertEqual(output.err, "segment-size should be positive\n") output.clear() with self.assertRaises(SystemExit): - argv = ["", "upload", "-S", "-40M", "container", "object"] + argv = ["", "upload", "-S=-40M", "container", "object"] swiftclient.shell.main(argv) self.assertEqual(output.err, "segment-size should be positive\n") output.clear() with self.assertRaises(SystemExit): - argv = ["", "upload", "-S", "-40G", "container", "object"] + argv = ["", "upload", "-S=-40G", "container", "object"] swiftclient.shell.main(argv) self.assertEqual(output.err, "segment-size should be positive\n") output.clear() @@ -1692,17 +1695,17 @@ def test_insufficient_env_vars_v3(self): def test_help(self): # --help returns condensed help message - opts = {"help": ""} + opts = {"help": None} os_opts = {} - args = _make_args("stat", opts, os_opts) + args = _make_args(None, opts, os_opts) with CaptureOutput() as out: self.assertRaises(SystemExit, swiftclient.shell.main, args) self.assertTrue(out.find('[--key ]') > 0) self.assertEqual(-1, out.find('--os-username=')) # --help returns condensed help message, overrides --os-help - opts = {"help": ""} - os_opts = {"help": ""} + opts = {"help": None} + os_opts = {"help": None} args = _make_args("", opts, os_opts) with CaptureOutput() as out: self.assertRaises(SystemExit, swiftclient.shell.main, args) @@ -1711,8 +1714,8 @@ def test_help(self): # --os-password, --os-username and --os-auth_url should be ignored # because --help overrides it - opts = {"help": ""} - os_opts = {"help": "", + opts = {"help": None} + os_opts = {"help": None, "password": "secret", "username": "user", "auth_url": "http://example.com:5000/v3"} From 4a6fa02c2832086d3ea5f7fff694c844da980928 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Fri, 8 Apr 2016 13:29:37 -0700 Subject: [PATCH 118/454] Identify segments uploaded via swiftclient ...using a new "application/swiftclient-segment" content-type. Segments uploaded by swiftclient are expected to have a many-to-one relationship to large objects, rather than the more-general many-to-many relationship that SLO and DLO generally allow. Later, we may use this information to make more intelligent decisions, such as when to automatically clean up segments. Change-Id: Ie56a3aa10065db754ac572cc37d93f2c901aac60 --- swiftclient/service.py | 11 +++++++---- tests/unit/test_service.py | 33 ++++++++++++++++++--------------- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/swiftclient/service.py b/swiftclient/service.py index 99c833e0..67ea2e3a 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -1657,10 +1657,13 @@ def _upload_segment_job(conn, path, container, segment_name, segment_start, fp.seek(segment_start) contents = LengthWrapper(fp, segment_size, md5=options['checksum']) - etag = conn.put_object(segment_container, - segment_name, contents, - content_length=segment_size, - response_dict=results_dict) + etag = conn.put_object( + segment_container, + segment_name, + contents, + content_length=segment_size, + content_type='application/swiftclient-segment', + response_dict=results_dict) if options['checksum'] and etag and etag != contents.get_md5sum(): raise SwiftError('Segment {0}: upload verification failed: ' diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index e9310aa7..d8de581a 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -945,11 +945,12 @@ def test_upload_segment_job(self): self.assertEqual(r, expected_r) self.assertEqual(mock_conn.put_object.call_count, 1) - mock_conn.put_object.assert_called_with('test_c_segments', - 'test_s_1', - mock.ANY, - content_length=10, - response_dict={}) + mock_conn.put_object.assert_called_with( + 'test_c_segments', 'test_s_1', + mock.ANY, + content_length=10, + content_type='application/swiftclient-segment', + response_dict={}) contents = mock_conn.put_object.call_args[0][2] self.assertIsInstance(contents, utils.LengthWrapper) self.assertEqual(len(contents), 10) @@ -988,11 +989,12 @@ def _consuming_conn(*a, **kw): self.assertIsNone(r.get('error')) self.assertEqual(mock_conn.put_object.call_count, 1) - mock_conn.put_object.assert_called_with('test_c_segments', - 'test_s_1', - mock.ANY, - content_length=10, - response_dict={}) + mock_conn.put_object.assert_called_with( + 'test_c_segments', 'test_s_1', + mock.ANY, + content_length=10, + content_type='application/swiftclient-segment', + response_dict={}) contents = mock_conn.put_object.call_args[0][2] # Check that md5sum is not calculated. self.assertEqual(contents.get_md5sum(), '') @@ -1028,11 +1030,12 @@ def _consuming_conn(*a, **kw): self.assertIn('md5 mismatch', str(r.get('error'))) self.assertEqual(mock_conn.put_object.call_count, 1) - mock_conn.put_object.assert_called_with('test_c_segments', - 'test_s_1', - mock.ANY, - content_length=10, - response_dict={}) + mock_conn.put_object.assert_called_with( + 'test_c_segments', 'test_s_1', + mock.ANY, + content_length=10, + content_type='application/swiftclient-segment', + response_dict={}) contents = mock_conn.put_object.call_args[0][2] self.assertEqual(contents.get_md5sum(), md5(b'b' * 10).hexdigest()) From 9fd537a08245cf6a34c1abdf8b7c42bfd3669c74 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Tue, 8 Dec 2015 10:45:07 -0800 Subject: [PATCH 119/454] Use application/directory content-type for dir markers Previously, we were using a content-type of text/directory, but that is already defined in RFC 2425 and doesn't reflect our usage: The text/directory Content-Type is defined for holding a variety of directory information, for example, name, or email address, or logo. (From there it goes on to describe a superset of the vCard format defined in RFC 2426.) application/directory, on the other hand, is used by Static Web [1] and is used by cloudfuse [2]. Seems like as sane a choice as any to standardize on. [1] https://github.com/openstack/swift/blob/2.5.0/swift/common/middleware/staticweb.py#L71-L75 [2] https://github.com/redbo/cloudfuse/blob/1.0/README#L105-L106 Change-Id: I19e30484270886292d83f50e7ee997b6e1623ec7 --- swiftclient/service.py | 15 ++-- tests/unit/test_service.py | 136 +++++++++++++++++++++++++++++++++++++ 2 files changed, 145 insertions(+), 6 deletions(-) diff --git a/swiftclient/service.py b/swiftclient/service.py index 99c833e0..232815ff 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -189,6 +189,10 @@ def _build_default_global_options(): } POLICY = 'X-Storage-Policy' +KNOWN_DIR_MARKERS = ( + 'application/directory', # Preferred + 'text/directory', # Historically relevant +) def get_from_queue(q, timeout=864000): @@ -1130,9 +1134,8 @@ def _download_object_job(self, conn, container, obj, options): fp = None try: - content_type = headers.get('content-type') - if (content_type and - content_type.split(';', 1)[0] == 'text/directory'): + content_type = headers.get('content-type', '').split(';', 1)[0] + if content_type in KNOWN_DIR_MARKERS: make_dir = not no_file and out_file != "-" if make_dir and not isdir(path): mkdirs(path) @@ -1590,12 +1593,12 @@ def _create_dir_marker_job(conn, container, obj, options, path=None): if options['changed']: try: headers = conn.head_object(container, obj) - ct = headers.get('content-type') + ct = headers.get('content-type', '').split(';', 1)[0] cl = int(headers.get('content-length')) et = headers.get('etag') mt = headers.get('x-object-meta-mtime') - if (ct.split(';', 1)[0] == 'text/directory' and + if (ct in KNOWN_DIR_MARKERS and cl == 0 and et == EMPTY_ETAG and mt == put_headers['x-object-meta-mtime']): @@ -1614,7 +1617,7 @@ def _create_dir_marker_job(conn, container, obj, options, path=None): return res try: conn.put_object(container, obj, '', content_length=0, - content_type='text/directory', + content_type=KNOWN_DIR_MARKERS[0], headers=put_headers, response_dict=results_dict) res.update({ diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index e9310aa7..8651b057 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -1348,6 +1348,142 @@ def test_make_upload_objects(self): errors.append(msg) self.assertFalse(errors, "\nERRORS:\n%s" % '\n'.join(errors)) + def test_create_dir_marker_job_unchanged(self): + mock_conn = mock.Mock() + mock_conn.head_object.return_value = { + 'content-type': 'application/directory', + 'content-length': '0', + 'x-object-meta-mtime': '1.234000', + 'etag': md5().hexdigest()} + + s = SwiftService() + with mock.patch('swiftclient.service.get_conn', + return_value=mock_conn): + with mock.patch('swiftclient.service.getmtime', + return_value=1.234): + r = s._create_dir_marker_job(conn=mock_conn, + container='test_c', + obj='test_o', + path='test', + options={'changed': True, + 'skip_identical': True, + 'leave_segments': True, + 'header': '', + 'segment_size': 10}) + self.assertEqual({ + 'action': 'create_dir_marker', + 'container': 'test_c', + 'object': 'test_o', + 'path': 'test', + 'headers': {'x-object-meta-mtime': '1.234000'}, + # NO response dict! + 'success': True, + }, r) + self.assertEqual([], mock_conn.put_object.mock_calls) + + def test_create_dir_marker_job_unchanged_old_type(self): + mock_conn = mock.Mock() + mock_conn.head_object.return_value = { + 'content-type': 'text/directory', + 'content-length': '0', + 'x-object-meta-mtime': '1.000000', + 'etag': md5().hexdigest()} + + s = SwiftService() + with mock.patch('swiftclient.service.get_conn', + return_value=mock_conn): + with mock.patch('swiftclient.service.time', + return_value=1.234): + r = s._create_dir_marker_job(conn=mock_conn, + container='test_c', + obj='test_o', + options={'changed': True, + 'skip_identical': True, + 'leave_segments': True, + 'header': '', + 'segment_size': 10}) + self.assertEqual({ + 'action': 'create_dir_marker', + 'container': 'test_c', + 'object': 'test_o', + 'path': None, + 'headers': {'x-object-meta-mtime': '1.000000'}, + # NO response dict! + 'success': True, + }, r) + self.assertEqual([], mock_conn.put_object.mock_calls) + + def test_create_dir_marker_job_overwrites_bad_type(self): + mock_conn = mock.Mock() + mock_conn.head_object.return_value = { + 'content-type': 'text/plain', + 'content-length': '0', + 'x-object-meta-mtime': '1.000000', + 'etag': md5().hexdigest()} + + s = SwiftService() + with mock.patch('swiftclient.service.get_conn', + return_value=mock_conn): + with mock.patch('swiftclient.service.time', + return_value=1.234): + r = s._create_dir_marker_job(conn=mock_conn, + container='test_c', + obj='test_o', + options={'changed': True, + 'skip_identical': True, + 'leave_segments': True, + 'header': '', + 'segment_size': 10}) + self.assertEqual({ + 'action': 'create_dir_marker', + 'container': 'test_c', + 'object': 'test_o', + 'path': None, + 'headers': {'x-object-meta-mtime': '1.000000'}, + 'response_dict': {}, + 'success': True, + }, r) + self.assertEqual([mock.call( + 'test_c', 'test_o', '', + content_length=0, + content_type='application/directory', + headers={'x-object-meta-mtime': '1.000000'}, + response_dict={})], mock_conn.put_object.mock_calls) + + def test_create_dir_marker_job_missing(self): + mock_conn = mock.Mock() + mock_conn.head_object.side_effect = \ + ClientException('Not Found', http_status=404) + + s = SwiftService() + with mock.patch('swiftclient.service.get_conn', + return_value=mock_conn): + with mock.patch('swiftclient.service.time', + return_value=1.234): + r = s._create_dir_marker_job(conn=mock_conn, + container='test_c', + obj='test_o', + options={'changed': True, + 'skip_identical': True, + 'leave_segments': True, + 'header': '', + 'segment_size': 10}) + self.assertEqual({ + 'action': 'create_dir_marker', + 'container': 'test_c', + 'object': 'test_o', + 'path': None, + 'headers': {'x-object-meta-mtime': '1.000000'}, + 'response_dict': {}, + 'success': True, + }, r) + self.assertEqual([mock.call( + 'test_c', 'test_o', '', + content_length=0, + content_type='application/directory', + headers={'x-object-meta-mtime': '1.000000'}, + response_dict={})], mock_conn.put_object.mock_calls) + class TestServiceDownload(_TestServiceBase): From 909bdf89542a9d021eb2ddcb951950f36245bc8b Mon Sep 17 00:00:00 2001 From: Sergey Gotliv Date: Sun, 3 Apr 2016 07:37:33 +0300 Subject: [PATCH 120/454] Fix downloading from "marker" item The documentation of "swift download" hints that "marker" option is supported, but in reality we forgot to patch it through, so all downloads were always done with the default, empty marker. Closes-Bug: #1565393 Change-Id: I38bd29d2baa9188b61397dec75ce1d864041653c --- swiftclient/service.py | 2 +- swiftclient/shell.py | 4 ++-- tests/unit/test_service.py | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/swiftclient/service.py b/swiftclient/service.py index 99c833e0..7d6f6bef 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -883,7 +883,7 @@ def _list_account_job(conn, options, result_queue): @staticmethod def _list_container_job(conn, container, options, result_queue): - marker = '' + marker = options.get('marker', '') error = None try: while True: diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 2c5bca82..c1b893d0 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -203,7 +203,7 @@ def st_delete(parser, args, output_manager): output_manager.error(err.value) -st_download_options = '''[--all] [--marker] [--prefix ] +st_download_options = '''[--all] [--marker ] [--prefix ] [--output ] [--output-dir ] [--object-threads ] [--container-threads ] [--no-download] @@ -225,7 +225,7 @@ def st_delete(parser, args, output_manager): Optional arguments: -a, --all Indicates that you really want to download everything in the account. - -m, --marker Marker to use when starting a container or account + -m, --marker Marker to use when starting a container or account download. -p, --prefix Only download items beginning with -r, --remove-prefix An optional flag for --prefix , use this diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py index e9310aa7..30947807 100644 --- a/tests/unit/test_service.py +++ b/tests/unit/test_service.py @@ -687,6 +687,41 @@ def test_list_container(self): self.assertEqual(expected_r_long, self._get_queue(mock_q)) self.assertIsNone(self._get_queue(mock_q)) + def test_list_container_marker(self): + mock_q = Queue() + mock_conn = self._get_mock_connection() + + get_container_returns = [ + (None, [{'name': 'b'}, {'name': 'c'}]), + (None, []) + ] + mock_get_cont = Mock(side_effect=get_container_returns) + mock_conn.get_container = mock_get_cont + + expected_r = self._get_expected({ + 'action': 'list_container_part', + 'container': 'test_c', + 'success': True, + 'listing': [{'name': 'b'}, {'name': 'c'}], + 'marker': 'b' + }) + + _opts = self.opts.copy() + _opts['marker'] = 'b' + SwiftService._list_container_job(mock_conn, 'test_c', _opts, mock_q) + + # This does not test if the marker is propagated, because we always + # get the final call to the get_container with the final item 'c', + # even if marker wasn't set. This test just makes sure the whole + # stack works in a sane way. + mock_kw = mock_get_cont.call_args[1] + self.assertEqual(mock_kw['marker'], 'c') + + # This tests that the lower levels get the marker delivered. + self.assertEqual(expected_r, self._get_queue(mock_q)) + + self.assertIsNone(self._get_queue(mock_q)) + def test_list_container_exception(self): mock_q = Queue() mock_conn = self._get_mock_connection() From 450f505c35f8762cca29d56b6e928490288ec166 Mon Sep 17 00:00:00 2001 From: Cedric Brandily Date: Sun, 10 Apr 2016 23:18:17 +0200 Subject: [PATCH 121/454] Support client certificate/key This change enables to specify a client certificate/key with: * usual CLI options (--os-cert/--os-key) * usual environment variables ($OS_CERT/$OS_KEY) Closes-Bug: #1565112 Change-Id: I12e151adcb6084d801c6dfed21d82232a3259aea --- swiftclient/client.py | 39 +++++++++++++++++++++++++++++++--- swiftclient/service.py | 4 ++++ swiftclient/shell.py | 12 +++++++++++ tests/unit/test_shell.py | 4 +++- tests/unit/test_swiftclient.py | 39 ++++++++++++++++++++++++++++++++-- tests/unit/utils.py | 6 ++++++ 6 files changed, 98 insertions(+), 6 deletions(-) diff --git a/swiftclient/client.py b/swiftclient/client.py index 4dbbd49c..0726f352 100644 --- a/swiftclient/client.py +++ b/swiftclient/client.py @@ -322,7 +322,8 @@ def read(self, length=None): class HTTPConnection(object): def __init__(self, url, proxy=None, cacert=None, insecure=False, - ssl_compression=False, default_user_agent=None, timeout=None): + cert=None, cert_key=None, ssl_compression=False, + default_user_agent=None, timeout=None): """ Make an HTTPConnection or HTTPSConnection @@ -333,6 +334,9 @@ def __init__(self, url, proxy=None, cacert=None, insecure=False, certificate. :param insecure: Allow to access servers without checking SSL certs. The server's certificate will not be verified. + :param cert: Client certificate file to connect on SSL server + requiring SSL client certificate. + :param cert_key: Client certificate private key file. :param ssl_compression: SSL compression should be disabled by default and this setting is not usable as of now. The parameter is kept for backward compatibility. @@ -362,6 +366,14 @@ def __init__(self, url, proxy=None, cacert=None, insecure=False, # verify requests parameter is used to pass the CA_BUNDLE file # see: http://docs.python-requests.org/en/latest/user/advanced/ self.requests_args['verify'] = cacert + if cert: + # NOTE(cbrandily): cert requests parameter is used to pass client + # cert path or a tuple with client certificate/key paths. + if cert_key: + self.requests_args['cert'] = cert, cert_key + else: + self.requests_args['cert'] = cert + if proxy: proxy_parsed = urlparse(proxy) if not proxy_parsed.scheme: @@ -448,8 +460,11 @@ def http_connection(*arg, **kwarg): def get_auth_1_0(url, user, key, snet, **kwargs): cacert = kwargs.get('cacert', None) insecure = kwargs.get('insecure', False) + cert = kwargs.get('cert') + cert_key = kwargs.get('cert_key') timeout = kwargs.get('timeout', None) parsed, conn = http_connection(url, cacert=cacert, insecure=insecure, + cert=cert, cert_key=cert_key, timeout=timeout) method = 'GET' headers = {'X-Auth-User': user, 'X-Auth-Key': key} @@ -530,6 +545,8 @@ def get_auth_keystone(auth_url, user, key, os_options, **kwargs): project_domain_id=os_options.get('project_domain_id'), debug=debug, cacert=kwargs.get('cacert'), + cert=kwargs.get('cert'), + key=kwargs.get('cert_key'), auth_url=auth_url, insecure=insecure, timeout=timeout) except exceptions.Unauthorized: msg = 'Unauthorized. Check username, password and tenant name/id.' @@ -580,6 +597,8 @@ def get_auth(auth_url, user, key, **kwargs): cacert = kwargs.get('cacert', None) insecure = kwargs.get('insecure', False) + cert = kwargs.get('cert') + cert_key = kwargs.get('cert_key') timeout = kwargs.get('timeout', None) if auth_version in AUTH_VERSIONS_V1: storage_url, token = get_auth_1_0(auth_url, @@ -588,6 +607,8 @@ def get_auth(auth_url, user, key, **kwargs): kwargs.get('snet'), cacert=cacert, insecure=insecure, + cert=cert, + cert_key=cert_key, timeout=timeout) elif auth_version in AUTH_VERSIONS_V2 + AUTH_VERSIONS_V3: # We are handling a special use case here where the user argument @@ -611,6 +632,8 @@ def get_auth(auth_url, user, key, **kwargs): key, os_options, cacert=cacert, insecure=insecure, + cert=cert, + cert_key=cert_key, timeout=timeout, auth_version=auth_version) else: @@ -1372,8 +1395,9 @@ def __init__(self, authurl=None, user=None, key=None, retries=5, preauthurl=None, preauthtoken=None, snet=False, starting_backoff=1, max_backoff=64, tenant_name=None, os_options=None, auth_version="1", cacert=None, - insecure=False, ssl_compression=True, - retry_on_ratelimit=False, timeout=None): + insecure=False, cert=None, cert_key=None, + ssl_compression=True, retry_on_ratelimit=False, + timeout=None): """ :param authurl: authentication URL :param user: user name to authenticate as @@ -1395,6 +1419,9 @@ def __init__(self, authurl=None, user=None, key=None, retries=5, service_username, service_project_name, service_key :param insecure: Allow to access servers without checking SSL certs. The server's certificate will not be verified. + :param cert: Client certificate file to connect on SSL server + requiring SSL client certificate. + :param cert_key: Client certificate private key file. :param ssl_compression: Whether to enable compression at the SSL layer. If set to 'False' and the pyOpenSSL library is present an attempt to disable SSL compression @@ -1430,6 +1457,8 @@ def __init__(self, authurl=None, user=None, key=None, retries=5, self.service_token = None self.cacert = cacert self.insecure = insecure + self.cert = cert + self.cert_key = cert_key self.ssl_compression = ssl_compression self.auth_end_time = 0 self.retry_on_ratelimit = retry_on_ratelimit @@ -1452,6 +1481,8 @@ def get_auth(self): os_options=self.os_options, cacert=self.cacert, insecure=self.insecure, + cert=self.cert, + cert_key=self.cert_key, timeout=self.timeout) return self.url, self.token @@ -1477,6 +1508,8 @@ def http_connection(self, url=None): return http_connection(url if url else self.url, cacert=self.cacert, insecure=self.insecure, + cert=self.cert, + cert_key=self.cert_key, ssl_compression=self.ssl_compression, timeout=self.timeout) diff --git a/swiftclient/service.py b/swiftclient/service.py index 99c833e0..d33d7bcf 100644 --- a/swiftclient/service.py +++ b/swiftclient/service.py @@ -148,6 +148,8 @@ def _build_default_global_options(): "os_service_type": environ.get('OS_SERVICE_TYPE'), "os_endpoint_type": environ.get('OS_ENDPOINT_TYPE'), "os_cacert": environ.get('OS_CACERT'), + "os_cert": environ.get('OS_CERT'), + "os_key": environ.get('OS_KEY'), "insecure": config_true_value(environ.get('SWIFTCLIENT_INSECURE')), "ssl_compression": False, 'segment_threads': 10, @@ -236,6 +238,8 @@ def get_conn(options): snet=options['snet'], cacert=options['os_cacert'], insecure=options['insecure'], + cert=options['os_cert'], + cert_key=options['os_key'], ssl_compression=options['ssl_compression']) diff --git a/swiftclient/shell.py b/swiftclient/shell.py index 53d7d994..c9e1b757 100755 --- a/swiftclient/shell.py +++ b/swiftclient/shell.py @@ -1294,6 +1294,8 @@ def main(arguments=None): [--os-service-type ] [--os-endpoint-type ] [--os-cacert ] [--insecure] + [--os-cert ] + [--os-key ] [--no-ssl-compression] [--help] [] @@ -1535,6 +1537,16 @@ def main(arguments=None): help='Specify a CA bundle file to use in verifying a ' 'TLS (https) server certificate. ' 'Defaults to env[OS_CACERT].') + os_grp.add_argument('--os-cert', + metavar='', + default=environ.get('OS_CERT'), + help='Specify a client certificate file (for client ' + 'auth). Defaults to env[OS_CERT].') + os_grp.add_argument('--os-key', + metavar='', + default=environ.get('OS_KEY'), + help='Specify a client certificate key file (for ' + 'client auth). Defaults to env[OS_KEY].') options, args = parse_args(parser, argv[1:], enforce_requires=False) if options.help or options.os_help: diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py index 236f1ef0..82a5590d 100644 --- a/tests/unit/test_shell.py +++ b/tests/unit/test_shell.py @@ -1781,7 +1781,9 @@ class TestKeystoneOptions(MockHttpTest): 'project-id': 'projectid', 'project-domain-id': 'projectdomainid', 'project-domain-name': 'projectdomain', - 'cacert': 'foo'} + 'cacert': 'foo', + 'cert': 'minnie', + 'key': 'mickey'} catalog_opts = {'service-type': 'my-object-store', 'endpoint-type': 'public', 'region-name': 'my-region'} diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py index f3bee3bf..5df54b16 100644 --- a/tests/unit/test_swiftclient.py +++ b/tests/unit/test_swiftclient.py @@ -512,6 +512,32 @@ def test_auth_v2_insecure(self): os_options=os_options, auth_version='2.0', insecure=False) + def test_auth_v2_cert(self): + os_options = {'tenant_name': 'foo'} + c.get_auth_keystone = fake_get_auth_keystone(os_options, None) + + auth_url_no_sslauth = 'https://www.tests.com' + auth_url_sslauth = 'https://www.tests.com/client-certificate' + + url, token = c.get_auth(auth_url_no_sslauth, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0') + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + url, token = c.get_auth(auth_url_sslauth, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0', + cert='minnie', cert_key='mickey') + self.assertTrue(url.startswith("http")) + self.assertTrue(token) + + self.assertRaises(c.ClientException, c.get_auth, + auth_url_sslauth, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0') + self.assertRaises(c.ClientException, c.get_auth, + auth_url_sslauth, 'asdf', 'asdf', + os_options=os_options, auth_version='2.0', + cert='minnie') + def test_auth_v3_with_tenant_name(self): # check the correct auth version is passed to get_auth_keystone os_options = {'tenant_name': 'asdf'} @@ -1511,6 +1537,15 @@ def test_insecure(self): conn = c.http_connection(u'http://www.test.com/', insecure=True) self.assertEqual(conn[1].requests_args['verify'], False) + def test_cert(self): + conn = c.http_connection(u'http://www.test.com/', cert='minnie') + self.assertEqual(conn[1].requests_args['cert'], 'minnie') + + def test_cert_key(self): + conn = c.http_connection( + u'http://www.test.com/', cert='minnie', cert_key='mickey') + self.assertEqual(conn[1].requests_args['cert'], ('minnie', 'mickey')) + def test_response_connection_released(self): _parsed_url, conn = c.http_connection(u'http://www.test.com/') conn.resp = MockHttpResponse() @@ -2018,8 +2053,8 @@ def read(self, *args, **kwargs): return '' def local_http_connection(url, proxy=None, cacert=None, - insecure=False, ssl_compression=True, - timeout=None): + insecure=False, cert=None, cert_key=None, + ssl_compression=True, timeout=None): parsed = urlparse(url) return parsed, LocalConnection() diff --git a/tests/unit/utils.py b/tests/unit/utils.py index 3b043bc7..d04583ff 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -57,6 +57,11 @@ def fake_get_auth_keystone(auth_url, actual_kwargs['cacert'] is None: from swiftclient import client as c raise c.ClientException("unverified-certificate") + if auth_url.startswith("https") and \ + auth_url.endswith("client-certificate") and \ + not (actual_kwargs['cert'] and actual_kwargs['cert_key']): + from swiftclient import client as c + raise c.ClientException("noclient-certificate") return storage_url, token return fake_get_auth_keystone @@ -215,6 +220,7 @@ def fake_http_connection(*args, **kwargs): on_request = kwargs.get('on_request') def wrapper(url, proxy=None, cacert=None, insecure=False, + cert=None, cert_key=None, ssl_compression=True, timeout=None): if storage_url: self.assertEqual(storage_url, url) From 3a5a25fe981817ba0e550d39d6e9863fa1539588 Mon Sep 17 00:00:00 2001 From: Joel Wright Date: Thu, 3 Mar 2016 17:22:33 +0000 Subject: [PATCH 122/454] Add new doc structure and contents for swiftclient As a result of the Hackathon we have produced a new documentation structure for the python-swiftclient. This patch introduces the new structure and adds the required content. The intention is to document the CLI, the SwiftService and Connection API. Importantly, we also provide guidance on important considerations when using a swift object store, such as which aspect of the python-swiftclient to use for various use cases, common authentication patterns and some useful examples. Co-Authored-By: Alexandra Settle Co-Authored-By: Mohit Motiani Co-Authored-By: Hisashi Osanai Change-Id: I9eb41f8e9137efa66cead67dc264a76a3c03fbda --- doc/source/cli.rst | 361 ++++++++++++-- doc/source/client-api.rst | 177 +++++++ doc/source/index.rst | 24 +- doc/source/introduction.rst | 94 ++++ doc/source/sdk.rst | 48 -- doc/source/{apis.rst => service-api.rst} | 601 +++++++++++++---------- examples/capabilities.py | 20 + examples/delete.py | 34 ++ examples/download.py | 37 ++ examples/list.py | 32 ++ examples/post.py | 31 ++ examples/stat.py | 25 + examples/upload.py | 71 +++ swiftclient/multithreading.py | 1 - 14 files changed, 1226 insertions(+), 330 deletions(-) create mode 100644 doc/source/client-api.rst create mode 100644 doc/source/introduction.rst delete mode 100644 doc/source/sdk.rst rename doc/source/{apis.rst => service-api.rst} (58%) create mode 100644 examples/capabilities.py create mode 100644 examples/delete.py create mode 100644 examples/download.py create mode 100644 examples/list.py create mode 100644 examples/post.py create mode 100644 examples/stat.py create mode 100644 examples/upload.py diff --git a/doc/source/cli.rst b/doc/source/cli.rst index 9527fbfc..12de02ff 100644 --- a/doc/source/cli.rst +++ b/doc/source/cli.rst @@ -1,29 +1,334 @@ -=== +==== CLI -=== - -Top-level commands -~~~~~~~~~~~~~~~~~~ - -.. TODO - - delete - download - list - post - stat - upload - info/capabilities - tempurl - auth - -Prescriptive examples -~~~~~~~~~~~~~~~~~~~~~ - -.. TODO - - A "Hello World" example - uploading an object - creating a tempurl - listing the contents of a container - downloading an object \ No newline at end of file +==== + +The ``swift`` tool is a command line utility for communicating with an OpenStack +Object Storage (swift) environment. It allows one to perform several types of +operations. + +Authentication +~~~~~~~~~~~~~~ + +This section covers the options for authenticating with a swift +object store. The combinations of options required for each authentication +version are detailed below, but are just a subset of those that can be used +to successfully authenticate. These are the most common and recommended +combinations. + +You should obtain the details of your authentication version and credentials +from your storage provider. These details should make it clearer which of the +authentication sections below are most likely to allow you to connect to your +storage account. + +Keystone v3 +----------- + +.. code-block:: bash + + swift --os-auth-url https://api.example.com:5000/v3 --auth-version 3 \ + --os-project-name project1 --os-project-domain-name domain1 \ + --os-username user --os-user-domain-name domain1 \ + --os-password password list + + swift --os-auth-url https://api.example.com:5000/v3 --auth-version 3 \ + --os-project-id 0123456789abcdef0123456789abcdef \ + --os-user-id abcdef0123456789abcdef0123456789 \ + --os-password password list + +Manually specifying the options above on the command line can be avoided by +setting the following combinations of environment variables: + +.. code-block:: bash + + ST_AUTH_VERSION=3 + OS_USERNAME=user + OS_USER_DOMAIN_NAME=domain1 + OS_PASSWORD=password + OS_PROJECT_NAME=project1 + OS_PROJECT_DOMAIN_NAME=domain1 + OS_AUTH_URL=https://api.example.com:5000/v3 + + ST_AUTH_VERSION=3 + OS_USER_ID=abcdef0123456789abcdef0123456789 + OS_PASSWORD=password + OS_PROJECT_ID=0123456789abcdef0123456789abcdef + OS_AUTH_URL=https://api.example.com:5000/v3 + +Keystone v2 +----------- + +.. code-block:: bash + + swift --os-auth-url https://api.example.com:5000/v2.0 \ + --os-tenant-name tenant \ + --os-username user --os-password password list + +Manually specifying the options above on the command line can be avoided by +setting the following environment variables: + +.. code-block:: bash + + ST_AUTH_VERSION=2.0 + OS_USERNAME=user + OS_PASSWORD=password + OS_TENANT_NAME=tenant + OS_AUTH_URL=https://api.example.com:5000/v2.0 + +Legacy auth systems +------------------- + +You can configure swift to work with any number of other authentication systems +that we will not cover in this document. If your storage provider is not using +Keystone to provide access tokens, please contact them for instructions on the +required options. It is likely that the options will need to be specified as +below: + +.. code-block:: bash + + swift -A https://auth.api.rackspacecloud.com/v1.0 -U user -K api_key list + +Specifying the options above manually on the command line can be avoided by +setting the following environment variables: + +.. code-block:: bash + + ST_AUTH_VERSION=1.0 + ST_AUTH=https://auth.api.rackspacecloud.com/v1.0 + ST_USER=user + ST_KEY=key + +It is also possible that you need to use a completely separate auth system, in which +case ``swiftclient`` cannot request a token for you. In this case you should make the +authentication request separately and access your storage using the token and +storage URL options shown below: + +.. code-block:: bash + + swift --os-auth-token 6ee5eb33efad4e45ab46806eac010566 \ + --os-storage-url https://10.1.5.2:8080/v1/AUTH_ced809b6a4baea7aeab61a \ + list + +.. We need the backslash below in order to indent the note +\ + + .. note:: + + Leftover environment variables are a common source of confusion when + authorization fails. + +CLI commands +~~~~~~~~~~~~ + +Stat +---- + + ``stat [container [object]]`` + + Displays information for the account, container, or object depending on + the arguments given (if any). In verbose mode, the storage URL and the + authentication token are displayed as well. + +List +---- + + ``list [command-options] [container]`` + + Lists the containers for the account or the objects for a container. + The ``-p `` or ``--prefix `` is an option that will only + list items beginning with that prefix. The ``-d `` or + ``--delimiter `` is an option (for container listings only) + that will roll up items with the given delimiter (see `OpenStack Swift + general documentation ` for + what this means). + + The ``-l`` and ``--lh`` options provide more detail, similar to ``ls -l`` + and ``ls -lh``, the latter providing sizes in human readable format + (For example: ``3K``, ``12M``, etc). The latter two switches use more + overhead to retrieve the displayed details, which is directly proportional + to the number of container or objects listed. + +Upload +------ + + ``upload [command-options] container file_or_directory [file_or_directory] [...]`` + + Uploads the files and directories specified by the remaining arguments to the + given container. The ``-c`` or ``--changed`` is an option that will only + upload files that have changed since the last upload. The + ``--object-name `` is an option that will upload a file and + name object to ```` or upload a directory and use ```` + as object prefix. The ``-S `` or ``--segment-size `` and + ``--leave-segments`` are options as well (see ``--help`` for more). + +Post +---- + + ``post [command-options] [container] [object]`` + + Updates meta information for the account, container, or object depending + on the arguments given. If the container is not found, the ``swiftclient`` + will create it automatically, but this is not true for accounts and + objects. Containers also allow the ``-r `` (or ``--read-acl + ``) and ``-w `` (or ``--write-acl ``) options. + The ``-m`` or ``--meta`` option is allowed on accounts, containers and objects, + and is used to define the user metadata items to set in the form ``Name:Value``. + You can repeat this option. For example: ``post -m Color:Blue -m Size:Large`` + + For more information about ACL formats see the documentation: + `ACLs `_. + +Download +-------- + + ``download [command-options] [container] [object] [object] [...]`` + + Downloads everything in the account (with ``--all``), or everything in a + container, or a list of objects depending on the arguments given. For a + single object download, you may use the ``-o `` or ``--output `` + option to redirect the output to a specific file or ``-`` to + redirect to stdout. You can specify optional headers with the repeatable + cURL-like option ``-H [--header ]``. + +Delete +------ + + ``delete [command-options] [container] [object] [object] [...]`` + + Deletes everything in the account (with ``--all``), or everything in a + container, or a list of objects depending on the arguments given. Segments + of manifest objects will be deleted as well, unless you specify the + ``--leave-segments`` option. + +Capabilities +------------ + + ``capabilities [proxy-url]`` + + Displays cluster capabilities. The output includes the list of the + activated Swift middlewares as well as relevant options for each ones. + Additionally the command displays relevant options for the Swift core. If + the ``proxy-url`` option is not provided, the storage URL retrieved after + authentication is used as ``proxy-url``. + + +Examples +~~~~~~~~ + +In this section we present some example usage of the ``swift`` CLI. To keep the +examples as short as possible, these examples assume that the relevant authentication +options have been set using environment variables. You can obtain the full list of +commands and options available in the ``swift`` CLI by executing the following: + +.. code-block:: bash + + > swift --help + > swift --help + +Simple examples +--------------- + +List the existing swift containers: + +.. code-block:: bash + + > swift list + + container_1 + +Create a new container: + +.. code-block:: bash + + > swift post TestContainer + +Upload an object into a container: + +.. code-block:: bash + + > swift upload TestContainer testSwift.txt + + testSwift.txt + +List the contents of a container: + +.. code-block:: bash + + > swift list TestContainer + + testSwift.txt + +Download an object from a container: + +.. code-block:: bash + + > swift download TestContainer testSwift.txt + + testSwift.txt [auth 0.028s, headers 0.045s, total 0.045s, 0.002 MB/s] + +.. We need the backslash below in order to indent the note +\ + + .. note:: + + To upload an object to a container, your current working directory must be + where the file is located or you must provide the complete path to the file. + In the case that you provide the complete path of the file, that complete + path will be the name of the uploaded object. + +For example: + +.. code-block:: bash + + > swift upload TestContainer /home/swift/testSwift/testSwift.txt + + home/swift/testSwift/testSwift.txt + + > swift list TestContainer + + home/swift/testSwift/testSwift.txt + +More complex examples +--------------------- + +Swift has a single object size limit of 5GiB. In order to upload files larger +than this, we must create a large object that consists of smaller segments. +The example below shows how to upload a large video file as a static large +object in 1GiB segments: + +.. code-block:: bash + + > swift upload videos --use-slo --segment-size 1G myvideo.mp4 + + myvideo.mp4 segment 8 + myvideo.mp4 segment 4 + myvideo.mp4 segment 2 + myvideo.mp4 segment 7 + myvideo.mp4 segment 0 + myvideo.mp4 segment 1 + myvideo.mp4 segment 3 + myvideo.mp4 segment 6 + myvideo.mp4 segment 5 + myvideo.mp4 + +This command will upload segments to a container named ``videos_segments``, and +create a manifest file describing the entire object in the ``videos`` container. +For more information on large objects, see the documentation `here +`_. + +.. code-block:: bash + + > swift list videos + + myvideo.mp4 + + > swift list videos_segments + + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000000 + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000001 + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000002 + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000003 + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000004 + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000005 + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000006 + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000007 + myvideo.mp4/slo/1460229233.679546/9341553868/1073741824/00000008 diff --git a/doc/source/client-api.rst b/doc/source/client-api.rst new file mode 100644 index 00000000..5677f70d --- /dev/null +++ b/doc/source/client-api.rst @@ -0,0 +1,177 @@ +============================== +The swiftclient.Connection API +============================== + +A low level API that provides methods for authentication and methods that +correspond to the individual REST API calls described in the swift +documentation. + +For usage details see the client docs: :mod:`swiftclient.client`. + +Authentication +-------------- + +This section covers the various combinations of kwargs required when creating +and instance of the ``Connection`` object for communicating with a swift +object store. The combinations of options required for each authentication +version are detailed below, but are +just a subset of those that can be used to successfully authenticate. These +are the most common and recommended combinations. + +Keystone v3 +~~~~~~~~~~~ + +.. code-block:: python + + _authurl = 'http://127.0.0.1:5000/v3/' + _auth_version = '3' + _user = 'tester' + _key = 'testing' + _os_options = { + 'user_domain_name': 'Default', + 'project_domain_name': 'Default', + 'project_name': 'Default' + } + + conn = Connection( + authurl=_authurl, + user=_user, + key=_key, + os_options=_os_options, + auth_version=_auth_version + ) + +.. code-block:: python + + _authurl = 'http://127.0.0.1:5000/v3/' + _auth_version = '3' + _user = 'tester' + _key = 'testing' + _os_options = { + 'user_domain_id': 'Default', + 'project_domain_id': 'Default', + 'project_id': 'Default' + } + + conn = Connection( + authurl=_authurl, + user=_user, + key=_key, + os_options=_os_options, + auth_version=_auth_version + ) + +Keystone v2 +~~~~~~~~~~~ + +.. code-block:: python + + _authurl = 'http://127.0.0.1:5000/v2.0/' + _auth_version = '2' + _user = 'tester' + _key = 'testing' + _tenant_name = 'test' + + conn = Connection( + authurl=_authurl, + user=_user, + key=_key, + tenant_name=_tenant_name, + auth_version=_auth_version + ) + +Legacy Auth +~~~~~~~~~~~ + +.. code-block:: python + + _authurl = 'http://127.0.0.1:8080/' + _auth_version = '1' + _user = 'tester' + _key = 'testing' + _tenant_name = 'test' + + conn = Connection( + authurl=_authurl, + user=_user, + key=_key, + tenant_name=_tenant_name, + auth_version=_auth_version + ) + +Examples +-------- + +In this section we present some simple code examples that demonstrate the usage +of the ``Connection`` API. You can find full details of the options and methods +available to the ``Connection`` API in the docstring generated documentation: +:mod:`swiftclient.client`. + +List the available containers: + +.. code-block:: python + + resp_headers, containers = conn.get_account() + print("Response headers: %s" % resp_headers) + for container in containers: + print(container) + +Create a new container: + +.. code-block:: python + + container = 'new-container' + conn.put_container(container) + resp_headers, containers = conn.get_account() + if container in containers: + print("The container was created") + +Create a new object with the contents of a local text file: + +.. code-block:: python + + container = 'new-container' + with open('local.txt', 'r') as local: + conn.put_object( + container, + 'local_object.txt', + contents=local, + content_type='text/plain' + ) + +Confirm presence of the object: + +.. code-block:: python + + obj = 'local_object.txt' + container = 'new-container' + try: + resp_headers = conn.head_object(container, obj) + print('The object was successfully created') + except ClientException as e: + if e.http_status = '404': + print('The object was not found') + else: + print('An error occurred checking for the existence of the object') + +Download the created object: + +.. code-block:: python + + obj = 'local_object.txt' + container = 'new-container' + resp_headers, obj_contents = conn.get_object(container, obj) + with open('local_copy.txt', 'w') as local: + local.write(obj_contents) + +Delete the created object: + +.. code-block:: python + + obj = 'local_object.txt' + container = 'new-container' + try: + conn.delete_object(container, obj) + print("Successfully deleted the object") + except ClientException as e: + print("Failed to delete the object with error: %s" % e) diff --git a/doc/source/index.rst b/doc/source/index.rst index da16a3ce..f123b7b1 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -1,18 +1,28 @@ +====================================== Welcome to the python-swiftclient Docs -************************************** +====================================== + +Introduction +~~~~~~~~~~~~ + +.. toctree:: + :maxdepth: 2 + + introduction Developer Documentation -======================= +~~~~~~~~~~~~~~~~~~~~~~~ .. toctree:: :maxdepth: 2 - apis cli - sdk + service-api + client-api + Code-Generated Documentation -============================ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. toctree:: :maxdepth: 2 @@ -20,14 +30,14 @@ Code-Generated Documentation swiftclient Indices and tables -================== +~~~~~~~~~~~~~~~~~~ * :ref:`genindex` * :ref:`modindex` * :ref:`search` License -======= +~~~~~~~ Copyright 2013 OpenStack, LLC. diff --git a/doc/source/introduction.rst b/doc/source/introduction.rst new file mode 100644 index 00000000..926b1b90 --- /dev/null +++ b/doc/source/introduction.rst @@ -0,0 +1,94 @@ +============ +Introduction +============ + +Where to Start? +~~~~~~~~~~~~~~~ + +The ``python-swiftclient`` project comprises a command line tool and two +separate APIs for accessing swift programmatically. Choosing the most +appropriate method for a given use case is the first problem a user needs to +solve. + +Use Cases +--------- + +Alongside the command line tool, the ``python-swiftclient`` includes two +levels of API: + + * A low level client API that provides simple Python wrappers around the + various authentication mechanisms and the individual HTTP requests. + * A high level service API that provides methods for performing common + operations in parallel on a thread pool. + +Example use cases: + + * Uploading and retrieving data + Use the command line tool if you are simply uploading and downloading + files and directories to and from your filesystem. The command line tool + can be integrated into a shell script to automate tasks. + + * Integrating into an automated Python workflow + Use the ``SwiftService`` API to perform operations offered by the CLI + if your use case requires integration with a Python-based workflow. + This method offers greater control and flexibility over individual object + operations, such as the metadata set on each object. The ``SwiftService`` + class provides methods to perform multiple sets of operations against a + swift object store using a configurable shared thread pool. A single + instance of the ``SwiftService`` class can be shared between multiple + threads in your own code. + + * Developing an application in Python to access a swift object store + Use the ``SwiftService`` API to develop Python applications that use + swift to store and retrieve objects. A ``SwiftService`` instance provides + a configurable thread pool for performing all operations supported by the + CLI. + + * Fine-grained control over threading or the requests being performed + Use the ``Connection`` API if your use case requires fine grained control + over advanced features or you wish to use your own existing threading + model. Examples of advanced features requiring the use of the + ``Connection`` API include creating an SLO manifest that references + already existing objects, or fine grained control over the query strings + supplied with each HTTP request. + +Important considerations +~~~~~~~~~~~~~~~~~~~~~~~~ + +This section covers some important considerations, helpful hints, and things to +avoid when integrating an object store into your workflow. + +An object store is not a filesystem +----------------------------------- + +It cannot be stressed enough that your usage of the object store should reflect +the proper use case, and not treat the storage like a traditional filesystem. +There are two main restrictions to bear in mind when designing an application +that uses an object store: + + * You cannot rename objects. Due to fact that the name of an object is one + of the factors that determines where the object and its replicas are stored, + renaming would require multiple copies of the data to be moved between + physical storage devices. If you want to rename an object you must upload + to the new location, or make a server side copy request to the new location, + and then delete the original. + + * You cannot modify objects. Objects are stored in multiple locations and + are checked for integrity based on the MD5 sum calculated during + upload. In order to modify the contents of an object, the entire desired + contents must be re-uploaded. In certain special cases it is possible to + work around this restriction using large objects, but no general + file-like access is available to modify a stored object. + +Objects cannot be locked +------------------------ + +There is no mechanism to perform a combination of reading the +data/metadata from an object and writing an update to that data/metadata in an +atomic way. Any user with access to a container could update the contents or +metadata associated with an object at any time. + +Workflows that assume that no updates have been made since the last read of an +object should be discouraged. Enabling a workflow of this type requires an +external object locking mechanism and/or cooperation between all clients +accessing the data. diff --git a/doc/source/sdk.rst b/doc/source/sdk.rst deleted file mode 100644 index aa152509..00000000 --- a/doc/source/sdk.rst +++ /dev/null @@ -1,48 +0,0 @@ -=== -SDK -=== - -Where to start? -~~~~~~~~~~~~~~~ - -.. TODO - - when to use SwiftService - when to use client.py - -SwiftService classes and methods -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. TODO - - docs for each method (autogen from docstrings?) - -Client classes and methods -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. TODO - - docs for each method (autogen from docstrings?) - -Guidelines for writing an app -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. TODO - - auth - how to use various features - when to use various features - pooling connections - concurrency - retries - -Prescriptive examples -~~~~~~~~~~~~~~~~~~~~~ - -.. TODO - - A "Hello World" example - connecting - uploading an object - uploading a directory - \ No newline at end of file diff --git a/doc/source/apis.rst b/doc/source/service-api.rst similarity index 58% rename from doc/source/apis.rst rename to doc/source/service-api.rst index 935b4a42..7d65fd15 100644 --- a/doc/source/apis.rst +++ b/doc/source/service-api.rst @@ -1,63 +1,93 @@ -====================== -python-swiftclient API -====================== +================================ +The swiftclient.SwiftService API +================================ -The python-swiftclient includes two levels of API. A low level client API that -provides simple python wrappers around the various authentication mechanisms, -the individual HTTP requests, and a high level service API that provides -methods for performing common operations in parallel on a thread pool. +A higher-level API aimed at allowing developers an easy way to perform multiple +operations asynchronously using a configurable thread pool. Documentation for +each service method call can be found here: :mod:`swiftclient.service`. -This document aims to provide guidance for choosing between these APIs and -examples of usage for the service API. +Authentication +-------------- +This section covers the various options for authenticating with a swift +object store. The combinations of options required for each authentication +version are detailed below. Once again, these are just a subset of those that +can be used to successfully authenticate, but they are the most common and +recommended. -Important Considerations -~~~~~~~~~~~~~~~~~~~~~~~~ +The relevant authentication options are presented as python dictionaries that +should be added to any other options you are supplying to your ``SwiftService`` +instance. As indicated in the python code, you can also set these options as +environment variables that will be loaded automatically if the relevant option +is not specified. -This section covers some important considerations, helpful hints, and things -to avoid when integrating an object store into your workflow. +The ``SwiftService`` authentication attempts to automatically select +the auth version based on the combination of options specified, but +supplying options from multiple different auth versions can cause unexpected +behaviour. -An Object Store is not a filesystem ------------------------------------ + .. note:: -.. important:: + Leftover environment variables are a common source of confusion when + authorization fails. - It cannot be stressed enough that your usage of the object store should reflect - the use case, and not treat the storage like a filesystem. +Keystone V3 +~~~~~~~~~~~ -There are 2 main restrictions to bear in mind here when designing your use of the object -store: +.. code-block:: python -#. Objects cannot be renamed due to the way in which objects are stored and - references by the object store. This usually requires multiple copies of - the data to be moved between physical storage devices. - As a result, a move operation is not provided. If the user wants to move an - object they must re-upload to the new location and delete the - original. -#. Objects cannot be modified. Objects are stored in multiple locations and are - checked for integrity based on the ``MD5 sum`` calculated during upload. - Object creation is a 1-shot event, and in order to modify the contents of an - object the entire new contents must be re-uploaded. In certain special cases - it is possible to work around this restriction using large objects, but no - general file-like access is available to modify a stored object. + { + ... + "auth_version": environ.get('ST_AUTH_VERSION'), # Should be '3' + "os_username": environ.get('OS_USERNAME'), + "os_password": environ.get('OS_PASSWORD'), + "os_project_name": environ.get('OS_PROJECT_NAME'), + "os_project_domain_name": environ.get('OS_PROJECT_DOMAIN_NAME'), + "os_auth_url": environ.get('OS_AUTH_URL'), + ... + } +.. code-block:: python -The swiftclient.Connection API -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + { + ... + "auth_version": environ.get('ST_AUTH_VERSION'), # Should be '3' + "os_username": environ.get('OS_USERNAME'), + "os_password": environ.get('OS_PASSWORD'), + "os_project_id": environ.get('OS_PROJECT_ID'), + "os_project_domain_id": environ.get('OS_PROJECT_DOMAIN_ID'), + "os_auth_url": environ.get('OS_AUTH_URL'), + ... + } -A low level API that provides methods for authentication and methods that -correspond to the individual REST API calls described in the swift -documentation. +Keystone V2 +~~~~~~~~~~~ -For usage details see the client docs: :mod:`swiftclient.client`. +.. code-block:: python + { + ... + "auth_version": environ.get('ST_AUTH_VERSION'), # Should be '2.0' + "os_username": environ.get('OS_USERNAME'), + "os_password": environ.get('OS_PASSWORD'), + "os_tenant_name": environ.get('OS_TENANT_NAME'), + "os_auth_url": environ.get('OS_AUTH_URL'), + ... + } -The swiftclient.SwiftService API -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Legacy Auth +~~~~~~~~~~~ -A higher level API aimed at allowing developers an easy way to perform multiple -operations asynchronously using a configurable thread pool. Documentation for each -service method call can be found here: :mod:`swiftclient.service`. +.. code-block:: python + + { + ... + "auth_version": environ.get('ST_AUTH_VERSION'), # Should be '1.0' + "auth": environ.get('ST_AUTH'), + "user": environ.get('ST_USER'), + "key": environ.get('ST_KEY'), + ... + } Configuration ------------- @@ -77,7 +107,7 @@ passed to the ``SwiftService`` during initialisation. The options available in this dictionary are described below, along with their defaults: Options -^^^^^^^ +~~~~~~~ ``retries``: ``5`` The number of times that the library should attempt to retry HTTP @@ -192,51 +222,8 @@ source code for ``python-swiftclient``. Each ``SwiftService`` method also allows for an optional dictionary to override those specified at init time, and the appropriate docstrings show which options modify each method's behaviour. -Authentication -~~~~~~~~~~~~~~ - -This section covers the various options for authenticating with a swift -object store. The combinations of options required for each authentication -version are detailed below. - -Version 1.0 Auth ----------------- - - ``auth_version``: ``environ.get('ST_AUTH_VERSION')`` - - ``auth``: ``environ.get('ST_AUTH')`` - - ``user``: ``environ.get('ST_USER')`` - - ``key``: ``environ.get('ST_KEY')`` - - -Version 2.0 and 3.0 Auth ------------------------- - - ``auth_version``: ``environ.get('ST_AUTH_VERSION')`` - - ``os_username``: ``environ.get('OS_USERNAME')`` - - ``os_password``: ``environ.get('OS_PASSWORD')`` - - ``os_tenant_name``: ``environ.get('OS_TENANT_NAME')`` - - ``os_auth_url``: ``environ.get('OS_AUTH_URL')`` - -As is evident from the default values, if these options are not set explicitly -in the options dictionary, then they will default to the values of the given -environment variables. The ``SwiftService`` authentication automatically selects -the auth version based on the combination of options specified, but -having options from different auth versions can cause unexpected behaviour. - - .. note:: - - Leftover environment variables are a common source of confusion when - authorization fails. - -Operation Return Values -~~~~~~~~~~~~~~~~~~~~~~~ +Available Operations +-------------------- Each operation provided by the service API may raise a ``SwiftError`` or ``ClientException`` for any call that fails completely (or a call which @@ -371,32 +358,14 @@ operation was not successful, and will include the keys below: } Example -------- +^^^^^^^ The code below demonstrates the use of ``stat`` to retrieve the headers for a given list of objects in a container using 20 threads. The code creates a -mapping from object name to headers. - -.. code-block:: python +mapping from object name to headers which is then pretty printed to the log. - import logging - - from swiftclient.service import SwiftService - - logger = logging.getLogger() - _opts = {'object_dd_threads': 20} - with SwiftService(options=_opts) as swift: - container = 'container1' - objects = [ 'object_%s' % n for n in range(0,100) ] - header_data = {} - stats_it = swift.stat(container=container, objects=objects) - for stat_res in stats_it: - if stat_res['success']: - header_data[stat_res['object']] = stat_res['headers'] - else: - logger.error( - 'Failed to retrieve stats for %s' % stat_res['object'] - ) +.. literalinclude:: ../../examples/stat.py + :language: python List ~~~~ @@ -456,55 +425,38 @@ dictionary as described below: } Example -------- +^^^^^^^ The code below demonstrates the use of ``list`` to list all items in a container that are over 10MiB in size: -.. code-block:: python - - container = 'example_container' - minimum_size = 10*1024**2 - with SwiftService() as swift: - try: - stats_parts_gen = swift.list(container=container) - for stats in stats_parts_gen: - if stats["success"]: - for item in stats["listing"]: - i_size = int(item["bytes"]) - if i_size > minimum_size: - i_name = item["name"] - i_etag = item["hash"] - print( - "%s [size: %s] [etag: %s]" % - (i_name, i_size, i_etag) - ) - else: - raise stats["error"] - except SwiftError as e: - output_manager.error(e.value) +.. literalinclude:: ../../examples/list.py + :language: python Post ~~~~ Post can be called against an account, container or list of objects in order to -update the metadata attached to the given items. Each element of the object list -may be a plain string of the object name, or a ``SwiftPostObject`` that -allows finer control over the options applied to each of the individual post -operations. In the first two cases a single dictionary is returned containing the -results of the operation, and in the case of a list of objects being supplied, -an iterator over the results generated for each object post is returned. If the -given container or account does not exist, the ``post`` method will raise a -``SwiftError``. - -.. When a string is given for the object name, the options - -Successful metadata update results are dictionaries as described below: +update the metadata attached to the given items. In the first two cases a single +dictionary is returned containing the results of the operation, and in the case +of a list of objects being supplied, an iterator over the results generated for +each object post is returned. + +Each element of the object list may be a plain string of the object name, or a +``SwiftPostObject`` that allows finer control over the options and metadata +applied to each of the individual post operations. When a string is given for +the object name, the options and metadata applied are a combination of those +supplied to the call to ``post()`` and the defaults of the ``SwiftService`` +object. + +If the given container or account does not exist, the ``post`` method will +raise a ``SwiftError``. Successful metadata update results are dictionaries as +described below: .. code-block:: python { - 'action': <'post_account'|<'post_container'>|'post_object'>, + 'action': <'post_account'|'post_container'|'post_object'>, 'success': True, 'container': , 'object': , @@ -513,28 +465,87 @@ Successful metadata update results are dictionaries as described below: } .. note:: - Updating user metadata keys will not only add any specified keys, but will also remove user metadata that has previously been set. This means that each time user metadata is updated, the complete set of desired key-value pairs must be specified. +Example +^^^^^^^ +The code below demonstrates the use of ``post`` to set an archive folder in a +given container to expire after a 24 hour delay: -.. Example -.. ------- +.. literalinclude:: ../../examples/post.py + :language: python -.. TBD +Download +~~~~~~~~ + +Download can be called against an entire account, a single container, or a list +of objects in a given container. Each element of the object list is a string +detailing the full name of an object to download. + +In order to download the full contents of an entire account, you must set the +value of ``yes_all`` to ``True`` in the ``options`` dictionary supplied to +either the ``SwiftService`` instance or the call to ``download``. + +If the given container or account does not exist, the ``download`` method will +raise a ``SwiftError``, otherwise an iterator over the results generated for +each object download is returned. + +See :mod:`swiftclient.service.SwiftService.download` for docs generated from the +method docstring. + +For each successfully downloaded object, the results returned by the iterator +will be a dictionary as described below (results are not returned for completed +container or object segment downloads): + +.. code-block:: python + + { + 'action': 'download_object', + 'container': , + 'object': , + 'success': True, + 'path': , + 'pseudodir': , + 'start_time':