Add functional tests to osc

Create a script that kicks off function tests that exercise
openstackclient commands against a cloud.

If no keystone/openstack process is detected, a devstack instance
is spun up and the tests are run against that.

There is also a hook added to tox.ini so that we can run these
tests easily from a gate job.

Change-Id: I3cc8b2b800de7ca74af506d2c7e8ee481fa985f0

Author: Steve Martinelli

functional/__init__.py

@@ -0,0 +1,26 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+class CommandFailed(Exception):
+    def __init__(self, returncode, cmd, output, stderr):
+        super(CommandFailed, self).__init__()
+        self.returncode = returncode
+        self.cmd = cmd
+        self.stdout = output
+        self.stderr = stderr
+
+    def __str__(self):
+        return ("Command '%s' returned non-zero exit status %d.\n"
+                "stdout:\n%s\n"
+                "stderr:\n%s" % (self.cmd, self.returncode,
+                                 self.stdout, self.stderr))

functional/common/__init__.py

@@ -0,0 +1,129 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import re
+import shlex
+import subprocess
+import testtools
+
+import six
+
+from functional.common import exceptions
+
+
+def execute(cmd, action, flags='', params='', fail_ok=False,
+            merge_stderr=False):
+    """Executes specified command for the given action."""
+    cmd = ' '.join([cmd, flags, action, params])
+    cmd = shlex.split(cmd.encode('utf-8'))
+    result = ''
+    result_err = ''
+    stdout = subprocess.PIPE
+    stderr = subprocess.STDOUT if merge_stderr else subprocess.PIPE
+    proc = subprocess.Popen(cmd, stdout=stdout, stderr=stderr)
+    result, result_err = proc.communicate()
+    if not fail_ok and proc.returncode != 0:
+        raise exceptions.CommandFailed(proc.returncode, cmd, result,
+                                       result_err)
+    return result
+
+
+class TestCase(testtools.TestCase):
+
+    delimiter_line = re.compile('^\+\-[\+\-]+\-\+$')
+
+    def openstack(self, action, flags='', params='', fail_ok=False):
+        """Executes openstackclient command for the given action."""
+        return execute('openstack', action, flags, params, fail_ok)
+
+    def assert_table_structure(self, items, field_names):
+        """Verify that all items have keys listed in field_names."""
+        for item in items:
+            for field in field_names:
+                self.assertIn(field, item)
+
+    def assert_show_fields(self, items, field_names):
+        """Verify that all items have keys listed in field_names."""
+        for item in items:
+            for key in six.iterkeys(item):
+                self.assertIn(key, field_names)
+
+    def parse_show(self, raw_output):
+        """Return list of dicts with item values parsed from cli output."""
+
+        items = []
+        table_ = self.table(raw_output)
+        for row in table_['values']:
+            item = {}
+            item[row[0]] = row[1]
+            items.append(item)
+        return items
+
+    def parse_listing(self, raw_output):
+        """Return list of dicts with basic item parsed from cli output."""
+
+        items = []
+        table_ = self.table(raw_output)
+        for row in table_['values']:
+            item = {}
+            for col_idx, col_key in enumerate(table_['headers']):
+                item[col_key] = row[col_idx]
+            items.append(item)
+        return items
+
+    def table(self, output_lines):
+        """Parse single table from cli output.
+
+        Return dict with list of column names in 'headers' key and
+        rows in 'values' key.
+        """
+        table_ = {'headers': [], 'values': []}
+        columns = None
+
+        if not isinstance(output_lines, list):
+            output_lines = output_lines.split('\n')
+
+        if not output_lines[-1]:
+            # skip last line if empty (just newline at the end)
+            output_lines = output_lines[:-1]
+
+        for line in output_lines:
+            if self.delimiter_line.match(line):
+                columns = self._table_columns(line)
+                continue
+            if '|' not in line:
+                continue
+            row = []
+            for col in columns:
+                row.append(line[col[0]:col[1]].strip())
+            if table_['headers']:
+                table_['values'].append(row)
+            else:
+                table_['headers'] = row
+
+        return table_
+
+    def _table_columns(self, first_table_row):
+        """Find column ranges in output line.
+
+        Return list of tuples (start,end) for each column
+        detected by plus (+) characters in delimiter line.
+        """
+        positions = []
+        start = 1  # there is '+' at 0
+        while start < len(first_table_row):
+            end = first_table_row.find('+', start)
+            if end == -1:
+                break
+            positions.append((start, end))
+            start = end + 1
+        return positions

functional/common/exceptions.py

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+FUNCTIONAL_TEST_DIR=$(cd $(dirname "$0") && pwd)
+source $FUNCTIONAL_TEST_DIR/harpoonrc
+
+OPENSTACKCLIENT_DIR=$FUNCTIONAL_TEST_DIR/..
+
+if [[ -z $DEVSTACK_DIR ]]; then
+    echo "guessing location of devstack"
+    DEVSTACK_DIR=$OPENSTACKCLIENT_DIR/../devstack
+fi
+
+function setup_credentials {
+    RC_FILE=$DEVSTACK_DIR/accrc/$HARPOON_USER/$HARPOON_TENANT
+    source $RC_FILE
+    echo 'sourcing' $RC_FILE
+    echo 'running tests with'
+    env | grep OS
+}
+
+function run_tests {
+    cd $FUNCTIONAL_TEST_DIR
+    python -m testtools.run discover
+    rvalue=$?
+    cd $OPENSTACKCLIENT_DIR
+    exit $rvalue
+}
+
+setup_credentials
+run_tests

functional/common/test.py

@@ -0,0 +1,14 @@
+# Global options
+#RECLONE=yes
+
+# Devstack options
+#ADMIN_PASSWORD=openstack
+#MYSQL_PASSWORD=openstack
+#RABBIT_PASSWORD=openstack
+#SERVICE_TOKEN=openstack
+#SERVICE_PASSWORD=openstack
+
+# Harpoon options
+HARPOON_USER=admin
+HARPOON_TENANT=admin
+#DEVSTACK_DIR=/opt/stack/devstack

functional/harpoon.sh

@@ -0,0 +1,35 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from functional.common import exceptions
+from functional.common import test
+
+
+class IdentityV2Tests(test.TestCase):
+    """Functional tests for Identity V2 commands. """
+
+    def test_user_list(self):
+        field_names = ['ID', 'Name']
+        raw_output = self.openstack('user list')
+        items = self.parse_listing(raw_output)
+        self.assert_table_structure(items, field_names)
+
+    def test_user_get(self):
+        field_names = ['email', 'enabled', 'id', 'name',
+                       'project_id', 'username']
+        raw_output = self.openstack('user show admin')
+        items = self.parse_show(raw_output)
+        self.assert_show_fields(items, field_names)
+
+    def test_bad_user_command(self):
+        self.assertRaises(exceptions.CommandFailed,
+                          self.openstack, 'user unlist')

functional/harpoonrc

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# This is a script that kicks off a series of functional tests against an
+# OpenStack cloud. It will attempt to create an instance if one is not
+# available. Do not run this script unless you know what you're doing.
+# For more information refer to:
+# http://docs.openstack.org/developer/python-openstackclient/
+
+set -xe
+
+OPENSTACKCLIENT_DIR=$(cd $(dirname "$0") && pwd)
+
+cd $OPENSTACKCLIENT_DIR
+echo "Running openstackclient functional test suite"
+sudo -H -u stack tox -e functional

functional/tests/__init__.py

@@ -11,10 +11,14 @@ setenv = VIRTUAL_ENV={envdir}
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
 commands = python setup.py testr --testr-args='{posargs}'
+whitelist_externals = bash
 
 [testenv:pep8]
 commands = flake8
 
+[testenv:functional]
+commands = bash -x {toxinidir}/functional/harpoon.sh
+
 [testenv:venv]
 commands = {posargs}