Use Python 3.8 in AzDo pipelines (microsoft#10276)

* Use Python 3.8 in AzDo pipelines
* Add python 3.8 condition
* Fail fast (single workspace)
* Try downgrading pytest
* Upgrade pytest, update pytest args
* Uncomment other PR validation stages
* Windis fail fast pytest 3.6.3 python 3.8
* Commenting everything out didn't work out
* Re-upgrade pytest to 4.6.9
* Use pathlib2 for py27 compatibility
* Save path as string
* Fix paths in the parent function (for all tests)
* Use Path.resolve() for projroot too
* Use pathlib in test_util too
* Uncomment accidental comment
* Add comments

Author: kimadeline

build/ci/templates/globals.yml

@@ -1,5 +1,5 @@
 variables:
-  PythonVersion: '3.7' # Always use latest version.
+  PythonVersion: '3.8' # Always use latest version.
   NodeVersion: '12.4.0' # Check version of node used in VS Code.
   NpmVersion: '6.13.4'
   MOCHA_FILE: '$(Build.ArtifactStagingDirectory)/test-junit.xml' # All test files will write their JUnit xml output to this file, clobbering the last time it was written.

build/test-requirements.txt

@@ -10,9 +10,9 @@ pycodestyle
 prospector
 pydocstyle
 nose
-pytest==3.6.3
+pytest==4.6.9 # Last version of pytest with Python 2.7 support
 rope
 flask
 django
 isort
-packaging==19.2
+pathlib2>=2.2.0 ; python_version<'3.6' # Python 2.7 compatibility (pytest)

news/3 Code Health/8298.md

@@ -0,0 +1 @@
+Use Python 3.8 in tests run on Azure DevOps.

pythonFiles/tests/testing_tools/adapter/pytest/test_discovery.py

@@ -20,6 +20,9 @@
 from testing_tools.adapter.pytest import _pytest_item as pytest_item
 from testing_tools.adapter.pytest._discovery import discover, TestCollector
 
+# In Python 3.8 __len__ is called twice, which impacts some of the test assertions we do below.
+PYTHON_38_OR_LATER = sys.version_info[0] >= 3 and sys.version_info[1] >= 8
+
 
 class StubPyTest(StubProxy):
     def __init__(self, stub=None):
@@ -255,20 +258,22 @@ def test_basic(self):
         plugin = StubPlugin(stub)
         expected = []
         plugin.discovered = expected
+        calls = [
+            ("pytest.main", None, {"args": self.DEFAULT_ARGS, "plugins": [plugin]}),
+            ("discovered.parents", None, None),
+            ("discovered.__len__", None, None),
+            ("discovered.__getitem__", (0,), None),
+        ]
+
+        # In Python 3.8 __len__ is called twice.
+        if PYTHON_38_OR_LATER:
+            calls.insert(3, ("discovered.__len__", None, None))
 
         parents, tests = discover([], _pytest_main=stubpytest.main, _plugin=plugin)
 
         self.assertEqual(parents, [])
         self.assertEqual(tests, expected)
-        self.assertEqual(
-            stub.calls,
-            [
-                ("pytest.main", None, {"args": self.DEFAULT_ARGS, "plugins": [plugin]}),
-                ("discovered.parents", None, None),
-                ("discovered.__len__", None, None),
-                ("discovered.__getitem__", (0,), None),
-            ],
-        )
+        self.assertEqual(stub.calls, calls)
 
     def test_failure(self):
         stub = Stub()
@@ -291,20 +296,22 @@ def test_no_tests_found(self):
         plugin = StubPlugin(stub)
         expected = []
         plugin.discovered = expected
+        calls = [
+            ("pytest.main", None, {"args": self.DEFAULT_ARGS, "plugins": [plugin]}),
+            ("discovered.parents", None, None),
+            ("discovered.__len__", None, None),
+            ("discovered.__getitem__", (0,), None),
+        ]
+
+        # In Python 3.8 __len__ is called twice.
+        if PYTHON_38_OR_LATER:
+            calls.insert(3, ("discovered.__len__", None, None))
 
         parents, tests = discover([], _pytest_main=pytest.main, _plugin=plugin)
 
         self.assertEqual(parents, [])
         self.assertEqual(tests, expected)
-        self.assertEqual(
-            stub.calls,
-            [
-                ("pytest.main", None, {"args": self.DEFAULT_ARGS, "plugins": [plugin]}),
-                ("discovered.parents", None, None),
-                ("discovered.__len__", None, None),
-                ("discovered.__getitem__", (0,), None),
-            ],
-        )
+        self.assertEqual(stub.calls, calls)
 
     def test_stdio_hidden(self):
         pytest_stdout = "spamspamspamspamspamspamspammityspam"
@@ -317,6 +324,17 @@ def fake_pytest_main(args, plugins):
 
         plugin = StubPlugin(stub)
         plugin.discovered = []
+        calls = [
+            ("pytest.main", None, {"args": self.DEFAULT_ARGS, "plugins": [plugin]}),
+            ("discovered.parents", None, None),
+            ("discovered.__len__", None, None),
+            ("discovered.__getitem__", (0,), None),
+        ]
+
+        # In Python 3.8 __len__ is called twice.
+        if PYTHON_38_OR_LATER:
+            calls.insert(3, ("discovered.__len__", None, None))
+
         buf = StringIO()
 
         sys.stdout = buf
@@ -327,15 +345,7 @@ def fake_pytest_main(args, plugins):
         captured = buf.getvalue()
 
         self.assertEqual(captured, "")
-        self.assertEqual(
-            stub.calls,
-            [
-                ("pytest.main", None, {"args": self.DEFAULT_ARGS, "plugins": [plugin]}),
-                ("discovered.parents", None, None),
-                ("discovered.__len__", None, None),
-                ("discovered.__getitem__", (0,), None),
-            ],
-        )
+        self.assertEqual(stub.calls, calls)
 
     def test_stdio_not_hidden(self):
         pytest_stdout = "spamspamspamspamspamspamspammityspam"
@@ -348,6 +358,17 @@ def fake_pytest_main(args, plugins):
 
         plugin = StubPlugin(stub)
         plugin.discovered = []
+        calls = [
+            ("pytest.main", None, {"args": self.DEFAULT_ARGS, "plugins": [plugin]}),
+            ("discovered.parents", None, None),
+            ("discovered.__len__", None, None),
+            ("discovered.__getitem__", (0,), None),
+        ]
+
+        # In Python 3.8 __len__ is called twice.
+        if PYTHON_38_OR_LATER:
+            calls.insert(3, ("discovered.__len__", None, None))
+
         buf = StringIO()
 
         sys.stdout = buf
@@ -358,15 +379,7 @@ def fake_pytest_main(args, plugins):
         captured = buf.getvalue()
 
         self.assertEqual(captured, pytest_stdout)
-        self.assertEqual(
-            stub.calls,
-            [
-                ("pytest.main", None, {"args": self.DEFAULT_ARGS, "plugins": [plugin]}),
-                ("discovered.parents", None, None),
-                ("discovered.__len__", None, None),
-                ("discovered.__getitem__", (0,), None),
-            ],
-        )
+        self.assertEqual(stub.calls, calls)
 
 
 class CollectorTests(unittest.TestCase):

pythonFiles/tests/testing_tools/adapter/test_functional.py

@@ -15,6 +15,12 @@
 from ...__main__ import TESTING_TOOLS_ROOT
 from testing_tools.adapter.util import fix_path, PATH_SEP
 
+# Pytest 3.7 and later uses pathlib/pathlib2 for path resolution.
+try:
+    from pathlib import Path
+except ImportError:
+    from pathlib2 import Path
+
 
 CWD = os.getcwd()
 DATA_DIR = os.path.join(os.path.dirname(__file__), ".data")
@@ -23,7 +29,8 @@
 
 def resolve_testroot(name):
     projroot = os.path.join(DATA_DIR, name)
-    return projroot, os.path.join(projroot, "tests")
+    testroot = os.path.join(projroot, "tests")
+    return str(Path(projroot).resolve()), str(Path(testroot).resolve())
 
 
 def run_adapter(cmd, tool, *cliargs):

pythonFiles/tests/testing_tools/adapter/test_util.py

@@ -13,6 +13,12 @@
 
 import pytest
 
+# Pytest 3.7 and later uses pathlib/pathlib2 for path resolution.
+try:
+    from pathlib import Path
+except ImportError:
+    from pathlib2 import Path
+
 from testing_tools.adapter.util import (
     fix_path,
     fix_relpath,
@@ -30,9 +36,9 @@ def test_isolated_imports(self):
         from . import test_functional
 
         ignored = {
-            os.path.abspath(__file__),
-            os.path.abspath(util.__file__),
-            os.path.abspath(test_functional.__file__),
+            str(Path(os.path.abspath(__file__)).resolve()),
+            str(Path(os.path.abspath(util.__file__)).resolve()),
+            str(Path(os.path.abspath(test_functional.__file__)).resolve()),
         }
         adapter = os.path.abspath(os.path.dirname(testing_tools.adapter.__file__))
         tests = os.path.join(

src/client/testing/pytest/services/argsService.ts

@@ -18,6 +18,7 @@ const OptionsWithArguments = [
     '-n', // -n is a pytest-xdist option
     '--assert',
     '--basetemp',
+    '--cache-show',
     '--capture',
     '--color',
     '--confcutdir',
@@ -31,6 +32,7 @@ const OptionsWithArguments = [
     '--doctest-report',
     '--durations',
     '--ignore',
+    '--ignore-glob',
     '--import-mode',
     '--junit-prefix',
     '--junit-xml',
@@ -65,7 +67,6 @@ const OptionsWithArguments = [
 
 const OptionsWithoutArguments = [
     '--cache-clear',
-    '--cache-show',
     '--collect-in-virtualenv',
     '--collect-only',
     '--continue-on-collection-errors',
@@ -106,7 +107,11 @@ const OptionsWithoutArguments = [
     '--setup-plan',
     '--setup-show',
     '--showlocals',
+    '--stepwise',
+    '--sw',
+    '--stepwise-skip',
     '--strict',
+    '--strict-markers',
     '--trace-config',
     '--verbose',
     '--version',