Codeception/tests/cli/DataProviderFailuresAndExceptionsCest.php at patch-2 · yesdevnull/Codeception · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
<?php

declare(strict_types=1);

final class DataProviderFailuresAndExceptionsCest
{
    private function moveToPath(CliGuy $I)
    {
        $I->amInPath('tests/data/dataprovider_failures_and_exceptions');
    }

    /**
     * This looks at only the contents of stdout when there is a failure in parsing a dataProvider annotation.
     * When there is a failure all the useful information should go to stderr, so stdout is left with
     * only the version headers.
     *
     * @param CliGuy $I
     * @before moveToPath
     */
    public function runTestWithDataProvidersFailureStdout(CliGuy $I)
    {
        /**
         * On windows /dev/null is NUL so detect running OS and return the appropriate string for redirection.
         * As some systems have php_uname and co disabled, we use the DIRECTORY_SEPARATOR constant to
         * figure out if we are running on windows or not.
         */
        $devNull = (DIRECTORY_SEPARATOR === '\\')?'NUL':'/dev/null';
        $I->executeCommand('run -n -v unit DataProvidersFailureCest 2> '.$devNull,false);
        // We should only see the version headers in stdout when there is this kind of failure.
        $I->canSeeShellOutputMatches('#^Codeception PHP Testing Framework v[0-9\.]+\nPowered by PHPUnit .+ by Sebastian Bergmann and contributors\.#');
        $I->seeResultCodeIs(1);
    }

    /**
     * This redirects stderr to stdout so that we can test the contents of stderr. Stderr is where all the interesting
     * information should be when there is a failure.
     *
     * @param CliGuy $I
     * @before moveToPath
     */
    public function runTestWithDataProvidersFailureStderr(CliGuy $I)
    {
        $I->executeCommand('run -n unit DataProvidersFailureCest 2>&1',false);
        $I->seeInShellOutput("Couldn't parse test");
        $I->seeInShellOutput("DataProvider 'rectangle' for DataProvidersFailureCest->testIsTriangle");
        $I->seeInShellOutput('Make sure that the dataprovider exist within the test class.');
        // For Unit tests PHPUnit throws the errors, this confirms that we haven't ended up running PHPUnit test Loader
        $I->dontSeeInShellOutput('PHPUnit_Framework_Warning');
        $I->dontSeeInShellOutput('The data provider specified for DataProvidersFailureCest::testIsTriangle');
        $I->dontSeeInShellOutput('Method rectangle does not exist');
        $I->dontSeeInShellOutput('FAILURES!');
        $I->dontSeeInShellOutput('WARNINGS!');
        $I->dontSeeInShellOutput('OK');
        $I->dontSeeInShellOutput('Tests: 1, Assertions: 0, Warnings: 1.');
        // In normal mode the Exception trace should not appear.
        $I->dontSeeInShellOutput('Exception trace');
        $I->seeResultCodeIs(1);
    }


    /**
     * This adds the -v to the stderr test which should just add the Exception Trace to the output.
     *
     * @param CliGuy $I
     * @before moveToPath
     */
    public function runTestWithDataProvidersFailureStderrVerbose(CliGuy $I)
    {
        $I->executeCommand('run -n unit DataProvidersFailureCest -v 2>&1',false);
        $I->seeInShellOutput("Couldn't parse test");
        $I->seeInShellOutput("DataProvider 'rectangle' for DataProvidersFailureCest->testIsTriangle");
        $I->seeInShellOutput('Make sure that the dataprovider exist within the test class.');
        // For Unit tests PHPUnit throws the errors, this confirms that we haven't ended up running PHPUnit test Loader
        $I->dontSeeInShellOutput('PHPUnit_Framework_Warning');
        $I->dontSeeInShellOutput('The data provider specified for DataProvidersFailureCest::testIsTriangle');
        $I->dontSeeInShellOutput('Method rectangle does not exist');
        $I->dontSeeInShellOutput('FAILURES!');
        $I->dontSeeInShellOutput('WARNINGS!');
        $I->dontSeeInShellOutput('OK');
        $I->dontSeeInShellOutput('Tests: 1, Assertions: 0, Warnings: 1.');
        // In verbose mode the Exception trace should be output.
        $I->seeInShellOutput('[Codeception\Exception\TestParseException]');
        $I->seeInShellOutput('Exception trace');
        $I->seeResultCodeIs(1);
    }

    /**
     * This looks at only the contents of stdout when there is an exception thrown when executing a dataProvider
     * function.
     * When exception thrown all the useful information should go to stderr, so stdout is left with nothing.
     *
     * @param CliGuy $I
     * @before moveToPath
     */
    public function runTestWithDataProvidersExceptionStdout(CliGuy $I)
    {
        /**
         * On windows /dev/null is NUL so detect running OS and return the appropriate string for redirection.
         * As some systems have php_uname and co disabled, we use the DIRECTORY_SEPARATOR constant to
         * figure out if we are running on windows or not.
         */
        $devNull = (DIRECTORY_SEPARATOR === '\\')?'NUL':'/dev/null';
        $I->executeCommand('run -n unit DataProvidersExceptionCest -v 2> '.$devNull, false);
        // Depending on the test environment, we either see nothing or just the headers here.
        $I->canSeeShellOutputMatches('/^Codeception PHP Testing Framework v[0-9\.]+\nPowered by PHPUnit .+ by Sebastian Bergmann and contributors\./');
        $I->seeResultCodeIs(1);
    }

    /**
     * This redirects stderr to stdout so that we can test the contents of stderr. Stderr is where all the interesting
     * information should be when there is a failure.
     *
     * @param CliGuy $I
     * @before moveToPath
     */
    public function runTestWithDataProvidersExceptionStderr(CliGuy $I)
    {
        $I->executeCommand('run -n unit DataProvidersExceptionCest 2>&1', false);
        // For Unit tests PHPUnit throws the errors, this confirms that we haven't ended up running PHPUnit test Loader
        $I->dontSeeInShellOutput('There was 1 warning');
        $I->dontSeeInShellOutput('PHPUnit_Framework_Warning');
        $I->dontSeeInShellOutput('The data provider specified for DataProvidersExceptionTest::testIsTriangle');
        $I->dontSeeInShellOutput('FAILURES!');
        $I->dontSeeInShellOutput('WARNINGS!');
        $I->dontSeeInShellOutput('OK');
        // We should not see the messages related to a failure to parse the dataProvider function
        $I->dontSeeInShellOutput('[Codeception\Exception\TestParseException]');
        $I->dontSeeInShellOutput("Couldn't parse test");
        $I->dontSeeInShellOutput("DataProvider 'rectangle' for DataProvidersFailureCest->testIsTriangle ");

        // We should just see the message
        $I->seeInShellOutput('Something went wrong!!!');
        // We don't have the verbose flag set, so there should be no trace.
        $I->dontSeeInShellOutput('Exception trace:');
        $I->seeResultCodeIs(1);
    }

    /**
     * This adds the -v to the stderr test which should just add the Exception Trace to the output of stderr.
     *
     * @param CliGuy $I
     * @before moveToPath
     */
    public function runTestWithDataProvidersExceptionStderrVerbose(CliGuy $I)
    {
        $I->executeCommand('run -n unit DataProvidersExceptionCest -v 2>&1', false);
        // For Unit tests PHPUnit throws the errors, this confirms that we haven't ended up running PHPUnit test Loader
        $I->dontSeeInShellOutput('There was 1 warning');
        $I->dontSeeInShellOutput('PHPUnit_Framework_Warning');
        $I->dontSeeInShellOutput('The data provider specified for DataProvidersExceptionTest::testIsTriangle');
        $I->dontSeeInShellOutput('FAILURES!');
        $I->dontSeeInShellOutput('WARNINGS!');
        $I->dontSeeInShellOutput('OK');
        // We should not see the messages related to a failure to parse the dataProvider function
        $I->dontSeeInShellOutput('[Codeception\Exception\TestParseException]');
        $I->dontSeeInShellOutput("Couldn't parse test");
        $I->dontSeeInShellOutput("DataProvider 'rectangle' for DataProvidersFailureCest->testIsTriangle is ");

        // We should just see the message
        $I->seeInShellOutput('Something went wrong!!!');
        // We have the verbose flag set, so there should be a trace.
        $I->seeInShellOutput('[Exception]');
        $I->seeInShellOutput('Exception trace:');
        $I->seeResultCodeIs(1);
    }
}