How to use method of org.easymock.tests.Util class

Best Easymock code snippet using org.easymock.tests.Util.

Run Easymock automation tests on LambdaTest cloud grid

Perform automation testing on 3000+ real desktop and mobile devices online.

copy
1/*
2 * Copyright (C) 2017 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16package com.android.tradefed.testtype.python;
17
18import static com.android.tradefed.testtype.python.PythonBinaryHostTest.TEST_OUTPUT_FILE_FLAG;
19import static com.android.tradefed.testtype.python.PythonBinaryHostTest.USE_TEST_OUTPUT_FILE_OPTION;
20
21import static com.google.common.truth.Truth.assertThat;
22
23import static org.easymock.EasyMock.anyLong;
24import static org.easymock.EasyMock.anyObject;
25import static org.easymock.EasyMock.capture;
26import static org.easymock.EasyMock.eq;
27import static org.easymock.EasyMock.expect;
28import static org.easymock.EasyMock.getCurrentArguments;
29
30import com.android.tradefed.build.IBuildInfo;
31import com.android.tradefed.config.OptionSetter;
32import com.android.tradefed.device.ITestDevice;
33import com.android.tradefed.device.StubDevice;
34import com.android.tradefed.invoker.ExecutionFiles.FilesKey;
35import com.android.tradefed.invoker.IInvocationContext;
36import com.android.tradefed.invoker.InvocationContext;
37import com.android.tradefed.invoker.TestInformation;
38import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
39import com.android.tradefed.result.FailureDescription;
40import com.android.tradefed.result.ITestInvocationListener;
41import com.android.tradefed.result.InputStreamSource;
42import com.android.tradefed.result.LogDataType;
43import com.android.tradefed.result.TestDescription;
44import com.android.tradefed.result.proto.TestRecordProto.FailureStatus;
45import com.android.tradefed.util.CommandResult;
46import com.android.tradefed.util.CommandStatus;
47import com.android.tradefed.util.FileUtil;
48import com.android.tradefed.util.IRunUtil;
49import com.android.tradefed.util.StreamUtil;
50import com.android.tradefed.util.IRunUtil.EnvPriority;
51
52import com.google.common.io.CharStreams;
53
54import org.easymock.Capture;
55import org.easymock.EasyMock;
56import org.easymock.IAnswer;
57import org.easymock.IArgumentMatcher;
58import org.junit.After;
59import org.junit.Before;
60import org.junit.Test;
61import org.junit.runner.RunWith;
62import org.junit.runners.JUnit4;
63
64import java.io.ByteArrayInputStream;
65import java.io.File;
66import java.io.IOException;
67import java.io.InputStream;
68import java.io.InputStreamReader;
69import java.io.OutputStream;
70import java.util.HashMap;
71
72/** Unit tests for {@link PythonBinaryHostTest}. */
73@RunWith(JUnit4.class)
74public final class PythonBinaryHostTestTest {
75
76    private static final String PYTHON_OUTPUT_FILE_1 = "python_output1.txt";
77
78    private PythonBinaryHostTest mTest;
79    private IRunUtil mMockRunUtil;
80    private IBuildInfo mMockBuildInfo;
81    private ITestDevice mMockDevice;
82    private TestInformation mTestInfo;
83    private ITestInvocationListener mMockListener;
84    private File mFakeAdb;
85    private File mPythonBinary;
86    private File mOutputFile;
87    private File mModuleDir;
88
89    @Before
90    public void setUp() throws Exception {
91        mFakeAdb = FileUtil.createTempFile("adb-python-tests", "");
92        mMockRunUtil = EasyMock.createMock(IRunUtil.class);
93        mMockBuildInfo = EasyMock.createMock(IBuildInfo.class);
94        mMockListener = EasyMock.createMock(ITestInvocationListener.class);
95        mMockDevice = EasyMock.createMock(ITestDevice.class);
96        mTest =
97                new PythonBinaryHostTest() {
98                    @Override
99                    IRunUtil getRunUtil() {
100                        return mMockRunUtil;
101                    }
102
103                    @Override
104                    String getAdbPath() {
105                        return mFakeAdb.getAbsolutePath();
106                    }
107                };
108        IInvocationContext context = new InvocationContext();
109        context.addAllocatedDevice("device", mMockDevice);
110        context.addDeviceBuildInfo("device", mMockBuildInfo);
111        mTestInfo = TestInformation.newBuilder().setInvocationContext(context).build();
112        EasyMock.expect(mMockDevice.getSerialNumber()).andStubReturn("SERIAL");
113        mMockRunUtil.setEnvVariable(PythonBinaryHostTest.ANDROID_SERIAL_VAR, "SERIAL");
114        mMockRunUtil.setWorkingDir(EasyMock.anyObject());
115        mMockRunUtil.setEnvVariablePriority(EnvPriority.SET);
116        mMockRunUtil.setEnvVariable(EasyMock.eq("PATH"), EasyMock.anyObject());
117
118        mModuleDir = FileUtil.createTempDir("python-module");
119        mPythonBinary = FileUtil.createTempFile("python-dir", "", mModuleDir);
120        mTestInfo.executionFiles().put(FilesKey.HOST_TESTS_DIRECTORY, new File("/path-not-exist"));
121    }
122
123    @After
124    public void tearDown() throws Exception {
125        FileUtil.deleteFile(mFakeAdb);
126        FileUtil.deleteFile(mPythonBinary);
127        FileUtil.deleteFile(mOutputFile);
128        FileUtil.recursiveDelete(mModuleDir);
129    }
130
131    /** Test that when running a python binary the output is parsed to obtain results. */
132    @Test
133    public void testRun() throws Exception {
134        mMockListener = EasyMock.createNiceMock(ITestInvocationListener.class);
135        mOutputFile = readInFile(PYTHON_OUTPUT_FILE_1);
136        try {
137            OptionSetter setter = new OptionSetter(mTest);
138            setter.setOptionValue("python-binaries", mPythonBinary.getAbsolutePath());
139            mMockRunUtil.setEnvVariable(EasyMock.eq("LD_LIBRARY_PATH"), EasyMock.anyObject());
140            expectedAdbPath(mFakeAdb);
141
142            CommandResult res = new CommandResult();
143            res.setStatus(CommandStatus.SUCCESS);
144            res.setStdout("python binary stdout.");
145            res.setStderr(FileUtil.readStringFromFile(mOutputFile));
146            EasyMock.expect(
147                            mMockRunUtil.runTimedCmd(
148                                    EasyMock.anyLong(),
149                                    EasyMock.isNull(),
150                                    (OutputStream) EasyMock.anyObject(),
151                                    EasyMock.eq(mPythonBinary.getAbsolutePath())))
152                    .andAnswer(
153                            new IAnswer<CommandResult>() {
154                                @Override
155                                public CommandResult answer() throws Throwable {
156                                    OutputStream stream = (OutputStream) getCurrentArguments()[2];
157                                    StreamUtil.copyFileToStream(mOutputFile, stream);
158                                    return res;
159                                }
160                            });
161            mMockListener.testRunStarted(
162                    EasyMock.eq(mPythonBinary.getName()),
163                    EasyMock.eq(11),
164                    EasyMock.eq(0),
165                    EasyMock.anyLong());
166            mMockListener.testLog(
167                    EasyMock.eq(mPythonBinary.getName() + "-stdout"),
168                    EasyMock.eq(LogDataType.TEXT),
169                    EasyMock.anyObject());
170            mMockListener.testLog(
171                    EasyMock.eq(mPythonBinary.getName() + "-stderr"),
172                    EasyMock.eq(LogDataType.TEXT),
173                    EasyMock.anyObject());
174            EasyMock.expect(mMockDevice.getIDevice()).andReturn(new StubDevice("serial"));
175
176            EasyMock.replay(mMockRunUtil, mMockBuildInfo, mMockListener, mMockDevice);
177            mTest.run(mTestInfo, mMockListener);
178            EasyMock.verify(mMockRunUtil, mMockBuildInfo, mMockListener, mMockDevice);
179        } finally {
180            FileUtil.deleteFile(mPythonBinary);
181        }
182    }
183
184    /** Test that when running a non-unittest python binary with any filter, the test shall fail. */
185    @Test
186    public void testRun_failWithIncludeFilters() throws Exception {
187        mOutputFile = readInFile(PYTHON_OUTPUT_FILE_1);
188        mMockListener = EasyMock.createNiceMock(ITestInvocationListener.class);
189        try {
190            OptionSetter setter = new OptionSetter(mTest);
191            setter.setOptionValue("python-binaries", mPythonBinary.getAbsolutePath());
192            mTest.addIncludeFilter("test1");
193            mMockRunUtil.setEnvVariable(EasyMock.eq("LD_LIBRARY_PATH"), EasyMock.anyObject());
194            expectedAdbPath(mFakeAdb);
195
196            CommandResult res = new CommandResult();
197            res.setStatus(CommandStatus.SUCCESS);
198            res.setStderr(FileUtil.readStringFromFile(mOutputFile));
199            EasyMock.expect(
200                            mMockRunUtil.runTimedCmd(
201                                    EasyMock.anyLong(),
202                                    EasyMock.isNull(),
203                                    (OutputStream) EasyMock.anyObject(),
204                                    EasyMock.eq(mPythonBinary.getAbsolutePath())))
205                    .andAnswer(
206                            new IAnswer<CommandResult>() {
207                                @Override
208                                public CommandResult answer() throws Throwable {
209                                    throw new RuntimeException("Parser error");
210                                }
211                            });
212            mMockListener.testRunStarted(EasyMock.eq(mPythonBinary.getName()), EasyMock.eq(0));
213            mMockListener.testRunFailed((FailureDescription) EasyMock.anyObject());
214            mMockListener.testRunEnded(
215                    EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>anyObject());
216            mMockListener.testLog(
217                    EasyMock.eq(mPythonBinary.getName() + "-stderr"),
218                    EasyMock.eq(LogDataType.TEXT),
219                    EasyMock.anyObject());
220            EasyMock.expect(mMockDevice.getIDevice()).andReturn(new StubDevice("serial"));
221
222            EasyMock.replay(mMockRunUtil, mMockBuildInfo, mMockListener, mMockDevice);
223            mTest.run(mTestInfo, mMockListener);
224            EasyMock.verify(mMockRunUtil, mMockBuildInfo, mMockListener, mMockDevice);
225        } finally {
226            FileUtil.deleteFile(mPythonBinary);
227        }
228    }
229
230    /**
231     * Test that when running a python binary with include filters, the output is parsed to obtain
232     * results.
233     */
234    @Test
235    public void testRun_withIncludeFilters() throws Exception {
236        try {
237            OptionSetter setter = new OptionSetter(mTest);
238            setter.setOptionValue("python-binaries", mPythonBinary.getAbsolutePath());
239            mTest.addIncludeFilter("__main__.Class1#test_1");
240            mMockRunUtil.setEnvVariable(EasyMock.eq("LD_LIBRARY_PATH"), EasyMock.anyObject());
241            expectedAdbPath(mFakeAdb);
242
243            CommandResult res = new CommandResult();
244            res.setStatus(CommandStatus.SUCCESS);
245            String output =
246                    "test_1 (__main__.Class1)\n"
247                        + "run first test. ... ok\n"
248                        + "test_2 (__main__.Class1)\n"
249                        + "run second test. ... ok\n"
250                        + "test_3 (__main__.Class1)\n"
251                        + "run third test. ... ok\n"
252                        + "----------------------------------------------------------------------\n"
253                        + "Ran 3 tests in 1s\n";
254            res.setStderr(output);
255            EasyMock.expect(
256                            mMockRunUtil.runTimedCmd(
257                                    EasyMock.anyLong(),
258                                    EasyMock.isNull(),
259                                    (OutputStream) EasyMock.anyObject(),
260                                    EasyMock.eq(mPythonBinary.getAbsolutePath())))
261                    .andAnswer(
262                            new IAnswer<CommandResult>() {
263                                @Override
264                                public CommandResult answer() throws Throwable {
265                                    OutputStream stream = (OutputStream) getCurrentArguments()[2];
266                                    StreamUtil.copyStreams(
267                                            new ByteArrayInputStream(output.getBytes()), stream);
268                                    return res;
269                                }
270                            });
271            // 3 tests are started and ended.
272            for (int i = 0; i < 3; i++) {
273                mMockListener.testStarted(
274                        EasyMock.<TestDescription>anyObject(), EasyMock.anyLong());
275                mMockListener.testEnded(
276                        EasyMock.<TestDescription>anyObject(),
277                        EasyMock.anyLong(),
278                        EasyMock.<HashMap<String, Metric>>anyObject());
279            }
280            // 2 tests are ignored.
281            mMockListener.testIgnored(EasyMock.<TestDescription>anyObject());
282            mMockListener.testIgnored(EasyMock.<TestDescription>anyObject());
283            mMockListener.testRunStarted(
284                    EasyMock.eq(mPythonBinary.getName()),
285                    EasyMock.eq(3),
286                    EasyMock.eq(0),
287                    EasyMock.anyLong());
288            mMockListener.testRunEnded(
289                    EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>anyObject());
290            mMockListener.testLog(
291                    EasyMock.eq(mPythonBinary.getName() + "-stderr"),
292                    EasyMock.eq(LogDataType.TEXT),
293                    EasyMock.anyObject());
294            EasyMock.expect(mMockDevice.getIDevice()).andReturn(new StubDevice("serial"));
295
296            EasyMock.replay(mMockRunUtil, mMockBuildInfo, mMockListener, mMockDevice);
297            mTest.run(mTestInfo, mMockListener);
298            EasyMock.verify(mMockRunUtil, mMockBuildInfo, mMockListener, mMockDevice);
299        } finally {
300            FileUtil.deleteFile(mPythonBinary);
301        }
302    }
303
304    /**
305     * Test that when running a python binary with exclude filters, the output is parsed to obtain
306     * results.
307     */
308    @Test
309    public void testRun_withExcludeFilters() throws Exception {
310        try {
311            OptionSetter setter = new OptionSetter(mTest);
312            setter.setOptionValue("python-binaries", mPythonBinary.getAbsolutePath());
313            mTest.addExcludeFilter("__main__.Class1#test_1");
314            mMockRunUtil.setEnvVariable(EasyMock.eq("LD_LIBRARY_PATH"), EasyMock.anyObject());
315            expectedAdbPath(mFakeAdb);
316
317            CommandResult res = new CommandResult();
318            res.setStatus(CommandStatus.SUCCESS);
319            String output =
320                    "test_1 (__main__.Class1)\n"
321                        + "run first test. ... ok\n"
322                        + "test_2 (__main__.Class1)\n"
323                        + "run second test. ... ok\n"
324                        + "test_3 (__main__.Class1)\n"
325                        + "run third test. ... ok\n"
326                        + "----------------------------------------------------------------------\n"
327                        + "Ran 3 tests in 1s\n";
328            res.setStderr(output);
329            EasyMock.expect(
330                            mMockRunUtil.runTimedCmd(
331                                    EasyMock.anyLong(),
332                                    EasyMock.isNull(),
333                                    (OutputStream) EasyMock.anyObject(),
334                                    EasyMock.eq(mPythonBinary.getAbsolutePath())))
335                    .andAnswer(
336                            new IAnswer<CommandResult>() {
337                                @Override
338                                public CommandResult answer() throws Throwable {
339                                    OutputStream stream = (OutputStream) getCurrentArguments()[2];
340                                    StreamUtil.copyStreams(
341                                            new ByteArrayInputStream(output.getBytes()), stream);
342                                    return res;
343                                }
344                            });
345            // 3 tests are started and ended.
346            for (int i = 0; i < 3; i++) {
347                mMockListener.testStarted(
348                        EasyMock.<TestDescription>anyObject(), EasyMock.anyLong());
349                mMockListener.testEnded(
350                        EasyMock.<TestDescription>anyObject(),
351                        EasyMock.anyLong(),
352                        EasyMock.<HashMap<String, Metric>>anyObject());
353            }
354            // 1 test is ignored.
355            mMockListener.testIgnored(EasyMock.<TestDescription>anyObject());
356            mMockListener.testRunStarted(
357                    EasyMock.eq(mPythonBinary.getName()),
358                    EasyMock.eq(3),
359                    EasyMock.eq(0),
360                    EasyMock.anyLong());
361            mMockListener.testRunEnded(
362                    EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>anyObject());
363            mMockListener.testLog(
364                    EasyMock.eq(mPythonBinary.getName() + "-stderr"),
365                    EasyMock.eq(LogDataType.TEXT),
366                    EasyMock.anyObject());
367            EasyMock.expect(mMockDevice.getIDevice()).andReturn(new StubDevice("serial"));
368
369            EasyMock.replay(mMockRunUtil, mMockBuildInfo, mMockListener, mMockDevice);
370            mTest.run(mTestInfo, mMockListener);
371            EasyMock.verify(mMockRunUtil, mMockBuildInfo, mMockListener, mMockDevice);
372        } finally {
373            FileUtil.deleteFile(mPythonBinary);
374        }
375    }
376
377    /**
378     * Test running the python tests when an adb path has been set. In that case we ensure the
379     * python script will use the provided adb.
380     */
381    @Test
382    public void testRun_withAdbPath() throws Exception {
383        mMockListener = EasyMock.createNiceMock(ITestInvocationListener.class);
384        mOutputFile = readInFile(PYTHON_OUTPUT_FILE_1);
385        mTestInfo.executionFiles().put(FilesKey.ADB_BINARY, new File("/test/adb"));
386
387        try {
388            OptionSetter setter = new OptionSetter(mTest);
389            setter.setOptionValue("python-binaries", mPythonBinary.getAbsolutePath());
390            mMockRunUtil.setEnvVariable(EasyMock.eq("LD_LIBRARY_PATH"), EasyMock.anyObject());
391            expectedAdbPath(new File("/test/adb"));
392
393            CommandResult res = new CommandResult();
394            res.setStatus(CommandStatus.SUCCESS);
395            res.setStderr(FileUtil.readStringFromFile(mOutputFile));
396            EasyMock.expect(
397                            mMockRunUtil.runTimedCmd(
398                                    EasyMock.anyLong(),
399                                    EasyMock.isNull(),
400                                    (OutputStream) EasyMock.anyObject(),
401                                    EasyMock.eq(mPythonBinary.getAbsolutePath())))
402                    .andAnswer(
403                            new IAnswer<CommandResult>() {
404                                @Override
405                                public CommandResult answer() throws Throwable {
406                                    OutputStream stream = (OutputStream) getCurrentArguments()[2];
407                                    StreamUtil.copyFileToStream(mOutputFile, stream);
408                                    return res;
409                                }
410                            });
411            mMockListener.testRunStarted(
412                    EasyMock.eq(mPythonBinary.getName()),
413                    EasyMock.eq(11),
414                    EasyMock.eq(0),
415                    EasyMock.anyLong());
416            mMockListener.testLog(
417                    EasyMock.eq(mPythonBinary.getName() + "-stderr"),
418                    EasyMock.eq(LogDataType.TEXT),
419                    EasyMock.anyObject());
420            EasyMock.expect(mMockDevice.getIDevice()).andReturn(new StubDevice("serial"));
421
422            EasyMock.replay(mMockRunUtil, mMockBuildInfo, mMockListener, mMockDevice);
423            mTest.run(mTestInfo, mMockListener);
424            EasyMock.verify(mMockRunUtil, mMockBuildInfo, mMockListener, mMockDevice);
425        } finally {
426            FileUtil.deleteFile(mPythonBinary);
427        }
428    }
429
430    /** Test running the python tests when shared lib is available in HOST_TESTS_DIRECTORY. */
431    @Test
432    public void testRun_withSharedLibInHostTestsDir() throws Exception {
433        mMockListener = EasyMock.createNiceMock(ITestInvocationListener.class);
434        mOutputFile = readInFile(PYTHON_OUTPUT_FILE_1);
435        File hostTestsDir = FileUtil.createTempDir("host-test-cases");
436        mTestInfo.executionFiles().put(FilesKey.HOST_TESTS_DIRECTORY, hostTestsDir);
437        File binary = FileUtil.createTempFile("python-dir", "", hostTestsDir);
438        File lib = new File(hostTestsDir, "lib");
439        lib.mkdirs();
440        File lib64 = new File(hostTestsDir, "lib64");
441        lib64.mkdirs();
442
443        try {
444            OptionSetter setter = new OptionSetter(mTest);
445            setter.setOptionValue("python-binaries", binary.getAbsolutePath());
446            mMockRunUtil.setEnvVariable(
447                    PythonBinaryHostTest.LD_LIBRARY_PATH,
448                    lib.getAbsolutePath()
449                            + ":"
450                            + lib64.getAbsolutePath()
451                            + ":"
452                            + hostTestsDir.getAbsolutePath());
453            expectedAdbPath(mFakeAdb);
454
455            CommandResult res = new CommandResult();
456            res.setStatus(CommandStatus.SUCCESS);
457            res.setStderr(FileUtil.readStringFromFile(mOutputFile));
458            EasyMock.expect(
459                            mMockRunUtil.runTimedCmd(
460                                    EasyMock.anyLong(),
461                                    EasyMock.isNull(),
462                                    (OutputStream) EasyMock.anyObject(),
463                                    EasyMock.eq(binary.getAbsolutePath())))
464                    .andAnswer(
465                            new IAnswer<CommandResult>() {
466                                @Override
467                                public CommandResult answer() throws Throwable {
468                                    OutputStream stream = (OutputStream) getCurrentArguments()[2];
469                                    StreamUtil.copyFileToStream(mOutputFile, stream);
470                                    return res;
471                                }
472                            });
473            mMockListener.testRunStarted(
474                    EasyMock.eq(binary.getName()),
475                    EasyMock.eq(11),
476                    EasyMock.eq(0),
477                    EasyMock.anyLong());
478            mMockListener.testLog(
479                    EasyMock.eq(binary.getName() + "-stderr"),
480                    EasyMock.eq(LogDataType.TEXT),
481                    EasyMock.anyObject());
482            EasyMock.expect(mMockDevice.getIDevice()).andReturn(new StubDevice("serial"));
483
484            EasyMock.replay(mMockRunUtil, mMockBuildInfo, mMockListener, mMockDevice);
485            mTest.run(mTestInfo, mMockListener);
486            EasyMock.verify(mMockRunUtil, mMockBuildInfo, mMockListener, mMockDevice);
487        } finally {
488            FileUtil.recursiveDelete(hostTestsDir);
489        }
490    }
491
492    /** Test running the python tests when shared lib is available in TESTS_DIRECTORY. */
493    @Test
494    public void testRun_withSharedLib() throws Exception {
495        mMockListener = EasyMock.createNiceMock(ITestInvocationListener.class);
496        mOutputFile = readInFile(PYTHON_OUTPUT_FILE_1);
497        File testsDir = FileUtil.createTempDir("host-test-cases");
498        mTestInfo.executionFiles().put(FilesKey.TESTS_DIRECTORY, testsDir);
499        File binary = FileUtil.createTempFile("python-dir", "", testsDir);
500        File lib = new File(testsDir, "lib");
501        lib.mkdirs();
502        File lib64 = new File(testsDir, "lib64");
503        lib64.mkdirs();
504
505        try {
506            OptionSetter setter = new OptionSetter(mTest);
507            setter.setOptionValue("python-binaries", binary.getAbsolutePath());
508            mMockRunUtil.setEnvVariable(
509                    PythonBinaryHostTest.LD_LIBRARY_PATH,
510                    lib.getAbsolutePath()
511                            + ":"
512                            + lib64.getAbsolutePath()
513                            + ":"
514                            + testsDir.getAbsolutePath());
515            expectedAdbPath(mFakeAdb);
516
517            CommandResult res = new CommandResult();
518            res.setStatus(CommandStatus.SUCCESS);
519            res.setStderr(FileUtil.readStringFromFile(mOutputFile));
520            EasyMock.expect(
521                            mMockRunUtil.runTimedCmd(
522                                    EasyMock.anyLong(),
523                                    EasyMock.isNull(),
524                                    (OutputStream) EasyMock.anyObject(),
525                                    EasyMock.eq(binary.getAbsolutePath())))
526                    .andAnswer(
527                            new IAnswer<CommandResult>() {
528                                @Override
529                                public CommandResult answer() throws Throwable {
530                                    OutputStream stream = (OutputStream) getCurrentArguments()[2];
531                                    StreamUtil.copyFileToStream(mOutputFile, stream);
532                                    return res;
533                                }
534                            });
535            mMockListener.testRunStarted(
536                    EasyMock.eq(binary.getName()),
537                    EasyMock.eq(11),
538                    EasyMock.eq(0),
539                    EasyMock.anyLong());
540            mMockListener.testLog(
541                    EasyMock.eq(binary.getName() + "-stderr"),
542                    EasyMock.eq(LogDataType.TEXT),
543                    EasyMock.anyObject());
544            EasyMock.expect(mMockDevice.getIDevice()).andReturn(new StubDevice("serial"));
545
546            EasyMock.replay(mMockRunUtil, mMockBuildInfo, mMockListener, mMockDevice);
547            mTest.run(mTestInfo, mMockListener);
548            EasyMock.verify(mMockRunUtil, mMockBuildInfo, mMockListener, mMockDevice);
549        } finally {
550            FileUtil.recursiveDelete(testsDir);
551        }
552    }
553
554    /**
555     * If the binary returns an exception status, we should throw a runtime exception since
556     * something went wrong with the binary setup.
557     */
558    @Test
559    public void testRunFail_exception() throws Exception {
560        try {
561            OptionSetter setter = new OptionSetter(mTest);
562            setter.setOptionValue("python-binaries", mPythonBinary.getAbsolutePath());
563            mMockRunUtil.setEnvVariable(EasyMock.eq("LD_LIBRARY_PATH"), EasyMock.anyObject());
564            expectedAdbPath(mFakeAdb);
565
566            CommandResult res = new CommandResult();
567            res.setStatus(CommandStatus.EXCEPTION);
568            res.setStderr("Could not execute.");
569            String output = "Could not execute.";
570            res.setStderr(output);
571            EasyMock.expect(
572                            mMockRunUtil.runTimedCmd(
573                                    EasyMock.anyLong(),
574                                    EasyMock.isNull(),
575                                    (OutputStream) EasyMock.anyObject(),
576                                    EasyMock.eq(mPythonBinary.getAbsolutePath())))
577                    .andAnswer(
578                            new IAnswer<CommandResult>() {
579                                @Override
580                                public CommandResult answer() throws Throwable {
581                                    OutputStream stream = (OutputStream) getCurrentArguments()[2];
582                                    StreamUtil.copyStreams(
583                                            new ByteArrayInputStream(output.getBytes()), stream);
584                                    return res;
585                                }
586                            });
587            EasyMock.expect(mMockDevice.getIDevice()).andReturn(new StubDevice("serial"));
588
589            mMockListener.testLog(
590                    EasyMock.eq(mPythonBinary.getName() + "-stderr"),
591                    EasyMock.eq(LogDataType.TEXT),
592                    EasyMock.anyObject());
593            // Report a failure if we cannot parse the logs
594            mMockListener.testRunStarted(mPythonBinary.getName(), 0);
595            FailureDescription failure =
596                    FailureDescription.create(
597                            "Failed to parse the python logs: Parser finished in unexpected "
598                                    + "state TEST_CASE. Please ensure that verbosity of output "
599                                    + "is high enough to be parsed.");
600            failure.setFailureStatus(FailureStatus.TEST_FAILURE);
601            mMockListener.testRunFailed(failure);
602            mMockListener.testRunEnded(
603                    EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>anyObject());
604
605            EasyMock.replay(mMockRunUtil, mMockBuildInfo, mMockListener, mMockDevice);
606            mTest.run(mTestInfo, mMockListener);
607            EasyMock.verify(mMockRunUtil, mMockBuildInfo, mMockListener, mMockDevice);
608        } finally {
609            FileUtil.deleteFile(mPythonBinary);
610        }
611    }
612
613    /**
614     * If the binary reports a FAILED status but the output actually have some tests, it most *
615     * likely means that some tests failed. So we simply continue with parsing the results.
616     */
617    @Test
618    public void testRunFail_failureOnly() throws Exception {
619        mMockListener = EasyMock.createNiceMock(ITestInvocationListener.class);
620        mOutputFile = readInFile(PYTHON_OUTPUT_FILE_1);
621        try {
622            OptionSetter setter = new OptionSetter(mTest);
623            setter.setOptionValue("python-binaries", mPythonBinary.getAbsolutePath());
624            mMockRunUtil.setEnvVariable(EasyMock.eq("LD_LIBRARY_PATH"), EasyMock.anyObject());
625            expectedAdbPath(mFakeAdb);
626
627            CommandResult res = new CommandResult();
628            res.setStatus(CommandStatus.FAILED);
629            res.setStderr(FileUtil.readStringFromFile(mOutputFile));
630            EasyMock.expect(
631                            mMockRunUtil.runTimedCmd(
632                                    EasyMock.anyLong(),
633                                    EasyMock.isNull(),
634                                    (OutputStream) EasyMock.anyObject(),
635                                    EasyMock.eq(mPythonBinary.getAbsolutePath())))
636                    .andAnswer(
637                            new IAnswer<CommandResult>() {
638                                @Override
639                                public CommandResult answer() throws Throwable {
640                                    OutputStream stream = (OutputStream) getCurrentArguments()[2];
641                                    StreamUtil.copyFileToStream(mOutputFile, stream);
642                                    return res;
643                                }
644                            });
645            mMockListener.testRunStarted(
646                    EasyMock.eq(mPythonBinary.getName()),
647                    EasyMock.eq(11),
648                    EasyMock.eq(0),
649                    EasyMock.anyLong());
650            mMockListener.testLog(
651                    EasyMock.eq(mPythonBinary.getName() + "-stderr"),
652                    EasyMock.eq(LogDataType.TEXT),
653                    EasyMock.anyObject());
654            EasyMock.expect(mMockDevice.getIDevice()).andReturn(new StubDevice("serial"));
655            EasyMock.replay(mMockRunUtil, mMockBuildInfo, mMockListener, mMockDevice);
656            mTest.run(mTestInfo, mMockListener);
657            EasyMock.verify(mMockRunUtil, mMockBuildInfo, mMockListener, mMockDevice);
658        } finally {
659            FileUtil.deleteFile(mPythonBinary);
660        }
661    }
662
663    @Test
664    public void testRun_useTestOutputFileOptionSet_parsesSubprocessOutputFile() throws Exception {
665        mMockRunUtil.setEnvVariable(EasyMock.eq("LD_LIBRARY_PATH"), EasyMock.anyObject());
666        mMockListener = EasyMock.createNiceMock(ITestInvocationListener.class);
667        mOutputFile = readInFile(PYTHON_OUTPUT_FILE_1);
668        newDefaultOptionSetter(mTest).setOptionValue(USE_TEST_OUTPUT_FILE_OPTION, "true");
669        expectRunThatWritesTestOutputFile(
670                newCommandResult(CommandStatus.SUCCESS, "NOT TEST OUTPUT"),
671                FileUtil.readStringFromFile(mOutputFile));
672        mMockListener.testRunStarted(anyObject(), eq(11), eq(0), anyLong());
673        replayAllMocks();
674
675        mTest.run(mTestInfo, mMockListener);
676
677        EasyMock.verify(mMockListener);
678    }
679
680    @Test
681    public void testRun_useTestOutputFileOptionSet_parsesUnitTestOutputFile() throws Exception {
682        mMockRunUtil.setEnvVariable(EasyMock.eq("LD_LIBRARY_PATH"), EasyMock.anyObject());
683        mMockListener = EasyMock.createNiceMock(ITestInvocationListener.class);
684        newDefaultOptionSetter(mTest).setOptionValue(USE_TEST_OUTPUT_FILE_OPTION, "true");
685        expectRunThatWritesTestOutputFile(
686                newCommandResult(CommandStatus.SUCCESS, "NOT TEST OUTPUT"),
687                "test_1 (__main__.Class1)\n"
688                        + "run first test. ... ok\n"
689                        + "test_2 (__main__.Class1)\n"
690                        + "run second test. ... ok\n"
691                        + "----------------------------------------------------------------------\n"
692                        + "Ran 2 tests in 1s");
693        mMockListener.testRunStarted(anyObject(), eq(2), eq(0), anyLong());
694        replayAllMocks();
695
696        mTest.run(mTestInfo, mMockListener);
697
698        EasyMock.verify(mMockListener);
699    }
700
701    @Test
702    public void testRun_useTestOutputFileOptionSet_logsErrorOutput() throws Exception {
703        mMockRunUtil.setEnvVariable(EasyMock.eq("LD_LIBRARY_PATH"), EasyMock.anyObject());
704        mMockListener = EasyMock.createNiceMock(ITestInvocationListener.class);
705        String errorOutput = "NOT TEST OUTPUT";
706        newDefaultOptionSetter(mTest).setOptionValue(USE_TEST_OUTPUT_FILE_OPTION, "true");
707        expectRunThatWritesTestOutputFile(
708                newCommandResult(CommandStatus.SUCCESS, errorOutput),
709                "TEST_RUN_STARTED {\"testCount\": 5, \"runName\": \"TestSuite\"}");
710        mMockListener.testLog(
711                anyObject(), eq(LogDataType.TEXT), inputStreamSourceContainsText(errorOutput));
712        replayAllMocks();
713
714        mTest.run(mTestInfo, mMockListener);
715
716        EasyMock.verify(mMockListener);
717    }
718
719    @Test
720    public void testRun_useTestOutputFileOptionSet_logsTestOutput() throws Exception {
721        mMockRunUtil.setEnvVariable(EasyMock.eq("LD_LIBRARY_PATH"), EasyMock.anyObject());
722        mMockListener = EasyMock.createNiceMock(ITestInvocationListener.class);
723        String testOutput = "TEST_RUN_STARTED {\"testCount\": 5, \"runName\": \"TestSuite\"}";
724        newDefaultOptionSetter(mTest).setOptionValue(USE_TEST_OUTPUT_FILE_OPTION, "true");
725        expectRunThatWritesTestOutputFile(
726                newCommandResult(CommandStatus.SUCCESS, "NOT TEST OUTPUT"), testOutput);
727        mMockListener.testLog(
728                EasyMock.eq(mPythonBinary.getName() + "-stderr"),
729                EasyMock.eq(LogDataType.TEXT),
730                EasyMock.anyObject());
731        mMockListener.testLog(
732                anyObject(), eq(LogDataType.TEXT), inputStreamSourceContainsText(testOutput));
733        replayAllMocks();
734
735        mTest.run(mTestInfo, mMockListener);
736
737        EasyMock.verify(mMockListener);
738    }
739
740    @Test
741    public void testRun_useTestOutputFileOptionSet_failureMessageContainsHints() throws Exception {
742        mMockRunUtil.setEnvVariable(EasyMock.eq("LD_LIBRARY_PATH"), EasyMock.anyObject());
743        mMockListener = EasyMock.createNiceMock(ITestInvocationListener.class);
744        newDefaultOptionSetter(mTest).setOptionValue(USE_TEST_OUTPUT_FILE_OPTION, "true");
745        expectRunThatWritesTestOutputFile(
746                newCommandResult(CommandStatus.SUCCESS, "NOT TEST OUTPUT"), "BAD OUTPUT FORMAT");
747        Capture<FailureDescription> description = new Capture<>();
748        mMockListener.testRunFailed(capture(description));
749        replayAllMocks();
750
751        mTest.run(mTestInfo, mMockListener);
752
753        String message = description.getValue().getErrorMessage();
754        EasyMock.verify(mMockListener);
755        assertThat(message).contains("--" + TEST_OUTPUT_FILE_FLAG);
756        assertThat(message).contains("verbosity");
757    }
758
759    private OptionSetter newDefaultOptionSetter(PythonBinaryHostTest test) throws Exception {
760        OptionSetter setter = new OptionSetter(test);
761        setter.setOptionValue("python-binaries", mPythonBinary.getAbsolutePath());
762        return setter;
763    }
764
765    private static CommandResult newCommandResult(CommandStatus status, String stderr) {
766        CommandResult res = new CommandResult();
767        res.setStatus(status);
768        res.setStderr(stderr);
769        return res;
770    }
771
772    private void expectRunThatWritesTestOutputFile(
773            CommandResult result, String testOutputFileContents) {
774        Capture<String> testOutputFilePath = new Capture<>();
775        expect(
776                        mMockRunUtil.runTimedCmd(
777                                anyLong(),
778                                eq(mPythonBinary.getAbsolutePath()),
779                                eq("--test-output-file"),
780                                capture(testOutputFilePath)))
781                .andStubAnswer(
782                        new IAnswer<CommandResult>() {
783                            @Override
784                            public CommandResult answer() {
785                                try {
786                                    FileUtil.writeToFile(
787                                            testOutputFileContents,
788                                            new File(testOutputFilePath.getValue()));
789                                } catch (IOException ex) {
790                                    throw new RuntimeException(ex);
791                                }
792                                return result;
793                            }
794                        });
795        expect(mMockDevice.getIDevice()).andReturn(new StubDevice("serial"));
796        expectedAdbPath(mFakeAdb);
797    }
798
799    private void replayAllMocks() {
800        EasyMock.replay(mMockRunUtil, mMockBuildInfo, mMockListener, mMockDevice);
801    }
802
803    private void expectedAdbPath(File adbPath) {
804        CommandResult pathRes = new CommandResult();
805        pathRes.setStatus(CommandStatus.SUCCESS);
806        pathRes.setStdout("bin/");
807        EasyMock.expect(mMockRunUtil.runTimedCmd(60000L, "/bin/bash", "-c", "echo $PATH"))
808                .andReturn(pathRes);
809        mMockRunUtil.setEnvVariable("PATH", String.format("%s:bin/", adbPath.getParent()));
810
811        CommandResult versionRes = new CommandResult();
812        versionRes.setStatus(CommandStatus.SUCCESS);
813        versionRes.setStdout("bin/");
814        EasyMock.expect(mMockRunUtil.runTimedCmd(60000L, "adb", "version")).andReturn(versionRes);
815    }
816
817    private static InputStreamSource inputStreamSourceContainsText(String text) {
818        EasyMock.reportMatcher(
819                new IArgumentMatcher() {
820                    @Override
821                    public boolean matches(Object actual) {
822                        if (!(actual instanceof InputStreamSource)) {
823                            return false;
824                        }
825
826                        InputStream is = ((InputStreamSource) actual).createInputStream();
827                        String contents;
828
829                        try {
830                            contents =
831                                    CharStreams.toString(
832                                            new InputStreamReader(is)); // Assumes default charset.
833                        } catch (IOException ex) {
834                            throw new RuntimeException(ex);
835                        }
836
837                        return contents.contains(text);
838                    }
839
840                    @Override
841                    public void appendTo(StringBuffer buffer) {
842                        buffer.append("inputStreamSourceContainsText(\"");
843                        buffer.append(text);
844                        buffer.append("\")");
845                    }
846                });
847        return null;
848    }
849
850    private File readInFile(String filename) throws IOException {
851        File output = FileUtil.createTempFile("python-host-test", ".txt");
852        InputStream stream =
853                getClass()
854                        .getResourceAsStream(
855                                File.separator + "testtype" + File.separator + filename);
856        FileUtil.writeToFile(stream, output);
857        return output;
858    }
859}
860
Full Screen
copy
1/*
2 * Copyright (C) 2019 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16package com.android.tradefed.testtype.rust;
17
18import com.android.tradefed.build.IBuildInfo;
19import com.android.tradefed.config.OptionSetter;
20import com.android.tradefed.invoker.InvocationContext;
21import com.android.tradefed.invoker.TestInformation;
22import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
23import com.android.tradefed.result.FailureDescription;
24import com.android.tradefed.result.ITestInvocationListener;
25import com.android.tradefed.result.LogDataType;
26import com.android.tradefed.result.TestDescription;
27import com.android.tradefed.util.CommandResult;
28import com.android.tradefed.util.CommandStatus;
29import com.android.tradefed.util.FileUtil;
30import com.android.tradefed.util.IRunUtil;
31
32import org.easymock.EasyMock;
33import org.junit.Before;
34import org.junit.Test;
35import org.junit.runner.RunWith;
36import org.junit.runners.JUnit4;
37
38import java.io.File;
39import java.util.HashMap;
40
41/** Unit tests for {@link RustBinaryHostTest}. */
42@RunWith(JUnit4.class)
43public class RustBinaryHostTestTest {
44    private RustBinaryHostTest mTest;
45    private IRunUtil mMockRunUtil;
46    private IBuildInfo mMockBuildInfo;
47    private TestInformation mTestInfo;
48    private ITestInvocationListener mMockListener;
49    // TODO(chh): maybe we will need mFakeAdb later like PythonBinaryHostTestTest.
50
51    @Before
52    public void setUp() throws Exception {
53        mMockRunUtil = EasyMock.createMock(IRunUtil.class);
54        mMockBuildInfo = EasyMock.createMock(IBuildInfo.class);
55        mMockListener = EasyMock.createMock(ITestInvocationListener.class);
56        mTest =
57                new RustBinaryHostTest() {
58                    @Override
59                    IRunUtil getRunUtil() {
60                        return mMockRunUtil;
61                    }
62                };
63        mTest.setBuild(mMockBuildInfo);
64        InvocationContext context = new InvocationContext();
65        context.addDeviceBuildInfo("device", mMockBuildInfo);
66        mTestInfo = TestInformation.newBuilder().setInvocationContext(context).build();
67    }
68
69    private CommandResult newCommandResult(CommandStatus status, String stderr, String stdout) {
70        CommandResult res = new CommandResult();
71        res.setStatus(status);
72        res.setStderr(stderr);
73        res.setStdout(stdout);
74        return res;
75    }
76
77    private String resultCount(int pass, int fail, int ignore) {
78        return "running 2 tests\ntest result: ok. "
79                + pass
80                + " passed; "
81                + fail
82                + " failed; "
83                + ignore
84                + " ignored;";
85    }
86
87    private CommandResult successResult(String stderr, String stdout) throws Exception {
88        return newCommandResult(CommandStatus.SUCCESS, stderr, stdout);
89    }
90
91    // shared with RustBinaryTestTest
92    static String runListOutput(int numTests) {
93        String listOutput = "";
94        for (int i = 1; i <= numTests; i++) {
95            listOutput += "test_case_" + i + ": test\n";
96        }
97        return listOutput + numTests + " tests, 0 benchmarks";
98    }
99
100    // shared with RustBinaryTestTest
101    static String runListOutput(String[] tests) {
102        String listOutput = "";
103        for (String name : tests) {
104            listOutput += name + ": test\n";
105        }
106        return listOutput + tests.length + " tests, 0 benchmarks";
107    }
108
109    // shared with RustBinaryTestTest
110    static String runListBenchmarksOutput(int numTests) {
111        String listOutput = "";
112        for (int i = 1; i <= numTests; i++) {
113            listOutput += "test_case_" + i + ": bench\n";
114        }
115        return listOutput;
116    }
117
118    /** Add mocked call "binary --list" to count the number of tests. */
119    private void mockCountTests(File binary, int numOfTest) throws Exception {
120        EasyMock.expect(
121                        mMockRunUtil.runTimedCmdSilently(
122                                EasyMock.anyLong(),
123                                EasyMock.eq(binary.getAbsolutePath()),
124                                EasyMock.eq("--list")))
125                .andReturn(successResult("", runListOutput(numOfTest)));
126    }
127
128    private void mockCountBenchmarks(File binary, int numOfTest) throws Exception {
129        EasyMock.expect(
130                        mMockRunUtil.runTimedCmdSilently(
131                                EasyMock.anyLong(),
132                                EasyMock.eq(binary.getAbsolutePath()),
133                                EasyMock.eq("--bench"),
134                                EasyMock.eq("--list")))
135                .andReturn(successResult("", runListBenchmarksOutput(numOfTest)));
136    }
137
138    /** Add mocked testRunStarted call to the listener. */
139    private void mockListenerStarted(File binary, int count) throws Exception {
140        mMockListener.testRunStarted(
141                EasyMock.eq(binary.getName()),
142                EasyMock.eq(count),
143                EasyMock.anyInt(),
144                EasyMock.anyLong());
145    }
146
147    /** Add mocked call to check listener log file. */
148    private void mockListenerLog(File binary, boolean error) {
149        if (error) {
150            mMockListener.testLog(
151                    EasyMock.eq(binary.getName() + "-stderr"),
152                    EasyMock.eq(LogDataType.TEXT),
153                    EasyMock.anyObject());
154        }
155        mMockListener.testLog(
156                EasyMock.eq(binary.getName() + "-stdout"),
157                EasyMock.eq(LogDataType.TEXT),
158                EasyMock.anyObject());
159    }
160
161    private void mockTestRunExpect(File binary, CommandResult res) throws Exception {
162        EasyMock.expect(
163                        mMockRunUtil.runTimedCmd(
164                                EasyMock.anyLong(), EasyMock.eq(binary.getAbsolutePath())))
165                .andReturn(res);
166    }
167
168    private void mockBenchmarkRunExpect(File binary, String output) throws Exception {
169        CommandResult res = newCommandResult(CommandStatus.SUCCESS, "", "");
170        EasyMock.expect(
171                        mMockRunUtil.runTimedCmd(
172                                EasyMock.anyLong(),
173                                EasyMock.eq(binary.getAbsolutePath()),
174                                EasyMock.eq("--bench"),
175                                EasyMock.eq("--color"),
176                                EasyMock.eq("never")))
177                .andReturn(successResult("", output));
178    }
179
180    /** Add mocked call to testRunEnded. */
181    private void mockTestRunEnded() {
182        mMockListener.testRunEnded(
183                EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>anyObject());
184    }
185
186    /** Call replay/run/verify. */
187    private void callReplayRunVerify() throws Exception {
188        EasyMock.replay(mMockRunUtil, mMockBuildInfo, mMockListener);
189        mTest.run(mTestInfo, mMockListener);
190        EasyMock.verify(mMockRunUtil, mMockBuildInfo, mMockListener);
191    }
192
193    /** Test that when running a rust binary the output is parsed to obtain results. */
194    @Test
195    public void testRun() throws Exception {
196        File binary = FileUtil.createTempFile("rust-dir", "");
197        try {
198            OptionSetter setter = new OptionSetter(mTest);
199            setter.setOptionValue("test-file", binary.getAbsolutePath());
200            mockCountTests(binary, 9);
201            mockListenerStarted(binary, 9);
202            mockListenerLog(binary, false);
203            CommandResult res = successResult("", resultCount(6, 1, 2));
204            mockTestRunExpect(binary, res);
205            mMockListener.testRunFailed("Test run incomplete. Started 2 tests, finished 0");
206            mockTestRunEnded();
207            callReplayRunVerify();
208        } finally {
209            FileUtil.deleteFile(binary);
210        }
211    }
212
213    /**
214     * Test running the rust tests when an adb path has been set. In that case we ensure the rust
215     * test will use the provided adb.
216     */
217    @Test
218    public void testRun_withAdbPath() throws Exception {
219        mMockBuildInfo = EasyMock.createMock(IBuildInfo.class);
220        mTest.setBuild(mMockBuildInfo);
221
222        File binary = FileUtil.createTempFile("rust-dir", "");
223        try {
224            OptionSetter setter = new OptionSetter(mTest);
225            setter.setOptionValue("test-file", binary.getAbsolutePath());
226            mockCountTests(binary, 9);
227            mockListenerStarted(binary, 9);
228            mockListenerLog(binary, false);
229            CommandResult res = successResult("", resultCount(6, 1, 2));
230            mockTestRunExpect(binary, res);
231            mMockListener.testRunFailed("Test run incomplete. Started 2 tests, finished 0");
232            mockTestRunEnded();
233            callReplayRunVerify();
234        } finally {
235            FileUtil.deleteFile(binary);
236        }
237    }
238
239    /** If the binary returns an exception status, it is treated as a failed test. */
240    @Test
241    public void testRunFail_exception() throws Exception {
242        File binary = FileUtil.createTempFile("rust-dir", "");
243        try {
244            OptionSetter setter = new OptionSetter(mTest);
245            setter.setOptionValue("test-file", binary.getAbsolutePath());
246            mockCountTests(binary, 2);
247            mockListenerStarted(binary, 2);
248            mockListenerLog(binary, true);
249            CommandResult res =
250                    newCommandResult(
251                            CommandStatus.EXCEPTION, "Err.", "running 2 tests\nException.");
252            mockTestRunExpect(binary, res);
253            mMockListener.testRunFailed("Test run incomplete. Started 2 tests, finished 0");
254            mMockListener.testRunFailed((FailureDescription) EasyMock.anyObject());
255            mockTestRunEnded();
256            callReplayRunVerify();
257        } finally {
258            FileUtil.deleteFile(binary);
259        }
260    }
261
262    /**
263     * If the binary reports a FAILED status when trying to count tests, it is treated as a failed
264     * test.
265     */
266    @Test
267    public void testRunFail_list() throws Exception {
268        File binary = FileUtil.createTempFile("rust-dir", "");
269        try {
270            OptionSetter setter = new OptionSetter(mTest);
271            setter.setOptionValue("test-file", binary.getAbsolutePath());
272            CommandResult listRes = newCommandResult(CommandStatus.FAILED, "", "");
273            EasyMock.expect(
274                            mMockRunUtil.runTimedCmdSilently(
275                                    EasyMock.anyLong(),
276                                    EasyMock.eq(binary.getAbsolutePath()),
277                                    EasyMock.eq("--list")))
278                    .andReturn(listRes);
279            mMockListener.testRunStarted(binary.getName(), 0);
280            mMockListener.testRunFailed((FailureDescription) EasyMock.anyObject());
281            mockTestRunEnded();
282            callReplayRunVerify();
283        } finally {
284            FileUtil.deleteFile(binary);
285        }
286    }
287
288    /** If the binary reports a FAILED status, it is treated as a failed test. */
289    @Test
290    public void testRunFail_failureOnly() throws Exception {
291        File binary = FileUtil.createTempFile("rust-dir", "");
292        try {
293            OptionSetter setter = new OptionSetter(mTest);
294            setter.setOptionValue("test-file", binary.getAbsolutePath());
295            mockCountTests(binary, 9);
296            mockListenerStarted(binary, 9);
297            mockListenerLog(binary, false);
298            CommandResult res = newCommandResult(CommandStatus.FAILED, "", resultCount(6, 1, 2));
299            mockTestRunExpect(binary, res);
300            mMockListener.testRunFailed("Test run incomplete. Started 2 tests, finished 0");
301            mMockListener.testRunFailed((FailureDescription) EasyMock.anyObject());
302            mockTestRunEnded();
303            callReplayRunVerify();
304        } finally {
305            FileUtil.deleteFile(binary);
306        }
307    }
308
309    /** Test the exclude filtering of test methods. */
310    @Test
311    public void testExcludeFilter() throws Exception {
312        File binary = FileUtil.createTempFile("rust-dir", "");
313        try {
314            OptionSetter setter = new OptionSetter(mTest);
315            setter.setOptionValue("test-file", binary.getAbsolutePath());
316            setter.setOptionValue("exclude-filter", "NotMe");
317            setter.setOptionValue("exclude-filter", "Long");
318            EasyMock.expect(
319                            mMockRunUtil.runTimedCmdSilently(
320                                    EasyMock.anyLong(),
321                                    EasyMock.eq(binary.getAbsolutePath()),
322                                    EasyMock.eq("--skip"),
323                                    EasyMock.eq("NotMe"),
324                                    EasyMock.eq("--skip"),
325                                    EasyMock.eq("Long"),
326                                    EasyMock.eq("--list")))
327                    .andReturn(successResult("", runListOutput(9)));
328            mockListenerStarted(binary, 9);
329            mockListenerLog(binary, false);
330            CommandResult res = successResult("", resultCount(6, 1, 2));
331            EasyMock.expect(
332                            mMockRunUtil.runTimedCmd(
333                                    EasyMock.anyLong(),
334                                    EasyMock.eq(binary.getAbsolutePath()),
335                                    EasyMock.eq("--skip"),
336                                    EasyMock.eq("NotMe"),
337                                    EasyMock.eq("--skip"),
338                                    EasyMock.eq("Long")))
339                    .andReturn(res);
340
341            mMockListener.testRunFailed("Test run incomplete. Started 2 tests, finished 0");
342            mockTestRunEnded();
343            callReplayRunVerify();
344        } finally {
345            FileUtil.deleteFile(binary);
346        }
347    }
348
349    /** Test both include and exclude filters. */
350    @Test
351    public void testIncludeExcludeFilter() throws Exception {
352        File binary = FileUtil.createTempFile("rust-dir", "");
353        try {
354            OptionSetter setter = new OptionSetter(mTest);
355            setter.setOptionValue("test-file", binary.getAbsolutePath());
356            setter.setOptionValue("exclude-filter", "MyTest#NotMe");
357            setter.setOptionValue("include-filter", "MyTest#OnlyMe");
358            setter.setOptionValue("exclude-filter", "Other");
359            // We always pass the include-filter before exclude-filter strings.
360            EasyMock.expect(
361                            mMockRunUtil.runTimedCmdSilently(
362                                    EasyMock.anyLong(),
363                                    EasyMock.eq(binary.getAbsolutePath()),
364                                    EasyMock.eq("OnlyMe"),
365                                    EasyMock.eq("--skip"),
366                                    EasyMock.eq("NotMe"),
367                                    EasyMock.eq("--skip"),
368                                    EasyMock.eq("Other"),
369                                    EasyMock.eq("--list")))
370                    .andReturn(successResult("", runListOutput(3)));
371            mockListenerStarted(binary, 3);
372
373            mockListenerLog(binary, false);
374            CommandResult res = successResult("", resultCount(3, 0, 0));
375            EasyMock.expect(
376                            mMockRunUtil.runTimedCmd(
377                                    EasyMock.anyLong(),
378                                    EasyMock.eq(binary.getAbsolutePath()),
379                                    EasyMock.eq("OnlyMe"),
380                                    EasyMock.eq("--skip"),
381                                    EasyMock.eq("NotMe"),
382                                    EasyMock.eq("--skip"),
383                                    EasyMock.eq("Other")))
384                    .andReturn(res);
385
386            mMockListener.testRunFailed("Test run incomplete. Started 2 tests, finished 0");
387            mockTestRunEnded();
388            callReplayRunVerify();
389        } finally {
390            FileUtil.deleteFile(binary);
391        }
392    }
393
394    /** Test multiple include and exclude filters. */
395    @Test
396    public void testMultipleIncludeExcludeFilter() throws Exception {
397        File binary = FileUtil.createTempFile("rust-dir", "");
398        try {
399            OptionSetter setter = new OptionSetter(mTest);
400            setter.setOptionValue("test-file", binary.getAbsolutePath());
401            setter.setOptionValue("exclude-filter", "NotMe");
402            setter.setOptionValue("include-filter", "MyTest#OnlyMe");
403            setter.setOptionValue("exclude-filter", "MyTest#Other");
404            setter.setOptionValue("include-filter", "Me2");
405            // Multiple include filters are run one by one with --list.
406            String[] selection1 = new String[] {"test1", "test2"};
407            EasyMock.expect(
408                            mMockRunUtil.runTimedCmdSilently(
409                                    EasyMock.anyLong(),
410                                    EasyMock.eq(binary.getAbsolutePath()),
411                                    EasyMock.eq("OnlyMe"),
412                                    EasyMock.eq("--skip"),
413                                    EasyMock.eq("NotMe"),
414                                    EasyMock.eq("--skip"),
415                                    EasyMock.eq("Other"),
416                                    EasyMock.eq("--list")))
417                    .andReturn(successResult("", runListOutput(selection1)));
418            String[] selection2 = new String[] {"test2", "test3", "test4"};
419            EasyMock.expect(
420                            mMockRunUtil.runTimedCmdSilently(
421                                    EasyMock.anyLong(),
422                                    EasyMock.eq(binary.getAbsolutePath()),
423                                    EasyMock.eq("Me2"),
424                                    EasyMock.eq("--skip"),
425                                    EasyMock.eq("NotMe"),
426                                    EasyMock.eq("--skip"),
427                                    EasyMock.eq("Other"),
428                                    EasyMock.eq("--list")))
429                    .andReturn(successResult("", runListOutput(selection2)));
430            // Union of selection1 and selection2 has 4 tests.
431            mockListenerStarted(binary, 4);
432
433            // Multiple include filters are run one by one.
434            mockListenerLog(binary, false);
435            CommandResult res = successResult("", resultCount(2, 0, 0));
436            EasyMock.expect(
437                            mMockRunUtil.runTimedCmd(
438                                    EasyMock.anyLong(),
439                                    EasyMock.eq(binary.getAbsolutePath()),
440                                    EasyMock.eq("OnlyMe"),
441                                    EasyMock.eq("--skip"),
442                                    EasyMock.eq("NotMe"),
443                                    EasyMock.eq("--skip"),
444                                    EasyMock.eq("Other")))
445                    .andReturn(res);
446            mMockListener.testRunFailed("Test run incomplete. Started 2 tests, finished 0");
447            mockListenerLog(binary, false);
448            res = successResult("", resultCount(3, 0, 0));
449            EasyMock.expect(
450                            mMockRunUtil.runTimedCmd(
451                                    EasyMock.anyLong(),
452                                    EasyMock.eq(binary.getAbsolutePath()),
453                                    EasyMock.eq("Me2"),
454                                    EasyMock.eq("--skip"),
455                                    EasyMock.eq("NotMe"),
456                                    EasyMock.eq("--skip"),
457                                    EasyMock.eq("Other")))
458                    .andReturn(res);
459
460            mMockListener.testRunFailed("Test run incomplete. Started 2 tests, finished 0");
461            mockTestRunEnded();
462            callReplayRunVerify();
463        } finally {
464            FileUtil.deleteFile(binary);
465        }
466    }
467
468    /** Test benchmark run */
469    @Test
470    public void testRun_benchmark() throws Exception {
471        File binary = FileUtil.createTempFile("rust-dir", "");
472        try {
473            OptionSetter setter = new OptionSetter(mTest);
474            setter.setOptionValue("test-file", binary.getAbsolutePath());
475            setter.setOptionValue("is-benchmark", "true");
476            mockCountBenchmarks(binary, 2);
477            mockListenerStarted(binary, 2);
478            mockListenerLog(binary, false);
479            mockBenchmarkRunExpect(
480                    binary,
481                    "Benchmarking test1\n"
482                            + "test                   time:   [0.1 ms 0.1 ms 0.1 ms]\n"
483                            + "Benchmarking test2\n"
484                            + "test                   time:   [0.1 ms 0.1 ms 0.1 ms]\n");
485
486            TestDescription desc1 = new TestDescription(binary.getName(), "test1");
487            TestDescription desc2 = new TestDescription(binary.getName(), "test2");
488            mMockListener.testStarted(desc1);
489            mMockListener.testEnded(
490                    EasyMock.eq(desc1), EasyMock.<HashMap<String, Metric>>anyObject());
491            mMockListener.testStarted(desc2);
492            mMockListener.testEnded(
493                    EasyMock.eq(desc2), EasyMock.<HashMap<String, Metric>>anyObject());
494            mockTestRunEnded();
495            callReplayRunVerify();
496        } finally {
497            FileUtil.deleteFile(binary);
498        }
499    }
500
501    @Test
502    public void testRun_benchmarkDoubleStart() throws Exception {
503        File binary = FileUtil.createTempFile("rust-dir", "");
504        try {
505            OptionSetter setter = new OptionSetter(mTest);
506            setter.setOptionValue("test-file", binary.getAbsolutePath());
507            setter.setOptionValue("is-benchmark", "true");
508            mockCountBenchmarks(binary, 2);
509            mockListenerStarted(binary, 2);
510            mockListenerLog(binary, false);
511            mockBenchmarkRunExpect(
512                    binary,
513                    "Benchmarking test1\n"
514                            + "Benchmarking test2\n"
515                            + "test                   time:   [0.1 ms 0.1 ms 0.1 ms]\n");
516
517            TestDescription desc1 = new TestDescription(binary.getName(), "test1");
518            TestDescription desc2 = new TestDescription(binary.getName(), "test2");
519            mMockListener.testStarted(desc1);
520            mMockListener.testFailed(EasyMock.eq(desc1), EasyMock.<String>anyObject());
521            mMockListener.testEnded(
522                    EasyMock.eq(desc1), EasyMock.<HashMap<String, Metric>>anyObject());
523            mMockListener.testStarted(desc2);
524            mMockListener.testEnded(
525                    EasyMock.eq(desc2), EasyMock.<HashMap<String, Metric>>anyObject());
526            mockTestRunEnded();
527            callReplayRunVerify();
528        } finally {
529            FileUtil.deleteFile(binary);
530        }
531    }
532
533    @Test
534    public void testRun_benchmarkNotFinished() throws Exception {
535        File binary = FileUtil.createTempFile("rust-dir", "");
536        try {
537            OptionSetter setter = new OptionSetter(mTest);
538            setter.setOptionValue("test-file", binary.getAbsolutePath());
539            setter.setOptionValue("is-benchmark", "true");
540            mockCountBenchmarks(binary, 2);
541            mockListenerStarted(binary, 2);
542            mockListenerLog(binary, false);
543            mockBenchmarkRunExpect(
544                    binary,
545                    "Benchmarking test1\n"
546                            + "test                   time:   [0.1 ms 0.1 ms 0.1 ms]\n"
547                            + "Benchmarking test2\n");
548
549            TestDescription desc1 = new TestDescription(binary.getName(), "test1");
550            TestDescription desc2 = new TestDescription(binary.getName(), "test2");
551            mMockListener.testStarted(desc1);
552            mMockListener.testEnded(
553                    EasyMock.eq(desc1), EasyMock.<HashMap<String, Metric>>anyObject());
554            mMockListener.testStarted(desc2);
555            mMockListener.testFailed(EasyMock.eq(desc2), EasyMock.<String>anyObject());
556            mMockListener.testEnded(
557                    EasyMock.eq(desc2), EasyMock.<HashMap<String, Metric>>anyObject());
558            mMockListener.testRunFailed(EasyMock.<String>anyObject());
559            mockTestRunEnded();
560            callReplayRunVerify();
561        } finally {
562            FileUtil.deleteFile(binary);
563        }
564    }
565}
566
Full Screen

Accelerate Your Automation Test Cycles With LambdaTest

Leverage LambdaTest’s cloud-based platform to execute your automation tests in parallel and trim down your test execution time significantly. Your first 100 automation testing minutes are on us.

Try LambdaTest

Trigger code on LambdaTest Cloud Grid

Execute automation tests with on a cloud-based Grid of 3000+ real browsers and operating systems for both web and mobile applications.

Test now for Free
LambdaTestX

We use cookies to give you the best experience. Cookies help to provide a more personalized experience and relevant advertising for you, and web analytics for us. Learn More in our Cookies policy, Privacy & Terms of service

Allow Cookie
Sarah

I hope you find the best code examples for your project.

If you want to accelerate automated browser testing, try LambdaTest. Your first 100 automation testing minutes are FREE.

Sarah Elson (Product & Growth Lead)