import java.io.BufferedReader; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.util.ArrayList; import java.util.List; public class TestCastRunner { private File projectFile; private String scriptName; private Logger logger; private StringBuffer cmdLineOutputErrors = new StringBuffer(); private StringBuffer cmdLineOutputAll = new StringBuffer(); private boolean accepted = true; private int testNumber; public enum verdictEnum { pass, inconc, fail, error, none } private verdictEnum testVerdict = verdictEnum.none; public TestCastRunner(File projectFile, String scriptName, Logger logger, int testNumber) { this.projectFile = projectFile; this.scriptName = scriptName; this.logger = logger; this.testNumber = testNumber; } public void run(boolean bExecute) { List args = new ArrayList(); File tempDir = new File(TestCastTester.config.get("TempPath") .toString()); System.setProperty("user.dir", tempDir.getAbsolutePath()); File tool = new File(TestCastTester.config.get( "TestCastCommandLineTool").toString()); args.add(tool.getAbsolutePath()); args.add("-proj"); args.add(projectFile.getAbsolutePath()); args.add("-s"); args.add(scriptName); if (bExecute) { args.add("-rb"); args.add("-r"); args.add("-log"); args.add("log"); } ProcessBuilder processBuilder = new ProcessBuilder(args); processBuilder.redirectErrorStream(true); try { Process process = processBuilder.start(); InputStream is = process.getInputStream(); InputStreamReader isr = new InputStreamReader(is); BufferedReader br = new BufferedReader(isr); String line; while ((line = br.readLine()) != null) { cmdLineOutputAll.append(" --->" + line); cmdLineOutputAll.append("\n"); // System.out.println(line); if (line.matches("^\\[ERR\\].*")) { cmdLineOutputErrors.append(" --->" + line + "\n"); accepted = false; } else if (line.contains("Compilation failed")) { accepted = false; } else if (line.contains("The test case") && line.contains("has ended with the result")) { if (line.contains("")) { updateVerdict(verdictEnum.pass); } else if (line.contains("")) { updateVerdict(verdictEnum.inconc); } else if (line.contains("")) { updateVerdict(verdictEnum.fail); } else if (line.contains("")) { updateVerdict(verdictEnum.error); } else if (line.contains("")) { updateVerdict(verdictEnum.none); } } // System.out.println(line); } } catch (IOException e) { e.printStackTrace(); } } public boolean isAccepted() { return accepted; } public void updateVerdict(verdictEnum newVerdict) { if (testVerdict == verdictEnum.none) { testVerdict = newVerdict; } else if (testVerdict == verdictEnum.pass) { if ((newVerdict != verdictEnum.pass) && (newVerdict != verdictEnum.none)) testVerdict = newVerdict; } else if (testVerdict == verdictEnum.inconc) { if ((newVerdict == verdictEnum.fail) || (newVerdict == verdictEnum.error)) testVerdict = newVerdict; } else if (testVerdict == verdictEnum.fail) { if (newVerdict == verdictEnum.error) testVerdict = newVerdict; } } public void evaluateResults(T3ExpectedOutput expectedOutput) { if (!accepted) { // testcast rejects the input if (expectedOutput.getExpectedOutput().contains("reject")) { logger.writeScreen(testNumber + ": [pass] " + scriptName); logger.writeAll(testNumber + ": [pass] " + scriptName + "\n"); } else if (expectedOutput.getExpectedOutput().contains("accept")) { logger.writeScreen(testNumber + ": [fail] " + scriptName + " - tool does not accept input, but acceptance is expected!"); logger.writeAll(testNumber + ": [fail] " + scriptName + " - tool does not accept input, but acceptance is expected!\n"); logger.writeCritical(testNumber + ": [fail] " + scriptName + " - tool does not accept input, but acceptance is expected!\n"); // logger.writeAll(cmdLineOutputErrors.toString()); // logger.writeCritical(cmdLineOutputErrors.toString()); logger.writeAll(cmdLineOutputAll.toString()); logger.writeCritical(cmdLineOutputAll.toString()); } else { logger.writeScreen(testNumber + ": [inconc] " + scriptName + " - cannot interpret expected output!"); logger.writeAll(testNumber + ": [inconc] " + scriptName + " - cannot interpret expected output!\n"); logger.writeCritical(testNumber + ": [inconc] " + scriptName + " - cannot interpret expected output!\n"); } } else { // testcast accepts the input if (expectedOutput.getExpectedOutput().contains("accept")) { // logger.writeScreen(testNumber + ": [pass] " + scriptName); // logger.writeAll(testNumber + ": [pass] " + scriptName + "\n"); // System.out.println("==================> have to check execution result!!"); // System.out.println(cmdLineOutputAll.toString()); if ((expectedOutput.getExecutionResult() == null) || (expectedOutput.getExecutionResult().contains("noexecution")) ){ // no execution result required... logger.writeScreen(testNumber + ": [pass] " + scriptName); logger.writeAll(testNumber + ": [pass] " + scriptName + "\n"); } else { // we must compare the execution result try { verdictEnum expectedVerdict; expectedVerdict = executionResultToVerdictEnum(expectedOutput.getExecutionResult()); if (expectedVerdict == testVerdict) { logger.writeScreen(testNumber + ": [pass] " + scriptName); logger.writeAll(testNumber + ": [pass] " + scriptName + "\n"); } else { logger.writeScreen(testNumber + ": [fail] " + scriptName + " - tool accepts the input, but expected execution results mismatch! Expected: " + expectedVerdict + ", Actual: " + testVerdict); logger.writeAll(testNumber + ": [fail] " + scriptName + " - tool accepts the input, but expected execution results mismatch! Expected: " + expectedVerdict + ", Actual: " + testVerdict + "\n"); logger.writeCritical(testNumber + ": [fail] " + scriptName + " - tool accepts the input, but expected execution results mismatch! Expected: " + expectedVerdict + ", Actual: " + testVerdict + "\n"); logger.writeAll(cmdLineOutputAll.toString()); logger.writeCritical(cmdLineOutputAll.toString()); } } catch (Exception e) { logger.writeScreen(testNumber + ": [fail] " + scriptName + " - tool accepts the input, but an error occured during the verdict conversion. Possible typo in test case!"); logger.writeAll(testNumber + ": [fail] " + scriptName + " - tool accepts the input, but an error occured during the verdict conversion. Possible typo in test case!\n"); logger.writeCritical(testNumber + ": [fail] " + scriptName + " - tool accepts the input, but an error occured during the verdict conversion. Possible typo in test case!\n"); logger.writeAll(cmdLineOutputAll.toString()); logger.writeCritical(cmdLineOutputAll.toString()); } } } else if (expectedOutput.getExpectedOutput().contains("reject")) { logger.writeScreen(testNumber + ": [fail] " + scriptName + " - tool accepts the input, but rejection is expected!"); logger.writeAll(testNumber + ": [fail] " + scriptName + " - tool accepts the input, but rejection is expected!\n"); logger.writeCritical(testNumber + ": [fail] " + scriptName + " - tool accepts the input, but rejection is expected!\n"); logger.writeAll(cmdLineOutputAll.toString()); logger.writeCritical(cmdLineOutputAll.toString()); } else { logger.writeScreen(testNumber + ": [inconc] " + scriptName + " - cannot interpret expected output!"); logger.writeAll(testNumber + ": [inconc] " + scriptName + " - cannot interpret expected output!\n"); logger.writeCritical(testNumber + ": [inconc] " + scriptName + " - cannot interpret expected output!\n"); } } } private verdictEnum executionResultToVerdictEnum(String str) throws Exception { if (str.contains("ttcn3verdict:pass")) { return verdictEnum.pass; } else if (str.contains("ttcn3verdict:inconc")) { return verdictEnum.inconc; } else if (str.contains("ttcn3verdict:fail")) { return verdictEnum.fail; } else if (str.contains("ttcn3verdict:error")) { return verdictEnum.error; } else if (str.contains("ttcn3verdict:none")) { return verdictEnum.none; } throw new Exception("Unknown execution result"); } }