Newer
Older
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;
public class TestCastRunner {
private File projectFile;
private String scriptName;
private Logger logger;
private StringBuffer cmdLineOutputErrors = new StringBuffer();
private StringBuffer cmdLineOutputAll = new StringBuffer();
private boolean accepted = true;
private int testNumber;
public enum verdictEnum {
pass, inconc, fail, error, none
}
private verdictEnum testVerdict = verdictEnum.none;
public TestCastRunner(File projectFile, String scriptName, Logger logger, int testNumber) {
this.projectFile = projectFile;
this.scriptName = scriptName;
this.logger = logger;
this.testNumber = testNumber;
}
List<String> args = new ArrayList<String>();
File tempDir = new File(TestCastTester.config.get("TempPath")
.toString());
System.setProperty("user.dir", tempDir.getAbsolutePath());
File tool = new File(TestCastTester.config.get(
"TestCastCommandLineTool").toString());
args.add(tool.getAbsolutePath());
args.add("-proj");
args.add(projectFile.getAbsolutePath());
args.add("-s");
args.add(scriptName);
if (bExecute) {
args.add("-rb");
args.add("-r");
args.add("-log");
args.add("log");
}
ProcessBuilder processBuilder = new ProcessBuilder(args);
processBuilder.redirectErrorStream(true);
try {
Process process = processBuilder.start();
InputStream is = process.getInputStream();
InputStreamReader isr = new InputStreamReader(is);
BufferedReader br = new BufferedReader(isr);
String line;
while ((line = br.readLine()) != null) {
cmdLineOutputAll.append(" --->" + line);
cmdLineOutputAll.append("\n");
if (line.matches("^\\[ERR\\].*")) {
cmdLineOutputErrors.append(" --->" + line + "\n");
accepted = false;
} else if (line.contains("Compilation failed")) {
accepted = false;
} else if (line.contains("The test case") && line.contains("has ended with the result")) {
if (line.contains("<pass>")) {
updateVerdict(verdictEnum.pass);
} else if (line.contains("<inconc>")) {
updateVerdict(verdictEnum.inconc);
} else if (line.contains("<fail>")) {
updateVerdict(verdictEnum.fail);
} else if (line.contains("<error>")) {
updateVerdict(verdictEnum.error);
} else if (line.contains("<none>")) {
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
updateVerdict(verdictEnum.none);
}
}
// System.out.println(line);
}
} catch (IOException e) {
e.printStackTrace();
}
}
public boolean isAccepted() {
return accepted;
}
public void updateVerdict(verdictEnum newVerdict) {
if (testVerdict == verdictEnum.none) {
testVerdict = newVerdict;
} else if (testVerdict == verdictEnum.pass) {
if ((newVerdict != verdictEnum.pass) && (newVerdict != verdictEnum.none))
testVerdict = newVerdict;
} else if (testVerdict == verdictEnum.inconc) {
if ((newVerdict == verdictEnum.fail) || (newVerdict == verdictEnum.error))
testVerdict = newVerdict;
} else if (testVerdict == verdictEnum.fail) {
if (newVerdict == verdictEnum.error)
testVerdict = newVerdict;
}
}
public void evaluateResults(T3ExpectedOutput expectedOutput) {
if (!accepted) { // testcast rejects the input
if (expectedOutput.getExpectedOutput().contains("reject")) {
logger.writeScreen(testNumber + ": [pass] " + scriptName);
logger.writeAll(testNumber + ": [pass] " + scriptName + "\n");
} else if (expectedOutput.getExpectedOutput().contains("accept")) {
logger.writeScreen(testNumber + ": [fail] " + scriptName + " - tool does not accept input, but acceptance is expected!");
logger.writeAll(testNumber + ": [fail] " + scriptName + " - tool does not accept input, but acceptance is expected!\n");
logger.writeCritical(testNumber + ": [fail] " + scriptName + " - tool does not accept input, but acceptance is expected!\n");
// logger.writeAll(cmdLineOutputErrors.toString());
// logger.writeCritical(cmdLineOutputErrors.toString());
logger.writeAll(cmdLineOutputAll.toString());
logger.writeCritical(cmdLineOutputAll.toString());
} else {
logger.writeScreen(testNumber + ": [inconc] " + scriptName + " - cannot interpret expected output!");
logger.writeAll(testNumber + ": [inconc] " + scriptName + " - cannot interpret expected output!\n");
logger.writeCritical(testNumber + ": [inconc] " + scriptName + " - cannot interpret expected output!\n");
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
}
} else { // testcast accepts the input
if (expectedOutput.getExpectedOutput().contains("accept")) {
// logger.writeScreen(testNumber + ": [pass] " + scriptName);
// logger.writeAll(testNumber + ": [pass] " + scriptName + "\n");
// System.out.println("==================> have to check execution result!!");
// System.out.println(cmdLineOutputAll.toString());
if ((expectedOutput.getExecutionResult() == null) || (expectedOutput.getExecutionResult().contains("noexecution")) ){ // no execution result required...
logger.writeScreen(testNumber + ": [pass] " + scriptName);
logger.writeAll(testNumber + ": [pass] " + scriptName + "\n");
} else { // we must compare the execution result
try {
verdictEnum expectedVerdict;
expectedVerdict = executionResultToVerdictEnum(expectedOutput.getExecutionResult());
if (expectedVerdict == testVerdict) {
logger.writeScreen(testNumber + ": [pass] " + scriptName);
logger.writeAll(testNumber + ": [pass] " + scriptName + "\n");
} else {
logger.writeScreen(testNumber + ": [fail] " + scriptName + " - tool accepts the input, but expected execution results mismatch! Expected: " + expectedVerdict + ", Actual: " + testVerdict);
logger.writeAll(testNumber + ": [fail] " + scriptName + " - tool accepts the input, but expected execution results mismatch! Expected: " + expectedVerdict + ", Actual: " + testVerdict + "\n");
logger.writeCritical(testNumber + ": [fail] " + scriptName + " - tool accepts the input, but expected execution results mismatch! Expected: " + expectedVerdict + ", Actual: " + testVerdict + "\n");
logger.writeAll(cmdLineOutputAll.toString());
logger.writeCritical(cmdLineOutputAll.toString());
}
} catch (Exception e) {
logger.writeScreen(testNumber + ": [fail] " + scriptName + " - tool accepts the input, but an error occured during the verdict conversion. Possible typo in test case!");
logger.writeAll(testNumber + ": [fail] " + scriptName + " - tool accepts the input, but an error occured during the verdict conversion. Possible typo in test case!\n");
logger.writeCritical(testNumber + ": [fail] " + scriptName + " - tool accepts the input, but an error occured during the verdict conversion. Possible typo in test case!\n");
logger.writeAll(cmdLineOutputAll.toString());
logger.writeCritical(cmdLineOutputAll.toString());
}
}
} else if (expectedOutput.getExpectedOutput().contains("reject")) {
logger.writeScreen(testNumber + ": [fail] " + scriptName + " - tool accepts the input, but rejection is expected!");
logger.writeAll(testNumber + ": [fail] " + scriptName + " - tool accepts the input, but rejection is expected!\n");
logger.writeCritical(testNumber + ": [fail] " + scriptName + " - tool accepts the input, but rejection is expected!\n");
logger.writeAll(cmdLineOutputAll.toString());
logger.writeCritical(cmdLineOutputAll.toString());
} else {
logger.writeScreen(testNumber + ": [inconc] " + scriptName + " - cannot interpret expected output!");
logger.writeAll(testNumber + ": [inconc] " + scriptName + " - cannot interpret expected output!\n");
logger.writeCritical(testNumber + ": [inconc] " + scriptName + " - cannot interpret expected output!\n");
}
}
}
private verdictEnum executionResultToVerdictEnum(String str) throws Exception {
if (str.contains("ttcn3verdict:pass")) {
return verdictEnum.pass;
} else if (str.contains("ttcn3verdict:inconc")) {
return verdictEnum.inconc;
} else if (str.contains("ttcn3verdict:fail")) {
return verdictEnum.fail;
} else if (str.contains("ttcn3verdict:error")) {
return verdictEnum.error;
} else if (str.contains("ttcn3verdict:none")) {
return verdictEnum.none;
}
throw new Exception("Unknown execution result");
}
}