diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/flakinessdetect/FlakinessDetectApplication.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/flakinessdetect/FlakinessDetectApplication.kt new file mode 100644 index 0000000000..9d9e47f29a --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/flakinessdetect/FlakinessDetectApplication.kt @@ -0,0 +1,15 @@ +package com.foo.rest.examples.spring.openapi.v3.flakinessdetect + +import org.springframework.boot.SpringApplication +import org.springframework.boot.autoconfigure.SpringBootApplication +import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration + +@SpringBootApplication(exclude = [SecurityAutoConfiguration::class]) +open class FlakinessDetectApplication { + companion object { + @JvmStatic + fun main(args: Array) { + SpringApplication.run(FlakinessDetectApplication::class.java, *args) + } + } +} \ No newline at end of file diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/flakinessdetect/FlakinessDetectRest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/flakinessdetect/FlakinessDetectRest.kt new file mode 100644 index 0000000000..26b5ef8abe --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/flakinessdetect/FlakinessDetectRest.kt @@ -0,0 +1,52 @@ +package com.foo.rest.examples.spring.openapi.v3.flakinessdetect + +import org.h2.util.MathUtils.randomInt +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PathVariable +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController +import java.time.LocalDateTime +import java.time.format.DateTimeFormatter +import kotlin.math.max +import kotlin.math.min + +@RestController +@RequestMapping(path = ["/api/flakinessdetect"]) +class FlakinessDetectRest { + + companion object{ + val formatter = DateTimeFormatter.ofPattern("HH:mm:ss.SSS yyyy-MM-dd EEEE 'Week' ww") + } + + @GetMapping(path = ["/stringfirst/{n}"]) + open fun getFirst( @PathVariable("n") n: Int) : ResponseEntity { + + return ResponseEntity.ok(getPartialDate(n)) + } + + @GetMapping(path = ["/next/{n}"]) + open fun getNext( @PathVariable("n") n: Int) : ResponseEntity { + + return ResponseEntity.ok(FlakinessDetectData(getPartialDate(n), randomInt(n))) + } + + @GetMapping(path = ["/multiplelines/{num}"]) + open fun getMultipleLines( @PathVariable("num") num: Int) : ResponseEntity { + + val num = min(20, max(2, randomInt(num))) + + val msg = (1..num).joinToString(System.lineSeparator()) { "LINE $it" } + return ResponseEntity.ok(FlakinessDetectData(msg, num)) + } + + + private fun getPartialDate(n: Int) : String { + val now = LocalDateTime.now().format(formatter) + val size = max(12, min(now.toString().length, n)) + + return now.substring(0, size) + } +} + +data class FlakinessDetectData(val first : String, val next : Int) \ No newline at end of file diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/flakinessdetect/FlakinessDetectController.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/flakinessdetect/FlakinessDetectController.kt new file mode 100644 index 0000000000..68f5854db6 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/flakinessdetect/FlakinessDetectController.kt @@ -0,0 +1,5 @@ +package com.foo.rest.examples.spring.openapi.v3.flakinessdetect + +import com.foo.rest.examples.spring.openapi.v3.SpringController + +class FlakinessDetectController : SpringController(FlakinessDetectApplication::class.java) \ No newline at end of file diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/flakinessdetect/FlakinessDetectEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/flakinessdetect/FlakinessDetectEMTest.kt new file mode 100644 index 0000000000..0daef4acf1 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/flakinessdetect/FlakinessDetectEMTest.kt @@ -0,0 +1,45 @@ +package org.evomaster.e2etests.spring.openapi.v3.flakinessdetect + +import com.foo.rest.examples.spring.openapi.v3.flakinessdetect.FlakinessDetectController +import org.evomaster.e2etests.spring.openapi.v3.SpringTestBase +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test +import java.nio.file.Files +import java.nio.file.Paths + +class FlakinessDetectEMTest : SpringTestBase() { + + companion object { + @BeforeAll + @JvmStatic + fun init() { + initClass(FlakinessDetectController()) + } + } + + + @Test + fun testRunEM() { + runTestHandlingFlakyAndCompilation( + "FlakinessDetectEM", + "org.foo.FlakinessDetectEM", + 5 + ) { args: MutableList -> + + val executedMainActionToFile = "target/em-tests/FlakinessDetectEM/org/foo/FlakinessDetectEM.kt" + + args.add("--minimize") + args.add("true") + args.add("--handleFlakiness") + args.add("true") + + + val solution = initAndRun(args) + + val size = Files.readAllLines(Paths.get(executedMainActionToFile)).count { !it.contains("Flaky") && it.isNotBlank() } + assertTrue(size >= 3) + } + } +} \ No newline at end of file diff --git a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt index eeee3f5d54..20b9e2ae63 100644 --- a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt +++ b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt @@ -2474,6 +2474,10 @@ class EMConfig { RANDOM } + @Experimental + @Cfg("Specify whether to detect flakiness and handle the flakiness in assertions during post handling of fuzzing.") + var handleFlakiness = false + @Experimental @Cfg("Specify a method to select the first external service spoof IP address.") var externalServiceIPSelectionStrategy = ExternalServiceIPSelectionStrategy.NONE diff --git a/core/src/main/kotlin/org/evomaster/core/output/Lines.kt b/core/src/main/kotlin/org/evomaster/core/output/Lines.kt index 81b387ddf0..e8611c37aa 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/Lines.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/Lines.kt @@ -75,6 +75,14 @@ class Lines(val format: OutputFormat) { buffer[buffer.lastIndex] = buffer.last().replace(regex, replacement) } + fun replaceFirstInCurrent(regex: Regex, replacement: String){ + if(buffer.isEmpty()){ + return + } + + buffer[buffer.lastIndex] = buffer.last().replaceFirst(regex, replacement) + } + /** * Is the current line just a comment // without any statement? */ diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt index b4764a062c..800a45620a 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt @@ -77,27 +77,35 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { /** * handle assertion with text plain */ - fun handleTextPlainTextAssertion(bodyString: String?, lines: Lines, bodyVarName: String?) { + fun handleTextPlainTextAssertion(bodyString: String?, flakyBodyString: String?, lines: Lines, bodyVarName: String?) { - if (bodyString.isNullOrBlank()) { - lines.add(emptyBodyCheck(bodyVarName)) + val assertion = if (bodyString.isNullOrBlank()) { + emptyBodyCheck(bodyVarName) } else { //TODO in the call above BODY was used... what's difference from TEXT? - lines.add(bodyIsString(bodyString, GeneUtils.EscapeMode.TEXT, bodyVarName)) + bodyIsString(bodyString, GeneUtils.EscapeMode.TEXT, bodyVarName) + } + if (flakyBodyString == null || flakyBodyString == bodyString) { + lines.add(assertion) + }else{ + lines.addSingleCommentLine(flakyInfo("response in plain text", bodyString?:"null", flakyBodyString)) + lines.addSingleCommentLine(assertion) } } /** * handle assertion with json body string */ - fun handleJsonStringAssertion(bodyString: String?, lines: Lines, bodyVarName: String?, isTooLargeBody: Boolean) { + fun handleJsonStringAssertion(bodyString: String?, flakyBodyString : String?, lines: Lines, bodyVarName: String?, isTooLargeBody: Boolean) { when (bodyString?.trim()?.first()) { //TODO this should be handled recursively, and not ad-hoc here... '[' -> { try{ // This would be run if the JSON contains an array of objects. val list = Gson().fromJson(bodyString, List::class.java) - handleAssertionsOnList(list, lines, "", bodyVarName) + val flakyList = flakyBodyString?.let { Gson().fromJson(it, List::class.java) } + + handleAssertionsOnList(list, flakyList, lines, "", bodyVarName) } catch (e: JsonSyntaxException) { lines.addSingleCommentLine("Failed to parse JSON response") } @@ -106,13 +114,21 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { // JSON contains an object try { val resContents = Gson().fromJson(bodyString, Map::class.java) - handleAssertionsOnObject(resContents as Map, lines, "", bodyVarName) + val flakyMap = flakyBodyString?.let { Gson().fromJson(it, Map::class.java) as Map } + handleAssertionsOnObject(resContents as Map, flakyMap, lines, "", bodyVarName) } catch (e: JsonSyntaxException) { lines.addSingleCommentLine("Failed to parse JSON response") } } '"' -> { - lines.add(bodyIsString(bodyString, GeneUtils.EscapeMode.BODY, bodyVarName)) + val isString = bodyIsString(bodyString, GeneUtils.EscapeMode.BODY, bodyVarName) + if (flakyBodyString == null || flakyBodyString == bodyString) { + lines.add(isString) + }else{ + lines.addSingleCommentLine(flakyInfo("Body", bodyString, flakyBodyString)) + lines.addSingleCommentLine(isString) + } + } else -> { /* @@ -127,14 +143,14 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { bodyString.isNullOrBlank() -> lines.add(emptyBodyCheck(bodyVarName)) - else -> handlePrimitive(lines, bodyString, "", bodyVarName) + else -> handlePrimitive(lines, bodyString, flakyBodyString,"", bodyVarName) } } } } - private fun handlePrimitive(lines: Lines, bodyString: String, fieldPath: String, responseVariableName: String?) { + private fun handlePrimitive(lines: Lines, bodyString: String, flakyBodyString: String?, fieldPath: String, responseVariableName: String?) { /* If we arrive here, it means we have free text. @@ -150,24 +166,36 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { when { format.isJavaOrKotlin() -> { + /* Unfortunately, a limitation of RestAssured is that for JSON it only handles object and array. The rest is either ignored or leads to crash */ - lines.add(bodyIsString(s, GeneUtils.EscapeMode.BODY, responseVariableName)) + val value = bodyIsString(s,GeneUtils.EscapeMode.BODY, responseVariableName) + + val fs = flakyBodyString?.trim() + if (fs == null || fs == s) + lines.add(value) + else{ + lines.addSingleCommentLine(flakyInfo("Body", s, fs)) + lines.addSingleCommentLine(value) + } + } format.isJavaScript() || format.isCsharp() || format.isPython() -> { try { val number = s.toDouble() - handleAssertionsOnField(number, lines, fieldPath, responseVariableName) + // TODO only support flaky for JVM + handleAssertionsOnField(number, null, lines, fieldPath, responseVariableName) return } catch (e: NumberFormatException) { } if (s.equals("true", true) || s.equals("false", true)) { val tf = bodyString.toBoolean() - handleAssertionsOnField(tf, lines, fieldPath, responseVariableName) + // TODO flaky + handleAssertionsOnField(tf, null, lines, fieldPath, responseVariableName) return } @@ -175,13 +203,18 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { Note: for JS, this will not work, as call would crash due to invalid JSON payload (Java and Python don't seem to have such issue) */ + // TODO flaky lines.add(bodyIsString(s, GeneUtils.EscapeMode.BODY, responseVariableName)) } else -> throw IllegalStateException("Format not supported yet: $format") } } - protected fun handleAssertionsOnObject(resContents: Map, lines: Lines, fieldPath: String, responseVariableName: String?) { + protected fun handleAssertionsOnObject(resContents: Map, flakyMap: Map?, lines: Lines, fieldPath: String, responseVariableName: String?) { + if (flakyMap != null && flakyMap.size != resContents.size) { + lines.addSingleCommentLine(flakyInfo("mismatched size of fields for Object $fieldPath", resContents.size.toString(), flakyMap.size.toString())) + } + if (resContents.isEmpty()) { val k = when { @@ -206,7 +239,11 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { else -> throw IllegalStateException("Format not supported yet: $format") } - lines.add(instruction) + if (flakyMap.isNullOrEmpty()) + lines.add(instruction) + else{ + lines.addSingleCommentLine(instruction) + } } resContents.entries @@ -244,7 +281,11 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { "${fieldPath}${fieldName}" } - handleAssertionsOnField(it.value, lines, extendedPath, responseVariableName) + if (flakyMap == null || flakyMap.containsKey(it.key)) { + handleAssertionsOnField(it.value, flakyMap?.get(it.key), lines, extendedPath, responseVariableName) + }else{ + lines.addSingleCommentLine(flakyInfo("mismatched field name", fieldName, "NONE")) + } } } /* @@ -255,7 +296,7 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { return text.replace("\$", "\\\$") } - private fun handleAssertionsOnField(value: Any?, lines: Lines, fieldPath: String, responseVariableName: String?) { + private fun handleAssertionsOnField(value: Any?, flakyValue: Any?, lines: Lines, fieldPath: String, responseVariableName: String?) { if (value == null) { val instruction = when { @@ -271,11 +312,11 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { when (value) { is Map<*, *> -> { - handleAssertionsOnObject(value as Map, lines, fieldPath, responseVariableName) + handleAssertionsOnObject(value as Map, flakyValue as? Map,lines, fieldPath, responseVariableName) return } is List<*> -> { - handleAssertionsOnList(value, lines, fieldPath, responseVariableName) + handleAssertionsOnList(value, flakyValue as? List<*>, lines, fieldPath, responseVariableName) return } } @@ -290,7 +331,12 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { else -> throw IllegalStateException("Unsupported type: ${value::class}") } if (isSuitableToPrint(left)) { - lines.add(".body(\"$fieldPath\", $left)") + if (flakyValue == null || flakyValue == value) + lines.add(".body(\"$fieldPath\", $left)") + else{ + lines.addSingleCommentLine(flakyInfo("value of field \"$fieldPath\"", value.toString(), flakyValue.toString())) + lines.addSingleCommentLine(".body(\"$fieldPath\", $left)") + } } return } @@ -322,9 +368,16 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { } - protected fun handleAssertionsOnList(list: List<*>, lines: Lines, fieldPath: String, responseVariableName: String?) { + protected fun handleAssertionsOnList(list: List<*>, flakyList: List<*>?, lines: Lines, fieldPath: String, responseVariableName: String?) { + + val checkSize = collectionSizeCheck(responseVariableName, fieldPath, list.size) + if (flakyList == null || flakyList.size == list.size) { + lines.add(checkSize) + }else{ + lines.addSingleCommentLine(flakyInfo("size of $fieldPath", list.size.toString(), flakyList.size.toString())) + lines.addSingleCommentLine(checkSize) + } - lines.add(collectionSizeCheck(responseVariableName, fieldPath, list.size)) //assertions on contents if (list.isEmpty()) { @@ -335,11 +388,26 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { TODO could do the same for numbers */ if (format.isJavaOrKotlin() && list.all { it is String } && list.isNotEmpty()) { - lines.add(".body(\"$fieldPath\", hasItems(${ - (list as List).joinToString { + val items = (list as List).joinToString { + "\"${GeneUtils.applyEscapes(it, mode = GeneUtils.EscapeMode.ASSERTION, format = format)}\"" + } + + if (flakyList != null && (!flakyList.containsAll(list) || flakyList.size != list.size)) { + + val flakyItems = (flakyList as List).joinToString { "\"${GeneUtils.applyEscapes(it, mode = GeneUtils.EscapeMode.ASSERTION, format = format)}\"" } - }))") + + lines.addSingleCommentLine(flakyInfo("Body", items, flakyItems)) + lines.addSingleCommentLine(".body(\"$fieldPath\", hasItems(${ + items + }))") + + }else{ + lines.add(".body(\"$fieldPath\", hasItems(${ + items + }))") + } return } @@ -356,7 +424,11 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { if (i == limit) { break } - handleAssertionsOnField(list[i], lines, "$fieldPath[$i]", responseVariableName) + if (flakyList != null && flakyList.size < i) + break + val flakyItem = if (flakyList != null) flakyList[i] else null + + handleAssertionsOnField(list[i], flakyItem, lines, "$fieldPath[$i]", responseVariableName) } if (skipped > 0) { lines.addSingleCommentLine("Skipping assertions on the remaining $skipped elements. This limit of $limit elements can be increased in the configurations") diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt index c47a22eaed..b8c91bff7d 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt @@ -697,7 +697,12 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { when { format.isJavaOrKotlin() -> { lines.add(".then()") - lines.add(".statusCode($code)") + if (res.getFlakyStatusCode() == null) { + lines.add(".statusCode($code)") + } else { + lines.addSingleCommentLine(flakyInfo("Status Code", code.toString(), res.getFlakyStatusCode().toString())) + lines.addSingleCommentLine(".statusCode($code)") + } } else -> throw IllegalStateException("No assertion in calls for format: $format") @@ -768,7 +773,12 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { format.isPython() -> "assert \"$bodyTypeSimplified\" in $responseVariableName.headers[\"content-type\"]" else -> throw IllegalStateException("Unsupported format $format") } - lines.add(instruction) + + // handle flaky body type + if (res.getFlakyBodyType() == null) + lines.add(instruction) + else + lines.addSingleCommentLine(instruction) } val type = res.getBodyType() @@ -789,7 +799,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.append("JsonConvert.DeserializeObject(await $responseVariableName.Content.ReadAsStringAsync());") } - handleJsonStringAssertion(bodyString, lines, bodyVarName, res.getTooLargeBody()) + handleJsonStringAssertion(bodyString, res.getFlakyBody(), lines, bodyVarName, res.getTooLargeBody()) } else if (type.isCompatible(MediaType.TEXT_PLAIN_TYPE)) { @@ -797,7 +807,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.append("await $responseVariableName.Content.ReadAsStringAsync();") } - handleTextPlainTextAssertion(bodyString, lines, bodyVarName) + handleTextPlainTextAssertion(bodyString, res.getFlakyBody(), lines, bodyVarName) } else { if (format.isCsharp()) { lines.append("await $responseVariableName.Content.ReadAsStringAsync();") @@ -849,7 +859,9 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.replaceInCurrent(Regex("\\s*//"), "; //") } - } else { + } else if (config.handleFlakiness && lines.isCurrentACommentLine()){ + lines.replaceInCurrent(Regex("(?<=\\s)//"), "; //") + }else { lines.appendSemicolon() } } diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt index a1712427b1..7a80fb24af 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt @@ -398,5 +398,9 @@ abstract class TestCaseWriter { // do nothing } + /** + * provide flaky info in a single-line comment + */ + fun flakyInfo(category : String?, value : String, flaky : String) = "Flaky${if (category == null) "" else " $category"}: ${value.replace("\n", "")} vs. ${flaky.replace("\n","")}" } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/graphql/service/GraphQLBlackBoxModule.kt b/core/src/main/kotlin/org/evomaster/core/problem/graphql/service/GraphQLBlackBoxModule.kt index 7de51793b0..7e27274fff 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/graphql/service/GraphQLBlackBoxModule.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/graphql/service/GraphQLBlackBoxModule.kt @@ -10,6 +10,7 @@ import org.evomaster.core.remote.service.RemoteController import org.evomaster.core.remote.service.RemoteControllerImplementation import org.evomaster.core.search.service.Archive import org.evomaster.core.search.service.FitnessFunction +import org.evomaster.core.search.service.FlakinessDetector import org.evomaster.core.search.service.Minimizer import org.evomaster.core.search.service.Sampler @@ -53,6 +54,12 @@ class GraphQLBlackBoxModule( bind(object : TypeLiteral>(){}) .asEagerSingleton() + bind(object : TypeLiteral>(){}) + .asEagerSingleton() + + bind(object : TypeLiteral>(){}) + .asEagerSingleton() + if(usingRemoteController) { bind(RemoteController::class.java) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/graphql/service/GraphQLModule.kt b/core/src/main/kotlin/org/evomaster/core/problem/graphql/service/GraphQLModule.kt index e9c773799d..c6cf794849 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/graphql/service/GraphQLModule.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/graphql/service/GraphQLModule.kt @@ -11,6 +11,7 @@ import org.evomaster.core.remote.service.RemoteController import org.evomaster.core.remote.service.RemoteControllerImplementation import org.evomaster.core.search.service.Archive import org.evomaster.core.search.service.FitnessFunction +import org.evomaster.core.search.service.FlakinessDetector import org.evomaster.core.search.service.Minimizer import org.evomaster.core.search.service.Sampler import org.evomaster.core.search.service.mutator.Mutator @@ -56,6 +57,11 @@ class GraphQLModule : EnterpriseModule() { bind(object : TypeLiteral>(){}) .asEagerSingleton() + bind(object : TypeLiteral>(){}) + .asEagerSingleton() + + bind(object : TypeLiteral>(){}) + .asEagerSingleton() bind(RemoteController::class.java) .to(RemoteControllerImplementation::class.java) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/httpws/HttpWsCallResult.kt b/core/src/main/kotlin/org/evomaster/core/problem/httpws/HttpWsCallResult.kt index 581704b72b..f8534147d6 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/httpws/HttpWsCallResult.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/httpws/HttpWsCallResult.kt @@ -29,6 +29,12 @@ abstract class HttpWsCallResult : EnterpriseActionResult { const val VULNERABLE_SSRF = "VULNERABLE_SSRF" const val VULNERABLE_SQLI = "VULNERABLE_SQLI" + + + const val FLAKY_STATUS_CODE = "FLAKY_STATUS_CODE" + const val FLAKY_BODY = "FLAKY_BODY" + const val FLAKY_BODY_TYPE = "FLAKY_BODY_TYPE" + const val FLAKY_ERROR_MESSAGE = "FLAKY_ERROR_MESSAGE" } /** @@ -134,4 +140,40 @@ abstract class HttpWsCallResult : EnterpriseActionResult { fun setResponseTimeMs(responseTime: Long) = addResultValue(RESPONSE_TIME_MS, responseTime.toString()) fun getResponseTimeMs(): Long? = getResultValue(RESPONSE_TIME_MS)?.toLong() + + + fun setFlakyErrorMessage(msg: String) = addResultValue(FLAKY_ERROR_MESSAGE, msg) + fun getFlakyErrorMessage() : String? = getResultValue(FLAKY_ERROR_MESSAGE) + + fun setFlakyStatusCode(code: Int) = addResultValue(FLAKY_STATUS_CODE, code.toString()) + fun getFlakyStatusCode() : Int? = getResultValue(FLAKY_STATUS_CODE)?.toInt() + + fun setFlakyBody(body: String) = addResultValue(FLAKY_BODY, body) + fun getFlakyBody() : String? = getResultValue(FLAKY_BODY) + + fun setFlakyBodyType(type: MediaType) = addResultValue(FLAKY_BODY_TYPE, type.toString()) + fun getFlakyBodyType() : MediaType? = getResultValue(FLAKY_BODY_TYPE)?.let { MediaType.valueOf(it) } + + + fun setFlakiness(previous: HttpWsCallResult){ + val pStatusCode = previous.getStatusCode() + if (pStatusCode != null && pStatusCode != getStatusCode()) { + setFlakyStatusCode(pStatusCode) + } + + val pBody = previous.getBody() + if (pBody != null && pBody != getBody()) { + setFlakyBody(pBody) + } + + val pBodyType = previous.getBodyType() + if (pBodyType != null && pBodyType != getBodyType()) { + setFlakyBodyType(pBodyType) + } + + val pMessage = previous.getErrorMessage() + if (pMessage != null && pMessage != getErrorMessage()) { + setFlakyErrorMessage(pMessage) + } + } } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/module/RestBaseModule.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/module/RestBaseModule.kt index 1c4b47be3a..f09079e759 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/module/RestBaseModule.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/module/RestBaseModule.kt @@ -12,6 +12,7 @@ import org.evomaster.core.problem.rest.service.HttpSemanticsService import org.evomaster.core.problem.rest.service.RestIndividualBuilder import org.evomaster.core.problem.rest.service.SecurityRest import org.evomaster.core.search.service.Archive +import org.evomaster.core.search.service.FlakinessDetector import org.evomaster.core.search.service.Minimizer import org.evomaster.core.seeding.service.rest.PirToRest @@ -42,6 +43,12 @@ open class RestBaseModule : EnterpriseModule() { bind(object : TypeLiteral>(){}) .asEagerSingleton() + bind(object : TypeLiteral>(){}) + .asEagerSingleton() + + bind(object : TypeLiteral>(){}) + .asEagerSingleton() + bind(object : TypeLiteral>() {}) .asEagerSingleton() diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rpc/service/RPCModule.kt b/core/src/main/kotlin/org/evomaster/core/problem/rpc/service/RPCModule.kt index c40d5611eb..29837dbc17 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rpc/service/RPCModule.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rpc/service/RPCModule.kt @@ -11,6 +11,7 @@ import org.evomaster.core.remote.service.RemoteController import org.evomaster.core.remote.service.RemoteControllerImplementation import org.evomaster.core.search.service.Archive import org.evomaster.core.search.service.FitnessFunction +import org.evomaster.core.search.service.FlakinessDetector import org.evomaster.core.search.service.Minimizer import org.evomaster.core.search.service.Sampler import org.evomaster.core.search.service.mutator.Mutator @@ -50,6 +51,12 @@ class RPCModule : EnterpriseModule(){ .to(RPCFitness::class.java) .asEagerSingleton() + bind(object : TypeLiteral>(){}) + .asEagerSingleton() + + bind(object : TypeLiteral>(){}) + .asEagerSingleton() + bind(object : TypeLiteral>() {}) .to(RPCFitness::class.java) .asEagerSingleton() diff --git a/core/src/main/kotlin/org/evomaster/core/problem/webfrontend/service/WebModule.kt b/core/src/main/kotlin/org/evomaster/core/problem/webfrontend/service/WebModule.kt index 42f5711906..f3fb1c9525 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/webfrontend/service/WebModule.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/webfrontend/service/WebModule.kt @@ -11,6 +11,7 @@ import org.evomaster.core.remote.service.RemoteController import org.evomaster.core.remote.service.RemoteControllerImplementation import org.evomaster.core.search.service.Archive import org.evomaster.core.search.service.FitnessFunction +import org.evomaster.core.search.service.FlakinessDetector import org.evomaster.core.search.service.Minimizer import org.evomaster.core.search.service.Sampler import org.evomaster.core.search.service.mutator.Mutator @@ -47,6 +48,12 @@ class WebModule: EnterpriseModule() { bind(object : TypeLiteral>(){}) .asEagerSingleton() + bind(object : TypeLiteral>(){}) + .asEagerSingleton() + + bind(object : TypeLiteral>(){}) + .asEagerSingleton() + bind(object : TypeLiteral>() {}) .to(WebFitness::class.java) .asEagerSingleton() diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/FlakinessDetector.kt b/core/src/main/kotlin/org/evomaster/core/search/service/FlakinessDetector.kt new file mode 100644 index 0000000000..f191341a43 --- /dev/null +++ b/core/src/main/kotlin/org/evomaster/core/search/service/FlakinessDetector.kt @@ -0,0 +1,77 @@ +package org.evomaster.core.search.service + +import com.google.inject.Inject +import org.evomaster.core.EMConfig +import org.evomaster.core.logging.LoggingUtil +import org.evomaster.core.problem.httpws.HttpWsAction +import org.evomaster.core.problem.httpws.HttpWsCallResult +import org.evomaster.core.search.EvaluatedIndividual +import org.evomaster.core.search.Individual +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +/** + * it is used to detect flaky tests by checking responses or return value + * currently, such a detection is performed during post-handling of fuzzing + */ +class FlakinessDetector { + + companion object{ + private val log : Logger = LoggerFactory.getLogger(FlakinessDetector::class.java) + } + + @Inject + private lateinit var config: EMConfig + + @Inject + private lateinit var archive: Archive + + @Inject + private lateinit var fitness : FitnessFunction + + /** + * re-execute individuals in archive for identifying flakiness + */ + fun reexecuteToDetectFlakiness() { + + val currentIndividuals = archive.extractSolution().individuals + + LoggingUtil.getInfoLogger().info("Reexecuting all individual ${currentIndividuals.size} for identifying flakiness.") + + currentIndividuals.mapNotNull { + + val ei = fitness.computeWholeAchievedCoverageForPostProcessing(it.individual) + if(ei == null){ + log.warn("Failed to re-evaluate individual during flakiness analysis.") + }else + checkConsistency(ei, it) + + } + + } + + /** + * compare [inArchive] with [other] to check if the action results are same, the inconsistent info will be saved in [inArchive] evaluated individual + * @param inArchive the evaluated individual which saves info of flakiness + */ + fun checkConsistency(other: EvaluatedIndividual, inArchive: EvaluatedIndividual){ + val previousActions = other.evaluatedMainActions() + val currentActions = inArchive.evaluatedMainActions() + + if(previousActions.size != currentActions.size){ + log.warn("Mismatch between number of actions in re-executed individual." + + " Previous =${previousActions.size}, Current =${currentActions.size}") + return + } + + currentActions.forEachIndexed { index, it -> + val action = it.action + if(action is HttpWsAction){ + if (it.result is HttpWsCallResult){ + it.result.setFlakiness(previousActions[index].result as HttpWsCallResult) + } + + } + } + } +} \ No newline at end of file diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/Minimizer.kt b/core/src/main/kotlin/org/evomaster/core/search/service/Minimizer.kt index 06670b62fe..328d6f0971 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/service/Minimizer.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/service/Minimizer.kt @@ -38,6 +38,9 @@ class Minimizer { @Inject private lateinit var idMapper: IdMapper + @Inject + private lateinit var flakinessDetector: FlakinessDetector + private var startTimer : Long = -1 @@ -235,6 +238,9 @@ class Minimizer { //don't check mismatch if possible issues, as then mismatches would be expected checkResultMismatches(it, ei) } + if (config.handleFlakiness && ei != null){ + flakinessDetector.checkConsistency(it, ei) + } ei } diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/SearchAlgorithm.kt b/core/src/main/kotlin/org/evomaster/core/search/service/SearchAlgorithm.kt index 695dbd693a..3cb340da1d 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/service/SearchAlgorithm.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/service/SearchAlgorithm.kt @@ -38,6 +38,9 @@ abstract class SearchAlgorithm where T : Individual { @Inject private lateinit var minimizer: Minimizer + @Inject + private lateinit var flakinessDetector: FlakinessDetector + @Inject private lateinit var ssu: SearchStatusUpdater @@ -111,6 +114,8 @@ abstract class SearchAlgorithm where T : Individual { minimizer.simplifyActions() val seconds = minimizer.passedTimeInSecond() LoggingUtil.getInfoLogger().info("Minimization phase took $seconds seconds") + } else if (config.handleFlakiness){ + flakinessDetector.reexecuteToDetectFlakiness() } if(config.addPreDefinedTests) { diff --git a/core/src/test/kotlin/org/evomaster/core/output/TestCaseWriterTest.kt b/core/src/test/kotlin/org/evomaster/core/output/TestCaseWriterTest.kt index 6be68db0fc..9d2431c9cd 100644 --- a/core/src/test/kotlin/org/evomaster/core/output/TestCaseWriterTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/output/TestCaseWriterTest.kt @@ -1392,4 +1392,183 @@ public void test() throws Exception { assertEquals(expectedLines, lines.toString()) } + + @Test + fun testTestFlakyBodyForRestCallResponses(){ + val fooAction = RestCallAction("1", HttpVerb.GET, RestPath("/foo"), mutableListOf()) + + val (format, baseUrlOfSut, ei) = buildResourceEvaluatedIndividual( + dbInitialization = mutableListOf(), + groups = mutableListOf( + (mutableListOf() to mutableListOf(fooAction)) + ), + format = OutputFormat.JAVA_JUNIT_5 + ) + + val fooResult = ei.seeResult(fooAction.getLocalId()) as RestCallResult + fooResult.setTimedout(false) + fooResult.setStatusCode(200) + fooResult.setBody(""" + { + "p0":[1,2], + "p1":{}, + "p2":{ + "id":"foo", + "properties":[ + {}, + { + "name":"mapProperty1", + "type":"string", + "value":"one" + }, + { + "name":"mapProperty2", + "type":"string", + "value":"two" + }], + "empty":{} + } + } + """.trimIndent()) + fooResult.setBodyType(MediaType.APPLICATION_JSON_TYPE) + + + val barResult = RestCallResult(fooAction.getLocalId()) + barResult.setTimedout(false) + barResult.setStatusCode(500) + barResult.setBody(""" + { + "p0":[1,2,3], + "p1":{}, + "p2":{ + "id":"foo", + "properties":[ + {}, + { + "name":"flaky1", + "type":"string", + "value":"flaky2" + }, + { + "name":"mapProperty2", + "type":"string", + "value":"two" + }, + { + "name":"flaky3", + "type":"string", + "value":"two" + }], + "empty":{ + "flakyField4": 42 + } + } + } + """.trimIndent()) + barResult.setBodyType(MediaType.APPLICATION_JSON_TYPE) + + fooResult.setFlakiness(barResult) + + val config = getConfig(format) + config.handleFlakiness = true + + val test = TestCase(test = ei, name = "test") + + val writer = RestTestCaseWriter(config, PartialOracles()) + val lines = writer.convertToCompilableTestCode( test, baseUrlOfSut) + + val expectedLines = """ +@Test +public void test() throws Exception { + + given().accept("*/*") + .get(baseUrlOfSut + "/foo") + .then() + // Flaky Status Code: 200 vs. 500 + // .statusCode(200) + .assertThat() + .contentType("application/json") + // Flaky size of 'p0': 2 vs. 3 + // .body("'p0'.size()", equalTo(2)) + .body("'p0'[0]", numberMatches(1.0)) + .body("'p0'[1]", numberMatches(2.0)) + .body("'p1'.isEmpty()", is(true)) + // Flaky size of 'p2'.'properties': 3 vs. 4 + // .body("'p2'.'properties'.size()", equalTo(3)) + .body("'p2'.'properties'[0].isEmpty()", is(true)) + // Flaky value of field "'p2'.'properties'[1].'name'": mapProperty1 vs. flaky1 + // .body("'p2'.'properties'[1].'name'", containsString("mapProperty1")) + .body("'p2'.'properties'[1].'type'", containsString("string")) + // Flaky value of field "'p2'.'properties'[1].'value'": one vs. flaky2 + // .body("'p2'.'properties'[1].'value'", containsString("one")) + .body("'p2'.'properties'[2].'name'", containsString("mapProperty2")) + .body("'p2'.'properties'[2].'type'", containsString("string")) + .body("'p2'.'properties'[2].'value'", containsString("two")) + // Flaky mismatched size of fields for Object 'p2'.'empty': 0 vs. 1 + ; // .body("'p2'.'empty'.isEmpty()", is(true)) +} + +""".trimIndent() + assertEquals(expectedLines, lines.toString()) + } + + @Test + fun testTestFlakyListForRestCallResponses(){ + val fooAction = RestCallAction("1", HttpVerb.GET, RestPath("/foo"), mutableListOf()) + + val (format, baseUrlOfSut, ei) = buildResourceEvaluatedIndividual( + dbInitialization = mutableListOf(), + groups = mutableListOf( + (mutableListOf() to mutableListOf(fooAction)) + ), + format = OutputFormat.JAVA_JUNIT_5 + ) + + val fooResult = ei.seeResult(fooAction.getLocalId()) as RestCallResult + fooResult.setTimedout(false) + fooResult.setStatusCode(200) + fooResult.setBody(""" + ["foo", "bar"] + """.trimIndent()) + fooResult.setBodyType(MediaType.APPLICATION_JSON_TYPE) + + + val barResult = RestCallResult(fooAction.getLocalId()) + barResult.setTimedout(false) + barResult.setStatusCode(500) + barResult.setBody(""" + ["foo", "abc", "bar"] + """.trimIndent()) + barResult.setBodyType(MediaType.APPLICATION_JSON_TYPE) + + fooResult.setFlakiness(barResult) + + val config = getConfig(format) + config.handleFlakiness = true + + val test = TestCase(test = ei, name = "test") + + val writer = RestTestCaseWriter(config, PartialOracles()) + val lines = writer.convertToCompilableTestCode( test, baseUrlOfSut) + + val expectedLines = """ +@Test +public void test() throws Exception { + + given().accept("*/*") + .get(baseUrlOfSut + "/foo") + .then() + // Flaky Status Code: 200 vs. 500 + // .statusCode(200) + .assertThat() + .contentType("application/json") + // Flaky size of : 2 vs. 3 + // .body("size()", equalTo(2)) + // Flaky Body: "foo", "bar" vs. "foo", "abc", "bar" + ; // .body("", hasItems("foo", "bar")) +} + +""".trimIndent() + assertEquals(expectedLines, lines.toString()) + } } diff --git a/core/src/test/kotlin/org/evomaster/core/problem/rest/RestCallResultTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/rest/RestCallResultTest.kt index fb182e77d6..f017f027a8 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/rest/RestCallResultTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/rest/RestCallResultTest.kt @@ -2,12 +2,44 @@ package org.evomaster.core.problem.rest import org.evomaster.core.problem.rest.data.RestCallResult import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotNull import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Test +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.Arguments +import org.junit.jupiter.params.provider.MethodSource +import java.util.stream.Stream import javax.ws.rs.core.MediaType internal class RestCallResultTest { + companion object{ + @JvmStatic + fun getFlakyBodyDataProvider(): Stream { + + return Stream.of( + Arguments.of("42", "-1"), + Arguments.of("{\"id\":\"42\"}", "{\"id\":\"735\"}"), + Arguments.of(""" + { + "solid": "a", + "timid": "b", + "fooId": "c", + "void": "d" + } + """, """ + { + "solid": "a", + "timid": "b", + "fooId": "d", + "void": "d" + } + """), + ) + + } + } + @Test fun givenAStringIdWhenGetResourceIdThenItIsReturnedAsString() { val rc = RestCallResult("", false) @@ -97,4 +129,53 @@ internal class RestCallResultTest { assertEquals("42", res.value) assertEquals("/", res.pointer) } + + @ParameterizedTest + @MethodSource("getFlakyBodyDataProvider") + fun testSetFlakinessInBody(same: String, diff: String){ + val body = createCallResult(same) + val same = createCallResult(same) + val diff = createCallResult(diff) + + body.setFlakiness(same) + assertNotNull(body.getBodyType()) + + assertNull(body.getFlakyBody()) + assertNull(body.getFlakyBodyType()) + + body.setFlakiness(diff) + assertEquals(diff.getBody(), body.getFlakyBody()) + assertNull(body.getFlakyBodyType()) + + } + + + @Test + fun testSetFlakiness(){ + val code = 201 + val diffcode = 500 + val msg = "hello" + val diffmsg = "hello!" + + val body = RestCallResult("1") + val same = RestCallResult("2") + val diff = RestCallResult("3") + + body.setStatusCode(code) + same.setStatusCode(code) + diff.setStatusCode(diffcode) + + body.setErrorMessage(msg) + same.setErrorMessage(msg) + diff.setErrorMessage(diffmsg) + + body.setFlakiness(same) + assertNull(body.getFlakyStatusCode()) + assertNull(body.getFlakyErrorMessage()) + + body.setFlakiness(diff) + assertEquals(diffcode, body.getFlakyStatusCode()) + assertEquals(diffmsg, body.getFlakyErrorMessage()) + + } } diff --git a/docs/options.md b/docs/options.md index a3ead34bf7..0e199a3dc6 100644 --- a/docs/options.md +++ b/docs/options.md @@ -276,6 +276,7 @@ There are 3 types of options: |`externalServiceIP`| __String__. User provided external service IP. When EvoMaster mocks external services, mock server instances will run on local addresses starting from this provided address. Min value is 127.0.0.4. Lower values like 127.0.0.2 and 127.0.0.3 are reserved. *Constraints*: `regex (?!^0*127(\.0*0){2}\.0*[0123]$)^0*127(\.0*(25[0-5]\|2[0-4][0-9]\|1?[0-9]?[0-9])){3}$`. *Default value*: `127.0.0.4`.| |`externalServiceIPSelectionStrategy`| __Enum__. Specify a method to select the first external service spoof IP address. *Valid values*: `NONE, DEFAULT, USER, RANDOM`. *Default value*: `NONE`.| |`generateSqlDataWithDSE`| __Boolean__. Enable EvoMaster to generate SQL data with direct accesses to the database. Use Dynamic Symbolic Execution. *Default value*: `false`.| +|`handleFlakiness`| __Boolean__. Specify whether to detect flakiness and handle the flakiness in assertions during post handling of fuzzing. *Default value*: `false`.| |`heuristicsForSQLAdvanced`| __Boolean__. If using SQL heuristics, enable more advanced version. *Default value*: `false`.| |`httpOracles`| __Boolean__. Extra checks on HTTP properties in returned responses, used as automated oracles to detect faults. *Default value*: `false`.| |`initStructureMutationProbability`| __Double__. Probability of applying a mutation that can change the structure of test's initialization if it has. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.0`.|