Visitors rework, anyOf allOf oneOf support

README.md

@@ -1,5 +1,5 @@
 # OpenApi Codec 
-![version](https://img.shields.io/badge/version-0.2.0-blue.svg) 
+![version](https://img.shields.io/badge/version-0.3.0-blue.svg) 
 
 This microservice can validate open api dictionary, encode th2 messages or decode http body.
 
@@ -140,10 +140,13 @@ Result of decode:
 
 ### Codec configs:
 
-* checkUndefinedFields - Enable or Disable warnings for all undefined fields inside object structures, true by default.
+* failOnUndefinedFields - fail on undefined fields inside object structures, (`true` by default).
+* dateFormat - setup format of processed date (`yyyy-MM-dd Z` by default)
+* dateTimeFormat - setup format of processed date-time (`yyyy/MM/dd HH:mm:ss` by default)
+
 
 **validationSettings (open api dictionary)**
-* enableRecommendations - Enable or Disable recommendations, true by default.
+* enableRecommendations - Enable or Disable recommendations, `true` by default.
 * enableApacheNginxUnderscoreRecommendation - Enable or Disable the recommendation check for Apache/Nginx potentially ignoring header with underscore by default.
 * enableOneOfWithPropertiesRecommendation - Enable or Disable the recommendation check for schemas containing properties and oneOf definitions.
 * enableUnusedSchemasRecommendation - Enable or Disable the recommendation check for unused schemas.
@@ -152,8 +155,8 @@ Result of decode:
 * enableInvalidTypeRecommendation - Enable or Disable the recommendation check for the 'type' attribute.
 
 **dictionaryParseOption (open api dictionary)**
-* resolve - true by default;
-* resolveCombinators - true by default;
+* resolve - `true` by default;
+* resolveCombinators - will combine some schemas into one due allOf statement (`false` by default);
 * resolveFully;
 * flatten;
 * flattenComposedSchemas;
@@ -175,6 +178,13 @@ May be empty due to missing required fields
 
 ## Release notes
 
+### 0.3.0
+
++ Feature: anyOf, allOf, oneOf support for objects and arrays
++ Feature: Date format support
++ Feature: DateTime format support
++ Fix: Reworked visitor interface
+
 ### 0.2.0
 
 + Feature: check for undefined fields and throw errors if they are found

gradle.properties

@@ -1,6 +1,6 @@
 kotlin.code.style=official
 
 kotlin_version=1.5.31
-release_version=0.2.0
+release_version=0.3.0
 
 vcs_url=https://github.com/th2-net/th2-codec-open-api

src/main/kotlin/com/exactpro/th2/codec/openapi/OpenApiCodec.kt

@@ -23,12 +23,14 @@ import com.exactpro.th2.codec.openapi.schemacontainer.RequestContainer
 import com.exactpro.th2.codec.openapi.schemacontainer.ResponseContainer
 import com.exactpro.th2.codec.openapi.throwable.DecodeException
 import com.exactpro.th2.codec.openapi.throwable.EncodeException
+import com.exactpro.th2.codec.openapi.utils.containingFormatOrNull
 import com.exactpro.th2.codec.openapi.utils.extractType
+import com.exactpro.th2.codec.openapi.utils.getByMethod
 import com.exactpro.th2.codec.openapi.utils.getEndPoint
 import com.exactpro.th2.codec.openapi.utils.getMethods
-import com.exactpro.th2.codec.openapi.utils.getSchema
 import com.exactpro.th2.codec.openapi.writer.SchemaWriter
 import com.exactpro.th2.codec.openapi.writer.visitors.VisitorFactory
+import com.exactpro.th2.codec.openapi.writer.visitors.VisitorSettings
 import com.exactpro.th2.common.grpc.MessageGroup
 import com.exactpro.th2.common.grpc.RawMessage
 import com.exactpro.th2.common.message.plusAssign
@@ -49,12 +51,16 @@ import com.exactpro.th2.common.message.sessionAlias
 import io.swagger.v3.oas.models.PathItem
 import io.swagger.v3.oas.models.parameters.Parameter
 import mu.KotlinLogging
+import java.lang.IllegalStateException
+import java.text.SimpleDateFormat
+import java.time.format.DateTimeFormatter
 
-class OpenApiCodec(private val dictionary: OpenAPI, settings: OpenApiCodecSettings) : IPipelineCodec {
+class OpenApiCodec(private val dictionary: OpenAPI, val settings: OpenApiCodecSettings) : IPipelineCodec {
 
     private val typeToSchema: Map<String, HttpRouteContainer>
     private val patternToPathItem: List<Pair<UriPattern, PathItem>>
-    private val schemaWriter = SchemaWriter(dictionary, settings.checkUndefinedFields)
+    private val schemaWriter = SchemaWriter(dictionary)
+    private val visitorSettings = VisitorSettings(dictionary, SimpleDateFormat(settings.dateFormat), DateTimeFormatter.ofPattern(settings.dateTimeFormat))
 
     init {
         val mapForName = mutableMapOf<String, HttpRouteContainer>()
@@ -125,11 +131,13 @@ class OpenApiCodec(private val dictionary: OpenAPI, settings: OpenApiCodecSettin
 
             val container = checkNotNull(typeToSchema[messageType]) { "There no message $messageType in dictionary" }
 
-            builder += createHeaderMessage(container, parsedMessage).apply {
-                if (parsedMessage.hasParentEventId()) parentEventId = parsedMessage.parentEventId
-                sessionAlias = parsedMessage.sessionAlias
-                metadataBuilder.putAllProperties(parsedMessage.metadata.propertiesMap)
-                LOGGER.trace { "Created header message for ${parsedMessage.messageType}: ${this.messageType}" }
+            if (container.headers.isNotEmpty() || container.body == null) {
+                builder += createHeaderMessage(container, parsedMessage).apply {
+                    if (parsedMessage.hasParentEventId()) parentEventId = parsedMessage.parentEventId
+                    sessionAlias = parsedMessage.sessionAlias
+                    metadataBuilder.putAllProperties(parsedMessage.metadata.propertiesMap)
+                    LOGGER.trace { "Created header message for ${parsedMessage.messageType}: ${this.messageType}" }
+                }
             }
 
             try {
@@ -150,10 +158,13 @@ class OpenApiCodec(private val dictionary: OpenAPI, settings: OpenApiCodecSettin
         }
 
         LOGGER.debug { "Start of message encoding: ${message.messageType}" }
-        checkNotNull(messageSchema.type) {"Type of schema [${messageSchema.name}] wasn't filled"}
 
-        val visitor = VisitorFactory.createEncodeVisitor(container.bodyFormat!!, messageSchema.type, message)
-        schemaWriter.traverse(visitor, messageSchema)
+        val visitor = try {
+            VisitorFactory.createEncodeVisitor(container.bodyFormat!!, messageSchema, message, visitorSettings)
+        } catch (e: Exception) {
+            throw IllegalStateException("Cannot create encode visitor for message: ${message.messageType} - ${container.body}", e)
+        }
+        schemaWriter.traverse(visitor, messageSchema, settings.checkUndefinedFields)
         val result = visitor.getResult()
 
         LOGGER.trace { "Result of encoded message ${message.messageType}: $result" }
@@ -165,9 +176,22 @@ class OpenApiCodec(private val dictionary: OpenAPI, settings: OpenApiCodecSettin
                 container.fillHttpMetadata(metadataBuilder)
                 metadataBuilder.apply {
                     putAllProperties(message.metadata.propertiesMap)
-                    this.id = metadata.id
-                    this.timestamp = metadata.timestamp
+                    this.id = message.metadata.id
+                    this.timestamp = message.metadata.timestamp
                     protocol = message.metadata.protocol
+
+                    when (container) {
+                        is ResponseContainer -> this.putProperties(CODE_FIELD, container.code)
+                        is RequestContainer -> {
+                            this.putProperties(URI_FIELD, if (container.params.isNotEmpty()) {
+                                container.uriPattern.resolve(container.params, message.getMessage(URI_PARAMS_FIELD).orEmpty().fieldsMap.mapValues { it.value.simpleValue })
+                            } else {
+                                container.uriPattern.pattern
+                            })
+                            this.putProperties(METHOD_FIELD, container.method)
+                        }
+                        else -> error("Wrong type of Http Route Container")
+                    }
                 }
                 body = result
             }.build()
@@ -189,7 +213,7 @@ class OpenApiCodec(private val dictionary: OpenAPI, settings: OpenApiCodecSettin
             builder += runCatching {
                 decodeBody(message, messages[1].rawMessage!!)
             }.getOrElse {
-                throw DecodeException("Cannot parse body of http message", it)
+                throw DecodeException("Cannot decode body of http message", it)
             }
         }
 
@@ -198,57 +222,77 @@ class OpenApiCodec(private val dictionary: OpenAPI, settings: OpenApiCodecSettin
 
     private fun decodeBody(header: Message, rawMessage: RawMessage): Message {
         val body = rawMessage.body
+        val (messageType, container) = searchContainer(header, rawMessage.metadata)
+        val schema = dictionary.getEndPoint(checkNotNull(container.body) { "Container: $messageType did not contain schema body" })
+        val format = checkNotNull(container.bodyFormat) { "Container: $messageType did not contain schema bodyFormat" }
+
+        val visitor = try {
+            VisitorFactory.createDecodeVisitor(format, schema, body, visitorSettings)
+        } catch (e: Exception) {
+            throw IllegalStateException("Cannot create decode visitor for message: ${header.messageType} - ${container.body}", e)
+        }
+
+        schemaWriter.traverse(visitor, schema, settings.checkUndefinedFields)
+        return visitor.getResult().apply {
+            if(rawMessage.hasParentEventId()) parentEventId = rawMessage.parentEventId
+            sessionAlias = rawMessage.sessionAlias
+            this.messageType = messageType
+            metadataBuilder.apply {
+                id = rawMessage.metadata.id
+                timestamp = rawMessage.metadata.timestamp
+                protocol = rawMessage.metadata.protocol
+                putAllProperties(rawMessage.metadata.propertiesMap)
+            }
+        }.build()
+    }
 
-        val bodyFormat = header.getList(HEADERS_FIELD)
+    private fun searchContainer(header: Message, rawMetadata: RawMessageMetadata): Pair<String, HttpRouteContainer> {
+        val headerFormat = header.getList(HEADERS_FIELD)
             ?.first { it.messageValue.getString("name") == "Content-Type" }
             ?.messageValue
             ?.getString("value")
             ?.extractType() ?: "null"
 
-        val uri: String
         val method: String
+        val schemaFormat: String
         var code = ""
 
-        val pairFound: Pair<UriPattern, PathItem>
+        val messageType: String
+
+        val patternToPathItem: Pair<UriPattern, PathItem>
 
-        val messageSchema = when (header.messageType) {
+        when (header.messageType) {
             RESPONSE_MESSAGE -> {
-                uri = requireNotNull(rawMessage.metadata.propertiesMap[URI_PROPERTY]) { "URI property in metadata from response is required" }
-                method = requireNotNull(rawMessage.metadata.propertiesMap[METHOD_PROPERTY]?.lowercase()) { "Method property in metadata from response is required" }
+                val uri = requireNotNull(rawMetadata.propertiesMap[URI_PROPERTY]) { "URI property in metadata from response is required" }
+                method = requireNotNull(rawMetadata.propertiesMap[METHOD_PROPERTY]?.lowercase()) { "Method property in metadata from response is required" }
                 code = requireNotNull(header.getString(STATUS_CODE_FIELD)) { "Code status field required inside of http response message" }
 
-                pairFound = checkNotNull(patternToPathItem.firstOrNull { it.first.matches(uri) }) { "Cannot find path-item by uri: $uri" }
-                dictionary.getEndPoint(pairFound.second.getSchema(method, code, bodyFormat))
+                patternToPathItem = checkNotNull(this.patternToPathItem.firstOrNull { it.first.matches(uri) }) { "Cannot find path-item by uri: $uri" }
+                val content = patternToPathItem.second.getByMethod(method)?.responses?.get(code)?.content
+                schemaFormat = checkNotNull(content?.containingFormatOrNull(headerFormat)) {
+                    "Schema Response [${patternToPathItem.first.pattern}] with method: [$method] and code: [$code] did not contain type $headerFormat"
+                }
+                messageType = combineName(patternToPathItem.first.pattern, method, code, schemaFormat)
             }
             REQUEST_MESSAGE -> {
-                uri = requireNotNull(header.getString(URI_FIELD)) { "URI field in request is required" }
+                val uri = requireNotNull(header.getString(URI_FIELD)) { "URI field in request is required" }
                 method = requireNotNull(header.getString(METHOD_FIELD)) { "Method field in request is required" }
 
-                pairFound = checkNotNull(patternToPathItem.firstOrNull { it.first.matches(uri) }) { "Cannot find path-item by uri: $uri" }
-                dictionary.getEndPoint(pairFound.second.getSchema(method, null, bodyFormat))
+                patternToPathItem = checkNotNull(this.patternToPathItem.firstOrNull { it.first.matches(uri) }) { "Cannot find path-item by uri: $uri" }
+                val content = patternToPathItem.second.getByMethod(method)?.requestBody?.content
+                schemaFormat = checkNotNull(content?.containingFormatOrNull(headerFormat)) {
+                    "Schema Response [${patternToPathItem.first.pattern}] with method: [$method] did not contain type $headerFormat"
+                }
+                messageType = combineName(patternToPathItem.first.pattern, method, code, schemaFormat)
             }
-            else -> error("Unsupported message type: ${header.messageType}")
+            else -> error("Unsupported header message type: ${header.messageType}")
         }
 
-        checkNotNull(messageSchema.type) { "Type of schema [${messageSchema.name}] wasn't filled" }
+        val container = checkNotNull(typeToSchema[messageType]) { "Container for path: [${patternToPathItem.first.pattern}] with method: [$method], code: [$code] and type [$schemaFormat] wasn't found" }
 
-        val type = combineName(pairFound.first.pattern, method, code, bodyFormat)
+        LOGGER.debug { "Container for ${header.messageType} with messageType: $messageType was found" }
 
-        LOGGER.debug { "Schema for ${header.messageType} with type-name: $type was found" }
-
-        val visitor = VisitorFactory.createDecodeVisitor(bodyFormat, messageSchema.type, body)
-        schemaWriter.traverse(visitor, messageSchema)
-        return visitor.getResult().apply {
-            if(rawMessage.hasParentEventId()) parentEventId = rawMessage.parentEventId
-            sessionAlias = rawMessage.sessionAlias
-            this.messageType = type
-            metadataBuilder.apply {
-                id = rawMessage.metadata.id
-                timestamp = metadata.timestamp
-                protocol = rawMessage.metadata.protocol
-                putAllProperties(rawMessage.metadata.propertiesMap)
-            }
-        }.build()
+        return messageType to container
     }
 
 
@@ -291,24 +335,22 @@ class OpenApiCodec(private val dictionary: OpenAPI, settings: OpenApiCodecSettin
                 })
             }
 
-            if (container.headers.isNotEmpty()) {
-                val headerMessage = message.getMessage(HEADER_PARAMS_FIELD).orEmpty()
-                container.headers.forEach { (name, value) ->
-                    headerMessage[name]?.let { header ->
-                        headers.add(message().apply {
-                            addField("name", name)
-                            addField("value", header.simpleValue)
-                        })
-                    } ?: run { if (value.required) error("Header param [$name] is required for ${message.messageType} message") }
-                }
+            val headerMessage = message.getMessage(HEADER_PARAMS_FIELD).orEmpty()
+            container.headers.forEach { (name, value) ->
+                headerMessage[name]?.let { header ->
+                    headers.add(message().apply {
+                        addField("name", name)
+                        addField("value", header.simpleValue)
+                    })
+                } ?: run { if (value.required) error("Header param [$name] is required for ${message.messageType} message") }
             }
 
             addField(HEADERS_FIELD, headers)
             this.metadataBuilder.protocol = HEADER_PROTOCOL
         }
     }
 
-    fun combineName(vararg steps: String) = steps.asSequence().flatMap { it.split(COMBINER_REGEX) }.joinToString("") { it.lowercase().capitalize() }
+    fun combineName(vararg steps: String) = steps.asSequence().flatMap { it.replace("*","Any").split(COMBINER_REGEX) }.joinToString("") { it.lowercase().capitalize() }
 
     companion object {
         private val LOGGER = KotlinLogging.logger { }

src/main/kotlin/com/exactpro/th2/codec/openapi/OpenApiCodecSettings.kt

@@ -27,7 +27,9 @@ class OpenApiCodecSettings : IPipelineCodecSettings {
     val dictionaryParseOption: ParseOptions = ParseOptions().apply {
         isResolve = true
         isResolveFully = true
-        isResolveCombinators = true
+        isResolveCombinators = false
     }
     val checkUndefinedFields: Boolean = true
+    val dateFormat: String = "yyyy-MM-dd Z"
+    val dateTimeFormat: String = "yyyy/MM/dd HH:mm:ss"
 }

src/main/kotlin/com/exactpro/th2/codec/openapi/utils/CommonUtils.kt

@@ -18,6 +18,7 @@ package com.exactpro.th2.codec.openapi.utils
 
 import com.exactpro.th2.common.grpc.Message
 import com.exactpro.th2.common.grpc.Value
+import com.exactpro.th2.common.grpc.Value.KindCase.NULL_VALUE
 import com.exactpro.th2.common.message.get
 import com.exactpro.th2.common.message.messageType
 import com.exactpro.th2.common.value.getString
@@ -27,7 +28,9 @@ import com.exactpro.th2.common.value.getString
  * @param required check if value is required, used only if extracted value was null
  */
 fun Message.getField(fieldName: String, required: Boolean): Value? = this[fieldName].apply {
-    if (required) checkNotNull(this) { "Field [$fieldName] is required for message [$messageType]" }
+    if (required) {
+        check(this != null && this.kindCase != NULL_VALUE) { "Field [$fieldName] is required for message [$messageType]" }
+    }
 }
 
 fun Value.getBoolean(): Boolean? = this.getString()?.toBooleanStrictOrNull()

src/main/kotlin/com/exactpro/th2/codec/openapi/utils/JsonUtils.kt

@@ -78,21 +78,6 @@ fun JsonNode.validateAsBigDecimal(): BigDecimal = when {
     else -> error("Cannot convert $this to BigDecimal")
 }
 
-fun JsonNode.validateAsInteger(): Int = when {
-    isNumber -> asInt()
-    else -> error("Cannot convert $this to Int")
-}
-
-fun JsonNode.validateAsDouble(): Double = when {
-    isNumber -> asDouble()
-    else -> error("Cannot convert $this to Double")
-}
-
-fun JsonNode.validateAsFloat(): Float = when {
-    isNumber -> asText().toFloat()
-    else -> error("Cannot convert $this to Float")
-}
-
 fun JsonNode.validateAsObject(): ObjectNode = when {
     isObject -> this as ObjectNode
     else -> error("Cannot convert $this to Object")