Aggregation on nested fields

.circleci/config.yml

@@ -27,8 +27,15 @@ jobs:
       - restore_cache:
           key: kp-dependency-test-cache-{{ checksum "pom.xml" }}
       - run:
-          name: Setup environment and run tests
-          command: bash vmsetup.sh
+          name: Save test results
+          command: |
+            mkdir -p ~/test-results/junit/
+            find . -type f -regex ".*/target/surefire-reports/.*xml" -exec cp {} ~/test-results/junit/ \;
+          when: always
+      - store_test_results:
+          path: ~/test-results
+      - store_artifacts:
+          path: ~/test-results/junit
       - save_cache:
           paths:
             - ~/.m2

content-api/content-actors/src/main/scala/org/sunbird/content/actors/ContentActor.scala

@@ -3,8 +3,8 @@ package org.sunbird.content.actors
 import java.util
 import java.util.concurrent.CompletionException
 import java.io.File
-
 import org.apache.commons.io.FilenameUtils
+
 import javax.inject.Inject
 import org.apache.commons.lang3.StringUtils
 import org.sunbird.`object`.importer.{ImportConfig, ImportManager}
@@ -16,6 +16,7 @@ import org.sunbird.common.{ContentParams, Platform, Slug}
 import org.sunbird.common.dto.{Request, Response, ResponseHandler}
 import org.sunbird.common.exception.ClientException
 import org.sunbird.content.dial.DIALManager
+import org.sunbird.content.mgr.ImportManager
 import org.sunbird.util.RequestUtil
 import org.sunbird.content.upload.mgr.UploadManager
 import org.sunbird.graph.OntologyEngineContext

content-api/content-actors/src/main/scala/org/sunbird/content/util/RetireManager.scala

@@ -57,7 +57,7 @@ object RetireManager {
     private def updateNodesToRetire(request: Request, updateMetadataMap: util.Map[String, AnyRef])(implicit ec: ExecutionContext, oec: OntologyEngineContext): Future[Response] = {
         RedisCache.delete(request.get(ContentConstants.IDENTIFIER).asInstanceOf[String])
         val updateReq = new Request(request)
-        updateReq.put(ContentConstants.IDENTIFIERS, java.util.Arrays.asList(request.get(ContentConstants.IDENTIFIER).asInstanceOf[String], request.get(ContentConstants.IDENTIFIER).asInstanceOf[String] + HierarchyConstants.IMAGE_SUFFIX))
+        updateReq.put(ContentConstants.IDENTIFIERS, java.util.Arrays.asList(request.get(ContentConstants.IDENTIFIER).asInstanceOf[String]))
         updateReq.put(ContentConstants.METADATA, updateMetadataMap)
         DataNode.bulkUpdate(updateReq).map(node => ResponseHandler.OK())
     }

content-api/content-service/app/controllers/v3/ContentController.scala

@@ -8,14 +8,16 @@ import org.sunbird.models.UploadParams
 import org.sunbird.common.dto.ResponseHandler
 import play.api.mvc.ControllerComponents
 import utils.{ActorNames, ApiId, JavaJsonUtils}
-
+import org.slf4j.LoggerFactory
+import com.fasterxml.jackson.databind.ObjectMapper
 import scala.collection.JavaConverters._
 
 import scala.concurrent.{ExecutionContext, Future}
 
 @Singleton
 class ContentController @Inject()(@Named(ActorNames.CONTENT_ACTOR) contentActor: ActorRef, @Named(ActorNames.COLLECTION_ACTOR) collectionActor: ActorRef, cc: ControllerComponents, actorSystem: ActorSystem)(implicit exec: ExecutionContext) extends BaseController(cc) {
-
+    val LOG = LoggerFactory.getLogger(classOf[ContentController])
+    val mapper = new ObjectMapper();
     val objectType = "Content"
     val schemaName: String = "content"
     val version = "1.0"
@@ -24,6 +26,7 @@ class ContentController @Inject()(@Named(ActorNames.CONTENT_ACTOR) contentActor:
         val headers = commonHeaders()
         val body = requestBody()
         val content = body.getOrDefault("content", new java.util.HashMap()).asInstanceOf[java.util.Map[String, Object]];
+        LOG.warn("Content create request :  {}", mapper.writeValueAsString(content))
         content.putAll(headers)
         val contentRequest = getRequest(content, headers, "createContent", true)
         setRequestContext(contentRequest, version, objectType, schemaName)
@@ -56,6 +59,7 @@ class ContentController @Inject()(@Named(ActorNames.CONTENT_ACTOR) contentActor:
         val headers = commonHeaders()
         val body = requestBody()
         val content = body.getOrDefault("content", new java.util.HashMap()).asInstanceOf[java.util.Map[String, Object]];
+        LOG.warn("Content Update request :  {}", mapper.writeValueAsString(content))
         content.putAll(headers)
         val contentRequest = getRequest(content, headers, "updateContent")
         setRequestContext(contentRequest, version, objectType, schemaName)
@@ -69,6 +73,7 @@ class ContentController @Inject()(@Named(ActorNames.CONTENT_ACTOR) contentActor:
         body.putAll(headers)
         val contentRequest = getRequest(body, headers, "addHierarchy")
         contentRequest.put("mode", "edit");
+        LOG.warn("Add Hierarchy request :  {}", mapper.writeValueAsString(contentRequest))
         setRequestContext(contentRequest, version, objectType, schemaName)
         getResult(ApiId.ADD_HIERARCHY, collectionActor, contentRequest)
     }
@@ -89,6 +94,7 @@ class ContentController @Inject()(@Named(ActorNames.CONTENT_ACTOR) contentActor:
         val data = body.getOrDefault("data", new java.util.HashMap()).asInstanceOf[java.util.Map[String, Object]]
         data.putAll(headers)
         val contentRequest = getRequest(data, headers, "updateHierarchy")
+        LOG.warn("Update Hierarchy request :  {}", mapper.writeValueAsString(contentRequest))
         setRequestContext(contentRequest, version, objectType, schemaName)
         getResult(ApiId.UPDATE_HIERARCHY, collectionActor, contentRequest)
     }

content-api/content-service/test/modules/TestModule.scala

@@ -18,6 +18,7 @@ class TestModule extends AbstractModule with AkkaGuiceSupport {
         bindActor(classOf[TestActor], ActorNames.CHANNEL_ACTOR)
         bindActor(classOf[TestActor], ActorNames.CATEGORY_ACTOR)
         bindActor(classOf[TestActor], ActorNames.ASSET_ACTOR)
+
         bindActor(classOf[TestActor], ActorNames.APP_ACTOR)
         bindActor(classOf[TestActor], ActorNames.EVENT_SET_ACTOR)
         bindActor(classOf[TestActor], ActorNames.EVENT_ACTOR)

definition-scripts/Goals.sh

@@ -49,4 +49,4 @@ curl -L -X POST '{{host}}/object/category/definition/v4/create' \
       }
     }
   }
-}'
+}'

definition-scripts/Playlist.sh

@@ -49,4 +49,4 @@ curl -L -X POST '{{host}}/object/category/definition/v4/create' \
       }
     }
   }
-}'
+}'

definition-scripts/master_category_create

@@ -20,6 +20,18 @@ curl --location --request POST '{{host}}/object/category/v4/create' \
     }
 }'
 
+curl --location --request POST '{{host}}/object/category/v4/create' \
+--header 'Content-Type: application/json' \
+--data-raw '{
+    "request": {
+        "objectCategory": {
+            "name": "Goals",
+            "description":"Goals for Users"
+        }
+    }
+}'
+
+
 curl --location --request POST '{{host}}/object/category/v4/create' \
 --header 'Content-Type: application/json' \
 --data-raw '{
@@ -374,4 +386,4 @@ curl --location --request POST '{{host}}/object/category/v4/create' \
             "description":"FTB Question"
         }
     }
-}'
+}'

platform-modules/mimetype-manager/pom.xml

@@ -13,8 +13,22 @@
         <scala.major.version>2.11</scala.major.version>
         <play2.version>2.7.2</play2.version>
     </properties>
-
+  <!-- <repositories>
+         <repository>
+                  <id>cloud-store-sdk_1.3.1-beta</id>
+                  <name>cloud-store-sdk_1.3.1-beta</name>
+                  <url>https://oss.sonatype.org/content/repositories/orgsunbird-1104/</url>
+                  <snapshots>
+                  <enabled>false</enabled>
+                  </snapshots>
+         </repository>
+    </repositories>-->
     <dependencies>
+          <!--<dependency>
+               <groupId>org.sunbird</groupId>
+                <artifactId>cloud-store-sdk</artifactId>
+                <version>1.3.1-beta</version>
+        </dependency> -->
         <dependency>
             <groupId>org.sunbird</groupId>
             <artifactId>platform-common</artifactId>
@@ -143,4 +157,4 @@
         </plugins>
     </build>
 
-</project>
+</project>

platform-modules/mimetype-manager/src/main/scala/org/sunbird/cloudstore/StorageService.scala

@@ -34,7 +34,6 @@ class StorageService {
                 val storageSecret = Platform.config.getString("cephs3_storage_secret")
                 val endpoint = Platform.config.getString("cephs3_storage_endpoint")
                 storageService = StorageServiceFactory.getStorageService(new StorageConfig(storageType, storageKey, storageSecret, Option(endpoint)))
-
             }
             else throw new ServerException("ERR_INVALID_CLOUD_STORAGE", "Error while initialising cloud storage")
         }

platform-modules/mimetype-manager/src/main/scala/org/sunbird/mimetype/mgr/impl/HtmlMimeTypeMgrImpl.scala

@@ -8,14 +8,17 @@ import org.sunbird.common.exception.ClientException
 import org.sunbird.graph.dac.model.Node
 import org.sunbird.mimetype.mgr.{BaseMimeTypeManager, MimeTypeManager}
 import org.sunbird.telemetry.logger.TelemetryManager
+import org.sunbird.common.Platform
 
 import scala.concurrent.{ExecutionContext, Future}
 
 class HtmlMimeTypeMgrImpl(implicit ss: StorageService) extends BaseMimeTypeManager with MimeTypeManager {
 
     override def upload(objectId: String, node: Node, uploadFile: File, filePath: Option[String], params: UploadParams)(implicit ec: ExecutionContext): Future[Map[String, AnyRef]] = {
         validateUploadRequest(objectId, node, uploadFile)
-        if (isValidPackageStructure(uploadFile, List[String]("index.html"))) {
+        val indexHtmlValidation: Boolean = if (Platform.config.hasPath("indexHtmlValidation.env")) Platform.config.getBoolean("indexHtmlValidation.env") else false
+        val flag: Boolean = if (indexHtmlValidation) isValidPackageStructure(uploadFile, List[String]("index.html")) else true
+        if (flag) {
             val urls = uploadArtifactToCloud(uploadFile, objectId, filePath)
             node.getMetadata.put("s3Key", urls(IDX_S3_KEY))
             node.getMetadata.put("artifactUrl", urls(IDX_S3_URL))

schemas/asset/1.0/schema.json

@@ -1249,4 +1249,4 @@
             }
         }
     }
-}
+}

schemas/collection/1.0/schema.json

@@ -1269,4 +1269,4 @@
       }
     }
   }
-}
+}

schemas/content/1.0/schema.json

@@ -87,7 +87,8 @@
                 "audio/webm",
                 "audio/x-wav",
                 "audio/wav",
-                "application/json"
+                "application/json",
+                "application/quiz"
             ]
         },
         "osId": {
@@ -1370,4 +1371,4 @@
             }
         }
     }
-}
+}

search-api/search-actors/src/main/java/org/sunbird/actors/SearchActor.java

@@ -106,6 +106,9 @@ private SearchDTO getSearchDTO(Request request) throws Exception {
                 wordChainsRequest = false;
             List<Map> properties = new ArrayList<Map>();
             Map<String, Object> filters = (Map<String, Object>) req.get(SearchConstants.filters);
+            List<Map> multiFilterProperties = new ArrayList<Map>();
+            Map<String, Object> multiFilters = (Map<String, Object>) req.get("multiFilters");
+
             if (null == filters)
                 filters = new HashMap<>();
             if (filters.containsKey("tags")) {
@@ -239,6 +242,10 @@ private SearchDTO getSearchDTO(Request request) throws Exception {
             searchObj.setSortBy(sortBy);
             searchObj.setFacets(facets);
             searchObj.setProperties(properties);
+            if(multiFilters!=null){
+                multiFilterProperties.addAll(getSearchFilterProperties(multiFilters, wordChainsRequest));
+                searchObj.setMultiFilterProperties(multiFilterProperties);
+            }
             // Added Implicit Filter Properties To Support Collection content tagging to reuse by tenants.
             setImplicitFilters(filters, searchObj);
             searchObj.setLimit(limit);

search-api/search-core/src/main/java/org/sunbird/search/client/ElasticSearchUtil.java

@@ -49,6 +49,7 @@
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.nested.Nested;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.sunbird.common.Platform;
 import org.sunbird.common.exception.ServerException;
@@ -613,43 +614,50 @@ private static SearchSourceBuilder buildJsonForWildCardQuery(String textKeyWord,
 
 	@SuppressWarnings("unchecked")
 	public static Object getCountFromAggregation(Aggregations aggregations, List<Map<String, Object>> groupByList,
-			IESResultTransformer transformer) {
-
-		Map<String, Object> countMap = new HashMap<String, Object>();
-		if (aggregations != null) {
-			for (Map<String, Object> aggregationsMap : groupByList) {
-				Map<String, Object> parentCountMap = new HashMap<String, Object>();
-				String groupByParent = (String) aggregationsMap.get("groupByParent");
-				Terms terms = aggregations.get(groupByParent);
-				List<Map<String, Object>> parentGroupList = new ArrayList<Map<String, Object>>();
-				List<Bucket> buckets = (List<Bucket>) terms.getBuckets();
-				for (Bucket bucket : buckets) {
-					Map<String, Object> parentCountObject = new HashMap<String, Object>();
-					parentCountObject.put("count", bucket.getDocCount());
-					List<String> groupByChildList = (List<String>) aggregationsMap.get("groupByChildList");
-					Aggregations subAggregations = bucket.getAggregations();
-					if (null != groupByChildList && !groupByChildList.isEmpty() && null != subAggregations) {
-						Map<String, Object> groupByChildMap = new HashMap<String, Object>();
-						for (String groupByChild : groupByChildList) {
-							Terms subTerms = subAggregations.get(groupByChild);
-							List<Bucket> childBuckets = (List<Bucket>) subTerms.getBuckets();
-							Map<String, Long> childCountMap = new HashMap<String, Long>();
-							for (Bucket childBucket : childBuckets) {
-								childCountMap.put(childBucket.getKeyAsString(), childBucket.getDocCount());
-								groupByChildMap.put(groupByChild, childCountMap);
-							}
-						}
-						parentCountObject.putAll(groupByChildMap);
-					}
-					parentCountMap.put(bucket.getKeyAsString(), parentCountObject);
-					parentGroupList.add(parentCountMap);
-				}
-
-				countMap.put(groupByParent, parentCountMap);
-			}
-		}
-		return transformer.getTransformedObject(countMap);
-	}
+                                                 IESResultTransformer transformer) {
+        Map<String, Object> countMap = new HashMap<String, Object>();	
+        if (aggregations != null) {
+            for (Map<String, Object> aggregationsMap : groupByList) {
+                Map<String, Object> parentCountMap = new HashMap<String, Object>();
+                String groupByParent = (String) aggregationsMap.get("groupByParent");
+                Terms terms = null;
+                List<Bucket> buckets = null;
+                if (groupByParent.contains(".")) {
+
+	            Nested nested = aggregations.get(groupByParent.split("\\.")[0]);
+                    terms = nested.getAggregations().get(groupByParent.split("\\.")[1]);
+                } else {
+                    terms = aggregations.get(groupByParent);
+                }
+		buckets = (List<Bucket>)terms.getBuckets();
+                List<Map<String, Object>> parentGroupList = new ArrayList<Map<String, Object>>();
+                for (Bucket bucket : buckets) {
+                    Map<String, Object> parentCountObject = new HashMap<String, Object>();
+                    parentCountObject.put("count", bucket.getDocCount());
+                    List<String> groupByChildList = (List<String>) aggregationsMap.get("groupByChildList");
+                    Aggregations subAggregations = bucket.getAggregations();
+                    if (null != groupByChildList && !groupByChildList.isEmpty() && null != subAggregations) {
+                        Map<String, Object> groupByChildMap = new HashMap<String, Object>();
+                        for (String groupByChild : groupByChildList) {
+                            Terms subTerms = subAggregations.get(groupByChild);
+                            List<Bucket> childBuckets = (List<Bucket>) subTerms.getBuckets();
+                            Map<String, Long> childCountMap = new HashMap<String, Long>();
+                            for (Bucket childBucket : childBuckets) {
+                                childCountMap.put(childBucket.getKeyAsString(), childBucket.getDocCount());
+                                groupByChildMap.put(groupByChild, childCountMap);
+                            }
+                        }
+                        parentCountObject.putAll(groupByChildMap);
+                    }
+                    parentCountMap.put(bucket.getKeyAsString(), parentCountObject);
+                    parentGroupList.add(parentCountMap);
+                }
+
+                countMap.put(groupByParent, parentCountMap);
+            }
+        }
+        return transformer.getTransformedObject(countMap);
+    }
 
 	private static void registerShutdownHook() {
 		Runtime.getRuntime().addShutdownHook(new Thread() {
@@ -711,4 +719,4 @@ public static void bulkDeleteDocumentById(String indexName, String documentType,
 		}
 	}
 
-}
+}

search-api/search-core/src/main/java/org/sunbird/search/dto/SearchDTO.java

@@ -9,6 +9,7 @@ public class SearchDTO {
 
     @SuppressWarnings("rawtypes")
     private List<Map> properties;
+    private List<Map> multiFilterProperties;
     private List<String> facets;
     private List<String> fields;
     private Map<String, String> sortBy;
@@ -33,6 +34,14 @@ public SearchDTO(List<Map> properties, String operation, int limit) {
         this.operation = operation;
         this.limit = limit;
     }
+    public List<Map> getMultiFilterProperties() {
+        return multiFilterProperties;
+    }
+
+    public void setMultiFilterProperties(List<Map> multiFilterProperties) {
+        this.multiFilterProperties = multiFilterProperties;
+    }
+
     @SuppressWarnings("rawtypes")
     public List<Map> getProperties() {
         return properties;