merge Release 5.6.0 into 5.7.0

.circleci/config.yml

@@ -11,7 +11,7 @@ jobs:
       - run:
           name: Run build
           command: |
-            mvn clean install -DskipTests
+            mvn clean install -DskipTests -DCLOUD_STORE_GROUP_ID=$CLOUD_STORE_GROUP_ID -DCLOUD_STORE_ARTIFACT_ID=$CLOUD_STORE_ARTIFACT_ID -DCLOUD_STORE_VERSION=$CLOUD_STORE_VERSION
       - save_cache:
           paths:
             - ~/.m2

build/content-service/Jenkinsfile

@@ -21,11 +21,14 @@ node('build-slave') {
                 commit_hash = sh(script: 'git rev-parse --short HEAD', returnStdout: true).trim()
                 build_tag = sh(script: "echo " + params.github_release_tag.split('/')[-1] + "_" + commit_hash + "_" + env.BUILD_NUMBER, returnStdout: true).trim()
                 echo "build_tag: " + build_tag
+                cloud_store_group_id = params.CLOUD_STORE_GROUP_ID
+                cloud_store_artifact_id = params.CLOUD_STORE_ARTIFACT_ID
+                cloud_store_version = params.CLOUD_STORE_VERSION
 
                 stage('Build') {
                     env.NODE_ENV = "build"
                     print "Environment will be : ${env.NODE_ENV}"
-                    sh 'mvn clean install -DskipTests=true '
+                    sh 'mvn clean install -DskipTests=true -DCLOUD_STORE_GROUP_ID=' + cloud_store_group_id + ' -DCLOUD_STORE_ARTIFACT_ID=' + cloud_store_artifact_id + ' -DCLOUD_STORE_VERSION=' + cloud_store_version
 
                 }
 
@@ -47,4 +50,4 @@ node('build-slave') {
         currentBuild.result = "FAILURE"
         throw err
     }
-}
+}

content-api/content-actors/src/test/resources/application.conf

@@ -490,7 +490,7 @@ content.copy.props_to_remove=["downloadUrl", "artifactUrl", "variants",
   "concepts", "keywords", "reservedDialcodes", "dialcodeRequired", "leafNodes", "sYS_INTERNAL_LAST_UPDATED_ON", "prevStatus", "lastPublishedBy", "streamingUrl"]
 
 content.copy.origin_data=["name", "author", "license", "organisation"]
-content.h5p.library.path="https://s3.ap-south-1.amazonaws.com/ekstep-public-dev/content/templates/h5p-library-latest.zip"
+content.h5p.library.path="https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/templates/h5p-library-latest.zip"
 
 # DIAL Link
 dial_service {

docker-compose.yml

@@ -47,7 +47,7 @@ services:
       - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://127.0.0.1:2181     
 
   kafka:
-    image: 'wurstmeister/kafka:2.12-1.0.1'
+    image: 'wurstmeister/kafka:2.12-2.5.1'
     container_name: kafka
     ports:
       - "9092:9092"

platform-modules/mimetype-manager/pom.xml

@@ -28,9 +28,9 @@
             <type>jar</type>
         </dependency>
         <dependency>
-            <groupId>org.sunbird</groupId>
-            <artifactId>cloud-store-sdk_2.12</artifactId>
-            <version>1.4.6</version>
+            <groupId>${CLOUD_STORE_GROUP_ID}</groupId>
+            <artifactId>${CLOUD_STORE_ARTIFACT_ID}</artifactId>
+            <version>${CLOUD_STORE_VERSION}</version>
         </dependency>
         <dependency>
             <groupId>org.scala-lang</groupId>

platform-modules/mimetype-manager/src/main/scala/org/sunbird/cloudstore/StorageService.scala

@@ -34,18 +34,29 @@ class StorageService {
         throw new ServerException("ERR_INVALID_CLOUD_STORAGE", "Cloud Storage Container name not configured.")
     }
 
+    def formatUrl(url: String): String = {
+      if (storageType == "oci") {
+        val newHostname: String = if (Platform.config.hasPath("cloud_storage_proxy_host")) Platform.config.getString("cloud_storage_proxy_host") else ""
+        val regex = "(?<=://)([^/]+)".r
+        val replacedUrl = regex.replaceAllIn(url, newHostname)
+        replacedUrl
+      } else {
+        url
+      }
+    }
+
     def uploadFile(folderName: String, file: File, slug: Option[Boolean] = Option(true)): Array[String] = {
         val slugFile = if (slug.getOrElse(true)) Slug.createSlugFile(file) else file
         val objectKey = folderName + "/" + slugFile.getName
         val url = getService.upload(getContainerName, slugFile.getAbsolutePath, objectKey, Option.apply(false), Option.apply(1), Option.apply(5), Option.empty)
-        Array[String](objectKey, url)
+        Array[String](objectKey, formatUrl(url))
     }
 
     def uploadDirectory(folderName: String, directory: File, slug: Option[Boolean] = Option(true)): Array[String] = {
         val slugFile = if (slug.getOrElse(true)) Slug.createSlugFile(directory) else directory
         val objectKey = folderName + File.separator
         val url = getService.upload(getContainerName, slugFile.getAbsolutePath, objectKey, Option.apply(true), Option.apply(1), Option.apply(5), Option.empty)
-        Array[String](objectKey, url)
+        Array[String](objectKey, formatUrl(url))
     }
 
     def uploadDirectoryAsync(folderName: String, directory: File, slug: Option[Boolean] = Option(true))(implicit ec: ExecutionContext): Future[List[String]] = {

platform-modules/mimetype-manager/src/test/resources/application.conf

@@ -470,7 +470,7 @@ collection.image.migration.enabled=true
 
 
 
-content.h5p.library.path="https://s3.ap-south-1.amazonaws.com/ekstep-public-dev/content/templates/h5p-library-latest.zip"
+content.h5p.library.path="https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/templates/h5p-library-latest.zip"
 # This is added to handle large artifacts sizes differently
 content.artifact.size.for_online=209715200
 cloud_storage_type="azure"

platform-modules/mimetype-manager/src/test/scala/org/sunbird/cloudstore/StorageServiceTest.scala

@@ -7,49 +7,49 @@ import java.io.File
 class StorageServiceTest extends AsyncFlatSpec with Matchers {
     val ss = new StorageService
 
-    "getService" should "return a Storage Service" in {
-        val service = ss.getService
-        assert(service != null)
-    }
-
-    "getContainerName" should "return the container name" in {
-        val container = ss.getContainerName
-        assert(container == "sunbird-content-dev")
-    }
-
-    "getSignedURL" should "return the signed url" in {
-        val objectKey = "content" + File.separator + "asset" + File.separator + "do_53245" + File.separator + "abc.png"
-        val preSignedURL = ss.getSignedURL(objectKey, Option.apply(600), Option.apply("w"))
-        assert(preSignedURL.contains(objectKey))
-    }
-
-    "getUri" should "return the signed url" in {
-        val uri = ss.getUri("content/abc.json")
-        assert(uri != null)
-    }
-
-    "getMimeType" should "return the mimetype application/epub+zip for epub" in {
-        val result = ss.getMimeType("test.alert.epub")
-        assert(result == "application/epub+zip")
-    }
-
-    "getMimeType" should "return the mimetype application/octet-stream for h5p" in {
-        val result = ss.getMimeType("test.alert.h5p")
-        assert(result == "application/octet-stream")
-    }
-
-    "getMimeType" should "return the mimetype text/csv for csv" in {
-        val result = ss.getMimeType("test.alert.csv")
-        assert(result == "text/csv")
-    }
-
-    "getMimeType" should "return the mimetype application/pdf for pdf" in {
-        val result = ss.getMimeType("test.alert.pdf")
-        assert(result == "application/pdf")
-    }
-
-    "getMimeType" should "return the mimetype application/zip for zip" in {
-        val result = ss.getMimeType("test.alert.zip")
-        assert(result == "application/zip")
-    }
+    // "getService" should "return a Storage Service" in {
+    //     val service = ss.getService
+    //     assert(service != null)
+    // }
+
+    // "getContainerName" should "return the container name" in {
+    //     val container = ss.getContainerName
+    //     assert(container == "sunbird-content-dev")
+    // }
+
+    // "getSignedURL" should "return the signed url" in {
+    //     val objectKey = "content" + File.separator + "asset" + File.separator + "do_53245" + File.separator + "abc.png"
+    //     val preSignedURL = ss.getSignedURL(objectKey, Option.apply(600), Option.apply("w"))
+    //     assert(preSignedURL.contains(objectKey))
+    // }
+
+    // "getUri" should "return the signed url" in {
+    //     val uri = ss.getUri("content/abc.json")
+    //     assert(uri != null)
+    // }
+
+    // "getMimeType" should "return the mimetype application/epub+zip for epub" in {
+    //     val result = ss.getMimeType("test.alert.epub")
+    //     assert(result == "application/epub+zip")
+    // }
+
+    // "getMimeType" should "return the mimetype application/octet-stream for h5p" in {
+    //     val result = ss.getMimeType("test.alert.h5p")
+    //     assert(result == "application/octet-stream")
+    // }
+
+    // "getMimeType" should "return the mimetype text/csv for csv" in {
+    //     val result = ss.getMimeType("test.alert.csv")
+    //     assert(result == "text/csv")
+    // }
+
+    // "getMimeType" should "return the mimetype application/pdf for pdf" in {
+    //     val result = ss.getMimeType("test.alert.pdf")
+    //     assert(result == "application/pdf")
+    // }
+
+    // "getMimeType" should "return the mimetype application/zip for zip" in {
+    //     val result = ss.getMimeType("test.alert.zip")
+    //     assert(result == "application/zip")
+    // }
 }

platform-modules/mimetype-manager/src/test/scala/org/sunbird/mimetype/mgr/impl/H5PMimeTypeMgrImplTest.scala

@@ -90,7 +90,7 @@ class H5PMimeTypeMgrImplTest extends AsyncFlatSpec with Matchers with AsyncMockF
         implicit val ss = mock[StorageService]
         (ss.uploadFile(_:String, _: File, _: Option[Boolean])).expects(*, *, *).returns(Array(identifier, identifier))
         (ss.uploadDirectoryAsync(_:String, _:File, _: Option[Boolean])(_: ExecutionContext)).expects(*, *, *, *).returns(Future(List(identifier, identifier)))
-        val resFuture = new H5PMimeTypeMgrImpl().upload(identifier, node,"https://ekstep-public-dev.s3-ap-south-1.amazonaws.com/content/do_112499826618179584111/artifact/1525857774447_do_112499826618179584111.zip", None, UploadParams(Some("composed-h5p-zip")))
+        val resFuture = new H5PMimeTypeMgrImpl().upload(identifier, node,"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_112499826618179584111/artifact/1525857774447_do_112499826618179584111.zip", None, UploadParams(Some("composed-h5p-zip")))
         resFuture.map(result => {
             assert("do_1234" == result.getOrElse("identifier", "do_1234"))
             assert(result.get("artifactUrl") != null)

platform-modules/mimetype-manager/src/test/scala/org/sunbird/mimetype/mgr/impl/PluginMimeTypeMgrImplTest.scala

@@ -44,7 +44,7 @@ class PluginMimeTypeMgrImplTest extends AsyncFlatSpec with Matchers with AsyncMo
     it should "upload Invalid plugin zip file url and Throw Client Exception" in {
         implicit val ss = new StorageService
         val exception = intercept[ClientException] {
-            new PluginMimeTypeMgrImpl().upload("org.ekstep.video", new Node(), "https://ekstep-public-dev.s3-ap-south-1.amazonaws.com/content/do_11218758555843788817/artifact/akshara_kan_1487743191313.zip", None, UploadParams())
+            new PluginMimeTypeMgrImpl().upload("org.ekstep.video", new Node(), "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11218758555843788817/artifact/akshara_kan_1487743191313.zip", None, UploadParams())
         }
         exception.getMessage shouldEqual "Error !Invalid Content Package File Structure. | [manifest.json should be at root location]"
     }

platform-modules/url-manager/src/test/java/org/sunbird/url/util/HttpUrlUtilTest.java

@@ -1,6 +1,7 @@
 package org.sunbird.url.util;
 
 import org.apache.commons.io.FileUtils;
+import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
@@ -57,6 +58,7 @@ public void testGetMetadataWithValidUrlExpectServerException() {
 		String url = "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/";
 		Map<String, Object> result = HTTPUrlUtil.getMetadata(url);
 	}
+
 	@Test
 	public void testDownloadFileWithValidUrl() {
 		String downloadFolder = "/tmp/content/" + System.currentTimeMillis() + "_temp/do_123";

taxonomy-api/taxonomy-actors/src/main/scala/org/sunbird/utils/CategoryCache.scala

@@ -21,7 +21,7 @@ object CategoryCache{
     if (null != framework && !framework.isEmpty) {
       val categories = framework.getOrDefault("categories", new util.ArrayList[util.Map[String, AnyRef]]).asInstanceOf[util.List[util.Map[String, AnyRef]]].toList
       categories.map(category => {
-        val catName = category.get("name").asInstanceOf[String]
+        val catName = category.get("code").asInstanceOf[String]
         val terms = getTerms(category, "terms")
         if (terms.nonEmpty) {
           val key = getKey(id, catName)
@@ -34,20 +34,6 @@ object CategoryCache{
 
   private def getKey(framework: String, category: String) = "cat_" + framework + category
 
-  private def setFramework(framework: String, categories: List[Map[String, AnyRef]]): Unit = {
-    if (null != categories && categories.isEmpty) {
-      categories.map(category =>{
-        val catName = category.get("code").asInstanceOf[String]
-        val terms = getTerms(category, "terms")
-        if (terms.nonEmpty) {
-          val key = getKey(framework, catName)
-          TelemetryManager.info("Setting framework category cache with key: " + key)
-          RedisCache.saveList(key, terms)
-        }
-      })
-    }
-  }
-
   private def getTerms(category: util.Map[String, AnyRef], key: String): List[String] = {
     val returnTerms = new util.ArrayList[String]
     if (category != null && category.nonEmpty) {