Merge pull request #323 from clowder-framework/release/1.20.0

Release/1.20.0
clowder-framework · Feb 10, 2022 · a9c2700 · a9c2700
2 parents 17651a4 + 951e16a
commit a9c2700
Show file tree

Hide file tree

Showing 20 changed files with 211 additions and 99 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,16 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## 1.20.0 - 2022-02-07
+
+### Fixed
+- Conf file and code had incosistent spelling of BagIt. Now all have capital B and I.
+- When event stream is disabled don't show for logged in user [#280](https://github.com/clowder-framework/clowder/issues/280)
+- three.js is no longer associated with application/octet-stream, now with models [#305](https://github.com/clowder-framework/clowder/issues/305)
+
+### Changed
+- Download of dataset/collection now has optional parameter bagit (default false) to download items in bagit format.
+
 ## 1.19.5 - 2022-01-21
 
 ### Fixed

diff --git a/app/Iterators/CollectionIterator.scala b/app/Iterators/CollectionIterator.scala
@@ -13,11 +13,12 @@ import scala.collection.mutable.ListBuffer
 
 //this is to download collections
 //that are not at the root level
-class CollectionIterator(pathToFolder : String, parent_collection : models.Collection,zip : ZipOutputStream, md5Files : scala.collection.mutable.HashMap[String, MessageDigest], user : Option[User],
-                         collections: CollectionService, datasets : DatasetService, files : FileService, folders : FolderService, metadataService : MetadataService,
+class CollectionIterator(pathToFolder: String, parent_collection: models.Collection, zip: ZipOutputStream,
+                         md5Files: scala.collection.mutable.HashMap[String, MessageDigest], user : Option[User],
+                         bagit: Boolean, collections: CollectionService, datasets: DatasetService, files: FileService,
+                         folders: FolderService, metadataService: MetadataService,
                          spaces : SpaceService) extends Iterator[Option[InputStream]] {
 
-
   def getNextGenerationCollections(currentCollections : List[Collection]) : List[Collection] = {
     var nextGenerationCollections : ListBuffer[Collection] = ListBuffer.empty[Collection]
     for (currentCollection <- currentCollections){
@@ -28,7 +29,7 @@ class CollectionIterator(pathToFolder : String, parent_collection : models.Colle
     nextGenerationCollections.toList
   }
 
-  val datasetIterator = new DatasetsInCollectionIterator(pathToFolder,parent_collection,zip,md5Files,user,
+  val datasetIterator = new DatasetsInCollectionIterator(pathToFolder,parent_collection,zip,bagit, md5Files,user,
     datasets,files, folders, metadataService,spaces)
 
   var currentCollectionIterator : Option[CollectionIterator] = None
@@ -40,7 +41,7 @@ class CollectionIterator(pathToFolder : String, parent_collection : models.Colle
   var childCollectionCount = 0
   var numChildCollections = child_collections.size
 
-  var file_type = 0
+  var file_type = if (bagit) 0 else 2
 
   // TODO: Repeat from api/Collections
   def jsonCollection(collection: Collection): JsValue = {
@@ -74,12 +75,12 @@ class CollectionIterator(pathToFolder : String, parent_collection : models.Colle
     if (file_type < 2){
       true
     }
-    else if (file_type ==2){
+    else if (file_type == 2){
       if (datasetIterator.hasNext()){
         true
       } else if (numChildCollections > 0){
-
-        currentCollectionIterator = Some(new CollectionIterator(pathToFolder+"/"+child_collections(childCollectionCount).name, child_collections(childCollectionCount),zip,md5Files,user,collections,datasets,files,
+        currentCollectionIterator = Some(new CollectionIterator(pathToFolder+"/"+child_collections(childCollectionCount).name,
+          child_collections(childCollectionCount),zip,md5Files,user, bagit, collections,datasets,files,
           folders,metadataService,spaces))
         file_type +=1
         true
@@ -94,7 +95,7 @@ class CollectionIterator(pathToFolder : String, parent_collection : models.Colle
           } else if (childCollectionCount < numChildCollections -2){
             childCollectionCount+=1
             currentCollectionIterator = Some(new CollectionIterator(pathToFolder+"/"+child_collections(childCollectionCount).name, child_collections(childCollectionCount),zip,md5Files,user,
-              collections,datasets,files,
+              bagit, collections,datasets,files,
               folders,metadataService,spaces))
             true
           } else {

diff --git a/app/Iterators/DatasetIterator.scala b/app/Iterators/DatasetIterator.scala
@@ -14,7 +14,7 @@ import scala.collection.mutable.ListBuffer
 
 //this iterator is used for downloading a dataset
 //it has a file iterator
-class DatasetIterator(pathToFolder : String, dataset : models.Dataset, zip: ZipOutputStream, md5Files :scala.collection.mutable.HashMap[String, MessageDigest],
+class DatasetIterator(pathToFolder : String, dataset : models.Dataset, zip: ZipOutputStream, bagit: Boolean, md5Files :scala.collection.mutable.HashMap[String, MessageDigest],
                       folders : FolderService, files: FileService, metadataService : MetadataService, datasets: DatasetService, spaces : SpaceService) extends Iterator[Option[InputStream]] {
 
   //get files in the dataset
@@ -111,10 +111,14 @@ class DatasetIterator(pathToFolder : String, dataset : models.Dataset, zip: ZipO
 
   var fileCounter = 0
 
-  var currentFileIterator : Option[FileIterator] = None
+  var currentFileIterator : Option[FileIterator] = if (numFiles > 0){
+    Some(new FileIterator(folderNameMap(inputFiles(fileCounter).id),inputFiles(fileCounter),bagit, zip,md5Files,files,folders,metadataService))
+  } else {
+    None
+  }
 
   var is : Option[InputStream] = None
-  var file_type : Int = 0
+  var file_type : Int = if (bagit) 0 else 2
 
   def hasNext() = {
     if (file_type < 2){
@@ -126,7 +130,7 @@ class DatasetIterator(pathToFolder : String, dataset : models.Dataset, zip: ZipO
             true
           } else if (fileCounter < numFiles -1){
             fileCounter +=1
-            currentFileIterator = Some(new FileIterator(folderNameMap(inputFiles(fileCounter).id),inputFiles(fileCounter),zip,md5Files,files,folders,metadataService))
+            currentFileIterator = Some(new FileIterator(folderNameMap(inputFiles(fileCounter).id),inputFiles(fileCounter),bagit, zip,md5Files,files,folders,metadataService))
             true
           } else {
             false
@@ -155,7 +159,6 @@ class DatasetIterator(pathToFolder : String, dataset : models.Dataset, zip: ZipO
         md5Files.put("_metadata.json",md5)
         if (numFiles > 0){
           file_type+=1
-          currentFileIterator = Some(new FileIterator(folderNameMap(inputFiles(fileCounter).id),inputFiles(fileCounter),zip,md5Files,files,folders,metadataService))
         } else {
           file_type+=2
         }

diff --git a/app/Iterators/DatasetsInCollectionIterator.scala b/app/Iterators/DatasetsInCollectionIterator.scala
@@ -11,7 +11,7 @@ import scala.collection.mutable.ListBuffer
 
 //this is used to download the datasets in a collection
 //it creates an iterator for each dataset in the collection
-class DatasetsInCollectionIterator(pathToFolder : String, collection : models.Collection, zip : ZipOutputStream, md5Files : scala.collection.mutable.HashMap[String, MessageDigest], user : Option[User],
+class DatasetsInCollectionIterator(pathToFolder : String, collection : models.Collection, zip : ZipOutputStream, bagit: Boolean, md5Files : scala.collection.mutable.HashMap[String, MessageDigest], user : Option[User],
                                   datasets : DatasetService, files : FileService, folders : FolderService, metadataService : MetadataService,
                                    spaces : SpaceService) extends Iterator[Option[InputStream]] {
 
@@ -36,8 +36,7 @@ class DatasetsInCollectionIterator(pathToFolder : String, collection : models.Co
   }
 
   var currentDatasetIterator : Option[DatasetIterator]  = if (numDatasets > 0){
-
-    Some(new DatasetIterator(pathToFolder+"/"+currentDataset.get.name,currentDataset.get, zip, md5Files,
+    Some(new DatasetIterator(pathToFolder+"/"+currentDataset.get.name,currentDataset.get, zip, bagit, md5Files,
     folders, files,metadataService,datasets,spaces))
   } else {
     None
@@ -56,7 +55,7 @@ class DatasetsInCollectionIterator(pathToFolder : String, collection : models.Co
             currentDataset = Some(datasetsInCollection(datasetCount))
             currentDataset match {
               case Some(cd) => {
-                currentDatasetIterator = Some(new DatasetIterator(pathToFolder+"/"+cd.name,cd, zip, md5Files,
+                currentDatasetIterator = Some(new DatasetIterator(pathToFolder+"/"+cd.name,cd, zip, bagit, md5Files,
                   folders, files,metadataService,datasets,spaces))
                 true
               }

diff --git a/app/Iterators/FileIterator.scala b/app/Iterators/FileIterator.scala
@@ -11,7 +11,7 @@ import util.JSONLD
 
 //this is used for file downloads
 //called by the dataset interator
-class FileIterator (pathToFile : String, file : models.File,zip : ZipOutputStream, md5Files :scala.collection.mutable.HashMap[String, MessageDigest], files : FileService, folders : FolderService , metadataService : MetadataService) extends Iterator[Option[InputStream]] {
+class FileIterator (pathToFile : String, file : models.File, bagit: Boolean, zip : ZipOutputStream, md5Files :scala.collection.mutable.HashMap[String, MessageDigest], files : FileService, folders : FolderService , metadataService : MetadataService) extends Iterator[Option[InputStream]] {
 
   def getFileInfoAsJson(file : models.File) : JsValue = {
     val rightsHolder = {
@@ -54,7 +54,7 @@ class FileIterator (pathToFile : String, file : models.File,zip : ZipOutputStrea
     Some(new ByteArrayInputStream(s.getBytes("UTF-8")))
   }
 
-  var file_type : Int = 0
+  var file_type : Int = if (bagit) 0 else 2
   var is : Option[InputStream] = None
   def hasNext() = {
     if ( file_type < 3){

diff --git a/app/Iterators/RootCollectionIterator.scala b/app/Iterators/RootCollectionIterator.scala
@@ -17,11 +17,11 @@ import scala.collection.mutable.ListBuffer
 class RootCollectionIterator(pathToFolder : String, root_collection : models.Collection,zip : ZipOutputStream,
                              md5Files : scala.collection.mutable.HashMap[String, MessageDigest],
                              md5Bag : scala.collection.mutable.HashMap[String, MessageDigest],
-                             user : Option[User],totalBytes : Long,bagit : Boolean,
+                             user : Option[User],totalBytes : Long, bagit : Boolean,
                              collections: CollectionService, datasets : DatasetService, files : FileService, folders : FolderService, metadataService : MetadataService,
                              spaces : SpaceService) extends Iterator[Option[InputStream]] {
 
-  val datasetIterator = new DatasetsInCollectionIterator(root_collection.name,root_collection,zip,md5Files,user,
+  val datasetIterator = new DatasetsInCollectionIterator(root_collection.name,root_collection,zip,bagit, md5Files,user,
     datasets,files,folders,metadataService,spaces)
 
   var currentCollectionIterator : Option[CollectionIterator] = None
@@ -34,7 +34,7 @@ class RootCollectionIterator(pathToFolder : String, root_collection : models.Col
   var numCollections = child_collections.size
 
   var bytesSoFar : Long  = 0L
-  var file_type = 0
+  var file_type = if (bagit) 0 else 2
 
 
   private def addCollectionInfoToZip(folderName: String, collection: models.Collection, zip: ZipOutputStream): Option[InputStream] = {
@@ -107,9 +107,9 @@ class RootCollectionIterator(pathToFolder : String, root_collection : models.Col
         true
       } else if (numCollections > 0){
 
-        currentCollectionIterator = Some(new CollectionIterator(pathToFolder+"/"+child_collections(collectionCount).name, child_collections(collectionCount),zip,md5Files,user,
-          collections,datasets,files,
-          folders,metadataService,spaces))
+        currentCollectionIterator = Some(new CollectionIterator(pathToFolder+"/"+child_collections(collectionCount).name,
+          child_collections(collectionCount), zip, md5Files, user, bagit,
+          collections, datasets,files, folders, metadataService, spaces))
         file_type +=1
         true
       } else if (bagit){
@@ -127,7 +127,7 @@ class RootCollectionIterator(pathToFolder : String, root_collection : models.Col
           } else if (collectionCount < numCollections -2){
             collectionCount+=1
             currentCollectionIterator = Some(new CollectionIterator(pathToFolder+"/"+child_collections(collectionCount).name, child_collections(collectionCount),zip,md5Files,user,
-              collections,datasets,files,
+              bagit, collections,datasets,files,
               folders,metadataService,spaces))
             true
           } else {

diff --git a/app/Iterators/SelectedIterator.scala b/app/Iterators/SelectedIterator.scala
@@ -21,7 +21,7 @@ class SelectedIterator(pathToFolder : String, selected : List[Dataset], zip : Zi
 
   var datasetCount = 0
   var currDs = selected(datasetCount)
-  var datasetIterator = new DatasetIterator(pathToFolder+"/"+currDs.name, currDs, zip, md5Files, folders, files,
+  var datasetIterator = new DatasetIterator(pathToFolder+"/"+currDs.name, currDs, zip, bagit, md5Files, folders, files,
     metadataService,datasets,spaces)
   var file_type = 0
 
@@ -54,7 +54,7 @@ class SelectedIterator(pathToFolder : String, selected : List[Dataset], zip : Zi
       } else if (selected.length > datasetCount+1){
         datasetCount += 1
         currDs = selected(datasetCount)
-        datasetIterator = new DatasetIterator(pathToFolder+"/"+currDs.name,currDs,zip,md5Files,folders,files,
+        datasetIterator = new DatasetIterator(pathToFolder+"/"+currDs.name,currDs,zip,bagit, md5Files,folders,files,
           metadataService,datasets,spaces)
         true
       } else if (bagit) {

diff --git a/app/api/Collections.scala b/app/api/Collections.scala
@@ -801,11 +801,10 @@ class Collections @Inject() (datasets: DatasetService,
     }
   }
 
-  def download(id: UUID, compression: Int) = PermissionAction(Permission.DownloadFiles, Some(ResourceRef(ResourceRef.collection, id))) { implicit request =>
+  def download(id: UUID, compression: Int, bagit: Boolean) = PermissionAction(Permission.DownloadFiles, Some(ResourceRef(ResourceRef.collection, id))) { implicit request =>
     implicit val user = request.user
     collections.get(id) match {
       case Some(collection) => {
-        val bagit = play.api.Play.configuration.getBoolean("downloadCollectionBagit").getOrElse(true)
         // Use custom enumerator to create the zip file on the fly
         // Use a 1MB in memory byte array
         Ok.chunked(enumeratorFromCollection(collection,1024*1024, compression,bagit,user)).withHeaders(
@@ -821,7 +820,9 @@ class Collections @Inject() (datasets: DatasetService,
   }
 
 
-  def enumeratorFromCollection(collection: Collection, chunkSize: Int = 1024 * 8, compression: Int = Deflater.DEFAULT_COMPRESSION, bagit: Boolean, user : Option[User])
+  def enumeratorFromCollection(collection: Collection, chunkSize: Int = 1024 * 8,
+                               compression: Int = Deflater.DEFAULT_COMPRESSION,
+                               bagit: Boolean, user : Option[User])
                               (implicit ec: ExecutionContext): Enumerator[Array[Byte]] = {
 
     implicit val pec = ec.prepare()
@@ -839,8 +840,6 @@ class Collections @Inject() (datasets: DatasetService,
     var current_iterator = new RootCollectionIterator(collection.name,collection,zip,md5Files,md5Bag,user, totalBytes,bagit,collections,
     datasets,files,folders,metadataService,spaces)
 
-
-
     //var current_iterator = new FileIterator(folderNameMap(inputFiles(1).id),inputFiles(1), zip,md5Files)
     var is = current_iterator.next()