Load BSBM from multiple (splitted) nt files instead of just one.

uzh · Aug 25, 2014 · 4c9e828 · 4c9e828
1 parent 68874b4
commit 4c9e828
Showing 1 changed file with 24 additions and 2 deletions.
diff --git a/src/main/scala/com/signalcollect/triplerush/evaluation/BerlinSparqlEvaluationParallel.scala b/src/main/scala/com/signalcollect/triplerush/evaluation/BerlinSparqlEvaluationParallel.scala
@@ -49,11 +49,17 @@ class BerlinSparqlEvaluationParallel extends TorqueDeployableAlgorithm {
     commonResults += "java.runtime.version" -> System.getProperty("java.runtime.version")
 
     val queriesObjectName = s"com.signalcollect.triplerush.evaluation.BerlinSparqlParameterized$datasetSize"
-    val ntriplesFileLocation = s"berlinsparql_$datasetSize-nt/dataset_$datasetSize.nt"
+    //val ntriplesFileLocation = s"berlinsparql_$datasetSize-nt/dataset_$datasetSize.nt"
+    val ntriplesFileLocation = s"berlinsparql_$datasetSize-nt"
 
-    val loadingTime = measureTime {
+    /*val loadingTime = measureTime {
       tr.loadNtriples(ntriplesFileLocation)
       tr.awaitIdle
+    }*/
+
+    val loadingTime = measureTime {
+      loadBSBMFromNTriples(ntriplesFileLocation, tr)
+      tr.awaitIdle
     }
 
     println(s"Finished loading")
@@ -126,6 +132,22 @@ class BerlinSparqlEvaluationParallel extends TorqueDeployableAlgorithm {
     tr.shutdown
   }
 
+  def loadBSBMFromNTriples(location: String, triplerush: TripleRush) {
+
+    val sourceFiles = filesIn(location).
+      filter(_.getName.endsWith(".nt")).
+      sorted
+
+    for (src <- sourceFiles) {
+      val ntFile = s"$src"
+      println(s"Loading file $ntFile")
+      triplerush.loadNtriples(ntFile)
+      println(s"Awaiting idle. Continuing graph loading...")
+      triplerush.awaitIdle
+      JvmWarmup.sleepUntilGcInactiveForXSeconds(60, 120)
+    }
+  }
+
   def executeEvaluationRun(queryString: String, queryRun: Int, queryDescription: String, tr: TripleRush, commonResults: Map[String, String]): Map[String, String] = {
     val gcs = ManagementFactory.getGarbageCollectorMXBeans.toList
     val compilations = ManagementFactory.getCompilationMXBean