Network latency correction between Vitro and the outsourced tripplet

api/pom.xml

@@ -7,20 +7,29 @@
 
     <groupId>org.vivoweb</groupId>
     <artifactId>vitro-api</artifactId>
-    <version>1.13.1-SNAPSHOT</version>
+    <version>1.14.1-SNAPSHOT</version>
     <packaging>jar</packaging>
 
     <parent>
         <groupId>org.vivoweb</groupId>
         <artifactId>vitro-project</artifactId>
-        <version>1.13.1-SNAPSHOT</version>
+        <version>1.14.1-SNAPSHOT</version>
         <relativePath>..</relativePath>
     </parent>
 
     <name>Vitro API</name>
 
     <build>
         <plugins>
+                <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-javadoc-plugin</artifactId>
+            <version>3.0.0</version>
+            <configuration>
+                <source>1.8</source>
+                <target>1.8</target>
+            </configuration>
+        </plugin>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-surefire-plugin</artifactId>
@@ -73,7 +82,7 @@
         <dependency>
             <groupId>org.vivoweb</groupId>
             <artifactId>vitro-dependencies</artifactId>
-            <version>1.13.1-SNAPSHOT</version>
+            <version>1.14.1-SNAPSHOT</version>
             <type>pom</type>
         </dependency>
         <dependency>

api/src/main/java/edu/cornell/mannlib/vitro/webapp/beans/IndividualImpl.java

@@ -11,7 +11,9 @@
 import java.util.List;
 import java.util.Map;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.JsonNodeFactory;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import edu.cornell.mannlib.vitro.webapp.dao.VitroVocabulary;
@@ -329,19 +331,75 @@ public int compareTo(Individual o2) {
        }
    }
 
+   public String toString_() {
+       ObjectMapper mapper = new ObjectMapper();
+       String jsonString = "";
+       try {
+           jsonString = (name != null                     ? "{ name = "+ mapper.writeValueAsString(name) + " },": "")
+                   + (rdfsLabel != null                   ? "{ rdfsLabel = "+ mapper.writeValueAsString(rdfsLabel)  + " },": "")
+                   + (vClassURI != null                   ? "{ vClassURI = "+ mapper.writeValueAsString(vClassURI)  + " },": "")
+                   + (vClass != null                      ? "{ vClass = "+ mapper.writeValueAsString(vClass)  + " },": "")
+                   + (directVClasses != null              ? "{ directVClasses = "+ mapper.writeValueAsString(directVClasses)  + " },": "")
+                   + (allVClasses != null                 ? "{ allVClasses = "+ mapper.writeValueAsString(allVClasses)  + " },": "")
+                   + (modTime != null                     ? "{ modTime = "+ mapper.writeValueAsString(modTime)  + " },": "")
+                   + (propertyList != null                ? "{ propertyList = "+ mapper.writeValueAsString(propertyList)  + " }," : "")
+                   + (populatedObjectPropertyList != null ? "{ populatedObjectPropertyList = "+ mapper.writeValueAsString(populatedObjectPropertyList)  + " },": "")
+                   + (objectPropertyMap != null           ? "{ objectPropertyMap = "+ mapper.writeValueAsString(objectPropertyMap)  + " },": "")
+                   + (datatypePropertyList != null        ? "{ datatypePropertyList = "+ mapper.writeValueAsString(datatypePropertyList)  + " },": "")
+                   + (populatedDataPropertyList != null   ? "{ populatedDataPropertyList = "+ mapper.writeValueAsString(populatedDataPropertyList)  + " },": "")
+                   + (dataPropertyMap != null             ? "{ dataPropertyMap = "+ mapper.writeValueAsString(dataPropertyMap)  + " },": "")
+                   + (dataPropertyStatements != null      ? "{ dataPropertyStatements = "+ mapper.writeValueAsString(dataPropertyStatements)  + " },": "")
+                   + (objectPropertyStatements != null    ? "{ objectPropertyStatements = "+ mapper.writeValueAsString(objectPropertyStatements)  + " },": "")
+                   + (rangeEnts2Ents != null              ? "{ rangeEnts2Ents = "+ mapper.writeValueAsString(rangeEnts2Ents)  + " },": "")
+                   + (externalIds != null                 ? "{ externalIds = "+ mapper.writeValueAsString(externalIds)  + " },": "")
+                   + (mainImageUri != null                ? "{ mainImageUri = "+ mapper.writeValueAsString(mainImageUri)  + " },": "")
+                   + (imageInfo != null                   ? "{ imageInfo = "+ mapper.writeValueAsString(imageInfo)  + " }": "");
+       } catch (JsonProcessingException e) {
+           // TODO Auto-generated catch block
+           e.printStackTrace();
+       }
+       return jsonString;
+   }
+
+
    public String toString(){
        if( getURI() == null ){
            return "uninitialized, null URI";
-       }else{
-           return getURI() + " " + getName();
+       } else {
+           return "IndividualImpl [\n" + 
+                   (name != null ? "name=" + name + ", \n" : "")
+                   + (rdfsLabel != null ? "rdfsLabel=" + rdfsLabel + ", \n" : "")
+                   + (vClassURI != null ? "vClassURI=" + vClassURI + ", \n" : "")
+                   + (vClass != null ? "vClass=" + vClass + ", \n" : "")
+                   + (directVClasses != null ? "directVClasses=" + directVClasses + ", \n" : "")
+                   + (allVClasses != null ? "allVClasses=" + allVClasses + ", \n" : "")
+                   + (modTime != null ? "modTime=" + modTime + ", " : "")
+                   + (propertyList != null ? "propertyList=" + propertyList + ", \n" : "")
+                   + (populatedObjectPropertyList != null
+                           ? "populatedObjectPropertyList=" + populatedObjectPropertyList + ", \n"
+                           : "")
+                   + (objectPropertyMap != null ? "objectPropertyMap=" + objectPropertyMap + ", \n" : "")
+                   + (datatypePropertyList != null ? "datatypePropertyList=" + datatypePropertyList + ", \n" : "")
+                   + (populatedDataPropertyList != null ? "populatedDataPropertyList=" + populatedDataPropertyList + ", \n"
+                           : "")
+                   + (dataPropertyMap != null ? "dataPropertyMap=" + dataPropertyMap + ", \n" : "")
+                   + (dataPropertyStatements != null ? "dataPropertyStatements=" + dataPropertyStatements + ", \n" : "")
+                   + (objectPropertyStatements != null ? "objectPropertyStatements=" + objectPropertyStatements + ", \n"
+                           : "")
+                   + (rangeEnts2Ents != null ? "rangeEnts2Ents=" + rangeEnts2Ents + ", \n" : "")
+                   + (externalIds != null ? "externalIds=" + externalIds + ", \n" : "")
+                   + (mainImageUri != null ? "mainImageUri=" + mainImageUri + ", \n" : "")
+                   + (imageInfo != null ? "imageInfo=" + imageInfo + ", \n" : "")
+                   + "]";
+//           return getURI() + " " + getName();
        }
    }
 
     public boolean hasThumb() {
         return getThumbUrl() != null && ! getThumbUrl().isEmpty();
     }
 
-	@Override
+    @Override
 	public void resolveAsFauxPropertyStatements(List<ObjectPropertyStatement> list) {
 		// No webappDaoFactory, so nothing to do.
 	}

api/src/main/java/edu/cornell/mannlib/vitro/webapp/beans/ObjectPropertyStatementImpl.java

@@ -31,12 +31,15 @@ public ObjectPropertyStatementImpl(String subjectUri, String propertyUri, String
      * @see edu.cornell.mannlib.vitro.webapp.beans.ObjectPropertyStatement#toString()
      */
     public String toString(){
-           String prop = (getProperty()!=null)?getProperty().getDomainPublic():"by propURI"+getPropertyURI();
+           String prop = (getProperty()!=null)?getProperty().getDomainPublic():"by propURI:"+getPropertyURI();
            String ran = (getObject()!= null)?getObject().getName():"objectURI:"+getObjectURI();
            String dom = (getSubject()!= null)?getSubject().getName():"subjectURI:"+getSubjectURI();
            return "Object Property Statements: "+dom+" "+prop+" to "+ran+" ";
     }
 
+    public String toStringByUri(){
+        return "<"+getSubjectURI()+"> <"+getPropertyURI()+"> <"+getObjectURI()+">";
+    }
     /* (non-Javadoc)
      * @see edu.cornell.mannlib.vitro.webapp.beans.ObjectPropertyStatement#getSubjectURI()
      */

api/src/main/java/edu/cornell/mannlib/vitro/webapp/config/RevisionInfoSetup.java

@@ -10,6 +10,7 @@
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.Reader;
+import java.nio.charset.StandardCharsets;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
@@ -99,7 +100,9 @@ private BufferedReader openRevisionInfoReader(ServletContext context)
 					"Can't find a resource in the webapp at '" + RESOURCE_PATH
 							+ "'.");
 		} else {
-			return new BufferedReader(new InputStreamReader(stream));
+		    BufferedReader br = new BufferedReader(
+                    new InputStreamReader(stream, StandardCharsets.UTF_8));
+			return br;
 		}
 	}
 

api/src/main/java/edu/cornell/mannlib/vitro/webapp/controller/VitroRequest.java

@@ -10,6 +10,7 @@
 import static edu.cornell.mannlib.vitro.webapp.modelaccess.ModelAccess.CONTENT;
 import static edu.cornell.mannlib.vitro.webapp.modelaccess.ModelAccess.LANGUAGE_NEUTRAL;
 import static edu.cornell.mannlib.vitro.webapp.modelaccess.ModelAccess.POLICY_NEUTRAL;
+import static edu.cornell.mannlib.vitro.webapp.modelaccess.ModelAccess.BUFFERED_TREATMENT;
 import static edu.cornell.mannlib.vitro.webapp.modelaccess.ModelNames.DISPLAY;
 
 import java.text.Collator;
@@ -41,11 +42,10 @@ public class VitroRequest extends HttpServletRequestWrapper {
     final static Log log = LogFactory.getLog(VitroRequest.class);
 
     //Attribute in case of special model editing such as display model editing
-    public static final String SPECIAL_WRITE_MODEL = "specialWriteModel";
-
-    public  static final String ID_FOR_WRITE_MODEL = "idForWriteModel";
-    public  static final String ID_FOR_TBOX_MODEL = "idForTboxModel";
-    public  static final String ID_FOR_ABOX_MODEL = "idForAboxModel";
+    public static final String SPECIAL_WRITE_MODEL  = "specialWriteModel";
+    public static final String ID_FOR_WRITE_MODEL   = "idForWriteModel";
+    public static final String ID_FOR_TBOX_MODEL    = "idForTboxModel";
+    public static final String ID_FOR_ABOX_MODEL    = "idForAboxModel";
     public static final String ID_FOR_DISPLAY_MODEL = "idForDisplayModel";
 
     private HttpServletRequest _req;
@@ -73,6 +73,10 @@ public WebappDaoFactory getUnfilteredWebappDaoFactory() {
     	return ModelAccess.on(this).getWebappDaoFactory(POLICY_NEUTRAL);
     }
 
+    public WebappDaoFactory getBufferedIndividualWebappDaoFactory() {
+        return ModelAccess.on(this).getWebappDaoFactory(POLICY_NEUTRAL, BUFFERED_TREATMENT);
+    }
+
     /** gets assertions-only WebappDaoFactory with no policy filtering */
     public WebappDaoFactory getUnfilteredAssertionsWebappDaoFactory() {
     	return ModelAccess.on(this).getWebappDaoFactory(POLICY_NEUTRAL, ASSERTIONS_ONLY);

api/src/main/java/edu/cornell/mannlib/vitro/webapp/controller/freemarker/IndividualListController.java

@@ -24,6 +24,7 @@
 import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.ResponseValues;
 import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.TemplateResponseValues;
 import edu.cornell.mannlib.vitro.webapp.controller.individuallist.IndividualListResults;
+import edu.cornell.mannlib.vitro.webapp.controller.json.JsonServlet;
 import edu.cornell.mannlib.vitro.webapp.dao.IndividualDao;
 import edu.cornell.mannlib.vitro.webapp.i18n.I18n;
 import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineException;
@@ -40,8 +41,10 @@ public class IndividualListController extends FreemarkerHttpServlet {
 
     private static final long serialVersionUID = 1L;
     private static final Log log = LogFactory.getLog(IndividualListController.class.getName());
-
-    private static final int INDIVIDUALS_PER_PAGE = 30;
+    /*
+     * The call to the attribution in JsonServlet ensures the uniformity of the assigned INDIVIDUALS_PER_PAGE value
+     */
+    private static final int INDIVIDUALS_PER_PAGE = JsonServlet.getIndividualsPerPage();
     private static final int MAX_PAGES = 40;  // must be even
 
     private static final String TEMPLATE_DEFAULT = "individualList.ftl";

api/src/main/java/edu/cornell/mannlib/vitro/webapp/controller/json/GetRenderedSearchIndividualsByVClass.java

@@ -5,19 +5,34 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import edu.cornell.mannlib.vitro.webapp.web.templatemodels.individual.IndividualTemplateModelBuilder;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.log4j.Level;
+import org.apache.log4j.LogManager;
 
 import edu.cornell.mannlib.vitro.webapp.beans.Individual;
 import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
 import edu.cornell.mannlib.vitro.webapp.dao.IndividualDao;
 import edu.cornell.mannlib.vitro.webapp.services.shortview.ShortViewService;
 import edu.cornell.mannlib.vitro.webapp.services.shortview.ShortViewService.ShortViewContext;
 import edu.cornell.mannlib.vitro.webapp.services.shortview.ShortViewServiceSetup;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.temporal.ChronoUnit;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 /**
  * Does a search for individuals, and uses the short view to render each of
@@ -59,31 +74,125 @@ protected ObjectNode process() throws Exception {
 		return rObj;
 	}
 
+//    /**
+//     * Look through the return object. For each individual, render the short view
+//     * and insert the resulting HTML into the object.
+//     * ORIGINAL CODE 
+//     * This code segment is never called. 
+//     * It is kept for the time being for performance comparison tests. 
+//     * It should be deleted when PR is accepted.
+//     */
+//	private void addShortViewRenderings_ORIGINAL(ObjectNode rObj) {
+//		ArrayNode individuals = (ArrayNode) rObj.get("individuals");
+//		String vclassName = rObj.get("vclass").get("name").asText();
+//		for (int i = 0; i < individuals.size(); i++) {
+//			ObjectNode individual = (ObjectNode) individuals.get(i);
+//			individual.put("shortViewHtml",
+//					renderShortView(individual.get("URI").asText(), vclassName));
+//		}
+//	}
+//    /**
+//     * Look through the return object. For each individual, render the short view
+//     * and insert the resulting HTML into the object.
+//     * The use of multi treading allows to submit the requests and the parallel 
+//     * processing of the elements necessary for the html objects of the page. 
+//     * Parallel processing allows maximum network bandwidth utilization.
+//     */
+//    private void addShortViewRenderings(ObjectNode rObj) {
+//
+//        ArrayNode individuals = (ArrayNode) rObj.get("individuals");
+//        String vclassName = rObj.get("vclass").get("name").asText();
+//        int indvSize = individuals.size();
+//        ExecutorService es = Executors.newFixedThreadPool(indvSize);
+//        for (int i = 0; i < indvSize; i++) {
+//            ObjectNode individual = (ObjectNode) individuals.get(i);
+//               ProcessIndividual pi = new ProcessIndividual();
+//               pi.setIndividual(individual);
+//               pi.setVclassName(vclassName);
+//               es.execute(pi);
+//        }
+//        es.shutdown();
+//        try {
+//            while(!es.awaitTermination(250, TimeUnit.MILLISECONDS)){
+//            }
+//        } catch (InterruptedException e1) {
+//        }
+//    }
+//    /*
+//     * The runnable class that executes the renderShortView 
+//     */
+//    private class ProcessIndividual implements Runnable {
+//        private ObjectNode individual = null;
+//        private String vclassName;
+//        public String getVclassName() {
+//            return vclassName;
+//        }
+//        public void setVclassName(String vclassName) {
+//            this.vclassName = vclassName;
+//        }
+//        // Method
+//        public void run() {
+//            individual.put("shortViewHtml", renderShortView(individual.get("URI").asText(), vclassName));
+//        }
+//        public void setIndividual(ObjectNode individual) {
+//            this.individual = individual;
+//        }
+//    }
+//    
+//    private String renderShortView(String individualUri, String vclassName) {
 	/**
 	 * Look through the return object. For each individual, render the short
 	 * view and insert the resulting HTML into the object.
 	 */
-	private void addShortViewRenderings(ObjectNode rObj) {
-		ArrayNode individuals = (ArrayNode) rObj.get("individuals");
-		String vclassName = rObj.get("vclass").get("name").asText();
-		for (int i = 0; i < individuals.size(); i++) {
-			ObjectNode individual = (ObjectNode) individuals.get(i);
-			individual.put("shortViewHtml",
-					renderShortView(individual.get("URI").asText(), vclassName));
-		}
-	}
+    private void addShortViewRenderings(ObjectNode rObj) {
+//        LogManager.getRootLogger().setLevel(Level.DEBUG);
+        ArrayNode individuals = (ArrayNode) rObj.get("individuals");
+        String vclassName = rObj.get("vclass").get("name").asText();
+        Instant d1 = Instant.now();
+        int totalIndv = individuals.size();
+        for (int i = 0; i < individuals.size(); i++) {
+            ObjectNode individual = (ObjectNode) individuals.get(i);
+            Instant t1 = Instant.now();
+            individual.put("shortViewHtml", renderShortView(individual.get("URI").asText(), vclassName));
+            Instant t2 = Instant.now();
+            long totalTime = ChronoUnit.MILLIS.between(t1, t2);
+            log.info("ANALYSER: The treatment at (" + t2 +") for (" + individual.get("URI").asText()+") took "+ totalTime/1000.0 + " seconds");
+
+        }
+        Instant d2 = Instant.now();
+        long totalTime = ChronoUnit.MILLIS.between(d1, d2);
+//        LogManager.getRootLogger().setLevel(Level.INFO);
+        log.info("ANALYSER: total indv:(" + totalIndv + 
+                ") total time (sec.):(" + totalTime / 1000.0 + 
+                ") avrg time (sec.): " + (totalTime / totalIndv) / 1000.0);
+    }
 
 	private String renderShortView(String individualUri, String vclassName) {
-		IndividualDao iDao = vreq.getWebappDaoFactory().getIndividualDao();
+//        Instant t1 = Instant.now();
+		IndividualDao iDao = vreq.getBufferedIndividualWebappDaoFactory().getIndividualDao();
+//        Instant t2 = Instant.now();
 		Individual individual = iDao.getIndividualByURI(individualUri);
+//        log.info("toString "+ individual);
+//        Instant t3 = Instant.now();
 
 		Map<String, Object> modelMap = new HashMap<String, Object>();
-		modelMap.put("individual",
-				IndividualTemplateModelBuilder.build(individual, vreq));
+		modelMap.put("individual", IndividualTemplateModelBuilder.build(individual, vreq));
 		modelMap.put("vclass", vclassName);
+//        Instant t4 = Instant.now();
 
 		ShortViewService svs = ShortViewServiceSetup.getService(ctx);
-		return svs.renderShortView(individual, ShortViewContext.BROWSE,
-				modelMap, vreq);
+
+//        Instant t5 = Instant.now();
+		String rsv = svs.renderShortView(individual, ShortViewContext.BROWSE,modelMap, vreq);
+//        Instant t6 = Instant.now();
+//        log.info("toString "+ individual);
+//        log.info("ANALYSER: "+
+//        " total-renderShortView="+ChronoUnit.MILLIS.between(t5,t6)+ 
+//        " total-iDao="+ChronoUnit.MILLIS.between(t1,t2)+ 
+//        " total-modelMap="+ChronoUnit.MILLIS.between(t3,t4)+ 
+//        " total-ShortViewService="+ChronoUnit.MILLIS.between(t4,t5)+ 
+//        " total="+ChronoUnit.MILLIS.between(t1,t6) 
+//        );
+		return rsv;
 	}
 }