Merge remote-tracking branch 'origin/develop'

# Conflicts:
#	build.gradle

.travis.yml

@@ -25,7 +25,8 @@ cache:
 before_install:
   - export TZ=Australia/Canberra
 after_success:
-  - '[ "${TRAVIS_PULL_REQUEST}" = "false" ] && ./gradlew clean && travis_retry ./gradlew publish'
+  - '[ "${TRAVIS_PULL_REQUEST}" = "false" ] && ./gradlew clean'
+  - ./gradlew publish
   - ./gradlew coveralls
 #after_success:
 #  - mvn coveralls:report

build.gradle

@@ -40,7 +40,7 @@ repositories {
 }
 
 group = 'au.org.ala'
-version = '3.2.1'
+version = '3.2.2'
 
 
 boolean inplace = false

src/main/java/au/org/ala/biocache/dao/SolrIndexDAOImpl.java

@@ -28,10 +28,7 @@
 import org.apache.solr.client.solrj.impl.*;
 import org.apache.solr.client.solrj.io.SolrClientCache;
 import org.apache.solr.client.solrj.io.Tuple;
-import org.apache.solr.client.solrj.io.stream.AlaCloudSolrStream;
-import org.apache.solr.client.solrj.io.stream.SolrStream;
-import org.apache.solr.client.solrj.io.stream.StreamContext;
-import org.apache.solr.client.solrj.io.stream.TupleStream;
+import org.apache.solr.client.solrj.io.stream.*;
 import org.apache.solr.client.solrj.request.UpdateRequest;
 import org.apache.solr.client.solrj.request.schema.SchemaRequest;
 import org.apache.solr.client.solrj.response.FieldStatsInfo;
@@ -1308,6 +1305,6 @@ private ModifiableSolrParams buildEndemicExpr(SolrQuery subset, SolrQuery supers
     }
 
     private String escapeDoubleQuote(String input) {
-        return input.replaceAll("\"", "\\\"");
+        return input.replaceAll("\"", "\\\\\"");
     }
 }

src/main/java/au/org/ala/biocache/dto/SpatialObjectDTO.java

@@ -0,0 +1,85 @@
+package au.org.ala.biocache.dto;
+
+public class SpatialObjectDTO {
+    Double area_km;
+    String bbox;
+    String name;
+    String description;
+    String fieldname;
+    String pid;
+    String id;
+    String fid;
+    String wmsurl;
+
+    public Double getArea_km() {
+        return area_km;
+    }
+
+    public void setArea_km(Double area_km) {
+        this.area_km = area_km;
+    }
+
+    public String getBbox() {
+        return bbox;
+    }
+
+    public void setBbox(String bbox) {
+        this.bbox = bbox;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    public String getDescription() {
+        return description;
+    }
+
+    public void setDescription(String description) {
+        this.description = description;
+    }
+
+    public String getFieldname() {
+        return fieldname;
+    }
+
+    public void setFieldname(String fieldname) {
+        this.fieldname = fieldname;
+    }
+
+    public String getPid() {
+        return pid;
+    }
+
+    public void setPid(String pid) {
+        this.pid = pid;
+    }
+
+    public String getId() {
+        return id;
+    }
+
+    public void setId(String id) {
+        this.id = id;
+    }
+
+    public String getFid() {
+        return fid;
+    }
+
+    public void setFid(String fid) {
+        this.fid = fid;
+    }
+
+    public String getWmsurl() {
+        return wmsurl;
+    }
+
+    public void setWmsurl(String wmsurl) {
+        this.wmsurl = wmsurl;
+    }
+}

src/main/java/au/org/ala/biocache/service/AlaLayersService.java

@@ -14,19 +14,25 @@
  ***************************************************************************/
 package au.org.ala.biocache.service;
 
+import au.org.ala.biocache.dto.SpatialObjectDTO;
 import com.fasterxml.jackson.core.type.TypeReference;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Value;
+import org.springframework.cache.annotation.Cacheable;
 import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Component;
+import org.springframework.util.StreamUtils;
 import org.springframework.web.client.RestOperations;
 
 import javax.annotation.PostConstruct;
 import javax.inject.Inject;
 import java.io.Reader;
+import java.net.URL;
+import java.net.URLConnection;
 import java.net.URLEncoder;
+import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -244,4 +250,24 @@ public Reader sample(String[] analysisLayers, double[][] points, Object o) {
         // TODO: for on the fly intersection of layers not indexed
         return null;
     }
+
+    @Cacheable("spatialObject")
+    @Override
+    public SpatialObjectDTO getObject(String spatialObjectId) {
+        String url = layersServiceUrl + "/object/" + Integer.parseInt(spatialObjectId);
+        return restTemplate.getForObject(url, SpatialObjectDTO.class);
+    }
+
+    @Cacheable("wkt")
+    @Override
+    public String getObjectWkt(String spatialObjectId) {
+        String url = layersServiceUrl + "/shape/wkt/" + Integer.parseInt(spatialObjectId);
+
+        try {
+            URLConnection con = new URL(url).openConnection();
+            return StreamUtils.copyToString(con.getInputStream(), Charset.defaultCharset());
+        } catch (Exception e) {
+            return null;
+        }
+    }
 }

src/main/java/au/org/ala/biocache/service/DownloadService.java

@@ -1125,6 +1125,9 @@ public String generateEmailContent(String template, Map<String, String> substitu
 
     public void cancel(DownloadDetailsDTO dd) throws InterruptedException {
 
+        // remove from persistent queue (disk)
+        persistentQueueDAO.remove(dd);
+
         // signal download to end
         dd.setInterrupt(true);
 
@@ -1134,9 +1137,6 @@ public void cancel(DownloadDetailsDTO dd) throws InterruptedException {
         // get executor for this user
         ThreadPoolExecutor ex = userExecutors.get(getUserId(dd));
         if (ex != null) {
-            // remove from persistent queue (disk)
-            persistentQueueDAO.remove(dd);
-
             // remove from executor queue
             for (Runnable r : ex.getQueue()) {
                 if (((DownloadRunnable) r).currentDownload.getUniqueId().equals(dd.getUniqueId())) {
@@ -1305,6 +1305,7 @@ public void run() {
                         }
                     }
                 }
+                persistentQueueDAO.remove(currentDownload);
             } catch (InterruptedException e) {
                 Thread.currentThread().interrupt();
                 //shutting down
@@ -1340,8 +1341,9 @@ public void run() {
                             + currentDownload.getFileLocation(), ex);
                 }
 
-                // If we ever want to retry on failure, enable this
+                // If we ever want to retry on failure, enable doRetry and disable queue.remove
                 //doRetry = true
+                persistentQueueDAO.remove(currentDownload);
             } finally {
                 // in case of server up/down, only remove from queue
                 // after emails are sent

src/main/java/au/org/ala/biocache/service/LayersService.java

@@ -1,7 +1,7 @@
 /**************************************************************************
  *  Copyright (C) 2013 Atlas of Living Australia
  *  All Rights Reserved.
- * 
+ *
  *  The contents of this file are subject to the Mozilla Public
  *  License Version 1.1 (the "License"); you may not use this file
  *  except in compliance with the License. You may obtain a copy of
@@ -14,6 +14,8 @@
  ***************************************************************************/
 package au.org.ala.biocache.service;
 
+import au.org.ala.biocache.dto.SpatialObjectDTO;
+
 import java.io.Reader;
 
 /**
@@ -49,4 +51,8 @@ public interface LayersService {
     String getLayersServiceUrl();
 
     Reader sample(String[] analysisLayers, double[][] points, Object o);
+
+    SpatialObjectDTO getObject(String spatialObjectId);
+
+    String getObjectWkt(String spatialObjectId);
 }

src/main/java/au/org/ala/biocache/util/QueryFormatUtils.java

@@ -71,6 +71,8 @@ public class QueryFormatUtils {
     //Patterns that are used to prepare a SOLR query for execution
     protected Pattern lsidPattern = Pattern.compile("(^|\\s|\"|\\(|\\[|'|-)taxonConceptID:\"?([a-zA-Z0-9/\\.:\\-_]*)\"?");
     protected Pattern speciesListPattern = Pattern.compile("(^|\\s|\"|\\(|\\[|'|-)species_list:\"?(dr[0-9]*)\"?");
+
+    protected Pattern spatialObjectPattern = Pattern.compile("(^|\\s|\"|\\(|\\[|'|-)spatialObject:\"?([0-9]*)\"?");
     protected Pattern urnPattern = Pattern.compile("\\burn:[a-zA-Z0-9\\.:-]*");
     protected Pattern httpPattern = Pattern.compile("http:[a-zA-Z0-9/\\.:\\-_]*");
     protected Pattern uidPattern = Pattern.compile("(?:[\"]*)?((?:[a-z_]*_uid:)|(?:[a-zA-Z]*Uid:))(\\w*)(?:[\"]*)?");
@@ -722,6 +724,68 @@ private boolean formatSpatial(String [] current) throws QidMissingException {
         return false;
     }
 
+    /**
+     * Insert spatialObject WKT.
+     *
+     * If the query string contains spatialObject: replace with the equivalent geohash:Intersects(WKT)
+     *
+     * @param current
+     * @return
+     * @throws QidMissingException
+     */
+    private void formatSpatialObject(String [] current) {
+        if (current == null || current.length < 2 || current[1] == null) {
+            return;
+        }
+
+        //if the query string contains spatialObject: replace with the equivalent geohash:Intersects(WKT)
+        StringBuffer sb = new StringBuffer();
+        Matcher m = spatialObjectPattern.matcher(current[1]);
+        int max = getMaxBooleanClauses();
+        HashSet<String> failedObjects = new HashSet<>();
+        while (m.find()) {
+            String spatialObjectId = m.group(2);
+            String prefix = m.group(1);
+            try {
+                String wkt = layersService.getObjectWkt(spatialObjectId);
+
+                if (wkt == null) {
+                    throw new Exception("invalid object id");
+                }
+                String q = prefix + spatialField + ":\"Intersects(" + wkt + ")\"";
+
+                m.appendReplacement(sb, q);
+            } catch (Exception e) {
+                logger.error("failed to get WKT for object: " + spatialObjectId);
+                m.appendReplacement(sb, prefix + "(NOT *:*)");
+                failedObjects.add(spatialObjectId);
+            }
+        }
+        m.appendTail(sb);
+        current[1] = sb.toString();
+
+        sb = new StringBuffer();
+        m = spatialObjectPattern.matcher(current[0]);
+        while (m.find()) {
+            String spatialObjectId = m.group(2);
+            String prefix = m.group(1);
+            if (failedObjects.contains(spatialObjectId)) {
+                m.appendReplacement(sb, prefix + "<span class=\"spatialObject failed\" id='" + HtmlEscapers.htmlEscaper().escape(spatialObjectId) + "'>" + HtmlEscapers.htmlEscaper().escape(spatialObjectId) + " (FAILED)</span>");
+            } else {
+                try {
+                    SpatialObjectDTO obj = layersService.getObject(spatialObjectId);
+                    String name = obj.getName();
+                    m.appendReplacement(sb, prefix + "<span class='spatialObject' id='" + HtmlEscapers.htmlEscaper().escape(spatialObjectId) + "'>" + HtmlEscapers.htmlEscaper().escape(name) + "</span>");
+                } catch (Exception e) {
+                    logger.error("Couldn't get spatial object name for " + spatialObjectId, e);
+                    m.appendReplacement(sb, prefix + "<span class='spatialObject' id='" + HtmlEscapers.htmlEscaper().escape(spatialObjectId) + "'>Species list</span>");
+                }
+            }
+        }
+        m.appendTail(sb);
+        current[0] = sb.toString();
+    }
+
     /**
      * General formatting for formattedQuery and displayString.
      *
@@ -818,6 +882,8 @@ public String[] formatQueryTerm(String query, SpatialSearchRequestDTO searchPara
         formatHttp(formatted);
         formatTitleMap(formatted);
 
+        formatSpatialObject(formatted);
+
         if (!formatSpatial(formatted)) {
             formatGeneral(formatted, searchParams);
         }