Merge pull request #4273 from vespa-engine/hmusum/move-test-4

Move test from internal repo

tests/config/filedistribution_uri/docs.json

@@ -0,0 +1,6 @@
+[
+  { "put": "id:tensor_from_uri:tensor_from_uri::0", "fields": {
+      "my_tensor": { "cells": [ { "address": { "x": "3" }, "value": 3 } ] }
+    }
+  }
+]

tests/config/filedistribution_uri/filedistribution_base.rb

@@ -0,0 +1,19 @@
+# Copyright Vespa.ai. All rights reserved.
+require 'config_test'
+require 'app_generator/container_app'
+
+module FileDistributionBase
+
+  def deploy(params)
+    deploy_app(create_app(), params)
+  end
+
+  def create_app
+    ContainerApp.new.
+      container(Container.new.
+                handler(Handler.new("com.yahoo.vespatest.VersionHandler").
+                        bundle("com.yahoo.vespatest.ExtraHitSearcher").
+                        binding("http://*/Version")))
+  end
+
+end

tests/config/filedistribution_uri/filedistribution_on_application_prepare.rb

@@ -0,0 +1,94 @@
+# Copyright Vespa.ai. All rights reserved.
+require 'config_test'
+require 'config/filedistribution_uri/filedistribution_base'
+
+# Note: 2 hosts are needed. If you want to run this by yourself you need to add "--configserverhost some_other_host"
+class FileDistributionOnApplicationPrepare < ConfigTest
+
+  include FileDistributionBase
+
+  def can_share_configservers?
+    true
+  end
+
+  def setup
+    set_owner("musum")
+    set_description("Tests file distribution happens when preparing an application")
+    @valgrind = false
+  end
+
+  # Check that file distribution works by verifying that a deployment
+  # with an updated bundle is distributed to the node, even though
+  # multiple deployments happened between preparing and activating
+  # that deployment
+  def test_filedistribution_on_application_prepare
+    initial = add_bundle_dir(selfdir+"initial", "com.yahoo.vespatest.ExtraHitSearcher", :name => 'initial')
+    updated = add_bundle_dir(selfdir+"updated", "com.yahoo.vespatest.ExtraHitSearcher", :name => 'updated')
+    compile_bundles(@vespa.nodeproxies.values.first)
+
+    # initial deployment
+    initial_output = deploy_and_activate(initial)
+    session_initial = get_generation(initial_output).to_i
+    start
+
+    #vespa.adminserver.execute("vespa-logctl configproxy:com.yahoo.vespa.filedistribution debug=on", :exceptiononfailure => false)
+    #vespa.adminserver.execute("vespa-logctl configproxy:com.yahoo.vespa.config.proxy.filedistribution.FileDistributionRpcServer debug=on", :exceptiononfailure => false)
+
+    # deployment with updated bundle
+    updated_output = deploy_do_not_activate(updated)
+    session_updated_bundle = get_generation_from_prepare(updated_output).to_i
+
+    start_session = session_updated_bundle + 1
+    # Deploy several times without activating (simulates internal redeployment with many model versions)
+    start_session.upto(start_session + 5) { |session|
+      deploy_from_active_app_do_not_activate
+      puts "Preparing #{session}"
+      deploy_prepare(session)
+    }
+
+    # Activate session with updated bundle
+    deploy_activate(session_updated_bundle)
+
+    @container = vespa.container.values.first
+    # Now, check that file distribution distributed the updated bundle (i.e. it got the expected config generation)
+    @container.wait_for_config_generation(session_updated_bundle)
+  end
+
+  def deploy_and_activate(bundle)
+    deploy({:bundles => [bundle]})
+  end
+
+  def deploy_do_not_activate(bundle)
+    deploy({:bundles => [bundle], :no_activate => true, :skip_create_model => true})
+  end
+
+  def deploy_from_active_app_do_not_activate
+    # TODO: Below is an attempt to make this work on just one host, need to look into why it failed
+    configserver = (configserverhostlist.length > 0 ? configserverhostlist[0] : vespa.nodeproxies.first[0])
+    from_url = "http://#{configserver}:19071/application/v2/tenant/#{@tenant_name}/application/#{@application_name}/environment/prod/region/default/instance/default"
+    deploy({:from_url => from_url, :no_activate => true, :skip_create_model => true})
+  end
+
+  def deploy_prepare(session_id)
+    deploy_with_command("prepare", session_id)
+  end
+
+  def deploy_activate(session_id)
+    deploy_with_command("activate", session_id)
+  end
+
+  def deploy_with_command(command, session_id)
+    node = vespa.adminserver
+    execute(node, "vespa-deploy -e #{@tenant_name} -a #{application_name} #{command} #{session_id}")
+  end
+
+  def get_generation_from_prepare(deploy_output)
+    deploy_output =~ /Session (\d+) for tenant/i
+    return $1;
+  end
+
+  def teardown
+    stop
+  end
+
+end

tests/config/filedistribution_uri/filedistribution_uri.rb

@@ -0,0 +1,31 @@
+# Copyright Vespa.ai. All rights reserved.
+require 'config_test'
+require 'json'
+require 'app_generator/container_app'
+require 'app_generator/search_app'
+
+# Note: 2 hosts are needed. If you want to run this by yourself you need to add "--configserverhost some_other_host"
+class FileDistributionUri < ConfigTest
+
+  def can_share_configservers?
+    true
+  end
+
+  def setup
+    set_owner("musum")
+    set_description("Tests file distribution with uri")
+    @valgrind = false
+  end
+
+  # Tests getting a file via a https uri (a constant tensor)
+  def test_filedistribution_uri_https
+    deploy_app(SearchApp.new().sd(selfdir + 'tensor-from-uri-https-sd/tensor_from_uri.sd'))
+    start
+    feed_and_wait_for_docs("tensor_from_uri", 1, :file => selfdir + "docs.json")
+  end
+
+  def teardown
+    stop
+  end
+
+end

tests/config/filedistribution_uri/initial/src/main/java/com/yahoo/vespatest/ExtraHitSearcher.java

@@ -0,0 +1,30 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespatest;
+
+import com.yahoo.search.Searcher;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.result.Hit;
+import com.yahoo.search.searchchain.Execution;
+import com.yahoo.vespatest.ExtraHitConfig;
+import com.yahoo.component.ComponentId;
+
+public class ExtraHitSearcher extends Searcher {
+
+    final String title;
+
+    public ExtraHitSearcher(ComponentId id, ExtraHitConfig config) {
+        super(id);
+        title = "Searcher says: " + config.exampleString();
+    }
+
+    @Override
+    public Result search(Query query, Execution execution) {
+        query.properties().set("query", query.properties().getString("query") + " AND demo");
+        Result result = execution.search(query);
+        Hit hit = new Hit("id");
+        hit.setField("title", title);
+        result.hits().add(hit);
+        return result;
+    }
+}

tests/config/filedistribution_uri/initial/src/main/java/com/yahoo/vespatest/VersionHandler.java

@@ -0,0 +1,17 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespatest;
+
+import com.yahoo.jdisc.Request;
+import com.yahoo.jdisc.Response;
+import com.yahoo.jdisc.handler.*;
+
+public class VersionHandler extends AbstractRequestHandler {
+
+    @Override
+    public ContentChannel handleRequest(Request request, ResponseHandler handler) {
+        FastContentWriter writer = ResponseDispatch.newInstance(Response.Status.OK).connectFastWriter(handler);
+        writer.write("Initial handler");
+        writer.close();
+        return null;
+    }
+}

tests/config/filedistribution_uri/initial/src/main/resources/configdefinitions/extra-hit.def

@@ -0,0 +1,3 @@
+namespace=vespatest
+
+exampleString string default="Hello World"

tests/config/filedistribution_uri/tensor-from-uri-https-sd/tensor_from_uri.sd

@@ -0,0 +1,18 @@
+search tensor_from_uri {
+  document tensor_from_uri {
+    field my_tensor type tensor(x{}) {
+      indexing: attribute | summary
+    }
+  }
+  constant my_constant {
+    uri: https://data.vespa-cloud.com/tests/data/constant_tensor_1.json
+    type: tensor(x{})
+  }
+  rank-profile default {
+    first-phase {
+      expression: sum(attribute(my_tensor)*constant(my_constant))
+    }
+  }
+
+
+}

tests/config/filedistribution_uri/updated/src/main/java/com/yahoo/vespatest/ExtraHitSearcher.java

@@ -0,0 +1,28 @@
+package com.yahoo.vespatest;
+
+import com.yahoo.search.Searcher;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.result.Hit;
+import com.yahoo.search.searchchain.Execution;
+import com.yahoo.vespatest.ExtraHitConfig;
+import com.yahoo.component.ComponentId;
+
+public class ExtraHitSearcher extends Searcher {
+
+    final String title;
+
+    public ExtraHitSearcher(ComponentId id, ExtraHitConfig config) {
+        super(id);
+        title = "Searcher says: " + config.exampleString();
+    }
+
+    public @Override Result search(Query query, Execution execution) {
+        query.properties().set("query", query.properties().getString("query") + " AND demo");
+        Result result = execution.search(query);
+        Hit hit = new Hit("id");
+        hit.setField("title", title);
+        result.hits().add(hit);
+        return result;
+    }
+}

tests/config/filedistribution_uri/updated/src/main/java/com/yahoo/vespatest/VersionHandler.java

@@ -0,0 +1,16 @@
+package com.yahoo.vespatest;
+
+import com.yahoo.jdisc.Request;
+import com.yahoo.jdisc.Response;
+import com.yahoo.jdisc.handler.*;
+
+public class VersionHandler extends AbstractRequestHandler {
+
+    @Override
+    public ContentChannel handleRequest(Request request, ResponseHandler handler) {
+        FastContentWriter writer = ResponseDispatch.newInstance(Response.Status.OK).connectFastWriter(handler);
+        writer.write("Updated handler");
+        writer.close();
+        return null;
+    }
+}

tests/config/filedistribution_uri/updated/src/main/resources/configdefinitions/extra-hit.def

@@ -0,0 +1,3 @@
+namespace=vespatest
+
+exampleString string default="Hello World"

tests/config/filedistribution_uri/with_files/src/main/java/com/yahoo/vespatest/FilesizeHandler.java

@@ -0,0 +1,36 @@
+package com.yahoo.vespatest;
+
+import com.yahoo.jdisc.Request;
+import com.yahoo.jdisc.Response;
+import com.yahoo.jdisc.handler.*;
+import com.yahoo.filedistribution.fileacquirer.FileAcquirer;
+import com.yahoo.test.FilesConfig;
+
+import java.io.File;
+import java.util.concurrent.TimeUnit;
+
+public class FilesizeHandler extends AbstractRequestHandler {
+
+    private final File file;
+
+    public FilesizeHandler(FilesConfig filesConfig, FileAcquirer fileAcquirer) {
+	try {
+	    //System.err.println(this + " waiting for fileref: " + filesConfig.myFile());
+	    file = fileAcquirer.waitFor(filesConfig.myFile(), 5, TimeUnit.MINUTES);
+	    //System.err.println(this + " got file with fileref: " + filesConfig.myFile());
+        } catch (InterruptedException e) {
+            throw new RuntimeException("InterruptedException: ", e);
+        }
+    }
+
+    public ContentChannel handleRequest(Request request, ResponseHandler handler) {
+        FastContentWriter writer = ResponseDispatch.newInstance(Response.Status.OK).connectFastWriter(handler);
+        try {
+	    writer.write(String.valueOf(file.length()));
+        } finally {
+            writer.close();
+        }
+        return null;
+    }
+
+}

tests/config/filedistribution_uri/with_files/src/main/resources/configdefinitions/files.def

@@ -0,0 +1,3 @@
+package=com.yahoo.test
+
+myFile file