Merge pull request #25150 from vespa-engine/arnej/add-tuning-of-logging

Arnej/add tuning of logging

config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java

@@ -100,6 +100,9 @@ private SentinelConfig.Service.Builder getServiceConfig(Service s, String startu
         for (var entry : s.getEnvVars().entrySet()) {
             serviceBuilder.environ(b -> b.varname(entry.getKey()).varvalue(entry.getValue().toString()));
         }
+        for (var entry : s.getLogctlSpecs()) {
+            serviceBuilder.logctl(b -> b.componentSpec(entry.componentSpec).levelsModSpec(entry.levelsModSpec));
+        }
         setPreShutdownCommand(serviceBuilder, s);
         return serviceBuilder;
     }

config-model/src/main/java/com/yahoo/vespa/model/LogctlSpec.java

@@ -0,0 +1,11 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model;
+
+public class LogctlSpec {
+    public final String componentSpec;
+    public final String levelsModSpec;
+    public LogctlSpec(String componentSpec, String levelsModSpec) {
+        this.componentSpec = componentSpec;
+        this.levelsModSpec = levelsModSpec;
+    }
+}

config-model/src/main/java/com/yahoo/vespa/model/Service.java

@@ -4,6 +4,7 @@
 import com.yahoo.config.model.api.ServiceInfo;
 
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 
@@ -24,6 +25,8 @@ public interface Service extends ConfigProducer, NetworkPortRequestor {
     // environment variables specific for this service:
     Map<String, Object> getEnvVars();
 
+    default List<LogctlSpec> getLogctlSpecs() { return List.of(); }
+
     /**
      * Services that wish that a command should be run before shutdown
      * should return the command here. The command will be executed

config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java

@@ -15,6 +15,7 @@
 import com.yahoo.vespa.model.ConfigProxy;
 import com.yahoo.vespa.model.ConfigSentinel;
 import com.yahoo.vespa.model.HostResource;
+import com.yahoo.vespa.model.LogctlSpec;
 import com.yahoo.vespa.model.Logd;
 import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerContainer;
 import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerContainerCluster;
@@ -68,6 +69,14 @@ public void setLogForwarderConfig(LogForwarder.Config cfg, boolean includeAdmin)
         this.logForwarderIncludeAdmin = includeAdmin;
     }
 
+    private final List<LogctlSpec> logctlSpecs = new ArrayList<>();
+    public List<LogctlSpec> getLogctlSpecs() {
+        return logctlSpecs;
+    }
+    public void addLogctlCommand(String componentSpec, String levelsModSpec) {
+        logctlSpecs.add(new LogctlSpec(componentSpec,  levelsModSpec));
+    }
+
     /**
      * The single cluster controller cluster shared by all content clusters by default when not multitenant.
      * If multitenant, this is null.

config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java

@@ -223,16 +223,6 @@ private Map<String, MetricsConsumer> getUserMetricsConsumers() {
                 .orElse(Collections.emptyMap());
     }
 
-    private Optional<Admin> getAdmin() {
-        if (parent != null) {
-            AbstractConfigProducerRoot r = parent.getRoot();
-            if (r instanceof VespaModel model) {
-                return Optional.ofNullable(model.getAdmin());
-            }
-        }
-        return Optional.empty();
-    }
-
     private Optional<String> getSystemName() {
         Monitoring monitoring = getMonitoringService();
         return monitoring != null && ! monitoring.getClustername().equals("") ?

config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java

@@ -5,6 +5,7 @@
 import com.yahoo.config.model.api.ConfigServerSpec;
 import com.yahoo.config.model.deploy.DeployState;
 import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.log.internal.LevelsModSpec;
 import com.yahoo.text.XML;
 import com.yahoo.vespa.model.Host;
 import com.yahoo.vespa.model.HostResource;
@@ -21,7 +22,9 @@
 import org.w3c.dom.Element;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 
 /**
@@ -110,4 +113,23 @@ void addLogForwarders(ModelElement logForwardingElement, Admin admin) {
         }
     }
 
+    private void addLoggingSpec(ModelElement loggingSpec, Admin admin) {
+        if (loggingSpec == null) return;
+        String componentSpec = loggingSpec.requiredStringAttribute("name");
+        String levels = loggingSpec.requiredStringAttribute("levels");
+        var levelSpec = new LevelsModSpec();
+        levelSpec.setLevels(levels);
+        admin.addLogctlCommand(componentSpec, levelSpec.toLogctlModSpec());
+    }
+
+    void addLoggingSpecs(ModelElement loggingElement, Admin admin) {
+        if (loggingElement == null) return;
+        for (ModelElement e : loggingElement.children("class")) {
+            addLoggingSpec(e, admin);
+        }
+        for (ModelElement e : loggingElement.children("package")) {
+            addLoggingSpec(e, admin);
+        }
+    }
+
 }

config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java

@@ -47,6 +47,7 @@ protected void doBuildAdmin(DeployState deployState, Admin admin, Element adminE
             admin.setClusterControllers(addConfiguredClusterControllers(deployState, admin, adminE), deployState);
 
         addLogForwarders(new ModelElement(adminE).child("logforwarding"), admin);
+        addLoggingSpecs(new ModelElement(adminE).child("logging"), admin);
     }
 
     private List<Configserver> parseConfigservers(DeployState deployState, Admin admin, Element adminE) {

config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java

@@ -56,6 +56,7 @@ protected void doBuildAdmin(DeployState deployState, Admin admin, Element w3cAdm
         assignLogserver(deployState, requestedLogservers.orElse(createNodesSpecificationForLogserver()), admin);
 
         addLogForwarders(adminElement.child("logforwarding"), admin);
+        addLoggingSpecs(adminElement.child("logging"), admin);
     }
 
     private void assignSlobroks(DeployState deployState, NodesSpecification nodesSpecification, Admin admin) {

config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java

@@ -9,8 +9,10 @@
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.NodeResources;
 import com.yahoo.search.config.QrStartConfig;
+import com.yahoo.vespa.model.LogctlSpec;
 import com.yahoo.vespa.model.container.component.SimpleComponent;
 
+import java.util.List;
 import java.util.Optional;
 
 import static com.yahoo.vespa.defaults.Defaults.getDefaults;
@@ -42,6 +44,15 @@ public ApplicationContainer(AbstractConfigProducer<?> parent, String name, boole
         addComponent(new SimpleComponent("com.yahoo.container.jdisc.ClusterInfoProvider"));
     }
 
+    private List<LogctlSpec> logctlSpecs = List.of();
+    void setLogctlSpecs(List<LogctlSpec> logctlSpecs) {
+        this.logctlSpecs = logctlSpecs;
+    }
+    @Override
+    public List<LogctlSpec> getLogctlSpecs() {
+        return logctlSpecs;
+    }
+
     @Override
     public void getConfig(QrStartConfig.Builder builder) {
         if (getHostResource() != null) {

config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java

@@ -31,6 +31,7 @@
 import com.yahoo.vespa.config.search.core.RankingConstantsConfig;
 import com.yahoo.vespa.config.search.core.RankingExpressionsConfig;
 import com.yahoo.vespa.model.AbstractService;
+import com.yahoo.vespa.model.VespaModel;
 import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer;
 import com.yahoo.vespa.model.container.component.BindingPattern;
 import com.yahoo.vespa.model.container.component.Component;
@@ -125,11 +126,19 @@ public ApplicationContainerCluster(AbstractConfigProducer<?> parent, String conf
                 : defaultHeapSizePercentageOfTotalNodeMemory;
     }
 
+    private void wireLogctlSpecs() {
+        getAdmin().ifPresent(admin -> {
+            for (var c : getContainers()) {
+                c.setLogctlSpecs(admin.getLogctlSpecs());
+            }});
+    }
+
     @Override
     protected void doPrepare(DeployState deployState) {
         addAndSendApplicationBundles(deployState);
         sendUserConfiguredFiles(deployState);
         createEndpointList(deployState);
+        wireLogctlSpecs();
     }
 
     private void addAndSendApplicationBundles(DeployState deployState) {

config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java

@@ -36,6 +36,8 @@
 import com.yahoo.vespa.configdefinition.IlscriptsConfig;
 import com.yahoo.vespa.model.PortsMeta;
 import com.yahoo.vespa.model.Service;
+import com.yahoo.vespa.model.VespaModel;
+import com.yahoo.vespa.model.admin.Admin;
 import com.yahoo.vespa.model.admin.monitoring.Monitoring;
 import com.yahoo.vespa.model.clients.ContainerDocumentApi;
 import com.yahoo.vespa.model.container.component.AccessLogComponent;
@@ -208,6 +210,17 @@ public Zone getZone() {
         return zone;
     }
 
+    protected Optional<Admin> getAdmin() {
+        var parent = getParent();
+        if (parent != null) {
+            var r = parent.getRoot();
+            if (r instanceof VespaModel model) {
+                return Optional.ofNullable(model.getAdmin());
+            }
+        }
+        return Optional.empty();
+    }
+
     public void addDefaultHandlersWithVip() {
         addDefaultHandlersExceptStatus();
         addVipHandler();

config-model/src/main/resources/schema/admin.rnc

@@ -12,6 +12,7 @@ AdminV2 =
    AdminMonitoring? &
    Metrics? &
    ClusterControllers? &
+   LoggingSpecs? &
    LogForwarding?
  }
 
@@ -30,6 +31,7 @@ AdminV4 =
     GenericConfig* &
     AdminMonitoring? &
     Metrics? &
+    LoggingSpecs? &
     LogForwarding?
   }
 
@@ -113,3 +115,17 @@ LogForwarding = element logforwarding {
         attribute phone-home-interval { xsd:positiveInteger }?
     }
 }
+
+LoggingSpecs = element logging {
+    (
+    element class {
+        attribute name { xsd:Name } &
+        attribute levels { xsd:string }
+    }
+    |
+    element package {
+        attribute name { xsd:Name } &
+        attribute levels { xsd:string }
+    }
+    )*
+}

configd/src/apps/sentinel/CMakeLists.txt

@@ -7,6 +7,7 @@ vespa_add_executable(configd_config-sentinel_app
     connectivity.cpp
     env.cpp
     line-splitter.cpp
+    logctl.cpp
     manager.cpp
     metrics.cpp
     model-owner.cpp

configd/src/apps/sentinel/logctl.cpp

@@ -0,0 +1,48 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "logctl.h"
+
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <unistd.h>
+#include <stdio.h>
+
+#include <vespa/log/log.h>
+LOG_SETUP(".sentinel.logctl");
+
+namespace config::sentinel {
+
+void justRunLogctl(const char *cspec, const char *lspec)
+{
+    const char *progName = "vespa-logctl";
+    pid_t pid = fork();
+    if (pid == 0) {
+        LOG(debug, "running '%s' '%s' '%s'", progName, cspec, lspec);
+        int rv = execlp(progName, progName, "-c", cspec, lspec, nullptr);
+        if (rv != 0) {
+            LOG(warning, "execlp of '%s' failed: %s", progName, strerror(errno));
+        }
+    } else if (pid > 0) {
+        int wstatus = 0;
+        pid_t got = waitpid(pid, &wstatus, 0);
+        if (got == pid) {
+            if (WIFEXITED(wstatus)) {
+                int exitCode = WEXITSTATUS(wstatus);
+                if (exitCode != 0) {
+                    LOG(warning, "running '%s' failed (exit code %d)", progName, exitCode);
+                }
+            } else if (WIFSIGNALED(wstatus)) {
+                int termSig = WTERMSIG(wstatus);
+                LOG(warning, "running '%s' failed (got signal %d)", progName, termSig);
+            } else {
+                LOG(warning, "'%s' failure (wait status was %d)", progName, wstatus);
+            }
+        } else {
+            LOG(error, "waitpid() failed: %s", strerror(errno));
+        }
+    } else {
+        LOG(error, "fork() failed: %s", strerror(errno));
+    }
+}
+
+}

configd/src/apps/sentinel/logctl.h

@@ -0,0 +1,8 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#pragma once
+
+namespace config::sentinel {
+
+void justRunLogctl(const char *cspec, const char *lspec);
+
+}

configd/src/apps/sentinel/service.cpp

@@ -2,6 +2,7 @@
 
 #include "service.h"
 #include "output-connection.h"
+#include "logctl.h"
 #include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/vespalib/util/signalhandler.h>
 
@@ -56,6 +57,21 @@ Service::Service(const SentinelConfig::Service& service, const SentinelConfig::A
     start();
 }
 
+void applyLogctl(const cloud::config::SentinelConfig::Service &config) {
+    for (const auto &logctl : config.logctl) {
+        const auto cspec = config.name + ":" + logctl.componentSpec;
+        const auto lspec = logctl.levelsModSpec;
+        justRunLogctl(cspec.c_str(), lspec.c_str());
+    }
+}
+
+void unApplyLogctl(const cloud::config::SentinelConfig::Service &config) {
+    for (const auto &logctl : config.logctl) {
+        const auto cspec = config.name + ":" + logctl.componentSpec;
+        justRunLogctl(cspec.c_str(), "all=on,debug=off,spam=off");
+    }
+}
+
 void
 Service::reconfigure(const SentinelConfig::Service& config)
 {
@@ -70,8 +86,10 @@ Service::reconfigure(const SentinelConfig::Service& config)
         terminate();
     }
 
+    unApplyLogctl(*_config);
     delete _config;
     _config = new SentinelConfig::Service(config);
+    applyLogctl(*_config);
 
     if ((_state == READY) || (_state == FINISHED) || (_state == RESTARTING)) {
         if (_isAutomatic) {
@@ -329,6 +347,7 @@ Service::runChild()
     for (const auto &envvar : _config->environ) {
         setenv(envvar.varname.c_str(), envvar.varvalue.c_str(), 1);
     }
+    applyLogctl(*_config);
 
     // Set up environment
     setenv("VESPA_SERVICE_NAME", _config->name.c_str(), 1);

configdefinitions/src/vespa/sentinel.def

@@ -43,6 +43,10 @@ service[].command string
 service[].environ[].varname string
 service[].environ[].varvalue string
 
+## Tune log-level settings for specific components
+service[].logctl[].componentSpec string
+service[].logctl[].levelsModSpec string
+
 ## The command to run before stopping service. The same properties as for
 ## startup command holds.
 service[].preShutdownCommand string default=""