Skip to content

Commit

Permalink
Merge pull request #20302 from vespa-engine/revert-20298-bratseth/aut…
Browse files Browse the repository at this point in the history 
…oscaling-in-dev

Revert "Account for capacity policies when autoscaling"
  • Loading branch information
@hakonhall
hakonhall authored Nov 30, 2021
2 parents 318edd1 + 612cba5 commit aa3ffac
Show file tree
Hide file tree
Showing 15 changed files with 102 additions and 209 deletions.
9 changes: 5 additions & 4 deletions node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,10 @@
package com.yahoo.vespa.hosted.provision.applications;

import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.Capacity;
import com.yahoo.config.provision.ClusterResources;
import com.yahoo.config.provision.ClusterSpec;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.HashMap;
import java.util.Optional;
Expand Down Expand Up @@ -58,12 +59,12 @@ public Application with(Cluster cluster) {
* Returns an application with the given cluster having the min and max resource limits of the given cluster.
* If the cluster has a target which is not inside the new limits, the target is removed.
*/
public Application withCluster(ClusterSpec.Id id, boolean exclusive, Capacity requested) {
public Application withCluster(ClusterSpec.Id id, boolean exclusive, ClusterResources min, ClusterResources max) {
Cluster cluster = clusters.get(id);
if (cluster == null)
cluster = Cluster.create(id, exclusive, requested);
cluster = new Cluster(id, exclusive, min, max, Optional.empty(), Optional.empty(), List.of(), AutoscalingStatus.empty());
else
cluster = cluster.withConfiguration(exclusive, requested);
cluster = cluster.withConfiguration(exclusive, min, max);
return with(cluster);
}

Expand Down
33 changes: 8 additions & 25 deletions node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.provision.applications;

import com.yahoo.config.provision.Capacity;
import com.yahoo.config.provision.ClusterResources;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.vespa.hosted.provision.autoscale.Autoscaler;
Expand All @@ -26,7 +25,6 @@ public class Cluster {
private final ClusterSpec.Id id;
private final boolean exclusive;
private final ClusterResources min, max;
private boolean required;
private final Optional<Suggestion> suggested;
private final Optional<ClusterResources> target;

Expand All @@ -38,7 +36,6 @@ public Cluster(ClusterSpec.Id id,
boolean exclusive,
ClusterResources minResources,
ClusterResources maxResources,
boolean required,
Optional<Suggestion> suggestedResources,
Optional<ClusterResources> targetResources,
List<ScalingEvent> scalingEvents,
Expand All @@ -47,7 +44,6 @@ public Cluster(ClusterSpec.Id id,
this.exclusive = exclusive;
this.min = Objects.requireNonNull(minResources);
this.max = Objects.requireNonNull(maxResources);
this.required = required;
this.suggested = Objects.requireNonNull(suggestedResources);
Objects.requireNonNull(targetResources);
if (targetResources.isPresent() && ! targetResources.get().isWithin(minResources, maxResources))
Expand All @@ -60,20 +56,14 @@ public Cluster(ClusterSpec.Id id,

public ClusterSpec.Id id() { return id; }

/** Returns whether the nodes allocated to this cluster must be on host exclusively dedicated to this application */
public boolean exclusive() { return exclusive; }

/** Returns the configured minimal resources in this cluster */
public ClusterResources minResources() { return min; }

/** Returns the configured maximal resources in this cluster */
public ClusterResources maxResources() { return max; }

/**
* Returns whether the resources of this cluster are required to be within the specified min and max.
* Otherwise they may be adjusted by capacity policies.
*/
public boolean required() { return required; }
/** Returns whether the nodes allocated to this cluster must be on host exclusively dedicated to this application */
public boolean exclusive() { return exclusive; }

/**
* Returns the computed resources (between min and max, inclusive) this cluster should
Expand Down Expand Up @@ -107,18 +97,16 @@ public Optional<ScalingEvent> lastScalingEvent() {
/** The latest autoscaling status of this cluster, or unknown (never null) if none */
public AutoscalingStatus autoscalingStatus() { return autoscalingStatus; }

public Cluster withConfiguration(boolean exclusive, Capacity capacity) {
return new Cluster(id, exclusive,
capacity.minResources(), capacity.maxResources(), capacity.isRequired(),
suggested, target, scalingEvents, autoscalingStatus);
public Cluster withConfiguration(boolean exclusive, ClusterResources min, ClusterResources max) {
return new Cluster(id, exclusive, min, max, suggested, target, scalingEvents, autoscalingStatus);
}

public Cluster withSuggested(Optional<Suggestion> suggested) {
return new Cluster(id, exclusive, min, max, required, suggested, target, scalingEvents, autoscalingStatus);
return new Cluster(id, exclusive, min, max, suggested, target, scalingEvents, autoscalingStatus);
}

public Cluster withTarget(Optional<ClusterResources> target) {
return new Cluster(id, exclusive, min, max, required, suggested, target, scalingEvents, autoscalingStatus);
return new Cluster(id, exclusive, min, max, suggested, target, scalingEvents, autoscalingStatus);
}

/** Add or update (based on "at" time) a scaling event */
Expand All @@ -132,12 +120,12 @@ public Cluster with(ScalingEvent scalingEvent) {
scalingEvents.add(scalingEvent);

prune(scalingEvents);
return new Cluster(id, exclusive, min, max, required, suggested, target, scalingEvents, autoscalingStatus);
return new Cluster(id, exclusive, min, max, suggested, target, scalingEvents, autoscalingStatus);
}

public Cluster with(AutoscalingStatus autoscalingStatus) {
if (autoscalingStatus.equals(this.autoscalingStatus)) return this;
return new Cluster(id, exclusive, min, max, required, suggested, target, scalingEvents, autoscalingStatus);
return new Cluster(id, exclusive, min, max, suggested, target, scalingEvents, autoscalingStatus);
}

@Override
Expand Down Expand Up @@ -168,11 +156,6 @@ private int eventIndexAt(Instant at) {
return -1;
}

public static Cluster create(ClusterSpec.Id id, boolean exclusive, Capacity requested) {
return new Cluster(id, exclusive, requested.minResources(), requested.maxResources(), requested.isRequired(),
Optional.empty(), Optional.empty(), List.of(), AutoscalingStatus.empty());
}

public static class Suggestion {

private final ClusterResources resources;
Expand Down
26 changes: 10 additions & 16 deletions ...src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
import com.yahoo.vespa.hosted.provision.Node;
import com.yahoo.vespa.hosted.provision.NodeList;
import com.yahoo.vespa.hosted.provision.NodeRepository;
import com.yahoo.vespa.hosted.provision.provisioning.CapacityPolicies;
import com.yahoo.vespa.hosted.provision.provisioning.NodeResourceLimits;

import java.util.List;
Expand Down Expand Up @@ -55,14 +54,14 @@ public AllocatableClusterResources(List<Node> nodes, NodeRepository nodeReposito

public AllocatableClusterResources(ClusterResources realResources,
NodeResources advertisedResources,
ClusterResources idealResources,
NodeResources idealResources,
ClusterSpec clusterSpec) {
this.nodes = realResources.nodes();
this.groups = realResources.groups();
this.realResources = realResources.nodeResources();
this.advertisedResources = advertisedResources;
this.clusterSpec = clusterSpec;
this.fulfilment = fulfilment(realResources, idealResources);
this.fulfilment = fulfilment(realResources.nodeResources(), idealResources);
}

/**
Expand Down Expand Up @@ -100,10 +99,10 @@ public int groupSize() {
*/
public double fulfilment() { return fulfilment; }

private static double fulfilment(ClusterResources realResources, ClusterResources idealResources) {
double vcpuFulfilment = Math.min(1, realResources.totalResources().vcpu() / idealResources.totalResources().vcpu());
double memoryGbFulfilment = Math.min(1, realResources.totalResources().memoryGb() / idealResources.totalResources().memoryGb());
double diskGbFulfilment = Math.min(1, realResources.totalResources().diskGb() / idealResources.totalResources().diskGb());
private static double fulfilment(NodeResources realResources, NodeResources idealResources) {
double vcpuFulfilment = Math.min(1, realResources.vcpu() / idealResources.vcpu());
double memoryGbFulfilment = Math.min(1, realResources.memoryGb() / idealResources.memoryGb());
double diskGbFulfilment = Math.min(1, realResources.diskGb() / idealResources.diskGb());
return (vcpuFulfilment + memoryGbFulfilment + diskGbFulfilment) / 3;
}

Expand Down Expand Up @@ -139,25 +138,21 @@ private static NodeResources averageRealResourcesOf(List<Node> nodes, NodeReposi
public static Optional<AllocatableClusterResources> from(ClusterResources wantedResources,
ClusterSpec clusterSpec,
Limits applicationLimits,
boolean required,
NodeList hosts,
NodeRepository nodeRepository) {
var capacityPolicies = new CapacityPolicies(nodeRepository);
var systemLimits = new NodeResourceLimits(nodeRepository);
boolean exclusive = clusterSpec.isExclusive();
int actualNodes = capacityPolicies.decideSize(wantedResources.nodes(), required, true, false, clusterSpec);
if ( !clusterSpec.isExclusive() && !nodeRepository.zone().getCloud().dynamicProvisioning()) {
// We decide resources: Add overhead to what we'll request (advertised) to make sure real becomes (at least) cappedNodeResources
var advertisedResources = nodeRepository.resourcesCalculator().realToRequest(wantedResources.nodeResources(), exclusive);
advertisedResources = systemLimits.enlargeToLegal(advertisedResources, clusterSpec.type(), exclusive); // Ask for something legal
advertisedResources = applicationLimits.cap(advertisedResources); // Overrides other conditions, even if it will then fail
advertisedResources = capacityPolicies.decideNodeResources(advertisedResources, required, clusterSpec); // Adjust to what we can request
var realResources = nodeRepository.resourcesCalculator().requestToReal(advertisedResources, exclusive); // What we'll really get
if ( ! systemLimits.isWithinRealLimits(realResources, clusterSpec.type())) return Optional.empty();
if (matchesAny(hosts, advertisedResources))
return Optional.of(new AllocatableClusterResources(wantedResources.withNodes(actualNodes).with(realResources),
return Optional.of(new AllocatableClusterResources(wantedResources.with(realResources),
advertisedResources,
wantedResources,
wantedResources.nodeResources(),
clusterSpec));
else
return Optional.empty();
Expand All @@ -168,7 +163,6 @@ public static Optional<AllocatableClusterResources> from(ClusterResources wanted
for (Flavor flavor : nodeRepository.flavors().getFlavors()) {
// Flavor decide resources: Real resources are the worst case real resources we'll get if we ask for these advertised resources
NodeResources advertisedResources = nodeRepository.resourcesCalculator().advertisedResourcesOf(flavor);
advertisedResources = capacityPolicies.decideNodeResources(advertisedResources, required, clusterSpec); // Adjust to what we can get
NodeResources realResources = nodeRepository.resourcesCalculator().requestToReal(advertisedResources, exclusive);

// Adjust where we don't need exact match to the flavor
Expand All @@ -184,9 +178,9 @@ public static Optional<AllocatableClusterResources> from(ClusterResources wanted

if ( ! between(applicationLimits.min().nodeResources(), applicationLimits.max().nodeResources(), advertisedResources)) continue;
if ( ! systemLimits.isWithinRealLimits(realResources, clusterSpec.type())) continue;
var candidate = new AllocatableClusterResources(wantedResources.withNodes(actualNodes).with(realResources),
var candidate = new AllocatableClusterResources(wantedResources.with(realResources),
advertisedResources,
wantedResources,
wantedResources.nodeResources(),
clusterSpec);
if (best.isEmpty() || candidate.preferableTo(best.get()))
best = Optional.of(candidate);
Expand Down
4 changes: 1 addition & 3 deletions ...ository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,9 +65,7 @@ public Optional<AllocatableClusterResources> findBestAllocation(ResourceTarget t
nodeResourcesWith(nodesAdjustedForRedundancy,
groupsAdjustedForRedundancy,
limits, target, current, clusterModel));
var allocatableResources = AllocatableClusterResources.from(next, current.clusterSpec(), limits,
clusterModel.cluster().required(),
hosts, nodeRepository);
var allocatableResources = AllocatableClusterResources.from(next, current.clusterSpec(), limits, hosts, nodeRepository);

if (allocatableResources.isEmpty()) continue;
if (bestAllocation.isEmpty() || allocatableResources.get().preferableTo(bestAllocation.get()))
Expand Down
5 changes: 0 additions & 5 deletions node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ public class ClusterModel {
static final double idealDiskLoad = 0.6;

private final Application application;
private final Cluster cluster;
/** The current nodes of this cluster, or empty if this models a new cluster not yet deployed */
private final NodeList nodes;
private final Clock clock;
Expand All @@ -51,7 +50,6 @@ public ClusterModel(Application application,
MetricsDb metricsDb,
Clock clock) {
this.application = application;
this.cluster = cluster;
this.nodes = clusterNodes;
this.clock = clock;
this.scalingDuration = computeScalingDuration(cluster, clusterSpec);
Expand All @@ -67,7 +65,6 @@ public ClusterModel(Application application,
ClusterTimeseries clusterTimeseries,
ClusterNodesTimeseries nodeTimeseries) {
this.application = application;
this.cluster = cluster;
this.nodes = null;
this.clock = clock;

Expand All @@ -76,8 +73,6 @@ public ClusterModel(Application application,
this.nodeTimeseries = nodeTimeseries;
}

public Cluster cluster() { return cluster; }

/** Returns the predicted duration of a rescaling of this cluster */
public Duration scalingDuration() { return scalingDuration; }

Expand Down
4 changes: 1 addition & 3 deletions ...ory/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,6 @@ public class ApplicationSerializer {
private static final String exclusiveKey = "exclusive";
private static final String minResourcesKey = "min";
private static final String maxResourcesKey = "max";
private static final String requiredKey = "required";
private static final String suggestedKey = "suggested";
private static final String resourcesKey = "resources";
private static final String targetResourcesKey = "target";
Expand Down Expand Up @@ -100,6 +99,7 @@ private static void toSlime(Status status, Cursor statusObject) {
}

private static Status statusFromSlime(Inspector statusObject) {
if ( ! statusObject.valid()) return Status.initial(); // TODO: Remove this line after March 2021
return new Status(statusObject.field(currentReadShareKey).asDouble(),
statusObject.field(maxReadShareKey).asDouble());
}
Expand All @@ -118,7 +118,6 @@ private static void toSlime(Cluster cluster, Cursor clusterObject) {
clusterObject.setBool(exclusiveKey, cluster.exclusive());
toSlime(cluster.minResources(), clusterObject.setObject(minResourcesKey));
toSlime(cluster.maxResources(), clusterObject.setObject(maxResourcesKey));
clusterObject.setBool(requiredKey, cluster.required());
cluster.suggestedResources().ifPresent(suggested -> toSlime(suggested, clusterObject.setObject(suggestedKey)));
cluster.targetResources().ifPresent(target -> toSlime(target, clusterObject.setObject(targetResourcesKey)));
scalingEventsToSlime(cluster.scalingEvents(), clusterObject.setArray(scalingEventsKey));
Expand All @@ -131,7 +130,6 @@ private static Cluster clusterFromSlime(String id, Inspector clusterObject) {
clusterObject.field(exclusiveKey).asBool(),
clusterResourcesFromSlime(clusterObject.field(minResourcesKey)),
clusterResourcesFromSlime(clusterObject.field(maxResourcesKey)),
clusterObject.field(requiredKey).asBool(),
optionalSuggestionFromSlime(clusterObject.field(suggestedKey)),
optionalClusterResourcesFromSlime(clusterObject.field(targetResourcesKey)),
scalingEventsFromSlime(clusterObject.field(scalingEventsKey)),
Expand Down
12 changes: 6 additions & 6 deletions ...ository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,11 @@ public CapacityPolicies(NodeRepository nodeRepository) {
this.sharedHosts = type -> PermanentFlags.SHARED_HOST.bindTo(nodeRepository.flagSource()).value().isEnabled(type.name());
}

public int decideSize(int requested, boolean required, boolean canFail, boolean isTester, ClusterSpec cluster) {
if (isTester) return 1;
public int decideSize(int requested, Capacity capacity, ClusterSpec cluster, ApplicationId application) {
if (application.instance().isTester()) return 1;

ensureRedundancy(requested, cluster, canFail);
if (required) return requested;
ensureRedundancy(requested, cluster, capacity.canFail());
if (capacity.isRequired()) return requested;
switch(zone.environment()) {
case dev : case test : return 1;
case perf : return Math.min(requested, 3);
Expand All @@ -43,11 +43,11 @@ public int decideSize(int requested, boolean required, boolean canFail, boolean
}
}

public NodeResources decideNodeResources(NodeResources target, boolean required, ClusterSpec cluster) {
public NodeResources decideNodeResources(NodeResources target, Capacity capacity, ClusterSpec cluster) {
if (target.isUnspecified())
target = defaultNodeResources(cluster.type());

if (required) return target;
if (capacity.isRequired()) return target;

// Dev does not cap the cpu or network of containers since usage is spotty: Allocate just a small amount exclusively
if (zone.environment() == Environment.dev && !zone.getCloud().dynamicProvisioning())
Expand Down
Loading

0 comments on commit aa3ffac

Please sign in to comment.