diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DeprovisionedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DeprovisionedExpirer.java index c2b383a5a54c..92062f13f1a4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DeprovisionedExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DeprovisionedExpirer.java @@ -3,12 +3,19 @@ import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.History; +import com.yahoo.vespa.hosted.provision.node.History.Event.Type; import java.time.Duration; +import java.time.Instant; +import java.util.ArrayDeque; +import java.util.Deque; import java.util.List; +import static java.util.Comparator.comparing; + /** * This removes hosts from {@link com.yahoo.vespa.hosted.provision.Node.State#deprovisioned}, in dynamically provisioned * zones, after a grace period. @@ -17,21 +24,37 @@ */ public class DeprovisionedExpirer extends Expirer { + private static final int maxDeprovisionedNodes = 1000; + DeprovisionedExpirer(NodeRepository nodeRepository, Duration expiryTime, Metric metric) { super(Node.State.deprovisioned, History.Event.Type.deprovisioned, nodeRepository, expiryTime, metric); } @Override protected boolean isExpired(Node node) { - return nodeRepository().zone().cloud().dynamicProvisioning() && - super.isExpired(node); + return nodeRepository().zone().cloud().dynamicProvisioning() && super.isExpired(node); } @Override - protected void expire(List expired) { - for (var node : expired) { - nodeRepository().nodes().forget(node); + protected NodeList getExpiredNodes() { + List deprovisioned = nodeRepository().nodes().list(Node.State.deprovisioned) + .sortedBy(comparing(node -> node.history().event(Type.deprovisioned) + .map(History.Event::at) + .orElse(Instant.EPOCH))) + .asList(); + Deque expired = new ArrayDeque<>(deprovisioned); + int kept = 0; + while ( ! expired.isEmpty()) { + if (isExpired(expired.getLast()) || kept++ >= maxDeprovisionedNodes) break; // If we encounter an expired node, the rest are also expired. + expired.removeLast(); } + return NodeList.copyOf(List.copyOf(expired)); + } + + @Override + protected void expire(List expired) { + nodeRepository().nodes().performOn(NodeList.copyOf(expired), + (node, lock) -> { nodeRepository().nodes().forget(node); return node; }); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java index a8929cf9d22f..1684ebbb38fa 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java @@ -40,7 +40,7 @@ public abstract class Expirer extends NodeRepositoryMaintainer { @Override protected double maintain() { - NodeList expired = nodeRepository().nodes().list(fromState).matching(this::isExpired); + NodeList expired = getExpiredNodes(); if ( ! expired.isEmpty()) { log.info(fromState + " expirer found " + expired.size() + " expired nodes: " + expired); @@ -51,6 +51,10 @@ protected double maintain() { return 1.0; } + protected NodeList getExpiredNodes() { + return nodeRepository().nodes().list(fromState).matching(this::isExpired); + } + protected boolean isExpired(Node node) { return isExpired(node, expiryTime); }