Limit the maximum nomber of deprovisioned nodes in a zone to 1000

vespa-engine · Jul 12, 2023 · 14f7bee · 14f7bee
1 parent ccb2515
commit 14f7bee
Show file tree

Hide file tree

Showing 2 changed files with 33 additions and 6 deletions.
diff --git a/...tory/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DeprovisionedExpirer.java b/...tory/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DeprovisionedExpirer.java
@@ -3,12 +3,19 @@
 
 import com.yahoo.jdisc.Metric;
 import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeList;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.node.History;
+import com.yahoo.vespa.hosted.provision.node.History.Event.Type;
 
 import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayDeque;
+import java.util.Deque;
 import java.util.List;
 
+import static java.util.Comparator.comparing;
+
 /**
  * This removes hosts from {@link com.yahoo.vespa.hosted.provision.Node.State#deprovisioned}, in dynamically provisioned
  * zones, after a grace period.
@@ -17,21 +24,37 @@
  */
 public class DeprovisionedExpirer extends Expirer {
 
+    private static final int maxDeprovisionedNodes = 1000;
+
     DeprovisionedExpirer(NodeRepository nodeRepository, Duration expiryTime, Metric metric) {
         super(Node.State.deprovisioned, History.Event.Type.deprovisioned, nodeRepository, expiryTime, metric);
     }
 
     @Override
     protected boolean isExpired(Node node) {
-        return nodeRepository().zone().cloud().dynamicProvisioning() &&
-               super.isExpired(node);
+        return nodeRepository().zone().cloud().dynamicProvisioning() && super.isExpired(node);
     }
 
     @Override
-    protected void expire(List<Node> expired) {
-        for (var node : expired) {
-            nodeRepository().nodes().forget(node);
+    protected NodeList getExpiredNodes() {
+        List<Node> deprovisioned = nodeRepository().nodes().list(Node.State.deprovisioned)
+                                                   .sortedBy(comparing(node -> node.history().event(Type.deprovisioned)
+                                                                                   .map(History.Event::at)
+                                                                                   .orElse(Instant.EPOCH)))
+                                                   .asList();
+        Deque<Node> expired = new ArrayDeque<>(deprovisioned);
+        int kept = 0;
+        while ( ! expired.isEmpty()) {
+            if (isExpired(expired.getLast()) || kept++ >= maxDeprovisionedNodes) break; // If we encounter an expired node, the rest are also expired.
+            expired.removeLast();
         }
+        return NodeList.copyOf(List.copyOf(expired));
+    }
+
+    @Override
+    protected void expire(List<Node> expired) {
+        nodeRepository().nodes().performOn(NodeList.copyOf(expired),
+                                           (node, lock) -> { nodeRepository().nodes().forget(node); return node; });
     }
 
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Expirer.java
@@ -40,7 +40,7 @@ public abstract class Expirer extends NodeRepositoryMaintainer {
 
     @Override
     protected double maintain() {
-        NodeList expired = nodeRepository().nodes().list(fromState).matching(this::isExpired);
+        NodeList expired = getExpiredNodes();
 
         if ( ! expired.isEmpty()) {
             log.info(fromState + " expirer found " + expired.size() + " expired nodes: " + expired);
@@ -51,6 +51,10 @@ protected double maintain() {
         return 1.0;
     }
 
+    protected NodeList getExpiredNodes() {
+        return nodeRepository().nodes().list(fromState).matching(this::isExpired);
+    }
+
     protected boolean isExpired(Node node) {
         return isExpired(node, expiryTime);
     }