From 728350622ab1e41198839cc22d73102967a9b67d Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Mon, 29 Aug 2022 10:44:59 +0200 Subject: [PATCH 1/7] Require non-null status page server --- ...troller.core.MasterElectionTest-output.txt | 859 ++++++++++++++++++ .../core/FleetController.java | 30 +- .../core/status/StatusHandler.java | 2 +- .../core/ClusterFeedBlockTest.java | 5 +- .../core/StateChangeTest.java | 5 +- 5 files changed, 879 insertions(+), 22 deletions(-) create mode 100644 clustercontroller-core/com.yahoo.vespa.clustercontroller.core.MasterElectionTest-output.txt diff --git a/clustercontroller-core/com.yahoo.vespa.clustercontroller.core.MasterElectionTest-output.txt b/clustercontroller-core/com.yahoo.vespa.clustercontroller.core.MasterElectionTest-output.txt new file mode 100644 index 000000000000..c4b87137f47e --- /dev/null +++ b/clustercontroller-core/com.yahoo.vespa.clustercontroller.core.MasterElectionTest-output.txt @@ -0,0 +1,859 @@ +STARTING TEST: MasterElectionTest::testMasterElection +1661243505.047218 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info STARTING TEST: MasterElectionTest::testMasterElection() +1661243505.048231 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.persistence.FileTxnSnapLog info zookeeper.snapshot.trust.empty : false +1661243505.049057 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.watch.WatchManagerFactory info Using org.apache.zookeeper.server.watch.WatchManager as watch manager +1661243505.049649 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.watch.WatchManagerFactory info Using org.apache.zookeeper.server.watch.WatchManager as watch manager +1661243505.050132 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ZKDatabase info zookeeper.snapshotSizeFactor = 0.33 +1661243505.050547 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ZKDatabase info zookeeper.commitLogCount=500 +1661243505.050983 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ZooKeeperServer info minSessionTimeout set to 4000 +1661243505.051315 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ZooKeeperServer info maxSessionTimeout set to 40000 +1661243505.051735 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ResponseCache info getData response cache size is initialized with value 400. +1661243505.052229 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ResponseCache info getChildren response cache size is initialized with value 400. +1661243505.052589 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.util.RequestPathMetricsCollector info zookeeper.pathStats.slotCapacity = 60 +1661243505.052878 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.util.RequestPathMetricsCollector info zookeeper.pathStats.slotDuration = 15 +1661243505.053195 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.util.RequestPathMetricsCollector info zookeeper.pathStats.maxDepth = 6 +1661243505.053450 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.util.RequestPathMetricsCollector info zookeeper.pathStats.initialDelay = 5 +1661243505.053698 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.util.RequestPathMetricsCollector info zookeeper.pathStats.delay = 5 +1661243505.053951 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.util.RequestPathMetricsCollector info zookeeper.pathStats.enabled = false +1661243505.054309 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ZooKeeperServer info The max bytes for all large requests are set to 104857600 +1661243505.054597 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ZooKeeperServer info The large request threshold is set to -1 +1661243505.054858 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.AuthenticationHelper info zookeeper.enforce.auth.enabled = false +1661243505.055152 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.AuthenticationHelper info zookeeper.enforce.auth.schemes = [] +1661243505.055426 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ZooKeeperServer info Created server with tickTime 2000 minSessionTimeout 4000 maxSessionTimeout 40000 clientPortListenBacklog -1 datadir /Users/musum/git/vespa/clustercontroller-core/target/surefire-reports/test_fltctrl_zk3840676619239160054sdir/version-2 snapdir /Users/musum/git/vespa/clustercontroller-core/target/surefire-reports/test_fltctrl_zk3840676619239160054sdir/version-2 +1661243505.056316 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ServerCnxnFactory warning maxCnxns is not configured, using default value 0. +1661243505.056699 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.NIOServerCnxnFactory info Configuring NIO connection handler with 10s sessionless connection timeout, 2 selector thread(s), 32 worker threads, and 64 kB direct buffers. +1661243505.057438 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.NIOServerCnxnFactory info binding to port 0.0.0.0/0.0.0.0:0 +1661243505.058607 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.persistence.FileTxnSnapLog info Snapshotting: 0x0 to /Users/musum/git/vespa/clustercontroller-core/target/surefire-reports/test_fltctrl_zk3840676619239160054sdir/version-2/snapshot.0 +1661243505.076253 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ZKDatabase info Snapshot loaded in 17 ms, highest zxid is 0x0, digest is 1371985504 +1661243505.076848 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.persistence.FileTxnSnapLog info Snapshotting: 0x0 to /Users/musum/git/vespa/clustercontroller-core/target/surefire-reports/test_fltctrl_zk3840676619239160054sdir/version-2/snapshot.0 +1661243505.078817 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ZooKeeperServer info Snapshot taken in 2 ms +1661243505.080001 localhost 75603/455 - fleetcontroller.org.apache.zookeeper.server.PrepRequestProcessor info PrepRequestProcessor (sid:0) started, reconfigEnabled=false +1661243505.082854 localhost 75603/1 - fleetcontroller.com.yahoo.jrt.slobrok.api.Mirror info no location brokers available, retrying: [] (in 0.5 seconds) +1661243505.083482 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 0 for cluster 'mycluster': Created +1661243505.084707 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Clearing master data as we lost connection on node 0 +1661243505.091045 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.rpc.RpcServer info Registering tcp/localhost:51707 with slobrok at ( tcp/localhost:51705 ) +1661243505.092054 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 0 for cluster 'mycluster': Starting tick loop +1661243505.092619 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 0 for cluster 'mycluster': Setting up new ZooKeeper session at localhost:51704 +1661243505.093164 localhost 75603/1 - fleetcontroller.com.yahoo.jrt.slobrok.api.Mirror info no location brokers available, retrying: [] (in 0.5 seconds) +1661243505.093445 localhost 75603/470 - fleetcontroller.org.apache.zookeeper.ZooKeeper info Initiating client connection, connectString=localhost:51704 sessionTimeout=30000 watcher=com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase$ZooKeeperWatcher@58f2bbcb +1661243505.093698 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 1 for cluster 'mycluster': Created +1661243505.094675 localhost 75603/470 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocket info jute.maxbuffer value is 1048575 Bytes +1661243505.095199 localhost 75603/470 - fleetcontroller.org.apache.zookeeper.ClientCnxn info zookeeper.request.timeout value is 0. feature enabled=false +1661243505.096099 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Clearing master data as we lost connection on node 1 +1661243505.096492 localhost 75603/477 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Opening socket connection to server localhost/[0:0:0:0:0:0:0:1]:51704. +1661243505.096923 localhost 75603/477 - fleetcontroller.org.apache.zookeeper.ClientCnxn info SASL config status: Will not attempt to authenticate using SASL (unknown error) +1661243505.101019 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.rpc.RpcServer info Registering tcp/localhost:51710 with slobrok at ( tcp/localhost:51705 ) +1661243505.101825 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 1 for cluster 'mycluster': Starting tick loop +1661243505.102322 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 1 for cluster 'mycluster': Setting up new ZooKeeper session at localhost:51704 +1661243505.103030 localhost 75603/1 - fleetcontroller.com.yahoo.jrt.slobrok.api.Mirror info no location brokers available, retrying: [] (in 0.5 seconds) +1661243505.103046 localhost 75603/485 - fleetcontroller.org.apache.zookeeper.ZooKeeper info Initiating client connection, connectString=localhost:51704 sessionTimeout=30000 watcher=com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase$ZooKeeperWatcher@173742bc +1661243505.103540 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 2 for cluster 'mycluster': Created +1661243505.103936 localhost 75603/485 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocket info jute.maxbuffer value is 1048575 Bytes +1661243505.104465 localhost 75603/485 - fleetcontroller.org.apache.zookeeper.ClientCnxn info zookeeper.request.timeout value is 0. feature enabled=false +1661243505.104824 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Clearing master data as we lost connection on node 2 +1661243505.105106 localhost 75603/492 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Opening socket connection to server localhost/[0:0:0:0:0:0:0:1]:51704. +1661243505.105512 localhost 75603/492 - fleetcontroller.org.apache.zookeeper.ClientCnxn info SASL config status: Will not attempt to authenticate using SASL (unknown error) +1661243505.106087 localhost 75603/484 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Socket connection established, initiating session, client: /[0:0:0:0:0:0:0:1]:51711, server: localhost/[0:0:0:0:0:0:0:1]:51704 +1661243505.106185 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.rpc.RpcServer info Registering tcp/localhost:51712 with slobrok at ( tcp/localhost:51705 ) +1661243505.106570 localhost 75603/484 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is connected: [id: 0xda4181f3, L:/[0:0:0:0:0:0:0:1]:51711 - R:localhost/[0:0:0:0:0:0:0:1]:51704] +1661243505.107208 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 2 for cluster 'mycluster': Starting tick loop +1661243505.107131 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Entering waitForMaster +1661243505.107811 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 2 for cluster 'mycluster': Setting up new ZooKeeper session at localhost:51704 +1661243505.108653 localhost 75603/500 - fleetcontroller.org.apache.zookeeper.ZooKeeper info Initiating client connection, connectString=localhost:51704 sessionTimeout=30000 watcher=com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase$ZooKeeperWatcher@6f891fab +1661243505.109231 localhost 75603/500 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocket info jute.maxbuffer value is 1048575 Bytes +1661243505.109635 localhost 75603/500 - fleetcontroller.org.apache.zookeeper.ClientCnxn info zookeeper.request.timeout value is 0. feature enabled=false +1661243505.110335 localhost 75603/502 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Opening socket connection to server localhost/[0:0:0:0:0:0:0:1]:51704. +1661243505.110731 localhost 75603/502 - fleetcontroller.org.apache.zookeeper.ClientCnxn info SASL config status: Will not attempt to authenticate using SASL (unknown error) +1661243505.121742 localhost 75603/499 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Socket connection established, initiating session, client: /[0:0:0:0:0:0:0:1]:51715, server: localhost/[0:0:0:0:0:0:0:1]:51704 +1661243505.122133 localhost 75603/454 - fleetcontroller.org.apache.zookeeper.server.persistence.FileTxnLog info Creating new log file: log.1 +1661243505.122262 localhost 75603/499 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is connected: [id: 0x35935cb2, L:/[0:0:0:0:0:0:0:1]:51715 - R:localhost/[0:0:0:0:0:0:0:1]:51704] +1661243505.125798 localhost 75603/504 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Socket connection established, initiating session, client: /[0:0:0:0:0:0:0:1]:51717, server: localhost/[0:0:0:0:0:0:0:1]:51704 +1661243505.126370 localhost 75603/504 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is connected: [id: 0x42017013, L:/[0:0:0:0:0:0:0:1]:51717 - R:localhost/[0:0:0:0:0:0:0:1]:51704] +1661243505.146831 localhost 75603/484 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Session establishment complete on server localhost/[0:0:0:0:0:0:0:1]:51704, session id = 0x1001abb14b10000, negotiated timeout = 30000 +1661243505.147373 localhost 75603/478 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 0 for cluster 'mycluster': Connection to zookeeper server established. Refetching master data +1661243505.165823 localhost 75603/504 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Session establishment complete on server localhost/[0:0:0:0:0:0:0:1]:51704, session id = 0x1001abb14b10002, negotiated timeout = 30000 +1661243505.165836 localhost 75603/499 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Session establishment complete on server localhost/[0:0:0:0:0:0:0:1]:51704, session id = 0x1001abb14b10001, negotiated timeout = 30000 +1661243505.166404 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 2 for cluster 'mycluster': Connection to zookeeper server established. Refetching master data +1661243505.166765 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 1 for cluster 'mycluster': Connection to zookeeper server established. Refetching master data +1661243505.402281 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 2 for cluster 'mycluster': Creating ephemeral master vote node with vote to self. +1661243505.402317 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 0 for cluster 'mycluster': Creating ephemeral master vote node with vote to self. +1661243505.403166 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 1 for cluster 'mycluster': Creating ephemeral master vote node with vote to self. +1661243505.422252 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 2 for cluster 'mycluster': Done setting up new ZooKeeper session at localhost:51704 +1661243505.425580 localhost 75603/479 - fleetcontroller.com.yahoo.jrt.slobrok.api.Mirror info successfully connected to location broker tcp/localhost:51705 (mirror initialized with 0 service names) +1661243505.425607 localhost 75603/494 - fleetcontroller.com.yahoo.jrt.slobrok.api.Mirror info successfully connected to location broker tcp/localhost:51705 (mirror initialized with 0 service names) +1661243505.425580 localhost 75603/465 - fleetcontroller.com.yahoo.jrt.slobrok.api.Mirror info successfully connected to location broker tcp/localhost:51705 (mirror initialized with 0 service names) +1661243505.440443 localhost 75603/468 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51707] registering storage/cluster.mycluster/fleetcontroller/0 with location broker tcp/localhost:51705 completed successfully +1661243505.440557 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 0 for cluster 'mycluster': Done setting up new ZooKeeper session at localhost:51704 +1661243505.444275 localhost 75603/482 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51710] registering storage/cluster.mycluster/fleetcontroller/1 with location broker tcp/localhost:51705 completed successfully +1661243505.444379 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got node list response from /vespa/fleetcontroller/mycluster/indexes version 0 with 3 nodes +1661243505.444359 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 1 for cluster 'mycluster': Done setting up new ZooKeeper session at localhost:51704 +1661243505.444275 localhost 75603/497 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51712] registering storage/cluster.mycluster/fleetcontroller/2 with location broker tcp/localhost:51705 completed successfully +1661243505.459490 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 2 for cluster 'mycluster': Stored new vote in ephemeral node. 2 -> 2 +1661243505.459493 localhost 75603/478 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got node list response from /vespa/fleetcontroller/mycluster/indexes version 0 with 3 nodes +1661243505.480063 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 0 for cluster 'mycluster': Stored new vote in ephemeral node. 0 -> 0 +1661243505.480951 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Node 0 is not master yet, sleeping more +1661243505.483964 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got node list response from /vespa/fleetcontroller/mycluster/indexes version 0 with 3 nodes +1661243505.484004 localhost 75603/478 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/0 with code 0 and data 0 +1661243505.484002 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 1 for cluster 'mycluster': Stored new vote in ephemeral node. 1 -> 1 +1661243505.484002 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/0 with code 0 and data 0 +1661243505.485008 localhost 75603/478 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote from fleetcontroller 0. Altering vote from null to 0. +1661243505.486023 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 0. Altering vote from null to 0. +1661243505.486580 localhost 75603/478 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 1 +1661243505.487037 localhost 75603/478 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote from fleetcontroller 1. Altering vote from null to 1. +1661243505.486697 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 1 +1661243505.487479 localhost 75603/478 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 2 +1661243505.488114 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 1. Altering vote from null to 1. +1661243505.488926 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 2 +1661243505.489344 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 2. Altering vote from null to 2. +1661243505.488509 localhost 75603/478 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote from fleetcontroller 2. Altering vote from null to 2. +1661243505.489793 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got new fleet data with 3 entries: {0=0, 1=1, 2=2} +1661243505.490119 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Got new fleet data with 3 entries: {0=0, 1=1, 2=2} +1661243505.490394 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Handling new master election, as we have received 3 entries +1661243505.490880 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Handling new master election, as we have received 3 entries +1661243505.491621 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/0 with code 0 and data 0 +1661243505.491428 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got master election state data(0 -> 0, 1 -> 1, 2 -> 2). +1661243505.492452 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote from fleetcontroller 0. Altering vote from null to 0. +1661243505.492040 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Got master election state data(0 -> 0, 1 -> 1, 2 -> 2). +1661243505.493323 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 1 +1661243505.494186 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote from fleetcontroller 1. Altering vote from null to 1. +1661243505.494616 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 2 +1661243505.495153 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote from fleetcontroller 2. Altering vote from null to 2. +1661243505.495547 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Currently too few followers for cluster controller candidate 0. No current master. (1/3 followers) +1661243505.495510 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Currently too few followers for cluster controller candidate 0. No current master. (1/3 followers) +1661243505.496407 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Altering our state to follow new fleet controller master candidate 0 +1661243505.495899 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got new fleet data with 3 entries: {0=0, 1=1, 2=2} +1661243505.497375 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Handling new master election, as we have received 3 entries +1661243505.496843 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type MASTER_ELECTION @1661243505496: This node just became node state gatherer as we are fleetcontroller master candidate. +1661243505.497804 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got master election state data(0 -> 0, 1 -> 1, 2 -> 2). +1661243505.498632 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Currently too few followers for cluster controller candidate 0. No current master. (1/3 followers) +1661243505.499225 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Altering our state to follow new fleet controller master candidate 0 +1661243505.512918 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 0 for cluster 'mycluster': Read cluster state version 0 from ZooKeeper (znode version 0) +1661243505.527492 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 2 for cluster 'mycluster': Stored new vote in ephemeral node. 2 -> 0 +1661243505.527648 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Altered data in node /vespa/fleetcontroller/mycluster/indexes/2. Requesting new vote +1661243505.527629 localhost 75603/478 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Altered data in node /vespa/fleetcontroller/mycluster/indexes/2. Requesting new vote +1661243505.527629 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Altered data in node /vespa/fleetcontroller/mycluster/indexes/2. Requesting new vote +1661243505.530796 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Altered data in node /vespa/fleetcontroller/mycluster/indexes/1. Requesting new vote +1661243505.530829 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 1 for cluster 'mycluster': Stored new vote in ephemeral node. 1 -> 0 +1661243505.530803 localhost 75603/478 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Altered data in node /vespa/fleetcontroller/mycluster/indexes/1. Requesting new vote +1661243505.530800 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Altered data in node /vespa/fleetcontroller/mycluster/indexes/1. Requesting new vote +1661243505.532218 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type MASTER_ELECTION @1661243505532: This node just became node state gatherer as we are fleetcontroller master candidate. +1661243505.553150 localhost 75603/478 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 0 +1661243505.553150 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 0 +1661243505.553150 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 0 +1661243505.554448 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 2. Altering vote from null to 0. +1661243505.553882 localhost 75603/478 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote from fleetcontroller 2. Altering vote from null to 0. +1661243505.554943 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote from fleetcontroller 2. Altering vote from null to 0. +1661243505.569127 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 0 +1661243505.569217 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 1 for cluster 'mycluster': Read cluster state version 0 from ZooKeeper (znode version 0) +1661243505.569133 localhost 75603/478 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 0 +1661243505.569127 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 0 +1661243505.570388 localhost 75603/478 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote from fleetcontroller 1. Altering vote from null to 0. +1661243505.569680 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote from fleetcontroller 1. Altering vote from null to 0. +1661243505.571413 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Got new fleet data with 3 entries: {0=0, 1=0, 2=0} +1661243505.570950 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 1. Altering vote from null to 0. +1661243505.572000 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Handling new master election, as we have received 3 entries +1661243505.571655 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got new fleet data with 3 entries: {0=0, 1=0, 2=0} +1661243505.572696 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Got master election state data(0 -> 0, 1 -> 0, 2 -> 0). +1661243505.572437 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got new fleet data with 3 entries: {0=0, 1=0, 2=0} +1661243505.573504 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': 0 is newly elected master +1661243505.573137 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Handling new master election, as we have received 3 entries +1661243505.573814 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Handling new master election, as we have received 3 entries +1661243505.574488 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got master election state data(0 -> 0, 1 -> 0, 2 -> 0). +1661243505.574847 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got master election state data(0 -> 0, 1 -> 0, 2 -> 0). +1661243505.575319 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': 0 is newly elected master +1661243505.575652 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': 0 is newly elected master +1661243505.604886 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 0 for cluster 'mycluster': Read cluster state version 0 from ZooKeeper (znode version 0) +1661243505.663509 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 0 for cluster 'mycluster': Loaded previous cluster state bundle from ZooKeeper: ClusterStateBundle('') +1661243505.663972 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type MASTER_ELECTION @1661243505663: This node just became fleetcontroller master. Bumped version to 1 to be in line. +1661243505.664600 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type SYSTEMSTATE @1661243505664: Too few storage nodes available in cluster. Setting cluster state down +1661243505.665034 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type SYSTEMSTATE @1661243505664: New cluster state version 2. Change from last: version: 0 => 2, cluster: Up => Down, storage: [0: description: => Node not seen in slobrok., 1: description: => Node not seen in slobrok., 2: description: => Node not seen in slobrok., 3: description: => Node not seen in slobrok., 4: description: => Node not seen in slobrok., 5: description: => Node not seen in slobrok., 6: description: => Node not seen in slobrok., 7: description: => Node not seen in slobrok., 8: description: => Node not seen in slobrok., 9: description: => Node not seen in slobrok.], distributor: [0: description: => Node not seen in slobrok., 1: description: => Node not seen in slobrok., 2: description: => Node not seen in slobrok., 3: description: => Node not seen in slobrok., 4: description: => Node not seen in slobrok., 5: description: => Node not seen in slobrok., 6: description: => Node not seen in slobrok., 7: description: => Node not seen in slobrok., 8: description: => Node not seen in slobrok., 9: description: => Node not seen in slobrok.] +1661243505.666904 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 0 for cluster 'mycluster': Storing new cluster state version in ZooKeeper: 2 +1661243505.750178 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 0 for cluster 'mycluster': Master moratorium complete: timed out waiting for all nodes to report in +1661243505.750779 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.SystemStateBroadcaster info FleetController 0 for cluster 'mycluster': Publishing cluster state version 2 +1661243505.751329 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Node 0 is master. Checking that no one else is master +1661243505.751614 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Leaving waitForMaster +1661243505.751836 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info SHUTTING DOWN FLEET CONTROLLER 0 +1661243505.752079 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 0 for cluster 'mycluster': Joining event thread. +1661243505.752352 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 0 for cluster 'mycluster': Tick loop stopped +1661243505.752640 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 0 for cluster 'mycluster': FleetController done shutting down event thread. +1661243505.752952 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 0 for cluster 'mycluster': Resetting database state +1661243505.772139 localhost 75603/478 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: A change occurred in the list of registered fleetcontrollers. Requesting new information +1661243505.774926 localhost 75603/484 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is disconnected: [id: 0xda4181f3, L:/[0:0:0:0:0:0:0:1]:51711 ! R:localhost/[0:0:0:0:0:0:0:1]:51704] +1661243505.774695 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is told closing +1661243505.772168 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: A change occurred in the list of registered fleetcontrollers. Requesting new information +1661243505.772139 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: A change occurred in the list of registered fleetcontrollers. Requesting new information +1661243505.775932 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.ZooKeeper info Session: 0x1001abb14b10000 closed +1661243505.775706 localhost 75603/484 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is told closing +1661243505.777078 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got node list response from /vespa/fleetcontroller/mycluster/indexes version 0 with 2 nodes +1661243505.775677 localhost 75603/478 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got node list response from /vespa/fleetcontroller/mycluster/indexes version -112 with 0 nodes +1661243505.776862 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Clearing master data as we lost connection on node 0 +1661243505.776826 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got node list response from /vespa/fleetcontroller/mycluster/indexes version 0 with 2 nodes +1661243505.778084 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type MASTER_ELECTION @1661243505778: This node is no longer fleetcontroller master. +1661243505.777856 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 0 +1661243505.777774 localhost 75603/478 - fleetcontroller.org.apache.zookeeper.ClientCnxn info EventThread shut down for session: 0x1001abb14b10000 +1661243505.779077 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 1. Altering vote from null to 0. +1661243505.778820 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 0 +1661243505.778711 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 0 for cluster 'mycluster': Done resetting database state +1661243505.779925 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote from fleetcontroller 1. Altering vote from null to 0. +1661243505.779610 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 0 +1661243505.781132 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 2. Altering vote from null to 0. +1661243505.780660 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 0 +1661243505.781540 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got new fleet data with 2 entries: {1=0, 2=0} +1661243505.781153 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Entering waitForMaster +1661243505.782065 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Handling new master election, as we have received 2 entries +1661243505.781774 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote from fleetcontroller 2. Altering vote from null to 0. +1661243505.782683 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got master election state data(1 -> 0, 2 -> 0). +1661243505.782382 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Node 1 is not master yet, sleeping more +1661243505.783351 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Master gone from ZooKeeper. Tagging timestamp. Will wait 100 ms. +1661243505.782975 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got new fleet data with 2 entries: {1=0, 2=0} +1661243505.783719 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': First index is not currently trying to become master. Waiting for it to change state +1661243505.783968 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Handling new master election, as we have received 2 entries +1661243505.784535 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got master election state data(1 -> 0, 2 -> 0). +1661243505.784798 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Master gone from ZooKeeper. Tagging timestamp. Will wait 100 ms. +1661243505.785075 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': First index is not currently trying to become master. Waiting for it to change state +1661243505.785371 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': We are next in line to become master. Altering our state to look for followers +1661243505.804787 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Altered data in node /vespa/fleetcontroller/mycluster/indexes/1. Requesting new vote +1661243505.804823 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 1 for cluster 'mycluster': Stored new vote in ephemeral node. 1 -> 1 +1661243505.804805 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Altered data in node /vespa/fleetcontroller/mycluster/indexes/1. Requesting new vote +1661243505.805941 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 1 +1661243505.806625 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote from fleetcontroller 1. Altering vote from null to 1. +1661243505.806771 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 1 +1661243505.807066 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got new fleet data with 2 entries: {1=1, 2=0} +1661243505.807437 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 1. Altering vote from null to 1. +1661243505.807718 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Handling new master election, as we have received 2 entries +1661243505.808122 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got new fleet data with 2 entries: {1=1, 2=0} +1661243505.808332 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got master election state data(1 -> 1, 2 -> 0). +1661243505.808594 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Handling new master election, as we have received 2 entries +1661243505.808899 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Currently too few followers for cluster controller candidate 1. No current master. (1/3 followers) +1661243505.809177 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got master election state data(1 -> 1, 2 -> 0). +1661243505.809768 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Currently too few followers for cluster controller candidate 1. No current master. (1/3 followers) +1661243505.810097 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Altering our state to follow new fleet controller master candidate 1 +1661243505.829036 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Altered data in node /vespa/fleetcontroller/mycluster/indexes/2. Requesting new vote +1661243505.829067 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 2 for cluster 'mycluster': Stored new vote in ephemeral node. 2 -> 1 +1661243505.829036 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Altered data in node /vespa/fleetcontroller/mycluster/indexes/2. Requesting new vote +1661243505.829809 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 1 +1661243505.830243 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 2. Altering vote from null to 1. +1661243505.830300 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 1 +1661243505.830647 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got new fleet data with 2 entries: {1=1, 2=1} +1661243505.830881 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote from fleetcontroller 2. Altering vote from null to 1. +1661243505.831161 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Handling new master election, as we have received 2 entries +1661243505.831483 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got new fleet data with 2 entries: {1=1, 2=1} +1661243505.831713 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got master election state data(1 -> 1, 2 -> 1). +1661243505.831978 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Handling new master election, as we have received 2 entries +1661243505.832379 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': 1 is new master candidate, but needs to wait before it can take over +1661243505.832542 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got master election state data(1 -> 1, 2 -> 1). +1661243505.833035 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': 1 is new master candidate, but needs to wait before it can take over +1661243505.911946 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Node 1 is not master yet, sleeping more +1661243505.956844 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 1 for cluster 'mycluster': Read cluster state version 2 from ZooKeeper (znode version 1) +1661243506.003215 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 1 for cluster 'mycluster': Loaded previous cluster state bundle from ZooKeeper: ClusterStateBundle('version:2 cluster:d', default 'version:2 cluster:d', global 'version:2 cluster:d' (deferred activation)) +1661243506.004137 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type MASTER_ELECTION @1661243506004: This node just became fleetcontroller master. Bumped version to 2 to be in line. +1661243506.004846 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type SYSTEMSTATE @1661243506004: New cluster state version 3. Change from last: version: 2 => 3, storage: [0: description: => Node not seen in slobrok., 1: description: => Node not seen in slobrok., 2: description: => Node not seen in slobrok., 3: description: => Node not seen in slobrok., 4: description: => Node not seen in slobrok., 5: description: => Node not seen in slobrok., 6: description: => Node not seen in slobrok., 7: description: => Node not seen in slobrok., 8: description: => Node not seen in slobrok., 9: description: => Node not seen in slobrok.], distributor: [0: description: => Node not seen in slobrok., 1: description: => Node not seen in slobrok., 2: description: => Node not seen in slobrok., 3: description: => Node not seen in slobrok., 4: description: => Node not seen in slobrok., 5: description: => Node not seen in slobrok., 6: description: => Node not seen in slobrok., 7: description: => Node not seen in slobrok., 8: description: => Node not seen in slobrok., 9: description: => Node not seen in slobrok.] +1661243506.006609 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 1 for cluster 'mycluster': Storing new cluster state version in ZooKeeper: 3 +1661243506.061811 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 1 for cluster 'mycluster': Master moratorium complete: timed out waiting for all nodes to report in +1661243506.062251 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.SystemStateBroadcaster info FleetController 1 for cluster 'mycluster': Publishing cluster state version 3 +1661243506.167506 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Node 1 is master. Checking that no one else is master +1661243506.167790 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Leaving waitForMaster +1661243506.167970 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info SHUTTING DOWN FLEET CONTROLLER 1 +1661243506.168287 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 1 for cluster 'mycluster': Joining event thread. +1661243506.168535 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 1 for cluster 'mycluster': Tick loop stopped +1661243506.168800 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 1 for cluster 'mycluster': FleetController done shutting down event thread. +1661243506.169077 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 1 for cluster 'mycluster': Resetting database state +1661243506.189449 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: A change occurred in the list of registered fleetcontrollers. Requesting new information +1661243506.189702 localhost 75603/499 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is disconnected: [id: 0x35935cb2, L:/[0:0:0:0:0:0:0:1]:51715 ! R:localhost/[0:0:0:0:0:0:0:1]:51704] +1661243506.189590 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is told closing +1661243506.189484 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: A change occurred in the list of registered fleetcontrollers. Requesting new information +1661243506.190411 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.ZooKeeper info Session: 0x1001abb14b10001 closed +1661243506.190333 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got node list response from /vespa/fleetcontroller/mycluster/indexes version 0 with 1 nodes +1661243506.190165 localhost 75603/499 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is told closing +1661243506.190936 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Clearing master data as we lost connection on node 1 +1661243506.190737 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got node list response from /vespa/fleetcontroller/mycluster/indexes version -112 with 0 nodes +1661243506.191684 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type MASTER_ELECTION @1661243506191: This node is no longer fleetcontroller master. +1661243506.191621 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 1 +1661243506.192225 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 1 for cluster 'mycluster': Done resetting database state +1661243506.191973 localhost 75603/493 - fleetcontroller.org.apache.zookeeper.ClientCnxn info EventThread shut down for session: 0x1001abb14b10001 +1661243506.192501 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 2. Altering vote from null to 1. +1661243506.193432 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got new fleet data with 1 entries: {2=1} +1661243506.193693 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Handling new master election, as we have received 1 entries +1661243506.193958 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got master election state data(2 -> 1). +1661243506.194191 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Master gone from ZooKeeper. Tagging timestamp. Will wait 100 ms. +1661243506.194434 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': First index is not currently trying to become master. Waiting for it to change state +1661243506.194697 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': We are next in line to become master. Altering our state to look for followers +1661243506.213596 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Altered data in node /vespa/fleetcontroller/mycluster/indexes/2. Requesting new vote +1661243506.213635 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 2 for cluster 'mycluster': Stored new vote in ephemeral node. 2 -> 2 +1661243506.214499 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 2 +1661243506.214841 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 2. Altering vote from null to 2. +1661243506.215220 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got new fleet data with 1 entries: {2=2} +1661243506.215529 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Handling new master election, as we have received 1 entries +1661243506.215876 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got master election state data(2 -> 2). +1661243506.216199 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Currently too few followers for cluster controller candidate 2. No current master. (1/3 followers) +1661243506.216601 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info STARTING FLEET CONTROLLER 0 +1661243506.217669 localhost 75603/1 - fleetcontroller.com.yahoo.jrt.slobrok.api.Mirror info no location brokers available, retrying: [] (in 0.5 seconds) +1661243506.218047 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 0 for cluster 'mycluster': Created +1661243506.218945 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Clearing master data as we lost connection on node 0 +1661243506.222737 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.rpc.RpcServer info Registering tcp/localhost:51887 with slobrok at ( tcp/localhost:51705 ) +1661243506.222912 localhost 75603/543 - fleetcontroller.com.yahoo.jrt.slobrok.api.Mirror info successfully connected to location broker tcp/localhost:51705 (mirror initialized with 1 service names) +1661243506.223312 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 0 for cluster 'mycluster': Starting tick loop +1661243506.223245 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Entering waitForMaster +1661243506.223670 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 0 for cluster 'mycluster': Setting up new ZooKeeper session at localhost:51704 +1661243506.224291 localhost 75603/547 - fleetcontroller.org.apache.zookeeper.ZooKeeper info Initiating client connection, connectString=localhost:51704 sessionTimeout=30000 watcher=com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase$ZooKeeperWatcher@7e118ccc +1661243506.224914 localhost 75603/547 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocket info jute.maxbuffer value is 1048575 Bytes +1661243506.225324 localhost 75603/547 - fleetcontroller.org.apache.zookeeper.ClientCnxn info zookeeper.request.timeout value is 0. feature enabled=false +1661243506.225880 localhost 75603/548 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Opening socket connection to server localhost/127.0.0.1:51704. +1661243506.226159 localhost 75603/548 - fleetcontroller.org.apache.zookeeper.ClientCnxn info SASL config status: Will not attempt to authenticate using SASL (unknown error) +1661243506.233788 localhost 75603/550 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Socket connection established, initiating session, client: /127.0.0.1:51890, server: localhost/127.0.0.1:51704 +1661243506.233971 localhost 75603/545 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51887] registering storage/cluster.mycluster/fleetcontroller/0 with location broker tcp/localhost:51705 completed successfully +1661243506.234315 localhost 75603/550 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is connected: [id: 0x54d96664, L:/127.0.0.1:51890 - R:localhost/127.0.0.1:51704] +1661243506.254801 localhost 75603/550 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Session establishment complete on server localhost/127.0.0.1:51704, session id = 0x1001abb14b10003, negotiated timeout = 30000 +1661243506.255438 localhost 75603/549 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 0 for cluster 'mycluster': Connection to zookeeper server established. Refetching master data +1661243506.269487 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 0 for cluster 'mycluster': Creating ephemeral master vote node with vote to self. +1661243506.299288 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 0 for cluster 'mycluster': Done setting up new ZooKeeper session at localhost:51704 +1661243506.299263 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: A change occurred in the list of registered fleetcontrollers. Requesting new information +1661243506.300011 localhost 75603/549 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got node list response from /vespa/fleetcontroller/mycluster/indexes version 0 with 2 nodes +1661243506.311990 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 0 for cluster 'mycluster': Stored new vote in ephemeral node. 0 -> 0 +1661243506.312534 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got node list response from /vespa/fleetcontroller/mycluster/indexes version 0 with 2 nodes +1661243506.312408 localhost 75603/549 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/0 with code 0 and data 0 +1661243506.312639 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Node 0 is not master yet, sleeping more +1661243506.313573 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/0 with code 0 and data 0 +1661243506.313417 localhost 75603/549 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote from fleetcontroller 0. Altering vote from null to 0. +1661243506.314162 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 0. Altering vote from null to 0. +1661243506.314502 localhost 75603/549 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 2 +1661243506.315320 localhost 75603/549 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote from fleetcontroller 2. Altering vote from null to 2. +1661243506.316030 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Got new fleet data with 2 entries: {0=0, 2=2} +1661243506.316392 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Handling new master election, as we have received 2 entries +1661243506.316794 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Got master election state data(0 -> 0, 2 -> 2). +1661243506.317275 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Currently too few followers for cluster controller candidate 0. No current master. (1/3 followers) +1661243506.317865 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type MASTER_ELECTION @1661243506317: This node just became node state gatherer as we are fleetcontroller master candidate. +1661243506.320823 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 2 +1661243506.321229 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 2. Altering vote from null to 2. +1661243506.321613 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got new fleet data with 2 entries: {0=0, 2=2} +1661243506.321962 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Handling new master election, as we have received 2 entries +1661243506.322304 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got master election state data(0 -> 0, 2 -> 2). +1661243506.322611 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Currently too few followers for cluster controller candidate 0. No current master. (1/3 followers) +1661243506.323020 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Altering our state to follow new fleet controller master candidate 0 +1661243506.328135 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 0 for cluster 'mycluster': Read cluster state version 3 from ZooKeeper (znode version 2) +1661243506.346667 localhost 75603/549 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Altered data in node /vespa/fleetcontroller/mycluster/indexes/2. Requesting new vote +1661243506.346674 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 2 for cluster 'mycluster': Stored new vote in ephemeral node. 2 -> 0 +1661243506.346667 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Altered data in node /vespa/fleetcontroller/mycluster/indexes/2. Requesting new vote +1661243506.347687 localhost 75603/549 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 0 +1661243506.348225 localhost 75603/549 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote from fleetcontroller 2. Altering vote from null to 0. +1661243506.348266 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 0 +1661243506.348604 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Got new fleet data with 2 entries: {0=0, 2=0} +1661243506.349279 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Handling new master election, as we have received 2 entries +1661243506.348937 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 2. Altering vote from null to 0. +1661243506.349622 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Got master election state data(0 -> 0, 2 -> 0). +1661243506.350047 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got new fleet data with 2 entries: {0=0, 2=0} +1661243506.350317 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': 0 is newly elected master +1661243506.350755 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Handling new master election, as we have received 2 entries +1661243506.351444 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got master election state data(0 -> 0, 2 -> 0). +1661243506.351681 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 0 for cluster 'mycluster': Read cluster state version 3 from ZooKeeper (znode version 2) +1661243506.351813 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': 0 is newly elected master +1661243506.353452 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 0 for cluster 'mycluster': Loaded previous cluster state bundle from ZooKeeper: ClusterStateBundle('version:3 cluster:d', default 'version:3 cluster:d', global 'version:3 cluster:d' (deferred activation)) +1661243506.353927 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type MASTER_ELECTION @1661243506353: This node just became fleetcontroller master. Bumped version to 3 to be in line. +1661243506.354498 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type SYSTEMSTATE @1661243506354: New cluster state version 4. Change from last: version: 3 => 4, storage: [0: description: => Node not seen in slobrok., 1: description: => Node not seen in slobrok., 2: description: => Node not seen in slobrok., 3: description: => Node not seen in slobrok., 4: description: => Node not seen in slobrok., 5: description: => Node not seen in slobrok., 6: description: => Node not seen in slobrok., 7: description: => Node not seen in slobrok., 8: description: => Node not seen in slobrok., 9: description: => Node not seen in slobrok.], distributor: [0: description: => Node not seen in slobrok., 1: description: => Node not seen in slobrok., 2: description: => Node not seen in slobrok., 3: description: => Node not seen in slobrok., 4: description: => Node not seen in slobrok., 5: description: => Node not seen in slobrok., 6: description: => Node not seen in slobrok., 7: description: => Node not seen in slobrok., 8: description: => Node not seen in slobrok., 9: description: => Node not seen in slobrok.] +1661243506.355864 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 0 for cluster 'mycluster': Storing new cluster state version in ZooKeeper: 4 +1661243506.393554 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 0 for cluster 'mycluster': Master moratorium complete: timed out waiting for all nodes to report in +1661243506.393869 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.SystemStateBroadcaster info FleetController 0 for cluster 'mycluster': Publishing cluster state version 4 +1661243506.420410 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Node 0 is master. Checking that no one else is master +1661243506.420858 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Leaving waitForMaster +1661243506.421181 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info STARTING FLEET CONTROLLER 1 +1661243506.422450 localhost 75603/1 - fleetcontroller.com.yahoo.jrt.slobrok.api.Mirror info no location brokers available, retrying: [] (in 0.5 seconds) +1661243506.422771 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 1 for cluster 'mycluster': Created +1661243506.423748 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Clearing master data as we lost connection on node 1 +1661243506.427680 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.rpc.RpcServer info Registering tcp/localhost:51912 with slobrok at ( tcp/localhost:51705 ) +1661243506.427920 localhost 75603/556 - fleetcontroller.com.yahoo.jrt.slobrok.api.Mirror info successfully connected to location broker tcp/localhost:51705 (mirror initialized with 2 service names) +1661243506.428330 localhost 75603/560 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 1 for cluster 'mycluster': Starting tick loop +1661243506.428246 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Entering waitForMaster +1661243506.428923 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Node 0 is master. Checking that no one else is master +1661243506.428705 localhost 75603/560 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 1 for cluster 'mycluster': Setting up new ZooKeeper session at localhost:51704 +1661243506.429609 localhost 75603/560 - fleetcontroller.org.apache.zookeeper.ZooKeeper info Initiating client connection, connectString=localhost:51704 sessionTimeout=30000 watcher=com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase$ZooKeeperWatcher@32c98e99 +1661243506.430223 localhost 75603/560 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocket info jute.maxbuffer value is 1048575 Bytes +1661243506.430619 localhost 75603/560 - fleetcontroller.org.apache.zookeeper.ClientCnxn info zookeeper.request.timeout value is 0. feature enabled=false +1661243506.431169 localhost 75603/561 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Opening socket connection to server localhost/[0:0:0:0:0:0:0:1]:51704. +1661243506.431449 localhost 75603/561 - fleetcontroller.org.apache.zookeeper.ClientCnxn info SASL config status: Will not attempt to authenticate using SASL (unknown error) +1661243506.438564 localhost 75603/563 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Socket connection established, initiating session, client: /[0:0:0:0:0:0:0:1]:51915, server: localhost/[0:0:0:0:0:0:0:1]:51704 +1661243506.438897 localhost 75603/558 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51912] registering storage/cluster.mycluster/fleetcontroller/1 with location broker tcp/localhost:51705 completed successfully +1661243506.439120 localhost 75603/563 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is connected: [id: 0x20d50555, L:/[0:0:0:0:0:0:0:1]:51915 - R:localhost/[0:0:0:0:0:0:0:1]:51704] +1661243506.460199 localhost 75603/563 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Session establishment complete on server localhost/[0:0:0:0:0:0:0:1]:51704, session id = 0x1001abb14b10004, negotiated timeout = 30000 +1661243506.460731 localhost 75603/562 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 1 for cluster 'mycluster': Connection to zookeeper server established. Refetching master data +1661243506.463849 localhost 75603/560 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 1 for cluster 'mycluster': Creating ephemeral master vote node with vote to self. +1661243506.482637 localhost 75603/560 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 1 for cluster 'mycluster': Done setting up new ZooKeeper session at localhost:51704 +1661243506.482640 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: A change occurred in the list of registered fleetcontrollers. Requesting new information +1661243506.482640 localhost 75603/549 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: A change occurred in the list of registered fleetcontrollers. Requesting new information +1661243506.483138 localhost 75603/562 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got node list response from /vespa/fleetcontroller/mycluster/indexes version 0 with 3 nodes +1661243506.501684 localhost 75603/560 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 1 for cluster 'mycluster': Stored new vote in ephemeral node. 1 -> 1 +1661243506.501759 localhost 75603/562 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/0 with code 0 and data 0 +1661243506.501735 localhost 75603/549 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got node list response from /vespa/fleetcontroller/mycluster/indexes version 0 with 3 nodes +1661243506.501683 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got node list response from /vespa/fleetcontroller/mycluster/indexes version 0 with 3 nodes +1661243506.502322 localhost 75603/562 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote from fleetcontroller 0. Altering vote from null to 0. +1661243506.502049 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Leaving waitForMaster +1661243506.503639 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from patTEST SUCCEEDED - cleaning up zookeeper directory +h /vespa/fleetcontroller/mycluster/indexes/0 with code 0 and data 0 +1661243506.503436 localhost 75603/562 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 1 +1661243506.503244 localhost 75603/549 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/0 with code 0 and data 0 +1661243506.504402 localhost 75603/562 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote from fleetcontroller 1. Altering vote from null to 1. +1661243506.504080 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 0. Altering vote from null to 0. +1661243506.503956 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ZooKeeperServer info shutting down +1661243506.505371 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 1 +1661243506.505040 localhost 75603/562 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 0 +1661243506.504689 localhost 75603/549 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote from fleetcontroller 0. Altering vote from null to 0. +1661243506.506135 localhost 75603/562 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote from fleetcontroller 2. Altering vote from null to 0. +1661243506.505835 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 1. Altering vote from null to 1. +1661243506.505533 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.RequestThrottler info Shutting down +1661243506.506920 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 0 +1661243506.506750 localhost 75603/560 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got new fleet data with 3 entries: {0=0, 1=1, 2=0} +1661243506.506408 localhost 75603/549 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 1 +1661243506.507640 localhost 75603/560 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Handling new master election, as we have received 3 entries +1661243506.507359 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 2. Altering vote from null to 0. +1661243506.507115 localhost 75603/456 - fleetcontroller.org.apache.zookeeper.server.RequestThrottler info Draining request throttler queue +1661243506.508549 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got new fleet data with 3 entries: {0=0, 1=1, 2=0} +1661243506.508211 localhost 75603/560 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got master election state data(0 -> 0, 1 -> 1, 2 -> 0). +1661243506.507923 localhost 75603/549 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote from fleetcontroller 1. Altering vote from null to 1. +1661243506.509191 localhost 75603/560 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': 0 is new master candidate, but needs to wait before it can take over +1661243506.508911 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Handling new master election, as we have received 3 entries +1661243506.508676 localhost 75603/456 - fleetcontroller.org.apache.zookeeper.server.RequestThrottler info RequestThrottler shutdown. Dropped 0 requests +1661243506.509985 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got master election state data(0 -> 0, 1 -> 1, 2 -> 0). +1661243506.509715 localhost 75603/560 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Altering our state to follow new fleet controller master candidate 0 +1661243506.509442 localhost 75603/549 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 0 +1661243506.510641 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': 0 is still the master +1661243506.510224 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.SessionTrackerImpl info Shutting down +1661243506.511137 localhost 75603/514 - fleetcontroller.org.apache.zookeeper.server.NIOServerCnxn warning Close of session 0x1001abb14b10004\nexception=\njava.io.IOException: ZooKeeperServer not running\n\tat org.apache.zookeeper.server.NIOServerCnxn.readLength(NIOServerCnxn.java:554)\n\tat org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:339)\n\tat org.apache.zookeeper.server.NIOServerCnxnFactory$IOWorkRequest.doWork(NIOServerCnxnFactory.java:508)\n\tat org.apache.zookeeper.server.WorkerService$ScheduledWorkRequest.run(WorkerService.java:154)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243506.511157 localhost 75603/549 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote from fleetcontroller 2. Altering vote from null to 0. +1661243506.511597 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.PrepRequestProcessor info Shutting down +1661243506.512802 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Got new fleet data with 3 entries: {0=0, 1=1, 2=0} +1661243506.512773 localhost 75603/563 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is disconnected: [id: 0x20d50555, L:/[0:0:0:0:0:0:0:1]:51915 ! R:localhost/[0:0:0:0:0:0:0:1]:51704] +1661243506.513198 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Handling new master election, as we have received 3 entries +1661243506.512939 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.SyncRequestProcessor info Shutting down +1661243506.512947 localhost 75603/455 - fleetcontroller.org.apache.zookeeper.server.PrepRequestProcessor info PrepRequestProcessor exited loop! +1661243506.513877 localhost 75603/454 - fleetcontroller.org.apache.zookeeper.server.SyncRequestProcessor info SyncRequestProcessor exited! +1661243506.513715 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Got master election state data(0 -> 0, 1 -> 1, 2 -> 0). +1661243506.513588 localhost 75603/561 - fleetcontroller.org.apache.zookeeper.ClientCnxn warning Session 0x1001abb14b10004 for server localhost/[0:0:0:0:0:0:0:1]:51704, Closing socket connection. Attempting reconnect except it is a SessionExpiredException.\nexception=\nEndOfStreamException: channel for sessionid 0x1001abb14b10004 is lost\n\tat org.apache.zookeeper.ClientCnxnSocketNetty.doTransport(ClientCnxnSocketNetty.java:285)\n\tat org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1283)\n +1661243506.513461 localhost 75603/563 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is told closing +1661243506.514561 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': 0 is still the master +1661243506.514292 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.FinalRequestProcessor info shutdown of request processor complete +1661243506.515605 localhost 75603/560 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase warning FleetController 1 for cluster 'mycluster': Failed to create our ephemeral node and store master vote. Exception: KeeperErrorCode = ConnectionLoss for /vespa/fleetcontroller/mycluster/indexes/1\norg.apache.zookeeper.KeeperException$ConnectionLossException: KeeperErrorCode = ConnectionLoss for /vespa/fleetcontroller/mycluster/indexes/1\n\tat org.apache.zookeeper.KeeperException.create(KeeperException.java:102)\n\tat org.apache.zookeeper.KeeperException.create(KeeperException.java:54)\n\tat org.apache.zookeeper.ZooKeeper.setData(ZooKeeper.java:2237)\n\tat com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase.storeMasterVote(ZooKeeperDatabase.java:188)\n\tat com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler.performZooKeeperWrites(DatabaseHandler.java:286)\n\tat com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler.doNextZooKeeperTask(DatabaseHandler.java:258)\n\tat com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler.setMasterVote(DatabaseHandler.java:355)\n\tat com.yahoo.vespa.clustercontroller.core.MasterElectionHandler.watchMasterElection(MasterElectionHandler.java:184)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.updateMasterElectionState(FleetController.java:654)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:596)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243506.515402 localhost 75603/562 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 1 for cluster 'mycluster': Lost connection to zookeeper server +1661243506.518043 localhost 75603/504 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is disconnected: [id: 0x42017013, L:/[0:0:0:0:0:0:0:1]:51717 ! R:localhost/[0:0:0:0:0:0:0:1]:51704] +1661243506.518061 localhost 75603/560 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler warning FleetController 1 for cluster 'mycluster': Failed to store master vote +1661243506.518005 localhost 75603/550 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is disconnected: [id: 0x54d96664, L:/127.0.0.1:51890 ! R:localhost/127.0.0.1:51704] +1661243506.518011 localhost 75603/452 - fleetcontroller.org.apache.zookeeper.server.NIOServerCnxnFactory info accept thread exitted run method +1661243506.517508 localhost 75603/449 - fleetcontroller.org.apache.zookeeper.server.NIOServerCnxnFactory info ConnnectionExpirerThread interrupted +1661243506.519749 localhost 75603/548 - fleetcontroller.org.apache.zookeeper.ClientCnxn warning Session 0x1001abb14b10003 for server localhost/127.0.0.1:51704, Closing socket connection. Attempting reconnect except it is a SessionExpiredException.\nexception=\nEndOfStreamException: channel for sessionid 0x1001abb14b10003 is lost\n\tat org.apache.zookeeper.ClientCnxnSocketNetty.doTransport(ClientCnxnSocketNetty.java:285)\n\tat org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1283)\n +1661243506.519708 localhost 75603/550 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is told closing +1661243506.519636 localhost 75603/560 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type MASTER_ELECTION @1661243506519: This node just became node state gatherer as we are fleetcontroller master candidate. +1661243506.519215 localhost 75603/502 - fleetcontroller.org.apache.zookeeper.ClientCnxn warning Session 0x1001abb14b10002 for server localhost/[0:0:0:0:0:0:0:1]:51704, Closing socket connection. Attempting reconnect except it is a SessionExpiredException.\nexception=\nEndOfStreamException: channel for sessionid 0x1001abb14b10002 is lost\n\tat org.apache.zookeeper.ClientCnxnSocketNetty.doTransport(ClientCnxnSocketNetty.java:285)\n\tat org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1283)\n +1661243506.519136 localhost 75603/504 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is told closing +1661243506.518737 localhost 75603/450 - fleetcontroller.org.apache.zookeeper.server.NIOServerCnxnFactory info selector thread exitted run method +1661243506.518624 localhost 75603/451 - fleetcontroller.org.apache.zookeeper.server.NIOServerCnxnFactory info selector thread exitted run method +1661243506.522573 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 2 for cluster 'mycluster': Lost connection to zookeeper server +1661243506.521409 localhost 75603/549 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 0 for cluster 'mycluster': Lost connection to zookeeper server +1661243506.523702 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Clearing master data as we lost connection on node 2 +1661243506.523709 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Clearing master data as we lost connection on node 0 +1661243506.524360 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type MASTER_ELECTION @1661243506524: This node is no longer fleetcontroller master. +STARTING TEST: MasterElectionTest::testClusterStateVersionIncreasesAcrossMasterElections +1661243506.526032 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.persistence.FileTxnSnapLog info zookeeper.snapshot.trust.empty : false +1661243506.526539 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.watch.WatchManagerFactory info Using org.apache.zookeeper.server.watch.WatchManager as watch manager +1661243506.526889 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.watch.WatchManagerFactory info Using org.apache.zookeeper.server.watch.WatchManager as watch manager +1661243506.527271 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ZKDatabase info zookeeper.snapshotSizeFactor = 0.33 +1661243506.527541 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ZKDatabase info zookeeper.commitLogCount=500 +1661243506.527781 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ZooKeeperServer info minSessionTimeout set to 4000 +1661243506.528053 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ZooKeeperServer info maxSessionTimeout set to 40000 +1661243506.528379 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ResponseCache info getData response cache size is initialized with value 400. +1661243506.528647 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ResponseCache info getChildren response cache size is initialized with value 400. +1661243506.528932 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.util.RequestPathMetricsCollector info zookeeper.pathStats.slotCapacity = 60 +1661243506.529178 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.util.RequestPathMetricsCollector info zookeeper.pathStats.slotDuration = 15 +1661243506.529418 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.util.RequestPathMetricsCollector info zookeeper.pathStats.maxDepth = 6 +1661243506.529653 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.util.RequestPathMetricsCollector info zookeeper.pathStats.initialDelay = 5 +1661243506.529886 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.util.RequestPathMetricsCollector info zookeeper.pathStats.delay = 5 +1661243506.530111 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.util.RequestPathMetricsCollector info zookeeper.pathStats.enabled = false +1661243506.530529 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ZooKeeperServer info The max bytes for all large requests are set to 104857600 +1661243506.530948 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ZooKeeperServer info The large request threshold is set to -1 +1661243506.531247 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.AuthenticationHelper info zookeeper.enforce.auth.enabled = false +1661243506.531729 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.AuthenticationHelper info zookeeper.enforce.auth.schemes = [] +1661243506.532173 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ZooKeeperServer info Created server with tickTime 2000 minSessionTimeout 4000 maxSessionTimeout 40000 clientPortListenBacklog -1 datadir /Users/musum/git/vespa/clustercontroller-core/target/surefire-reports/test_fltctrl_zk3602320267307376975sdir/version-2 snapdir /Users/musum/git/vespa/clustercontroller-core/target/surefire-reports/test_fltctrl_zk3602320267307376975sdir/version-2 +1661243506.532871 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ServerCnxnFactory warning maxCnxns is not configured, using default value 0. +1661243506.533208 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.NIOServerCnxnFactory info Configuring NIO connection handler with 10s sessionless connection timeout, 2 selector thread(s), 32 worker threads, and 64 kB direct buffers. +1661243506.533724 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.NIOServerCnxnFactory info binding to port 0.0.0.0/0.0.0.0:0 +1661243506.534770 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.persistence.FileTxnSnapLog info Snapshotting: 0x0 to /Users/musum/git/vespa/clustercontroller-core/target/surefire-reports/test_fltctrl_zk3602320267307376975sdir/version-2/snapshot.0 +1661243506.535745 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ZKDatabase info Snapshot loaded in 1 ms, highest zxid is 0x0, digest is 1371985504 +1661243506.536054 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.persistence.FileTxnSnapLog info Snapshotting: 0x0 to /Users/musum/git/vespa/clustercontroller-core/target/surefire-reports/test_fltctrl_zk3602320267307376975sdir/version-2/snapshot.0 +1661243506.536680 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ZooKeeperServer info Snapshot taken in 1 ms +1661243506.537359 localhost 75603/570 - fleetcontroller.org.apache.zookeeper.server.PrepRequestProcessor info PrepRequestProcessor (sid:0) started, reconfigEnabled=false +1661243506.538719 localhost 75603/1 - fleetcontroller.com.yahoo.jrt.slobrok.api.Mirror info no location brokers available, retrying: [] (in 0.5 seconds) +1661243506.538988 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 0 for cluster 'mycluster': Created +1661243506.539735 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Clearing master data as we lost connection on node 0 +1661243506.543591 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.rpc.RpcServer info Registering tcp/localhost:51919 with slobrok at ( tcp/localhost:51917 ) +1661243506.543768 localhost 75603/580 - fleetcontroller.com.yahoo.jrt.slobrok.api.Mirror info successfully connected to location broker tcp/localhost:51917 (mirror initialized with 0 service names) +1661243506.544149 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 0 for cluster 'mycluster': Starting tick loop +1661243506.544582 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 0 for cluster 'mycluster': Setting up new ZooKeeper session at localhost:51916 +1661243506.544790 localhost 75603/1 - fleetcontroller.com.yahoo.jrt.slobrok.api.Mirror info no location brokers available, retrying: [] (in 0.5 seconds) +1661243506.544964 localhost 75603/584 - fleetcontroller.org.apache.zookeeper.ZooKeeper info Initiating client connection, connectString=localhost:51916 sessionTimeout=30000 watcher=com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase$ZooKeeperWatcher@1a1d9ef +1661243506.545108 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 1 for cluster 'mycluster': Created +1661243506.545701 localhost 75603/584 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocket info jute.maxbuffer value is 1048575 Bytes +1661243506.546061 localhost 75603/584 - fleetcontroller.org.apache.zookeeper.ClientCnxn info zookeeper.request.timeout value is 0. feature enabled=false +1661243506.546496 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Clearing master data as we lost connection on node 1 +1661243506.546672 localhost 75603/589 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Opening socket connection to server localhost/127.0.0.1:51916. +1661243506.547030 localhost 75603/589 - fleetcontroller.org.apache.zookeeper.ClientCnxn info SASL config status: Will not attempt to authenticate using SASL (unknown error) +1661243506.547696 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.rpc.RpcServer info Registering tcp/localhost:51921 with slobrok at ( tcp/localhost:51917 ) +1661243506.548321 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 1 for cluster 'mycluster': Starting tick loop +1661243506.548771 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 1 for cluster 'mycluster': Setting up new ZooKeeper session at localhost:51916 +1661243506.549005 localhost 75603/1 - fleetcontroller.com.yahoo.jrt.slobrok.api.Mirror info no location brokers available, retrying: [] (in 0.5 seconds) +1661243506.549218 localhost 75603/598 - fleetcontroller.org.apache.zookeeper.ZooKeeper info Initiating client connection, connectString=localhost:51916 sessionTimeout=30000 watcher=com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase$ZooKeeperWatcher@26718f6b +1661243506.549385 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 2 for cluster 'mycluster': Created +1661243506.549842 localhost 75603/598 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocket info jute.maxbuffer value is 1048575 Bytes +1661243506.550195 localhost 75603/598 - fleetcontroller.org.apache.zookeeper.ClientCnxn info zookeeper.request.timeout value is 0. feature enabled=false +1661243506.550816 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Clearing master data as we lost connection on node 2 +1661243506.550825 localhost 75603/603 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Opening socket connection to server localhost/127.0.0.1:51916. +1661243506.551412 localhost 75603/603 - fleetcontroller.org.apache.zookeeper.ClientCnxn info SASL config status: Will not attempt to authenticate using SASL (unknown error) +1661243506.551768 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.rpc.RpcServer info Registering tcp/localhost:51923 with slobrok at ( tcp/localhost:51917 ) +1661243506.552524 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 2 for cluster 'mycluster': Starting tick loop +1661243506.552959 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 2 for cluster 'mycluster': Setting up new ZooKeeper session at localhost:51916 +1661243506.553309 localhost 75603/611 - fleetcontroller.org.apache.zookeeper.ZooKeeper info Initiating client connection, connectString=localhost:51916 sessionTimeout=30000 watcher=com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase$ZooKeeperWatcher@28fe0700 +1661243506.553815 localhost 75603/611 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocket info jute.maxbuffer value is 1048575 Bytes +1661243506.554122 localhost 75603/611 - fleetcontroller.org.apache.zookeeper.ClientCnxn info zookeeper.request.timeout value is 0. feature enabled=false +1661243506.554631 localhost 75603/617 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Opening socket connection to server localhost/127.0.0.1:51916. +1661243506.554855 localhost 75603/617 - fleetcontroller.org.apache.zookeeper.ClientCnxn info SASL config status: Will not attempt to authenticate using SASL (unknown error) +1661243506.562585 localhost 75603/597 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Socket connection established, initiating session, client: /127.0.0.1:51925, server: localhost/127.0.0.1:51916 +1661243506.563182 localhost 75603/597 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is connected: [id: 0xd4444b66, L:/127.0.0.1:51925 - R:localhost/127.0.0.1:51916] +1661243506.590086 localhost 75603/592 - fleetcontroller.com.yahoo.jrt.slobrok.api.Mirror info successfully connected to location broker tcp/localhost:51917 (mirror initialized with 0 service names) +1661243506.590892 localhost 75603/569 - fleetcontroller.org.apache.zookeeper.server.persistence.FileTxnLog info Creating new log file: log.1 +1661243506.598339 localhost 75603/619 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Socket connection established, initiating session, client: /127.0.0.1:51934, server: localhost/127.0.0.1:51916 +1661243506.598339 localhost 75603/620 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Socket connection established, initiating session, client: /127.0.0.1:51933, server: localhost/127.0.0.1:51916 +1661243506.599014 localhost 75603/619 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is connected: [id: 0x33554773, L:/127.0.0.1:51934 - R:localhost/127.0.0.1:51916] +1661243506.599325 localhost 75603/620 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is connected: [id: 0x1186c720, L:/127.0.0.1:51933 - R:localhost/127.0.0.1:51916] +1661243506.614415 localhost 75603/606 - fleetcontroller.com.yahoo.jrt.slobrok.api.Mirror info successfully connected to location broker tcp/localhost:51917 (mirror initialized with 0 service names) +1661243506.637838 localhost 75603/597 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Session establishment complete on server localhost/127.0.0.1:51916, session id = 0x1001abb1a620000, negotiated timeout = 30000 +1661243506.638720 localhost 75603/591 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 0 for cluster 'mycluster': Connection to zookeeper server established. Refetching master data +1661243506.662210 localhost 75603/582 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51919] registering storage/cluster.mycluster/fleetcontroller/0 with location broker tcp/localhost:51917 completed successfully +1661243506.662536 localhost 75603/620 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Session establishment complete on server localhost/127.0.0.1:51916, session id = 0x1001abb1a620001, negotiated timeout = 30000 +1661243506.663794 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 2 for cluster 'mycluster': Connection to zookeeper server established. Refetching master data +1661243506.690305 localhost 75603/619 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Session establishment complete on server localhost/127.0.0.1:51916, session id = 0x1001abb1a620002, negotiated timeout = 30000 +1661243506.691150 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 1 for cluster 'mycluster': Connection to zookeeper server established. Refetching master data +1661243506.737726 localhost 75603/595 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51921] registering storage/cluster.mycluster/fleetcontroller/1 with location broker tcp/localhost:51917 completed successfully +1661243506.746175 localhost 75603/609 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51923] registering storage/cluster.mycluster/fleetcontroller/2 with location broker tcp/localhost:51917 completed successfully +1661243506.749958 localhost 75603/615 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51928] registering storage/cluster.mycluster/distributor/0/default with location broker tcp/localhost:51917 completed successfully +1661243506.753935 localhost 75603/624 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51935] registering storage/cluster.mycluster/storage/0/default with location broker tcp/localhost:51917 completed successfully +1661243506.757980 localhost 75603/629 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51939] registering storage/cluster.mycluster/distributor/1/default with location broker tcp/localhost:51917 completed successfully +1661243506.757980 localhost 75603/634 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51941] registering storage/cluster.mycluster/storage/1/default with location broker tcp/localhost:51917 completed successfully +1661243506.768805 localhost 75603/638 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51942] registering storage/cluster.mycluster/distributor/2/default with location broker tcp/localhost:51917 completed successfully +1661243506.775945 localhost 75603/648 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51947] registering storage/cluster.mycluster/distributor/3/default with location broker tcp/localhost:51917 completed successfully +1661243506.775986 localhost 75603/642 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51945] registering storage/cluster.mycluster/storage/2/default with location broker tcp/localhost:51917 completed successfully +1661243506.775955 localhost 75603/652 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51949] registering storage/cluster.mycluster/storage/3/default with location broker tcp/localhost:51917 completed successfully +1661243506.783290 localhost 75603/656 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51951] registering storage/cluster.mycluster/distributor/4/default with location broker tcp/localhost:51917 completed successfully +1661243506.787144 localhost 75603/661 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51953] registering storage/cluster.mycluster/storage/4/default with location broker tcp/localhost:51917 completed successfully +1661243506.787144 localhost 75603/665 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51958] registering storage/cluster.mycluster/distributor/5/default with location broker tcp/localhost:51917 completed successfully +1661243506.794455 localhost 75603/672 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51960] registering storage/cluster.mycluster/storage/5/default with location broker tcp/localhost:51917 completed successfully +1661243506.794474 localhost 75603/678 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51962] registering storage/cluster.mycluster/distributor/6/default with location broker tcp/localhost:51917 completed successfully +1661243506.798134 localhost 75603/683 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51966] registering storage/cluster.mycluster/storage/6/default with location broker tcp/localhost:51917 completed successfully +1661243506.808148 localhost 75603/694 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51974] registering storage/cluster.mycluster/storage/7/default with location broker tcp/localhost:51917 completed successfully +1661243506.808147 localhost 75603/689 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51969] registering storage/cluster.mycluster/distributor/7/default with location broker tcp/localhost:51917 completed successfully +1661243506.811748 localhost 75603/701 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51976] registering storage/cluster.mycluster/distributor/8/default with location broker tcp/localhost:51917 completed successfully +1661243506.819119 localhost 75603/707 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51980] registering storage/cluster.mycluster/storage/8/default with location broker tcp/localhost:51917 completed successfully +1661243506.819132 localhost 75603/712 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51983] registering storage/cluster.mycluster/distributor/9/default with location broker tcp/localhost:51917 completed successfully +1661243506.819119 localhost 75603/717 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51985] registering storage/cluster.mycluster/storage/9/default with location broker tcp/localhost:51917 completed successfully +1661243506.830431 localhost 75603/615 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51928] registering storage/cluster.mycluster/distributor/0 with location broker tcp/localhost:51917 completed successfully +1661243506.830479 localhost 75603/629 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51939] registering storage/cluster.mycluster/distributor/1 with location broker tcp/localhost:51917 completed successfully +1661243506.830443 localhost 75603/624 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51935] registering storage/cluster.mycluster/storage/0 with location broker tcp/localhost:51917 completed successfully +1661243506.845705 localhost 75603/638 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51942] registering storage/cluster.mycluster/distributor/2 with location broker tcp/localhost:51917 completed successfully +1661243506.845753 localhost 75603/652 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51949] registering storage/cluster.mycluster/storage/3 with location broker tcp/localhost:51917 completed successfully +1661243506.845705 localhost 75603/634 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51941] registering storage/cluster.mycluster/storage/1 with location broker tcp/localhost:51917 completed successfully +1661243506.856584 localhost 75603/648 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51947] registering storage/cluster.mycluster/distributor/3 with location broker tcp/localhost:51917 completed successfully +1661243506.856628 localhost 75603/656 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51951] registering storage/cluster.mycluster/distributor/4 with location broker tcp/localhost:51917 completed successfully +1661243506.856613 localhost 75603/642 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51945] registering storage/cluster.mycluster/storage/2 with location broker tcp/localhost:51917 completed successfully +1661243506.860464 localhost 75603/661 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51953] registering storage/cluster.mycluster/storage/4 with location broker tcp/localhost:51917 completed successfully +1661243506.860895 localhost 75603/712 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51983] registering storage/cluster.mycluster/distributor/9 with location broker tcp/localhost:51917 completed successfully +1661243506.860880 localhost 75603/689 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51969] registering storage/cluster.mycluster/distributor/7 with location broker tcp/localhost:51917 completed successfully +1661243506.860851 localhost 75603/701 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51976] registering storage/cluster.mycluster/distributor/8 with location broker tcp/localhost:51917 completed successfully +1661243506.860847 localhost 75603/694 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51974] registering storage/cluster.mycluster/storage/7 with location broker tcp/localhost:51917 completed successfully +1661243506.860782 localhost 75603/683 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51966] registering storage/cluster.mycluster/storage/6 with location broker tcp/localhost:51917 completed successfully +1661243506.860752 localhost 75603/672 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51960] registering storage/cluster.mycluster/storage/5 with location broker tcp/localhost:51917 completed successfully +1661243506.860530 localhost 75603/678 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51962] registering storage/cluster.mycluster/distributor/6 with location broker tcp/localhost:51917 completed successfully +1661243506.860464 localhost 75603/665 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51958] registering storage/cluster.mycluster/distributor/5 with location broker tcp/localhost:51917 completed successfully +1661243506.861219 localhost 75603/707 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51980] registering storage/cluster.mycluster/storage/8 with location broker tcp/localhost:51917 completed successfully +1661243506.861184 localhost 75603/717 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51985] registering storage/cluster.mycluster/storage/9 with location broker tcp/localhost:51917 completed successfully +1661243506.939485 localhost 75603/453 - fleetcontroller.org.apache.zookeeper.server.SessionTrackerImpl info SessionTrackerImpl exited loop! +1661243506.939482 localhost 75603/227 - fleetcontroller.org.apache.zookeeper.server.SessionTrackerImpl info SessionTrackerImpl exited loop! +1661243507.075476 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 1 for cluster 'mycluster': Creating ephemeral master vote node with vote to self. +1661243507.075476 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 2 for cluster 'mycluster': Creating ephemeral master vote node with vote to self. +1661243507.075941 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 0 for cluster 'mycluster': Creating ephemeral master vote node with vote to self. +1661243507.105249 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 1 for cluster 'mycluster': Done setting up new ZooKeeper session at localhost:51916 +1661243507.125027 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 0 for cluster 'mycluster': Done setting up new ZooKeeper session at localhost:51916 +1661243507.125147 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got node list response from /vespa/fleetcontroller/mycluster/indexes version 0 with 3 nodes +1661243507.125040 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 2 for cluster 'mycluster': Done setting up new ZooKeeper session at localhost:51916 +1661243507.145058 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 1 for cluster 'mycluster': Stored new vote in ephemeral node. 1 -> 1 +1661243507.145126 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got node list response from /vespa/fleetcontroller/mycluster/indexes version 0 with 3 nodes +1661243507.145126 localhost 75603/591 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got node list response from /vespa/fleetcontroller/mycluster/indexes version 0 with 3 nodes +1661243507.164381 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 0 for cluster 'mycluster': Stored new vote in ephemeral node. 0 -> 0 +1661243507.164875 localhost 75603/591 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/0 with code 0 and data 0 +1661243507.164754 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/0 with code 0 and data 0 +1661243507.164719 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 2 for cluster 'mycluster': Stored new vote in ephemeral node. 2 -> 2 +1661243507.164567 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/0 with code 0 and data 0 +1661243507.165628 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 0. Altering vote from null to 0. +1661243507.166469 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 1 +1661243507.165319 localhost 75603/591 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote from fleetcontroller 0. Altering vote from null to 0. +1661243507.167058 localhost 75603/591 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 1 +1661243507.167329 localhost 75603/591 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote from fleetcontroller 1. Altering vote from null to 1. +1661243507.167572 localhost 75603/591 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 2 +1661243507.165034 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.testutils.Waiter$Impl info Waiting for RegexStateMatcher:\n wanted: 'version:\\d+ distributor:10 storage:10'\n last checked: 'null'\n current: '' with wait task GetNodeStateResender +1661243507.167844 localhost 75603/591 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote from fleetcontroller 2. Altering vote from null to 2. +1661243507.166796 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 1. Altering vote from null to 1. +1661243507.166198 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote from fleetcontroller 0. Altering vote from null to 0. +1661243507.169076 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 1 +1661243507.169473 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote from fleetcontroller 1. Altering vote from null to 1. +1661243507.168804 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 2 +1661243507.168587 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Got new fleet data with 3 entries: {0=0, 1=1, 2=2} +1661243507.170397 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Handling new master election, as we have received 3 entries +1661243507.168273 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.testutils.Waiter$Impl info Wait condition not met: No cluster state defined yet +1661243507.170700 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Got master election state data(0 -> 0, 1 -> 1, 2 -> 2). +1661243507.170038 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 2. Altering vote from null to 2. +1661243507.169758 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 2 +1661243507.171491 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got new fleet data with 3 entries: {0=0, 1=1, 2=2} +1661243507.171172 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Currently too few followers for cluster controller candidate 0. No current master. (1/3 followers) +1661243507.172005 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Handling new master election, as we have received 3 entries +1661243507.171758 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote from fleetcontroller 2. Altering vote from null to 2. +1661243507.172648 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got master election state data(0 -> 0, 1 -> 1, 2 -> 2). +1661243507.173228 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got new fleet data with 3 entries: {0=0, 1=1, 2=2} +1661243507.173201 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Altered data in node /vespa/fleetcontroller/mycluster/indexes/2. Requesting new vote +1661243507.173966 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Handling new master election, as we have received 3 entries +1661243507.173736 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type MASTER_ELECTION @1661243507173: This node just became node state gatherer as we are fleetcontroller master candidate. +1661243507.173664 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Currently too few followers for cluster controller candidate 0. No current master. (1/3 followers) +1661243507.174536 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got master election state data(0 -> 0, 1 -> 1, 2 -> 2). +1661243507.175447 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Altering our state to follow new fleet controller master candidate 0 +1661243507.175725 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Currently too few followers for cluster controller candidate 0. No current master. (1/3 followers) +1661243507.176648 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Altering our state to follow new fleet controller master candidate 0 +1661243507.236938 localhost 75603/591 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Altered data in node /vespa/fleetcontroller/mycluster/indexes/2. Requesting new vote +1661243507.237154 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 2 for cluster 'mycluster': Stored new vote in ephemeral node. 2 -> 0 +1661243507.237150 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Altered data in node /vespa/fleetcontroller/mycluster/indexes/2. Requesting new vote +1661243507.237144 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 0 for cluster 'mycluster': Read cluster state version 0 from ZooKeeper (znode version 0) +1661243507.236938 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 0 +1661243507.239223 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243507.239776 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote from fleetcontroller 2. Altering vote from null to 0. +1661243507.242825 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243507.244876 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 1 for cluster 'mycluster': Stored new vote in ephemeral node. 1 -> 0 +1661243507.244815 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Altered data in node /vespa/fleetcontroller/mycluster/indexes/1. Requesting new vote +1661243507.244815 localhost 75603/591 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Altered data in node /vespa/fleetcontroller/mycluster/indexes/1. Requesting new vote +1661243507.247572 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type MASTER_ELECTION @1661243507247: This node just became node state gatherer as we are fleetcontroller master candidate. +1661243507.247334 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 0 +1661243507.246575 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243507.249030 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 2. Altering vote from null to 0. +1661243507.247900 localhost 75603/591 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 0 +1661243507.252522 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243507.253123 localhost 75603/591 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote from fleetcontroller 2. Altering vote from null to 0. +1661243507.256577 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243507.259765 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243507.262772 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243507.265782 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243507.268632 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243507.271328 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243507.274020 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243507.276987 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243507.280101 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243507.283056 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243507.285980 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243507.288942 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243507.291763 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243507.294817 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243507.296872 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 1 for cluster 'mycluster': Read cluster state version 0 from ZooKeeper (znode version 0) +1661243507.298037 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243507.298239 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got new fleet data with 3 entries: {0=0, 1=1, 2=0} +1661243507.298192 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Altered data in node /vespa/fleetcontroller/mycluster/indexes/1. Requesting new vote +1661243507.301219 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243507.301209 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Handling new master election, as we have received 3 entries +1661243507.304692 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got master election state data(0 -> 0, 1 -> 1, 2 -> 0). +1661243507.305134 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': 0 is newly elected master +1661243507.305486 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Altering our state to follow new fleet controller master candidate 0 +1661243507.307166 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 0 +1661243507.307720 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 1. Altering vote from null to 0. +1661243507.308367 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got new fleet data with 3 entries: {0=0, 1=0, 2=0} +1661243507.308888 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Handling new master election, as we have received 3 entries +1661243507.309261 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got master election state data(0 -> 0, 1 -> 0, 2 -> 0). +1661243507.309636 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': 0 is newly elected master +1661243507.318165 localhost 75603/591 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 0 +1661243507.318624 localhost 75603/591 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote from fleetcontroller 1. Altering vote from null to 0. +1661243507.319044 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Got new fleet data with 3 entries: {0=0, 1=0, 2=0} +1661243507.319487 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Handling new master election, as we have received 3 entries +1661243507.319808 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Got master election state data(0 -> 0, 1 -> 0, 2 -> 0). +1661243507.320135 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': 0 is newly elected master +1661243507.324715 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 0 +1661243507.324770 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 0 for cluster 'mycluster': Read cluster state version 0 from ZooKeeper (znode version 0) +1661243507.325691 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243507.325517 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote from fleetcontroller 1. Altering vote from null to 0. +1661243507.329094 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243507.332316 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got new fleet data with 3 entries: {0=0, 1=0, 2=0} +1661243507.332946 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Handling new master election, as we have received 3 entries +1661243507.333340 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got master election state data(0 -> 0, 1 -> 0, 2 -> 0). +1661243507.333758 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': 0 is still the master +1661243507.334132 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243507.336963 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243507.337582 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 0 for cluster 'mycluster': Loaded previous cluster state bundle from ZooKeeper: ClusterStateBundle('') +1661243507.340246 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243507.340309 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type MASTER_ELECTION @1661243507340: This node just became fleetcontroller master. Bumped version to 1 to be in line. +1661243507.343430 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243507.343874 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: storage.0: Altered node state in cluster state from 'D' to 'U, t 1661243506' +1661243507.346302 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243507.346415 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: distributor.0: Altered node state in cluster state from 'D' to 'U, t 1661243506' +1661243507.349827 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: storage.1: Altered node state in cluster state from 'D' to 'U, t 1661243506' +1661243507.350170 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: distributor.1: Altered node state in cluster state from 'D' to 'U, t 1661243506' +1661243507.350464 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: storage.2: Altered node state in cluster state from 'D' to 'U, t 1661243506' +1661243507.350774 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: distributor.2: Altered node state in cluster state from 'D' to 'U, t 1661243506' +1661243507.349780 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243507.351056 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: storage.3: Altered node state in cluster state from 'D' to 'U, t 1661243506' +1661243507.353651 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: distributor.3: Altered node state in cluster state from 'D' to 'U, t 1661243506' +1661243507.353955 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: storage.4: Altered node state in cluster state from 'D' to 'U, t 1661243506' +1661243507.354220 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: distributor.4: Altered node state in cluster state from 'D' to 'U, t 1661243506' +1661243507.354492 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: storage.5: Altered node state in cluster state from 'D' to 'U, t 1661243506' +1661243507.354858 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: distributor.5: Altered node state in cluster state from 'D' to 'U, t 1661243506' +1661243507.355139 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: storage.6: Altered node state in cluster state from 'D' to 'U, t 1661243506' +1661243507.355411 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: distributor.6: Altered node state in cluster state from 'D' to 'U, t 1661243506' +1661243507.355700 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: storage.7: Altered node state in cluster state from 'D' to 'U, t 1661243506' +1661243507.356001 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: distributor.7: Altered node state in cluster state from 'D' to 'U, t 1661243506' +1661243507.353517 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243507.356298 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: storage.8: Altered node state in cluster state from 'D' to 'U, t 1661243506' +1661243507.358952 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243507.358996 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: distributor.8: Altered node state in cluster state from 'D' to 'U, t 1661243506' +1661243507.361430 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: storage.9: Altered node state in cluster state from 'D' to 'U, t 1661243506' +1661243507.361303 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243507.361720 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: distributor.9: Altered node state in cluster state from 'D' to 'U, t 1661243506' +1661243507.363843 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243507.364021 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type SYSTEMSTATE @1661243507343: New cluster state version 2. Change from last: version: 0 => 2, storage: [0: [Down => Up, startTimestamp: 0 => 1661243506], 1: [Down => Up, startTimestamp: 0 => 1661243506], 2: [Down => Up, startTimestamp: 0 => 1661243506], 3: [Down => Up, startTimestamp: 0 => 1661243506], 4: [Down => Up, startTimestamp: 0 => 1661243506], 5: [Down => Up, startTimestamp: 0 => 1661243506], 6: [Down => Up, startTimestamp: 0 => 1661243506], 7: [Down => Up, startTimestamp: 0 => 1661243506], 8: [Down => Up, startTimestamp: 0 => 1661243506], 9: [Down => Up, startTimestamp: 0 => 1661243506]], distributor: [0: [Down => Up, startTimestamp: 0 => 1661243506], 1: [Down => Up, startTimestamp: 0 => 1661243506], 2: [Down => Up, startTimestamp: 0 => 1661243506], 3: [Down => Up, startTimestamp: 0 => 1661243506], 4: [Down => Up, startTimestamp: 0 => 1661243506], 5: [Down => Up, startTimestamp: 0 => 1661243506], 6: [Down => Up, startTimestamp: 0 => 1661243506], 7: [Down => Up, startTimestamp: 0 => 1661243506], 8: [Down => Up, startTimestamp: 0 => 1661243506], 9: [Down => Up, startTimestamp: 0 => 1661243506]] +1661243507.366138 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243507.367454 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 0 for cluster 'mycluster': Storing new cluster state version in ZooKeeper: 2 +1661243507.369765 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243507.372025 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243507.374386 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243507.376627 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243507.378923 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243507.381198 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243507.383551 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243507.408998 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 0 for cluster 'mycluster': Master moratorium complete: timed out waiting for all nodes to report in +1661243507.409328 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.SystemStateBroadcaster info FleetController 0 for cluster 'mycluster': Publishing cluster state version 2 +1661243507.414145 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type SYSTEMSTATE @1661243507414: Reset 20 start timestamps as all available distributors have seen newest cluster state. +1661243507.435110 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.testutils.Waiter$Impl info Wait condition not met: Cluster state mismatch +1661243507.435443 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: storage.0: Altered node state in cluster state from 'U, t 1661243506' to 'U' +1661243507.435703 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: distributor.0: Altered node state in cluster state from 'U, t 1661243506' to 'U' +1661243507.435949 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: storage.1: Altered node state in cluster state from 'U, t 1661243506' to 'U' +1661243507.436191 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: distributor.1: Altered node state in cluster state from 'U, t 1661243506' to 'U' +1661243507.436460 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: storage.2: Altered node state in cluster state from 'U, t 1661243506' to 'U' +1661243507.436722 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: distributor.2: Altered node state in cluster state from 'U, t 1661243506' to 'U' +1661243507.437016 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: storage.3: Altered node state in cluster state from 'U, t 1661243506' to 'U' +1661243507.437349 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: distributor.3: Altered node state in cluster state from 'U, t 1661243506' to 'U' +1661243507.437668 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: storage.4: Altered node state in cluster state from 'U, t 1661243506' to 'U' +1661243507.437944 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: distributor.4: Altered node state in cluster state from 'U, t 1661243506' to 'U' +1661243507.438197 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: storage.5: Altered node state in cluster state from 'U, t 1661243506' to 'U' +1661243507.438440 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: distributor.5: Altered node state in cluster state from 'U, t 1661243506' to 'U' +1661243507.438690 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: storage.6: Altered node state in cluster state from 'U, t 1661243506' to 'U' +1661243507.438941 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: distributor.6: Altered node state in cluster state from 'U, t 1661243506' to 'U' +1661243507.439198 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: storage.7: Altered node state in cluster state from 'U, t 1661243506' to 'U' +1661243507.439459 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: distributor.7: Altered node state in cluster state from 'U, t 1661243506' to 'U' +1661243507.439814 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: storage.8: Altered node state in cluster state from 'U, t 1661243506' to 'U' +1661243507.440057 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: distributor.8: Altered node state in cluster state from 'U, t 1661243506' to 'U' +1661243507.440320 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: storage.9: Altered node state in cluster state from 'U, t 1661243506' to 'U' +1661243507.440621 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: distributor.9: Altered node state in cluster state from 'U, t 1661243506' to 'U' +1661243507.441004 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type SYSTEMSTATE @1661243507435: New cluster state version 3. Change from last: version: 2 => 3, storage: [0: startTimestamp: 1661243506 => 0, 1: startTimestamp: 1661243506 => 0, 2: startTimestamp: 1661243506 => 0, 3: startTimestamp: 1661243506 => 0, 4: startTimestamp: 1661243506 => 0, 5: startTimestamp: 1661243506 => 0, 6: startTimestamp: 1661243506 => 0, 7: startTimestamp: 1661243506 => 0, 8: startTimestamp: 1661243506 => 0, 9: startTimestamp: 1661243506 => 0], distributor: [0: startTimestamp: 1661243506 => 0, 1: startTimestamp: 1661243506 => 0, 2: startTimestamp: 1661243506 => 0, 3: startTimestamp: 1661243506 => 0, 4: startTimestamp: 1661243506 => 0, 5: startTimestamp: 1661243506 => 0, 6: startTimestamp: 1661243506 => 0, 7: startTimestamp: 1661243506 => 0, 8: startTimestamp: 1661243506 => 0, 9: startTimestamp: 1661243506 => 0] +1661243507.442408 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 0 for cluster 'mycluster': Storing new cluster state version in ZooKeeper: 3 +1661243507.485739 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.SystemStateBroadcaster info FleetController 0 for cluster 'mycluster': Publishing cluster state version 3 +1661243507.701429 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.testutils.Waiter$Impl info Condition met. Returning +1661243507.701749 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Entering waitForMaster +1661243507.702019 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Node 0 is master. Checking that no one else is master +1661243507.702286 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Leaving waitForMaster +1661243507.725313 localhost 75603/548 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Opening socket connection to server localhost/127.0.0.1:51704. +1661243507.725686 localhost 75603/548 - fleetcontroller.org.apache.zookeeper.ClientCnxn info SASL config status: Will not attempt to authenticate using SASL (unknown error) +1661243507.727789 localhost 75603/550 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty warning future isn't success.\nexception=\nio.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: localhost/127.0.0.1:51704\nCaused by: java.net.ConnectException: Connection refused\n\tat java.base/sun.nio.ch.Net.pollConnect(Native Method)\n\tat java.base/sun.nio.ch.Net.pollConnectNow(Net.java:672)\n\tat java.base/sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946)\n\tat io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)\n\tat io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:710)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:658)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:584)\n\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496)\n\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)\n\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243507.730593 localhost 75603/548 - fleetcontroller.org.apache.zookeeper.ClientCnxn warning Session 0x1001abb14b10003 for server localhost/127.0.0.1:51704, Closing socket connection. Attempting reconnect except it is a SessionExpiredException.\nexception=\nEndOfStreamException: channel for sessionid 0x1001abb14b10003 is lost\n\tat org.apache.zookeeper.ClientCnxnSocketNetty.doTransport(ClientCnxnSocketNetty.java:285)\n\tat org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1283)\n +1661243508.017421 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 0 for cluster 'mycluster': Joining event thread. +1661243508.017768 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 0 for cluster 'mycluster': Tick loop stopped +1661243508.018146 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 0 for cluster 'mycluster': FleetController done shutting down event thread. +1661243508.018493 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 0 for cluster 'mycluster': Resetting database state +1661243508.068105 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: A change occurred in the list of registered fleetcontrollers. Requesting new information +1661243508.068712 localhost 75603/597 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is disconnected: [id: 0xd4444b66, L:/127.0.0.1:51925 ! R:localhost/127.0.0.1:51916] +1661243508.068378 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is told closing +1661243508.068326 localhost 75603/591 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: A change occurred in the list of registered fleetcontrollers. Requesting new information +1661243508.068144 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: A change occurred in the list of registered fleetcontrollers. Requesting new information +1661243508.070558 localhost 75603/591 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got node list response from /vespa/fleetcontroller/mycluster/indexes version -112 with 0 nodes +1661243508.069868 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.ZooKeeper info Session: 0x1001abb1a620000 closed +1661243508.069499 localhost 75603/597 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is told closing +1661243508.071980 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Clearing master data as we lost connection on node 0 +1661243508.071719 localhost 75603/591 - fleetcontroller.org.apache.zookeeper.ClientCnxn info EventThread shut down for session: 0x1001abb1a620000 +1661243508.072885 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type MASTER_ELECTION @1661243508072: This node is no longer fleetcontroller master. +1661243508.074140 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 0 for cluster 'mycluster': Done resetting database state +1661243508.078643 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Entering waitForMaster +1661243508.079373 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Node 1 is not master yet, sleeping more +1661243508.132075 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got node list response from /vespa/fleetcontroller/mycluster/indexes version 0 with 2 nodes +1661243508.132074 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got node list response from /vespa/fleetcontroller/mycluster/indexes version 0 with 2 nodes +1661243508.153050 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 0 +1661243508.153050 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 0 +1661243508.153731 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote from fleetcontroller 1. Altering vote from null to 0. +1661243508.154793 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 0 +1661243508.154262 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 1. Altering vote from null to 0. +1661243508.155463 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote from fleetcontroller 2. Altering vote from null to 0. +1661243508.156080 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 0 +1661243508.156688 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got new fleet data with 2 entries: {1=0, 2=0} +1661243508.157194 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 2. Altering vote from null to 0. +1661243508.157852 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Handling new master election, as we have received 2 entries +1661243508.158860 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got new fleet data with 2 entries: {1=0, 2=0} +1661243508.159575 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got master election state data(1 -> 0, 2 -> 0). +1661243508.160109 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Handling new master election, as we have received 2 entries +1661243508.160600 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Master gone from ZooKeeper. Tagging timestamp. Will wait 1 ms. +1661243508.161198 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got master election state data(1 -> 0, 2 -> 0). +1661243508.161603 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': First index is not currently trying to become master. Waiting for it to change state +1661243508.162088 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Master gone from ZooKeeper. Tagging timestamp. Will wait 1 ms. +1661243508.162693 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': We are next in line to become master. Altering our state to look for followers +1661243508.163348 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': First index is not currently trying to become master. Waiting for it to change state +1661243508.185815 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Altered data in node /vespa/fleetcontroller/mycluster/indexes/1. Requesting new vote +1661243508.185853 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 1 for cluster 'mycluster': Stored new vote in ephemeral node. 1 -> 1 +1661243508.185815 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Altered data in node /vespa/fleetcontroller/mycluster/indexes/1. Requesting new vote +1661243508.186946 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 1 +1661243508.187820 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote from fleetcontroller 1. Altering vote from null to 1. +1661243508.187828 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 1 +1661243508.188768 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 1. Altering vote from null to 1. +1661243508.188441 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got new fleet data with 2 entries: {1=1, 2=0} +1661243508.189206 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got new fleet data with 2 entries: {1=1, 2=0} +1661243508.189489 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Handling new master election, as we have received 2 entries +1661243508.189858 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Handling new master election, as we have received 2 entries +1661243508.190241 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got master election state data(1 -> 1, 2 -> 0). +1661243508.190638 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got master election state data(1 -> 1, 2 -> 0). +1661243508.191013 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Currently too few followers for cluster controller candidate 1. No current master. (1/3 followers) +1661243508.191604 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Currently too few followers for cluster controller candidate 1. No current master. (1/3 followers) +1661243508.192970 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Altering our state to follow new fleet controller master candidate 1 +1661243508.206683 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Altered data in node /vespa/fleetcontroller/mycluster/indexes/2. Requesting new vote +1661243508.206704 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 2 for cluster 'mycluster': Stored new vote in ephemeral node. 2 -> 1 +1661243508.206689 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Altered data in node /vespa/fleetcontroller/mycluster/indexes/2. Requesting new vote +1661243508.218344 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 1 +1661243508.218344 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 1 +1661243508.218941 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 2. Altering vote from null to 1. +1661243508.219369 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote from fleetcontroller 2. Altering vote from null to 1. +1661243508.219862 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got new fleet data with 2 entries: {1=1, 2=1} +1661243508.220267 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got new fleet data with 2 entries: {1=1, 2=1} +1661243508.220562 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Handling new master election, as we have received 2 entries +1661243508.221046 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Handling new master election, as we have received 2 entries +1661243508.221403 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got master election state data(1 -> 1, 2 -> 1). +1661243508.221797 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got master election state data(1 -> 1, 2 -> 1). +1661243508.222986 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': 1 is newly elected master +1661243508.222251 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': 1 is newly elected master +1661243508.259410 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 1 for cluster 'mycluster': Read cluster state version 3 from ZooKeeper (znode version 2) +1661243508.270984 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 1 for cluster 'mycluster': Loaded previous cluster state bundle from ZooKeeper: ClusterStateBundle('version:3 distributor:10 storage:10', default 'version:3 distributor:10 storage:10', global 'version:3 distributor:10 storage:10' (deferred activation)) +1661243508.272095 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type MASTER_ELECTION @1661243508272: This node just became fleetcontroller master. Bumped version to 3 to be in line. +1661243508.273069 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type SYSTEMSTATE @1661243508272: New cluster state version 4. Change from last: version: 3 => 4 +1661243508.273621 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 1 for cluster 'mycluster': Storing new cluster state version in ZooKeeper: 4 +1661243508.301543 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 1 for cluster 'mycluster': Master moratorium complete: timed out waiting for all nodes to report in +1661243508.302386 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.SystemStateBroadcaster info FleetController 1 for cluster 'mycluster': Publishing cluster state version 4 +1661243508.303245 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Node 1 is master. Checking that no one else is master +1661243508.303650 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Leaving waitForMaster +1661243508.403410 localhost 75603/561 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Opening socket connection to server localhost/[0:0:0:0:0:0:0:1]:51704. +1661243508.403888 localhost 75603/561 - fleetcontroller.org.apache.zookeeper.ClientCnxn info SASL config status: Will not attempt to authenticate using SASL (unknown error) +1661243508.421342 localhost 75603/563 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty warning future isn't success.\nexception=\nio.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: localhost/[0:0:0:0:0:0:0:1]:51704\nCaused by: java.net.ConnectException: Connection refused\n\tat java.base/sun.nio.ch.Net.pollConnect(Native Method)\n\tat java.base/sun.nio.ch.Net.pollConnectNow(Net.java:672)\n\tat java.base/sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946)\n\tat io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)\n\tat io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:710)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:658)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:584)\n\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496)\n\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)\n\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n +1661243508.424373 localhost 75603/561 - fleetcontroller.org.apache.zookeeper.ClientCnxn warning Session 0x1001abb14b10004 for server localhost/[0:0:0:0:0:0:0:1]:51704, Closing socket connection. Attempting reconnect except it is a SessionExpiredException.\nexception=\nEndOfStreamException: channel for sessionid 0x1001abb14b10004 is lost\n\tat org.apache.zookeeper.ClientCnxnSocketNetty.doTransport(ClientCnxnSocketNetty.java:285)\n\tat org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1283)\n +java.lang.RuntimeException: Failed to get /vespa/fleetcontroller/mycluster/latestversion + at com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase.retrieveLatestSystemStateVersion(ZooKeeperDatabase.java:228) + at com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler.getLatestSystemStateVersion(DatabaseHandler.java:378) + at com.yahoo.vespa.clustercontroller.core.FleetController.resyncLocallyCachedState(FleetController.java:908) + at com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:607) + at com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120) + at java.base/java.lang.Thread.run(Thread.java:833) +Caused by: org.apache.zookeeper.KeeperException$ConnectionLossException: KeeperErrorCode = ConnectionLoss for /vespa/fleetcontroller/mycluster/latestversion + at org.apache.zookeeper.KeeperException.create(KeeperException.java:102) + at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) + at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:2028) + at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:2056) + at com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase.retrieveLatestSystemStateVersion(ZooKeeperDatabase.java:221) + ... 5 more +1661243508.427229 localhost 75603/560 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController error FleetController 1 for cluster 'mycluster': Fatal error killed fleet controller\nexception=\njava.lang.RuntimeException: Failed to get /vespa/fleetcontroller/mycluster/latestversion\n\tat com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase.retrieveLatestSystemStateVersion(ZooKeeperDatabase.java:228)\n\tat com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler.getLatestSystemStateVersion(DatabaseHandler.java:378)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.resyncLocallyCachedState(FleetController.java:908)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:607)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\nCaused by: org.apache.zookeeper.KeeperException$ConnectionLossException: KeeperErrorCode = ConnectionLoss for /vespa/fleetcontroller/mycluster/latestversion\n\tat org.apache.zookeeper.KeeperException.create(KeeperException.java:102)\n\tat org.apache.zookeeper.KeeperException.create(KeeperException.java:54)\n\tat org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:2028)\n\tat org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:2056)\n\tat com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase.retrieveLatestSystemStateVersion(ZooKeeperDatabase.java:221)\n\t... 5 more\n diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java index 3f3bf62bf4d2..b3dd0d738e43 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java @@ -37,6 +37,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Queue; import java.util.Set; import java.util.TimeZone; @@ -130,12 +131,9 @@ public FleetController(FleetControllerContext context, this.systemStateBroadcaster = systemStateBroadcaster; this.stateVersionTracker = new StateVersionTracker(options.minMergeCompletionRatio); this.metricUpdater = metricUpdater; - - this.statusPageServer = statusPage; + this.statusPageServer = Objects.requireNonNull(statusPage, "statusPage cannot be null"); this.rpcServer = server; - this.masterElectionHandler = masterElectionHandler; - this.statusRequestRouter.addHandler( "^/node=([a-z]+)\\.(\\d+)$", new LegacyNodePageRequestHandler(timer, eventLog, cluster)); @@ -277,9 +275,7 @@ public void shutdown() throws InterruptedException, java.io.IOException { controllerThreadId = Thread.currentThread().getId(); database.shutdown(databaseContext); - if (statusPageServer != null) { - statusPageServer.shutdown(); - } + statusPageServer.shutdown(); if (rpcServer != null) { rpcServer.shutdown(); } @@ -530,12 +526,10 @@ private void propagateOptions() { } } - if (statusPageServer != null) { - try{ - statusPageServer.setPort(options.httpPort); - } catch (Exception e) { - context.log(logger, Level.WARNING, "Failed to initialize status server socket. This may be natural if cluster has altered the services running on this node: " + e.getMessage()); - } + try { + statusPageServer.setPort(options.httpPort); + } catch (Exception e) { + context.log(logger, Level.WARNING, "Failed to initialize status server socket. This may be natural if cluster has altered the services running on this node: " + e.getMessage()); } long currentTime = timer.getCurrentTimeInMillis(); @@ -679,12 +673,10 @@ private void switchToNewConfig() { } private boolean processAnyPendingStatusPageRequest() { - if (statusPageServer != null) { - StatusPageServer.HttpRequest statusRequest = statusPageServer.getCurrentHttpRequest(); - if (statusRequest != null) { - statusPageServer.answerCurrentStatusRequest(fetchStatusPage(statusRequest)); - return true; - } + StatusPageServer.HttpRequest statusRequest = statusPageServer.getCurrentHttpRequest(); + if (statusRequest != null) { + statusPageServer.answerCurrentStatusRequest(fetchStatusPage(statusRequest)); + return true; } return false; } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/StatusHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/StatusHandler.java index 69144d23fe53..79e6a91f5612 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/StatusHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/StatusHandler.java @@ -31,7 +31,7 @@ public static class ContainerStatusPageServer implements StatusPageServerInterfa StatusPageServer.HttpRequest request; StatusPageResponse response; - // Ensure only only one use the server at a time + // Ensure only one use the server at a time private final Object queueMonitor = new Object(); // Lock safety with fleetcontroller. Wait until completion private final Object answerMonitor = new Object(); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFeedBlockTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFeedBlockTest.java index ec6255fd13bf..5a9d28bc3278 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFeedBlockTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFeedBlockTest.java @@ -9,6 +9,7 @@ import com.yahoo.vdslib.state.State; import com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler; import com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabaseFactory; +import com.yahoo.vespa.clustercontroller.core.status.StatusHandler; import com.yahoo.vespa.clustercontroller.utils.util.NoMetricReporter; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -58,7 +59,9 @@ private void initialize(FleetControllerOptions options) throws Exception { var stateGenerator = new StateChangeHandler(context, timer, eventLog); var stateBroadcaster = new SystemStateBroadcaster(context, timer, timer); var masterElectionHandler = new MasterElectionHandler(context, options.fleetControllerIndex, options.fleetControllerCount, timer, timer); - ctrl = new FleetController(context, timer, eventLog, cluster, stateGatherer, communicator, null, null, communicator, database, stateGenerator, stateBroadcaster, masterElectionHandler, metricUpdater, options); + var status = new StatusHandler.ContainerStatusPageServer(); + ctrl = new FleetController(context, timer, eventLog, cluster, stateGatherer, communicator, status, null, communicator, database, + stateGenerator, stateBroadcaster, masterElectionHandler, metricUpdater, options); ctrl.tick(); markAllNodesAsUp(options); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java index 426ac42fe9e6..fd6ff30a08fb 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java @@ -11,6 +11,7 @@ import com.yahoo.vdslib.state.State; import com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler; import com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabaseFactory; +import com.yahoo.vespa.clustercontroller.core.status.StatusHandler; import com.yahoo.vespa.clustercontroller.core.testutils.StateWaiter; import com.yahoo.vespa.clustercontroller.utils.util.NoMetricReporter; import org.junit.jupiter.api.BeforeEach; @@ -57,7 +58,9 @@ private void initialize(FleetControllerOptions options) throws Exception { var stateGenerator = new StateChangeHandler(context, timer, eventLog); var stateBroadcaster = new SystemStateBroadcaster(context, timer, timer); var masterElectionHandler = new MasterElectionHandler(context, options.fleetControllerIndex, options.fleetControllerCount, timer, timer); - ctrl = new FleetController(context, timer, eventLog, cluster, stateGatherer, communicator, null, null, communicator, database, stateGenerator, stateBroadcaster, masterElectionHandler, metricUpdater, options); + var status = new StatusHandler.ContainerStatusPageServer(); + ctrl = new FleetController(context, timer, eventLog, cluster, stateGatherer, communicator, status, null, communicator, database, + stateGenerator, stateBroadcaster, masterElectionHandler, metricUpdater, options); ctrl.tick(); if (options.fleetControllerCount == 1) { From 9dcf293c912bf40a4caedbc6fa7bc5b7f3fc2b4e Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Mon, 29 Aug 2022 15:35:27 +0200 Subject: [PATCH 2/7] Remove dead code and some cosmetic changes --- .../model/content/cluster/ContentCluster.java | 27 ++++++++++--------- .../storagecluster/FileStorProducer.java | 16 +++++------ .../IntegrityCheckerProducer.java | 6 ++--- .../storagecluster/StorServerProducer.java | 16 +++-------- .../storagecluster/StorageCluster.java | 4 +-- 5 files changed, 30 insertions(+), 39 deletions(-) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 74017b6b821c..bf5770681ef8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -117,20 +117,23 @@ public ContentCluster build(Collection containers, ConfigModelCo RedundancyBuilder redundancyBuilder = new RedundancyBuilder(contentElement); Set globallyDistributedDocuments = new GlobalDistributionBuilder(documentDefinitions).build(documentsElement); - ContentCluster c = new ContentCluster(context.getParentProducer(), getClusterId(contentElement), documentDefinitions, + String clusterId = getClusterId(contentElement); + ContentCluster c = new ContentCluster(context.getParentProducer(), clusterId, documentDefinitions, globallyDistributedDocuments, routingSelection, deployState.zone(), deployState.isHosted()); var resourceLimits = new ClusterResourceLimits.Builder(stateIsHosted(deployState), deployState.featureFlags().resourceLimitDisk(), deployState.featureFlags().resourceLimitMemory()) .build(contentElement); - c.clusterControllerConfig = new ClusterControllerConfig.Builder(getClusterId(contentElement), - contentElement, - resourceLimits.getClusterControllerLimits()).build(deployState, c, contentElement.getXml()); + c.clusterControllerConfig = new ClusterControllerConfig.Builder(clusterId, + contentElement, + resourceLimits.getClusterControllerLimits()) + .build(deployState, c, contentElement.getXml()); c.search = new ContentSearchCluster.Builder(documentDefinitions, - globallyDistributedDocuments, - fractionOfMemoryReserved(getClusterId(contentElement), containers), - resourceLimits.getContentNodeLimits()).build(deployState, c, contentElement.getXml()); + globallyDistributedDocuments, + fractionOfMemoryReserved(clusterId, containers), + resourceLimits.getContentNodeLimits()) + .build(deployState, c, contentElement.getXml()); c.persistenceFactory = new EngineFactoryBuilder().build(contentElement, c); c.storageNodes = new StorageCluster.Builder().build(deployState, c, w3cContentElement); c.distributorNodes = new DistributorCluster.Builder(c).build(deployState, c, w3cContentElement); @@ -173,9 +176,8 @@ private void setupSearchCluster(ContentSearchCluster csc, ModelElement element, if (csc.hasIndexedCluster()) { setupIndexedCluster(csc.getIndexed(), search, element, logger); } - - } + private void setupIndexedCluster(IndexedSearchCluster index, ContentSearch search, ModelElement element, DeployLogger logger) { Double queryTimeout = search.getQueryTimeout(); if (queryTimeout != null) { @@ -267,9 +269,8 @@ private void validateGroupSiblings(String cluster, StorageGroup group) { } private void validateThatGroupSiblingsAreUnique(String cluster, StorageGroup group) { - if (group == null) { - return; // Unit testing case - } + if (group == null) return; // Unit testing case + validateGroupSiblings(cluster, group); for (StorageGroup g : group.getSubgroups()) { validateThatGroupSiblingsAreUnique(cluster, g); @@ -301,7 +302,7 @@ else if (admin.multitenant()) { // system tests: cluster controllers on logserve } clusterControllers = admin.getClusterControllers(); } - else { // self-hosted: Put cluser controller on config servers or use explicit cluster controllers + else { // self-hosted: Put cluster controller on config servers or use explicit cluster controllers if (admin.getClusterControllers() == null) { var hosts = admin.getConfigservers().stream().map(s -> s.getHostResource()).collect(toList()); if (hosts.size() > 1) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java index fb4016f4cf47..ff9051879694 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java @@ -44,11 +44,11 @@ private Integer getThreads(ModelElement clusterElem) { private final Integer numThreads; private final ContentCluster cluster; - private final int reponseNumThreads; + private final int responseNumThreads; private final StorFilestorConfig.Response_sequencer_type.Enum responseSequencerType; private final double persistenceThrottlingWsDecrementFactor; private final double persistenceThrottlingWsBackoff; - private final int persistenceThrottingWindowSize; + private final int persistenceThrottlingWindowSize; private final double persistenceThrottlingWsResizeRate; private final boolean persistenceThrottlingOfMergeFeedOps; private final boolean useAsyncMessageHandlingOnSchedule; @@ -64,11 +64,11 @@ private static StorFilestorConfig.Response_sequencer_type.Enum convertResponseSe public FileStorProducer(ModelContext.FeatureFlags featureFlags, ContentCluster parent, Integer numThreads) { this.numThreads = numThreads; this.cluster = parent; - this.reponseNumThreads = featureFlags.defaultNumResponseThreads(); + this.responseNumThreads = featureFlags.defaultNumResponseThreads(); this.responseSequencerType = convertResponseSequencerType(featureFlags.responseSequencerType()); this.persistenceThrottlingWsDecrementFactor = featureFlags.persistenceThrottlingWsDecrementFactor(); this.persistenceThrottlingWsBackoff = featureFlags.persistenceThrottlingWsBackoff(); - this.persistenceThrottingWindowSize = featureFlags.persistenceThrottlingWindowSize(); + this.persistenceThrottlingWindowSize = featureFlags.persistenceThrottlingWindowSize(); this.persistenceThrottlingWsResizeRate = featureFlags.persistenceThrottlingWsResizeRate(); this.persistenceThrottlingOfMergeFeedOps = featureFlags.persistenceThrottlingOfMergeFeedOps(); this.useAsyncMessageHandlingOnSchedule = featureFlags.useAsyncMessageHandlingOnSchedule(); @@ -80,15 +80,15 @@ public void getConfig(StorFilestorConfig.Builder builder) { builder.num_threads(numThreads); } builder.enable_multibit_split_optimalization(cluster.getPersistence().enableMultiLevelSplitting()); - builder.num_response_threads(reponseNumThreads); + builder.num_response_threads(responseNumThreads); builder.response_sequencer_type(responseSequencerType); builder.use_async_message_handling_on_schedule(useAsyncMessageHandlingOnSchedule); var throttleBuilder = new StorFilestorConfig.Async_operation_throttler.Builder(); throttleBuilder.window_size_decrement_factor(persistenceThrottlingWsDecrementFactor); throttleBuilder.window_size_backoff(persistenceThrottlingWsBackoff); - if (persistenceThrottingWindowSize > 0) { - throttleBuilder.min_window_size(persistenceThrottingWindowSize); - throttleBuilder.max_window_size(persistenceThrottingWindowSize); + if (persistenceThrottlingWindowSize > 0) { + throttleBuilder.min_window_size(persistenceThrottlingWindowSize); + throttleBuilder.max_window_size(persistenceThrottlingWindowSize); } throttleBuilder.resize_rate(persistenceThrottlingWsResizeRate); throttleBuilder.throttle_individual_merge_feed_ops(persistenceThrottlingOfMergeFeedOps); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/IntegrityCheckerProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/IntegrityCheckerProducer.java index 588e7c55ab92..4f81bbf165f0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/IntegrityCheckerProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/IntegrityCheckerProducer.java @@ -16,9 +16,9 @@ protected IntegrityCheckerProducer build(ContentCluster cluster, ModelElement cl } } - private Integer startTime; - private Integer stopTime; - private String weeklyCycle; + private final Integer startTime; + private final Integer stopTime; + private final String weeklyCycle; IntegrityCheckerProducer(Integer startTime, Integer stopTime, String weeklyCycle) { this.startTime = startTime; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java index 2fca964a9955..e66f2c48f267 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java @@ -3,10 +3,8 @@ import com.yahoo.config.model.api.ModelContext; import com.yahoo.vespa.config.content.core.StorServerConfig; -import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; - -import java.util.Optional; +import com.yahoo.vespa.model.content.cluster.ContentCluster; /** * Serves config for stor-server for storage clusters (clusters of storage nodes). @@ -28,11 +26,10 @@ StorServerProducer build(ModelContext.Properties properties, ModelElement elemen } } - private String clusterName; + private final String clusterName; private Integer maxMergesPerNode; private Integer queueSize; - private Integer bucketDBStripeBits; - private StorServerConfig.Merge_throttling_policy.Type.Enum mergeThrottlingPolicyType; + private final StorServerConfig.Merge_throttling_policy.Type.Enum mergeThrottlingPolicyType; private StorServerProducer setMaxMergesPerNode(Integer value) { if (value != null) { @@ -46,10 +43,6 @@ private StorServerProducer setMaxQueueSize(Integer value) { } return this; } - private StorServerProducer setBucketDBStripeBits(Integer value) { - bucketDBStripeBits = value; - return this; - } private static StorServerConfig.Merge_throttling_policy.Type.Enum toThrottlePolicyType(String policyType) { try { @@ -80,9 +73,6 @@ public void getConfig(StorServerConfig.Builder builder) { if (queueSize != null) { builder.max_merge_queue_size(queueSize); } - if (bucketDBStripeBits != null) { - builder.content_node_bucket_db_stripe_bits(bucketDBStripeBits); - } // TODO set throttle policy params based on existing or separate flags builder.merge_throttling_policy(new StorServerConfig.Merge_throttling_policy.Builder().type(mergeThrottlingPolicyType)); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java index a3f32fcb44ba..da82a69842ab 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java @@ -30,7 +30,7 @@ public class StorageCluster extends AbstractConfigProducer { public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder { @Override - protected StorageCluster doBuild(DeployState deployState, AbstractConfigProducer ancestor, Element producerSpec) { + protected StorageCluster doBuild(DeployState deployState, AbstractConfigProducer ancestor, Element producerSpec) { final ModelElement clusterElem = new ModelElement(producerSpec); final ContentCluster cluster = (ContentCluster)ancestor; @@ -51,7 +51,7 @@ protected StorageCluster doBuild(DeployState deployState, AbstractConfigProducer private final StorVisitorProducer storVisitorProducer; private final PersistenceProducer persistenceProducer; - StorageCluster(AbstractConfigProducer parent, + StorageCluster(AbstractConfigProducer parent, String clusterName, FileStorProducer fileStorProducer, IntegrityCheckerProducer integrityCheckerProducer, From 95b8488c614ac13cac912cedce6af6c239126b11 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Tue, 30 Aug 2022 09:43:54 +0200 Subject: [PATCH 3/7] Move html generation of options to request handler --- .../core/FleetControllerOptions.java | 85 +----------------- .../status/LegacyIndexPageRequestHandler.java | 86 ++++++++++++++++++- 2 files changed, 85 insertions(+), 86 deletions(-) diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetControllerOptions.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetControllerOptions.java index 16e0e1c06735..750d44c8fbfb 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetControllerOptions.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetControllerOptions.java @@ -5,18 +5,13 @@ import com.yahoo.vdslib.distribution.ConfiguredNode; import com.yahoo.vdslib.distribution.Distribution; import com.yahoo.vdslib.state.NodeType; - -import java.text.DecimalFormat; -import java.text.DecimalFormatSymbols; import java.time.Duration; import java.util.Collection; import java.util.Collections; -import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; -import java.util.stream.Collectors; /** * This class represents all the options that can be set in the fleetcontroller. @@ -125,7 +120,7 @@ public class FleetControllerOptions implements Cloneable { // TODO: Get rid of this by always getting nodes by distribution.getNodes() public Set nodes; - private Duration maxDeferredTaskVersionWaitTime = Duration.ofSeconds(30); + public Duration maxDeferredTaskVersionWaitTime = Duration.ofSeconds(30); public boolean clusterHasGlobalDocumentTypes = false; @@ -175,82 +170,4 @@ public FleetControllerOptions clone() { } } - public static String splitZooKeeperAddress(String s) { - StringBuilder sb = new StringBuilder(); - while (true) { - int index = s.indexOf(','); - if (index > 0) { - sb.append(s.substring(0, index + 1)).append(' '); - s = s.substring(index+1); - } else { - break; - } - } - sb.append(s); - return sb.toString(); - } - - static DecimalFormat DecimalDot2 = new DecimalFormat("0.00", new DecimalFormatSymbols(Locale.ENGLISH)); - - public void writeHtmlState(StringBuilder sb) { - String slobrokspecs = ""; - for (int i=0; iCurrent config\n") - .append("

Fleet controller config id: ").append(fleetControllerConfigId == null ? null : fleetControllerConfigId.replaceAll("\n", "
\n")).append("

\n") - .append("

Slobrok config id: ").append(slobrokConfigId == null ? null : slobrokConfigId.replaceAll("\n", "
\n")).append("

\n") - .append("\n"); - - sb.append(""); - sb.append(""); - sb.append(""); - - sb.append(""); - sb.append(""); - sb.append(""); - sb.append(""); - String zooKeeperAddress = (zooKeeperServerAddress == null ? "Not using Zookeeper" : splitZooKeeperAddress(zooKeeperServerAddress)); - sb.append(""); - sb.append(""); - - sb.append(""); - sb.append(""); - sb.append(""); - sb.append(""); - - sb.append(""); - sb.append(""); - sb.append(""); - sb.append(""); - sb.append(""); - sb.append(""); - sb.append(""); - sb.append(""); - - sb.append(""); - sb.append(""); - sb.append(""); - sb.append(""); - sb.append(""); - sb.append(""); - - sb.append(""); - sb.append(""); - sb.append(""); - sb.append(""); - sb.append(""); - sb.append(""); - sb.append(""); - sb.append(""); - sb.append(""); - - sb.append("
PropertyValue
Cluster name").append(clusterName).append("
Fleet controller index").append(fleetControllerIndex).append("/").append(fleetControllerCount).append("
Number of fleetcontrollers gathering states from nodes").append(stateGatherCount).append("
Slobrok connection spec").append(slobrokspecs).append("
RPC port").append(rpcPort == 0 ? "Pick random available" : rpcPort).append("
HTTP port").append(httpPort == 0 ? "Pick random available" : httpPort).append("
Master cooldown period").append(RealTimer.printDuration(masterZooKeeperCooldownPeriod)).append("
Zookeeper server address").append(zooKeeperAddress).append("
Zookeeper session timeout").append(RealTimer.printDuration(zooKeeperSessionTimeout)).append("
Cycle wait time").append(cycleWaitTime).append(" ms
Minimum time before first clusterstate broadcast as master").append(RealTimer.printDuration(minTimeBeforeFirstSystemStateBroadcast)).append("
Minimum time between official cluster states").append(RealTimer.printDuration(minTimeBetweenNewSystemStates)).append("
Slobrok mirror backoff policy").append(slobrokBackOffPolicy == null ? "default" : "overridden").append("
Node state request timeout").append(RealTimer.printDuration(nodeStateRequestTimeoutMS)).append("
VDS 4.1 node state polling frequency").append(RealTimer.printDuration(statePollingFrequency)).append("
Maximum distributor transition time").append(RealTimer.printDuration(maxTransitionTime.get(NodeType.DISTRIBUTOR))).append("
Maximum storage transition time").append(RealTimer.printDuration(maxTransitionTime.get(NodeType.STORAGE))).append("
Maximum initialize without progress time").append(RealTimer.printDuration(maxInitProgressTime)).append("
Maximum premature crashes").append(maxPrematureCrashes).append("
Stable state time period").append(RealTimer.printDuration(stableStateTimePeriod)).append("
Slobrok disconnect grace period").append(RealTimer.printDuration(maxSlobrokDisconnectGracePeriod)).append("
Number of distributor nodes").append(nodes == null ? "Autodetect" : nodes.size()).append("
Number of storage nodes").append(nodes == null ? "Autodetect" : nodes.size()).append("
Minimum distributor nodes being up for cluster to be up").append(minDistributorNodesUp).append("
Minimum storage nodes being up for cluster to be up").append(minStorageNodesUp).append("
Minimum percentage of distributor nodes being up for cluster to be up").append(DecimalDot2.format(100 * minRatioOfDistributorNodesUp)).append(" %
Minimum percentage of storage nodes being up for cluster to be up").append(DecimalDot2.format(100 * minRatioOfStorageNodesUp)).append(" %
Show local cluster state changes").append(showLocalSystemStatesInEventLog).append("
Maximum event log size").append(eventLogMaxSize).append("
Maximum node event log size").append(eventNodeLogMaxSize).append("
Wanted distribution bits").append(distributionBits).append("
Max deferred task version wait time").append(maxDeferredTaskVersionWaitTime.toMillis()).append("ms
Cluster has global document types configured").append(clusterHasGlobalDocumentTypes).append("
Enable 2-phase cluster state activation protocol").append(enableTwoPhaseClusterStateActivation).append("
Cluster auto feed block on resource exhaustion enabled") - .append(clusterFeedBlockEnabled).append("
Feed block limits") - .append(clusterFeedBlockLimit.entrySet().stream() - .map(kv -> String.format("%s: %.2f%%", kv.getKey(), kv.getValue() * 100.0)) - .collect(Collectors.joining("
"))).append("
"); - } - } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java index 4f20b3d0cdca..d0aeb639ea26 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java @@ -20,16 +20,22 @@ import com.yahoo.vespa.clustercontroller.core.status.statuspage.StatusPageResponse; import com.yahoo.vespa.clustercontroller.core.status.statuspage.StatusPageServer; import com.yahoo.vespa.clustercontroller.core.status.statuspage.VdsClusterHtmlRenderer; +import java.text.DecimalFormat; +import java.text.DecimalFormatSymbols; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.TimeZone; import java.util.TreeMap; +import java.util.stream.Collectors; /** * @author Haakon Humberset */ public class LegacyIndexPageRequestHandler implements StatusPageServer.RequestHandler { + private static final DecimalFormat DecimalDot2 = new DecimalFormat("0.00", new DecimalFormatSymbols(Locale.ENGLISH)); + private final Timer timer; private final ContentCluster cluster; private final MasterElectionHandler masterElectionHandler; @@ -78,12 +84,12 @@ public StatusPageResponse handle(StatusPageServer.HttpRequest request) { writeHtmlState(stateVersionTracker, content); } else { // Overview of current config - options.writeHtmlState(content); + writeHtmlState(content, options); } // State of master election masterElectionHandler.writeHtmlState(content, options.stateGatherCount); // Overview of current config - options.writeHtmlState(content); + writeHtmlState(content, options); // Event log eventLog.writeHtmlState(content, null); response.writeHtmlFooter(content, ""); @@ -200,4 +206,80 @@ private void storeNodeInfo(ContentCluster cluster, int nodeIndex, NodeType nodeT nodeInfoByIndex.put(nodeIndex, nodeInfo); } + public void writeHtmlState(StringBuilder sb, FleetControllerOptions options) { + String slobrokspecs = ""; + for (int i = 0; i < options.slobrokConnectionSpecs.length; ++i) { + if (i != 0) slobrokspecs += "
"; + slobrokspecs += options.slobrokConnectionSpecs[i]; + } + sb.append("

Current config

\n") + .append("

Fleet controller config id: ").append(options.fleetControllerConfigId == null ? null : options.fleetControllerConfigId.replaceAll("\n", "
\n")).append("

\n") + .append("

Slobrok config id: ").append(options.slobrokConfigId == null ? null : options.slobrokConfigId.replaceAll("\n", "
\n")).append("

\n") + .append("\n"); + + sb.append(""); + sb.append(""); + sb.append(""); + + sb.append(""); + sb.append(""); + sb.append(""); + sb.append(""); + String zooKeeperAddress = (options.zooKeeperServerAddress == null ? "Not using Zookeeper" : splitZooKeeperAddress(options.zooKeeperServerAddress)); + sb.append(""); + sb.append(""); + + sb.append(""); + sb.append(""); + sb.append(""); + sb.append(""); + + sb.append(""); + sb.append(""); + sb.append(""); + sb.append(""); + sb.append(""); + sb.append(""); + sb.append(""); + sb.append(""); + + sb.append(""); + sb.append(""); + sb.append(""); + sb.append(""); + sb.append(""); + sb.append(""); + + sb.append(""); + sb.append(""); + sb.append(""); + sb.append(""); + sb.append(""); + sb.append(""); + sb.append(""); + sb.append(""); + sb.append(""); + + sb.append("
PropertyValue
Cluster name").append(options.clusterName).append("
Fleet controller index").append(options.fleetControllerIndex).append("/").append(options.fleetControllerCount).append("
Number of fleetcontrollers gathering states from nodes").append(options.stateGatherCount).append("
Slobrok connection spec").append(slobrokspecs).append("
RPC port").append(options.rpcPort == 0 ? "Pick random available" : options.rpcPort).append("
HTTP port").append(options.httpPort == 0 ? "Pick random available" : options.httpPort).append("
Master cooldown period").append(RealTimer.printDuration(options.masterZooKeeperCooldownPeriod)).append("
Zookeeper server address").append(zooKeeperAddress).append("
Zookeeper session timeout").append(RealTimer.printDuration(options.zooKeeperSessionTimeout)).append("
Cycle wait time").append(options.cycleWaitTime).append(" ms
Minimum time before first clusterstate broadcast as master").append(RealTimer.printDuration(options.minTimeBeforeFirstSystemStateBroadcast)).append("
Minimum time between official cluster states").append(RealTimer.printDuration(options.minTimeBetweenNewSystemStates)).append("
Slobrok mirror backoff policy").append(options.slobrokBackOffPolicy == null ? "default" : "overridden").append("
Node state request timeout").append(RealTimer.printDuration(options.nodeStateRequestTimeoutMS)).append("
VDS 4.1 node state polling frequency").append(RealTimer.printDuration(options.statePollingFrequency)).append("
Maximum distributor transition time").append(RealTimer.printDuration(options.maxTransitionTime.get(NodeType.DISTRIBUTOR))).append("
Maximum storage transition time").append(RealTimer.printDuration(options.maxTransitionTime.get(NodeType.STORAGE))).append("
Maximum initialize without progress time").append(RealTimer.printDuration(options.maxInitProgressTime)).append("
Maximum premature crashes").append(options.maxPrematureCrashes).append("
Stable state time period").append(RealTimer.printDuration(options.stableStateTimePeriod)).append("
Slobrok disconnect grace period").append(RealTimer.printDuration(options.maxSlobrokDisconnectGracePeriod)).append("
Number of distributor nodes").append(options.nodes == null ? "Autodetect" : options.nodes.size()).append("
Number of storage nodes").append(options.nodes == null ? "Autodetect" : options.nodes.size()).append("
Minimum distributor nodes being up for cluster to be up").append(options.minDistributorNodesUp).append("
Minimum storage nodes being up for cluster to be up").append(options.minStorageNodesUp).append("
Minimum percentage of distributor nodes being up for cluster to be up").append(DecimalDot2.format(100 * options.minRatioOfDistributorNodesUp)).append(" %
Minimum percentage of storage nodes being up for cluster to be up").append(DecimalDot2.format(100 * options.minRatioOfStorageNodesUp)).append(" %
Show local cluster state changes").append(options.showLocalSystemStatesInEventLog).append("
Maximum event log size").append(options.eventLogMaxSize).append("
Maximum node event log size").append(options.eventNodeLogMaxSize).append("
Wanted distribution bits").append(options.distributionBits).append("
Max deferred task version wait time").append(options.maxDeferredTaskVersionWaitTime.toMillis()).append("ms
Cluster has global document types configured").append(options.clusterHasGlobalDocumentTypes).append("
Enable 2-phase cluster state activation protocol").append(options.enableTwoPhaseClusterStateActivation).append("
Cluster auto feed block on resource exhaustion enabled") + .append(options.clusterFeedBlockEnabled).append("
Feed block limits") + .append(options.clusterFeedBlockLimit.entrySet().stream() + .map(kv -> String.format("%s: %.2f%%", kv.getKey(), kv.getValue() * 100.0)) + .collect(Collectors.joining("
"))).append("
"); + } + + private static String splitZooKeeperAddress(String s) { + StringBuilder sb = new StringBuilder(); + while (true) { + int index = s.indexOf(','); + if (index > 0) { + sb.append(s.substring(0, index + 1)).append(' '); + s = s.substring(index+1); + } else { + break; + } + } + sb.append(s); + return sb.toString(); + } + } From 2d3deea19851a168c19046b74517a82ef1859449 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Tue, 30 Aug 2022 10:04:13 +0200 Subject: [PATCH 4/7] Split out method --- .../core/MasterElectionHandler.java | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MasterElectionHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MasterElectionHandler.java index 637aca16ee7c..3c4959ea5d3e 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MasterElectionHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MasterElectionHandler.java @@ -77,16 +77,15 @@ public boolean inMasterMoratorium() { @Override public Integer getMaster() { - // If too few followers there can be no master - if (2 * followers <= totalCount) { + if (tooFewFollowersToHaveAMaster()) { return null; } - // If all are following master candidate, it is master if it exists. + // If all are following master candidate, it is master if it exists. if (followers == totalCount) { return masterCandidate; } - // If not all are following we only accept master candidate if old master - // disappeared sufficient time ago + // If not all are following we only accept master candidate if old master + // disappeared sufficient time ago if (masterGoneFromZooKeeperTime + masterZooKeeperCooldownPeriod > timer.getCurrentTimeInMillis()) { return null; } @@ -97,8 +96,7 @@ public String getMasterReason() { if (masterCandidate == null) { return "There is currently no master candidate."; } - // If too few followers there can be no master - if (2 * followers <= totalCount) { + if (tooFewFollowersToHaveAMaster()) { return "More than half of the nodes must agree for there to be a master. Only " + followers + " of " + totalCount + " nodes agree on current master candidate (" + masterCandidate + ")."; } @@ -118,6 +116,10 @@ public String getMasterReason() { return followers + " of " + totalCount + " nodes agree " + masterCandidate + " is master."; } + private boolean tooFewFollowersToHaveAMaster() { + return 2 * followers <= totalCount; + } + public boolean isAmongNthFirst(int first) { return (nextInLineCount < first); } public boolean watchMasterElection(DatabaseHandler database, @@ -256,7 +258,7 @@ public void writeHtmlState(StringBuilder sb, int stateGatherCount) { if (master.intValue() == index) sb.append(" (This node)"); sb.append("

"); } else { - if (2 * followers <= totalCount) { + if (tooFewFollowersToHaveAMaster()) { sb.append("

There is currently no master. Less than half the fleet controllers (") .append(followers).append(") are following master candidate ").append(masterCandidate) .append(".

"); @@ -267,7 +269,7 @@ public void writeHtmlState(StringBuilder sb, int stateGatherCount) { .append(" before electing new master unless all possible master candidates are online.

"); } } - if ((master == null || master.intValue() != index) && nextInLineCount < stateGatherCount) { + if ((master == null || master != index) && nextInLineCount < stateGatherCount) { sb.append("

As we are number ").append(nextInLineCount) .append(" in line for taking over as master, we're gathering state from nodes.

"); sb.append("

As we are not the master, we don't know about nodes current system state" From 302251d45e1dc4f63580dba7d55b455f4d86d5b1 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Tue, 30 Aug 2022 11:14:26 +0200 Subject: [PATCH 5/7] Remove unused method --- .../com/yahoo/vespa/clustercontroller/core/FleetController.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java index b3dd0d738e43..a1e213fc1f95 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java @@ -1216,8 +1216,6 @@ public void waitForNodesInSlobrok(int distNodeCount, int storNodeCount, Duration public ContentCluster getCluster() { return cluster; } - public List getNodeEvents(Node n) { return eventLog.getNodeEvents(n); } - public EventLog getEventLog() { return eventLog; } From 3c810a80546353e7ef011a397bdb294e2bdd7888 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Tue, 30 Aug 2022 11:23:59 +0200 Subject: [PATCH 6/7] Add some convenience methods, reindent --- .../core/MasterElectionHandler.java | 24 ++++++++++--------- .../status/LegacyIndexPageRequestHandler.java | 1 - 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MasterElectionHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MasterElectionHandler.java index 3c4959ea5d3e..dfc328346bb3 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MasterElectionHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MasterElectionHandler.java @@ -35,11 +35,9 @@ public MasterElectionHandler(FleetControllerContext context, int index, int tota this.index = index; this.totalCount = totalCount; this.nextInLineCount = Integer.MAX_VALUE; - // Only a given set of nodes can ever become master - if (index > (totalCount - 1) / 2) { + if (cannotBecomeMaster()) context.log(logger, Level.FINE, () -> "We can never become master and will always stay a follower."); - } - // Tag current time as when we have not seen any other master. Make sure we're not taking over at once for master that is on the way down + // Tag current time as when we have not seen any other master. Make sure we're not taking over at once for master that is on the way down masterGoneFromZooKeeperTime = timer.getCurrentTimeInMillis(); } @@ -133,8 +131,8 @@ public boolean watchMasterElection(DatabaseHandler database, } return false; // Nothing have happened since last time. } - // Move next data to temporary, such that we don't need to keep lock, and such that we don't retry - // if we happen to fail processing the data. + // Move next data to temporary, such that we don't need to keep lock, and such that we don't retry + // if we happen to fail processing the data. Map state; context.log(logger, Level.INFO, "Handling new master election, as we have received " + nextMasterData.size() + " entries"); synchronized (monitor) { @@ -186,8 +184,7 @@ public boolean watchMasterElection(DatabaseHandler database, database.setMasterVote(dbContext, first.getKey()); } } - // Only a given set of nodes can ever become master - if (index <= (totalCount - 1) / 2) { + if (canBecomeMaster()) { int ourPosition = 0; for (Map.Entry entry : state.entrySet()) { if (entry.getKey() != index) { @@ -207,6 +204,11 @@ public boolean watchMasterElection(DatabaseHandler database, return true; } + // Only a given set of nodes can ever become master + private boolean canBecomeMaster() {return index <= (totalCount - 1) / 2;} + + private boolean cannotBecomeMaster() {return ! canBecomeMaster();} + private static String toString(Map data) { StringBuilder sb = new StringBuilder(); for (Map.Entry entry : data.entrySet()) { @@ -255,7 +257,7 @@ public void writeHtmlState(StringBuilder sb, int stateGatherCount) { Integer master = getMaster(); if (master != null) { sb.append("

Current cluster controller master is node " + master + "."); - if (master.intValue() == index) sb.append(" (This node)"); + if (master == index) sb.append(" (This node)"); sb.append("

"); } else { if (tooFewFollowersToHaveAMaster()) { @@ -276,12 +278,12 @@ public void writeHtmlState(StringBuilder sb, int stateGatherCount) { + " or wanted states, so some statistics below may be stale. Look at status page on master " + "for updated data.

"); } - if (index * 2 > totalCount) { + if (cannotBecomeMaster()) { sb.append("

As lowest index fleet controller is prioritized to become master, and more than half " + "of the fleet controllers need to be available to select a master, we can never become master.

"); } - // Debug data + // Debug data sb.append("

Master election handler internal state:") .append("
Index: " + index) .append("
Fleet controller count: " + totalCount) diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java index d0aeb639ea26..0611d754b698 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/LegacyIndexPageRequestHandler.java @@ -75,7 +75,6 @@ public StatusPageResponse handle(StatusPageServer.HttpRequest request) { .append(" | Event log") .append(" ]

\n"); content.append(""); - //content.append(""); content.append("
UTC time when creating this page:").append(RealTimer.printDateNoMilliSeconds(currentTime, tz)).append("
Fleetcontroller version:" + Vtag.V_TAG_PKG + "
Cluster controller uptime:" + RealTimer.printDuration(currentTime - startedTime) + "
"); if (masterElectionHandler.isAmongNthFirst(options.stateGatherCount)) { // Table overview of all the nodes From 7455d58e8eeae70e8909964b9584a4e81e501bd4 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Tue, 30 Aug 2022 16:36:27 +0200 Subject: [PATCH 7/7] Revert unintended check-in --- ...troller.core.MasterElectionTest-output.txt | 859 ------------------ 1 file changed, 859 deletions(-) delete mode 100644 clustercontroller-core/com.yahoo.vespa.clustercontroller.core.MasterElectionTest-output.txt diff --git a/clustercontroller-core/com.yahoo.vespa.clustercontroller.core.MasterElectionTest-output.txt b/clustercontroller-core/com.yahoo.vespa.clustercontroller.core.MasterElectionTest-output.txt deleted file mode 100644 index c4b87137f47e..000000000000 --- a/clustercontroller-core/com.yahoo.vespa.clustercontroller.core.MasterElectionTest-output.txt +++ /dev/null @@ -1,859 +0,0 @@ -STARTING TEST: MasterElectionTest::testMasterElection -1661243505.047218 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info STARTING TEST: MasterElectionTest::testMasterElection() -1661243505.048231 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.persistence.FileTxnSnapLog info zookeeper.snapshot.trust.empty : false -1661243505.049057 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.watch.WatchManagerFactory info Using org.apache.zookeeper.server.watch.WatchManager as watch manager -1661243505.049649 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.watch.WatchManagerFactory info Using org.apache.zookeeper.server.watch.WatchManager as watch manager -1661243505.050132 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ZKDatabase info zookeeper.snapshotSizeFactor = 0.33 -1661243505.050547 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ZKDatabase info zookeeper.commitLogCount=500 -1661243505.050983 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ZooKeeperServer info minSessionTimeout set to 4000 -1661243505.051315 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ZooKeeperServer info maxSessionTimeout set to 40000 -1661243505.051735 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ResponseCache info getData response cache size is initialized with value 400. -1661243505.052229 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ResponseCache info getChildren response cache size is initialized with value 400. -1661243505.052589 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.util.RequestPathMetricsCollector info zookeeper.pathStats.slotCapacity = 60 -1661243505.052878 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.util.RequestPathMetricsCollector info zookeeper.pathStats.slotDuration = 15 -1661243505.053195 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.util.RequestPathMetricsCollector info zookeeper.pathStats.maxDepth = 6 -1661243505.053450 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.util.RequestPathMetricsCollector info zookeeper.pathStats.initialDelay = 5 -1661243505.053698 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.util.RequestPathMetricsCollector info zookeeper.pathStats.delay = 5 -1661243505.053951 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.util.RequestPathMetricsCollector info zookeeper.pathStats.enabled = false -1661243505.054309 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ZooKeeperServer info The max bytes for all large requests are set to 104857600 -1661243505.054597 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ZooKeeperServer info The large request threshold is set to -1 -1661243505.054858 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.AuthenticationHelper info zookeeper.enforce.auth.enabled = false -1661243505.055152 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.AuthenticationHelper info zookeeper.enforce.auth.schemes = [] -1661243505.055426 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ZooKeeperServer info Created server with tickTime 2000 minSessionTimeout 4000 maxSessionTimeout 40000 clientPortListenBacklog -1 datadir /Users/musum/git/vespa/clustercontroller-core/target/surefire-reports/test_fltctrl_zk3840676619239160054sdir/version-2 snapdir /Users/musum/git/vespa/clustercontroller-core/target/surefire-reports/test_fltctrl_zk3840676619239160054sdir/version-2 -1661243505.056316 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ServerCnxnFactory warning maxCnxns is not configured, using default value 0. -1661243505.056699 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.NIOServerCnxnFactory info Configuring NIO connection handler with 10s sessionless connection timeout, 2 selector thread(s), 32 worker threads, and 64 kB direct buffers. -1661243505.057438 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.NIOServerCnxnFactory info binding to port 0.0.0.0/0.0.0.0:0 -1661243505.058607 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.persistence.FileTxnSnapLog info Snapshotting: 0x0 to /Users/musum/git/vespa/clustercontroller-core/target/surefire-reports/test_fltctrl_zk3840676619239160054sdir/version-2/snapshot.0 -1661243505.076253 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ZKDatabase info Snapshot loaded in 17 ms, highest zxid is 0x0, digest is 1371985504 -1661243505.076848 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.persistence.FileTxnSnapLog info Snapshotting: 0x0 to /Users/musum/git/vespa/clustercontroller-core/target/surefire-reports/test_fltctrl_zk3840676619239160054sdir/version-2/snapshot.0 -1661243505.078817 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ZooKeeperServer info Snapshot taken in 2 ms -1661243505.080001 localhost 75603/455 - fleetcontroller.org.apache.zookeeper.server.PrepRequestProcessor info PrepRequestProcessor (sid:0) started, reconfigEnabled=false -1661243505.082854 localhost 75603/1 - fleetcontroller.com.yahoo.jrt.slobrok.api.Mirror info no location brokers available, retrying: [] (in 0.5 seconds) -1661243505.083482 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 0 for cluster 'mycluster': Created -1661243505.084707 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Clearing master data as we lost connection on node 0 -1661243505.091045 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.rpc.RpcServer info Registering tcp/localhost:51707 with slobrok at ( tcp/localhost:51705 ) -1661243505.092054 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 0 for cluster 'mycluster': Starting tick loop -1661243505.092619 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 0 for cluster 'mycluster': Setting up new ZooKeeper session at localhost:51704 -1661243505.093164 localhost 75603/1 - fleetcontroller.com.yahoo.jrt.slobrok.api.Mirror info no location brokers available, retrying: [] (in 0.5 seconds) -1661243505.093445 localhost 75603/470 - fleetcontroller.org.apache.zookeeper.ZooKeeper info Initiating client connection, connectString=localhost:51704 sessionTimeout=30000 watcher=com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase$ZooKeeperWatcher@58f2bbcb -1661243505.093698 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 1 for cluster 'mycluster': Created -1661243505.094675 localhost 75603/470 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocket info jute.maxbuffer value is 1048575 Bytes -1661243505.095199 localhost 75603/470 - fleetcontroller.org.apache.zookeeper.ClientCnxn info zookeeper.request.timeout value is 0. feature enabled=false -1661243505.096099 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Clearing master data as we lost connection on node 1 -1661243505.096492 localhost 75603/477 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Opening socket connection to server localhost/[0:0:0:0:0:0:0:1]:51704. -1661243505.096923 localhost 75603/477 - fleetcontroller.org.apache.zookeeper.ClientCnxn info SASL config status: Will not attempt to authenticate using SASL (unknown error) -1661243505.101019 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.rpc.RpcServer info Registering tcp/localhost:51710 with slobrok at ( tcp/localhost:51705 ) -1661243505.101825 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 1 for cluster 'mycluster': Starting tick loop -1661243505.102322 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 1 for cluster 'mycluster': Setting up new ZooKeeper session at localhost:51704 -1661243505.103030 localhost 75603/1 - fleetcontroller.com.yahoo.jrt.slobrok.api.Mirror info no location brokers available, retrying: [] (in 0.5 seconds) -1661243505.103046 localhost 75603/485 - fleetcontroller.org.apache.zookeeper.ZooKeeper info Initiating client connection, connectString=localhost:51704 sessionTimeout=30000 watcher=com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase$ZooKeeperWatcher@173742bc -1661243505.103540 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 2 for cluster 'mycluster': Created -1661243505.103936 localhost 75603/485 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocket info jute.maxbuffer value is 1048575 Bytes -1661243505.104465 localhost 75603/485 - fleetcontroller.org.apache.zookeeper.ClientCnxn info zookeeper.request.timeout value is 0. feature enabled=false -1661243505.104824 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Clearing master data as we lost connection on node 2 -1661243505.105106 localhost 75603/492 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Opening socket connection to server localhost/[0:0:0:0:0:0:0:1]:51704. -1661243505.105512 localhost 75603/492 - fleetcontroller.org.apache.zookeeper.ClientCnxn info SASL config status: Will not attempt to authenticate using SASL (unknown error) -1661243505.106087 localhost 75603/484 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Socket connection established, initiating session, client: /[0:0:0:0:0:0:0:1]:51711, server: localhost/[0:0:0:0:0:0:0:1]:51704 -1661243505.106185 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.rpc.RpcServer info Registering tcp/localhost:51712 with slobrok at ( tcp/localhost:51705 ) -1661243505.106570 localhost 75603/484 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is connected: [id: 0xda4181f3, L:/[0:0:0:0:0:0:0:1]:51711 - R:localhost/[0:0:0:0:0:0:0:1]:51704] -1661243505.107208 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 2 for cluster 'mycluster': Starting tick loop -1661243505.107131 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Entering waitForMaster -1661243505.107811 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 2 for cluster 'mycluster': Setting up new ZooKeeper session at localhost:51704 -1661243505.108653 localhost 75603/500 - fleetcontroller.org.apache.zookeeper.ZooKeeper info Initiating client connection, connectString=localhost:51704 sessionTimeout=30000 watcher=com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase$ZooKeeperWatcher@6f891fab -1661243505.109231 localhost 75603/500 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocket info jute.maxbuffer value is 1048575 Bytes -1661243505.109635 localhost 75603/500 - fleetcontroller.org.apache.zookeeper.ClientCnxn info zookeeper.request.timeout value is 0. feature enabled=false -1661243505.110335 localhost 75603/502 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Opening socket connection to server localhost/[0:0:0:0:0:0:0:1]:51704. -1661243505.110731 localhost 75603/502 - fleetcontroller.org.apache.zookeeper.ClientCnxn info SASL config status: Will not attempt to authenticate using SASL (unknown error) -1661243505.121742 localhost 75603/499 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Socket connection established, initiating session, client: /[0:0:0:0:0:0:0:1]:51715, server: localhost/[0:0:0:0:0:0:0:1]:51704 -1661243505.122133 localhost 75603/454 - fleetcontroller.org.apache.zookeeper.server.persistence.FileTxnLog info Creating new log file: log.1 -1661243505.122262 localhost 75603/499 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is connected: [id: 0x35935cb2, L:/[0:0:0:0:0:0:0:1]:51715 - R:localhost/[0:0:0:0:0:0:0:1]:51704] -1661243505.125798 localhost 75603/504 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Socket connection established, initiating session, client: /[0:0:0:0:0:0:0:1]:51717, server: localhost/[0:0:0:0:0:0:0:1]:51704 -1661243505.126370 localhost 75603/504 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is connected: [id: 0x42017013, L:/[0:0:0:0:0:0:0:1]:51717 - R:localhost/[0:0:0:0:0:0:0:1]:51704] -1661243505.146831 localhost 75603/484 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Session establishment complete on server localhost/[0:0:0:0:0:0:0:1]:51704, session id = 0x1001abb14b10000, negotiated timeout = 30000 -1661243505.147373 localhost 75603/478 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 0 for cluster 'mycluster': Connection to zookeeper server established. Refetching master data -1661243505.165823 localhost 75603/504 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Session establishment complete on server localhost/[0:0:0:0:0:0:0:1]:51704, session id = 0x1001abb14b10002, negotiated timeout = 30000 -1661243505.165836 localhost 75603/499 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Session establishment complete on server localhost/[0:0:0:0:0:0:0:1]:51704, session id = 0x1001abb14b10001, negotiated timeout = 30000 -1661243505.166404 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 2 for cluster 'mycluster': Connection to zookeeper server established. Refetching master data -1661243505.166765 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 1 for cluster 'mycluster': Connection to zookeeper server established. Refetching master data -1661243505.402281 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 2 for cluster 'mycluster': Creating ephemeral master vote node with vote to self. -1661243505.402317 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 0 for cluster 'mycluster': Creating ephemeral master vote node with vote to self. -1661243505.403166 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 1 for cluster 'mycluster': Creating ephemeral master vote node with vote to self. -1661243505.422252 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 2 for cluster 'mycluster': Done setting up new ZooKeeper session at localhost:51704 -1661243505.425580 localhost 75603/479 - fleetcontroller.com.yahoo.jrt.slobrok.api.Mirror info successfully connected to location broker tcp/localhost:51705 (mirror initialized with 0 service names) -1661243505.425607 localhost 75603/494 - fleetcontroller.com.yahoo.jrt.slobrok.api.Mirror info successfully connected to location broker tcp/localhost:51705 (mirror initialized with 0 service names) -1661243505.425580 localhost 75603/465 - fleetcontroller.com.yahoo.jrt.slobrok.api.Mirror info successfully connected to location broker tcp/localhost:51705 (mirror initialized with 0 service names) -1661243505.440443 localhost 75603/468 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51707] registering storage/cluster.mycluster/fleetcontroller/0 with location broker tcp/localhost:51705 completed successfully -1661243505.440557 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 0 for cluster 'mycluster': Done setting up new ZooKeeper session at localhost:51704 -1661243505.444275 localhost 75603/482 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51710] registering storage/cluster.mycluster/fleetcontroller/1 with location broker tcp/localhost:51705 completed successfully -1661243505.444379 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got node list response from /vespa/fleetcontroller/mycluster/indexes version 0 with 3 nodes -1661243505.444359 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 1 for cluster 'mycluster': Done setting up new ZooKeeper session at localhost:51704 -1661243505.444275 localhost 75603/497 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51712] registering storage/cluster.mycluster/fleetcontroller/2 with location broker tcp/localhost:51705 completed successfully -1661243505.459490 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 2 for cluster 'mycluster': Stored new vote in ephemeral node. 2 -> 2 -1661243505.459493 localhost 75603/478 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got node list response from /vespa/fleetcontroller/mycluster/indexes version 0 with 3 nodes -1661243505.480063 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 0 for cluster 'mycluster': Stored new vote in ephemeral node. 0 -> 0 -1661243505.480951 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Node 0 is not master yet, sleeping more -1661243505.483964 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got node list response from /vespa/fleetcontroller/mycluster/indexes version 0 with 3 nodes -1661243505.484004 localhost 75603/478 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/0 with code 0 and data 0 -1661243505.484002 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 1 for cluster 'mycluster': Stored new vote in ephemeral node. 1 -> 1 -1661243505.484002 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/0 with code 0 and data 0 -1661243505.485008 localhost 75603/478 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote from fleetcontroller 0. Altering vote from null to 0. -1661243505.486023 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 0. Altering vote from null to 0. -1661243505.486580 localhost 75603/478 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 1 -1661243505.487037 localhost 75603/478 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote from fleetcontroller 1. Altering vote from null to 1. -1661243505.486697 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 1 -1661243505.487479 localhost 75603/478 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 2 -1661243505.488114 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 1. Altering vote from null to 1. -1661243505.488926 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 2 -1661243505.489344 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 2. Altering vote from null to 2. -1661243505.488509 localhost 75603/478 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote from fleetcontroller 2. Altering vote from null to 2. -1661243505.489793 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got new fleet data with 3 entries: {0=0, 1=1, 2=2} -1661243505.490119 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Got new fleet data with 3 entries: {0=0, 1=1, 2=2} -1661243505.490394 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Handling new master election, as we have received 3 entries -1661243505.490880 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Handling new master election, as we have received 3 entries -1661243505.491621 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/0 with code 0 and data 0 -1661243505.491428 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got master election state data(0 -> 0, 1 -> 1, 2 -> 2). -1661243505.492452 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote from fleetcontroller 0. Altering vote from null to 0. -1661243505.492040 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Got master election state data(0 -> 0, 1 -> 1, 2 -> 2). -1661243505.493323 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 1 -1661243505.494186 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote from fleetcontroller 1. Altering vote from null to 1. -1661243505.494616 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 2 -1661243505.495153 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote from fleetcontroller 2. Altering vote from null to 2. -1661243505.495547 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Currently too few followers for cluster controller candidate 0. No current master. (1/3 followers) -1661243505.495510 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Currently too few followers for cluster controller candidate 0. No current master. (1/3 followers) -1661243505.496407 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Altering our state to follow new fleet controller master candidate 0 -1661243505.495899 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got new fleet data with 3 entries: {0=0, 1=1, 2=2} -1661243505.497375 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Handling new master election, as we have received 3 entries -1661243505.496843 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type MASTER_ELECTION @1661243505496: This node just became node state gatherer as we are fleetcontroller master candidate. -1661243505.497804 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got master election state data(0 -> 0, 1 -> 1, 2 -> 2). -1661243505.498632 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Currently too few followers for cluster controller candidate 0. No current master. (1/3 followers) -1661243505.499225 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Altering our state to follow new fleet controller master candidate 0 -1661243505.512918 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 0 for cluster 'mycluster': Read cluster state version 0 from ZooKeeper (znode version 0) -1661243505.527492 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 2 for cluster 'mycluster': Stored new vote in ephemeral node. 2 -> 0 -1661243505.527648 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Altered data in node /vespa/fleetcontroller/mycluster/indexes/2. Requesting new vote -1661243505.527629 localhost 75603/478 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Altered data in node /vespa/fleetcontroller/mycluster/indexes/2. Requesting new vote -1661243505.527629 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Altered data in node /vespa/fleetcontroller/mycluster/indexes/2. Requesting new vote -1661243505.530796 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Altered data in node /vespa/fleetcontroller/mycluster/indexes/1. Requesting new vote -1661243505.530829 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 1 for cluster 'mycluster': Stored new vote in ephemeral node. 1 -> 0 -1661243505.530803 localhost 75603/478 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Altered data in node /vespa/fleetcontroller/mycluster/indexes/1. Requesting new vote -1661243505.530800 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Altered data in node /vespa/fleetcontroller/mycluster/indexes/1. Requesting new vote -1661243505.532218 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type MASTER_ELECTION @1661243505532: This node just became node state gatherer as we are fleetcontroller master candidate. -1661243505.553150 localhost 75603/478 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 0 -1661243505.553150 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 0 -1661243505.553150 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 0 -1661243505.554448 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 2. Altering vote from null to 0. -1661243505.553882 localhost 75603/478 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote from fleetcontroller 2. Altering vote from null to 0. -1661243505.554943 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote from fleetcontroller 2. Altering vote from null to 0. -1661243505.569127 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 0 -1661243505.569217 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 1 for cluster 'mycluster': Read cluster state version 0 from ZooKeeper (znode version 0) -1661243505.569133 localhost 75603/478 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 0 -1661243505.569127 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 0 -1661243505.570388 localhost 75603/478 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote from fleetcontroller 1. Altering vote from null to 0. -1661243505.569680 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote from fleetcontroller 1. Altering vote from null to 0. -1661243505.571413 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Got new fleet data with 3 entries: {0=0, 1=0, 2=0} -1661243505.570950 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 1. Altering vote from null to 0. -1661243505.572000 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Handling new master election, as we have received 3 entries -1661243505.571655 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got new fleet data with 3 entries: {0=0, 1=0, 2=0} -1661243505.572696 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Got master election state data(0 -> 0, 1 -> 0, 2 -> 0). -1661243505.572437 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got new fleet data with 3 entries: {0=0, 1=0, 2=0} -1661243505.573504 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': 0 is newly elected master -1661243505.573137 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Handling new master election, as we have received 3 entries -1661243505.573814 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Handling new master election, as we have received 3 entries -1661243505.574488 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got master election state data(0 -> 0, 1 -> 0, 2 -> 0). -1661243505.574847 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got master election state data(0 -> 0, 1 -> 0, 2 -> 0). -1661243505.575319 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': 0 is newly elected master -1661243505.575652 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': 0 is newly elected master -1661243505.604886 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 0 for cluster 'mycluster': Read cluster state version 0 from ZooKeeper (znode version 0) -1661243505.663509 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 0 for cluster 'mycluster': Loaded previous cluster state bundle from ZooKeeper: ClusterStateBundle('') -1661243505.663972 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type MASTER_ELECTION @1661243505663: This node just became fleetcontroller master. Bumped version to 1 to be in line. -1661243505.664600 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type SYSTEMSTATE @1661243505664: Too few storage nodes available in cluster. Setting cluster state down -1661243505.665034 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type SYSTEMSTATE @1661243505664: New cluster state version 2. Change from last: version: 0 => 2, cluster: Up => Down, storage: [0: description: => Node not seen in slobrok., 1: description: => Node not seen in slobrok., 2: description: => Node not seen in slobrok., 3: description: => Node not seen in slobrok., 4: description: => Node not seen in slobrok., 5: description: => Node not seen in slobrok., 6: description: => Node not seen in slobrok., 7: description: => Node not seen in slobrok., 8: description: => Node not seen in slobrok., 9: description: => Node not seen in slobrok.], distributor: [0: description: => Node not seen in slobrok., 1: description: => Node not seen in slobrok., 2: description: => Node not seen in slobrok., 3: description: => Node not seen in slobrok., 4: description: => Node not seen in slobrok., 5: description: => Node not seen in slobrok., 6: description: => Node not seen in slobrok., 7: description: => Node not seen in slobrok., 8: description: => Node not seen in slobrok., 9: description: => Node not seen in slobrok.] -1661243505.666904 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 0 for cluster 'mycluster': Storing new cluster state version in ZooKeeper: 2 -1661243505.750178 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 0 for cluster 'mycluster': Master moratorium complete: timed out waiting for all nodes to report in -1661243505.750779 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.SystemStateBroadcaster info FleetController 0 for cluster 'mycluster': Publishing cluster state version 2 -1661243505.751329 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Node 0 is master. Checking that no one else is master -1661243505.751614 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Leaving waitForMaster -1661243505.751836 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info SHUTTING DOWN FLEET CONTROLLER 0 -1661243505.752079 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 0 for cluster 'mycluster': Joining event thread. -1661243505.752352 localhost 75603/470 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 0 for cluster 'mycluster': Tick loop stopped -1661243505.752640 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 0 for cluster 'mycluster': FleetController done shutting down event thread. -1661243505.752952 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 0 for cluster 'mycluster': Resetting database state -1661243505.772139 localhost 75603/478 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: A change occurred in the list of registered fleetcontrollers. Requesting new information -1661243505.774926 localhost 75603/484 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is disconnected: [id: 0xda4181f3, L:/[0:0:0:0:0:0:0:1]:51711 ! R:localhost/[0:0:0:0:0:0:0:1]:51704] -1661243505.774695 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is told closing -1661243505.772168 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: A change occurred in the list of registered fleetcontrollers. Requesting new information -1661243505.772139 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: A change occurred in the list of registered fleetcontrollers. Requesting new information -1661243505.775932 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.ZooKeeper info Session: 0x1001abb14b10000 closed -1661243505.775706 localhost 75603/484 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is told closing -1661243505.777078 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got node list response from /vespa/fleetcontroller/mycluster/indexes version 0 with 2 nodes -1661243505.775677 localhost 75603/478 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got node list response from /vespa/fleetcontroller/mycluster/indexes version -112 with 0 nodes -1661243505.776862 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Clearing master data as we lost connection on node 0 -1661243505.776826 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got node list response from /vespa/fleetcontroller/mycluster/indexes version 0 with 2 nodes -1661243505.778084 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type MASTER_ELECTION @1661243505778: This node is no longer fleetcontroller master. -1661243505.777856 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 0 -1661243505.777774 localhost 75603/478 - fleetcontroller.org.apache.zookeeper.ClientCnxn info EventThread shut down for session: 0x1001abb14b10000 -1661243505.779077 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 1. Altering vote from null to 0. -1661243505.778820 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 0 -1661243505.778711 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 0 for cluster 'mycluster': Done resetting database state -1661243505.779925 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote from fleetcontroller 1. Altering vote from null to 0. -1661243505.779610 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 0 -1661243505.781132 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 2. Altering vote from null to 0. -1661243505.780660 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 0 -1661243505.781540 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got new fleet data with 2 entries: {1=0, 2=0} -1661243505.781153 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Entering waitForMaster -1661243505.782065 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Handling new master election, as we have received 2 entries -1661243505.781774 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote from fleetcontroller 2. Altering vote from null to 0. -1661243505.782683 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got master election state data(1 -> 0, 2 -> 0). -1661243505.782382 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Node 1 is not master yet, sleeping more -1661243505.783351 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Master gone from ZooKeeper. Tagging timestamp. Will wait 100 ms. -1661243505.782975 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got new fleet data with 2 entries: {1=0, 2=0} -1661243505.783719 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': First index is not currently trying to become master. Waiting for it to change state -1661243505.783968 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Handling new master election, as we have received 2 entries -1661243505.784535 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got master election state data(1 -> 0, 2 -> 0). -1661243505.784798 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Master gone from ZooKeeper. Tagging timestamp. Will wait 100 ms. -1661243505.785075 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': First index is not currently trying to become master. Waiting for it to change state -1661243505.785371 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': We are next in line to become master. Altering our state to look for followers -1661243505.804787 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Altered data in node /vespa/fleetcontroller/mycluster/indexes/1. Requesting new vote -1661243505.804823 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 1 for cluster 'mycluster': Stored new vote in ephemeral node. 1 -> 1 -1661243505.804805 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Altered data in node /vespa/fleetcontroller/mycluster/indexes/1. Requesting new vote -1661243505.805941 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 1 -1661243505.806625 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote from fleetcontroller 1. Altering vote from null to 1. -1661243505.806771 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 1 -1661243505.807066 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got new fleet data with 2 entries: {1=1, 2=0} -1661243505.807437 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 1. Altering vote from null to 1. -1661243505.807718 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Handling new master election, as we have received 2 entries -1661243505.808122 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got new fleet data with 2 entries: {1=1, 2=0} -1661243505.808332 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got master election state data(1 -> 1, 2 -> 0). -1661243505.808594 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Handling new master election, as we have received 2 entries -1661243505.808899 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Currently too few followers for cluster controller candidate 1. No current master. (1/3 followers) -1661243505.809177 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got master election state data(1 -> 1, 2 -> 0). -1661243505.809768 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Currently too few followers for cluster controller candidate 1. No current master. (1/3 followers) -1661243505.810097 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Altering our state to follow new fleet controller master candidate 1 -1661243505.829036 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Altered data in node /vespa/fleetcontroller/mycluster/indexes/2. Requesting new vote -1661243505.829067 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 2 for cluster 'mycluster': Stored new vote in ephemeral node. 2 -> 1 -1661243505.829036 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Altered data in node /vespa/fleetcontroller/mycluster/indexes/2. Requesting new vote -1661243505.829809 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 1 -1661243505.830243 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 2. Altering vote from null to 1. -1661243505.830300 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 1 -1661243505.830647 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got new fleet data with 2 entries: {1=1, 2=1} -1661243505.830881 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote from fleetcontroller 2. Altering vote from null to 1. -1661243505.831161 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Handling new master election, as we have received 2 entries -1661243505.831483 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got new fleet data with 2 entries: {1=1, 2=1} -1661243505.831713 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got master election state data(1 -> 1, 2 -> 1). -1661243505.831978 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Handling new master election, as we have received 2 entries -1661243505.832379 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': 1 is new master candidate, but needs to wait before it can take over -1661243505.832542 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got master election state data(1 -> 1, 2 -> 1). -1661243505.833035 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': 1 is new master candidate, but needs to wait before it can take over -1661243505.911946 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Node 1 is not master yet, sleeping more -1661243505.956844 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 1 for cluster 'mycluster': Read cluster state version 2 from ZooKeeper (znode version 1) -1661243506.003215 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 1 for cluster 'mycluster': Loaded previous cluster state bundle from ZooKeeper: ClusterStateBundle('version:2 cluster:d', default 'version:2 cluster:d', global 'version:2 cluster:d' (deferred activation)) -1661243506.004137 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type MASTER_ELECTION @1661243506004: This node just became fleetcontroller master. Bumped version to 2 to be in line. -1661243506.004846 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type SYSTEMSTATE @1661243506004: New cluster state version 3. Change from last: version: 2 => 3, storage: [0: description: => Node not seen in slobrok., 1: description: => Node not seen in slobrok., 2: description: => Node not seen in slobrok., 3: description: => Node not seen in slobrok., 4: description: => Node not seen in slobrok., 5: description: => Node not seen in slobrok., 6: description: => Node not seen in slobrok., 7: description: => Node not seen in slobrok., 8: description: => Node not seen in slobrok., 9: description: => Node not seen in slobrok.], distributor: [0: description: => Node not seen in slobrok., 1: description: => Node not seen in slobrok., 2: description: => Node not seen in slobrok., 3: description: => Node not seen in slobrok., 4: description: => Node not seen in slobrok., 5: description: => Node not seen in slobrok., 6: description: => Node not seen in slobrok., 7: description: => Node not seen in slobrok., 8: description: => Node not seen in slobrok., 9: description: => Node not seen in slobrok.] -1661243506.006609 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 1 for cluster 'mycluster': Storing new cluster state version in ZooKeeper: 3 -1661243506.061811 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 1 for cluster 'mycluster': Master moratorium complete: timed out waiting for all nodes to report in -1661243506.062251 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.SystemStateBroadcaster info FleetController 1 for cluster 'mycluster': Publishing cluster state version 3 -1661243506.167506 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Node 1 is master. Checking that no one else is master -1661243506.167790 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Leaving waitForMaster -1661243506.167970 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info SHUTTING DOWN FLEET CONTROLLER 1 -1661243506.168287 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 1 for cluster 'mycluster': Joining event thread. -1661243506.168535 localhost 75603/485 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 1 for cluster 'mycluster': Tick loop stopped -1661243506.168800 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 1 for cluster 'mycluster': FleetController done shutting down event thread. -1661243506.169077 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 1 for cluster 'mycluster': Resetting database state -1661243506.189449 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: A change occurred in the list of registered fleetcontrollers. Requesting new information -1661243506.189702 localhost 75603/499 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is disconnected: [id: 0x35935cb2, L:/[0:0:0:0:0:0:0:1]:51715 ! R:localhost/[0:0:0:0:0:0:0:1]:51704] -1661243506.189590 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is told closing -1661243506.189484 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: A change occurred in the list of registered fleetcontrollers. Requesting new information -1661243506.190411 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.ZooKeeper info Session: 0x1001abb14b10001 closed -1661243506.190333 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got node list response from /vespa/fleetcontroller/mycluster/indexes version 0 with 1 nodes -1661243506.190165 localhost 75603/499 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is told closing -1661243506.190936 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Clearing master data as we lost connection on node 1 -1661243506.190737 localhost 75603/493 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got node list response from /vespa/fleetcontroller/mycluster/indexes version -112 with 0 nodes -1661243506.191684 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type MASTER_ELECTION @1661243506191: This node is no longer fleetcontroller master. -1661243506.191621 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 1 -1661243506.192225 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 1 for cluster 'mycluster': Done resetting database state -1661243506.191973 localhost 75603/493 - fleetcontroller.org.apache.zookeeper.ClientCnxn info EventThread shut down for session: 0x1001abb14b10001 -1661243506.192501 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 2. Altering vote from null to 1. -1661243506.193432 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got new fleet data with 1 entries: {2=1} -1661243506.193693 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Handling new master election, as we have received 1 entries -1661243506.193958 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got master election state data(2 -> 1). -1661243506.194191 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Master gone from ZooKeeper. Tagging timestamp. Will wait 100 ms. -1661243506.194434 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': First index is not currently trying to become master. Waiting for it to change state -1661243506.194697 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': We are next in line to become master. Altering our state to look for followers -1661243506.213596 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Altered data in node /vespa/fleetcontroller/mycluster/indexes/2. Requesting new vote -1661243506.213635 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 2 for cluster 'mycluster': Stored new vote in ephemeral node. 2 -> 2 -1661243506.214499 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 2 -1661243506.214841 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 2. Altering vote from null to 2. -1661243506.215220 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got new fleet data with 1 entries: {2=2} -1661243506.215529 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Handling new master election, as we have received 1 entries -1661243506.215876 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got master election state data(2 -> 2). -1661243506.216199 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Currently too few followers for cluster controller candidate 2. No current master. (1/3 followers) -1661243506.216601 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info STARTING FLEET CONTROLLER 0 -1661243506.217669 localhost 75603/1 - fleetcontroller.com.yahoo.jrt.slobrok.api.Mirror info no location brokers available, retrying: [] (in 0.5 seconds) -1661243506.218047 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 0 for cluster 'mycluster': Created -1661243506.218945 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Clearing master data as we lost connection on node 0 -1661243506.222737 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.rpc.RpcServer info Registering tcp/localhost:51887 with slobrok at ( tcp/localhost:51705 ) -1661243506.222912 localhost 75603/543 - fleetcontroller.com.yahoo.jrt.slobrok.api.Mirror info successfully connected to location broker tcp/localhost:51705 (mirror initialized with 1 service names) -1661243506.223312 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 0 for cluster 'mycluster': Starting tick loop -1661243506.223245 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Entering waitForMaster -1661243506.223670 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 0 for cluster 'mycluster': Setting up new ZooKeeper session at localhost:51704 -1661243506.224291 localhost 75603/547 - fleetcontroller.org.apache.zookeeper.ZooKeeper info Initiating client connection, connectString=localhost:51704 sessionTimeout=30000 watcher=com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase$ZooKeeperWatcher@7e118ccc -1661243506.224914 localhost 75603/547 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocket info jute.maxbuffer value is 1048575 Bytes -1661243506.225324 localhost 75603/547 - fleetcontroller.org.apache.zookeeper.ClientCnxn info zookeeper.request.timeout value is 0. feature enabled=false -1661243506.225880 localhost 75603/548 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Opening socket connection to server localhost/127.0.0.1:51704. -1661243506.226159 localhost 75603/548 - fleetcontroller.org.apache.zookeeper.ClientCnxn info SASL config status: Will not attempt to authenticate using SASL (unknown error) -1661243506.233788 localhost 75603/550 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Socket connection established, initiating session, client: /127.0.0.1:51890, server: localhost/127.0.0.1:51704 -1661243506.233971 localhost 75603/545 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51887] registering storage/cluster.mycluster/fleetcontroller/0 with location broker tcp/localhost:51705 completed successfully -1661243506.234315 localhost 75603/550 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is connected: [id: 0x54d96664, L:/127.0.0.1:51890 - R:localhost/127.0.0.1:51704] -1661243506.254801 localhost 75603/550 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Session establishment complete on server localhost/127.0.0.1:51704, session id = 0x1001abb14b10003, negotiated timeout = 30000 -1661243506.255438 localhost 75603/549 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 0 for cluster 'mycluster': Connection to zookeeper server established. Refetching master data -1661243506.269487 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 0 for cluster 'mycluster': Creating ephemeral master vote node with vote to self. -1661243506.299288 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 0 for cluster 'mycluster': Done setting up new ZooKeeper session at localhost:51704 -1661243506.299263 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: A change occurred in the list of registered fleetcontrollers. Requesting new information -1661243506.300011 localhost 75603/549 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got node list response from /vespa/fleetcontroller/mycluster/indexes version 0 with 2 nodes -1661243506.311990 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 0 for cluster 'mycluster': Stored new vote in ephemeral node. 0 -> 0 -1661243506.312534 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got node list response from /vespa/fleetcontroller/mycluster/indexes version 0 with 2 nodes -1661243506.312408 localhost 75603/549 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/0 with code 0 and data 0 -1661243506.312639 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Node 0 is not master yet, sleeping more -1661243506.313573 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/0 with code 0 and data 0 -1661243506.313417 localhost 75603/549 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote from fleetcontroller 0. Altering vote from null to 0. -1661243506.314162 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 0. Altering vote from null to 0. -1661243506.314502 localhost 75603/549 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 2 -1661243506.315320 localhost 75603/549 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote from fleetcontroller 2. Altering vote from null to 2. -1661243506.316030 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Got new fleet data with 2 entries: {0=0, 2=2} -1661243506.316392 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Handling new master election, as we have received 2 entries -1661243506.316794 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Got master election state data(0 -> 0, 2 -> 2). -1661243506.317275 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Currently too few followers for cluster controller candidate 0. No current master. (1/3 followers) -1661243506.317865 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type MASTER_ELECTION @1661243506317: This node just became node state gatherer as we are fleetcontroller master candidate. -1661243506.320823 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 2 -1661243506.321229 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 2. Altering vote from null to 2. -1661243506.321613 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got new fleet data with 2 entries: {0=0, 2=2} -1661243506.321962 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Handling new master election, as we have received 2 entries -1661243506.322304 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got master election state data(0 -> 0, 2 -> 2). -1661243506.322611 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Currently too few followers for cluster controller candidate 0. No current master. (1/3 followers) -1661243506.323020 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Altering our state to follow new fleet controller master candidate 0 -1661243506.328135 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 0 for cluster 'mycluster': Read cluster state version 3 from ZooKeeper (znode version 2) -1661243506.346667 localhost 75603/549 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Altered data in node /vespa/fleetcontroller/mycluster/indexes/2. Requesting new vote -1661243506.346674 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 2 for cluster 'mycluster': Stored new vote in ephemeral node. 2 -> 0 -1661243506.346667 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Altered data in node /vespa/fleetcontroller/mycluster/indexes/2. Requesting new vote -1661243506.347687 localhost 75603/549 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 0 -1661243506.348225 localhost 75603/549 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote from fleetcontroller 2. Altering vote from null to 0. -1661243506.348266 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 0 -1661243506.348604 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Got new fleet data with 2 entries: {0=0, 2=0} -1661243506.349279 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Handling new master election, as we have received 2 entries -1661243506.348937 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 2. Altering vote from null to 0. -1661243506.349622 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Got master election state data(0 -> 0, 2 -> 0). -1661243506.350047 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got new fleet data with 2 entries: {0=0, 2=0} -1661243506.350317 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': 0 is newly elected master -1661243506.350755 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Handling new master election, as we have received 2 entries -1661243506.351444 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got master election state data(0 -> 0, 2 -> 0). -1661243506.351681 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 0 for cluster 'mycluster': Read cluster state version 3 from ZooKeeper (znode version 2) -1661243506.351813 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': 0 is newly elected master -1661243506.353452 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 0 for cluster 'mycluster': Loaded previous cluster state bundle from ZooKeeper: ClusterStateBundle('version:3 cluster:d', default 'version:3 cluster:d', global 'version:3 cluster:d' (deferred activation)) -1661243506.353927 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type MASTER_ELECTION @1661243506353: This node just became fleetcontroller master. Bumped version to 3 to be in line. -1661243506.354498 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type SYSTEMSTATE @1661243506354: New cluster state version 4. Change from last: version: 3 => 4, storage: [0: description: => Node not seen in slobrok., 1: description: => Node not seen in slobrok., 2: description: => Node not seen in slobrok., 3: description: => Node not seen in slobrok., 4: description: => Node not seen in slobrok., 5: description: => Node not seen in slobrok., 6: description: => Node not seen in slobrok., 7: description: => Node not seen in slobrok., 8: description: => Node not seen in slobrok., 9: description: => Node not seen in slobrok.], distributor: [0: description: => Node not seen in slobrok., 1: description: => Node not seen in slobrok., 2: description: => Node not seen in slobrok., 3: description: => Node not seen in slobrok., 4: description: => Node not seen in slobrok., 5: description: => Node not seen in slobrok., 6: description: => Node not seen in slobrok., 7: description: => Node not seen in slobrok., 8: description: => Node not seen in slobrok., 9: description: => Node not seen in slobrok.] -1661243506.355864 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 0 for cluster 'mycluster': Storing new cluster state version in ZooKeeper: 4 -1661243506.393554 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 0 for cluster 'mycluster': Master moratorium complete: timed out waiting for all nodes to report in -1661243506.393869 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.SystemStateBroadcaster info FleetController 0 for cluster 'mycluster': Publishing cluster state version 4 -1661243506.420410 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Node 0 is master. Checking that no one else is master -1661243506.420858 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Leaving waitForMaster -1661243506.421181 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info STARTING FLEET CONTROLLER 1 -1661243506.422450 localhost 75603/1 - fleetcontroller.com.yahoo.jrt.slobrok.api.Mirror info no location brokers available, retrying: [] (in 0.5 seconds) -1661243506.422771 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 1 for cluster 'mycluster': Created -1661243506.423748 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Clearing master data as we lost connection on node 1 -1661243506.427680 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.rpc.RpcServer info Registering tcp/localhost:51912 with slobrok at ( tcp/localhost:51705 ) -1661243506.427920 localhost 75603/556 - fleetcontroller.com.yahoo.jrt.slobrok.api.Mirror info successfully connected to location broker tcp/localhost:51705 (mirror initialized with 2 service names) -1661243506.428330 localhost 75603/560 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 1 for cluster 'mycluster': Starting tick loop -1661243506.428246 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Entering waitForMaster -1661243506.428923 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Node 0 is master. Checking that no one else is master -1661243506.428705 localhost 75603/560 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 1 for cluster 'mycluster': Setting up new ZooKeeper session at localhost:51704 -1661243506.429609 localhost 75603/560 - fleetcontroller.org.apache.zookeeper.ZooKeeper info Initiating client connection, connectString=localhost:51704 sessionTimeout=30000 watcher=com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase$ZooKeeperWatcher@32c98e99 -1661243506.430223 localhost 75603/560 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocket info jute.maxbuffer value is 1048575 Bytes -1661243506.430619 localhost 75603/560 - fleetcontroller.org.apache.zookeeper.ClientCnxn info zookeeper.request.timeout value is 0. feature enabled=false -1661243506.431169 localhost 75603/561 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Opening socket connection to server localhost/[0:0:0:0:0:0:0:1]:51704. -1661243506.431449 localhost 75603/561 - fleetcontroller.org.apache.zookeeper.ClientCnxn info SASL config status: Will not attempt to authenticate using SASL (unknown error) -1661243506.438564 localhost 75603/563 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Socket connection established, initiating session, client: /[0:0:0:0:0:0:0:1]:51915, server: localhost/[0:0:0:0:0:0:0:1]:51704 -1661243506.438897 localhost 75603/558 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51912] registering storage/cluster.mycluster/fleetcontroller/1 with location broker tcp/localhost:51705 completed successfully -1661243506.439120 localhost 75603/563 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is connected: [id: 0x20d50555, L:/[0:0:0:0:0:0:0:1]:51915 - R:localhost/[0:0:0:0:0:0:0:1]:51704] -1661243506.460199 localhost 75603/563 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Session establishment complete on server localhost/[0:0:0:0:0:0:0:1]:51704, session id = 0x1001abb14b10004, negotiated timeout = 30000 -1661243506.460731 localhost 75603/562 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 1 for cluster 'mycluster': Connection to zookeeper server established. Refetching master data -1661243506.463849 localhost 75603/560 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 1 for cluster 'mycluster': Creating ephemeral master vote node with vote to self. -1661243506.482637 localhost 75603/560 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 1 for cluster 'mycluster': Done setting up new ZooKeeper session at localhost:51704 -1661243506.482640 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: A change occurred in the list of registered fleetcontrollers. Requesting new information -1661243506.482640 localhost 75603/549 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: A change occurred in the list of registered fleetcontrollers. Requesting new information -1661243506.483138 localhost 75603/562 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got node list response from /vespa/fleetcontroller/mycluster/indexes version 0 with 3 nodes -1661243506.501684 localhost 75603/560 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 1 for cluster 'mycluster': Stored new vote in ephemeral node. 1 -> 1 -1661243506.501759 localhost 75603/562 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/0 with code 0 and data 0 -1661243506.501735 localhost 75603/549 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got node list response from /vespa/fleetcontroller/mycluster/indexes version 0 with 3 nodes -1661243506.501683 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got node list response from /vespa/fleetcontroller/mycluster/indexes version 0 with 3 nodes -1661243506.502322 localhost 75603/562 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote from fleetcontroller 0. Altering vote from null to 0. -1661243506.502049 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Leaving waitForMaster -1661243506.503639 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from patTEST SUCCEEDED - cleaning up zookeeper directory -h /vespa/fleetcontroller/mycluster/indexes/0 with code 0 and data 0 -1661243506.503436 localhost 75603/562 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 1 -1661243506.503244 localhost 75603/549 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/0 with code 0 and data 0 -1661243506.504402 localhost 75603/562 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote from fleetcontroller 1. Altering vote from null to 1. -1661243506.504080 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 0. Altering vote from null to 0. -1661243506.503956 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ZooKeeperServer info shutting down -1661243506.505371 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 1 -1661243506.505040 localhost 75603/562 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 0 -1661243506.504689 localhost 75603/549 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote from fleetcontroller 0. Altering vote from null to 0. -1661243506.506135 localhost 75603/562 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote from fleetcontroller 2. Altering vote from null to 0. -1661243506.505835 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 1. Altering vote from null to 1. -1661243506.505533 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.RequestThrottler info Shutting down -1661243506.506920 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 0 -1661243506.506750 localhost 75603/560 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got new fleet data with 3 entries: {0=0, 1=1, 2=0} -1661243506.506408 localhost 75603/549 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 1 -1661243506.507640 localhost 75603/560 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Handling new master election, as we have received 3 entries -1661243506.507359 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 2. Altering vote from null to 0. -1661243506.507115 localhost 75603/456 - fleetcontroller.org.apache.zookeeper.server.RequestThrottler info Draining request throttler queue -1661243506.508549 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got new fleet data with 3 entries: {0=0, 1=1, 2=0} -1661243506.508211 localhost 75603/560 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got master election state data(0 -> 0, 1 -> 1, 2 -> 0). -1661243506.507923 localhost 75603/549 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote from fleetcontroller 1. Altering vote from null to 1. -1661243506.509191 localhost 75603/560 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': 0 is new master candidate, but needs to wait before it can take over -1661243506.508911 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Handling new master election, as we have received 3 entries -1661243506.508676 localhost 75603/456 - fleetcontroller.org.apache.zookeeper.server.RequestThrottler info RequestThrottler shutdown. Dropped 0 requests -1661243506.509985 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got master election state data(0 -> 0, 1 -> 1, 2 -> 0). -1661243506.509715 localhost 75603/560 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Altering our state to follow new fleet controller master candidate 0 -1661243506.509442 localhost 75603/549 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 0 -1661243506.510641 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': 0 is still the master -1661243506.510224 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.SessionTrackerImpl info Shutting down -1661243506.511137 localhost 75603/514 - fleetcontroller.org.apache.zookeeper.server.NIOServerCnxn warning Close of session 0x1001abb14b10004\nexception=\njava.io.IOException: ZooKeeperServer not running\n\tat org.apache.zookeeper.server.NIOServerCnxn.readLength(NIOServerCnxn.java:554)\n\tat org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:339)\n\tat org.apache.zookeeper.server.NIOServerCnxnFactory$IOWorkRequest.doWork(NIOServerCnxnFactory.java:508)\n\tat org.apache.zookeeper.server.WorkerService$ScheduledWorkRequest.run(WorkerService.java:154)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243506.511157 localhost 75603/549 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote from fleetcontroller 2. Altering vote from null to 0. -1661243506.511597 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.PrepRequestProcessor info Shutting down -1661243506.512802 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Got new fleet data with 3 entries: {0=0, 1=1, 2=0} -1661243506.512773 localhost 75603/563 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is disconnected: [id: 0x20d50555, L:/[0:0:0:0:0:0:0:1]:51915 ! R:localhost/[0:0:0:0:0:0:0:1]:51704] -1661243506.513198 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Handling new master election, as we have received 3 entries -1661243506.512939 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.SyncRequestProcessor info Shutting down -1661243506.512947 localhost 75603/455 - fleetcontroller.org.apache.zookeeper.server.PrepRequestProcessor info PrepRequestProcessor exited loop! -1661243506.513877 localhost 75603/454 - fleetcontroller.org.apache.zookeeper.server.SyncRequestProcessor info SyncRequestProcessor exited! -1661243506.513715 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Got master election state data(0 -> 0, 1 -> 1, 2 -> 0). -1661243506.513588 localhost 75603/561 - fleetcontroller.org.apache.zookeeper.ClientCnxn warning Session 0x1001abb14b10004 for server localhost/[0:0:0:0:0:0:0:1]:51704, Closing socket connection. Attempting reconnect except it is a SessionExpiredException.\nexception=\nEndOfStreamException: channel for sessionid 0x1001abb14b10004 is lost\n\tat org.apache.zookeeper.ClientCnxnSocketNetty.doTransport(ClientCnxnSocketNetty.java:285)\n\tat org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1283)\n -1661243506.513461 localhost 75603/563 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is told closing -1661243506.514561 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': 0 is still the master -1661243506.514292 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.FinalRequestProcessor info shutdown of request processor complete -1661243506.515605 localhost 75603/560 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase warning FleetController 1 for cluster 'mycluster': Failed to create our ephemeral node and store master vote. Exception: KeeperErrorCode = ConnectionLoss for /vespa/fleetcontroller/mycluster/indexes/1\norg.apache.zookeeper.KeeperException$ConnectionLossException: KeeperErrorCode = ConnectionLoss for /vespa/fleetcontroller/mycluster/indexes/1\n\tat org.apache.zookeeper.KeeperException.create(KeeperException.java:102)\n\tat org.apache.zookeeper.KeeperException.create(KeeperException.java:54)\n\tat org.apache.zookeeper.ZooKeeper.setData(ZooKeeper.java:2237)\n\tat com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase.storeMasterVote(ZooKeeperDatabase.java:188)\n\tat com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler.performZooKeeperWrites(DatabaseHandler.java:286)\n\tat com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler.doNextZooKeeperTask(DatabaseHandler.java:258)\n\tat com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler.setMasterVote(DatabaseHandler.java:355)\n\tat com.yahoo.vespa.clustercontroller.core.MasterElectionHandler.watchMasterElection(MasterElectionHandler.java:184)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.updateMasterElectionState(FleetController.java:654)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:596)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243506.515402 localhost 75603/562 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 1 for cluster 'mycluster': Lost connection to zookeeper server -1661243506.518043 localhost 75603/504 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is disconnected: [id: 0x42017013, L:/[0:0:0:0:0:0:0:1]:51717 ! R:localhost/[0:0:0:0:0:0:0:1]:51704] -1661243506.518061 localhost 75603/560 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler warning FleetController 1 for cluster 'mycluster': Failed to store master vote -1661243506.518005 localhost 75603/550 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is disconnected: [id: 0x54d96664, L:/127.0.0.1:51890 ! R:localhost/127.0.0.1:51704] -1661243506.518011 localhost 75603/452 - fleetcontroller.org.apache.zookeeper.server.NIOServerCnxnFactory info accept thread exitted run method -1661243506.517508 localhost 75603/449 - fleetcontroller.org.apache.zookeeper.server.NIOServerCnxnFactory info ConnnectionExpirerThread interrupted -1661243506.519749 localhost 75603/548 - fleetcontroller.org.apache.zookeeper.ClientCnxn warning Session 0x1001abb14b10003 for server localhost/127.0.0.1:51704, Closing socket connection. Attempting reconnect except it is a SessionExpiredException.\nexception=\nEndOfStreamException: channel for sessionid 0x1001abb14b10003 is lost\n\tat org.apache.zookeeper.ClientCnxnSocketNetty.doTransport(ClientCnxnSocketNetty.java:285)\n\tat org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1283)\n -1661243506.519708 localhost 75603/550 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is told closing -1661243506.519636 localhost 75603/560 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type MASTER_ELECTION @1661243506519: This node just became node state gatherer as we are fleetcontroller master candidate. -1661243506.519215 localhost 75603/502 - fleetcontroller.org.apache.zookeeper.ClientCnxn warning Session 0x1001abb14b10002 for server localhost/[0:0:0:0:0:0:0:1]:51704, Closing socket connection. Attempting reconnect except it is a SessionExpiredException.\nexception=\nEndOfStreamException: channel for sessionid 0x1001abb14b10002 is lost\n\tat org.apache.zookeeper.ClientCnxnSocketNetty.doTransport(ClientCnxnSocketNetty.java:285)\n\tat org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1283)\n -1661243506.519136 localhost 75603/504 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is told closing -1661243506.518737 localhost 75603/450 - fleetcontroller.org.apache.zookeeper.server.NIOServerCnxnFactory info selector thread exitted run method -1661243506.518624 localhost 75603/451 - fleetcontroller.org.apache.zookeeper.server.NIOServerCnxnFactory info selector thread exitted run method -1661243506.522573 localhost 75603/503 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 2 for cluster 'mycluster': Lost connection to zookeeper server -1661243506.521409 localhost 75603/549 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 0 for cluster 'mycluster': Lost connection to zookeeper server -1661243506.523702 localhost 75603/500 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Clearing master data as we lost connection on node 2 -1661243506.523709 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Clearing master data as we lost connection on node 0 -1661243506.524360 localhost 75603/547 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type MASTER_ELECTION @1661243506524: This node is no longer fleetcontroller master. -STARTING TEST: MasterElectionTest::testClusterStateVersionIncreasesAcrossMasterElections -1661243506.526032 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.persistence.FileTxnSnapLog info zookeeper.snapshot.trust.empty : false -1661243506.526539 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.watch.WatchManagerFactory info Using org.apache.zookeeper.server.watch.WatchManager as watch manager -1661243506.526889 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.watch.WatchManagerFactory info Using org.apache.zookeeper.server.watch.WatchManager as watch manager -1661243506.527271 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ZKDatabase info zookeeper.snapshotSizeFactor = 0.33 -1661243506.527541 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ZKDatabase info zookeeper.commitLogCount=500 -1661243506.527781 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ZooKeeperServer info minSessionTimeout set to 4000 -1661243506.528053 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ZooKeeperServer info maxSessionTimeout set to 40000 -1661243506.528379 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ResponseCache info getData response cache size is initialized with value 400. -1661243506.528647 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ResponseCache info getChildren response cache size is initialized with value 400. -1661243506.528932 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.util.RequestPathMetricsCollector info zookeeper.pathStats.slotCapacity = 60 -1661243506.529178 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.util.RequestPathMetricsCollector info zookeeper.pathStats.slotDuration = 15 -1661243506.529418 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.util.RequestPathMetricsCollector info zookeeper.pathStats.maxDepth = 6 -1661243506.529653 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.util.RequestPathMetricsCollector info zookeeper.pathStats.initialDelay = 5 -1661243506.529886 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.util.RequestPathMetricsCollector info zookeeper.pathStats.delay = 5 -1661243506.530111 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.util.RequestPathMetricsCollector info zookeeper.pathStats.enabled = false -1661243506.530529 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ZooKeeperServer info The max bytes for all large requests are set to 104857600 -1661243506.530948 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ZooKeeperServer info The large request threshold is set to -1 -1661243506.531247 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.AuthenticationHelper info zookeeper.enforce.auth.enabled = false -1661243506.531729 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.AuthenticationHelper info zookeeper.enforce.auth.schemes = [] -1661243506.532173 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ZooKeeperServer info Created server with tickTime 2000 minSessionTimeout 4000 maxSessionTimeout 40000 clientPortListenBacklog -1 datadir /Users/musum/git/vespa/clustercontroller-core/target/surefire-reports/test_fltctrl_zk3602320267307376975sdir/version-2 snapdir /Users/musum/git/vespa/clustercontroller-core/target/surefire-reports/test_fltctrl_zk3602320267307376975sdir/version-2 -1661243506.532871 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ServerCnxnFactory warning maxCnxns is not configured, using default value 0. -1661243506.533208 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.NIOServerCnxnFactory info Configuring NIO connection handler with 10s sessionless connection timeout, 2 selector thread(s), 32 worker threads, and 64 kB direct buffers. -1661243506.533724 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.NIOServerCnxnFactory info binding to port 0.0.0.0/0.0.0.0:0 -1661243506.534770 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.persistence.FileTxnSnapLog info Snapshotting: 0x0 to /Users/musum/git/vespa/clustercontroller-core/target/surefire-reports/test_fltctrl_zk3602320267307376975sdir/version-2/snapshot.0 -1661243506.535745 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ZKDatabase info Snapshot loaded in 1 ms, highest zxid is 0x0, digest is 1371985504 -1661243506.536054 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.persistence.FileTxnSnapLog info Snapshotting: 0x0 to /Users/musum/git/vespa/clustercontroller-core/target/surefire-reports/test_fltctrl_zk3602320267307376975sdir/version-2/snapshot.0 -1661243506.536680 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.server.ZooKeeperServer info Snapshot taken in 1 ms -1661243506.537359 localhost 75603/570 - fleetcontroller.org.apache.zookeeper.server.PrepRequestProcessor info PrepRequestProcessor (sid:0) started, reconfigEnabled=false -1661243506.538719 localhost 75603/1 - fleetcontroller.com.yahoo.jrt.slobrok.api.Mirror info no location brokers available, retrying: [] (in 0.5 seconds) -1661243506.538988 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 0 for cluster 'mycluster': Created -1661243506.539735 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Clearing master data as we lost connection on node 0 -1661243506.543591 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.rpc.RpcServer info Registering tcp/localhost:51919 with slobrok at ( tcp/localhost:51917 ) -1661243506.543768 localhost 75603/580 - fleetcontroller.com.yahoo.jrt.slobrok.api.Mirror info successfully connected to location broker tcp/localhost:51917 (mirror initialized with 0 service names) -1661243506.544149 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 0 for cluster 'mycluster': Starting tick loop -1661243506.544582 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 0 for cluster 'mycluster': Setting up new ZooKeeper session at localhost:51916 -1661243506.544790 localhost 75603/1 - fleetcontroller.com.yahoo.jrt.slobrok.api.Mirror info no location brokers available, retrying: [] (in 0.5 seconds) -1661243506.544964 localhost 75603/584 - fleetcontroller.org.apache.zookeeper.ZooKeeper info Initiating client connection, connectString=localhost:51916 sessionTimeout=30000 watcher=com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase$ZooKeeperWatcher@1a1d9ef -1661243506.545108 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 1 for cluster 'mycluster': Created -1661243506.545701 localhost 75603/584 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocket info jute.maxbuffer value is 1048575 Bytes -1661243506.546061 localhost 75603/584 - fleetcontroller.org.apache.zookeeper.ClientCnxn info zookeeper.request.timeout value is 0. feature enabled=false -1661243506.546496 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Clearing master data as we lost connection on node 1 -1661243506.546672 localhost 75603/589 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Opening socket connection to server localhost/127.0.0.1:51916. -1661243506.547030 localhost 75603/589 - fleetcontroller.org.apache.zookeeper.ClientCnxn info SASL config status: Will not attempt to authenticate using SASL (unknown error) -1661243506.547696 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.rpc.RpcServer info Registering tcp/localhost:51921 with slobrok at ( tcp/localhost:51917 ) -1661243506.548321 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 1 for cluster 'mycluster': Starting tick loop -1661243506.548771 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 1 for cluster 'mycluster': Setting up new ZooKeeper session at localhost:51916 -1661243506.549005 localhost 75603/1 - fleetcontroller.com.yahoo.jrt.slobrok.api.Mirror info no location brokers available, retrying: [] (in 0.5 seconds) -1661243506.549218 localhost 75603/598 - fleetcontroller.org.apache.zookeeper.ZooKeeper info Initiating client connection, connectString=localhost:51916 sessionTimeout=30000 watcher=com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase$ZooKeeperWatcher@26718f6b -1661243506.549385 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 2 for cluster 'mycluster': Created -1661243506.549842 localhost 75603/598 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocket info jute.maxbuffer value is 1048575 Bytes -1661243506.550195 localhost 75603/598 - fleetcontroller.org.apache.zookeeper.ClientCnxn info zookeeper.request.timeout value is 0. feature enabled=false -1661243506.550816 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Clearing master data as we lost connection on node 2 -1661243506.550825 localhost 75603/603 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Opening socket connection to server localhost/127.0.0.1:51916. -1661243506.551412 localhost 75603/603 - fleetcontroller.org.apache.zookeeper.ClientCnxn info SASL config status: Will not attempt to authenticate using SASL (unknown error) -1661243506.551768 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.rpc.RpcServer info Registering tcp/localhost:51923 with slobrok at ( tcp/localhost:51917 ) -1661243506.552524 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 2 for cluster 'mycluster': Starting tick loop -1661243506.552959 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 2 for cluster 'mycluster': Setting up new ZooKeeper session at localhost:51916 -1661243506.553309 localhost 75603/611 - fleetcontroller.org.apache.zookeeper.ZooKeeper info Initiating client connection, connectString=localhost:51916 sessionTimeout=30000 watcher=com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase$ZooKeeperWatcher@28fe0700 -1661243506.553815 localhost 75603/611 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocket info jute.maxbuffer value is 1048575 Bytes -1661243506.554122 localhost 75603/611 - fleetcontroller.org.apache.zookeeper.ClientCnxn info zookeeper.request.timeout value is 0. feature enabled=false -1661243506.554631 localhost 75603/617 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Opening socket connection to server localhost/127.0.0.1:51916. -1661243506.554855 localhost 75603/617 - fleetcontroller.org.apache.zookeeper.ClientCnxn info SASL config status: Will not attempt to authenticate using SASL (unknown error) -1661243506.562585 localhost 75603/597 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Socket connection established, initiating session, client: /127.0.0.1:51925, server: localhost/127.0.0.1:51916 -1661243506.563182 localhost 75603/597 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is connected: [id: 0xd4444b66, L:/127.0.0.1:51925 - R:localhost/127.0.0.1:51916] -1661243506.590086 localhost 75603/592 - fleetcontroller.com.yahoo.jrt.slobrok.api.Mirror info successfully connected to location broker tcp/localhost:51917 (mirror initialized with 0 service names) -1661243506.590892 localhost 75603/569 - fleetcontroller.org.apache.zookeeper.server.persistence.FileTxnLog info Creating new log file: log.1 -1661243506.598339 localhost 75603/619 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Socket connection established, initiating session, client: /127.0.0.1:51934, server: localhost/127.0.0.1:51916 -1661243506.598339 localhost 75603/620 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Socket connection established, initiating session, client: /127.0.0.1:51933, server: localhost/127.0.0.1:51916 -1661243506.599014 localhost 75603/619 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is connected: [id: 0x33554773, L:/127.0.0.1:51934 - R:localhost/127.0.0.1:51916] -1661243506.599325 localhost 75603/620 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is connected: [id: 0x1186c720, L:/127.0.0.1:51933 - R:localhost/127.0.0.1:51916] -1661243506.614415 localhost 75603/606 - fleetcontroller.com.yahoo.jrt.slobrok.api.Mirror info successfully connected to location broker tcp/localhost:51917 (mirror initialized with 0 service names) -1661243506.637838 localhost 75603/597 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Session establishment complete on server localhost/127.0.0.1:51916, session id = 0x1001abb1a620000, negotiated timeout = 30000 -1661243506.638720 localhost 75603/591 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 0 for cluster 'mycluster': Connection to zookeeper server established. Refetching master data -1661243506.662210 localhost 75603/582 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51919] registering storage/cluster.mycluster/fleetcontroller/0 with location broker tcp/localhost:51917 completed successfully -1661243506.662536 localhost 75603/620 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Session establishment complete on server localhost/127.0.0.1:51916, session id = 0x1001abb1a620001, negotiated timeout = 30000 -1661243506.663794 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 2 for cluster 'mycluster': Connection to zookeeper server established. Refetching master data -1661243506.690305 localhost 75603/619 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Session establishment complete on server localhost/127.0.0.1:51916, session id = 0x1001abb1a620002, negotiated timeout = 30000 -1661243506.691150 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 1 for cluster 'mycluster': Connection to zookeeper server established. Refetching master data -1661243506.737726 localhost 75603/595 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51921] registering storage/cluster.mycluster/fleetcontroller/1 with location broker tcp/localhost:51917 completed successfully -1661243506.746175 localhost 75603/609 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51923] registering storage/cluster.mycluster/fleetcontroller/2 with location broker tcp/localhost:51917 completed successfully -1661243506.749958 localhost 75603/615 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51928] registering storage/cluster.mycluster/distributor/0/default with location broker tcp/localhost:51917 completed successfully -1661243506.753935 localhost 75603/624 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51935] registering storage/cluster.mycluster/storage/0/default with location broker tcp/localhost:51917 completed successfully -1661243506.757980 localhost 75603/629 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51939] registering storage/cluster.mycluster/distributor/1/default with location broker tcp/localhost:51917 completed successfully -1661243506.757980 localhost 75603/634 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51941] registering storage/cluster.mycluster/storage/1/default with location broker tcp/localhost:51917 completed successfully -1661243506.768805 localhost 75603/638 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51942] registering storage/cluster.mycluster/distributor/2/default with location broker tcp/localhost:51917 completed successfully -1661243506.775945 localhost 75603/648 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51947] registering storage/cluster.mycluster/distributor/3/default with location broker tcp/localhost:51917 completed successfully -1661243506.775986 localhost 75603/642 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51945] registering storage/cluster.mycluster/storage/2/default with location broker tcp/localhost:51917 completed successfully -1661243506.775955 localhost 75603/652 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51949] registering storage/cluster.mycluster/storage/3/default with location broker tcp/localhost:51917 completed successfully -1661243506.783290 localhost 75603/656 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51951] registering storage/cluster.mycluster/distributor/4/default with location broker tcp/localhost:51917 completed successfully -1661243506.787144 localhost 75603/661 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51953] registering storage/cluster.mycluster/storage/4/default with location broker tcp/localhost:51917 completed successfully -1661243506.787144 localhost 75603/665 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51958] registering storage/cluster.mycluster/distributor/5/default with location broker tcp/localhost:51917 completed successfully -1661243506.794455 localhost 75603/672 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51960] registering storage/cluster.mycluster/storage/5/default with location broker tcp/localhost:51917 completed successfully -1661243506.794474 localhost 75603/678 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51962] registering storage/cluster.mycluster/distributor/6/default with location broker tcp/localhost:51917 completed successfully -1661243506.798134 localhost 75603/683 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51966] registering storage/cluster.mycluster/storage/6/default with location broker tcp/localhost:51917 completed successfully -1661243506.808148 localhost 75603/694 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51974] registering storage/cluster.mycluster/storage/7/default with location broker tcp/localhost:51917 completed successfully -1661243506.808147 localhost 75603/689 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51969] registering storage/cluster.mycluster/distributor/7/default with location broker tcp/localhost:51917 completed successfully -1661243506.811748 localhost 75603/701 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51976] registering storage/cluster.mycluster/distributor/8/default with location broker tcp/localhost:51917 completed successfully -1661243506.819119 localhost 75603/707 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51980] registering storage/cluster.mycluster/storage/8/default with location broker tcp/localhost:51917 completed successfully -1661243506.819132 localhost 75603/712 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51983] registering storage/cluster.mycluster/distributor/9/default with location broker tcp/localhost:51917 completed successfully -1661243506.819119 localhost 75603/717 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51985] registering storage/cluster.mycluster/storage/9/default with location broker tcp/localhost:51917 completed successfully -1661243506.830431 localhost 75603/615 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51928] registering storage/cluster.mycluster/distributor/0 with location broker tcp/localhost:51917 completed successfully -1661243506.830479 localhost 75603/629 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51939] registering storage/cluster.mycluster/distributor/1 with location broker tcp/localhost:51917 completed successfully -1661243506.830443 localhost 75603/624 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51935] registering storage/cluster.mycluster/storage/0 with location broker tcp/localhost:51917 completed successfully -1661243506.845705 localhost 75603/638 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51942] registering storage/cluster.mycluster/distributor/2 with location broker tcp/localhost:51917 completed successfully -1661243506.845753 localhost 75603/652 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51949] registering storage/cluster.mycluster/storage/3 with location broker tcp/localhost:51917 completed successfully -1661243506.845705 localhost 75603/634 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51941] registering storage/cluster.mycluster/storage/1 with location broker tcp/localhost:51917 completed successfully -1661243506.856584 localhost 75603/648 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51947] registering storage/cluster.mycluster/distributor/3 with location broker tcp/localhost:51917 completed successfully -1661243506.856628 localhost 75603/656 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51951] registering storage/cluster.mycluster/distributor/4 with location broker tcp/localhost:51917 completed successfully -1661243506.856613 localhost 75603/642 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51945] registering storage/cluster.mycluster/storage/2 with location broker tcp/localhost:51917 completed successfully -1661243506.860464 localhost 75603/661 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51953] registering storage/cluster.mycluster/storage/4 with location broker tcp/localhost:51917 completed successfully -1661243506.860895 localhost 75603/712 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51983] registering storage/cluster.mycluster/distributor/9 with location broker tcp/localhost:51917 completed successfully -1661243506.860880 localhost 75603/689 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51969] registering storage/cluster.mycluster/distributor/7 with location broker tcp/localhost:51917 completed successfully -1661243506.860851 localhost 75603/701 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51976] registering storage/cluster.mycluster/distributor/8 with location broker tcp/localhost:51917 completed successfully -1661243506.860847 localhost 75603/694 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51974] registering storage/cluster.mycluster/storage/7 with location broker tcp/localhost:51917 completed successfully -1661243506.860782 localhost 75603/683 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51966] registering storage/cluster.mycluster/storage/6 with location broker tcp/localhost:51917 completed successfully -1661243506.860752 localhost 75603/672 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51960] registering storage/cluster.mycluster/storage/5 with location broker tcp/localhost:51917 completed successfully -1661243506.860530 localhost 75603/678 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51962] registering storage/cluster.mycluster/distributor/6 with location broker tcp/localhost:51917 completed successfully -1661243506.860464 localhost 75603/665 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51958] registering storage/cluster.mycluster/distributor/5 with location broker tcp/localhost:51917 completed successfully -1661243506.861219 localhost 75603/707 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51980] registering storage/cluster.mycluster/storage/8 with location broker tcp/localhost:51917 completed successfully -1661243506.861184 localhost 75603/717 - fleetcontroller.com.yahoo.jrt.slobrok.api.Register info [RPC @ tcp/localhost:51985] registering storage/cluster.mycluster/storage/9 with location broker tcp/localhost:51917 completed successfully -1661243506.939485 localhost 75603/453 - fleetcontroller.org.apache.zookeeper.server.SessionTrackerImpl info SessionTrackerImpl exited loop! -1661243506.939482 localhost 75603/227 - fleetcontroller.org.apache.zookeeper.server.SessionTrackerImpl info SessionTrackerImpl exited loop! -1661243507.075476 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 1 for cluster 'mycluster': Creating ephemeral master vote node with vote to self. -1661243507.075476 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 2 for cluster 'mycluster': Creating ephemeral master vote node with vote to self. -1661243507.075941 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 0 for cluster 'mycluster': Creating ephemeral master vote node with vote to self. -1661243507.105249 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 1 for cluster 'mycluster': Done setting up new ZooKeeper session at localhost:51916 -1661243507.125027 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 0 for cluster 'mycluster': Done setting up new ZooKeeper session at localhost:51916 -1661243507.125147 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got node list response from /vespa/fleetcontroller/mycluster/indexes version 0 with 3 nodes -1661243507.125040 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 2 for cluster 'mycluster': Done setting up new ZooKeeper session at localhost:51916 -1661243507.145058 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 1 for cluster 'mycluster': Stored new vote in ephemeral node. 1 -> 1 -1661243507.145126 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got node list response from /vespa/fleetcontroller/mycluster/indexes version 0 with 3 nodes -1661243507.145126 localhost 75603/591 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got node list response from /vespa/fleetcontroller/mycluster/indexes version 0 with 3 nodes -1661243507.164381 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 0 for cluster 'mycluster': Stored new vote in ephemeral node. 0 -> 0 -1661243507.164875 localhost 75603/591 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/0 with code 0 and data 0 -1661243507.164754 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/0 with code 0 and data 0 -1661243507.164719 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 2 for cluster 'mycluster': Stored new vote in ephemeral node. 2 -> 2 -1661243507.164567 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/0 with code 0 and data 0 -1661243507.165628 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 0. Altering vote from null to 0. -1661243507.166469 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 1 -1661243507.165319 localhost 75603/591 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote from fleetcontroller 0. Altering vote from null to 0. -1661243507.167058 localhost 75603/591 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 1 -1661243507.167329 localhost 75603/591 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote from fleetcontroller 1. Altering vote from null to 1. -1661243507.167572 localhost 75603/591 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 2 -1661243507.165034 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.testutils.Waiter$Impl info Waiting for RegexStateMatcher:\n wanted: 'version:\\d+ distributor:10 storage:10'\n last checked: 'null'\n current: '' with wait task GetNodeStateResender -1661243507.167844 localhost 75603/591 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote from fleetcontroller 2. Altering vote from null to 2. -1661243507.166796 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 1. Altering vote from null to 1. -1661243507.166198 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote from fleetcontroller 0. Altering vote from null to 0. -1661243507.169076 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 1 -1661243507.169473 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote from fleetcontroller 1. Altering vote from null to 1. -1661243507.168804 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 2 -1661243507.168587 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Got new fleet data with 3 entries: {0=0, 1=1, 2=2} -1661243507.170397 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Handling new master election, as we have received 3 entries -1661243507.168273 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.testutils.Waiter$Impl info Wait condition not met: No cluster state defined yet -1661243507.170700 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Got master election state data(0 -> 0, 1 -> 1, 2 -> 2). -1661243507.170038 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 2. Altering vote from null to 2. -1661243507.169758 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 2 -1661243507.171491 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got new fleet data with 3 entries: {0=0, 1=1, 2=2} -1661243507.171172 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Currently too few followers for cluster controller candidate 0. No current master. (1/3 followers) -1661243507.172005 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Handling new master election, as we have received 3 entries -1661243507.171758 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote from fleetcontroller 2. Altering vote from null to 2. -1661243507.172648 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got master election state data(0 -> 0, 1 -> 1, 2 -> 2). -1661243507.173228 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got new fleet data with 3 entries: {0=0, 1=1, 2=2} -1661243507.173201 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Altered data in node /vespa/fleetcontroller/mycluster/indexes/2. Requesting new vote -1661243507.173966 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Handling new master election, as we have received 3 entries -1661243507.173736 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type MASTER_ELECTION @1661243507173: This node just became node state gatherer as we are fleetcontroller master candidate. -1661243507.173664 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Currently too few followers for cluster controller candidate 0. No current master. (1/3 followers) -1661243507.174536 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got master election state data(0 -> 0, 1 -> 1, 2 -> 2). -1661243507.175447 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Altering our state to follow new fleet controller master candidate 0 -1661243507.175725 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Currently too few followers for cluster controller candidate 0. No current master. (1/3 followers) -1661243507.176648 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Altering our state to follow new fleet controller master candidate 0 -1661243507.236938 localhost 75603/591 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Altered data in node /vespa/fleetcontroller/mycluster/indexes/2. Requesting new vote -1661243507.237154 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 2 for cluster 'mycluster': Stored new vote in ephemeral node. 2 -> 0 -1661243507.237150 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Altered data in node /vespa/fleetcontroller/mycluster/indexes/2. Requesting new vote -1661243507.237144 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 0 for cluster 'mycluster': Read cluster state version 0 from ZooKeeper (znode version 0) -1661243507.236938 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 0 -1661243507.239223 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243507.239776 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote from fleetcontroller 2. Altering vote from null to 0. -1661243507.242825 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243507.244876 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 1 for cluster 'mycluster': Stored new vote in ephemeral node. 1 -> 0 -1661243507.244815 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Altered data in node /vespa/fleetcontroller/mycluster/indexes/1. Requesting new vote -1661243507.244815 localhost 75603/591 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Altered data in node /vespa/fleetcontroller/mycluster/indexes/1. Requesting new vote -1661243507.247572 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type MASTER_ELECTION @1661243507247: This node just became node state gatherer as we are fleetcontroller master candidate. -1661243507.247334 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 0 -1661243507.246575 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243507.249030 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 2. Altering vote from null to 0. -1661243507.247900 localhost 75603/591 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 0 -1661243507.252522 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243507.253123 localhost 75603/591 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote from fleetcontroller 2. Altering vote from null to 0. -1661243507.256577 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243507.259765 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243507.262772 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243507.265782 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243507.268632 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243507.271328 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243507.274020 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243507.276987 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243507.280101 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243507.283056 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243507.285980 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243507.288942 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243507.291763 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243507.294817 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243507.296872 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 1 for cluster 'mycluster': Read cluster state version 0 from ZooKeeper (znode version 0) -1661243507.298037 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243507.298239 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got new fleet data with 3 entries: {0=0, 1=1, 2=0} -1661243507.298192 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Altered data in node /vespa/fleetcontroller/mycluster/indexes/1. Requesting new vote -1661243507.301219 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243507.301209 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Handling new master election, as we have received 3 entries -1661243507.304692 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got master election state data(0 -> 0, 1 -> 1, 2 -> 0). -1661243507.305134 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': 0 is newly elected master -1661243507.305486 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Altering our state to follow new fleet controller master candidate 0 -1661243507.307166 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 0 -1661243507.307720 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 1. Altering vote from null to 0. -1661243507.308367 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got new fleet data with 3 entries: {0=0, 1=0, 2=0} -1661243507.308888 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Handling new master election, as we have received 3 entries -1661243507.309261 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got master election state data(0 -> 0, 1 -> 0, 2 -> 0). -1661243507.309636 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': 0 is newly elected master -1661243507.318165 localhost 75603/591 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 0 -1661243507.318624 localhost 75603/591 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got vote from fleetcontroller 1. Altering vote from null to 0. -1661243507.319044 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Got new fleet data with 3 entries: {0=0, 1=0, 2=0} -1661243507.319487 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Handling new master election, as we have received 3 entries -1661243507.319808 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Got master election state data(0 -> 0, 1 -> 0, 2 -> 0). -1661243507.320135 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': 0 is newly elected master -1661243507.324715 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 0 -1661243507.324770 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 0 for cluster 'mycluster': Read cluster state version 0 from ZooKeeper (znode version 0) -1661243507.325691 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243507.325517 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote from fleetcontroller 1. Altering vote from null to 0. -1661243507.329094 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243507.332316 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got new fleet data with 3 entries: {0=0, 1=0, 2=0} -1661243507.332946 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Handling new master election, as we have received 3 entries -1661243507.333340 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got master election state data(0 -> 0, 1 -> 0, 2 -> 0). -1661243507.333758 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': 0 is still the master -1661243507.334132 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243507.336963 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243507.337582 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 0 for cluster 'mycluster': Loaded previous cluster state bundle from ZooKeeper: ClusterStateBundle('') -1661243507.340246 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243507.340309 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type MASTER_ELECTION @1661243507340: This node just became fleetcontroller master. Bumped version to 1 to be in line. -1661243507.343430 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243507.343874 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: storage.0: Altered node state in cluster state from 'D' to 'U, t 1661243506' -1661243507.346302 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243507.346415 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: distributor.0: Altered node state in cluster state from 'D' to 'U, t 1661243506' -1661243507.349827 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: storage.1: Altered node state in cluster state from 'D' to 'U, t 1661243506' -1661243507.350170 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: distributor.1: Altered node state in cluster state from 'D' to 'U, t 1661243506' -1661243507.350464 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: storage.2: Altered node state in cluster state from 'D' to 'U, t 1661243506' -1661243507.350774 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: distributor.2: Altered node state in cluster state from 'D' to 'U, t 1661243506' -1661243507.349780 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243507.351056 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: storage.3: Altered node state in cluster state from 'D' to 'U, t 1661243506' -1661243507.353651 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: distributor.3: Altered node state in cluster state from 'D' to 'U, t 1661243506' -1661243507.353955 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: storage.4: Altered node state in cluster state from 'D' to 'U, t 1661243506' -1661243507.354220 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: distributor.4: Altered node state in cluster state from 'D' to 'U, t 1661243506' -1661243507.354492 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: storage.5: Altered node state in cluster state from 'D' to 'U, t 1661243506' -1661243507.354858 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: distributor.5: Altered node state in cluster state from 'D' to 'U, t 1661243506' -1661243507.355139 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: storage.6: Altered node state in cluster state from 'D' to 'U, t 1661243506' -1661243507.355411 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: distributor.6: Altered node state in cluster state from 'D' to 'U, t 1661243506' -1661243507.355700 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: storage.7: Altered node state in cluster state from 'D' to 'U, t 1661243506' -1661243507.356001 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: distributor.7: Altered node state in cluster state from 'D' to 'U, t 1661243506' -1661243507.353517 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243507.356298 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: storage.8: Altered node state in cluster state from 'D' to 'U, t 1661243506' -1661243507.358952 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243507.358996 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: distributor.8: Altered node state in cluster state from 'D' to 'U, t 1661243506' -1661243507.361430 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: storage.9: Altered node state in cluster state from 'D' to 'U, t 1661243506' -1661243507.361303 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243507.361720 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: distributor.9: Altered node state in cluster state from 'D' to 'U, t 1661243506' -1661243507.363843 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243507.364021 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type SYSTEMSTATE @1661243507343: New cluster state version 2. Change from last: version: 0 => 2, storage: [0: [Down => Up, startTimestamp: 0 => 1661243506], 1: [Down => Up, startTimestamp: 0 => 1661243506], 2: [Down => Up, startTimestamp: 0 => 1661243506], 3: [Down => Up, startTimestamp: 0 => 1661243506], 4: [Down => Up, startTimestamp: 0 => 1661243506], 5: [Down => Up, startTimestamp: 0 => 1661243506], 6: [Down => Up, startTimestamp: 0 => 1661243506], 7: [Down => Up, startTimestamp: 0 => 1661243506], 8: [Down => Up, startTimestamp: 0 => 1661243506], 9: [Down => Up, startTimestamp: 0 => 1661243506]], distributor: [0: [Down => Up, startTimestamp: 0 => 1661243506], 1: [Down => Up, startTimestamp: 0 => 1661243506], 2: [Down => Up, startTimestamp: 0 => 1661243506], 3: [Down => Up, startTimestamp: 0 => 1661243506], 4: [Down => Up, startTimestamp: 0 => 1661243506], 5: [Down => Up, startTimestamp: 0 => 1661243506], 6: [Down => Up, startTimestamp: 0 => 1661243506], 7: [Down => Up, startTimestamp: 0 => 1661243506], 8: [Down => Up, startTimestamp: 0 => 1661243506], 9: [Down => Up, startTimestamp: 0 => 1661243506]] -1661243507.366138 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243507.367454 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 0 for cluster 'mycluster': Storing new cluster state version in ZooKeeper: 2 -1661243507.369765 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243507.372025 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243507.374386 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243507.376627 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243507.378923 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243507.381198 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243507.383551 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo warning Problem parsing Dummy node host info\nexception=\ncom.fasterxml.jackson.core.JsonParseException: Unrecognized token 'Dummy': was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')\n at [Source: (String)"Dummy node host info"; line: 1, column: 6]\n\tat com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:2391)\n\tat com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:745)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._reportInvalidToken(ReaderBasedJsonParser.java:2961)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser._handleOddValue(ReaderBasedJsonParser.java:2002)\n\tat com.fasterxml.jackson.core.json.ReaderBasedJsonParser.nextToken(ReaderBasedJsonParser.java:802)\n\tat com.fasterxml.jackson.databind.ObjectMapper._initForReading(ObjectMapper.java:4761)\n\tat com.fasterxml.jackson.databind.ObjectMapper._readMapAndClose(ObjectMapper.java:4667)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3629)\n\tat com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3597)\n\tat com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo.createHostInfo(HostInfo.java:54)\n\tat com.yahoo.vespa.clustercontroller.core.NodeStateGatherer.processResponses(NodeStateGatherer.java:144)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.lambda$tick$6(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.MetricUpdater.forWork(MetricUpdater.java:115)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:602)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243507.408998 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 0 for cluster 'mycluster': Master moratorium complete: timed out waiting for all nodes to report in -1661243507.409328 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.SystemStateBroadcaster info FleetController 0 for cluster 'mycluster': Publishing cluster state version 2 -1661243507.414145 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type SYSTEMSTATE @1661243507414: Reset 20 start timestamps as all available distributors have seen newest cluster state. -1661243507.435110 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.testutils.Waiter$Impl info Wait condition not met: Cluster state mismatch -1661243507.435443 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: storage.0: Altered node state in cluster state from 'U, t 1661243506' to 'U' -1661243507.435703 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: distributor.0: Altered node state in cluster state from 'U, t 1661243506' to 'U' -1661243507.435949 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: storage.1: Altered node state in cluster state from 'U, t 1661243506' to 'U' -1661243507.436191 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: distributor.1: Altered node state in cluster state from 'U, t 1661243506' to 'U' -1661243507.436460 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: storage.2: Altered node state in cluster state from 'U, t 1661243506' to 'U' -1661243507.436722 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: distributor.2: Altered node state in cluster state from 'U, t 1661243506' to 'U' -1661243507.437016 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: storage.3: Altered node state in cluster state from 'U, t 1661243506' to 'U' -1661243507.437349 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: distributor.3: Altered node state in cluster state from 'U, t 1661243506' to 'U' -1661243507.437668 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: storage.4: Altered node state in cluster state from 'U, t 1661243506' to 'U' -1661243507.437944 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: distributor.4: Altered node state in cluster state from 'U, t 1661243506' to 'U' -1661243507.438197 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: storage.5: Altered node state in cluster state from 'U, t 1661243506' to 'U' -1661243507.438440 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: distributor.5: Altered node state in cluster state from 'U, t 1661243506' to 'U' -1661243507.438690 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: storage.6: Altered node state in cluster state from 'U, t 1661243506' to 'U' -1661243507.438941 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: distributor.6: Altered node state in cluster state from 'U, t 1661243506' to 'U' -1661243507.439198 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: storage.7: Altered node state in cluster state from 'U, t 1661243506' to 'U' -1661243507.439459 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: distributor.7: Altered node state in cluster state from 'U, t 1661243506' to 'U' -1661243507.439814 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: storage.8: Altered node state in cluster state from 'U, t 1661243506' to 'U' -1661243507.440057 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: distributor.8: Altered node state in cluster state from 'U, t 1661243506' to 'U' -1661243507.440320 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: storage.9: Altered node state in cluster state from 'U, t 1661243506' to 'U' -1661243507.440621 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Added node only event: Event: distributor.9: Altered node state in cluster state from 'U, t 1661243506' to 'U' -1661243507.441004 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type SYSTEMSTATE @1661243507435: New cluster state version 3. Change from last: version: 2 => 3, storage: [0: startTimestamp: 1661243506 => 0, 1: startTimestamp: 1661243506 => 0, 2: startTimestamp: 1661243506 => 0, 3: startTimestamp: 1661243506 => 0, 4: startTimestamp: 1661243506 => 0, 5: startTimestamp: 1661243506 => 0, 6: startTimestamp: 1661243506 => 0, 7: startTimestamp: 1661243506 => 0, 8: startTimestamp: 1661243506 => 0, 9: startTimestamp: 1661243506 => 0], distributor: [0: startTimestamp: 1661243506 => 0, 1: startTimestamp: 1661243506 => 0, 2: startTimestamp: 1661243506 => 0, 3: startTimestamp: 1661243506 => 0, 4: startTimestamp: 1661243506 => 0, 5: startTimestamp: 1661243506 => 0, 6: startTimestamp: 1661243506 => 0, 7: startTimestamp: 1661243506 => 0, 8: startTimestamp: 1661243506 => 0, 9: startTimestamp: 1661243506 => 0] -1661243507.442408 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 0 for cluster 'mycluster': Storing new cluster state version in ZooKeeper: 3 -1661243507.485739 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.SystemStateBroadcaster info FleetController 0 for cluster 'mycluster': Publishing cluster state version 3 -1661243507.701429 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.testutils.Waiter$Impl info Condition met. Returning -1661243507.701749 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Entering waitForMaster -1661243507.702019 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Node 0 is master. Checking that no one else is master -1661243507.702286 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Leaving waitForMaster -1661243507.725313 localhost 75603/548 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Opening socket connection to server localhost/127.0.0.1:51704. -1661243507.725686 localhost 75603/548 - fleetcontroller.org.apache.zookeeper.ClientCnxn info SASL config status: Will not attempt to authenticate using SASL (unknown error) -1661243507.727789 localhost 75603/550 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty warning future isn't success.\nexception=\nio.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: localhost/127.0.0.1:51704\nCaused by: java.net.ConnectException: Connection refused\n\tat java.base/sun.nio.ch.Net.pollConnect(Native Method)\n\tat java.base/sun.nio.ch.Net.pollConnectNow(Net.java:672)\n\tat java.base/sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946)\n\tat io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)\n\tat io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:710)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:658)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:584)\n\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496)\n\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)\n\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243507.730593 localhost 75603/548 - fleetcontroller.org.apache.zookeeper.ClientCnxn warning Session 0x1001abb14b10003 for server localhost/127.0.0.1:51704, Closing socket connection. Attempting reconnect except it is a SessionExpiredException.\nexception=\nEndOfStreamException: channel for sessionid 0x1001abb14b10003 is lost\n\tat org.apache.zookeeper.ClientCnxnSocketNetty.doTransport(ClientCnxnSocketNetty.java:285)\n\tat org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1283)\n -1661243508.017421 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 0 for cluster 'mycluster': Joining event thread. -1661243508.017768 localhost 75603/584 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 0 for cluster 'mycluster': Tick loop stopped -1661243508.018146 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 0 for cluster 'mycluster': FleetController done shutting down event thread. -1661243508.018493 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 0 for cluster 'mycluster': Resetting database state -1661243508.068105 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: A change occurred in the list of registered fleetcontrollers. Requesting new information -1661243508.068712 localhost 75603/597 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is disconnected: [id: 0xd4444b66, L:/127.0.0.1:51925 ! R:localhost/127.0.0.1:51916] -1661243508.068378 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is told closing -1661243508.068326 localhost 75603/591 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: A change occurred in the list of registered fleetcontrollers. Requesting new information -1661243508.068144 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: A change occurred in the list of registered fleetcontrollers. Requesting new information -1661243508.070558 localhost 75603/591 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 0: Got node list response from /vespa/fleetcontroller/mycluster/indexes version -112 with 0 nodes -1661243508.069868 localhost 75603/1 - fleetcontroller.org.apache.zookeeper.ZooKeeper info Session: 0x1001abb1a620000 closed -1661243508.069499 localhost 75603/597 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty info channel is told closing -1661243508.071980 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 0 for cluster 'mycluster': Clearing master data as we lost connection on node 0 -1661243508.071719 localhost 75603/591 - fleetcontroller.org.apache.zookeeper.ClientCnxn info EventThread shut down for session: 0x1001abb1a620000 -1661243508.072885 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type MASTER_ELECTION @1661243508072: This node is no longer fleetcontroller master. -1661243508.074140 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler info FleetController 0 for cluster 'mycluster': Done resetting database state -1661243508.078643 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Entering waitForMaster -1661243508.079373 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Node 1 is not master yet, sleeping more -1661243508.132075 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got node list response from /vespa/fleetcontroller/mycluster/indexes version 0 with 2 nodes -1661243508.132074 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got node list response from /vespa/fleetcontroller/mycluster/indexes version 0 with 2 nodes -1661243508.153050 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 0 -1661243508.153050 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 0 -1661243508.153731 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote from fleetcontroller 1. Altering vote from null to 0. -1661243508.154793 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 0 -1661243508.154262 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 1. Altering vote from null to 0. -1661243508.155463 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote from fleetcontroller 2. Altering vote from null to 0. -1661243508.156080 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 0 -1661243508.156688 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got new fleet data with 2 entries: {1=0, 2=0} -1661243508.157194 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 2. Altering vote from null to 0. -1661243508.157852 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Handling new master election, as we have received 2 entries -1661243508.158860 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got new fleet data with 2 entries: {1=0, 2=0} -1661243508.159575 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got master election state data(1 -> 0, 2 -> 0). -1661243508.160109 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Handling new master election, as we have received 2 entries -1661243508.160600 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Master gone from ZooKeeper. Tagging timestamp. Will wait 1 ms. -1661243508.161198 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got master election state data(1 -> 0, 2 -> 0). -1661243508.161603 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': First index is not currently trying to become master. Waiting for it to change state -1661243508.162088 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Master gone from ZooKeeper. Tagging timestamp. Will wait 1 ms. -1661243508.162693 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': We are next in line to become master. Altering our state to look for followers -1661243508.163348 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': First index is not currently trying to become master. Waiting for it to change state -1661243508.185815 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Altered data in node /vespa/fleetcontroller/mycluster/indexes/1. Requesting new vote -1661243508.185853 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 1 for cluster 'mycluster': Stored new vote in ephemeral node. 1 -> 1 -1661243508.185815 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Altered data in node /vespa/fleetcontroller/mycluster/indexes/1. Requesting new vote -1661243508.186946 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 1 -1661243508.187820 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote from fleetcontroller 1. Altering vote from null to 1. -1661243508.187828 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/1 with code 0 and data 1 -1661243508.188768 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 1. Altering vote from null to 1. -1661243508.188441 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got new fleet data with 2 entries: {1=1, 2=0} -1661243508.189206 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got new fleet data with 2 entries: {1=1, 2=0} -1661243508.189489 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Handling new master election, as we have received 2 entries -1661243508.189858 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Handling new master election, as we have received 2 entries -1661243508.190241 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got master election state data(1 -> 1, 2 -> 0). -1661243508.190638 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got master election state data(1 -> 1, 2 -> 0). -1661243508.191013 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Currently too few followers for cluster controller candidate 1. No current master. (1/3 followers) -1661243508.191604 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Currently too few followers for cluster controller candidate 1. No current master. (1/3 followers) -1661243508.192970 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Altering our state to follow new fleet controller master candidate 1 -1661243508.206683 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Altered data in node /vespa/fleetcontroller/mycluster/indexes/2. Requesting new vote -1661243508.206704 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 2 for cluster 'mycluster': Stored new vote in ephemeral node. 2 -> 1 -1661243508.206689 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Altered data in node /vespa/fleetcontroller/mycluster/indexes/2. Requesting new vote -1661243508.218344 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 1 -1661243508.218344 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote data from path /vespa/fleetcontroller/mycluster/indexes/2 with code 0 and data 1 -1661243508.218941 localhost 75603/618 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 2: Got vote from fleetcontroller 2. Altering vote from null to 1. -1661243508.219369 localhost 75603/604 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.MasterDataGatherer info Fleetcontroller 1: Got vote from fleetcontroller 2. Altering vote from null to 1. -1661243508.219862 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got new fleet data with 2 entries: {1=1, 2=1} -1661243508.220267 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got new fleet data with 2 entries: {1=1, 2=1} -1661243508.220562 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Handling new master election, as we have received 2 entries -1661243508.221046 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Handling new master election, as we have received 2 entries -1661243508.221403 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': Got master election state data(1 -> 1, 2 -> 1). -1661243508.221797 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': Got master election state data(1 -> 1, 2 -> 1). -1661243508.222986 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 1 for cluster 'mycluster': 1 is newly elected master -1661243508.222251 localhost 75603/611 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionHandler info FleetController 2 for cluster 'mycluster': 1 is newly elected master -1661243508.259410 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 1 for cluster 'mycluster': Read cluster state version 3 from ZooKeeper (znode version 2) -1661243508.270984 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 1 for cluster 'mycluster': Loaded previous cluster state bundle from ZooKeeper: ClusterStateBundle('version:3 distributor:10 storage:10', default 'version:3 distributor:10 storage:10', global 'version:3 distributor:10 storage:10' (deferred activation)) -1661243508.272095 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type MASTER_ELECTION @1661243508272: This node just became fleetcontroller master. Bumped version to 3 to be in line. -1661243508.273069 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.EventLog info Cluster event type SYSTEMSTATE @1661243508272: New cluster state version 4. Change from last: version: 3 => 4 -1661243508.273621 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase info FleetController 1 for cluster 'mycluster': Storing new cluster state version in ZooKeeper: 4 -1661243508.301543 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController info FleetController 1 for cluster 'mycluster': Master moratorium complete: timed out waiting for all nodes to report in -1661243508.302386 localhost 75603/598 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.SystemStateBroadcaster info FleetController 1 for cluster 'mycluster': Publishing cluster state version 4 -1661243508.303245 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Node 1 is master. Checking that no one else is master -1661243508.303650 localhost 75603/1 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.MasterElectionTest info Leaving waitForMaster -1661243508.403410 localhost 75603/561 - fleetcontroller.org.apache.zookeeper.ClientCnxn info Opening socket connection to server localhost/[0:0:0:0:0:0:0:1]:51704. -1661243508.403888 localhost 75603/561 - fleetcontroller.org.apache.zookeeper.ClientCnxn info SASL config status: Will not attempt to authenticate using SASL (unknown error) -1661243508.421342 localhost 75603/563 - fleetcontroller.org.apache.zookeeper.ClientCnxnSocketNetty warning future isn't success.\nexception=\nio.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: localhost/[0:0:0:0:0:0:0:1]:51704\nCaused by: java.net.ConnectException: Connection refused\n\tat java.base/sun.nio.ch.Net.pollConnect(Native Method)\n\tat java.base/sun.nio.ch.Net.pollConnectNow(Net.java:672)\n\tat java.base/sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946)\n\tat io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)\n\tat io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:710)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:658)\n\tat io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:584)\n\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496)\n\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986)\n\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n -1661243508.424373 localhost 75603/561 - fleetcontroller.org.apache.zookeeper.ClientCnxn warning Session 0x1001abb14b10004 for server localhost/[0:0:0:0:0:0:0:1]:51704, Closing socket connection. Attempting reconnect except it is a SessionExpiredException.\nexception=\nEndOfStreamException: channel for sessionid 0x1001abb14b10004 is lost\n\tat org.apache.zookeeper.ClientCnxnSocketNetty.doTransport(ClientCnxnSocketNetty.java:285)\n\tat org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1283)\n -java.lang.RuntimeException: Failed to get /vespa/fleetcontroller/mycluster/latestversion - at com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase.retrieveLatestSystemStateVersion(ZooKeeperDatabase.java:228) - at com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler.getLatestSystemStateVersion(DatabaseHandler.java:378) - at com.yahoo.vespa.clustercontroller.core.FleetController.resyncLocallyCachedState(FleetController.java:908) - at com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:607) - at com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120) - at java.base/java.lang.Thread.run(Thread.java:833) -Caused by: org.apache.zookeeper.KeeperException$ConnectionLossException: KeeperErrorCode = ConnectionLoss for /vespa/fleetcontroller/mycluster/latestversion - at org.apache.zookeeper.KeeperException.create(KeeperException.java:102) - at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) - at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:2028) - at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:2056) - at com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase.retrieveLatestSystemStateVersion(ZooKeeperDatabase.java:221) - ... 5 more -1661243508.427229 localhost 75603/560 - fleetcontroller.com.yahoo.vespa.clustercontroller.core.FleetController error FleetController 1 for cluster 'mycluster': Fatal error killed fleet controller\nexception=\njava.lang.RuntimeException: Failed to get /vespa/fleetcontroller/mycluster/latestversion\n\tat com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase.retrieveLatestSystemStateVersion(ZooKeeperDatabase.java:228)\n\tat com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler.getLatestSystemStateVersion(DatabaseHandler.java:378)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.resyncLocallyCachedState(FleetController.java:908)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.tick(FleetController.java:607)\n\tat com.yahoo.vespa.clustercontroller.core.FleetController.run(FleetController.java:1120)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\nCaused by: org.apache.zookeeper.KeeperException$ConnectionLossException: KeeperErrorCode = ConnectionLoss for /vespa/fleetcontroller/mycluster/latestversion\n\tat org.apache.zookeeper.KeeperException.create(KeeperException.java:102)\n\tat org.apache.zookeeper.KeeperException.create(KeeperException.java:54)\n\tat org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:2028)\n\tat org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:2056)\n\tat com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabase.retrieveLatestSystemStateVersion(ZooKeeperDatabase.java:221)\n\t... 5 more\n