From d3920de967062abc616c760e3bce1514c51820fd Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Wed, 13 Nov 2024 10:48:18 +0100 Subject: [PATCH] Move test from internal repo Only change is /home/y/ -> /opt/vespa/ --- tests/vds/vdsstat/1234.dump | 104 +++++++++++++++++++++++++++++++ tests/vds/vdsstat/doc.dump | 46 ++++++++++++++ tests/vds/vdsstat/gid.dump | 46 ++++++++++++++ tests/vds/vdsstat/vdsstat.rb | 117 +++++++++++++++++++++++++++++++++++ 4 files changed, 313 insertions(+) create mode 100644 tests/vds/vdsstat/1234.dump create mode 100644 tests/vds/vdsstat/doc.dump create mode 100644 tests/vds/vdsstat/gid.dump create mode 100644 tests/vds/vdsstat/vdsstat.rb diff --git a/tests/vds/vdsstat/1234.dump b/tests/vds/vdsstat/1234.dump new file mode 100644 index 000000000..b32b78555 --- /dev/null +++ b/tests/vds/vdsstat/1234.dump @@ -0,0 +1,104 @@ +Generated 32-bit bucket id: BucketId(0x80000000000004d2) +Bucket maps to the following actual files: + BucketInfo(BucketId(0x84000000000004d2): [distributor:1] [node(idx=0,crc=0xd4f18fd8,docs=2,bytes=164,trusted=true), node(idx=1,crc=0xd4f18fd8,docs=2,bytes=164,trusted=true)]) + BucketInfo(BucketId(0x84000001000004d2): [distributor:1] [node(idx=0,crc=0x5cb52fd5,docs=1,bytes=82,trusted=true), node(idx=1,crc=0x5cb52fd5,docs=1,bytes=82,trusted=true)]) + +Details for BucketId(0x84000000000004d2): + Bucket information from node 0: + + Filename: '/opt/vespa/var/db/vespa/vds/storage/storage/0/disks/d0/0004/84000000000004d2.0' (fd 34) + Filesize: 1048576 + SlotFileHeader( + version: abcd0001 + meta data list size: 512 + header block size: 102848b + checksum: fd80f89 (OK) + file checksum: d4f18fd8 + ) + Slot(gid(0xd2040000eb8696646878fec0) + Header pos: 0 - 82, Body pos: 0 - 0 + Timestamp: 2010/02/28-09:01:11, Checksum 0x4158, + Flags: 0x 1) + Name: id:storage_test:music:n=1234:1 + Bucket: BucketId(0xe8fe7868000004d2) + Slot(gid(0xd2040000ba5509d4d4685abb) + Header pos: 82 - 82, Body pos: 0 - 0 + Timestamp: 2010/02/28-09:01:12, Checksum 0x1d04, + Flags: 0x 1) + Name: id:storage_test:music:n=1234:2 + Bucket: BucketId(0xeb5a68d4000004d2) + Header block: (102848b) + Content block: (925184b) + + + Bucket information from node 1: + + Filename: '/opt/vespa/var/db/vespa/vds/storage/storage/1/disks/d0/0004/84000000000004d2.0' (fd 34) + Filesize: 1048576 + SlotFileHeader( + version: abcd0001 + meta data list size: 512 + header block size: 102848b + checksum: fd80f89 (OK) + file checksum: d4f18fd8 + ) + Slot(gid(0xd2040000eb8696646878fec0) + Header pos: 0 - 82, Body pos: 0 - 0 + Timestamp: 2010/02/28-09:01:11, Checksum 0x4158, + Flags: 0x 1) + Name: id:storage_test:music:n=1234:1 + Bucket: BucketId(0xe8fe7868000004d2) + Slot(gid(0xd2040000ba5509d4d4685abb) + Header pos: 82 - 82, Body pos: 0 - 0 + Timestamp: 2010/02/28-09:01:12, Checksum 0x1d04, + Flags: 0x 1) + Name: id:storage_test:music:n=1234:2 + Bucket: BucketId(0xeb5a68d4000004d2) + Header block: (102848b) + Content block: (925184b) + + + +Details for BucketId(0x84000001000004d2): + Bucket information from node 0: + + Filename: '/opt/vespa/var/db/vespa/vds/storage/storage/0/disks/d0/00f0/84000001000004d2.0' (fd 34) + Filesize: 1048576 + SlotFileHeader( + version: abcd0001 + meta data list size: 512 + header block size: 102848b + checksum: fd80f89 (OK) + file checksum: 5cb52fd5 + ) + Slot(gid(0xd2040000050dfa54ab6befc9) + Header pos: 0 - 82, Body pos: 0 - 0 + Timestamp: 2010/02/28-09:01:12, Checksum 0x4585, + Flags: 0x 1) + Name: id:storage_test:music:n=1234:3 + Bucket: BucketId(0xe9ef6bab000004d2) + Header block: (102848b) + Content block: (925184b) + + + Bucket information from node 1: + + Filename: '/opt/vespa/var/db/vespa/vds/storage/storage/1/disks/d0/00f0/84000001000004d2.0' (fd 34) + Filesize: 1048576 + SlotFileHeader( + version: abcd0001 + meta data list size: 512 + header block size: 102848b + checksum: fd80f89 (OK) + file checksum: 5cb52fd5 + ) + Slot(gid(0xd2040000050dfa54ab6befc9) + Header pos: 0 - 82, Body pos: 0 - 0 + Timestamp: 2010/02/28-09:01:12, Checksum 0x4585, + Flags: 0x 1) + Name: id:storage_test:music:n=1234:3 + Bucket: BucketId(0xe9ef6bab000004d2) + Header block: (102848b) + Content block: (925184b) + + diff --git a/tests/vds/vdsstat/doc.dump b/tests/vds/vdsstat/doc.dump new file mode 100644 index 000000000..e6a25a197 --- /dev/null +++ b/tests/vds/vdsstat/doc.dump @@ -0,0 +1,46 @@ +Bucket maps to the following actual files: + BucketInfo(BucketId(0x840000003a7455d7): [distributor:1] [node(idx=2,crc=0xe5beb5bd,docs=1,bytes=90,trusted=true), node(idx=0,crc=0xe5beb5bd,docs=1,bytes=90,trusted=true)]) + +Details for BucketId(0x840000003a7455d7): + Bucket information from node 0: + + Filename: '/opt/vespa/var/db/vespa/vds/storage/storage/0/disks/d0/00a5/840000003a7455d7.0' (fd 35) + Filesize: 1048576 + SlotFileHeader( + version: abcd0001 + meta data list size: 512 + header block size: 102848b + checksum: fd80f89 (OK) + file checksum: e5beb5bd + ) + Slot(gid(0xd755743a3d5cf9efd03deb76) + Header pos: 0 - 90, Body pos: 0 - 0 + Timestamp: 2010/02/28-09:01:12, Checksum 0xc1b2, + Flags: 0x 1) + Name: id:storage_test:music:g=mygroup:2 + Bucket: BucketId(0xeaeb3dd03a7455d7) + Header block: (102848b) + Content block: (925184b) + + + Bucket information from node 2: + + Filename: '/opt/vespa/var/db/vespa/vds/storage/storage/2/disks/d0/00a5/840000003a7455d7.0' (fd 34) + Filesize: 1048576 + SlotFileHeader( + version: abcd0001 + meta data list size: 512 + header block size: 102848b + checksum: fd80f89 (OK) + file checksum: e5beb5bd + ) + Slot(gid(0xd755743a3d5cf9efd03deb76) + Header pos: 0 - 90, Body pos: 0 - 0 + Timestamp: 2010/02/28-09:01:12, Checksum 0xc1b2, + Flags: 0x 1) + Name: id:storage_test:music:g=mygroup:2 + Bucket: BucketId(0xeaeb3dd03a7455d7) + Header block: (102848b) + Content block: (925184b) + + diff --git a/tests/vds/vdsstat/gid.dump b/tests/vds/vdsstat/gid.dump new file mode 100644 index 000000000..e6a25a197 --- /dev/null +++ b/tests/vds/vdsstat/gid.dump @@ -0,0 +1,46 @@ +Bucket maps to the following actual files: + BucketInfo(BucketId(0x840000003a7455d7): [distributor:1] [node(idx=2,crc=0xe5beb5bd,docs=1,bytes=90,trusted=true), node(idx=0,crc=0xe5beb5bd,docs=1,bytes=90,trusted=true)]) + +Details for BucketId(0x840000003a7455d7): + Bucket information from node 0: + + Filename: '/opt/vespa/var/db/vespa/vds/storage/storage/0/disks/d0/00a5/840000003a7455d7.0' (fd 35) + Filesize: 1048576 + SlotFileHeader( + version: abcd0001 + meta data list size: 512 + header block size: 102848b + checksum: fd80f89 (OK) + file checksum: e5beb5bd + ) + Slot(gid(0xd755743a3d5cf9efd03deb76) + Header pos: 0 - 90, Body pos: 0 - 0 + Timestamp: 2010/02/28-09:01:12, Checksum 0xc1b2, + Flags: 0x 1) + Name: id:storage_test:music:g=mygroup:2 + Bucket: BucketId(0xeaeb3dd03a7455d7) + Header block: (102848b) + Content block: (925184b) + + + Bucket information from node 2: + + Filename: '/opt/vespa/var/db/vespa/vds/storage/storage/2/disks/d0/00a5/840000003a7455d7.0' (fd 34) + Filesize: 1048576 + SlotFileHeader( + version: abcd0001 + meta data list size: 512 + header block size: 102848b + checksum: fd80f89 (OK) + file checksum: e5beb5bd + ) + Slot(gid(0xd755743a3d5cf9efd03deb76) + Header pos: 0 - 90, Body pos: 0 - 0 + Timestamp: 2010/02/28-09:01:12, Checksum 0xc1b2, + Flags: 0x 1) + Name: id:storage_test:music:g=mygroup:2 + Bucket: BucketId(0xeaeb3dd03a7455d7) + Header block: (102848b) + Content block: (925184b) + + diff --git a/tests/vds/vdsstat/vdsstat.rb b/tests/vds/vdsstat/vdsstat.rb new file mode 100644 index 000000000..09645f5c5 --- /dev/null +++ b/tests/vds/vdsstat/vdsstat.rb @@ -0,0 +1,117 @@ +# Copyright Vespa.ai. All rights reserved. +require 'vds_test' + +class VdsStat < VdsTest + + def setup + set_owner("vekterli") + deploy_app(default_app. + bucket_split_count(2). + num_nodes(3). + redundancy(2)) + start + end + + def cleanstring(str) + return str.gsub(/crc=0x[0-9a-f]+,/, "crc=,"). + gsub(/docs=([0-9]+)\/[0-9]+/, "docs=\\1/"). + gsub(/bytes=[0-9]+\/[0-9]+/, "bytes="). + gsub(/idx=[0-9]+/, "idx="). + gsub(/active=[a-z]+/, "active="). + gsub(/distributor:[0-9]+/, "distributor:x"). + gsub(/node [0-9]+/,""). + gsub(/checksum: [0-9a-f]+/, ""). + gsub(/Timestamp:[^,]*,/,"Timestamp:,"). + gsub(/Checksum 0x[0-9a-f]+,/, ""). + gsub(/ \(fd [0-9]+\)/, ""). + gsub(/^\s+/, ""). + gsub(/Header block: \([0-9]+b\)/, "Header block: (X b)"). + gsub(/Header pos: ([^-]+)/, "Header pos: X -"). + gsub(/header block size: [0-9]+b/, "header block size: X b"). + gsub(/meta data list size: [0-9]+/, "meta data list size: X"). + gsub(/Content block: \([0-9]+b\)/, "Content block: (X b)"). + gsub(/Filesize: ([0-9]+)/, "Filesize: X"). + gsub(/storage\/[0-9]\/disks/, "storage/X/disks") + end + + def test_stat + doc = Document.new("music", "id:storage_test:music:n=1234:1") + vespa.document_api_v1.put(doc) + + doc = Document.new("music", "id:storage_test:music:n=1234:2") + vespa.document_api_v1.put(doc) + + doc = Document.new("music", "id:storage_test:music:n=1234:3") + vespa.document_api_v1.put(doc) + + # 66770 is in same bucket as 1234 + doc = Document.new("music", "id:storage_test:music:n=66770:1") + vespa.document_api_v1.put(doc) + + doc = Document.new("music", "id:storage_test:music:n=66770:2") + vespa.document_api_v1.put(doc) + + doc = Document.new("music", "id:storage_test:music:n=66770:3") + vespa.document_api_v1.put(doc) + + # Feed a group as well. + doc = Document.new("music", "id:storage_test:music:g=mygroup:1") + vespa.document_api_v1.put(doc) + + doc = Document.new("music", "id:storage_test:music:g=mygroup:2") + vespa.document_api_v1.put(doc) + + doc = Document.new("music", "id:storage_test:music:g=mygroup:3") + vespa.document_api_v1.put(doc) + + vespa.storage["storage"].wait_until_ready + + value = vespa.storage["storage"].storage["0"].execute("vespa-stat --user 1234") + + correct = "Generated 32-bit bucket id: BucketId(0x80000000000004d2)\n" + + "Bucket maps to the following actual files:\n" + + "\tBucketInfo(BucketId(0x84000000000004d2): [distributor:0] [node(idx=1,crc=0x2bfee69e,docs=1/1,bytes=82/82,trusted=true,active=false,ready=true), node(idx=2,crc=0x2bfee69e,docs=1/1,bytes=82/82,trusted=true,active=false,ready=true)])\n" + + "\tBucketInfo(BucketId(0x84000001000004d2): [distributor:0] [node(idx=0,crc=0x62e9afd6,docs=2/2,bytes=164/164,trusted=true,active=false,ready=true), node(idx=1,crc=0x62e9afd6,docs=2/2,bytes=164/164,trusted=true,active=false,ready=true)])\n"; + + assert_equal(cleanstring(correct), cleanstring(value)) + + value = vespa.storage["storage"].storage["0"].execute("vespa-stat --bucket 0x40000000000004d2") + + correct = "Bucket maps to the following actual files:\n" + + "\tBucketInfo(BucketId(0x84000000000004d2): [distributor:0] [node(idx=1,crc=0x2bfee69e,docs=1/1,bytes=82/82,trusted=true,active=false,ready=true), node(idx=2,crc=0x2bfee69e,docs=1/1,bytes=82/82,trusted=true,active=false,ready=true)])\n" + + "\tBucketInfo(BucketId(0x84000001000004d2): [distributor:0] [node(idx=0,crc=0x62e9afd6,docs=2/2,bytes=164/164,trusted=true,active=false,ready=true), node(idx=1,crc=0x62e9afd6,docs=2/2,bytes=164/164,trusted=true,active=false,ready=true)])\n" + + "\tBucketInfo(BucketId(0x84000000000104d2): [distributor:0] [node(idx=0,crc=0xa5fc1520,docs=1/1,bytes=84/84,trusted=true,active=false,ready=true), node(idx=1,crc=0xa5fc1520,docs=1/1,bytes=84/84,trusted=true,active=false,ready=true)])\n" + + "\tBucketInfo(BucketId(0x84000001000104d2): [distributor:0] [node(idx=1,crc=0x3b66fcd2,docs=2/2,bytes=168/168,trusted=true,active=false,ready=true), node(idx=2,crc=0x3b66fcd2,docs=2/2,bytes=168/168,trusted=true,active=false,ready=true)])\n" + + assert_equal(cleanstring(correct), cleanstring(value)) + + value = vespa.storage["storage"].storage["0"].execute("vespa-stat --group mygroup") + + correct = "Generated 32-bit bucket id: BucketId(0x800000003a7455d7)\n" + + "Bucket maps to the following actual files:\n" + + "\tBucketInfo(BucketId(0x840000003a7455d7): [distributor:0] [node(idx=0,crc=0x3c6277a7,docs=1/1,bytes=90/90,trusted=true,active=false,ready=true), node(idx=2,crc=0x3c6277a7,docs=1/1,bytes=90/90,trusted=true,active=false,ready=true)])\n" + + "\tBucketInfo(BucketId(0x840000013a7455d7): [distributor:0] [node(idx=0,crc=0x643a2091,docs=2/2,bytes=180/180,trusted=true,active=false,ready=true), node(idx=1,crc=0x643a2091,docs=2/2,bytes=180/180,trusted=true,active=false,ready=true)])\n" + + assert_equal(cleanstring(correct), cleanstring(value)) + + value = vespa.storage["storage"].storage["0"].execute("vespa-stat --user 1234 --dump") + fname = selfdir + "/1234.dump" + correct = File.open(fname) { |f| f.read } + assert(value =~ /Doc/) + + value = vespa.storage["storage"].storage["0"].execute("vespa-stat --document id:storage_test:music:g=mygroup:2") + fname = selfdir + "/doc.dump" + correct = File.open(fname) { |f| f.read } + assert(value =~ /Doc/) + + value = vespa.storage["storage"].storage["0"].execute("vespa-stat --gid 0xd755743a4624d818b89abe0f") + fname = selfdir + "/gid.dump" + correct = File.open(fname) { |f| f.read } + assert(value =~ /Doc/) + end + + def teardown + stop + end + +end