From 5cbc6baf0e1154ad3f88d91d30fcead6893e9b0c Mon Sep 17 00:00:00 2001 From: Amey Varhade Date: Thu, 7 Nov 2024 16:11:06 +0530 Subject: [PATCH 1/6] add vertices in sorted order --- src/index.cpp | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/src/index.cpp b/src/index.cpp index bf93344fa..5d3dfd086 100644 --- a/src/index.cpp +++ b/src/index.cpp @@ -1268,6 +1268,18 @@ void Index::inter_insert(uint32_t n, std::vector &pru inter_insert(n, pruned_list, _indexingRange, scratch); } +template +bool comp(const std::pair &a, const std::pair &b) +{ + if (a.first < b.first){ + return true; + } + else if (a.first == b.first){ + return a.second < b.second; + } + return false; +} + template void Index::link() { uint32_t num_threads = _indexingThreads; @@ -1279,10 +1291,23 @@ template void Index pool, tmp; tsl::robin_set visited; visit_order.reserve(_nd + _num_frozen_pts); + + // if there are frozen points, the first such one is set to be the _start + if (_num_frozen_pts > 0) + _start = (uint32_t)_max_points; + else + _start = calculate_entry_point(); + + std::vector> distances_to_mediod; for (uint32_t i = 0; i < (uint32_t)_nd; i++) { - visit_order.emplace_back(i); + float cur_distance = _data_store->get_distance(i, _start); + distances_to_mediod.emplace_back(std::make_pair(cur_distance, i)); } + std::sort(distances_to_mediod.begin(), distances_to_mediod.end(), comp); + + for (uint32_t i = 0; i < (uint32_t)_nd; i++) + visit_order.emplace_back(distances_to_mediod[i].second); // If there are any frozen points, add them all. for (uint32_t frozen = (uint32_t)_max_points; frozen < _max_points + _num_frozen_pts; frozen++) @@ -1290,12 +1315,6 @@ template void Index 0) - _start = (uint32_t)_max_points; - else - _start = calculate_entry_point(); - diskann::Timer link_timer; #pragma omp parallel for schedule(dynamic, 2048) From 4afeed41413788e02f1f6196d003522602a3ec77 Mon Sep 17 00:00:00 2001 From: Amey Varhade Date: Thu, 7 Nov 2024 16:11:58 +0530 Subject: [PATCH 2/6] update gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index f80e5c682..a9582aaa6 100644 --- a/.gitignore +++ b/.gitignore @@ -113,6 +113,9 @@ ipch/ *.vspx *.sap +# Visual Studio Code Settings +.vscode + # Visual Studio Trace Files *.e2e From 051c69f8e7eb60804f8fdad6bc60f1801f79f8d2 Mon Sep 17 00:00:00 2001 From: Amey Varhade Date: Thu, 7 Nov 2024 16:19:41 +0530 Subject: [PATCH 3/6] add logs --- src/index.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/index.cpp b/src/index.cpp index 5d3dfd086..eee571b4f 100644 --- a/src/index.cpp +++ b/src/index.cpp @@ -1321,6 +1321,9 @@ template void Index> manager(_query_scratch); From 1b1661bb87ac61abb53ec3a60256ab6b61e15419 Mon Sep 17 00:00:00 2001 From: ravishankar Date: Thu, 7 Nov 2024 04:23:01 -0800 Subject: [PATCH 4/6] add sorted results --- prec40M/baseline_r64_k10.txt | 84 +++++++ prec40M/baseline_r64_k100.txt | 21 ++ prec40M/baseline_r64_k25.txt | 70 ++++++ prec40M/baseline_r64_k50.txt | 56 +++++ prec40M/baseline_r64_l100.txt | 10 + prec40M/sorted_r64_k50.txt | 56 +++++ prec40M/sorted_r64_l100.txt | 413 ++++++++++++++++++++++++++++++++++ run-search.sh | 2 + src/index.cpp | 8 +- 9 files changed, 716 insertions(+), 4 deletions(-) create mode 100644 prec40M/baseline_r64_k10.txt create mode 100644 prec40M/baseline_r64_k100.txt create mode 100644 prec40M/baseline_r64_k25.txt create mode 100644 prec40M/baseline_r64_k50.txt create mode 100644 prec40M/baseline_r64_l100.txt create mode 100644 prec40M/sorted_r64_k50.txt create mode 100644 prec40M/sorted_r64_l100.txt create mode 100644 run-search.sh diff --git a/prec40M/baseline_r64_k10.txt b/prec40M/baseline_r64_k10.txt new file mode 100644 index 000000000..6b9af1d0e --- /dev/null +++ b/prec40M/baseline_r64_k10.txt @@ -0,0 +1,84 @@ +Reading (with alignment) bin file /nvmessd1/fbv4/queries384d.bin ...Metadata: #pts = 13265, #dims = 384, aligned_dim = 384... allocating aligned memory of 5093760 bytes... done. Copying data to mem_aligned buffer... done. +Opened: /nvmessd1/fbv4/gt100_prec40M, size: 10612008, cache_size: 10612008 +Reading truthset file /nvmessd1/fbv4/gt100_prec40M ... +Metadata: #pts = 13265, #dims = 100... +Using AVX2 distance computation DistanceL2Int8. +Resizing took: 1.40021s +From graph header, expected_file_size: 7438757948, _max_observed_degree: 64, _start: 11890341, file_frozen_pts: 0 +Loading vamana graph /nvmessd1/fbv4/avarhade/prec40M_memory_index_r64_l100.......done. Index has 40469609 nodes and 1819219872 out-edges, _start is set to 11890341 +Num frozen points:0 _nd: 40469609 _start: 11890341 size(_location_to_tag): 0 size(_tag_to_location):0 Max points: 40469609 +Index loaded +Using 48 threads to search + Ls QPS Avg dist cmps Mean Latency (mus) 99.9 Latency Recall@10 +================================================================================= + 10 51557.93 1324.67 913.18 9792.77 43.13 + 20 33640.81 2021.14 1420.26 3249.95 58.38 + 30 25210.59 2617.97 1896.59 4469.27 66.24 + 40 20547.19 3168.06 2326.82 5450.12 71.25 + 50 17328.33 3687.22 2760.19 6599.82 74.44 + 60 14980.50 4195.96 3193.32 7739.67 76.98 + 70 13341.03 4685.40 3588.47 8306.07 78.78 + 80 11462.47 5171.15 4176.58 11172.33 80.21 + 90 10723.69 5654.48 4462.36 10865.74 81.53 + 100 9782.57 6129.37 4892.21 12162.22 82.58 +Done searching. Now saving results +Writing bin: /home/rakri/avarhade/Dump/tmp_10_idx_uint32.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_10_dists_float.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_20_idx_uint32.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_20_dists_float.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_30_idx_uint32.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_30_dists_float.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_40_idx_uint32.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_40_dists_float.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_50_idx_uint32.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_50_dists_float.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_60_idx_uint32.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_60_dists_float.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_70_idx_uint32.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_70_dists_float.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_80_idx_uint32.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_80_dists_float.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_90_idx_uint32.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_90_dists_float.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_100_idx_uint32.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_100_dists_float.bin +bin: #pts = 13265, #dims = 10, size = 530608B +Finished writing bin. diff --git a/prec40M/baseline_r64_k100.txt b/prec40M/baseline_r64_k100.txt new file mode 100644 index 000000000..b6c634e16 --- /dev/null +++ b/prec40M/baseline_r64_k100.txt @@ -0,0 +1,21 @@ +Reading (with alignment) bin file /nvmessd1/fbv4/queries384d.bin ...Metadata: #pts = 13265, #dims = 384, aligned_dim = 384... allocating aligned memory of 5093760 bytes... done. Copying data to mem_aligned buffer... done. +Opened: /nvmessd1/fbv4/gt100_prec40M, size: 10612008, cache_size: 10612008 +Reading truthset file /nvmessd1/fbv4/gt100_prec40M ... +Metadata: #pts = 13265, #dims = 100... +Using AVX2 distance computation DistanceL2Int8. +Resizing took: 1.40809s +From graph header, expected_file_size: 7438757948, _max_observed_degree: 64, _start: 11890341, file_frozen_pts: 0 +Loading vamana graph /nvmessd1/fbv4/avarhade/prec40M_memory_index_r64_l100.......done. Index has 40469609 nodes and 1819219872 out-edges, _start is set to 11890341 +Num frozen points:0 _nd: 40469609 _start: 11890341 size(_location_to_tag): 0 size(_tag_to_location):0 Max points: 40469609 +Index loaded +Using 48 threads to search + Ls QPS Avg dist cmps Mean Latency (mus) 99.9 Latency Recall@100 +================================================================================= + 100 10331.70 6129.37 4626.05 11214.14 66.81 +Done searching. Now saving results +Writing bin: /home/rakri/avarhade/Dump/tmp_100_idx_uint32.bin +bin: #pts = 13265, #dims = 100, size = 5306008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_100_dists_float.bin +bin: #pts = 13265, #dims = 100, size = 5306008B +Finished writing bin. diff --git a/prec40M/baseline_r64_k25.txt b/prec40M/baseline_r64_k25.txt new file mode 100644 index 000000000..eaf63e5de --- /dev/null +++ b/prec40M/baseline_r64_k25.txt @@ -0,0 +1,70 @@ +Reading (with alignment) bin file /nvmessd1/fbv4/queries384d.bin ...Metadata: #pts = 13265, #dims = 384, aligned_dim = 384... allocating aligned memory of 5093760 bytes... done. Copying data to mem_aligned buffer... done. +Opened: /nvmessd1/fbv4/gt100_prec40M, size: 10612008, cache_size: 10612008 +Reading truthset file /nvmessd1/fbv4/gt100_prec40M ... +Metadata: #pts = 13265, #dims = 100... +Using AVX2 distance computation DistanceL2Int8. +Resizing took: 1.48129s +From graph header, expected_file_size: 7438757948, _max_observed_degree: 64, _start: 11890341, file_frozen_pts: 0 +Loading vamana graph /nvmessd1/fbv4/avarhade/prec40M_memory_index_r64_l100.......done. Index has 40469609 nodes and 1819219872 out-edges, _start is set to 11890341 +Num frozen points:0 _nd: 40469609 _start: 11890341 size(_location_to_tag): 0 size(_tag_to_location):0 Max points: 40469609 +Index loaded +Using 48 threads to search + Ls QPS Avg dist cmps Mean Latency (mus) 99.9 Latency Recall@25 +================================================================================= + 30 25253.53 2617.97 1885.01 10497.99 60.18 + 40 20654.60 3168.06 2316.21 5201.40 65.38 + 50 17413.17 3687.22 2747.47 6720.12 68.82 + 60 15010.45 4195.96 3187.87 7331.08 71.60 + 70 13300.10 4685.40 3599.56 8146.48 73.66 + 80 11990.02 5171.15 3991.75 8909.45 75.35 + 90 10918.86 5654.48 4385.05 9846.92 76.87 + 100 9976.54 6129.37 4799.29 10763.89 78.14 +Done searching. Now saving results +Writing bin: /home/rakri/avarhade/Dump/tmp_30_idx_uint32.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_30_dists_float.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_40_idx_uint32.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_40_dists_float.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_50_idx_uint32.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_50_dists_float.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_60_idx_uint32.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_60_dists_float.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_70_idx_uint32.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_70_dists_float.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_80_idx_uint32.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_80_dists_float.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_90_idx_uint32.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_90_dists_float.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_100_idx_uint32.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_100_dists_float.bin +bin: #pts = 13265, #dims = 25, size = 1326508B +Finished writing bin. diff --git a/prec40M/baseline_r64_k50.txt b/prec40M/baseline_r64_k50.txt new file mode 100644 index 000000000..af184dc2b --- /dev/null +++ b/prec40M/baseline_r64_k50.txt @@ -0,0 +1,56 @@ +Reading (with alignment) bin file /nvmessd1/fbv4/queries384d.bin ...Metadata: #pts = 13265, #dims = 384, aligned_dim = 384... allocating aligned memory of 5093760 bytes... done. Copying data to mem_aligned buffer... done. +Opened: /nvmessd1/fbv4/gt100_prec40M, size: 10612008, cache_size: 10612008 +Reading truthset file /nvmessd1/fbv4/gt100_prec40M ... +Metadata: #pts = 13265, #dims = 100... +Using AVX2 distance computation DistanceL2Int8. +Resizing took: 1.41607s +From graph header, expected_file_size: 7438757948, _max_observed_degree: 64, _start: 11890341, file_frozen_pts: 0 +Loading vamana graph /nvmessd1/fbv4/avarhade/prec40M_memory_index_r64_l100.......done. Index has 40469609 nodes and 1819219872 out-edges, _start is set to 11890341 +Num frozen points:0 _nd: 40469609 _start: 11890341 size(_location_to_tag): 0 size(_tag_to_location):0 Max points: 40469609 +Index loaded +Using 48 threads to search + Ls QPS Avg dist cmps Mean Latency (mus) 99.9 Latency Recall@50 +================================================================================= + 50 17629.05 3687.22 2705.62 14582.74 62.96 + 60 14420.19 4195.96 3319.41 9059.50 65.92 + 70 12430.30 4685.40 3850.11 10474.53 68.19 + 80 12256.81 5171.15 3903.45 9068.07 70.07 + 90 11310.12 5654.48 4230.41 9502.88 71.76 + 100 10462.90 6129.37 4574.90 9994.75 73.17 +Done searching. Now saving results +Writing bin: /home/rakri/avarhade/Dump/tmp_50_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_50_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_60_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_60_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_70_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_70_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_80_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_80_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_90_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_90_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_100_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_100_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. diff --git a/prec40M/baseline_r64_l100.txt b/prec40M/baseline_r64_l100.txt new file mode 100644 index 000000000..b86d0c55f --- /dev/null +++ b/prec40M/baseline_r64_l100.txt @@ -0,0 +1,10 @@ +Starting index build with R: 64 Lbuild: 100 alpha: 1.2 #threads: 48 +Using AVX2 distance computation DistanceL2Int8. +Using only first 40469609 from file.. +Starting index build with 40469609 points... + 0% of index build completed. 0.247099% of index build completed. 0.494198% of index build completed. 0.741297% of index build completed. 0.988396% of index build completed. 1.2355% of index build completed. 1.48259% of index build completed. 1.72969% of index build completed. 1.97679% of index build completed. 2.22389% of index build completed. 2.47099% of index build completed. 2.71809% of index build completed. 2.96519% of index build completed. 3.21229% of index build completed. 3.45939% of index build completed. 3.70649% of index build completed. 3.95358% of index build completed. 4.20068% of index build completed. 4.44778% of index build completed. 4.69488% of index build completed. 4.94198% of index build completed. 5.18908% of index build completed. 5.43618% of index build completed. 5.68328% of index build completed. 5.93038% of index build completed. 6.17748% of index build completed. 6.42457% of index build completed. 6.67167% of index build completed. 6.91877% of index build completed. 7.16587% of index build completed. 7.41297% of index build completed. 7.66007% of index build completed. 7.90717% of index build completed. 8.15427% of index build completed. 8.40137% of index build completed. 8.64847% of index build completed. 8.89556% of index build completed. 9.14266% of index build completed. 9.38976% of index build completed. 9.63686% of index build completed. 9.88396% of index build completed. 10.1311% of index build completed. 10.3782% of index build completed. 10.6253% of index build completed. 10.8724% of index build completed. 11.1195% of index build completed. 11.3666% of index build completed. 11.6137% of index build completed. 11.8608% of index build completed. 12.1079% of index build completed. 12.355% of index build completed. 12.602% of index build completed. 12.8491% of index build completed. 13.0962% of index build completed. 13.3433% of index build completed. 13.5904% of index build completed. 13.8375% of index build completed. 14.0846% of index build completed. 14.3317% of index build completed. 14.5788% of index build completed. 14.8259% of index build completed. 15.073% of index build completed. 15.3201% of index build completed. 15.5672% of index build completed. 15.8143% of index build completed. 16.0614% of index build completed. 16.3085% of index build completed. 16.5556% of index build completed. 16.8027% of index build completed. 17.0498% of index build completed. 17.2969% of index build completed. 17.544% of index build completed. 17.7911% of index build completed. 18.0382% of index build completed. 18.2853% of index build completed. 18.5324% of index build completed. 18.7795% of index build completed. 19.0266% of index build completed. 19.2737% of index build completed. 19.5208% of index build completed. 19.7679% of index build completed. 20.015% of index build completed. 20.2621% of index build completed. 20.5092% of index build completed. 20.7563% of index build completed. 21.0034% of index build completed. 21.2505% of index build completed. 21.4976% of index build completed. 21.7447% of index build completed. 21.9918% of index build completed. 22.2389% of index build completed. 22.486% of index build completed. 22.7331% of index build completed. 22.9802% of index build completed. 23.2273% of index build completed. 23.4744% of index build completed. 23.7215% of index build completed. 23.9686% of index build completed. 24.2157% of index build completed. 24.4628% of index build completed. 24.7099% of index build completed. 24.957% of index build completed. 25.2041% of index build completed. 25.4512% of index build completed. 25.6983% of index build completed. 25.9454% of index build completed. 26.1925% of index build completed. 26.4396% of index build completed. 26.6867% of index build completed. 26.9338% of index build completed. 27.1809% of index build completed. 27.428% of index build completed. 27.6751% of index build completed. 27.9222% of index build completed. 28.1693% of index build completed. 28.4164% of index build completed. 28.6635% of index build completed. 28.9106% of index build completed. 29.1577% of index build completed. 29.4048% of index build completed. 29.6519% of index build completed. 29.899% of index build completed. 30.1461% of index build completed. 30.3932% of index build completed. 30.6403% of index build completed. 30.8874% of index build completed. 31.1345% of index build completed. 31.3816% of index build completed. 31.6287% of index build completed. 31.8758% of index build completed. 32.1229% of index build completed. 32.37% of index build completed. 32.6171% of index build completed. 32.8642% of index build completed. 33.1113% of index build completed. 33.3584% of index build completed. 33.6055% of index build completed. 33.8526% of index build completed. 34.0997% of index build completed. 34.3468% of index build completed. 34.5939% of index build completed. 34.841% of index build completed. 35.0881% of index build completed. 35.3352% of index build completed. 35.5823% of index build completed. 35.8294% of index build completed. 36.0765% of index build completed. 36.3236% of index build completed. 36.5707% of index build completed. 36.8178% of index build completed. 37.0649% of index build completed. 37.3119% of index build completed. 37.559% of index build completed. 37.8061% of index build completed. 38.0532% of index build completed. 38.3003% of index build completed. 38.5474% of index build completed. 38.7945% of index build completed. 39.0416% of index build completed. 39.2887% of index build completed. 39.5358% of index build completed. 39.7829% of index build completed. 40.03% of index build completed. 40.2771% of index build completed. 40.5242% of index build completed. 40.7713% of index build completed. 41.0184% of index build completed. 41.2655% of index build completed. 41.5126% of index build completed. 41.7597% of index build completed. 42.0068% of index build completed. 42.2539% of index build completed. 42.501% of index build completed. 42.7481% of index build completed. 42.9952% of index build completed. 43.2423% of index build completed. 43.4894% of index build completed. 43.7365% of index build completed. 43.9836% of index build completed. 44.2307% of index build completed. 44.4778% of index build completed. 44.7249% of index build completed. 44.972% of index build completed. 45.2191% of index build completed. 45.4662% of index build completed. 45.7133% of index build completed. 45.9604% of index build completed. 46.2075% of index build completed. 46.4546% of index build completed. 46.7017% of index build completed. 46.9488% of index build completed. 47.1959% of index build completed. 47.443% of index build completed. 47.6901% of index build completed. 47.9372% of index build completed. 48.1843% of index build completed. 48.4314% of index build completed. 48.6785% of index build completed. 48.9256% of index build completed. 49.1727% of index build completed. 49.4198% of index build completed. 49.6669% of index build completed. 49.914% of index build completed. 50.1611% of index build completed. 50.4082% of index build completed. 50.6553% of index build completed. 50.9024% of index build completed. 51.1495% of index build completed. 51.3966% of index build completed. 51.6437% of index build completed. 51.8908% of index build completed. 52.1379% of index build completed. 52.385% of index build completed. 52.6321% of index build completed. 52.8792% of index build completed. 53.1263% of index build completed. 53.3734% of index build completed. 53.6205% of index build completed. 53.8676% of index build completed. 54.1147% of index build completed. 54.3618% of index build completed. 54.6089% of index build completed. 54.856% of index build completed. 55.1031% of index build completed. 55.3502% of index build completed. 55.5973% of index build completed. 55.8444% of index build completed. 56.0915% of index build completed. 56.3386% of index build completed. 56.5857% of index build completed. 56.8328% of index build completed. 57.0799% of index build completed. 57.327% of index build completed. 57.5741% of index build completed. 57.8212% of index build completed. 58.0683% of index build completed. 58.3154% of index build completed. 58.5625% of index build completed. 58.8096% of index build completed. 59.0567% of index build completed. 59.3038% of index build completed. 59.5509% of index build completed. 59.798% of index build completed. 60.0451% of index build completed. 60.2922% of index build completed. 60.5393% of index build completed. 60.7864% of index build completed. 61.0335% of index build completed. 61.2806% of index build completed. 61.5277% of index build completed. 61.7748% of index build completed. 62.0218% of index build completed. 62.2689% of index build completed. 62.516% of index build completed. 62.7631% of index build completed. 63.0102% of index build completed. 63.2573% of index build completed. 63.5044% of index build completed. 63.7515% of index build completed. 63.9986% of index build completed. 64.2457% of index build completed. 64.4928% of index build completed. 64.7399% of index build completed. 64.987% of index build completed. 65.2341% of index build completed. 65.4812% of index build completed. 65.7283% of index build completed. 65.9754% of index build completed. 66.2225% of index build completed. 66.4696% of index build completed. 66.7167% of index build completed. 66.9638% of index build completed. 67.2109% of index build completed. 67.458% of index build completed. 67.7051% of index build completed. 67.9522% of index build completed. 68.1993% of index build completed. 68.4464% of index build completed. 68.6935% of index build completed. 68.9406% of index build completed. 69.1877% of index build completed. 69.4348% of index build completed. 69.6819% of index build completed. 69.929% of index build completed. 70.1761% of index build completed. 70.4232% of index build completed. 70.6703% of index build completed. 70.9174% of index build completed. 71.1645% of index build completed. 71.4116% of index build completed. 71.6587% of index build completed. 71.9058% of index build completed. 72.1529% of index build completed. 72.4% of index build completed. 72.6471% of index build completed. 72.8942% of index build completed. 73.1413% of index build completed. 73.3884% of index build completed. 73.6355% of index build completed. 73.8826% of index build completed. 74.1297% of index build completed. 74.3768% of index build completed. 74.6239% of index build completed. 74.871% of index build completed. 75.1181% of index build completed. 75.3652% of index build completed. 75.6123% of index build completed. 75.8594% of index build completed. 76.1065% of index build completed. 76.3536% of index build completed. 76.6007% of index build completed. 76.8478% of index build completed. 77.0949% of index build completed. 77.342% of index build completed. 77.5891% of index build completed. 77.8362% of index build completed. 78.0833% of index build completed. 78.3304% of index build completed. 78.5775% of index build completed. 78.8246% of index build completed. 79.0717% of index build completed. 79.3188% of index build completed. 79.5659% of index build completed. 79.813% of index build completed. 80.0601% of index build completed. 80.3072% of index build completed. 80.5543% of index build completed. 80.8014% of index build completed. 81.0485% of index build completed. 81.2956% of index build completed. 81.5427% of index build completed. 81.7898% of index build completed. 82.0369% of index build completed. 82.284% of index build completed. 82.5311% of index build completed. 82.7782% of index build completed. 83.0253% of index build completed. 83.2724% of index build completed. 83.5195% of index build completed. 83.7666% of index build completed. 84.0137% of index build completed. 84.2608% of index build completed. 84.5079% of index build completed. 84.755% of index build completed. 85.0021% of index build completed. 85.2492% of index build completed. 85.4963% of index build completed. 85.7434% of index build completed. 85.9905% of index build completed. 86.2376% of index build completed. 86.4847% of index build completed. 86.7317% of index build completed. 86.9788% of index build completed. 87.2259% of index build completed. 87.473% of index build completed. 87.7201% of index build completed. 87.9672% of index build completed. 88.2143% of index build completed. 88.4614% of index build completed. 88.7085% of index build completed. 88.9556% of index build completed. 89.2027% of index build completed. 89.4498% of index build completed. 89.6969% of index build completed. 89.944% of index build completed. 90.1911% of index build completed. 90.4382% of index build completed. 90.6853% of index build completed. 90.9324% of index build completed. 91.1795% of index build completed. 91.4266% of index build completed. 91.6737% of index build completed. 91.9208% of index build completed. 92.1679% of index build completed. 92.415% of index build completed. 92.6621% of index build completed. 92.9092% of index build completed. 93.1563% of index build completed. 93.4034% of index build completed. 93.6505% of index build completed. 93.8976% of index build completed. 94.1447% of index build completed. 94.3918% of index build completed. 94.6389% of index build completed. 94.886% of index build completed. 95.1331% of index build completed. 95.3802% of index build completed. 95.6273% of index build completed. 95.8744% of index build completed. 96.1215% of index build completed. 96.3686% of index build completed. 96.6157% of index build completed. 96.8628% of index build completed. 97.1099% of index build completed. 97.357% of index build completed. 97.6041% of index build completed. 97.8512% of index build completed. 98.0983% of index build completed. 98.3454% of index build completed. 98.5925% of index build completed. 98.8396% of index build completed. 99.0867% of index build completed. 99.3338% of index build completed. 99.5809% of index build completed. 99.828% of index build completed.Starting final cleanup..done. Link time: 3495.63s +Index built with degree: max:64 avg:44.9527 min:1 count(deg<2):1019259 +Robust Prune Calls: 108594089 +Indexing time: 3504.72 +Not saving tags as they are not enabled. +Time taken for save: 19.6696s. diff --git a/prec40M/sorted_r64_k50.txt b/prec40M/sorted_r64_k50.txt new file mode 100644 index 000000000..ccee78802 --- /dev/null +++ b/prec40M/sorted_r64_k50.txt @@ -0,0 +1,56 @@ +Reading (with alignment) bin file /nvmessd1/fbv4/queries384d.bin ...Metadata: #pts = 13265, #dims = 384, aligned_dim = 384... allocating aligned memory of 5093760 bytes... done. Copying data to mem_aligned buffer... done. +Opened: /nvmessd1/fbv4/gt100_prec40M, size: 10612008, cache_size: 10612008 +Reading truthset file /nvmessd1/fbv4/gt100_prec40M ... +Metadata: #pts = 13265, #dims = 100... +Using AVX2 distance computation DistanceL2Int8. +Resizing took: 1.39236s +From graph header, expected_file_size: 7568750436, _max_observed_degree: 64, _start: 11890341, file_frozen_pts: 0 +Loading vamana graph /nvmessd1/fbv4/avarhade/prec40M_sorted_memory_index_r64_l100.......done. Index has 40469609 nodes and 1851717994 out-edges, _start is set to 11890341 +Num frozen points:0 _nd: 40469609 _start: 11890341 size(_location_to_tag): 0 size(_tag_to_location):0 Max points: 40469609 +Index loaded +Using 48 threads to search + Ls QPS Avg dist cmps Mean Latency (mus) 99.9 Latency Recall@50 +================================================================================= + 50 18082.22 3627.93 2637.09 9457.87 64.98 + 60 15311.68 4148.06 3124.79 6752.18 67.80 + 70 13356.11 4659.91 3582.58 7538.96 70.08 + 80 11921.17 5170.73 4012.49 8238.08 72.01 + 90 10913.02 5674.54 4378.83 8900.64 73.67 + 100 9333.17 6176.31 5126.38 13903.87 75.07 +Done searching. Now saving results +Writing bin: /home/rakri/avarhade/Dump/tmp_50_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_50_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_60_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_60_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_70_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_70_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_80_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_80_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_90_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_90_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_100_idx_uint32.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. +Writing bin: /home/rakri/avarhade/Dump/tmp_100_dists_float.bin +bin: #pts = 13265, #dims = 50, size = 2653008B +Finished writing bin. diff --git a/prec40M/sorted_r64_l100.txt b/prec40M/sorted_r64_l100.txt new file mode 100644 index 000000000..d7bbdfc66 --- /dev/null +++ b/prec40M/sorted_r64_l100.txt @@ -0,0 +1,413 @@ +Starting index build with R: 64 Lbuild: 100 alpha: 1.2 #threads: 48 +Using AVX2 distance computation DistanceL2Int8. +Using only first 40469609 from file.. +Starting index build with 40469609 points... + 0% of index build completed. Distance to mediod: -128 + 0.247099% of index build completed. Distance to mediod: -127 + 0.494198% of index build completed. Distance to mediod: -127 + 0.741297% of index build completed. Distance to mediod: -126 + 0.988396% of index build completed. Distance to mediod: -125 + 1.2355% of index build completed. Distance to mediod: -125 + 1.48259% of index build completed. Distance to mediod: -124 + 1.72969% of index build completed. Distance to mediod: -123 + 1.97679% of index build completed. Distance to mediod: -123 + 2.22389% of index build completed. Distance to mediod: -122 + 2.47099% of index build completed. Distance to mediod: -122 + 2.71809% of index build completed. Distance to mediod: -121 + 2.96519% of index build completed. Distance to mediod: -120 + 3.21229% of index build completed. Distance to mediod: -120 + 3.45939% of index build completed. Distance to mediod: -119 + 3.70649% of index build completed. Distance to mediod: -118 + 3.95358% of index build completed. Distance to mediod: -118 + 4.20068% of index build completed. Distance to mediod: -117 + 4.44778% of index build completed. Distance to mediod: -116 + 4.69488% of index build completed. Distance to mediod: -116 + 4.94198% of index build completed. Distance to mediod: -115 + 5.18908% of index build completed. Distance to mediod: -115 + 5.43618% of index build completed. Distance to mediod: -114 + 5.68328% of index build completed. Distance to mediod: -113 + 5.93038% of index build completed. Distance to mediod: -113 + 6.17748% of index build completed. Distance to mediod: -112 + 6.42457% of index build completed. Distance to mediod: -111 + 6.67167% of index build completed. Distance to mediod: -111 + 6.91877% of index build completed. Distance to mediod: -110 + 7.16587% of index build completed. Distance to mediod: -109 + 7.41297% of index build completed. Distance to mediod: -109 + 7.66007% of index build completed. Distance to mediod: -108 + 7.90717% of index build completed. Distance to mediod: -108 + 8.15427% of index build completed. Distance to mediod: -107 + 8.40137% of index build completed. Distance to mediod: -106 + 8.64847% of index build completed. Distance to mediod: -106 + 8.89556% of index build completed. Distance to mediod: -105 + 9.14266% of index build completed. Distance to mediod: -104 + 9.38976% of index build completed. Distance to mediod: -104 + 9.63686% of index build completed. Distance to mediod: -103 + 9.88396% of index build completed. Distance to mediod: -103 + 10.1311% of index build completed. Distance to mediod: -102 + 10.3782% of index build completed. Distance to mediod: -101 + 10.6253% of index build completed. Distance to mediod: -101 + 10.8724% of index build completed. Distance to mediod: -100 + 11.1195% of index build completed. Distance to mediod: -99 + 11.3666% of index build completed. Distance to mediod: -99 + 11.6137% of index build completed. Distance to mediod: -98 + 11.8608% of index build completed. Distance to mediod: -97 + 12.1079% of index build completed. Distance to mediod: -97 + 12.355% of index build completed. Distance to mediod: -96 + 12.602% of index build completed. Distance to mediod: -96 + 12.8491% of index build completed. Distance to mediod: -95 + 13.0962% of index build completed. Distance to mediod: -94 + 13.3433% of index build completed. Distance to mediod: -94 + 13.5904% of index build completed. Distance to mediod: -93 + 13.8375% of index build completed. Distance to mediod: -92 + 14.0846% of index build completed. Distance to mediod: -92 + 14.3317% of index build completed. Distance to mediod: -91 + 14.5788% of index build completed. Distance to mediod: -90 + 14.8259% of index build completed. Distance to mediod: -90 + 15.073% of index build completed. Distance to mediod: -89 + 15.3201% of index build completed. Distance to mediod: -89 + 15.5672% of index build completed. Distance to mediod: -88 + 15.8143% of index build completed. Distance to mediod: -87 + 16.0614% of index build completed. Distance to mediod: -87 + 16.3085% of index build completed. Distance to mediod: -86 + 16.5556% of index build completed. Distance to mediod: -85 + 16.8027% of index build completed. Distance to mediod: -85 + 17.0498% of index build completed. Distance to mediod: -84 + 17.2969% of index build completed. Distance to mediod: -84 + 17.544% of index build completed. Distance to mediod: -83 + 17.7911% of index build completed. Distance to mediod: -82 + 18.0382% of index build completed. Distance to mediod: -82 + 18.2853% of index build completed. Distance to mediod: -81 + 18.5324% of index build completed. Distance to mediod: -80 + 18.7795% of index build completed. Distance to mediod: -80 + 19.0266% of index build completed. Distance to mediod: -79 + 19.2737% of index build completed. Distance to mediod: -79 + 19.5208% of index build completed. Distance to mediod: -78 + 19.7679% of index build completed. Distance to mediod: -77 + 20.015% of index build completed. Distance to mediod: -77 + 20.2621% of index build completed. Distance to mediod: -76 + 20.5092% of index build completed. Distance to mediod: -75 + 20.7563% of index build completed. Distance to mediod: -75 + 21.0034% of index build completed. Distance to mediod: -74 + 21.2505% of index build completed. Distance to mediod: -74 + 21.4976% of index build completed. Distance to mediod: -73 + 21.7447% of index build completed. Distance to mediod: -72 + 21.9918% of index build completed. Distance to mediod: -72 + 22.2389% of index build completed. Distance to mediod: -71 + 22.486% of index build completed. Distance to mediod: -70 + 22.7331% of index build completed. Distance to mediod: -70 + 22.9802% of index build completed. Distance to mediod: -69 + 23.2273% of index build completed. Distance to mediod: -68 + 23.4744% of index build completed. Distance to mediod: -68 + 23.7215% of index build completed. Distance to mediod: -67 + 23.9686% of index build completed. Distance to mediod: -67 + 24.2157% of index build completed. Distance to mediod: -66 + 24.4628% of index build completed. Distance to mediod: -65 + 24.7099% of index build completed. Distance to mediod: -65 + 24.957% of index build completed. Distance to mediod: -64 + 25.2041% of index build completed. Distance to mediod: -63 + 25.4512% of index build completed. Distance to mediod: -63 + 25.6983% of index build completed. Distance to mediod: -62 + 25.9454% of index build completed. Distance to mediod: -62 + 26.1925% of index build completed. Distance to mediod: -61 + 26.4396% of index build completed. Distance to mediod: -60 + 26.6867% of index build completed. Distance to mediod: -60 + 26.9338% of index build completed. Distance to mediod: -59 + 27.1809% of index build completed. Distance to mediod: -58 + 27.428% of index build completed. Distance to mediod: -58 + 27.6751% of index build completed. Distance to mediod: -57 + 27.9222% of index build completed. Distance to mediod: -56 + 28.1693% of index build completed. Distance to mediod: -56 + 28.4164% of index build completed. Distance to mediod: -55 + 28.6635% of index build completed. Distance to mediod: -55 + 28.9106% of index build completed. Distance to mediod: -54 + 29.1577% of index build completed. Distance to mediod: -53 + 29.4048% of index build completed. Distance to mediod: -53 + 29.6519% of index build completed. Distance to mediod: -52 + 29.899% of index build completed. Distance to mediod: -51 + 30.1461% of index build completed. Distance to mediod: -51 + 30.3932% of index build completed. Distance to mediod: -50 + 30.6403% of index build completed. Distance to mediod: -49 + 30.8874% of index build completed. Distance to mediod: -49 + 31.1345% of index build completed. Distance to mediod: -48 + 31.3816% of index build completed. Distance to mediod: -48 + 31.6287% of index build completed. Distance to mediod: -47 + 31.8758% of index build completed. Distance to mediod: -46 + 32.1229% of index build completed. Distance to mediod: -46 + 32.37% of index build completed. Distance to mediod: -45 + 32.6171% of index build completed. Distance to mediod: -44 + 32.8642% of index build completed. Distance to mediod: -44 + 33.1113% of index build completed. Distance to mediod: -43 + 33.3584% of index build completed. Distance to mediod: -42 + 33.6055% of index build completed. Distance to mediod: -42 + 33.8526% of index build completed. Distance to mediod: -41 + 34.0997% of index build completed. Distance to mediod: -41 + 34.3468% of index build completed. Distance to mediod: -40 + 34.5939% of index build completed. Distance to mediod: -39 + 34.841% of index build completed. Distance to mediod: -39 + 35.0881% of index build completed. Distance to mediod: -38 + 35.3352% of index build completed. Distance to mediod: -37 + 35.5823% of index build completed. Distance to mediod: -37 + 35.8294% of index build completed. Distance to mediod: -36 + 36.0765% of index build completed. Distance to mediod: -35 + 36.3236% of index build completed. Distance to mediod: -35 + 36.5707% of index build completed. Distance to mediod: -34 + 36.8178% of index build completed. Distance to mediod: -34 + 37.0649% of index build completed. Distance to mediod: -33 + 37.3119% of index build completed. Distance to mediod: -33 + 37.559% of index build completed. Distance to mediod: -32 + 37.8061% of index build completed. Distance to mediod: -31 + 38.0532% of index build completed. Distance to mediod: -31 + 38.3003% of index build completed. Distance to mediod: -30 + 38.5474% of index build completed. Distance to mediod: -29 + 38.7945% of index build completed. Distance to mediod: -29 + 39.0416% of index build completed. Distance to mediod: -28 + 39.2887% of index build completed. Distance to mediod: -28 + 39.5358% of index build completed. Distance to mediod: -27 + 39.7829% of index build completed. Distance to mediod: -26 + 40.03% of index build completed. Distance to mediod: -26 + 40.2771% of index build completed. Distance to mediod: -25 + 40.5242% of index build completed. Distance to mediod: -24 + 40.7713% of index build completed. Distance to mediod: -24 + 41.0184% of index build completed. Distance to mediod: -23 + 41.2655% of index build completed. Distance to mediod: -22 + 41.5126% of index build completed. Distance to mediod: -22 + 41.7597% of index build completed. Distance to mediod: -21 + 42.0068% of index build completed. Distance to mediod: -21 + 42.2539% of index build completed. Distance to mediod: -20 + 42.501% of index build completed. Distance to mediod: -19 + 42.7481% of index build completed. Distance to mediod: -19 + 42.9952% of index build completed. Distance to mediod: -18 + 43.2423% of index build completed. Distance to mediod: -17 + 43.4894% of index build completed. Distance to mediod: -17 + 43.7365% of index build completed. Distance to mediod: -16 + 43.9836% of index build completed. Distance to mediod: -15 + 44.2307% of index build completed. Distance to mediod: -15 + 44.4778% of index build completed. Distance to mediod: -14 + 44.7249% of index build completed. Distance to mediod: -14 + 44.972% of index build completed. Distance to mediod: -13 + 45.2191% of index build completed. Distance to mediod: -12 + 45.4662% of index build completed. Distance to mediod: -12 + 45.7133% of index build completed. Distance to mediod: -11 + 45.9604% of index build completed. Distance to mediod: -10 + 46.2075% of index build completed. Distance to mediod: -10 + 46.4546% of index build completed. Distance to mediod: -9 + 46.7017% of index build completed. Distance to mediod: -8 + 46.9488% of index build completed. Distance to mediod: -8 + 47.1959% of index build completed. Distance to mediod: -7 + 47.443% of index build completed. Distance to mediod: -7 + 47.6901% of index build completed. Distance to mediod: -6 + 47.9372% of index build completed. Distance to mediod: -5 + 48.1843% of index build completed. Distance to mediod: -5 + 48.4314% of index build completed. Distance to mediod: -4 + 48.6785% of index build completed. Distance to mediod: -3 + 48.9256% of index build completed. Distance to mediod: -3 + 49.1727% of index build completed. Distance to mediod: -2 + 49.4198% of index build completed. Distance to mediod: -1 + 49.6669% of index build completed. Distance to mediod: -1 + 49.914% of index build completed. Distance to mediod: 0 + 50.1611% of index build completed. Distance to mediod: 0 + 50.4082% of index build completed. Distance to mediod: 1 + 50.6553% of index build completed. Distance to mediod: 2 + 50.9024% of index build completed. Distance to mediod: 2 + 51.1495% of index build completed. Distance to mediod: 3 + 51.3966% of index build completed. Distance to mediod: 4 + 51.6437% of index build completed. Distance to mediod: 4 + 51.8908% of index build completed. Distance to mediod: 5 + 52.1379% of index build completed. Distance to mediod: 5 + 52.385% of index build completed. Distance to mediod: 6 + 52.6321% of index build completed. Distance to mediod: 7 + 52.8792% of index build completed. Distance to mediod: 7 + 53.1263% of index build completed. Distance to mediod: 8 + 53.3734% of index build completed. Distance to mediod: 9 + 53.6205% of index build completed. Distance to mediod: 9 + 53.8676% of index build completed. Distance to mediod: 10 + 54.1147% of index build completed. Distance to mediod: 10 + 54.3618% of index build completed. Distance to mediod: 11 + 54.6089% of index build completed. Distance to mediod: 12 + 54.856% of index build completed. Distance to mediod: 12 + 55.1031% of index build completed. Distance to mediod: 13 + 55.3502% of index build completed. Distance to mediod: 14 + 55.5973% of index build completed. Distance to mediod: 14 + 55.8444% of index build completed. Distance to mediod: 15 + 56.0915% of index build completed. Distance to mediod: 16 + 56.3386% of index build completed. Distance to mediod: 16 + 56.5857% of index build completed. Distance to mediod: 17 + 56.8328% of index build completed. Distance to mediod: 17 + 57.0799% of index build completed. Distance to mediod: 18 + 57.327% of index build completed. Distance to mediod: 19 + 57.5741% of index build completed. Distance to mediod: 19 + 57.8212% of index build completed. Distance to mediod: 20 + 58.0683% of index build completed. Distance to mediod: 21 + 58.3154% of index build completed. Distance to mediod: 21 + 58.5625% of index build completed. Distance to mediod: 22 + 58.8096% of index build completed. Distance to mediod: 23 + 59.0567% of index build completed. Distance to mediod: 23 + 59.3038% of index build completed. Distance to mediod: 24 + 59.5509% of index build completed. Distance to mediod: 24 + 59.798% of index build completed. Distance to mediod: 25 + 60.0451% of index build completed. Distance to mediod: 26 + 60.2922% of index build completed. Distance to mediod: 26 + 60.5393% of index build completed. Distance to mediod: 27 + 60.7864% of index build completed. Distance to mediod: 28 + 61.0335% of index build completed. Distance to mediod: 28 + 61.2806% of index build completed. Distance to mediod: 29 + 61.5277% of index build completed. Distance to mediod: 30 + 61.7748% of index build completed. Distance to mediod: 30 + 62.0218% of index build completed. Distance to mediod: 31 + 62.2689% of index build completed. Distance to mediod: 31 + 62.516% of index build completed. Distance to mediod: 32 + 62.7631% of index build completed. Distance to mediod: 33 + 63.0102% of index build completed. Distance to mediod: 33 + 63.2573% of index build completed. Distance to mediod: 34 + 63.5044% of index build completed. Distance to mediod: 35 + 63.7515% of index build completed. Distance to mediod: 35 + 63.9986% of index build completed. Distance to mediod: 36 + 64.2457% of index build completed. Distance to mediod: 36 + 64.4928% of index build completed. Distance to mediod: 37 + 64.7399% of index build completed. Distance to mediod: 38 + 64.987% of index build completed. Distance to mediod: 38 + 65.2341% of index build completed. Distance to mediod: 39 + 65.4812% of index build completed. Distance to mediod: 40 + 65.7283% of index build completed. Distance to mediod: 40 + 65.9754% of index build completed. Distance to mediod: 41 + 66.2225% of index build completed. Distance to mediod: 42 + 66.4696% of index build completed. Distance to mediod: 42 + 66.7167% of index build completed. Distance to mediod: 43 + 66.9638% of index build completed. Distance to mediod: 43 + 67.2109% of index build completed. Distance to mediod: 44 + 67.458% of index build completed. Distance to mediod: 45 + 67.7051% of index build completed. Distance to mediod: 45 + 67.9522% of index build completed. Distance to mediod: 46 + 68.1993% of index build completed. Distance to mediod: 47 + 68.4464% of index build completed. Distance to mediod: 47 + 68.6935% of index build completed. Distance to mediod: 48 + 68.9406% of index build completed. Distance to mediod: 49 + 69.1877% of index build completed. Distance to mediod: 49 + 69.4348% of index build completed. Distance to mediod: 50 + 69.6819% of index build completed. Distance to mediod: 50 + 69.929% of index build completed. Distance to mediod: 51 + 70.1761% of index build completed. Distance to mediod: 52 + 70.4232% of index build completed. Distance to mediod: 52 + 70.6703% of index build completed. Distance to mediod: 53 + 70.9174% of index build completed. Distance to mediod: 54 + 71.1645% of index build completed. Distance to mediod: 54 + 71.4116% of index build completed. Distance to mediod: 55 + 71.6587% of index build completed. Distance to mediod: 56 + 71.9058% of index build completed. Distance to mediod: 56 + 72.1529% of index build completed. Distance to mediod: 57 + 72.4% of index build completed. Distance to mediod: 57 + 72.6471% of index build completed. Distance to mediod: 58 + 72.8942% of index build completed. Distance to mediod: 59 + 73.1413% of index build completed. Distance to mediod: 59 + 73.3884% of index build completed. Distance to mediod: 60 + 73.6355% of index build completed. Distance to mediod: 61 + 73.8826% of index build completed. Distance to mediod: 61 + 74.1297% of index build completed. Distance to mediod: 62 + 74.3768% of index build completed. Distance to mediod: 62 + 74.6239% of index build completed. Distance to mediod: 63 + 74.871% of index build completed. Distance to mediod: 64 + 75.1181% of index build completed. Distance to mediod: 64 + 75.3652% of index build completed. Distance to mediod: 65 + 75.6123% of index build completed. Distance to mediod: 66 + 75.8594% of index build completed. Distance to mediod: 66 + 76.1065% of index build completed. Distance to mediod: 67 + 76.3536% of index build completed. Distance to mediod: 68 + 76.6007% of index build completed. Distance to mediod: 68 + 76.8478% of index build completed. Distance to mediod: 69 + 77.0949% of index build completed. Distance to mediod: 69 + 77.342% of index build completed. Distance to mediod: 70 + 77.5891% of index build completed. Distance to mediod: 71 + 77.8362% of index build completed. Distance to mediod: 71 + 78.0833% of index build completed. Distance to mediod: 72 + 78.3304% of index build completed. Distance to mediod: 73 + 78.5775% of index build completed. Distance to mediod: 73 + 78.8246% of index build completed. Distance to mediod: 74 + 79.0717% of index build completed. Distance to mediod: 75 + 79.3188% of index build completed. Distance to mediod: 75 + 79.5659% of index build completed. Distance to mediod: 76 + 79.813% of index build completed. Distance to mediod: 76 + 80.0601% of index build completed. Distance to mediod: 77 + 80.3072% of index build completed. Distance to mediod: 78 + 80.5543% of index build completed. Distance to mediod: 78 + 80.8014% of index build completed. Distance to mediod: 79 + 81.0485% of index build completed. Distance to mediod: 80 + 81.2956% of index build completed. Distance to mediod: 80 + 81.5427% of index build completed. Distance to mediod: 81 + 81.7898% of index build completed. Distance to mediod: 81 + 82.0369% of index build completed. Distance to mediod: 82 + 82.284% of index build completed. Distance to mediod: 83 + 82.5311% of index build completed. Distance to mediod: 83 + 82.7782% of index build completed. Distance to mediod: 84 + 83.0253% of index build completed. Distance to mediod: 85 + 83.2724% of index build completed. Distance to mediod: 85 + 83.5195% of index build completed. Distance to mediod: 86 + 83.7666% of index build completed. Distance to mediod: 87 + 84.0137% of index build completed. Distance to mediod: 87 + 84.2608% of index build completed. Distance to mediod: 88 + 84.5079% of index build completed. Distance to mediod: 88 + 84.755% of index build completed. Distance to mediod: 89 + 85.0021% of index build completed. Distance to mediod: 90 + 85.2492% of index build completed. Distance to mediod: 90 + 85.4963% of index build completed. Distance to mediod: 91 + 85.7434% of index build completed. Distance to mediod: 92 + 85.9905% of index build completed. Distance to mediod: 92 + 86.2376% of index build completed. Distance to mediod: 93 + 86.4847% of index build completed. Distance to mediod: 94 + 86.7317% of index build completed. Distance to mediod: 94 + 86.9788% of index build completed. Distance to mediod: 95 + 87.2259% of index build completed. Distance to mediod: 95 + 87.473% of index build completed. Distance to mediod: 96 + 87.7201% of index build completed. Distance to mediod: 97 + 87.9672% of index build completed. Distance to mediod: 97 + 88.2143% of index build completed. Distance to mediod: 98 + 88.4614% of index build completed. Distance to mediod: 99 + 88.7085% of index build completed. Distance to mediod: 99 + 88.9556% of index build completed. Distance to mediod: 100 + 89.2027% of index build completed. Distance to mediod: 100 + 89.4498% of index build completed. Distance to mediod: 101 + 89.6969% of index build completed. Distance to mediod: 102 + 89.944% of index build completed. Distance to mediod: 102 + 90.1911% of index build completed. Distance to mediod: 103 + 90.4382% of index build completed. Distance to mediod: 104 + 90.6853% of index build completed. Distance to mediod: 104 + 90.9324% of index build completed. Distance to mediod: 105 + 91.1795% of index build completed. Distance to mediod: 106 + 91.4266% of index build completed. Distance to mediod: 106 + 91.6737% of index build completed. Distance to mediod: 107 + 91.9208% of index build completed. Distance to mediod: 107 + 92.1679% of index build completed. Distance to mediod: 108 + 92.415% of index build completed. Distance to mediod: 109 + 92.6621% of index build completed. Distance to mediod: 109 + 92.9092% of index build completed. Distance to mediod: 110 + 93.1563% of index build completed. Distance to mediod: 111 + 93.4034% of index build completed. Distance to mediod: 111 + 93.6505% of index build completed. Distance to mediod: 112 + 93.8976% of index build completed. Distance to mediod: 112 + 94.1447% of index build completed. Distance to mediod: 113 + 94.3918% of index build completed. Distance to mediod: 114 + 94.6389% of index build completed. Distance to mediod: 114 + 94.886% of index build completed. Distance to mediod: 115 + 95.1331% of index build completed. Distance to mediod: 116 + 95.3802% of index build completed. Distance to mediod: 116 + 95.6273% of index build completed. Distance to mediod: 117 + 95.8744% of index build completed. Distance to mediod: 118 + 96.1215% of index build completed. Distance to mediod: 118 + 96.3686% of index build completed. Distance to mediod: 119 + 96.6157% of index build completed. Distance to mediod: 119 + 96.8628% of index build completed. Distance to mediod: 120 + 97.1099% of index build completed. Distance to mediod: 121 + 97.357% of index build completed. Distance to mediod: 121 + 97.6041% of index build completed. Distance to mediod: 122 + 97.8512% of index build completed. Distance to mediod: 123 + 98.0983% of index build completed. Distance to mediod: 123 + 98.3454% of index build completed. Distance to mediod: 124 + 98.5925% of index build completed. Distance to mediod: 125 + 98.8396% of index build completed. Distance to mediod: 125 + 99.0867% of index build completed. Distance to mediod: 126 + 99.3338% of index build completed. Distance to mediod: 126 + 99.5809% of index build completed. Distance to mediod: 127 + 99.828% of index build completed.Starting final cleanup..done. Link time: 3629.97s +Index built with degree: max:64 avg:45.7558 min:1 count(deg<2):1018463 +Indexing time: 3648.44 +Not saving tags as they are not enabled. +Time taken for save: 19.4831s. diff --git a/run-search.sh b/run-search.sh new file mode 100644 index 000000000..c1efe77b4 --- /dev/null +++ b/run-search.sh @@ -0,0 +1,2 @@ +cd build +./apps/search_memory_index --data_type int8 --dist_fn l2 --index_path_prefix /nvmessd1/fbv4/avarhade/prec40M_sorted_memory_index_r64_l100 --gt_file /nvmessd1/fbv4/gt100_prec40M --query_file /nvmessd1/fbv4/queries384d.bin --result_path /home/rakri/avarhade/Dump/tmp -K 50 -L 50 60 70 80 90 100 -T 48 > /home/rakri/avarhade/DiskANN/prec40M/sorted_r64_k50.txt \ No newline at end of file diff --git a/src/index.cpp b/src/index.cpp index eee571b4f..85e9f3bd6 100644 --- a/src/index.cpp +++ b/src/index.cpp @@ -1269,7 +1269,7 @@ void Index::inter_insert(uint32_t n, std::vector &pru } template -bool comp(const std::pair &a, const std::pair &b) +bool comp(const std::pair &a, const std::pair &b) { if (a.first < b.first){ return true; @@ -1304,7 +1304,7 @@ template void Indexget_distance(i, _start); distances_to_mediod.emplace_back(std::make_pair(cur_distance, i)); } - std::sort(distances_to_mediod.begin(), distances_to_mediod.end(), comp); + std::sort(distances_to_mediod.begin(), distances_to_mediod.end(), comp); for (uint32_t i = 0; i < (uint32_t)_nd; i++) visit_order.emplace_back(distances_to_mediod[i].second); @@ -1321,8 +1321,8 @@ template void Index 0){ + diskann::cout <<" Distance to mediod: " << (float)(distances_to_mediod[node_ctr].first) << std::endl; } // Find and add appropriate graph edges From 28f4ac4ce49c006cdec2992cdbc39706a69ed896 Mon Sep 17 00:00:00 2001 From: ravishankar Date: Thu, 7 Nov 2024 05:12:46 -0800 Subject: [PATCH 5/6] add baseline results --- prec40M/baseline_r64_k50.txt | 16 ++++++++-------- run-search.sh | 3 ++- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/prec40M/baseline_r64_k50.txt b/prec40M/baseline_r64_k50.txt index af184dc2b..45e6b863f 100644 --- a/prec40M/baseline_r64_k50.txt +++ b/prec40M/baseline_r64_k50.txt @@ -3,20 +3,20 @@ Opened: /nvmessd1/fbv4/gt100_prec40M, size: 10612008, cache_size: 10612008 Reading truthset file /nvmessd1/fbv4/gt100_prec40M ... Metadata: #pts = 13265, #dims = 100... Using AVX2 distance computation DistanceL2Int8. -Resizing took: 1.41607s +Resizing took: 1.39346s From graph header, expected_file_size: 7438757948, _max_observed_degree: 64, _start: 11890341, file_frozen_pts: 0 Loading vamana graph /nvmessd1/fbv4/avarhade/prec40M_memory_index_r64_l100.......done. Index has 40469609 nodes and 1819219872 out-edges, _start is set to 11890341 Num frozen points:0 _nd: 40469609 _start: 11890341 size(_location_to_tag): 0 size(_tag_to_location):0 Max points: 40469609 Index loaded -Using 48 threads to search +Using 1 threads to search Ls QPS Avg dist cmps Mean Latency (mus) 99.9 Latency Recall@50 ================================================================================= - 50 17629.05 3687.22 2705.62 14582.74 62.96 - 60 14420.19 4195.96 3319.41 9059.50 65.92 - 70 12430.30 4685.40 3850.11 10474.53 68.19 - 80 12256.81 5171.15 3903.45 9068.07 70.07 - 90 11310.12 5654.48 4230.41 9502.88 71.76 - 100 10462.90 6129.37 4574.90 9994.75 73.17 + 50 750.71 3687.22 1332.03 15081.17 62.96 + 60 664.89 4195.96 1503.96 13729.61 65.92 + 70 595.61 4685.40 1678.89 12513.98 68.19 + 80 539.72 5171.15 1852.77 12737.32 70.07 + 90 493.28 5654.48 2027.21 12803.06 71.76 + 100 453.69 6129.37 2204.12 15871.16 73.17 Done searching. Now saving results Writing bin: /home/rakri/avarhade/Dump/tmp_50_idx_uint32.bin bin: #pts = 13265, #dims = 50, size = 2653008B diff --git a/run-search.sh b/run-search.sh index c1efe77b4..6d048b834 100644 --- a/run-search.sh +++ b/run-search.sh @@ -1,2 +1,3 @@ cd build -./apps/search_memory_index --data_type int8 --dist_fn l2 --index_path_prefix /nvmessd1/fbv4/avarhade/prec40M_sorted_memory_index_r64_l100 --gt_file /nvmessd1/fbv4/gt100_prec40M --query_file /nvmessd1/fbv4/queries384d.bin --result_path /home/rakri/avarhade/Dump/tmp -K 50 -L 50 60 70 80 90 100 -T 48 > /home/rakri/avarhade/DiskANN/prec40M/sorted_r64_k50.txt \ No newline at end of file +./apps/search_memory_index --data_type int8 --dist_fn l2 --index_path_prefix /nvmessd1/fbv4/avarhade/prec40M_sorted_memory_index_r64_l100 --gt_file /nvmessd1/fbv4/gt100_prec40M --query_file /nvmessd1/fbv4/queries384d.bin --result_path /home/rakri/avarhade/Dump/tmp -K 50 -L 50 60 70 80 90 100 -T 1 > /home/rakri/avarhade/DiskANN/prec40M/sorted_r64_k50.txt + From dc42a6a35d279200fc695b783638235ae8bc5ff0 Mon Sep 17 00:00:00 2001 From: ravishankar Date: Thu, 7 Nov 2024 21:02:45 -0800 Subject: [PATCH 6/6] add sorted results --- prec40M/baseline_r64_k50.txt | 14 +- prec40M/sorted_r64_k50.txt | 20 +- prec40M/sorted_r64_l100.txt | 413 +---------------------------------- src/index.cpp | 19 +- 4 files changed, 32 insertions(+), 434 deletions(-) diff --git a/prec40M/baseline_r64_k50.txt b/prec40M/baseline_r64_k50.txt index 45e6b863f..fbda8d361 100644 --- a/prec40M/baseline_r64_k50.txt +++ b/prec40M/baseline_r64_k50.txt @@ -3,7 +3,7 @@ Opened: /nvmessd1/fbv4/gt100_prec40M, size: 10612008, cache_size: 10612008 Reading truthset file /nvmessd1/fbv4/gt100_prec40M ... Metadata: #pts = 13265, #dims = 100... Using AVX2 distance computation DistanceL2Int8. -Resizing took: 1.39346s +Resizing took: 1.70302s From graph header, expected_file_size: 7438757948, _max_observed_degree: 64, _start: 11890341, file_frozen_pts: 0 Loading vamana graph /nvmessd1/fbv4/avarhade/prec40M_memory_index_r64_l100.......done. Index has 40469609 nodes and 1819219872 out-edges, _start is set to 11890341 Num frozen points:0 _nd: 40469609 _start: 11890341 size(_location_to_tag): 0 size(_tag_to_location):0 Max points: 40469609 @@ -11,12 +11,12 @@ Index loaded Using 1 threads to search Ls QPS Avg dist cmps Mean Latency (mus) 99.9 Latency Recall@50 ================================================================================= - 50 750.71 3687.22 1332.03 15081.17 62.96 - 60 664.89 4195.96 1503.96 13729.61 65.92 - 70 595.61 4685.40 1678.89 12513.98 68.19 - 80 539.72 5171.15 1852.77 12737.32 70.07 - 90 493.28 5654.48 2027.21 12803.06 71.76 - 100 453.69 6129.37 2204.12 15871.16 73.17 + 50 321.67 3687.22 3108.74 9766.01 62.96 + 60 426.77 4195.96 2343.13 6894.97 65.92 + 70 459.30 4685.40 2177.17 21952.75 68.19 + 80 416.92 5171.15 2398.49 22705.72 70.07 + 90 384.96 5654.48 2597.62 23008.43 71.76 + 100 353.87 6129.37 2825.84 23359.16 73.17 Done searching. Now saving results Writing bin: /home/rakri/avarhade/Dump/tmp_50_idx_uint32.bin bin: #pts = 13265, #dims = 50, size = 2653008B diff --git a/prec40M/sorted_r64_k50.txt b/prec40M/sorted_r64_k50.txt index ccee78802..a5b45ce43 100644 --- a/prec40M/sorted_r64_k50.txt +++ b/prec40M/sorted_r64_k50.txt @@ -3,20 +3,20 @@ Opened: /nvmessd1/fbv4/gt100_prec40M, size: 10612008, cache_size: 10612008 Reading truthset file /nvmessd1/fbv4/gt100_prec40M ... Metadata: #pts = 13265, #dims = 100... Using AVX2 distance computation DistanceL2Int8. -Resizing took: 1.39236s -From graph header, expected_file_size: 7568750436, _max_observed_degree: 64, _start: 11890341, file_frozen_pts: 0 -Loading vamana graph /nvmessd1/fbv4/avarhade/prec40M_sorted_memory_index_r64_l100.......done. Index has 40469609 nodes and 1851717994 out-edges, _start is set to 11890341 +Resizing took: 1.55789s +From graph header, expected_file_size: 7427476424, _max_observed_degree: 64, _start: 11890341, file_frozen_pts: 0 +Loading vamana graph /nvmessd1/fbv4/avarhade/prec40M_sorted_memory_index_r64_l100.......done. Index has 40469609 nodes and 1816399491 out-edges, _start is set to 11890341 Num frozen points:0 _nd: 40469609 _start: 11890341 size(_location_to_tag): 0 size(_tag_to_location):0 Max points: 40469609 Index loaded -Using 48 threads to search +Using 1 threads to search Ls QPS Avg dist cmps Mean Latency (mus) 99.9 Latency Recall@50 ================================================================================= - 50 18082.22 3627.93 2637.09 9457.87 64.98 - 60 15311.68 4148.06 3124.79 6752.18 67.80 - 70 13356.11 4659.91 3582.58 7538.96 70.08 - 80 11921.17 5170.73 4012.49 8238.08 72.01 - 90 10913.02 5674.54 4378.83 8900.64 73.67 - 100 9333.17 6176.31 5126.38 13903.87 75.07 + 50 335.17 3644.75 2983.48 10038.98 62.96 + 60 411.36 4146.50 2430.86 7139.40 65.80 + 70 459.29 4637.39 2177.20 12186.82 68.10 + 80 423.70 5122.83 2360.07 22461.92 70.02 + 90 389.42 5604.18 2567.85 22611.97 71.66 + 100 358.39 6079.79 2790.22 23125.17 73.07 Done searching. Now saving results Writing bin: /home/rakri/avarhade/Dump/tmp_50_idx_uint32.bin bin: #pts = 13265, #dims = 50, size = 2653008B diff --git a/prec40M/sorted_r64_l100.txt b/prec40M/sorted_r64_l100.txt index d7bbdfc66..575b57e37 100644 --- a/prec40M/sorted_r64_l100.txt +++ b/prec40M/sorted_r64_l100.txt @@ -2,412 +2,9 @@ Starting index build with R: 64 Lbuild: 100 alpha: 1.2 #threads: 48 Using AVX2 distance computation DistanceL2Int8. Using only first 40469609 from file.. Starting index build with 40469609 points... - 0% of index build completed. Distance to mediod: -128 - 0.247099% of index build completed. Distance to mediod: -127 - 0.494198% of index build completed. Distance to mediod: -127 - 0.741297% of index build completed. Distance to mediod: -126 - 0.988396% of index build completed. Distance to mediod: -125 - 1.2355% of index build completed. Distance to mediod: -125 - 1.48259% of index build completed. Distance to mediod: -124 - 1.72969% of index build completed. Distance to mediod: -123 - 1.97679% of index build completed. Distance to mediod: -123 - 2.22389% of index build completed. Distance to mediod: -122 - 2.47099% of index build completed. Distance to mediod: -122 - 2.71809% of index build completed. Distance to mediod: -121 - 2.96519% of index build completed. Distance to mediod: -120 - 3.21229% of index build completed. Distance to mediod: -120 - 3.45939% of index build completed. Distance to mediod: -119 - 3.70649% of index build completed. Distance to mediod: -118 - 3.95358% of index build completed. Distance to mediod: -118 - 4.20068% of index build completed. Distance to mediod: -117 - 4.44778% of index build completed. Distance to mediod: -116 - 4.69488% of index build completed. Distance to mediod: -116 - 4.94198% of index build completed. Distance to mediod: -115 - 5.18908% of index build completed. Distance to mediod: -115 - 5.43618% of index build completed. Distance to mediod: -114 - 5.68328% of index build completed. Distance to mediod: -113 - 5.93038% of index build completed. Distance to mediod: -113 - 6.17748% of index build completed. Distance to mediod: -112 - 6.42457% of index build completed. Distance to mediod: -111 - 6.67167% of index build completed. Distance to mediod: -111 - 6.91877% of index build completed. Distance to mediod: -110 - 7.16587% of index build completed. Distance to mediod: -109 - 7.41297% of index build completed. Distance to mediod: -109 - 7.66007% of index build completed. Distance to mediod: -108 - 7.90717% of index build completed. Distance to mediod: -108 - 8.15427% of index build completed. Distance to mediod: -107 - 8.40137% of index build completed. Distance to mediod: -106 - 8.64847% of index build completed. Distance to mediod: -106 - 8.89556% of index build completed. Distance to mediod: -105 - 9.14266% of index build completed. Distance to mediod: -104 - 9.38976% of index build completed. Distance to mediod: -104 - 9.63686% of index build completed. Distance to mediod: -103 - 9.88396% of index build completed. Distance to mediod: -103 - 10.1311% of index build completed. Distance to mediod: -102 - 10.3782% of index build completed. Distance to mediod: -101 - 10.6253% of index build completed. Distance to mediod: -101 - 10.8724% of index build completed. Distance to mediod: -100 - 11.1195% of index build completed. Distance to mediod: -99 - 11.3666% of index build completed. Distance to mediod: -99 - 11.6137% of index build completed. Distance to mediod: -98 - 11.8608% of index build completed. Distance to mediod: -97 - 12.1079% of index build completed. Distance to mediod: -97 - 12.355% of index build completed. Distance to mediod: -96 - 12.602% of index build completed. Distance to mediod: -96 - 12.8491% of index build completed. Distance to mediod: -95 - 13.0962% of index build completed. Distance to mediod: -94 - 13.3433% of index build completed. Distance to mediod: -94 - 13.5904% of index build completed. Distance to mediod: -93 - 13.8375% of index build completed. Distance to mediod: -92 - 14.0846% of index build completed. Distance to mediod: -92 - 14.3317% of index build completed. Distance to mediod: -91 - 14.5788% of index build completed. Distance to mediod: -90 - 14.8259% of index build completed. Distance to mediod: -90 - 15.073% of index build completed. Distance to mediod: -89 - 15.3201% of index build completed. Distance to mediod: -89 - 15.5672% of index build completed. Distance to mediod: -88 - 15.8143% of index build completed. Distance to mediod: -87 - 16.0614% of index build completed. Distance to mediod: -87 - 16.3085% of index build completed. Distance to mediod: -86 - 16.5556% of index build completed. Distance to mediod: -85 - 16.8027% of index build completed. Distance to mediod: -85 - 17.0498% of index build completed. Distance to mediod: -84 - 17.2969% of index build completed. Distance to mediod: -84 - 17.544% of index build completed. Distance to mediod: -83 - 17.7911% of index build completed. Distance to mediod: -82 - 18.0382% of index build completed. Distance to mediod: -82 - 18.2853% of index build completed. Distance to mediod: -81 - 18.5324% of index build completed. Distance to mediod: -80 - 18.7795% of index build completed. Distance to mediod: -80 - 19.0266% of index build completed. Distance to mediod: -79 - 19.2737% of index build completed. Distance to mediod: -79 - 19.5208% of index build completed. Distance to mediod: -78 - 19.7679% of index build completed. Distance to mediod: -77 - 20.015% of index build completed. Distance to mediod: -77 - 20.2621% of index build completed. Distance to mediod: -76 - 20.5092% of index build completed. Distance to mediod: -75 - 20.7563% of index build completed. Distance to mediod: -75 - 21.0034% of index build completed. Distance to mediod: -74 - 21.2505% of index build completed. Distance to mediod: -74 - 21.4976% of index build completed. Distance to mediod: -73 - 21.7447% of index build completed. Distance to mediod: -72 - 21.9918% of index build completed. Distance to mediod: -72 - 22.2389% of index build completed. Distance to mediod: -71 - 22.486% of index build completed. Distance to mediod: -70 - 22.7331% of index build completed. Distance to mediod: -70 - 22.9802% of index build completed. Distance to mediod: -69 - 23.2273% of index build completed. Distance to mediod: -68 - 23.4744% of index build completed. Distance to mediod: -68 - 23.7215% of index build completed. Distance to mediod: -67 - 23.9686% of index build completed. Distance to mediod: -67 - 24.2157% of index build completed. Distance to mediod: -66 - 24.4628% of index build completed. Distance to mediod: -65 - 24.7099% of index build completed. Distance to mediod: -65 - 24.957% of index build completed. Distance to mediod: -64 - 25.2041% of index build completed. Distance to mediod: -63 - 25.4512% of index build completed. Distance to mediod: -63 - 25.6983% of index build completed. Distance to mediod: -62 - 25.9454% of index build completed. Distance to mediod: -62 - 26.1925% of index build completed. Distance to mediod: -61 - 26.4396% of index build completed. Distance to mediod: -60 - 26.6867% of index build completed. Distance to mediod: -60 - 26.9338% of index build completed. Distance to mediod: -59 - 27.1809% of index build completed. Distance to mediod: -58 - 27.428% of index build completed. Distance to mediod: -58 - 27.6751% of index build completed. Distance to mediod: -57 - 27.9222% of index build completed. Distance to mediod: -56 - 28.1693% of index build completed. Distance to mediod: -56 - 28.4164% of index build completed. Distance to mediod: -55 - 28.6635% of index build completed. Distance to mediod: -55 - 28.9106% of index build completed. Distance to mediod: -54 - 29.1577% of index build completed. Distance to mediod: -53 - 29.4048% of index build completed. Distance to mediod: -53 - 29.6519% of index build completed. Distance to mediod: -52 - 29.899% of index build completed. Distance to mediod: -51 - 30.1461% of index build completed. Distance to mediod: -51 - 30.3932% of index build completed. Distance to mediod: -50 - 30.6403% of index build completed. Distance to mediod: -49 - 30.8874% of index build completed. Distance to mediod: -49 - 31.1345% of index build completed. Distance to mediod: -48 - 31.3816% of index build completed. Distance to mediod: -48 - 31.6287% of index build completed. Distance to mediod: -47 - 31.8758% of index build completed. Distance to mediod: -46 - 32.1229% of index build completed. Distance to mediod: -46 - 32.37% of index build completed. Distance to mediod: -45 - 32.6171% of index build completed. Distance to mediod: -44 - 32.8642% of index build completed. Distance to mediod: -44 - 33.1113% of index build completed. Distance to mediod: -43 - 33.3584% of index build completed. Distance to mediod: -42 - 33.6055% of index build completed. Distance to mediod: -42 - 33.8526% of index build completed. Distance to mediod: -41 - 34.0997% of index build completed. Distance to mediod: -41 - 34.3468% of index build completed. Distance to mediod: -40 - 34.5939% of index build completed. Distance to mediod: -39 - 34.841% of index build completed. Distance to mediod: -39 - 35.0881% of index build completed. Distance to mediod: -38 - 35.3352% of index build completed. Distance to mediod: -37 - 35.5823% of index build completed. Distance to mediod: -37 - 35.8294% of index build completed. Distance to mediod: -36 - 36.0765% of index build completed. Distance to mediod: -35 - 36.3236% of index build completed. Distance to mediod: -35 - 36.5707% of index build completed. Distance to mediod: -34 - 36.8178% of index build completed. Distance to mediod: -34 - 37.0649% of index build completed. Distance to mediod: -33 - 37.3119% of index build completed. Distance to mediod: -33 - 37.559% of index build completed. Distance to mediod: -32 - 37.8061% of index build completed. Distance to mediod: -31 - 38.0532% of index build completed. Distance to mediod: -31 - 38.3003% of index build completed. Distance to mediod: -30 - 38.5474% of index build completed. Distance to mediod: -29 - 38.7945% of index build completed. Distance to mediod: -29 - 39.0416% of index build completed. Distance to mediod: -28 - 39.2887% of index build completed. Distance to mediod: -28 - 39.5358% of index build completed. Distance to mediod: -27 - 39.7829% of index build completed. Distance to mediod: -26 - 40.03% of index build completed. Distance to mediod: -26 - 40.2771% of index build completed. Distance to mediod: -25 - 40.5242% of index build completed. Distance to mediod: -24 - 40.7713% of index build completed. Distance to mediod: -24 - 41.0184% of index build completed. Distance to mediod: -23 - 41.2655% of index build completed. Distance to mediod: -22 - 41.5126% of index build completed. Distance to mediod: -22 - 41.7597% of index build completed. Distance to mediod: -21 - 42.0068% of index build completed. Distance to mediod: -21 - 42.2539% of index build completed. Distance to mediod: -20 - 42.501% of index build completed. Distance to mediod: -19 - 42.7481% of index build completed. Distance to mediod: -19 - 42.9952% of index build completed. Distance to mediod: -18 - 43.2423% of index build completed. Distance to mediod: -17 - 43.4894% of index build completed. Distance to mediod: -17 - 43.7365% of index build completed. Distance to mediod: -16 - 43.9836% of index build completed. Distance to mediod: -15 - 44.2307% of index build completed. Distance to mediod: -15 - 44.4778% of index build completed. Distance to mediod: -14 - 44.7249% of index build completed. Distance to mediod: -14 - 44.972% of index build completed. Distance to mediod: -13 - 45.2191% of index build completed. Distance to mediod: -12 - 45.4662% of index build completed. Distance to mediod: -12 - 45.7133% of index build completed. Distance to mediod: -11 - 45.9604% of index build completed. Distance to mediod: -10 - 46.2075% of index build completed. Distance to mediod: -10 - 46.4546% of index build completed. Distance to mediod: -9 - 46.7017% of index build completed. Distance to mediod: -8 - 46.9488% of index build completed. Distance to mediod: -8 - 47.1959% of index build completed. Distance to mediod: -7 - 47.443% of index build completed. Distance to mediod: -7 - 47.6901% of index build completed. Distance to mediod: -6 - 47.9372% of index build completed. Distance to mediod: -5 - 48.1843% of index build completed. Distance to mediod: -5 - 48.4314% of index build completed. Distance to mediod: -4 - 48.6785% of index build completed. Distance to mediod: -3 - 48.9256% of index build completed. Distance to mediod: -3 - 49.1727% of index build completed. Distance to mediod: -2 - 49.4198% of index build completed. Distance to mediod: -1 - 49.6669% of index build completed. Distance to mediod: -1 - 49.914% of index build completed. Distance to mediod: 0 - 50.1611% of index build completed. Distance to mediod: 0 - 50.4082% of index build completed. Distance to mediod: 1 - 50.6553% of index build completed. Distance to mediod: 2 - 50.9024% of index build completed. Distance to mediod: 2 - 51.1495% of index build completed. Distance to mediod: 3 - 51.3966% of index build completed. Distance to mediod: 4 - 51.6437% of index build completed. Distance to mediod: 4 - 51.8908% of index build completed. Distance to mediod: 5 - 52.1379% of index build completed. Distance to mediod: 5 - 52.385% of index build completed. Distance to mediod: 6 - 52.6321% of index build completed. Distance to mediod: 7 - 52.8792% of index build completed. Distance to mediod: 7 - 53.1263% of index build completed. Distance to mediod: 8 - 53.3734% of index build completed. Distance to mediod: 9 - 53.6205% of index build completed. Distance to mediod: 9 - 53.8676% of index build completed. Distance to mediod: 10 - 54.1147% of index build completed. Distance to mediod: 10 - 54.3618% of index build completed. Distance to mediod: 11 - 54.6089% of index build completed. Distance to mediod: 12 - 54.856% of index build completed. Distance to mediod: 12 - 55.1031% of index build completed. Distance to mediod: 13 - 55.3502% of index build completed. Distance to mediod: 14 - 55.5973% of index build completed. Distance to mediod: 14 - 55.8444% of index build completed. Distance to mediod: 15 - 56.0915% of index build completed. Distance to mediod: 16 - 56.3386% of index build completed. Distance to mediod: 16 - 56.5857% of index build completed. Distance to mediod: 17 - 56.8328% of index build completed. Distance to mediod: 17 - 57.0799% of index build completed. Distance to mediod: 18 - 57.327% of index build completed. Distance to mediod: 19 - 57.5741% of index build completed. Distance to mediod: 19 - 57.8212% of index build completed. Distance to mediod: 20 - 58.0683% of index build completed. Distance to mediod: 21 - 58.3154% of index build completed. Distance to mediod: 21 - 58.5625% of index build completed. Distance to mediod: 22 - 58.8096% of index build completed. Distance to mediod: 23 - 59.0567% of index build completed. Distance to mediod: 23 - 59.3038% of index build completed. Distance to mediod: 24 - 59.5509% of index build completed. Distance to mediod: 24 - 59.798% of index build completed. Distance to mediod: 25 - 60.0451% of index build completed. Distance to mediod: 26 - 60.2922% of index build completed. Distance to mediod: 26 - 60.5393% of index build completed. Distance to mediod: 27 - 60.7864% of index build completed. Distance to mediod: 28 - 61.0335% of index build completed. Distance to mediod: 28 - 61.2806% of index build completed. Distance to mediod: 29 - 61.5277% of index build completed. Distance to mediod: 30 - 61.7748% of index build completed. Distance to mediod: 30 - 62.0218% of index build completed. Distance to mediod: 31 - 62.2689% of index build completed. Distance to mediod: 31 - 62.516% of index build completed. Distance to mediod: 32 - 62.7631% of index build completed. Distance to mediod: 33 - 63.0102% of index build completed. Distance to mediod: 33 - 63.2573% of index build completed. Distance to mediod: 34 - 63.5044% of index build completed. Distance to mediod: 35 - 63.7515% of index build completed. Distance to mediod: 35 - 63.9986% of index build completed. Distance to mediod: 36 - 64.2457% of index build completed. Distance to mediod: 36 - 64.4928% of index build completed. Distance to mediod: 37 - 64.7399% of index build completed. Distance to mediod: 38 - 64.987% of index build completed. Distance to mediod: 38 - 65.2341% of index build completed. Distance to mediod: 39 - 65.4812% of index build completed. Distance to mediod: 40 - 65.7283% of index build completed. Distance to mediod: 40 - 65.9754% of index build completed. Distance to mediod: 41 - 66.2225% of index build completed. Distance to mediod: 42 - 66.4696% of index build completed. Distance to mediod: 42 - 66.7167% of index build completed. Distance to mediod: 43 - 66.9638% of index build completed. Distance to mediod: 43 - 67.2109% of index build completed. Distance to mediod: 44 - 67.458% of index build completed. Distance to mediod: 45 - 67.7051% of index build completed. Distance to mediod: 45 - 67.9522% of index build completed. Distance to mediod: 46 - 68.1993% of index build completed. Distance to mediod: 47 - 68.4464% of index build completed. Distance to mediod: 47 - 68.6935% of index build completed. Distance to mediod: 48 - 68.9406% of index build completed. Distance to mediod: 49 - 69.1877% of index build completed. Distance to mediod: 49 - 69.4348% of index build completed. Distance to mediod: 50 - 69.6819% of index build completed. Distance to mediod: 50 - 69.929% of index build completed. Distance to mediod: 51 - 70.1761% of index build completed. Distance to mediod: 52 - 70.4232% of index build completed. Distance to mediod: 52 - 70.6703% of index build completed. Distance to mediod: 53 - 70.9174% of index build completed. Distance to mediod: 54 - 71.1645% of index build completed. Distance to mediod: 54 - 71.4116% of index build completed. Distance to mediod: 55 - 71.6587% of index build completed. Distance to mediod: 56 - 71.9058% of index build completed. Distance to mediod: 56 - 72.1529% of index build completed. Distance to mediod: 57 - 72.4% of index build completed. Distance to mediod: 57 - 72.6471% of index build completed. Distance to mediod: 58 - 72.8942% of index build completed. Distance to mediod: 59 - 73.1413% of index build completed. Distance to mediod: 59 - 73.3884% of index build completed. Distance to mediod: 60 - 73.6355% of index build completed. Distance to mediod: 61 - 73.8826% of index build completed. Distance to mediod: 61 - 74.1297% of index build completed. Distance to mediod: 62 - 74.3768% of index build completed. Distance to mediod: 62 - 74.6239% of index build completed. Distance to mediod: 63 - 74.871% of index build completed. Distance to mediod: 64 - 75.1181% of index build completed. Distance to mediod: 64 - 75.3652% of index build completed. Distance to mediod: 65 - 75.6123% of index build completed. Distance to mediod: 66 - 75.8594% of index build completed. Distance to mediod: 66 - 76.1065% of index build completed. Distance to mediod: 67 - 76.3536% of index build completed. Distance to mediod: 68 - 76.6007% of index build completed. Distance to mediod: 68 - 76.8478% of index build completed. Distance to mediod: 69 - 77.0949% of index build completed. Distance to mediod: 69 - 77.342% of index build completed. Distance to mediod: 70 - 77.5891% of index build completed. Distance to mediod: 71 - 77.8362% of index build completed. Distance to mediod: 71 - 78.0833% of index build completed. Distance to mediod: 72 - 78.3304% of index build completed. Distance to mediod: 73 - 78.5775% of index build completed. Distance to mediod: 73 - 78.8246% of index build completed. Distance to mediod: 74 - 79.0717% of index build completed. Distance to mediod: 75 - 79.3188% of index build completed. Distance to mediod: 75 - 79.5659% of index build completed. Distance to mediod: 76 - 79.813% of index build completed. Distance to mediod: 76 - 80.0601% of index build completed. Distance to mediod: 77 - 80.3072% of index build completed. Distance to mediod: 78 - 80.5543% of index build completed. Distance to mediod: 78 - 80.8014% of index build completed. Distance to mediod: 79 - 81.0485% of index build completed. Distance to mediod: 80 - 81.2956% of index build completed. Distance to mediod: 80 - 81.5427% of index build completed. Distance to mediod: 81 - 81.7898% of index build completed. Distance to mediod: 81 - 82.0369% of index build completed. Distance to mediod: 82 - 82.284% of index build completed. Distance to mediod: 83 - 82.5311% of index build completed. Distance to mediod: 83 - 82.7782% of index build completed. Distance to mediod: 84 - 83.0253% of index build completed. Distance to mediod: 85 - 83.2724% of index build completed. Distance to mediod: 85 - 83.5195% of index build completed. Distance to mediod: 86 - 83.7666% of index build completed. Distance to mediod: 87 - 84.0137% of index build completed. Distance to mediod: 87 - 84.2608% of index build completed. Distance to mediod: 88 - 84.5079% of index build completed. Distance to mediod: 88 - 84.755% of index build completed. Distance to mediod: 89 - 85.0021% of index build completed. Distance to mediod: 90 - 85.2492% of index build completed. Distance to mediod: 90 - 85.4963% of index build completed. Distance to mediod: 91 - 85.7434% of index build completed. Distance to mediod: 92 - 85.9905% of index build completed. Distance to mediod: 92 - 86.2376% of index build completed. Distance to mediod: 93 - 86.4847% of index build completed. Distance to mediod: 94 - 86.7317% of index build completed. Distance to mediod: 94 - 86.9788% of index build completed. Distance to mediod: 95 - 87.2259% of index build completed. Distance to mediod: 95 - 87.473% of index build completed. Distance to mediod: 96 - 87.7201% of index build completed. Distance to mediod: 97 - 87.9672% of index build completed. Distance to mediod: 97 - 88.2143% of index build completed. Distance to mediod: 98 - 88.4614% of index build completed. Distance to mediod: 99 - 88.7085% of index build completed. Distance to mediod: 99 - 88.9556% of index build completed. Distance to mediod: 100 - 89.2027% of index build completed. Distance to mediod: 100 - 89.4498% of index build completed. Distance to mediod: 101 - 89.6969% of index build completed. Distance to mediod: 102 - 89.944% of index build completed. Distance to mediod: 102 - 90.1911% of index build completed. Distance to mediod: 103 - 90.4382% of index build completed. Distance to mediod: 104 - 90.6853% of index build completed. Distance to mediod: 104 - 90.9324% of index build completed. Distance to mediod: 105 - 91.1795% of index build completed. Distance to mediod: 106 - 91.4266% of index build completed. Distance to mediod: 106 - 91.6737% of index build completed. Distance to mediod: 107 - 91.9208% of index build completed. Distance to mediod: 107 - 92.1679% of index build completed. Distance to mediod: 108 - 92.415% of index build completed. Distance to mediod: 109 - 92.6621% of index build completed. Distance to mediod: 109 - 92.9092% of index build completed. Distance to mediod: 110 - 93.1563% of index build completed. Distance to mediod: 111 - 93.4034% of index build completed. Distance to mediod: 111 - 93.6505% of index build completed. Distance to mediod: 112 - 93.8976% of index build completed. Distance to mediod: 112 - 94.1447% of index build completed. Distance to mediod: 113 - 94.3918% of index build completed. Distance to mediod: 114 - 94.6389% of index build completed. Distance to mediod: 114 - 94.886% of index build completed. Distance to mediod: 115 - 95.1331% of index build completed. Distance to mediod: 116 - 95.3802% of index build completed. Distance to mediod: 116 - 95.6273% of index build completed. Distance to mediod: 117 - 95.8744% of index build completed. Distance to mediod: 118 - 96.1215% of index build completed. Distance to mediod: 118 - 96.3686% of index build completed. Distance to mediod: 119 - 96.6157% of index build completed. Distance to mediod: 119 - 96.8628% of index build completed. Distance to mediod: 120 - 97.1099% of index build completed. Distance to mediod: 121 - 97.357% of index build completed. Distance to mediod: 121 - 97.6041% of index build completed. Distance to mediod: 122 - 97.8512% of index build completed. Distance to mediod: 123 - 98.0983% of index build completed. Distance to mediod: 123 - 98.3454% of index build completed. Distance to mediod: 124 - 98.5925% of index build completed. Distance to mediod: 125 - 98.8396% of index build completed. Distance to mediod: 125 - 99.0867% of index build completed. Distance to mediod: 126 - 99.3338% of index build completed. Distance to mediod: 126 - 99.5809% of index build completed. Distance to mediod: 127 - 99.828% of index build completed.Starting final cleanup..done. Link time: 3629.97s -Index built with degree: max:64 avg:45.7558 min:1 count(deg<2):1018463 -Indexing time: 3648.44 + 0% of index build completed. Distance to mediod: -128 0.247099% of index build completed. Distance to mediod: -128 0.494198% of index build completed. Distance to mediod: -127 0.741297% of index build completed. Distance to mediod: -127 0.988396% of index build completed. Distance to mediod: -126 1.2355% of index build completed. Distance to mediod: -125 1.48259% of index build completed. Distance to mediod: -125 1.72969% of index build completed. Distance to mediod: -124 1.97679% of index build completed. Distance to mediod: -123 2.22389% of index build completed. Distance to mediod: -123 2.47099% of index build completed. Distance to mediod: -122 2.71809% of index build completed. Distance to mediod: -122 2.96519% of index build completed. Distance to mediod: -121 3.21229% of index build completed. Distance to mediod: -120 3.45939% of index build completed. Distance to mediod: -120 3.70649% of index build completed. Distance to mediod: -119 3.95358% of index build completed. Distance to mediod: -118 4.20068% of index build completed. Distance to mediod: -118 4.44778% of index build completed. Distance to mediod: -117 4.69488% of index build completed. Distance to mediod: -116 4.94198% of index build completed. Distance to mediod: -116 5.18908% of index build completed. Distance to mediod: -115 5.43618% of index build completed. Distance to mediod: -115 5.68328% of index build completed. Distance to mediod: -114 5.93038% of index build completed. Distance to mediod: -113 6.17748% of index build completed. Distance to mediod: -113 6.42457% of index build completed. Distance to mediod: -112 6.67167% of index build completed. Distance to mediod: -111 6.91877% of index build completed. Distance to mediod: -111 7.16587% of index build completed. Distance to mediod: -110 7.41297% of index build completed. Distance to mediod: -109 7.66007% of index build completed. Distance to mediod: -109 7.90717% of index build completed. Distance to mediod: -108 8.15427% of index build completed. Distance to mediod: -108 8.40137% of index build completed. Distance to mediod: -107 8.64847% of index build completed. Distance to mediod: -106 8.89556% of index build completed. Distance to mediod: -106 9.14266% of index build completed. Distance to mediod: -105 9.38976% of index build completed. Distance to mediod: -104 9.63686% of index build completed. Distance to mediod: -104 9.88396% of index build completed. Distance to mediod: -103 10.1311% of index build completed. Distance to mediod: -103 10.3782% of index build completed. Distance to mediod: -102 10.6253% of index build completed. Distance to mediod: -101 10.8724% of index build completed. Distance to mediod: -101 11.1195% of index build completed. Distance to mediod: -100 11.3666% of index build completed. Distance to mediod: -99 11.6137% of index build completed. Distance to mediod: -99 11.8608% of index build completed. Distance to mediod: -98 12.1079% of index build completed. Distance to mediod: -97 12.355% of index build completed. Distance to mediod: -97 12.602% of index build completed. Distance to mediod: -96 12.8491% of index build completed. Distance to mediod: -96 13.0962% of index build completed. Distance to mediod: -95 13.3433% of index build completed. Distance to mediod: -94 13.5904% of index build completed. Distance to mediod: -94 13.8375% of index build completed. Distance to mediod: -93 14.0846% of index build completed. Distance to mediod: -92 14.3317% of index build completed. Distance to mediod: -92 14.5788% of index build completed. Distance to mediod: -91 14.8259% of index build completed. Distance to mediod: -90 15.073% of index build completed. Distance to mediod: -90 15.3201% of index build completed. Distance to mediod: -89 15.5672% of index build completed. Distance to mediod: -89 15.8143% of index build completed. Distance to mediod: -88 16.0614% of index build completed. Distance to mediod: -87 16.3085% of index build completed. Distance to mediod: -87 16.5556% of index build completed. Distance to mediod: -86 16.8027% of index build completed. Distance to mediod: -85 17.0498% of index build completed. Distance to mediod: -85 17.2969% of index build completed. Distance to mediod: -84 17.544% of index build completed. Distance to mediod: -84 17.7911% of index build completed. Distance to mediod: -83 18.0382% of index build completed. Distance to mediod: -82 18.2853% of index build completed. Distance to mediod: -82 18.5324% of index build completed. Distance to mediod: -81 18.7795% of index build completed. Distance to mediod: -80 19.0266% of index build completed. Distance to mediod: -80 19.2737% of index build completed. Distance to mediod: -79 19.5208% of index build completed. Distance to mediod: -79 19.7679% of index build completed. Distance to mediod: -78 20.015% of index build completed. Distance to mediod: -77 20.2621% of index build completed. Distance to mediod: -77 20.5092% of index build completed. Distance to mediod: -76 20.7563% of index build completed. Distance to mediod: -75 21.0034% of index build completed. Distance to mediod: -75 21.2505% of index build completed. Distance to mediod: -74 21.4976% of index build completed. Distance to mediod: -74 21.7447% of index build completed. Distance to mediod: -73 21.9918% of index build completed. Distance to mediod: -72 22.2389% of index build completed. Distance to mediod: -72 22.486% of index build completed. Distance to mediod: -71 22.7331% of index build completed. Distance to mediod: -70 22.9802% of index build completed. Distance to mediod: -70 23.2273% of index build completed. Distance to mediod: -69 23.4744% of index build completed. Distance to mediod: -68 23.7215% of index build completed. Distance to mediod: -68 23.9686% of index build completed. Distance to mediod: -67 24.2157% of index build completed. Distance to mediod: -67 24.4628% of index build completed. Distance to mediod: -66 24.7099% of index build completed. Distance to mediod: -65 24.957% of index build completed. Distance to mediod: -65 25.2041% of index build completed. Distance to mediod: -64 25.4512% of index build completed. Distance to mediod: -63 25.6983% of index build completed. Distance to mediod: -63 25.9454% of index build completed. Distance to mediod: -62 26.1925% of index build completed. Distance to mediod: -62 26.4396% of index build completed. Distance to mediod: -61 26.6867% of index build completed. Distance to mediod: -60 26.9338% of index build completed. Distance to mediod: -60 27.1809% of index build completed. Distance to mediod: -59 27.428% of index build completed. Distance to mediod: -58 27.6751% of index build completed. Distance to mediod: -58 27.9222% of index build completed. Distance to mediod: -57 28.1693% of index build completed. Distance to mediod: -56 28.4164% of index build completed. Distance to mediod: -56 28.6635% of index build completed. Distance to mediod: -55 28.9106% of index build completed. Distance to mediod: -55 29.1577% of index build completed. Distance to mediod: -54 29.4048% of index build completed. Distance to mediod: -53 29.6519% of index build completed. Distance to mediod: -53 29.899% of index build completed. Distance to mediod: -52 30.1461% of index build completed. Distance to mediod: -51 30.3932% of index build completed. Distance to mediod: -51 30.6403% of index build completed. Distance to mediod: -50 30.8874% of index build completed. Distance to mediod: -49 31.1345% of index build completed. Distance to mediod: -49 31.3816% of index build completed. Distance to mediod: -48 31.6287% of index build completed. Distance to mediod: -48 31.8758% of index build completed. Distance to mediod: -47 32.1229% of index build completed. Distance to mediod: -46 32.37% of index build completed. Distance to mediod: -46 32.6171% of index build completed. Distance to mediod: -45 32.8642% of index build completed. Distance to mediod: -44 33.1113% of index build completed. Distance to mediod: -44 33.3584% of index build completed. Distance to mediod: -43 33.6055% of index build completed. Distance to mediod: -42 33.8526% of index build completed. Distance to mediod: -42 34.0997% of index build completed. Distance to mediod: -41 34.3468% of index build completed. Distance to mediod: -41 34.5939% of index build completed. Distance to mediod: -40 34.841% of index build completed. Distance to mediod: -39 35.0881% of index build completed. Distance to mediod: -39 35.3352% of index build completed. Distance to mediod: -38 35.5823% of index build completed. Distance to mediod: -37 35.8294% of index build completed. Distance to mediod: -37 36.0765% of index build completed. Distance to mediod: -36 36.3236% of index build completed. Distance to mediod: -35 36.5707% of index build completed. Distance to mediod: -35 36.8178% of index build completed. Distance to mediod: -34 37.0649% of index build completed. Distance to mediod: -34 37.3119% of index build completed. Distance to mediod: -33 37.559% of index build completed. Distance to mediod: -33 37.8061% of index build completed. Distance to mediod: -32 38.0532% of index build completed. Distance to mediod: -31 38.3003% of index build completed. Distance to mediod: -31 38.5474% of index build completed. Distance to mediod: -30 38.7945% of index build completed. Distance to mediod: -29 39.0416% of index build completed. Distance to mediod: -29 39.2887% of index build completed. Distance to mediod: -28 39.5358% of index build completed. Distance to mediod: -28 39.7829% of index build completed. Distance to mediod: -27 40.03% of index build completed. Distance to mediod: -26 40.2771% of index build completed. Distance to mediod: -26 40.5242% of index build completed. Distance to mediod: -25 40.7713% of index build completed. Distance to mediod: -24 41.0184% of index build completed. Distance to mediod: -24 41.2655% of index build completed. Distance to mediod: -23 41.5126% of index build completed. Distance to mediod: -22 41.7597% of index build completed. Distance to mediod: -22 42.0068% of index build completed. Distance to mediod: -21 42.2539% of index build completed. Distance to mediod: -21 42.501% of index build completed. Distance to mediod: -20 42.7481% of index build completed. Distance to mediod: -19 42.9952% of index build completed. Distance to mediod: -19 43.2423% of index build completed. Distance to mediod: -18 43.4894% of index build completed. Distance to mediod: -17 43.7365% of index build completed. Distance to mediod: -17 43.9836% of index build completed. Distance to mediod: -16 44.2307% of index build completed. Distance to mediod: -15 44.4778% of index build completed. Distance to mediod: -15 44.7249% of index build completed. Distance to mediod: -14 44.972% of index build completed. Distance to mediod: -14 45.2191% of index build completed. Distance to mediod: -13 45.4662% of index build completed. Distance to mediod: -12 45.7133% of index build completed. Distance to mediod: -12 45.9604% of index build completed. Distance to mediod: -11 46.2075% of index build completed. Distance to mediod: -10 46.4546% of index build completed. Distance to mediod: -10 46.7017% of index build completed. Distance to mediod: -9 46.9488% of index build completed. Distance to mediod: -8 47.1959% of index build completed. Distance to mediod: -8 47.443% of index build completed. Distance to mediod: -7 47.6901% of index build completed. Distance to mediod: -7 47.9372% of index build completed. Distance to mediod: -6 48.1843% of index build completed. Distance to mediod: -5 48.4314% of index build completed. Distance to mediod: -5 48.6785% of index build completed. Distance to mediod: -4 48.9256% of index build completed. Distance to mediod: -3 49.1727% of index build completed. Distance to mediod: -3 49.4198% of index build completed. Distance to mediod: -2 49.6669% of index build completed. Distance to mediod: -1 49.914% of index build completed. Distance to mediod: -1 50.1611% of index build completed. Distance to mediod: 0 50.4082% of index build completed. Distance to mediod: 0 50.6553% of index build completed. Distance to mediod: 1 50.9024% of index build completed. Distance to mediod: 2 51.1495% of index build completed. Distance to mediod: 2 51.3966% of index build completed. Distance to mediod: 3 51.6437% of index build completed. Distance to mediod: 4 51.8908% of index build completed. Distance to mediod: 4 52.1379% of index build completed. Distance to mediod: 5 52.385% of index build completed. Distance to mediod: 5 52.6321% of index build completed. Distance to mediod: 6 52.8792% of index build completed. Distance to mediod: 7 53.1263% of index build completed. Distance to mediod: 7 53.3734% of index build completed. Distance to mediod: 8 53.6205% of index build completed. Distance to mediod: 9 53.8676% of index build completed. Distance to mediod: 9 54.1147% of index build completed. Distance to mediod: 10 54.3618% of index build completed. Distance to mediod: 10 54.6089% of index build completed. Distance to mediod: 11 54.856% of index build completed. Distance to mediod: 12 55.1031% of index build completed. Distance to mediod: 12 55.3502% of index build completed. Distance to mediod: 13 55.5973% of index build completed. Distance to mediod: 14 55.8444% of index build completed. Distance to mediod: 14 56.0915% of index build completed. Distance to mediod: 15 56.3386% of index build completed. Distance to mediod: 16 56.5857% of index build completed. Distance to mediod: 16 56.8328% of index build completed. Distance to mediod: 17 57.0799% of index build completed. Distance to mediod: 17 57.327% of index build completed. Distance to mediod: 18 57.5741% of index build completed. Distance to mediod: 19 57.8212% of index build completed. Distance to mediod: 19 58.0683% of index build completed. Distance to mediod: 20 58.3154% of index build completed. Distance to mediod: 21 58.5625% of index build completed. Distance to mediod: 21 58.8096% of index build completed. Distance to mediod: 22 59.0567% of index build completed. Distance to mediod: 23 59.3038% of index build completed. Distance to mediod: 23 59.5509% of index build completed. Distance to mediod: 24 59.798% of index build completed. Distance to mediod: 24 60.0451% of index build completed. Distance to mediod: 25 60.2922% of index build completed. Distance to mediod: 26 60.5393% of index build completed. Distance to mediod: 26 60.7864% of index build completed. Distance to mediod: 27 61.0335% of index build completed. Distance to mediod: 28 61.2806% of index build completed. Distance to mediod: 28 61.5277% of index build completed. Distance to mediod: 29 61.7748% of index build completed. Distance to mediod: 30 62.0218% of index build completed. Distance to mediod: 30 62.2689% of index build completed. Distance to mediod: 31 62.516% of index build completed. Distance to mediod: 31 62.7631% of index build completed. Distance to mediod: 32 63.0102% of index build completed. Distance to mediod: 33 63.2573% of index build completed. Distance to mediod: 33 63.5044% of index build completed. Distance to mediod: 34 63.7515% of index build completed. Distance to mediod: 35 63.9986% of index build completed. Distance to mediod: 35 64.2457% of index build completed. Distance to mediod: 36 64.4928% of index build completed. Distance to mediod: 36 64.7399% of index build completed. Distance to mediod: 37 64.987% of index build completed. Distance to mediod: 38 65.2341% of index build completed. Distance to mediod: 38 65.4812% of index build completed. Distance to mediod: 39 65.7283% of index build completed. Distance to mediod: 40 65.9754% of index build completed. Distance to mediod: 40 66.2225% of index build completed. Distance to mediod: 41 66.4696% of index build completed. Distance to mediod: 42 66.7167% of index build completed. Distance to mediod: 42 66.9638% of index build completed. Distance to mediod: 43 67.2109% of index build completed. Distance to mediod: 43 67.458% of index build completed. Distance to mediod: 44 67.7051% of index build completed. Distance to mediod: 45 67.9522% of index build completed. Distance to mediod: 45 68.1993% of index build completed. Distance to mediod: 46 68.4464% of index build completed. Distance to mediod: 47 68.6935% of index build completed. Distance to mediod: 47 68.9406% of index build completed. Distance to mediod: 48 69.1877% of index build completed. Distance to mediod: 49 69.4348% of index build completed. Distance to mediod: 49 69.6819% of index build completed. Distance to mediod: 50 69.929% of index build completed. Distance to mediod: 50 70.1761% of index build completed. Distance to mediod: 51 70.4232% of index build completed. Distance to mediod: 52 70.6703% of index build completed. Distance to mediod: 52 70.9174% of index build completed. Distance to mediod: 53 71.1645% of index build completed. Distance to mediod: 54 71.4116% of index build completed. Distance to mediod: 54 71.6587% of index build completed. Distance to mediod: 55 71.9058% of index build completed. Distance to mediod: 56 72.1529% of index build completed. Distance to mediod: 56 72.4% of index build completed. Distance to mediod: 57 72.6471% of index build completed. Distance to mediod: 57 72.8942% of index build completed. Distance to mediod: 58 73.1413% of index build completed. Distance to mediod: 59 73.3884% of index build completed. Distance to mediod: 59 73.6355% of index build completed. Distance to mediod: 60 73.8826% of index build completed. Distance to mediod: 61 74.1297% of index build completed. Distance to mediod: 61 74.3768% of index build completed. Distance to mediod: 62 74.6239% of index build completed. Distance to mediod: 62 74.871% of index build completed. Distance to mediod: 63 75.1181% of index build completed. Distance to mediod: 64 75.3652% of index build completed. Distance to mediod: 64 75.6123% of index build completed. Distance to mediod: 65 75.8594% of index build completed. Distance to mediod: 66 76.1065% of index build completed. Distance to mediod: 66 76.3536% of index build completed. Distance to mediod: 67 76.6007% of index build completed. Distance to mediod: 68 76.8478% of index build completed. Distance to mediod: 68 77.0949% of index build completed. Distance to mediod: 69 77.342% of index build completed. Distance to mediod: 69 77.5891% of index build completed. Distance to mediod: 70 77.8362% of index build completed. Distance to mediod: 71 78.0833% of index build completed. Distance to mediod: 71 78.3304% of index build completed. Distance to mediod: 72 78.5775% of index build completed. Distance to mediod: 73 78.8246% of index build completed. Distance to mediod: 73 79.0717% of index build completed. Distance to mediod: 74 79.3188% of index build completed. Distance to mediod: 75 79.5659% of index build completed. Distance to mediod: 75 79.813% of index build completed. Distance to mediod: 76 80.0601% of index build completed. Distance to mediod: 76 80.3072% of index build completed. Distance to mediod: 77 80.5543% of index build completed. Distance to mediod: 78 80.8014% of index build completed. Distance to mediod: 78 81.0485% of index build completed. Distance to mediod: 79 81.2956% of index build completed. Distance to mediod: 80 81.5427% of index build completed. Distance to mediod: 80 81.7898% of index build completed. Distance to mediod: 81 82.0369% of index build completed. Distance to mediod: 81 82.284% of index build completed. Distance to mediod: 82 82.5311% of index build completed. Distance to mediod: 83 82.7782% of index build completed. Distance to mediod: 83 83.0253% of index build completed. Distance to mediod: 84 83.2724% of index build completed. Distance to mediod: 85 83.5195% of index build completed. Distance to mediod: 85 83.7666% of index build completed. Distance to mediod: 86 84.0137% of index build completed. Distance to mediod: 87 84.2608% of index build completed. Distance to mediod: 87 84.5079% of index build completed. Distance to mediod: 88 84.755% of index build completed. Distance to mediod: 88 85.0021% of index build completed. Distance to mediod: 89 85.2492% of index build completed. Distance to mediod: 90 85.4963% of index build completed. Distance to mediod: 90 85.7434% of index build completed. Distance to mediod: 91 85.9905% of index build completed. Distance to mediod: 92 86.2376% of index build completed. Distance to mediod: 92 86.4847% of index build completed. Distance to mediod: 93 86.7317% of index build completed. Distance to mediod: 94 86.9788% of index build completed. Distance to mediod: 94 87.2259% of index build completed. Distance to mediod: 95 87.473% of index build completed. Distance to mediod: 95 87.7201% of index build completed. Distance to mediod: 96 87.9672% of index build completed. Distance to mediod: 97 88.2143% of index build completed. Distance to mediod: 97 88.4614% of index build completed. Distance to mediod: 98 88.7085% of index build completed. Distance to mediod: 99 88.9556% of index build completed. Distance to mediod: 99 89.2027% of index build completed. Distance to mediod: 100 89.4498% of index build completed. Distance to mediod: 100 89.6969% of index build completed. Distance to mediod: 101 89.944% of index build completed. Distance to mediod: 102 90.1911% of index build completed. Distance to mediod: 102 90.4382% of index build completed. Distance to mediod: 103 90.6853% of index build completed. Distance to mediod: 104 90.9324% of index build completed. Distance to mediod: 104 91.1795% of index build completed. Distance to mediod: 105 91.4266% of index build completed. Distance to mediod: 106 91.6737% of index build completed. Distance to mediod: 106 91.9208% of index build completed. Distance to mediod: 107 92.1679% of index build completed. Distance to mediod: 107 92.415% of index build completed. Distance to mediod: 108 92.6621% of index build completed. Distance to mediod: 109 92.9092% of index build completed. Distance to mediod: 109 93.1563% of index build completed. Distance to mediod: 110 93.4034% of index build completed. Distance to mediod: 111 93.6505% of index build completed. Distance to mediod: 111 93.8976% of index build completed. Distance to mediod: 112 94.1447% of index build completed. Distance to mediod: 112 94.3918% of index build completed. Distance to mediod: 113 94.6389% of index build completed. Distance to mediod: 114 94.886% of index build completed. Distance to mediod: 114 95.1331% of index build completed. Distance to mediod: 115 95.3802% of index build completed. Distance to mediod: 116 95.6273% of index build completed. Distance to mediod: 116 95.8744% of index build completed. Distance to mediod: 117 96.1215% of index build completed. Distance to mediod: 118 96.3686% of index build completed. Distance to mediod: 118 96.6157% of index build completed. Distance to mediod: 119 96.8628% of index build completed. Distance to mediod: 119 97.1099% of index build completed. Distance to mediod: 120 97.357% of index build completed. Distance to mediod: 121 97.6041% of index build completed. Distance to mediod: 121 97.8512% of index build completed. Distance to mediod: 122 98.0983% of index build completed. Distance to mediod: 123 98.3454% of index build completed. Distance to mediod: 123 98.5925% of index build completed. Distance to mediod: 124 98.8396% of index build completed. Distance to mediod: 125 99.0867% of index build completed. Distance to mediod: 125 99.3338% of index build completed. Distance to mediod: 126 99.5809% of index build completed. Distance to mediod: 126 99.828% of index build completed. Distance to mediod: 127Starting final cleanup..done. Link time: 3825.28s +Index built with degree: max:64 avg:44.8831 min:1 count(deg<2):1012594 +Robust Prune Calls: 108442436 +Indexing time: 3850.67 Not saving tags as they are not enabled. -Time taken for save: 19.4831s. +Time taken for save: 28.4491s. diff --git a/src/index.cpp b/src/index.cpp index 85e9f3bd6..fda819422 100644 --- a/src/index.cpp +++ b/src/index.cpp @@ -4,7 +4,7 @@ #include #include - +#include #include "boost/dynamic_bitset.hpp" #include "index_factory.h" #include "memory_mapper.h" @@ -25,6 +25,9 @@ #define MAX_POINTS_FOR_USING_BITSET 10000000 +//bool reduce_prune = false; +std::atomic count_prune = 0; + namespace diskann { // Initialize an index with metric m, load the data of type T with filename @@ -1058,7 +1061,8 @@ void Index::occlude_list(const uint32_t location, std::vector &result, InMemQueryScratch *scratch, const tsl::robin_set *const delete_set_ptr) -{ +{ + count_prune++; if (pool.size() == 0) return; @@ -1074,7 +1078,8 @@ void Index::occlude_list(const uint32_t location, std::vector void Index 0){ - diskann::cout <<" Distance to mediod: " << (float)(distances_to_mediod[node_ctr].first) << std::endl; - } // Find and add appropriate graph edges ScratchStoreManager> manager(_query_scratch); @@ -1350,8 +1352,7 @@ template void Index::build_with_data_populated(const std::vector & } diskann::cout << "Index built with degree: max:" << max << " avg:" << (float)total / (float)(_nd + _num_frozen_pts) << " min:" << min << " count(deg<2):" << cnt << std::endl; - + diskann::cout << "Robust Prune Calls: " << count_prune << std::endl; _has_built = true; } template