From 85c01412946ed1e2632cd2dfae4c672efd38caba Mon Sep 17 00:00:00 2001 From: Paul Irwin Date: Sun, 22 Dec 2024 10:05:52 -0700 Subject: [PATCH] Port Packed Producing Python Programs, #1008 (#1067) --- .../Lucene-Net-Util-Packed-Python-Scripts.yml | 54 +++ src/Lucene.Net/Util/Packed/BulkOperation.cs | 16 +- .../Util/Packed/BulkOperationPacked1.cs | 16 +- .../Util/Packed/BulkOperationPacked10.cs | 12 +- .../Util/Packed/BulkOperationPacked11.cs | 10 +- .../Util/Packed/BulkOperationPacked12.cs | 10 +- .../Util/Packed/BulkOperationPacked13.cs | 10 +- .../Util/Packed/BulkOperationPacked14.cs | 10 +- .../Util/Packed/BulkOperationPacked15.cs | 10 +- .../Util/Packed/BulkOperationPacked16.cs | 10 +- .../Util/Packed/BulkOperationPacked17.cs | 10 +- .../Util/Packed/BulkOperationPacked18.cs | 10 +- .../Util/Packed/BulkOperationPacked19.cs | 10 +- .../Util/Packed/BulkOperationPacked2.cs | 14 +- .../Util/Packed/BulkOperationPacked20.cs | 10 +- .../Util/Packed/BulkOperationPacked21.cs | 10 +- .../Util/Packed/BulkOperationPacked22.cs | 10 +- .../Util/Packed/BulkOperationPacked23.cs | 10 +- .../Util/Packed/BulkOperationPacked24.cs | 10 +- .../Util/Packed/BulkOperationPacked3.cs | 10 +- .../Util/Packed/BulkOperationPacked4.cs | 14 +- .../Util/Packed/BulkOperationPacked5.cs | 10 +- .../Util/Packed/BulkOperationPacked6.cs | 10 +- .../Util/Packed/BulkOperationPacked7.cs | 10 +- .../Util/Packed/BulkOperationPacked8.cs | 10 +- .../Util/Packed/BulkOperationPacked9.cs | 12 +- src/Lucene.Net/Util/Packed/Direct16.cs | 19 +- src/Lucene.Net/Util/Packed/Direct32.cs | 21 +- src/Lucene.Net/Util/Packed/Direct64.cs | 25 +- src/Lucene.Net/Util/Packed/Direct8.cs | 21 +- .../Util/Packed/Packed16ThreeBlocks.cs | 17 +- .../Util/Packed/Packed64SingleBlock.cs | 55 +-- .../Util/Packed/Packed8ThreeBlocks.cs | 23 +- src/Lucene.Net/Util/Packed/README.md | 28 ++ .../Util/Packed/gen_BulkOperation.py | 421 ++++++++++++++++++ src/Lucene.Net/Util/Packed/gen_Direct.py | 231 ++++++++++ .../Util/Packed/gen_Packed64SingleBlock.py | 345 ++++++++++++++ .../Util/Packed/gen_PackedThreeBlocks.py | 199 +++++++++ src/Lucene.Net/Util/Packed/package.md | 16 +- 39 files changed, 1607 insertions(+), 142 deletions(-) create mode 100644 .github/workflows/Lucene-Net-Util-Packed-Python-Scripts.yml create mode 100644 src/Lucene.Net/Util/Packed/README.md create mode 100644 src/Lucene.Net/Util/Packed/gen_BulkOperation.py create mode 100644 src/Lucene.Net/Util/Packed/gen_Direct.py create mode 100644 src/Lucene.Net/Util/Packed/gen_Packed64SingleBlock.py create mode 100644 src/Lucene.Net/Util/Packed/gen_PackedThreeBlocks.py diff --git a/.github/workflows/Lucene-Net-Util-Packed-Python-Scripts.yml b/.github/workflows/Lucene-Net-Util-Packed-Python-Scripts.yml new file mode 100644 index 0000000000..65a688d2c6 --- /dev/null +++ b/.github/workflows/Lucene-Net-Util-Packed-Python-Scripts.yml @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: 'Lucene.Net.Util.Packed Python Scripts' + +# This will: +# checkout this repo +# run the python scripts in the src/Lucene.Net/Util/Packed folder +# check for any pending git changes + +on: + workflow_dispatch: + pull_request: + paths: + - '.github/workflows/Lucene-Net-Packed-Python-Scripts.yml' + - 'src/Lucene.Net/Util/Packed/*.py' + - 'src/Lucene.Net/Util/Packed/BulkOperation*.cs' + - 'src/Lucene.Net/Util/Packed/Direct*.cs' + - 'src/Lucene.Net/Util/Packed/Packed*.cs' + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout Lucene.Net source + uses: actions/checkout@v3 + + - name: Execute Python scripts + working-directory: src/Lucene.Net/Util/Packed + run: | + python3 ./gen_BulkOperation.py + python3 ./gen_Direct.py + python3 ./gen_Packed64SingleBlock.py + python3 ./gen_PackedThreeBlocks.py + shell: bash + + - name: Check for changes + run: | + git diff --exit-code + shell: bash diff --git a/src/Lucene.Net/Util/Packed/BulkOperation.cs b/src/Lucene.Net/Util/Packed/BulkOperation.cs index 6dc0f49c5e..70b4da34ef 100644 --- a/src/Lucene.Net/Util/Packed/BulkOperation.cs +++ b/src/Lucene.Net/Util/Packed/BulkOperation.cs @@ -1,7 +1,14 @@ -using Lucene.Net.Diagnostics; +using Lucene.Net.Diagnostics; using System; -// this file has been automatically generated, DO NOT EDIT +//------------------------------------------------------------------------------ +// +// This code was generated by the gen_BulkOperation.py script. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ namespace Lucene.Net.Util.Packed { @@ -121,7 +128,7 @@ internal abstract class BulkOperation : PackedInt32s.IDecoder, PackedInt32s.IEnc new BulkOperationPacked(61), new BulkOperationPacked(62), new BulkOperationPacked(63), - new BulkOperationPacked(64) + new BulkOperationPacked(64), }; // NOTE: this is sparse (some entries are null): @@ -157,9 +164,10 @@ internal abstract class BulkOperation : PackedInt32s.IDecoder, PackedInt32s.IEnc null, null, null, - new BulkOperationPackedSingleBlock(32) + new BulkOperationPackedSingleBlock(32), }; + public static BulkOperation Of(PackedInt32s.Format format, int bitsPerValue) { if (format == PackedInt32s.Format.PACKED) diff --git a/src/Lucene.Net/Util/Packed/BulkOperationPacked1.cs b/src/Lucene.Net/Util/Packed/BulkOperationPacked1.cs index 49f8870580..1c44030987 100644 --- a/src/Lucene.Net/Util/Packed/BulkOperationPacked1.cs +++ b/src/Lucene.Net/Util/Packed/BulkOperationPacked1.cs @@ -1,4 +1,11 @@ -// this file has been automatically generated, DO NOT EDIT +//------------------------------------------------------------------------------ +// +// This code was generated by the gen_BulkOperation.py script. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ namespace Lucene.Net.Util.Packed { @@ -45,7 +52,7 @@ public override void Decode(byte[] blocks, int blocksOffset, int[] values, int v { for (int j = 0; j < iterations; ++j) { - var block = blocks[blocksOffset++]; + byte block = blocks[blocksOffset++]; values[valuesOffset++] = (block >>> 7) & 1; values[valuesOffset++] = (block >>> 6) & 1; values[valuesOffset++] = (block >>> 5) & 1; @@ -56,7 +63,6 @@ public override void Decode(byte[] blocks, int blocksOffset, int[] values, int v values[valuesOffset++] = block & 1; } } - public override void Decode(long[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) { for (int i = 0; i < iterations; ++i) @@ -64,7 +70,7 @@ public override void Decode(long[] blocks, int blocksOffset, long[] values, int long block = blocks[blocksOffset++]; for (int shift = 63; shift >= 0; shift -= 1) { - values[valuesOffset++] = ((block >>> shift)) & 1; + values[valuesOffset++] = (block >>> shift) & 1; } } } @@ -73,7 +79,7 @@ public override void Decode(byte[] blocks, int blocksOffset, long[] values, int { for (int j = 0; j < iterations; ++j) { - var block = blocks[blocksOffset++]; + byte block = blocks[blocksOffset++]; values[valuesOffset++] = (block >>> 7) & 1; values[valuesOffset++] = (block >>> 6) & 1; values[valuesOffset++] = (block >>> 5) & 1; diff --git a/src/Lucene.Net/Util/Packed/BulkOperationPacked10.cs b/src/Lucene.Net/Util/Packed/BulkOperationPacked10.cs index c1caa2396f..f8ce2c6f0c 100644 --- a/src/Lucene.Net/Util/Packed/BulkOperationPacked10.cs +++ b/src/Lucene.Net/Util/Packed/BulkOperationPacked10.cs @@ -1,4 +1,11 @@ -// this file has been automatically generated, DO NOT EDIT +//------------------------------------------------------------------------------ +// +// This code was generated by the gen_BulkOperation.py script. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ namespace Lucene.Net.Util.Packed { @@ -88,13 +95,12 @@ public override void Decode(byte[] blocks, int blocksOffset, int[] values, int v values[valuesOffset++] = ((byte3 & 3) << 8) | byte4; } } - public override void Decode(long[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) { for (int i = 0; i < iterations; ++i) { long block0 = blocks[blocksOffset++]; - values[valuesOffset++] = (block0 >>> 54); + values[valuesOffset++] = block0 >>> 54; values[valuesOffset++] = (block0 >>> 44) & 1023L; values[valuesOffset++] = (block0 >>> 34) & 1023L; values[valuesOffset++] = (block0 >>> 24) & 1023L; diff --git a/src/Lucene.Net/Util/Packed/BulkOperationPacked11.cs b/src/Lucene.Net/Util/Packed/BulkOperationPacked11.cs index d840038aa3..ede8063939 100644 --- a/src/Lucene.Net/Util/Packed/BulkOperationPacked11.cs +++ b/src/Lucene.Net/Util/Packed/BulkOperationPacked11.cs @@ -1,4 +1,11 @@ -// this file has been automatically generated, DO NOT EDIT +//------------------------------------------------------------------------------ +// +// This code was generated by the gen_BulkOperation.py script. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ namespace Lucene.Net.Util.Packed { @@ -136,7 +143,6 @@ public override void Decode(byte[] blocks, int blocksOffset, int[] values, int v values[valuesOffset++] = ((byte9 & 7) << 8) | byte10; } } - public override void Decode(long[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) { for (int i = 0; i < iterations; ++i) diff --git a/src/Lucene.Net/Util/Packed/BulkOperationPacked12.cs b/src/Lucene.Net/Util/Packed/BulkOperationPacked12.cs index 9e81409705..e89f1cb913 100644 --- a/src/Lucene.Net/Util/Packed/BulkOperationPacked12.cs +++ b/src/Lucene.Net/Util/Packed/BulkOperationPacked12.cs @@ -1,4 +1,11 @@ -// this file has been automatically generated, DO NOT EDIT +//------------------------------------------------------------------------------ +// +// This code was generated by the gen_BulkOperation.py script. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ namespace Lucene.Net.Util.Packed { @@ -66,7 +73,6 @@ public override void Decode(byte[] blocks, int blocksOffset, int[] values, int v values[valuesOffset++] = ((byte1 & 15) << 8) | byte2; } } - public override void Decode(long[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) { for (int i = 0; i < iterations; ++i) diff --git a/src/Lucene.Net/Util/Packed/BulkOperationPacked13.cs b/src/Lucene.Net/Util/Packed/BulkOperationPacked13.cs index dd912fec73..c69b177400 100644 --- a/src/Lucene.Net/Util/Packed/BulkOperationPacked13.cs +++ b/src/Lucene.Net/Util/Packed/BulkOperationPacked13.cs @@ -1,4 +1,11 @@ -// this file has been automatically generated, DO NOT EDIT +//------------------------------------------------------------------------------ +// +// This code was generated by the gen_BulkOperation.py script. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ namespace Lucene.Net.Util.Packed { @@ -140,7 +147,6 @@ public override void Decode(byte[] blocks, int blocksOffset, int[] values, int v values[valuesOffset++] = ((byte11 & 31) << 8) | byte12; } } - public override void Decode(long[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) { for (int i = 0; i < iterations; ++i) diff --git a/src/Lucene.Net/Util/Packed/BulkOperationPacked14.cs b/src/Lucene.Net/Util/Packed/BulkOperationPacked14.cs index 1afe982c63..ad3b91b64a 100644 --- a/src/Lucene.Net/Util/Packed/BulkOperationPacked14.cs +++ b/src/Lucene.Net/Util/Packed/BulkOperationPacked14.cs @@ -1,4 +1,11 @@ -// this file has been automatically generated, DO NOT EDIT +//------------------------------------------------------------------------------ +// +// This code was generated by the gen_BulkOperation.py script. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ namespace Lucene.Net.Util.Packed { @@ -92,7 +99,6 @@ public override void Decode(byte[] blocks, int blocksOffset, int[] values, int v values[valuesOffset++] = ((byte5 & 63) << 8) | byte6; } } - public override void Decode(long[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) { for (int i = 0; i < iterations; ++i) diff --git a/src/Lucene.Net/Util/Packed/BulkOperationPacked15.cs b/src/Lucene.Net/Util/Packed/BulkOperationPacked15.cs index f22146ac88..1752c90d9f 100644 --- a/src/Lucene.Net/Util/Packed/BulkOperationPacked15.cs +++ b/src/Lucene.Net/Util/Packed/BulkOperationPacked15.cs @@ -1,4 +1,11 @@ -// this file has been automatically generated, DO NOT EDIT +//------------------------------------------------------------------------------ +// +// This code was generated by the gen_BulkOperation.py script. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ namespace Lucene.Net.Util.Packed { @@ -144,7 +151,6 @@ public override void Decode(byte[] blocks, int blocksOffset, int[] values, int v values[valuesOffset++] = ((byte13 & 127) << 8) | byte14; } } - public override void Decode(long[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) { for (int i = 0; i < iterations; ++i) diff --git a/src/Lucene.Net/Util/Packed/BulkOperationPacked16.cs b/src/Lucene.Net/Util/Packed/BulkOperationPacked16.cs index 13a4743b87..93f50da8fd 100644 --- a/src/Lucene.Net/Util/Packed/BulkOperationPacked16.cs +++ b/src/Lucene.Net/Util/Packed/BulkOperationPacked16.cs @@ -1,4 +1,11 @@ -// this file has been automatically generated, DO NOT EDIT +//------------------------------------------------------------------------------ +// +// This code was generated by the gen_BulkOperation.py script. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ namespace Lucene.Net.Util.Packed { @@ -48,7 +55,6 @@ public override void Decode(byte[] blocks, int blocksOffset, int[] values, int v values[valuesOffset++] = ((blocks[blocksOffset++] & 0xFF) << 8) | (blocks[blocksOffset++] & 0xFF); } } - public override void Decode(long[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) { for (int i = 0; i < iterations; ++i) diff --git a/src/Lucene.Net/Util/Packed/BulkOperationPacked17.cs b/src/Lucene.Net/Util/Packed/BulkOperationPacked17.cs index 33d00ad985..9e643dc2fe 100644 --- a/src/Lucene.Net/Util/Packed/BulkOperationPacked17.cs +++ b/src/Lucene.Net/Util/Packed/BulkOperationPacked17.cs @@ -1,4 +1,11 @@ -// this file has been automatically generated, DO NOT EDIT +//------------------------------------------------------------------------------ +// +// This code was generated by the gen_BulkOperation.py script. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ namespace Lucene.Net.Util.Packed { @@ -148,7 +155,6 @@ public override void Decode(byte[] blocks, int blocksOffset, int[] values, int v values[valuesOffset++] = ((byte14 & 1) << 16) | (byte15 << 8) | byte16; } } - public override void Decode(long[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) { for (int i = 0; i < iterations; ++i) diff --git a/src/Lucene.Net/Util/Packed/BulkOperationPacked18.cs b/src/Lucene.Net/Util/Packed/BulkOperationPacked18.cs index 8f1a5b9552..d428a876bf 100644 --- a/src/Lucene.Net/Util/Packed/BulkOperationPacked18.cs +++ b/src/Lucene.Net/Util/Packed/BulkOperationPacked18.cs @@ -1,4 +1,11 @@ -// this file has been automatically generated, DO NOT EDIT +//------------------------------------------------------------------------------ +// +// This code was generated by the gen_BulkOperation.py script. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ namespace Lucene.Net.Util.Packed { @@ -96,7 +103,6 @@ public override void Decode(byte[] blocks, int blocksOffset, int[] values, int v values[valuesOffset++] = ((byte6 & 3) << 16) | (byte7 << 8) | byte8; } } - public override void Decode(long[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) { for (int i = 0; i < iterations; ++i) diff --git a/src/Lucene.Net/Util/Packed/BulkOperationPacked19.cs b/src/Lucene.Net/Util/Packed/BulkOperationPacked19.cs index de93d3b886..d216eafb68 100644 --- a/src/Lucene.Net/Util/Packed/BulkOperationPacked19.cs +++ b/src/Lucene.Net/Util/Packed/BulkOperationPacked19.cs @@ -1,4 +1,11 @@ -// this file has been automatically generated, DO NOT EDIT +//------------------------------------------------------------------------------ +// +// This code was generated by the gen_BulkOperation.py script. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ namespace Lucene.Net.Util.Packed { @@ -152,7 +159,6 @@ public override void Decode(byte[] blocks, int blocksOffset, int[] values, int v values[valuesOffset++] = ((byte16 & 7) << 16) | (byte17 << 8) | byte18; } } - public override void Decode(long[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) { for (int i = 0; i < iterations; ++i) diff --git a/src/Lucene.Net/Util/Packed/BulkOperationPacked2.cs b/src/Lucene.Net/Util/Packed/BulkOperationPacked2.cs index da7f3f1854..cb85da908a 100644 --- a/src/Lucene.Net/Util/Packed/BulkOperationPacked2.cs +++ b/src/Lucene.Net/Util/Packed/BulkOperationPacked2.cs @@ -1,4 +1,11 @@ -// this file has been automatically generated, DO NOT EDIT +//------------------------------------------------------------------------------ +// +// This code was generated by the gen_BulkOperation.py script. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ namespace Lucene.Net.Util.Packed { @@ -45,14 +52,13 @@ public override void Decode(byte[] blocks, int blocksOffset, int[] values, int v { for (int j = 0; j < iterations; ++j) { - var block = blocks[blocksOffset++]; + byte block = blocks[blocksOffset++]; values[valuesOffset++] = (block >>> 6) & 3; values[valuesOffset++] = (block >>> 4) & 3; values[valuesOffset++] = (block >>> 2) & 3; values[valuesOffset++] = block & 3; } } - public override void Decode(long[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) { for (int i = 0; i < iterations; ++i) @@ -69,7 +75,7 @@ public override void Decode(byte[] blocks, int blocksOffset, long[] values, int { for (int j = 0; j < iterations; ++j) { - var block = blocks[blocksOffset++]; + byte block = blocks[blocksOffset++]; values[valuesOffset++] = (block >>> 6) & 3; values[valuesOffset++] = (block >>> 4) & 3; values[valuesOffset++] = (block >>> 2) & 3; diff --git a/src/Lucene.Net/Util/Packed/BulkOperationPacked20.cs b/src/Lucene.Net/Util/Packed/BulkOperationPacked20.cs index 859e2a1369..5fe170a444 100644 --- a/src/Lucene.Net/Util/Packed/BulkOperationPacked20.cs +++ b/src/Lucene.Net/Util/Packed/BulkOperationPacked20.cs @@ -1,4 +1,11 @@ -// this file has been automatically generated, DO NOT EDIT +//------------------------------------------------------------------------------ +// +// This code was generated by the gen_BulkOperation.py script. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ namespace Lucene.Net.Util.Packed { @@ -70,7 +77,6 @@ public override void Decode(byte[] blocks, int blocksOffset, int[] values, int v values[valuesOffset++] = ((byte2 & 15) << 16) | (byte3 << 8) | byte4; } } - public override void Decode(long[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) { for (int i = 0; i < iterations; ++i) diff --git a/src/Lucene.Net/Util/Packed/BulkOperationPacked21.cs b/src/Lucene.Net/Util/Packed/BulkOperationPacked21.cs index b215928dcc..0aab769ed0 100644 --- a/src/Lucene.Net/Util/Packed/BulkOperationPacked21.cs +++ b/src/Lucene.Net/Util/Packed/BulkOperationPacked21.cs @@ -1,4 +1,11 @@ -// this file has been automatically generated, DO NOT EDIT +//------------------------------------------------------------------------------ +// +// This code was generated by the gen_BulkOperation.py script. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ namespace Lucene.Net.Util.Packed { @@ -156,7 +163,6 @@ public override void Decode(byte[] blocks, int blocksOffset, int[] values, int v values[valuesOffset++] = ((byte18 & 31) << 16) | (byte19 << 8) | byte20; } } - public override void Decode(long[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) { for (int i = 0; i < iterations; ++i) diff --git a/src/Lucene.Net/Util/Packed/BulkOperationPacked22.cs b/src/Lucene.Net/Util/Packed/BulkOperationPacked22.cs index 162538fd33..df4e1ad6e0 100644 --- a/src/Lucene.Net/Util/Packed/BulkOperationPacked22.cs +++ b/src/Lucene.Net/Util/Packed/BulkOperationPacked22.cs @@ -1,4 +1,11 @@ -// this file has been automatically generated, DO NOT EDIT +//------------------------------------------------------------------------------ +// +// This code was generated by the gen_BulkOperation.py script. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ namespace Lucene.Net.Util.Packed { @@ -100,7 +107,6 @@ public override void Decode(byte[] blocks, int blocksOffset, int[] values, int v values[valuesOffset++] = ((byte8 & 63) << 16) | (byte9 << 8) | byte10; } } - public override void Decode(long[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) { for (int i = 0; i < iterations; ++i) diff --git a/src/Lucene.Net/Util/Packed/BulkOperationPacked23.cs b/src/Lucene.Net/Util/Packed/BulkOperationPacked23.cs index 92e7ef0160..81db68e729 100644 --- a/src/Lucene.Net/Util/Packed/BulkOperationPacked23.cs +++ b/src/Lucene.Net/Util/Packed/BulkOperationPacked23.cs @@ -1,4 +1,11 @@ -// this file has been automatically generated, DO NOT EDIT +//------------------------------------------------------------------------------ +// +// This code was generated by the gen_BulkOperation.py script. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ namespace Lucene.Net.Util.Packed { @@ -160,7 +167,6 @@ public override void Decode(byte[] blocks, int blocksOffset, int[] values, int v values[valuesOffset++] = ((byte20 & 127) << 16) | (byte21 << 8) | byte22; } } - public override void Decode(long[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) { for (int i = 0; i < iterations; ++i) diff --git a/src/Lucene.Net/Util/Packed/BulkOperationPacked24.cs b/src/Lucene.Net/Util/Packed/BulkOperationPacked24.cs index ce1a6833de..50b47b7104 100644 --- a/src/Lucene.Net/Util/Packed/BulkOperationPacked24.cs +++ b/src/Lucene.Net/Util/Packed/BulkOperationPacked24.cs @@ -1,4 +1,11 @@ -// this file has been automatically generated, DO NOT EDIT +//------------------------------------------------------------------------------ +// +// This code was generated by the gen_BulkOperation.py script. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ namespace Lucene.Net.Util.Packed { @@ -57,7 +64,6 @@ public override void Decode(byte[] blocks, int blocksOffset, int[] values, int v values[valuesOffset++] = (byte0 << 16) | (byte1 << 8) | byte2; } } - public override void Decode(long[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) { for (int i = 0; i < iterations; ++i) diff --git a/src/Lucene.Net/Util/Packed/BulkOperationPacked3.cs b/src/Lucene.Net/Util/Packed/BulkOperationPacked3.cs index c6bd250457..7fb95b6d45 100644 --- a/src/Lucene.Net/Util/Packed/BulkOperationPacked3.cs +++ b/src/Lucene.Net/Util/Packed/BulkOperationPacked3.cs @@ -1,4 +1,11 @@ -// this file has been automatically generated, DO NOT EDIT +//------------------------------------------------------------------------------ +// +// This code was generated by the gen_BulkOperation.py script. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ namespace Lucene.Net.Util.Packed { @@ -120,7 +127,6 @@ public override void Decode(byte[] blocks, int blocksOffset, int[] values, int v values[valuesOffset++] = byte2 & 7; } } - public override void Decode(long[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) { for (int i = 0; i < iterations; ++i) diff --git a/src/Lucene.Net/Util/Packed/BulkOperationPacked4.cs b/src/Lucene.Net/Util/Packed/BulkOperationPacked4.cs index 8784bbe51f..f1f4b17514 100644 --- a/src/Lucene.Net/Util/Packed/BulkOperationPacked4.cs +++ b/src/Lucene.Net/Util/Packed/BulkOperationPacked4.cs @@ -1,4 +1,11 @@ -// this file has been automatically generated, DO NOT EDIT +//------------------------------------------------------------------------------ +// +// This code was generated by the gen_BulkOperation.py script. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ namespace Lucene.Net.Util.Packed { @@ -45,12 +52,11 @@ public override void Decode(byte[] blocks, int blocksOffset, int[] values, int v { for (int j = 0; j < iterations; ++j) { - var block = blocks[blocksOffset++]; + byte block = blocks[blocksOffset++]; values[valuesOffset++] = (block >>> 4) & 15; values[valuesOffset++] = block & 15; } } - public override void Decode(long[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) { for (int i = 0; i < iterations; ++i) @@ -67,7 +73,7 @@ public override void Decode(byte[] blocks, int blocksOffset, long[] values, int { for (int j = 0; j < iterations; ++j) { - var block = blocks[blocksOffset++]; + byte block = blocks[blocksOffset++]; values[valuesOffset++] = (block >>> 4) & 15; values[valuesOffset++] = block & 15; } diff --git a/src/Lucene.Net/Util/Packed/BulkOperationPacked5.cs b/src/Lucene.Net/Util/Packed/BulkOperationPacked5.cs index 1abf6f71a6..62354c501b 100644 --- a/src/Lucene.Net/Util/Packed/BulkOperationPacked5.cs +++ b/src/Lucene.Net/Util/Packed/BulkOperationPacked5.cs @@ -1,4 +1,11 @@ -// this file has been automatically generated, DO NOT EDIT +//------------------------------------------------------------------------------ +// +// This code was generated by the gen_BulkOperation.py script. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ namespace Lucene.Net.Util.Packed { @@ -124,7 +131,6 @@ public override void Decode(byte[] blocks, int blocksOffset, int[] values, int v values[valuesOffset++] = byte4 & 31; } } - public override void Decode(long[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) { for (int i = 0; i < iterations; ++i) diff --git a/src/Lucene.Net/Util/Packed/BulkOperationPacked6.cs b/src/Lucene.Net/Util/Packed/BulkOperationPacked6.cs index dff7f5cb8d..3a65ff1603 100644 --- a/src/Lucene.Net/Util/Packed/BulkOperationPacked6.cs +++ b/src/Lucene.Net/Util/Packed/BulkOperationPacked6.cs @@ -1,4 +1,11 @@ -// this file has been automatically generated, DO NOT EDIT +//------------------------------------------------------------------------------ +// +// This code was generated by the gen_BulkOperation.py script. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ namespace Lucene.Net.Util.Packed { @@ -84,7 +91,6 @@ public override void Decode(byte[] blocks, int blocksOffset, int[] values, int v values[valuesOffset++] = byte2 & 63; } } - public override void Decode(long[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) { for (int i = 0; i < iterations; ++i) diff --git a/src/Lucene.Net/Util/Packed/BulkOperationPacked7.cs b/src/Lucene.Net/Util/Packed/BulkOperationPacked7.cs index 764c56bdf1..4755a9c5b4 100644 --- a/src/Lucene.Net/Util/Packed/BulkOperationPacked7.cs +++ b/src/Lucene.Net/Util/Packed/BulkOperationPacked7.cs @@ -1,4 +1,11 @@ -// this file has been automatically generated, DO NOT EDIT +//------------------------------------------------------------------------------ +// +// This code was generated by the gen_BulkOperation.py script. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ namespace Lucene.Net.Util.Packed { @@ -128,7 +135,6 @@ public override void Decode(byte[] blocks, int blocksOffset, int[] values, int v values[valuesOffset++] = byte6 & 127; } } - public override void Decode(long[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) { for (int i = 0; i < iterations; ++i) diff --git a/src/Lucene.Net/Util/Packed/BulkOperationPacked8.cs b/src/Lucene.Net/Util/Packed/BulkOperationPacked8.cs index 3a7313c4d5..ebc4b5716a 100644 --- a/src/Lucene.Net/Util/Packed/BulkOperationPacked8.cs +++ b/src/Lucene.Net/Util/Packed/BulkOperationPacked8.cs @@ -1,4 +1,11 @@ -// this file has been automatically generated, DO NOT EDIT +//------------------------------------------------------------------------------ +// +// This code was generated by the gen_BulkOperation.py script. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ namespace Lucene.Net.Util.Packed { @@ -48,7 +55,6 @@ public override void Decode(byte[] blocks, int blocksOffset, int[] values, int v values[valuesOffset++] = blocks[blocksOffset++] & 0xFF; } } - public override void Decode(long[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) { for (int i = 0; i < iterations; ++i) diff --git a/src/Lucene.Net/Util/Packed/BulkOperationPacked9.cs b/src/Lucene.Net/Util/Packed/BulkOperationPacked9.cs index ae43de9ef4..e220590030 100644 --- a/src/Lucene.Net/Util/Packed/BulkOperationPacked9.cs +++ b/src/Lucene.Net/Util/Packed/BulkOperationPacked9.cs @@ -1,4 +1,11 @@ -// this file has been automatically generated, DO NOT EDIT +//------------------------------------------------------------------------------ +// +// This code was generated by the gen_BulkOperation.py script. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ namespace Lucene.Net.Util.Packed { @@ -132,13 +139,12 @@ public override void Decode(byte[] blocks, int blocksOffset, int[] values, int v values[valuesOffset++] = ((byte7 & 1) << 8) | byte8; } } - public override void Decode(long[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) { for (int i = 0; i < iterations; ++i) { long block0 = blocks[blocksOffset++]; - values[valuesOffset++] = (long)((ulong)block0 >> 55); + values[valuesOffset++] = block0 >>> 55; values[valuesOffset++] = (block0 >>> 46) & 511L; values[valuesOffset++] = (block0 >>> 37) & 511L; values[valuesOffset++] = (block0 >>> 28) & 511L; diff --git a/src/Lucene.Net/Util/Packed/Direct16.cs b/src/Lucene.Net/Util/Packed/Direct16.cs index 2418337894..ffd2616324 100644 --- a/src/Lucene.Net/Util/Packed/Direct16.cs +++ b/src/Lucene.Net/Util/Packed/Direct16.cs @@ -2,11 +2,18 @@ using Lucene.Net.Support; using System; using System.Runtime.CompilerServices; - -// this file has been automatically generated, DO NOT EDIT +//------------------------------------------------------------------------------ +// +// This code was generated by the gen_Direct.py script. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ namespace Lucene.Net.Util.Packed { + /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -33,7 +40,7 @@ namespace Lucene.Net.Util.Packed /// internal sealed class Direct16 : PackedInt32s.MutableImpl { - internal readonly short[] values; + private readonly short[] values; internal Direct16(int valueCount) : base(valueCount, 16) @@ -71,8 +78,8 @@ public override void Set(int index, long value) public override long RamBytesUsed() { return RamUsageEstimator.AlignObjectSize( - RamUsageEstimator.NUM_BYTES_OBJECT_HEADER - + 2 * RamUsageEstimator.NUM_BYTES_INT32 // valueCount,bitsPerValue + RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + + 2 * RamUsageEstimator.NUM_BYTES_INT32 // valueCount,bitsPerValue + RamUsageEstimator.NUM_BYTES_OBJECT_REF) // values ref + RamUsageEstimator.SizeOf(values); } @@ -132,4 +139,4 @@ public override void Fill(int fromIndex, int toIndex, long val) Arrays.Fill(values, fromIndex, toIndex, (short)val); } } -} \ No newline at end of file +} diff --git a/src/Lucene.Net/Util/Packed/Direct32.cs b/src/Lucene.Net/Util/Packed/Direct32.cs index 9dd7dc6670..d853bb071b 100644 --- a/src/Lucene.Net/Util/Packed/Direct32.cs +++ b/src/Lucene.Net/Util/Packed/Direct32.cs @@ -2,11 +2,18 @@ using Lucene.Net.Support; using System; using System.Runtime.CompilerServices; - -// this file has been automatically generated, DO NOT EDIT +//------------------------------------------------------------------------------ +// +// This code was generated by the gen_Direct.py script. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ namespace Lucene.Net.Util.Packed { + /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -33,7 +40,7 @@ namespace Lucene.Net.Util.Packed /// internal sealed class Direct32 : PackedInt32s.MutableImpl { - internal readonly int[] values; + private readonly int[] values; internal Direct32(int valueCount) : base(valueCount, 32) @@ -71,9 +78,9 @@ public override void Set(int index, long value) public override long RamBytesUsed() { return RamUsageEstimator.AlignObjectSize( - RamUsageEstimator.NUM_BYTES_OBJECT_HEADER - + 2 * RamUsageEstimator.NUM_BYTES_INT32 // valueCount,bitsPerValue - + RamUsageEstimator.NUM_BYTES_OBJECT_REF) // values ref + RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + + 2 * RamUsageEstimator.NUM_BYTES_INT32 // valueCount,bitsPerValue + + RamUsageEstimator.NUM_BYTES_OBJECT_REF) // values ref + RamUsageEstimator.SizeOf(values); } @@ -132,4 +139,4 @@ public override void Fill(int fromIndex, int toIndex, long val) Arrays.Fill(values, fromIndex, toIndex, (int)val); } } -} \ No newline at end of file +} diff --git a/src/Lucene.Net/Util/Packed/Direct64.cs b/src/Lucene.Net/Util/Packed/Direct64.cs index 21ab51a814..797b924781 100644 --- a/src/Lucene.Net/Util/Packed/Direct64.cs +++ b/src/Lucene.Net/Util/Packed/Direct64.cs @@ -1,12 +1,19 @@ -using Lucene.Net.Diagnostics; +using Lucene.Net.Diagnostics; using Lucene.Net.Support; using System; using System.Runtime.CompilerServices; - -// this file has been automatically generated, DO NOT EDIT +//------------------------------------------------------------------------------ +// +// This code was generated by the gen_Direct.py script. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ namespace Lucene.Net.Util.Packed { + /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -33,7 +40,7 @@ namespace Lucene.Net.Util.Packed /// internal sealed class Direct64 : PackedInt32s.MutableImpl { - internal readonly long[] values; + private readonly long[] values; internal Direct64(int valueCount) : base(valueCount, 64) @@ -65,10 +72,10 @@ public override void Set(int index, long value) public override long RamBytesUsed() { return RamUsageEstimator.AlignObjectSize( - RamUsageEstimator.NUM_BYTES_OBJECT_HEADER - + 2 * RamUsageEstimator.NUM_BYTES_INT32 // valueCount,bitsPerValue - + RamUsageEstimator.NUM_BYTES_OBJECT_REF) // values ref - + RamUsageEstimator.SizeOf(values); + RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + + 2 * RamUsageEstimator.NUM_BYTES_INT32 // valueCount,bitsPerValue + + RamUsageEstimator.NUM_BYTES_OBJECT_REF) // values ref + + RamUsageEstimator.SizeOf(values); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -119,4 +126,4 @@ public override void Fill(int fromIndex, int toIndex, long val) Arrays.Fill(values, fromIndex, toIndex, val); } } -} \ No newline at end of file +} diff --git a/src/Lucene.Net/Util/Packed/Direct8.cs b/src/Lucene.Net/Util/Packed/Direct8.cs index 220bae7e3e..23c75abe87 100644 --- a/src/Lucene.Net/Util/Packed/Direct8.cs +++ b/src/Lucene.Net/Util/Packed/Direct8.cs @@ -2,11 +2,18 @@ using Lucene.Net.Support; using System; using System.Runtime.CompilerServices; - -// this file has been automatically generated, DO NOT EDIT +//------------------------------------------------------------------------------ +// +// This code was generated by the gen_Direct.py script. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ namespace Lucene.Net.Util.Packed { + /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -68,10 +75,10 @@ public override void Set(int index, long value) public override long RamBytesUsed() { return RamUsageEstimator.AlignObjectSize( - RamUsageEstimator.NUM_BYTES_OBJECT_HEADER - + 2 * RamUsageEstimator.NUM_BYTES_INT32 // valueCount,bitsPerValue - + RamUsageEstimator.NUM_BYTES_OBJECT_REF) // values ref - + RamUsageEstimator.SizeOf(values); + RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + + 2 * RamUsageEstimator.NUM_BYTES_INT32 // valueCount,bitsPerValue + + RamUsageEstimator.NUM_BYTES_OBJECT_REF) // values ref + + RamUsageEstimator.SizeOf(values); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -129,4 +136,4 @@ public override void Fill(int fromIndex, int toIndex, long val) Arrays.Fill(values, fromIndex, toIndex, (byte)val); } } -} \ No newline at end of file +} diff --git a/src/Lucene.Net/Util/Packed/Packed16ThreeBlocks.cs b/src/Lucene.Net/Util/Packed/Packed16ThreeBlocks.cs index 5ead66c9b9..b4602f8cbe 100644 --- a/src/Lucene.Net/Util/Packed/Packed16ThreeBlocks.cs +++ b/src/Lucene.Net/Util/Packed/Packed16ThreeBlocks.cs @@ -1,9 +1,16 @@ -using Lucene.Net.Diagnostics; +using Lucene.Net.Diagnostics; using Lucene.Net.Support; using System; using System.Runtime.CompilerServices; -// this file has been automatically generated, DO NOT EDIT +//------------------------------------------------------------------------------ +// +// This code was generated by the gen_PackedThreeBlocks.py script. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ namespace Lucene.Net.Util.Packed { @@ -33,7 +40,7 @@ namespace Lucene.Net.Util.Packed /// internal sealed class Packed16ThreeBlocks : PackedInt32s.MutableImpl { - internal readonly short[] blocks; + private readonly short[] blocks; public const int MAX_SIZE = int.MaxValue / 3; @@ -42,7 +49,7 @@ internal Packed16ThreeBlocks(int valueCount) { if (valueCount > MAX_SIZE) { - throw new IndexOutOfRangeException("MAX_SIZE exceeded"); + throw new ArgumentOutOfRangeException(nameof(valueCount), "MAX_SIZE exceeded"); } blocks = new short[valueCount * 3]; } @@ -137,7 +144,7 @@ public override long RamBytesUsed() { return RamUsageEstimator.AlignObjectSize( RamUsageEstimator.NUM_BYTES_OBJECT_HEADER - + 2 * RamUsageEstimator.NUM_BYTES_INT32 // valueCount,bitsPerValue + + 2 * RamUsageEstimator.NUM_BYTES_INT32 // valueCount,bitsPerValue + RamUsageEstimator.NUM_BYTES_OBJECT_REF) // blocks ref + RamUsageEstimator.SizeOf(blocks); } diff --git a/src/Lucene.Net/Util/Packed/Packed64SingleBlock.cs b/src/Lucene.Net/Util/Packed/Packed64SingleBlock.cs index 7d29504a40..19312dd173 100644 --- a/src/Lucene.Net/Util/Packed/Packed64SingleBlock.cs +++ b/src/Lucene.Net/Util/Packed/Packed64SingleBlock.cs @@ -1,27 +1,35 @@ -using Lucene.Net.Diagnostics; +using Lucene.Net.Diagnostics; using Lucene.Net.Support; using System; using System.Runtime.CompilerServices; -// this file has been automatically generated, DO NOT EDIT +//------------------------------------------------------------------------------ +// +// This code was generated by the gen_Packed64SingleBlock.py script. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ namespace Lucene.Net.Util.Packed { + /* * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. */ using DataInput = Lucene.Net.Store.DataInput; @@ -68,7 +76,7 @@ public override long RamBytesUsed() { return RamUsageEstimator.AlignObjectSize( RamUsageEstimator.NUM_BYTES_OBJECT_HEADER - + 2 * RamUsageEstimator.NUM_BYTES_INT32 // valueCount,bitsPerValue + + 2 * RamUsageEstimator.NUM_BYTES_INT32 // valueCount,bitsPerValue + RamUsageEstimator.NUM_BYTES_OBJECT_REF) // blocks ref + RamUsageEstimator.SizeOf(blocks); } @@ -113,8 +121,7 @@ public override int Get(int index, long[] arr, int off, int len) int nblocks = (index + len) / valuesPerBlock - blockIndex; decoder.Decode(blocks, blockIndex, arr, off, nblocks); int diff = nblocks * valuesPerBlock; - index += diff; - len -= diff; + index += diff; len -= diff; if (index > originalIndex) { @@ -167,8 +174,7 @@ public override int Set(int index, long[] arr, int off, int len) int nblocks = (index + len) / valuesPerBlock - blockIndex; op.Encode(arr, off, blocks, blockIndex, nblocks); int diff = nblocks * valuesPerBlock; - index += diff; - len -= diff; + index += diff; len -= diff; if (index > originalIndex) { @@ -255,46 +261,32 @@ public static Packed64SingleBlock Create(int valueCount, int bitsPerValue) { case 1: return new Packed64SingleBlock1(valueCount); - case 2: return new Packed64SingleBlock2(valueCount); - case 3: return new Packed64SingleBlock3(valueCount); - case 4: return new Packed64SingleBlock4(valueCount); - case 5: return new Packed64SingleBlock5(valueCount); - case 6: return new Packed64SingleBlock6(valueCount); - case 7: return new Packed64SingleBlock7(valueCount); - case 8: return new Packed64SingleBlock8(valueCount); - case 9: return new Packed64SingleBlock9(valueCount); - case 10: return new Packed64SingleBlock10(valueCount); - case 12: return new Packed64SingleBlock12(valueCount); - case 16: return new Packed64SingleBlock16(valueCount); - case 21: return new Packed64SingleBlock21(valueCount); - case 32: return new Packed64SingleBlock32(valueCount); - default: throw new ArgumentException("Unsupported number of bits per value: " + 32); } @@ -635,5 +627,6 @@ public override void Set(int index, long value) blocks[o] = (blocks[o] & ~(4294967295L << shift)) | (value << shift); } } + } } diff --git a/src/Lucene.Net/Util/Packed/Packed8ThreeBlocks.cs b/src/Lucene.Net/Util/Packed/Packed8ThreeBlocks.cs index 4c635f8ee9..6bcea7ccb6 100644 --- a/src/Lucene.Net/Util/Packed/Packed8ThreeBlocks.cs +++ b/src/Lucene.Net/Util/Packed/Packed8ThreeBlocks.cs @@ -1,9 +1,16 @@ -using Lucene.Net.Diagnostics; +using Lucene.Net.Diagnostics; using Lucene.Net.Support; using System; using System.Runtime.CompilerServices; -// this file has been automatically generated, DO NOT EDIT +//------------------------------------------------------------------------------ +// +// This code was generated by the gen_PackedThreeBlocks.py script. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ namespace Lucene.Net.Util.Packed { @@ -42,7 +49,7 @@ internal Packed8ThreeBlocks(int valueCount) { if (valueCount > MAX_SIZE) { - throw new IndexOutOfRangeException("MAX_SIZE exceeded"); + throw new ArgumentOutOfRangeException(nameof(valueCount), "MAX_SIZE exceeded"); } blocks = new byte[valueCount * 3]; } @@ -52,7 +59,7 @@ internal Packed8ThreeBlocks(int packedIntsVersion, DataInput @in, int valueCount { @in.ReadBytes(blocks, 0, 3 * valueCount); // because packed ints have not always been byte-aligned - var remaining = (int)(PackedInt32s.Format.PACKED.ByteCount(packedIntsVersion, valueCount, 24) - 3L * valueCount * 1); + int remaining = (int)(PackedInt32s.Format.PACKED.ByteCount(packedIntsVersion, valueCount, 24) - 3L * valueCount * 1); for (int i = 0; i < remaining; ++i) { @in.ReadByte(); @@ -113,9 +120,9 @@ public override int Set(int index, long[] arr, int off, int len) public override void Fill(int fromIndex, int toIndex, long val) { - var block1 = (byte)(val >>> 16); - var block2 = (byte)(val >>> 8); - var block3 = (byte)val; + byte block1 = (byte)(val >>> 16); + byte block2 = (byte)(val >>> 8); + byte block3 = (byte)val; for (int i = fromIndex * 3, end = toIndex * 3; i < end; i += 3) { blocks[i] = block1; @@ -134,7 +141,7 @@ public override long RamBytesUsed() { return RamUsageEstimator.AlignObjectSize( RamUsageEstimator.NUM_BYTES_OBJECT_HEADER - + 2 * RamUsageEstimator.NUM_BYTES_INT32 // valueCount,bitsPerValue + + 2 * RamUsageEstimator.NUM_BYTES_INT32 // valueCount,bitsPerValue + RamUsageEstimator.NUM_BYTES_OBJECT_REF) // blocks ref + RamUsageEstimator.SizeOf(blocks); } diff --git a/src/Lucene.Net/Util/Packed/README.md b/src/Lucene.Net/Util/Packed/README.md new file mode 100644 index 0000000000..ada4404f0d --- /dev/null +++ b/src/Lucene.Net/Util/Packed/README.md @@ -0,0 +1,28 @@ +# Lucene.Net.Util.Packed + +## Generating the files + +There are four Python scripts in this folder: +[`gen_BulkOperation.py`](gen_BulkOperation.py), +[`gen_Direct.py`](gen_Direct.py), +[`gen_Packed64SingleBlock.py`](gen_Packed64SingleBlock.py), and +[`gen_PackedThreeBlocks.py`](gen_PackedThreeBlocks.py). +These scripts were ported from the original Lucene 4.8.1 code, and modified to generate the corresponding C# code instead of Java. + +To generate the files, run the following commands from this directory: + +```sh +python3 ./gen_BulkOperation.py +python3 ./gen_Direct.py +python3 ./gen_Packed64SingleBlock.py +python3 ./gen_PackedThreeBlocks.py +``` + +These scripts do not require any virtual environment or additional packages to run. +All have been tested with Python 3 and updated as necessary from the original Python 2 code. +As Python 2 is now out of support, they have not been tested against Python 2. + +As noted in the comment on the generated files, please do not modify the generated files directly, now that we have these scripts. +Instead, modify the scripts as needed and re-generate the files. + +A good test to ensure the generated files are in sync with the scripts is to run the scripts, and then see if there are any pending Git changes in the repository. diff --git a/src/Lucene.Net/Util/Packed/gen_BulkOperation.py b/src/Lucene.Net/Util/Packed/gen_BulkOperation.py new file mode 100644 index 0000000000..2390e61346 --- /dev/null +++ b/src/Lucene.Net/Util/Packed/gen_BulkOperation.py @@ -0,0 +1,421 @@ +#! /usr/bin/env python + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ## LUCENENET PORTING NOTES +# This script was originally written for Python 2, but has been modified to work with Python 3. +# It is noted below where changes were made to support Python 3. +# Some of the changes have been backported from newer versions of Lucene, +# so when upgrading this script in the future, feel free to remove those comments. + +# LUCENENET backport - Python 3 support (lucene 945b1cb) +try: + # python 3.9+ + from math import gcd +except ImportError: + # old python + from fractions import gcd + +"""Code generation for bulk operations""" + +MAX_SPECIALIZED_BITS_PER_VALUE = 24 +PACKED_64_SINGLE_BLOCK_BPV = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 16, 21, 32] +OUTPUT_FILE = "BulkOperation.cs" + +# LUCENENET specific - we write the usings before the header, since +# the numbered files don't need them +HEADER = """//------------------------------------------------------------------------------ +// +// This code was generated by the gen_BulkOperation.py script. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ + +namespace Lucene.Net.Util.Packed +{ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +""" + +FOOTER=""" + /// + /// NOTE: This was writeLong() in Lucene. + /// + protected virtual int WriteInt64(long block, byte[] blocks, int blocksOffset) + { + for (int j = 1; j <= 8; ++j) + { + blocks[blocksOffset++] = (byte)(block >>> (64 - (j << 3))); + } + return blocksOffset; + } + + /// + /// For every number of bits per value, there is a minimum number of + /// blocks (b) / values (v) you need to write in order to reach the next block + /// boundary: + /// - 16 bits per value -> b=2, v=1 + /// - 24 bits per value -> b=3, v=1 + /// - 50 bits per value -> b=25, v=4 + /// - 63 bits per value -> b=63, v=8 + /// - ... + /// + /// A bulk read consists in copying iterations*v values that are + /// contained in iterations*b blocks into a long[] + /// (higher values of iterations are likely to yield a better + /// throughput) => this requires n * (b + 8v) bytes of memory. + /// + /// This method computes iterations as + /// ramBudget / (b + 8v) (since a long is 8 bytes). + /// + public int ComputeIterations(int valueCount, int ramBudget) + { + int iterations = ramBudget / (ByteBlockCount + 8 * ByteValueCount); + if (iterations == 0) + { + // at least 1 + return 1; + } + else if ((iterations - 1) * ByteValueCount >= valueCount) + { + // don't allocate for more than the size of the reader + return (int)Math.Ceiling((double)valueCount / ByteValueCount); + } + else + { + return iterations; + } + } + } +} +""" + +def is_power_of_two(n): + return n & (n - 1) == 0 + +def casts(typ): + cast_start = "(%s)(" %typ # LUCENENET specific - removed space to match existing C# port style + cast_end = ")" + if typ == "long": + cast_start = "" + cast_end = "" + return cast_start, cast_end + +def hexNoLSuffix(n): + # On 32 bit Python values > (1 << 31)-1 will have L appended by hex function: + s = hex(n) + if s.endswith('L'): + s = s[:-1] + return s + +def masks(bits): + if bits == 64: + return "", "" + return "(", " & %sL)" %(hexNoLSuffix((1 << bits) - 1)) + +def get_type(bits): + if bits == 8: + return "byte" + elif bits == 16: + return "short" + elif bits == 32: + return "int" + elif bits == 64: + return "long" + else: + assert False + +def block_value_count(bpv, bits=64): + blocks = bpv + values = blocks * bits // bpv # LUCENENET backport - Python 3 support (lucene 9302eee) + while blocks % 2 == 0 and values % 2 == 0: + blocks //= 2 # LUCENENET backport - Python 3 support (lucene 9302eee) + values //= 2 # LUCENENET backport - Python 3 support (lucene 9302eee) + assert values * bpv == bits * blocks, "%d values, %d blocks, %d bits per value" %(values, blocks, bpv) + return (blocks, values) + +def packed64(bpv, f): + mask = (1 << bpv) - 1 + + #f.write("\n") // LUCENENET: Not needed + f.write(" public BulkOperationPacked%d()\n" %bpv) + f.write(" : base(%d)\n" %bpv) + f.write(" {\n") + f.write(" }\n\n") + + if bpv == 64: + # LUCENENET NOTE - these don't seem to be used? + f.write(""" public override void Decode(long[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) + { + Array.Copy(blocks, blocksOffset, values, valuesOffset, valueCount() * iterations); + } + + public override void Decode(long[] blocks, int blocksOffset, int[] values, int valuesOffset, int iterations) + { + throw new NotSupportedException(); + } + + public override void Decode(byte[] blocks, int blocksOffset, int[] values, int valuesOffset, int iterations) + { + throw new NotSupportedException(); + } + + public override void Decode(byte[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) + { + LongBuffer.wrap(values, valuesOffset, iterations * valueCount()).put(ByteBuffer.wrap(blocks, blocksOffset, 8 * iterations * blockCount()).asLongBuffer()); + } +""") + else: + p64_decode(bpv, f, 32) + p64_decode(bpv, f, 64) + +def p64_decode(bpv, f, bits): + blocks, values = block_value_count(bpv) + typ = get_type(bits) + cast_start, cast_end = casts(typ) + + f.write(" public override void Decode(long[] blocks, int blocksOffset, %s[] values, int valuesOffset, int iterations)\n" %typ) + f.write(" {\n") + if bits < bpv: + f.write(" throw new NotSupportedException();\n") + else: + f.write(" for (int i = 0; i < iterations; ++i)\n") + f.write(" {\n") + mask = (1 << bpv) - 1 + + if is_power_of_two(bpv): + f.write(" long block = blocks[blocksOffset++];\n") + f.write(" for (int shift = %d; shift >= 0; shift -= %d)\n" %(64 - bpv, bpv)) + f.write(" {\n") + f.write(" values[valuesOffset++] = %s(block >>> shift) & %d%s;\n" %(cast_start, mask, cast_end)) + f.write(" }\n") + else: + for i in range(0, values): # LUCENENET backport - Python 3 support (lucene 9302eee) + block_offset = i * bpv // 64 # LUCENENET backport - Python 3 support (lucene 9302eee) + bit_offset = (i * bpv) % 64 + if bit_offset == 0: + # start of block + f.write(" long block%d = blocks[blocksOffset++];\n" %block_offset) + f.write(" values[valuesOffset++] = %sblock%d >>> %d%s;\n" %(cast_start, block_offset, 64 - bpv, cast_end)) + elif bit_offset + bpv == 64: + # end of block + f.write(" values[valuesOffset++] = %sblock%d & %dL%s;\n" %(cast_start, block_offset, mask, cast_end)) + elif bit_offset + bpv < 64: + # middle of block + f.write(" values[valuesOffset++] = %s(block%d >>> %d) & %dL%s;\n" %(cast_start, block_offset, 64 - bit_offset - bpv, mask, cast_end)) + else: + # value spans across 2 blocks + mask1 = (1 << (64 - bit_offset)) -1 + shift1 = bit_offset + bpv - 64 + shift2 = 64 - shift1 + f.write(" long block%d = blocks[blocksOffset++];\n" %(block_offset + 1)) + f.write(" values[valuesOffset++] = %s((block%d & %dL) << %d) | (block%d >>> %d)%s;\n" %(cast_start, block_offset, mask1, shift1, block_offset + 1, shift2, cast_end)) + f.write(" }\n") + f.write(" }\n\n") + + byte_blocks, byte_values = block_value_count(bpv, 8) + + f.write(" public override void Decode(byte[] blocks, int blocksOffset, %s[] values, int valuesOffset, int iterations)\n" %typ) + f.write(" {\n") + if bits < bpv: + f.write(" throw new NotSupportedException();\n") + else: + if is_power_of_two(bpv) and bpv < 8: + f.write(" for (int j = 0; j < iterations; ++j)\n") + f.write(" {\n") + f.write(" byte block = blocks[blocksOffset++];\n") + for shift in range(8 - bpv, 0, -bpv): # LUCENENET backport - Python 3 support (lucene 9302eee) + f.write(" values[valuesOffset++] = (block >>> %d) & %d;\n" %(shift, mask)) + f.write(" values[valuesOffset++] = block & %d;\n" %mask) + f.write(" }\n") + elif bpv == 8: + f.write(" for (int j = 0; j < iterations; ++j)\n") + f.write(" {\n") + f.write(" values[valuesOffset++] = blocks[blocksOffset++] & 0xFF;\n") + f.write(" }\n") + elif is_power_of_two(bpv) and bpv > 8: + f.write(" for (int j = 0; j < iterations; ++j)\n") + f.write(" {\n") + m = bits <= 32 and "0xFF" or "0xFFL" + f.write(" values[valuesOffset++] =") + for i in range(bpv // 8 - 1): # LUCENENET backport - Python 3 support (lucene 9302eee) + f.write(" ((blocks[blocksOffset++] & %s) << %d) |" %(m, bpv - 8)) + f.write(" (blocks[blocksOffset++] & %s);\n" %m) + f.write(" }\n") + else: + f.write(" for (int i = 0; i < iterations; ++i)\n") + f.write(" {\n") + for i in range(0, byte_values): # LUCENENET backport - Python 3 support (lucene 9302eee) + byte_start = i * bpv // 8 # LUCENENET backport - Python 3 support (lucene 9302eee) + bit_start = (i * bpv) % 8 + byte_end = ((i + 1) * bpv - 1) // 8 # LUCENENET backport - Python 3 support (lucene 9302eee) + bit_end = ((i + 1) * bpv - 1) % 8 + shift = lambda b: 8 * (byte_end - b - 1) + 1 + bit_end + if bit_start == 0: + f.write(" %s byte%d = blocks[blocksOffset++] & 0xFF;\n" %(typ, byte_start)) + for b in range(byte_start + 1, byte_end + 1): # LUCENENET backport - Python 3 support (lucene 9302eee) + f.write(" %s byte%d = blocks[blocksOffset++] & 0xFF;\n" %(typ, b)) + f.write(" values[valuesOffset++] =") + if byte_start == byte_end: + if bit_start == 0: + if bit_end == 7: + f.write(" byte%d" %byte_start) + else: + f.write(" byte%d >>> %d" %(byte_start, 7 - bit_end)) + else: + if bit_end == 7: + f.write(" byte%d & %d" %(byte_start, 2 ** (8 - bit_start) - 1)) + else: + f.write(" (byte%d >>> %d) & %d" %(byte_start, 7 - bit_end, 2 ** (bit_end - bit_start + 1) - 1)) + else: + if bit_start == 0: + f.write(" (byte%d << %d)" %(byte_start, shift(byte_start))) + else: + f.write(" ((byte%d & %d) << %d)" %(byte_start, 2 ** (8 - bit_start) - 1, shift(byte_start))) + for b in range(byte_start + 1, byte_end): # LUCENENET backport - Python 3 support (lucene 9302eee) + f.write(" | (byte%d << %d)" %(b, shift(b))) + if bit_end == 7: + f.write(" | byte%d" %byte_end) + else: + f.write(" | (byte%d >>> %d)" %(byte_end, 7 - bit_end)) + f.write(";\n") + f.write(" }\n") + f.write(" }\n") # LUCENENET specific - removed extra newline + +if __name__ == '__main__': + f = open(OUTPUT_FILE, 'w') + + # LUCENENET specific - we write the usings before the header, see top of file + f.write('using Lucene.Net.Diagnostics;\n') + f.write('using System;\n\n') + + f.write(HEADER) + #f.write('\n') // LUCENENET: Not needed + f.write(''' /// + /// Efficient sequential read/write of packed integers. + /// \n''') + + f.write(' internal abstract class BulkOperation : PackedInt32s.IDecoder, PackedInt32s.IEncoder\n') + f.write(' {') + + # LUCENENET specific - write abstract methods + f.write(""" + public abstract void Encode(int[] values, int valuesOffset, byte[] blocks, int blocksOffset, int iterations); + + public abstract void Encode(int[] values, int valuesOffset, long[] blocks, int blocksOffset, int iterations); + + public abstract void Encode(long[] values, int valuesOffset, byte[] blocks, int blocksOffset, int iterations); + + public abstract void Encode(long[] values, int valuesOffset, long[] blocks, int blocksOffset, int iterations); + + public abstract void Decode(byte[] blocks, int blocksOffset, int[] values, int valuesOffset, int iterations); + + public abstract void Decode(long[] blocks, int blocksOffset, int[] values, int valuesOffset, int iterations); + + public abstract void Decode(byte[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations); + + public abstract void Decode(long[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations); + + public abstract int ByteValueCount { get; } + + public abstract int ByteBlockCount { get; } + + /// + /// NOTE: This was longValueCount() in Lucene. + /// + public abstract int Int64ValueCount { get; } + + /// + /// NOTE: This was longBlockCount() in Lucene. + /// + public abstract int Int64BlockCount { get; } + +""") + + f.write(' private static readonly BulkOperation[] packedBulkOps = new BulkOperation[] {\n') + + for bpv in range(1, 65): # LUCENENET backport - Python 3 support (lucene 9302eee) + if bpv > MAX_SPECIALIZED_BITS_PER_VALUE: + f.write(' new BulkOperationPacked(%d),\n' % bpv) + continue + f2 = open('BulkOperationPacked%d.cs' % bpv, 'w') + f2.write(HEADER) + if bpv == 64: + # LUCENENET NOTE - these don't seem to be used? + f2.write('//import java.nio.LongBuffer;\n') + f2.write('//import java.nio.ByteBuffer;\n') + f2.write('\n') + f2.write(''' /// + /// Efficient sequential read/write of packed integers. + /// \n''') + f2.write(' internal sealed class BulkOperationPacked%d : BulkOperationPacked\n' % bpv) + f2.write(' {\n') + packed64(bpv, f2) + f2.write(' }\n') + f2.write('}\n') + f2.close() + f.write(' new BulkOperationPacked%d(),\n' % bpv) + + f.write(' };\n') + f.write('\n') + + f.write(' // NOTE: this is sparse (some entries are null):\n') + f.write(' private static readonly BulkOperation[] packedSingleBlockBulkOps = new BulkOperation[] {\n') + for bpv in range(1, max(PACKED_64_SINGLE_BLOCK_BPV)+1): # LUCENENET backport - Python 3 support (lucene 9302eee) + if bpv in PACKED_64_SINGLE_BLOCK_BPV: + f.write(' new BulkOperationPackedSingleBlock(%d),\n' % bpv) + else: + f.write(' null,\n') + f.write(' };\n') + f.write('\n') + + f.write("\n") + f.write(" public static BulkOperation Of(PackedInt32s.Format format, int bitsPerValue)\n") + f.write(" {\n") + f.write(" if (format == PackedInt32s.Format.PACKED)\n") + f.write(" {\n") + f.write(" if (Debugging.AssertsEnabled) Debugging.Assert(packedBulkOps[bitsPerValue - 1] != null);\n") + f.write(" return packedBulkOps[bitsPerValue - 1];\n") + f.write(" }\n") + f.write(" else if (format == PackedInt32s.Format.PACKED_SINGLE_BLOCK)\n") + f.write(" {\n") + f.write(" if (Debugging.AssertsEnabled) Debugging.Assert(packedSingleBlockBulkOps[bitsPerValue - 1] != null);\n") + f.write(" return packedSingleBlockBulkOps[bitsPerValue - 1];\n") + f.write(" }\n") + f.write(" else\n") + f.write(" {\n") + f.write(" throw AssertionError.Create();\n") + f.write(" }\n") + f.write(" }\n") + f.write(FOOTER) + f.close() diff --git a/src/Lucene.Net/Util/Packed/gen_Direct.py b/src/Lucene.Net/Util/Packed/gen_Direct.py new file mode 100644 index 0000000000..af0264526c --- /dev/null +++ b/src/Lucene.Net/Util/Packed/gen_Direct.py @@ -0,0 +1,231 @@ +#! /usr/bin/env python + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ## LUCENENET PORTING NOTES +# This script was originally written for Python 2, but has been tested against Python 3. +# No changes were necessary to run this script in Python 3. + +# LUCENENET specific - we write the usings before the header +HEADER="""//------------------------------------------------------------------------------ +// +// This code was generated by the gen_Direct.py script. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ + +namespace Lucene.Net.Util.Packed +{ + + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + using DataInput = Lucene.Net.Store.DataInput; + +""" + +TYPES = {8: "byte", 16: "short", 32: "int", 64: "long"} +DOTNET_READ_TYPES = {8: "Byte", 16: "Int16", 32: "Int32", 64: "Int64"} # LUCENENET specific +MASKS = {8: " & 0xFFL", 16: " & 0xFFFFL", 32: " & 0xFFFFFFFFL", 64: ""} +CASTS = {8: "(byte)", 16: "(short)", 32: "(int)", 64: ""} # LUCENENET specific - removed space from casts to match existing C# port style + +if __name__ == '__main__': + for bpv in TYPES.keys(): + type + f = open("Direct%d.cs" %bpv, 'w') + + # LUCENENET specific - we write the usings before the header + f.write("using Lucene.Net.Diagnostics;\n") + f.write("using Lucene.Net.Support;\n") + f.write("using System;\n") + f.write("using System.Runtime.CompilerServices;\n") + + f.write(HEADER) + f.write(""" /// + /// Direct wrapping of %d-bits values to a backing array. + /// + /// @lucene.internal + /// \n""" %bpv) + f.write(" internal sealed class Direct%d : PackedInt32s.MutableImpl\n" %bpv) + f.write(" {\n") + f.write(" private readonly %s[] values;\n\n" %TYPES[bpv]) + + f.write(" internal Direct%d(int valueCount)\n" %bpv) + f.write(" : base(valueCount, %d)\n" %bpv) + f.write(" {\n") + f.write(" values = new %s[valueCount];\n" %TYPES[bpv]) + f.write(" }\n\n") + + # LUCENENET specific - remove unused parameter for Direct64 + if bpv == 64: + f.write(" internal Direct%d(/*int packedIntsVersion,*/ DataInput @in, int valueCount) // LUCENENET specific - removed unused parameter\n" %bpv) + else: + f.write(" internal Direct%d(int packedIntsVersion, DataInput @in, int valueCount)\n" %bpv) + f.write(" : this(valueCount)\n") + f.write(" {\n") + if bpv == 8: + f.write(" @in.ReadBytes(values, 0, valueCount);\n") + else: + f.write(" for (int i = 0; i < valueCount; ++i)\n") + f.write(" {\n") + f.write(" values[i] = @in.Read%s();\n" %DOTNET_READ_TYPES[bpv].title()) # LUCENENET specific + f.write(" }\n") + if bpv != 64: + f.write(" // because packed ints have not always been byte-aligned\n") + f.write(" int remaining = (int)(PackedInt32s.Format.PACKED.ByteCount(packedIntsVersion, valueCount, %d) - %dL * valueCount);\n" %(bpv, bpv / 8)) + f.write(" for (int i = 0; i < remaining; ++i)\n") + f.write(" {\n") + f.write(" @in.ReadByte();\n") + f.write(" }\n") + f.write(" }\n") + + f.write(""" + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public override long Get(int index) + { + return values[index]%s; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public override void Set(int index, long value) + { + values[index] = %s(value); + } + + public override long RamBytesUsed() + { + return RamUsageEstimator.AlignObjectSize( + RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + + 2 * RamUsageEstimator.NUM_BYTES_INT32 // valueCount,bitsPerValue + + RamUsageEstimator.NUM_BYTES_OBJECT_REF) // values ref + + RamUsageEstimator.SizeOf(values); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public override void Clear() + { + Arrays.Fill(values, %s0L); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public override object GetArray() + { + return values; + } + + public override bool HasArray => true; +""" %(MASKS[bpv], CASTS[bpv], CASTS[bpv])) + + if bpv == 64: + f.write(""" + public override int Get(int index, long[] arr, int off, int len) + { + if (Debugging.AssertsEnabled) + { + Debugging.Assert(len > 0, "len must be > 0 (got {0})", len); + Debugging.Assert(index >= 0 && index < m_valueCount); + Debugging.Assert(off + len <= arr.Length); + } + + int gets = Math.Min(m_valueCount - index, len); + Arrays.Copy(values, index, arr, off, gets); + return gets; + } + + public override int Set(int index, long[] arr, int off, int len) + { + if (Debugging.AssertsEnabled) + { + Debugging.Assert(len > 0, "len must be > 0 (got {0})", len); + Debugging.Assert(index >= 0 && index < m_valueCount); + Debugging.Assert(off + len <= arr.Length); + } + + int sets = Math.Min(m_valueCount - index, len); + Arrays.Copy(arr, off, values, index, sets); + return sets; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public override void Fill(int fromIndex, int toIndex, long val) + { + Arrays.Fill(values, fromIndex, toIndex, val); + } +""") + else: + f.write(""" + public override int Get(int index, long[] arr, int off, int len) + { + if (Debugging.AssertsEnabled) + { + Debugging.Assert(len > 0, "len must be > 0 (got {0})", len); + Debugging.Assert(index >= 0 && index < m_valueCount); + Debugging.Assert(off + len <= arr.Length); + } + + int gets = Math.Min(m_valueCount - index, len); + for (int i = index, o = off, end = index + gets; i < end; ++i, ++o) + { + arr[o] = values[i]%s; + } + return gets; + } + + public override int Set(int index, long[] arr, int off, int len) + { + if (Debugging.AssertsEnabled) + { + Debugging.Assert(len > 0, "len must be > 0 (got {0})", len); + Debugging.Assert(index >= 0 && index < m_valueCount); + Debugging.Assert(off + len <= arr.Length); + } + + int sets = Math.Min(m_valueCount - index, len); + for (int i = index, o = off, end = index + sets; i < end; ++i, ++o) + { + values[i] = %sarr[o]; + } + return sets; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public override void Fill(int fromIndex, int toIndex, long val) + { + if (Debugging.AssertsEnabled) Debugging.Assert(val == (val%s)); + Arrays.Fill(values, fromIndex, toIndex, %sval); + } +""" %(MASKS[bpv], CASTS[bpv], MASKS[bpv], CASTS[bpv])) + + f.write(" }\n") + f.write("}\n") + + f.close() diff --git a/src/Lucene.Net/Util/Packed/gen_Packed64SingleBlock.py b/src/Lucene.Net/Util/Packed/gen_Packed64SingleBlock.py new file mode 100644 index 0000000000..a2c40bc5b6 --- /dev/null +++ b/src/Lucene.Net/Util/Packed/gen_Packed64SingleBlock.py @@ -0,0 +1,345 @@ +#! /usr/bin/env python + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ## LUCENENET PORTING NOTES +# This script was originally written for Python 2, but has been tested against Python 3. +# No changes were necessary to run this script in Python 3. + +SUPPORTED_BITS_PER_VALUE = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 16, 21, 32] + +HEADER="""using Lucene.Net.Diagnostics; +using Lucene.Net.Support; +using System; +using System.Runtime.CompilerServices; + +//------------------------------------------------------------------------------ +// +// This code was generated by the gen_Packed64SingleBlock.py script. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ + +namespace Lucene.Net.Util.Packed +{ + + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + + using DataInput = Lucene.Net.Store.DataInput; + + /// + /// This class is similar to except that it trades space for + /// speed by ensuring that a single block needs to be read/written in order to + /// read/write a value. + /// + internal abstract class Packed64SingleBlock : PackedInt32s.MutableImpl + { + public const int MAX_SUPPORTED_BITS_PER_VALUE = %d; + private static readonly int[] SUPPORTED_BITS_PER_VALUE = new int[] { %s }; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool IsSupported(int bitsPerValue) + { + return Array.BinarySearch(SUPPORTED_BITS_PER_VALUE, bitsPerValue) >= 0; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int RequiredCapacity(int valueCount, int valuesPerBlock) + { + return valueCount / valuesPerBlock + (valueCount %% valuesPerBlock == 0 ? 0 : 1); + } + + internal readonly long[] blocks; + + private protected Packed64SingleBlock(int valueCount, int bitsPerValue) // LUCENENET: Changed from internal to private protected + : base(valueCount, bitsPerValue) + { + if (Debugging.AssertsEnabled) Debugging.Assert(IsSupported(bitsPerValue)); + int valuesPerBlock = 64 / bitsPerValue; + blocks = new long[RequiredCapacity(valueCount, valuesPerBlock)]; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public override void Clear() + { + Arrays.Fill(blocks, 0L); + } + + public override long RamBytesUsed() + { + return RamUsageEstimator.AlignObjectSize( + RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + + 2 * RamUsageEstimator.NUM_BYTES_INT32 // valueCount,bitsPerValue + + RamUsageEstimator.NUM_BYTES_OBJECT_REF) // blocks ref + + RamUsageEstimator.SizeOf(blocks); + } + + public override int Get(int index, long[] arr, int off, int len) + { + if (Debugging.AssertsEnabled) + { + Debugging.Assert(len > 0, "len must be > 0 (got {0})", len); + Debugging.Assert(index >= 0 && index < m_valueCount); + } + len = Math.Min(len, m_valueCount - index); + if (Debugging.AssertsEnabled) Debugging.Assert(off + len <= arr.Length); + + int originalIndex = index; + + // go to the next block boundary + int valuesPerBlock = 64 / m_bitsPerValue; + int offsetInBlock = index %% valuesPerBlock; + if (offsetInBlock != 0) + { + for (int i = offsetInBlock; i < valuesPerBlock && len > 0; ++i) + { + arr[off++] = Get(index++); + --len; + } + if (len == 0) + { + return index - originalIndex; + } + } + + // bulk get + if (Debugging.AssertsEnabled) Debugging.Assert(index %% valuesPerBlock == 0); + PackedInt32s.IDecoder decoder = BulkOperation.Of(PackedInt32s.Format.PACKED_SINGLE_BLOCK, m_bitsPerValue); + if (Debugging.AssertsEnabled) + { + Debugging.Assert(decoder.Int64BlockCount == 1); + Debugging.Assert(decoder.Int64ValueCount == valuesPerBlock); + } + int blockIndex = index / valuesPerBlock; + int nblocks = (index + len) / valuesPerBlock - blockIndex; + decoder.Decode(blocks, blockIndex, arr, off, nblocks); + int diff = nblocks * valuesPerBlock; + index += diff; len -= diff; + + if (index > originalIndex) + { + // stay at the block boundary + return index - originalIndex; + } + else + { + // no progress so far => already at a block boundary but no full block to + // get + if (Debugging.AssertsEnabled) Debugging.Assert(index == originalIndex); + return base.Get(index, arr, off, len); + } + } + + public override int Set(int index, long[] arr, int off, int len) + { + if (Debugging.AssertsEnabled) + { + Debugging.Assert(len > 0, "len must be > 0 (got {0})", len); + Debugging.Assert(index >= 0 && index < m_valueCount); + } + len = Math.Min(len, m_valueCount - index); + if (Debugging.AssertsEnabled) Debugging.Assert(off + len <= arr.Length); + + int originalIndex = index; + + // go to the next block boundary + int valuesPerBlock = 64 / m_bitsPerValue; + int offsetInBlock = index %% valuesPerBlock; + if (offsetInBlock != 0) + { + for (int i = offsetInBlock; i < valuesPerBlock && len > 0; ++i) + { + Set(index++, arr[off++]); + --len; + } + if (len == 0) + { + return index - originalIndex; + } + } + + // bulk set + if (Debugging.AssertsEnabled) Debugging.Assert(index %% valuesPerBlock == 0); + BulkOperation op = BulkOperation.Of(PackedInt32s.Format.PACKED_SINGLE_BLOCK, m_bitsPerValue); + if (Debugging.AssertsEnabled) Debugging.Assert(op.Int64BlockCount == 1); + if (Debugging.AssertsEnabled) Debugging.Assert(op.Int64ValueCount == valuesPerBlock); + int blockIndex = index / valuesPerBlock; + int nblocks = (index + len) / valuesPerBlock - blockIndex; + op.Encode(arr, off, blocks, blockIndex, nblocks); + int diff = nblocks * valuesPerBlock; + index += diff; len -= diff; + + if (index > originalIndex) + { + // stay at the block boundary + return index - originalIndex; + } + else + { + // no progress so far => already at a block boundary but no full block to + // set + if (Debugging.AssertsEnabled) Debugging.Assert(index == originalIndex); + return base.Set(index, arr, off, len); + } + } + + public override void Fill(int fromIndex, int toIndex, long val) + { + if (Debugging.AssertsEnabled) + { + Debugging.Assert(fromIndex >= 0); + Debugging.Assert(fromIndex <= toIndex); + Debugging.Assert(PackedInt32s.BitsRequired(val) <= m_bitsPerValue); + } + + int valuesPerBlock = 64 / m_bitsPerValue; + if (toIndex - fromIndex <= valuesPerBlock << 1) + { + // there needs to be at least one full block to set for the block + // approach to be worth trying + base.Fill(fromIndex, toIndex, val); + return; + } + + // set values naively until the next block start + int fromOffsetInBlock = fromIndex %% valuesPerBlock; + if (fromOffsetInBlock != 0) + { + for (int i = fromOffsetInBlock; i < valuesPerBlock; ++i) + { + Set(fromIndex++, val); + } + if (Debugging.AssertsEnabled) Debugging.Assert(fromIndex %% valuesPerBlock == 0); + } + + // bulk set of the inner blocks + int fromBlock = fromIndex / valuesPerBlock; + int toBlock = toIndex / valuesPerBlock; + if (Debugging.AssertsEnabled) Debugging.Assert(fromBlock * valuesPerBlock == fromIndex); + + long blockValue = 0L; + for (int i = 0; i < valuesPerBlock; ++i) + { + blockValue |= (val << (i * m_bitsPerValue)); + } + Arrays.Fill(blocks, fromBlock, toBlock, blockValue); + + // fill the gap + for (int i = valuesPerBlock * toBlock; i < toIndex; ++i) + { + Set(i, val); + } + } + + internal override PackedInt32s.Format Format => PackedInt32s.Format.PACKED_SINGLE_BLOCK; + + public override string ToString() + { + return this.GetType().Name + "(bitsPerValue=" + m_bitsPerValue + ", size=" + Count + ", elements.length=" + blocks.Length + ")"; + } + + public static Packed64SingleBlock Create(DataInput @in, int valueCount, int bitsPerValue) + { + Packed64SingleBlock reader = Create(valueCount, bitsPerValue); + for (int i = 0; i < reader.blocks.Length; ++i) + { + reader.blocks[i] = @in.ReadInt64(); + } + return reader; + } + +""" %(SUPPORTED_BITS_PER_VALUE[-1], ", ".join(map(str, SUPPORTED_BITS_PER_VALUE))) + +### LUCENENET specific - add namespace closing brace and EOF newline too +FOOTER = " }\n}\n" + +if __name__ == '__main__': + + f = open("Packed64SingleBlock.cs", 'w') + f.write(HEADER) + f.write(" public static Packed64SingleBlock Create(int valueCount, int bitsPerValue)\n") + f.write(" {\n") + f.write(" switch (bitsPerValue)\n") + f.write(" {\n") + for bpv in SUPPORTED_BITS_PER_VALUE: + f.write(" case %d:\n" %bpv) + f.write(" return new Packed64SingleBlock%d(valueCount);\n" %bpv) + f.write(" default:\n") + f.write(" throw new ArgumentException(\"Unsupported number of bits per value: \" + %d);\n" %bpv) # LUCENENET NOTE: suspicious/incorrect use of range variable, but porting as-is + f.write(" }\n") + f.write(" }\n\n") + + for bpv in SUPPORTED_BITS_PER_VALUE: + log_2 = 0 + while (1 << log_2) < bpv: + log_2 = log_2 + 1 + if (1 << log_2) != bpv: + log_2 = None + + f.write(" internal class Packed64SingleBlock%d : Packed64SingleBlock\n" %bpv) + f.write(" {\n") + + f.write(" internal Packed64SingleBlock%d(int valueCount)\n" %bpv) + f.write(" : base(valueCount, %d)\n" %bpv) + f.write(" {\n") + f.write(" }\n\n") + + f.write(" public override long Get(int index)\n") + f.write(" {\n") + if log_2 is not None: + f.write(" int o = index >>> %d;\n" %(6 - log_2)) + f.write(" int b = index & %d;\n" %((1 << (6 - log_2)) - 1)) + f.write(" int shift = b << %d;\n" %log_2) + else: + f.write(" int o = index / %d;\n" %(64 / bpv)) + f.write(" int b = index %% %d;\n" %(64 / bpv)) + f.write(" int shift = b * %d;\n" %bpv) + f.write(" return (blocks[o] >>> shift) & %dL;\n" %((1 << bpv) - 1)) + f.write(" }\n\n") + + f.write(" public override void Set(int index, long value)\n") + f.write(" {\n") + if log_2 is not None: + f.write(" int o = index >>> %d;\n" %(6 - log_2)) + f.write(" int b = index & %d;\n" %((1 << (6 - log_2)) - 1)) + f.write(" int shift = b << %d;\n" %log_2) + else: + f.write(" int o = index / %d;\n" %(64 / bpv)) + f.write(" int b = index %% %d;\n" %(64 / bpv)) + f.write(" int shift = b * %d;\n" %bpv) + f.write(" blocks[o] = (blocks[o] & ~(%dL << shift)) | (value << shift);\n" % ((1 << bpv) - 1)) + f.write(" }\n") + f.write(" }\n\n") + + f.write(FOOTER) + f.close() diff --git a/src/Lucene.Net/Util/Packed/gen_PackedThreeBlocks.py b/src/Lucene.Net/Util/Packed/gen_PackedThreeBlocks.py new file mode 100644 index 0000000000..f14964c988 --- /dev/null +++ b/src/Lucene.Net/Util/Packed/gen_PackedThreeBlocks.py @@ -0,0 +1,199 @@ +#! /usr/bin/env python + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ## LUCENENET PORTING NOTES +# This script was originally written for Python 2, but has been tested against Python 3. +# No changes were necessary to run this script in Python 3. + +HEADER="""using Lucene.Net.Diagnostics; +using Lucene.Net.Support; +using System; +using System.Runtime.CompilerServices; + +//------------------------------------------------------------------------------ +// +// This code was generated by the gen_PackedThreeBlocks.py script. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ + +namespace Lucene.Net.Util.Packed +{ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + using DataInput = Lucene.Net.Store.DataInput; + +""" + +TYPES = {8: "byte", 16: "short"} +DOTNET_READ_TYPES = {8: "Byte", 16: "Int16"} # LUCENENET specific +MASKS = {8: " & 0xFFL", 16: " & 0xFFFFL", 32: " & 0xFFFFFFFFL", 64: ""} +CASTS = {8: "(byte)", 16: "(short)", 32: "(int)", 64: ""} # LUCENENET specific - removed space from casts to match existing C# port style + +if __name__ == '__main__': + for bpv in TYPES.keys(): + type + f = open("Packed%dThreeBlocks.cs" %bpv, 'w') + f.write(HEADER) + f.write(""" /// + /// Packs integers into 3 %ss (%d bits per value). + /// + /// @lucene.internal + /// \n""" %(TYPES[bpv], bpv*3)) + f.write(" internal sealed class Packed%dThreeBlocks : PackedInt32s.MutableImpl\n" %bpv) + f.write(" {\n") + f.write(" private readonly %s[] blocks;\n\n" %TYPES[bpv]) + + f.write(" public const int MAX_SIZE = int.MaxValue / 3;\n\n") + + f.write(" internal Packed%dThreeBlocks(int valueCount)\n" %bpv) + f.write(" : base(valueCount, %d)\n" %(bpv*3)) + f.write(" {\n") + f.write(" if (valueCount > MAX_SIZE)\n") + f.write(" {\n") + f.write(" throw new ArgumentOutOfRangeException(nameof(valueCount), \"MAX_SIZE exceeded\");\n") + f.write(" }\n") + f.write(" blocks = new %s[valueCount * 3];\n" %TYPES[bpv]) + f.write(" }\n\n") + + f.write(" internal Packed%dThreeBlocks(int packedIntsVersion, DataInput @in, int valueCount)\n" %bpv) + f.write(" : this(valueCount)\n") + f.write(" {\n") + if bpv == 8: + f.write(" @in.ReadBytes(blocks, 0, 3 * valueCount);\n") + else: + f.write(" for (int i = 0; i < 3 * valueCount; ++i)\n") + f.write(" {\n") + f.write(" blocks[i] = @in.Read%s();\n" %DOTNET_READ_TYPES[bpv].title()) # LUCENENET specific + f.write(" }\n") + f.write(" // because packed ints have not always been byte-aligned\n") + f.write(" int remaining = (int)(PackedInt32s.Format.PACKED.ByteCount(packedIntsVersion, valueCount, %d) - 3L * valueCount * %d);\n" %(3 * bpv, bpv / 8)) + f.write(" for (int i = 0; i < remaining; ++i)\n") + f.write(" {\n") + f.write(" @in.ReadByte();\n") + f.write(" }\n") + f.write(" }\n") + + f.write(""" + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public override long Get(int index) + { + int o = index * 3; + return (blocks[o]%s) << %d | (blocks[o + 1]%s) << %d | (blocks[o + 2]%s); + } + + public override int Get(int index, long[] arr, int off, int len) + { + if (Debugging.AssertsEnabled) + { + Debugging.Assert(len > 0, "len must be > 0 (got {0})", len); + Debugging.Assert(index >= 0 && index < m_valueCount); + Debugging.Assert(off + len <= arr.Length); + } + + int gets = Math.Min(m_valueCount - index, len); + for (int i = index * 3, end = (index + gets) * 3; i < end; i += 3) + { + arr[off++] = (blocks[i]%s) << %d | (blocks[i + 1]%s) << %d | (blocks[i + 2]%s); + } + return gets; + } + + public override void Set(int index, long value) + { + int o = index * 3; + blocks[o] = %s(value >>> %d); + blocks[o + 1] = %s(value >>> %d); + blocks[o + 2] = %svalue; + } + + public override int Set(int index, long[] arr, int off, int len) + { + if (Debugging.AssertsEnabled) + { + Debugging.Assert(len > 0, "len must be > 0 (got {0})", len); + Debugging.Assert(index >= 0 && index < m_valueCount); + Debugging.Assert(off + len <= arr.Length); + } + + int sets = Math.Min(m_valueCount - index, len); + for (int i = off, o = index * 3, end = off + sets; i < end; ++i) + { + long value = arr[i]; + blocks[o++] = %s(value >>> %d); + blocks[o++] = %s(value >>> %d); + blocks[o++] = %svalue; + } + return sets; + } + + public override void Fill(int fromIndex, int toIndex, long val) + { + %s block1 = %s(val >>> %d); + %s block2 = %s(val >>> %d); + %s block3 = %sval; + for (int i = fromIndex * 3, end = toIndex * 3; i < end; i += 3) + { + blocks[i] = block1; + blocks[i + 1] = block2; + blocks[i + 2] = block3; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public override void Clear() + { + Arrays.Fill(blocks, %s0); + } + + public override long RamBytesUsed() + { + return RamUsageEstimator.AlignObjectSize( + RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + + 2 * RamUsageEstimator.NUM_BYTES_INT32 // valueCount,bitsPerValue + + RamUsageEstimator.NUM_BYTES_OBJECT_REF) // blocks ref + + RamUsageEstimator.SizeOf(blocks); + } + + public override string ToString() + { + return this.GetType().Name + "(bitsPerValue=" + m_bitsPerValue + ", size=" + Count + ", elements.length=" + blocks.Length + ")"; + } + } +} +""" %(MASKS[bpv], 2*bpv, MASKS[bpv], bpv, MASKS[bpv], MASKS[bpv], 2*bpv, MASKS[bpv], bpv, MASKS[bpv], CASTS[bpv], 2*bpv, CASTS[bpv], bpv, CASTS[bpv], CASTS[bpv], + 2*bpv, CASTS[bpv], bpv, CASTS[bpv], TYPES[bpv], CASTS[bpv], 2*bpv, TYPES[bpv], + CASTS[bpv], bpv, TYPES[bpv], CASTS[bpv], CASTS[bpv])) + + f.close() diff --git a/src/Lucene.Net/Util/Packed/package.md b/src/Lucene.Net/Util/Packed/package.md index 18d003b1cb..8c26bef454 100644 --- a/src/Lucene.Net/Util/Packed/package.md +++ b/src/Lucene.Net/Util/Packed/package.md @@ -23,13 +23,13 @@ summary: *content ## Packed integer arrays and streams. The packed package provides - + * sequential and random access capable arrays of positive longs, * routines for efficient serialization and deserialization of streams of packed integers. - - The implementations provide different trade-offs between memory usage and access speed. The standard usage scenario is replacing large int or long arrays in order to reduce the memory footprint. - The main access point is the factory. + The implementations provide different trade-offs between memory usage and access speed. The standard usage scenario is replacing large int or long arrays in order to reduce the memory footprint. + + The main access point is the factory. ### In-memory structures @@ -43,9 +43,9 @@ summary: *content * ____ - * Same as PackedInts.Mutable but grows the number of bits per values when needed. + * Same as PackedInt32s.Mutable but grows the number of bits per values when needed. - * Useful to build a PackedInts.Mutable from a read-once stream of longs. + * Useful to build a PackedInt32s.Mutable from a read-once stream of longs. * ____ @@ -53,7 +53,7 @@ summary: *content * Supports more than 2B values. - * You should use Appending(Delta)PackedLongBuffer instead if you don't need random write access. + * You should use Appending(Delta)PackedInt64Buffer instead if you don't need random write access. * ____ @@ -99,4 +99,4 @@ summary: *content * __, __ - * Writes sequences of longs where each long can use any number of bits. \ No newline at end of file + * Writes sequences of longs where each long can use any number of bits.