From b24b41678f392dca42432395feb650dc686865d0 Mon Sep 17 00:00:00 2001
From: Jon Bratseth <bratseth@vespa.ai>
Date: Mon, 28 Oct 2024 22:03:24 +0100
Subject: [PATCH] Document pack_bits and binarize

---
 en/reference/indexing-language-reference.html | 83 ++++++++++++++-----
 1 file changed, 62 insertions(+), 21 deletions(-)

diff --git a/en/reference/indexing-language-reference.html b/en/reference/indexing-language-reference.html
index 72dd176740..639ad30594 100644
--- a/en/reference/indexing-language-reference.html
+++ b/en/reference/indexing-language-reference.html
@@ -201,12 +201,9 @@ <h3 id="arithmetic">Arithmetics</h3>
 </p>
 
 
-
 <h3 id="converter">Converters</h3>
 
-<p>There are several expressions that allow you to convert from one data type to another.
-These are often used within a <code>for_each</code> to convert
-e.g. an array of strings to an array of integers.</p>
+<p>These expressions lets you convert from one data type to another.</p>
 
 <table class="table">
   <thead>
@@ -217,21 +214,65 @@ <h3 id="converter">Converters</h3>
     <th>Description</th>
   </tr>
   </thead><tbody>
-  <tr>
-    <td><code>embed</code></td>
-    <td>String</td>
-    <td>A tensor of the type of the receiving field</td>
-    <td><p id="embed">Invokes an <a href="../embedding.html">embedder</a> to convert a text to a point in a tensor space.
-        Arguments are given space separated, as in <code>embed colbert chunk</code>.
-        The first argument is the id of the embedder, and can be omitted when only one is configured.
-        Any additional arguments are passed to the embedder implementation.</p></td>
-  </tr>
-  <tr>
-    <td><code>hash</code></td>
-    <td>String</td>
-    <td>Any string</td>
-    <td><p id="hash">Converts the input to a hash value (using SipHash).
+<tr>
+  <td><code>binarize [threshold]</code></td>
+  <td>Any tensor</td>
+  <td>Any tensor</td>
+  <td>
+    <p id="binarize">
+      Replaces all values in a tensor by 0 or 1.
+      This takes an optional argument specifying the threshold a value needs to be larger than to be
+      replaced by 1 instead of 0. The default threshold is 0.
+      This is useful to create a suitable input to <a href="#pack_bits">pack_bits</a>.
+    </p>
+  </td>
+</tr>
+<tr>
+  <td><code>embed [id]</code></td>
+  <td>String</td>
+  <td>A tensor</td>
+  <td><p id="embed">Invokes an <a href="../embedding.html">embedder</a> to convert a text to one or more vector embeddings.
+      The type of the output tensor is what is required by the following expression (as supported by the specific embedder).
+      Arguments are given space separated, as in <code>embed colbert chunk</code>.
+      The first argument is the id of the embedder, and can be omitted when only one is configured.
+      Any additional arguments are passed to the embedder implementation.</p></td>
+</tr>
+<tr>
+  <td><code>hash</code></td>
+  <td>String</td>
+  <td>int or long</td>
+  <td><p id="hash">Converts the input to a hash value (using SipHash).
       The hash will be int or long depending on the target field.</p></td>
+</tr>
+<tr>
+    <td><code>pack_bits</code></td>
+    <td>A tensor</td>
+    <td>A tensor</td>
+    <td>
+      <p id="pack_bits">
+        Packs the values of a binary tensor into bytes with 1 bit per value in big-endian order.
+      </p>
+      <p>
+        The input tensor must:
+      <ul>
+        <li>Only have values that are 0 or 1</li>
+        <li>Have a single dense dimension</li>
+      </ul>
+      It can have any value type and any number of sparse dimensions.
+      </p>
+      <p>
+        The output tensor will have:
+      <ul>
+        <li><code>int8</code> as the value type.</li>
+        <li>The dense dimension size divided by 8 (rounded upwards to integer).</li>
+        <li>The same sparse dimensions as before.</li>
+      </ul>
+      The resulting tensor can be unpacked during ranking using
+      <a href="ranking-expressions.html#unpack-bits">unpack_bits</a>.
+      A tensor can be converted to binary form suitable as input to this by the
+      <a href="#binarize">binarize function</a>.
+      </p>
+    </td>
   </tr>
   <tr>
     <td><code>to_array</code></td>
@@ -541,9 +582,9 @@ <h3 id="other">Other expressions</h3>
     <td><code>random [ &lt;max&gt; ]</code></td>
     <td>
       <p id="random">
-      Returns a random integer value.
-      Lowest value is 0 and the highest value is determined either by the argument or,
-      if no argument is given, the execution value.
+        Returns a random integer value.
+        Lowest value is 0 and the highest value is determined either by the argument or,
+        if no argument is given, the execution value.
       </p>
     </td>
   </tr>