From c7a6e1da3905add10d847d78384f1aad9c4a1708 Mon Sep 17 00:00:00 2001
From: Jon Bratseth <bratseth@vespa.ai>
Date: Fri, 4 Oct 2024 14:51:12 +0200
Subject: [PATCH] Parse 'binarize'

---
 .../src/main/javacc/IndexingParser.jj         | 11 ++++----
 .../ccc/indexinglanguage/IndexingParser.ccc   | 27 +++++++++++++------
 2 files changed, 25 insertions(+), 13 deletions(-)

diff --git a/indexinglanguage/src/main/javacc/IndexingParser.jj b/indexinglanguage/src/main/javacc/IndexingParser.jj
index 94143bd8f7ec..09aa3829ab91 100644
--- a/indexinglanguage/src/main/javacc/IndexingParser.jj
+++ b/indexinglanguage/src/main/javacc/IndexingParser.jj
@@ -816,7 +816,8 @@ String identifier() :
       ( <ATTRIBUTE>              |
         <BASE64_DECODE>          |
         <BASE64_ENCODE>          |
-        <BUSY_WAIT>                   |
+        <BINARIZE>               |
+        <BUSY_WAIT>              |
         <CASE>                   |
         <CASE_DEFAULT>           |
         <CLEAR_STATE>            |
@@ -831,15 +832,15 @@ String identifier() :
         <GET_VAR>                |
         <GUARD>                  |
         <HASH>                   |
-        <HEX_DECODE>             |
-        <HEX_ENCODE>             |
-        <HOST_NAME>              |
+        <HEX_DECODE>              |
+        <HEX_ENCODE>              |
+        <HOST_NAME>               |
         <IDENTIFIER>             |
         <IF>                     |
         <INDEX>                  |
         <INPUT>                  |
         <JOIN>                   |
-        <LOWER_CASE>             |
+        <LOWER_CASE>              |
         <MAX_LENGTH>             |
         <NGRAM>                  |
         <NORMALIZE>              |
diff --git a/integration/schema-language-server/language-server/src/main/ccc/indexinglanguage/IndexingParser.ccc b/integration/schema-language-server/language-server/src/main/ccc/indexinglanguage/IndexingParser.ccc
index 23d3ced8cf8f..a398b9d9199f 100644
--- a/integration/schema-language-server/language-server/src/main/ccc/indexinglanguage/IndexingParser.ccc
+++ b/integration/schema-language-server/language-server/src/main/ccc/indexinglanguage/IndexingParser.ccc
@@ -79,18 +79,18 @@ INJECT IndexingParser:
         return this;
     }
 
-    private static FieldValue parseDouble(String str) {
+    private static DoubleFieldValue parseDouble(String str) {
         return new DoubleFieldValue(new BigDecimal(str).doubleValue());
     }
 
-    private static FieldValue parseFloat(String str) {
+    private static FloatFieldValue parseFloat(String str) {
         if (str.endsWith("f") || str.endsWith("F")) {
             str = str.substring(0, str.length() - 1);
         }
         return new FloatFieldValue(new BigDecimal(str).floatValue());
     }
 
-    private static FieldValue parseInteger(String str) {
+    private static IntegerFieldValue parseInteger(String str) {
         if (str.startsWith("0x")) {
             return new IntegerFieldValue(new BigInteger(str.substring(2), 16).intValue());
         } else {
@@ -98,7 +98,7 @@ INJECT IndexingParser:
         }
     }
 
-    private static FieldValue parseLong(String str) {
+    private static LongFieldValue parseLong(String str) {
         if (str.endsWith("l") || str.endsWith("L")) {
             str = str.substring(0, str.length() - 1);
         }
@@ -174,6 +174,7 @@ TOKEN :
     <ATTRIBUTE: "attribute"> |
     <BASE64_DECODE: "base64decode"> |
     <BASE64_ENCODE: "base64encode"> |
+    <BINARIZE: "binarize"> |
     <BUSY_WAIT: "busy_wait"> |
     <CASE: "case"> |
     <CASE_DEFAULT: "default"> |
@@ -330,6 +331,7 @@ Expression value() :
     ( val = attributeExp()          |
       val = base64DecodeExp()       |
       val = base64EncodeExp()       |
+      val = binarizeExp()           |
       val = busy_waitExp()          |
       val = clearStateExp()         |
       val = echoExp()               |
@@ -407,6 +409,14 @@ Expression base64EncodeExp() : { }
     { return new Base64EncodeExpression(); }
 ;
 
+Expression binarizeExp() :
+{
+    NumericFieldValue threshold = new DoubleFieldValue(0);
+}
+    ( <BINARIZE> [ threshold = numericValue() ] )
+    { return new BinarizeExpression(threshold.getNumber().doubleValue()); }
+;
+
 Expression busy_waitExp() : { }
  
     ( <BUSY_WAIT> )
@@ -835,11 +845,12 @@ String identifierStr() :
     String val;
 }
  
-    ( val = stringLiteral()             |
+    ( val = stringLiteral()      |
       ( <ATTRIBUTE>              |
         <BASE64_DECODE>          |
         <BASE64_ENCODE>          |
-        <BUSY_WAIT>                   |
+        <BINARIZE>               |
+        <BUSY_WAIT>              |
         <CASE>                   |
         <CASE_DEFAULT>           |
         <CLEAR_STATE>            |
@@ -920,9 +931,9 @@ FieldValue fieldValue() :
     { return val; }
 ;
 
-FieldValue numericValue() :
+NumericFieldValue numericValue() :
 {
-    FieldValue val;
+    NumericFieldValue val;
     String pre = "";
 }