Update to newer smui + new SMUI features (#168)

* it's 2024, it's time to ditch docker-compose * now showing rules usage Signed-off-by: Eric Pugh <[email protected]>
querqy · Sep 18, 2024 · 1905206 · 1905206
1 parent 419469a
commit 1905206
Show file tree

Hide file tree

Showing 12 changed files with 36 additions and 29 deletions.
diff --git a/README.md b/README.md
@@ -86,7 +86,7 @@ There is also a video series that is very closely related called [Meet Pete](htt
 To start your environment, but still run each command to setup the integrations manually, run:
 
 ```
-docker-compose up --build -d
+docker compose up --build -d
 ```
 
 The quickstart command will launch a Solr cluster, load the configsets and product data for the _ecommerce_ index, and launch the SMUI user interface:
@@ -115,18 +115,18 @@ To include the observability features, run:
 
 To see what is happening in the Chorus stack you can tail the logs for all the components via:
 ```
-docker-compose logs -tf
+docker compose logs -tf
 ```
 
 If you want to narrow down to just one component of the Chorus stack do:
 ```
-docker-compose ps                       # list out the names of the components
-docker-compose logs -tf solr1 solr2     # tail solr1 and solr2 only
+docker compose ps                       # list out the names of the components
+docker compose logs -tf solr1 solr2     # tail solr1 and solr2 only
 ```
 
 To destroy your environment (including any volumes created like the mysql db), just run:
 ```
-docker-compose down -v
+docker compose down -v
 ```
 
 or

diff --git a/TECHNICAL_DETAILS.md b/TECHNICAL_DETAILS.md
@@ -43,7 +43,7 @@ curl http://localhost:8983/solr/ecommerce/config -H 'Content-type:application/js
 
 Create a user with the email `[email protected]` and the password `password`:
 ```sh
-docker-compose run quepid bundle exec thor user:create -a [email protected] "Demo User" password
+docker compose run quepid bundle exec thor user:create -a [email protected] "Demo User" password
 ```
 
 For Quepid case, pick `name` for title, and `id` for identifier.  Add `thumb:imageUrl` to the list of fields.

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,4 +1,3 @@
-version: '3'
 services:
 
   blacklight:
@@ -58,6 +57,9 @@ services:
       - SMUI_TOGGLE_EVENTHISTORY=true
       - SMUI_RULE_TAGGING_ACTIVE=true
       - SMUI_PREDEFINED_TAGS_FILE=/smui/conf/predefined_tags.json
+      - SMUI_RULE_USAGE_STATISTICS_LOCATION=file:///smui/data/rules-usage-stats.csv    
+    volumes:
+        - ./smui/data/:/smui/data      
     depends_on:
       - mysql
 
@@ -125,7 +127,7 @@ services:
       - keycloak
 
   zoo1:
-    image: zookeeper:3.7.0
+    image: zookeeper:3.9.2
     container_name: zoo1
     hostname: zoo1
     ports:
@@ -139,7 +141,7 @@ services:
 
 
   zoo2:
-    image: zookeeper:3.7.0
+    image: zookeeper:3.9.2
     container_name: zoo2
     hostname: zoo2
     ports:
@@ -153,7 +155,7 @@ services:
 
 
   zoo3:
-    image: zookeeper:3.7.0
+    image: zookeeper:3.9.2
     container_name: zoo3
     hostname: zoo3
     ports:

diff --git a/embeddings/app/main.py b/embeddings/app/main.py
@@ -1,11 +1,11 @@
 from fastapi import FastAPI
 from clipL14 import router as router_clip
 from minilm import router as router_minilm
-from finetuned import router as router_finetuned
+#from finetuned import router as router_finetuned
 
 app = FastAPI()
 
 app.include_router(router_clip.router, prefix="/clip")
 app.include_router(router_minilm.router, prefix="/minilm")
-app.include_router(router_finetuned.sbert_router, prefix="/ftbert")
-app.include_router(router_finetuned.clip_router, prefix="/ftclip")
+#app.include_router(router_finetuned.sbert_router, prefix="/ftbert")
+#app.include_router(router_finetuned.clip_router, prefix="/ftclip")
diff --git a/embeddings/requirements.txt b/embeddings/requirements.txt
@@ -1,3 +1,4 @@
+docarray==0.21.0 # See https://github.com/querqy/chorus/issues/169
 sentence-transformers
 fastapi
 uvicorn[standard]

diff --git a/helpers.sh b/helpers.sh
@@ -58,7 +58,7 @@ function log_prerequisite_missing() {
 
 function check_prerequisites() {
     set +e
-    PREREQUISITES=("curl" "jq" "docker-compose" "zip")
+    PREREQUISITES=("curl" "jq" "docker" "zip")
     log_major "Checking availability of prerequisites..."
     COUNT_MISSING_PREQUISITES=0
     for PREREQUISITE in "${PREREQUISITES[@]}"
@@ -81,10 +81,10 @@ function check_prerequisites() {
 }
 
 function display_help_message() {
-    echo -e "Use the option --with-offline-lab | -lab to include Quepid and RRE services in Chorus."
+    echo -e "Use the option --with-offline-lab | -lab to include Quepid in Chorus."
     echo -e "Use the option --with-observability | -obs to include Grafana, Prometheus, and Solr Exporter services in Chorus."
+    echo -e "Use the option --with-active-search-management | -active to include Active Search Management via SMUI in Chorus."    
     echo -e "Use the option --with-vector-search | -vector to include Vector Search services in Chorus."
-    echo -e "Use the option --with-active-search-management | -active to include Active Search Management in Chorus."
     echo -e "Use the option --shutdown | -s to shutdown and remove the Docker containers and data."
     echo -e "Use the option --online-deployment | -online to update configuration to run on chorus.dev.o19s.com environment."
 }
diff --git a/katas/000_setting_up_chorus.md b/katas/000_setting_up_chorus.md
@@ -7,7 +7,7 @@ We use a Docker Compose based environment to manage firing up all the components
 Open up a terminal window and run:
 
 ```sh
-docker-compose up --build
+docker compose up --build
 ```
 
 Wait a while, because you'll be downloading and building quite a few images!  You may think it's frozen at various points, but go for a walk and come back and it'll be up and running.
@@ -139,13 +139,13 @@ We'll start with Quepid and then move on to RRE.
 First we need to create the database for Quepid:
 
 ```sh
-docker-compose run --rm quepid bin/rake db:setup
+docker compose run --rm quepid bin/rake db:setup
 ```
 
 We also need to create you an account with Administrator permissions:
 
 ```sh
-docker-compose run quepid bundle exec thor user:create -a [email protected] "Chorus Admin" password
+docker compose run quepid bundle exec thor user:create -a [email protected] "Chorus Admin" password
 ```
 
 Visit Quepid at http://localhost:3000 and log in with the email and password you just set up.

diff --git a/katas/012_vector_search.md b/katas/012_vector_search.md
@@ -55,7 +55,7 @@ Also, notice that this also triggers an additional 'embedding service' based on
 ```bash
 if $vector_search; then
   log_major "Setting up Embeddings service"
-  docker-compose up -d --build embeddings
+  docker compose up -d --build embeddings
   ./embeddings/wait-for-api.sh
 fi
 ```

diff --git a/keycloak/Dockerfile b/keycloak/Dockerfile
@@ -1,4 +1,4 @@
-FROM quay.io/keycloak/keycloak:20.0.5 as builder
+FROM quay.io/keycloak/keycloak:20.0.5 AS builder
 
 # Enable health and metrics support
 ENV KC_HEALTH_ENABLED=true

diff --git a/quickstart.sh b/quickstart.sh
@@ -51,7 +51,7 @@ do
   shift
 done
 
-# Function check_prerequisites makes sure that you have curl, jq, docker-compose, and zip installed. See helpers.sh for details.
+# Function check_prerequisites makes sure that you have curl, jq, docker, and zip installed. See helpers.sh for details.
 check_prerequisites
 
 services="blacklight solr1 solr2 solr3 keycloak"
@@ -91,7 +91,7 @@ if ! $local_deploy; then
   sed -i.bu 's/keycloak:9080/chorus.dev.o19s.com:9080/g'  ./docker-compose.yml
 fi
 
-docker-compose down -t 30 -v
+docker compose down -t 30 -v
 if $shutdown; then
   exit
 fi
@@ -105,7 +105,7 @@ if $active_search_management; then
 
 fi
 
-docker-compose up -d --build ${services}
+docker compose up -d --build ${services}
 
 echo -e "${MAJOR}Waiting for Solr cluster to start up and all three nodes to be online.${RESET}"
 ./solr/wait-for-solr-cluster.sh # Wait for all three Solr nodes to be online
@@ -400,15 +400,15 @@ if $offline_lab; then
   log_major "Setting up Quepid"
   ./mysql/wait-for-mysql.sh
 
-  docker-compose run --rm quepid bundle exec bin/rake db:setup
-  docker-compose run quepid bundle exec thor user:create -a [email protected] "Chorus Admin" password
+  docker compose run --rm quepid bundle exec bin/rake db:setup
+  docker compose run quepid bundle exec thor user:create -a [email protected] "Chorus Admin" password
   log_minor "Setting up Chorus Baseline Relevance case"
   if $local_deploy; then
     solr_collection_url=http://localhost:8983/solr/ecommerce/select
   else
     solr_collection_url=http://chorus.dev.o19s.com:8983/solr/ecommerce/select
   fi
-  docker-compose run quepid bundle exec thor case:create "Chorus Baseline Relevance" solr ${solr_collection_url} JSONP "id:id, title:title, thumb:img_500x500, name, brand, product_type" "q=#\$query##&useParams=visible_products,querqy_algo" nDCG@10 [email protected]
+  docker compose run quepid bundle exec thor case:create "Chorus Baseline Relevance" solr ${solr_collection_url} JSONP "id:id, title:title, thumb:img_500x500, name, brand, product_type" "q=#\$query##&useParams=visible_products,querqy_algo" nDCG@10 [email protected]
 
   docker cp ./katas/Broad_Query_Set_rated.csv quepid:/srv/app/Broad_Query_Set_rated.csv
   docker exec quepid bundle exec thor ratings:import 1 /srv/app/Broad_Query_Set_rated.csv >> /dev/null
@@ -424,7 +424,7 @@ fi
 
 if $vector_search; then
   log_major "Setting up Embeddings service"
-  docker-compose up -d --build embeddings
+  docker compose up -d --build embeddings
   ./embeddings/wait-for-api.sh
 fi
 

diff --git a/smui/Dockerfile b/smui/Dockerfile
@@ -1,4 +1,4 @@
-FROM querqy/smui:4.0.11
+FROM querqy/smui:4.3.0
 
 COPY --chown=smui:smui conf/* conf/
 

diff --git a/smui/data/rules-usage-stats.csv b/smui/data/rules-usage-stats.csv
@@ -0,0 +1,4 @@
+SMUI_GUID,USER_QUERY,FREQUENCY
+c4b10dea-1cd5-4361-83c6-d5c72691e343,laptop,6
+c4b10dea-1cd5-4361-83c6-d5c72691e343,notebook,2
+7bc66070-7d88-43d4-8d09-0dd0210c4ba3,laptop,20