From 57a11d01eb1899a76a78427bd806c7f0c84c8a8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Thu, 21 Sep 2023 15:35:08 +0200 Subject: [PATCH 1/7] Mini langstream --- bin/mini-langstream/README.md | 40 ++ bin/mini-langstream/get-mini-langstream.sh | 94 +++++ bin/mini-langstream/mini-langstream | 465 +++++++++++++++++++++ 3 files changed, 599 insertions(+) create mode 100644 bin/mini-langstream/README.md create mode 100755 bin/mini-langstream/get-mini-langstream.sh create mode 100755 bin/mini-langstream/mini-langstream diff --git a/bin/mini-langstream/README.md b/bin/mini-langstream/README.md new file mode 100644 index 000000000..48f73dc98 --- /dev/null +++ b/bin/mini-langstream/README.md @@ -0,0 +1,40 @@ +# mini-langstream + +## Get started + +### MacOS/Linux/WSL + +``` +curl -Ls "https://raw.githubusercontent.com/LangStream/langstream/main/bin/mini-langstream/get-mini-langstream.sh" | bash +``` + +## Start the cluster + +``` +$ mini-langstream start +``` + +## Use the CLI to deploy an application + +``` +$ mini-langstream cli apps deploy app -i \$(mini-langstream get-instance)" -s https://raw.githubusercontent.com/LangStream/langstream/main/examples/secrets/secrets.yaml -app https://github.com/LangStream/langstream/tree/main/examples/applications/python-processor-exclamation +``` + +## Start the cluster + +``` +$ mini-langstream start +``` + + +## Stop the cluster + +``` +$ mini-langstream delete +``` + +## Get help + +``` +mini-langstream help +``` diff --git a/bin/mini-langstream/get-mini-langstream.sh b/bin/mini-langstream/get-mini-langstream.sh new file mode 100755 index 000000000..189696b68 --- /dev/null +++ b/bin/mini-langstream/get-mini-langstream.sh @@ -0,0 +1,94 @@ +#!/bin/bash + +set -e + +track_last_command() { + last_command=$current_command + current_command=$BASH_COMMAND +} +trap track_last_command DEBUG + +echo_failed_command() { + local exit_code="$?" + if [[ "$exit_code" != "0" ]]; then + echo "'$last_command': command failed with exit code $exit_code." + fi +} + +trap echo_failed_command EXIT + +clear + + +echo "Installing $(tput setaf 6)mini-langstream$(tput setaf 7) please wait" + +URL="https://raw.githubusercontent.com/LangStream/langstream/mini-langstream/bin/mini-langstream/mini-langstream?$(date +%s)" + +app_home_dir=$HOME/.mini-langstream +mkdir -p $app_home_dir +app_bin=$app_home_dir/bin +mkdir -p $app_bin + +darwin=false +case "$(uname)" in + Darwin*) + darwin=true + ;; +esac + +echo "$(tput setaf 2)[OK]$(tput setaf 7) - Ready to install." + +if ! command -v curl > /dev/null; then + echo "Not found." + echo "" + echo "======================================================================================================" + echo " Please install curl on your system using your favourite package manager." + echo "" + echo " Restart after installing curl." + echo "======================================================================================================" + echo "" + exit 1 +fi +echo "$(tput setaf 2)[OK]$(tput setaf 7) - curl command is available" + + +echo "" +echo "$(tput setaf 6)Downloading:$(tput setaf 7)" +curl -H 'Cache-Control: no-cache, no-store' --fail --location --progress-bar "$URL" > "$app_bin/mini-langstream" +chmod +x $app_bin/* +echo "$(tput setaf 2)[OK]$(tput setaf 7) - Downloaded & Installed" + +function inject_if_not_found() { + local file=$1 + touch "$file" + if [[ -z $(grep 'mini-langstream/bin' "$file") ]]; then + echo -e "\n$init_snipped" >> "$file" + echo "$(tput setaf 2)[OK]$(tput setaf 7) - mini-langstream bin added to ${file}" + fi +} + + + + +bash_profile="${HOME}/.bash_profile" +bashrc="${HOME}/.bashrc" +zshrc="${ZDOTDIR:-${HOME}}/.zshrc" +init_snipped=$( cat << EOF +export PATH=\$PATH:$app_bin +EOF +) + +if [[ $darwin == true ]]; then + inject_if_not_found $bash_profile +else + inject_if_not_found $bashrc +fi + +if [[ -s "$zshrc" ]]; then + inject_if_not_found $zshrc +fi + +echo "$(tput setaf 2)[OK]$(tput setaf 7) - Installation Successful" +echo "Open $(tput setaf 2)a new terminal$(tput setaf 7) and run: $(tput setaf 3)mini-langstream start$(tput setaf 7)" +echo "" +echo "You can close this window." \ No newline at end of file diff --git a/bin/mini-langstream/mini-langstream b/bin/mini-langstream/mini-langstream new file mode 100755 index 000000000..3e3a479e6 --- /dev/null +++ b/bin/mini-langstream/mini-langstream @@ -0,0 +1,465 @@ +#!/bin/bash + +set -e +set -o errexit -o pipefail -o nounset +check_command() { + if ! command -v $1 &> /dev/null + then + echo "Command $1 could not be found. $2" + exit + fi +} + +check_command docker +check_command minikube +check_command helm +check_command kubectl +check_command langstream "Please install LangStream CLI using the instructions at https://github.com/LangStream/langstream#installation" + + +help() { + echo """ +mini-langstream, LangStream local cluster on Minikube. + +Syntax: $0 [options] + +Options: + start Start the LangStream local cluster. + delete Delete the LangStream local cluster and local data. + langstream|cli Run the LangStream CLI with the given command. + get-instance Get the path to the LangStream app instance YAML file. + get-config Get the path to the Kubernetes configuration file. + get-config Get the path to the Kubernetes configuration file. + get-values Get the path to the Helm values. + kubectl Run kubectl with the given command. + k9s Run K9s with the given command. + helm Run Helm with the given command. + help Print this help message. + dev Dev commands. It's required to run the command from the LangStream repository root directory. + start [--load] Start the LangStream local cluster in development mode. This will use the latest development images that are built locally. + --load Load LangStream images instead of building them. + load Load a specific Docker image into Minikube. + build Build docker image for a specific LangStream component and restart it. No arguments to rebuild all the images. + get-values Get the path to the Helm values. This values file is used when the cluster is started in dev mode. + +""" +} + + +# Constants +minikube_profile=mini-langstream +k8s_namespace=langstream +app_home=$HOME/.mini-langstream +data_dir=$app_home/data +conf_dir=$app_home/conf +kubeconfig_path=$data_dir/kube-config +cli_config=$data_dir/cli.yaml + +mkdir -p $app_home +mkdir -p $data_dir +touch $cli_config + + +cat > $conf_dir/app-instance.yaml < $conf_dir/dev-values.yaml < $conf_dir/values.yaml </dev/null + kubectl_cmd config set-context --current --namespace=$k8s_namespace >/dev/null + echo "✅" +} + +delete_minikube() { + echo -n "Deleting minikube: " + minikube_cmd delete --purge + echo "✅" +} + +helm_cmd() { + helm --kubeconfig $kubeconfig_path "$@" +} +kubectl_cmd() { + KUBECONFIG=$kubeconfig_path kubectl "$@" +} + +load_image_if_not_exists() { + local image=$1 + echo -n "Image $image: " + (minikube_cmd image list | grep -q $image) || load_image $image + echo "✅" +} +load_image() { + local image=$1 + minikube_cmd image load $image +} + +install_langstream() { + local dev="$1" + local load="$2" + helm_cmd repo add langstream https://datastax.github.io/langstream &> /dev/null || true + helm_cmd repo update &> /dev/null + if [ "$dev" == "true" ]; then + if [ "$load" == "false" ]; then + echo -n "Building images: " + build_no_restart + echo "✅" + else + load_image_if_not_exists langstream/langstream-control-plane:latest-dev + load_image_if_not_exists langstream/langstream-cli:latest-dev + load_image_if_not_exists langstream/langstream-deployer:latest-dev + load_image_if_not_exists langstream/langstream-runtime:latest-dev + load_image_if_not_exists langstream/langstream-api-gateway:latest-dev + fi + + echo -n "LangStream: " + helm_cmd upgrade --install langstream -n $k8s_namespace --create-namespace langstream/langstream --values $data_dir/dev-values.yaml > /dev/null + else + echo -n "LangStream: " + helm_cmd upgrade --install langstream langstream/langstream -n $k8s_namespace --create-namespace --values $data_dir/values.yaml > /dev/null + fi + echo "✅" + +} + +start_port_forward() { + local service=$1 + local file=$(mktemp) + + minikube_cmd service $1 --url -n $k8s_namespace &> $file & + grep -q 'http' <(tail -f $file) + url=$(grep -e http $file) + echo "$url" +} + +kafka_hostname=langstream-kafka + +install_kafka() { + if [ "$(docker ps -q -f name=$kafka_hostname)" ]; then + echo "Kafka: ✅" + return + fi + + if [ "$(docker ps -a -q -f name=$kafka_hostname)" ]; then + docker rm -f $kafka_hostname + fi + + + (docker run \ + -d \ + --name $kafka_hostname \ + -p 39092:39092 \ + -e KAFKA_LISTENERS=BROKER://0.0.0.0:19092,EXTERNAL://0.0.0.0:39092,CONTROLLER://0.0.0.0:9093 \ + -e KAFKA_ADVERTISED_LISTENERS=BROKER://localhost:19092,EXTERNAL://host.minikube.internal:39092 \ + -e KAFKA_INTER_BROKER_LISTENER_NAME=BROKER \ + -e KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER \ + -e KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,BROKER:PLAINTEXT,EXTERNAL:PLAINTEXT \ + -e KAFKA_PROCESS_ROLES='controller,broker' \ + -e KAFKA_NODE_ID=1 \ + -e KAFKA_CONTROLLER_QUORUM_VOTERS='1@0.0.0.0:9093' \ + -e KAFKA_LOG_DIRS='/tmp/kraft-combined-logs' \ + -e CLUSTER_ID=ciWo7IWazngRchmPES6q5A== \ + -e KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 \ + -v $data_dir/kafka:/var/lib/kafka/data \ + confluentinc/cp-kafka:7.5.0) > /dev/null + echo "Kafka: ✅" +} +delete_kafka() { + delete_docker_container $kafka_hostname + rm -rf $data_dir/kafka +} + +minio_hostname=langstream-minio + +install_minio() { + + if [ "$(docker ps -q -f name=$minio_hostname)" ]; then + echo "Minio: ✅" + return + fi + + if [ "$(docker ps -a -q -f name=$minio_hostname)" ]; then + docker rm -f $minio_hostname + fi + (docker run \ + -d \ + --name $minio_hostname \ + -p 9090:9090 \ + -p 9000:9000 \ + -v $data_dir/minio:/data \ + quay.io/minio/minio:latest \ + server /data --console-address :9090) > /dev/null + echo "Minio: ✅" +} +delete_minio() { + delete_docker_container $minio_hostname + rm -rf $data_dir/minio +} + +configure_cli() { + control_plane_url=$(start_port_forward langstream-control-plane) + api_gateway_url=$(start_port_forward langstream-api-gateway) + langstream -V + langstream profiles delete local-langstream-cluster &> /dev/null || true + langstream profiles create local-langstream-cluster --web-service-url $control_plane_url --api-gateway-url ${api_gateway_url/http/ws} --tenant default >/dev/null + echo "CLI: ✅" +} + +run_cli() { + langstream -p local-langstream-cluster "$@" +} +cleanup_docker_env() { + echo -n "Docker: " + docker system prune -f &> /dev/null + docker volume prune -f &> /dev/null + echo "✅" +} +start() { + local dev="false" + local load="false" + while [[ "$#" -gt 0 ]]; do + case $1 in + --dev) dev="true"; shift ;; + --load) load="true"; shift ;; + *) echo "Unknown parameter passed: $1"; exit 1 ;; + esac + done + cleanup_docker_env + start_minikube + install_minio + install_langstream "$dev" "$load" + install_kafka + configure_cli + echo "Ready 🚀" + echo "Deploy your first app with:" + echo "mini-langstream cli apps deploy app -i \$(mini-langstream get-instance)" -s https://raw.githubusercontent.com/LangStream/langstream/main/examples/secrets/secrets.yaml -app https://github.com/LangStream/langstream/tree/main/examples/applications/python-processor-exclamation + echo "mini-langstream cli apps get app" +} + +delete_docker_container() { + local container=$1 + echo -n "Deleting $container: " + docker rm -f $container &> /dev/null || true + echo "✅" +} + +delete() { + delete_kafka + delete_minikube + delete_minio +} + +handle_load() { + local image=$1 + if [[ $image == "control-plane" ]]; then + image=langstream/langstream-control-plane:latest-dev + elif [[ $image == "deployer" ]]; then + image=langstream/langstream-deployer:latest-dev + elif [[ $image == "api-gateway" ]]; then + image=langstream/langstream-api-gateway:latest-dev + elif [[ $image == "cli" ]]; then + image=langstream/langstream-cli:latest-dev + elif [[ $image == "runtime" ]]; then + image=langstream/langstream-runtime:latest-dev + fi + load_image $image + label="" + if [[ $image == *"langstream-control-plane"* ]]; then + label=langstream-control-plane + elif [[ $image == *"langstream-deployer"* ]]; then + label=langstream-deployer + elif [[ $image == *"langstream-api-gateway"* ]]; then + label=langstream-api-gateway + elif [[ $image == *"langstream-cli"* ]]; then + label=langstream-client + fi + if [ ! -z "$label" ]; then + kubectl_cmd -n $k8s_namespace delete pod -l app.kubernetes.io/name=$label + echo "Pod $label: ✅" + fi +} +build_no_restart() { + eval $(minikube_cmd docker-env) + ./docker/build.sh "$@" + eval $(minikube_cmd docker-env -u) +} + +handle_build() { + eval $(minikube_cmd docker-env) + ./docker/build.sh "$@" + eval $(minikube_cmd docker-env -u) + + docker_build_input=$1 + if [[ "$docker_build_input" == "control-plane" ]]; then + labels=(langstream-control-plane) + elif [[ "$docker_build_input" == "operator" || "$docker_build_input" == "deployer" ]]; then + labels=(langstream-deployer) + elif [[ "$docker_build_input" == "cli" ]]; then + labels=(langstream-client) + elif [[ "$docker_build_input" == "api-gateway" ]]; then + labels=(langstream-api-gateway) + elif [[ "$docker_build_input" == "runtime" ]]; then + labels=() + else + labels=(langstream-control-plane langstream-deployer langstream-api-gateway langstream-client) + fi + + if (( ${#labels[@]} )); then + for label in "${labels[@]}"; do + deploy_name=$(kubectl_cmd -n $k8s_namespace get deployment -l app.kubernetes.io/name=$label | awk '{print $1}' | grep langstream) + output=$(kubectl_cmd -n $k8s_namespace patch deployment $deploy_name -p "{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"name\":\"langstream\",\"image\":\"langstream/$label:latest-dev\"}]}}}}") + if echo "$output" | grep -q "(no change)"; then + kubectl_cmd -n $k8s_namespace delete pod -l app.kubernetes.io/name=$label > /dev/null + fi + echo "Pod $label: ✅" + done + fi +} + + +arg1=${1:-""} +if [ "$arg1" == "" ] || [ "$arg1" == "help" ]; then + help +elif [ "$arg1" == "start" ]; then + shift + start "$@" +elif [ "$arg1" == "delete" ]; then + shift + delete "$@" +elif [ "$arg1" == "langstream" ] || [ "$arg1" == "cli" ]; then + shift + run_cli "$@" +elif [ "$arg1" == "get-instance" ]; then + shift + echo "$(realpath $data_dir/app-instance.yaml)" +elif [ "$arg1" == "get-config" ]; then + shift + echo "$kubeconfig_path" +elif [ "$arg1" == "kubectl" ]; then + shift + kubectl_cmd "$@" +elif [ "$arg1" == "k9s" ]; then + shift + KUBECONFIG=$kubeconfig_path k9s "$@" +elif [ "$arg1" == "helm" ]; then + shift + helm_cmd "$@" +elif [ "$arg1" == "get-values" ]; then + shift + echo "$data_dir/values.yaml" +elif [ "$arg1" == "dev" ]; then + shift + devarg1=${1:-""} + if [ "$devarg1" == "start" ]; then + shift + start --dev "$@" + elif [ "$devarg1" == "load" ]; then + shift + handle_load "$@" + elif [ "$devarg1" == "build" ]; then + shift + handle_build "$@" + elif [ "$devarg1" == "get-values" ]; then + shift + echo "$data_dir/dev-values.yaml" + else + echo "Unknown command $arg1" + help + exit 1 + fi +else + echo "Unknown command $arg1" + help + exit 1 +fi From d7aca2447dc36085fc59725fb8c750af46d99374 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Thu, 21 Sep 2023 15:42:52 +0200 Subject: [PATCH 2/7] fixes --- bin/mini-langstream/mini-langstream | 31 ++++++++++++++++------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/bin/mini-langstream/mini-langstream b/bin/mini-langstream/mini-langstream index 3e3a479e6..efaa77339 100755 --- a/bin/mini-langstream/mini-langstream +++ b/bin/mini-langstream/mini-langstream @@ -21,16 +21,15 @@ help() { echo """ mini-langstream, LangStream local cluster on Minikube. -Syntax: $0 [options] +Syntax: $0 [command] [options] -Options: +Commands: start Start the LangStream local cluster. delete Delete the LangStream local cluster and local data. langstream|cli Run the LangStream CLI with the given command. get-instance Get the path to the LangStream app instance YAML file. get-config Get the path to the Kubernetes configuration file. - get-config Get the path to the Kubernetes configuration file. - get-values Get the path to the Helm values. + get-values Get the path to the Helm values. kubectl Run kubectl with the given command. k9s Run K9s with the given command. helm Run Helm with the given command. @@ -54,13 +53,17 @@ data_dir=$app_home/data conf_dir=$app_home/conf kubeconfig_path=$data_dir/kube-config cli_config=$data_dir/cli.yaml +app_instance_file=$conf_dir/app-instance.yaml +values_file=$conf_dir/values.yaml +dev_values_file=$conf_dir/dev-values.yaml mkdir -p $app_home mkdir -p $data_dir +mkdir -p $conf_dir touch $cli_config -cat > $conf_dir/app-instance.yaml < $app_instance_file < $conf_dir/dev-values.yaml < $dev_values_file < $conf_dir/values.yaml < $values_file < /dev/null + helm_cmd upgrade --install langstream -n $k8s_namespace --create-namespace langstream/langstream --values $dev_values_file > /dev/null else echo -n "LangStream: " - helm_cmd upgrade --install langstream langstream/langstream -n $k8s_namespace --create-namespace --values $data_dir/values.yaml > /dev/null + helm_cmd upgrade --install langstream langstream/langstream -n $k8s_namespace --create-namespace --values $values_file > /dev/null fi echo "✅" @@ -422,7 +425,7 @@ elif [ "$arg1" == "langstream" ] || [ "$arg1" == "cli" ]; then run_cli "$@" elif [ "$arg1" == "get-instance" ]; then shift - echo "$(realpath $data_dir/app-instance.yaml)" + echo "$(realpath $app_instance_file)" elif [ "$arg1" == "get-config" ]; then shift echo "$kubeconfig_path" @@ -437,7 +440,7 @@ elif [ "$arg1" == "helm" ]; then helm_cmd "$@" elif [ "$arg1" == "get-values" ]; then shift - echo "$data_dir/values.yaml" + echo "$(realpath $values_file)" elif [ "$arg1" == "dev" ]; then shift devarg1=${1:-""} @@ -452,7 +455,7 @@ elif [ "$arg1" == "dev" ]; then handle_build "$@" elif [ "$devarg1" == "get-values" ]; then shift - echo "$data_dir/dev-values.yaml" + echo "$(realpath $dev_values_file)" else echo "Unknown command $arg1" help From 9af3cf9aa7b7904eed1830dce2c083f3c778b1fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Thu, 21 Sep 2023 15:45:23 +0200 Subject: [PATCH 3/7] fix helo --- bin/mini-langstream/mini-langstream | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/mini-langstream/mini-langstream b/bin/mini-langstream/mini-langstream index efaa77339..04facfac5 100755 --- a/bin/mini-langstream/mini-langstream +++ b/bin/mini-langstream/mini-langstream @@ -37,8 +37,8 @@ Commands: dev Dev commands. It's required to run the command from the LangStream repository root directory. start [--load] Start the LangStream local cluster in development mode. This will use the latest development images that are built locally. --load Load LangStream images instead of building them. - load Load a specific Docker image into Minikube. - build Build docker image for a specific LangStream component and restart it. No arguments to rebuild all the images. + load Load a specific Docker image into Minikube. + build Build docker image for a specific LangStream component and restart it. No arguments to rebuild all the images. get-values Get the path to the Helm values. This values file is used when the cluster is started in dev mode. """ From 5a945ffc724e572a17e3ac24cf694dd92075ac23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Thu, 21 Sep 2023 15:46:34 +0200 Subject: [PATCH 4/7] fix helo --- bin/mini-langstream/mini-langstream | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/mini-langstream/mini-langstream b/bin/mini-langstream/mini-langstream index 04facfac5..015145eb8 100755 --- a/bin/mini-langstream/mini-langstream +++ b/bin/mini-langstream/mini-langstream @@ -21,7 +21,7 @@ help() { echo """ mini-langstream, LangStream local cluster on Minikube. -Syntax: $0 [command] [options] +Syntax: mini-langstream [command] [options] Commands: start Start the LangStream local cluster. From aaffc2753976b1899d1001abe6c8d3be96acdb70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Thu, 21 Sep 2023 16:05:03 +0200 Subject: [PATCH 5/7] create package --- .github/workflows/release.yml | 8 +- bin/mini-langstream/get-mini-langstream.sh | 122 ++++++++++++++++++--- bin/mini-langstream/package.sh | 16 +++ 3 files changed, 131 insertions(+), 15 deletions(-) create mode 100755 bin/mini-langstream/package.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 613224838..422266f64 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -56,9 +56,15 @@ jobs: tag_and_push langstream-control-plane tag_and_push langstream-api-gateway + - name: Package mini-langstream + run: | + version=${GITHUB_REF/refs\/tags\/v/} + ./bin/mini-langstream/package.sh $version + + - uses: ncipollo/release-action@v1 with: - artifacts: "langstream-cli/target/langstream-*.zip,helm/crds/*.yml" + artifacts: "langstream-cli/target/langstream-*.zip,helm/crds/*.yml,target/mini-langstream-*.zip" token: ${{ secrets.GITHUB_TOKEN }} generateReleaseNotes: true prerelease: false diff --git a/bin/mini-langstream/get-mini-langstream.sh b/bin/mini-langstream/get-mini-langstream.sh index 189696b68..fac8cc26f 100755 --- a/bin/mini-langstream/get-mini-langstream.sh +++ b/bin/mini-langstream/get-mini-langstream.sh @@ -1,4 +1,20 @@ #!/bin/bash +# +# +# Copyright DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# set -e @@ -20,14 +36,60 @@ trap echo_failed_command EXIT clear -echo "Installing $(tput setaf 6)mini-langstream$(tput setaf 7) please wait" +echo " _ ____ _ "; +echo " | | __ _ _ __ __ _/ ___|| |_ _ __ ___ __ _ _ __ ___ "; +echo " | | / _\` | '_ \ / _\` \___ \| __| '__/ _ \/ _\` | '_ \` _ \ "; +echo " | |__| (_| | | | | (_| |___) | |_| | | __/ (_| | | | | | |"; +echo " |_____\__,_|_| |_|\__, |____/ \__|_| \___|\__,_|_| |_| |_|"; +echo " |___/ "; + + +get_latest_release_tarball_url() { + if ! command -v jq > /dev/null; then + echo "Not found." + echo "======================================================================================================" + echo " Please install jq on your system using your favourite package manager." + echo "" + echo " Restart after installing jq." + echo "======================================================================================================" + echo " In alternative you can set a fixed mini-langstream version by setting MINILANGSTREAM_URL." + echo "======================================================================================================" + echo "" + exit 1 + fi + curl -Ss https://api.github.com/repos/LangStream/langstream/releases/latest | jq -r '.assets[] | select((.name | contains("mini-langstream"))) | .browser_download_url' +} + + +echo "Installing $(tput setaf 6)mini-langstream$(tput setaf 7) please wait... " +# Local installation +BIN_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +ROOT_DIR=$( cd -P "$( dirname "$BIN_DIR" )" >/dev/null 2>&1 && pwd ) +echo "" +echo "$(tput setaf 6)Checking archive:$(tput setaf 7)" +if [ -z "$MINILANGSTREAM_URL" ]; then + echo "$(tput setaf 2)MINILANGSTREAM_URL$(tput setaf 7) environment not set, checking for the latest release" + MINILANGSTREAM_URL=$(get_latest_release_tarball_url) + echo "$(tput setaf 2)[OK]$(tput setaf 7) - Using $MINILANGSTREAM_URL" +else + echo "$(tput setaf 2)[OK]$(tput setaf 7) - mini-langstream url set to $MINILANGSTREAM_URL (from MINILANGSTREAM_URL)" +fi +candidate_base_name=$(basename $MINILANGSTREAM_URL) -URL="https://raw.githubusercontent.com/LangStream/langstream/mini-langstream/bin/mini-langstream/mini-langstream?$(date +%s)" -app_home_dir=$HOME/.mini-langstream -mkdir -p $app_home_dir -app_bin=$app_home_dir/bin -mkdir -p $app_bin +langstream_root_dir="$HOME/.mini-langstream" +mkdir -p $langstream_root_dir +langstream_downloads_dir="$langstream_root_dir/downloads" +mkdir -p $langstream_downloads_dir +langstream_candidates_dir="$langstream_root_dir/candidates" +mkdir -p $langstream_candidates_dir + +langstream_current_symlink="$langstream_candidates_dir/current" +mkdir -p $langstream_candidates_dir + +downloaded_zip_path=$langstream_downloads_dir/$candidate_base_name +downloaded_extracted_dir=${MINILANGSTREAM_URL//\.zip} +downloaded_extracted_path="$langstream_candidates_dir/$(basename $downloaded_extracted_dir)" darwin=false case "$(uname)" in @@ -36,7 +98,19 @@ case "$(uname)" in ;; esac -echo "$(tput setaf 2)[OK]$(tput setaf 7) - Ready to install." +echo "$(tput setaf 2)[OK]$(tput setaf 7) - Ready to install $(basename $downloaded_extracted_dir)." + +if ! command -v unzip > /dev/null; then + echo "Not found." + echo "======================================================================================================" + echo " Please install unzip on your system using your favourite package manager." + echo "" + echo " Restart after installing unzip." + echo "======================================================================================================" + echo "" + exit 1 +fi +echo "$(tput setaf 2)[OK]$(tput setaf 7) - unzip command is available" if ! command -v curl > /dev/null; then echo "Not found." @@ -51,17 +125,37 @@ if ! command -v curl > /dev/null; then fi echo "$(tput setaf 2)[OK]$(tput setaf 7) - curl command is available" +echo "" +echo "$(tput setaf 6)Downloading archive:$(tput setaf 7)" +if [ -f "$downloaded_zip_path" ]; then + echo "$(tput setaf 2)[OK]$(tput setaf 7) - Archive is already there" +else + curl --fail --location --progress-bar "$MINILANGSTREAM_URL" > "$downloaded_zip_path" + echo "$(tput setaf 2)[OK]$(tput setaf 7) - File downloaded" +fi + +# check integrity +ARCHIVE_OK=$(unzip -qt "$downloaded_zip_path" | grep 'No errors detected in compressed data') +if [[ -z "$ARCHIVE_OK" ]]; then + echo "Downloaded zip archive corrupt. Are you connected to the internet?" + exit +fi +echo "$(tput setaf 2)[OK]$(tput setaf 7) - Integrity of the archive checked" echo "" -echo "$(tput setaf 6)Downloading:$(tput setaf 7)" -curl -H 'Cache-Control: no-cache, no-store' --fail --location --progress-bar "$URL" > "$app_bin/mini-langstream" -chmod +x $app_bin/* -echo "$(tput setaf 2)[OK]$(tput setaf 7) - Downloaded & Installed" +echo "$(tput setaf 6)Extracting and installation:$(tput setaf 7)" +unzip -qo "$downloaded_zip_path" -d "$langstream_candidates_dir" +echo "$(tput setaf 2)[OK]$(tput setaf 7) - Extraction is successful" + +rm -rf $langstream_current_symlink +ln -s $downloaded_extracted_path $langstream_current_symlink + +echo "$(tput setaf 2)[OK]$(tput setaf 7) - mini-langstream installed at $langstream_candidates_dir" function inject_if_not_found() { local file=$1 touch "$file" - if [[ -z $(grep 'mini-langstream/bin' "$file") ]]; then + if [[ -z $(grep 'mini-langstream/candidates' "$file") ]]; then echo -e "\n$init_snipped" >> "$file" echo "$(tput setaf 2)[OK]$(tput setaf 7) - mini-langstream bin added to ${file}" fi @@ -74,7 +168,7 @@ bash_profile="${HOME}/.bash_profile" bashrc="${HOME}/.bashrc" zshrc="${ZDOTDIR:-${HOME}}/.zshrc" init_snipped=$( cat << EOF -export PATH=\$PATH:$app_bin +export PATH=\$PATH:$langstream_current_symlink/bin EOF ) @@ -91,4 +185,4 @@ fi echo "$(tput setaf 2)[OK]$(tput setaf 7) - Installation Successful" echo "Open $(tput setaf 2)a new terminal$(tput setaf 7) and run: $(tput setaf 3)mini-langstream start$(tput setaf 7)" echo "" -echo "You can close this window." \ No newline at end of file +echo "You can close this window." diff --git a/bin/mini-langstream/package.sh b/bin/mini-langstream/package.sh new file mode 100755 index 000000000..f93e67364 --- /dev/null +++ b/bin/mini-langstream/package.sh @@ -0,0 +1,16 @@ +#!/bin/bash +version=$1 +if [ -z "$version" ]; then + echo "version is not set" + exit 1 +fi +temp_dir=$(mktemp -d) +mkdir -p $temp_dir/bin +cp bin/mini-langstream/mini-langstream $temp_dir/bin/mini-langstream + +mkdir -p target +target_filename=$(realpath target/mini-langstream-$version.zip) +rm -rf $target_filename +cd $temp_dir +zip -r $target_filename . +echo "Created $target_filename" \ No newline at end of file From 0ed9077520fc63845320452b05c4b04456137bcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Thu, 21 Sep 2023 16:11:37 +0200 Subject: [PATCH 6/7] move upper dir --- {bin/mini-langstream => mini-langstream}/README.md | 0 {bin/mini-langstream => mini-langstream}/get-mini-langstream.sh | 0 {bin/mini-langstream => mini-langstream}/mini-langstream | 0 {bin/mini-langstream => mini-langstream}/package.sh | 2 +- 4 files changed, 1 insertion(+), 1 deletion(-) rename {bin/mini-langstream => mini-langstream}/README.md (100%) rename {bin/mini-langstream => mini-langstream}/get-mini-langstream.sh (100%) rename {bin/mini-langstream => mini-langstream}/mini-langstream (100%) rename {bin/mini-langstream => mini-langstream}/package.sh (81%) diff --git a/bin/mini-langstream/README.md b/mini-langstream/README.md similarity index 100% rename from bin/mini-langstream/README.md rename to mini-langstream/README.md diff --git a/bin/mini-langstream/get-mini-langstream.sh b/mini-langstream/get-mini-langstream.sh similarity index 100% rename from bin/mini-langstream/get-mini-langstream.sh rename to mini-langstream/get-mini-langstream.sh diff --git a/bin/mini-langstream/mini-langstream b/mini-langstream/mini-langstream similarity index 100% rename from bin/mini-langstream/mini-langstream rename to mini-langstream/mini-langstream diff --git a/bin/mini-langstream/package.sh b/mini-langstream/package.sh similarity index 81% rename from bin/mini-langstream/package.sh rename to mini-langstream/package.sh index f93e67364..29e148570 100755 --- a/bin/mini-langstream/package.sh +++ b/mini-langstream/package.sh @@ -6,7 +6,7 @@ if [ -z "$version" ]; then fi temp_dir=$(mktemp -d) mkdir -p $temp_dir/bin -cp bin/mini-langstream/mini-langstream $temp_dir/bin/mini-langstream +cp mini-langstream/mini-langstream $temp_dir/bin/mini-langstream mkdir -p target target_filename=$(realpath target/mini-langstream-$version.zip) From b93a2929c8521bcb106ac87c7c3fcd41e364fcd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Thu, 21 Sep 2023 16:11:59 +0200 Subject: [PATCH 7/7] fix --- mini-langstream/package.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/mini-langstream/package.sh b/mini-langstream/package.sh index 29e148570..0f714bdcc 100755 --- a/mini-langstream/package.sh +++ b/mini-langstream/package.sh @@ -1,4 +1,20 @@ #!/bin/bash +# +# Copyright DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + version=$1 if [ -z "$version" ]; then echo "version is not set"