From 3a2343cb6fe13ecb8bc0225183d61aa74b445c2d Mon Sep 17 00:00:00 2001 From: lvliang-intel Date: Fri, 31 May 2024 23:12:29 +0800 Subject: [PATCH] Update HF token environment variable (#116) * Update TF token environment variable Signed-off-by: lvliang-intel * add token Signed-off-by: lvliang-intel --------- Signed-off-by: lvliang-intel --- .github/workflows/megaservice-test.yaml | 2 +- .github/workflows/microservice-test.yml | 2 +- comps/guardrails/README.md | 2 +- .../langchain/docker/docker_compose_guardrails.yaml | 3 +++ comps/llms/README.md | 6 +++--- comps/llms/summarization/tgi/docker_compose_llm.yaml | 4 +++- comps/llms/text-generation/ray_serve/README.md | 6 +++--- comps/llms/text-generation/tgi/docker_compose_llm.yaml | 2 +- comps/llms/text-generation/vllm/README.md | 6 +++--- comps/reranks/README.md | 6 +++--- .../reranks/langchain/docker/docker_compose_reranking.yaml | 2 +- tests/test_llms.sh | 6 +++--- tests/test_reranks.sh | 6 +++--- 13 files changed, 29 insertions(+), 24 deletions(-) diff --git a/.github/workflows/megaservice-test.yaml b/.github/workflows/megaservice-test.yaml index abe80a714..f6f2a14c4 100644 --- a/.github/workflows/megaservice-test.yaml +++ b/.github/workflows/megaservice-test.yaml @@ -63,7 +63,7 @@ jobs: # - name: Run Workflow # env: - # HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }} + # HF_TOKEN: ${{ secrets.HF_TOKEN }} # run: | # xxx diff --git a/.github/workflows/microservice-test.yml b/.github/workflows/microservice-test.yml index ad93e971a..25b87ac7c 100644 --- a/.github/workflows/microservice-test.yml +++ b/.github/workflows/microservice-test.yml @@ -62,7 +62,7 @@ jobs: - name: Run microservice test env: - HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }} + HF_TOKEN: ${{ secrets.HF_TOKEN }} service: ${{ matrix.service }} hardware: ${{ matrix.hardware }} run: | diff --git a/comps/guardrails/README.md b/comps/guardrails/README.md index a29cf53cb..6b0a72ab3 100644 --- a/comps/guardrails/README.md +++ b/comps/guardrails/README.md @@ -33,7 +33,7 @@ pip install -r requirements.txt ## 1.2 Start TGI Gaudi Service ```bash -export HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token} +export HF_TOKEN=${your_hf_api_token} export LANGCHAIN_TRACING_V2=true export LANGCHAIN_API_KEY=${your_langchain_api_key} export LANGCHAIN_PROJECT="opea/gaurdrails" diff --git a/comps/guardrails/langchain/docker/docker_compose_guardrails.yaml b/comps/guardrails/langchain/docker/docker_compose_guardrails.yaml index 6ea31bb56..c398c05e3 100644 --- a/comps/guardrails/langchain/docker/docker_compose_guardrails.yaml +++ b/comps/guardrails/langchain/docker/docker_compose_guardrails.yaml @@ -11,6 +11,8 @@ services: - "8088:80" volumes: - "./data:/data" + environment: + HF_TOKEN: ${HF_TOKEN} shm_size: 1g command: --model-id ${LLM_MODEL_ID} guardrails: @@ -24,6 +26,7 @@ services: https_proxy: ${https_proxy} SAFETY_GUARD_ENDPOINT: ${SAFETY_GUARD_ENDPOINT} LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY} + HUGGINGFACEHUB_API_TOKEN: ${HF_TOKEN} restart: unless-stopped networks: diff --git a/comps/llms/README.md b/comps/llms/README.md index e4475e12f..1892844b6 100644 --- a/comps/llms/README.md +++ b/comps/llms/README.md @@ -21,7 +21,7 @@ pip install -r requirements.txt ### 1.2.1 Start TGI Service ```bash -export HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token} +export HF_TOKEN=${your_hf_api_token} export LANGCHAIN_TRACING_V2=true export LANGCHAIN_API_KEY=${your_langchain_api_key} export LANGCHAIN_PROJECT="opea/gen-ai-comps:llms" @@ -115,7 +115,7 @@ If you start an LLM microservice with docker, the `docker_compose_llm.yaml` file In order to start TGI and LLM services, you need to setup the following environment variables first. ```bash -export HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token} +export HF_TOKEN=${your_hf_api_token} export TGI_LLM_ENDPOINT="http://${your_ip}:8008" export LLM_MODEL_ID=${your_hf_llm_model} export LANGCHAIN_TRACING_V2=true @@ -180,7 +180,7 @@ You can choose one as needed. ### 2.3.1 TGI ```bash -docker run -d --name="llm-tgi-server" -p 9000:9000 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e TGI_LLM_ENDPOINT=$TGI_LLM_ENDPOINT -e HUGGINGFACEHUB_API_TOKEN=$HUGGINGFACEHUB_API_TOKEN opea/llm-tgi:latest +docker run -d --name="llm-tgi-server" -p 9000:9000 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e TGI_LLM_ENDPOINT=$TGI_LLM_ENDPOINT -e HF_TOKEN=$HF_TOKEN opea/llm-tgi:latest ``` ### 2.3.2 vLLM diff --git a/comps/llms/summarization/tgi/docker_compose_llm.yaml b/comps/llms/summarization/tgi/docker_compose_llm.yaml index df8c433d5..085526f66 100644 --- a/comps/llms/summarization/tgi/docker_compose_llm.yaml +++ b/comps/llms/summarization/tgi/docker_compose_llm.yaml @@ -11,6 +11,8 @@ services: - "8008:80" volumes: - "./data:/data" + environment: + HF_TOKEN: ${HF_TOKEN} shm_size: 1g command: --model-id ${LLM_MODEL_ID} llm: @@ -23,7 +25,7 @@ services: http_proxy: ${http_proxy} https_proxy: ${https_proxy} TGI_LLM_ENDPOINT: ${TGI_LLM_ENDPOINT} - HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + HUGGINGFACEHUB_API_TOKEN: ${HF_TOKEN} LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY} restart: unless-stopped diff --git a/comps/llms/text-generation/ray_serve/README.md b/comps/llms/text-generation/ray_serve/README.md index e44f33ce2..71384aab6 100644 --- a/comps/llms/text-generation/ray_serve/README.md +++ b/comps/llms/text-generation/ray_serve/README.md @@ -10,12 +10,12 @@ bash ./launch_ray_service.sh ``` -For gated models such as `LLAMA-2`, you need set the environment variable `HUGGING_FACE_HUB_TOKEN=` to access the Hugging Face Hub. +For gated models such as `LLAMA-2`, you need set the environment variable `HF_TOKEN=` to access the Hugging Face Hub. -Please follow this link [huggingface token](https://huggingface.co/docs/hub/security-tokens) to get the access token and export `HUGGINGFACEHUB_API_TOKEN` environment with the token. +Please follow this link [huggingface token](https://huggingface.co/docs/hub/security-tokens) to get the access token and export `HF_TOKEN` environment with the token. ```bash -export HUGGINGFACEHUB_API_TOKEN= +export HF_TOKEN= ``` And then you can make requests with the OpenAI-compatible APIs like below to check the service status: diff --git a/comps/llms/text-generation/tgi/docker_compose_llm.yaml b/comps/llms/text-generation/tgi/docker_compose_llm.yaml index df8c433d5..a76433d66 100644 --- a/comps/llms/text-generation/tgi/docker_compose_llm.yaml +++ b/comps/llms/text-generation/tgi/docker_compose_llm.yaml @@ -23,7 +23,7 @@ services: http_proxy: ${http_proxy} https_proxy: ${https_proxy} TGI_LLM_ENDPOINT: ${TGI_LLM_ENDPOINT} - HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + HF_TOKEN: ${HF_TOKEN} LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY} restart: unless-stopped diff --git a/comps/llms/text-generation/vllm/README.md b/comps/llms/text-generation/vllm/README.md index de98e7521..af5343da3 100644 --- a/comps/llms/text-generation/vllm/README.md +++ b/comps/llms/text-generation/vllm/README.md @@ -12,12 +12,12 @@ bash ./serving/vllm/launch_vllm_service.sh ``` -For gated models such as `LLAMA-2`, you will have to pass -e HUGGING_FACE_HUB_TOKEN=\ to the docker run command above with a valid Hugging Face Hub read token. +For gated models such as `LLAMA-2`, you will have to pass -e HF_TOKEN=\ to the docker run command above with a valid Hugging Face Hub read token. -Please follow this link [huggingface token](https://huggingface.co/docs/hub/security-tokens) to get the access token and export `HUGGINGFACEHUB_API_TOKEN` environment with the token. +Please follow this link [huggingface token](https://huggingface.co/docs/hub/security-tokens) to get the access token and export `HF_TOKEN` environment with the token. ```bash -export HUGGINGFACEHUB_API_TOKEN= +export HF_TOKEN= ``` And then you can make requests like below to check the service status: diff --git a/comps/reranks/README.md b/comps/reranks/README.md index 46493ca72..f8f12251e 100644 --- a/comps/reranks/README.md +++ b/comps/reranks/README.md @@ -15,7 +15,7 @@ pip install -r requirements.txt ## 1.2 Start TEI Service ```bash -export HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token} +export HF_TOKEN=${your_hf_api_token} export LANGCHAIN_TRACING_V2=true export LANGCHAIN_API_KEY=${your_langchain_api_key} export LANGCHAIN_PROJECT="opea/reranks" @@ -48,7 +48,7 @@ If you start an Reranking microservice with docker, the `docker_compose_rerankin ## 2.1 Setup Environment Variables ```bash -export HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token} +export HF_TOKEN=${your_hf_api_token} export LANGCHAIN_TRACING_V2=true export LANGCHAIN_API_KEY=${your_langchain_api_key} export LANGCHAIN_PROJECT="opea/reranks" @@ -72,7 +72,7 @@ You can choose one as needed. ## 2.3 Run Docker with CLI (Option A) ```bash -docker run -d --name="reranking-tei-server" -p 8000:8000 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e TEI_RERANKING_ENDPOINT=$TEI_RERANKING_ENDPOINT -e HUGGINGFACEHUB_API_TOKEN=$HUGGINGFACEHUB_API_TOKEN -e LANGCHAIN_API_KEY=$LANGCHAIN_API_KEY opea/reranking-tei:latest +docker run -d --name="reranking-tei-server" -p 8000:8000 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e TEI_RERANKING_ENDPOINT=$TEI_RERANKING_ENDPOINT -e HF_TOKEN=$HF_TOKEN -e LANGCHAIN_API_KEY=$LANGCHAIN_API_KEY opea/reranking-tei:latest ``` ## 2.4 Run Docker with Docker Compose (Option B) diff --git a/comps/reranks/langchain/docker/docker_compose_reranking.yaml b/comps/reranks/langchain/docker/docker_compose_reranking.yaml index c42c43f4d..b0929d594 100644 --- a/comps/reranks/langchain/docker/docker_compose_reranking.yaml +++ b/comps/reranks/langchain/docker/docker_compose_reranking.yaml @@ -23,7 +23,7 @@ services: http_proxy: ${http_proxy} https_proxy: ${https_proxy} TEI_RERANKING_ENDPOINT: ${TEI_RERANKING_ENDPOINT} - HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN} + HF_TOKEN: ${HF_TOKEN} LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY} restart: unless-stopped diff --git a/tests/test_llms.sh b/tests/test_llms.sh index 0a29932f7..ffbcdc657 100644 --- a/tests/test_llms.sh +++ b/tests/test_llms.sh @@ -15,14 +15,14 @@ function build_docker_images() { function start_service() { tgi_endpoint_port=5004 export your_hf_llm_model="Intel/neural-chat-7b-v3-3" - # Remember to set HUGGINGFACEHUB_API_TOKEN before invoking this test! - export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} + # Remember to set HF_TOKEN before invoking this test! + export HF_TOKEN=${HF_TOKEN} docker run -d --name="test-comps-llm-tgi-endpoint" -p $tgi_endpoint_port:80 -v ./data:/data --shm-size 1g ghcr.io/huggingface/text-generation-inference:1.4 --model-id ${your_hf_llm_model} export TGI_LLM_ENDPOINT="http://${ip_address}:${tgi_endpoint_port}" tei_service_port=5005 unset http_proxy - docker run -d --name="test-comps-llm-tgi-server" -p ${tei_service_port}:9000 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e TGI_LLM_ENDPOINT=$TGI_LLM_ENDPOINT -e HUGGINGFACEHUB_API_TOKEN=$HUGGINGFACEHUB_API_TOKEN opea/llm-tgi:comps + docker run -d --name="test-comps-llm-tgi-server" -p ${tei_service_port}:9000 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e TGI_LLM_ENDPOINT=$TGI_LLM_ENDPOINT -e HF_TOKEN=$HF_TOKEN opea/llm-tgi:comps # check whether tgi is fully ready n=0 diff --git a/tests/test_reranks.sh b/tests/test_reranks.sh index fd8ff7fb9..9fdf58725 100644 --- a/tests/test_reranks.sh +++ b/tests/test_reranks.sh @@ -13,8 +13,8 @@ function build_docker_images() { function start_service() { tei_endpoint=5006 - # Remember to set HUGGINGFACEHUB_API_TOKEN before invoking this test! - export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} + # Remember to set HF_TOKEN before invoking this test! + export HF_TOKEN=${HF_TOKEN} model=BAAI/bge-reranker-large revision=refs/pr/4 volume=$PWD/data @@ -23,7 +23,7 @@ function start_service() { export TEI_RERANKING_ENDPOINT="http://${ip_address}:${tei_endpoint}" tei_service_port=5007 unset http_proxy - docker run -d --name="test-comps-reranking-tei-server" -p ${tei_service_port}:8000 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e TEI_RERANKING_ENDPOINT=$TEI_RERANKING_ENDPOINT -e HUGGINGFACEHUB_API_TOKEN=$HUGGINGFACEHUB_API_TOKEN opea/reranking-tei:comps + docker run -d --name="test-comps-reranking-tei-server" -p ${tei_service_port}:8000 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e TEI_RERANKING_ENDPOINT=$TEI_RERANKING_ENDPOINT -e HF_TOKEN=$HF_TOKEN opea/reranking-tei:comps sleep 1m }