feed.yml

name: Vespa Sampleapps Search Feed
on:
  push:
    branches: [ master ]
    paths-ignore:
      - Gemfile.lock

env:
  DATA_PLANE_PUBLIC_KEY     : ${{ secrets.VESPA_TEAM_DATA_PLANE_PUBLIC_CERT }}
  DATA_PLANE_PRIVATE_KEY    : ${{ secrets.VESPA_TEAM_DATA_PLANE_PRIVATE_KEY }}
  VESPA_CLI_DATA_PLANE_CERT : ${{ secrets.VESPA_TEAM_VESPA_CLI_DATA_PLANE_CERT }}
  VESPA_CLI_DATA_PLANE_KEY  : ${{ secrets.VESPA_TEAM_VESPA_CLI_DATA_PLANE_KEY }}

jobs:
  build:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
    - uses: ruby/setup-ruby@v1
      with:
        ruby-version: 3.1
        bundler-cache: true

    - name: Generate feed-file (add empty frontmatter to README files first)
      run: |
        find . -not -path './_site/*' -not -path './test/*' -not -path './vendor/*' -name \*README.md | \
          while read f; do (echo -e "---\nrender_with_liquid: false\n---\n"; cat ${f})>${f}.new; mv ${f}.new ${f}; done
        bundle exec jekyll build -p _plugins-vespafeed

    - name: Setup Python
      uses: actions/setup-python@v4
      with:
        python-version: '3.x'

    - name: Install dependencies
      run: |
        pip3 install PyYAML mmh3 requests html5lib beautifulsoup4 markdownify tiktoken

    - name: Get Vespa CLI - update to later versions as needed
      run: |
        apt update && apt -y install curl
        mkdir -p opt
        VESPA_CLI_VERSION=$(curl -fsSL https://api.github.com/repos/vespa-engine/vespa/releases/latest | grep -Po '"tag_name": "v\K.*?(?=")')
        curl -fsSL https://github.com/vespa-engine/vespa/releases/download/v${VESPA_CLI_VERSION}/vespa-cli_${VESPA_CLI_VERSION}_linux_amd64.tar.gz | \
          tar -zxf - -C opt
        ln -fs ./opt/*/bin/vespa

    - name: Feed site
      run: |
        # The python scripts below uses the Vespa CLI for feeding / data access.
        # See https://docs.vespa.ai/en/vespa-cli.html.
        # The environment variables below have credentials for endpoint access -
        # use the key/cert files in .vespa and paste their content into GitHub Secrets.
        # VESPA_CLI_API_KEY must be set and empty as below.
        export VESPA_CLI_DATA_PLANE_CERT
        export VESPA_CLI_DATA_PLANE_KEY
        # export VESPA_CLI_API_KEY=
        ./feed_to_vespa.py _config.yml

    - name: Feed paragraphs site
      run: |
        export VESPA_CLI_DATA_PLANE_CERT
        export VESPA_CLI_DATA_PLANE_KEY
        export VESPA_CLI_API_KEY=
        ./feed-split.py vespaapps_index.json https://github.com/vespa-engine/sample-apps/tree/master questions.jsonl
        ./feed_to_vespa.py _paragraphs_config.yml

    - name: Feed suggestions
      run: |
        export VESPA_CLI_DATA_PLANE_CERT
        export VESPA_CLI_DATA_PLANE_KEY
        export VESPA_CLI_API_KEY=
        ./feed_to_vespa.py _suggestions_config.yml