scrape-epa #870
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: scrape-epa | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '17 4 * * *' # Run once a day at 4:17am | |
| jobs: | |
| scheduled: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: checkout repo content | |
| uses: actions/checkout@v3 # checkout the repository content to github runner | |
| - name: setup Bun | |
| uses: oven-sh/setup-bun@v1 | |
| - name: Bun install npm packages | |
| run: | | |
| bun install | |
| wget https://epa-rss.s3.eu-west-1.amazonaws.com/latest/epa-rss.sqlite -P sqlite | |
| - name: bun run TS code | |
| env: | |
| EPA_RSS_ACCESS_KEY_ID: ${{ secrets.AWS_KEY_ID }} | |
| EPA_RSS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY}} | |
| EPA_RSS_BUCKET: ${{ secrets.AWS_BUCKET }} | |
| run: | | |
| bun generate_bun.ts | |
| - name: commit files | |
| run: | | |
| git config --local user.email "[email protected]" | |
| git config --local user.name "GitHub Action" | |
| git add -A | |
| git commit --allow-empty -m "update EPS data" -a | |
| git push | |
| # - name: install and run feediverse for Mastodon | |
| # run: | | |
| # pip install feediverse | |
| # sed 's/MASTODON_ACCESS_TOKEN/'$MASTODON_ACCESS_TOKEN'/' .feediverse_template > /home/runner/.feediverse | |
| # sed -i 's/MASTODON_CLIENT_ID/'$MASTODON_CLIENT_ID'/' /home/runner/.feediverse | |
| # sed -i 's/MASTODON_CLIENT_SECRET/'$MASTODON_CLIENT_SECRET'/' /home/runner/.feediverse | |
| # /home/runner/.local/bin/feediverse --verbose -c /home/runner/.feediverse | |
| # rm /home/runner/.feediverse | |
| # shell: bash | |
| # env: | |
| # MASTODON_ACCESS_TOKEN: ${{secrets.MASTODON_ACCESS_TOKEN}} | |
| # MASTODON_CLIENT_ID: ${{secrets.MASTODON_CLIENT_ID}} | |
| # MASTODON_CLIENT_SECRET: ${{secrets.MASTODON_CLIENT_SECRET}} | |
| - name: upload sqlite3 to s3 | |
| uses: shallwefootball/s3-upload-action@master | |
| with: | |
| aws_key_id: ${{ secrets.AWS_KEY_ID }} | |
| aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY}} | |
| aws_bucket: ${{ secrets.AWS_BUCKET }} | |
| source_dir: 'sqlite' | |
| destination_dir: 'latest' | |
| - name: Get 2 days ago date | |
| id: date | |
| run: | | |
| echo "twodaysago=$(date --date="2 day ago" +"%Y-%m-%d")" >> $GITHUB_ENV | |
| - name: Create an issue | |
| run: | | |
| issueID="$(gh issue create --title "${{env.ISSUE_TITLE}}" --body "${{env.ISSUE_BODY}}" --repo $GITHUB_REPOSITORY)" | |
| echo 'ISSUE_ID='$issueID >> $GITHUB_ENV | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| ISSUE_TITLE: 'EPA Submissions for $twodaysago' | |
| ISSUE_BODY: 'If any submissions, they will be here: https://github.com/EPA-Ireland-Updates-Unofficial/epa-rss/blob/main/output/csv/daily/$twodaysago.csv' | |
| - name: Close that issue | |
| run: | | |
| gh issue close $ISSUE_ID | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| REPO: ${{ github.event.repository.name }} |