Save scraper logs in csv

.github/workflows/bitcoinops.yml

@@ -25,6 +25,21 @@ jobs:
         run: |
           mkdir /tmp/data
           python bitcoinops/main.py
+
+      - name: Configure Git
+        run: |
+          git config user.email "${{ secrets.GIT_AUTHOR_EMAIL }}"
+          git config user.name "${{ secrets.GIT_AUTHOR_NAME }}"
+
+      - name: Add and commit changes
+        run: |
+          git add .
+          if git diff --staged --quiet; then
+            echo "No changes to commit"
+          else
+            git commit -m "bitcoinops: updated scraper logs"
+            git push
+          fi
         env:
             ES_ENGINE: ${{ secrets.ES_ENGINE }}
             ES_URL: ${{ secrets.ES_URL }}

.github/workflows/bitcointalk.yml

@@ -21,6 +21,21 @@ jobs:
         run: |
           mkdir /tmp/data
           python bitcointalk/main.py
+
+      - name: Configure Git
+        run: |
+          git config user.email "${{ secrets.GIT_AUTHOR_EMAIL }}"
+          git config user.name "${{ secrets.GIT_AUTHOR_NAME }}"
+
+      - name: Add and commit changes
+        run: |
+          git add .
+          if git diff --staged --quiet; then
+            echo "No changes to commit"
+          else
+            git commit -m "bitcointalk: updated scraper logs"
+            git push
+          fi
         env:
             ES_ENGINE: ${{ secrets.ES_ENGINE }}
             ES_URL: ${{ secrets.ES_URL }}

.github/workflows/bitcointranscripts.yml

@@ -21,6 +21,21 @@ jobs:
         run: |
           mkdir /tmp/data
           python bitcointranscripts/main.py
+
+      - name: Configure Git
+        run: |
+          git config user.email "${{ secrets.GIT_AUTHOR_EMAIL }}"
+          git config user.name "${{ secrets.GIT_AUTHOR_NAME }}"
+
+      - name: Add and commit changes
+        run: |
+          git add .
+          if git diff --staged --quiet; then
+            echo "No changes to commit"
+          else
+            git commit -m "bitcointranscripts: updated scraper logs"
+            git push
+          fi
         env:
             ES_ENGINE: ${{ secrets.ES_ENGINE }}
             ES_URL: ${{ secrets.ES_URL }}

.github/workflows/delving-bitcoin.yml

@@ -18,6 +18,21 @@ jobs:
       - name: Fetch data
         run: |
           python delvingbitcoin_2_elasticsearch/delvingbitcoin_2_elasticsearch.py
+
+      - name: Configure Git
+        run: |
+          git config user.email "${{ secrets.GIT_AUTHOR_EMAIL }}"
+          git config user.name "${{ secrets.GIT_AUTHOR_NAME }}"
+
+      - name: Add and commit changes
+        run: |
+          git add .
+          if git diff --staged --quiet; then
+            echo "No changes to commit"
+          else
+            git commit -m "delving-bitcoin: updated scraper logs"
+            git push
+          fi
         env:
             ES_ENGINE: ${{ secrets.ES_ENGINE }}
             ES_URL: ${{ secrets.ES_URL }}

.github/workflows/mailing-list-bitcoin-new.yml

@@ -18,6 +18,21 @@ jobs:
       - name: Fetch data
         run: |
           python mailing-list/main.py
+
+      - name: Configure Git
+        run: |
+          git config user.email "${{ secrets.GIT_AUTHOR_EMAIL }}"
+          git config user.name "${{ secrets.GIT_AUTHOR_NAME }}"
+
+      - name: Add and commit changes
+        run: |
+          git add .
+          if git diff --staged --quiet; then
+            echo "No changes to commit"
+          else
+            git commit -m "mailing-list-bitcoin-new: updated scraper logs"
+            git push
+          fi
         env:
             ES_ENGINE: ${{ secrets.ES_ENGINE }}
             ES_URL: ${{ secrets.ES_URL }}

.github/workflows/stackexchange.yml

@@ -6,25 +6,44 @@ on:
 jobs:
   fetch:
     runs-on: ubuntu-latest
+
+    env:
+        DATA_DIR: /tmp/stackexchange
+        ES_ENGINE: ${{ secrets.ES_ENGINE }}
+        ES_URL: ${{ secrets.ES_URL }}
+        ES_TOKEN: ${{ secrets.ES_TOKEN }}
+        CLOUD_ID: ${{ secrets.CLOUD_ID }}
+        USER_PASSWORD: ${{ secrets.USER_PASSWORD }}
+        USERNAME: ${{ secrets.USERNAME }}
+        INDEX: ${{ secrets.INDEX }}
+
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
         with:
           python-version: 3.9
+
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
           pip install -r requirements.txt
+
       - name: Fetch data
         run: |
           mkdir /tmp/stackexchange
           python bitcoin.stackexchange.com/main.py
-        env:
-            DATA_DIR: /tmp/stackexchange
-            ES_ENGINE: ${{ secrets.ES_ENGINE }}
-            ES_URL: ${{ secrets.ES_URL }}
-            ES_TOKEN: ${{ secrets.ES_TOKEN }}
-            CLOUD_ID: ${{ secrets.CLOUD_ID }}
-            USER_PASSWORD: ${{ secrets.USER_PASSWORD }}
-            USERNAME: ${{ secrets.USERNAME }}
-            INDEX: ${{ secrets.INDEX }}
+
+      - name: Configure Git
+        run: |
+          git config user.email "${{ secrets.GIT_AUTHOR_EMAIL }}"
+          git config user.name "${{ secrets.GIT_AUTHOR_NAME }}"
+
+      - name: Add and commit changes
+        run: |
+          git add .
+          if git diff --staged --quiet; then
+            echo "No changes to commit"
+          else
+            git commit -m "stackexchange: updated scraper logs"
+            git push
+          fi