misc: [+] literalCol value getter & git hooks (#288)

.git_hooks/README.md

@@ -0,0 +1,17 @@
+# Optional git hooks
+
+# Add one of these predefined hooks to your project hooks
+A symlink can be added to your repo:
+```
+ln -s .git_hooks/<SOURCE_FILE> <LINK_NAME_NO_EXTENSION>
+```
+> Example:
+> 
+> `ln -s .git_hooks/pre-push.sh .git/hooks/pre-push`
+
+Alternatively you can change your hooks' path config:
+```
+git config core.hooksPath .git_hooks
+```
+
+In general, the path may be absolute, or relative to the directory where the hooks are run (usually the working tree root; see DESCRIPTION section of [man githooks](https://git-scm.com/docs/githooks)).

.git_hooks/pre-commit.sh

@@ -0,0 +1,32 @@
+#!/bin/sh
+
+set -e
+
+cd "$(git rev-parse --git-dir)/.."
+
+#files=$(git diff --diff-filter=AM -STODO --name-only)
+#echo "$files"
+#if [ -n "$files" ]; then
+#    if grep -Hn TODO "${files}"; then
+#        echo "Blocking commit as TODO was found."
+#        exit 1
+#    fi
+#fi
+
+todos=$(git diff --staged --diff-filter=AM -U0 --no-prefix --pickaxe-regex -S"((//|(^|( )+\*)|#)( )*TODO)|\*( )*(@todo)" HEAD)
+
+echo "${todos}" | awk '
+  /^diff / {f="?"; next}
+  f=="?" {if (/^\+\+\+ /) f=$0"\n"; next}
+  /^@@/ {n=$3; sub(/,.*/,"",n); n=0+$3; next}
+  /^\+.*(\/\/( )*TODO|(^|( )+\*)( )*TODO|#( )*TODO|@todo)/ {print f n ":" substr($0,2); f=""}
+  substr($0,1,1)~/[ +]/ {n++}'
+
+if [ -n "${todos}" ]; then
+  echo ""
+  echo "You have staged TODOs. You have to:"
+  echo "  * Make sure you didn't add a TODO, staged it, removed it and forgot to stage the removal. This script only checks staged files"
+  echo "  * Fix them"
+  echo "  * Use --no-verify flag to avoid this check"
+  exit 1
+fi

.git_hooks/pre-push.sh

@@ -0,0 +1,40 @@
+
+CHECK_TODOS=true
+CHECK_FMT=false
+
+#############################
+#        CHECK TODOS        #
+#############################
+
+todos=$(git diff --diff-filter=AM -U0 --no-prefix --pickaxe-regex -S"((//|(^|( )+\*)|#)( )*TODO)|(@todo)" upstream/main HEAD)
+
+echo "${todos}" | awk '
+  /^diff / {f="?"; next}
+  f=="?" {if (/^\+\+\+ /) f=$0"\n"; next}
+  /^@@/ {n=$3; sub(/,.*/,"",n); n=0+$3; next}
+  /^\+.*(\/\/( )*TODO|(^|( )+\*)( )*TODO|#( )*TODO|@todo)/ {print f n ":" substr($0,2); f=""}
+  substr($0,1,1)~/[ +]/ {n++}'
+
+if [ ${CHECK_TODOS} = true ] && [ -n "${todos}" ]; then
+  echo ""
+  echo "You have TODOs. You have to:"
+  echo "  * Fix them"
+  echo "  * Use --no-verify flag to avoid this check"
+  exit 1
+fi
+
+#############################
+#       CHECK SCALAFMT      #
+#############################
+
+if [ ${CHECK_FMT} = true ]; then
+  echo "sbt scalafmtCheckAll"
+fi
+
+# Check & autoformat --> Disabled: I don't like not knowing the changes
+#if `sbt scalafmtCheckAll`; then
+#  exit 0
+#else
+#  sbt scalafmtAll
+#  exit 1
+#fi

.github/ISSUE_TEMPLATE/bug_report_form.yml

@@ -1,5 +1,5 @@
 name: 🐛 Bug Report form
-description: Report a bug using a form. Use this option by default
+description: Report a bug using a form. >> Use this option by default <<
 title: "[Bug report]: "
 labels: ["bug"]
 #assignees:

.github/workflows/ci.yml

@@ -29,9 +29,48 @@ jobs:
         uses: JoshuaTheMiller/[email protected]
         with:
           filter: '[]'
+  context_dump:
+    runs-on: ubuntu-latest
+    steps:
+#      - name: <DEBUG> Dump GitHub context
+#        env:
+#          GITHUB_CONTEXT: ${{ toJson(github) }}
+#        run: |
+#          echo "$GITHUB_CONTEXT"
+#        shell: bash
+
+      - name: PR json
+        shell: bash
+        run: |
+          echo "{
+            \"pr_number\": ${{ github.event.number }},
+            \"head\": {
+              \"branch\": \"${{ github.head_ref }}\",
+              \"sha\": \"${{ github.event.after }}\"
+            },
+            \"remote\": {
+              \"branch\": \"${{ github.base_ref }}\",
+              \"sha\": \"${{ github.event.pull_request.base.sha }}\"
+            },
+            \"html\": \"${{ github.event.pull_request._links.html.href }}\",
+            \"additions\": ${{ github.event.pull_request.additions }},
+            \"deletions\": ${{ github.event.pull_request.deletions }},
+            \"changed_files\": ${{ github.event.pull_request.changed_files }},
+            \"commits\": ${{ github.event.pull_request.commits }},
+            \"created_at\": \"${{ github.event.pull_request.created_at }}\",
+            \"is_draft\": ${{ github.event.pull_request.draft }}
+          }" > pr_context.json
+
+      - name: Upload PR context
+        uses: actions/upload-artifact@v3
+        with:
+          name: "pr_context.json"
+          path: "pr_context.json"
+          if-no-files-found: error
+          retention-days: 1
   build:
     #if: github.event.pull_request.draft == false
-    needs: ["matrix_prep"]
+    needs: ["matrix_prep", "context_dump"]
     strategy:
       fail-fast: false
       matrix: ${{fromJson(needs.matrix_prep.outputs.matrix)}}

.github/workflows/pr_comment.yml

@@ -11,35 +11,45 @@ on:
 
 jobs:
   create_test_summary_report:
+    if: ${{ github.event.workflow_run.conclusion == 'success' || github.event.workflow_run.conclusion == 'failure'}}
 #    if: github.repository == 'hablapps/doric'
     runs-on: ubuntu-latest
     name: Create testing summary comment
     steps:
-      - uses: actions/checkout@v3
-
-      - name: Find Current Pull Request
-        uses: jwalton/[email protected]
-        id: findPr
+#      - name: <DEBUG> Dump GitHub context
+#        env:
+#          GITHUB_CONTEXT: ${{ toJson(github) }}
+#        run: echo "$GITHUB_CONTEXT"
+      - name: Download PR context
+        uses: dawidd6/[email protected] # marcofaggian/[email protected]
         with:
-          state: open
-
-      - run: echo "Your PR is ${PR}"
-        if: success() && steps.findPr.outputs.number
-        env:
-          PR: ${{ steps.findPr.outputs.pr }}
+          name: pr_context.json
+          workflow_conclusion: ""
+          github_token: ${{secrets.GITHUB_TOKEN}}
+          workflow: ${{ github.event.workflow.id }}
+          commit: ${{ github.event.workflow_run.head_commit.id }}
+          repo: ${{github.repository}}
+          if_no_artifact_found: fail
 
-      - name: <DEBUG> Dump GitHub context
-        env:
-          GITHUB_CONTEXT: ${{ toJson(github) }}
-        run: echo "$GITHUB_CONTEXT"
+      - name: Output PR context
+        id: output_pr_context
+        shell: bash
+        run: |
+          content="$(cat pr_context.json)"
+          echo "${content}"
+          # the following lines are only required for multi line json
+          content="${content//'%'/'%25'}"
+          content="${content//$'\n'/'%0A'}"
+          content="${content//$'\r'/'%0D'}"
+          echo "::set-output name=pr_context::$(echo "${content}")"
 
       - name: Find Comment
 #        if: github.event_name == 'pull_request'
 #        if: ${{ github.event.workflow_run.conclusion == 'success/failure' }}
         uses: peter-evans/find-comment@v2
         id: fc
         with:
-          issue-number: ${{ github.event.workflow_run.pull_requests[0].number }}
+          issue-number: ${{ fromJson(steps.output_pr_context.outputs.pr_context).pr_number }}
           body-includes: "This is an auto-generated comment"
 
 #      - uses: actions/checkout@v3
@@ -142,9 +152,21 @@ jobs:
           body="${body//'%'/'%25'}"
           body="${body//$'\n'/'%0A'}"
           body="${body//$'\r'/'%0D'}"
-          echo "::set-output name=summary::$body"
+          echo "::set-output name=summary::$(echo "${body}")"
         shell: bash
 
+      - name: Output PR context
+        id: output_pr_context
+        shell: bash
+        run: |
+          content="$(cat pr_context.json/pr_context.json)"
+          echo "${content}"
+          # the following lines are only required for multi line json
+          content="${content//'%'/'%25'}"
+          content="${content//$'\n'/'%0A'}"
+          content="${content//$'\r'/'%0D'}"
+          echo "::set-output name=pr_context::$(echo "${content}")"
+
       - name: Find Comment
         if: ${{ always() }}
         uses: peter-evans/find-comment@v2
@@ -159,7 +181,7 @@ jobs:
         uses: peter-evans/create-or-update-comment@v2
         with:
           comment-id: ${{ steps.fc.outputs.comment-id }}
-          issue-number: ${{ github.event.workflow_run.pull_requests[0].number }}
+          issue-number: ${{ fromJson(steps.output_pr_context.outputs.pr_context).pr_number }}
           token: "${{ secrets.GITHUB_TOKEN }}"
           edit-mode: append
           body: |

build.sbt

@@ -4,7 +4,7 @@ import sbt.Compile
 
 val stableVersion = "0.0.5"
 
-val sparkDefaultShortVersion = "3.2"
+val sparkDefaultShortVersion = "3.3"
 val spark24Version           = "2.4.8"
 val spark30Version           = "3.0.3"
 val spark31Version           = "3.1.3"

core/src/main/scala/doric/syntax/LiteralConversions.scala

@@ -2,6 +2,7 @@ package doric
 package syntax
 
 import doric.sem.Location
+import doric.types.SparkType.Primitive
 import doric.types.{LiteralSparkType, SparkType}
 
 private[syntax] trait LiteralConversions {
@@ -37,4 +38,12 @@ private[syntax] trait LiteralConversions {
     )
   }
 
+  implicit class DoricColLiteralGetter[T: Primitive](dCol: DoricColumn[T]) {
+    @inline def getValueIfLiteral: Option[T] = dCol match {
+      case literal: LiteralDoricColumn[T] =>
+        Some(literal.columnValue)
+      case _ => None
+    }
+  }
+
 }

core/src/test/scala/doric/syntax/LiteralConversionsSpec.scala

@@ -0,0 +1,31 @@
+package doric
+package syntax
+
+class LiteralConversionsSpec extends DoricTestElements {
+
+  describe("Primitive doric columns") {
+
+    it("should extract the original value of a literal(str)") {
+      val literal = "myValue"
+      // This casting is necessary, otherwise it would be a LiteralDoricColumn and we could access to its value with another method
+      val litColumn: StringColumn = literal.lit
+
+      litColumn.getValueIfLiteral shouldBe Some(literal)
+    }
+
+    it("should extract the original value of a literal(int)") {
+      val literal = 123
+      // This casting is necessary, otherwise it would be a LiteralDoricColumn and we could access to its value with another method
+      val litColumn: IntegerColumn = literal.lit
+
+      litColumn.getValueIfLiteral shouldBe Some(literal)
+    }
+
+    it("should avoid an error if it is not a literal") {
+      val colStr = colString("myColumn")
+
+      colStr.getValueIfLiteral shouldBe None
+    }
+  }
+
+}