diff --git a/.git_hooks/README.md b/.git_hooks/README.md new file mode 100755 index 000000000..4fd933418 --- /dev/null +++ b/.git_hooks/README.md @@ -0,0 +1,17 @@ +# Optional git hooks + +# Add one of these predefined hooks to your project hooks +A symlink can be added to your repo: +``` +ln -s .git_hooks/ +``` +> Example: +> +> `ln -s .git_hooks/pre-push.sh .git/hooks/pre-push` + +Alternatively you can change your hooks' path config: +``` +git config core.hooksPath .git_hooks +``` + +In general, the path may be absolute, or relative to the directory where the hooks are run (usually the working tree root; see DESCRIPTION section of [man githooks](https://git-scm.com/docs/githooks)). diff --git a/.git_hooks/pre-commit.sh b/.git_hooks/pre-commit.sh new file mode 100755 index 000000000..c813ff460 --- /dev/null +++ b/.git_hooks/pre-commit.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +set -e + +cd "$(git rev-parse --git-dir)/.." + +#files=$(git diff --diff-filter=AM -STODO --name-only) +#echo "$files" +#if [ -n "$files" ]; then +# if grep -Hn TODO "${files}"; then +# echo "Blocking commit as TODO was found." +# exit 1 +# fi +#fi + +todos=$(git diff --staged --diff-filter=AM -U0 --no-prefix --pickaxe-regex -S"((//|(^|( )+\*)|#)( )*TODO)|\*( )*(@todo)" HEAD) + +echo "${todos}" | awk ' + /^diff / {f="?"; next} + f=="?" {if (/^\+\+\+ /) f=$0"\n"; next} + /^@@/ {n=$3; sub(/,.*/,"",n); n=0+$3; next} + /^\+.*(\/\/( )*TODO|(^|( )+\*)( )*TODO|#( )*TODO|@todo)/ {print f n ":" substr($0,2); f=""} + substr($0,1,1)~/[ +]/ {n++}' + +if [ -n "${todos}" ]; then + echo "" + echo "You have staged TODOs. You have to:" + echo " * Make sure you didn't add a TODO, staged it, removed it and forgot to stage the removal. This script only checks staged files" + echo " * Fix them" + echo " * Use --no-verify flag to avoid this check" + exit 1 +fi diff --git a/.git_hooks/pre-push.sh b/.git_hooks/pre-push.sh new file mode 100755 index 000000000..79fe936b4 --- /dev/null +++ b/.git_hooks/pre-push.sh @@ -0,0 +1,40 @@ + +CHECK_TODOS=true +CHECK_FMT=false + +############################# +# CHECK TODOS # +############################# + +todos=$(git diff --diff-filter=AM -U0 --no-prefix --pickaxe-regex -S"((//|(^|( )+\*)|#)( )*TODO)|(@todo)" upstream/main HEAD) + +echo "${todos}" | awk ' + /^diff / {f="?"; next} + f=="?" {if (/^\+\+\+ /) f=$0"\n"; next} + /^@@/ {n=$3; sub(/,.*/,"",n); n=0+$3; next} + /^\+.*(\/\/( )*TODO|(^|( )+\*)( )*TODO|#( )*TODO|@todo)/ {print f n ":" substr($0,2); f=""} + substr($0,1,1)~/[ +]/ {n++}' + +if [ ${CHECK_TODOS} = true ] && [ -n "${todos}" ]; then + echo "" + echo "You have TODOs. You have to:" + echo " * Fix them" + echo " * Use --no-verify flag to avoid this check" + exit 1 +fi + +############################# +# CHECK SCALAFMT # +############################# + +if [ ${CHECK_FMT} = true ]; then + echo "sbt scalafmtCheckAll" +fi + +# Check & autoformat --> Disabled: I don't like not knowing the changes +#if `sbt scalafmtCheckAll`; then +# exit 0 +#else +# sbt scalafmtAll +# exit 1 +#fi diff --git a/.github/ISSUE_TEMPLATE/bug_report_form.yml b/.github/ISSUE_TEMPLATE/bug_report_form.yml index 7471cccd3..bc33d2624 100644 --- a/.github/ISSUE_TEMPLATE/bug_report_form.yml +++ b/.github/ISSUE_TEMPLATE/bug_report_form.yml @@ -1,5 +1,5 @@ name: 🐛 Bug Report form -description: Report a bug using a form. Use this option by default +description: Report a bug using a form. >> Use this option by default << title: "[Bug report]: " labels: ["bug"] #assignees: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dbf0da16d..5b7f4da21 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,9 +29,48 @@ jobs: uses: JoshuaTheMiller/conditional-build-matrix@v1.0.1 with: filter: '[]' + context_dump: + runs-on: ubuntu-latest + steps: +# - name: Dump GitHub context +# env: +# GITHUB_CONTEXT: ${{ toJson(github) }} +# run: | +# echo "$GITHUB_CONTEXT" +# shell: bash + + - name: PR json + shell: bash + run: | + echo "{ + \"pr_number\": ${{ github.event.number }}, + \"head\": { + \"branch\": \"${{ github.head_ref }}\", + \"sha\": \"${{ github.event.after }}\" + }, + \"remote\": { + \"branch\": \"${{ github.base_ref }}\", + \"sha\": \"${{ github.event.pull_request.base.sha }}\" + }, + \"html\": \"${{ github.event.pull_request._links.html.href }}\", + \"additions\": ${{ github.event.pull_request.additions }}, + \"deletions\": ${{ github.event.pull_request.deletions }}, + \"changed_files\": ${{ github.event.pull_request.changed_files }}, + \"commits\": ${{ github.event.pull_request.commits }}, + \"created_at\": \"${{ github.event.pull_request.created_at }}\", + \"is_draft\": ${{ github.event.pull_request.draft }} + }" > pr_context.json + + - name: Upload PR context + uses: actions/upload-artifact@v3 + with: + name: "pr_context.json" + path: "pr_context.json" + if-no-files-found: error + retention-days: 1 build: #if: github.event.pull_request.draft == false - needs: ["matrix_prep"] + needs: ["matrix_prep", "context_dump"] strategy: fail-fast: false matrix: ${{fromJson(needs.matrix_prep.outputs.matrix)}} diff --git a/.github/workflows/pr_comment.yml b/.github/workflows/pr_comment.yml index 996a653e6..3c6744bed 100644 --- a/.github/workflows/pr_comment.yml +++ b/.github/workflows/pr_comment.yml @@ -11,27 +11,37 @@ on: jobs: create_test_summary_report: + if: ${{ github.event.workflow_run.conclusion == 'success' || github.event.workflow_run.conclusion == 'failure'}} # if: github.repository == 'hablapps/doric' runs-on: ubuntu-latest name: Create testing summary comment steps: - - uses: actions/checkout@v3 - - - name: Find Current Pull Request - uses: jwalton/gh-find-current-pr@v1.3.2 - id: findPr +# - name: Dump GitHub context +# env: +# GITHUB_CONTEXT: ${{ toJson(github) }} +# run: echo "$GITHUB_CONTEXT" + - name: Download PR context + uses: dawidd6/action-download-artifact@v2.23.0 # marcofaggian/action-download-multiple-artifacts@v3.0.8 with: - state: open - - - run: echo "Your PR is ${PR}" - if: success() && steps.findPr.outputs.number - env: - PR: ${{ steps.findPr.outputs.pr }} + name: pr_context.json + workflow_conclusion: "" + github_token: ${{secrets.GITHUB_TOKEN}} + workflow: ${{ github.event.workflow.id }} + commit: ${{ github.event.workflow_run.head_commit.id }} + repo: ${{github.repository}} + if_no_artifact_found: fail - - name: Dump GitHub context - env: - GITHUB_CONTEXT: ${{ toJson(github) }} - run: echo "$GITHUB_CONTEXT" + - name: Output PR context + id: output_pr_context + shell: bash + run: | + content="$(cat pr_context.json)" + echo "${content}" + # the following lines are only required for multi line json + content="${content//'%'/'%25'}" + content="${content//$'\n'/'%0A'}" + content="${content//$'\r'/'%0D'}" + echo "::set-output name=pr_context::$(echo "${content}")" - name: Find Comment # if: github.event_name == 'pull_request' @@ -39,7 +49,7 @@ jobs: uses: peter-evans/find-comment@v2 id: fc with: - issue-number: ${{ github.event.workflow_run.pull_requests[0].number }} + issue-number: ${{ fromJson(steps.output_pr_context.outputs.pr_context).pr_number }} body-includes: "This is an auto-generated comment" # - uses: actions/checkout@v3 @@ -142,9 +152,21 @@ jobs: body="${body//'%'/'%25'}" body="${body//$'\n'/'%0A'}" body="${body//$'\r'/'%0D'}" - echo "::set-output name=summary::$body" + echo "::set-output name=summary::$(echo "${body}")" shell: bash + - name: Output PR context + id: output_pr_context + shell: bash + run: | + content="$(cat pr_context.json/pr_context.json)" + echo "${content}" + # the following lines are only required for multi line json + content="${content//'%'/'%25'}" + content="${content//$'\n'/'%0A'}" + content="${content//$'\r'/'%0D'}" + echo "::set-output name=pr_context::$(echo "${content}")" + - name: Find Comment if: ${{ always() }} uses: peter-evans/find-comment@v2 @@ -159,7 +181,7 @@ jobs: uses: peter-evans/create-or-update-comment@v2 with: comment-id: ${{ steps.fc.outputs.comment-id }} - issue-number: ${{ github.event.workflow_run.pull_requests[0].number }} + issue-number: ${{ fromJson(steps.output_pr_context.outputs.pr_context).pr_number }} token: "${{ secrets.GITHUB_TOKEN }}" edit-mode: append body: | diff --git a/build.sbt b/build.sbt index 3187d4d32..0a9cb06d3 100644 --- a/build.sbt +++ b/build.sbt @@ -4,7 +4,7 @@ import sbt.Compile val stableVersion = "0.0.5" -val sparkDefaultShortVersion = "3.2" +val sparkDefaultShortVersion = "3.3" val spark24Version = "2.4.8" val spark30Version = "3.0.3" val spark31Version = "3.1.3" diff --git a/core/src/main/scala/doric/syntax/LiteralConversions.scala b/core/src/main/scala/doric/syntax/LiteralConversions.scala index 3c4d207b9..1908626ce 100644 --- a/core/src/main/scala/doric/syntax/LiteralConversions.scala +++ b/core/src/main/scala/doric/syntax/LiteralConversions.scala @@ -2,6 +2,7 @@ package doric package syntax import doric.sem.Location +import doric.types.SparkType.Primitive import doric.types.{LiteralSparkType, SparkType} private[syntax] trait LiteralConversions { @@ -37,4 +38,12 @@ private[syntax] trait LiteralConversions { ) } + implicit class DoricColLiteralGetter[T: Primitive](dCol: DoricColumn[T]) { + @inline def getValueIfLiteral: Option[T] = dCol match { + case literal: LiteralDoricColumn[T] => + Some(literal.columnValue) + case _ => None + } + } + } diff --git a/core/src/test/scala/doric/syntax/LiteralConversionsSpec.scala b/core/src/test/scala/doric/syntax/LiteralConversionsSpec.scala new file mode 100644 index 000000000..9ab8f937e --- /dev/null +++ b/core/src/test/scala/doric/syntax/LiteralConversionsSpec.scala @@ -0,0 +1,31 @@ +package doric +package syntax + +class LiteralConversionsSpec extends DoricTestElements { + + describe("Primitive doric columns") { + + it("should extract the original value of a literal(str)") { + val literal = "myValue" + // This casting is necessary, otherwise it would be a LiteralDoricColumn and we could access to its value with another method + val litColumn: StringColumn = literal.lit + + litColumn.getValueIfLiteral shouldBe Some(literal) + } + + it("should extract the original value of a literal(int)") { + val literal = 123 + // This casting is necessary, otherwise it would be a LiteralDoricColumn and we could access to its value with another method + val litColumn: IntegerColumn = literal.lit + + litColumn.getValueIfLiteral shouldBe Some(literal) + } + + it("should avoid an error if it is not a literal") { + val colStr = colString("myColumn") + + colStr.getValueIfLiteral shouldBe None + } + } + +}