.gitlab-ci.yml

workflow:
  rules:
    - if: '$CI_PIPELINE_SOURCE=="schedule" && $CI_COMMIT_REF_NAME=="master"' ## nightly scheduled pipeline at 4:15 UTC
    - if: '$CI_PIPELINE_SOURCE=="web"'                                       ## manually started from web UI
    - if: '$CI_PIPELINE_SOURCE=="push" && $CI_COMMIT_REF_NAME!="master"'     ## branches pushed to GL directly, mirror is set for master branch only

variables:
  CRAN_MIRROR: "https://cloud.r-project.org"
  _R_CHECK_FORCE_SUGGESTS_: "false"
  _R_CHECK_NO_STOP_ON_TEST_ERROR_: "true"
  _R_CHECK_SYSTEM_CLOCK_: "false"  ## https://stackoverflow.com/questions/63613301/r-cmd-check-note-unable-to-verify-current-time
  _R_CHECK_TESTS_NLINES_: "0"
  TZ: "UTC"  ## to avoid 'Failed to create bus connection' from timedatectl via Sys.timezone() on Docker with R 3.4.
             ## Setting TZ for all GLCI jobs to isolate them from timezone. We could have a new GLCI job to test under
             ## a non-UTC timezone, although, that's what we do routinely in dev.
  R_REL_VERSION:  "4.4" # only raise when RTOOLS for REL is available
  R_REL_WIN_BIN:  "https://cloud.r-project.org/bin/windows/base/old/4.4.1/R-4.4.1-win.exe"
  R_DEV_VERSION:  "4.5"
  R_DEV_WIN_BIN:  "https://cloud.r-project.org/bin/windows/base/R-devel-win.exe"
  R_OLD_VERSION:  "4.3"
  R_OLD_WIN_BIN:  "https://cloud.r-project.org/bin/windows/base/old/4.3.3/R-4.3.3-win.exe"  
  R_REL_MAC_BIN:  "https://cloud.r-project.org/bin/macosx/big-sur-arm64/base/R-4.4.1-arm64.pkg"
  R_OLD_MAC_BIN:  "https://cloud.r-project.org/bin/macosx/big-sur-arm64/base/R-4.3.3-arm64.pkg"

stages:
  - dependencies
  - build
  - test
  - integration
  - deploy

.artifacts-template: &artifacts
  artifacts:
    expire_in: 2 weeks
    when: always
    paths:
      - bus/$CI_JOB_NAME

## mirror packages
# download all recursive dependencies once to be used across multiple test jobs
# sources and binaries for r-release, r-devel and r-oldrel
# cache between runs
mirror-packages:
  stage: dependencies
  tags:
    - saas-linux-medium-amd64
  image: registry.gitlab.com/jangorecki/dockerfiles/r-base-minimal
  cache:
    paths:
      - bus/$CI_JOB_NAME/cran
  script:
    - echo 'source(".ci/ci.R")' >> .Rprofile
    - mkdir -p bus/$CI_JOB_NAME/cran/src/contrib
    - Rscript -e 'mirror.packages(dcf.dependencies("DESCRIPTION", "all"), repos=Sys.getenv("CRAN_MIRROR"), repodir="bus/mirror-packages/cran")'
    - Rscript -e 'sapply(simplify=FALSE, setNames(nm=Sys.getenv(c("R_REL_VERSION","R_DEV_VERSION","R_OLD_VERSION"))), function(binary.ver) mirror.packages(type="win.binary", dcf.dependencies("DESCRIPTION", "all"), repos=Sys.getenv("CRAN_MIRROR"), repodir="bus/mirror-packages/cran", binary.ver=binary.ver))'
    - Rscript -e 'sapply(simplify=FALSE, setNames(nm=Sys.getenv(c("R_REL_VERSION","R_OLD_VERSION"))), function(binary.ver) mirror.packages(type="mac.binary.big-sur-arm64", dcf.dependencies("DESCRIPTION", "all"), repos=Sys.getenv("CRAN_MIRROR"), repodir="bus/mirror-packages/cran", binary.ver=binary.ver))'
  <<: *artifacts

## install deps alias
.test-install-deps: &install-deps
  - Rscript -e 'source(".ci/ci.R"); install.packages(dcf.dependencies("DESCRIPTION", which="all"), repos=file.path("file:", normalizePath("bus/mirror-packages/cran", mustWork=FALSE)), quiet=TRUE)'

## build
# sources as tar.gz archive
# build vignettes
build:
  stage: build
  tags:
    - saas-linux-medium-amd64
  image: registry.gitlab.com/jangorecki/dockerfiles/r-base-gcc
  needs: ["mirror-packages"]
  before_script:
    - *install-deps
    - rm -r bus
  script:
    - sed -i '/^[[:space:]]*$/d' ./DESCRIPTION ## make last line end abruptly; i.e. without a final \n
    - echo "Revision:" $CI_COMMIT_SHA >> ./DESCRIPTION
    - R CMD build .
    - mkdir -p bus/$CI_JOB_NAME/
    - mv $(ls -1t data.table_*.tar.gz | head -n 1) bus/$CI_JOB_NAME/
  <<: *artifacts

.test-template: &test
  stage: test
  needs: ["mirror-packages","build"]
  allow_failure: true
  <<: *artifacts

.test-lin-template: &test-lin
  <<: *test
  tags:
    - saas-linux-medium-amd64
  before_script:
    - cp $(ls -1t bus/build/data.table_*.tar.gz | head -n 1) .
    - mkdir -p ~/.R
  after_script:
    - mkdir -p bus/$CI_JOB_NAME
    - echo $CI_JOB_ID > bus/$CI_JOB_NAME/id
    - echo $CI_JOB_STATUS > bus/$CI_JOB_NAME/status
    - echo $CI_JOB_IMAGE > bus/$CI_JOB_NAME/image
    - '[ -d data.table.Rcheck ] && mv data.table.Rcheck bus/$CI_JOB_NAME/'

## most comprehensive tests
# force all suggests
# flags: gcc -O3 -flto=auto -fno-common -Wunused-result
# tests for compilation warnings
test-lin-rel:
  <<: *test-lin
  image: registry.gitlab.com/jangorecki/dockerfiles/r-data.table
  variables:
    _R_CHECK_COMPILATION_FLAGS_KNOWN_: "-Wvla"
    _R_CHECK_CRAN_INCOMING_: "FALSE"
    _R_CHECK_CRAN_INCOMING_REMOTE_: "FALSE"
    _R_CHECK_FORCE_SUGGESTS_: "TRUE"
    OPENBLAS_MAIN_FREE: "1"
  script:
    - *install-deps
    - echo 'CFLAGS=-g -O3 -flto=auto -fno-common -fopenmp -Wall -Wvla -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' > ~/.R/Makevars
    - echo 'CXXFLAGS=-g -O3 -flto=auto -fno-common -fopenmp -Wall -Wvla -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' >> ~/.R/Makevars
    - R CMD check $(ls -1t data.table_*.tar.gz | head -n 1)
    - (! grep "warning:" data.table.Rcheck/00install.out)

## vanilla minimal
# no zlib
# no suggested deps
# no vignettes or manuals
# no openmp
# flags: gcc -O0 -fno-openmp
test-lin-rel-vanilla:
  <<: *test-lin
  image: registry.gitlab.com/jangorecki/dockerfiles/r-base-gcc
  variables:
    _R_CHECK_COMPILATION_FLAGS_KNOWN_: "-Wvla"
  script:
    - echo 'CFLAGS=-g -O0 -fno-openmp -Wall -Wvla -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' > ~/.R/Makevars
    - echo 'CXXFLAGS=-g -O0 -fno-openmp -Wall -Wvla -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' >> ~/.R/Makevars
    - R CMD check --no-manual --ignore-vignettes $(ls -1t data.table_*.tar.gz | head -n 1)

## R-release on Linux
# strict checks for 0 NOTEs
# extra NOTEs check and build pdf manual thus not from cran-lin template
test-lin-rel-cran:
  <<: *test-lin
  image: registry.gitlab.com/jangorecki/dockerfiles/r-base
  variables:
    _R_CHECK_CRAN_INCOMING_: "TRUE"           ## stricter --as-cran checks should run in dev pipelines continuously (not sure what they are though)
    _R_CHECK_CRAN_INCOMING_REMOTE_: "FALSE"   ## Other than no URL checking (takes many minutes) or 'Days since last update 0' NOTEs needed, #3284
    _R_CHECK_CRAN_INCOMING_TARBALL_THRESHOLD_: "7500000" ## bytes
    _R_CHECK_PKG_SIZES_THRESHOLD_: "10"        ## MB 'checking installed package size' NOTE increased due to po
  script:
    - *install-deps
    - echo 'CFLAGS=-g -O2 -fopenmp -Wall -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' > ~/.R/Makevars
    - echo 'CXXFLAGS=-g -O2 -fopenmp -Wall -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' >> ~/.R/Makevars
    - R CMD check --as-cran $(ls -1t data.table_*.tar.gz | head -n 1)
    - >-
        Rscript -e 'l=tail(readLines("data.table.Rcheck/00check.log"), 1L); if (!identical(l, "Status: OK")) stop("Last line of ", shQuote("00check.log"), " is not ", shQuote("Status: OK"), " but ", shQuote(l)) else q("no")'

## R-devel on Linux gcc strict
# R built with --enable-strict-barrier --disable-long-double
# tests for compilation warnings
# tests for new notes
test-lin-dev-gcc-strict-cran:
  <<: *test-lin
  image: registry.gitlab.com/jangorecki/dockerfiles/r-devel-gcc-strict
  variables:
    _R_CHECK_COMPILATION_FLAGS_KNOWN_: "-Wvla"
    _R_CHECK_CRAN_INCOMING_: "TRUE"
    _R_CHECK_CRAN_INCOMING_REMOTE_: "FALSE"
    _R_S3_METHOD_LOOKUP_BASEENV_AFTER_GLOBALENV_: "FALSE" ## detects S3 method lookup found on search path #4777
    _R_S3_METHOD_LOOKUP_REPORT_SEARCH_PATH_USES_: "TRUE"
  script:
    - echo 'CFLAGS=-g -O2 -flto=auto -fno-common -fopenmp -Wall -Wvla -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' > ~/.R/Makevars
    - echo 'CXXFLAGS=-g -O2 -flto=auto -fno-common -fopenmp -Wall -Wvla -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' >> ~/.R/Makevars
    - *install-deps
    - R CMD check --as-cran $(ls -1t data.table_*.tar.gz | head -n 1)
    - (! grep "warning:" data.table.Rcheck/00install.out)
    - >-
        Rscript -e 'l=tail(readLines("data.table.Rcheck/00check.log"), 1L); notes<-"Status: 3 NOTEs"; if (!identical(l, notes)) stop("Last line of ", shQuote("00check.log"), " is not ", shQuote(notes), " (size of tarball, installed package size, non-API calls) but ", shQuote(l)) else q("no")'

## R-devel on Linux clang
# R compiled with clang, flags removed: -flto=auto -fopenmp
# tests for compilation warnings
# tests for new notes
test-lin-dev-clang-cran:
  <<: *test-lin
  image: registry.gitlab.com/jangorecki/dockerfiles/r-devel-clang
  variables:
    _R_CHECK_COMPILATION_FLAGS_KNOWN_: "-Wvla"
    _R_CHECK_CRAN_INCOMING_: "TRUE"
    _R_CHECK_CRAN_INCOMING_REMOTE_: "FALSE"
    _R_S3_METHOD_LOOKUP_BASEENV_AFTER_GLOBALENV_: "FALSE"
    _R_S3_METHOD_LOOKUP_REPORT_SEARCH_PATH_USES_: "TRUE"
  script:
    - echo 'CFLAGS=-g -O2 -fno-common -Wall -Wvla -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' > ~/.R/Makevars
    - echo 'CXXFLAGS=-g -O2 -fno-common -Wall -Wvla -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' >> ~/.R/Makevars
    - *install-deps
    - R CMD check --as-cran $(ls -1t data.table_*.tar.gz | head -n 1)
    - (! grep "warning:" data.table.Rcheck/00install.out)
    - >-
        Rscript -e 'l=tail(readLines("data.table.Rcheck/00check.log"), 1L); notes<-"Status: 2 NOTEs"; if (!identical(l, notes)) stop("Last line of ", shQuote("00check.log"), " is not ", shQuote(notes), " (size of tarball, non-API calls) but ", shQuote(l)) else q("no")'

# stated dependency on R
test-lin-ancient-cran:
  <<: *test-lin
  image: registry.gitlab.com/jangorecki/dockerfiles/r-3.3.0
  variables:
    _R_CHECK_FORCE_SUGGESTS_: "FALSE" # can be removed if all dependencies are available (knitr, xts, etc.)
  script:
    - *install-deps
    # knitr requires evaluate, which requires R 3.6.0.
    # Restore checking vignettes if upgrading our R dependency means knitr can be installed.
    - R CMD check --no-manual --no-build-vignettes --ignore-vignettes $(ls -1t data.table_*.tar.gz | head -n 1)

.test-win-template: &test-win
  <<: *test
  tags:
    - saas-windows-medium-amd64
  before_script:
    - Invoke-WebRequest -Uri $R_BIN -OutFile ../R-win.exe -ErrorAction Stop; if (!(Test-Path -Path ..\R-win.exe)) {Write-Error "R-win.exe not found, download failed?"}
    - Start-Process -FilePath ..\R-win.exe -ArgumentList "/VERYSILENT /DIR=C:\R" -NoNewWindow -Wait
    - $VERSION = $RTOOLS_VERSION -replace '\.', ''
    # persist RTOOLSXX_HOME="/c/rtools" which is needed for finding gcc #6552
    - Add-Content -Path ".\.Renviron" -Value "RTOOLS${VERSION}_HOME='/c/rtools'"
    - $RTOOLS_BIN="https://github.com/r-hub/rtools${VERSION}/releases/download/latest/rtools${VERSION}.exe"
    - Invoke-WebRequest -Uri $RTOOLS_BIN -OutFile ../rtools.exe -ErrorAction Stop; if (!(Test-Path -Path ..\rtools.exe)) {Write-Error "rtools.exe not found, download failed?"}
    - Start-Process -FilePath ..\rtools.exe -ArgumentList "/VERYSILENT /DIR=C:\rtools" -NoNewWindow -Wait
    - $env:PATH = "C:\R\bin;C:\rtools\usr\bin;$env:PATH"
    - Rscript.exe -e "source('.ci/ci.R'); install.packages(dcf.dependencies('DESCRIPTION', which='all'), repos=file.path('file://',getwd(),'bus/mirror-packages/cran'), quiet=TRUE)"
    - cp.exe $(ls.exe -1t bus/build/data.table_*.tar.gz | head.exe -n 1) .
  script:
    - R.exe CMD check --no-manual $(ls.exe -1t data.table_*.tar.gz | head.exe -n 1)
    - R.exe CMD INSTALL --build $(ls.exe -1t data.table_*.tar.gz | head.exe -n 1)
    - -not (grep.exe "warning:" data.table.Rcheck\00install.out)
  after_script:
    - $env:PATH = "C:\R\bin;C:\rtools\usr\bin;$env:PATH"
    - mkdir.exe -p bus/$CI_JOB_NAME
    - Rscript.exe -e "cat(Sys.getenv('CI_JOB_ID'), file=file.path('bus', Sys.getenv('CI_JOB_NAME'), 'id'))"
    - Rscript.exe -e "cat(Sys.getenv('CI_JOB_STATUS'), file=file.path('bus', Sys.getenv('CI_JOB_NAME'), 'status'))"
    - Rscript.exe -e "cat(Sys.getenv('CI_JOB_IMAGE'), file=file.path('bus', Sys.getenv('CI_JOB_NAME'), 'image'))"
    - Rscript.exe -e "to<-file.path('bus', Sys.getenv('CI_JOB_NAME'), 'data.table.Rcheck'); if (dir.exists(from<-'data.table.Rcheck')) invisible(file.rename(from, to)); dir.exists(to)"
    - Rscript.exe -e "from<-tail(list.files(pattern='^data\\.table_.*\\.zip$'), 1L); to<-file.path('bus', Sys.getenv('CI_JOB_NAME'), from); if (length(from)) invisible(file.rename(from, to)); length(to)&&file.exists(to)"

## R-release on Windows
# test and build binaries
test-win-rel:
  <<: *test-win
  variables:
    R_BIN: "$R_REL_WIN_BIN"
    RTOOLS_VERSION: "$R_REL_VERSION"

## R-devel on Windows
# test and build binaries
test-win-dev:
  <<: *test-win
  variables:
    R_BIN: "$R_DEV_WIN_BIN"
    RTOOLS_VERSION: "$R_REL_VERSION"

## R-oldrel on Windows
# test and build binaries
test-win-old:
  <<: *test-win
  variables:
    R_BIN: "$R_OLD_WIN_BIN"
    RTOOLS_VERSION: "$R_OLD_VERSION"

.test-mac-template: &test-mac
  <<: *test
  tags:
    - saas-macos-medium-m1
  before_script:
    - curl -O $R_BIN
    - sudo installer -pkg "$(ls -1t R-*-arm64.pkg | head -n 1)" -target /
    - *install-deps
    - cp $(ls -1t bus/build/data.table_*.tar.gz | head -n 1) .
  script:
    - R CMD check --no-manual $(ls -1t data.table_*.tar.gz | head -n 1)
    - R CMD INSTALL --build $(ls -1t data.table_*.tar.gz | head -n 1)
  after_script:
    - mkdir -p bus/$CI_JOB_NAME
    - '[ -d data.table.Rcheck ] && mv data.table.Rcheck bus/$CI_JOB_NAME/'
    - '[ -f data.table_*.tgz ] && mv $(ls -1t data.table_*.tgz | head -n 1) bus/$CI_JOB_NAME/'
    - echo $CI_JOB_ID > bus/$CI_JOB_NAME/id
    - echo $CI_JOB_STATUS > bus/$CI_JOB_NAME/status
    - echo $CI_JOB_IMAGE > bus/$CI_JOB_NAME/image

## R-release on MacOS
test-mac-rel:
  <<: *test-mac
  variables:
    R_VERSION: "$R_REL_VERSION"
    R_BIN: "$R_REL_MAC_BIN"

## R-oldrel on MacOS
test-mac-old:
  <<: *test-mac
  variables:
    R_VERSION: "$R_OLD_VERSION"
    R_BIN: "$R_OLD_MAC_BIN"

## integrate artifacts
# merging package tarballs and binaries into single R repository
# rendering documentation
# setting up CRAN-like structure
# generating pkgdown website
integration:
  stage: integration
  image: registry.gitlab.com/jangorecki/dockerfiles/r-pkgdown
  tags:
    - saas-linux-medium-amd64
  only:
    - master
  needs: ["mirror-packages","build","test-lin-rel","test-lin-rel-cran","test-lin-dev-gcc-strict-cran","test-lin-dev-clang-cran","test-lin-rel-vanilla","test-lin-ancient-cran","test-win-rel","test-win-dev" ,"test-win-old","test-mac-rel","test-mac-old"]
  script:
    - R --version
    - *install-deps ## markdown pkg not present in r-pkgdown image
    - mkdir -p ./pkgdown/favicon/ && cp .graphics/favicon/* ./pkgdown/favicon/ ## copy favicons
    - Rscript -e 'pkgdown::build_site(override=list(destination="./website"))'
    ## html manual, vignettes, repos, cran_web, cran_checks
    - echo 'source(".ci/ci.R"); source(".ci/publish.R")' >> .Rprofile
    ## list of available test-* jobs dynamically based on bus/test-* directories
    - Rscript -e 'cat("\ntest.jobs <- c(\n"); cat(paste0("  \"",list.files("bus",pattern="^test-"),"\" = \"data.table\""), sep=",\n"); cat(")\n")' >> .Rprofile
    - Rscript -e 'sapply(names(test.jobs), check.test, pkg="data.table", simplify=FALSE)'
    - mkdir -p bus/$CI_JOB_NAME
    ## delete any existing non-dev version of data.table
    - rm -f bus/mirror-packages/cran/src/contrib/data.table_*.tar.gz
    - rm -f bus/mirror-packages/cran/bin/windows/contrib/$R_REL_VERSION/data.table_*.zip
    - rm -f bus/mirror-packages/cran/bin/windows/contrib/$R_DEV_VERSION/data.table_*.zip
    - rm -f bus/mirror-packages/cran/bin/windows/contrib/$R_OLD_VERSION/data.table_*.zip
    - rm -f bus/mirror-packages/cran/bin/macosx/big-sur-arm64/contrib/$R_REL_VERSION/data.table_*.tgz
    # - rm -f bus/mirror-packages/cran/bin/macosx/big-sur-arm64/contrib/$R_DEV_VERSION/data.table_*.tgz
    - rm -f bus/mirror-packages/cran/bin/macosx/big-sur-arm64/contrib/$R_OLD_VERSION/data.table_*.tgz
    ## merge mirror-packages and R devel packages
    - mv bus/mirror-packages/cran bus/$CI_JOB_NAME/
    ## publish package sources
    - mkdir -p bus/$CI_JOB_NAME/cran/library bus/$CI_JOB_NAME/cran/doc
    - mv $(ls -1t bus/build/data.table_*.tar.gz | head -n 1) bus/$CI_JOB_NAME/cran/src/contrib
    - Rscript -e 'tools::write_PACKAGES(contrib.url("bus/integration/cran", type="source"), type="source", fields="Revision", addFiles=TRUE)'
    ## publish binaries
    - mkdir -p bus/integration/cran/bin/windows/contrib/$R_REL_VERSION/
    - mkdir -p bus/integration/cran/bin/windows/contrib/$R_DEV_VERSION/
    - mkdir -p bus/integration/cran/bin/windows/contrib/$R_OLD_VERSION/
    - '[ -f bus/test-win-rel/data.table_*.zip ] && cp bus/test-win-rel/data.table_*.zip bus/integration/cran/bin/windows/contrib/$R_REL_VERSION/'
    - ls -1 "bus/integration/cran/bin/windows/contrib/$R_REL_VERSION"/data.table_*.zip || true
    - '[ -f bus/test-win-dev/data.table_*.zip ] && cp bus/test-win-dev/data.table_*.zip bus/integration/cran/bin/windows/contrib/$R_DEV_VERSION/'
    - ls -1 "bus/integration/cran/bin/windows/contrib/$R_DEV_VERSION"/data.table_*.zip || true
    - '[ -f bus/test-win-old/data.table_*.zip ] && cp bus/test-win-old/data.table_*.zip bus/integration/cran/bin/windows/contrib/$R_OLD_VERSION/'
    - ls -1 "bus/integration/cran/bin/windows/contrib/$R_OLD_VERSION"/data.table_*.zip || true
    - Rscript -e 'tools::write_PACKAGES(contrib.url("bus/integration/cran", type="win.binary", ver=Sys.getenv("R_REL_VERSION")), type="win.binary", fields="Revision", addFiles=TRUE)'
    - Rscript -e 'tools::write_PACKAGES(contrib.url("bus/integration/cran", type="win.binary", ver=Sys.getenv("R_DEV_VERSION")), type="win.binary", fields="Revision", addFiles=TRUE)'
    - Rscript -e 'tools::write_PACKAGES(contrib.url("bus/integration/cran", type="win.binary", ver=Sys.getenv("R_OLD_VERSION")), type="win.binary", fields="Revision", addFiles=TRUE)'
    #### macos mkdir cran/bin/.../contrib/...
    - mkdir -p bus/integration/cran/bin/macosx/big-sur-arm64/contrib/$R_REL_VERSION/
    # - mkdir -p bus/integration/cran/bin/macosx/big-sur-arm64/contrib/$R_DEV_VERSION/
    - mkdir -p bus/integration/cran/bin/macosx/big-sur-arm64/contrib/$R_OLD_VERSION/
    #### macos move binaries
    - '[ -f bus/test-mac-rel/data.table_*.tgz ] && cp bus/test-mac-rel/data.table_*.tgz bus/integration/cran/bin/macosx/big-sur-arm64/contrib/$R_REL_VERSION/'
    - ls -1 "bus/integration/cran/bin/macosx/big-sur-arm64/contrib/$R_REL_VERSION"/data.table_*.tgz || true
    # - '[ -f bus/test-mac-dev/data.table_*.tgz ] && cp bus/test-mac-dev/data.table_*.tgz bus/integration/cran/bin/macosx/big-sur-arm64/contrib/$R_DEV_VERSION/'
    # - ls -1 "bus/integration/cran/bin/macosx/big-sur-arm64/contrib/$R_DEV_VERSION"/data.table_*.tgz || true
    - '[ -f bus/test-mac-old/data.table_*.tgz ] && cp bus/test-mac-old/data.table_*.tgz bus/integration/cran/bin/macosx/big-sur-arm64/contrib/$R_OLD_VERSION/'
    - ls -1 "bus/integration/cran/bin/macosx/big-sur-arm64/contrib/$R_OLD_VERSION"/data.table_*.tgz || true
    #### macos write_PACKAGES
    - Rscript -e 'tools::write_PACKAGES(contrib.url("bus/integration/cran", type="mac.binary.big-sur-arm64", ver=Sys.getenv("R_REL_VERSION")), type="mac.binary", fields="Revision", addFiles=TRUE)'
    # - Rscript -e 'tools::write_PACKAGES(contrib.url("bus/integration/cran", type="mac.binary.big-sur-arm64", ver=Sys.getenv("R_DEV_VERSION")), type="mac.binary", fields="Revision", addFiles=TRUE)'
    - Rscript -e 'tools::write_PACKAGES(contrib.url("bus/integration/cran", type="mac.binary.big-sur-arm64", ver=Sys.getenv("R_OLD_VERSION")), type="mac.binary", fields="Revision", addFiles=TRUE)'
    ## install pkg to render html
    - mkdir -p /tmp/opencran/library /tmp/opencran/doc/html
    - Rscript -e 'install.packages("data.table", lib="/tmp/opencran/library", repos=file.path("file:",normalizePath("bus/integration/cran")), INSTALL_opts="--html", quiet=TRUE)'
    - Rscript -e 'packageVersion("data.table", lib.loc="/tmp/opencran/library")'
    ## CRAN style web/CRAN_web.css
    - wget -q -P bus/integration/cran/web https://cran.r-project.org/web/CRAN_web.css
    ## web/packages/$pkg/index.html
    - Rscript -e 'sapply(setNames(nm=rownames(installed.packages(lib.loc="/tmp/opencran/library", priority="NA"))), package.index, lib.loc="/tmp/opencran/library")'
    ## R docs, html, css, icons
    - Rscript -e 'doc.copy(repodir="/tmp/opencran")'
    ## Update packages.html, fix paths
    - Rscript -e 'setwd("/tmp/opencran/doc/html"); make.packages.html(lib.loc="../../library", docdir="/tmp/opencran/doc"); tmp<-readLines(f<-"/tmp/opencran/doc/html/packages.html"); writeLines(gsub("file:///../../library","../../library", tmp, fixed=TRUE), f)'
    - mv /tmp/opencran/doc bus/integration/cran/
    ## library html manual, vignettes
    - Rscript -e 'lib.copy(lib.from="/tmp/opencran/library")'
    ## web/checks/$pkg/$job 00install.out, 00check.log, *.Rout
    - Rscript -e 'sapply(names(test.jobs), check.copy)'
    ## web/packages/$pkg/$pkg.pdf
    - Rscript -e 'pdf.copy("data.table", "test-lin-rel")'
    ## web/checks/check_results_$pkg.html
    - Rscript -e 'check.index("data.table", names(test.jobs))'
    ## web/checks/check_flavors.html
    - Rscript -e 'check.flavors(names(test.jobs))'
    ## pkgdown merge
    - Rscript -e 'common_files<-function(path1, path2) intersect(list.files(path1, all.files=TRUE, no..=TRUE), list.files(path2, all.files=TRUE, no..=TRUE)); msg = if (length(f<-common_files("website","bus/integration/cran"))) paste(c("Following artifacts will be overwritten by pkgdown artifacts:", paste0("  ", f)), collapse="\n") else "No overlapping files from pkgdown artifacts"; message(msg); q("no")'
    - mv website/* bus/integration/cran/
    ## add plausible.io stats
    - find bus/integration/cran -type f -iname "*.html" | xargs sed -i 's!<head>!<head><script defer data-domain="rdatatable.gitlab.io/data.table" src="https://plausible.io/js/script.js"></script>!g'
  <<: *artifacts

## publish
# R repository
# test jobs summaries
# html documentation of all packages in repo
# pkgdown website
pages:
  stage: deploy
  environment: production
  tags:
    - saas-linux-medium-amd64
  only:
    - master
  image: ubuntu
  needs: ["integration"]
  script:
    - mkdir -p public
    - cp -r bus/integration/cran/* public
    - cat public/src/contrib/PACKAGES
  artifacts:
    expire_in: 2 weeks
    paths:
      - public