RedisLabsModules · chayim · Jun 9, 2021 · Jun 9, 2021 · Jun 9, 2021 · Jun 10, 2021
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,85 +1,42 @@
 version: 2.1
 
 commands:
-  setup-executor:
-    steps:
-      - run:
-          name: Setup executor
-          command: |
-            apt-get -qq update
-            apt-get -q install -y git openssh-client curl ca-certificates make tar gzip
-            bash <(curl -fsSL https://get.docker.com)
-      - setup_remote_docker:
-          docker_layer_caching: true
 
-  platform-build-steps:
-    parameters:
-      platform:
-        type: string
-      publish:
-        type: boolean
-        default: false
+  checkout-all:
     steps:
-      - setup-executor
       - checkout
       - run:
-          name: Setup
+          name: Checkout submodules
           command: |
             git submodule update --init --recursive
-            ./deps/readies/bin/getpy3
-      - run:
-          name: Build for platform
-          command: |
-            docker login -u redisfab -p $DOCKER_REDISFAB_PWD
-            make build OSNICK=<<parameters.platform>>
-            <<# parameters.publish >> make publish OSNICK=<<parameters.platform>> <</ parameters.publish >>
-          no_output_timeout: 40m
 
-  edge-build-steps:
-    parameters:
-      platform:
-        type: string
+  setup-build-system:
     steps:
-      - setup-executor
-      - checkout
-      - run:
-          name: Setup
-          command: |
-            git submodule update --init --recursive
-            ./deps/readies/bin/getpy3
       - run:
-          name: Build for platform
-          command: |
-            docker login -u redisfab -p $DOCKER_REDISFAB_PWD
-            for i in cpu gpu; do
-              make -C edge build publish OSNICK=<<parameters.platform>> PUSH_EDGE=1 DOCKER_ORG=redislabs REDISGEARS_DOCKERDEP=edge REDISTIMESERIES_DOCKERDEP=edge REDISAI_DOCKERDEP=edge-$i VERSION=edge VARIANT=x64-$i
-            done
-          no_output_timeout: 40m
+          name: system setup
+          command: ./deps/readies/bin/getpy3
 
 jobs:
-  platform-build:
-    parameters:
-      platform:
-        type: string
-      publish:
-        type: boolean
-        default: false
-    docker:
-      - image: debian:buster
-    steps:
-      - platform-build-steps:
-          platform: <<parameters.platform>>
-          publish: <<parameters.publish>>
 
   edge-build:
     parameters:
-      platform:
+      variant:
+        type: string
+      publish:
         type: string
+        default: ""
     docker:
-      - image: debian:buster
+      - image: cimg/python:3.9
     steps:
-      - edge-build-steps:
-          platform: <<parameters.platform>>
+      - setup_remote_docker
+      - checkout-all
+      - setup-build-system
+      - run:
+          name: Build edge for variant
+          command: |
+            docker login -u redisfab -p $DOCKER_REDISFAB_PWD
+            make -C edge build DOCKER_SUFFIX="<<parameters.variant>>" VERBOSE=1 <<parameters.variant>> <<parameters.publish>>
+          no_output_timeout: 40m
 
 on-any-branch: &on-any-branch
   filters:
@@ -120,20 +77,29 @@ workflows:
   version: 2
   default_flow:
     jobs:
-      - platform-build:
-          name: build-bionic
-          platform: bionic
-          publish: true
+      - edge-build:
+          context: common
+          matrix:
+              parameters:
+                variant:
+                  - "GPU=1"
+                  - "CPU=1"
           <<: *on-any-branch
 
-  edge_dailies:
+  edge_nightly:
     triggers:
       - schedule:
-          cron: "2 9,18 * * *"
+          cron: "2 3 * * *"
           filters:
             branches:
               only: master
     jobs:
-      - platform-build:
-          name: build-edge
-          platform: bionic
+      - edge-build:
+          context: common
+          matrix:
+            parameters:
+              variant:
+                - "GPU=1"
+                - "CPU=1"
+              publish:
+                - "PUBLISH=1"
diff --git a/README.md b/README.md
@@ -1,4 +1,4 @@
-[![CircleCI](https://circleci.com/gh/RedisLabs/RedisEdge/tree/master.svg?style=svg)](https://circleci.com/gh/RedisLabs/RedisEdge/tree/master)
+[![CircleCI](https://circleci.com/gh/RedisLabsModules/RedisEdge/tree/master.svg?style=svg)](https://circleci.com/gh/RedisLabsModules/RedisEdge/tree/master)
 [![Docker Cloud Build Status](https://img.shields.io/docker/cloud/build/redislabs/redisedge.svg)](https://hub.docker.com/r/redislabs/redisedge/builds/)
 
 # redisedge - a Docker image with select Redis Labs modules for the Edge

diff --git a/deps/readies b/deps/readies
diff --git a/docker-compose.debug.yml b/docker-compose.debug.yml
diff --git a/docker-compose.yml b/docker-compose.yml
diff --git a/edge/Dockerfile b/edge/Dockerfile
diff --git a/edge/Makefile b/edge/Makefile
@@ -1,53 +1,22 @@
-
-ROOT=..
-MK.pyver:=3
-include $(ROOT)/deps/readies/mk/main
-
-# OSNICK=buster|stretch|bionic
-OSNICK ?= bionic # buster
-
-REPO=redisedge
-
-ifeq ($(JETSON),1)
-DOCKER_SUFFIX=.jetson
-endif
-
-DOCKERFILE_STEM=$(ROOT)/edge/Dockerfile$(DOCKER_SUFFIX)
-
-include ../versions
-
-DOCKER_BUILD_ARGS += \
-	REDISAI_VERSION=$(REDISAI_VERSION) \
-	REDISTIMESERIES_VERSION=$(REDISTIMESERIES_VERSION) \
-	REDISGEARS_VERSION=$(REDISGEARS_VERSION)
-
-# override defines for the case
-ifdef REDISGEARS_DOCKERDEP
-DOCKER_BUILD_ARGS += \
-	VERSION=edge \
-	REDISGEARS_DOCKERDEP=$(REDISGEARS_DOCKERDEP)
+DOCKER_ORG=redislabs
+PRODUCT=redisedge
+VARIANT=cpu
+OSNICK=bionic
+DEBIAN_OSNICK=buster
+
+VERSION ?= edge
+VARIANT=cpu
+ifeq ($(GPU),1)
+VARIANT=gpu
 endif
 
-ifdef REDISAI_DOCKERDEP
-DOCKER_BUILD_ARGS += \
-	REDISAI_DOCKERDEP=$(REDISAI_DOCKERDEP)
-endif
-
-ifdef REDISTIMESERIES_DOCKERDEP
-DOCKER_BUILD_ARGS += \
-	REDISTIMESERIES_DOCKERDEP=$(REDISTIMESERIES_DOCKERDEP)
-endif
-
-ifdef REDIS_DOCKERDEP
-DOCKER_BUILD_ARGS += \
-	REDIS_DOCKERDEP=$(REDIS_DOCKERDEP)
-endif
-
-ifdef DOCKER_ORG
-DOCKER_BUILD_ARGS += \
-	DOCKER_ORG=$(DOCKER_ORG)
-endif
-
-
+export REDIS_DOCKER_ORG=redislabs
+export REDISAI_DOCKERTAG=${VERSION}-${VARIANT}-${OSNICK}
+export REDISGEARS_DOCKERTAG=${VERSION}
+export REDISTIMESERIES_DOCKERTAG=${VERSION}
+export REDIS_DOCKERTAG=latest
+DEFAULT_TAG=${DOCKER_ORG}/${PRODUCT}:${VERSION}-${VARIANT}
 
-include $(MK)/docker.native
+ROOT=../
+READIES=${ROOT}/deps/readies
+include ${READIES}/mk/docker.rules
diff --git a/edge/dockerfile.tmpl b/edge/dockerfile.tmpl
@@ -0,0 +1,39 @@
+# BUILD {{REDIS_DOCKER_ORG}}/redisedge:{{VERSION}}-cpu-{{REDIS_ARCH}}-{{REDIS_OSNICK}}
+
+#----------------------------------------------------------------------------------------------
+FROM {{REDIS_DOCKER_ORG}}/redisai:{{REDISAI_DOCKERTAG}} as ai
+FROM {{REDIS_DOCKER_ORG}}/redistimeseries:{{REDISTIMESERIES_DOCKERTAG}} as timeseries
+FROM {{REDIS_DOCKER_ORG}}/redisgears:{{REDISGEARS_DOCKERTAG}} as gears
+
+#----------------------------------------------------------------------------------------------
+FROM {{REDIS_DOCKER_ORG}}/redis:{{REDIS_DOCKERTAG}}
+
+COPY --chown=root:root ./edge/sudofile /etc/sudoers.d/99-redislabs
+
+RUN echo "Building redisedge-{{REDIS_OSNICK}}:{{VERSION}}-{{DEVICE}}-{{REDIS_ARCH}} with:" ;\
+    echo "  RedisAI={{REDISAI_DOCKERTAG}}";\
+    echo "  RedisTimeSeries={{REDISTIMESERIES_DOCKERTAG}}" ;\
+    echo "  RedisGears={{REDISGEARS_DOCKERTAG}}" ;\
+    echo "  Redis={{REDIS_DOCKERTAG}}"
+
+ENV LIBDIR /usr/lib/redis/modules
+ENV LD_LIBRARY_PATH $LIBDIR
+
+RUN set -e; if [ ! -z $(command -v apt-get) ]; then sudo apt-get -qq update; sudo apt-get -q install -y libgomp1 git; fi
+RUN set -e; if [ ! -z $(command -v yum) ]; then sudo yum install -y libgomp git; fi
+
+RUN sudo mkdir -p ${LIBDIR}
+RUN sudo chown -R redislabs:redislabs ${LIBDIR}
+
+COPY --from=timeseries --chown=redislabs:redislabs ${LIBDIR}/*.so ${LIBDIR}/
+COPY --from=ai --chown=redislabs:redislabs ${LIBDIR}/ ${LIBDIR}/
+COPY --from=gears --chown=redislabs:redislabs /var/opt/redislabs/lib/modules/redisgears.so ${LIBDIR}/
+COPY --from=gears --chown=redislabs:redislabs /var/opt/redislabs/modules/ /var/opt/redislabs/modules/
+
+WORKDIR /var/opt/redislabs/modules/rg
+RUN ln -s python3 python3_`cat /var/opt/redislabs/artifacts/VERSION`
+
+RUN sudo rm /etc/sudoers.d/99-redislabs
+
+ADD redisedge.conf /etc
+CMD ["/etc/redisedge.conf"]
diff --git a/edge/sudofile b/edge/sudofile
@@ -0,0 +1 @@
+redislabs ALL=(root) NOPASSWD: ALL
+152 −0		bin/circleci-setvars
+99 −0		bin/dockerhub-ls
+164 −0		bin/dockerwrapper
+1 −1		bin/enable-utf8
+50 −0		bin/generateversion
+7 −0		bin/getbash
+3 −1		bin/getbazel
+10 −5		bin/getcmake
+8 −0		bin/getgcc
+1 −0		bin/getget
+15 −7		bin/getgolang
+58 −0		bin/getpudb
+8 −0		bin/getpy2
+18 −5		bin/getpy3
+5 −5		bin/getredis
+12 −1		bin/getrust
+42 −0		bin/getterraform
+30 −0		bin/getupdates
+29 −0		bin/getvalgrind
+4 −1		mk/docker
+4 −1		mk/docker.cross
+18 −1		mk/docker.native
+67 −0		mk/docker.rules
+32 −0		mk/osnick.defs
+5 −2		paella/debug.py
+9 −0		paella/platform.py
+29 −5		paella/setup.py
+5 −3		shibumi/functions