diff --git a/.circleci/config.yml b/.circleci/config.yml index 24f6a1a..2734275 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,85 +1,42 @@ version: 2.1 commands: - setup-executor: - steps: - - run: - name: Setup executor - command: | - apt-get -qq update - apt-get -q install -y git openssh-client curl ca-certificates make tar gzip - bash <(curl -fsSL https://get.docker.com) - - setup_remote_docker: - docker_layer_caching: true - platform-build-steps: - parameters: - platform: - type: string - publish: - type: boolean - default: false + checkout-all: steps: - - setup-executor - checkout - run: - name: Setup + name: Checkout submodules command: | git submodule update --init --recursive - ./deps/readies/bin/getpy3 - - run: - name: Build for platform - command: | - docker login -u redisfab -p $DOCKER_REDISFAB_PWD - make build OSNICK=<> - <<# parameters.publish >> make publish OSNICK=<> <> - no_output_timeout: 40m - edge-build-steps: - parameters: - platform: - type: string + setup-build-system: steps: - - setup-executor - - checkout - - run: - name: Setup - command: | - git submodule update --init --recursive - ./deps/readies/bin/getpy3 - run: - name: Build for platform - command: | - docker login -u redisfab -p $DOCKER_REDISFAB_PWD - for i in cpu gpu; do - make -C edge build publish OSNICK=<> PUSH_EDGE=1 DOCKER_ORG=redislabs REDISGEARS_DOCKERDEP=edge REDISTIMESERIES_DOCKERDEP=edge REDISAI_DOCKERDEP=edge-$i VERSION=edge VARIANT=x64-$i - done - no_output_timeout: 40m + name: system setup + command: ./deps/readies/bin/getpy3 jobs: - platform-build: - parameters: - platform: - type: string - publish: - type: boolean - default: false - docker: - - image: debian:buster - steps: - - platform-build-steps: - platform: <> - publish: <> edge-build: parameters: - platform: + variant: + type: string + publish: type: string + default: "" docker: - - image: debian:buster + - image: cimg/python:3.9 steps: - - edge-build-steps: - platform: <> + - setup_remote_docker + - checkout-all + - setup-build-system + - run: + name: Build edge for variant + command: | + docker login -u redisfab -p $DOCKER_REDISFAB_PWD + make -C edge build DOCKER_SUFFIX="<>" VERBOSE=1 <> <> + no_output_timeout: 40m on-any-branch: &on-any-branch filters: @@ -120,20 +77,29 @@ workflows: version: 2 default_flow: jobs: - - platform-build: - name: build-bionic - platform: bionic - publish: true + - edge-build: + context: common + matrix: + parameters: + variant: + - "GPU=1" + - "CPU=1" <<: *on-any-branch - edge_dailies: + edge_nightly: triggers: - schedule: - cron: "2 9,18 * * *" + cron: "2 3 * * *" filters: branches: only: master jobs: - - platform-build: - name: build-edge - platform: bionic + - edge-build: + context: common + matrix: + parameters: + variant: + - "GPU=1" + - "CPU=1" + publish: + - "PUBLISH=1" diff --git a/README.md b/README.md index 4bf1c3b..c2760d9 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![CircleCI](https://circleci.com/gh/RedisLabs/RedisEdge/tree/master.svg?style=svg)](https://circleci.com/gh/RedisLabs/RedisEdge/tree/master) +[![CircleCI](https://circleci.com/gh/RedisLabsModules/RedisEdge/tree/master.svg?style=svg)](https://circleci.com/gh/RedisLabsModules/RedisEdge/tree/master) [![Docker Cloud Build Status](https://img.shields.io/docker/cloud/build/redislabs/redisedge.svg)](https://hub.docker.com/r/redislabs/redisedge/builds/) # redisedge - a Docker image with select Redis Labs modules for the Edge diff --git a/deps/readies b/deps/readies index 05baec3..6f14530 160000 --- a/deps/readies +++ b/deps/readies @@ -1 +1 @@ -Subproject commit 05baec34fbde293eed1b63b5e43edb4d9259e98e +Subproject commit 6f145308b740b8108f1902ef53519b10dbe59fa9 diff --git a/docker-compose.debug.yml b/docker-compose.debug.yml deleted file mode 100644 index f619e54..0000000 --- a/docker-compose.debug.yml +++ /dev/null @@ -1,10 +0,0 @@ -version: '2.1' - -services: - redisedge: - image: redisedge - build: - context: . - dockerfile: Dockerfile - ports: - - 6379:6379 diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index b252dd6..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,8 +0,0 @@ -version: '2.1' - -services: - redisedge: - image: redisedge - build: . - ports: - - 6379:6379 diff --git a/edge/Dockerfile b/edge/Dockerfile deleted file mode 100755 index e478e9d..0000000 --- a/edge/Dockerfile +++ /dev/null @@ -1,68 +0,0 @@ -# BUILD redisfab/redisedge:${VERSION}-cpu-${ARCH}-${OSNICK} - -ARG VERSION=0.4.0 - -# OSNICK=stretch|bionic|buster -ARG OSNICK=bionic - -# ARCH=x64|arm64v8|arm32v7 -ARG ARCH=x64 - -ARG REDISAI_VERSION=1.0.2 -ARG REDISTIMESERIES_VERSION=1.4.6 -ARG REDISGEARS_VERSION=1.0.2 -ARG REDIS_VER=6.2.1 -ARG DEVICE=cpu - -ARG DOCKER_ORG=redisfab - -# done to specifically allow for ease of optional override with --build-arg -ARG REDISAI_DOCKERDEP=${REDISAI_VERSION}-${DEVICE}-${ARCH}-${OSNICK} -ARG REDISTIMESERIES_DOCKERDEP=${REDISTIMESERIES_VERSION}-${ARCH}-${OSNICK} -ARG REDISGEARS_DOCKERDEP=${REDISGEARS_VERSION}-${ARCH}-${OSNICK} -ARG REDIS_DOCKERDEP=${REDIS_VER}-${ARCH}-${OSNICK} - -#---------------------------------------------------------------------------------------------- -FROM ${DOCKER_ORG}/redisai:${REDISAI_DOCKERDEP} as ai -FROM ${DOCKER_ORG}/redistimeseries:${REDISTIMESERIES_DOCKERDEP} as timeseries -FROM ${DOCKER_ORG}/redisgears:${REDISGEARS_DOCKERDEP} as gears - -#---------------------------------------------------------------------------------------------- -FROM redisfab/redis:${REDIS_DOCKERDEP} - -ARG OSNICK -ARG ARCH -ARG VERSION -ARG REDISAI_VERSION -ARG REDISTIMESERIES_VERSION -ARG REDISGEARS_VERSION -ARG REDIS_VER -ARG REDISTIMESERIES_DOCKERDEP -ARG REDISGEARS_DOCKERDEP -ARG REDIS_DOCKERDEP -ARG REDISAI_DOCKERDEP - -RUN echo "Building redisedge-${OSNICK}:${VERSION}-${DEVICE}-${ARCH} with:" ;\ - echo " RedisAI=${REDISAI_DOCKERDEP}-${DEVICE}" ;\ - echo " RedisTimeSeries=${REDISTIMESERIES_DOCKERDEP}" ;\ - echo " RedisGears=${REDISGEARS_DOCKERDEP}" ;\ - echo " Redis=${REDIS_DOCKERDEP}" - -RUN set -e; if [ ! -z $(command -v apt-get) ]; then apt-get -qq update; apt-get -q install -y libgomp1 git; fi -RUN set -e; if [ ! -z $(command -v yum) ]; then yum install -y libgomp git; fi - -ENV LIBDIR /usr/lib/redis/modules -ENV LD_LIBRARY_PATH $LIBDIR -WORKDIR /data -RUN mkdir -p ${LIBDIR} - -COPY --from=timeseries --chown=redis:redis ${LIBDIR}/*.so ${LIBDIR}/ -COPY --from=ai --chown=redis:redis ${LIBDIR}/ ${LIBDIR}/ -COPY --from=gears --chown=redis:redis /var/opt/redislabs/lib/modules/redisgears.so ${LIBDIR}/ -COPY --from=gears --chown=redis:redis /var/opt/redislabs/modules/ /var/opt/redislabs/modules/ - -WORKDIR /var/opt/redislabs/modules/rg -RUN ln -s python3 python3_`cat /var/opt/redislabs/artifacts/VERSION` - -ADD redisedge.conf /etc -CMD ["/etc/redisedge.conf"] diff --git a/edge/Makefile b/edge/Makefile index 79c207e..d634069 100755 --- a/edge/Makefile +++ b/edge/Makefile @@ -1,53 +1,22 @@ - -ROOT=.. -MK.pyver:=3 -include $(ROOT)/deps/readies/mk/main - -# OSNICK=buster|stretch|bionic -OSNICK ?= bionic # buster - -REPO=redisedge - -ifeq ($(JETSON),1) -DOCKER_SUFFIX=.jetson -endif - -DOCKERFILE_STEM=$(ROOT)/edge/Dockerfile$(DOCKER_SUFFIX) - -include ../versions - -DOCKER_BUILD_ARGS += \ - REDISAI_VERSION=$(REDISAI_VERSION) \ - REDISTIMESERIES_VERSION=$(REDISTIMESERIES_VERSION) \ - REDISGEARS_VERSION=$(REDISGEARS_VERSION) - -# override defines for the case -ifdef REDISGEARS_DOCKERDEP -DOCKER_BUILD_ARGS += \ - VERSION=edge \ - REDISGEARS_DOCKERDEP=$(REDISGEARS_DOCKERDEP) +DOCKER_ORG=redislabs +PRODUCT=redisedge +VARIANT=cpu +OSNICK=bionic +DEBIAN_OSNICK=buster + +VERSION ?= edge +VARIANT=cpu +ifeq ($(GPU),1) +VARIANT=gpu endif -ifdef REDISAI_DOCKERDEP -DOCKER_BUILD_ARGS += \ - REDISAI_DOCKERDEP=$(REDISAI_DOCKERDEP) -endif - -ifdef REDISTIMESERIES_DOCKERDEP -DOCKER_BUILD_ARGS += \ - REDISTIMESERIES_DOCKERDEP=$(REDISTIMESERIES_DOCKERDEP) -endif - -ifdef REDIS_DOCKERDEP -DOCKER_BUILD_ARGS += \ - REDIS_DOCKERDEP=$(REDIS_DOCKERDEP) -endif - -ifdef DOCKER_ORG -DOCKER_BUILD_ARGS += \ - DOCKER_ORG=$(DOCKER_ORG) -endif - - +export REDIS_DOCKER_ORG=redislabs +export REDISAI_DOCKERTAG=${VERSION}-${VARIANT}-${OSNICK} +export REDISGEARS_DOCKERTAG=${VERSION} +export REDISTIMESERIES_DOCKERTAG=${VERSION} +export REDIS_DOCKERTAG=latest +DEFAULT_TAG=${DOCKER_ORG}/${PRODUCT}:${VERSION}-${VARIANT} -include $(MK)/docker.native +ROOT=../ +READIES=${ROOT}/deps/readies +include ${READIES}/mk/docker.rules diff --git a/edge/dockerfile.tmpl b/edge/dockerfile.tmpl new file mode 100755 index 0000000..fa814ee --- /dev/null +++ b/edge/dockerfile.tmpl @@ -0,0 +1,39 @@ +# BUILD {{REDIS_DOCKER_ORG}}/redisedge:{{VERSION}}-cpu-{{REDIS_ARCH}}-{{REDIS_OSNICK}} + +#---------------------------------------------------------------------------------------------- +FROM {{REDIS_DOCKER_ORG}}/redisai:{{REDISAI_DOCKERTAG}} as ai +FROM {{REDIS_DOCKER_ORG}}/redistimeseries:{{REDISTIMESERIES_DOCKERTAG}} as timeseries +FROM {{REDIS_DOCKER_ORG}}/redisgears:{{REDISGEARS_DOCKERTAG}} as gears + +#---------------------------------------------------------------------------------------------- +FROM {{REDIS_DOCKER_ORG}}/redis:{{REDIS_DOCKERTAG}} + +COPY --chown=root:root ./edge/sudofile /etc/sudoers.d/99-redislabs + +RUN echo "Building redisedge-{{REDIS_OSNICK}}:{{VERSION}}-{{DEVICE}}-{{REDIS_ARCH}} with:" ;\ + echo " RedisAI={{REDISAI_DOCKERTAG}}";\ + echo " RedisTimeSeries={{REDISTIMESERIES_DOCKERTAG}}" ;\ + echo " RedisGears={{REDISGEARS_DOCKERTAG}}" ;\ + echo " Redis={{REDIS_DOCKERTAG}}" + +ENV LIBDIR /usr/lib/redis/modules +ENV LD_LIBRARY_PATH $LIBDIR + +RUN set -e; if [ ! -z $(command -v apt-get) ]; then sudo apt-get -qq update; sudo apt-get -q install -y libgomp1 git; fi +RUN set -e; if [ ! -z $(command -v yum) ]; then sudo yum install -y libgomp git; fi + +RUN sudo mkdir -p ${LIBDIR} +RUN sudo chown -R redislabs:redislabs ${LIBDIR} + +COPY --from=timeseries --chown=redislabs:redislabs ${LIBDIR}/*.so ${LIBDIR}/ +COPY --from=ai --chown=redislabs:redislabs ${LIBDIR}/ ${LIBDIR}/ +COPY --from=gears --chown=redislabs:redislabs /var/opt/redislabs/lib/modules/redisgears.so ${LIBDIR}/ +COPY --from=gears --chown=redislabs:redislabs /var/opt/redislabs/modules/ /var/opt/redislabs/modules/ + +WORKDIR /var/opt/redislabs/modules/rg +RUN ln -s python3 python3_`cat /var/opt/redislabs/artifacts/VERSION` + +RUN sudo rm /etc/sudoers.d/99-redislabs + +ADD redisedge.conf /etc +CMD ["/etc/redisedge.conf"] diff --git a/edge/sudofile b/edge/sudofile new file mode 100644 index 0000000..75be111 --- /dev/null +++ b/edge/sudofile @@ -0,0 +1 @@ +redislabs ALL=(root) NOPASSWD: ALL