ARM support and build system refactoring

.circleci/config.yml

@@ -1,114 +1,72 @@
-version: 2
-jobs:
-  build:
-    docker:
-      - image: 'redislabsmodules/rmbuilder:latest'
+version: 2.1
+
+commands:
+  ci_steps:
+    parameters:
+      platform:
+        type: string
     steps:
       - checkout
       - run:
-          name: "Pull Submodules"
-          command: |
-            git submodule init
-            git submodule update --remote
+          name: Pull Submodules
+          command: make fetch
       - run:
-          name: Install prerequisite
-          command: >-
-            apt-get update; apt-get install -y cmake; pip install -r src/tests/requirements.txt
+          name: Install prerequisites
+          command: make setup
       - run:
           name: Build
-          command: cd src && make -j 8
+          command: make build SHOW=1
       - run:
           name: Test
-          command: cd src && mkdir -p /workspace/tests && make tests EXTRA_ARGS="--junitxml=/workspace/tests/tests_report.xml"
+          command: make test SHOW=1
       - run:
-          name: Persist Artifacts
-          command: >-
-            mkdir -p /workspace/build && cp src/$MODULE_ARTIFACT /workspace/ &&
-            cp ramp.yml /workspace/ &&
-            cp src/clean_ramp.py /workspace; ls -la /workspace
+          name: Package
+          command: make pack BRANCH="$(CIRCLE_BRANCH)" INTO=/workspace/packages SHOW=1
       - persist_to_workspace:
-          root: /workspace
+          root: /workspace/packages
           paths:
-            - '*.so'
-            - '*.yml'
-            - '*.py'
-            - build
+            - 'release/*.zip'
+            - 'branch/*.zip'
       - store_test_results:
           path: /workspace/tests
-  package_branch:
-    docker:
-      - image: 'redislabsmodules/rmbuilder:latest'
+  deploy:
+    parameters:
+      from:
+        type: string
     steps:
-      - attach_workspace:
-          at: /workspace
       - run:
-          name: Package
+          name: Deploy to S3
           command: >-
-            ramp pack -m /workspace/ramp.yml -o
-            /workspace/build/$PACKAGE_NAME.{os}-{architecture}.$CIRCLE_BRANCH.zip
-            /workspace/$MODULE_ARTIFACT
-      - run:
-          name: Clean RAMP file
-          shell: /bin/bash
-          command: |
-            /workspace/clean_ramp.py /workspace/build/*.zip
-      - persist_to_workspace:
-          root: /workspace
-          paths:
-            - build
-      - store_artifacts:
-          path: /workspace/build
-  package_release:
+            aws s3 cp <<parameters.from>>/ s3://redismodules/$PACKAGE_NAME/ --acl public-read --recursive --exclude "*" --include "*.zip"
+
+jobs:
+  build:
     docker:
       - image: 'redislabsmodules/rmbuilder:latest'
     steps:
-      - attach_workspace:
-          at: /workspace
-      - run:
-          name: Package
-          command: >-
-            ramp pack -m /workspace/ramp.yml -o
-            /workspace/build/$PACKAGE_NAME.{os}-{architecture}.{semantic_version}.zip
-            /workspace/$MODULE_ARTIFACT
-      - run:
-          name: Package
-          command: >-
-            ramp pack -m /workspace/ramp.yml -o
-            /workspace/build/$PACKAGE_NAME.{os}-{architecture}.latest.zip
-            /workspace/$MODULE_ARTIFACT
-      - run:
-          name: Clean RAMP file
-          shell: /bin/bash
-          command: |
-            /workspace/clean_ramp.py /workspace/build/*.zip
-      - persist_to_workspace:
-          root: /workspace
-          paths:
-            - build
-      - store_artifacts:
-          path: /workspace/build
-  deploy_branch:
-    docker:
-      - image: 'redislabsmodules/rmbuilder:latest'
+      - ci_steps:
+          platform: debian 
+  build-macos:
+    macos:
+      xcode: 10.2.1
     steps:
-      - attach_workspace:
-          at: /workspace
-      - run:
-          name: Deploy to S3
-          command: >-
-            aws s3 cp /workspace/build/ s3://redismodules/$PACKAGE_NAME/ --acl
-            public-read --recursive --exclude "*" --include "*.zip"
-  deploy_release:
+      - ci_steps:
+          platform: macosx
+
+  deploy_package:
+    parameters:
+      package:
+        type: string
     docker:
       - image: 'redislabsmodules/rmbuilder:latest'
     steps:
       - attach_workspace:
           at: /workspace
-      - run:
-          name: Deploy to S3
-          command: >-
-            aws s3 cp /workspace/build/ s3://redismodules/$PACKAGE_NAME/ --acl
-            public-read --recursive --exclude "*" --include "*.zip"
+      - deploy:
+          from: /workspace/packages/<<parameters.package>>
+      - store_artifacts:
+          path: /workspace/packages/<<parameters.package>>
+
   deploy_docs:
     docker:
       - image: 'redislabsmodules/rmbuilder:latest'
@@ -120,9 +78,8 @@ jobs:
       - run:
           name: Deploy Docs to S3
           command: >-
-            aws s3 cp site s3://oss.redislabs.com/$WEBSITE_FOLDER/ --acl
-            public-read --recursive
-
+            aws s3 cp site s3://oss.redislabs.com/$WEBSITE_FOLDER/ --acl public-read --recursive
+
   run_automation:
     docker:
       - image: 'redislabsmodules/rmbuilder:latest'
@@ -135,7 +92,7 @@ jobs:
             -H "Content-Type: application/json"
             -d '{"service_id":"single_module_test_cycle_sanity_and_extended", "name":"redistimeseries automation-testing", "properties":{"sut_version":"master", "email_recipients":"t9b6i7v6e2q3d0s4@redislabs.slack.com", "sut_environments":[], "tools_environment":{}, "module_name": "redistimeseries", "module_version":"master", "cycle_environments_setup":[{"teardown":true, "name":"xenial-amd64-aws", "concurrency":1}, {"teardown":true, "name":"bionic-amd64-aws", "concurrency":1}]}}'
             https://qa-automation-center.redislabs.com/processes
-            
+
 workflows:
   version: 2
   build_and_package:
@@ -144,32 +101,31 @@ workflows:
           filters:
             tags:
               only: /.*/
-      - package_branch:
+#      - build-macos:
+#          filters:
+#            tags:
+#              only: /.*/
+      - deploy_package:
+          name: deploy_branch
+          package: branch
           requires:
             - build
           filters:
             branches:
               only: master
-      - package_release:
+      - deploy_package:
+          name: deploy_release
+          package: release
           requires:
             - build
           filters:
             branches:
               ignore: /.*/
             tags:
               only: '/^v[0-9].*/'
-      - deploy_branch:
-          requires:
-            - package_branch
       - run_automation:
           requires:
             - deploy_branch
-      - deploy_release:
-          filters:
-            tags:
-              only: '/^v[0-9].*/'
-          requires:
-            - package_release
       - deploy_docs:
           filters:
             branches:

.dockerignore

@@ -0,0 +1,4 @@
+bin/
+/VARIANT
+/BINDIR
+

.gitignore

@@ -9,4 +9,8 @@ __pycache__
 venv/
 .idea/
 .DS_Store
- tests/.cache/
+/tests/.cache/
+/bin/
+/VARIANT
+/BINDIR
+/tests/tests_report.xml

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "RedisModulesSDK"]
 	path = RedisModulesSDK
 	url = https://github.com/RedisLabs/RedisModulesSDK.git
+[submodule "deps/RedisModulesSDK"]
+	path = deps/RedisModulesSDK
+	url = https://github.com/RedisLabs/RedisModulesSDK.git

Dockerfile

@@ -1,36 +1,35 @@
-FROM redis:latest as builder
-
-ENV DEPS "build-essential"
-
-# Set up a build environment
-RUN set -ex;\
-    deps="$DEPS";\
-    apt-get update;\
-    apt-get install -y --no-install-recommends $deps;
-    
-# Build the source
-ADD ./ /redis-timeseries
-
-# Build Redis-TimeSeries
-WORKDIR /redis-timeseries
-RUN set -ex;\
-	cd RedisModulesSDK/rmutil;\
-	make clean;\
-	make;\
-	cd -;
-RUN set -ex;\
-	cd src;\
-    make clean; \
-    make;
-
-# Package the runner
-FROM redis:latest
+# BUILD redisfab/redistimeseries-${OSNICK}:M.m.b-${ARCH}
+
+# stretch|bionic|buster
+ARG OSNICK=buster
+
+# ARCH=x64|arm64v8|arm32v7
+ARG ARCH=x64
+
+#----------------------------------------------------------------------------------------------
+# FROM redis:latest AS builder
+FROM redisfab/redis-${ARCH}-${OSNICK}:5.0.5 AS builder
+
+ADD ./ /build
+WORKDIR /build
+
+RUN ./deps/readies/bin/getpy2
+RUN ./system-setup.py
+RUN make fetch
+
+ENV X_NPROC "cat /proc/cpuinfo|grep processor|wc -l"
+RUN echo nproc=$(nproc); echo NPROC=$(eval "$X_NPROC")
+RUN make build
+
+#----------------------------------------------------------------------------------------------
+# FROM redis:latest
+FROM redisfab/redis-${ARCH}-${OSNICK}:5.0.5
+
 ENV LIBDIR /usr/lib/redis/modules
 WORKDIR /data
-RUN set -ex;\
-    mkdir -p "$LIBDIR";
+RUN mkdir -p "$LIBDIR"
 
-COPY --from=builder /redis-timeseries/src/redistimeseries.so "$LIBDIR"
+COPY --from=builder /build/bin/redistimeseries.so "$LIBDIR"
 
 EXPOSE 6379
 CMD ["redis-server", "--loadmodule", "/usr/lib/redis/modules/redistimeseries.so"]

Dockerfile.arm

@@ -0,0 +1,41 @@
+# BUILD redisfab/redistimeseries-${OSNICK}:M.m.b-${ARCH}
+
+# stretch|bionic|buster
+ARG OSNICK=buster
+
+# ARCH=arm64v8|arm32v7
+ARG ARCH=arm64v8
+
+#----------------------------------------------------------------------------------------------
+FROM redisfab/redis-${ARCH}-${OSNICK}-xbuild:5.0.5 as builder
+
+RUN [ "cross-build-start" ]
+
+ADD ./ /build
+WORKDIR /build
+
+RUN ./deps/readies/bin/getpy2
+RUN ./system-setup.py
+RUN make fetch
+
+ENV X_NPROC "cat /proc/cpuinfo|grep processor|wc -l"
+RUN echo nproc=$(nproc); echo NPROC=$(eval "$X_NPROC")
+RUN make build
+
+RUN [ "cross-build-end" ]
+
+#----------------------------------------------------------------------------------------------
+FROM redisfab/redis-${ARCH}-${OSNICK}-xbuild:5.0.5
+
+RUN [ "cross-build-start" ]
+
+ENV LIBDIR /usr/lib/redis/modules
+WORKDIR /data
+RUN mkdir -p "$LIBDIR"
+
+COPY --from=builder /build/bin/redistimeseries.so "$LIBDIR"
+
+EXPOSE 6379
+CMD ["redis-server", "--loadmodule", "/usr/lib/redis/modules/redistimeseries.so"]
+
+RUN [ "cross-build-end" ]

Makefile

@@ -0,0 +1,23 @@
+
+.PHONY: all setup fetch build test pack
+
+all: fetch build
+
+setup:
+	@./deps/readies/bin/getpy2
+	@python ./system-setup.py
+
+fetch:
+	@git submodule update --init --recursive
+
+build:
+	@make -C src all -j
+
+test:
+	@make -C src tests
+
+pack:
+	@make -C src package
+
+# deploy:
+#	@make -C src deploy