Numerous automation changes (#264)

* Amazon Linux support

* Fix 1

* Fix 2

* Fix 3

* Fix 4

* Fix 5

* Fix 6

* Fix 7

* Added CircleCI macOS build + automation fixes

* Fix 8

.circleci/config.yml

@@ -68,12 +68,12 @@ jobs:
     steps:
       - ci_steps:
           platform: debian
-  # build-macos:
-  #   macos:
-  #     xcode: 10.2.1
-  #   steps:
-  #     - ci_steps:
-  #         platform: macosx
+  build-macos:
+    macos:
+      xcode: 10.2.1
+    steps:
+      - ci_steps:
+          platform: macosx
   build-multiarch-docker:
     machine:
       enabled: true
@@ -159,10 +159,13 @@ workflows:
           filters:
             tags:
               only: /.*/
-      # - build-macos:
-      #     filters:
-      #       tags:
-      #         only: /.*/
+      - build-macos:
+          filters:
+            branches:
+              # only: /.*/
+              ignore: /.*/
+            tags:
+              only: /^v[0-9].*/
       #- build-multiarch-docker:
       #    filters:
       #      tags:
@@ -198,3 +201,14 @@ workflows:
           filters:
             branches:
               only: master
+
+  nightly:
+    triggers:
+      - schedule:
+          cron: "20 17 * * *"
+          filters:
+             branches:
+               only:
+                 - master
+    jobs:
+      - build-macos

get_deps.sh

@@ -120,7 +120,7 @@ if [[ $WITH_TF != 0 ]]; then
 
 		LIBTF_ARCHIVE=libtensorflow-${TF_BUILD}-${TF_OS}-${TF_ARCH}-${TF_VERSION}.tar.gz
 
-		[[ ! -f $LIBTF_ARCHIVE || $FORCE == 1 ]] && wget --quiet $LIBTF_URL_BASE/$LIBTF_ARCHIVE
+		[[ ! -f $LIBTF_ARCHIVE || $FORCE == 1 ]] && wget -q $LIBTF_URL_BASE/$LIBTF_ARCHIVE
 
 		rm -rf $LIBTENSORFLOW.x
 		mkdir $LIBTENSORFLOW.x
@@ -169,7 +169,7 @@ if [[ $WITH_TFLITE != 0 ]]; then
 
 		LIBTFLITE_ARCHIVE=libtensorflowlite-${TFLITE_OS}-${TFLITE_ARCH}-${TFLITE_VERSION}.tar.gz
 
-		[[ ! -f $LIBTFLITE_ARCHIVE || $FORCE == 1 ]] && wget $LIBTF_URL_BASE/$LIBTFLITE_ARCHIVE
+		[[ ! -f $LIBTFLITE_ARCHIVE || $FORCE == 1 ]] && wget -q $LIBTF_URL_BASE/$LIBTFLITE_ARCHIVE
 
 		rm -rf $LIBTFLITE.x
 		mkdir $LIBTFLITE.x

opt/readies/bin/platform

@@ -12,6 +12,7 @@ parser = argparse.ArgumentParser(description='Report platform characteristics.')
 parser.add_argument('--os', action="store_true", help='Operating system')
 parser.add_argument('--version', action="store_true", help='OS/Distribution version')
 parser.add_argument('--dist', action="store_true", help='Linux distribution (if applicable)')
+parser.add_argument('--osnick', action="store_true", help='OS/distribution/version is a single word')
 parser.add_argument('--arch', action="store_true", help='CPU Architecture')
 parser.add_argument('--kernel', action="store_true", help='Kernel version (if applicable)')
 parser.add_argument('--glibc', action="store_true", help='GLIBC version (if applicable)')
@@ -26,6 +27,8 @@ except:
 ret = ""
 if args.os:
 	ret += " " + platform.os
+if args.osnick:
+	ret += " " + platform.osnick
 if args.dist:
 	ret += " " + platform.dist
 if args.version:
@@ -41,5 +44,9 @@ if ret == "":
     dist = platform.dist
     if dist != "":
         os = dist + " " + os
-    ret = os + " " + platform.os_ver + " " + platform.arch
+    if platform.osnick != "":
+        nick = " (" + platform.osnick + ")"
+    else:
+        nick = ""
+    ret = os + " " + platform.os_ver + nick + " " + platform.arch
 print(ret.strip())

opt/readies/paella/platform.py

@@ -24,7 +24,10 @@ def version(self):
             return self.defs["VERSION_ID"]
 
         def osnick(self):
-            return self.defs["VERSION_CODENAME"]
+            try:
+                return self.defs["VERSION_CODENAME"]
+            except:
+                return ""
 
     #------------------------------------------------------------------------------------------
 
@@ -42,6 +45,9 @@ def __init__(self, strict=False):
                     os_release = Platform.OSRelease()
                     distname = os_release.distname()
                     self.os_ver = self.full_os_ver = os_release.version()
+                    self.osnick = os_release.osnick()
+                    if self.osnick == "":
+                        self.osnick = distname + str(self.os_ver)
                 except:
                     if strict:
                         assert(False), "Cannot determine distribution"
@@ -55,6 +61,9 @@ def __init__(self, strict=False):
                 distname = 'redhat'
             elif distname.startswith('suse'):
                 distname = 'suse'
+            elif distname.startswith('amzn'):
+                distname = 'amzn'
+                self.osnick = 'amzn' + str(os_release.version())
             else:
                 if strict:
                     assert(False), "Cannot determine distribution"
@@ -66,6 +75,7 @@ def __init__(self, strict=False):
             self.full_os_ver = mac_ver[0] # e.g. 10.14, but also 10.5.8
             self.os_ver = '.'.join(self.full_os_ver.split('.')[:2]) # major.minor
             # self.arch = mac_ver[2] # e.g. x64_64
+            self.osnick = self.os + str(self.full_os_ver.split('.')[1])
         elif self.os == 'windows':
             self.dist = self.os
             self.os_ver = platform.release()
@@ -94,7 +104,7 @@ def is_debian_compat(self):
         return self.dist == 'debian' or self.dist == 'ubuntu'
 
     def is_redhat_compat(self):
-        return self.dist == 'redhat' or self.dist == 'centos'
+        return self.dist == 'redhat' or self.dist == 'centos' or self.dist == 'amzn'
 
     def is_container(self):
         with open('/proc/1/cgroups', 'r') as conf:
@@ -144,6 +154,8 @@ def invoke(self):
                     self.suse()
                 elif dist == 'arch':
                     self.arch()
+                elif dist == 'amzn':
+                    self.amzn()
                 else:
                     assert(False), "Cannot determine installer"
             elif os == 'macosx':
@@ -201,3 +213,6 @@ def bsd_compat(self):
 
     def freebsd(self):
         pass
+
+    def amzn(self):
+        pass

opt/readies/paella/setup.py

@@ -61,16 +61,13 @@ def __init__(self, nop=False):
         self.dist = self.platform.dist
         self.ver = self.platform.os_ver
 
-        if self.has_command("python3"):
-            self.python = "python3"
-        elif self.has_command("python"):
-            self.python = "python"
-        elif self.has_command("python2"):
-            self.python = "python2"
+        self.python = sys.executable
 
         if self.os == 'macosx':
             # this is required because osx pip installed are done with --user
-            os.environ["PATH"] = os.environ["PATH"] + ':' + '$HOME/Library/Python/2.7/bin'
+            os.environ["PATH"] = os.environ["PATH"] + ':' + os.environ["HOME"] + '/Library/Python/2.7/bin'
+            # this prevents brew updating before each install
+            os.environ["HOMEBREW_NO_AUTO_UPDATE"] = "1"
 
         if self.platform.is_debian_compat():
             # prevents apt-get from interactively prompting
@@ -118,11 +115,11 @@ def brew_install(self, packs, group=False, _try=False):
 
     def install(self, packs, group=False, _try=False):
         if self.os == 'linux':
-            if self.dist == 'fedora':
+            if self.dist == 'fedora': # also include centos 8
                 self.dnf_install(packs, group=group, _try=_try)
-            elif self.dist == 'ubuntu' or self.dist == 'debian':
+            elif self.platform.is_debian_compat():
                 self.apt_install(packs, group=group, _try=_try)
-            elif self.dist == 'centos' or self.dist == 'redhat':
+            elif self.platform.is_redhat_compat():
                 self.yum_install(packs, group=group, _try=_try)
             elif self.dist == 'suse':
                 self.zypper_install(packs, group=group, _try=_try)
@@ -135,48 +132,48 @@ def install(self, packs, group=False, _try=False):
         else:
             Assert(False), "Cannot determine installer"
 
-    def group_install(self, packs):
-        self.install(packs, group=True)
+    def group_install(self, packs, _try=False):
+        self.install(packs, group=True, _try=_try)
 
     #------------------------------------------------------------------------------------------
 
-    def yum_add_repo(self, repourl, repo=""):
+    def yum_add_repo(self, repourl, repo="", _try=False):
         if not self.has_command("yum-config-manager"):
             self.install("yum-utils")
-        self.run("yum-config-manager -y --add-repo {}".format(repourl))
+        self.run("yum-config-manager -y --add-repo {}".format(repourl), _try=_try)
 
-    def apt_add_repo(self, repourl, repo=""):
+    def apt_add_repo(self, repourl, repo="", _try=False):
         if not self.has_command("yum-config-manager"):
             self.install("software-properties-common")
-        self.run("add-apt-repository -y {}".format(repourl))
-        self.run("apt-get -qq update")
+        self.run("add-apt-repository -y {}".format(repourl), _try=_try)
+        self.run("apt-get -qq update", _try=_try)
 
-    def dnf_add_repo(self, repourl, repo=""):
+    def dnf_add_repo(self, repourl, repo="", _try=False):
         if self.run("dnf config-manager 2>/dev/null", _try=True):
-            self.install("dnf-plugins-core")
-        self.run("dnf config-manager -y --add-repo {}".format(repourl))
+            self.install("dnf-plugins-core", _try=_try)
+        self.run("dnf config-manager -y --add-repo {}".format(repourl), _try=_try)
 
-    def zypper_add_repo(self, repourl, repo=""):
+    def zypper_add_repo(self, repourl, repo="", _try=False):
         pass
 
-    def pacman_add_repo(self, repourl, repo=""):
+    def pacman_add_repo(self, repourl, repo="", _try=False):
         pass
 
-    def brew_add_repo(self, repourl, repo=""):
+    def brew_add_repo(self, repourl, repo="", _try=False):
         pass
 
-    def add_repo(self, repourl, repo=""):
+    def add_repo(self, repourl, repo="", _try=False):
         if self.os == 'linux':
             if self.dist == 'fedora':
-                self.dnf_add_repo(repourl, repo=repo)
+                self.dnf_add_repo(repourl, repo=repo, _try=_try)
             elif self.dist == 'ubuntu' or self.dist == 'debian':
-                self.apt_add_repo(repourl, repo=repo)
+                self.apt_add_repo(repourl, repo=repo, _try=_try)
             elif self.dist == 'centos' or self.dist == 'redhat':
-                self.yum_add_repo(repourl, repo=repo)
+                self.yum_add_repo(repourl, repo=repo, _try=_try)
             elif self.dist == 'suse':
-                self.zypper_add_repo(repourl, repo=repo)
+                self.zypper_add_repo(repourl, repo=repo, _try=_try)
             elif self.dist == 'arch':
-                self.pacman_add_repo(repourl, repo=repo)
+                self.pacman_add_repo(repourl, repo=repo, _try=_try)
             else:
                 Assert(False), "Cannot determine installer"
         elif self.os == 'macosx':
@@ -190,26 +187,35 @@ def pip_install(self, cmd, _try=False):
         pip_user = ''
         if self.os == 'macosx':
             pip_user = '--user '
-        self.run("pip install --disable-pip-version-check " + pip_user + cmd, output_on_error=True, _try=_try)
+        self.run(self.python + " -m pip install --disable-pip-version-check " + pip_user + cmd, output_on_error=True, _try=_try)
 
     def pip3_install(self, cmd, _try=False):
         pip_user = ''
         if self.os == 'macosx':
             pip_user = '--user '
-        self.run("pip3 install --disable-pip-version-check " + pip_user + cmd, output_on_error=True, _try=_try)
+        self.run(self.python + " -m pip install --disable-pip-version-check " + pip_user + cmd, output_on_error=True, _try=_try)
 
-    def setup_pip(self):
+    def setup_pip(self, _try=False):
         get_pip = "set -e; wget -q https://bootstrap.pypa.io/get-pip.py -O /tmp/get-pip.py"
-        if not self.has_command("pip3"):
-            self.install("python3-distutils", _try=True)
+        if self.run(self.python + " -m pip --version", _try=True, output_on_error=False) != 0:
+            if sys.version_info.major == 3:
+                self.install("python3-distutils")
             self.install_downloaders()
-            self.run(get_pip + "; " + self.python + " /tmp/get-pip.py", output_on_error=True)
+            pip_user = ' --user' if self.os == 'macosx' else ''
+            self.run(get_pip + "; " + self.python + " /tmp/get-pip.py" + pip_user + " 'pip==19.3.1'", output_on_error=True, _try=_try)
 
-    def install_downloaders(self):
+    def install_downloaders(self, _try=False):
         if self.os == 'linux':
-            self.install("ca-certificates")
-        self.install("curl wget")
-
-    def install_git_lfs_on_linux(self):
-        self.run("curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash")
-        self.install("git-lfs")
+            self.install("ca-certificates", _try=_try)
+        self.install("curl wget", _try=_try)
+
+    def install_git_lfs_on_linux(self, _try=False):
+        self.run("set -e; wget -q https://github.com/git-lfs/git-lfs/releases/download/v2.9.2/git-lfs-linux-amd64-v2.9.2.tar.gz -O /tmp/git-lfs.tar.gz")
+        self.run("cd /tmp; tar xzf git-lfs.tar.gz; ./install.sh")
+
+#        cmd = "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.{}.sh | bash"
+#        if self.platform.is_redhat_compat():
+#            self.run(cmd.format('rpm'), _try=_try)
+#        elif self.platform.is_debian_compat():
+#            self.run(cmd.format('deb'), _try=_try)
+#        self.install("git-lfs", _try=_try)

opt/system-setup.py

@@ -34,9 +34,14 @@ def redhat_compat(self):
         self.group_install("'Development Tools'")
         self.install("redhat-lsb-core")
 
-        self.install("epel-release")
-        self.install("python36 python36-pip")
-        self.install("python36-psutil")
+        if not self.dist == "amzn":
+            self.install("epel-release")
+            self.install("python36 python36-pip")
+            self.install("python36-psutil")
+        else:
+            self.run("amazon-linux-extras install epel", output_on_error=True)
+            self.install("python3 python3-devel")
+            self.pip_install("psutil")
 
         self.install_git_lfs_on_linux()
 
@@ -53,10 +58,6 @@ def macosx(self):
         self.install("git-lfs")
         self.install("redis")
 
-    def install_git_lfs_on_linux(self):
-        self.run("curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash")
-        self.install("git-lfs")
-
     def common_last(self):
         if not self.has_command("RLTest"):
             self.pip3_install("git+https://github.com/RedisLabsModules/RLTest.git@master")