build: create release package for CLIs (#3635)

Summary:

The current release mechanism only allows releasing everything.
Introduce a new option to release just the CLIs: cqlsh, redis-cli, and
ysqlsh.

Do so by creating a package level in `yb_release_manifest.json`, as
suggested by Bogdan Matican.  At that level, split off to `all`,
containing the old configuration, and `cli`, containing only what's
necessary for the CLIs.  Update all the scripts to respect this new
manifest structure.  Don't change build: regardless of release package,
build the whole `packaged` target.

Also, move `postgres/bin/*` from `get_seed_executable_patterns` to
`yb_release_manifest.json` because it seems to work and is more uniform.

Depends on D7970

Close: #3635

Test Plan:

```sh
./yb_release --force
# verify a release tar.gz is created that has full features as before
git clean -dX # use at your own risk
./yb_release --force --package_name all
# verify a release tar.gz is created that has full features as before
git clean -dX # use at your own risk
./yb_release --force --package_name cli
tar xzf build/yugabyte-*.tar.gz -C /tmp
cd /tmp/yugabyte-*
if [ "$(uname)" = linux ]; then
  ./bin/post_install.sh
fi
# create a cluster with default IP and port
./bin/cqlsh --execute "SHOW VERSION"
./bin/redis-cli "INFO"
./bin/ysqlsh --command "SELECT version()"
```

Reviewers: mikhail, neha, sanketh, bogdan

Reviewed By: bogdan

Subscribers: jenkins-bot, ybase

Differential Revision: https://phabricator.dev.yugabyte.com/D8031

build-support/list_packaged_targets.py

@@ -4,8 +4,8 @@
 
 # Lists the targets that need to be built for a YB release.
 
-import os
 import json
+import os
 import re
 
 
@@ -15,7 +15,7 @@
 
 def main():
     with open(os.path.join(YB_SRC_ROOT, 'yb_release_manifest.json')) as manifest_input_file:
-        release_manifest = json.loads(manifest_input_file.read())
+        release_manifest = json.loads(manifest_input_file.read())['all']
     found_matches = False
     for bin_path in release_manifest['bin']:
         match = LATEST_BINARY_RE.match(bin_path)

build-support/yb_release.py

@@ -63,6 +63,9 @@ def main():
     parser.add_argument('--no_reinitdb', action='store_true',
                         help='Do not re-create the initial sys catalog snapshot. Useful when '
                              'debugging the release process.')
+    parser.add_argument('--package_name',
+                        default='all',
+                        help='Name of package to release (e.g. cli).')
     add_common_arguments(parser)
     args = parser.parse_args()
 
@@ -209,7 +212,8 @@ def main():
     # This points to the release manifest within the release_manager, and we are modifying that
     # directly.
     release_util = ReleaseUtil(
-            YB_SRC_ROOT, build_type, build_target, args.force, args.commit, build_root)
+        YB_SRC_ROOT, build_type, build_target, args.force, args.commit, build_root,
+        args.package_name)
 
     system = platform.system().lower()
     library_packager_args = dict(

python/yb/release_util.py

@@ -25,8 +25,8 @@
 
 class ReleaseUtil(object):
     """Packages a YugaByte package with the appropriate file naming schema."""
-    def __init__(self, repository, build_type, distribution_path, force, commit,
-                 build_root):
+    def __init__(self, repository, build_type, distribution_path, force, commit, build_root,
+                 package_name):
         self.repo = repository
         self.build_type = build_type
         self.build_path = os.path.join(self.repo, 'build')
@@ -41,7 +41,7 @@ def __init__(self, repository, build_type, distribution_path, force, commit,
             'Unable to read {0} file'.format(RELEASE_VERSION_FILE)
 
         with open(os.path.join(self.repo, RELEASE_MANIFEST_NAME)) as f:
-            self.release_manifest = json.load(f)
+            self.release_manifest = json.load(f)[package_name]
         assert self.release_manifest is not None, \
             'Unable to read {0} file'.format(RELEASE_MANIFEST_NAME)
         self.build_root = build_root
@@ -55,8 +55,7 @@ def get_release_manifest(self):
         return self.release_manifest
 
     def get_seed_executable_patterns(self):
-        return self.release_manifest['bin'] + [
-            os.path.join(self.build_root, 'postgres', 'bin', '*')]
+        return self.release_manifest['bin']
 
     def expand_value(self, old_value):
         """

yb_release_manifest.json

@@ -1,43 +1,62 @@
 {
-  "%symlinks%": {
-    "bin/psql": "../postgres/bin/ysqlsh",
-    "bin/ysqlsh": "../postgres/bin/ysqlsh"
+  "all": {
+    "%symlinks%": {
+      "bin/psql": "../postgres/bin/ysqlsh",
+      "bin/ysqlsh": "../postgres/bin/ysqlsh"
+    },
+    ".": ["$BUILD_ROOT/version_metadata.json"],
+    "bin": [
+      "$BUILD_ROOT/bin/ldb",
+      "$BUILD_ROOT/bin/log-dump",
+      "$BUILD_ROOT/bin/sst_dump",
+      "$BUILD_ROOT/bin/yb-admin",
+      "$BUILD_ROOT/bin/yb-bulk_load",
+      "$BUILD_ROOT/bin/yb-generate_partitions_main",
+      "$BUILD_ROOT/bin/yb-master",
+      "$BUILD_ROOT/bin/yb-pbc-dump",
+      "$BUILD_ROOT/bin/yb-ts-cli",
+      "$BUILD_ROOT/bin/yb-tserver",
+      "$BUILD_ROOT/bin/yb-ysck",
+      "$BUILD_ROOT/bin/yb_load_test_tool",
+      "$BUILD_ROOT/postgres/bin/*",
+      "$THIRDPARTY_BUILD_SPECIFIC_DIR/bin/pprof",
+      "bin/bulk_load_cleanup.sh",
+      "bin/bulk_load_helper.sh",
+      "bin/configure",
+      "bin/log_cleanup.sh",
+      "bin/yb-check-consistency.py",
+      "bin/yb-check-failed-tablets.sh",
+      "bin/yb-prof.py",
+      "bin/yugabyted",
+      "build-support/post_install.sh",
+      "submodules/yugabyte-installation/bin/yb-ctl",
+      "thirdparty/installed/common/bin/redis-cli",
+      "thirdparty/installed/common/cqlsh/bin/cqlsh",
+      "thirdparty/installed/common/cqlsh/bin/cqlsh.py"
+    ],
+    "lib": ["thirdparty/installed/common/cqlsh/lib/*"],
+    "postgres": ["$BUILD_ROOT/postgres/*"],
+    "pylib": ["thirdparty/installed/common/cqlsh/pylib/*"],
+    "share": [
+      "$BUILD_ROOT/share/initial_sys_catalog_snapshot",
+      "sample/*.sql"
+    ],
+    "www": ["www/*"]
   },
-  ".": ["$BUILD_ROOT/version_metadata.json"],
-  "bin": [
-    "$BUILD_ROOT/bin/ldb",
-    "$BUILD_ROOT/bin/log-dump",
-    "$BUILD_ROOT/bin/sst_dump",
-    "$BUILD_ROOT/bin/yb-admin",
-    "$BUILD_ROOT/bin/yb-bulk_load",
-    "$BUILD_ROOT/bin/yb-generate_partitions_main",
-    "$BUILD_ROOT/bin/yb-master",
-    "$BUILD_ROOT/bin/yb-pbc-dump",
-    "$BUILD_ROOT/bin/yb-ts-cli",
-    "$BUILD_ROOT/bin/yb-tserver",
-    "$BUILD_ROOT/bin/yb-ysck",
-    "$BUILD_ROOT/bin/yb_load_test_tool",
-    "$THIRDPARTY_BUILD_SPECIFIC_DIR/bin/pprof",
-    "bin/bulk_load_cleanup.sh",
-    "bin/bulk_load_helper.sh",
-    "bin/configure",
-    "bin/log_cleanup.sh",
-    "bin/yb-check-consistency.py",
-    "bin/yb-check-failed-tablets.sh",
-    "bin/yb-prof.py",
-    "bin/yugabyted",
-    "build-support/post_install.sh",
-    "submodules/yugabyte-installation/bin/yb-ctl",
-    "thirdparty/installed/common/bin/redis-cli",
-    "thirdparty/installed/common/cqlsh/bin/cqlsh",
-    "thirdparty/installed/common/cqlsh/bin/cqlsh.py"
-  ],
-  "lib": ["thirdparty/installed/common/cqlsh/lib/*"],
-  "postgres": ["$BUILD_ROOT/postgres/*"],
-  "pylib": ["thirdparty/installed/common/cqlsh/pylib/*"],
-  "share": [
-    "$BUILD_ROOT/share/initial_sys_catalog_snapshot",
-    "sample/*.sql"
-  ],
-  "www": ["www/*"]
+  "cli": {
+    "%symlinks%": {
+      "bin/psql": "../postgres/bin/ysqlsh",
+      "bin/ysqlsh": "../postgres/bin/ysqlsh"
+    },
+    ".": ["$BUILD_ROOT/version_metadata.json"],
+    "bin": [
+      "$BUILD_ROOT/postgres/bin/ysqlsh",
+      "build-support/post_install.sh",
+      "thirdparty/installed/common/bin/redis-cli",
+      "thirdparty/installed/common/cqlsh/bin/cqlsh",
+      "thirdparty/installed/common/cqlsh/bin/cqlsh.py"
+    ],
+    "lib": ["thirdparty/installed/common/cqlsh/lib/*"],
+    "pylib": ["thirdparty/installed/common/cqlsh/pylib/*"]
+  }
 }