Make new pip package for building/repacking client templates, reforma…

…t proto files, make rel-db flows more efficient + other changes.

.travis.yml

@@ -31,7 +31,7 @@ matrix:
         - pip install --upgrade virtualenv
         - virtualenv "${HOME}/INSTALL"
       install:
-        - travis/install_client_only.sh
+        - travis/install_client_builder.sh
       script:
         - source "${HOME}/INSTALL/bin/activate"
         - travis/build_templates.sh
@@ -55,7 +55,7 @@ matrix:
         - pip install --upgrade virtualenv
         - virtualenv "${HOME}/INSTALL"
       install:
-        - travis/install_client_only.sh
+        - travis/install_client_builder.sh
       script:
         - source "${HOME}/INSTALL/bin/activate"
         - travis/build_templates.sh
@@ -96,7 +96,7 @@ matrix:
       install:
         - >-
           sudo docker exec --user "${DOCKER_USER}" "${DOCKER_CONTAINER}"
-          travis/install_client_only.sh
+          travis/install_client_builder.sh
       script:
         - >-
           sudo docker exec --user "${DOCKER_USER}" "${DOCKER_CONTAINER}"
@@ -141,7 +141,7 @@ matrix:
       install:
         - >-
           sudo docker exec --user "${DOCKER_USER}" "${DOCKER_CONTAINER}"
-          travis/install_client_only.sh
+          travis/install_client_builder.sh
       script:
         - >-
           sudo docker exec --user "${DOCKER_USER}" "${DOCKER_CONTAINER}"
@@ -185,7 +185,7 @@ matrix:
       install:
         - >-
           sudo docker exec --user "${DOCKER_USER}" "${DOCKER_CONTAINER}"
-          linux32 travis/install_client_only.sh
+          linux32 travis/install_client_builder.sh
       script:
         - >-
           sudo docker exec --user "${DOCKER_USER}" "${DOCKER_CONTAINER}"
@@ -229,7 +229,7 @@ matrix:
       install:
         - >-
           sudo docker exec --user "${DOCKER_USER}" "${DOCKER_CONTAINER}"
-          linux32 travis/install_client_only.sh
+          linux32 travis/install_client_builder.sh
       script:
         - >-
           sudo docker exec --user "${DOCKER_USER}" "${DOCKER_CONTAINER}"

appveyor/windows_templates/build_windows_templates.py

@@ -26,25 +26,6 @@
     default=r"C:\grrbuild\output",
     help="Destination directory for the templates.")
 
-parser.add_argument(
-    "--cloud_storage_sdist_bucket",
-    default=None,
-    help="If defined, copy sdists from this bucket rather than"
-    " building them.")
-
-parser.add_argument(
-    "--cloud_storage_output_bucket",
-    default=None,
-    help="If defined, build products will be copied to this "
-    "cloud storage bucket.")
-
-parser.add_argument(
-    "--gsutil",
-    default=(r"C:\Program Files (x86)\Google\Cloud "
-             r"SDK\google-cloud-sdk\bin\gsutil.cmd"),
-    help="gsutil binary. The default is the SDK install location since that's "
-    "likely the one the user has authorized their creds for")
-
 parser.add_argument(
     "--test_repack_install",
     action="store_true",
@@ -203,32 +184,15 @@ def MakeClientSdist(self):
     return glob.glob(os.path.join(args.build_dir,
                                   "grr-response-client-*.zip")).pop()
 
-  def CopySdistsFromCloudStorage(self):
-    """Use gsutil to copy sdists from cloud storage."""
-    subprocess.check_call([
-        args.gsutil, "cp",
-        "gs://%s/grr-response-proto-*.zip" % args.cloud_storage_sdist_bucket,
-        args.build_dir
-    ])
-    proto = glob.glob(os.path.join(args.build_dir,
-                                   "grr-response-proto-*.zip")).pop()
-
-    subprocess.check_call([
-        args.gsutil, "cp",
-        "gs://%s/grr-response-core-*.zip" % args.cloud_storage_sdist_bucket,
-        args.build_dir
-    ])
-    core = glob.glob(os.path.join(args.build_dir,
-                                  "grr-response-core-*.zip")).pop()
-
+  def MakeClientBuilderSdist(self):
+    os.chdir(os.path.join(args.grr_src, "grr/client_builder/"))
     subprocess.check_call([
-        args.gsutil, "cp",
-        "gs://%s/grr-response-client-*.zip" % args.cloud_storage_sdist_bucket,
-        args.build_dir
+        self.virtualenv_python64, "setup.py", "sdist", "--formats=zip",
+        "--dist-dir=%s" % args.build_dir
     ])
-    client = glob.glob(
-        os.path.join(args.build_dir, "grr-response-client-*.zip")).pop()
-    return proto, core, client
+    return glob.glob(
+        os.path.join(args.build_dir,
+                     "grr-response-client-builder-*.zip")).pop()
 
   def InstallGRR(self, path):
     """Installs GRR."""
@@ -346,33 +310,26 @@ def _InstallInstallers(self):
     subprocess.check_call([installer_amd64])
     self._CheckInstallSuccess()
 
-  def CopyResultsToCloudStorage(self):
-    paths = glob.glob("%s\\*" % args.output_dir)
-    subprocess.check_call([args.gsutil, "-m", "cp"] + paths +
-                          ["gs://%s/" % args.cloud_storage_output_bucket])
-
   def Build(self):
     """Build templates."""
     self.SetupVars()
     self.Clean()
-    if args.cloud_storage_sdist_bucket:
-      proto_sdist, core_sdist, client_sdist = self.CopySdistsFromCloudStorage()
-    else:
-      if not os.path.exists(args.grr_src):
-        self.GitCheckoutGRR()
-      proto_sdist = self.MakeProtoSdist()
-      core_sdist = self.MakeCoreSdist()
-      client_sdist = self.MakeClientSdist()
+
+    if not os.path.exists(args.grr_src):
+      self.GitCheckoutGRR()
+    proto_sdist = self.MakeProtoSdist()
+    core_sdist = self.MakeCoreSdist()
+    client_sdist = self.MakeClientSdist()
+    client_builder_sdist = self.MakeClientBuilderSdist()
 
     self.InstallGRR(proto_sdist)
     self.InstallGRR(core_sdist)
     self.InstallGRR(client_sdist)
+    self.InstallGRR(client_builder_sdist)
     self.BuildTemplates()
     if args.test_repack_install:
       self._RepackTemplates()
       self._InstallInstallers()
-    if args.cloud_storage_output_bucket:
-      self.CopyResultsToCloudStorage()
 
 
 def main():

debian/rules

@@ -67,5 +67,5 @@ override_dh_virtualenv:
 		--skip-install \
 		--preinstall "${LOCAL_DEB_PYINDEX}/${API_SDIST}" \
 		--preinstall "${LOCAL_DEB_PYINDEX}/${TEMPLATES_SDIST}" \
+		--preinstall "${LOCAL_DEB_PYINDEX}/${CLIENT_BUILDER_SDIST}" \
 		--preinstall "${LOCAL_DEB_PYINDEX}/${SERVER_SDIST}" \
-		--preinstall "${LOCAL_DEB_PYINDEX}/${CLIENT_SDIST}"

grr/client/grr_response_client/distro_entry.py

@@ -9,11 +9,6 @@
 # pylint: disable=g-import-not-at-top
 
 
-def ClientBuild():
-  from grr_response_client import client_build
-  client_build.Run()
-
-
 def Client():
   from grr_response_client import client
   app.run(client.main)

grr/client/setup.py

@@ -77,7 +77,6 @@ def make_release_tree(self, base_dir, files):
             "grr_client = grr_response_client.distro_entry:Client",
             ("grr_fleetspeak_client = "
              "grr_response_client.distro_entry:FleetspeakClient"),
-            "grr_client_build = grr_response_client.distro_entry:ClientBuild",
             "grr_pool_client = grr_response_client.distro_entry:PoolClient"
         ]
     },

grr/client_builder/grr_response_client_builder/__init__.py

@@ -0,0 +1,2 @@
+#!/usr/bin/env python
+

grr/core/grr_response_core/lib/build.py → grr/client_builder/grr_response_client_builder/build.py

@@ -47,10 +47,8 @@
 from grr_response_core import config
 from grr_response_core import version
 from grr_response_core.config import contexts
-from grr_response_core.lib import config_lib
 from grr_response_core.lib import config_validator_base
 from grr_response_core.lib import rdfvalue
-from grr_response_core.lib import registry
 from grr_response_core.lib import utils
 # Pull in local config validators.
 from grr_response_core.lib.local import plugins
@@ -246,20 +244,17 @@ def CopyMissingModules(self):
   def MakeExecutableTemplate(self, output_file=None):
     """Create the executable template.
 
-    Args:
-      output_file: string filename where we will write the template.
-
-    The client is build in two phases. First an executable template is created
+    The client is built in two phases. First an executable template is created
     with the client binaries contained inside a zip file. Then the installation
     package is created by appending the SFX extractor to this template and
-    writing a config file into the zip file.
-
-    This technique allows the client build to be carried out once on the
-    supported platform (e.g. windows with MSVS), but the deployable installer
-    can be build on any platform which supports python.
+    writing a config file into the zip file.  This technique allows the
+    client build to be carried out once on the supported platform (e.g.
+    windows with MSVS), but the deployable installer can be build on any
+    platform which supports python.  Subclasses for each OS do the actual
+    work, we just make sure the output directory is set up correctly here.
 
-    Subclasses for each OS do the actual work, we just make sure the output
-    directory is set up correctly here.
+    Args:
+      output_file: string filename where we will write the template.
     """
     self.template_file = output_file or config.CONFIG.Get(
         "ClientBuilder.template_path", context=self.context)
@@ -564,6 +559,7 @@ def MakeSelfExtractingZip(self, payload_data, output_path):
     Args:
       payload_data: data payload for zip file
       output_path: filename for the zip output
+
     Raises:
       RuntimeError: if the ClientBuilder.unzipsfx_stub doesn't require admin.
     Returns:

grr/core/grr_response_core/lib/build_test.py → grr/client_builder/grr_response_client_builder/build_test.py

@@ -13,8 +13,8 @@
 from future.builtins import str
 import mock
 
+from grr_response_client_builder import build
 from grr_response_core import config
-from grr_response_core.lib import build
 from grr_response_core.lib import config_lib
 from grr_response_core.lib.rdfvalues import client as rdf_client
 from grr_response_core.lib.util import yaml

grr/core/grr_response_core/lib/builders/__init__.py → grr/client_builder/grr_response_client_builder/builders/__init__.py

@@ -9,14 +9,14 @@
 # pylint: disable=unused-import,g-import-not-at-top,g-bad-name
 
 if platform.system() == "Linux":
-  from grr_response_core.lib.builders import linux
+  from grr_response_client_builder.builders import linux
   LinuxClientBuilder = linux.LinuxClientBuilder
   CentosClientBuilder = linux.CentosClientBuilder
 
 elif platform.system() == "Windows":
-  from grr_response_core.lib.builders import windows
+  from grr_response_client_builder.builders import windows
   WindowsClientBuilder = windows.WindowsClientBuilder
 
 elif platform.system() == "Darwin":
-  from grr_response_core.lib.builders import osx
+  from grr_response_client_builder.builders import osx
   DarwinClientBuilder = osx.DarwinClientBuilder

grr/core/grr_response_core/lib/builders/linux.py → grr/client_builder/grr_response_client_builder/builders/linux.py

@@ -11,8 +11,8 @@
 import subprocess
 import zipfile
 
+from grr_response_client_builder import build
 from grr_response_core import config
-from grr_response_core.lib import build
 from grr_response_core.lib import config_lib
 from grr_response_core.lib import package
 from grr_response_core.lib import utils
@@ -44,8 +44,8 @@ def BuildWithPyInstaller(self):
     self.StripLibraries(self.output_dir)
 
   def MakeExecutableTemplate(self, output_file=None):
-    super(LinuxClientBuilder, self).MakeExecutableTemplate(
-        output_file=output_file)
+    super(LinuxClientBuilder,
+          self).MakeExecutableTemplate(output_file=output_file)
     self.MakeBuildDirectory()
     self.CleanDirectory(
         config.CONFIG.Get("PyInstaller.dpkg_root", context=self.context))
@@ -74,20 +74,23 @@ def CopyFiles(self):
     # Copy upstart files
     outdir = os.path.join(dpkg_dir, "debian/upstart.in")
     utils.EnsureDirExists(outdir)
-    shutil.copy(config_lib.Resource().Filter(
-        "install_data/debian/dpkg_client/upstart/grr-client.conf"), outdir)
+    shutil.copy(
+        config_lib.Resource().Filter(
+            "install_data/debian/dpkg_client/upstart/grr-client.conf"), outdir)
 
     # Copy init files
     outdir = os.path.join(dpkg_dir, "debian/initd.in")
     utils.EnsureDirExists(outdir)
-    shutil.copy(config_lib.Resource().Filter(
-        "install_data/debian/dpkg_client/initd/grr-client"), outdir)
+    shutil.copy(
+        config_lib.Resource().Filter(
+            "install_data/debian/dpkg_client/initd/grr-client"), outdir)
 
     # Copy systemd unit file
     outdir = os.path.join(dpkg_dir, "debian/systemd.in")
     utils.EnsureDirExists(outdir)
-    shutil.copy(config_lib.Resource().Filter(
-        "install_data/systemd/client/grr-client.service"), outdir)
+    shutil.copy(
+        config_lib.Resource().Filter(
+            "install_data/systemd/client/grr-client.service"), outdir)
 
   def MakeZip(self, input_dir, output_file):
     """Creates a ZIP archive of the files in the input directory.
@@ -126,11 +129,13 @@ def CopyFiles(self):
         os.path.join(build_dir, "rpmbuild/grr-client.initd.in"))
 
     # Copy systemd unit file
-    shutil.copy(config_lib.Resource().Filter(
-        "install_data/systemd/client/grr-client.service"),
-                os.path.join(build_dir, "rpmbuild/grr-client.service.in"))
+    shutil.copy(
+        config_lib.Resource().Filter(
+            "install_data/systemd/client/grr-client.service"),
+        os.path.join(build_dir, "rpmbuild/grr-client.service.in"))
 
     # Copy prelink blacklist file
-    shutil.copy(config_lib.Resource().Filter(
-        "install_data/centos/prelink_blacklist.conf.in"),
-                os.path.join(build_dir, "rpmbuild/prelink_blacklist.conf.in"))
+    shutil.copy(
+        config_lib.Resource().Filter(
+            "install_data/centos/prelink_blacklist.conf.in"),
+        os.path.join(build_dir, "rpmbuild/prelink_blacklist.conf.in"))

grr/core/grr_response_core/lib/builders/osx.py → grr/client_builder/grr_response_client_builder/builders/osx.py

@@ -12,8 +12,8 @@
 import subprocess
 import zipfile
 
+from grr_response_client_builder import build
 from grr_response_core import config
-from grr_response_core.lib import build
 from grr_response_core.lib import package
 from grr_response_core.lib import utils
 

grr/core/grr_response_core/lib/builders/signing.py → grr/client_builder/grr_response_client_builder/builders/signing.py

@@ -90,6 +90,7 @@ def SignBuffer(self, in_buffer):
 
     Args:
       in_buffer: data to sign
+
     Returns:
       signed data
     """
@@ -107,7 +108,8 @@ def SignFile(self, in_filename, out_filename=None):
     Args:
       in_filename: file to read from
       out_filename: file to output to, if none we output to the same filename as
-                    the input with a .signed suffix.
+        the input with a .signed suffix.
+
     Returns:
       output filename string
     Raises:

grr/core/grr_response_core/lib/builders/signing_test.py → grr/client_builder/grr_response_client_builder/builders/signing_test.py

@@ -15,7 +15,7 @@
 import mock
 import pexpect
 
-from grr_response_core.lib.builders import signing
+from grr_response_client_builder.builders import signing
 from grr.test_lib import test_lib
 
 

grr/core/grr_response_core/lib/builders/windows.py → grr/client_builder/grr_response_client_builder/builders/windows.py

@@ -15,8 +15,8 @@
 
 import win32process
 
+from grr_response_client_builder import build
 from grr_response_core import config
-from grr_response_core.lib import build
 
 MODULE_PATTERNS = [
     # Visual Studio runtime libs.
@@ -50,9 +50,9 @@ def EnumMissingModules():
   """
   module_handle = ctypes.c_ulong()
   count = ctypes.c_ulong()
-  process_handle = ctypes.windll.kernel32.OpenProcess(PROCESS_QUERY_INFORMATION
-                                                      | PROCESS_VM_READ, 0,
-                                                      os.getpid())
+  process_handle = ctypes.windll.kernel32.OpenProcess(
+      PROCESS_QUERY_INFORMATION
+      | PROCESS_VM_READ, 0, os.getpid())
   ctypes.windll.psapi.EnumProcessModules(process_handle,
                                          ctypes.byref(module_handle),
                                          ctypes.sizeof(module_handle),