More Python3-compatibility changes, add back the config directory + o…

…ther changes.
google · Sep 12, 2018 · e8766d1 · e8766d1
1 parent 7c3a0ab
commit e8766d1
Show file tree

Hide file tree

Showing 241 changed files with 610 additions and 137 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -20,7 +20,7 @@ build
 dist
 executables
 !executables/windows/templates/unzipsfx/*.exe
-grr/config/grr-response-templates
+grr/config/grr_response_templates
 grr/gui/static/bower_components
 grr/gui/static/node_modules
 grr/gui/static/tmp

diff --git a/grr/config/grr_response_templates/.gitignore b/grr/config/grr_response_templates/.gitignore
@@ -0,0 +1,2 @@
+components/
+templates/
diff --git a/grr/config/grr_response_templates/MANIFEST.in b/grr/config/grr_response_templates/MANIFEST.in
@@ -0,0 +1,3 @@
+include version.ini
+recursive-include components *.bin
+recursive-include templates *.zip
diff --git a/grr/config/grr_response_templates/index.html b/grr/config/grr_response_templates/index.html
@@ -0,0 +1,4 @@
+<html><head><title>Simple Index</title><meta name="api-version" value="2" /></head><body>
+<a href='grr-response-templates-3.1.0.tar.gz#md5=71525271a5fdbf0c72580ce56194d8a9'>grr-response-templates-3.1.0</a><br/>
+<a href="grr-response-templates-3.1.0post1.tar.gz#md5=93703b0a2c8fb385706970708af0b24b">grr-response-templates-3.1.0post1</a><br/>
+</body></html>
diff --git a/grr/config/grr_response_templates/setup.py b/grr/config/grr_response_templates/setup.py
@@ -0,0 +1,118 @@
+#!/usr/bin/env python
+"""This package contains GRR client templates."""
+from __future__ import print_function
+import ConfigParser
+import glob
+import os
+import re
+import shutil
+from setuptools import setup
+from setuptools.command.sdist import sdist
+
+THIS_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
+
+# If you run setup.py from the root GRR dir you get very different results since
+# setuptools uses the MANIFEST.in from the root dir.  Make sure we are in the
+# package dir.
+os.chdir(THIS_DIRECTORY)
+
+
+def get_config():
+  """Get INI parser with version.ini data."""
+  ini_path = os.path.join(THIS_DIRECTORY, "version.ini")
+  if not os.path.exists(ini_path):
+    ini_path = os.path.join(THIS_DIRECTORY, "../../../version.ini")
+    if not os.path.exists(ini_path):
+      raise RuntimeError("Couldn't find version.ini")
+
+  config = ConfigParser.SafeConfigParser()
+  config.read(ini_path)
+  return config
+
+
+VERSION = get_config()
+
+
+class Sdist(sdist):
+  """Make a sdist release."""
+
+  REQUIRED_TEMPLATES = [
+      "GRR_maj.minor_amd64.exe.zip",
+      "GRR_maj.minor_i386.exe.zip",
+      "grr_maj.minor_amd64.deb.zip",
+      "grr_maj.minor_amd64.xar.zip",
+      "grr_maj.minor_amd64.rpm.zip",
+      "grr_maj.minor_i386.deb.zip",
+      "grr_maj.minor_i386.rpm.zip",
+  ]
+
+  def CheckTemplates(self, base_dir, version):
+    """Verify we have at least one template that matches maj.minor version."""
+    major_minor = ".".join(version.split(".")[0:2])
+    templates = glob.glob(
+        os.path.join(base_dir, "templates/*%s*.zip" % major_minor))
+    required_templates = set(
+        [x.replace("maj.minor", major_minor) for x in self.REQUIRED_TEMPLATES])
+
+    # Client templates have an extra version digit, e.g. 3.1.0.0
+    templates_present = set([
+        re.sub(r"_%s[^_]+_" % major_minor, "_%s_" % major_minor,
+               os.path.basename(x)) for x in templates
+    ])
+
+    difference = required_templates - templates_present
+    if difference:
+      raise RuntimeError("Missing templates %s" % difference)
+
+  def run(self):
+    base_dir = os.getcwd()
+    self.CheckTemplates(base_dir, setup_args["version"])
+    sdist.run(self)
+    print("To upload a release, run upload.sh [version]")
+
+  def make_release_tree(self, base_dir, files):
+    sdist.make_release_tree(self, base_dir, files)
+    sdist_version_ini = os.path.join(base_dir, "version.ini")
+    if os.path.exists(sdist_version_ini):
+      os.unlink(sdist_version_ini)
+    shutil.copy(
+        os.path.join(THIS_DIRECTORY, "../../../version.ini"), sdist_version_ini)
+
+
+def find_data_files(source, prefix=None):
+  result = []
+  for directory, _, files in os.walk(source):
+    files = [os.path.join(directory, x) for x in files]
+    if prefix:
+      result.append((os.path.join(prefix, directory), files))
+    else:
+      result.append((directory, files))
+
+  return result
+
+
+if "VIRTUAL_ENV" not in os.environ:
+  print("*****************************************************")
+  print("  WARNING: You are not installing in a virtual")
+  print("  environment. This configuration is not supported!!!")
+  print("  Expect breakage.")
+  print("*****************************************************")
+
+setup_args = dict(
+    name="grr-response-templates",
+    version=VERSION.get("Version", "packageversion"),
+    description="GRR Rapid Response client templates",
+    long_description=("This PyPi package is just a placeholder. The package"
+                      " itself is too large to distribute on PyPi so it is "
+                      "available from google cloud storage. See"
+                      " https://github.com/google/grr-doc/blob/master/"
+                      "installfrompip.adoc for installation instructions."),
+    license="Apache License, Version 2.0",
+    url="https://github.com/google/grr",
+    data_files=(find_data_files("templates", prefix="grr-response-templates") +
+                ["version.ini"]),
+    cmdclass={
+        "sdist": Sdist,
+    })
+
+setup(**setup_args)
diff --git a/grr/config/grr_response_templates/upload.sh b/grr/config/grr_response_templates/upload.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+# Helper script to release client templates.
+
+# Upload to the test bucket:
+# ./upload.sh 3.1.0 grr-releases-testing
+
+# Upload a release:
+# ./upload.sh 3.1.0
+
+set -e
+
+VERSION=$1
+RELEASE_NAME="grr-response-templates-${VERSION}"
+RELEASE_TAR="${RELEASE_NAME}.tar.gz"
+RELEASE_FILE="dist/${RELEASE_TAR}"
+if [[ $# -eq 1 ]] ; then
+  BUCKET="releases.grr-response.com"
+else
+  BUCKET=$2
+fi
+
+md5fingerprint=$(md5sum "${RELEASE_FILE}" | cut -d" " -f1)
+
+# Upload tarball, make public
+gsutil cp "${RELEASE_FILE}" "gs://${BUCKET}/"
+gsutil acl ch -u AllUsers:R "gs://${BUCKET}/${RELEASE_TAR}"
+
+sed -i -e "s!</body></html>!<a href=\"${RELEASE_TAR}#md5=${md5fingerprint}\">${RELEASE_NAME}</a><br/>\n</body></html>!" index.html
+
+gsutil cp index.html "gs://${BUCKET}/"
+gsutil acl ch -u AllUsers:R "gs://${BUCKET}/index.html"
+
+echo "Test install with:"
+echo "pip install --no-cache-dir -f https://storage.googleapis.com/${BUCKET}/index.html grr-response-templates==${VERSION}"
diff --git a/grr/core/grr_response_core/lib/rdfvalues/crypto.py b/grr/core/grr_response_core/lib/rdfvalues/crypto.py
@@ -390,6 +390,8 @@ def GetPublicKey(self):
 
   def Sign(self, message, use_pss=False):
     """Sign a given message."""
+    utils.AssertType(message, bytes)
+
     # TODO(amoser): This should use PSS by default at some point.
     if not use_pss:
       padding_algorithm = padding.PKCS1v15()

diff --git a/grr/core/grr_response_core/lib/utils.py b/grr/core/grr_response_core/lib/utils.py
@@ -28,6 +28,7 @@
 import tempfile
 import threading
 import time
+import types
 import weakref
 import zipfile
 import zlib
@@ -2022,8 +2023,8 @@ def MakeType(name, base_classes, namespace):
   return type(name, base_classes, namespace)
 
 
-def GetName(cls):
-  """A compatibility wrapper for getting class name.
+def GetName(obj):
+  """A compatibility wrapper for getting object's name.
 
   In Python 2 class names are returned as `bytes` (since class names can contain
   only ASCII characters) whereas in Python 3 they are `unicode` (since class
@@ -2036,20 +2037,70 @@ def GetName(cls):
   replaced with ordinary `__name__` access.
 
   Args:
-    cls: A class object to get the name for.
+    obj: A type or function object to get the name for.
 
   Returns:
     Name of the specified class as unicode string.
   """
+  AssertType(obj, (types.TypeType, types.FunctionType))
+
+  # TODO(hanuszczak): According to pytype, `sys.version_info` is a tuple of two
+  # elements which is not true.
+  # pytype: disable=attribute-error
+  if sys.version_info.major == 2:
+    return obj.__name__.decode("ascii")
+  else:
+    return obj.__name__
+  # pytype: enable=attribute-error
+
+
+def SetName(obj, name):
+  """A compatibility wrapper for setting object's name.
+
+  See documentation for `GetName` for more information.
+
+  Args:
+    obj: A type or function object to set the name for.
+    name: A name to set.
+  """
+  AssertType(obj, (types.TypeType, types.FunctionType))
+  AssertType(name, unicode)
+
+  # TODO(hanuszczak): According to pytype, `sys.version_info` is a tuple of two
+  # elements which is not true.
+  # pytype: disable=attribute-error
+  if sys.version_info.major == 2:
+    obj.__name__ = name.encode("ascii")
+  else:
+    obj.__name__ = name
+  # pytype: disable=attribute-error
+
+
+def ListAttrs(cls):
+  """A compatibility wrapper for listing class attributes.
+
+  This method solves similar Python 2 compatibility issues for `dir` function as
+  `GetName` does for `__name__` invocations. See documentation for `GetName` for
+  more details.
+
+  Once support for Python 2 is dropped all invocations of this function should
+  be replaced with ordinary `dir` calls.
+
+  Args:
+    cls: A class object to list the attributes for.
+
+  Returns:
+    A list of attribute names as unicode strings.
+  """
   AssertType(cls, type)
 
   # TODO(hanuszczak): According to pytype, `sys.version_info` is a tuple of two
   # elements which is not true.
   # pytype: disable=attribute-error
   if sys.version_info.major == 2:
-    return cls.__name__.decode("ascii")
+    return [item.decode("ascii") for item in dir(cls)]
   else:
-    return cls.__name__
+    return dir(cls)
   # pytype: enable=attribute-error
 
 

diff --git a/grr/server/grr_response_server/bin/api_shell_raw_access.py b/grr/server/grr_response_server/bin/api_shell_raw_access.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 """Raw access server-side only API shell."""
 from __future__ import print_function
+from __future__ import unicode_literals
 
 import os
 import sys

diff --git a/grr/server/grr_response_server/bin/api_shell_raw_access_lib.py b/grr/server/grr_response_server/bin/api_shell_raw_access_lib.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 """Raw access server-side only API shell."""
+from __future__ import unicode_literals
 
 import traceback
 

diff --git a/grr/server/grr_response_server/bin/api_shell_raw_access_lib_test.py b/grr/server/grr_response_server/bin/api_shell_raw_access_lib_test.py
@@ -1,4 +1,6 @@
 #!/usr/bin/env python
+from __future__ import unicode_literals
+
 import io
 
 

diff --git a/grr/server/grr_response_server/bin/config_updater.py b/grr/server/grr_response_server/bin/config_updater.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 """Util for modifying the GRR server configuration."""
 from __future__ import print_function
+from __future__ import unicode_literals
 
 import argparse
 import os
@@ -26,7 +27,6 @@
 from grr_response_server import aff4
 from grr_response_server import artifact
 from grr_response_server import artifact_registry
-from grr_response_server import data_migration
 from grr_response_server import maintenance_utils
 from grr_response_server import rekall_profile_server
 from grr_response_server import server_startup
@@ -315,11 +315,6 @@
     help="The key length for the new server key. "
     "Defaults to the Server.rsa_key_length config option.")
 
-parser_migrate_data = subparsers.add_parser(
-    "migrate_data",
-    parents=[],
-    help="Migrates data to the relational database.")
-
 
 def main(argv):
   """Main."""
@@ -499,8 +494,6 @@ def main(argv):
 
       maintenance_utils.RotateServerKey(
           cn=flags.FLAGS.common_name, keylength=keylength)
-  elif flags.FLAGS.subparser_name == "migrate_data":
-    data_migration.Migrate()
 
 
 if __name__ == "__main__":

diff --git a/grr/server/grr_response_server/bin/config_updater_util.py b/grr/server/grr_response_server/bin/config_updater_util.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 """Utililies for modifying the GRR server configuration."""
 from __future__ import print_function
+from __future__ import unicode_literals
 
 import getpass
 import os
@@ -266,9 +267,12 @@ def ConfigureMySQLDatastore(config):
         "MySQL Database", "^[A-Za-z0-9-]+$", config["Mysql.database_name"])
     db_options["Mysql.database_username"] = RetryQuestion(
         "MySQL Username", "[A-Za-z0-9-@]+$", config["Mysql.database_username"])
+    # TODO(hanuszczak): Incorrect type specification for `getpass`.
+    # pytype: disable=wrong-arg-types
     db_options["Mysql.database_password"] = getpass.getpass(
         prompt="Please enter password for database user %s: " %
         db_options["Mysql.database_username"])
+    # pytype: enable=wrong-arg-types
 
     if CheckMySQLConnection(db_options):
       print("Successfully connected to MySQL with the provided details.")

diff --git a/grr/server/grr_response_server/bin/config_updater_util_test.py b/grr/server/grr_response_server/bin/config_updater_util_test.py
@@ -1,4 +1,6 @@
 #!/usr/bin/env python
+from __future__ import unicode_literals
+
 import getpass
 
 import builtins

diff --git a/grr/server/grr_response_server/bin/console.py b/grr/server/grr_response_server/bin/console.py
@@ -4,6 +4,7 @@
 We can schedule a new flow for a specific client.
 """
 from __future__ import print_function
+from __future__ import unicode_literals
 
 # pylint: disable=unused-import
 # Import things that are useful from the console.

diff --git a/grr/server/grr_response_server/bin/fleetspeak_frontend.py b/grr/server/grr_response_server/bin/fleetspeak_frontend.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 """This is the GRR frontend FS Server."""
 from __future__ import print_function
+from __future__ import unicode_literals
 
 import logging
 import time