A number of changes.

 * Add configurable email for GRR users.
 * Increment version to 3.4.0.4.
 * WIP: Create API endpoint to trigger Interrogate.
 * Restore support for pre-Vista Windows versions for the timezone artifact.
 * Fixed: GRR tests fail with updated WindowsTimestamp artifact definition
 * Adding the Yara version to the list of reported client library versions.
 * Bumping Fleetspeak version used to 0.1.6.
 * Use 'unknown' for inconsistent flows instead of null creator.
 * Improved expansion of Windows environment variables.
 * WIP: Have default limits for YaraProcessScan/Dump.
 * WIP: Make Timeline Feature more robust
 * Implement Approval UI.
 * Various other/small improvements.

api_client/python/grr_api_client/client.py

@@ -7,6 +7,7 @@
 from grr_api_client import flow
 from grr_api_client import utils
 from grr_api_client import vfs
+from grr_response_core.lib.util import compatibility
 from grr_response_proto.api import client_pb2
 from grr_response_proto.api import flow_pb2
 from grr_response_proto.api import user_pb2
@@ -158,6 +159,13 @@ def CreateFlow(self, name=None, args=None, runner_args=None):
     data = self._context.SendRequest("CreateFlow", request)
     return flow.Flow(data=data, context=self._context)
 
+  def Interrogate(self):
+    """Run an Interrogate Flow on this client."""
+    request = client_pb2.ApiInterrogateClientArgs(client_id=self.client_id)
+    data = self._context.SendRequest("InterrogateClient", request)
+    # Return a populated Flow, similar to the behavior of CreateFlow().
+    return self.Flow(data.operation_id).Get()
+
   def ListFlows(self):
     """List flows that ran on this client."""
 
@@ -250,6 +258,9 @@ def Get(self):
 class ClientRef(ClientBase):
   """Ref to the client."""
 
+  def __repr__(self):
+    return "ClientRef(client_id={!r})".format(self.client_id)
+
 
 class Client(ClientBase):
   """Client object with fetched data."""
@@ -264,6 +275,10 @@ def __init__(self, data=None, context=None):
 
     self.data = data
 
+  def __repr__(self):
+    return "Client(data=<{} client_id={!r}>, ...)".format(
+        compatibility.GetName(type(self.data)), self.data.client_id)
+
 
 def SearchClients(query=None, context=None):
   """List clients conforming to a givent query."""

api_client/python/grr_api_client/flow.py

@@ -6,7 +6,9 @@
 
 from grr_api_client import errors
 from grr_api_client import utils
+from grr_response_core.lib.util import compatibility
 from grr_response_proto.api import flow_pb2
+from grr_response_proto.api import timeline_pb2
 
 
 class FlowResult(object):
@@ -82,6 +84,11 @@ def GetFilesArchive(self):
         client_id=self.client_id, flow_id=self.flow_id)
     return self._context.SendStreamingRequest("GetFlowFilesArchive", args)
 
+  def GetCollectedTimeline(self, fmt):
+    args = timeline_pb2.ApiGetCollectedTimelineArgs(
+        client_id=self.client_id, flow_id=self.flow_id, format=fmt)
+    return self._context.SendStreamingRequest("GetCollectedTimeline", args)
+
   def Get(self):
     """Fetch flow's data and return proper Flow object."""
 
@@ -118,6 +125,10 @@ def WaitUntilDone(self, timeout=None):
 class FlowRef(FlowBase):
   """Flow reference (points to the flow, but has no data)."""
 
+  def __repr__(self):
+    return "FlowRef(client_id={!r}, flow_id={!r})".format(
+        self.client_id, self.flow_id)
+
 
 class Flow(FlowBase):
   """Flow object with fetched data."""
@@ -137,3 +148,10 @@ def __init__(self, data=None, context=None):
   @property
   def args(self):
     return utils.UnpackAny(self.data.args)
+
+  def __repr__(self):
+    return ("Flow(data=<{} client_id={!r}, flow_id={!r}, name={!r}, "
+            "state={}, ...>)").format(
+                compatibility.GetName(type(self.data)), self.data.client_id,
+                self.data.flow_id, self.data.name,
+                flow_pb2.ApiFlow.State.Name(self.data.state))

api_client/python/grr_api_client/root.py

@@ -41,7 +41,7 @@ def Delete(self):
     args = user_management_pb2.ApiDeleteGrrUserArgs(username=self.username)
     self._context.SendRequest("DeleteGrrUser", args)
 
-  def Modify(self, user_type=None, password=None):
+  def Modify(self, user_type=None, password=None, email=None):
     """Modifies user's type and/or password."""
 
     args = user_management_pb2.ApiModifyGrrUserArgs(
@@ -53,6 +53,9 @@ def Modify(self, user_type=None, password=None):
     if password is not None:
       args.password = password
 
+    if email is not None:
+      args.email = email
+
     data = self._context.SendRequest("ModifyGrrUser", args)
     return GrrUser(data=data, context=self._context)
 
@@ -141,7 +144,11 @@ def __init__(self, context=None):
     super(RootGrrApi, self).__init__()
     self._context = context
 
-  def CreateGrrUser(self, username=None, user_type=None, password=None):
+  def CreateGrrUser(self,
+                    username=None,
+                    user_type=None,
+                    password=None,
+                    email=None):
     """Creates a new GRR user of a given type with a given username/password."""
 
     if not username:
@@ -155,6 +162,9 @@ def CreateGrrUser(self, username=None, user_type=None, password=None):
     if password is not None:
       args.password = password
 
+    if email is not None:
+      args.email = email
+
     data = self._context.SendRequest("CreateGrrUser", args)
     return GrrUser(data=data, context=self._context)
 

api_client/python/grr_api_client/utils.py

@@ -19,6 +19,7 @@
 from grr_response_proto import flows_pb2
 from grr_response_proto import jobs_pb2
 from grr_response_proto import osquery_pb2
+from grr_response_proto import timeline_pb2
 
 from grr_response_proto.api import artifact_pb2
 from grr_response_proto.api import client_pb2
@@ -234,6 +235,7 @@ def RegisterProtoDescriptors(db, *additional_descriptors):
   db.RegisterFileDescriptor(flows_pb2.DESCRIPTOR)
   db.RegisterFileDescriptor(jobs_pb2.DESCRIPTOR)
   db.RegisterFileDescriptor(osquery_pb2.DESCRIPTOR)
+  db.RegisterFileDescriptor(timeline_pb2.DESCRIPTOR)
   db.RegisterFileDescriptor(wrappers_pb2.DESCRIPTOR)
 
   for d in additional_descriptors:

colab/grr_colab/client_test.py

@@ -345,6 +345,7 @@ def testLastSeen(self):
   def testRequestApproval(self):
     data_store.REL_DB.WriteClientMetadata(
         client_id=ClientTest.FAKE_CLIENT_ID, fleetspeak_enabled=False)
+    data_store.REL_DB.WriteGRRUser('foo')
 
     client = grr_colab.Client.with_id(ClientTest.FAKE_CLIENT_ID)
     client.request_approval(reason='test', approvers=['foo'])
@@ -361,6 +362,7 @@ def testRequestApproval(self):
   def testRequestApprovalAndWait(self):
     data_store.REL_DB.WriteClientMetadata(
         client_id=ClientTest.FAKE_CLIENT_ID, fleetspeak_enabled=False)
+    data_store.REL_DB.WriteGRRUser('foo')
 
     client = grr_colab.Client.with_id(ClientTest.FAKE_CLIENT_ID)
 

grr/client/grr_response_client/actions.py

@@ -75,6 +75,8 @@ class ActionPlugin(object):
 
   _PROGRESS_THROTTLE_INTERVAL = rdfvalue.Duration.From(2, rdfvalue.SECONDS)
 
+  last_progress_time = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(0)
+
   def __init__(self, grr_worker=None):
     """Initializes the action plugin.
 
@@ -94,7 +96,6 @@ def __init__(self, grr_worker=None):
     self.proc = psutil.Process()
     self.cpu_start = self.proc.cpu_times()
     self.cpu_limit = rdf_flows.GrrMessage().cpu_limit
-    self.last_progress_time = rdfvalue.RDFDatetime.Now()
     self.start_time = None
     self.runtime_limit = None
 
@@ -155,6 +156,7 @@ def Execute(self, message):
         used = self.proc.cpu_times()
         self.cpu_used = (used.user - self.cpu_start.user,
                          used.system - self.cpu_start.system)
+        self.status.runtime_us = rdfvalue.RDFDatetime.Now() - self.start_time
 
     except NetworkBytesExceededError as e:
       self.grr_worker.SendClientAlert("Network limit exceeded.")
@@ -288,15 +290,16 @@ def Progress(self):
       RuntimeExceededError: Runtime limit exceeded.
     """
     now = rdfvalue.RDFDatetime.Now()
+    time_since_last_progress = now - ActionPlugin.last_progress_time
 
-    if now - self.last_progress_time <= self._PROGRESS_THROTTLE_INTERVAL:
+    if time_since_last_progress <= self._PROGRESS_THROTTLE_INTERVAL:
       return
 
     if self.runtime_limit and now - self.start_time > self.runtime_limit:
       raise RuntimeExceededError("{} exceeded runtime limit of {}.".format(
           compatibility.GetName(type(self)), self.runtime_limit))
 
-    self.last_progress_time = now
+    ActionPlugin.last_progress_time = now
 
     # Prevent the machine from sleeping while the action is running.
     client_utils.KeepAlive()

grr/client/grr_response_client/client_actions/action_test.py

@@ -269,12 +269,13 @@ class MockWorker(object):
       def Heartbeat(self):
         pass
 
-    action = ProgressAction(grr_worker=MockWorker())
+    worker = MockWorker()
 
     with test_lib.Instrument(client_utils, "KeepAlive") as instrument:
       for time, expected_count in [(100, 1), (101, 1), (102, 1), (103, 2),
                                    (104, 2), (105, 2), (106, 3)]:
         with test_lib.FakeTime(time):
+          action = ProgressAction(grr_worker=worker)
           action.Progress()
           self.assertEqual(instrument.call_count, expected_count)
 

grr/client/grr_response_client/client_actions/admin.py

@@ -8,18 +8,17 @@
 import os
 import platform
 import socket
-import time
 import traceback
 
 import cryptography
 from cryptography.hazmat.backends import openssl
 from future.builtins import map
-from future.builtins import range
 from future.builtins import str
 from future.utils import iteritems
 import pkg_resources
 import psutil
 import pytsk3
+import yara
 
 from grr_response_client import actions
 from grr_response_client.client_actions import tempfiles
@@ -91,47 +90,6 @@ def Run(self, unused_arg):
     os._exit(242)  # pylint: disable=protected-access
 
 
-class Hang(actions.ActionPlugin):
-  """A client action for simulating the client becoming unresponsive (hanging).
-
-  Used for testing nanny terminating the client.
-  """
-  in_rdfvalue = rdf_protodict.DataBlob
-
-  def Run(self, arg):
-    # Sleep a really long time.
-    time.sleep(arg.integer or 6000)
-
-
-class BusyHang(actions.ActionPlugin):
-  """A client action that burns cpu cycles. Used for testing cpu limits."""
-  in_rdfvalue = rdf_protodict.DataBlob
-
-  def Run(self, arg):
-    duration = 5
-    if arg and arg.integer:
-      duration = arg.integer
-    end = time.time() + duration
-    while time.time() < end:
-      pass
-
-
-class Bloat(actions.ActionPlugin):
-  """A client action that uses lots of memory for testing."""
-  in_rdfvalue = rdf_protodict.DataBlob
-
-  def Run(self, arg):
-
-    iterations = arg.integer or 1024  # Gives 1 gb.
-
-    l = []
-
-    for _ in range(iterations):
-      l.append("X" * 1048576)  # 1 mb.
-
-    time.sleep(60)
-
-
 class GetConfiguration(actions.ActionPlugin):
   """Retrieves the running configuration parameters."""
   in_rdfvalue = None
@@ -182,12 +140,16 @@ def GetTSKVersion(self):
   def GetPyTSKVersion(self):
     return pytsk3.get_version()
 
+  def GetYaraVersion(self):
+    return yara.YARA_VERSION
+
   library_map = {
       "pytsk": GetPyTSKVersion,
       "TSK": GetTSKVersion,
       "cryptography": GetCryptographyVersion,
       "SSL": GetSSLVersion,
       "psutil": GetPSUtilVersion,
+      "yara": GetYaraVersion,
   }
 
   error_str = "Unable to determine library version: %s"

grr/client/grr_response_client/client_actions/memory.py

@@ -94,6 +94,10 @@ class YaraProcessScan(actions.ActionPlugin):
   in_rdfvalue = rdf_memory.YaraProcessScanRequest
   out_rdfvalues = [rdf_memory.YaraProcessScanResponse]
 
+  def __init__(self, grr_worker=None):
+    super(YaraProcessScan, self).__init__(grr_worker=grr_worker)
+    self._rules = None
+
   def _ScanRegion(self, rules, chunks, deadline):
     for chunk in chunks:
       if not chunk.data:
@@ -127,7 +131,8 @@ def _GetMatches(self, psutil_process, scan_request):
       deadline = rdfvalue.RDFDatetime.Now() + rdfvalue.Duration.From(
           1, rdfvalue.WEEKS)
 
-    rules = scan_request.yara_signature.GetRules()
+    if self._rules is None:
+      self._rules = scan_request.yara_signature.GetRules()
 
     process = client_utils.OpenProcessForMemoryAccess(pid=psutil_process.pid)
     with process:
@@ -140,7 +145,7 @@ def _GetMatches(self, psutil_process, scan_request):
         for region in client_utils.MemoryRegions(process, scan_request):
           chunks = streamer.StreamMemory(
               process, offset=region.start, amount=region.size)
-          for m in self._ScanRegion(rules, chunks, deadline):
+          for m in self._ScanRegion(self._rules, chunks, deadline):
             matches.append(m)
             if 0 < scan_request.max_results_per_process <= len(matches):
               return matches

grr/client/grr_response_client/client_actions/registry_init.py

@@ -20,6 +20,7 @@
 from grr_response_client.client_actions import searching
 from grr_response_client.client_actions import standard
 from grr_response_client.client_actions import tempfiles
+from grr_response_client.client_actions import timeline
 from grr_response_client.client_actions import vfs_file_finder
 
 
@@ -28,7 +29,6 @@ def RegisterClientActions():
 
   client_actions.Register("ArtifactCollector",
                           artifact_collector.ArtifactCollector)
-  client_actions.Register("Bloat", admin.Bloat)
   client_actions.Register("CheckFreeGRRTempSpace",
                           tempfiles.CheckFreeGRRTempSpace)
   client_actions.Register("DeleteGRRTempFiles", tempfiles.DeleteGRRTempFiles)
@@ -49,7 +49,6 @@ def RegisterClientActions():
   client_actions.Register("GetMemorySize", standard.GetMemorySize)
   client_actions.Register("GetPlatformInfo", admin.GetPlatformInfo)
   client_actions.Register("Grep", searching.Grep)
-  client_actions.Register("Hang", admin.Hang)
   client_actions.Register("HashBuffer", standard.HashBuffer)
   client_actions.Register("HashFile", standard.HashFile)
   client_actions.Register("Kill", admin.Kill)
@@ -64,6 +63,7 @@ def RegisterClientActions():
   client_actions.Register("SendFile", standard.SendFile)
   client_actions.Register("SendStartupInfo", admin.SendStartupInfo)
   client_actions.Register("StatFS", standard.StatFS)
+  client_actions.Register("Timeline", timeline.Timeline)
   client_actions.Register("TransferBuffer", standard.TransferBuffer)
   client_actions.Register("UpdateConfiguration", admin.UpdateConfiguration)
   client_actions.Register("VfsFileFinder", vfs_file_finder.VfsFileFinder)

grr/client/grr_response_client/client_actions/searching_test.py

@@ -433,7 +433,7 @@ def testExtAttrsCollection(self):
       self.assertCountEqual(values, [b"foo", b"bar", b"baz"])
 
 
-class GrepTest(client_test_lib.EmptyActionTest):
+class GrepTest(vfs_test_lib.VfsTestCase, client_test_lib.EmptyActionTest):
   """Test the find client Actions."""
 
   XOR_IN_KEY = 0

grr/client/grr_response_client/client_vfs_test.py

@@ -33,7 +33,7 @@
 # pylint: mode=test
 
 
-class VFSTest(test_lib.GRRBaseTest):
+class VFSTest(vfs_test_lib.VfsTestCase, test_lib.GRRBaseTest):
   """Test the client VFS switch."""
 
   def GetNumbers(self):