A number of changes:

* More work on sandboxin and Windows MSI installers.
* Windows named pipe collections.
* Artifact collection UIv2 work.
* pytsk3 dependency moved from grr-response-core to grr-response-client.

api_client/python/grr_api_client/metadata.py

@@ -5,14 +5,14 @@
 from __future__ import unicode_literals
 
 import json
-from typing import Dict, Any
+from typing import Dict, Any, Optional
 
 from grr_api_client import context as api_context
 from grr_response_proto.api import metadata_pb2
 
 
 def GetOpenApiDescription(
-    context: api_context.GrrApiContext = None,) -> Dict[str, Any]:
+    context: Optional[api_context.GrrApiContext] = None,) -> Dict[str, Any]:
   """Returns the OpenAPI description of the GRR API as a dictionary."""
   if not context:
     raise ValueError("context can't be empty")

appveyor/windows_templates/build_windows_templates.py

@@ -5,12 +5,16 @@
 from __future__ import unicode_literals
 
 import argparse
+import errno
 import glob
+import logging
 import os
 import shutil
 import subprocess
 import time
 
+from typing import Callable
+
 parser = argparse.ArgumentParser(description="Build windows templates.")
 
 parser.add_argument(
@@ -97,7 +101,60 @@
 args = parser.parse_args()
 
 
-_SC_STOP_WAIT_TIME_SECS = 5
+_SC_STOP_WAIT_TIME_SECS = 10
+_FILE_RETRY_LOOP_RETRY_TIME_SECS = 30
+
+
+def _FileRetryLoop(path: str, f: Callable[[], None]) -> None:
+  """If `path` exists, calls `f` in a retry loop."""
+  if not os.path.exists(path):
+    return
+  attempts = 0
+  while True:
+    try:
+      f()
+      return
+    except OSError as e:
+      attempts += 1
+      if (e.errno == errno.EACCES and
+          attempts < _FILE_RETRY_LOOP_RETRY_TIME_SECS):
+        # The currently installed GRR process may stick around for a few
+        # seconds after the service is terminated (keeping the contents of
+        # the installation directory locked).
+        logging.info("Permission-denied error while trying to process %s.",
+                     path)
+        time.sleep(1)
+      else:
+        raise
+
+
+def _RmTree(path: str) -> None:
+  _FileRetryLoop(path, lambda: shutil.rmtree(path))
+
+
+def _Rename(src: str, dst: str) -> None:
+  _FileRetryLoop(src, lambda: os.rename(src, dst))
+
+
+def _RmTreePseudoTransactional(path: str) -> None:
+  """Removes `path`.
+
+  Makes sure that either `path` is gone or that it is still present as
+  it was.
+
+  Args:
+    path: The path to remove.
+  """
+  temp_path = f"{path}_orphaned_{int(time.time())}"
+  logging.info("Trying to rename %s -> %s.", path, temp_path)
+
+  _Rename(path, temp_path)
+
+  try:
+    logging.info("Trying to remove %s.", temp_path)
+    _RmTree(temp_path)
+  except:  # pylint: disable=bare-except
+    logging.info("Failed to remove %s. Ignoring.", temp_path, exc_info=True)
 
 
 class WindowsTemplateBuilder(object):
@@ -303,9 +360,22 @@ def _RepackTemplates(self):
           "repack", "--template", template_i386, "--output_dir", args.output_dir
       ])
 
+  def _WaitForServiceToStop(self) -> bool:
+    """Waits for the GRR monitor service to stop."""
+    logging.info("Waiting for service %s to stop.", self.service_name)
+    for _ in range(_SC_STOP_WAIT_TIME_SECS):
+      command = ["sc", "query", self.service_name]
+      output = subprocess.check_output(command, encoding="utf-8")
+      logging.info("Command %s returned %s.", command, output)
+      if "STOPPED" in output:
+        return True
+      time.sleep(1.0)
+    return False
+
   def _CleanupInstall(self):
     """Cleanup from any previous installer enough for _CheckInstallSuccess."""
 
+    logging.info("Stoping service %s.", self.service_name)
     subprocess.check_call(["sc", "stop", self.service_name])
 
     if args.build_msi:
@@ -315,19 +385,12 @@ def _CleanupInstall(self):
           "/x",
           glob.glob(os.path.join(args.output_dir, "dbg_*_amd64.msi")).pop(),
       ]
-      print("Running:", msiexec_args)
+      logging.info("Running: %s.", msiexec_args)
       subprocess.check_call(msiexec_args)
     else:
-      # Wait for service to stop.
-      for _ in range(_SC_STOP_WAIT_TIME_SECS):
-        output = subprocess.check_output(["sc", "query", self.service_name],
-                                         encoding="utf-8")
-        stopped = "STOPPED" in output
-        if stopped:
-          break
-        time.sleep(1.0)
+      self._WaitForServiceToStop()
       if os.path.exists(self.install_path):
-        shutil.rmtree(self.install_path)
+        _RmTreePseudoTransactional(self.install_path)
         if os.path.exists(self.install_path):
           raise RuntimeError("Install path still exists: %s" %
                              self.install_path)
@@ -375,7 +438,7 @@ def _InstallInstallers(self):
           glob.glob(os.path.join(args.output_dir, "dbg_*_amd64.exe")).pop()
       ]
     # The exit code is always 0, test to see if install was actually successful.
-    print("Running:", installer_amd64_args)
+    logging.info("Running: %s.", installer_amd64_args)
     subprocess.check_call(installer_amd64_args)
     self._CheckInstallSuccess()
     self._CleanupInstall()

colab/grr_colab/convert.py

@@ -127,9 +127,10 @@ def _get_pretty_value(value: Any, desc: descriptor.FieldDescriptor,
   return data
 
 
-def reindex_dataframe(df: pd.DataFrame,
-                      priority_columns: List[Text] = None,
-                      ignore_columns: List[Text] = None) -> pd.DataFrame:
+def reindex_dataframe(
+    df: pd.DataFrame,
+    priority_columns: Optional[List[Text]] = None,
+    ignore_columns: Optional[List[Text]] = None) -> pd.DataFrame:
   """Reorders and removes dataframe columns according to the given priorities.
 
   Args:

colab/grr_colab/vfs.py

@@ -4,7 +4,7 @@
 The module contains classes that interact with VFS.
 """
 import io
-from typing import Text, List, Iterator, Callable
+from typing import Optional, Text, List, Iterator, Callable
 
 from grr_api_client import client
 from grr_api_client import errors as api_errors
@@ -110,7 +110,7 @@ def tell(self) -> int:
     self._ensure_not_closed()
     return self._pos
 
-  def truncate(self, size: int = None) -> None:
+  def truncate(self, size: Optional[int] = None) -> None:
     raise io.UnsupportedOperation()
 
   def writable(self) -> bool:

grr/client/grr_response_client/client_actions/registry_init.py

@@ -23,6 +23,7 @@
 from grr_response_client.client_actions import tempfiles
 from grr_response_client.client_actions import timeline
 from grr_response_client.client_actions import vfs_file_finder
+from grr_response_client.client_actions.windows import pipes
 
 
 def RegisterClientActions():
@@ -97,6 +98,7 @@ def RegisterClientActions():
     client_actions.Register("WmiQuery", windows.WmiQuery)
     client_actions.Register("Uninstall", windows.Uninstall)
     client_actions.Register("UpdateAgent", windows.UpdateAgent)
+    client_actions.Register("ListNamedPipes", pipes.ListNamedPipesAction)
 
   elif platform.system() == "Darwin":
     from grr_response_client.client_actions.osx import osx  # pylint: disable=g-import-not-at-top

grr/client/grr_response_client/client_actions/windows/pipes.py

@@ -0,0 +1,160 @@
+#!/usr/bin/env python
+"""A module with an action for collecting named pipes."""
+import contextlib
+import logging
+import os
+import platform
+from typing import Iterator
+
+from grr_response_client import actions
+from grr_response_core.lib.rdfvalues import client as rdf_client
+
+
+def ListNamedPipes() -> Iterator[rdf_client.NamedPipe]:
+  """Yields all named pipes available in the system."""
+  if platform.system() != "Windows":
+    raise RuntimeError(f"Unsupported platform: {platform.system()}")
+
+  # pylint: disable=g-import-not-at-top
+  # pytype: disable=import-error
+  import ctypes
+  import ctypes.wintypes
+  import win32api
+  import win32file
+  import win32pipe
+  import winerror
+  # pytype: enable=import-error
+  # pylint: enable=g-import-not-at-top
+
+  # The `GetNamedPipeHandleState` function provided by the `win32pipe` module is
+  # broken (calling it results in invalid function exception). Hence, we need to
+  # go to a lower level and use raw Windows API calls to get this information.
+  #
+  # https://docs.microsoft.com/en-us/windows/win32/api/namedpipeapi/nf-namedpipeapi-getnamedpipehandlestatew
+  # pytype: disable=module-attr
+  GetNamedPipeHandleStateW = ctypes.windll.kernel32.GetNamedPipeHandleStateW  # pylint: disable=invalid-name
+  # pytype: enable=module-attr
+  GetNamedPipeHandleStateW.argtypes = [
+      ctypes.wintypes.HANDLE,
+      ctypes.wintypes.LPDWORD,
+      ctypes.wintypes.LPDWORD,
+      ctypes.wintypes.LPDWORD,
+      ctypes.wintypes.LPDWORD,
+      ctypes.wintypes.LPWSTR,
+      ctypes.wintypes.DWORD,
+  ]
+  GetNamedPipeHandleStateW.restype = ctypes.wintypes.BOOL
+
+  # For some reason the `GetNamedPipeClientComputerName` function does not exist
+  # in `win32pipe`. Hence, we implement a low-level wrapper for Windows API for
+  # it ourselves.
+  #
+  # https://docs.microsoft.com/en-us/windows/win32/api/namedpipeapi/nf-namedpipeapi-getnamedpipeclientcomputernamew
+  # pytype: disable=module-attr
+  GetNamedPipeClientComputerNameW = ctypes.windll.kernel32.GetNamedPipeClientComputerNameW  # pylint: disable=invalid-name
+  # pytype: enable=module-attr
+  GetNamedPipeClientComputerNameW.argtypes = [
+      ctypes.wintypes.HANDLE,
+      ctypes.wintypes.LPWSTR,
+      ctypes.wintypes.ULONG,
+  ]
+  GetNamedPipeClientComputerNameW.restype = ctypes.wintypes.BOOL
+
+  # https://docs.microsoft.com/en-us/windows/win32/ipc/pipe-names
+  for name in os.listdir(r"\\.\pipe"):
+    pipe = rdf_client.NamedPipe()
+    pipe.name = name
+
+    try:
+      handle = win32file.CreateFile(f"\\\\.\\pipe\\{name}", 0, 0, None,
+                                    win32file.OPEN_EXISTING, 0, None)
+    except win32file.error as error:
+      # There might be some permission issues. We log the error and skip getting
+      # pipe details, but still yield a result with at least the name filled-in.
+      logging.error("Cannot open pipe '%s': %s", name, error)
+      yield pipe
+      continue
+
+    with contextlib.closing(handle):
+      try:
+        pipe_info = win32pipe.GetNamedPipeInfo(handle)
+        flags, in_buffer_size, out_buffer_size, max_instance_count = pipe_info
+
+        pipe.flags = flags
+        pipe.in_buffer_size = in_buffer_size
+        pipe.out_buffer_size = out_buffer_size
+        pipe.max_instance_count = max_instance_count
+      except win32pipe.error as error:
+        # Getting the information might fail (for whatever reason), but we don't
+        # want to fail action execution as other probing calls might succeed.
+        logging.error("Failed to get info about pipe '%s': '%s'", name, error)
+
+      try:
+        pipe.server_pid = win32pipe.GetNamedPipeServerProcessId(handle)
+      except win32pipe.error as error:
+        # See similar comment for `GetNamedPipeInfo` for more information.
+        message = "Failed to get server pid of pipe '%s': '%s'"
+        logging.error(message, name, error)
+
+      try:
+        pipe.client_pid = win32pipe.GetNamedPipeClientProcessId(handle)
+      except win32pipe.error as error:
+        # See similar comment for `GetNamedPipeInfo` for more information.
+        message = "Failed to get client pid of pipe '%s': '%s'"
+        logging.error(message, name, error)
+
+      cur_instance_count = ctypes.wintypes.DWORD()
+      status = GetNamedPipeHandleStateW(
+          ctypes.wintypes.HANDLE(int(handle)),
+          None,
+          ctypes.byref(cur_instance_count),
+          None,
+          None,
+          None,
+          0,
+      )
+
+      if status == 0:
+        # See similar comment for `GetNamedPipeInfo` for more information.
+        error = win32api.GetLastError()
+        logging.error("Failed to get state of pipe '%s': %s", name, error)
+      else:
+        pipe.cur_instance_count = cur_instance_count.value
+
+      client_computer_name = (ctypes.wintypes.WCHAR * _COMPUTER_NAME_MAX_SIZE)()  # pytype: disable=not-callable
+      status = GetNamedPipeClientComputerNameW(
+          ctypes.wintypes.HANDLE(int(handle)),
+          client_computer_name,
+          _COMPUTER_NAME_MAX_SIZE,
+      )
+
+      if status == 0:
+        # See similar comment for `GetNamedPipeInfo` for more information.
+        error = win32api.GetLastError()
+        # Not being able to get computer name of a local pipe is expected, there
+        # is no need to log errors in such cases.
+        if error != winerror.ERROR_PIPE_LOCAL:
+          logging.error("Failed to get hostname of pipe '%s': %s", name, error)
+      else:
+        pipe.client_computer_name = client_computer_name.value
+
+      yield pipe
+
+
+class ListNamedPipesAction(actions.ActionPlugin):
+  """An action for collecting named pipes."""
+
+  in_rdfvalue = None
+  out_rdfvalues = [rdf_client.NamedPipe]
+
+  def Run(self, args: None) -> None:
+    """Executes the action."""
+    for result in ListNamedPipes():
+      self.SendReply(result)
+
+
+# The length is 15 characters but we also might need one extra byte for the null
+# character.
+#
+# https://docs.microsoft.com/en-us/troubleshoot/windows-server/identity/naming-conventions-for-computer-domain-site-ou
+_COMPUTER_NAME_MAX_SIZE = 16