Implement sgr cloud tunnel (without provisioning)

The `sgr cloud tunnel` command now works with the following caveats:
- The rathole client binary must be copied into SG_CONFIG_DIR manually.
- When the provisioning endpoint works, the
  server's TLS cert hostname and
  public management address will be provided via
  GQL call, currently its hardcoded.

splitgraph/cloud/__init__.py

@@ -58,6 +58,7 @@
     GET_PLUGINS,
     GET_REPO_METADATA,
     GET_REPO_SOURCE,
+    GET_TUNNEL_PROVISIONING_TOKEN,
     INGESTION_JOB_STATUS,
     JOB_LOGS,
     PROFILE_UPSERT,
@@ -1104,3 +1105,15 @@ def load_all_repositories(self, limit_to: List[str] = None) -> List[Repository]:
             parsed_external = ExternalResponse.from_response(external_r.json())
 
         return make_repositories(parsed_metadata, parsed_external)
+
+    def get_tunnel_provisioning_token(self, namespace: str, repository: str) -> str:
+        response = self._gql(
+            {
+                "query": GET_TUNNEL_PROVISIONING_TOKEN,
+                "operationName": "GetTunnelProvisioningToken",
+                "variables": {"namespace": namespace, "repository": repository},
+            },
+            handle_errors=True,
+            anonymous_ok=False,
+        )
+        return str(response.json()["data"]["getTunnelProvisioningToken"])

splitgraph/cloud/queries.py

@@ -280,6 +280,11 @@
 }
 """
 
+GET_TUNNEL_PROVISIONING_TOKEN = """mutation GetTunnelProvisioningToken($namespace: String!, $repository: String!) {
+  getTunnelProvisioningToken(namespace:$namespace, repository:$repository)
+}
+"""
+
 EXPORT_JOB_STATUS = """query ExportJobStatus($taskId: UUID!) {
   exportJobStatus(taskId: $taskId) {
     taskId

splitgraph/cloud/tunnel_client.py

@@ -0,0 +1,97 @@
+import os
+import subprocess
+import sys
+from os import path
+from typing import IO, Optional, cast
+
+from splitgraph.cloud.project.models import Repository
+
+RATHOLE_CLIENT_FILENAME = "rathole-client.toml"
+
+RATHOLE_CLIENT_TEMPLATE = """
+[client]
+remote_addr = "{rathole_server_management_address}"
+
+[client.transport]
+type = "tls"
+
+[client.transport.tls]
+{trusted_root_line}
+hostname = "{hostname}"
+
+[client.services.{servicename}]
+local_addr = "{local_address}"
+# token is provisioner JWT token
+token = "{provisioning_token}"
+
+"""
+
+
+def get_rathole_client_config(
+    rathole_server_management_address: str,
+    hostname: str,
+    local_address: str,
+    provisioning_token: str,
+    namespace: str,
+    repository: str,
+    trusted_root: Optional[str],
+) -> str:
+    trusted_root_line = f'trusted_root = "{trusted_root}"' if trusted_root else ""
+    # 'servicename' is the suffix of a section header in the rathol TOML config.
+    # It must match in the rathole client and server configs.
+    # The rathole server config has sections for _all_ users' tunnels. If
+    # namespace: 'a' repository: 'bc' and
+    # namespace: 'ab' repository 'c' result in the same section config then
+    # users could query each others' tunnels, so we need to disambiguate these
+    # scenarios. Unfortuantely, TOML section header characters are as
+    # restrictive as valid namespace / repo names. As a workaround, the length
+    # of each field is prefixed with the field padded to 4 chars,
+    # avoiding the collision.
+    servicename = f"{len(namespace):04}{namespace}{len(repository):04}{repository}"
+    return RATHOLE_CLIENT_TEMPLATE.format(
+        rathole_server_management_address=rathole_server_management_address,
+        hostname=hostname,
+        local_address=local_address,
+        provisioning_token=provisioning_token,
+        servicename=servicename,
+        trusted_root_line=trusted_root_line,
+    )
+
+
+def write_rathole_client_config(
+    provisioning_token: str, repository: Repository, config_dir: str
+) -> str:
+    # verify repository is external
+    if not repository.external or not repository.external.tunnel:
+        raise Exception("Repository %s not a tunneled external repository" % (repository))
+    # TODO: instead of printing token, make gql call to provision tunnel
+    print("Got provisioning token %s" % provisioning_token)
+    # in production, this will be None, but for dev instances, we need to
+    # specify rootCA.pem
+    trusted_root = os.environ["REQUESTS_CA_BUNDLE"] or os.environ["SSL_CERT_FILE"]
+    rathole_client_config = get_rathole_client_config(
+        # TODO: replace these stub values with response of provisioning call
+        rathole_server_management_address="34.70.46.40:2333",
+        hostname="www.splitgraph.test",
+        local_address=f"{repository.external.params['host']}:{repository.external.params['port']}",
+        provisioning_token=provisioning_token,
+        namespace=repository.namespace,
+        repository=repository.repository,
+        trusted_root=trusted_root,
+    )
+    config_filename = path.join(config_dir, RATHOLE_CLIENT_FILENAME)
+    with open(config_filename, "w") as f:
+        f.write(rathole_client_config)
+    return config_filename
+
+
+# inspired by https://stackoverflow.com/questions/18421757/live-output-from-subprocess-command
+def launch_rathole_client(rathole_client_binary_path, rathole_client_config_path):
+    process = subprocess.Popen(
+        [rathole_client_binary_path, "--client", rathole_client_config_path],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    )
+    # pipe rathole process output to stdout
+    for c in iter(lambda: cast(IO[bytes], process.stdout).read(1), b""):  # nomypy
+        sys.stdout.buffer.write(c)

splitgraph/commandline/cloud.py

@@ -17,6 +17,10 @@
 
 from splitgraph.cloud.models import AddExternalRepositoryRequest, IntrospectionMode
 from splitgraph.cloud.project.models import Metadata, SplitgraphYAML
+from splitgraph.cloud.tunnel_client import (
+    launch_rathole_client,
+    write_rathole_client_config,
+)
 from splitgraph.commandline.common import (
     ImageType,
     RepositoryType,
@@ -26,7 +30,8 @@
     wait_for_job,
 )
 from splitgraph.commandline.engine import inject_config_into_engines
-from splitgraph.config.config import get_from_subsection
+from splitgraph.config import CONFIG
+from splitgraph.config.config import get_from_subsection, get_singleton
 from splitgraph.config.management import patch_and_save_config
 from splitgraph.core.output import Color, pluralise
 
@@ -1221,12 +1226,12 @@ def seed_c(remote, seed, github_repository, directory):
 @click.argument("repository", type=str)
 def tunnel_c(remote, repositories_file, repository):
     """
-    Start the tunnel client to make it available
+    Start the tunnel client to make tunneled external repo available.
 
     This will load a splitgraph.yml file and tunnel the host:port address of the
     external repository specified in the argument.
     """
-    from splitgraph.cloud import GQLAPIClient, RESTAPIClient
+
     from splitgraph.cloud.project.utils import load_project
 
     repo_yaml = load_project(repositories_file)
@@ -1240,16 +1245,21 @@ def tunnel_c(remote, repositories_file, repository):
             "Repository %s not found in %s" % (repository, ", ".join(repositories_file))
         )
 
-    # verify repository is external
-    # TODO: unit test
-    if not tunneled_repo.external or not tunneled_repo.external.tunnel:
-        raise click.UsageError("Repository %s not a tunneled external repository" % (repository))
+    config_dir = os.path.dirname(get_singleton(CONFIG, "SG_CONFIG_FILE"))
+    # TODO: Get current version of rathole client for architecture.
+    # user must manually download rathole for now
+    rathole_client_binary_path = os.path.join(config_dir, "rathole")
 
-    # TODO: Get rathole client for architecture (manually downloading rathole for now)
-    #
+    from splitgraph.cloud import GQLAPIClient
 
-    # gql_client = GQLAPIClient(remote)
-    print("next step: start client!")
+    client = GQLAPIClient(remote)
+    provisioning_token = client.get_tunnel_provisioning_token(
+        tunneled_repo.namespace, tunneled_repo.repository
+    )
+    rathole_client_config_path = write_rathole_client_config(
+        provisioning_token, tunneled_repo, config_dir
+    )
+    launch_rathole_client(rathole_client_binary_path, rathole_client_config_path)
 
 
 @click.group("cloud")