Dynamically set API_SERVER_URL (#708)

.github/workflows/local_tests.yaml

@@ -5,6 +5,9 @@ on:
     branches: [ main ]
   pull_request:
 
+env:
+  API_SERVER_URL: https://api.run.house
+
 jobs:
   # TODO: THESE ARE ONLY SEPARATE JOBS BECAUSE THERE ARE
   # DEVELOPMENT INCONSISTENCIES WHEN RUNNING ALL THE LOCAL
@@ -38,6 +41,7 @@ jobs:
         with:
           username: ${{ secrets.CI_ACCOUNT_USERNAME }}
           token: ${{ secrets.CI_ACCOUNT_TOKEN }}
+          api_server_url: ${{ env.API_SERVER_URL }}
 
       - name: pytest -v --level local tests/test_servers/
         env:
@@ -75,6 +79,7 @@ jobs:
         with:
           username: ${{ secrets.CI_ACCOUNT_USERNAME }}
           token: ${{ secrets.CI_ACCOUNT_TOKEN }}
+          api_server_url: ${{ env.API_SERVER_URL }}
 
       - name: pytest -v --level local -k "not servertest and not secrettest and not moduletest and not functiontest and not envtest"
         env:
@@ -112,6 +117,7 @@ jobs:
         with:
           username: ${{ secrets.CI_ACCOUNT_USERNAME }}
           token: ${{ secrets.CI_ACCOUNT_TOKEN }}
+          api_server_url: ${{ env.API_SERVER_URL }}
 
       - name: pytest -v --level local -k "secrettest"
         env:
@@ -134,6 +140,7 @@ jobs:
         with:
           username: ${{ secrets.CI_ACCOUNT_USERNAME }}
           token: ${{ secrets.CI_ACCOUNT_TOKEN }}
+          api_server_url: ${{ env.API_SERVER_URL }}
 
       - name: pytest -v --level local -k "moduletest"
         env:
@@ -156,6 +163,7 @@ jobs:
         with:
           username: ${{ secrets.CI_ACCOUNT_USERNAME }}
           token: ${{ secrets.CI_ACCOUNT_TOKEN }}
+          api_server_url: ${{ env.API_SERVER_URL }}
 
       - name: pytest -v --level local -k "functiontest"
         env:
@@ -178,6 +186,7 @@ jobs:
         with:
           username: ${{ secrets.CI_ACCOUNT_USERNAME }}
           token: ${{ secrets.CI_ACCOUNT_TOKEN }}
+          api_server_url: ${{ env.API_SERVER_URL }}
 
       - name: pytest -v --level local -k "envtest"
         env:

.github/workflows/local_tests_den_dev.yaml

@@ -0,0 +1,195 @@
+name: Tests with level "local" on Den Dev
+
+on:
+  workflow_dispatch:
+
+env:
+  API_SERVER_URL: https://api-dev.run.house
+
+jobs:
+  # TODO: THESE ARE ONLY SEPARATE JOBS BECAUSE THERE ARE
+  # DEVELOPMENT INCONSISTENCIES WHEN RUNNING ALL THE LOCAL
+  # TESTS TOGETHER.
+  # server-tests-logged-out-level-local:
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - name: Check out repository code
+  #       uses: actions/checkout@v3
+
+  #     - name: Setup Runhouse
+  #       uses: ./.github/workflows/setup_runhouse
+
+  #     - name: pytest -v --level local tests/test_servers/
+  #       env:
+  #         TEST_TOKEN: ${{ secrets.TEST_TOKEN }}
+  #         TEST_USERNAME: ${{ secrets.TEST_USERNAME }}
+  #       run: pytest -v --level local tests/test_servers/
+
+  server-tests-logged-in-level-local:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v3
+
+      - name: Setup Runhouse
+        uses: ./.github/workflows/setup_runhouse
+
+      - name: Setup ~/.rh/config.yaml
+        uses: ./.github/workflows/setup_rh_config
+        with:
+          username: ${{ secrets.CI_ACCOUNT_USERNAME }}
+          token: ${{ secrets.CI_ACCOUNT_TOKEN }}
+          api_server_url: ${{ env.API_SERVER_URL }}
+
+
+      - name: pytest -v --level local tests/test_servers/
+        env:
+          TEST_TOKEN: ${{ secrets.TEST_TOKEN }}
+          TEST_USERNAME: ${{ secrets.TEST_USERNAME }}
+        run: pytest -v --level local tests/test_servers/ --api-server-url $API_SERVER_URL
+        timeout-minutes: 60
+
+  # most-tests-logged-out-level-local:
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - name: Check out repository code
+  #       uses: actions/checkout@v3
+
+  #     - name: Setup Runhouse
+  #       uses: ./.github/workflows/setup_runhouse
+
+  #     - name: pytest -v --level local -k "not servertest and not secrettest"
+  #       env:
+  #         TEST_TOKEN: ${{ secrets.TEST_TOKEN }}
+  #         TEST_USERNAME: ${{ secrets.TEST_USERNAME }}
+  #       run: pytest -v --level local -k "not servertest and not secrettest"
+
+  most-tests-logged-in-level-local:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v3
+
+      - name: Setup Runhouse
+        uses: ./.github/workflows/setup_runhouse
+
+      - name: Setup ~/.rh/config.yaml
+        uses: ./.github/workflows/setup_rh_config
+        with:
+          username: ${{ secrets.CI_ACCOUNT_USERNAME }}
+          token: ${{ secrets.CI_ACCOUNT_TOKEN }}
+          api_server_url: ${{ env.API_SERVER_URL }}
+
+      - name: pytest -v --level local -k "not servertest and not secrettest and not moduletest and not functiontest and not envtest"
+        env:
+          TEST_TOKEN: ${{ secrets.TEST_TOKEN }}
+          TEST_USERNAME: ${{ secrets.TEST_USERNAME }}
+        run: pytest -v --level local -k "not servertest and not secrettest and not moduletest and not functiontest and not envtest" --api-server-url $API_SERVER_URL
+        timeout-minutes: 60
+
+  # secret-tests-logged-out-level-local:
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - name: Check out repository code
+  #       uses: actions/checkout@v3
+
+  #     - name: Setup Runhouse
+  #       uses: ./.github/workflows/setup_runhouse
+
+  #     - name: pytest -v --level local -k "secrettest"
+  #       env:
+  #         TEST_TOKEN: ${{ secrets.TEST_TOKEN }}
+  #         TEST_USERNAME: ${{ secrets.TEST_USERNAME }}
+  #       run: pytest -v --level local -k "secrettest"
+
+  secret-tests-logged-in-level-local:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v3
+
+      - name: Setup Runhouse
+        uses: ./.github/workflows/setup_runhouse
+
+      - name: Setup ~/.rh/config.yaml
+        uses: ./.github/workflows/setup_rh_config
+        with:
+          username: ${{ secrets.CI_ACCOUNT_USERNAME }}
+          token: ${{ secrets.CI_ACCOUNT_TOKEN }}
+          api_server_url: ${{ env.API_SERVER_URL }}
+
+      - name: pytest -v --level local -k "secrettest"
+        env:
+          TEST_TOKEN: ${{ secrets.TEST_TOKEN }}
+          TEST_USERNAME: ${{ secrets.TEST_USERNAME }}
+        run: pytest -v --level local -k "secrettest" --api-server-url $API_SERVER_URL
+        timeout-minutes: 60
+
+  module-tests-logged-in-level-local:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v3
+
+      - name: Setup Runhouse
+        uses: ./.github/workflows/setup_runhouse
+
+      - name: Setup ~/.rh/config.yaml
+        uses: ./.github/workflows/setup_rh_config
+        with:
+          username: ${{ secrets.CI_ACCOUNT_USERNAME }}
+          token: ${{ secrets.CI_ACCOUNT_TOKEN }}
+          api_server_url: ${{ env.API_SERVER_URL }}
+
+      - name: pytest -v --level local -k "moduletest"
+        env:
+          TEST_TOKEN: ${{ secrets.TEST_TOKEN }}
+          TEST_USERNAME: ${{ secrets.TEST_USERNAME }}
+        run: pytest -v --level local -k "moduletest" --api-server-url $API_SERVER_URL
+        timeout-minutes: 60
+
+  function-tests-logged-in-level-local:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v3
+
+      - name: Setup Runhouse
+        uses: ./.github/workflows/setup_runhouse
+
+      - name: Setup ~/.rh/config.yaml
+        uses: ./.github/workflows/setup_rh_config
+        with:
+          username: ${{ secrets.CI_ACCOUNT_USERNAME }}
+          token: ${{ secrets.CI_ACCOUNT_TOKEN }}
+          api_server_url: ${{ env.API_SERVER_URL }}
+
+      - name: pytest -v --level local -k "functiontest"
+        env:
+          TEST_TOKEN: ${{ secrets.TEST_TOKEN }}
+          TEST_USERNAME: ${{ secrets.TEST_USERNAME }}
+        run: pytest -v --level local -k "functiontest" --api-server-url $API_SERVER_URL
+        timeout-minutes: 60
+
+  env-tests-logged-in-level-local:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v3
+
+      - name: Setup Runhouse
+        uses: ./.github/workflows/setup_runhouse
+
+      - name: Setup ~/.rh/config.yaml
+        uses: ./.github/workflows/setup_rh_config
+        with:
+          username: ${{ secrets.CI_ACCOUNT_USERNAME }}
+          token: ${{ secrets.CI_ACCOUNT_TOKEN }}
+          api_server_url: ${{ env.API_SERVER_URL }}
+
+      - name: pytest -v --level local -k "envtest"
+        env:
+          TEST_TOKEN: ${{ secrets.TEST_TOKEN }}
+          TEST_USERNAME: ${{ secrets.TEST_USERNAME }}
+        run: pytest -v --level local -k "envtest" --api-server-url $API_SERVER_URL
+        timeout-minutes: 60

.github/workflows/setup_rh_config/action.yaml

@@ -11,6 +11,10 @@ inputs:
     description: 'The token of the logged in username'
     required: true
 
+  api_server_url:
+    description: 'The den api server to send the requests to'
+    required: true
+
 runs:
   using: composite
   steps:
@@ -22,3 +26,4 @@ runs:
         echo "disable_data_collection: true" >> ~/.rh/config.yaml
         echo "token: ${{ inputs.token }}" >> ~/.rh/config.yaml
         echo "username: ${{ inputs.username }}" >> ~/.rh/config.yaml
+        echo "api_server_url: ${{ inputs.api_server_url }}" >> ~/.rh/config.yaml

runhouse/main.py

@@ -296,6 +296,7 @@ def _start_server(
     domain=None,
     certs_address=None,
     use_local_telemetry=False,
+    api_server_url=None,
 ):
     ############################################
     # Build CLI commands to start the server
@@ -369,6 +370,13 @@ def _start_server(
         logger.info("Configuring local telemetry on the cluster.")
         flags.append(use_local_telemetry_flag)
 
+    api_server_url_flag = (
+        f" --api-server-url {api_server_url}" if api_server_url else ""
+    )
+    if api_server_url_flag:
+        logger.info(f"Setting api_server url to {api_server_url}")
+        flags.append(api_server_url_flag)
+
     # Check if screen or nohup are available
     screen = screen and _check_if_command_exists("screen")
     nohup = not screen and nohup and _check_if_command_exists("nohup")
@@ -535,6 +543,10 @@ def restart(
         False,
         help="Whether to use local telemetry",
     ),
+    api_server_url: str = typer.Option(
+        default="https://api.run.house",
+        help="URL of Runhouse Den",
+    ),
 ):
     """Restart the HTTP server on the cluster."""
     if name:
@@ -559,6 +571,7 @@ def restart(
         domain=domain,
         certs_address=certs_address,
         use_local_telemetry=use_local_telemetry,
+        api_server_url=api_server_url,
     )
 
 

runhouse/resources/hardware/cluster.py

@@ -113,6 +113,7 @@ def creds_values(self) -> Dict:
         return self._creds.values
 
     def save_config_to_cluster(self, node: str = None):
+
         config = self.config(condensed=False)
         config.pop("creds")
         json_config = f"{json.dumps(config)}"
@@ -196,6 +197,7 @@ def config(self, condensed=True):
             creds = creds.replace("loaded_secret_", "")
 
         config["creds"] = creds
+        config["api_server_url"] = rns_client.api_server_url
 
         if self._use_custom_certs:
             config["ssl_certfile"] = self.cert_config.cert_path
@@ -743,11 +745,12 @@ def restart_server(
             + (f" --domain {domain}" if domain else "")
             + (" --use-local-telemetry" if use_local_telemetry else "")
             + f" --port {self.server_port}"
+            + f" --api-server-url {rns_client.api_server_url}"
         )
 
         status_codes = self.run(commands=[cmd], env=env)
         if not status_codes[0][0] == 0:
-            raise ValueError(f"Failed to restart server {self.name}.")
+            raise ValueError(f"Failed to restart server {self.name}")
 
         if https_flag:
             rns_address = self.rns_address or self.name

runhouse/rns/defaults.py

@@ -233,6 +233,11 @@ def set_many(self, key_value_pairs: Dict, config_path: Optional[str] = None):
 
     # TODO [DG] allow hierarchical defaults from folders and groups
     def get(self, key: str, alt: Any = None) -> Any:
+        # Prioritize env vars
+        env_var = os.getenv(key.upper())
+        if env_var is not None:
+            return env_var
+
         res = self.defaults_cache.get(key, alt)
         if not res and key in self.BASE_DEFAULTS:
             res = self.BASE_DEFAULTS[key]

runhouse/rns/rns_client.py

@@ -153,6 +153,9 @@ def username(self):
 
     @property
     def api_server_url(self):
+        url_as_env_var = os.getenv("API_SERVER_URL")
+        if url_as_env_var:
+            return url_as_env_var
         return self._configs.get("api_server_url", None)
 
     def _index_base_folders(self, lst):

runhouse/servers/cluster_servlet.py

@@ -28,7 +28,7 @@ async def __init__(
         )
         self._initialized_env_servlet_names: Set[str] = set()
         self._key_to_env_servlet_name: Dict[Any, str] = {}
-        self._auth_cache: AuthCache = AuthCache()
+        self._auth_cache: AuthCache = AuthCache(cluster_config)
 
         if self.cluster_config.get("resource_subtype", None) == "OnDemandCluster":
             self._last_activity = time.time()

runhouse/servers/http/auth.py

@@ -10,9 +10,10 @@
 
 class AuthCache:
     # Maps a user's token to all the resources they have access to
-    def __init__(self):
+    def __init__(self, cluster_config: dict):
         self.CACHE = {}
         self.USERNAMES = {}
+        self.cluster_config = cluster_config
 
     def get_username(self, token: str) -> Optional[str]:
         """Get username associated with a particular token"""
@@ -41,7 +42,7 @@ def lookup_access_level(
         else:
             resource_uri_to_send = resource_uri.replace("/", ":")
 
-        uri = f"{rns_client.api_server_url}/resource/{resource_uri_to_send}"
+        uri = f"{self.cluster_config.get('api_server_url')}/resource/{resource_uri_to_send}"
         resp = rns_client.session.get(
             uri,
             headers={"Authorization": f"Bearer {token}"},