Skip to content

Commit

Permalink
Bug fix in "genv remote llm attach"
Browse files Browse the repository at this point in the history
* Supporting new environment name format

- Showing port in "ps"
  • Loading branch information
razrotenberg committed May 7, 2024
1 parent 78026fd commit c4c39f5
Showing 1 changed file with 20 additions and 4 deletions.
24 changes: 20 additions & 4 deletions genv/cli/remote.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import dataclasses
import itertools
import os
import re
import shutil
import sys
from typing import Iterable, NoReturn, Optional, Tuple
Expand Down Expand Up @@ -241,6 +242,14 @@ async def do_envs(
)


def _find_llm_port(env: genv.Env) -> Optional[int]:
"""Finds any port an LLM server environment listens on."""

match = re.match(r"^llm/[^/]+/(\d+)$", env.config.name)
if match:
return int(match.group(1))


async def do_llm(config: genv.remote.Config, args: argparse.Namespace):
"""Runs the "genv llm" logic."""

Expand All @@ -265,9 +274,16 @@ async def do_llm_attach(config: genv.remote.Config, model: str) -> NoReturn:
hosts, snapshots = await genv.remote.core.envs.snapshot(config)

for host, snapshot in zip(hosts, snapshots):
replicas = len(snapshot.filter(name=f"llm/{model}"))
for env in snapshot:
if not env.config.name:
continue

if not (
env.config.name.startswith(f"llm/{model}/")
or env.config.name == f"llm/{model}" # before 1.4.1
):
continue

if replicas > 0:
# TODO(raz): we currently attach to the first node; we should pick a node in a smarter way.
_exec_ssh(host, f"genv llm attach {model}")

Expand Down Expand Up @@ -309,8 +325,8 @@ async def do_llm_ps(

total += 1

model = env.config.name.split("llm/")[1]
port = "N/A"
model = env.config.name.split("/")[1]
port = _find_llm_port(env) or "N/A"
created = env.creation if timestamp else env.time_since
eid = env.eid
user = env.username or ""
Expand Down

0 comments on commit c4c39f5

Please sign in to comment.