v2.19.5

.github/workflows/python-package.yml

@@ -41,18 +41,14 @@ jobs:
         poetry run pip list
 
     # Style/format checks.
-    - name: Run Black (code formatter)
-      run: |
-        poetry run black --check --diff .
-
-    - name: Run isort (import sorting)
-      run: |
-        poetry run isort --check-only --diff --profile black .
-
     - name: Run Ruff (linter)
       run: |
         poetry run ruff check --diff .
 
+    - name: Run Ruff (formatter)
+      run: |
+        poetry run ruff format --check --diff .
+
     - name: Test with pytest
       run: |
         # run tests in tests/ dir and only fail if there are failures or errors

.pre-commit-config.yaml

@@ -20,18 +20,4 @@ repos:
   hooks:
     - id: ruff
       args: [--fix]
-
-- repo: https://github.com/psf/black
-  rev: 23.7.0
-  hooks:
-    - id: black
-      name: black (code formatter)
-      language_version: python3.10
-      additional_dependencies: ["black[jupyter]"]
-
-- repo: https://github.com/pycqa/isort
-  rev: 5.13.2
-  hooks:
-    - id: isort
-      name: isort (import sorting)
-      args: ["--profile", "black"]
+    - id: ruff-format

containerized_job/Dockerfile

@@ -0,0 +1,24 @@
+FROM python:3.10-slim
+
+WORKDIR /app
+
+RUN apt-get update && apt-get install -y \
+    git build-essential \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY download_model.py .
+
+ARG LLM_MODEL
+ENV MODEL_PATH=./downloaded_model
+
+RUN python download_model.py --model-name "$LLM_MODEL" --model-path "$MODEL_PATH"
+
+COPY . .
+COPY --from=external_context /vllm_llm.py .
+
+EXPOSE 8000
+
+CMD ["python", "app.py"]

containerized_job/app.py

@@ -0,0 +1,50 @@
+import os
+
+import uvicorn
+from fastapi import FastAPI
+from fastapi.responses import JSONResponse
+from schema import ChatRequest, LogitsRequest
+from vllm_llm import ReproducibleVLLM
+
+MODEL_PATH = os.getenv("MODEL_PATH")
+
+
+class ReproducibleVllmApp:
+    def __init__(self):
+        self.llm = ReproducibleVLLM(model_id=MODEL_PATH)
+        self.app = FastAPI()
+        self.app.post("/generate")(self.generate)
+        self.app.post("/generate_logits")(self.generate_logits)
+
+    async def generate(self, request: ChatRequest):
+        try:
+            result = await self.llm.generate(
+                messages=[m.dict() for m in request.messages],
+                sampling_params=request.sampling_parameters.dict(),
+                seed=request.seed,
+                continue_last_message=request.continue_last_message,
+            )
+            return {"result": result}
+        except Exception as e:
+            return JSONResponse(status_code=500, content={"error": str(e)})
+
+    async def generate_logits(self, request: LogitsRequest):
+        try:
+            logits, prompt = await self.llm.generate_logits(
+                messages=[m.dict() for m in request.messages],
+                top_logprobs=request.top_logprobs,
+                sampling_params=request.sampling_parameters.dict(),
+                seed=request.seed,
+                continue_last_message=request.continue_last_message,
+            )
+            return {"logits": logits, "prompt": prompt}
+        except Exception as e:
+            return JSONResponse(status_code=500, content={"error": str(e)})
+
+    def run(self):
+        uvicorn.run(self.app, host="0.0.0.0", port=8000)
+
+
+if __name__ == "__main__":
+    server = ReproducibleVllmApp()
+    server.run()

containerized_job/build.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+IMAGE_NAME="sn1-validator-api"
+MODEL_NAME="mrfakename/mistral-small-3.1-24b-instruct-2503-hf"
+
+DOCKER_BUILDKIT=1 docker build \
+    --build-arg LLM_MODEL="$MODEL_NAME" \
+    -t "$IMAGE_NAME" \
+    --build-context external_context=../prompting/llms \
+    .

containerized_job/download_model.py

@@ -0,0 +1,24 @@
+import argparse
+
+from huggingface_hub import snapshot_download
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Download model files")
+    parser.add_argument(
+        "--model-name",
+        type=str,
+        help="Model name to use",
+    )
+    parser.add_argument(
+        "--model-path",
+        type=str,
+        help="Path to save the model files",
+    )
+
+    args = parser.parse_args()
+
+    print(f"Downloading Model {args.model_name}, files downloaded to {args.model_path}")
+
+    snapshot_download(repo_id=args.model_name, local_dir=args.model_path)
+
+    print(f"Model files downloaded to {args.model_path}")

containerized_job/requirements.txt

@@ -0,0 +1,8 @@
+fastapi==0.115.0
+uvicorn==0.23.2
+pydantic==2.9.0
+vllm==0.8.3
+torch==2.6.0
+numpy==1.26.4
+loguru==0.7.2
+huggingface-hub==0.30.0

containerized_job/schema.py

@@ -0,0 +1,29 @@
+from typing import List, Literal, Optional
+
+from pydantic import BaseModel
+
+
+class ChatMessage(BaseModel):
+    content: str
+    role: Literal["user", "assistant", "system"]
+
+
+class SamplingParameters(BaseModel):
+    temperature: Optional[float] = 1.0
+    top_p: Optional[float] = 1.0
+    max_tokens: Optional[int] = 512
+    presence_penalty: Optional[float] = 0.0
+    frequency_penalty: Optional[float] = 0.0
+    top_k: Optional[int] = -1
+    logprobs: Optional[int] = None
+
+
+class ChatRequest(BaseModel):
+    messages: List[ChatMessage]
+    seed: Optional[int]
+    sampling_parameters: Optional[SamplingParameters] = SamplingParameters()
+    continue_last_message: Optional[bool] = False
+
+
+class LogitsRequest(ChatRequest):
+    top_logprobs: Optional[int] = 10

docs/API_docs.md

@@ -87,11 +87,11 @@ bash run_api.sh
 
 **Endpoint:** `POST /miner_availabilities/miner_availabilities`
 
-**Description:** Fetches miner availabilities based on provided UIDs.
+**Description:** Fetches miner availabilities based on provided UIDs. **Note: Specifying UIDs is deprecated.**
 
 **Request Body:**
 
-- JSON array of integers or null (optional).
+- JSON array of integers or null (optional, deprecated).
 
 ---
 
@@ -169,13 +169,13 @@ Web Retrieval
 
 **Endpoint:** `GET /web_retrieval`
 
-**Description:** Retrieves a list websites about a search query
+**Description:** Retrieves a list websites about a search query. **Note: The `uids` parameter is deprecated.**
 
 **Parameters:**
 
 - **search_query** (str): The search term you'd like to look up
 - **n_miners** (int, optional): How many miners to query
-- **uids**: (list[int], optional): which specific uids to query (Deprecated)
+- **uids**: (list[int], optional, deprecated): which specific uids to query
 
 ---
 

prompting/api/weight_syncing/api.py

@@ -22,7 +22,7 @@ async def verify_weight_signature(request: Request):
         raise HTTPException(status_code=400, detail="Bad Request, message is not intended for self")
     validator_hotkeys = [shared_settings.METAGRAPH.hotkeys[uid] for uid in WHITELISTED_VALIDATORS_UIDS]
     if signed_by not in validator_hotkeys:
-        logger.error("Signer not the expected ss58 address")
+        logger.error(f"Signer not the expected ss58 address: {signed_by}")
         raise HTTPException(status_code=401, detail="Signer not the expected ss58 address")
     now = time.time()
     body = await request.body()

prompting/datasets/sn13.py

@@ -10,10 +10,8 @@
 class SN13Dataset(BaseDataset):
     _url: ClassVar[str] = "arrmlet/x_dataset_218"
     name: ClassVar[str] = "x_dataset_218"
-    _chance_word_synonym: ClassVar[float] = 0.10
-    _chance_char_typo: ClassVar[float] = 0.02
     exception: Exception | None = None
-    dataset: datasets.Dataset = None
+    dataset: datasets.Dataset | None = None
 
     class Config:
         arbitrary_types_allowed = True

prompting/llms/utils.py

@@ -117,7 +117,7 @@ def gpu_utilization(cls):
         return cls.used_memory / cls.total_memory
 
 
-TEXT_MODELS = ["mrfakename/mistral-small-3.1-24b-instruct-2503-hf"]
+TEXT_MODELS: set[str | None] = set([None, "mrfakename/mistral-small-3.1-24b-instruct-2503-hf"])
 
 
 def model_factory(model_name: str) -> type[ReproducibleHF]:

prompting/rewards/exact_match.py

@@ -18,12 +18,14 @@
 TOP_LOGPROBS = 10
 MIN_VERIFY_TOKENS = 10
 MAX_VERIFY_TOKENS = 51
-PARTIAL_PENALTY = -1.0
+# Partial completion is much more harmful from API perspective, compared to no response.
+# TODO: Experimental aggressive value, revisit once the network is clean.
+PARTIAL_PENALTY = -100.0
 INCORRECT_PENALTY = -2.0
 NOT_ENOUGH_TOKENS_PENALTY_SCALE = 0.1
 MIN_SMOOTH_PENALTY_SCALE = 0.3
 MIN_TIME_PENALTY_SCALE = 0.3
-VERIFICATION_THRESH_CONTAINS = 0.92
+VERIFICATION_THRESH_CONTAINS = 0.90
 VERIFICATION_THRESH_SIM = 0.83
 VERIFICATION_SIM_EXP_SCALE = 2.0
 
@@ -108,6 +110,7 @@ async def reward(  # noqa: C901
                     to_complete = "".join(chunks[:check_idx])
                     if to_complete:
                         messages.extend([{"role": "assistant", "content": to_complete}])
+
                     verification_logits, _ = await model_manager.generate_logits(
                         model=task.llm_model_id,
                         messages=messages,

prompting/rewards/scoring.py

@@ -108,44 +108,62 @@ async def run_step(self) -> RewardLoggingEvent:
                 model_manager=self.model_scheduler.llm_model_manager,
                 task_queue=self.task_queue,
             )
-            if scoring_config.task.organic:
-                logger.debug(f"Reward events size: {len(reward_events)}")
+
         self.reward_events.append(reward_events)
 
         logger.debug(
             f"Scored {scoring_config.task.__class__.__name__} {scoring_config.task.task_id} with model "
             f"{scoring_config.task.llm_model_id}"
         )
-        if not scoring_config.task.organic:
-            # Reduce log size for raw chunks, wandb fails to log any data when overloaded.
-            response = copy.deepcopy(scoring_config.response)
-            response.stream_results_all_chunk_dicts_raw = []
-            for idx in range(len(response.stream_results)):
-                response.stream_results[idx].accumulated_chunk_dicts_raw = []
-
-            if isinstance(scoring_config.task, MSRv2Task):
-                if scoring_config.task.ground_truth is not None:
-                    reference_value = str(scoring_config.task.ground_truth)  # "0" or "1"
-                else:
-                    reference_value = None
+
+        # Reduce log size for raw chunks, wandb fails to log any data when overloaded.
+        response = copy.deepcopy(scoring_config.response)
+        response.stream_results_all_chunk_dicts_raw = []
+        for idx in range(len(response.stream_results)):
+            response.stream_results[idx].accumulated_chunk_dicts_raw = []
+
+        if isinstance(scoring_config.task, MSRv2Task):
+            if scoring_config.task.ground_truth is not None:
+                reference_value = str(scoring_config.task.ground_truth)  # "0" or "1"
             else:
-                reference_value = scoring_config.task.reference
-
-            log_event(
-                RewardLoggingEvent(
-                    response_event=response,
-                    reward_events=reward_events,
-                    reference=reference_value,
-                    challenge=scoring_config.task.query,
-                    task=scoring_config.task.name,
-                    block=scoring_config.block,
-                    step=scoring_config.step,
-                    task_id=scoring_config.task_id,
-                    task_dict=scoring_config.task.model_dump(),
-                    source=scoring_config.dataset_entry.source,
-                )
+                reference_value = None
+        else:
+            reference_value = scoring_config.task.reference
+
+        if scoring_config.task.organic:
+            response.stream_results = []
+            response.axons = []
+            response.completions = []
+            response.stream_results_all_chunks = []
+            response.stream_results_all_tokens_per_chunk = []
+            reward_events = copy.deepcopy(reward_events)
+            for event in reward_events:
+                event.task = event.task.__class__()
+
+            reference = None
+            challenge = ""
+            task_dict = {}
+            source = "organic"
+        else:
+            reference = reference_value
+            challenge = scoring_config.task.query
+            task_dict = scoring_config.task.model_dump()
+            source = scoring_config.dataset_entry.source
+
+        log_event(
+            RewardLoggingEvent(
+                response_event=response,
+                reward_events=reward_events,
+                reference=reference,
+                challenge=challenge,
+                task=scoring_config.task.name,
+                block=scoring_config.block,
+                step=scoring_config.step,
+                task_id=scoring_config.task_id,
+                task_dict=task_dict,
+                source=source,
             )
-
+        )
         self.model_scheduler.llm_model_manager.lock.release()
         await asyncio.sleep(0.01)