From 27929363f644b783caaad71f131302a0822784e3 Mon Sep 17 00:00:00 2001 From: woWoosuk Kwon Date: Wed, 10 May 2023 08:30:57 +0000 Subject: [PATCH 1/6] Minor --- cacheflow/core/scheduler.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/cacheflow/core/scheduler.py b/cacheflow/core/scheduler.py index 7656ae978ec0..046b07dc7931 100644 --- a/cacheflow/core/scheduler.py +++ b/cacheflow/core/scheduler.py @@ -1,8 +1,6 @@ import enum -import os -import pickle import time -from typing import Any, Dict, List, Optional, Tuple +from typing import Dict, List, Optional, Tuple from cacheflow.core.block_manager import BlockSpaceManager from cacheflow.logger import init_logger @@ -347,7 +345,6 @@ def _allocate(self, seq_group: SequenceGroup) -> None: self.block_manager.allocate(seq_group) for seq in seq_group.seqs: seq.status = SequenceStatus.RUNNING - # FIXME(woosuk): Support interactive generation. if seq_group.group_id not in self.num_steps: self.num_steps[seq_group.group_id] = 0 From d3bbe68ec28500a5f71d1264cb739b4333b67a16 Mon Sep 17 00:00:00 2001 From: woWoosuk Kwon Date: Wed, 10 May 2023 08:42:44 +0000 Subject: [PATCH 2/6] Avoid sorting the waiting queue --- cacheflow/core/scheduler.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cacheflow/core/scheduler.py b/cacheflow/core/scheduler.py index 046b07dc7931..bb2f0d2a31b3 100644 --- a/cacheflow/core/scheduler.py +++ b/cacheflow/core/scheduler.py @@ -127,7 +127,6 @@ def _schedule( # Swap in the sequence groups in the SWAPPED state if possible. self.swapped = self.policy.sort_by_priority(now, self.swapped) - # FCFS while self.swapped and not blocks_to_swap_out: seq_group = self.swapped[0] # If the sequence group has been preempted in this step, stop. @@ -160,7 +159,6 @@ def _schedule( # This is because we want to bound the amount of CPU memory taken by # the swapped sequence groups. if not self.swapped: - self.waiting = self.policy.sort_by_priority(now, self.waiting) while self.waiting: seq_group = self.waiting[0] # If the sequence group has been preempted in this step, stop. @@ -401,7 +399,9 @@ def _preempt_by_recompute( for seq in seqs: seq.status = SequenceStatus.WAITING self.block_manager.free(seq) - self.waiting.append(seq_group) + # For FCFS, we insert the preempted sequence group to the front of the + # waiting queue. + self.waiting.insert(0, seq_group) def _preempt_by_swap( self, From ffa03cabdd810529235253d63ce74c9469c3a8f1 Mon Sep 17 00:00:00 2001 From: woWoosuk Kwon Date: Wed, 10 May 2023 08:43:21 +0000 Subject: [PATCH 3/6] Fix imports --- cacheflow/core/scheduler.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/cacheflow/core/scheduler.py b/cacheflow/core/scheduler.py index bb2f0d2a31b3..c6d6cae4a62b 100644 --- a/cacheflow/core/scheduler.py +++ b/cacheflow/core/scheduler.py @@ -6,11 +6,8 @@ from cacheflow.logger import init_logger from cacheflow.core.policy import PolicyFactory from cacheflow.sampling_params import SamplingParams -from cacheflow.sequence import Sequence -from cacheflow.sequence import SequenceGroup -from cacheflow.sequence import SequenceGroupMetadata -from cacheflow.sequence import SequenceOutputs -from cacheflow.sequence import SequenceStatus +from cacheflow.sequence import (Sequence, SequenceGroup, SequenceGroupMetadata, + SequenceOutputs, SequenceStatus) logger = init_logger(__name__) From ab542eebc41eb487f7b9e8443e519739d99438a1 Mon Sep 17 00:00:00 2001 From: woWoosuk Kwon Date: Wed, 10 May 2023 08:49:05 +0000 Subject: [PATCH 4/6] Add a comment --- cacheflow/core/scheduler.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cacheflow/core/scheduler.py b/cacheflow/core/scheduler.py index c6d6cae4a62b..ac54efe0265a 100644 --- a/cacheflow/core/scheduler.py +++ b/cacheflow/core/scheduler.py @@ -156,6 +156,9 @@ def _schedule( # This is because we want to bound the amount of CPU memory taken by # the swapped sequence groups. if not self.swapped: + # Optimization: We do not sort the waiting queue since the preempted + # sequence groups are added to the front and the new sequence groups + # are added to the back. while self.waiting: seq_group = self.waiting[0] # If the sequence group has been preempted in this step, stop. From c7c248d3a5c47fd87324a60d5b78a996d7d7eedf Mon Sep 17 00:00:00 2001 From: woWoosuk Kwon Date: Wed, 10 May 2023 08:49:52 +0000 Subject: [PATCH 5/6] Minor --- cacheflow/core/scheduler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cacheflow/core/scheduler.py b/cacheflow/core/scheduler.py index ac54efe0265a..aaa51ea3ad6b 100644 --- a/cacheflow/core/scheduler.py +++ b/cacheflow/core/scheduler.py @@ -399,8 +399,8 @@ def _preempt_by_recompute( for seq in seqs: seq.status = SequenceStatus.WAITING self.block_manager.free(seq) - # For FCFS, we insert the preempted sequence group to the front of the - # waiting queue. + # NOTE: For FCFS, we insert the preempted sequence group to the front + # of the waiting queue. self.waiting.insert(0, seq_group) def _preempt_by_swap( From 45d5530715362c6abbb78463c542808d23aff4f2 Mon Sep 17 00:00:00 2001 From: woWoosuk Kwon Date: Wed, 10 May 2023 08:55:06 +0000 Subject: [PATCH 6/6] Minor --- cacheflow/core/scheduler.py | 2 +- cacheflow/core/server.py | 1 - cacheflow/frontend/simple_frontend.py | 1 - cacheflow/model_executor/memory_analyzer.py | 1 - 4 files changed, 1 insertion(+), 4 deletions(-) diff --git a/cacheflow/core/scheduler.py b/cacheflow/core/scheduler.py index aaa51ea3ad6b..02e864e626c3 100644 --- a/cacheflow/core/scheduler.py +++ b/cacheflow/core/scheduler.py @@ -9,8 +9,8 @@ from cacheflow.sequence import (Sequence, SequenceGroup, SequenceGroupMetadata, SequenceOutputs, SequenceStatus) - logger = init_logger(__name__) + _LOGGING_INTERVAL_SEC = 10 diff --git a/cacheflow/core/server.py b/cacheflow/core/server.py index 2f968f8b4e24..a35a27bcb129 100644 --- a/cacheflow/core/server.py +++ b/cacheflow/core/server.py @@ -17,7 +17,6 @@ from cacheflow.utils import get_gpu_memory, get_cpu_memory from cacheflow.worker.controller import Controller, DeviceID - logger = init_logger(__name__) diff --git a/cacheflow/frontend/simple_frontend.py b/cacheflow/frontend/simple_frontend.py index da3639530cd6..9d65e4f0de55 100644 --- a/cacheflow/frontend/simple_frontend.py +++ b/cacheflow/frontend/simple_frontend.py @@ -7,7 +7,6 @@ from cacheflow.sequence import Sequence, SequenceGroup from cacheflow.utils import Counter - logger = init_logger(__name__) diff --git a/cacheflow/model_executor/memory_analyzer.py b/cacheflow/model_executor/memory_analyzer.py index bc85d6586ae0..fb910e6403de 100644 --- a/cacheflow/model_executor/memory_analyzer.py +++ b/cacheflow/model_executor/memory_analyzer.py @@ -4,7 +4,6 @@ from cacheflow.logger import init_logger from cacheflow.model_executor.utils import get_dtype_size - logger = init_logger(__name__) _GiB = 1 << 30