From f65b92ec39d76f10f9a18ae6a2baf96bd09b4538 Mon Sep 17 00:00:00 2001 From: wufeisheng Date: Thu, 11 Sep 2025 11:12:02 +0800 Subject: [PATCH 1/3] Add token processor plugin support --- fastdeploy/engine/common_engine.py | 8 +++++- fastdeploy/plugins/__init__.py | 3 ++- .../plugins/token_processor/__init__.py | 27 +++++++++++++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 fastdeploy/plugins/token_processor/__init__.py diff --git a/fastdeploy/engine/common_engine.py b/fastdeploy/engine/common_engine.py index 929e093c4cd..3c244796e16 100644 --- a/fastdeploy/engine/common_engine.py +++ b/fastdeploy/engine/common_engine.py @@ -42,10 +42,16 @@ from fastdeploy.metrics.metrics import main_process_metrics from fastdeploy.metrics.trace_util import start_span, start_span_request from fastdeploy.model_executor.guided_decoding import schema_checker -from fastdeploy.output.token_processor import TokenProcessor +from fastdeploy.plugins.input_processor import load_token_processor_plugins from fastdeploy.splitwise.splitwise_connector import SplitwiseConnector from fastdeploy.utils import EngineError, envs, llm_logger +try: + TokenProcessor = load_token_processor_plugins() + llm_logger.info(f"TokenProcessor plugin {TokenProcessor} loaded") +except: + from fastdeploy.output.token_processor import TokenProcessor + class EngineService: """ diff --git a/fastdeploy/plugins/__init__.py b/fastdeploy/plugins/__init__.py index 5972f1b4aba..73cee0bbf92 100644 --- a/fastdeploy/plugins/__init__.py +++ b/fastdeploy/plugins/__init__.py @@ -14,7 +14,7 @@ # limitations under the License. """ -from .input_processor import load_input_processor_plugins +from .input_processor import load_input_processor_plugins, load_token_processor_plugins from .model_register import load_model_register_plugins from .model_runner import load_model_runner_plugins from .reasoning_parser import load_reasoning_parser_plugins @@ -24,4 +24,5 @@ "load_model_runner_plugins", "load_input_processor_plugins", "load_reasoning_parser_plugins", + "load_token_processor_plugins", ] diff --git a/fastdeploy/plugins/token_processor/__init__.py b/fastdeploy/plugins/token_processor/__init__.py new file mode 100644 index 00000000000..2b5c76b7a18 --- /dev/null +++ b/fastdeploy/plugins/token_processor/__init__.py @@ -0,0 +1,27 @@ +""" +# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License" +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" + +from fastdeploy.plugins.utils import load_plugins_by_group + +# make sure one process only loads plugins once +PLUGINS_GROUP = "fastdeploy.token_processor_plugins" + + +def load_token_processor_plugins(): + """load_token_processor_plugins""" + plugins = load_plugins_by_group(group=PLUGINS_GROUP) + assert len(plugins) <= 1, "Most one plugin is allowed to be loaded." + return next(iter(plugins.values()))() From f8413964b9e2763930bd5df09baef644bbf87eab Mon Sep 17 00:00:00 2001 From: wufeisheng Date: Thu, 11 Sep 2025 14:34:30 +0800 Subject: [PATCH 2/3] fix import --- fastdeploy/plugins/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fastdeploy/plugins/__init__.py b/fastdeploy/plugins/__init__.py index 73cee0bbf92..08c2922968a 100644 --- a/fastdeploy/plugins/__init__.py +++ b/fastdeploy/plugins/__init__.py @@ -14,10 +14,11 @@ # limitations under the License. """ -from .input_processor import load_input_processor_plugins, load_token_processor_plugins +from .input_processor import load_input_processor_plugins from .model_register import load_model_register_plugins from .model_runner import load_model_runner_plugins from .reasoning_parser import load_reasoning_parser_plugins +from .token_processor import load_token_processor_plugins __all__ = [ "load_model_register_plugins", From 008e6200932196b100e47c14c61e4da5e9cb6f07 Mon Sep 17 00:00:00 2001 From: wufeisheng Date: Thu, 11 Sep 2025 15:52:24 +0800 Subject: [PATCH 3/3] fix import --- fastdeploy/engine/common_engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastdeploy/engine/common_engine.py b/fastdeploy/engine/common_engine.py index 3c244796e16..002e27566ba 100644 --- a/fastdeploy/engine/common_engine.py +++ b/fastdeploy/engine/common_engine.py @@ -42,7 +42,7 @@ from fastdeploy.metrics.metrics import main_process_metrics from fastdeploy.metrics.trace_util import start_span, start_span_request from fastdeploy.model_executor.guided_decoding import schema_checker -from fastdeploy.plugins.input_processor import load_token_processor_plugins +from fastdeploy.plugins.token_processor import load_token_processor_plugins from fastdeploy.splitwise.splitwise_connector import SplitwiseConnector from fastdeploy.utils import EngineError, envs, llm_logger