Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[HF][streaming][2/n] Text Translation #853

Merged
merged 2 commits into from
Jan 10, 2024
Merged

[HF][streaming][2/n] Text Translation #853

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
074b768
[HF][streaming][1/n] Text Summarization
Jan 10, 2024
13a4c6e
[HF][streaming][2/n] Text Translation
Jan 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions extensions/HuggingFace/python/requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,12 @@ huggingface_hub

#Hugging Face Libraries - Local Inference Tranformers & Diffusors
accelerate # Used to help speed up image generation
diffusers # Used for image + audio generation
diffusers # Used for image generation
scipy # array -> wav file, text-speech. torchaudio.save seems broken.
sentencepiece # Used for text translation
torch
torchvision
torchaudio
scipy # array -> wav file, text-speech. torchaudio.save seems broken.
transformers # Used for text generation

#Other
Expand Down
2 changes: 1 addition & 1 deletion ...HuggingFace/python/src/aiconfig_extension_hugging_face/local_inference/text_generation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -251,7 +251,7 @@ async def run_inference(
not "stream" in completion_data or completion_data.get("stream") != False
)
if should_stream:
tokenizer : AutoTokenizer = AutoTokenizer.from_pretrained(model_name)
tokenizer: AutoTokenizer = AutoTokenizer.from_pretrained(model_name)
streamer = TextIteratorStreamer(tokenizer)
completion_data["streamer"] = streamer

Expand Down
11 changes: 9 additions & 2 deletions ...gingFace/python/src/aiconfig_extension_hugging_face/local_inference/text_summarization.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,13 +128,18 @@ def construct_stream_output(
"metadata": {},
}
)

accumulated_message = ""
for new_text in streamer:
if isinstance(new_text, str):
# For some reason these symbols aren't filtered out by the streamer
new_text = new_text.replace("</s>", "")
new_text = new_text.replace("<s>", "")

accumulated_message += new_text
options.stream_callback(new_text, accumulated_message, 0)

output.data = accumulated_message

return output


Expand Down Expand Up @@ -245,7 +250,9 @@ async def run_inference(self, prompt: Prompt, aiconfig: "AIConfigRuntime", optio

# if stream enabled in runtime options and config, then stream. Otherwise don't stream.
streamer = None
should_stream = (options.stream if options else False) and (not "stream" in completion_data or completion_data.get("stream") != False)
should_stream = (options.stream if options else False) and (
not "stream" in completion_data or completion_data.get("stream") != False
)
if should_stream:
tokenizer: AutoTokenizer = AutoTokenizer.from_pretrained(model_name)
streamer = TextIteratorStreamer(tokenizer)
Expand Down
24 changes: 19 additions & 5 deletions ...uggingFace/python/src/aiconfig_extension_hugging_face/local_inference/text_translation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,12 +129,19 @@ def construct_stream_output(
"metadata": {},
}
)

accumulated_message = ""
for new_text in streamer:
if isinstance(new_text, str):
# For some reason these symbols aren't filtered out by the streamer
new_text = new_text.replace("</s>", "")
new_text = new_text.replace("<s>", "")
new_text = new_text.replace("<pad>", "")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this needed in the summarization model parser too? That one only replaces </s> and <s>

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I didn't personally see it in summarization. If it does show up, we can fix forward but this was just me doing some manual testing so could be I missed some use cases


accumulated_message += new_text
options.stream_callback(new_text, accumulated_message, 0)
output.data = accumulated_message

return output


Expand Down Expand Up @@ -240,19 +247,26 @@ async def run_inference(self, prompt: Prompt, aiconfig: "AIConfigRuntime", optio

model_name: str = aiconfig.get_model_name(prompt)
if isinstance(model_name, str) and model_name not in self.translators:
self.translators[model_name] = pipeline(model_name)
self.translators[model_name] = pipeline("translation", model_name)
translator = self.translators[model_name]

# if stream enabled in runtime options and config, then stream. Otherwise don't stream.
streamer = None
should_stream = (options.stream if options else False) and (not "stream" in completion_data or completion_data.get("stream") != False)
should_stream = (options.stream if options else False) and (
not "stream" in completion_data or completion_data.get("stream") != False
)
if should_stream:
raise NotImplementedError("Streaming is not supported for HuggingFace Text Translation")
tokenizer: AutoTokenizer = AutoTokenizer.from_pretrained(model_name)
streamer = TextIteratorStreamer(tokenizer)
completion_data["streamer"] = streamer

def _translate():
return translator(inputs, **completion_data)

outputs: List[Output] = []
output = None
if not should_stream:
response: List[Any] = translator(inputs, **completion_data)
response: List[Any] = _translate()
for count, result in enumerate(response):
output = construct_regular_output(result, count)
outputs.append(output)
Expand All @@ -263,7 +277,7 @@ async def run_inference(self, prompt: Prompt, aiconfig: "AIConfigRuntime", optio
raise ValueError("Stream option is selected but streamer is not initialized")

# For streaming, cannot call `translator` directly otherwise response will be blocking
thread = threading.Thread(target=translator, kwargs=completion_data)
thread = threading.Thread(target=_translate)
thread.start()
output = construct_stream_output(streamer, options)
if output is not None:
Expand Down