Skip to content

Rename image_encoder to vision_encoder to match HF naming convention #14392

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Sep 17, 2025
Merged

Rename image_encoder to vision_encoder to match HF naming convention #14392

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions examples/models/llava/export_llava.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -224,12 +224,12 @@ def export_all(llava_model: LlavaModel):

lowered_and_edge = to_edge_transform_and_lower(
{
"image_encoder": image_encoder_ep,
"vision_encoder": image_encoder_ep,
"token_embedding": token_embedding_ep,
"text_decoder": text_model_ep,
},
partitioner={
"image_encoder": [XnnpackPartitioner()],
"vision_encoder": [XnnpackPartitioner()],
"text_decoder": [
# First partition the DQLinear nodes, then partition the rest of the nodes,
# to avoid multiple DQLinear nodes in the same partition,
Expand All @@ -254,7 +254,7 @@ def export_all(llava_model: LlavaModel):
],
memory_planning_pass=MemoryPlanningPass(alloc_graph_input=False),
sym_shape_eval_pass={
"image_encoder": ConstraintBasedSymShapeEvalPass(),
"vision_encoder": ConstraintBasedSymShapeEvalPass(),
"text_decoder": ConstraintBasedSymShapeEvalPass(),
"token_embedding": HintBasedSymShapeEvalPass(),
},
Expand Down
2 changes: 1 addition & 1 deletion examples/models/llava/test/test_llava.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ def test_llava_export(self):
start_pos += pte_embeds_before_img.shape[1]

# pte prefill image
pte_embeds_img = llava_module.run_method("image_encoder", (resized,))[0]
pte_embeds_img = llava_module.run_method("vision_encoder", (resized,))[0]
llava_module.run_method(
"text_decoder",
(
Expand Down
2 changes: 1 addition & 1 deletion examples/models/llava/test/test_pte.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ def main():

# pte prefill image
logging.warning("Image encoder started")
pte_embeds_img = llava_module.run_method("image_encoder", (resized,))[0]
pte_embeds_img = llava_module.run_method("vision_encoder", (resized,))[0]
logging.warning("Image encoder finished")
logging.warning("Image token prefill started")
pte_prefill_img = llava_module.run_method(
Expand Down
2 changes: 1 addition & 1 deletion extension/llm/runner/constants.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ inline constexpr auto kUseKVCache = "use_kv_cache";
inline constexpr auto kUseSDPAWithKVCache = "use_sdpa_with_kv_cache";

// Multimodal method name conventions
inline constexpr auto kImageEncoderMethod = "image_encoder";
inline constexpr auto kVisionEncoderMethod = "vision_encoder";
inline constexpr auto kAudioEncoderMethod = "audio_encoder";
inline constexpr auto kTokenEmbeddingMethod = "token_embedding";
inline constexpr auto kTextModelMethod = "text_decoder";
Expand Down
14 changes: 7 additions & 7 deletions extension/llm/runner/multimodal_prefiller.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,9 +43,9 @@ Result<uint64_t> MultimodalPrefiller::prefill(
Image image = input.get_image();

auto method_meta = ET_UNWRAP(
module_->method_meta(kImageEncoderMethod),
module_->method_meta(kVisionEncoderMethod),
"Failed to get method_meta for %s",
kImageEncoderMethod);
kVisionEncoderMethod);

ET_CHECK_MSG(
method_meta.num_inputs() > 0,
Expand Down Expand Up @@ -80,7 +80,7 @@ Result<uint64_t> MultimodalPrefiller::prefill(

// Run image encoder
auto image_encoder_outputs =
ET_UNWRAP(module_->execute(kImageEncoderMethod, image_tensor));
ET_UNWRAP(module_->execute(kVisionEncoderMethod, image_tensor));

encoder_output = image_encoder_outputs[0];
} else if (input.is_audio()) {
Expand Down Expand Up @@ -175,8 +175,8 @@ ::executorch::runtime::Error MultimodalPrefiller::load() {
ET_UNWRAP(module_->method_names(), "Failed to get method names");

// Load image_encoder method if exists.
if (methods.find(kImageEncoderMethod) != methods.end()) {
ET_CHECK_OK_OR_RETURN_ERROR(module_->load_method(kImageEncoderMethod));
if (methods.find(kVisionEncoderMethod) != methods.end()) {
ET_CHECK_OK_OR_RETURN_ERROR(module_->load_method(kVisionEncoderMethod));
}

if (methods.find(kAudioEncoderMethod) != methods.end()) {
Expand All @@ -203,8 +203,8 @@ bool MultimodalPrefiller::is_method_loaded() {
ET_CHECK_MSG(false, "Failed to get method names");
}
std::unordered_set<std::string> methods = methods_res.get();
if (methods.find(kImageEncoderMethod) != methods.end()) {
return module_->is_method_loaded(kImageEncoderMethod);
if (methods.find(kVisionEncoderMethod) != methods.end()) {
return module_->is_method_loaded(kVisionEncoderMethod);
}
return true;
}
Expand Down
Loading