Repository navigation
Expand file tree
/
Copy pathflow.py
More file actions
139 lines (115 loc) · 4.24 KB
/
Copy pathflow.py
File metadata and controls
139 lines (115 loc) · 4.24 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
from metaflow import (
card,
FlowSpec,
step,
current,
resources,
pypi,
Parameter,
trigger_on_finish,
profile,
Flow,
)
from metaflow.cards import Markdown as MD, Image
from obproject import ProjectFlow, project_trigger, highlight
from xkcd_utils import get_img
MODEL = "HuggingFaceTB/SmolVLM-Instruct"
PROMPT = "Explain what is funny about this XKCD comic strip?"
def prompt(img_url):
"""
This prompting example is from
https://huggingface.co/HuggingFaceTB/SmolVLM-Instruct
"""
import torch
from transformers import AutoProcessor, AutoModelForVision2Seq
from transformers.image_utils import load_image
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
image1 = load_image(img_url)
with profile(f"Loading model"):
processor = AutoProcessor.from_pretrained(MODEL)
model = AutoModelForVision2Seq.from_pretrained(
MODEL,
torch_dtype=torch.bfloat16,
_attn_implementation="flash_attention_2" if DEVICE == "cuda" else "eager",
).to(DEVICE)
# Create input messages
messages = [
{
"role": "user",
"content": [{"type": "image"}, {"type": "text", "text": PROMPT}],
},
]
# Prepare inputs
prompt = processor.apply_chat_template(messages, add_generation_prompt=True)
inputs = processor(text=prompt, images=[image1], return_tensors="pt")
inputs = inputs.to(DEVICE)
# Generate outputs
with profile("Prompting"):
generated_ids = model.generate(**inputs, max_new_tokens=500)
generated_texts = processor.batch_decode(
generated_ids,
skip_special_tokens=True,
)
return generated_texts[0]
@project_trigger(event="explain")
class XKCDExplainer(ProjectFlow):
xkcd_url = Parameter("xkcd_url", help="Image url of an XKCD comic", default="null")
@card(type="blank")
@step
def start(self):
if self.xkcd_url and self.xkcd_url not in ("null", "None", ""):
self.img_url = self.xkcd_url
print(f"Using an image passed in as a parameter, {self.img_url}")
else:
try:
self.img_url = self.prj.get_data("xkcd")
except Exception:
ref = self.prj._write_asset.peek_data_asset("xkcd")
from metaflow import get_namespace, namespace, Task
ns = get_namespace()
try:
namespace(None)
task = Task(ref["created_by"]["entity_id"])
self.img_url = task[ref["data_properties"]["annotations"]["artifact"]].data
finally:
namespace(ns)
print(f"Using an image from the latest data asset, {self.img_url}")
try:
print(self.prj.asset.consume_model_asset("explainer-vlm"))
except Exception:
print(f"No 'explainer-vlm' model on read branch '{self.prj.read_branch}' yet.")
self.next(self.prompt_vlm)
# ⬇️ add gpu=1 to @resources if you have GPU compute pools configured
@resources(cpu=4, memory=16000)
@card(type="blank", id="model", refresh_interval=2)
@pypi(
python="3.11.11",
packages={"transformers": "4.55.2", "torch": "2.8.0", "pillow": "11.3.0"},
)
@highlight
@step
def prompt_vlm(self):
msg = f"Starting model `{MODEL}`.. This may take 3-5 minutes! ⌛"
title = MD(f"## {msg}")
current.card["model"].append(title)
current.card["model"].refresh()
print(msg)
explanation = prompt(self.img_url)
print(explanation)
print("🔍 See a card attached for the explanation in context")
img = get_img(self.img_url)
self.highlight.title = "Click to see an explanation of the comic"
self.highlight.add_line(f"The latest XKCD comic")
self.highlight.set_image(img)
title.update("### ✅ Prompting done!")
current.card["model"].append(
MD(f"## 🤖 `{MODEL}`'s interpretation of the comic")
)
current.card["model"].append(Image(img))
current.card["model"].append(MD(explanation))
self.next(self.end)
@step
def end(self):
pass
if __name__ == "__main__":
XKCDExplainer()