-
Notifications
You must be signed in to change notification settings - Fork 1
/
DeforumSD.py
486 lines (406 loc) · 17.8 KB
/
DeforumSD.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
import typing
import uuid
from daras_ai_v2.pydantic_validation import FieldHttpUrl
from django.db.models import TextChoices
from pydantic import BaseModel
from typing_extensions import TypedDict
import gooey_gui as gui
from bots.models import Workflow
from daras_ai_v2.base import BasePage
from daras_ai_v2.enum_selector_widget import enum_selector
from daras_ai_v2.exceptions import UserError
from daras_ai_v2.gpu_server import call_celery_task_outfile
from daras_ai_v2.loom_video_widget import youtube_video
from daras_ai_v2.safety_checker import safety_checker
DEFAULT_DEFORUMSD_META_IMG = "https://storage.googleapis.com/dara-c1b52.appspot.com/daras_ai/media/7dc25196-93fe-11ee-9e3a-02420a0001ce/AI%20Animation%20generator.jpg.png"
class AnimationModels(TextChoices):
protogen_2_2 = ("Protogen_V2.2.ckpt", "Protogen V2.2 (darkstorm2150)")
epicdream = ("epicdream.safetensors", "epiCDream (epinikion)")
class _AnimationPrompt(TypedDict):
frame: str
prompt: str
AnimationPrompts = list[_AnimationPrompt]
CREDITS_PER_FRAME = 1.5
MODEL_ESTIMATED_TIME_PER_FRAME = 2.4 # seconds
def input_prompt_to_animation_prompts(input_prompt: str):
animation_prompts = []
for fp in input_prompt.split("|"):
split = fp.split(":")
if len(split) == 2:
frame = int(split[0])
prompt = split[1].strip()
else:
frame = 0
prompt = fp
animation_prompts.append({"frame": frame, "prompt": prompt})
return animation_prompts
def animation_prompts_to_st_list(animation_prompts: AnimationPrompts):
return [
{"frame": fp["frame"], "prompt": fp["prompt"], "key": str(uuid.uuid1())}
for fp in animation_prompts
]
def st_list_to_animation_prompt(prompt_st_list) -> AnimationPrompts:
return [
{"frame": fp["frame"], "prompt": prompt}
for fp in prompt_st_list
if (prompt := fp["prompt"].strip())
]
def animation_prompts_editor(
animation_prompts_key: str = "animation_prompts",
input_prompt_key: str = "input_prompt",
):
st_list_key = f"{animation_prompts_key}/st_list"
if st_list_key in gui.session_state:
prompt_st_list = gui.session_state[st_list_key]
else:
animation_prompts = gui.session_state.get(
animation_prompts_key
) or input_prompt_to_animation_prompts(
gui.session_state.get(input_prompt_key, "0:")
)
prompt_st_list = animation_prompts_to_st_list(animation_prompts)
gui.session_state[st_list_key] = prompt_st_list
gui.write("#### 👩💻 Animation Prompts")
gui.caption(
"""
Describe the scenes or series of images that you want to generate into an animation. You can add as many prompts as you like. Mention the keyframe number for each prompt i.e. the transition point from the first prompt to the next.
View the ‘Details’ drop down menu to get started.
"""
)
updated_st_list = []
for idx, fp in enumerate(prompt_st_list):
fp_key = fp["key"]
frame_key = f"{st_list_key}/frame/{fp_key}"
prompt_key = f"{st_list_key}/prompt/{fp_key}"
if frame_key not in gui.session_state:
gui.session_state[frame_key] = fp["frame"]
if prompt_key not in gui.session_state:
gui.session_state[prompt_key] = fp["prompt"]
col1, col2 = gui.columns([8, 3], responsive=False)
with col1:
gui.text_area(
label="*Prompt*",
key=prompt_key,
height=100,
)
with col2:
gui.number_input(
label="*Frame*",
key=frame_key,
min_value=0,
step=1,
)
if gui.button("🗑️", help=f"Remove Frame {idx + 1}"):
prompt_st_list.pop(idx)
gui.rerun()
updated_st_list.append(
{
"frame": gui.session_state.get(frame_key),
"prompt": gui.session_state.get(prompt_key),
"key": fp_key,
}
)
prompt_st_list.clear()
prompt_st_list.extend(updated_st_list)
if gui.button("➕ Add a Prompt"):
max_frames = gui.session_state.get("max_frames", 100)
if prompt_st_list:
next_frame = get_last_frame(prompt_st_list)
next_frame += max(min(max_frames - next_frame, 10), 1)
else:
next_frame = 0
if next_frame > max_frames:
gui.error("Please increase Frame Count")
else:
prompt_st_list.append(
{
"frame": next_frame,
"prompt": "",
"key": str(uuid.uuid1()),
}
)
gui.rerun()
gui.session_state[animation_prompts_key] = st_list_to_animation_prompt(
prompt_st_list
)
gui.caption(
"""
Pro-tip: To avoid abrupt endings on your animation, ensure that the last keyframe prompt is set for a higher number of keyframes/time than the previous transition rate. There should be an ample number of frames between the last frame and the total frame count of the animation.
"""
)
def get_last_frame(prompt_list: list) -> int:
return max(fp["frame"] for fp in prompt_list)
DEFAULT_ANIMATION_META_IMG = "https://storage.googleapis.com/dara-c1b52.appspot.com/daras_ai/media/assets/cropped_animation_meta.gif"
class DeforumSDPage(BasePage):
title = "AI Animation Generator"
explore_image = "https://storage.googleapis.com/dara-c1b52.appspot.com/media/users/kxmNIYAOJbfOURxHBKNCWeUSKiP2/dd88c110-88d6-11ee-9b4f-2b58bd50e819/animation.gif"
workflow = Workflow.DEFORUM_SD
slug_versions = ["DeforumSD", "animation-generator"]
sane_defaults = dict(
zoom="0: (1.004)",
animation_mode="2D",
translation_x="0:(10*sin(2*3.14*t/10))",
translation_y="0:(0)",
rotation_3d_x="0:(0)",
rotation_3d_y="0:(0)",
rotation_3d_z="0:(0)",
fps=12,
seed=42,
selected_model=AnimationModels.protogen_2_2.name,
)
class RequestModel(BasePage.RequestModel):
# input_prompt: str
animation_prompts: AnimationPrompts
max_frames: int | None
selected_model: typing.Literal[tuple(e.name for e in AnimationModels)] | None
animation_mode: str | None
zoom: str | None
translation_x: str | None
translation_y: str | None
rotation_3d_x: str | None
rotation_3d_y: str | None
rotation_3d_z: str | None
fps: int | None
seed: int | None
class ResponseModel(BaseModel):
output_video: FieldHttpUrl
def preview_image(self, state: dict) -> str | None:
return DEFAULT_DEFORUMSD_META_IMG
def related_workflows(self) -> list:
from recipes.VideoBots import VideoBotsPage
from recipes.LipsyncTTS import LipsyncTTSPage
from recipes.CompareText2Img import CompareText2ImgPage
from recipes.QRCodeGenerator import QRCodeGeneratorPage
return [
QRCodeGeneratorPage,
VideoBotsPage,
LipsyncTTSPage,
CompareText2ImgPage,
]
def render_form_v2(self):
animation_prompts_editor()
col1, col2 = gui.columns(2)
with col1:
gui.slider(
"""
#### Frame Count
Choose the number of frames in your animation.
""",
min_value=10,
max_value=500,
step=10,
key="max_frames",
)
gui.caption(
"""
Pro-tip: The more frames you add, the longer it will take to render the animation. Test your prompts before adding more frames.
"""
)
def get_cost_note(self) -> str | None:
return f"{CREDITS_PER_FRAME} / frame"
def additional_notes(self) -> str | None:
return "Render Time ≈ 3s / frame"
def get_raw_price(self, state: dict) -> float:
max_frames = state.get("max_frames", 100) or 0
return max_frames * CREDITS_PER_FRAME
def validate_form_v2(self):
prompt_list = gui.session_state.get("animation_prompts")
assert prompt_list, "Please provide animation prompts"
max_frames = gui.session_state["max_frames"]
assert (
get_last_frame(prompt_list) <= max_frames
), "Please make sure that Frame Count matches the Animation Prompts"
def render_usage_guide(self):
youtube_video("sUvica6UuQU")
def render_settings(self):
col1, col2 = gui.columns(2)
with col1:
enum_selector(
AnimationModels,
label="""
Choose your preferred AI Animation Model
""",
key="selected_model",
use_selectbox=True,
)
animation_mode = gui.selectbox(
"Animation Mode", key="animation_mode", options=["2D", "3D"]
)
gui.text_input(
"""
###### Zoom
How should the camera zoom in or out? This setting scales the canvas size, multiplicatively.
1 is static, with numbers greater than 1 moving forward (or zooming in) and numbers less than 1 moving backwards (or zooming out).
""",
key="zoom",
)
gui.caption(
"""
With 0 as the starting keyframe, the input of 0: (1.004) can be used to zoom in moderately, starting at frame 0 and continuing until the end.
"""
)
gui.text_input(
"""
###### Horizontal Pan
How should the camera pan horizontally? This parameter uses positive values to move right and negative values to move left.
""",
key="translation_x",
)
gui.text_input(
"""
###### Vertical Pan
How should the camera pan vertically? This parameter uses positive values to move up and negative values to move down.
""",
key="translation_y",
)
if animation_mode == "3D":
gui.text_input(
"""
###### Roll Clockwise/Counterclockwise
Gradually moves the camera on a focal axis. Roll the camera clockwise or counterclockwise in a specific degree per frame. This parameter uses positive values to roll counterclockwise and negative values to roll clockwise. E.g. use `0:(-1), 20:(0)` to roll the camera 1 degree clockwise for the first 20 frames.
""",
key="rotation_3d_z",
)
gui.text_input(
"""
###### Pan Left/Right
Pans the canvas left or right in degrees per frame. This parameter uses positive values to pan right and negative values to pan left.
""",
key="rotation_3d_y",
)
gui.text_input(
"""
###### Tilt Up/Down
Tilts the camera up or down in degrees per frame. This parameter uses positive values to tilt up and negative values to tilt down.
""",
key="rotation_3d_x",
)
gui.slider(
"""
###### FPS (Frames per second)
Choose fps for the video.
""",
min_value=10,
max_value=60,
step=1,
key="fps",
)
# gui.selectbox(
# """
# ###### Sampler
# What Stable Diffusion sampler should be used.
# """,
# key="sampler",
# options=[
# "euler_ancestral",
# "klms",
# "dpm2",
# "dpm2_ancestral",
# "heun",
# "euler",
# "plms",
# "ddim",
# "dpm_fast",
# "dpm_adaptive",
# "dpmpp_2s_a",
# "dpmpp_2m",
# ],
# )
def fallback_preivew_image(self) -> str:
return DEFAULT_ANIMATION_META_IMG
def preview_description(self, state: dict) -> str:
return "Create AI-generated Animation without relying on complex CoLab notebooks. Input your prompts + keyframes and bring your ideas to life using the animation capabilities of Gooey & Stable Diffusion's Deforum. For more help on how to use the tool visit https://www.help.gooey.ai/learn-animation"
def render_description(self):
gui.markdown(
f"""
- Every Submit will require approximately 3-5 minutes to render.
- Animation is complex: Please watch the video and review our decks to help you.
- Test your image prompts BEFORE adding lots of frames e.g. Tweak key frame images with just 10 frames between them AND then increase the FPS or frame count between them once you like the outputs. This will save you time and credits.
- No lost work! All your animations or previously generated versions are in the History tab. If they don't appear here, it likely means they aren't done rendering yet.
"""
)
gui.markdown(
"""
#### Resources:
[Learn Animation](https://www.help.gooey.ai/learn-animation)
[Gooey Guide to Prompting](https://docs.google.com/presentation/d/1RaoMP0l7FnBZovDAR42zVmrUND9W5DW6eWet-pi6kiE/edit?usp=sharing)
Here’s a comprehensive style guide to assist you with different stylized animation prompts:
[StableDiffusion CheatSheet](https://supagruen.github.io/StableDiffusion-CheatSheet/)
"""
)
gui.write("---")
gui.markdown(
"""
Animation Length: You can indicate how long you want your animation to be by increasing or decreasing your frame count.
FPS: Every Animation is set at 12 frames per second by default. You can change this default frame rate/ frames per second (FPS) on the Settings menu.
Prompts: Within your sequence you can input multiple text Prompts for your visuals. Each prompt can be defined for a specific keyframe number.
##### What are keyframes?
Keyframes define the transition points from one prompt to the next, or the start and end points of a prompted action set in between the total frame count or sequence. These keyframes or markers are necessary to establish smooth transitions or jump cuts, whatever you prefer.
Use the Camera Settings to generate animations with depth and other 3D parameters.
"""
)
gui.markdown(
"""
Prompt Construction Tip:
Changing Elements transition better from a visual prompt that is artefact or object heavy to another busy visual prompt. For example: Prompt 1: a busy street transitions to Prompt 2: a busy interior of a park. This transition will render interesting and beautiful imagery.
`Transitions from a simpler or plain visual prompt to a more complex visual might be challenging to generate. For example: Prompt 1: a blue sky to Prompt 2: a crowded market. This is because there are fewer artefacts for the generator to transition.`
This recipe takes any text and creates animation. It's based on the Deforum notebook with lots of details at http://deforum.art.
"""
)
def render_output(self):
output_video = gui.session_state.get("output_video")
if output_video:
gui.write("#### Output Video")
gui.video(output_video, autoplay=True, show_download_button=True)
def estimate_run_duration(self):
# in seconds
return gui.session_state.get("max_frames", 100) * MODEL_ESTIMATED_TIME_PER_FRAME
def render_example(self, state: dict):
display = self.preview_input(state)
gui.markdown("```lua\n" + display + "\n```")
gui.video(state.get("output_video"), autoplay=True)
@classmethod
def preview_input(cls, state: dict) -> str:
input_prompt = state.get("input_prompt")
if input_prompt:
animation_prompts = input_prompt_to_animation_prompts(input_prompt)
else:
animation_prompts = state.get("animation_prompts", [])
display = "\n\n".join(
[f"{fp['prompt']} [{fp['frame']}]" for fp in animation_prompts]
)
return display
def run(self, state: dict):
request: DeforumSDPage.RequestModel = self.RequestModel.parse_obj(state)
yield
if not self.request.user.disable_safety_checker:
safety_checker(text=self.preview_input(state))
try:
state["output_video"] = call_celery_task_outfile(
"deforum",
pipeline=dict(
model_id=AnimationModels[request.selected_model].value,
seed=request.seed,
),
inputs=dict(
animation_mode=request.animation_mode,
animation_prompts={
fp["frame"]: fp["prompt"] for fp in request.animation_prompts
},
max_frames=request.max_frames,
zoom=request.zoom or "0: (1.004)",
translation_x=request.translation_x or "0:(10*sin(2*3.14*t/10))",
translation_y=request.translation_y or "0:(0)",
rotation_3d_x=request.rotation_3d_x or "0:(0)",
rotation_3d_y=request.rotation_3d_y or "0:(0)",
rotation_3d_z=request.rotation_3d_z or "0:(0)",
translation_z="0:(0)",
fps=request.fps,
),
content_type="video/mp4",
filename=f"gooey.ai animation {request.animation_prompts}.mp4",
)[0]
except RuntimeError as e:
msg = "\n\n".join(e.args).lower()
if "key frame string not correctly formatted" in msg:
raise UserError(str(e)) from e