Picsart-AI-Research · vigneshrajmohan · Apr 23, 2023 · Apr 23, 2023 · Apr 23, 2023 · Apr 23, 2023
diff --git a/1deer_as_bunny.mp4 b/1deer_as_bunny.mp4
diff --git a/__assets__/canny_videos_mp4/big_deer.mp4 b/__assets__/canny_videos_mp4/big_deer.mp4
diff --git a/__assets__/canny_videos_mp4/interski.mp4 b/__assets__/canny_videos_mp4/interski.mp4
diff --git a/__assets__/canny_videos_mp4/kayak_draw.mp4 b/__assets__/canny_videos_mp4/kayak_draw.mp4
diff --git a/__assets__/canny_videos_mp4/lion.png b/__assets__/canny_videos_mp4/lion.png
diff --git a/__assets__/canny_videos_mp4/myvideo.mp4 b/__assets__/canny_videos_mp4/myvideo.mp4
diff --git a/__assets__/canny_videos_mp4/myvideo_new.mp4 b/__assets__/canny_videos_mp4/myvideo_new.mp4
diff --git a/__assets__/canny_videos_mp4/myvideo_new_4X_1fps (1).mp4 b/__assets__/canny_videos_mp4/myvideo_new_4X_1fps (1).mp4
diff --git a/__assets__/canny_videos_mp4/skib_sun_none.mp4 b/__assets__/canny_videos_mp4/skib_sun_none.mp4
diff --git a/__assets__/canny_videos_mp4/vig_deer_inter.mp4 b/__assets__/canny_videos_mp4/vig_deer_inter.mp4
diff --git a/__assets__/canny_videos_mp4/vig_deer_skribble_no.mp4 b/__assets__/canny_videos_mp4/vig_deer_skribble_no.mp4
diff --git a/__assets__/frames/deer_pic.jpeg b/__assets__/frames/deer_pic.jpeg
diff --git a/__assets__/frames/deer_pic2.jpeg b/__assets__/frames/deer_pic2.jpeg
diff --git a/__assets__/frames/deer_pic3.jpeg b/__assets__/frames/deer_pic3.jpeg
diff --git a/__assets__/frames/deer_pic4.jpeg b/__assets__/frames/deer_pic4.jpeg
diff --git a/__assets__/frames/kayak_a.png b/__assets__/frames/kayak_a.png
diff --git a/__assets__/frames/kayak_b.png b/__assets__/frames/kayak_b.png
diff --git a/__assets__/frames/kayak_c.png b/__assets__/frames/kayak_c.png
diff --git a/__assets__/frames/old/a.jpg b/__assets__/frames/old/a.jpg
diff --git a/__assets__/frames/old/aa.jpg b/__assets__/frames/old/aa.jpg
diff --git a/__assets__/frames/old/aaa.jpg b/__assets__/frames/old/aaa.jpg
diff --git a/__assets__/frames/old/b.jpg b/__assets__/frames/old/b.jpg
diff --git a/__assets__/frames/old/bb.jpg b/__assets__/frames/old/bb.jpg
diff --git a/__assets__/frames/old/bba.jpg b/__assets__/frames/old/bba.jpg
diff --git a/__assets__/frames/old/c.jpg b/__assets__/frames/old/c.jpg
diff --git a/__assets__/frames/old/cc.jpg b/__assets__/frames/old/cc.jpg
diff --git a/__assets__/frames/old/cca.jpg b/__assets__/frames/old/cca.jpg
diff --git a/__assets__/frames/old/d.jpg b/__assets__/frames/old/d.jpg
diff --git a/__assets__/frames/old/dd.jpg b/__assets__/frames/old/dd.jpg
diff --git a/__assets__/frames/old/dda.jpg b/__assets__/frames/old/dda.jpg
diff --git a/__assets__/frames/old/deer1.png b/__assets__/frames/old/deer1.png
diff --git a/__assets__/frames/old/deer2.png b/__assets__/frames/old/deer2.png
diff --git a/__assets__/frames/old/deer3.png b/__assets__/frames/old/deer3.png
diff --git a/__assets__/frames/old/sun_a.jpg b/__assets__/frames/old/sun_a.jpg
diff --git a/__assets__/frames/old/sun_b.jpg b/__assets__/frames/old/sun_b.jpg
diff --git a/__assets__/frames/old/sun_c.jpg b/__assets__/frames/old/sun_c.jpg
diff --git a/__assets__/frames/old/sun_dd.jpg b/__assets__/frames/old/sun_dd.jpg
diff --git a/__assets__/frames/old/sun_e.jpg b/__assets__/frames/old/sun_e.jpg
diff --git a/__init__.py b/__init__.py
diff --git a/banana.mp4 b/banana.mp4
diff --git a/banana_new.mp4 b/banana_new.mp4
diff --git a/big_deer_16X_20fps.mp4 b/big_deer_16X_20fps.mp4
diff --git a/big_deer_256X_30fps.mp4 b/big_deer_256X_30fps.mp4
diff --git a/big_deer_4X_1fps.mp4 b/big_deer_4X_1fps.mp4
diff --git a/big_deer_no.mp4 b/big_deer_no.mp4
diff --git a/big_deer_pre.mp4 b/big_deer_pre.mp4
diff --git a/deer_as_bunny.mp4 b/deer_as_bunny.mp4
diff --git a/deer_as_ccentaur.mp4 b/deer_as_ccentaur.mp4
diff --git a/deer_as_lion.mp4 b/deer_as_lion.mp4
diff --git a/deer_as_rocketship.mp4 b/deer_as_rocketship.mp4
diff --git a/deer_as_workingman.mp4 b/deer_as_workingman.mp4
diff --git a/deer_pic.mp4 b/deer_pic.mp4
diff --git a/final_oil painting of a deer, a high-quality, detailed, and professional photo.mp4 b/final_oil painting of a deer, a high-quality, detailed, and professional photo.mp4
diff --git a/highfps_deer.mp4 b/highfps_deer.mp4
diff --git a/highfpsplus_deer.mp4 b/highfpsplus_deer.mp4
diff --git a/kayak.mp4 b/kayak.mp4
diff --git a/kayak_draw_2X_1fps.mp4 b/kayak_draw_2X_1fps.mp4
diff --git a/kayak_draw_4X_1fps.mp4 b/kayak_draw_4X_1fps.mp4
diff --git a/model.py b/model.py
@@ -4,6 +4,9 @@
 import tomesd
 import torch
 
+from einops import rearrange
+
+
 from diffusers import StableDiffusionInstructPix2PixPipeline, StableDiffusionControlNetPipeline, ControlNetModel, UNet2DConditionModel
 from diffusers.schedulers import EulerAncestralDiscreteScheduler, DDIMScheduler
 from text_to_video_pipeline import TextToVideoPipeline
@@ -69,13 +72,19 @@ def inference_chunk(self, frame_ids, **kwargs):
         negative_prompt = np.array(kwargs.pop('negative_prompt', ''))
         latents = None
         if 'latents' in kwargs:
+            print("FOUND latents!!!!")
             latents = kwargs.pop('latents')[frame_ids]
         if 'image' in kwargs:
             kwargs['image'] = kwargs['image'][frame_ids]
         if 'video_length' in kwargs:
             kwargs['video_length'] = len(frame_ids)
         if self.model_type == ModelType.Text2Video:
             kwargs["frame_ids"] = frame_ids
+
+        print("frameids:!!!")
+        print(frame_ids)
+        print(prompt)
+        print(prompt[frame_ids])
         return self.pipe(prompt=prompt[frame_ids].tolist(),
                          negative_prompt=negative_prompt[frame_ids].tolist(),
                          latents=latents,
@@ -119,12 +128,15 @@ def inference(self, split_to_chunks=False, chunk_size=8, **kwargs):
                 print(f'Processing chunk {i + 1} / {len(chunk_ids)}')
                 result.append(self.inference_chunk(frame_ids=frame_ids,
                                                    prompt=prompt,
-                                                   negative_prompt=negative_prompt,
+                                                 negative_prompt=negative_prompt,
                                                    **kwargs).images[1:])
+                print(result[0].shape)
                 frames_counter += len(chunk_ids)-1
                 if on_huggingspace and frames_counter >= 80:
                     break
             result = np.concatenate(result)
+            print("shape in inference")
+            print(result.shape)
             return result
         else:
             self.generator.manual_seed(seed)
@@ -133,7 +145,7 @@ def inference(self, split_to_chunks=False, chunk_size=8, **kwargs):
     def process_controlnet_canny(self,
                                  video_path,
                                  prompt,
-                                 chunk_size=8,
+                                 chunk_size=2,
                                  watermark='Picsart AI Research',
                                  merging_ratio=0.0,
                                  num_inference_steps=20,
@@ -150,7 +162,7 @@ def process_controlnet_canny(self,
         video_path = gradio_utils.edge_path_to_video_path(video_path)
         if self.model_type != ModelType.ControlNetCanny:
             controlnet = ControlNetModel.from_pretrained(
-                "lllyasviel/sd-controlnet-canny")
+                "lllyasviel/sd-controlnet-scribble")
             self.set_model(ModelType.ControlNetCanny,
                            model_id="runwayml/stable-diffusion-v1-5", controlnet=controlnet)
             self.pipe.scheduler = DDIMScheduler.from_config(
@@ -169,8 +181,8 @@ def process_controlnet_canny(self,
         control = utils.pre_process_canny(
             video, low_threshold, high_threshold).to(self.device).to(self.dtype)
 
-        # canny_to_save = list(rearrange(control, 'f c w h -> f w h c').cpu().detach().numpy())
-        # _ = utils.create_video(canny_to_save, 4, path="ddxk.mp4", watermark=None)
+        canny_to_save = list(rearrange(control, 'f c w h -> f w h c').cpu().detach().numpy())
+        _ = utils.create_video(canny_to_save, 4, path="deer_pic.mp4", watermark=None)
 
         f, _, h, w = video.shape
         self.generator.manual_seed(seed)
@@ -193,7 +205,7 @@ def process_controlnet_canny(self,
                                 chunk_size=chunk_size,
                                 merging_ratio=merging_ratio,
                                 )
-        return utils.create_video(result, fps, path=save_path, watermark=gradio_utils.logo_name_to_path(watermark))
+        return utils.create_video(result, fps, path=save_path, watermark=None)
 
     def process_controlnet_depth(self,
                                  video_path,
@@ -490,6 +502,159 @@ def process_text2video(self,
                                 negative_prompt=negative_prompt,
                                 merging_ratio=merging_ratio,
                                 split_to_chunks=True,
+                                chunk_size=2,
+                                )
+        return utils.create_video(result, fps, path=path, watermark=None)
+
+    def process_text2video_with_draw(self,
+                           video_path,
+                           prompt,
+                           model_name="dreamlike-art/dreamlike-photoreal-2.0",
+                           motion_field_strength_x=12,
+                           motion_field_strength_y=12,
+                           t0=44,
+                           t1=47,
+                           n_prompt="",
+                           chunk_size=8,
+                           video_length=8,
+                           watermark='Picsart AI Research',
+                           merging_ratio=0.0,
+                           seed=0,
+                           resolution=512,
+                           fps=2,
+                           use_cf_attn=True,
+                           use_motion_field=True,
+                           smooth_bg=False,
+                           smooth_bg_strength=0.4,
+                           path=None):
+        print("Module Text2Video")
+        if self.model_type != ModelType.Text2Video or model_name != self.model_name:
+            print("Model update")
+            unet = UNet2DConditionModel.from_pretrained(
+                model_name, subfolder="unet")
+            controlnet = ControlNetModel.from_pretrained(
+                "lllyasviel/sd-controlnet-canny")
+            controlnet.cuda()
+
+            video, fps = utils.prepare_image(
+            video_path, resolution, self.device, self.dtype, False)
+            control = utils.pre_process_canny(
+            video, 100, 200).to(self.device).to(self.dtype)
+
+            canny_to_save = list(rearrange(control, 'f c w h -> f w h c').cpu().detach().numpy())
+            _ = utils.create_video(canny_to_save, 4, path="deer_pic.mp4", watermark=None)
+
+            self.set_model(ModelType.Text2Video,
+                           model_id=model_name, unet=unet, controlnet=controlnet, condition=control)
+            self.pipe.scheduler = DDIMScheduler.from_config(
+                self.pipe.scheduler.config)
+            if use_cf_attn and False:
+                self.pipe.controlnet.set_attn_processor(
+                    processor=self.text2video_attn_proc)
+                self.pipe.unet.set_attn_processor(
+                    processor=self.text2video_attn_proc)
+        self.generator.manual_seed(seed)
+
+        added_prompt = "high quality, HD, 8K, trending on artstation, high focus, dramatic lighting"
+        negative_prompts = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, fewer difits, cropped, worst quality, low quality, deformed body, bloated, ugly, unrealistic'
+
+        prompt = prompt.rstrip()
+        if len(prompt) > 0 and (prompt[-1] == "," or prompt[-1] == "."):
+            prompt = prompt.rstrip()[:-1]
+        prompt = prompt.rstrip()
+        prompt = prompt + ", "+added_prompt
+        if len(n_prompt) > 0:
+            negative_prompt = n_prompt
+        else:
+            negative_prompt = None
+
+        result = self.inference(prompt=prompt,
+                                video_length=video_length,
+                                height=resolution,
+                                width=resolution,
+                                num_inference_steps=50,
+                                guidance_scale=7.5,
+                                guidance_stop_step=1.0,
+                                t0=t0,
+                                t1=t1,
+                                motion_field_strength_x=motion_field_strength_x,
+                                motion_field_strength_y=motion_field_strength_y,
+                                use_motion_field=use_motion_field,
+                                smooth_bg=smooth_bg,
+                                smooth_bg_strength=smooth_bg_strength,
+                                seed=seed,
+                                output_type='numpy',
+                                negative_prompt=negative_prompt,
+                                merging_ratio=merging_ratio,
+                                split_to_chunks=True,
+                                chunk_size=3,
+                                control=control
+                                )
+        fps = 1
+        print(result.shape)
+        return utils.create_video(result, fps, path=path, watermark=None)
+
+    def process_controlnet_draw_frames(self,
+                                 video_path,
+                                 prompt,
+                                 chunk_size=8,
+                                 watermark='Picsart AI Research',
+                                 merging_ratio=0.0,
+                                 num_inference_steps=20,
+                                 controlnet_conditioning_scale=1.0,
+                                 guidance_scale=9.0,
+                                 seed=42,
+                                 eta=0.0,
+                                 low_threshold=100,
+                                 high_threshold=200,
+                                 resolution=512,
+                                 use_cf_attn=True,
+                                 save_path=None):
+        print("Module Canny")
+        #video_path = gradio_utils.edge_path_to_video_path(video_path)
+        if self.model_type != ModelType.ControlNetCanny:
+            controlnet = ControlNetModel.from_pretrained(
+                "lllyasviel/sd-controlnet-canny")
+            self.set_model(ModelType.ControlNetCanny,
+                           model_id="runwayml/stable-diffusion-v1-5", controlnet=controlnet)
+            self.pipe.scheduler = DDIMScheduler.from_config(
+                self.pipe.scheduler.config)
+            if use_cf_attn:
+                self.pipe.unet.set_attn_processor(
+                    processor=self.controlnet_attn_proc)
+                self.pipe.controlnet.set_attn_processor(
+                    processor=self.controlnet_attn_proc)
+
+        added_prompt = 'best quality, extremely detailed'
+        negative_prompts = 'longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality'
+
+        video, fps = utils.prepare_images(
+            video_path, resolution, self.device, self.dtype, False)
+        control = utils.pre_process_canny(
+            video, low_threshold, high_threshold).to(self.device).to(self.dtype)
+
+        canny_to_save = list(rearrange(control, 'f c w h -> f w h c').cpu().detach().numpy())
+        _ = utils.create_video(canny_to_save, 4, path="deer_pic.mp4", watermark=None)
+
+        f, _, h, w = video.shape
+        self.generator.manual_seed(seed)
+        latents = torch.randn((1, 4, h//8, w//8), dtype=self.dtype,
+                              device=self.device, generator=self.generator)
+        latents = latents.repeat(f*2, 1, 1, 1)
+        result = self.inference(image=control,
+                                prompt=prompt + ', ' + added_prompt,
+                                height=h,
+                                width=w,
+                                negative_prompt=negative_prompts,
+                                num_inference_steps=num_inference_steps,
+                                guidance_scale=guidance_scale,
+                                controlnet_conditioning_scale=controlnet_conditioning_scale,
+                                eta=eta,
+                                latents=latents,
+                                seed=seed,
+                                output_type='numpy',
+                                split_to_chunks=True,
                                 chunk_size=chunk_size,
+                                merging_ratio=merging_ratio,
                                 )
-        return utils.create_video(result, fps, path=path, watermark=gradio_utils.logo_name_to_path(watermark))
+        return utils.create_video(result, fps, path=save_path, watermark=gradio_utils.logo_name_to_path(watermark))
diff --git a/new_final_oil painting of a deer, a high-quality, detailed, and professional photo.mp4 b/new_final_oil painting of a deer, a high-quality, detailed, and professional photo.mp4
diff --git a/output_vid/test.txt b/output_vid/test.txt
diff --git a/requirements.txt b/requirements.txt
@@ -11,7 +11,7 @@ imageio==2.9.0
 imageio-ffmpeg==0.4.2
 invisible-watermark>=0.1.5
 moviepy==1.0.3
-numpy==1.24.1
+numpy<1.24
 omegaconf==2.3.0
 open_clip_torch==2.16.0
 opencv_python==4.7.0.72

diff --git a/run_tests.py b/run_tests.py
@@ -0,0 +1,64 @@
+import torch
+from model import Model
+import cv2
+import os
+from moviepy.editor import ImageSequenceClip
+import argparse
+
+arg_parser = argparse.ArgumentParser(description="Video-to-video with custom prompt")
+arg_parser.add_argument('--prompt', type=str, required=True, help="The prompt for the video generation")
+args = arg_parser.parse_args()
+
+model = Model(device="cuda", dtype=torch.float16)
+
+
+def images_to_video(input_dir, output_path, fps=30):
+    # Get all image file names in the input directory
+    image_files = [os.path.join(input_dir, f) for f in os.listdir(input_dir) if f.endswith('.jpg') or f.endswith('.png') or f.endswith('jpeg')]
+
+    # Sort the file names alphabetically
+    image_files.sort()
+
+    # Determine the width and height of the images
+    img = cv2.imread(image_files[0])
+    height, width, channels = img.shape
+
+    # Define the video codec and create a VideoWriter object
+    fourcc = cv2.VideoWriter_fourcc(*'mp4v')
+    video_writer = cv2.VideoWriter(output_path, fourcc, fps, (width, height))
+
+    # Iterate over all images and add them to the video
+    for image_file in image_files:
+        img = cv2.imread(image_file)
+        video_writer.write(img)
+
+    # Release the VideoWriter object
+    video_writer.release()
+
+def images_to_video_new(input_dir, output_path, fps=30):
+    # Get all image file names in the input directory
+    image_files = [f for f in os.listdir(input_dir) if f.endswith('.jpg') or f.endswith('.png')]
+
+    # Sort the file names alphabetically
+    image_files.sort()
+
+    # Create an ImageSequenceClip from the images
+    clip = ImageSequenceClip([os.path.join(input_dir, f) for f in image_files], fps=fps)
+
+    # Write the clip to a video file
+    clip.write_videofile(output_path, fps=fps, codec='libx264')
+
+def video2video(input_path, output_path, prompt):
+    model = Model(device = "cuda", dtype = torch.float16)
+    model.process_controlnet_canny(input_path, prompt=prompt, save_path=output_path)
+
+#video_path = 'vid2vid/__assets__/canny_videos_mp4/deer_pic.jpeg'
+#prompt = "Deer walking in the street"
+params = {"t0": 44, "t1": 47 , "motion_field_strength_x" : 12, "motion_field_strength_y" : 12, "video_length": 2}
+
+prompt = args.prompt
+images_to_video_new('../rife-interop/vid_out', 'output_vid/o.mp4', 1)
+video_path = 'output_vid/o.mp4'
+out_path = 'output_vid/output.mp4'
+model.process_controlnet_canny(video_path, prompt=prompt, save_path=out_path)
+
diff --git a/skib_sun_none_4X_1fps.mp4 b/skib_sun_none_4X_1fps.mp4
diff --git a/suns_none.mp4 b/suns_none.mp4
diff --git a/suns_none_not_a_deer.mp4 b/suns_none_not_a_deer.mp4
diff --git a/suns_pre_int.mp4 b/suns_pre_int.mp4
diff --git a/text2video_edge_guidance_Deer in the streets.mp4 b/text2video_edge_guidance_Deer in the streets.mp4
diff --git a/...dge_guidance_oil painting of a deer, a high-quality, detailed, and professional photo.mp4 b/...dge_guidance_oil painting of a deer, a high-quality, detailed, and professional photo.mp4