Preprocessing videos: extracting RGB images + flow data #2

FrederikSchorr · 2018-06-09T19:56:50Z

Thank you @Rhythmblue for great work!
I guess you preprocess the UCF videos upfront to extract RGB images + flow data. Can you share these as well? I am particularly interested in which tools you use to extract the flow data.
Thank you very much,
Frederik

Rhythmblue · 2018-06-10T05:42:58Z

Hi, Frederik Schorr:

A. I have use two ways to extract flow

this is a repo which can extract flow and warped flow(which is modified by RANSAC to remove motion of background)
And I use the branch of opencv3.1.
https://github.com/yjxiong/dense_flow/tree/opencv-3.1
(CPU or GPU)
api of python-opencv
you need to install by:

pip install opencv-python
pip install opencv-contrib-python

Then, you can use the TVL1 api to extract optical flow (Note: opencv-python cannot use GPU to compute flow. You can use multi-processing)
I add a py file in this mail.

B. There are some ways to extract rgb.
Sometimes I use ffmpeg. Sometimes use VideoCapture api of opencv-python.

Finally, we have create a new repo about i3d. You can try it.
https://github.com/USTC-Video-Understanding/I3D_Finetune

Rhythmblue · 2018-06-10T05:53:02Z

import os
import numpy as np
import cv2
from glob import glob
from multiprocessing import Pool


_IMAGE_SIZE = 256


def cal_for_frames(video_path):
    frames = glob(os.path.join(video_path, '*.jpg'))
    frames.sort()

    flow = []
    prev = cv2.imread(frames[0])
    prev = cv2.cvtColor(prev, cv2.COLOR_BGR2GRAY)
    for i, frame_curr in enumerate(frames):
        curr = cv2.imread(frame_curr)
        curr = cv2.cvtColor(curr, cv2.COLOR_BGR2GRAY)
        tmp_flow = compute_TVL1(prev, curr)
        flow.append(tmp_flow)
        prev = curr

    return flow


def compute_TVL1(prev, curr, bound=15):
    """Compute the TV-L1 optical flow."""
    TVL1 = cv2.DualTVL1OpticalFlow_create()
    flow = TVL1.calc(prev, curr, None)
    assert flow.dtype == np.float32

    flow = (flow + bound) * (255.0 / (2*bound))
    flow = np.round(flow).astype(int)
    flow[flow >= 255] = 255
    flow[flow <= 0] = 0

    return flow


def save_flow(video_flows, flow_path):
    for i, flow in enumerate(video_flows):
        cv2.imwrite(os.path.join(flow_path.format('u'), "{:06d}.jpg".format(i)),
                    flow[:, :, 0])
        cv2.imwrite(os.path.join(flow_path.format('v'), "{:06d}.jpg".format(i)),
                    flow[:, :, 1])


def gen_video_path():
    path = []
    flow_path = []
    length = []
    base = ''
    flow_base = ''
    for task in ['train', 'dev', 'test']:
        videos = os.listdir(os.path.join(base, task))
        for video in videos:
            tmp_path = os.path.join(base, task, video, '1')
            tmp_flow = os.path.join(flow_base, task, '{:s}', video)
            tmp_len = len(glob(os.path.join(tmp_path, '*.png')))
            u = False
            v = False
            if os.path.exists(tmp_flow.format('u')):
                if len(glob(os.path.join(tmp_flow.format('u'), '*.jpg'))) == tmp_len:
                    u = True
            else:
                os.makedirs(tmp_flow.format('u'))
            if os.path.exists(tmp_flow.format('v')):
                if len(glob(os.path.join(tmp_flow.format('v'), '*.jpg'))) == tmp_len:
                    v = True
            else:
                os.makedirs(tmp_flow.format('v'))
            if u and v:
                print('skip:' + tmp_flow)
                continue

            path.append(tmp_path)
            flow_path.append(tmp_flow)
            length.append(tmp_len)
    return path, flow_path, length


def extract_flow(args):
    video_path, flow_path = args
    flow = cal_for_frames(video_path)
    save_flow(flow, flow_path)
    print('complete:' + flow_path)
    return


if __name__ =='__main__':
    pool = Pool(2)   # multi-processing

    video_paths, flow_paths, video_lengths = gen_video_path()

    pool.map(extract_flow, zip(video_paths, flow_paths))

FrederikSchorr · 2018-06-10T13:29:41Z

Thank you Rythmblue - this is exactly what I was looking for! I will re-use in a project on sign language recognition (for deaf-mute people). Thx! kind regards, Frederik

chanajianyu · 2019-01-17T05:16:36Z

import os
import numpy as np
import cv2
from glob import glob
from multiprocessing import Pool


_IMAGE_SIZE = 256


def cal_for_frames(video_path):
    frames = glob(os.path.join(video_path, '*.jpg'))
    frames.sort()

    flow = []
    prev = cv2.imread(frames[0])
    prev = cv2.cvtColor(prev, cv2.COLOR_BGR2GRAY)
    for i, frame_curr in enumerate(frames):
        curr = cv2.imread(frame_curr)
        curr = cv2.cvtColor(curr, cv2.COLOR_BGR2GRAY)
        tmp_flow = compute_TVL1(prev, curr)
        flow.append(tmp_flow)
        prev = curr

    return flow


def compute_TVL1(prev, curr, bound=15):
    """Compute the TV-L1 optical flow."""
    TVL1 = cv2.DualTVL1OpticalFlow_create()
    flow = TVL1.calc(prev, curr, None)
    assert flow.dtype == np.float32

    flow = (flow + bound) * (255.0 / (2*bound))
    flow = np.round(flow).astype(int)
    flow[flow >= 255] = 255
    flow[flow <= 0] = 0

    return flow


def save_flow(video_flows, flow_path):
    for i, flow in enumerate(video_flows):
        cv2.imwrite(os.path.join(flow_path.format('u'), "{:06d}.jpg".format(i)),
                    flow[:, :, 0])
        cv2.imwrite(os.path.join(flow_path.format('v'), "{:06d}.jpg".format(i)),
                    flow[:, :, 1])


def gen_video_path():
    path = []
    flow_path = []
    length = []
    base = ''
    flow_base = ''
    for task in ['train', 'dev', 'test']:
        videos = os.listdir(os.path.join(base, task))
        for video in videos:
            tmp_path = os.path.join(base, task, video, '1')
            tmp_flow = os.path.join(flow_base, task, '{:s}', video)
            tmp_len = len(glob(os.path.join(tmp_path, '*.png')))
            u = False
            v = False
            if os.path.exists(tmp_flow.format('u')):
                if len(glob(os.path.join(tmp_flow.format('u'), '*.jpg'))) == tmp_len:
                    u = True
            else:
                os.makedirs(tmp_flow.format('u'))
            if os.path.exists(tmp_flow.format('v')):
                if len(glob(os.path.join(tmp_flow.format('v'), '*.jpg'))) == tmp_len:
                    v = True
            else:
                os.makedirs(tmp_flow.format('v'))
            if u and v:
                print('skip:' + tmp_flow)
                continue

            path.append(tmp_path)
            flow_path.append(tmp_flow)
            length.append(tmp_len)
    return path, flow_path, length


def extract_flow(args):
    video_path, flow_path = args
    flow = cal_for_frames(video_path)
    save_flow(flow, flow_path)
    print('complete:' + flow_path)
    return


if __name__ =='__main__':
    pool = Pool(2)   # multi-processing

    video_paths, flow_paths, video_lengths = gen_video_path()

    pool.map(extract_flow, zip(video_paths, flow_paths))

but , TVL1 = cv2.DualTVL1OpticalFlow_create() ,do not work in the lastest opencv-python==0.4.0

JwDong2019 · 2019-02-13T03:35:48Z

lastest
Do you know how to solve the problem?
"AttributeError: module 'cv2.cv2' has no attribute 'DualTVL1OpticalFlow_create"
Looking forward to your answer! Thank you

Rhythmblue · 2019-02-22T12:22:55Z

@jianweidong

pip uninstall opencv-python
pip install opencv-contrib-python
reason in section 2.a
modify TVL1 = cv2.DualTVL1OpticalFlow_create()
to TVL1 = cv2.optflow.DualTVL1OpticalFlow_create()
This way suits for version 4.x.x.

VeeranjaneyuluThoka · 2019-07-08T01:48:59Z

Hi Rhythmblue,
I am using the same code that you have given in your reply (https://github.com/USTC-Video-Understanding/I3D_Finetune) to fine tune UCF101 and HMDB51. When i have finetuned model of UCF101, the accuracy is as below
RGB data: 0.921, Flow data: 0.9984 and Fused: 0.792

With finetuned model of HMDB51, the accuracy is as below
RGB data: 0.7577, flow data: 0.6749 and fused: 0.5957
Note: i have generated flow data for HMDB51 using PWCNET

I am trying to understand why i am getting such a low accuracy rate using HMDB51, and high accuracy rate with UCF101 flow data, Could you plz suggest me any directions to find out the same?

Thanks,
Veeru.

Rhythmblue · 2019-09-06T15:10:07Z

@VeeranjaneyuluThoka ，sorry for the late response. I was busy doing experiments and tried to catch the deadline for a conference.
Now, I have difficulty to review the codes in TensorFlow. So I consider to re-implement i3d codes in PyTorch.

About your question, I think 0.99 of flow on UCF101 is an unexpected result. So, the code used for evaluation may have some problems. (Sometimes problem happens in the file list of video names and ground-truths. )

yuanzhedong · 2019-10-07T19:27:06Z

Hi @Rhythmblue , thank you for sharing the code to compute optical flow, could you explain more why you use the bound and also convert the flow to int?

Rhythmblue · 2019-10-09T11:48:34Z

Hi @Rhythmblue , thank you for sharing the code to compute optical flow, could you explain more why you use the bound and also convert the flow to int?

@yuanzhedong , I suggest that you can change bound to 20, following denseflow.

These operations follow the settings of the paper TSN and the codes in denseflow.
bound: limit the maximum movement of one pixel. It's an optional setting.
int: before to int, it is first changed to 0-255. The reason for this operation is to save storage space of the flow. An int8 JPEG can save much more space than the original float32 array.
Actually, the bound is also served to save the array of flow as a grey image. It can help limit the range and achieve normalization.

yuanzhedong · 2019-11-13T00:55:11Z

Hi @Rhythmblue , thank you for sharing the code to compute optical flow, could you explain more why you use the bound and also convert the flow to int?

@yuanzhedong , I suggest that you can change bound to 20, following denseflow.

These operations follow the settings of the paper TSN and the codes in denseflow.

bound: limit the maximum movement of one pixel. It's an optional setting.

int: before to int, it is first changed to 0-255. The reason for this operation is to save storage space of the flow. An int8 JPEG can save much more space than the original float32 array.

Actually, the bound is also served to save the array of flow as a grey image. It can help limit the range and achieve normalization.

Thank you for the detailed answer, it's very helpful!!

Rhythmblue closed this as completed Jun 15, 2018

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Preprocessing videos: extracting RGB images + flow data #2

Preprocessing videos: extracting RGB images + flow data #2

FrederikSchorr commented Jun 9, 2018

Rhythmblue commented Jun 10, 2018

Rhythmblue commented Jun 10, 2018

FrederikSchorr commented Jun 10, 2018

chanajianyu commented Jan 17, 2019

JwDong2019 commented Feb 13, 2019

Rhythmblue commented Feb 22, 2019

VeeranjaneyuluThoka commented Jul 8, 2019

Rhythmblue commented Sep 6, 2019

yuanzhedong commented Oct 7, 2019

Rhythmblue commented Oct 9, 2019 •

edited

yuanzhedong commented Nov 13, 2019

Preprocessing videos: extracting RGB images + flow data #2

Preprocessing videos: extracting RGB images + flow data #2

Comments

FrederikSchorr commented Jun 9, 2018

Rhythmblue commented Jun 10, 2018

Rhythmblue commented Jun 10, 2018

FrederikSchorr commented Jun 10, 2018

chanajianyu commented Jan 17, 2019

JwDong2019 commented Feb 13, 2019

Rhythmblue commented Feb 22, 2019

VeeranjaneyuluThoka commented Jul 8, 2019

Rhythmblue commented Sep 6, 2019

yuanzhedong commented Oct 7, 2019

Rhythmblue commented Oct 9, 2019 • edited

yuanzhedong commented Nov 13, 2019

Rhythmblue commented Oct 9, 2019 •

edited