Preprocessing DVS Gestures

[giacomop96]

Hi, I was trying to preprocessing the DVS Gesture dataset with the code given in #10, but I get this error.
The path is right since I have tried to read the data with the readAedatevent function and everything works well.
Any suggestions?

0 user01_fluorescent
ValueError Traceback (most recent call last)
in ()
23 #for i in range(20):
24 # print(x[i], y[i], p[i],t[i])
---> 25 splitData(filename, path)
26 count += 1

3 frames
in splitData(filename, path)
1 def splitData(filename, path):
----> 2 x, y, p, t = scipy.io.loadmat(path + filename + '.aedat')
3 print(x,y,p,t)
4 labels = np.loadtxt(path + filename + '_labels.csv', delimiter=',', skiprows=1)
5 labels[:,0] -= 1

/usr/local/lib/python3.6/dist-packages/scipy/io/matlab/mio.py in loadmat(file_name, mdict, appendmat, **kwargs)
215 variable_names = kwargs.pop('variable_names', None)
216 with _open_file_context(file_name, appendmat) as f:
--> 217 MR, _ = mat_reader_factory(f, **kwargs)
218 matfile_dict = MR.get_variables(variable_names)
219

/usr/local/lib/python3.6/dist-packages/scipy/io/matlab/mio.py in mat_reader_factory(file_name, appendmat, **kwargs)
70 """
71 byte_stream, file_opened = _open_file(file_name, appendmat)
---> 72 mjv, mnv = get_matfile_version(byte_stream)
73 if mjv == 0:
74 return MatFile4Reader(byte_stream, **kwargs), file_opened

/usr/local/lib/python3.6/dist-packages/scipy/io/matlab/miobase.py in get_matfile_version(fileobj)
239 if maj_val in (1, 2):
240 return ret
--> 241 raise ValueError('Unknown mat file type, version %s, %s' % ret)
242
243

ValueError: Unknown mat file type, version 0, 154

[bamsumit]

Replace scipy.io.loadmat with readAedatEvent.

[giacomop96]

That worked, but later when

ind = (t >= tst) & (t < ten)
TD = snn.io.event(x[ind],y[ind], p[ind], (t[ind] - tst)/1000)

It says that ind should be scalar integer since it is user as an index, instead it is an array of bool I think. Should I cast I somehow or I have to redefine it in some other way?

[albertopolito]

I think this can be the right code:

import numpy as np
import matplotlib.pyplot as plt
import slayerSNN as snn
from dv import LegacyAedatFile
import os

path = './raw/'

actionName = [
    'hand_clapping',
    'right_hand_wave',
    'left_hand_wave',
    'right_arm_clockwise',
    'right_arm_counter_clockwise',
    'left_arm_clockwise', 
    'left_arm_counter_clockwise',
    'arm_roll',
    'air_drums',
    'air_guitar',
    'other_gestures',
]

def readAedatEvent(filename):
    xEvent = []
    yEvent = []
    pEvent = []
    tEvent = []
    with LegacyAedatFile(filename) as f:
        for event in f:
            xEvent.append(event.x)
            yEvent.append(event.y)
            pEvent.append(event.polarity)
            tEvent.append(event.timestamp/1000)

    return xEvent, yEvent, pEvent, tEvent

def splitData(filename, path):
    
    x, y, p, t = readAedatEvent(path + filename + '.aedat')
    labels = np.loadtxt(path + filename + '_labels.csv', delimiter=',', skiprows=1)
    labels[:,0]  -= 1
    labels[:,1:]

    if not os.path.isdir('data/' + filename):
        os.mkdir('data/' + filename)

    lastAction = 100
    for action, tst, ten in labels:
        if action == lastAction:    continue # This is to ignore second arm_roll samples
        print(actionName[int(action)])
        #remember t is in ms and tst and ten are in us
        #This give you a boolean vector to indicate where it is the action, so the time (t) >= time start (tst) and time (t) < time end (ten)
        #where it is True, it indicate the part of the stream (vector) that represent the action
        ind = (t >= tst/1000) & (t < ten/1000) 
        #with this function we catch the index of the first True item, so the first event of the action                      
        ind_in = np.argmax(ind) 
        #with this function we catch the index of the first False item after the action stream, so the end of the action
        ind_end = ind_in+np.argmin(ind[(ind.argmax()):-1])
        #this is because the True stream can be untill the end of the t vector, 
        #so argmin() give you zero, and ind_end = ind_in that is uncorrect
        if ind_end==ind_in: 
               ind_end=0
        TD = snn.io.event(x[ind_in:ind_end-1], y[ind_in:ind_end-1], p[ind_in:ind_end-1], (t[ind_in:ind_end-1] - tst/1000)) #collect the event of the action with t in ms
        # snn.io.showTD(TD)
        lastAction = action

        snn.io.encodeNpSpikes('data/'+ filename + '/{:g}.npy'.format(action), TD)

if __name__ == '__main__':
    user = np.arange(29) + 1
    lighting = [
        'fluorescent',
        'fluorescent_led',
        'lab',
        'led',
        'natural',
    ]

    count = 0
    for id in user:
        for light in lighting:
            filename = 'user{:02d}_{}'.format(id, light)      
            if os.path.isfile(path + filename + '.aedat'):
                print(count, filename)
                splitData(filename, path)
                count += 1

I hope it can help you.

[bamsumit]

Thanks @albertopolito .
@giacomop96 Hope that solved your issue.

[giacomop96]

Hi, I have tried and it seems to work perfectly. Thanks a lot for the help.

[bamsumit]

Great.

[chenjiefighting]

I think this can be the right code:

import numpy as np
import matplotlib.pyplot as plt
import slayerSNN as snn
from dv import LegacyAedatFile
import os

path = './raw/'

actionName = [
    'hand_clapping',
    'right_hand_wave',
    'left_hand_wave',
    'right_arm_clockwise',
    'right_arm_counter_clockwise',
    'left_arm_clockwise', 
    'left_arm_counter_clockwise',
    'arm_roll',
    'air_drums',
    'air_guitar',
    'other_gestures',
]

def readAedatEvent(filename):
    xEvent = []
    yEvent = []
    pEvent = []
    tEvent = []
    with LegacyAedatFile(filename) as f:
        for event in f:
            xEvent.append(event.x)
            yEvent.append(event.y)
            pEvent.append(event.polarity)
            tEvent.append(event.timestamp/1000)

    return xEvent, yEvent, pEvent, tEvent

def splitData(filename, path):
    
    x, y, p, t = readAedatEvent(path + filename + '.aedat')
    labels = np.loadtxt(path + filename + '_labels.csv', delimiter=',', skiprows=1)
    labels[:,0]  -= 1
    labels[:,1:]

    if not os.path.isdir('data/' + filename):
        os.mkdir('data/' + filename)

    lastAction = 100
    for action, tst, ten in labels:
        if action == lastAction:    continue # This is to ignore second arm_roll samples
        print(actionName[int(action)])
        #remember t is in ms and tst and ten are in us
        #This give you a boolean vector to indicate where it is the action, so the time (t) >= time start (tst) and time (t) < time end (ten)
        #where it is True, it indicate the part of the stream (vector) that represent the action
        ind = (t >= tst/1000) & (t < ten/1000) 
        #with this function we catch the index of the first True item, so the first event of the action                      
        ind_in = np.argmax(ind) 
        #with this function we catch the index of the first False item after the action stream, so the end of the action
        ind_end = ind_in+np.argmin(ind[(ind.argmax()):-1])
        #this is because the True stream can be untill the end of the t vector, 
        #so argmin() give you zero, and ind_end = ind_in that is uncorrect
        if ind_end==ind_in: 
               ind_end=0
        TD = snn.io.event(x[ind_in:ind_end-1], y[ind_in:ind_end-1], p[ind_in:ind_end-1], (t[ind_in:ind_end-1] - tst/1000)) #collect the event of the action with t in ms
        # snn.io.showTD(TD)
        lastAction = action

        snn.io.encodeNpSpikes('data/'+ filename + '/{:g}.npy'.format(action), TD)

if __name__ == '__main__':
    user = np.arange(29) + 1
    lighting = [
        'fluorescent',
        'fluorescent_led',
        'lab',
        'led',
        'natural',
    ]

    count = 0
    for id in user:
        for light in lighting:
            filename = 'user{:02d}_{}'.format(id, light)      
            if os.path.isfile(path + filename + '.aedat'):
                print(count, filename)
                splitData(filename, path)
                count += 1

I hope it can help you.

Thanks for your work. Can other data sets be preprocessed in this way, such as ASL-DVS？