OMX image_fx method OMX_ImageFilterDeInterlaceLineDouble broken

[tp-m]

When configuring the OMX image_fx element to use the OMX_ImageFilterDeInterlaceLineDouble filter method, the image gets completely garbled (green/red noise mostly). Other methods such as OMX_ImageFilterDeInterlaceAdvanced or OMX_ImageFilterSolarize work just fine though.

I will attach a modified hello_video/video.c

[tp-m]

// Video deocode demo using OpenMAX IL though the ilcient helper library
#include 
#include 
#include 
#include "bcm_host.h"
#include "ilclient.h"
static int video_decode_test(char *filename)
{
   OMX_VIDEO_PARAM_PORTFORMATTYPE format;
   OMX_TIME_CONFIG_CLOCKSTATETYPE cstate;
   COMPONENT_T *video_decode = NULL, *video_scheduler = NULL, *video_render = NULL, *clock = NULL;
   COMPONENT_T *list[6], *image_filter = NULL;
   TUNNEL_T tunnel[5];
   ILCLIENT_T *client;
   FILE *in;
   int status = 0;
   unsigned char *data = NULL;
   unsigned int data_len = 0;
   int find_start_codes = 0;
   int packet_size = 16<<10;   
   memset(list, 0, sizeof(list));
   memset(tunnel, 0, sizeof(tunnel));
   if((in = fopen(filename, "rb")) == NULL)
      return -2;
   if((client = ilclient_init()) == NULL)
   {
      fclose(in);
      return -3;
   }
   if(OMX_Init() != OMX_ErrorNone)
   {
      ilclient_destroy(client);
      fclose(in);
      return -4;
   }
   if(find_start_codes && (data = malloc(packet_size+4)) == NULL)
   {
      status = -16;
      if(OMX_Deinit() != OMX_ErrorNone)
         status = -17;
      ilclient_destroy(client);
      fclose(in);
      return status;
   }
   // create video_decode
   if(ilclient_create_component(client, &video_decode, "video_decode", ILCLIENT_DISABLE_ALL_PORTS | ILCLIENT_ENABLE_INPUT_BUFFERS) != 0)
      status = -14;
   list[0] = video_decode;
   // create video_render
   if(status == 0 && ilclient_create_component(client, &video_render, "video_render", ILCLIENT_DISABLE_ALL_PORTS) != 0)
      status = -14;
   list[1] = video_render;
   // create clock
   if(status == 0 && ilclient_create_component(client, &clock, "clock", ILCLIENT_DISABLE_ALL_PORTS) != 0)
      status = -14;
   list[2] = clock;
   memset(&cstate, 0, sizeof(cstate));
   cstate.nSize = sizeof(cstate);
   cstate.nVersion.nVersion = OMX_VERSION;
   cstate.eState = OMX_TIME_ClockStateWaitingForStartTime;
   cstate.nWaitMask = 1;
   if(clock != NULL && OMX_SetParameter(ILC_GET_HANDLE(clock), OMX_IndexConfigTimeClockState, &cstate) != OMX_ErrorNone)
      status = -13;
   // create video_scheduler
   if(status == 0 && ilclient_create_component(client, &video_scheduler, "video_scheduler", ILCLIENT_DISABLE_ALL_PORTS) != 0)
      status = -14;
   list[3] = video_scheduler;
   // create image_filter
   if(ilclient_create_component(client, &image_filter, "image_fx", ILCLIENT_DISABLE_ALL_PORTS) != 0)
      status = -14;
   list[4] = image_filter;
   set_tunnel(tunnel, video_decode, 131, image_filter, 190);
   set_tunnel(tunnel+1, image_filter, 191, video_scheduler, 10);
   set_tunnel(tunnel+2, video_scheduler, 11, video_render, 90);
   set_tunnel(tunnel+3, clock, 80, video_scheduler, 12);
   // setup clock tunnel first
   if(status == 0 && ilclient_setup_tunnel(tunnel+3, 0, 0) != 0)
      status = -15;
   else
      ilclient_change_component_state(clock, OMX_StateExecuting);
   if(status == 0)
      ilclient_change_component_state(video_decode, OMX_StateIdle);
   memset(&format, 0, sizeof(OMX_VIDEO_PARAM_PORTFORMATTYPE));
   format.nSize = sizeof(OMX_VIDEO_PARAM_PORTFORMATTYPE);
   format.nVersion.nVersion = OMX_VERSION;
   format.nPortIndex = 130;
   format.eCompressionFormat = OMX_VIDEO_CodingAVC;
   if(status == 0 &&
      OMX_SetParameter(ILC_GET_HANDLE(video_decode), OMX_IndexParamVideoPortFormat, &format) == OMX_ErrorNone &&
      ilclient_enable_port_buffers(video_decode, 130, NULL, NULL, NULL) == 0)
   {
      OMX_BUFFERHEADERTYPE *buf;
      int port_settings_changed = 0;
      int first_packet = 1;
      ilclient_change_component_state(video_decode, OMX_StateExecuting);
      while((buf = ilclient_get_input_buffer(video_decode, 130, 1)) != NULL)
      {
         // feed data and wait until we get port settings changed
         unsigned char *dest = find_start_codes ? data + data_len : buf->pBuffer;
         data_len += fread(dest, 1, packet_size+(find_start_codes*4)-data_len, in);
         if(port_settings_changed == 0 &&
            ((data_len > 0 && ilclient_remove_event(video_decode, OMX_EventPortSettingsChanged, 131, 0, 0, 1) == 0) ||
             (data_len == 0 && ilclient_wait_for_event(video_decode, OMX_EventPortSettingsChanged, 131, 0, 0, 1,
                                                       ILCLIENT_EVENT_ERROR | ILCLIENT_PARAMETER_CHANGED, 10000) == 0)))
         {
            port_settings_changed = 1;
            if(ilclient_setup_tunnel(tunnel, 0, 0) != 0)
            {
               status = -7;
               break;
            }
            {
                OMX_CONFIG_IMAGEFILTERTYPE filtertype;
                memset(&filtertype, 0, sizeof(OMX_VIDEO_PARAM_PORTFORMATTYPE));
                filtertype.nSize = sizeof(OMX_VIDEO_PARAM_PORTFORMATTYPE);
                filtertype.nVersion.nVersion = OMX_VERSION;
                filtertype.nPortIndex = 191;
                filtertype.eImageFilter = OMX_ImageFilterDeInterlaceLineDouble; //Advanced; //OMX_ImageFilterNone;
                if (OMX_SetParameter(ILC_GET_HANDLE(image_filter), OMX_IndexConfigCommonImageFilter, &filtertype) != OMX_ErrorNone)
                {
                    status = -99;
                    break;
                }
            }
            ilclient_change_component_state(image_filter, OMX_StateExecuting);
            // now setup tunnel to video_scheduler
            if(ilclient_setup_tunnel(tunnel+1, 0, 1000) != 0)
            {
               status = -12;
               break;
            }
            ilclient_change_component_state(video_scheduler, OMX_StateExecuting);
            // now setup tunnel to video_render
            if(ilclient_setup_tunnel(tunnel+2, 0, 1000) != 0)
            {
               status = -12;
               break;
            }
            
            ilclient_change_component_state(video_render, OMX_StateExecuting);
         }
         if(!data_len)
            break;
         if(find_start_codes)
         {
            int i, start = -1, len = 0;
            int max_len = data_len > packet_size ? packet_size : data_len;
            for(i=2; i 2 && data[i-3] == 0)
                  {
                     len++;
                     start--;
                  }
                  break;
               }
            }
            if(start == 0)
            {
               // start code is next, so just send that
               buf->nFilledLen = len;
            }
            else if(start == -1)
            {
               // no start codes seen, send the first block
               buf->nFilledLen = max_len;
            }
            else
            {
               // start code in the middle of the buffer, send up to the code
               buf->nFilledLen = start;
            }
            memcpy(buf->pBuffer, data, buf->nFilledLen);
            memmove(data, data + buf->nFilledLen, data_len - buf->nFilledLen);
            data_len -= buf->nFilledLen;
         }
         else
         {
            buf->nFilledLen = data_len;
            data_len = 0;
         }
         buf->nOffset = 0;
         if(first_packet)
         {
            buf->nFlags = OMX_BUFFERFLAG_STARTTIME;
            first_packet = 0;
         }
         else
            buf->nFlags = OMX_BUFFERFLAG_TIME_UNKNOWN;
         if(OMX_EmptyThisBuffer(ILC_GET_HANDLE(video_decode), buf) != OMX_ErrorNone)
         {
            status = -6;
            break;
         }
         
      }
      buf->nFilledLen = 0;
      buf->nFlags = OMX_BUFFERFLAG_TIME_UNKNOWN | OMX_BUFFERFLAG_EOS;
      
      if(OMX_EmptyThisBuffer(ILC_GET_HANDLE(video_decode), buf) != OMX_ErrorNone)
         status = -20;
      
      // wait for EOS from render
      ilclient_wait_for_event(video_render, OMX_EventBufferFlag, 90, 0, OMX_BUFFERFLAG_EOS, 0,
                              ILCLIENT_BUFFER_FLAG_EOS, 10000);
      
      // need to flush the renderer to allow video_decode to disable its input port
      ilclient_flush_tunnels(tunnel, 0);
      ilclient_disable_port_buffers(video_decode, 130, NULL, NULL, NULL);
   }
   fclose(in);
   ilclient_disable_tunnel(tunnel);
   ilclient_disable_tunnel(tunnel+2);
   ilclient_disable_tunnel(tunnel+3);
   ilclient_teardown_tunnels(tunnel);
   ilclient_state_transition(list, OMX_StateIdle);
   ilclient_state_transition(list, OMX_StateLoaded);
   ilclient_cleanup_components(list);
   OMX_Deinit();
   ilclient_destroy(client);
   return status;
}
int main (int argc, char **argv)
{
   if (argc < 2) {
      printf("Usage: %s \n", argv[0]);
      exit(1);
   }
   bcm_host_init();
   return video_decode_test(argv[1]);
}

[julianscheel]

I can confirm this issue. Getting the same garbled issues.

[6by9]

Confirmed. The GPU is mismanaging buffers in this case and ends up not doing the conversion before deinterlacing. The output is just uninitialised memory.

I don't fully understand the algorithm, but think I understand the buffer handling so I'll look at fixing it up.

[6by9]

Firmware changes pushed internally that fix the buffer mismanagement. I now get an output image that looks reasonable, but I can't comment on the deinterlacing being correct.
Test firmware at https://github.com/6by9/RPiTest/blob/master/mmal_image_fx/start_x.elf if you want to experiment, but should be released in the next couple of days.

[popcornmix]

Firmware has been updated. Please run rpi-update and test.

[tp-m]

Thanks a lot for the fix. I won't be able to test this until next week at the earliest, so please feel free to close if it works for you now.

[julianscheel]

I'm going to test it tomorrow

[julianscheel]

Works for me. Can be closed.