Skip to content

/ libav

Permalink
master
Switch branches/tags

libav/libavcodec/x86/hevc_mc.asm

Go to file
 
 
Cannot retrieve contributors at this time
851 lines (699 sloc) 17.6 KB
Raw Blame
;*****************************************************************************
;* x86-optimized HEVC MC
;* Copyright 2015 Anton Khirnov
;*
;* This file is part of Libav.
;*
;* Libav is free software; you can redistribute it and/or
;* modify it under the terms of the GNU Lesser General Public
;* License as published by the Free Software Foundation; either
;* version 2.1 of the License, or (at your option) any later version.
;*
;* Libav is distributed in the hope that it will be useful,
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
;* Lesser General Public License for more details.
;*
;* You should have received a copy of the GNU Lesser General Public
;* License along with Libav; if not, write to the Free Software
;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
;******************************************************************************
%include "libavutil/x86/x86util.asm"
SECTION_RODATA
pw_1023: times 8 dw 1023
cextern hevc_qpel_coeffs
cextern hevc_qpel_coeffs8
cextern hevc_epel_coeffs
cextern hevc_epel_coeffs8
cextern pw_8
cextern pw_16
cextern pw_32
cextern pw_64
SECTION .text
; %1: width
; %2: bit depth
%macro COMMON_DEFS 2
%assign blocksize 8
%assign nb_blocks ((%1 + blocksize - 1) / blocksize)
%define last_block_truncated (blocksize * nb_blocks > %1)
%if %2 > 8
%define LOAD_BLOCK movu
%define LOAD_HALFBLOCK movq
%assign pixelsize 2
%else
%define LOAD_BLOCK movq
%define LOAD_HALFBLOCK movd
%assign pixelsize 1
%endif
%define STORE_BLOCK mova
%define STORE_HALFBLOCK movq
%endmacro
; %1: block index
%macro BLOCK_DEFS 1
%if last_block_truncated && %1 == nb_blocks - 1
%define block_truncated 1
%define LOAD LOAD_HALFBLOCK
%define STORE STORE_HALFBLOCK
%else
%define block_truncated 0
%define LOAD LOAD_BLOCK
%define STORE STORE_BLOCK
%endif
%endmacro
; hevc_get_pixels_<w>_<d>(int16_t *dst, ptrdiff_t dststride,
; pixel *src, ptrdiff_t srcstride,
; int height, int mx, int my, int *mcbuffer)
; %1: block width
; %2: bit depth
; %3: log2 of height unroll
%macro GET_PIXELS 3
cglobal hevc_get_pixels_ %+ %1 %+ _ %+ %2, 5, 5, 2, dst, dststride, src, srcstride, height ; rest of the args unused
%assign shift 14 - %2
COMMON_DEFS %1, %2
%if pixelsize == 1
pxor m0, m0
%endif
shr heightd, %3
.loop:
%assign i 0
%rep (1 << %3)
%assign j 0
%rep nb_blocks
BLOCK_DEFS j
LOAD m1, [srcq + j * pixelsize * blocksize]
%if pixelsize == 1
punpcklbw m1, m0
%endif
psllw m1, shift
STORE [dstq + j * 2 * blocksize], m1
%assign j (j + 1)
%endrep
add dstq, dststrideq
add srcq, srcstrideq
%assign i (i + 1)
%endrep
dec heightd
jg .loop
RET
%endmacro
INIT_XMM sse2
GET_PIXELS 4, 8, 1
GET_PIXELS 8, 8, 1
GET_PIXELS 12, 8, 3
GET_PIXELS 16, 8, 2
GET_PIXELS 24, 8, 3
GET_PIXELS 32, 8, 3
GET_PIXELS 48, 8, 3
GET_PIXELS 64, 8, 3
GET_PIXELS 4, 10, 1
GET_PIXELS 8, 10, 1
GET_PIXELS 12, 10, 3
GET_PIXELS 16, 10, 2
GET_PIXELS 24, 10, 3
GET_PIXELS 32, 10, 3
GET_PIXELS 48, 10, 3
GET_PIXELS 64, 10, 3
; hevc_qpel_h/v_<w>_8(int16_t *dst, ptrdiff_t dststride,
; uint8_t *src, ptrdiff_t srcstride,
; int height, int mx, int my, int *mcbuffer)
; 8-bit qpel interpolation
; %1: block width
; %2: 0 - horizontal; 1 - vertical
%macro QPEL_8 2
%if %2
%define postfix v
%define mvfrac myq
%define coeffsaddr r5q
%define pixstride srcstrideq
%define pixstride3 r5q
%define src_m3 r6q
%else
%define postfix h
%define mvfrac mxq
%define coeffsaddr r6q
%define pixstride 1
%define pixstride3 3
%define src_m3 (srcq - 3)
%endif
COMMON_DEFS %1, 8
cglobal hevc_qpel_ %+ postfix %+ _ %+ %1 %+ _8, 7, 7, 7, dst, dststride, src, srcstride, height, mx, my
and mvfrac, 0x3
dec mvfrac
shl mvfrac, 4
lea coeffsaddr, [hevc_qpel_coeffs8]
mova m0, [coeffsaddr + mvfrac]
SPLATW m1, m0, 1
SPLATW m2, m0, 2
SPLATW m3, m0, 3
SPLATW m0, m0, 0
%if %2
lea pixstride3, [srcstrideq + 2 * srcstrideq]
mov src_m3, srcq
sub src_m3, pixstride3
%endif
.loop:
%assign i 0
%rep nb_blocks
BLOCK_DEFS i
LOAD m4, [src_m3 + i * blocksize]
LOAD m5, [src_m3 + i * blocksize + 1 * pixstride]
punpcklbw m4, m5
pmaddubsw m4, m0
LOAD m5, [src_m3 + i * blocksize + 2 * pixstride]
LOAD m6, [srcq + i * blocksize]
punpcklbw m5, m6
pmaddubsw m5, m1
paddsw m4, m5
LOAD m5, [srcq + i * blocksize + 1 * pixstride]
LOAD m6, [srcq + i * blocksize + 2 * pixstride]
punpcklbw m5, m6
pmaddubsw m5, m2
paddsw m4, m5
LOAD m5, [srcq + i * blocksize + pixstride3]
LOAD m6, [srcq + i * blocksize + 4 * pixstride]
punpcklbw m5, m6
pmaddubsw m5, m3
paddsw m4, m5
STORE [dstq + i * 2 * blocksize], m4
%assign i (i + 1)
%endrep
add dstq, dststrideq
add srcq, srcstrideq
%if %2
add src_m3, srcstrideq
%endif
dec heightd
jg .loop
RET
%endmacro
INIT_XMM ssse3
QPEL_8 4, 0
QPEL_8 8, 0
QPEL_8 12, 0
QPEL_8 16, 0
QPEL_8 24, 0
QPEL_8 32, 0
QPEL_8 48, 0
QPEL_8 64, 0
QPEL_8 4, 1
QPEL_8 8, 1
QPEL_8 12, 1
QPEL_8 16, 1
QPEL_8 24, 1
QPEL_8 32, 1
QPEL_8 48, 1
QPEL_8 64, 1
; 16-bit qpel interpolation
; %1: block width
; %2: shift applied to the result
; %3: 0 - horizontal; 1 - vertical
%macro QPEL_16 3
%if %3
%define mvfrac myq
%define pixstride srcstrideq
%define pixstride3 sstride3q
%define src_m3 srcm3q
%else
%define mvfrac mxq
%define pixstride 2
%define pixstride3 6
%define src_m3 (srcq - 6)
%endif
COMMON_DEFS %1, 16
and mvfrac, 0x3
dec mvfrac
shl mvfrac, 4
lea coeffsregq, [hevc_qpel_coeffs]
mova m0, [coeffsregq + mvfrac]
pshufd m1, m0, 0x55
pshufd m2, m0, 0xaa
pshufd m3, m0, 0xff
pshufd m0, m0, 0x00
%if %3
lea sstride3q, [srcstrideq + 2 * srcstrideq]
mov srcm3q, srcq
sub srcm3q, sstride3q
%endif
.loop:
%assign i 0
%rep nb_blocks
BLOCK_DEFS i
LOAD m4, [src_m3 + i * 2 * blocksize]
LOAD m5, [src_m3 + i * 2 * blocksize + 1 * pixstride]
LOAD m6, [src_m3 + i * 2 * blocksize + 2 * pixstride]
LOAD m7, [srcq + i * 2 * blocksize + 0 * pixstride]
LOAD m8, [srcq + i * 2 * blocksize + 1 * pixstride]
LOAD m9, [srcq + i * 2 * blocksize + 2 * pixstride]
LOAD m10, [srcq + i * 2 * blocksize + pixstride3]
LOAD m11, [srcq + i * 2 * blocksize + 4 * pixstride]
punpcklwd m12, m4, m5
pmaddwd m12, m0
punpcklwd m13, m6, m7
pmaddwd m13, m1
paddd m12, m13
punpcklwd m13, m8, m9
pmaddwd m13, m2
paddd m12, m13
punpcklwd m13, m10, m11
pmaddwd m13, m3
paddd m12, m13
psrad m12, %2
%if block_truncated == 0
punpckhwd m4, m5
pmaddwd m4, m0
punpckhwd m6, m7
pmaddwd m6, m1
paddd m4, m6
punpckhwd m8, m9
pmaddwd m8, m2
paddd m4, m8
punpckhwd m10, m11
pmaddwd m10, m3
paddd m4, m10
psrad m4, %2
%endif
packssdw m12, m4
STORE [dstq + i * 2 * blocksize], m12
%assign i (i + 1)
%endrep
add dstq, dststrideq
add srcq, srcstrideq
%if %3
add srcm3q, srcstrideq
%endif
dec heightd
jg .loop
RET
%endmacro
%if ARCH_X86_64
%macro QPEL_H_10 1
cglobal hevc_qpel_h_ %+ %1 %+ _10, 7, 9, 14, dst, dststride, src, srcstride, height, mx, my, mcbuffer, coeffsreg
QPEL_16 %1, 2, 0
%endmacro
INIT_XMM avx
QPEL_H_10 4
QPEL_H_10 8
QPEL_H_10 12
QPEL_H_10 16
QPEL_H_10 24
QPEL_H_10 32
QPEL_H_10 48
QPEL_H_10 64
%macro QPEL_V_10 1
cglobal hevc_qpel_v_ %+ %1 %+ _10, 7, 10, 14, dst, dststride, src, srcstride, height, mx, my, sstride3, srcm3, coeffsreg
QPEL_16 %1, 2, 1
%endmacro
INIT_XMM avx
QPEL_V_10 4
QPEL_V_10 8
QPEL_V_10 12
QPEL_V_10 16
QPEL_V_10 24
QPEL_V_10 32
QPEL_V_10 48
QPEL_V_10 64
; hevc_qpel_hv_<w>(int16_t *dst, ptrdiff_t dststride,
; uint8_t *src, ptrdiff_t srcstride,
; int height, int mx, int my, int *mcbuffer)
%macro QPEL_HV 1
cglobal hevc_qpel_hv_ %+ %1, 7, 10, 14, dst, dststride, src, srcstride, height, mx, my, sstride3, srcm3, coeffsreg
QPEL_16 %1, 6, 1
%endmacro
INIT_XMM avx
QPEL_HV 4
QPEL_HV 8
QPEL_HV 12
QPEL_HV 16
QPEL_HV 24
QPEL_HV 32
QPEL_HV 48
QPEL_HV 64
%endif ; ARCH_X86_64
; hevc_epel_h/v_<w>_8(int16_t *dst, ptrdiff_t dststride,
; uint8_t *src, ptrdiff_t srcstride,
; int height, int mx, int my, int *mcbuffer)
; 8-bit epel interpolation
; %1: block width
; %2: 0 - horizontal; 1 - vertical
%macro EPEL_8 2
%if %2
%define postfix v
%define mvfrac myq
%define coeffsaddr r5q
%define pixstride srcstrideq
%define pixstride3 r5q
%else
%define postfix h
%define mvfrac mxq
%define coeffsaddr r6q
%define pixstride 1
%define pixstride3 3
%endif
COMMON_DEFS %1, 8
cglobal hevc_epel_ %+ postfix %+ _ %+ %1 %+ _8, 7, 7, 6, dst, dststride, src, srcstride, height, mx, my
and mvfrac, 0x7
dec mvfrac
shl mvfrac, 4
lea coeffsaddr, [hevc_epel_coeffs8]
movq m0, [coeffsaddr + mvfrac]
SPLATW m1, m0, 1
SPLATW m0, m0, 0
%if %2
lea pixstride3, [srcstrideq + 2 * srcstrideq]
%endif
sub srcq, pixstride
.loop:
%assign i 0
%rep nb_blocks
BLOCK_DEFS i
LOAD m2, [srcq + i * blocksize + 0 * pixstride]
LOAD m3, [srcq + i * blocksize + 1 * pixstride]
LOAD m4, [srcq + i * blocksize + 2 * pixstride]
LOAD m5, [srcq + i * blocksize + pixstride3]
punpcklbw m2, m3
punpcklbw m4, m5
pmaddubsw m2, m0
pmaddubsw m4, m1
paddsw m2, m4
STORE [dstq + i * 2 * blocksize], m2
%assign i (i + 1)
%endrep
add dstq, dststrideq
add srcq, srcstrideq
dec heightd
jg .loop
RET
%endmacro
INIT_XMM ssse3
EPEL_8 4, 0
EPEL_8 8, 0
EPEL_8 12, 0
EPEL_8 16, 0
EPEL_8 24, 0
EPEL_8 32, 0
EPEL_8 4, 1
EPEL_8 8, 1
EPEL_8 12, 1
EPEL_8 16, 1
EPEL_8 24, 1
EPEL_8 32, 1
%macro EPEL_16 3
%if %3
%define mvfrac myq
%define pixstride srcstrideq
%define pixstride3 sstride3q
%else
%define mvfrac mxq
%define pixstride 2
%define pixstride3 6
%endif
COMMON_DEFS %1, 16
and mvfrac, 0x7
dec mvfrac
shl mvfrac, 5
lea coeffsregq, [hevc_epel_coeffs]
mova m0, [coeffsregq + mvfrac]
pshufd m1, m0, 0x55
pshufd m0, m0, 0x00
%if %3
lea sstride3q, [srcstrideq + 2 * srcstrideq]
%endif
sub srcq, pixstride
.loop:
%assign i 0
%rep nb_blocks
BLOCK_DEFS i
LOAD m2, [srcq + i * 2 * blocksize + 0 * pixstride]
LOAD m3, [srcq + i * 2 * blocksize + 1 * pixstride]
LOAD m4, [srcq + i * 2 * blocksize + 2 * pixstride]
LOAD m5, [srcq + i * 2 * blocksize + pixstride3]
punpcklwd m6, m2, m3
punpcklwd m7, m4, m5
pmaddwd m6, m0
pmaddwd m7, m1
paddd m6, m7
psrad m6, %2
%if block_truncated == 0
punpckhwd m2, m3
punpckhwd m4, m5
pmaddwd m2, m0
pmaddwd m4, m1
paddd m2, m4
psrad m2, %2
%endif
packssdw m6, m2
STORE [dstq + i * 2 * blocksize], m6
%assign i (i + 1)
%endrep
add dstq, dststrideq
add srcq, srcstrideq
dec heightd
jg .loop
RET
%endmacro
%if ARCH_X86_64
%macro EPEL_H_10 1
cglobal hevc_epel_h_ %+ %1 %+ _10, 8, 9, 8, dst, dststride, src, srcstride, height, mx, my, sstride3, coeffsreg
EPEL_16 %1, 2, 0
%endmacro
INIT_XMM avx
EPEL_H_10 4
EPEL_H_10 8
EPEL_H_10 12
EPEL_H_10 16
EPEL_H_10 24
EPEL_H_10 32
%macro EPEL_V_10 1
cglobal hevc_epel_v_ %+ %1 %+ _10, 8, 9, 8, dst, dststride, src, srcstride, height, mx, my, sstride3, coeffsreg
EPEL_16 %1, 2, 1
%endmacro
INIT_XMM avx
EPEL_V_10 4
EPEL_V_10 8
EPEL_V_10 12
EPEL_V_10 16
EPEL_V_10 24
EPEL_V_10 32
; hevc_epel_hv_<w>_8(int16_t *dst, ptrdiff_t dststride,
; int16_t *src, ptrdiff_t srcstride,
; int height, int mx, int my, int *mcbuffer)
%macro EPEL_HV 1
cglobal hevc_epel_hv_ %+ %1, 8, 9, 8, dst, dststride, src, srcstride, height, mx, my, sstride3, coeffsreg
EPEL_16 %1, 6, 1
%endmacro
INIT_XMM avx
EPEL_HV 4
EPEL_HV 8
EPEL_HV 12
EPEL_HV 16
EPEL_HV 24
EPEL_HV 32
%endif ; ARCH_X86_64
; hevc_put_unweighted_pred_<w>_<d>(pixel *dst, ptrdiff_t dststride,
; int16_t *src, ptrdiff_t srcstride,
; int height)
%macro AVG 5
%if %3
%if %4 == 4
movq %5, %2
paddsw %1, %5
%else
paddsw %1, %2
%endif
%endif
%endmacro
; %1: 0 - one source; 1 - two sources
; %2: width
; %3: bit depth
%macro PUT_PRED 3
%if %1
cglobal hevc_put_unweighted_pred_avg_ %+ %2 %+ _ %+ %3, 6, 6, 4, dst, dststride, src, src2, srcstride, height
%else
cglobal hevc_put_unweighted_pred_ %+ %2 %+ _ %+ %3, 5, 5, 4, dst, dststride, src, srcstride, height
%endif
%assign shift 14 + %1 - %3
%assign offset (1 << (shift - 1))
%define offset_data pw_ %+ offset
mova m0, [offset_data]
%if %3 > 8
%define STORE_BLOCK movu
%define STORE_HALF movq
%assign pixel_max ((1 << %3) - 1)
%define pw_pixel_max pw_ %+ pixel_max
pxor m1, m1
mova m2, [pw_pixel_max]
%else
%define STORE_BLOCK movq
%define STORE_HALF movd
%endif
.loop:
%assign i 0
%rep (%2 + 7) / 8
%if (i + 1) * 8 > %2
%define LOAD movq
%define STORE STORE_HALF
%else
%define LOAD mova
%define STORE STORE_BLOCK
%endif
LOAD m3, [srcq + 16 * i]
AVG m3, [src2q + 16 * i], %1, %3 - i * 8, m4
paddsw m3, m0
psraw m3, shift
%if %3 == 8
packuswb m3, m3
STORE [dstq + 8 * i], m3
%else
CLIPW m3, m1, m2
STORE [dstq + 16 * i], m3
%endif
%assign i (i + 1)
%endrep
add dstq, dststrideq
add srcq, srcstrideq
%if %1
add src2q, srcstrideq
%endif
dec heightd
jg .loop
RET
%endmacro
INIT_XMM sse2
PUT_PRED 0, 4, 8
PUT_PRED 1, 4, 8
PUT_PRED 0, 8, 8
PUT_PRED 1, 8, 8
PUT_PRED 0, 12, 8
PUT_PRED 1, 12, 8
PUT_PRED 0, 16, 8
PUT_PRED 1, 16, 8
PUT_PRED 0, 24, 8
PUT_PRED 1, 24, 8
PUT_PRED 0, 32, 8
PUT_PRED 1, 32, 8
PUT_PRED 0, 48, 8
PUT_PRED 1, 48, 8
PUT_PRED 0, 64, 8
PUT_PRED 1, 64, 8
PUT_PRED 0, 4, 10
PUT_PRED 1, 4, 10
PUT_PRED 0, 8, 10
PUT_PRED 1, 8, 10
PUT_PRED 0, 12, 10
PUT_PRED 1, 12, 10
PUT_PRED 0, 16, 10
PUT_PRED 1, 16, 10
PUT_PRED 0, 24, 10
PUT_PRED 1, 24, 10
PUT_PRED 0, 32, 10
PUT_PRED 1, 32, 10
PUT_PRED 0, 48, 10
PUT_PRED 1, 48, 10
PUT_PRED 0, 64, 10
PUT_PRED 1, 64, 10
%macro PUT_WEIGHTED_PRED 3
%if %1
cglobal hevc_put_weighted_pred_avg_ %+ %2 %+ _ %+ %3, 11, 11, 8, denom, weight0, weight1, offset0, offset1, dst, dststride, src0, src1, srcstride, height
%else
cglobal hevc_put_weighted_pred_ %+ %2 %+ _ %+ %3, 8, 8, 8, denom, weight0, offset0, dst, dststride, src0, srcstride, height
%endif
and denomd, 0xff
movsx weight0d, weight0w
movsx offset0d, offset0w
%if %1
movsx weight1d, weight1w
movsx offset1d, offset1w
%endif
add denomd, 14 + %1 - %3
movd m0, denomd
%if %3 > 8
%assign pixel_max ((1 << %3) - 1)
%define pw_pixel_max pw_ %+ pixel_max
pxor m4, m4
mova m5, [pw_pixel_max]
shl offset0d, %3 - 8
%if %1
shl offset1d, %3 - 8
%endif
%endif
%if %1
lea offset0d, [offset0d + offset1d + 1]
%else
lea offset0d, [2 * offset0d + 1]
%endif
movd m1, offset0d
SPLATD m1
pslld m1, m0
psrad m1, 1
movd m2, weight0d
SPLATD m2
%if %1
movd m3, weight1d
SPLATD m3
%endif
.loop:
%assign i 0
%rep (%2 + 3) / 4
pmovsxwd m6, [src0q + 8 * i]
pmulld m6, m2
%if %1
pmovsxwd m7, [src1q + 8 * i]
pmulld m7, m3
paddd m6, m7
%endif
paddd m6, m1
psrad m6, m0
packssdw m6, m6
%if %3 > 8
CLIPW m6, m4, m5
movq [dstq + 8 * i], m6
%else
packuswb m6, m6
movd [dstq + 4 * i], m6
%endif
%assign i (i + 1)
%endrep
add dstq, dststrideq
add src0q, srcstrideq
%if %1
add src1q, srcstrideq
%endif
dec heightd
jg .loop
RET
%endmacro
%if ARCH_X86_64
INIT_XMM sse4
PUT_WEIGHTED_PRED 0, 4, 8
PUT_WEIGHTED_PRED 1, 4, 8
PUT_WEIGHTED_PRED 0, 8, 8
PUT_WEIGHTED_PRED 1, 8, 8
PUT_WEIGHTED_PRED 0, 12, 8
PUT_WEIGHTED_PRED 1, 12, 8
PUT_WEIGHTED_PRED 0, 16, 8
PUT_WEIGHTED_PRED 1, 16, 8
PUT_WEIGHTED_PRED 0, 24, 8
PUT_WEIGHTED_PRED 1, 24, 8
PUT_WEIGHTED_PRED 0, 32, 8
PUT_WEIGHTED_PRED 1, 32, 8
PUT_WEIGHTED_PRED 0, 48, 8
PUT_WEIGHTED_PRED 1, 48, 8
PUT_WEIGHTED_PRED 0, 64, 8
PUT_WEIGHTED_PRED 1, 64, 8
PUT_WEIGHTED_PRED 0, 4, 10
PUT_WEIGHTED_PRED 1, 4, 10
PUT_WEIGHTED_PRED 0, 8, 10
PUT_WEIGHTED_PRED 1, 8, 10
PUT_WEIGHTED_PRED 0, 12, 10
PUT_WEIGHTED_PRED 1, 12, 10
PUT_WEIGHTED_PRED 0, 16, 10
PUT_WEIGHTED_PRED 1, 16, 10
PUT_WEIGHTED_PRED 0, 24, 10
PUT_WEIGHTED_PRED 1, 24, 10
PUT_WEIGHTED_PRED 0, 32, 10
PUT_WEIGHTED_PRED 1, 32, 10
PUT_WEIGHTED_PRED 0, 48, 10
PUT_WEIGHTED_PRED 1, 48, 10
PUT_WEIGHTED_PRED 0, 64, 10
PUT_WEIGHTED_PRED 1, 64, 10
%endif ; ARCH_X86_64