Revert "Revert "[NEON] Optimize vp9_diamond_search_sad() for NEON""
This reverts commit 9f1329f8ac
and fixes a dumb mistake in evaluation of vfcmv. Used vdupq_n_s16,
instead of vdupq_n_s32.
Change-Id: Ie236c878c166405c49bc0f93f6d63a6715534a0a
This commit is contained in:
Konstantinos Margaritis
@@ -175,7 +175,7 @@ if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") ne "yes") {
|
||||
# Motion search
|
||||
#
|
||||
add_proto qw/int vp9_diamond_search_sad/, "const struct macroblock *x, const struct search_site_config *cfg, struct mv *ref_mv, struct mv *best_mv, int search_param, int sad_per_bit, int *num00, const struct vp9_variance_vtable *fn_ptr, const struct mv *center_mv";
|
||||
specialize qw/vp9_diamond_search_sad avx/;
|
||||
specialize qw/vp9_diamond_search_sad avx neon/;
|
||||
|
||||
#
|
||||
# Apply temporal filter
|
||||
|
||||
@@ -0,0 +1,322 @@
|
||||
/*
|
||||
* Copyright (c) 2022 The WebM project authors. All Rights Reserved.
|
||||
*
|
||||
* Use of this source code is governed by a BSD-style license
|
||||
* that can be found in the LICENSE file in the root of the source
|
||||
* tree. An additional intellectual property rights grant can be found
|
||||
* in the file PATENTS. All contributing project authors may
|
||||
* be found in the AUTHORS file in the root of the source tree.
|
||||
*/
|
||||
|
||||
#include <assert.h>
|
||||
#include <arm_neon.h>
|
||||
|
||||
#include "vpx_dsp/vpx_dsp_common.h"
|
||||
#include "vp9/encoder/vp9_encoder.h"
|
||||
#include "vpx_ports/mem.h"
|
||||
|
||||
#ifdef __GNUC__
|
||||
#define LIKELY(v) __builtin_expect(v, 1)
|
||||
#define UNLIKELY(v) __builtin_expect(v, 0)
|
||||
#else
|
||||
#define LIKELY(v) (v)
|
||||
#define UNLIKELY(v) (v)
|
||||
#endif
|
||||
|
||||
static INLINE int_mv pack_int_mv(int16_t row, int16_t col) {
|
||||
int_mv result;
|
||||
result.as_mv.row = row;
|
||||
result.as_mv.col = col;
|
||||
return result;
|
||||
}
|
||||
|
||||
static INLINE MV_JOINT_TYPE get_mv_joint(const int_mv mv) {
|
||||
// This is simplified from the C implementation to utilise that
|
||||
// x->nmvjointsadcost[1] == x->nmvjointsadcost[2] and
|
||||
// x->nmvjointsadcost[1] == x->nmvjointsadcost[3]
|
||||
return mv.as_int == 0 ? 0 : 1;
|
||||
}
|
||||
|
||||
static INLINE int mv_cost(const int_mv mv, const int *joint_cost,
|
||||
int *const comp_cost[2]) {
|
||||
assert(mv.as_mv.row >= -MV_MAX && mv.as_mv.row < MV_MAX);
|
||||
assert(mv.as_mv.col >= -MV_MAX && mv.as_mv.col < MV_MAX);
|
||||
return joint_cost[get_mv_joint(mv)] + comp_cost[0][mv.as_mv.row] +
|
||||
comp_cost[1][mv.as_mv.col];
|
||||
}
|
||||
|
||||
static int mvsad_err_cost(const MACROBLOCK *x, const int_mv mv, const MV *ref,
|
||||
int sad_per_bit) {
|
||||
const int_mv diff =
|
||||
pack_int_mv(mv.as_mv.row - ref->row, mv.as_mv.col - ref->col);
|
||||
return ROUND_POWER_OF_TWO(
|
||||
(unsigned)mv_cost(diff, x->nmvjointsadcost, x->nmvsadcost) * sad_per_bit,
|
||||
VP9_PROB_COST_SHIFT);
|
||||
}
|
||||
|
||||
/*****************************************************************************
|
||||
* This function utilizes 3 properties of the cost function lookup tables, *
|
||||
* constructed in using 'cal_nmvjointsadcost' and 'cal_nmvsadcosts' in *
|
||||
* vp9_encoder.c. *
|
||||
* For the joint cost: *
|
||||
* - mvjointsadcost[1] == mvjointsadcost[2] == mvjointsadcost[3] *
|
||||
* For the component costs: *
|
||||
* - For all i: mvsadcost[0][i] == mvsadcost[1][i] *
|
||||
* (Equal costs for both components) *
|
||||
* - For all i: mvsadcost[0][i] == mvsadcost[0][-i] *
|
||||
* (Cost function is even) *
|
||||
* If these do not hold, then this function cannot be used without *
|
||||
* modification, in which case you can revert to using the C implementation, *
|
||||
* which does not rely on these properties. *
|
||||
*****************************************************************************/
|
||||
int vp9_diamond_search_sad_neon(const MACROBLOCK *x,
|
||||
const search_site_config *cfg, MV *ref_mv,
|
||||
MV *best_mv, int search_param, int sad_per_bit,
|
||||
int *num00, const vp9_variance_fn_ptr_t *fn_ptr,
|
||||
const MV *center_mv) {
|
||||
static const uint32_t data[4] = { 0, 1, 2, 3 };
|
||||
const uint32x4_t v_idx_d = vld1q_u32((const uint32_t *)data);
|
||||
|
||||
const int32x4_t zero_s32 = vdupq_n_s32(0);
|
||||
const int_mv maxmv = pack_int_mv(x->mv_limits.row_max, x->mv_limits.col_max);
|
||||
const int16x8_t v_max_mv_w = vreinterpretq_s16_s32(vdupq_n_s32(maxmv.as_int));
|
||||
const int_mv minmv = pack_int_mv(x->mv_limits.row_min, x->mv_limits.col_min);
|
||||
const int16x8_t v_min_mv_w = vreinterpretq_s16_s32(vdupq_n_s32(minmv.as_int));
|
||||
|
||||
const int32x4_t v_spb_d = vdupq_n_s32(sad_per_bit);
|
||||
|
||||
const int32x4_t v_joint_cost_0_d = vdupq_n_s32(x->nmvjointsadcost[0]);
|
||||
const int32x4_t v_joint_cost_1_d = vdupq_n_s32(x->nmvjointsadcost[1]);
|
||||
|
||||
// search_param determines the length of the initial step and hence the number
|
||||
// of iterations.
|
||||
// 0 = initial step (MAX_FIRST_STEP) pel
|
||||
// 1 = (MAX_FIRST_STEP/2) pel,
|
||||
// 2 = (MAX_FIRST_STEP/4) pel...
|
||||
const MV *ss_mv = &cfg->ss_mv[cfg->searches_per_step * search_param];
|
||||
const intptr_t *ss_os = &cfg->ss_os[cfg->searches_per_step * search_param];
|
||||
const int tot_steps = cfg->total_steps - search_param;
|
||||
|
||||
const int_mv fcenter_mv =
|
||||
pack_int_mv(center_mv->row >> 3, center_mv->col >> 3);
|
||||
const int16x8_t vfcmv = vdupq_n_s16(fcenter_mv.as_int);
|
||||
|
||||
const int ref_row = clamp(ref_mv->row, minmv.as_mv.row, maxmv.as_mv.row);
|
||||
const int ref_col = clamp(ref_mv->col, minmv.as_mv.col, maxmv.as_mv.col);
|
||||
|
||||
int_mv bmv = pack_int_mv(ref_row, ref_col);
|
||||
int_mv new_bmv = bmv;
|
||||
int16x8_t v_bmv_w = vreinterpretq_s16_s32(vdupq_n_s32(bmv.as_int));
|
||||
|
||||
const int what_stride = x->plane[0].src.stride;
|
||||
const int in_what_stride = x->e_mbd.plane[0].pre[0].stride;
|
||||
const uint8_t *const what = x->plane[0].src.buf;
|
||||
const uint8_t *const in_what =
|
||||
x->e_mbd.plane[0].pre[0].buf + ref_row * in_what_stride + ref_col;
|
||||
|
||||
// Work out the start point for the search
|
||||
const uint8_t *best_address = in_what;
|
||||
const uint8_t *new_best_address = best_address;
|
||||
#if defined(__aarch64__)
|
||||
int64x2_t v_ba_q = vdupq_n_s64((intptr_t)best_address);
|
||||
#else
|
||||
int32x4_t v_ba_d = vdupq_n_s32((intptr_t)best_address);
|
||||
#endif
|
||||
unsigned int best_sad = INT_MAX;
|
||||
int i, j, step;
|
||||
|
||||
// Check the prerequisite cost function properties that are easy to check
|
||||
// in an assert. See the function-level documentation for details on all
|
||||
// prerequisites.
|
||||
assert(x->nmvjointsadcost[1] == x->nmvjointsadcost[2]);
|
||||
assert(x->nmvjointsadcost[1] == x->nmvjointsadcost[3]);
|
||||
|
||||
// Check the starting position
|
||||
best_sad = fn_ptr->sdf(what, what_stride, in_what, in_what_stride);
|
||||
best_sad += mvsad_err_cost(x, bmv, &fcenter_mv.as_mv, sad_per_bit);
|
||||
|
||||
*num00 = 0;
|
||||
|
||||
for (i = 0, step = 0; step < tot_steps; step++) {
|
||||
for (j = 0; j < cfg->searches_per_step; j += 4, i += 4) {
|
||||
int16x8_t v_diff_mv_w;
|
||||
int8x16_t v_inside_d;
|
||||
uint32x4_t v_outside_d;
|
||||
int32x4_t v_cost_d, v_sad_d;
|
||||
#if defined(__aarch64__)
|
||||
int64x2_t v_blocka[2];
|
||||
#else
|
||||
int32x4_t v_blocka[1];
|
||||
uint32x2_t horiz_max_0, horiz_max_1;
|
||||
#endif
|
||||
|
||||
uint32_t horiz_max;
|
||||
// Compute the candidate motion vectors
|
||||
const int16x8_t v_ss_mv_w = vld1q_s16((const int16_t *)&ss_mv[i]);
|
||||
const int16x8_t v_these_mv_w = vaddq_s16(v_bmv_w, v_ss_mv_w);
|
||||
// Clamp them to the search bounds
|
||||
int16x8_t v_these_mv_clamp_w = v_these_mv_w;
|
||||
v_these_mv_clamp_w = vminq_s16(v_these_mv_clamp_w, v_max_mv_w);
|
||||
v_these_mv_clamp_w = vmaxq_s16(v_these_mv_clamp_w, v_min_mv_w);
|
||||
// The ones that did not change are inside the search area
|
||||
v_inside_d = vreinterpretq_s8_u32(
|
||||
vceqq_s32(vreinterpretq_s32_s16(v_these_mv_clamp_w),
|
||||
vreinterpretq_s32_s16(v_these_mv_w)));
|
||||
|
||||
// If none of them are inside, then move on
|
||||
#if defined(__aarch64__)
|
||||
horiz_max = vmaxvq_u32(vreinterpretq_u32_s8(v_inside_d));
|
||||
#else
|
||||
horiz_max_0 = vmax_u32(vget_low_u32(vreinterpretq_u32_s8(v_inside_d)),
|
||||
vget_high_u32(vreinterpretq_u32_s8(v_inside_d)));
|
||||
horiz_max_1 = vpmax_u32(horiz_max_0, horiz_max_0);
|
||||
vst1_lane_u32(&horiz_max, horiz_max_1, 0);
|
||||
#endif
|
||||
if (LIKELY(horiz_max == 0)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// The inverse mask indicates which of the MVs are outside
|
||||
v_outside_d =
|
||||
vreinterpretq_u32_s8(veorq_s8(v_inside_d, vdupq_n_s8((int8_t)0xff)));
|
||||
// Shift right to keep the sign bit clear, we will use this later
|
||||
// to set the cost to the maximum value.
|
||||
v_outside_d = vshrq_n_u32(v_outside_d, 1);
|
||||
|
||||
// Compute the difference MV
|
||||
v_diff_mv_w = vsubq_s16(v_these_mv_clamp_w, vfcmv);
|
||||
// We utilise the fact that the cost function is even, and use the
|
||||
// absolute difference. This allows us to use unsigned indexes later
|
||||
// and reduces cache pressure somewhat as only a half of the table
|
||||
// is ever referenced.
|
||||
v_diff_mv_w = vabsq_s16(v_diff_mv_w);
|
||||
|
||||
// Compute the SIMD pointer offsets.
|
||||
{
|
||||
#if defined(__aarch64__) // sizeof(intptr_t) == 8
|
||||
// Load the offsets
|
||||
int64x2_t v_bo10_q = vld1q_s64((const int64_t *)&ss_os[i + 0]);
|
||||
int64x2_t v_bo32_q = vld1q_s64((const int64_t *)&ss_os[i + 2]);
|
||||
// Set the ones falling outside to zero
|
||||
v_bo10_q = vandq_s64(
|
||||
v_bo10_q,
|
||||
vmovl_s32(vget_low_s32(vreinterpretq_s32_s8(v_inside_d))));
|
||||
v_bo32_q = vandq_s64(
|
||||
v_bo32_q,
|
||||
vmovl_s32(vget_high_s32(vreinterpretq_s32_s8(v_inside_d))));
|
||||
// Compute the candidate addresses
|
||||
v_blocka[0] = vaddq_s64(v_ba_q, v_bo10_q);
|
||||
v_blocka[1] = vaddq_s64(v_ba_q, v_bo32_q);
|
||||
#else // sizeof(intptr_t) == 4
|
||||
int32x4_t v_bo_d = vld1q_s32((const int32_t *)&ss_os[i]);
|
||||
v_bo_d = vandq_s32(v_bo_d, vreinterpretq_s32_s8(v_inside_d));
|
||||
v_blocka[0] = vaddq_s32(v_ba_d, v_bo_d);
|
||||
#endif
|
||||
}
|
||||
|
||||
fn_ptr->sdx4df(what, what_stride, (const uint8_t **)&v_blocka[0],
|
||||
in_what_stride, (uint32_t *)&v_sad_d);
|
||||
|
||||
// Look up the component cost of the residual motion vector
|
||||
{
|
||||
uint32_t cost[4];
|
||||
int16_t __attribute__((aligned(16))) rowcol[8];
|
||||
vst1q_s16(rowcol, v_diff_mv_w);
|
||||
|
||||
// Note: This is a use case for gather instruction
|
||||
cost[0] = x->nmvsadcost[0][rowcol[0]] + x->nmvsadcost[0][rowcol[1]];
|
||||
cost[1] = x->nmvsadcost[0][rowcol[2]] + x->nmvsadcost[0][rowcol[3]];
|
||||
cost[2] = x->nmvsadcost[0][rowcol[4]] + x->nmvsadcost[0][rowcol[5]];
|
||||
cost[3] = x->nmvsadcost[0][rowcol[6]] + x->nmvsadcost[0][rowcol[7]];
|
||||
|
||||
v_cost_d = vld1q_s32((int32_t *)cost);
|
||||
}
|
||||
|
||||
// Now add in the joint cost
|
||||
{
|
||||
const uint32x4_t v_sel_d =
|
||||
vceqq_s32(vreinterpretq_s32_s16(v_diff_mv_w), zero_s32);
|
||||
const int32x4_t v_joint_cost_d = vreinterpretq_s32_u8(
|
||||
vbslq_u8(vreinterpretq_u8_u32(v_sel_d),
|
||||
vreinterpretq_u8_s32(v_joint_cost_0_d),
|
||||
vreinterpretq_u8_s32(v_joint_cost_1_d)));
|
||||
v_cost_d = vaddq_s32(v_cost_d, v_joint_cost_d);
|
||||
}
|
||||
|
||||
// Multiply by sad_per_bit
|
||||
v_cost_d = vmulq_s32(v_cost_d, v_spb_d);
|
||||
// ROUND_POWER_OF_TWO(v_cost_d, VP9_PROB_COST_SHIFT)
|
||||
v_cost_d =
|
||||
vaddq_s32(v_cost_d, vdupq_n_s32(1 << (VP9_PROB_COST_SHIFT - 1)));
|
||||
v_cost_d = vshrq_n_s32(v_cost_d, VP9_PROB_COST_SHIFT);
|
||||
// Add the cost to the sad
|
||||
v_sad_d = vaddq_s32(v_sad_d, v_cost_d);
|
||||
|
||||
// Make the motion vectors outside the search area have max cost
|
||||
// by or'ing in the comparison mask, this way the minimum search won't
|
||||
// pick them.
|
||||
v_sad_d = vorrq_s32(v_sad_d, vreinterpretq_s32_u32(v_outside_d));
|
||||
|
||||
// Find the minimum value and index horizontally in v_sad_d
|
||||
{
|
||||
uint32_t local_best_sad;
|
||||
#if defined(__aarch64__)
|
||||
local_best_sad = vminvq_u32(vreinterpretq_u32_s32(v_sad_d));
|
||||
#else
|
||||
uint32x2_t horiz_min_0 =
|
||||
vmin_u32(vget_low_u32(vreinterpretq_u32_s32(v_sad_d)),
|
||||
vget_high_u32(vreinterpretq_u32_s32(v_sad_d)));
|
||||
uint32x2_t horiz_min_1 = vpmin_u32(horiz_min_0, horiz_min_0);
|
||||
vst1_lane_u32(&local_best_sad, horiz_min_1, 0);
|
||||
#endif
|
||||
|
||||
// Update the global minimum if the local minimum is smaller
|
||||
if (LIKELY(local_best_sad < best_sad)) {
|
||||
#if defined(__GNUC__) && __GNUC__ >= 4 && !defined(__clang__)
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
|
||||
#endif
|
||||
uint32_t local_best_idx;
|
||||
const uint32x4_t v_sel_d =
|
||||
vceqq_s32(v_sad_d, vdupq_n_s32(local_best_sad));
|
||||
uint32x4_t v_mask_d = vandq_u32(v_sel_d, v_idx_d);
|
||||
v_mask_d = vbslq_u32(v_sel_d, v_mask_d, vdupq_n_u32(0xffffffff));
|
||||
|
||||
#if defined(__aarch64__)
|
||||
local_best_idx = vminvq_u32(v_mask_d);
|
||||
#else
|
||||
horiz_min_0 =
|
||||
vmin_u32(vget_low_u32(v_mask_d), vget_high_u32(v_mask_d));
|
||||
horiz_min_1 = vpmin_u32(horiz_min_0, horiz_min_0);
|
||||
vst1_lane_u32(&local_best_idx, horiz_min_1, 0);
|
||||
#endif
|
||||
|
||||
new_bmv = ((const int_mv *)&v_these_mv_w)[local_best_idx];
|
||||
#if defined(__GNUC__) && __GNUC__ >= 4 && !defined(__clang__)
|
||||
#pragma GCC diagnostic pop
|
||||
#endif
|
||||
new_best_address = ((const uint8_t **)v_blocka)[local_best_idx];
|
||||
|
||||
best_sad = local_best_sad;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bmv = new_bmv;
|
||||
best_address = new_best_address;
|
||||
|
||||
v_bmv_w = vreinterpretq_s16_s32(vdupq_n_s32(bmv.as_int));
|
||||
#if defined(__aarch64__)
|
||||
v_ba_q = vdupq_n_s64((intptr_t)best_address);
|
||||
#else
|
||||
v_ba_d = vdupq_n_s32((intptr_t)best_address);
|
||||
#endif
|
||||
|
||||
if (UNLIKELY(best_address == in_what)) {
|
||||
(*num00)++;
|
||||
}
|
||||
}
|
||||
|
||||
*best_mv = bmv.as_mv;
|
||||
return best_sad;
|
||||
}
|
||||
@@ -113,6 +113,7 @@ VP9_CX_SRCS-$(HAVE_SSE4_1) += encoder/x86/temporal_filter_constants.h
|
||||
VP9_CX_SRCS-$(HAVE_SSE2) += encoder/x86/vp9_quantize_sse2.c
|
||||
VP9_CX_SRCS-$(HAVE_AVX2) += encoder/x86/vp9_quantize_avx2.c
|
||||
VP9_CX_SRCS-$(HAVE_AVX) += encoder/x86/vp9_diamond_search_sad_avx.c
|
||||
VP9_CX_SRCS-$(HAVE_NEON) += encoder/arm/neon/vp9_diamond_search_sad_neon.c
|
||||
ifeq ($(CONFIG_VP9_HIGHBITDEPTH),yes)
|
||||
VP9_CX_SRCS-$(HAVE_SSE2) += encoder/x86/vp9_highbd_block_error_intrin_sse2.c
|
||||
VP9_CX_SRCS-$(HAVE_SSE4_1) += encoder/x86/highbd_temporal_filter_sse4.c
|
||||
|
||||
Reference in New Issue
Block a user