AOMedia AV1 Codec: av1/encoder/temporal_filter.h Source File

| AOMedia AV1 Codec

temporal_filter.h
1/*
2 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
3 *
4 * This source code is subject to the terms of the BSD 2 Clause License and
5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6 * was not distributed with this source code in the LICENSE file, you can
7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8 * Media Patent License 1.0 was not distributed with this source code in the
9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10 */
11
12#ifndef AOM_AV1_ENCODER_TEMPORAL_FILTER_H_
13#define AOM_AV1_ENCODER_TEMPORAL_FILTER_H_
14
15#include <stdbool.h>
16
17#ifdef __cplusplus
18extern "C" {
19#endif
21struct AV1_COMP;
22struct AV1EncoderConfig;
23struct ThreadData;
24// TODO(any): These two variables are only used in avx2, sse2, sse4
25// implementations, where the block size is still hard coded. This should be
26// fixed to align with the c implementation.
27#define BH 32
28#define BW 32
29
30// Block size used in temporal filtering.
31#define TF_BLOCK_SIZE BLOCK_32X32
32
33// Window size for temporal filtering.
34#define TF_WINDOW_LENGTH 5
35
36// Hyper-parameters used to compute filtering weight. These hyper-parameters can
37// be tuned for a better performance.
38// 0. A scale factor used in temporal filtering to raise the filter weight from
39// `double` with range [0, 1] to `int` with range [0, 1000].
40#define TF_WEIGHT_SCALE 1000
41// 1. Weight factor used to balance the weighted-average between window error
42// and block error. The weight is for window error while the weight for block
43// error is always set as 1.
44#define TF_WINDOW_BLOCK_BALANCE_WEIGHT 5
45// 2. Threshold for using q to adjust the filtering weight. Concretely, when
46// using a small q (high bitrate), we would like to reduce the filtering
47// strength such that more detailed information can be preserved. Hence, when
48// q is smaller than this threshold, we will adjust the filtering weight
49// based on the q-value.
50#define TF_Q_DECAY_THRESHOLD 20
51// 3. Normalization factor used to normalize the motion search error. Since the
52// motion search error can be large and uncontrollable, we will simply
53// normalize it before using it to compute the filtering weight.
54#define TF_SEARCH_ERROR_NORM_WEIGHT 20
55// 4. Threshold for using `arnr_strength` to adjust the filtering strength.
56// Concretely, users can use `arnr_strength` arguments to control the
57// strength of temporal filtering. When `arnr_strength` is small enough (
58// i.e., smaller than this threshold), we will adjust the filtering weight
59// based on the strength value.
60#define TF_STRENGTH_THRESHOLD 4
61// 5. Threshold for using motion search distance to adjust the filtering weight.
62// Concretely, larger motion search vector leads to a higher probability of
63// unreliable search. Hence, we would like to reduce the filtering strength
64// when the distance is large enough. Considering that the distance actually
65// relies on the frame size, this threshold is also a resolution-based
66// threshold. Taking 720p videos as an instance, if this field equals to 0.1,
67// then the actual threshold will be 720 * 0.1 = 72. Similarly, the threshold
68// for 360p videos will be 360 * 0.1 = 36.
69#define TF_SEARCH_DISTANCE_THRESHOLD 0.1
70// 6. Threshold to identify if the q is in a relative high range.
71// Above this cutoff q, a stronger filtering is applied.
72// For a high q, the quantization throws away more information, and thus a
73// stronger filtering is less likely to distort the encoded quality, while a
74// stronger filtering could reduce bit rates.
75// Ror a low q, more details are expected to be retained. Filtering is thus
76// more conservative.
77#define TF_QINDEX_CUTOFF 128
78
79#define NOISE_ESTIMATION_EDGE_THRESHOLD 50
80
81// Sum and SSE source vs filtered frame difference returned by
82// temporal filter.
83typedef struct {
84 int64_t sum;
85 int64_t sse;
86} FRAME_DIFF;
87
93 typedef struct {
97 YV12_BUFFER_CONFIG *frames[MAX_LAG_BUFFERS];
101 int num_frames;
102
106 YV12_BUFFER_CONFIG *output_frame;
107
111 int filter_frame_idx;
115 int compute_frame_diff;
119 struct scale_factors sf;
123 double noise_levels[MAX_MB_PLANE];
127 int num_pels;
131 int mb_rows;
135 int mb_cols;
139 int is_highbitdepth;
143 int q_factor;
144} TemporalFilterCtx;
145
151#define TF_INFO_BUF_COUNT 2
152
156 typedef struct TEMPORAL_FILTER_INFO {
162 int is_temporal_filter_on;
167 YV12_BUFFER_CONFIG tf_buf[TF_INFO_BUF_COUNT];
168
175 YV12_BUFFER_CONFIG tf_buf_second_arf;
179 FRAME_DIFF frame_diff[TF_INFO_BUF_COUNT];
183 int tf_buf_gf_index[TF_INFO_BUF_COUNT];
187 int tf_buf_display_index_offset[TF_INFO_BUF_COUNT];
191 int tf_buf_valid[TF_INFO_BUF_COUNT];
192} TEMPORAL_FILTER_INFO;
193
199int av1_is_temporal_filter_on(const struct AV1EncoderConfig *oxcf);
200
205void av1_tf_info_alloc(TEMPORAL_FILTER_INFO *tf_info,
206 const struct AV1_COMP *cpi);
207
211void av1_tf_info_free(TEMPORAL_FILTER_INFO *tf_info);
212
216void av1_tf_info_reset(TEMPORAL_FILTER_INFO *tf_info);
217
223void av1_tf_info_filtering(TEMPORAL_FILTER_INFO *tf_info, struct AV1_COMP *cpi,
224 const GF_GROUP *gf_group);
225
232 YV12_BUFFER_CONFIG *av1_tf_info_get_filtered_buf(TEMPORAL_FILTER_INFO *tf_info,
233 int gf_index,
234 FRAME_DIFF *frame_diff);
235
238// Data related to temporal filtering.
239typedef struct {
240 // Source vs filtered frame error.
241 FRAME_DIFF diff;
242 // Pointer to temporary block info used to store state in temporal filtering
243 // process.
244 MB_MODE_INFO *tmp_mbmi;
245 // Pointer to accumulator buffer used in temporal filtering process.
246 uint32_t *accum;
247 // Pointer to count buffer used in temporal filtering process.
248 uint16_t *count;
249 // Pointer to predictor used in temporal filtering process.
250 uint8_t *pred;
251} TemporalFilterData;
252
253// Data related to temporal filter multi-thread synchronization.
254typedef struct {
255#if CONFIG_MULTITHREAD
256 // Mutex lock used for dispatching jobs.
257 pthread_mutex_t *mutex_;
258#endif // CONFIG_MULTITHREAD
259 // Next temporal filter block row to be filtered.
260 int next_tf_row;
261} AV1TemporalFilterSync;
262
263// Estimates noise level from a given frame using a single plane (Y, U, or V).
264// This is an adaptation of the mehtod in the following paper:
265// Shen-Chuan Tai, Shih-Ming Yang, "A fast method for image noise
266// estimation using Laplacian operator and adaptive edge detection",
267// Proc. 3rd International Symposium on Communications, Control and
268// Signal Processing, 2008, St Julians, Malta.
269// Inputs:
270// frame: Pointer to the frame to estimate noise level from.
271// plane: Index of the plane used for noise estimation. Commonly, 0 for
272// Y-plane, 1 for U-plane, and 2 for V-plane.
273// bit_depth: Actual bit-depth instead of the encoding bit-depth of the frame.
274// Returns:
275// The estimated noise, or -1.0 if there are too few smooth pixels.
276double av1_estimate_noise_from_single_plane(const YV12_BUFFER_CONFIG *frame,
277 const int plane,
278 const int bit_depth,
279 const int edge_thresh);
293void av1_tf_do_filtering_row(struct AV1_COMP *cpi, struct ThreadData *td,
294 int mb_row);
295
320 void av1_temporal_filter(struct AV1_COMP *cpi,
321 const int filter_frame_lookahead_idx,
322 int gf_frame_index, FRAME_DIFF *frame_diff,
323 YV12_BUFFER_CONFIG *output_frame);
324
339 int av1_check_show_filtered_frame(const YV12_BUFFER_CONFIG *frame,
340 const FRAME_DIFF *frame_diff, int q_index,
341 aom_bit_depth_t bit_depth);
342
344// Helper function to get `q` used for encoding.
345int av1_get_q(const struct AV1_COMP *cpi);
346
347// Allocates memory for members of TemporalFilterData.
348// Inputs:
349// tf_data: Pointer to the structure containing temporal filter related data.
350// num_pels: Number of pixels in the block across all planes.
351// is_high_bitdepth: Whether the frame is high-bitdepth or not.
352// Returns:
353// Nothing will be returned. But the contents of tf_data will be modified.
354static AOM_INLINE bool tf_alloc_and_reset_data(TemporalFilterData *tf_data,
355 int num_pels,
356 int is_high_bitdepth) {
357 tf_data->tmp_mbmi = (MB_MODE_INFO *)malloc(sizeof(*tf_data->tmp_mbmi));
358 memset(tf_data->tmp_mbmi, 0, sizeof(*tf_data->tmp_mbmi));
359 tf_data->accum =
360 (uint32_t *)aom_memalign(16, num_pels * sizeof(*tf_data->accum));
361 tf_data->count =
362 (uint16_t *)aom_memalign(16, num_pels * sizeof(*tf_data->count));
363 memset(&tf_data->diff, 0, sizeof(tf_data->diff));
364 if (is_high_bitdepth)
365 tf_data->pred = CONVERT_TO_BYTEPTR(
366 aom_memalign(32, num_pels * 2 * sizeof(*tf_data->pred)));
367 else
368 tf_data->pred =
369 (uint8_t *)aom_memalign(32, num_pels * sizeof(*tf_data->pred));
370 if (!(tf_data->accum && tf_data->count && tf_data->pred)) {
371 aom_free(tf_data->accum);
372 aom_free(tf_data->count);
373 aom_free(tf_data->pred);
374 return false;
375 }
376 return true;
377}
378
379// Setup macroblockd params for temporal filtering process.
380// Inputs:
381// mbd: Pointer to the block for filtering.
382// tf_data: Pointer to the structure containing temporal filter related data.
383// scale: Scaling factor.
384// Returns:
385// Nothing will be returned. Contents of mbd will be modified.
386static AOM_INLINE void tf_setup_macroblockd(MACROBLOCKD *mbd,
387 TemporalFilterData *tf_data,
388 const struct scale_factors *scale) {
389 mbd->block_ref_scale_factors[0] = scale;
390 mbd->block_ref_scale_factors[1] = scale;
391 mbd->mi = &tf_data->tmp_mbmi;
392 mbd->mi[0]->motion_mode = SIMPLE_TRANSLATION;
393}
394
395// Deallocates the memory allocated for members of TemporalFilterData.
396// Inputs:
397// tf_data: Pointer to the structure containing temporal filter related data.
398// is_high_bitdepth: Whether the frame is high-bitdepth or not.
399// Returns:
400// Nothing will be returned.
401static AOM_INLINE void tf_dealloc_data(TemporalFilterData *tf_data,
402 int is_high_bitdepth) {
403 if (is_high_bitdepth)
404 tf_data->pred = (uint8_t *)CONVERT_TO_SHORTPTR(tf_data->pred);
405 free(tf_data->tmp_mbmi);
406 aom_free(tf_data->accum);
407 aom_free(tf_data->count);
408 aom_free(tf_data->pred);
409}
410
411// Saves the state prior to temporal filter process.
412// Inputs:
413// mbd: Pointer to the block for filtering.
414// input_mbmi: Backup block info to save input state.
415// input_buffer: Backup buffer pointer to save input state.
416// num_planes: Number of planes.
417// Returns:
418// Nothing will be returned. Contents of input_mbmi and input_buffer will be
419// modified.
420static INLINE void tf_save_state(MACROBLOCKD *mbd, MB_MODE_INFO ***input_mbmi,
421 uint8_t **input_buffer, int num_planes) {
422 for (int i = 0; i < num_planes; i++) {
423 input_buffer[i] = mbd->plane[i].pre[0].buf;
424 }
425 *input_mbmi = mbd->mi;
426}
427
428// Restores the initial state after temporal filter process.
429// Inputs:
430// mbd: Pointer to the block for filtering.
431// input_mbmi: Backup block info from where input state is restored.
432// input_buffer: Backup buffer pointer from where input state is restored.
433// num_planes: Number of planes.
434// Returns:
435// Nothing will be returned. Contents of mbd will be modified.
436static INLINE void tf_restore_state(MACROBLOCKD *mbd, MB_MODE_INFO **input_mbmi,
437 uint8_t **input_buffer, int num_planes) {
438 for (int i = 0; i < num_planes; i++) {
439 mbd->plane[i].pre[0].buf = input_buffer[i];
440 }
441 mbd->mi = input_mbmi;
442}
443
445#ifdef __cplusplus
446} // extern "C"
447#endif
448
449#endif // AOM_AV1_ENCODER_TEMPORAL_FILTER_H_
aom_bit_depth_t
enum aom_bit_depth aom_bit_depth_t
Bit depth for codecThis enumeration determines the bit depth of the codec.
av1_check_show_filtered_frame
int av1_check_show_filtered_frame(const YV12_BUFFER_CONFIG *frame, const FRAME_DIFF *frame_diff, int q_index, aom_bit_depth_t bit_depth)
Check whether a filtered frame can be show directly.
av1_temporal_filter
void av1_temporal_filter(struct AV1_COMP *cpi, const int filter_frame_lookahead_idx, int gf_frame_index, FRAME_DIFF *frame_diff, YV12_BUFFER_CONFIG *output_frame)
Performs temporal filtering if needed on a source frame. For example to create a filtered alternate r...
av1_tf_do_filtering_row
void av1_tf_do_filtering_row(struct AV1_COMP *cpi, struct ThreadData *td, int mb_row)
Does temporal filter for a given macroblock row.
Definition: temporal_filter.c:792
AV1EncoderConfig
Main encoder configuration data structure.
Definition: encoder.h:915
AV1_COMP
Top level encoder structure.
Definition: encoder.h:2759
GF_GROUP
Data related to the current GF/ARF group and the individual frames within the group.
Definition: firstpass.h:344
MB_MODE_INFO
Stores the prediction/txfm mode of the current coding block.
Definition: blockd.h:222
MB_MODE_INFO::motion_mode
MOTION_MODE motion_mode
The motion mode used by the inter prediction.
Definition: blockd.h:250
TEMPORAL_FILTER_INFO
Temporal filter info for a gop.
Definition: temporal_filter.h:156
TEMPORAL_FILTER_INFO::tf_buf_display_index_offset
int tf_buf_display_index_offset[2]
Definition: temporal_filter.h:187
TEMPORAL_FILTER_INFO::tf_buf
YV12_BUFFER_CONFIG tf_buf[2]
Definition: temporal_filter.h:167
TEMPORAL_FILTER_INFO::tf_buf_gf_index
int tf_buf_gf_index[2]
Definition: temporal_filter.h:183
TEMPORAL_FILTER_INFO::frame_diff
FRAME_DIFF frame_diff[2]
Definition: temporal_filter.h:179
TEMPORAL_FILTER_INFO::is_temporal_filter_on
int is_temporal_filter_on
Definition: temporal_filter.h:162
TEMPORAL_FILTER_INFO::tf_buf_valid
int tf_buf_valid[2]
Definition: temporal_filter.h:191
TEMPORAL_FILTER_INFO::tf_buf_second_arf
YV12_BUFFER_CONFIG tf_buf_second_arf
Definition: temporal_filter.h:175
TemporalFilterCtx
Parameters related to temporal filtering.
Definition: temporal_filter.h:93
TemporalFilterCtx::output_frame
YV12_BUFFER_CONFIG * output_frame
Definition: temporal_filter.h:106
TemporalFilterCtx::q_factor
int q_factor
Definition: temporal_filter.h:143
TemporalFilterCtx::num_pels
int num_pels
Definition: temporal_filter.h:127
TemporalFilterCtx::num_frames
int num_frames
Definition: temporal_filter.h:101
TemporalFilterCtx::compute_frame_diff
int compute_frame_diff
Definition: temporal_filter.h:115
TemporalFilterCtx::mb_rows
int mb_rows
Definition: temporal_filter.h:131
TemporalFilterCtx::mb_cols
int mb_cols
Definition: temporal_filter.h:135
TemporalFilterCtx::is_highbitdepth
int is_highbitdepth
Definition: temporal_filter.h:139
TemporalFilterCtx::filter_frame_idx
int filter_frame_idx
Definition: temporal_filter.h:111
macroblockd
Variables related to current coding block.
Definition: blockd.h:574
macroblockd::plane
struct macroblockd_plane plane[3]
Definition: blockd.h:610
macroblockd::block_ref_scale_factors
const struct scale_factors * block_ref_scale_factors[2]
Definition: blockd.h:691
macroblockd::mi
MB_MODE_INFO ** mi
Definition: blockd.h:621
yv12_buffer_config
YV12 frame buffer data structure.
Definition: yv12config.h:39