AOMedia AV1 Codec
tpl_model.h
1 /*
2  * Copyright (c) 2019, Alliance for Open Media. All rights reserved
3  *
4  * This source code is subject to the terms of the BSD 2 Clause License and
5  * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6  * was not distributed with this source code in the LICENSE file, you can
7  * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8  * Media Patent License 1.0 was not distributed with this source code in the
9  * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10  */
11 
12 #ifndef AOM_AV1_ENCODER_TPL_MODEL_H_
13 #define AOM_AV1_ENCODER_TPL_MODEL_H_
14 
15 #ifdef __cplusplus
16 extern "C" {
17 #endif
18 
21 struct AV1_PRIMARY;
22 struct AV1_COMP;
23 struct AV1_SEQ_CODING_TOOLS;
24 struct EncodeFrameParams;
25 struct EncodeFrameInput;
26 struct GF_GROUP;
27 
28 #include "config/aom_config.h"
29 
30 #include "aom_scale/yv12config.h"
31 
32 #include "av1/common/mv.h"
33 #include "av1/common/scale.h"
34 #include "av1/encoder/block.h"
35 #include "av1/encoder/lookahead.h"
36 #include "av1/encoder/ratectrl.h"
37 
38 static INLINE BLOCK_SIZE convert_length_to_bsize(int length) {
39  switch (length) {
40  case 64: return BLOCK_64X64;
41  case 32: return BLOCK_32X32;
42  case 16: return BLOCK_16X16;
43  case 8: return BLOCK_8X8;
44  case 4: return BLOCK_4X4;
45  default:
46  assert(0 && "Invalid block size for tpl model");
47  return BLOCK_16X16;
48  }
49 }
50 
51 typedef struct AV1TplRowMultiThreadSync {
52 #if CONFIG_MULTITHREAD
53  // Synchronization objects for top-right dependency.
54  pthread_mutex_t *mutex_;
55  pthread_cond_t *cond_;
56 #endif
57  // Buffer to store the macroblock whose encoding is complete.
58  // num_finished_cols[i] stores the number of macroblocks which finished
59  // encoding in the ith macroblock row.
60  int *num_finished_cols;
61  // Number of extra macroblocks of the top row to be complete for encoding
62  // of the current macroblock to start. A value of 1 indicates top-right
63  // dependency.
64  int sync_range;
65  // Number of macroblock rows.
66  int rows;
67  // Number of threads processing the current tile.
68  int num_threads_working;
69 } AV1TplRowMultiThreadSync;
70 
71 typedef struct AV1TplRowMultiThreadInfo {
72  // Row synchronization related function pointers.
73  void (*sync_read_ptr)(AV1TplRowMultiThreadSync *tpl_mt_sync, int r, int c);
74  void (*sync_write_ptr)(AV1TplRowMultiThreadSync *tpl_mt_sync, int r, int c,
75  int cols);
76 } AV1TplRowMultiThreadInfo;
77 
78 // TODO(jingning): This needs to be cleaned up next.
79 
80 // TPL stats buffers are prepared for every frame in the GOP,
81 // including (internal) overlays and (internal) arfs.
82 // In addition, frames in the lookahead that are outside of the GOP
83 // are also used.
84 // Thus it should use
85 // (gop_length) + (# overlays) + (MAX_LAG_BUFFERS - gop_len) =
86 // MAX_LAG_BUFFERS + (# overlays)
87 // 2 * MAX_LAG_BUFFERS is therefore a safe estimate.
88 // TODO(bohanli): test setting it to 1.5 * MAX_LAG_BUFFER
89 #define MAX_TPL_FRAME_IDX (2 * MAX_LAG_BUFFERS)
90 // The first REF_FRAMES + 1 buffers are reserved.
91 // tpl_data->tpl_frame starts after REF_FRAMES + 1
92 #define MAX_LENGTH_TPL_FRAME_STATS (MAX_TPL_FRAME_IDX + REF_FRAMES + 1)
93 #define TPL_DEP_COST_SCALE_LOG2 4
94 
95 #define TPL_EPSILON 0.0000001
96 
97 typedef struct TplTxfmStats {
98  double abs_coeff_sum[256]; // Assume we are using 16x16 transform block
99  int txfm_block_count;
100  int coeff_num;
101 } TplTxfmStats;
102 
103 typedef struct TplDepStats {
104  int64_t intra_cost;
105  int64_t inter_cost;
106  int64_t srcrf_dist;
107  int64_t recrf_dist;
108  int64_t cmp_recrf_dist[2];
109  int64_t srcrf_rate;
110  int64_t recrf_rate;
111  int64_t srcrf_sse;
112  int64_t cmp_recrf_rate[2];
113  int64_t mc_dep_rate;
114  int64_t mc_dep_dist;
115  int_mv mv[INTER_REFS_PER_FRAME];
116  int ref_frame_index[2];
117  int64_t pred_error[INTER_REFS_PER_FRAME];
118 } TplDepStats;
119 
120 typedef struct TplDepFrame {
121  uint8_t is_valid;
122  TplDepStats *tpl_stats_ptr;
123  const YV12_BUFFER_CONFIG *gf_picture;
124  YV12_BUFFER_CONFIG *rec_picture;
125  int ref_map_index[REF_FRAMES];
126  int stride;
127  int width;
128  int height;
129  int mi_rows;
130  int mi_cols;
131  int base_rdmult;
132  uint32_t frame_display_index;
133 } TplDepFrame;
134 
139 typedef struct TplParams {
143  int ready;
144 
149 
153  uint8_t tpl_bsize_1d;
154 
160  TplDepFrame tpl_stats_buffer[MAX_LENGTH_TPL_FRAME_STATS];
161 
167  TplDepStats *tpl_stats_pool[MAX_LAG_BUFFERS];
168 
173  TplTxfmStats txfm_stats_list[MAX_LENGTH_TPL_FRAME_STATS];
174 
180 
184  TplDepFrame *tpl_frame;
185 
189  struct scale_factors sf;
190 
195 
201  const YV12_BUFFER_CONFIG *src_ref_frame[INTER_REFS_PER_FRAME];
202 
208  const YV12_BUFFER_CONFIG *ref_frame[INTER_REFS_PER_FRAME];
209 
214  AV1TplRowMultiThreadSync tpl_mt_sync;
215 
220 
221 #if CONFIG_BITRATE_ACCURACY
222  /*
223  * Estimated and actual GOP bitrate.
224  */
225  double estimated_gop_bitrate;
226  double actual_gop_bitrate;
227 #endif
228 } TplParams;
229 
230 #if CONFIG_BITRATE_ACCURACY
231 
235 typedef struct {
236  double keyframe_bitrate;
237  double total_bit_budget; // The total bit budget of the entire video
238  int show_frame_count; // Number of show frames in the entire video
239 
240  int gop_showframe_count; // The number of show frames in the current gop
241  double gop_bit_budget; // The bitbudget for the current gop
242  double scale_factors[FRAME_UPDATE_TYPES]; // Scale factors to improve the
243  // budget estimation
244  double mv_scale_factors[FRAME_UPDATE_TYPES]; // Scale factors to improve
245  // MV entropy estimation
246 
247  // === Below this line are GOP related data that will be updated per GOP ===
248  int base_q_index; // Stores the base q index.
249  int q_index_list_ready;
250  int q_index_list[MAX_LENGTH_TPL_FRAME_STATS]; // q indices for the current
251  // GOP
252  // Arrays to store frame level bitrate accuracy data.
253  double estimated_bitrate_byframe[MAX_LENGTH_TPL_FRAME_STATS];
254  double estimated_mv_bitrate_byframe[MAX_LENGTH_TPL_FRAME_STATS];
255  int actual_bitrate_byframe[MAX_LENGTH_TPL_FRAME_STATS];
256  int actual_mv_bitrate_byframe[MAX_LENGTH_TPL_FRAME_STATS];
257  int actual_coeff_bitrate_byframe[MAX_LENGTH_TPL_FRAME_STATS];
258 
259  // Array to store qstep_ratio for each frame in a GOP
260  double qstep_ratio_list[MAX_LENGTH_TPL_FRAME_STATS];
261 } VBR_RATECTRL_INFO;
262 
263 static INLINE void vbr_rc_reset_gop_data(VBR_RATECTRL_INFO *vbr_rc_info) {
264  vbr_rc_info->q_index_list_ready = 0;
265  av1_zero(vbr_rc_info->q_index_list);
266  av1_zero(vbr_rc_info->estimated_bitrate_byframe);
267  av1_zero(vbr_rc_info->estimated_mv_bitrate_byframe);
268  av1_zero(vbr_rc_info->actual_bitrate_byframe);
269  av1_zero(vbr_rc_info->actual_mv_bitrate_byframe);
270  av1_zero(vbr_rc_info->actual_coeff_bitrate_byframe);
271 }
272 
273 static INLINE void vbr_rc_init(VBR_RATECTRL_INFO *vbr_rc_info,
274  double total_bit_budget, int show_frame_count) {
275  vbr_rc_info->total_bit_budget = total_bit_budget;
276  vbr_rc_info->show_frame_count = show_frame_count;
277  vbr_rc_info->keyframe_bitrate = 0;
278  const double scale_factors[FRAME_UPDATE_TYPES] = { 0.94559, 0.12040, 1,
279  1.10199, 1, 1,
280  0.16393 };
281  const double mv_scale_factors[FRAME_UPDATE_TYPES] = { 3, 3, 3, 3, 3, 3, 3 };
282  memcpy(vbr_rc_info->scale_factors, scale_factors,
283  sizeof(scale_factors[0]) * FRAME_UPDATE_TYPES);
284  memcpy(vbr_rc_info->mv_scale_factors, mv_scale_factors,
285  sizeof(mv_scale_factors[0]) * FRAME_UPDATE_TYPES);
286 
287  vbr_rc_reset_gop_data(vbr_rc_info);
288 }
289 
290 static INLINE void vbr_rc_set_gop_bit_budget(VBR_RATECTRL_INFO *vbr_rc_info,
291  int gop_showframe_count) {
292  vbr_rc_info->gop_showframe_count = gop_showframe_count;
293  vbr_rc_info->gop_bit_budget = vbr_rc_info->total_bit_budget *
294  gop_showframe_count /
295  vbr_rc_info->show_frame_count;
296 }
297 
298 static INLINE void vbr_rc_set_keyframe_bitrate(VBR_RATECTRL_INFO *vbr_rc_info,
299  double keyframe_bitrate) {
300  vbr_rc_info->keyframe_bitrate = keyframe_bitrate;
301 }
302 
303 static INLINE void vbr_rc_info_log(const VBR_RATECTRL_INFO *vbr_rc_info,
304  int gf_frame_index, int gf_group_size,
305  FRAME_UPDATE_TYPE *update_type) {
306  // Add +2 here because this is the last frame this method is called at.
307  if (gf_frame_index + 2 >= gf_group_size) {
308  printf(
309  "\ni, \test_bitrate, \test_mv_bitrate, \tact_bitrate, "
310  "\tact_mv_bitrate, \tact_coeff_bitrate, \tq, \tupdate_type\n");
311  for (int i = 0; i < gf_group_size; i++) {
312  printf("%d, \t%f, \t%f, \t%d, \t%d, \t%d, \t%d, \t%d\n", i,
313  vbr_rc_info->estimated_bitrate_byframe[i],
314  vbr_rc_info->estimated_mv_bitrate_byframe[i],
315  vbr_rc_info->actual_bitrate_byframe[i],
316  vbr_rc_info->actual_mv_bitrate_byframe[i],
317  vbr_rc_info->actual_coeff_bitrate_byframe[i],
318  vbr_rc_info->q_index_list[i], update_type[i]);
319  }
320  }
321 }
322 
323 #endif // CONFIG_BITRATE_ACCURACY
324 
325 #if CONFIG_RD_COMMAND
326 typedef enum {
327  RD_OPTION_NONE,
328  RD_OPTION_SET_Q,
329  RD_OPTION_SET_Q_RDMULT
330 } RD_OPTION;
331 
332 typedef struct RD_COMMAND {
333  RD_OPTION option_ls[MAX_LENGTH_TPL_FRAME_STATS];
334  int q_index_ls[MAX_LENGTH_TPL_FRAME_STATS];
335  int rdmult_ls[MAX_LENGTH_TPL_FRAME_STATS];
336  int frame_count;
337  int frame_index;
338 } RD_COMMAND;
339 
340 void av1_read_rd_command(const char *filepath, RD_COMMAND *rd_command);
341 #endif // CONFIG_RD_COMMAND
342 
351 void av1_setup_tpl_buffers(struct AV1_PRIMARY *const ppi,
352  CommonModeInfoParams *const mi_params, int width,
353  int height, int byte_alignment, int lag_in_frames);
354 
366 int av1_tpl_setup_stats(struct AV1_COMP *cpi, int gop_eval,
367  const struct EncodeFrameParams *const frame_params);
368 
371 void av1_tpl_preload_rc_estimate(
372  struct AV1_COMP *cpi, const struct EncodeFrameParams *const frame_params);
373 
374 int av1_tpl_ptr_pos(int mi_row, int mi_col, int stride, uint8_t right_shift);
375 
376 void av1_init_tpl_stats(TplParams *const tpl_data);
377 
378 int av1_tpl_stats_ready(const TplParams *tpl_data, int gf_frame_index);
379 
380 void av1_tpl_rdmult_setup(struct AV1_COMP *cpi);
381 
382 void av1_tpl_rdmult_setup_sb(struct AV1_COMP *cpi, MACROBLOCK *const x,
383  BLOCK_SIZE sb_size, int mi_row, int mi_col);
384 
385 void av1_mc_flow_dispenser_row(struct AV1_COMP *cpi,
386  TplTxfmStats *tpl_txfm_stats, MACROBLOCK *x,
387  int mi_row, BLOCK_SIZE bsize, TX_SIZE tx_size);
388 
401 double av1_exponential_entropy(double q_step, double b);
402 
416 double av1_laplace_entropy(double q_step, double b, double zero_bin_ratio);
417 
435 double av1_laplace_estimate_frame_rate(int q_index, int block_count,
436  const double *abs_coeff_mean,
437  int coeff_num);
438 
439 /*
440  *!\brief Compute the number of bits needed to encode a GOP
441  *
442  * \param[in] q_index_list array of q_index, one per frame
443  * \param[in] frame_count number of frames in the GOP
444  * \param[in] stats array of transform stats, one per frame
445  * \param[in] stats_valid_list List indicates whether transform stats
446  * exists
447  * \param[out] bitrate_byframe_list Array to keep track of frame bitrate
448  *
449  * \return The estimated GOP bitrate.
450  *
451  */
452 double av1_estimate_gop_bitrate(const int *q_index_list, const int frame_count,
453  const TplTxfmStats *stats,
454  const int *stats_valid_list,
455  double *bitrate_byframe_list);
456 
457 /*
458  *!\brief Init TplTxfmStats
459  *
460  * \param[in] tpl_txfm_stats a structure for storing transform stats
461  *
462  */
463 void av1_init_tpl_txfm_stats(TplTxfmStats *tpl_txfm_stats);
464 
465 /*
466  *!\brief Accumulate TplTxfmStats
467  *
468  * \param[in] sub_stats a structure for storing sub transform stats
469  * \param[out] accumulated_stats a structure for storing accumulated transform
470  *stats
471  *
472  */
473 void av1_accumulate_tpl_txfm_stats(const TplTxfmStats *sub_stats,
474  TplTxfmStats *accumulated_stats);
475 
476 /*
477  *!\brief Record a transform block into TplTxfmStats
478  *
479  * \param[in] tpl_txfm_stats A structure for storing transform stats
480  * \param[out] coeff An array of transform coefficients. Its size
481  * should equal to tpl_txfm_stats.coeff_num.
482  *
483  */
484 void av1_record_tpl_txfm_block(TplTxfmStats *tpl_txfm_stats,
485  const tran_low_t *coeff);
486 
502 double av1_estimate_coeff_entropy(double q_step, double b,
503  double zero_bin_ratio, int qcoeff);
504 
517 double av1_estimate_txfm_block_entropy(int q_index,
518  const double *abs_coeff_mean,
519  int *qcoeff_arr, int coeff_num);
520 
521 // TODO(angiebird): Add doxygen description here.
522 int64_t av1_delta_rate_cost(int64_t delta_rate, int64_t recrf_dist,
523  int64_t srcrf_dist, int pix_num);
524 
540 int av1_get_overlap_area(int row_a, int col_a, int row_b, int col_b, int width,
541  int height);
542 
563 int av1_q_mode_estimate_base_q(const struct GF_GROUP *gf_group,
564  const TplTxfmStats *txfm_stats_list,
565  const int *stats_valid_list, double bit_budget,
566  int gf_frame_index, aom_bit_depth_t bit_depth,
567  double scale_factor,
568  const double *qstep_ratio_list,
569  int *q_index_list,
570  double *estimated_bitrate_byframe);
571 
581 int av1_tpl_get_q_index(const TplParams *tpl_data, int gf_frame_index,
582  int leaf_qindex, aom_bit_depth_t bit_depth);
583 
591 double av1_tpl_get_frame_importance(const TplParams *tpl_data,
592  int gf_frame_index);
593 
604 double av1_tpl_get_qstep_ratio(const TplParams *tpl_data, int gf_frame_index);
605 
615 int av1_get_q_index_from_qstep_ratio(int leaf_qindex, double qstep_ratio,
616  aom_bit_depth_t bit_depth);
617 
618 #if CONFIG_BITRATE_ACCURACY
619 
628 void av1_vbr_rc_update_q_index_list(VBR_RATECTRL_INFO *vbr_rc_info,
629  const TplParams *tpl_data,
630  const struct GF_GROUP *gf_group,
631  int gf_frame_index,
632  aom_bit_depth_t bit_depth);
633 
644 double av1_tpl_compute_mv_bits(const TplParams *tpl_data, int gf_group_size,
645  int gf_frame_index, int gf_update_type,
646  VBR_RATECTRL_INFO *vbr_rc_info);
647 #endif // CONFIG_BITRATE_ACCURACY
648 
661 int_mv av1_compute_mv_difference(const TplDepFrame *tpl_frame, int row, int col,
662  int step, int tpl_stride, int right_shift);
663 
671 double av1_tpl_compute_frame_mv_entropy(const TplDepFrame *tpl_frame,
672  uint8_t right_shift);
673 
675 #ifdef __cplusplus
676 } // extern "C"
677 #endif
678 
679 #endif // AOM_AV1_ENCODER_TPL_MODEL_H_
TplParams::tpl_bsize_1d
uint8_t tpl_bsize_1d
Definition: tpl_model.h:153
block.h
TplParams::src_ref_frame
const YV12_BUFFER_CONFIG * src_ref_frame[INTER_REFS_PER_FRAME]
Definition: tpl_model.h:201
GF_GROUP
Data related to the current GF/ARF group and the individual frames within the group.
Definition: firstpass.h:344
CommonModeInfoParams
Params related to MB_MODE_INFO arrays and related info.
Definition: av1_common_int.h:505
AV1_PRIMARY
Top level primary encoder structure.
Definition: encoder.h:2332
TplParams
Params related to temporal dependency model.
Definition: tpl_model.h:139
AV1_COMP
Top level encoder structure.
Definition: encoder.h:2632
TplParams::tpl_stats_buffer
TplDepFrame tpl_stats_buffer[MAX_LENGTH_TPL_FRAME_STATS]
Definition: tpl_model.h:160
TplParams::txfm_stats_list
TplTxfmStats txfm_stats_list[MAX_LENGTH_TPL_FRAME_STATS]
Definition: tpl_model.h:173
TplParams::ready
int ready
Definition: tpl_model.h:143
TplParams::ref_frame
const YV12_BUFFER_CONFIG * ref_frame[INTER_REFS_PER_FRAME]
Definition: tpl_model.h:208
lookahead.h
Describes look ahead buffer operations.
TplParams::tpl_stats_pool
TplDepStats * tpl_stats_pool[MAX_LAG_BUFFERS]
Definition: tpl_model.h:167
TplParams::tpl_rec_pool
YV12_BUFFER_CONFIG tpl_rec_pool[MAX_LAG_BUFFERS]
Definition: tpl_model.h:179
av1_tpl_setup_stats
int av1_tpl_setup_stats(struct AV1_COMP *cpi, int gop_eval, const struct EncodeFrameParams *const frame_params)
Implements temporal dependency modelling for a GOP (GF/ARF group) and selects between 16 and 32 frame...
TplParams::tpl_mt_sync
AV1TplRowMultiThreadSync tpl_mt_sync
Definition: tpl_model.h:214
TplParams::sf
struct scale_factors sf
Definition: tpl_model.h:189
yv12_buffer_config
YV12 frame buffer data structure.
Definition: yv12config.h:39
EncodeFrameInput
Input frames and last input frame.
Definition: encoder.h:3290
EncodeFrameParams
contains per-frame encoding parameters decided upon by av1_encode_strategy() and passed down to av1_e...
Definition: encoder.h:3302
TplParams::tpl_stats_block_mis_log2
uint8_t tpl_stats_block_mis_log2
Definition: tpl_model.h:148
TplParams::frame_idx
int frame_idx
Definition: tpl_model.h:194
TplParams::tpl_frame
TplDepFrame * tpl_frame
Definition: tpl_model.h:184
TplParams::border_in_pixels
int border_in_pixels
Definition: tpl_model.h:219
aom_bit_depth_t
enum aom_bit_depth aom_bit_depth_t
Bit depth for codecThis enumeration determines the bit depth of the codec.
macroblock
Encoder's parameters related to the current coding block.
Definition: block.h:778