Android视频防抖开发 安卓视频防抖

FFmpeg在avfilter模块提供各种音视频滤镜。本篇文章主要介绍deshake抗抖动，又称为去抖动，用于修复水平和/或垂直移动中的小变化。运用SAD块匹配运动补偿来消除垂直或水平方向漂移带来的微小偏差。此滤波器有助于消除手持相机、在车辆上移动时产生的相机抖动。涉及的运动估计算法，可参考：GPU_Motion_Estimation。

关于视频滤镜的详细介绍，可查看FFmpeg官方文档：Video-Filters。如文档介绍所说，deshake支持的参数选项如下：

• x、y、w、h：指定搜索的矩形区域，xy为左上角坐标，wh为宽高
• rx、ry：x轴和y轴方向移动的最大像素点，范围为[0, 64]，默认为16
• edge：指定在视频帧边沿生成像素模式，可用模式如下：
•     ‘blank, 0’：空白位置填充0
•     ‘original, 1’：空白位置填充原始图像像素
•     ‘clamp, 2’：空白位置拉伸
•     ‘mirror, 3’：空白位置镜像
• blocksize：指定运动搜索的块大小，范围为[4, 128], 默认为8
• contrast：指定搜索块的对比度阈值，范围为[1, 255], 默认为125
• search：搜索策略，默认为详细搜索，可用策略如下：
•     ‘exhaustive, 0’：详细搜索
•     ‘less, 1’：模糊搜索

1、抗抖动整体流程

deshake抗抖动的源码位于libavfilter/vf_deshake.c，整体流程为：从缓冲区读取视频帧数据，寻找最相似的全局运动，生成亮度变换矩阵，生成色度变换矩阵，亮度与色度变换。关键代码如下：

static int filter_frame(AVFilterLink *link, AVFrame *in)
{
	// 从缓冲区读取视频帧数据
    out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
    if (!out) {
        av_frame_free(&in);
        return AVERROR(ENOMEM);
    }
    av_frame_copy_props(out, in);

    aligned = !((intptr_t)in->data[0] & 15 | in->linesize[0] & 15);
    deshake->sad = av_pixelutils_get_sad_fn(4, 4, aligned, deshake);
    if (!deshake->sad)
        return AVERROR(EINVAL);

    if (deshake->cx < 0 || deshake->cy < 0 || deshake->cw < 0 || deshake->ch < 0) {
        // 寻找最相似的全局运动
        find_motion(deshake, (deshake->ref == NULL) ? 
		    in->data[0] : deshake->ref->data[0], 
		    in->data[0], link->w, link->h, in->linesize[0], &t);
    } else {
        uint8_t *src1 = (deshake->ref == NULL) ? in->data[0] : deshake->ref->data[0];
        uint8_t *src2 = in->data[0];
        deshake->cx = FFMIN(deshake->cx, link->w);
        deshake->cy = FFMIN(deshake->cy, link->h);

        if ((unsigned)deshake->cx + (unsigned)deshake->cw > link->w) 
			deshake->cw = link->w - deshake->cx;
        if ((unsigned)deshake->cy + (unsigned)deshake->ch > link->h) 
			deshake->ch = link->h - deshake->cy;

        deshake->cw &= ~15;
        src1 += deshake->cy * in->linesize[0] + deshake->cx;
        src2 += deshake->cy * in->linesize[0] + deshake->cx;
		// 寻找最相似的全局运动
        find_motion(deshake, src1, src2, deshake->cw, deshake->ch, in->linesize[0], &t);
    }
    ......
    // 生成亮度变换矩阵
    ff_get_matrix(t.vec.x, t.vec.y, t.angle, transform_zoom, transform_zoom, matrix_y);
    // 生成色度变换矩阵
    ff_get_matrix(t.vec.x / (link->w / chroma_width), t.vec.y / (link->h / chroma_height), 
	t.angle, transform_zoom, transform_zoom, matrix_uv);
    // 亮度与色度变换
    ret = deshake->transform(link->dst, link->w, link->h, chroma_width, chroma_height,
                             matrix_y, matrix_uv, INTERPOLATE_BILINEAR, deshake->edge, in, out);

    av_frame_free(&deshake->ref);
    if (ret < 0)
        goto fail;

    deshake->ref = in;
    return ff_filter_frame(outlink, out);
fail:
    av_frame_free(&out);
    return ret;
}

2、寻找最相似的全局运动

通过逐块运动搜索(菱形搜索，又称为钻石搜索)，找出最相似的全局运动，最后计算出偏移值与偏移角度，相关代码如下：

static void find_motion(DeshakeContext *deshake, uint8_t *src1, uint8_t *src2,
                        int width, int height, int stride, Transform *t)
{
    // 抖动计数清零
    for (x = 0; x < deshake->rx * 2 + 1; x++) {
        for (y = 0; y < deshake->ry * 2 + 1; y++) {
            deshake->counts[x][y] = 0;
        }
    }
    // 1、逐块运动搜索
    for (y = deshake->ry; y < height - deshake->ry - (deshake->blocksize * 2); 
	    y += deshake->blocksize * 2) {
        // 2、使用宽为16来匹配sad函数
        for (x = deshake->rx; x < width - deshake->rx - 16; x += 16) {
            // 3、计算块对比度
            contrast = block_contrast(src2, x, y, stride, deshake->blocksize);
            if (contrast > deshake->contrast) {
				// 4、找出块运动
                find_block_motion(deshake, src1, src2, x, y, stride, &mv);
                if (mv.x != -1 && mv.y != -1) {
                    deshake->counts[mv.x + deshake->rx][mv.y + deshake->ry] += 1;
					// 5、计算块角度
                    if (x > deshake->rx && y > deshake->ry)
                        deshake->angles[pos++] = block_angle(x, y, 0, 0, &mv);

                    center_x += mv.x;
                    center_y += mv.y;
                }
            }
        }
    }

    if (pos) {
         center_x /= pos;
         center_y /= pos;
         t->angle = clean_mean(deshake->angles, pos);
         if (t->angle < 0.001)
              t->angle = 0;
    } else {
         t->angle = 0;
    }
    // 6、找出当前帧最相似的运动矢量
    for (y = deshake->ry * 2; y >= 0; y--) {
        for (x = 0; x < deshake->rx * 2 + 1; x++) {
            if (deshake->counts[x][y] > count_max_value) {
                t->vec.x = x - deshake->rx;
                t->vec.y = y - deshake->ry;
                count_max_value = deshake->counts[x][y];
            }
        }
    }

    p_x = (center_x - width / 2.0);
    p_y = (center_y - height / 2.0);
    t->vec.x += (cos(t->angle)-1)*p_x  - sin(t->angle)*p_y;
    t->vec.y += sin(t->angle)*p_x  + (cos(t->angle)-1)*p_y;
    // 7、计算偏移值与角度
    t->vec.x = av_clipf(t->vec.x, -deshake->rx * 2, deshake->rx * 2);
    t->vec.y = av_clipf(t->vec.y, -deshake->ry * 2, deshake->ry * 2);
    t->angle = av_clipf(t->angle, -0.1, 0.1);
}

3、计算块对比度

计算给定块的对比度。如果对比度大，那么会进行下一步处理；如果对比度小，直接跳过当前块。block_contrast()函数如下：

static int block_contrast(uint8_t *src, int x, int y, int stride, int blocksize)
{
    int highest = 0;
    int lowest = 255;
    int i, j, pos;

    for (i = 0; i <= blocksize * 2; i++) {
        // 使用宽为16来匹配sad函数
        for (j = 0; j <= 15; j++) {
            pos = (y + i) * stride + (x + j);
            if (src[pos] < lowest)
                lowest = src[pos];
            else if (src[pos] > highest) {
                highest = src[pos];
            }
        }
    }

    return highest - lowest;
}

4、寻找块运动

查找给定宏块的两帧之间最相似运动偏移。使用这些移位矩阵进行搜索，按块中的最小差异选择最可能的偏移。find_block_motion()函数如下：

static void find_block_motion(DeshakeContext *deshake, uint8_t *src1,
                              uint8_t *src2, int cx, int cy, int stride,
                              IntMotionVector *mv)
{
    int x, y;
    int diff;
    int smallest = INT_MAX;
    int tmp, tmp2;

    #define CMP(i, j) deshake->sad(src1 + cy  * stride + cx,  stride,\
                                   src2 + (j) * stride + (i), stride)

    if (deshake->search == EXHAUSTIVE) {
        // 比较相似位置
        for (y = -deshake->ry; y <= deshake->ry; y++) {
            for (x = -deshake->rx; x <= deshake->rx; x++) {
                diff = CMP(cx - x, cy - y);
                if (diff < smallest) {
                    smallest = diff;
                    mv->x = x;
                    mv->y = y;
                }
            }
        }
    } else if (deshake->search == SMART_EXHAUSTIVE) {
        // 比较相似位置，找出最佳匹配
        for (y = -deshake->ry + 1; y < deshake->ry; y += 2) {
            for (x = -deshake->rx + 1; x < deshake->rx; x += 2) {
                diff = CMP(cx - x, cy - y);
                if (diff < smallest) {
                    smallest = diff;
                    mv->x = x;
                    mv->y = y;
                }
            }
        }

        tmp = mv->x;
        tmp2 = mv->y;

        for (y = tmp2 - 1; y <= tmp2 + 1; y++) {
            for (x = tmp - 1; x <= tmp + 1; x++) {
                if (x == tmp && y == tmp2)
                    continue;

                diff = CMP(cx - x, cy - y);
                if (diff < smallest) {
                    smallest = diff;
                    mv->x = x;
                    mv->y = y;
                }
            }
        }
    }

    if (smallest > 512) {
        mv->x = -1;
        mv->y = -1;
    }
    emms_c();
}

5、计算块角度

计算给定块的偏移角度，block_angle()函数如下：

static double block_angle(int x, int y, int cx, int cy, IntMotionVector *shift)
{
    double a1, a2, diff;

    a1 = atan2(y - cy, x - cx);
    a2 = atan2(y - cy + shift->y, x - cx + shift->x);
    diff = a2 - a1;

    return (diff > M_PI)  ? diff - 2 * M_PI :
           (diff < -M_PI) ? diff + 2 * M_PI :
           diff;
}

6、亮度与色度变换

deshake->transform是个函数指针，在初始化时指向deshake_transform_c()，函数如下：

static int deshake_transform_c(AVFilterContext *ctx,
                                    int width, int height, int cw, int ch,
                                    const float *matrix_y, const float *matrix_uv,
                                    enum InterpolateMethod interpolate,
                                    enum FillMethod fill, AVFrame *in, AVFrame *out)
{
    int i = 0, ret = 0;
    const float *matrixs[3];
    int plane_w[3], plane_h[3];
    matrixs[0] = matrix_y;
    matrixs[1] =  matrixs[2] = matrix_uv;
    plane_w[0] = width;
    plane_w[1] = plane_w[2] = cw;
    plane_h[0] = height;
    plane_h[1] = plane_h[2] = ch;

    for (i = 0; i < 3; i++) {
        // 转换亮度与色度分量
        ret = avfilter_transform(in->data[i], out->data[i], in->linesize[i], out->linesize[i],
                                 plane_w[i], plane_h[i], matrixs[i], interpolate, fill);
        if (ret < 0)
            return ret;
    }
    return ret;
}

其中，avfilter_transform()函数位于transform.c源文件中，用给定的插值方法进行仿射变换。函数实现如下：

int avfilter_transform(const uint8_t *src, uint8_t *dst,
                        int src_stride, int dst_stride,
                        int width, int height, const float *matrix,
                        enum InterpolateMethod interpolate,
                        enum FillMethod fill)
{
    int x, y;
    float x_s, y_s;
    uint8_t def = 0;
    uint8_t (*func)(float, float, const uint8_t *, int, int, int, uint8_t) = NULL;

    switch(interpolate) {
        case INTERPOLATE_NEAREST:     // 最近邻插值
            func = interpolate_nearest;
            break;
        case INTERPOLATE_BILINEAR:    // 双线性插值
            func = interpolate_bilinear;
            break;
        case INTERPOLATE_BIQUADRATIC: // 双四次插值
            func = interpolate_biquadratic;
            break;
        default:
            return AVERROR(EINVAL);
    }

    for (y = 0; y < height; y++) {
        for(x = 0; x < width; x++) {
            x_s = x * matrix[0] + y * matrix[1] + matrix[2];
            y_s = x * matrix[3] + y * matrix[4] + matrix[5];

            switch(fill) {
                case FILL_ORIGINAL: // 原始
                    def = src[y * src_stride + x];
                    break;
                case FILL_CLAMP:    // 截取
                    y_s = av_clipf(y_s, 0, height - 1);
                    x_s = av_clipf(x_s, 0, width - 1);
                    def = src[(int)y_s * src_stride + (int)x_s];
                    break;
                case FILL_MIRROR:   // 镜像
                    x_s = avpriv_mirror(x_s,  width-1);
                    y_s = avpriv_mirror(y_s, height-1);

                    av_assert2(x_s >= 0 && y_s >= 0);
                    av_assert2(x_s < width && y_s < height);
                    def = src[(int)y_s * src_stride + (int)x_s];
            }

            dst[y * dst_stride + x] = func(x_s, y_s, src, width, height, src_stride, def);
        }
    }
    return 0;
}