当前位置: 萬仟网 > IT编程>移动开发>Android > Android 视频通信,低延时解决方案

Android 视频通信,低延时解决方案

2019年01月12日 09:35  | 萬仟网IT编程  | 我要评论

背景:

  由于,项目需要,需要进行视频通信,把a的画面,转给b。

运维部署:

  app1:编码摄像头采集的数据,并且发送数据到服务端

  app2:从服务端,拉取数据,并且进行解码显示

  服务端:接收app1提交的数据,发送app1提交数据到app2

应用说明:

  app1:camera = camera.open(camera.camerainfo.camera_facing_front);

        camera.parameters parameters = camera.getparameters();
        parameters.setpreviewformat(imageformat.nv21);
        parameters.setpreviewsize(width, height);

        // 设置屏幕亮度
        parameters.setexposurecompensation(parameters.getmaxexposurecompensation() / 2);
        camera.setparameters(parameters);
        camera.setdisplayorientation(90);
        camera.setpreviewcallback(new camera.previewcallback() {
            @override
            public void onpreviewframe(byte[] data, camera camera) {
// 采集视频数据,同时记录采集视频的时间点,解码需要(保证视频连续,流畅,且不花屏需要) stamptime = system.nanotime(); yuv_data = data; } });
  1 public class avckeyframeencoder {
  2     private final static string tag = "meidacodec";
  3     private int timeout_usec = 12000;
  4 
  5     private mediacodec mediacodec;
  6     int m_width;
  7     int m_height;
  8     int m_framerate;
  9 
 10     public byte[] configbyte;
 11 
 12     //待解码视频缓冲队列,静态成员!
 13     public byte[] yuv_data = null;
 14     public long stamptime = 0;
 15 
 16     public avckeyframeencoder(int width, int height, int framerate) {
 17         m_width = width;
 18         m_height = height;
 19         m_framerate = framerate;
 20 
 21         //正常的编码出来是横屏的。因为手机本身采集的数据默认就是横屏的
 22         // mediaformat mediaformat = mediaformat.createvideoformat(mime, width, height);
 23         //如果你需要旋转90度或者270度,那么需要把宽和高对调。否则会花屏。因为比如你320 x 240,图像旋转90°之后宽高变成了240 x 320。
 24         mediaformat mediaformat = mediaformat.createvideoformat("video/avc", width, height);
 25         mediaformat.setinteger(mediaformat.key_color_format, mediacodecinfo.codeccapabilities.color_formatyuv420semiplanar);
 26         mediaformat.setinteger(mediaformat.key_bit_rate, 125000);
 27         mediaformat.setinteger(mediaformat.key_frame_rate, framerate); // 30
 28         mediaformat.setinteger(mediaformat.key_i_frame_interval, 1);
 29         try {
 30             mediacodec = mediacodec.createencoderbytype("video/avc");
 31         } catch (ioexception e) {
 32             e.printstacktrace();
 33         }
 34 
 35         //配置编码器参数
 36         mediacodec.configure(mediaformat, null, null, mediacodec.configure_flag_encode);
 37 
 38         //启动编码器
 39         mediacodec.start();
 40     }
 41 
 42     public void stopencoder() {
 43         try {
 44             mediacodec.stop();
 45             mediacodec.release();
 46         } catch (exception e) {
 47             e.printstacktrace();
 48         }
 49     }
 50 
 51     public boolean isruning = false;
 52 
 53     public void startencoderthread(final isavevideo savevideo, final icall callback) {
 54         isruning = true;
 55         new thread(new runnable() {
 56             @override
 57             public void run() {
 58                 byte[] input = null;
 59                 long pts = 0;
 60                 while (isruning) {
 61                     // 访问mainactivity用来缓冲待解码数据的队列
 62                     if(yuv_data == null){
 63                         continue;
 64                     }
 65 
 66                     if (yuv_data != null) {
 67                         //从缓冲队列中取出一帧
 68                         input = yuv_data;
 69                         pts = stamptime;
 70                         yuv_data = null;
 71                         byte[] yuv420sp = new byte[m_width * m_height * 3 / 2];
 72 
 73                         nv21tonv12(input, yuv420sp, m_width, m_height);
 74                         input = yuv420sp;
 75                     }
 76 
 77                     if (input != null) {
 78                         try {
 79                             //编码器输入缓冲区
 80                             bytebuffer[] inputbuffers = mediacodec.getinputbuffers();
 81 
 82                             //编码器输出缓冲区
 83                             bytebuffer[] outputbuffers = mediacodec.getoutputbuffers();
 84                             int inputbufferindex = mediacodec.dequeueinputbuffer(-1);
 85                             if (inputbufferindex >= 0) {
 86                                 bytebuffer inputbuffer = inputbuffers[inputbufferindex];
 87                                 inputbuffer.clear();
 88                                 //把转换后的yuv420格式的视频帧放到编码器输入缓冲区中
 89                                 inputbuffer.put(input);
 90                                 mediacodec.queueinputbuffer(inputbufferindex, 0, input.length, pts, 0);
 91                             }
 92 
 93                             mediacodec.bufferinfo bufferinfo = new mediacodec.bufferinfo();
 94                             int outputbufferindex = mediacodec.dequeueoutputbuffer(bufferinfo, timeout_usec);
 95                             while (outputbufferindex >= 0) {
 96                                 //log.i("avcencoder", "get h264 buffer success! flag = "+bufferinfo.flags+",pts = "+bufferinfo.presentationtimeus+"");
 97                                 bytebuffer outputbuffer = outputbuffers[outputbufferindex];
 98                                 byte[] outdata = new byte[bufferinfo.size];
 99                                 outputbuffer.get(outdata);
100                                 if (bufferinfo.flags == buffer_flag_codec_config) {
101                                     configbyte = new byte[bufferinfo.size];
102                                     configbyte = outdata;
103                                 } else if (bufferinfo.flags == buffer_flag_key_frame) {
104                                     byte[] keyframe = new byte[bufferinfo.size + configbyte.length];
105                                     system.arraycopy(configbyte, 0, keyframe, 0, configbyte.length);
106                                     //把编码后的视频帧从编码器输出缓冲区中拷贝出来
107                                     system.arraycopy(outdata, 0, keyframe, configbyte.length, outdata.length);
108 
109                                     logs.i("上传i帧 " + keyframe.length);
110                                     byte[] send_data = new byte[13 + keyframe.length];
111                                     system.arraycopy(new byte[]{0x01}, 0, send_data, 0, 1);
112                                     system.arraycopy(intbytes.longtobytes(pts), 0, send_data, 1, 8);
113                                     system.arraycopy(intbytes.inttobytearray(keyframe.length), 0, send_data, 9, 4);
114                                     system.arraycopy(keyframe, 0, send_data, 13, keyframe.length);
115                                     if(savevideo != null){
116                                         savevideo.savevideodata(send_data);
117                                     }
118 
119                                     if(callback != null){
120                                         callback.callback(keyframe, pts);
121                                     }
122                                 } else {
123                                     byte[] send_data = new byte[13 + outdata.length];
124                                     system.arraycopy(new byte[]{0x02}, 0, send_data, 0, 1);
125                                     system.arraycopy(intbytes.longtobytes(pts), 0, send_data, 1, 8);
126                                     system.arraycopy(intbytes.inttobytearray(outdata.length), 0, send_data, 9, 4);
127                                     system.arraycopy(outdata, 0, send_data, 13, outdata.length);
128                                     if(savevideo != null){
129                                         savevideo.savevideodata(send_data);
130                                     }
131 
132                                     if(callback != null){
133                                         callback.callback(outdata, pts);
134                                     }
135                                 }
136 
137                                 mediacodec.releaseoutputbuffer(outputbufferindex, false);
138                                 outputbufferindex = mediacodec.dequeueoutputbuffer(bufferinfo, timeout_usec);
139                             }
140 
141                         } catch (throwable t) {
142                             t.printstacktrace();
143                             break;
144                         }
145                     }
146                 }
147             }
148         }).start();
149     }
150 
151     private void nv21tonv12(byte[] nv21, byte[] nv12, int width, int height) {
152         if (nv21 == null || nv12 == null) return;
153         int framesize = width * height;
154         int i = 0, j = 0;
155         system.arraycopy(nv21, 0, nv12, 0, framesize);
156         for (i = 0; i < framesize; i++) {
157             nv12[i] = nv21[i];
158         }
159 
160         for (j = 0; j < framesize / 2; j += 2) {
161             nv12[framesize + j - 1] = nv21[j + framesize];
162         }
163 
164         for (j = 0; j < framesize / 2; j += 2) {
165             nv12[framesize + j] = nv21[j + framesize - 1];
166         }
167     }
168 }
视频编码类encoder

其中使用到了,接口用于,把采集和编码后的数据,往外部传递,通过线程提交到服务端。或者通过本地解码显示,查看,编码解码时间差。

通过使用 arrayblockingqueue<byte[]> h264queue = new arrayblockingqueue<byte[]>(10); 队列,对接口提交数据,进行暂时保存,在后台对数据,进行解码或提交到服务端。

  app2:接入服务端,然后从i帧数据开始拿数据,(且数据是最新的i帧开始保存的数据)。同时需要把,之前采集得到的时间点传给:

mediacodec 对象的 queueinputbuffer 方法的时间戳参数(第四个)。

 服务端:一帧一帧接收app1传入数据,对i帧开始的数据进行记录,同时对非i帧开始的数据,进行丢弃。一次只保存一帧内容。读取数据,并且移除已经添加数据,循环发送给app2

public class videodecoder {
    private thread mdecodethread;
    private mediacodec mcodec;
    private boolean mstopflag = false;
    private int video_width = 640;
    private int video_height = 480;
    private int framerate = 25;
    private boolean isuseppsandsps = false;
    private receivevideothread runthread = null;

    public videodecoder(string ip, int port, byte type, int roomid){
        runthread = new receivevideothread(ip, port, type, roomid);
        new thread(runthread).start();
    }

    public void initreaddata(surface surface){
        try {
            //通过多媒体格式名创建一个可用的解码器
            mcodec = mediacodec.createdecoderbytype("video/avc");
        } catch (ioexception e) {
            e.printstacktrace();
        }

        //初始化编码器
        final mediaformat mediaformat = mediaformat.createvideoformat("video/avc", video_width, video_height);

        //设置帧率
        mediaformat.setinteger(mediaformat.key_frame_rate, framerate);

        //https://developer.android.com/reference/android/media/mediaformat.html#key_max_input_size
        //设置配置参数,参数介绍 :
        // format   如果为解码器,此处表示输入数据的格式;如果为编码器,此处表示输出数据的格式。
        //surface   指定一个surface,可用作decode的输出渲染。
        //crypto    如果需要给媒体数据加密,此处指定一个crypto类.
        //   flags  如果正在配置的对象是用作编码器,此处加上configure_flag_encode 标签。
        mcodec.configure(mediaformat, surface, null, 0);
        startdecodingthread();
    }

    private void startdecodingthread() {
        mcodec.start();

        mdecodethread = new thread(new decodeh264thread());
        mdecodethread.start();
    }

    @requiresapi(api = build.version_codes.lollipop)
    private class decodeh264thread implements runnable {
        @override
        public void run() {
            try {
                // savedataloop();
                decodeloop_new();
            } catch (exception e) {
                e.printstacktrace();
            }
        }

        private void decodeloop_new() {
            // 存放目标文件的数据
            bytebuffer[] inputbuffers = mcodec.getinputbuffers();
            // 解码后的数据,包含每一个buffer的元数据信息,例如偏差,在相关解码器中有效的数据大小
            mediacodec.bufferinfo info = new mediacodec.bufferinfo();
            long timeoutus = 1000;
            byte[] marker0 = new byte[]{0, 0, 0, 1};
            byte[] dummyframe = new byte[]{0x00, 0x00, 0x01, 0x20};
            byte[] streambuffer = null;
            while (true) {
                if(runthread.h264queue.size() > 0){
                    streambuffer = runthread.h264queue.poll();
                }else{
                    try {
                        thread.sleep(20);
                    }catch (exception ex){
                    }

                    continue;
                }

                byte[] time_data = new byte[8];
                system.arraycopy(streambuffer, 0, time_data, 0, 8);
                long pts = intbytes.bytestolong(time_data);
                byte[] video_data = new byte[streambuffer.length - 8];
                system.arraycopy(streambuffer, 8, video_data, 0, video_data.length);
                streambuffer = video_data;

                logs.i("得到 streambuffer " + streambuffer.length + " pts " + pts);
                int bytes_cnt = 0;
                mstopflag = false;
                while (mstopflag == false) {
                    bytes_cnt = streambuffer.length;
                    if (bytes_cnt == 0) {
                        streambuffer = dummyframe;
                    }

                    int startindex = 0;
                    int remaining = bytes_cnt;
                    while (true) {
                        if (remaining == 0 || startindex >= remaining) {
                            break;
                        }
                        int nextframestart = kmpmatch(marker0, streambuffer, startindex + 2, remaining);
                        if (nextframestart == -1) {
                            nextframestart = remaining;
                        } else {
                        }

                        int inindex = mcodec.dequeueinputbuffer(timeoutus);
                        if (inindex >= 0) {
                            bytebuffer bytebuffer = inputbuffers[inindex];
                            bytebuffer.clear();
                            bytebuffer.put(streambuffer, startindex, nextframestart - startindex);
                            //在给指定index的inputbuffer[]填充数据后,调用这个函数把数据传给解码器
                            mcodec.queueinputbuffer(inindex, 0, nextframestart - startindex, pts, 0);
                            startindex = nextframestart;
                        } else {
                            continue;
                        }

                        int outindex = mcodec.dequeueoutputbuffer(info, timeoutus);
                        if (outindex >= 0) {
                            //帧控制是不在这种情况下工作,因为没有pts h264是可用的
                            /*
                            while (info.presentationtimeus / 1000 > system.currenttimemillis() - startms) {
                                try {
                                    thread.sleep(100);
                                } catch (interruptedexception e) {
                                    e.printstacktrace();
                                }
                            }
                            */
                            boolean dorender = (info.size != 0);
                            //对outputbuffer的处理完后,调用这个函数把buffer重新返回给codec类。

                            // todo:添加处理,保存原始帧数据
                            if (dorender) {
                                image image = mcodec.getoutputimage(outindex);
                                if (image != null) {
                                    // 通过反射
                                    // 发送数据到指定接口
                                    byte[] data = getdatafromimage(image, color_formatnv21);
                                }
                            }

                            mcodec.releaseoutputbuffer(outindex, dorender);
                        } else {
                            // log.e(tag, "bbbb");
                        }
                    }
                    mstopflag = true;
                }

                // logs.i("处理单帧视频耗时:" + (system.currenttimemillis() - c_start));
            }
        }
    }

    private static final boolean verbose = false;
    private static final long default_timeout_us = 10000;

    private static final int color_formati420 = 1;
    private static final int color_formatnv21 = 2;

    private static boolean isimageformatsupported(image image) {
        int format = image.getformat();
        switch (format) {
            case imageformat.yuv_420_888:
            case imageformat.nv21:
            case imageformat.yv12:
                return true;
        }
        return false;
    }

    @requiresapi(api = build.version_codes.lollipop)
    private static byte[] getdatafromimage(image image, int colorformat) {
        if (colorformat != color_formati420 && colorformat != color_formatnv21) {
            throw new illegalargumentexception("only support color_formati420 " + "and color_formatnv21");
        }
        if (!isimageformatsupported(image)) {
            throw new runtimeexception("can't convert image to byte array, format " + image.getformat());
        }

        rect crop = image.getcroprect();
        int format = image.getformat();
        int width = crop.width();
        int height = crop.height();
        image.plane[] planes = image.getplanes();
        byte[] data = new byte[width * height * imageformat.getbitsperpixel(format) / 8];
        byte[] rowdata = new byte[planes[0].getrowstride()];
        if (verbose) logs.i("get data from " + planes.length + " planes");
        int channeloffset = 0;
        int outputstride = 1;
        for (int i = 0; i < planes.length; i++) {
            switch (i) {
                case 0:
                    channeloffset = 0;
                    outputstride = 1;
                    break;
                case 1:
                    if (colorformat == color_formati420) {
                        channeloffset = width * height;
                        outputstride = 1;
                    } else if (colorformat == color_formatnv21) {
                        channeloffset = width * height + 1;
                        outputstride = 2;
                    }
                    break;
                case 2:
                    if (colorformat == color_formati420) {
                        channeloffset = (int) (width * height * 1.25);
                        outputstride = 1;
                    } else if (colorformat == color_formatnv21) {
                        channeloffset = width * height;
                        outputstride = 2;
                    }
                    break;
            }
            bytebuffer buffer = planes[i].getbuffer();
            int rowstride = planes[i].getrowstride();
            int pixelstride = planes[i].getpixelstride();
            if (verbose) {
                logs.i("pixelstride " + pixelstride);
                logs.i("rowstride " + rowstride);
                logs.i("width " + width);
                logs.i("height " + height);
                logs.i("buffer size " + buffer.remaining());
            }
            int shift = (i == 0) ? 0 : 1;
            int w = width >> shift;
            int h = height >> shift;
            buffer.position(rowstride * (crop.top >> shift) + pixelstride * (crop.left >> shift));
            for (int row = 0; row < h; row++) {
                int length;
                if (pixelstride == 1 && outputstride == 1) {
                    length = w;
                    buffer.get(data, channeloffset, length);
                    channeloffset += length;
                } else {
                    length = (w - 1) * pixelstride + 1;
                    buffer.get(rowdata, 0, length);
                    for (int col = 0; col < w; col++) {
                        data[channeloffset] = rowdata[col * pixelstride];
                        channeloffset += outputstride;
                    }
                }
                if (row < h - 1) {
                    buffer.position(buffer.position() + rowstride - length);
                }
            }
            if (verbose) logs.i("finished reading data from plane " + i);
        }
        return data;
    }

    private int kmpmatch(byte[] pattern, byte[] bytes, int start, int remain) {
        try {
            thread.sleep(30);
        } catch (interruptedexception e) {
            e.printstacktrace();
        }

        int[] lsp = computelsptable(pattern);

        int j = 0;  // number of chars matched in pattern
        for (int i = start; i < remain; i++) {
            while (j > 0 && bytes[i] != pattern[j]) {
                // fall back in the pattern
                j = lsp[j - 1];  // strictly decreasing
            }
            if (bytes[i] == pattern[j]) {
                // next char matched, increment position
                j++;
                if (j == pattern.length)
                    return i - (j - 1);
            }
        }
        return -1;  // not found
    }

    private int[] computelsptable(byte[] pattern) {
        int[] lsp = new int[pattern.length];
        lsp[0] = 0;  // base case
        for (int i = 1; i < pattern.length; i++) {
            // start by assuming we're extending the previous lsp
            int j = lsp[i - 1];
            while (j > 0 && pattern[i] != pattern[j])
                j = lsp[j - 1];
            if (pattern[i] == pattern[j])
                j++;
            lsp[i] = j;
        }
        return lsp;
    }

    public void stopdecode() {
        if(runthread != null){
            runthread.stopreceive();
        }
    }
}
视频解码类decoder

总结:

  通过对视频的处理,学习到了,一些处理视频的细节点。同时加深了,依赖导致在实际项目中的使用。to android.

 

如对本文有疑问,请在下面进行留言讨论,广大热心网友会与你互动!! 点击进行留言回复

相关文章:

◎已有 0 人评论

Copyright © 2019  萬仟网 保留所有权利. 粤ICP备17035492号-1
站长QQ:2386932994 | 联系邮箱:2386932994@qq.com