From 449ca7b28ef5c398170d358b9a84234459ae444a Mon Sep 17 00:00:00 2001 From: xiaowusky Date: Fri, 3 Nov 2023 14:17:19 +0800 Subject: [PATCH] =?UTF-8?q?desc:yuv=E6=B7=BB=E5=8A=A0=E6=B0=B4=E5=8D=B0?= =?UTF-8?q?=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../safetywatcher/watcher/ui/HomeActivity.kt | 49 +- .../watcher/utils/RecordHelper.kt | 101 +- .../safetywatcher/watcher/utils/SnUtils.java | 401 ++++++ app/src/main/res/layout/activity_home.xml | 2 +- .../com/common/commonlib/utils/BitmapUtils.kt | 12 +- .../library/vlc/encoder/AndroidMuxer.java | 9 +- .../vlc/encoder/MediaCodecManager.java | 336 +++++ .../main/java/org/easydarwin/TxtOverlay.kt | 12 +- .../easydarwin/video/EasyPlayerClient.java | 1141 +++++++---------- 9 files changed, 1276 insertions(+), 787 deletions(-) create mode 100644 library-push/src/main/java/com/yinuo/library/vlc/encoder/MediaCodecManager.java diff --git a/app/src/main/java/com/yinuo/safetywatcher/watcher/ui/HomeActivity.kt b/app/src/main/java/com/yinuo/safetywatcher/watcher/ui/HomeActivity.kt index 9a32d08..1e3f605 100644 --- a/app/src/main/java/com/yinuo/safetywatcher/watcher/ui/HomeActivity.kt +++ b/app/src/main/java/com/yinuo/safetywatcher/watcher/ui/HomeActivity.kt @@ -3,13 +3,12 @@ package com.yinuo.safetywatcher.watcher.ui import android.content.Intent import android.os.Build import android.os.Process -import android.view.SurfaceHolder -import android.view.SurfaceHolder.Callback import android.view.View import androidx.annotation.RequiresApi import androidx.lifecycle.lifecycleScope import com.common.commonlib.db.DBUtils import com.common.commonlib.utils.LogUtils +import com.common.commonlib.utils.MMKVUtils import com.common.commonlib.utils.NetworkHelper import com.yinuo.safetywatcher.R import com.yinuo.safetywatcher.databinding.ActivityHomeBinding @@ -99,37 +98,33 @@ class HomeActivity : NoOptionsActivity() { } } + private var lastUpdateTime = 0L + /** * 设置摄像头 */ private fun setForCamera() { - mBinding.surface.holder.addCallback(object : Callback{ - override fun surfaceCreated(holder: SurfaceHolder) { - mClient = EasyPlayerClient(this@HomeActivity, mBinding.surface.holder.surface, null) { - RecordHelper.onFrameAvailable(mBinding.surface) - if (!AppData.hasCameraData()) { - AppData.setCameraData(true) - changeViewStatus() - closeLoadingDialog() - } - watchCamera(DELAY_TIME_CHECK_CAMERA) + mClient = EasyPlayerClient( + this@HomeActivity, + mBinding.surface, + null + ) { + LogUtils.w("onI420Data c0") + RecordHelper.onFrameAvailable(mBinding.surface, it) + LogUtils.w("onI420Data c1") + val currentTimeMillis = System.currentTimeMillis() + if (currentTimeMillis - lastUpdateTime > 1000) { + lastUpdateTime = currentTimeMillis + if (!AppData.hasCameraData()) { + AppData.setCameraData(true) + changeViewStatus() + closeLoadingDialog() } - mClient?.play(CAMERA_URL) - } - - override fun surfaceChanged( - holder: SurfaceHolder, - format: Int, - width: Int, - height: Int - ) { + watchCamera(DELAY_TIME_CHECK_CAMERA) } - - override fun surfaceDestroyed(holder: SurfaceHolder) { - mClient?.stop() - } - - }) + LogUtils.w("onI420Data c2") + } + mClient?.play(CAMERA_URL) // 第一次很慢，所以10秒 watchCamera(DELAY_TIME_OPEN_CAMERA) showLoadingDialog(R.string.connecting_camera) diff --git a/app/src/main/java/com/yinuo/safetywatcher/watcher/utils/RecordHelper.kt b/app/src/main/java/com/yinuo/safetywatcher/watcher/utils/RecordHelper.kt index 54f735b..ca02fac 100644 --- a/app/src/main/java/com/yinuo/safetywatcher/watcher/utils/RecordHelper.kt +++ b/app/src/main/java/com/yinuo/safetywatcher/watcher/utils/RecordHelper.kt @@ -1,92 +1,65 @@ package com.yinuo.safetywatcher.watcher.utils import android.graphics.Bitmap -import android.opengl.EGL14 -import android.os.Handler -import android.os.HandlerThread -import android.view.PixelCopy -import android.view.PixelCopy.OnPixelCopyFinishedListener -import android.view.SurfaceView -import com.common.commonlib.CommonApplication -import com.common.commonlib.utils.BitmapUtils -import com.yinuo.library.vlc.encoder.BaseMovieEncoder.EncoderConfig -import com.yinuo.library.vlc.encoder.MovieEncoder1 +import android.view.TextureView +import com.common.commonlib.utils.LogUtils +import com.yinuo.library.vlc.encoder.MediaCodecManager import org.easydarwin.TxtOverlay -import java.nio.ByteBuffer object RecordHelper { - private val mVideoEncoder: MovieEncoder1 + private val codecManager: MediaCodecManager = MediaCodecManager.getInstance() private const val width = 1920 private const val height = 1080 - - val utils by lazy { - NV21ToBitmap(CommonApplication.getContext()) - } - - private val workHandler by lazy { - val mHandlerThread = HandlerThread("recordAndEncode") - mHandlerThread.start() - Handler(mHandlerThread.looper) - } + private var recording = false; init { - mVideoEncoder = MovieEncoder1(CommonApplication.getContext(), width, height, true) + codecManager.initCodecManager(width, height, 0) } - fun onFrameAvailable(view: SurfaceView) { - if (!mVideoEncoder.isRecording) { + fun onFrameAvailable(view: TextureView, nv12Data: ByteArray) { + if (!recording) { return } -// workHandler.post { -// val nanoTime = System.nanoTime() -// var bitmap = view.bitmap -// bitmap?.let { -// val overLayBitmap: Bitmap? = TxtOverlay.getOverlayBitmap() -// overLayBitmap?.let { -// bitmap = BitmapUtils.mergeBitmap(bitmap!!, overLayBitmap) -// } -// } -// val buffer = ByteBuffer.allocate(bitmap!!.getByteCount()) -// bitmap!!.copyPixelsToBuffer(buffer) -// bitmap!!.recycle() -// mVideoEncoder.frameAvailable(buffer.array(), nanoTime) -// } - workHandler.post { - var bitmap = Bitmap.createBitmap( - width, - height, - Bitmap.Config.ARGB_8888 - ) - PixelCopy.request( - view.holder.surface, bitmap, { copyResult -> - val nanoTime = System.nanoTime() - if (copyResult == PixelCopy.SUCCESS) { - bitmap?.let { - val overLayBitmap: Bitmap? = TxtOverlay.getOverlayBitmap() - overLayBitmap?.let { - bitmap = BitmapUtils.mergeBitmap(bitmap!!, overLayBitmap) - } + LogUtils.w("onI420Data 11111") + val overLayBitmap: Bitmap? = TxtOverlay.getOverlayBitmap() + overLayBitmap?.let { + val yuv = TxtOverlay.getOverlayYuv() + yuv?.let { + for ((j, i) in (100 until overLayBitmap.height + 100).withIndex()) { + for (c in 0 until overLayBitmap.width) { + //去掉PNG水印的黑边 + if (yuv[j * overLayBitmap.width + c] + .toInt() != 0x10 && yuv[j * overLayBitmap.width + c] + .toInt() != 0x80 && yuv[j * overLayBitmap.width + c].toInt() != 0xeb + ) { + System.arraycopy( + yuv, + j * overLayBitmap.width + c, + nv12Data, + 100 + i * 1920 + c, + 1 + ) } - val buffer = ByteBuffer.allocate(bitmap!!.byteCount) - bitmap!!.copyPixelsToBuffer(buffer) - bitmap!!.recycle() - mVideoEncoder.frameAvailable(buffer.array(), nanoTime) } - }, workHandler - ) + } + } } + codecManager.addFrameData(nv12Data) + LogUtils.w("onI420Data 44444") } fun startRecording() { - if (!mVideoEncoder.isRecording) { - mVideoEncoder.startRecording(EncoderConfig(EGL14.eglGetCurrentContext())) + if (!recording) { + recording = true + codecManager.startMediaCodec() } } fun stopRecording() { - if (mVideoEncoder.isRecording) { - mVideoEncoder.stopRecording() + if (recording) { + recording = false + codecManager.pauseMediaCodec() } } } diff --git a/app/src/main/java/com/yinuo/safetywatcher/watcher/utils/SnUtils.java b/app/src/main/java/com/yinuo/safetywatcher/watcher/utils/SnUtils.java index ad98a88..53a32d9 100644 --- a/app/src/main/java/com/yinuo/safetywatcher/watcher/utils/SnUtils.java +++ b/app/src/main/java/com/yinuo/safetywatcher/watcher/utils/SnUtils.java @@ -1,6 +1,17 @@ package com.yinuo.safetywatcher.watcher.utils; +import android.graphics.Bitmap; +import android.graphics.BitmapFactory; +import android.graphics.Color; +import android.graphics.ImageFormat; +import android.graphics.Rect; +import android.graphics.YuvImage; + +import java.io.ByteArrayOutputStream; import java.lang.reflect.Method; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.IntBuffer; public class SnUtils { public static String getDeviceSN() { @@ -14,4 +25,394 @@ public class SnUtils { } return serial; } + + public static int[] yuvI420toARGB(byte[] i420, int width, int height) { + int framesize = width * height; + + int[] rgb = new int[framesize];//新图大小 + for (int i = 0; i < framesize; i++) { + int index = (i / width) / 2 * (width / 2) + (i % width) / 2; + int y = i420[i] & 0x000000ff;//i420 y分量的值大小是一个字节，准换成int + int u = i420[framesize + index] & 0x000000ff;//i420 u分量 + int v = i420[framesize + framesize / 4 + index] & 0x000000ff;//i420 v + + /*yuv----rgb 转换公式*/ + int b = (int) (y + 1.8556 * (u - 128)); + int g = (int) (y - (0.4681 * (v - 128) + 0.1872 * (u - 128))); + int r = (int) (y + 1.5748 * (v - 128)); + /*防止越界*/ + b = (b > 255) ? 255 : ((b < 0) ? 0 : b); + g = (g > 255) ? 255 : ((g < 0) ? 0 : g); + r = (r > 255) ? 255 : ((r < 0) ? 0 : r); + rgb[i] = (0xff000000) | (0x00ff0000 & r << 16) | (0x0000ff00 & g << 8) | (0x000000ff & b); + } + return rgb; + } + + static byte[] intArrToByteArrLittleEndianBySystemApi(int[] array) { + ByteBuffer byteBuffer = ByteBuffer.allocate(array.length * 4); + byteBuffer.order(ByteOrder.LITTLE_ENDIAN); + IntBuffer intBuffer = byteBuffer.asIntBuffer(); + intBuffer.put(array); + return byteBuffer.array(); + } + + static byte[] intArrToByteArrBigEndianBySystemApi(int[] array) { + ByteBuffer byteBuffer = ByteBuffer.allocate(array.length * 4); + byteBuffer.order(ByteOrder.BIG_ENDIAN); + IntBuffer intBuffer = byteBuffer.asIntBuffer(); + intBuffer.put(array); + return byteBuffer.array(); + } + + static byte[] yuv = new byte[1920 * 1080 * 3 / 2]; + + public static byte[] rgb2YCbCr420(int[] pixels, int width, int height) { + int len = width * height; + // yuv格式数组大小，y亮度占len长度，u、v各占len/4长度 + int y, u, v; + for (int i = 0; i < height; i++) { + for (int j = 0; j < width; j++) { + // 屏蔽ARGB的透明度 + int rgb = pixels[i * width + j] & 0x00FFFFFF; + // 像素的颜色顺序为bgr，移位运算 + int r = rgb & 0xFF; + int g = (rgb >> 8) & 0xFF; + int b = (rgb >> 16) & 0xFF; + // 套用公式 + y = ((66 * r + 129 * g + 25 * b + 128) >> 8) + 16; + u = ((-38 * r - 74 * g + 112 * b + 128) >> 8) + 128; + v = ((112 * r - 94 * g - 18 * b + 128) >> 8) + 128; + // 调整 + y = y < 16 ? 16 : (y > 255 ? 255 : y); + u = u < 0 ? 0 : (u > 255 ? 255 : u); + v = v < 0 ? 0 : (v > 255 ? 255 : v); + // 复制 + yuv[i * width + j] = (byte) y; + yuv[len + (i >> 1) * width + (j & ~1) + 0] = (byte) u; + yuv[len + +(i >> 1) * width + (j & ~1) + 1] = (byte) v; + } + } + return yuv; + } + + public static void nv21ToRgba(byte[] input, int width, int height, byte[] output, boolean isRGB) { + int nvOff = width * height; + int i, j, yIndex = 0; + int y, u, v; + int r, g, b, nvIndex = 0; + //遍历图像每一个像素点，依次计算r,g,b的值。 + for (i = 0; i < height; i++) { + for (j = 0; j < width; j++, ++yIndex) { + //对于位置为i,j的像素，找到对应的y,u,v的值 + nvIndex = (i / 2) * width + j - j % 2; + y = input[yIndex] & 0xff; + u = input[nvOff + nvIndex] & 0xff; + v = input[nvOff + nvIndex + 1] & 0xff; + + //对于位置为i,j的像素，根据其yuv的值计算出r,g,b的值 + r = y + ((351 * (v - 128)) >> 8); //r + g = y - ((179 * (v - 128) + 86 * (u - 128)) >> 8); //g + b = y + ((443 * (u - 128)) >> 8); //b + + //要求的rgb值是byte类型，范围是0--255，消除越界的值。 + r = ((r > 255) ? 255 : (r < 0) ? 0 : r); + g = ((g > 255) ? 255 : (g < 0) ? 0 : g); + b = ((b > 255) ? 255 : (b < 0) ? 0 : b); + // 对结果rgb/bgr的值赋值，a通道表示透明度，这里都给255（根据你自己的场景设置） + //由于rgba格式和bgra格式只是r,g,b的顺序不同，因此这里通过bool值判断，既可以得到rgba的格式也可以得到bgra格式的数组。 + if (isRGB) { + output[yIndex * 4 + 0] = (byte) b; + output[yIndex * 4 + 1] = (byte) g; + output[yIndex * 4 + 2] = (byte) r; + output[yIndex * 4 + 3] = (byte) 255; + } else { + output[yIndex * 4 + 0] = (byte) r; + output[yIndex * 4 + 1] = (byte) g; + output[yIndex * 4 + 2] = (byte) b; + output[yIndex * 4 + 3] = (byte) 255; + } + } + } + } + + public static Bitmap nv12ToBitmap2(byte[] nv12Data, int width, int height) { + int[] rgbaData = new int[width * height]; + + int frameSize = width * height; + int yOffset = 0; + int uvOffset = frameSize; + + for (int j = 0; j < height; j++) { + int rgbIndex = j * width; + int uvIndex = uvOffset + (j >> 1) * width; + + for (int i = 0; i < width; i++) { + int y = nv12Data[yOffset + i] & 0xFF; + + int uv = (nv12Data[uvIndex + (i & ~1)] & 0xFF) - 128; + + int v = (nv12Data[uvIndex + (i & ~1) + 1] & 0xFF) - 128; + + int r = (int) (y + 1.402 * v); + int g = (int) (y - 0.344136 * uv - 0.714136 * v); + int b = (int) (y + 1.772 * uv); + + r = Math.max(0, Math.min(r, 255)); + g = Math.max(0, Math.min(g, 255)); + b = Math.max(0, Math.min(b, 255)); + + rgbaData[rgbIndex++] = Color.argb(255, r, g, b); + } + + yOffset += width; + } + + return Bitmap.createBitmap(rgbaData, width, height, Bitmap.Config.ARGB_8888); + } + + public static Bitmap nv12ToBitmap(byte[] data, int w, int h) { + return spToBitmap(data, w, h, 0, 1); + } + + public static Bitmap nv21ToBitmap(byte[] data, int w, int h) { + return spToBitmap(data, w, h, 1, 0); + } + + static int[] colors = null; + private static Bitmap spToBitmap(byte[] data, int w, int h, int uOff, int vOff) { + int plane = w * h; + if (colors == null){ + colors = new int[plane]; + } + int yPos = 0, uvPos = plane; + for (int j = 0; j < h; j++) { + for (int i = 0; i < w; i++) { + // YUV byte to RGB int + final int y1 = data[yPos] & 0xff; + final int u = (data[uvPos + uOff] & 0xff) - 128; + final int v = (data[uvPos + vOff] & 0xff) - 128; + final int y1192 = 1192 * y1; + int r = (y1192 + 1634 * v); + int g = (y1192 - 833 * v - 400 * u); + int b = (y1192 + 2066 * u); + + r = (r < 0) ? 0 : ((r > 262143) ? 262143 : r); + g = (g < 0) ? 0 : ((g > 262143) ? 262143 : g); + b = (b < 0) ? 0 : ((b > 262143) ? 262143 : b); + colors[yPos] = ((r << 6) & 0xff0000) | + ((g >> 2) & 0xff00) | + ((b >> 10) & 0xff); + + if ((yPos++ & 1) == 1) uvPos += 2; + } + if ((j & 1) == 0) uvPos -= w; + } + return Bitmap.createBitmap(colors, w, h, Bitmap.Config.RGB_565); + } + + public static Bitmap i420ToBitmap(byte[] data, int w, int h) { + return pToBitmap(data, w, h, true); + } + + public static Bitmap yv12ToBitmap(byte[] data, int w, int h) { + return pToBitmap(data, w, h, false); + } + + private static Bitmap pToBitmap(byte[] data, int w, int h, boolean uv) { + int plane = w * h; + int[] colors = new int[plane]; + int off = plane >> 2; + int yPos = 0, uPos = plane + (uv ? 0 : off), vPos = plane + (uv ? off : 0); + for (int j = 0; j < h; j++) { + for (int i = 0; i < w; i++) { + // YUV byte to RGB int + final int y1 = data[yPos] & 0xff; + final int u = (data[uPos] & 0xff) - 128; + final int v = (data[vPos] & 0xff) - 128; + final int y1192 = 1192 * y1; + int r = (y1192 + 1634 * v); + int g = (y1192 - 833 * v - 400 * u); + int b = (y1192 + 2066 * u); + + r = (r < 0) ? 0 : ((r > 262143) ? 262143 : r); + g = (g < 0) ? 0 : ((g > 262143) ? 262143 : g); + b = (b < 0) ? 0 : ((b > 262143) ? 262143 : b); + colors[yPos] = ((r << 6) & 0xff0000) | + ((g >> 2) & 0xff00) | + ((b >> 10) & 0xff); + + if ((yPos++ & 1) == 1) { + uPos++; + vPos++; + } + } + if ((j & 1) == 0) { + uPos -= (w >> 1); + vPos -= (w >> 1); + } + } + return Bitmap.createBitmap(colors, w, h, Bitmap.Config.RGB_565); +// return Bitmap.createBitmap(colors, w, h, Bitmap.Config.ARGB_8888); + } + +// public static byte[] nv12ToRgba(byte[] nv12Data, int width, int height) { +// int nv12Size = width * height * 12; +// byte[] rgbaData = new byte[nv12Size + 4 * (width * height)]; +// +// int offset = 0; +// for (int y = 0; y < height; y++) { +// for (int x = 0; x < width; x++) { +// int nv12Offset = (y * width + x) * 12; +// int yValue = (nv12Data[nv12Offset] & 0xFF) << 8 | (nv12Data[nv12Offset + 1] & 0xFF); +// int uValue = (nv12Data[nv12Offset + 2] & 0xFF) << 8 | (nv12Data[nv12Offset + 3] & 0xFF); +// int vValue = (nv12Data[nv12Offset + 4] & 0xFF) << 8 | (nv12Data[nv12Offset + 5] & 0xFF); +// +// int r = (yValue + 16) >> 16; +// int g = ((66 * uValue + 128) >> 16) + r; +// int b = ((49 * vValue + 128) >> 16) + g; +// int a = 255; +// +// rgbaData[offset++] = (byte) r; +// rgbaData[offset++] = (byte) g; +// rgbaData[offset++] = (byte) b; +// rgbaData[offset++] = (byte) a; +// } +// } +// +// return rgbaData; +// } + + public static byte[] nv12ToRGBA(byte[] nv12Data, int width, int height) { + byte[] rgbaData = new byte[width * height * 4]; + int frameSize = width * height; + + int nv12Index = 0; + int rgbaIndex = 0; + + int yOffset = 0; + int uvOffset = frameSize; + + for (int j = 0; j < height; j++) { + for (int i = 0; i < width; i++) { + int y = (nv12Data[yOffset + i] & 0xFF); + + int uvIndex = uvOffset + (i & ~1) + (j >> 1) * width; + int u = (nv12Data[uvIndex] & 0xFF) - 128; + int v = (nv12Data[uvIndex + 1] & 0xFF) - 128; + + int r = (int) (y + 1.402 * v); + int g = (int) (y - 0.344136 * u - 0.714136 * v); + int b = (int) (y + 1.772 * u); + + r = Math.max(0, Math.min(r, 255)); + g = Math.max(0, Math.min(g, 255)); + b = Math.max(0, Math.min(b, 255)); + + rgbaData[rgbaIndex++] = (byte) r; + rgbaData[rgbaIndex++] = (byte) g; + rgbaData[rgbaIndex++] = (byte) b; + rgbaData[rgbaIndex++] = (byte) 255; + } + yOffset += width; + } + + return rgbaData; + } + + public static void convertYUV420SemiPlanarToNV12(byte[] yuv420spData, byte[] nv12Data, int width, int height) { + int frameSize = width * height; + int chromaOffset = frameSize; + + // 将 Y 分量复制到 NV12 数据中 + System.arraycopy(yuv420spData, 0, nv12Data, 0, frameSize); + + // 分离 U 和 V 分量，并交错复制到 NV12 数据中 + for (int i = 0, j = 0; i < frameSize / 2; i += 2, j++) { + nv12Data[chromaOffset + j] = yuv420spData[frameSize + i]; // 复制 U 分量 + nv12Data[chromaOffset + j + 1] = yuv420spData[frameSize + i + 1]; // 复制 V 分量 + } + } + + public static Bitmap i420To(byte[] i420, int width, int height){ + YuvImage yuvImage = new YuvImage(i420, ImageFormat.NV21, width, height, null); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + yuvImage.compressToJpeg(new Rect(0, 0, width, height), 100, outputStream); + byte[] jpegData = outputStream.toByteArray(); + Bitmap bitmap = BitmapFactory.decodeByteArray(jpegData, 0, jpegData.length); + return bitmap; + } + + public static Bitmap i420To2(byte[] yuv420spData){ + int width = 1920; // 输入视频的宽度 + int height = 1080; // 输入视频的高度 + int uvPixelStride = 2; // 对于U和V分量交错存储的情况，PixelStride为2 + + int[] argbData = new int[width * height]; + + int yIndex = 0; + int uvIndex = width * height; + + for (int y = 0; y < height; y++) { + int pY = yIndex; + int pUV = uvIndex; + + for (int x = 0; x < width; x++) { + int uv = (y >> 1) * 2 * width + (x & ~1) * uvPixelStride; + + int Y = yuv420spData[pY++] & 0xFF; + int U = (yuv420spData[pUV] & 0xFF) - 128; + int V = (yuv420spData[pUV + 1] & 0xFF) - 128; + + int R = Math.max(0, Math.min(255, Y + (int) (1.402f * V))); + int G = Math.max(0, Math.min(255, Y - (int) (0.344f * U + 0.714f * V))); + int B = Math.max(0, Math.min(255, Y + (int) (1.772f * U))); + + int color = 0xFF000000 | (R << 16) | (G << 8) | B; + argbData[y * width + x] = color; + + if (x % 2 == 1) { + pUV += uvPixelStride; + } + } + + if (y % 2 == 1) { + uvIndex += width * uvPixelStride; + } + } + + return Bitmap.createBitmap(argbData, width, height, Bitmap.Config.ARGB_8888); + } + + public static Bitmap i420To3(byte[] yuv420spData){ + int width = 1024; + int height = 576; + + int[] argb = new int[width * height]; + int pY = 0; + int pUV = width * height; + + for (int i = 0, startY = 0; i < height; i++, startY += width) { + int uvIndex = (i / 2) * width; + + for (int j = 0; j < width; j++) { + int uvOffset = (j / 2) << 1; + int u = (yuv420spData[pUV + uvIndex + uvOffset] & 0xFF) - 128; + int v = (yuv420spData[pUV + uvIndex + uvOffset + 1] & 0xFF) - 128; + + int y = yuv420spData[pY + j] & 0xFF; + + int r = Math.round(y + 1.402f * v); + int g = Math.round(y - 0.3441f * u - 0.7141f * v); + int b = Math.round(y + 1.772f * u); + + r = Math.max(0, Math.min(255, r)); + g = Math.max(0, Math.min(255, g)); + b = Math.max(0, Math.min(255, b)); + + argb[startY + j] = 0xFF000000 | (r << 16) | (g << 8) | b; + } + } + return Bitmap.createBitmap(argb, width, height, Bitmap.Config.ARGB_8888); + } } diff --git a/app/src/main/res/layout/activity_home.xml b/app/src/main/res/layout/activity_home.xml index 1498a72..b2e4e80 100644 --- a/app/src/main/res/layout/activity_home.xml +++ b/app/src/main/res/layout/activity_home.xml @@ -3,7 +3,7 @@ android:layout_width="match_parent" android:layout_height="match_parent"> - 0 && mRecordTime < System.currentTimeMillis() && !TextUtils.isEmpty(mCurrentPath)) { - insertToDB(mRecordTime, mCurrentPath); + if (mStartRecordTime > 0 && mStartRecordTime < System.currentTimeMillis() && !TextUtils.isEmpty(mCurrentPath)) { + insertToDB(mStartRecordTime, mCurrentPath); } } } diff --git a/library-push/src/main/java/com/yinuo/library/vlc/encoder/MediaCodecManager.java b/library-push/src/main/java/com/yinuo/library/vlc/encoder/MediaCodecManager.java new file mode 100644 index 0000000..625a7ed --- /dev/null +++ b/library-push/src/main/java/com/yinuo/library/vlc/encoder/MediaCodecManager.java @@ -0,0 +1,336 @@ +package com.yinuo.library.vlc.encoder; + +import android.annotation.TargetApi; +import android.media.MediaCodec; +import android.media.MediaCodecInfo; +import android.media.MediaFormat; +import android.os.Build; +import android.os.Handler; +import android.os.HandlerThread; + +import androidx.annotation.NonNull; + +import com.common.commonlib.utils.LogUtils; + +import org.easydarwin.PushHelper; + +import java.nio.ByteBuffer; +import java.text.SimpleDateFormat; +import java.util.concurrent.ArrayBlockingQueue; + +@TargetApi(Build.VERSION_CODES.M) +public class MediaCodecManager { + public static boolean DEBUG = false; + + private static final String TAG = "MediaCodecManager"; + // parameters for the encoder + private static final String MIME_TYPE = "video/hevc"; // H.264 Advanced Video + // + private ArrayBlockingQueue frameBytes; + private MediaCodec mMediaCodec; + private int mColorFormat; + private long mStartTime = 0; + private MediaFormat mediaFormat; + private volatile boolean isStart = false; + private volatile boolean isPause = false; + private int dstWidth, dstHeight; + + private static MediaCodecManager sInstance; + private HandlerThread mHandlerThread; + + private boolean isInitCodec; + + private boolean isFlush = false; + + private long lastPauseTime = -1;//上次暂停时间 + private boolean isHasKeyFrame; + + private Handler mHandler; + + private int off_y = 50, off_x = 100; + private int rotation; + + private AndroidMuxer androidMuxer; + protected int mTrackIndex = -1; + + private MediaCodecManager() { + mHandlerThread = new HandlerThread("codecThread"); + mHandlerThread.start(); + mHandler = new Handler(mHandlerThread.getLooper()); + } + + public static MediaCodecManager getInstance() { + if (sInstance == null) { + synchronized (MediaCodecManager.class) { + if (sInstance == null) + sInstance = new MediaCodecManager(); + } + } + return sInstance; + } + + /** + * 初始化编码器 + * + * @param dstWidth + * @param dstHeight + */ + public void initCodecManager(int dstWidth, int dstHeight, int rotation) { + isFlush = false; + if (!isInitCodec) { + mHandler.post(() -> { + frameBytes = new ArrayBlockingQueue<>(100); + MediaCodecManager.this.dstWidth = dstWidth; + MediaCodecManager.this.dstHeight = dstHeight; + MediaCodecManager.this.rotation = rotation; + prepare(); + isInitCodec = true; + }); + } + } + + /** + * 开始编码器 + */ + public void startMediaCodec() { + if (androidMuxer == null) { + androidMuxer = new AndroidMuxer(); + } + if (isStart) { + LogUtils.w("startMediaCodec: was started"); + return; + } + mHandler.post(() -> { + start(); + }); + } + + long pauseTime; + + /** + * 暂停录制 + */ + public void pauseMediaCodec() { + if (!isStart || isPause) { + LogUtils.w(TAG, "MediaCodec: isn't started"); + return; + } + if (androidMuxer != null) { + androidMuxer.release(); + androidMuxer = null; + } + isPause = true; + pauseTime = System.nanoTime(); + frameBytes.clear(); + } + + private long mTime; + + /** + * 继续录制 + */ + public void resumeMediaCodec() { + if (!isStart || !isPause) { + LogUtils.w(TAG, "MediaCodec: was started"); + return; + } + isPause = false; + mTime += System.nanoTime() - pauseTime; + } + + public void releaseManager() { + mHandler.post(() -> { + if (mMediaCodec != null) { + frameBytes.clear(); + mHandlerThread.quit(); +// YuvOsdUtils.releaseOsd(); + stopMediaCodec(); + sInstance = null; + if (androidMuxer != null) { + androidMuxer.release(); + } + } + }); + + + } + + public void addFrameData(byte[] data) { + if (isStart && !isPause) { + boolean isOffer = frameBytes.offer(data); +// Logger1.i(TAG, "addFrameData: isOffer=%s", isOffer); + if (!isOffer) { + frameBytes.poll(); + frameBytes.offer(data); + } + } + } + + /** + * 准备一些需要的参数 + *

+ * YV12: YYYYYYYY VV UU =>YUV420P + * NV12: YYYYYYYY UVUV =>YUV420SP + * NV21: YYYYYYYY VUVU =>YUV420SP + * create at 2017/3/22 18:13 + */ + private void prepare() { + + mColorFormat = MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar;//nv12 最终数据需要nv12 + + int videoW = rotation == 90 || rotation == 270 ? dstHeight : dstWidth; + int videoH = rotation == 90 || rotation == 270 ? dstWidth : dstHeight; + mediaFormat = MediaFormat.createVideoFormat(MIME_TYPE,//注意这里旋转后有一个大坑，就是要交换mHeight，和mWidth的位置。否则录制出来的视频时花屏的。 + videoW, videoH); + int frameRate = 25; // 15fps + int compressRatio = 256; + int bitRate = dstWidth * dstHeight * 3 * 8 * frameRate / compressRatio; + + mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, bitRate); + mediaFormat.setInteger(MediaFormat.KEY_FRAME_RATE, frameRate); + mediaFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, mColorFormat); + mediaFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1); + LogUtils.w("prepare format: " + mediaFormat); + } + + public static final int NAL_I = 19; + public static final int NAL_VPS = 32; + + private byte[] vps_sps_pps_buf; + byte[] outData; + + private void start() { + if (!isInitCodec) + throw new RuntimeException("initCodec is false,please call initCodecManager() before"); + if (isStart) { + LogUtils.w(TAG, "startMediaCodec: was started"); + return; + } + try { + LogUtils.w(TAG, "startMediaCodec: starting"); + + mMediaCodec = MediaCodec.createEncoderByType(MIME_TYPE); + mMediaCodec.configure(mediaFormat, null, null, + MediaCodec.CONFIGURE_FLAG_ENCODE); + + mMediaCodec.setCallback(new MediaCodec.Callback() { + @Override + public void onInputBufferAvailable(@NonNull MediaCodec codec, int index) { + + byte[] data = null; + try { + data = frameBytes.take(); + } catch (InterruptedException e) { + e.printStackTrace(); + return; + } + + + ByteBuffer inputBuffer = codec.getInputBuffer(index); + if (inputBuffer == null) + return; + inputBuffer.clear(); + inputBuffer.put(data); + + long currentTimeUs = (System.nanoTime() - mTime) / 1000;//通过控制时间轴，达到暂停录制，继续录制的效果 + + codec.queueInputBuffer(index, 0, data.length, currentTimeUs, 0); + } + + @Override + public void onOutputBufferAvailable(@NonNull MediaCodec codec, int index, @NonNull MediaCodec.BufferInfo info) { + ByteBuffer outputBuffer = codec.getOutputBuffer(index); + if (!isHasKeyFrame && info.flags == MediaCodec.BUFFER_FLAG_KEY_FRAME) { + isHasKeyFrame = true; + } + if (outData == null){ + outData = new byte[info.size]; + } + if (outputBuffer == null) + return; + else if (info.presentationTimeUs < lastPauseTime || !isHasKeyFrame) {//上一视频的数据，或者无关键帧，丢弃 + //视频第一帧一定要是关键帧 + outputBuffer.get(outData); + } else if ((info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) { + LogUtils.w(TAG, "ignoring BUFFER_FLAG_CODEC_CONFIG"); + } else { + outputBuffer.position(info.offset); + outputBuffer.limit(info.offset + info.size); + if (androidMuxer != null) { + androidMuxer.writeSampleData(mTrackIndex, outputBuffer, info); + } + int offset = 4; + if (outData[2] == 0x01) { + offset = 3; + } + int type = (outData[offset] & 0x7E) >> 1; + if (type == NAL_VPS) { + vps_sps_pps_buf = outData; + PushHelper.INSTANCE.pushData(outData, outData.length, info.presentationTimeUs / 1000); + } else if (type == NAL_I) { + if (vps_sps_pps_buf != null) { + byte[] newBuf = new byte[vps_sps_pps_buf.length + outData.length]; + System.arraycopy(vps_sps_pps_buf, 0, newBuf, 0, vps_sps_pps_buf.length); + System.arraycopy(outData, 0, newBuf, vps_sps_pps_buf.length, outData.length); + outData = newBuf; + PushHelper.INSTANCE.pushData(outData, outData.length, info.presentationTimeUs / 1000); + } + } + } + codec.releaseOutputBuffer(index, false); + } + + @Override + public void onError(@NonNull MediaCodec codec, @NonNull MediaCodec.CodecException e) { + + } + + @Override + public void onOutputFormatChanged(@NonNull MediaCodec codec, @NonNull MediaFormat format) { + MediaFormat newFormat = mMediaCodec.getOutputFormat(); + mTrackIndex = androidMuxer.addTrack(newFormat); + } + }, mHandler); + mMediaCodec.start(); + } catch (Exception e) { + return; + } + isStart = true; + } + + /** + * 返回到接收数据状态 + */ + public synchronized void flushMediaCodec() { + LogUtils.w(TAG, "flushMediaCodec"); + frameBytes.clear(); + isFlush = true; + lastPauseTime = (System.nanoTime()) / 1000;//记录 + + isHasKeyFrame = false; + + } + + private void stopMediaCodec() { + if (isStart && mMediaCodec != null) { + mMediaCodec.stop(); + mMediaCodec.release(); + mMediaCodec = null; + } + isStart = false; + isPause = true; + LogUtils.w(TAG, "stopMediaCodec video"); + } + + private int getIndex(char c) { + if (c >= '0' && c <= '9') + return c - '0'; + else if (c == '-') + return 10; + else if (c == ' ') + return 11; + else if (c == ':') + return 12; + return 11; + } +} diff --git a/library-push/src/main/java/org/easydarwin/TxtOverlay.kt b/library-push/src/main/java/org/easydarwin/TxtOverlay.kt index 1285ba8..c25d42c 100644 --- a/library-push/src/main/java/org/easydarwin/TxtOverlay.kt +++ b/library-push/src/main/java/org/easydarwin/TxtOverlay.kt @@ -32,12 +32,14 @@ object TxtOverlay { // 文字生成的bitmap private var bmp: Bitmap? = null + private var yuv : ByteArray?= null + // 时间格式化字符串 private val dateFormat = SimpleDateFormat("yy-MM-dd HH:mm:ss") - fun buildOverlayBitmap(): Bitmap? { + fun buildOverlayBitmap() { if (TextUtils.isEmpty(mToDoShowTip)) { - return null + return } val currentTimeMillis = System.currentTimeMillis() // 限制获取bitmap的频率，保证性能 @@ -53,14 +55,18 @@ object TxtOverlay { ) // 缩放旋转bitmap // bmp = YUVUtils.rotateImage(bmp, 0); + yuv = YUVUtils.getYUVByBitmap(bmp) } - return bmp } fun getOverlayBitmap(): Bitmap? { return bmp; } + fun getOverlayYuv(): ByteArray? { + return yuv; + } + fun setTipChangeListener(onChange: () -> Unit) { mTipChangeListener = onChange } diff --git a/library-rtsp/src/main/java/org/easydarwin/video/EasyPlayerClient.java b/library-rtsp/src/main/java/org/easydarwin/video/EasyPlayerClient.java index f62c02b..3110a6e 100644 --- a/library-rtsp/src/main/java/org/easydarwin/video/EasyPlayerClient.java +++ b/library-rtsp/src/main/java/org/easydarwin/video/EasyPlayerClient.java @@ -1,10 +1,13 @@ package org.easydarwin.video; +import static android.media.MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible; +import static android.media.MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420PackedSemiPlanar; +import static android.media.MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar; +import static android.media.MediaCodecInfo.CodecCapabilities.COLOR_TI_FormatYUV420PackedSemiPlanar; +import static org.easydarwin.video.Client.TRANSTYPE_TCP; + import android.annotation.TargetApi; import android.content.Context; -import android.media.AudioFormat; -import android.media.AudioManager; -import android.media.AudioTrack; import android.media.MediaCodec; import android.media.MediaCodecInfo; import android.media.MediaCodecList; @@ -13,48 +16,29 @@ import android.os.Build; import android.os.Bundle; import android.os.Process; import android.os.ResultReceiver; -import android.os.SystemClock; import android.preference.PreferenceManager; -import android.text.TextUtils; import android.util.Log; import android.view.Surface; import android.view.TextureView; -import org.easydarwin.audio.AudioCodec; -import org.easydarwin.audio.EasyAACMuxer; +import androidx.lifecycle.Lifecycle; +import androidx.lifecycle.LifecycleObserver; +import androidx.lifecycle.LifecycleOwner; +import androidx.lifecycle.OnLifecycleEvent; + import org.easydarwin.sw.JNIUtil; import org.easydarwin.util.CodecSpecificDataUtil; import org.easydarwin.util.TextureLifecycler; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.security.InvalidParameterException; -import java.util.ArrayList; import java.util.Comparator; import java.util.PriorityQueue; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantLock; -import static android.media.AudioManager.AUDIOFOCUS_REQUEST_GRANTED; -import static android.media.MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible; -import static android.media.MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420PackedPlanar; -import static android.media.MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Planar; -import static android.media.MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420PackedSemiPlanar; -import static android.media.MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar; -import static android.media.MediaCodecInfo.CodecCapabilities.COLOR_TI_FormatYUV420PackedSemiPlanar; -import static org.easydarwin.util.CodecSpecificDataUtil.AUDIO_SPECIFIC_CONFIG_SAMPLING_RATE_TABLE; -import static org.easydarwin.video.Client.TRANSTYPE_TCP; -import static org.easydarwin.video.EasyMuxer2.VIDEO_TYPE_H264; -import static org.easydarwin.video.EasyMuxer2.VIDEO_TYPE_H265; - -import androidx.lifecycle.Lifecycle; -import androidx.lifecycle.LifecycleObserver; -import androidx.lifecycle.LifecycleOwner; -import androidx.lifecycle.OnLifecycleEvent; - /** * Created by John on 2016/3/17. */ @@ -64,8 +48,6 @@ public class EasyPlayerClient implements Client.SourceCallBack { /* 视频编码 */ public static final int EASY_SDK_VIDEO_CODEC_H264 = 0x1C; /* H264 */ public static final int EASY_SDK_VIDEO_CODEC_H265 = 0x48323635; /* H265 */ - public static final int EASY_SDK_VIDEO_CODEC_MJPEG = 0x08; /* MJPEG */ - public static final int EASY_SDK_VIDEO_CODEC_MPEG4 = 0x0D; /* MPEG4 */ /* 音频编码 */ public static final int EASY_SDK_AUDIO_CODEC_AAC = 0x15002; /* AAC */ @@ -96,11 +78,6 @@ public class EasyPlayerClient implements Client.SourceCallBack { public static final int RESULT_RECORD_BEGIN = 7; public static final int RESULT_RECORD_END = 8; - /** - * 表示第一帧数据已经收到 - */ - public static final int RESULT_FRAME_RECVED = 9; - private static final String TAG = EasyPlayerClient.class.getSimpleName(); /** * 表示视频的宽度 @@ -122,30 +99,27 @@ public class EasyPlayerClient implements Client.SourceCallBack { private Client mClient; private boolean mAudioEnable = true; private volatile long mReceivedDataLength; - private AudioTrack mAudioTrack; - private String mRecordingPath; - private EasyAACMuxer mObject; - private EasyMuxer2 muxer2; - private Client.MediaInfo mMediaInfo; + // private AudioTrack mAudioTrack; +// private EasyAACMuxer mObject; +// private EasyMuxer2 muxer2; +// private Client.MediaInfo mMediaInfo; private short mHeight = 0; short mWidth = 0; private ByteBuffer mCSD0; private ByteBuffer mCSD1; private final I420DataCallback i420callback; - private boolean mMuxerWaitingKeyVideo; +// private boolean mMuxerWaitingKeyVideo; - /** - * -1 表示暂停中，0表示正常录像中，1表示恢复中。 - */ - private int mRecordingStatus; - private long muxerPausedMillis = 0L; - private long mMuxerCuttingMillis = 0L; +// /** +// * -1 表示暂停中，0表示正常录像中，1表示恢复中。 +// */ +// private int mRecordingStatus; +// private long muxerPausedMillis = 0L; +// private long mMuxerCuttingMillis = 0L; -// private RtmpClient mRTMPClient = new RtmpClient(); - - public boolean isRecording() { - return !TextUtils.isEmpty(mRecordingPath); - } +// public boolean isRecording() { +// return !TextUtils.isEmpty(mRecordingPath); +// } private static class FrameInfoQueue extends PriorityQueue { public static final int CAPACITY = 500; @@ -295,10 +269,6 @@ public class EasyPlayerClient implements Client.SourceCallBack { private final Context mContext; - /** - * 最新的视频时间戳 - */ - private volatile long mNewestStample; private boolean mWaitingKeyFrame; private boolean mTimeout; private boolean mNotSupportedVideoCB, mNotSupportedAudioCB; @@ -378,10 +348,6 @@ public class EasyPlayerClient implements Client.SourceCallBack { * @return */ public void play(final String url) { - if (lifecycler == null){ - start(url, TRANSTYPE_TCP, 0, Client.EASY_SDK_VIDEO_FRAME_FLAG | Client.EASY_SDK_AUDIO_FRAME_FLAG, "", "", null); - return; - } if (lifecycler.getLifecycle().getCurrentState().isAtLeast(Lifecycle.State.CREATED)) { start(url, TRANSTYPE_TCP, 0, Client.EASY_SDK_VIDEO_FRAME_FLAG | Client.EASY_SDK_AUDIO_FRAME_FLAG, "", "", null); } else { @@ -426,18 +392,16 @@ public class EasyPlayerClient implements Client.SourceCallBack { } if (type == 0) type = TRANSTYPE_TCP; - mNewestStample = 0; mWaitingKeyFrame = PreferenceManager.getDefaultSharedPreferences(mContext).getBoolean("waiting_i_frame", true); mWidth = mHeight = 0; mQueue.clear(); startCodec(); - startAudio(); +// startAudio(); mTimeout = false; mNotSupportedVideoCB = mNotSupportedAudioCB = false; mReceivedDataLength = 0; mClient = new Client(mContext); int channel = mClient.registerCallback(this); - mRecordingPath = recordPath; Log.i(TAG, String.format("playing url:\n%s\n", url)); return mClient.openStream(channel, url, type, sendOption, mediaType, user, pwd); } @@ -446,25 +410,25 @@ public class EasyPlayerClient implements Client.SourceCallBack { return mAudioEnable; } - public void setAudioEnable(boolean enable) { - mAudioEnable = enable; - AudioTrack at = mAudioTrack; - if (at != null) { - Log.i(TAG, String.format("audio will be %s", enable ? "enabled" : "disabled")); - synchronized (at) { - if (!enable) { - at.pause(); - at.flush(); - } else { - at.flush(); - at.play(); - } - } - } - } +// public void setAudioEnable(boolean enable) { +// mAudioEnable = enable; +// AudioTrack at = mAudioTrack; +// if (at != null) { +// Log.i(TAG, String.format("audio will be %s", enable ? "enabled" : "disabled")); +// synchronized (at) { +// if (!enable) { +// at.pause(); +// at.flush(); +// } else { +// at.flush(); +// at.play(); +// } +// } +// } +// } public static interface I420DataCallback { - public void onI420Data(ByteBuffer buffer); + public void onI420Data(byte[] buffer); } public void pause() { @@ -506,7 +470,7 @@ public class EasyPlayerClient implements Client.SourceCallBack { e.printStackTrace(); } } - stopRecord(); +// stopRecord(); mQueue.clear(); if (mClient != null) { @@ -520,146 +484,145 @@ public class EasyPlayerClient implements Client.SourceCallBack { } mQueue.clear(); mClient = null; - mNewestStample = 0; } public long receivedDataLength() { return mReceivedDataLength; } - private void startAudio() { - mAudioThread = new Thread("AUDIO_CONSUMER") { - - @TargetApi(Build.VERSION_CODES.JELLY_BEAN) - @Override - public void run() { - { - Process.setThreadPriority(Process.THREAD_PRIORITY_AUDIO); - Client.FrameInfo frameInfo; - long handle = 0; - final AudioManager am = (AudioManager) mContext.getSystemService(Context.AUDIO_SERVICE); - AudioManager.OnAudioFocusChangeListener l = new AudioManager.OnAudioFocusChangeListener() { - @Override - public void onAudioFocusChange(int focusChange) { - if (focusChange == AudioManager.AUDIOFOCUS_GAIN) { - AudioTrack audioTrack = mAudioTrack; - if (audioTrack != null) { - audioTrack.setStereoVolume(1.0f, 1.0f); - if (audioTrack.getPlayState() == AudioTrack.PLAYSTATE_PAUSED) { - audioTrack.flush(); - audioTrack.play(); - } - } - } else if (focusChange == AudioManager.AUDIOFOCUS_LOSS) { - AudioTrack audioTrack = mAudioTrack; - if (audioTrack != null) { - if (audioTrack.getPlayState() == AudioTrack.PLAYSTATE_PLAYING) { - audioTrack.pause(); - } - } - } else if (focusChange == AudioManager.AUDIOFOCUS_LOSS_TRANSIENT_CAN_DUCK) { - AudioTrack audioTrack = mAudioTrack; - if (audioTrack != null) { - audioTrack.setStereoVolume(0.5f, 0.5f); - } - } - } - }; - try { - int requestCode = am.requestAudioFocus(l, AudioManager.STREAM_MUSIC, AudioManager.AUDIOFOCUS_GAIN); - if (requestCode != AUDIOFOCUS_REQUEST_GRANTED) { - return; - } - do { - frameInfo = mQueue.takeAudioFrame(); - if (mMediaInfo != null) break; - } while (true); - final Thread t = Thread.currentThread(); - - if (mAudioTrack == null) { - int sampleRateInHz = (int) (mMediaInfo.sample * 1.001); - int channelConfig = mMediaInfo.channel == 1 ? AudioFormat.CHANNEL_OUT_MONO : AudioFormat.CHANNEL_OUT_STEREO; - int audioFormat = AudioFormat.ENCODING_PCM_16BIT; - int bfSize = AudioTrack.getMinBufferSize(mMediaInfo.sample, channelConfig, audioFormat) * 8; - mAudioTrack = new AudioTrack(AudioManager.STREAM_MUSIC, sampleRateInHz, channelConfig, audioFormat, bfSize, AudioTrack.MODE_STREAM); - } - mAudioTrack.play(); - handle = AudioCodec.create(frameInfo.codec, frameInfo.sample_rate, frameInfo.channels, frameInfo.bits_per_sample); - - Log.w(TAG, String.format("POST VIDEO_DISPLAYED IN AUDIO THREAD!!!")); - ResultReceiver rr = mRR; - if (rr != null) rr.send(RESULT_VIDEO_DISPLAYED, null); - - // 半秒钟的数据缓存 - byte[] mBufferReuse = new byte[16000]; - int[] outLen = new int[1]; - while (mAudioThread != null) { - if (frameInfo == null) { - frameInfo = mQueue.takeAudioFrame(); - } - if (frameInfo.codec == EASY_SDK_AUDIO_CODEC_AAC && false) { - pumpAACSample(frameInfo); - } - outLen[0] = mBufferReuse.length; - long ms = SystemClock.currentThreadTimeMillis(); - int nRet = AudioCodec.decode(handle, frameInfo.buffer, 0, frameInfo.length, mBufferReuse, outLen); - if (nRet == 0) { -// if (frameInfo.codec != EASY_SDK_AUDIO_CODEC_AAC ) - { -// save2path(mBufferReuse, 0, outLen[0],"/sdcard/111.pcm", true); - pumpPCMSample(mBufferReuse, outLen[0], frameInfo.stamp); - } - if (mAudioEnable) - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { - mAudioTrack.write(mBufferReuse, 0, outLen[0], AudioTrack.WRITE_NON_BLOCKING); - } else { - mAudioTrack.write(mBufferReuse, 0, outLen[0]); - } - - } - frameInfo = null; - } - } catch (Exception ex) { - ex.printStackTrace(); - } finally { - am.abandonAudioFocus(l); - if (handle != 0) { - AudioCodec.close(handle); - } - AudioTrack track = mAudioTrack; - if (track != null) { - synchronized (track) { - mAudioTrack = null; - track.release(); - } - } - } - } - } - }; - - mAudioThread.start(); - } - - private static void save2path(byte[] buffer, int offset, int length, String path, boolean append) { - FileOutputStream fos = null; - try { - fos = new FileOutputStream(path, append); - fos.write(buffer, offset, length); - } catch (FileNotFoundException e) { - e.printStackTrace(); - } catch (IOException e) { - e.printStackTrace(); - } finally { - if (fos != null) { - try { - fos.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - } - } +// private void startAudio() { +// mAudioThread = new Thread("AUDIO_CONSUMER") { +// +// @TargetApi(Build.VERSION_CODES.JELLY_BEAN) +// @Override +// public void run() { +// { +// Process.setThreadPriority(Process.THREAD_PRIORITY_AUDIO); +// Client.FrameInfo frameInfo; +// long handle = 0; +// final AudioManager am = (AudioManager) mContext.getSystemService(Context.AUDIO_SERVICE); +// AudioManager.OnAudioFocusChangeListener l = new AudioManager.OnAudioFocusChangeListener() { +// @Override +// public void onAudioFocusChange(int focusChange) { +// if (focusChange == AudioManager.AUDIOFOCUS_GAIN) { +// AudioTrack audioTrack = mAudioTrack; +// if (audioTrack != null) { +// audioTrack.setStereoVolume(1.0f, 1.0f); +// if (audioTrack.getPlayState() == AudioTrack.PLAYSTATE_PAUSED) { +// audioTrack.flush(); +// audioTrack.play(); +// } +// } +// } else if (focusChange == AudioManager.AUDIOFOCUS_LOSS) { +// AudioTrack audioTrack = mAudioTrack; +// if (audioTrack != null) { +// if (audioTrack.getPlayState() == AudioTrack.PLAYSTATE_PLAYING) { +// audioTrack.pause(); +// } +// } +// } else if (focusChange == AudioManager.AUDIOFOCUS_LOSS_TRANSIENT_CAN_DUCK) { +// AudioTrack audioTrack = mAudioTrack; +// if (audioTrack != null) { +// audioTrack.setStereoVolume(0.5f, 0.5f); +// } +// } +// } +// }; +// try { +// int requestCode = am.requestAudioFocus(l, AudioManager.STREAM_MUSIC, AudioManager.AUDIOFOCUS_GAIN); +// if (requestCode != AUDIOFOCUS_REQUEST_GRANTED) { +// return; +// } +// do { +// frameInfo = mQueue.takeAudioFrame(); +// if (mMediaInfo != null) break; +// } while (true); +// final Thread t = Thread.currentThread(); +// +// if (mAudioTrack == null) { +// int sampleRateInHz = (int) (mMediaInfo.sample * 1.001); +// int channelConfig = mMediaInfo.channel == 1 ? AudioFormat.CHANNEL_OUT_MONO : AudioFormat.CHANNEL_OUT_STEREO; +// int audioFormat = AudioFormat.ENCODING_PCM_16BIT; +// int bfSize = AudioTrack.getMinBufferSize(mMediaInfo.sample, channelConfig, audioFormat) * 8; +// mAudioTrack = new AudioTrack(AudioManager.STREAM_MUSIC, sampleRateInHz, channelConfig, audioFormat, bfSize, AudioTrack.MODE_STREAM); +// } +// mAudioTrack.play(); +// handle = AudioCodec.create(frameInfo.codec, frameInfo.sample_rate, frameInfo.channels, frameInfo.bits_per_sample); +// +// Log.w(TAG, String.format("POST VIDEO_DISPLAYED IN AUDIO THREAD!!!")); +// ResultReceiver rr = mRR; +// if (rr != null) rr.send(RESULT_VIDEO_DISPLAYED, null); +// +// // 半秒钟的数据缓存 +// byte[] mBufferReuse = new byte[16000]; +// int[] outLen = new int[1]; +// while (mAudioThread != null) { +// if (frameInfo == null) { +// frameInfo = mQueue.takeAudioFrame(); +// } +// if (frameInfo.codec == EASY_SDK_AUDIO_CODEC_AAC && false) { +// pumpAACSample(frameInfo); +// } +// outLen[0] = mBufferReuse.length; +// long ms = SystemClock.currentThreadTimeMillis(); +// int nRet = AudioCodec.decode(handle, frameInfo.buffer, 0, frameInfo.length, mBufferReuse, outLen); +// if (nRet == 0) { +//// if (frameInfo.codec != EASY_SDK_AUDIO_CODEC_AAC ) +// { +//// save2path(mBufferReuse, 0, outLen[0],"/sdcard/111.pcm", true); +// pumpPCMSample(mBufferReuse, outLen[0], frameInfo.stamp); +// } +// if (mAudioEnable) +// if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { +// mAudioTrack.write(mBufferReuse, 0, outLen[0], AudioTrack.WRITE_NON_BLOCKING); +// } else { +// mAudioTrack.write(mBufferReuse, 0, outLen[0]); +// } +// +// } +// frameInfo = null; +// } +// } catch (Exception ex) { +// ex.printStackTrace(); +// } finally { +// am.abandonAudioFocus(l); +// if (handle != 0) { +// AudioCodec.close(handle); +// } +// AudioTrack track = mAudioTrack; +// if (track != null) { +// synchronized (track) { +// mAudioTrack = null; +// track.release(); +// } +// } +// } +// } +// } +// }; +// +// mAudioThread.start(); +// } + +// private static void save2path(byte[] buffer, int offset, int length, String path, boolean append) { +// FileOutputStream fos = null; +// try { +// fos = new FileOutputStream(path, append); +// fos.write(buffer, offset, length); +// } catch (FileNotFoundException e) { +// e.printStackTrace(); +// } catch (IOException e) { +// e.printStackTrace(); +// } finally { +// if (fos != null) { +// try { +// fos.close(); +// } catch (IOException e) { +// e.printStackTrace(); +// } +// } +// } +// } private static int getXPS(byte[] data, int offset, int length, byte[] dataOut, int[] outLen, int type) { int i; @@ -795,32 +758,32 @@ public class EasyPlayerClient implements Client.SourceCallBack { return false; } - private static String codecName() { - ArrayList array = new ArrayList<>(); - int numCodecs = MediaCodecList.getCodecCount(); - for (int i1 = 0; i1 < numCodecs; i1++) { - MediaCodecInfo codecInfo = MediaCodecList.getCodecInfoAt(i1); - if (codecInfo.isEncoder()) { - continue; - } - - if (codecMatch("video/avc", codecInfo)) { - String name = codecInfo.getName(); - Log.d("DECODER", String.format("decoder:%s", name)); - array.add(name); - } - } -// if (array.remove("OMX.qcom.video.decoder.avc")) { -// array.add("OMX.qcom.video.decoder.avc"); +// private static String codecName() { +// ArrayList array = new ArrayList<>(); +// int numCodecs = MediaCodecList.getCodecCount(); +// for (int i1 = 0; i1 < numCodecs; i1++) { +// MediaCodecInfo codecInfo = MediaCodecList.getCodecInfoAt(i1); +// if (codecInfo.isEncoder()) { +// continue; +// } +// +// if (codecMatch("video/avc", codecInfo)) { +// String name = codecInfo.getName(); +// Log.d("DECODER", String.format("decoder:%s", name)); +// array.add(name); +// } // } -// if (array.remove("OMX.amlogic.avc.decoder.awesome")) { -// array.add("OMX.amlogic.avc.decoder.awesome"); +//// if (array.remove("OMX.qcom.video.decoder.avc")) { +//// array.add("OMX.qcom.video.decoder.avc"); +//// } +//// if (array.remove("OMX.amlogic.avc.decoder.awesome")) { +//// array.add("OMX.amlogic.avc.decoder.awesome"); +//// } +// if (array.isEmpty()) { +// return ""; // } - if (array.isEmpty()) { - return ""; - } - return array.get(0); - } +// return array.get(0); +// } private static MediaCodecInfo selectCodec(String mimeType) { int numCodecs = MediaCodecList.getCodecCount(); @@ -841,6 +804,9 @@ public class EasyPlayerClient implements Client.SourceCallBack { return null; } + byte[] in = new byte[1920 * 1080 * 3 / 2]; + byte[] nv12Data = new byte[in.length]; + private void startCodec() { mThread = new Thread("VIDEO_CONSUMER") { @@ -857,18 +823,9 @@ public class EasyPlayerClient implements Client.SourceCallBack { int index = 0; // previous - long previousStampUs = 0l; - long lastFrameStampUs = 0l; long differ = 0; int realWidth = mWidth; int realHeight = mHeight; - int sliceHeight = realHeight; - - int frameWidth = 0; - int frameHeight = 0; -// -// long decodeBegin = 0; -// long current = 0; MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); @@ -880,14 +837,8 @@ public class EasyPlayerClient implements Client.SourceCallBack { if (frameInfo == null) { frameInfo = mQueue.takeVideoFrame(); } - initFrameInfo = frameInfo; - try { - if (PreferenceManager.getDefaultSharedPreferences(mContext).getBoolean("use-sw-codec", false)) { - throw new IllegalStateException("user set sw codec"); - } - final String mime = frameInfo.codec == EASY_SDK_VIDEO_CODEC_H264 ? "video/avc" : "video/hevc"; MediaFormat format = MediaFormat.createVideoFormat(mime, mWidth, mHeight); format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 0); @@ -930,16 +881,16 @@ public class EasyPlayerClient implements Client.SourceCallBack { Log.i(TAG, String.format("config codec:%s", format)); MediaCodec codec = MediaCodec.createByCodecName(ci.getName()); - codec.configure(format, mSurface, null, 0); + codec.configure(format, i420callback != null ? null : mSurface, null, 0); codec.setVideoScalingMode(MediaCodec.VIDEO_SCALING_MODE_SCALE_TO_FIT); codec.start(); mCodec = codec; -// if (i420callback != null) { -// final VideoCodec.VideoDecoderLite decoder = new VideoCodec.VideoDecoderLite(); -// decoder.create(mSurface, frameInfo.codec == EASY_SDK_VIDEO_CODEC_H264); -// displayer = decoder; -// } + if (i420callback != null) { + final VideoCodec.VideoDecoderLite decoder = new VideoCodec.VideoDecoderLite(); + decoder.create(mSurface, frameInfo.codec == EASY_SDK_VIDEO_CODEC_H264); + displayer = decoder; + } } catch (Throwable e) { if (mCodec != null) { mCodec.release(); @@ -960,273 +911,103 @@ public class EasyPlayerClient implements Client.SourceCallBack { decoder.create(mSurface, frameInfo.codec == EASY_SDK_VIDEO_CODEC_H264); mDecoder = decoder; } - -// previewTickUs = mTexture.getTimestamp(); -// differ = previewTickUs - frameInfo.stamp; -// index = mCodec.dequeueInputBuffer(0); -// if (index >= 0) { -// ByteBuffer buffer = mCodec.getInputBuffers()[index]; -// buffer.clear(); -// mCSD0.clear(); -// mCSD1.clear(); -// buffer.put(mCSD0.array(), 0, mCSD0.remaining()); -// buffer.put(mCSD1.array(), 0, mCSD1.remaining()); -// mCodec.queueInputBuffer(index, 0, buffer.position(), 0, MediaCodec.BUFFER_FLAG_CODEC_CONFIG); -// } } else { frameInfo = mQueue.takeVideoFrame(5); } - - if (frameInfo != null) { - Log.d(TAG, "video " + frameInfo.stamp + " take[" + (frameInfo.stamp - lastFrameStampUs) + "]"); - if (frameHeight != 0 && frameWidth != 0) { - if (frameInfo.width != 0 && frameInfo.height != 0) { - if (frameInfo.width != frameWidth || frameInfo.height != frameHeight) { - frameHeight = frameInfo.height; - frameWidth = frameInfo.width; - stopRecord(); - if (mCodec != null) { - mCodec.release(); - mCodec = null; - continue; - } - } - } - } - frameHeight = frameInfo.height; - frameWidth = frameInfo.width; - pumpVideoSample(frameInfo); - lastFrameStampUs = frameInfo.stamp; - } - do { - if (mDecoder != null) { - if (frameInfo != null) { - long decodeBegin = SystemClock.elapsedRealtime(); - int[] size = new int[2]; - -// mDecoder.decodeFrame(frameInfo, size); - ByteBuffer buf = mDecoder.decodeFrameYUV(frameInfo, size); - - if (i420callback != null && buf != null) { - i420callback.onI420Data(buf); - } - - if (buf != null) { - mDecoder.releaseBuffer(buf); - Log.i(TAG, "AAAA 1022 releaseBuffer "); - } - - long decodeSpend = SystemClock.elapsedRealtime() - decodeBegin; - - boolean firstFrame = previousStampUs == 0l; - if (firstFrame) { - Log.i(TAG, String.format("POST VIDEO_DISPLAYED!!!")); - ResultReceiver rr = mRR; - if (rr != null) { - Bundle data = new Bundle(); - data.putInt(KEY_VIDEO_DECODE_TYPE, 0); - rr.send(RESULT_VIDEO_DISPLAYED, data); - } - } - - //Log.d(TAG, String.format("timestamp=%d diff=%d",current, current - previousStampUs )); - - if (previousStampUs != 0l) { - long sleepTime = frameInfo.stamp - previousStampUs - decodeSpend * 1000; - if (sleepTime > 100000) { - Log.w(TAG, "sleep time.too long:" + sleepTime); - sleepTime = 100000; - } - if (sleepTime > 0) { - sleepTime %= 100000; - long cache = mNewestStample - frameInfo.stamp; - sleepTime = fixSleepTime(sleepTime, cache, 50000); - if (sleepTime > 0) { - Thread.sleep(sleepTime / 1000); + try { + do { + if (frameInfo != null) { + byte[] pBuf = frameInfo.buffer; + index = mCodec.dequeueInputBuffer(10); + if (index >= 0) { + ByteBuffer buffer = mCodec.getInputBuffer(index); + buffer.clear(); + if (pBuf.length > buffer.remaining()) { + mCodec.queueInputBuffer(index, 0, 0, frameInfo.stamp, 0); + } else { + buffer.put(pBuf, frameInfo.offset, frameInfo.length); + mCodec.queueInputBuffer(index, 0, buffer.position(), frameInfo.stamp + differ, 0); } - Log.d(TAG, "cache:" + cache); + frameInfo = null; } } - previousStampUs = frameInfo.stamp; - } - } else { - try { - do { - if (frameInfo != null) { - byte[] pBuf = frameInfo.buffer; - index = mCodec.dequeueInputBuffer(10); - if (false) - throw new IllegalStateException("fake state"); - if (index >= 0) { - ByteBuffer buffer = mCodec.getInputBuffers()[index]; - buffer.clear(); - if (pBuf.length > buffer.remaining()) { - mCodec.queueInputBuffer(index, 0, 0, frameInfo.stamp, 0); - } else { - buffer.put(pBuf, frameInfo.offset, frameInfo.length); - mCodec.queueInputBuffer(index, 0, buffer.position(), frameInfo.stamp + differ, 0); - } - frameInfo = null; + index = mCodec.dequeueOutputBuffer(info, 10); // + switch (index) { + case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED: + break; + case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED: + MediaFormat mf = mCodec.getOutputFormat(); + int width = mf.getInteger(MediaFormat.KEY_WIDTH); + if (mf.containsKey("crop-left") && mf.containsKey("crop-right")) { + width = mf.getInteger("crop-right") + 1 - mf.getInteger("crop-left"); } - } - index = mCodec.dequeueOutputBuffer(info, 10); // - switch (index) { - case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED: - Log.i(TAG, "INFO_OUTPUT_BUFFERS_CHANGED"); - break; - case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED: - MediaFormat mf = mCodec.getOutputFormat(); - Log.i(TAG, "INFO_OUTPUT_FORMAT_CHANGED ：" + mf); - - int width = mf.getInteger(MediaFormat.KEY_WIDTH); - if (mf.containsKey("crop-left") && mf.containsKey("crop-right")) { - width = mf.getInteger("crop-right") + 1 - mf.getInteger("crop-left"); - } - - int height = mf.getInteger(MediaFormat.KEY_HEIGHT); - if (mf.containsKey("crop-top") && mf.containsKey("crop-bottom")) { - height = mf.getInteger("crop-bottom") + 1 - mf.getInteger("crop-top"); - } - - realWidth = width; - realHeight = height; - - if (mf.containsKey(MediaFormat.KEY_SLICE_HEIGHT)) { - sliceHeight = mf.getInteger(MediaFormat.KEY_SLICE_HEIGHT); - } else { - sliceHeight = realHeight; - } - break; - case MediaCodec.INFO_TRY_AGAIN_LATER: - // 输出为空 - break; - default: - // 输出队列不为空 - // -1表示为第一帧数据 - long newSleepUs = -1; - boolean firstTime = previousStampUs == 0l; - if (!firstTime) { - long sleepUs = (info.presentationTimeUs - previousStampUs); - if (sleepUs > 100000) { - // 时间戳异常，可能服务器丢帧了。 - Log.w(TAG, "sleep time.too long:" + sleepUs); - sleepUs = 100000; - } else if (sleepUs < 0) { - Log.w(TAG, "sleep time.too short:" + sleepUs); - sleepUs = 0; - } - - { - long cache = mNewestStample - lastFrameStampUs; - newSleepUs = fixSleepTime(sleepUs, cache, 100000); - // Log.d(TAG, String.format("sleepUs:%d,newSleepUs:%d,Cache:%d", sleepUs, newSleepUs, cache)); - } - } - - //previousStampUs = info.presentationTimeUs; - ByteBuffer outputBuffer; - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { - outputBuffer = mCodec.getOutputBuffer(index); - } else { - outputBuffer = mCodec.getOutputBuffers()[index]; - } - - if (i420callback != null && outputBuffer != null) { -// if (sliceHeight != realHeight) { -// ByteBuffer tmp = ByteBuffer.allocateDirect(realWidth * realHeight * 3 / 2); -// outputBuffer.clear(); -// outputBuffer.limit(realWidth * realHeight); -// tmp.put(outputBuffer); -// -// outputBuffer.clear(); -// outputBuffer.position(realWidth * sliceHeight); -// outputBuffer.limit((realWidth * sliceHeight + realWidth * realHeight / 4)); -// tmp.put(outputBuffer); -// -// outputBuffer.clear(); -// outputBuffer.position(realWidth * sliceHeight + realWidth * realHeight / 4); -// outputBuffer.limit((realWidth * sliceHeight + realWidth * realHeight / 4 + realWidth * realHeight / 4)); -// tmp.put(outputBuffer); -// -// tmp.clear(); -// outputBuffer = tmp; -// } - - if (mColorFormat == COLOR_FormatYUV420SemiPlanar - || mColorFormat == COLOR_FormatYUV420PackedSemiPlanar - || mColorFormat == COLOR_TI_FormatYUV420PackedSemiPlanar) { - -// byte[] in = new byte[realWidth * realHeight * 3 / 2]; -// outputBuffer.clear(); -// outputBuffer.get(in); -// -// // yuvuv_to_yuv -// JNIUtil.yuvConvert(in, realWidth, realHeight, 4); -//// // 旋转90或180或270度 -//// yuvRotate(in, 0, realWidth, realHeight, 90); -// -// ByteBuffer tmp = ByteBuffer.allocateDirect(realWidth * realHeight * 3 / 2); -// tmp.clear(); -// tmp.put(in); - - i420callback.onI420Data(null); - -// // 旋转90或270度，则宽高需要互换 -// displayer.decoder_decodeBuffer(tmp, realWidth, realHeight); - } - } - - //previewStampUs = info.presentationTimeUs; - if (false && Build.VERSION.SDK_INT >= 21) { - Log.d(TAG, String.format("releaseoutputbuffer:%d,stampUs:%d", index, previousStampUs)); - mCodec.releaseOutputBuffer(index, previousStampUs); - } else { - if (newSleepUs < 0) { - newSleepUs = 0; - } -// Log.d(TAG,String.format("sleep:%d", newSleepUs/1000)); - Thread.sleep(newSleepUs / 1000); - mCodec.releaseOutputBuffer(index, true); - } - - if (firstTime) { - Log.i(TAG, String.format("POST VIDEO_DISPLAYED!!!")); - ResultReceiver rr = mRR; - if (rr != null) { - Bundle data = new Bundle(); - data.putInt(KEY_VIDEO_DECODE_TYPE, 1); - rr.send(RESULT_VIDEO_DISPLAYED, data); + int height = mf.getInteger(MediaFormat.KEY_HEIGHT); + if (mf.containsKey("crop-top") && mf.containsKey("crop-bottom")) { + height = mf.getInteger("crop-bottom") + 1 - mf.getInteger("crop-top"); + } + realWidth = width; + realHeight = height; + break; + case MediaCodec.INFO_TRY_AGAIN_LATER: + // 输出为空 + break; + default: + // 输出队列不为空 + ByteBuffer outputBuffer; + outputBuffer = mCodec.getOutputBuffer(index); + if (i420callback != null && outputBuffer != null) { + if (mColorFormat == COLOR_FormatYUV420SemiPlanar + || mColorFormat == COLOR_FormatYUV420PackedSemiPlanar + || mColorFormat == COLOR_TI_FormatYUV420PackedSemiPlanar) { + mIndex = mIndex + 1; + if (mIndex < 3) { + outputBuffer.clear(); + outputBuffer.get(in); + System.arraycopy(in, 0, nv12Data, 0, in.length); + // nv12 + i420callback.onI420Data(nv12Data); + // yuvuv_to_yuv + JNIUtil.yuvConvert(in, realWidth, realHeight, 4); +// // 旋转90或180或270度 +// yuvRotate(in, 0, realWidth, realHeight, 90); + + ByteBuffer tmp = ByteBuffer.allocateDirect(realWidth * realHeight * 3 / 2); + tmp.clear(); + tmp.put(in); + // 旋转90或270度，则宽高需要互换 + displayer.decoder_decodeBuffer(tmp, realWidth, realHeight); + } else { + mIndex = 0; } } - previousStampUs = info.presentationTimeUs; - } - + } + mCodec.releaseOutputBuffer(index, false); } - while (frameInfo != null || index < MediaCodec.INFO_TRY_AGAIN_LATER); - } catch (IllegalStateException ex) { - // mediacodec error... - ex.printStackTrace(); + } + while (frameInfo != null || index < MediaCodec.INFO_TRY_AGAIN_LATER); + } catch (IllegalStateException ex) { + // mediacodec error... - Log.e(TAG, String.format("init codec error due to %s", ex.getMessage())); + ex.printStackTrace(); - if (mCodec != null) - mCodec.release(); - mCodec = null; + Log.e(TAG, String.format("init codec error due to %s", ex.getMessage())); - if (displayer != null) { - displayer.close(); - Log.i(TAG, "AAAA 1217 displayer.close()"); - } - displayer = null; + if (mCodec != null) + mCodec.release(); + mCodec = null; - final VideoCodec.VideoDecoderLite decoder = new VideoCodec.VideoDecoderLite(); - decoder.create(mSurface, initFrameInfo.codec == EASY_SDK_VIDEO_CODEC_H264); - mDecoder = decoder; - continue; + if (displayer != null) { + displayer.close(); + Log.i(TAG, "AAAA 1217 displayer.close()"); } + displayer = null; + final VideoCodec.VideoDecoderLite decoder = new VideoCodec.VideoDecoderLite(); + decoder.create(mSurface, initFrameInfo.codec == EASY_SDK_VIDEO_CODEC_H264); + mDecoder = decoder; + continue; } break; } while (true); @@ -1255,178 +1036,176 @@ public class EasyPlayerClient implements Client.SourceCallBack { mThread.start(); } - private static final long fixSleepTime(long sleepTimeUs, long totalTimestampDifferUs, long delayUs) { - if (totalTimestampDifferUs < 0l) { - Log.w(TAG, String.format("totalTimestampDifferUs is:%d, this should not be happen.", totalTimestampDifferUs)); - totalTimestampDifferUs = 0; - } - - double dValue = ((double) (delayUs - totalTimestampDifferUs)) / 1000000d; - double radio = Math.exp(dValue); - double r = sleepTimeUs * radio + 0.5f; - Log.i(TAG, String.format("%d,%d,%d->%d", sleepTimeUs, totalTimestampDifferUs, delayUs, (int) r)); - return (long) r; - } +// private static final long fixSleepTime(long sleepTimeUs, long totalTimestampDifferUs, long delayUs) { +// if (totalTimestampDifferUs < 0l) { +// Log.w(TAG, String.format("totalTimestampDifferUs is:%d, this should not be happen.", totalTimestampDifferUs)); +// totalTimestampDifferUs = 0; +// } +// +// double dValue = ((double) (delayUs - totalTimestampDifferUs)) / 1000000d; +// double radio = Math.exp(dValue); +// double r = sleepTimeUs * radio + 0.5f; +// Log.i(TAG, String.format("%d,%d,%d->%d", sleepTimeUs, totalTimestampDifferUs, delayUs, (int) r)); +// return (long) r; +// } @TargetApi(Build.VERSION_CODES.JELLY_BEAN_MR2) - public synchronized void startRecord(String path) { - if (mMediaInfo == null || mWidth == 0 || mHeight == 0 || mCSD0 == null) - return; - - mRecordingPath = path; - EasyMuxer2 muxer2 = new EasyMuxer2(); - mMuxerCuttingMillis = 0l; - mRecordingStatus = 0; - muxerPausedMillis = 0; - ByteBuffer csd1 = this.mCSD1; - if (csd1 == null) csd1 = ByteBuffer.allocate(0); - byte[] extra = new byte[mCSD0.capacity() + csd1.capacity()]; - mCSD0.clear(); - csd1.clear(); - mCSD0.get(extra, 0, mCSD0.capacity()); - csd1.get(extra, mCSD0.capacity(), csd1.capacity()); - - int r = muxer2.create(path, mMediaInfo.videoCodec == EASY_SDK_VIDEO_CODEC_H265 ? VIDEO_TYPE_H265 : VIDEO_TYPE_H264, mWidth, mHeight, extra, mMediaInfo.sample, mMediaInfo.channel); - if (r != 0) { - Log.w(TAG, "create muxer2:" + r); - return; - } - - mMuxerWaitingKeyVideo = true; - this.muxer2 = muxer2; - - ResultReceiver rr = mRR; - if (rr != null) { - rr.send(RESULT_RECORD_BEGIN, null); - } - } - - public synchronized void pauseRecord() { - if (mRecordingStatus != -1) { - mRecordingStatus = -1; - muxerPausedMillis = SystemClock.elapsedRealtime(); - } - } - - public synchronized void resumeRecord() { - if (mRecordingStatus == -1) { - mMuxerWaitingKeyVideo = true; - mRecordingStatus = 1; - } - } - - private static int getSampleIndex(int sample) { - for (int i = 0; i < AUDIO_SPECIFIC_CONFIG_SAMPLING_RATE_TABLE.length; i++) { - if (sample == AUDIO_SPECIFIC_CONFIG_SAMPLING_RATE_TABLE[i]) { - return i; - } - } - return -1; - } - - private void pumpAACSample(Client.FrameInfo frameInfo) { - EasyMuxer muxer = mObject; - if (muxer == null) return; - MediaCodec.BufferInfo bi = new MediaCodec.BufferInfo(); - bi.offset = frameInfo.offset; - bi.size = frameInfo.length; - ByteBuffer buffer = ByteBuffer.wrap(frameInfo.buffer, bi.offset, bi.size); - bi.presentationTimeUs = frameInfo.stamp; - - try { - if (!frameInfo.audio) { - throw new IllegalArgumentException("frame should be audio!"); - } - if (frameInfo.codec != EASY_SDK_AUDIO_CODEC_AAC) { - throw new IllegalArgumentException("audio codec should be aac!"); - } - bi.offset += 7; - bi.size -= 7; - muxer.pumpStream(buffer, bi, false); - } catch (IllegalStateException ex) { - ex.printStackTrace(); - } - } - - private synchronized void pumpPCMSample(byte[] pcm, int length, long stampUS) { - EasyMuxer2 muxer2 = this.muxer2; - if (muxer2 == null) - return; - - if (mRecordingStatus < 0) - return; - - if (mMuxerWaitingKeyVideo) { - Log.i(TAG, "writeFrame ignore due to no key frame!"); - return; - } - - long timeStampMillis = stampUS / 1000; - timeStampMillis -= mMuxerCuttingMillis; - timeStampMillis = Math.max(0, timeStampMillis); - int r = muxer2.writeFrame(EasyMuxer2.AVMEDIA_TYPE_AUDIO, pcm, 0, length, timeStampMillis); - Log.i(TAG, "writeFrame audio ret:" + r); - } - +// public synchronized void startRecord(String path) { +// if (mMediaInfo == null || mWidth == 0 || mHeight == 0 || mCSD0 == null) +// return; +// +// mRecordingPath = path; +// EasyMuxer2 muxer2 = new EasyMuxer2(); +// mMuxerCuttingMillis = 0l; +// mRecordingStatus = 0; +// muxerPausedMillis = 0; +// ByteBuffer csd1 = this.mCSD1; +// if (csd1 == null) csd1 = ByteBuffer.allocate(0); +// byte[] extra = new byte[mCSD0.capacity() + csd1.capacity()]; +// mCSD0.clear(); +// csd1.clear(); +// mCSD0.get(extra, 0, mCSD0.capacity()); +// csd1.get(extra, mCSD0.capacity(), csd1.capacity()); +// +// int r = muxer2.create(path, mMediaInfo.videoCodec == EASY_SDK_VIDEO_CODEC_H265 ? VIDEO_TYPE_H265 : VIDEO_TYPE_H264, mWidth, mHeight, extra, mMediaInfo.sample, mMediaInfo.channel); +// if (r != 0) { +// Log.w(TAG, "create muxer2:" + r); +// return; +// } +// +// mMuxerWaitingKeyVideo = true; +// this.muxer2 = muxer2; +// +// ResultReceiver rr = mRR; +// if (rr != null) { +// rr.send(RESULT_RECORD_BEGIN, null); +// } +// } - private synchronized void pumpVideoSample(Client.FrameInfo frameInfo) { - EasyMuxer2 muxer2 = this.muxer2; - if (muxer2 == null) return; - if (mRecordingStatus < 0) return; - if (mMuxerWaitingKeyVideo) { - if (frameInfo.type == 1) { - mMuxerWaitingKeyVideo = false; - if (mRecordingStatus == 1) { - mMuxerCuttingMillis += SystemClock.elapsedRealtime() - muxerPausedMillis; - mRecordingStatus = 0; - } - } - } - if (mMuxerWaitingKeyVideo) { - Log.i(TAG, "writeFrame ignore due to no key frame!"); - return; - } - if (frameInfo.type == 1) { -// frameInfo.offset = 60; -// frameInfo.length -= 60; - } - long timeStampMillis = frameInfo.stamp / 1000; - timeStampMillis -= mMuxerCuttingMillis; - timeStampMillis = Math.max(0, timeStampMillis); - int r = muxer2.writeFrame(EasyMuxer2.AVMEDIA_TYPE_VIDEO, frameInfo.buffer, frameInfo.offset, frameInfo.length, timeStampMillis); - Log.i(TAG, "writeFrame video ret:" + r); - } +// public synchronized void pauseRecord() { +// if (mRecordingStatus != -1) { +// mRecordingStatus = -1; +// muxerPausedMillis = SystemClock.elapsedRealtime(); +// } +// } +// public synchronized void resumeRecord() { +// if (mRecordingStatus == -1) { +// mMuxerWaitingKeyVideo = true; +// mRecordingStatus = 1; +// } +// } - public synchronized void stopRecord() { - mRecordingPath = null; - mMuxerCuttingMillis = 0l; - mRecordingStatus = 0; - muxerPausedMillis = 0; - EasyMuxer2 muxer2 = this.muxer2; - if (muxer2 == null) return; - this.muxer2 = null; - muxer2.close(); - mObject = null; - ResultReceiver rr = mRR; - if (rr != null) { - rr.send(RESULT_RECORD_END, null); - } - } +// private static int getSampleIndex(int sample) { +// for (int i = 0; i < AUDIO_SPECIFIC_CONFIG_SAMPLING_RATE_TABLE.length; i++) { +// if (sample == AUDIO_SPECIFIC_CONFIG_SAMPLING_RATE_TABLE[i]) { +// return i; +// } +// } +// return -1; +// } + +// private void pumpAACSample(Client.FrameInfo frameInfo) { +// EasyMuxer muxer = mObject; +// if (muxer == null) return; +// MediaCodec.BufferInfo bi = new MediaCodec.BufferInfo(); +// bi.offset = frameInfo.offset; +// bi.size = frameInfo.length; +// ByteBuffer buffer = ByteBuffer.wrap(frameInfo.buffer, bi.offset, bi.size); +// bi.presentationTimeUs = frameInfo.stamp; +// +// try { +// if (!frameInfo.audio) { +// throw new IllegalArgumentException("frame should be audio!"); +// } +// if (frameInfo.codec != EASY_SDK_AUDIO_CODEC_AAC) { +// throw new IllegalArgumentException("audio codec should be aac!"); +// } +// bi.offset += 7; +// bi.size -= 7; +// muxer.pumpStream(buffer, bi, false); +// } catch (IllegalStateException ex) { +// ex.printStackTrace(); +// } +// } +// +// private synchronized void pumpPCMSample(byte[] pcm, int length, long stampUS) { +// EasyMuxer2 muxer2 = this.muxer2; +// if (muxer2 == null) +// return; +// +// if (mRecordingStatus < 0) +// return; +// +// if (mMuxerWaitingKeyVideo) { +// Log.i(TAG, "writeFrame ignore due to no key frame!"); +// return; +// } +// +// long timeStampMillis = stampUS / 1000; +// timeStampMillis -= mMuxerCuttingMillis; +// timeStampMillis = Math.max(0, timeStampMillis); +// int r = muxer2.writeFrame(EasyMuxer2.AVMEDIA_TYPE_AUDIO, pcm, 0, length, timeStampMillis); +// Log.i(TAG, "writeFrame audio ret:" + r); +// } +// +// +// private synchronized void pumpVideoSample(Client.FrameInfo frameInfo) { +// EasyMuxer2 muxer2 = this.muxer2; +// if (muxer2 == null) return; +// if (mRecordingStatus < 0) return; +// if (mMuxerWaitingKeyVideo) { +// if (frameInfo.type == 1) { +// mMuxerWaitingKeyVideo = false; +// if (mRecordingStatus == 1) { +// mMuxerCuttingMillis += SystemClock.elapsedRealtime() - muxerPausedMillis; +// mRecordingStatus = 0; +// } +// } +// } +// if (mMuxerWaitingKeyVideo) { +// Log.i(TAG, "writeFrame ignore due to no key frame!"); +// return; +// } +// if (frameInfo.type == 1) { +//// frameInfo.offset = 60; +//// frameInfo.length -= 60; +// } +// long timeStampMillis = frameInfo.stamp / 1000; +// timeStampMillis -= mMuxerCuttingMillis; +// timeStampMillis = Math.max(0, timeStampMillis); +// int r = muxer2.writeFrame(EasyMuxer2.AVMEDIA_TYPE_VIDEO, frameInfo.buffer, frameInfo.offset, frameInfo.length, timeStampMillis); +// Log.i(TAG, "writeFrame video ret:" + r); +// } +// +// +// public synchronized void stopRecord() { +// mRecordingPath = null; +// mMuxerCuttingMillis = 0l; +// mRecordingStatus = 0; +// muxerPausedMillis = 0; +// EasyMuxer2 muxer2 = this.muxer2; +// if (muxer2 == null) return; +// this.muxer2 = null; +// muxer2.close(); +// mObject = null; +// ResultReceiver rr = mRR; +// if (rr != null) { +// rr.send(RESULT_RECORD_END, null); +// } +// } - @TargetApi(Build.VERSION_CODES.JELLY_BEAN) @Override public void onSourceCallBack(int _channelId, int _channelPtr, int _frameType, Client.FrameInfo frameInfo) { -// long begin = SystemClock.elapsedRealtime(); try { onRTSPSourceCallBack1(_channelId, _channelPtr, _frameType, frameInfo); } catch (Throwable e) { e.printStackTrace(); - } finally { -// Log.d(TAG, String.format("onRTSPSourceCallBack %d", SystemClock.elapsedRealtime() - begin)); } } + int mIndex = 0; + public void onRTSPSourceCallBack1(int _channelId, int _channelPtr, int _frameType, Client.FrameInfo frameInfo) { Thread.currentThread().setName("PRODUCER_THREAD"); if (frameInfo != null) { @@ -1467,9 +1246,6 @@ public class EasyPlayerClient implements Client.SourceCallBack { if (frameInfo.type == 1) { Log.i(TAG, String.format("recv I frame")); } - -// boolean firstFrame = mNewestStample == 0; - mNewestStample = frameInfo.stamp; frameInfo.audio = false; if (mWaitingKeyFrame) { @@ -1522,11 +1298,11 @@ public class EasyPlayerClient implements Client.SourceCallBack { return; } mWaitingKeyFrame = false; - synchronized (this) { - if (!TextUtils.isEmpty(mRecordingPath) && mObject == null) { - startRecord(mRecordingPath); - } - } +// synchronized (this) { +// if (!TextUtils.isEmpty(mRecordingPath) && mObject == null) { +// startRecord(mRecordingPath); +// } +// } } else { int width = frameInfo.width; int height = frameInfo.height; @@ -1543,14 +1319,14 @@ public class EasyPlayerClient implements Client.SourceCallBack { if (rr != null) rr.send(RESULT_VIDEO_SIZE, bundle); } } -// Log.d(TAG, String.format("queue size :%d", mQueue.size())); + Log.d(TAG, String.format("queue size :%d", mQueue.size())); try { mQueue.put(frameInfo); } catch (InterruptedException e) { e.printStackTrace(); } } else if (_frameType == Client.EASY_SDK_AUDIO_FRAME_FLAG) { - mNewestStample = frameInfo.stamp; +// mNewestStample = frameInfo.stamp; frameInfo.audio = true; if (true) { if (frameInfo.codec != EASY_SDK_AUDIO_CODEC_AAC && @@ -1568,13 +1344,14 @@ public class EasyPlayerClient implements Client.SourceCallBack { } } - Log.d(TAG, String.format("queue size :%d", mQueue.size())); - try { - mQueue.put(frameInfo); - } catch (InterruptedException e) { - e.printStackTrace(); - } +// Log.d(TAG, String.format("queue size :%d", mQueue.size())); +// try { +// mQueue.put(frameInfo); +// } catch (InterruptedException e) { +// e.printStackTrace(); +// } } else if (_frameType == 0) { + Log.d(TAG, String.format("time out... :%d", mQueue.size())); // time out... if (!mTimeout) { mTimeout = true; @@ -1592,7 +1369,7 @@ public class EasyPlayerClient implements Client.SourceCallBack { @Override public void onMediaInfoCallBack(int _channelId, Client.MediaInfo mi) { - mMediaInfo = mi; +// mMediaInfo = mi; Log.i(TAG, String.format("MediaInfo fetchd\n%s", mi)); }