mirror of
https://github.com/DKJone/DKWechatHelper.git
synced 2026-07-30 06:44:32 +08:00
[v1.0.7](https://github.com/DKWechatHelper/DKWechatHelper/releases/tag/1.0.7) / 2021-01-29
what's new * 动态启动图 * 动态聊天背景 * 支持8.0.1 * 更新越狱包8.0.1 * 更新已注入助手的8.0.1未签名包 * 更新越狱源安装包
This commit is contained in:
DKJone
@@ -0,0 +1,54 @@
|
||||
// QGHWDMetalRenderer.h
|
||||
// Tencent is pleased to support the open source community by making vap available.
|
||||
//
|
||||
// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved.
|
||||
//
|
||||
// Licensed under the MIT License (the "License"); you may not use this file except in
|
||||
// compliance with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://opensource.org/licenses/MIT
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License is
|
||||
// distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
|
||||
// either express or implied. See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#import <UIKit/UIKit.h>
|
||||
#import <Metal/Metal.h>
|
||||
#import "VAPMacros.h"
|
||||
|
||||
UIKIT_EXTERN NSString *const kHWDVertexFunctionName;
|
||||
UIKIT_EXTERN NSString *const kHWDYUVFragmentFunctionName;
|
||||
extern matrix_float3x3 const kColorConversionMatrix601Default;
|
||||
extern matrix_float3x3 const kColorConversionMatrix601FullRangeDefault;
|
||||
extern matrix_float3x3 const kColorConversionMatrix709Default;
|
||||
extern matrix_float3x3 const kColorConversionMatrix709FullRangeDefault;
|
||||
extern matrix_float3x3 const kBlurWeightMatrixDefault;
|
||||
extern id<MTLDevice> kQGHWDMetalRendererDevice;
|
||||
|
||||
#if TARGET_OS_SIMULATOR//模拟器
|
||||
|
||||
@interface QGHWDMetalRenderer : NSObject
|
||||
|
||||
@property (nonatomic, assign) QGHWDTextureBlendMode blendMode;
|
||||
|
||||
- (instancetype)initWithMetalLayer:(id)layer blendMode:(QGHWDTextureBlendMode)mode;
|
||||
- (void)renderPixelBuffer:(CVPixelBufferRef)pixelBuffer metalLayer:(id)layer;
|
||||
- (void)dispose;
|
||||
|
||||
@end
|
||||
#else
|
||||
|
||||
@interface QGHWDMetalRenderer : NSObject
|
||||
|
||||
@property (nonatomic, assign) QGHWDTextureBlendMode blendMode;
|
||||
|
||||
- (instancetype)initWithMetalLayer:(CAMetalLayer *)layer blendMode:(QGHWDTextureBlendMode)mode;
|
||||
|
||||
- (void)renderPixelBuffer:(CVPixelBufferRef)pixelBuffer metalLayer:(CAMetalLayer *)layer;
|
||||
|
||||
- (void)dispose;
|
||||
|
||||
@end
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,321 @@
|
||||
// QGHWDMetalRenderer.m
|
||||
// Tencent is pleased to support the open source community by making vap available.
|
||||
//
|
||||
// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved.
|
||||
//
|
||||
// Licensed under the MIT License (the "License"); you may not use this file except in
|
||||
// compliance with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://opensource.org/licenses/MIT
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License is
|
||||
// distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
|
||||
// either express or implied. See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#import "QGHWDMetalRenderer.h"
|
||||
#import "QGHWDShaderTypes.h"
|
||||
#import "QGVAPLogger.h"
|
||||
#import <simd/simd.h>
|
||||
#import <MetalKit/MetalKit.h>
|
||||
#import "UIDevice+VAPUtil.h"
|
||||
#import "QGVAPMetalUtil.h"
|
||||
#import "QGVAPMetalShaderFunctionLoader.h"
|
||||
|
||||
#pragma mark - constants
|
||||
|
||||
NSString *const kHWDVertexFunctionName = @"hwd_vertexShader";
|
||||
NSString *const kHWDYUVFragmentFunctionName = @"hwd_yuvFragmentShader";
|
||||
|
||||
static NSInteger const kQuadVerticesConstantsRow = 4;
|
||||
static NSInteger const kQuadVerticesConstantsColumn = 32;
|
||||
static NSInteger const kHWDVertexCount = 4;
|
||||
|
||||
id<MTLDevice> kQGHWDMetalRendererDevice;
|
||||
|
||||
// BT.601, which is the standard for SDTV.
|
||||
matrix_float3x3 const kColorConversionMatrix601Default = {{
|
||||
{1.164, 1.164, 1.164},
|
||||
{0.0, -0.392, 2.017},
|
||||
{1.596, -0.813, 0.0}
|
||||
}};
|
||||
|
||||
/*矩阵形式！！！
|
||||
1.0 0.0 1.4
|
||||
[1.0 -0.343 -0.711 ]
|
||||
1.0 1.765 0.0
|
||||
*/
|
||||
//ITU BT.601 Full Range
|
||||
matrix_float3x3 const kColorConversionMatrix601FullRangeDefault = {{
|
||||
{1.0, 1.0, 1.0},
|
||||
{0.0, -0.34413, 1.772},
|
||||
{1.402, -0.71414, 0.0}
|
||||
}};
|
||||
|
||||
// BT.709, which is the standard for HDTV.
|
||||
matrix_float3x3 const kColorConversionMatrix709Default = {{
|
||||
{1.164, 1.164, 1.164},
|
||||
{0.0, -0.213, 2.112},
|
||||
{1.793, -0.533, 0.0}
|
||||
}};
|
||||
|
||||
// BT.709 Full Range.
|
||||
matrix_float3x3 const kColorConversionMatrix709FullRangeDefault = {{
|
||||
{1.0, 1.0, 1.0},
|
||||
{0.0, -.18732, 1.8556},
|
||||
{1.57481, -.46813, 0.0}
|
||||
}};
|
||||
|
||||
// Blur weight matrix.
|
||||
matrix_float3x3 const kBlurWeightMatrixDefault = {{
|
||||
{0.0625, 0.125, 0.0625},
|
||||
{0.125, 0.25, 0.125},
|
||||
{0.0625, 0.125, 0.0625}
|
||||
}};
|
||||
|
||||
//QGHWDVertex 顶点坐标+纹理坐标（rgb+alpha）
|
||||
static const float kQuadVerticesConstants[kQuadVerticesConstantsRow][kQuadVerticesConstantsColumn] = {
|
||||
//左侧alpha
|
||||
{-1.0, -1.0, 0.0, 1.0, 0.5, 1.0, 0.0, 1.0,
|
||||
-1.0, 1.0, 0.0, 1.0, 0.5, 0.0, 0.0, 0.0,
|
||||
1.0, -1.0, 0.0, 1.0, 1.0, 1.0, 0.5, 1.0,
|
||||
1.0, 1.0, 0.0, 1.0, 1.0, 0.0, 0.5, 0.0},
|
||||
//右侧alpha
|
||||
{-1.0, -1.0, 0.0, 1.0, 0.0, 1.0, 0.5, 1.0,
|
||||
-1.0, 1.0, 0.0, 1.0, 0.0, 0.0, 0.5, 0.0,
|
||||
1.0, -1.0, 0.0, 1.0, 0.5, 1.0, 1.0, 1.0,
|
||||
1.0, 1.0, 0.0, 1.0, 0.5, 0.0, 1.0, 0.0},
|
||||
//顶部alpha
|
||||
{-1.0, -1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.5,
|
||||
-1.0, 1.0, 0.0, 1.0, 0.0, 0.5, 0.0, 0.0,
|
||||
1.0, -1.0, 0.0, 1.0, 1.0, 1.0, 1.0, 0.5,
|
||||
1.0, 1.0, 0.0, 1.0, 1.0, 0.5, 1.0, 0.0},
|
||||
//底部alpha
|
||||
{-1.0, -1.0, 0.0, 1.0, 0.0, 0.5, 0.0, 1.0,
|
||||
-1.0, 1.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.5,
|
||||
1.0, -1.0, 0.0, 1.0, 1.0, 0.5, 1.0, 1.0,
|
||||
1.0, 1.0, 0.0, 1.0, 1.0, 0.0, 1.0, 0.5}
|
||||
};
|
||||
|
||||
#if TARGET_OS_SIMULATOR//模拟器
|
||||
#else
|
||||
|
||||
@interface QGHWDMetalRenderer () {
|
||||
BOOL _renderingResourcesDisposed; //用以标记渲染资源是否被回收
|
||||
matrix_float3x3 _currentColorConversionMatrix;
|
||||
}
|
||||
|
||||
@property (nonatomic, strong) id<MTLBuffer> vertexBuffer;
|
||||
@property (nonatomic, strong) id<MTLBuffer> yuvMatrixBuffer;
|
||||
@property (nonatomic, strong) id<MTLRenderPipelineState> pipelineState;//This will keep track of the compiled render pipeline you’re about to create.
|
||||
@property (nonatomic, strong) id<MTLCommandQueue> commandQueue;
|
||||
@property (nonatomic, assign) int vertexCount;
|
||||
@property (nonatomic, assign) CVMetalTextureCacheRef videoTextureCache;//need release
|
||||
@property (nonatomic, strong) QGVAPMetalShaderFunctionLoader *shaderFuncLoader;
|
||||
|
||||
@end
|
||||
|
||||
@implementation QGHWDMetalRenderer
|
||||
|
||||
#pragma mark - main
|
||||
- (instancetype)initWithMetalLayer:(CAMetalLayer *)layer blendMode:(QGHWDTextureBlendMode)mode {
|
||||
self = [super init];
|
||||
if (self) {
|
||||
_blendMode = mode;
|
||||
if (!kQGHWDMetalRendererDevice) {
|
||||
kQGHWDMetalRendererDevice = MTLCreateSystemDefaultDevice();
|
||||
}
|
||||
layer.device = kQGHWDMetalRendererDevice;
|
||||
[self setupConstants];
|
||||
[self setupPipelineStatesWithMetalLayer:layer];
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
/**
|
||||
回收渲染数据，减少内存占用
|
||||
*/
|
||||
- (void)dispose {
|
||||
|
||||
_commandQueue = nil;
|
||||
_pipelineState = nil;
|
||||
_vertexBuffer = nil;
|
||||
_yuvMatrixBuffer = nil;
|
||||
_shaderFuncLoader = nil;
|
||||
if (_videoTextureCache) {
|
||||
CVMetalTextureCacheFlush(_videoTextureCache, 0);
|
||||
CFRelease(_videoTextureCache);
|
||||
_videoTextureCache = NULL;
|
||||
}
|
||||
_renderingResourcesDisposed = YES;
|
||||
}
|
||||
|
||||
- (void)dealloc {
|
||||
[self dispose];
|
||||
}
|
||||
|
||||
- (void)setupConstants {
|
||||
//buffers
|
||||
const void *vertices = [self suitableQuadVertices];
|
||||
NSUInteger allocationSize = kQuadVerticesConstantsColumn * sizeof(float);
|
||||
_vertexBuffer = [kQGHWDMetalRendererDevice newBufferWithBytes:vertices length:allocationSize options:kDefaultMTLResourceOption];
|
||||
_vertexCount = kHWDVertexCount;
|
||||
_currentColorConversionMatrix = kColorConversionMatrix601FullRangeDefault;
|
||||
struct ColorParameters yuvMatrixs[] = {{_currentColorConversionMatrix,{0.5, 0.5}}};
|
||||
NSUInteger yuvMatrixsDataSize = sizeof(struct ColorParameters);
|
||||
_yuvMatrixBuffer = [kQGHWDMetalRendererDevice newBufferWithBytes:yuvMatrixs length:yuvMatrixsDataSize options:kDefaultMTLResourceOption];
|
||||
}
|
||||
|
||||
- (void)updateMetalPropertiesIfNeed:(CVPixelBufferRef)pixelBuffer {
|
||||
|
||||
if (!pixelBuffer) {
|
||||
return ;
|
||||
}
|
||||
CFTypeRef yCbCrMatrixType = CVBufferGetAttachment(pixelBuffer, kCVImageBufferYCbCrMatrixKey, NULL);
|
||||
matrix_float3x3 matrix = kColorConversionMatrix601FullRangeDefault;
|
||||
if (CFStringCompare(yCbCrMatrixType, kCVImageBufferYCbCrMatrix_ITU_R_709_2, 0) == kCFCompareEqualTo) {
|
||||
matrix = kColorConversionMatrix709FullRangeDefault;
|
||||
}
|
||||
if (simd_equal(_currentColorConversionMatrix, matrix)) {
|
||||
return ;
|
||||
}
|
||||
_currentColorConversionMatrix = matrix;
|
||||
struct ColorParameters yuvMatrixs[] = {{_currentColorConversionMatrix,{0.5, 0.5}}};
|
||||
NSUInteger yuvMatrixsDataSize = sizeof(struct ColorParameters);
|
||||
_yuvMatrixBuffer = [kQGHWDMetalRendererDevice newBufferWithBytes:yuvMatrixs length:yuvMatrixsDataSize options:kDefaultMTLResourceOption];
|
||||
}
|
||||
|
||||
- (void)setupPipelineStatesWithMetalLayer:(CAMetalLayer *)metalLayer {
|
||||
|
||||
self.shaderFuncLoader = [[QGVAPMetalShaderFunctionLoader alloc] initWithDevice:kQGHWDMetalRendererDevice];
|
||||
id<MTLFunction> vertexProgram = [self.shaderFuncLoader loadFunctionWithName:kHWDVertexFunctionName];
|
||||
id<MTLFunction> fragmentProgram = [self.shaderFuncLoader loadFunctionWithName:kHWDYUVFragmentFunctionName];
|
||||
|
||||
if (!vertexProgram || !fragmentProgram) {
|
||||
VAP_Error(kQGVAPModuleCommon, @"setupPipelineStatesWithMetalLayer fail! cuz: shader load fail");
|
||||
NSAssert(0, @"check if .metal files been compiled to correct target!");
|
||||
return ;
|
||||
}
|
||||
MTLRenderPipelineDescriptor *pipelineStateDescriptor = [MTLRenderPipelineDescriptor new];
|
||||
pipelineStateDescriptor.vertexFunction = vertexProgram;
|
||||
pipelineStateDescriptor.fragmentFunction = fragmentProgram;
|
||||
pipelineStateDescriptor.colorAttachments[0].pixelFormat = metalLayer.pixelFormat;
|
||||
NSError *psError = nil;
|
||||
id<MTLRenderPipelineState> pipelineState = [kQGHWDMetalRendererDevice newRenderPipelineStateWithDescriptor:pipelineStateDescriptor error:&psError];
|
||||
if (!pipelineState || psError) {
|
||||
VAP_Error(kQGVAPModuleCommon, @"newRenderPipelineStateWithDescriptor error!:%@", psError);
|
||||
return ;
|
||||
}
|
||||
self.pipelineState = pipelineState;
|
||||
self.commandQueue = [kQGHWDMetalRendererDevice newCommandQueue];
|
||||
CVReturn textureCacheError = CVMetalTextureCacheCreate(kCFAllocatorDefault, nil, kQGHWDMetalRendererDevice, nil, &_videoTextureCache);
|
||||
if (textureCacheError != kCVReturnSuccess) {
|
||||
VAP_Error(kQGVAPModuleCommon, @"create texture cache fail!:%@", textureCacheError);
|
||||
return ;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
使用metal渲染管线渲染CVPixelBufferRef，若有需要融合的图层则通过对应的管线一并渲染
|
||||
|
||||
@param pixelBuffer 图像数据
|
||||
@param layer metalLayer
|
||||
*/
|
||||
- (void)renderPixelBuffer:(CVPixelBufferRef)pixelBuffer metalLayer:(CAMetalLayer *)layer {
|
||||
|
||||
if (!layer.superlayer || layer.bounds.size.width <= 0 || layer.bounds.size.height <= 0) {
|
||||
//https://forums.developer.apple.com/thread/26278
|
||||
VAP_Error(kQGVAPModuleCommon, @"quit rendering cuz layer.superlayer or size error is nil! superlayer:%@ height:%@ width:%@", layer.superlayer, @(layer.bounds.size.height), @(layer.bounds.size.width));
|
||||
return ;
|
||||
}
|
||||
[self reconstructIfNeed:layer];
|
||||
if (pixelBuffer == NULL || !self.commandQueue || !self.pipelineState) {
|
||||
VAP_Error(kQGVAPModuleCommon, @"quit rendering cuz pixelbuffer is nil!");
|
||||
return ;
|
||||
}
|
||||
[self updateMetalPropertiesIfNeed:pixelBuffer];
|
||||
CVMetalTextureCacheFlush(_videoTextureCache, 0);
|
||||
CVMetalTextureRef yTextureRef = nil, uvTextureRef = nil;
|
||||
size_t yWidth = CVPixelBufferGetWidthOfPlane(pixelBuffer, 0);
|
||||
size_t yHeight = CVPixelBufferGetHeightOfPlane(pixelBuffer, 0);
|
||||
size_t uvWidth = CVPixelBufferGetWidthOfPlane(pixelBuffer, 1);
|
||||
size_t uvHeight = CVPixelBufferGetHeightOfPlane(pixelBuffer, 1);
|
||||
//注意格式！r8Unorm
|
||||
CVReturn yStatus = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, _videoTextureCache, pixelBuffer, nil, MTLPixelFormatR8Unorm, yWidth, yHeight, 0, &yTextureRef);
|
||||
//注意格式！rg8Unorm
|
||||
CVReturn uvStatus = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, _videoTextureCache, pixelBuffer, nil, MTLPixelFormatRG8Unorm, uvWidth, uvHeight, 1, &uvTextureRef);
|
||||
if (yStatus != kCVReturnSuccess || uvStatus != kCVReturnSuccess) {
|
||||
VAP_Error(kQGVAPModuleCommon, @"quit rendering cuz failing getting yuv texture-yStatus%@:uvStatus%@", @(yStatus), @(uvStatus));
|
||||
return ;
|
||||
}
|
||||
id<MTLTexture> yTexture = CVMetalTextureGetTexture(yTextureRef);
|
||||
id<MTLTexture> uvTexture = CVMetalTextureGetTexture(uvTextureRef);
|
||||
CVBufferRelease(yTextureRef);
|
||||
CVBufferRelease(uvTextureRef);
|
||||
CVMetalTextureCacheFlush(_videoTextureCache, 0);
|
||||
yTextureRef = NULL;
|
||||
uvTextureRef = NULL;
|
||||
if (!yTexture || !uvTexture || !layer) {
|
||||
VAP_Error(kQGVAPModuleCommon, @"quit rendering cuz content is nil! y:%@ uv:%@, layer:%@", @(yTexture != nil), @(uvTexture != nil), @(layer != nil));
|
||||
return ;
|
||||
}
|
||||
if (layer.drawableSize.width <= 0 || layer.drawableSize.height <= 0) {
|
||||
VAP_Error(kQGVAPModuleCommon, @"quit rendering cuz drawableSize is 0");
|
||||
return ;
|
||||
}
|
||||
id<CAMetalDrawable> drawable = layer.nextDrawable;
|
||||
if (!drawable) {
|
||||
VAP_Error(kQGVAPModuleCommon, @"quit rendering cuz nextDrawable is nil!");
|
||||
return ;
|
||||
}
|
||||
MTLRenderPassDescriptor *renderPassDescriptor = [MTLRenderPassDescriptor new];
|
||||
renderPassDescriptor.colorAttachments[0].texture = drawable.texture; //which returns the texture in which you need to draw in order for something to appear on the screen.
|
||||
renderPassDescriptor.colorAttachments[0].loadAction = MTLLoadActionClear; //“set the texture to the clear color before doing any drawing,”
|
||||
renderPassDescriptor.colorAttachments[0].clearColor =MTLClearColorMake(1.0, 1.0, 1.0, 1.0);
|
||||
id<MTLCommandBuffer> commandBuffer = [self.commandQueue commandBuffer];
|
||||
id<MTLRenderCommandEncoder> renderEncoder = [commandBuffer renderCommandEncoderWithDescriptor:renderPassDescriptor];
|
||||
[renderEncoder setRenderPipelineState:self.pipelineState];
|
||||
[renderEncoder setVertexBuffer:self.vertexBuffer offset:0 atIndex:0];
|
||||
[renderEncoder setFragmentBuffer:self.yuvMatrixBuffer offset:0 atIndex:0];
|
||||
[renderEncoder setFragmentTexture:yTexture atIndex:QGHWDYUVFragmentTextureIndexLuma];
|
||||
[renderEncoder setFragmentTexture:uvTexture atIndex:QGHWDYUVFragmentTextureIndexChroma];
|
||||
[renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip vertexStart:0 vertexCount:self.vertexCount instanceCount:1];
|
||||
[renderEncoder endEncoding];
|
||||
[commandBuffer presentDrawable:drawable];
|
||||
[commandBuffer commit];
|
||||
}
|
||||
|
||||
#pragma mark - private
|
||||
|
||||
/**
|
||||
在必要的时候重建渲染数据，以便渲染
|
||||
|
||||
@param layer metalLayer
|
||||
*/
|
||||
- (void)reconstructIfNeed:(CAMetalLayer *)layer {
|
||||
if (_renderingResourcesDisposed) {
|
||||
[self setupConstants];
|
||||
[self setupPipelineStatesWithMetalLayer:layer];
|
||||
_renderingResourcesDisposed = NO;
|
||||
}
|
||||
}
|
||||
|
||||
- (const void *)suitableQuadVertices {
|
||||
|
||||
switch (self.blendMode) {
|
||||
case QGHWDTextureBlendMode_AlphaLeft:
|
||||
return kQuadVerticesConstants[0];
|
||||
case QGHWDTextureBlendMode_AlphaRight:
|
||||
return kQuadVerticesConstants[1];
|
||||
case QGHWDTextureBlendMode_AlphaTop:
|
||||
return kQuadVerticesConstants[2];
|
||||
case QGHWDTextureBlendMode_AlphaBottom:
|
||||
return kQuadVerticesConstants[3];
|
||||
default:
|
||||
break;
|
||||
}
|
||||
return kQuadVerticesConstants[0];
|
||||
}
|
||||
|
||||
@end
|
||||
#endif
|
||||
@@ -0,0 +1,47 @@
|
||||
// QGHWDMetalView.h
|
||||
// Tencent is pleased to support the open source community by making vap available.
|
||||
//
|
||||
// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved.
|
||||
//
|
||||
// Licensed under the MIT License (the "License"); you may not use this file except in
|
||||
// compliance with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://opensource.org/licenses/MIT
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License is
|
||||
// distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
|
||||
// either express or implied. See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#import <UIKit/UIKit.h>
|
||||
#import "VAPMacros.h"
|
||||
|
||||
//https://developer.apple.com/library/archive/documentation/Miscellaneous/Conceptual/MetalProgrammingGuide/Device/Device.html#//apple_ref/doc/uid/TP40014221-CH2-SW1
|
||||
/*
|
||||
2018-12-31 00:01:51.349229+0800 MetalTest[28134:2050088] [DYMTLInitPlatform] platform initialization successful
|
||||
2018-12-31 00:01:51.413574+0800 MetalTest[28134:2050043] Metal GPU Frame Capture Enabled
|
||||
2018-12-31 00:01:51.414037+0800 MetalTest[28134:2050043] Metal API Validation Enabled
|
||||
2018-12-31 00:01:54.008682+0800 MetalTest[28134:2050086] Execution of the command buffer was aborted due to an error during execution. Insufficient Permission (to submit GPU work from background) (IOAF code 6)
|
||||
2018-12-31 00:01:54.009053+0800 MetalTest[28134:2050086] Execution of the command buffer was aborted due to an error during execution. Insufficient Permission (to submit GPU work from background) (IOAF code 6)
|
||||
2018-12-31 00:01:54.011370+0800 MetalTest[28134:2050086] Execution of the command buffer was aborted due to an error during execution. Insufficient Permission (to submit GPU work from background) (IOAF code 6)
|
||||
2018-12-31 00:01:54.011710+0800 MetalTest[28134:2050086] Execution of the command buffer was aborted due to an error during execution. Insufficient Permission (to submit GPU work from background) (IOAF code 6)
|
||||
*/
|
||||
@protocol QGHWDMetelViewDelegate <NSObject>
|
||||
|
||||
- (void)onMetalViewUnavailable;
|
||||
|
||||
@end
|
||||
|
||||
@interface QGHWDMetalView : UIView
|
||||
|
||||
@property (nonatomic, weak) id<QGHWDMetelViewDelegate> delegate;
|
||||
@property (nonatomic, assign) QGHWDTextureBlendMode blendMode;
|
||||
|
||||
- (instancetype)initWithFrame:(CGRect)frame blendMode:(QGHWDTextureBlendMode)mode;
|
||||
|
||||
- (void)display:(CVPixelBufferRef)pixelBuffer;
|
||||
|
||||
- (void)dispose;
|
||||
|
||||
@end
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
// QGHWDMetalView.m
|
||||
// Tencent is pleased to support the open source community by making vap available.
|
||||
//
|
||||
// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved.
|
||||
//
|
||||
// Licensed under the MIT License (the "License"); you may not use this file except in
|
||||
// compliance with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://opensource.org/licenses/MIT
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License is
|
||||
// distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
|
||||
// either express or implied. See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#import "QGHWDMetalView.h"
|
||||
#import "QGVAPLogger.h"
|
||||
#import "QGHWDMetalRenderer.h"
|
||||
|
||||
#if TARGET_OS_SIMULATOR//模拟器
|
||||
|
||||
@implementation QGHWDMetalView
|
||||
|
||||
- (instancetype)initWithFrame:(CGRect)frame blendMode:(QGHWDTextureBlendMode)mode {
|
||||
return [self initWithFrame:frame];
|
||||
}
|
||||
|
||||
- (void)display:(CVPixelBufferRef)pixelBuffer {}
|
||||
|
||||
-(void)dispose {}
|
||||
|
||||
@end
|
||||
|
||||
#else
|
||||
|
||||
@interface QGHWDMetalView ()
|
||||
|
||||
@property (nonatomic, strong) CAMetalLayer *metalLayer;
|
||||
@property (nonatomic, strong) QGHWDMetalRenderer *renderer;
|
||||
@property (nonatomic, assign) BOOL drawableSizeShouldUpdate;
|
||||
|
||||
@end
|
||||
|
||||
@implementation QGHWDMetalView
|
||||
|
||||
#pragma mark - override
|
||||
|
||||
+ (Class)layerClass {
|
||||
return [CAMetalLayer class];
|
||||
}
|
||||
|
||||
- (instancetype)initWithCoder:(NSCoder *)aDecoder {
|
||||
NSAssert(0, @"initWithCoder: has not been implemented");
|
||||
return nil;
|
||||
}
|
||||
|
||||
- (instancetype)initWithFrame:(CGRect)frame {
|
||||
|
||||
if (self = [super initWithFrame:frame]) {
|
||||
_drawableSizeShouldUpdate = YES;
|
||||
_blendMode = QGHWDTextureBlendMode_AlphaLeft;
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
- (void)didMoveToWindow {
|
||||
[super didMoveToWindow];
|
||||
self.drawableSizeShouldUpdate = YES;
|
||||
}
|
||||
|
||||
- (void)layoutSubviews {
|
||||
[super layoutSubviews];
|
||||
self.drawableSizeShouldUpdate = YES;
|
||||
}
|
||||
|
||||
- (void)dealloc {
|
||||
[self onMetalViewUnavailable];
|
||||
}
|
||||
|
||||
#pragma mark - main
|
||||
|
||||
- (instancetype)initWithFrame:(CGRect)frame blendMode:(QGHWDTextureBlendMode)mode {
|
||||
|
||||
if (self = [super initWithFrame:frame]) {
|
||||
_drawableSizeShouldUpdate = YES;
|
||||
_blendMode = QGHWDTextureBlendMode_AlphaLeft;
|
||||
_metalLayer = (CAMetalLayer *)self.layer;
|
||||
_metalLayer.frame = self.frame;
|
||||
_metalLayer.opaque = NO;
|
||||
_blendMode = mode;
|
||||
_renderer = [[QGHWDMetalRenderer alloc] initWithMetalLayer:_metalLayer blendMode:mode];
|
||||
_metalLayer.contentsScale = [UIScreen mainScreen].scale;
|
||||
_metalLayer.pixelFormat = MTLPixelFormatBGRA8Unorm;
|
||||
_metalLayer.framebufferOnly = YES;
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
- (void)display:(CVPixelBufferRef)pixelBuffer {
|
||||
if (!self.window) {
|
||||
VAP_Event(kQGVAPModuleCommon, @"quit display pixelbuffer, cuz window is nil!");
|
||||
[self onMetalViewUnavailable];
|
||||
return ;
|
||||
}
|
||||
if (self.drawableSizeShouldUpdate) {
|
||||
CGFloat nativeScale = [UIScreen mainScreen].nativeScale;
|
||||
CGSize drawableSize = CGSizeMake(CGRectGetWidth(self.bounds)*nativeScale, CGRectGetHeight(self.bounds)*nativeScale);
|
||||
self.metalLayer.drawableSize = drawableSize;
|
||||
VAP_Event(kQGVAPModuleCommon, @"update drawablesize :%@", [NSValue valueWithCGSize:drawableSize]);
|
||||
self.drawableSizeShouldUpdate = NO;
|
||||
}
|
||||
self.renderer.blendMode = self.blendMode;
|
||||
[self.renderer renderPixelBuffer:pixelBuffer metalLayer:self.metalLayer];
|
||||
}
|
||||
|
||||
/**
|
||||
资源回收
|
||||
*/
|
||||
- (void)dispose {
|
||||
[self.renderer dispose];
|
||||
}
|
||||
|
||||
#pragma mark - private
|
||||
|
||||
- (void)onMetalViewUnavailable{
|
||||
|
||||
if ([self.delegate respondsToSelector:@selector(onMetalViewUnavailable)]) {
|
||||
[self.delegate onMetalViewUnavailable];
|
||||
}
|
||||
}
|
||||
|
||||
@end
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,48 @@
|
||||
// QGVAPMetalRenderer.h
|
||||
// Tencent is pleased to support the open source community by making vap available.
|
||||
//
|
||||
// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved.
|
||||
//
|
||||
// Licensed under the MIT License (the "License"); you may not use this file except in
|
||||
// compliance with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://opensource.org/licenses/MIT
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License is
|
||||
// distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
|
||||
// either express or implied. See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#import <Foundation/Foundation.h>
|
||||
#import "QGVAPConfigModel.h"
|
||||
#import <Metal/Metal.h>
|
||||
#import "VAPMacros.h"
|
||||
|
||||
#if TARGET_OS_SIMULATOR//模拟器
|
||||
|
||||
@interface QGVAPMetalRenderer : NSObject
|
||||
|
||||
@property (nonatomic, strong) QGVAPCommonInfo *commonInfo;
|
||||
|
||||
- (instancetype)initWithMetalLayer:(id)layer;
|
||||
- (void)renderPixelBuffer:(CVPixelBufferRef)pixelBuffer metalLayer:(id)layer mergeInfos:(NSArray<QGVAPMergedInfo *> *)infos;
|
||||
- (void)dispose;
|
||||
|
||||
@end
|
||||
|
||||
#else
|
||||
|
||||
@interface QGVAPMetalRenderer : NSObject
|
||||
|
||||
@property (nonatomic, strong) QGVAPCommonInfo *commonInfo;
|
||||
@property (nonatomic, strong) QGVAPMaskInfo *maskInfo;
|
||||
|
||||
- (instancetype)initWithMetalLayer:(CAMetalLayer *)layer;
|
||||
|
||||
- (void)renderPixelBuffer:(CVPixelBufferRef)pixelBuffer metalLayer:(CAMetalLayer *)layer mergeInfos:(NSArray<QGVAPMergedInfo *> *)infos;
|
||||
|
||||
- (void)dispose;
|
||||
|
||||
@end
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,425 @@
|
||||
// QGVAPMetalRenderer.m
|
||||
// Tencent is pleased to support the open source community by making vap available.
|
||||
//
|
||||
// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved.
|
||||
//
|
||||
// Licensed under the MIT License (the "License"); you may not use this file except in
|
||||
// compliance with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://opensource.org/licenses/MIT
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License is
|
||||
// distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
|
||||
// either express or implied. See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#import "QGVAPMetalRenderer.h"
|
||||
#import "QGHWDMetalRenderer.h"
|
||||
#import <MetalKit/MetalKit.h>
|
||||
#import "QGVAPLogger.h"
|
||||
#import <simd/simd.h>
|
||||
#import "UIDevice+VAPUtil.h"
|
||||
#import "QGVAPMetalUtil.h"
|
||||
#import "QGVAPMetalShaderFunctionLoader.h"
|
||||
|
||||
#if TARGET_OS_SIMULATOR//模拟器
|
||||
#else
|
||||
|
||||
@interface QGVAPMetalRenderer () {
|
||||
BOOL _renderingResourcesDisposed; //用以标记渲染资源是否被回收
|
||||
matrix_float3x3 _currentColorConversionMatrix;
|
||||
}
|
||||
|
||||
@property (nonatomic, strong) id<MTLBuffer> vertexBuffer;
|
||||
@property (nonatomic, strong) id<MTLBuffer> yuvMatrixBuffer;
|
||||
@property (nonatomic, strong) id<MTLBuffer> maskBlurBuffer;
|
||||
@property (nonatomic, strong) id<MTLRenderPipelineState> attachmentPipelineState;
|
||||
@property (nonatomic, strong) id<MTLRenderPipelineState> defaultMainPipelineState;
|
||||
@property (nonatomic, strong) id<MTLRenderPipelineState> mainPipelineStateForMask; //带遮罩处理的主流程管线
|
||||
@property (nonatomic, strong) id<MTLRenderPipelineState> mainPipelineStateForMaskBlur; //带遮罩模糊处理的主流程管线
|
||||
@property (nonatomic, strong) id<MTLCommandQueue> commandQueue;
|
||||
@property (nonatomic, assign) CVMetalTextureCacheRef videoTextureCache;//need release
|
||||
@property (nonatomic, strong) QGVAPMetalShaderFunctionLoader *shaderFuncLoader;
|
||||
@property (nonatomic, weak) CAMetalLayer *metalLayer;
|
||||
|
||||
@end
|
||||
|
||||
@implementation QGVAPMetalRenderer
|
||||
|
||||
- (instancetype)initWithMetalLayer:(CAMetalLayer *)layer {
|
||||
|
||||
if (self = [super init]) {
|
||||
if (!kQGHWDMetalRendererDevice) {
|
||||
kQGHWDMetalRendererDevice = MTLCreateSystemDefaultDevice();
|
||||
}
|
||||
layer.device = kQGHWDMetalRendererDevice;
|
||||
_metalLayer = layer;
|
||||
[self setupRenderContext];
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
#pragma mark - main
|
||||
|
||||
- (void)renderPixelBuffer:(CVPixelBufferRef)pixelBuffer metalLayer:(CAMetalLayer *)layer mergeInfos:(NSArray<QGVAPMergedInfo *> *)infos {
|
||||
|
||||
if (!layer.superlayer || layer.bounds.size.width <= 0 || layer.bounds.size.height <= 0) {
|
||||
//https://forums.developer.apple.com/thread/26278
|
||||
VAP_Error(kQGVAPModuleCommon, @"quit rendering cuz layer.superlayer or size error is nil! superlayer:%@ height:%@ width:%@", layer.superlayer, @(layer.bounds.size.height), @(layer.bounds.size.width));
|
||||
return ;
|
||||
}
|
||||
[self reconstructIfNeed:layer];
|
||||
if (pixelBuffer == NULL || !self.commandQueue) {
|
||||
VAP_Error(kQGVAPModuleCommon, @"quit rendering cuz pixelbuffer is nil!");
|
||||
return ;
|
||||
}
|
||||
[self updateMetalPropertiesIfNeed:pixelBuffer];
|
||||
CVMetalTextureCacheFlush(_videoTextureCache, 0);
|
||||
CVMetalTextureRef yTextureRef = nil, uvTextureRef = nil;
|
||||
size_t yWidth = CVPixelBufferGetWidthOfPlane(pixelBuffer, 0);
|
||||
size_t yHeight = CVPixelBufferGetHeightOfPlane(pixelBuffer, 0);
|
||||
size_t uvWidth = CVPixelBufferGetWidthOfPlane(pixelBuffer, 1);
|
||||
size_t uvHeight = CVPixelBufferGetHeightOfPlane(pixelBuffer, 1);
|
||||
//注意格式！r8Unorm
|
||||
CVReturn yStatus = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, _videoTextureCache, pixelBuffer, nil, MTLPixelFormatR8Unorm, yWidth, yHeight, 0, &yTextureRef);
|
||||
//注意格式！rg8Unorm
|
||||
CVReturn uvStatus = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, _videoTextureCache, pixelBuffer, nil, MTLPixelFormatRG8Unorm, uvWidth, uvHeight, 1, &uvTextureRef);
|
||||
if (yStatus != kCVReturnSuccess || uvStatus != kCVReturnSuccess) {
|
||||
VAP_Error(kQGVAPModuleCommon, @"quit rendering cuz failing getting yuv texture-yStatus%@:uvStatus%@", @(yStatus), @(uvStatus));
|
||||
return ;
|
||||
}
|
||||
id<MTLTexture> yTexture = CVMetalTextureGetTexture(yTextureRef);
|
||||
id<MTLTexture> uvTexture = CVMetalTextureGetTexture(uvTextureRef);
|
||||
CVBufferRelease(yTextureRef);
|
||||
CVBufferRelease(uvTextureRef);
|
||||
CVMetalTextureCacheFlush(_videoTextureCache, 0);
|
||||
yTextureRef = NULL;
|
||||
uvTextureRef = NULL;
|
||||
if (!yTexture || !uvTexture || !layer) {
|
||||
VAP_Error(kQGVAPModuleCommon, @"quit rendering cuz content is nil! y:%@ uv:%@, layer:%@", @(yTexture != nil), @(uvTexture != nil), @(layer != nil));
|
||||
return ;
|
||||
}
|
||||
if (layer.drawableSize.width <= 0 || layer.drawableSize.height <= 0) {
|
||||
VAP_Error(kQGVAPModuleCommon, @"quit rendering cuz drawableSize is 0");
|
||||
return ;
|
||||
}
|
||||
id<CAMetalDrawable> drawable = layer.nextDrawable;
|
||||
if (!drawable) {
|
||||
VAP_Error(kQGVAPModuleCommon, @"quit rendering cuz nextDrawable is nil!");
|
||||
return ;
|
||||
}
|
||||
MTLRenderPassDescriptor *renderPassDescriptor = [MTLRenderPassDescriptor new];
|
||||
renderPassDescriptor.colorAttachments[0].texture = drawable.texture; //which returns the texture in which you need to draw in order for something to appear on the screen.
|
||||
renderPassDescriptor.colorAttachments[0].loadAction = MTLLoadActionClear; //“set the texture to the clear color before doing any drawing,”
|
||||
renderPassDescriptor.colorAttachments[0].clearColor =MTLClearColorMake(0.0, 0.0, 0.0, 0.0);
|
||||
id<MTLCommandBuffer> commandBuffer = [self.commandQueue commandBuffer];
|
||||
id<MTLRenderCommandEncoder> renderEncoder = [commandBuffer renderCommandEncoderWithDescriptor:renderPassDescriptor];
|
||||
if (renderEncoder == nil) {
|
||||
VAP_Error(kQGVAPModuleCommon, @"quit rendering cuz renderEncoder:%p or self.pipelineState:%p is nil!", renderEncoder);
|
||||
return ;
|
||||
}
|
||||
if (self.vertexBuffer == nil || self.yuvMatrixBuffer == nil) {
|
||||
VAP_Error(kQGVAPModuleCommon, @"quit rendering cuz vertexBuffer:%p or yuvMatrixBuffer:%p is nil!", self.vertexBuffer, self.yuvMatrixBuffer);
|
||||
return ;
|
||||
}
|
||||
|
||||
[self drawBackground:yTexture uvTexture:uvTexture encoder:renderEncoder];
|
||||
[self drawMergedAttachments:infos yTexture:yTexture uvTexture:uvTexture renderEncoder:renderEncoder metalLayer:layer];
|
||||
[renderEncoder endEncoding];
|
||||
[commandBuffer presentDrawable:drawable];
|
||||
[commandBuffer commit];
|
||||
}
|
||||
|
||||
- (void)drawBackground:(id<MTLTexture>)yTexture uvTexture:(id<MTLTexture>)uvTexture encoder:(id<MTLRenderCommandEncoder>)renderEncoder {
|
||||
|
||||
if (self.maskInfo) {
|
||||
id<MTLTexture> maskTexture = self.maskInfo.texture;
|
||||
if (!maskTexture) {
|
||||
VAP_Error(kQGVAPModuleCommon, @"maskTexture error! maskTexture is nil");
|
||||
return;
|
||||
}
|
||||
|
||||
if (!self.mainPipelineStateForMask) {
|
||||
VAP_Error(kQGVAPModuleCommon, @"maskPipelineState error! maskTexture is nil");
|
||||
return;
|
||||
}
|
||||
|
||||
if (self.maskInfo.blurLength > 0) {
|
||||
[renderEncoder setRenderPipelineState:self.mainPipelineStateForMaskBlur];
|
||||
[renderEncoder setVertexBuffer:self.vertexBuffer offset:0 atIndex:0];
|
||||
[renderEncoder setFragmentBuffer:self.yuvMatrixBuffer offset:0 atIndex:0];
|
||||
[renderEncoder setFragmentBuffer:self.maskBlurBuffer offset:0 atIndex:1];
|
||||
[renderEncoder setFragmentTexture:yTexture atIndex:QGHWDYUVFragmentTextureIndexLuma];
|
||||
[renderEncoder setFragmentTexture:uvTexture atIndex:QGHWDYUVFragmentTextureIndexChroma];
|
||||
[renderEncoder setFragmentTexture:maskTexture atIndex:QGHWDYUVFragmentTextureIndexAttachmentStart];
|
||||
[renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip vertexStart:0 vertexCount:4 instanceCount:1];
|
||||
} else {
|
||||
[renderEncoder setRenderPipelineState:self.mainPipelineStateForMask];
|
||||
[renderEncoder setVertexBuffer:self.vertexBuffer offset:0 atIndex:0];
|
||||
[renderEncoder setFragmentBuffer:self.yuvMatrixBuffer offset:0 atIndex:0];
|
||||
[renderEncoder setFragmentTexture:yTexture atIndex:QGHWDYUVFragmentTextureIndexLuma];
|
||||
[renderEncoder setFragmentTexture:uvTexture atIndex:QGHWDYUVFragmentTextureIndexChroma];
|
||||
[renderEncoder setFragmentTexture:maskTexture atIndex:QGHWDYUVFragmentTextureIndexAttachmentStart];
|
||||
[renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip vertexStart:0 vertexCount:4 instanceCount:1];
|
||||
}
|
||||
} else {
|
||||
if (!self.defaultMainPipelineState) {
|
||||
VAP_Error(kQGVAPModuleCommon, @"yuvPipelineState error! maskTexture is nil");
|
||||
return;
|
||||
}
|
||||
[renderEncoder setRenderPipelineState:self.defaultMainPipelineState];
|
||||
[renderEncoder setVertexBuffer:self.vertexBuffer offset:0 atIndex:0];
|
||||
[renderEncoder setFragmentBuffer:self.yuvMatrixBuffer offset:0 atIndex:0];
|
||||
[renderEncoder setFragmentTexture:yTexture atIndex:QGHWDYUVFragmentTextureIndexLuma];
|
||||
[renderEncoder setFragmentTexture:uvTexture atIndex:QGHWDYUVFragmentTextureIndexChroma];
|
||||
[renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip vertexStart:0 vertexCount:4 instanceCount:1];
|
||||
}
|
||||
}
|
||||
|
||||
- (void)dispose {
|
||||
|
||||
_commandQueue = nil;
|
||||
_vertexBuffer = nil;
|
||||
_yuvMatrixBuffer = nil;
|
||||
_attachmentPipelineState = nil;
|
||||
_shaderFuncLoader = nil;
|
||||
if (_videoTextureCache) {
|
||||
CVMetalTextureCacheFlush(_videoTextureCache, 0);
|
||||
CFRelease(_videoTextureCache);
|
||||
_videoTextureCache = NULL;
|
||||
}
|
||||
_renderingResourcesDisposed = YES;
|
||||
_mainPipelineStateForMask = nil;
|
||||
_defaultMainPipelineState = nil;
|
||||
}
|
||||
|
||||
-(void)dealloc {
|
||||
[self dispose];
|
||||
}
|
||||
|
||||
- (void)setupRenderContext {
|
||||
|
||||
//constants
|
||||
_currentColorConversionMatrix = kColorConversionMatrix601FullRangeDefault;
|
||||
struct ColorParameters yuvMatrixs[] = {{_currentColorConversionMatrix,{0.5, 0.5}}};
|
||||
NSUInteger yuvMatrixsDataSize = sizeof(struct ColorParameters);
|
||||
_yuvMatrixBuffer = [kQGHWDMetalRendererDevice newBufferWithBytes:yuvMatrixs length:yuvMatrixsDataSize options:kDefaultMTLResourceOption];
|
||||
|
||||
//function loader
|
||||
self.shaderFuncLoader = [[QGVAPMetalShaderFunctionLoader alloc] initWithDevice:kQGHWDMetalRendererDevice];
|
||||
|
||||
//command queue
|
||||
self.commandQueue = [kQGHWDMetalRendererDevice newCommandQueue];
|
||||
|
||||
//texture cache
|
||||
CVReturn textureCacheError = CVMetalTextureCacheCreate(kCFAllocatorDefault, nil, kQGHWDMetalRendererDevice, nil, &_videoTextureCache);
|
||||
if (textureCacheError != kCVReturnSuccess) {
|
||||
VAP_Error(kQGVAPModuleCommon, @"create texture cache fail!:%@", textureCacheError);
|
||||
}
|
||||
}
|
||||
|
||||
- (void)drawMergedAttachments:(NSArray<QGVAPMergedInfo *> *)infos
|
||||
yTexture:(id<MTLTexture>)yTexture
|
||||
uvTexture:(id<MTLTexture>)uvTexture
|
||||
renderEncoder:(id<MTLRenderCommandEncoder>)encoder
|
||||
metalLayer:(CAMetalLayer *)layer {
|
||||
|
||||
if (infos.count == 0) {
|
||||
return ;
|
||||
}
|
||||
if (!encoder || !self.commonInfo || !self.attachmentPipelineState) {
|
||||
VAP_Error(kQGVAPModuleCommon, @"renderMergedAttachments error! infos:%@ encoder:%p commonInfo:%@ attachmentPipelineState:%p", @(infos.count), encoder, @(self.commonInfo != nil), self.attachmentPipelineState);
|
||||
return ;
|
||||
}
|
||||
if (yTexture == nil || uvTexture == nil) {
|
||||
VAP_Error(kQGVAPModuleCommon, @"renderMergedAttachments error! cuz yTexture:%p or uvTexture:%p is nil!", yTexture, uvTexture);
|
||||
return ;
|
||||
}
|
||||
[infos enumerateObjectsUsingBlock:^(QGVAPMergedInfo * _Nonnull mergeInfo, NSUInteger idx, BOOL * _Nonnull stop) {
|
||||
|
||||
[encoder setRenderPipelineState:self.attachmentPipelineState];
|
||||
id<MTLTexture> sourceTexture = mergeInfo.source.texture;//图片纹理
|
||||
id<MTLBuffer> vertexBuffer = [mergeInfo vertexBufferWithContainerSize:self.commonInfo.size maskContianerSize:self.commonInfo.videoSize device:kQGHWDMetalRendererDevice];
|
||||
id<MTLBuffer> colorParamsBuffer = mergeInfo.source.colorParamsBuffer;
|
||||
id<MTLBuffer> yuvMatrixBuffer = self.yuvMatrixBuffer;
|
||||
if (!sourceTexture || !vertexBuffer || !colorParamsBuffer || !yuvMatrixBuffer) {
|
||||
//VAP_Error(kQGVAPModuleCommon, @"quit attachment:%p cuz-source:%p vertex:%p",mergeInfo, sourceTexture, vertexBuffer);
|
||||
return ;
|
||||
}
|
||||
[encoder setVertexBuffer:vertexBuffer offset:0 atIndex:0];
|
||||
[encoder setFragmentBuffer:yuvMatrixBuffer offset:0 atIndex:0];
|
||||
[encoder setFragmentBuffer:colorParamsBuffer offset:0 atIndex:1];
|
||||
//遮罩信息在视频流中
|
||||
[encoder setFragmentTexture:yTexture atIndex:QGHWDYUVFragmentTextureIndexLuma];
|
||||
[encoder setFragmentTexture:uvTexture atIndex:QGHWDYUVFragmentTextureIndexChroma];
|
||||
[encoder setFragmentTexture:sourceTexture atIndex:QGHWDYUVFragmentTextureIndexAttachmentStart];
|
||||
[encoder drawPrimitives:MTLPrimitiveTypeTriangleStrip vertexStart:0 vertexCount:4 instanceCount:1];
|
||||
}];
|
||||
}
|
||||
|
||||
#pragma mark - setter&getter
|
||||
|
||||
- (id<MTLBuffer>)maskBlurBuffer {
|
||||
if (!_maskBlurBuffer) {
|
||||
struct MaskParameters parameters[] = {{kBlurWeightMatrixDefault, 3, 0.01}};
|
||||
NSUInteger parametersSize = sizeof(struct MaskParameters);
|
||||
_maskBlurBuffer = [kQGHWDMetalRendererDevice newBufferWithBytes:parameters length:parametersSize options:kDefaultMTLResourceOption];
|
||||
}
|
||||
return _maskBlurBuffer;
|
||||
}
|
||||
|
||||
- (void)setCommonInfo:(QGVAPCommonInfo *)commonInfo {
|
||||
_commonInfo = commonInfo;
|
||||
[self updateMainVertexBuffer];
|
||||
}
|
||||
|
||||
- (void)setMaskInfo:(QGVAPMaskInfo *)maskInfo {
|
||||
|
||||
if (maskInfo && (!maskInfo.data || maskInfo.dataSize.width <= 0 || maskInfo.dataSize.height <= 0)) {
|
||||
VAP_Error(kQGVAPModuleCommon, @"setMaskInfo fail: data:%@, size:%@", maskInfo.data, NSStringFromCGSize(maskInfo.dataSize));
|
||||
return;
|
||||
}
|
||||
if (_maskInfo == maskInfo) {
|
||||
return ;
|
||||
}
|
||||
_maskInfo = maskInfo;
|
||||
if (_vertexBuffer) {
|
||||
[self updateMainVertexBuffer];
|
||||
}
|
||||
}
|
||||
|
||||
#pragma mark - pipelines
|
||||
|
||||
- (id<MTLRenderPipelineState>)createPipelineState:(NSString *)vertexFunction fragmentFunction:(NSString *)fragmentFunction {
|
||||
|
||||
id<MTLFunction> vertexProgram = [self.shaderFuncLoader loadFunctionWithName:vertexFunction];
|
||||
id<MTLFunction> fragmentProgram = [self.shaderFuncLoader loadFunctionWithName:fragmentFunction];
|
||||
|
||||
if (!vertexProgram || !fragmentProgram) {
|
||||
VAP_Error(kQGVAPModuleCommon, @"setupPipelineStatesWithMetalLayer fail! cuz: shader load fail!");
|
||||
NSAssert(0, @"check if .metal files been compiled to correct target!");
|
||||
return nil;
|
||||
}
|
||||
|
||||
//融混方程
|
||||
//https://objccn.io/issue-3-1/
|
||||
//https://www.andersriggelsen.dk/glblendfunc.php
|
||||
MTLRenderPipelineDescriptor *pipelineStateDescriptor = [MTLRenderPipelineDescriptor new];
|
||||
pipelineStateDescriptor.vertexFunction = vertexProgram;
|
||||
pipelineStateDescriptor.fragmentFunction = fragmentProgram;
|
||||
pipelineStateDescriptor.colorAttachments[0].pixelFormat = _metalLayer.pixelFormat;
|
||||
[pipelineStateDescriptor.colorAttachments[0] setBlendingEnabled:YES];
|
||||
pipelineStateDescriptor.colorAttachments[0].rgbBlendOperation = MTLBlendOperationAdd;
|
||||
pipelineStateDescriptor.colorAttachments[0].alphaBlendOperation = MTLBlendOperationAdd;
|
||||
pipelineStateDescriptor.colorAttachments[0].sourceRGBBlendFactor = MTLBlendFactorSourceAlpha;
|
||||
pipelineStateDescriptor.colorAttachments[0].sourceAlphaBlendFactor = MTLBlendFactorSourceAlpha;
|
||||
pipelineStateDescriptor.colorAttachments[0].destinationRGBBlendFactor = MTLBlendFactorOneMinusSourceAlpha;
|
||||
pipelineStateDescriptor.colorAttachments[0].destinationAlphaBlendFactor = MTLBlendFactorOneMinusSourceAlpha;
|
||||
NSError *psError = nil;
|
||||
id<MTLRenderPipelineState> pipelineState = [kQGHWDMetalRendererDevice newRenderPipelineStateWithDescriptor:pipelineStateDescriptor error:&psError];
|
||||
if (!pipelineState || psError) {
|
||||
VAP_Error(kQGVAPModuleCommon, @"newRenderPipelineStateWithDescriptor error!:%@", psError);
|
||||
return nil;
|
||||
}
|
||||
|
||||
return pipelineState;
|
||||
}
|
||||
|
||||
- (id<MTLRenderPipelineState>)defaultMainPipelineState {
|
||||
if (!_defaultMainPipelineState) {
|
||||
_defaultMainPipelineState = [self createPipelineState:kVAPVertexFunctionName fragmentFunction:kVAPYUVFragmentFunctionName];
|
||||
}
|
||||
return _defaultMainPipelineState;
|
||||
}
|
||||
|
||||
- (id<MTLRenderPipelineState>)mainPipelineStateForMask {
|
||||
if (!_mainPipelineStateForMask) {
|
||||
_mainPipelineStateForMask = [self createPipelineState:kVAPVertexFunctionName fragmentFunction:kVAPMaskFragmentFunctionName];
|
||||
}
|
||||
return _mainPipelineStateForMask;
|
||||
}
|
||||
|
||||
- (id<MTLRenderPipelineState>)attachmentPipelineState {
|
||||
if (!_attachmentPipelineState) {
|
||||
_attachmentPipelineState = [self createPipelineState:kVAPAttachmentVertexFunctionName fragmentFunction:kVAPAttachmentFragmentFunctionName];
|
||||
}
|
||||
return _attachmentPipelineState;
|
||||
}
|
||||
|
||||
- (id<MTLRenderPipelineState>)mainPipelineStateForMaskBlur {
|
||||
if (!_mainPipelineStateForMaskBlur) {
|
||||
_mainPipelineStateForMaskBlur = [self createPipelineState:kVAPVertexFunctionName fragmentFunction:kVAPMaskBlurFragmentFunctionName];
|
||||
}
|
||||
return _mainPipelineStateForMaskBlur;
|
||||
}
|
||||
|
||||
#pragma mark - private
|
||||
|
||||
- (void)reconstructIfNeed:(CAMetalLayer *)layer {
|
||||
|
||||
if (_renderingResourcesDisposed) {
|
||||
[self setupRenderContext];
|
||||
_renderingResourcesDisposed = NO;
|
||||
}
|
||||
}
|
||||
|
||||
- (void)updateMetalPropertiesIfNeed:(CVPixelBufferRef)pixelBuffer {
|
||||
|
||||
if (!pixelBuffer) {
|
||||
return ;
|
||||
}
|
||||
CFTypeRef yCbCrMatrixType = CVBufferGetAttachment(pixelBuffer, kCVImageBufferYCbCrMatrixKey, NULL);
|
||||
matrix_float3x3 matrix = kColorConversionMatrix601FullRangeDefault;
|
||||
if (CFStringCompare(yCbCrMatrixType, kCVImageBufferYCbCrMatrix_ITU_R_709_2, 0) == kCFCompareEqualTo) {
|
||||
matrix = kColorConversionMatrix709FullRangeDefault;
|
||||
}
|
||||
if (simd_equal(_currentColorConversionMatrix, matrix)) {
|
||||
return ;
|
||||
}
|
||||
_currentColorConversionMatrix = matrix;
|
||||
struct ColorParameters yuvMatrixs[] = {{_currentColorConversionMatrix,{0.5, 0.5}}};
|
||||
NSUInteger yuvMatrixsDataSize = sizeof(struct ColorParameters);
|
||||
_yuvMatrixBuffer = [kQGHWDMetalRendererDevice newBufferWithBytes:yuvMatrixs length:yuvMatrixsDataSize options:kDefaultMTLResourceOption];
|
||||
}
|
||||
|
||||
- (void)updateMainVertexBuffer {
|
||||
|
||||
const int colunmCountForVertices = 4, colunmCountForCoordinate = 2, vertexDataLength = 40; //顶点(x,y,z,w),纹理坐标(x,x),数组长度
|
||||
static float vertexData[vertexDataLength]; //顶点+纹理坐标数据
|
||||
float rgbCoordinates[8], alphaCoordinates[8];
|
||||
float maskCoordinates[8] = {0};
|
||||
const void *vertices = kVAPMTLVerticesIdentity;
|
||||
|
||||
genMTLTextureCoordinates(self.commonInfo.rgbAreaRect, self.commonInfo.videoSize, rgbCoordinates, NO, 0);
|
||||
genMTLTextureCoordinates(self.commonInfo.alphaAreaRect, self.commonInfo.videoSize, alphaCoordinates, NO, 0);
|
||||
if (self.maskInfo) {
|
||||
genMTLTextureCoordinates(self.maskInfo.sampleRect, self.maskInfo.dataSize, maskCoordinates, NO, 0);
|
||||
}
|
||||
|
||||
int indexForVertexData = 0;
|
||||
//顶点数据+坐标。==> 这里的写法需有优化一下
|
||||
for (int i = 0; i < 4 * colunmCountForVertices; i ++) {
|
||||
//顶点数据
|
||||
vertexData[indexForVertexData++] = ((float*)vertices)[i];
|
||||
//逐行处理
|
||||
if (i%colunmCountForVertices == colunmCountForVertices-1) {
|
||||
int row = i/colunmCountForVertices;
|
||||
//rgb纹理坐标
|
||||
vertexData[indexForVertexData++] = ((float*)rgbCoordinates)[row*colunmCountForCoordinate];
|
||||
vertexData[indexForVertexData++] = ((float*)rgbCoordinates)[row*colunmCountForCoordinate+1];
|
||||
//alpha纹理坐标
|
||||
vertexData[indexForVertexData++] = ((float*)alphaCoordinates)[row*colunmCountForCoordinate];
|
||||
vertexData[indexForVertexData++] = ((float*)alphaCoordinates)[row*colunmCountForCoordinate+1];
|
||||
//mask纹理坐标
|
||||
vertexData[indexForVertexData++] = ((float*)maskCoordinates)[row*colunmCountForCoordinate];
|
||||
vertexData[indexForVertexData++] = ((float*)maskCoordinates)[row*colunmCountForCoordinate+1];
|
||||
}
|
||||
}
|
||||
NSUInteger allocationSize = vertexDataLength * sizeof(float);
|
||||
id<MTLBuffer> vertexBuffer = [kQGHWDMetalRendererDevice newBufferWithBytes:vertexData length:allocationSize options:kDefaultMTLResourceOption];
|
||||
_vertexBuffer = vertexBuffer;
|
||||
}
|
||||
|
||||
@end
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,37 @@
|
||||
// QGVAPMetalView.h
|
||||
// Tencent is pleased to support the open source community by making vap available.
|
||||
//
|
||||
// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved.
|
||||
//
|
||||
// Licensed under the MIT License (the "License"); you may not use this file except in
|
||||
// compliance with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://opensource.org/licenses/MIT
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License is
|
||||
// distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
|
||||
// either express or implied. See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#import <UIKit/UIKit.h>
|
||||
#import "QGVAPConfigModel.h"
|
||||
|
||||
@class QGVAPMaskInfo;
|
||||
|
||||
@protocol QGVAPMetalViewDelegate <NSObject>
|
||||
|
||||
- (void)onMetalViewUnavailable;
|
||||
|
||||
@end
|
||||
|
||||
@interface QGVAPMetalView : UIView
|
||||
|
||||
@property (nonatomic, weak) id<QGVAPMetalViewDelegate> delegate;
|
||||
@property (nonatomic, strong) QGVAPCommonInfo *commonInfo;
|
||||
@property (nonatomic, strong) QGVAPMaskInfo *maskInfo;
|
||||
|
||||
- (void)display:(CVPixelBufferRef)pixelBuffer mergeInfos:(NSArray<QGVAPMergedInfo *> *)infos;
|
||||
|
||||
- (void)dispose;
|
||||
|
||||
@end
|
||||
@@ -0,0 +1,132 @@
|
||||
// QGVAPMetalView.m
|
||||
// Tencent is pleased to support the open source community by making vap available.
|
||||
//
|
||||
// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved.
|
||||
//
|
||||
// Licensed under the MIT License (the "License"); you may not use this file except in
|
||||
// compliance with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://opensource.org/licenses/MIT
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License is
|
||||
// distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
|
||||
// either express or implied. See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#import "QGVAPMetalView.h"
|
||||
#import "QGVAPMetalRenderer.h"
|
||||
#import "QGVAPLogger.h"
|
||||
|
||||
#if TARGET_OS_SIMULATOR//模拟器
|
||||
|
||||
@implementation QGVAPMetalView
|
||||
|
||||
- (void)display:(CVPixelBufferRef)pixelBuffer mergeInfos:(NSArray<QGVAPMergedInfo *> *)infos {}
|
||||
|
||||
- (void)dispose {}
|
||||
|
||||
@end
|
||||
|
||||
#else
|
||||
|
||||
@interface QGVAPMetalView ()
|
||||
|
||||
@property (nonatomic, strong) CAMetalLayer *metalLayer;
|
||||
@property (nonatomic, strong) QGVAPMetalRenderer *renderer;
|
||||
@property (nonatomic, assign) BOOL drawableSizeShouldUpdate;
|
||||
|
||||
@end
|
||||
|
||||
@implementation QGVAPMetalView
|
||||
|
||||
#pragma mark - override
|
||||
|
||||
+ (Class)layerClass {
|
||||
return [CAMetalLayer class];
|
||||
}
|
||||
|
||||
- (instancetype)initWithCoder:(NSCoder *)aDecoder {
|
||||
NSAssert(0, @"initWithCoder: has not been implemented");
|
||||
if (self = [super initWithCoder:aDecoder]) {
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
- (instancetype)initWithFrame:(CGRect)frame {
|
||||
|
||||
if (self = [super initWithFrame:frame]) {
|
||||
_drawableSizeShouldUpdate = YES;
|
||||
_metalLayer = (CAMetalLayer *)self.layer;
|
||||
_metalLayer.frame = self.frame;
|
||||
_metalLayer.opaque = NO;
|
||||
_renderer = [[QGVAPMetalRenderer alloc] initWithMetalLayer:_metalLayer];
|
||||
_metalLayer.contentsScale = [UIScreen mainScreen].scale;
|
||||
_metalLayer.pixelFormat = MTLPixelFormatBGRA8Unorm;
|
||||
_metalLayer.framebufferOnly = YES;
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
- (void)didMoveToWindow {
|
||||
[super didMoveToWindow];
|
||||
self.drawableSizeShouldUpdate = YES;
|
||||
}
|
||||
|
||||
- (void)layoutSubviews {
|
||||
[super layoutSubviews];
|
||||
self.drawableSizeShouldUpdate = YES;
|
||||
}
|
||||
|
||||
- (void)dealloc {
|
||||
[self onMetalViewUnavailable];
|
||||
}
|
||||
|
||||
#pragma mark - getter&setter
|
||||
|
||||
- (QGVAPCommonInfo *)commonInfo {
|
||||
return self.renderer.commonInfo;
|
||||
}
|
||||
|
||||
- (void)setCommonInfo:(QGVAPCommonInfo *)commonInfo {
|
||||
[self.renderer setCommonInfo:commonInfo];
|
||||
}
|
||||
|
||||
- (void)setMaskInfo:(QGVAPMaskInfo *)maskInfo {
|
||||
[self.renderer setMaskInfo:maskInfo];
|
||||
}
|
||||
|
||||
#pragma mark - main
|
||||
|
||||
- (void)display:(CVPixelBufferRef)pixelBuffer mergeInfos:(NSArray<QGVAPMergedInfo *> *)infos {
|
||||
|
||||
if (!self.window) {
|
||||
VAP_Event(kQGVAPModuleCommon, @"quit display pixelbuffer, cuz window is nil!");
|
||||
[self onMetalViewUnavailable];
|
||||
return ;
|
||||
}
|
||||
if (self.drawableSizeShouldUpdate) {
|
||||
CGFloat nativeScale = [UIScreen mainScreen].nativeScale;
|
||||
CGSize drawableSize = CGSizeMake(CGRectGetWidth(self.bounds)*nativeScale, CGRectGetHeight(self.bounds)*nativeScale);
|
||||
self.metalLayer.drawableSize = drawableSize;
|
||||
VAP_Event(kQGVAPModuleCommon, @"update drawablesize :%@", [NSValue valueWithCGSize:drawableSize]);
|
||||
self.drawableSizeShouldUpdate = NO;
|
||||
}
|
||||
[self.renderer renderPixelBuffer:pixelBuffer metalLayer:self.metalLayer mergeInfos:infos];
|
||||
}
|
||||
|
||||
- (void)dispose {
|
||||
[self.renderer dispose];
|
||||
}
|
||||
|
||||
#pragma mark - private
|
||||
|
||||
- (void)onMetalViewUnavailable{
|
||||
|
||||
if ([self.delegate respondsToSelector:@selector(onMetalViewUnavailable)]) {
|
||||
[self.delegate onMetalViewUnavailable];
|
||||
}
|
||||
}
|
||||
|
||||
@end
|
||||
|
||||
#endif
|
||||
Reference in New Issue
Block a user