ApsaraVideo Live:Custom video rendering

Créer un processeur de retouche

Les interfaces de rappel précédentes peuvent être implémentées en appelant la méthode handleBeautyProcess.

- (bool)processVideoBeauty:(AliRtcVideoDataSample *)videoSample {
    
    if(videoSample) {
        if(videoSample.type == AliRtcBufferType_CVPixelBuffer) {
            if(videoSample.pixelBuffer) {
                return [self handleBeautyProcessBuffer:videoSample.pixelBuffer];
            }
            return false ;
        }
    }
    
    return false ;
}

- (bool)handleBeautyProcessBuffer:(CVPixelBufferRef)pixelBufferRef {
    if (!self.beautyEngine) {
        [self initBeautyEngine:YES];
    }
    if (self.beautyEngine && pixelBufferRef)
    {
        QEPixelBufferData *bufferData = [QEPixelBufferData new];
        bufferData.bufferIn = pixelBufferRef;
        bufferData.bufferOut = pixelBufferRef;
        // Perform image editing on the pixelBuffer and output the processed buffer.
        kQueenResultCode resultCode = [self.beautyEngine processPixelBuffer:bufferData]; // The thread that executes this method must always be the same.
        if (resultCode == kQueenResultCodeOK && bufferData.bufferOut)
        {
            return YES;
        }
    }
    return NO;
}

La méthode initBeautyEngine crée le engine de retouche.

Définir les paramètres de retouche

#pragma mark Retouching-related processing
- (void)initBeautyEngine:(BOOL)processBuffer
{
    if (self.beautyEngine != nil) {
        return;
    }
    // Initialize the engine configuration information object.
    QueenEngineConfigInfo *configInfo = [QueenEngineConfigInfo new];

    // Specify whether to automatically set the image rotation angle. If the device screen is locked and the image is captured from the camera by default, you can set the image rotation angle automatically.
    configInfo.autoSettingImgAngle = YES;
    configInfo.runOnCustomThread = processBuffer ? NO : YES; // Set to YES for texture calls and NO for buffer calls. NO indicates that the call does not run on the current caller thread, and a new thread is created internally.
    configInfo.withContext = processBuffer ? YES : NO;        // Set to NO for texture calls and YES for buffer calls. YES indicates that a glContext needs to be created internally.

    // Debugging interface: Enable the logging feature of the Queen SDK.
    // configInfo.enableDebugLog = YES;
    
    // Initialize the engine.
    self.beautyEngine = [[QueenEngine alloc] initWithConfigInfo:configInfo];
    
    // Basic retouching.
    // Skin smoothing.
    [_beautyEngine setQueenBeautyType:kQueenBeautyTypeSkinBuffing enable:YES mode:kQueenBeautyFilterModeSkinBuffing_Natural];
    // Set the skin smoothing coefficient.
    [self.beautyEngine setQueenBeautyParams:kQueenBeautyParamsSkinBuffing value:0.5f];
    // Set the sharpening coefficient.
    [self.beautyEngine setQueenBeautyParams:kQueenBeautyParamsSharpen value:0.5f];
    
    // Skin whitening.
    [_beautyEngine setQueenBeautyType:kQueenBeautyTypeSkinWhiting enable:YES];
    // Set the skin whitening coefficient.
    [_beautyEngine setQueenBeautyParams:kQueenBeautyParamsWhitening value:0.5f];
    // Advanced retouching.
    [_beautyEngine setQueenBeautyType:kQueenBeautyTypeFaceBuffing enable:YES];
    // Wrinkle removal.
    [_beautyEngine setQueenBeautyParams:kQueenBeautyParamsWrinkles value:0.5f];
    // Set the eye bag removal coefficient.
    [_beautyEngine setQueenBeautyParams:kQueenBeautyParamsPouch value:0.5f];
    // Set the nasolabial fold removal coefficient.
    [_beautyEngine setQueenBeautyParams:kQueenBeautyParamsNasolabialFolds value:0.5f];
    // Set the teeth whitening coefficient.
    [_beautyEngine setQueenBeautyParams:kQueenBeautyParamsWhiteTeeth value:0.5f];
    
    // Enable the face shaping feature.
    [_beautyEngine setQueenBeautyType:kQueenBeautyTypeFaceShape enable:YES mode:kQueenBeautyFilterModeFaceShape_Main];
    // Face slimming.
    [_beautyEngine setFaceShape:kQueenBeautyFaceShapeTypeCutFace value:0.5f];
    // Eye enlarging.
    [_beautyEngine setFaceShape:kQueenBeautyFaceShapeTypeBigEye value:0.9f];
    
    // Debugging interface: Display facial recognition feature points.
    //[self.beautyEngine showFaceDetectPoint:YES];
    
}

- (bool)handleBeautyProcessBuffer:(CVPixelBufferRef)pixelBufferRef {
    if (!self.beautyEngine) {
        [self initBeautyEngine];
    }
    if (self.beautyEngine && pixelBufferRef)
    {
        QEPixelBufferData *bufferData = [QEPixelBufferData new];
        bufferData.bufferIn = pixelBufferRef;
        bufferData.bufferOut = pixelBufferRef;
        // Perform image editing on the pixelBuffer and output the processed buffer.
        kQueenResultCode resultCode = [self.beautyEngine processPixelBuffer:bufferData]; // The thread that executes this method must always be the same.
        if (resultCode == kQueenResultCodeOK && bufferData.bufferOut)
        {
            return YES;
        }
    }
    return NO;
}

- (bool)handleBeautyProcessTexture:(int)textureID withWidth:(int)width withHeight:(int)height {
    // See the pure texture processing mode section below.
}

- (void)initBeautyEngine:(BOOL)processBuffer
{
    if (self.beautyEngine != nil) {
        return;
    }
    // Initialize the engine configuration object.
    QueenEngineConfigInfo *configInfo = [QueenEngineConfigInfo new];

    // Specify whether to automatically set the image rotation angle. Set this to YES if the device screen is locked and the default image is captured from the camera.
    configInfo.autoSettingImgAngle = YES;
    
    configInfo.runOnCustomThread = processBuffer ? NO : YES; // Set to YES for texture calls and NO for buffer calls. If set to NO, the process does not run on the current caller thread. A new thread is created internally.
    configInfo.withContext = processBuffer ? YES : NO;        // Set to NO for texture calls and YES for buffer calls. If set to YES, an internal glContext is created.
    
    // A debugging API to enable the log feature of the retouching SDK.
    // configInfo.enableDebugLog = YES;

    // Initialize the engine.
    self.beautyEngine = [[QueenEngine alloc] initWithConfigInfo:configInfo];
    
    ...
    // The rest of the code remains the same.
}

- (int)handleBeautyProcessTexture:(int)textureID withWidth:(int)width withHeight:(int)height {
    if (!self.beautyEngine) {
        [self initBeautyEngine:NO];
    }

    // Advanced retouching features require a face detection algorithm, so you must pass bufData.
    // If you enable only basic retouching, do not call the updateInputDataAndRunAlg method.
    uint8_t* bufData = [self extractI420DataFromPixelBuffer:videoSample.pixelBuffer];
    [self.beautyEngine updateInputDataAndRunAlg:bufData
                                            withImgFormat:kQueenImageFormatNV12
                                            withWidth:width
                                            withHeight:height
                                             withStride:0
                                         withInputAngle:0
                                        withOutputAngle:0
                                           withFlipAxis:0];
    free(bufData);
    
    QETextureData* textureData = [[QETextureData alloc] init];
    textureData.inputTextureID = textureID;
    textureData.width = width;
    textureData.height = height;
    kQueenResultCode result = [self.beautyEngine processTexture:textureData];
    if (result != kQueenResultCodeOK)
    {
        return textureID;
    }
    return textureData.outputTextureID;
}

-(uint8_t *)extractI420DataFromPixelBuffer:(CVPixelBufferRef) pixelBuffer {
    uint8_t *bufData = nil;
    // Get the format information.
    OSType pixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer);
    size_t bufWidth = CVPixelBufferGetWidth(pixelBuffer);
    size_t bufHeight = CVPixelBufferGetHeight(pixelBuffer);
        
    // Lock the buffer.
    CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
    
    if (CVPixelBufferIsPlanar(pixelBuffer)) {
        if (pixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) {
            uint8_t *yPlane = (uint8_t *)CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0);
            uint8_t *uvPlane = (uint8_t *)CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1);

            size_t yStride = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0);
            size_t uvStride = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1);

            size_t width = CVPixelBufferGetWidth(pixelBuffer);
            size_t height = CVPixelBufferGetHeight(pixelBuffer);
            // Allocate the standard NV12 size (no padding).
            size_t bufDataSize = width * height * 3 / 2;
            bufData = (uint8_t *)malloc(bufDataSize);
            // Copy the Y plane row by row (remove stride padding).
            for (size_t i = 0; i < height; i++) {
                memcpy(bufData + i * width, yPlane + i * yStride, width);
            }
            // Copy the UV plane row by row.
            for (size_t i = 0; i < height / 2; i++) {
                memcpy(bufData + width * height + i * width,
                       uvPlane + i * uvStride,
                       width);
            }
        }
        
    } else {
        // Packed format, such as BGRA.
        void *baseAddress = CVPixelBufferGetBaseAddress(pixelBuffer);
        size_t dataSize = CVPixelBufferGetDataSize(pixelBuffer);
        // Process the baseAddress.
        bufData = (uint8_t *)malloc(dataSize);
        memcpy(bufData, baseAddress, dataSize);
    }
    CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);

    return bufData;
}