Custom video rendering

Process frames

Process buffer callbacks:

- (bool)handleBeautyProcessBuffer:(CVPixelBufferRef)pixelBufferRef {
    if (!self.beautyEngine) {
        [self initBeautyEngine];
    }
    if (self.beautyEngine && pixelBufferRef)
    {
        QEPixelBufferData *bufferData = [QEPixelBufferData new];
        bufferData.bufferIn = pixelBufferRef;
        bufferData.bufferOut = pixelBufferRef;
        // Perform image editing on the pixelBuffer and output the processed buffer.
        kQueenResultCode resultCode = [self.beautyEngine processPixelBuffer:bufferData]; // The thread that executes this method must always be the same.
        if (resultCode == kQueenResultCodeOK && bufferData.bufferOut)
        {
            return YES;
        }
    }
    return NO;
}

Process texture callbacks:

- (bool)handleBeautyProcessTexture:(int)textureID withWidth:(int)width withHeight:(int)height {
    // See the pure texture processing mode section below.
}

3. Create a retouching processor

- (void)initBeautyEngine:(BOOL)processBuffer
{
    if (self.beautyEngine != nil) {
        return;
    }
    // Initialize the engine configuration object.
    QueenEngineConfigInfo *configInfo = [QueenEngineConfigInfo new];

    // Specify whether to automatically set the image rotation angle. Set this to YES if the device screen is locked and the default image is captured from the camera.
    configInfo.autoSettingImgAngle = YES;
    
    configInfo.runOnCustomThread = processBuffer ? NO : YES; // Set to YES for texture calls and NO for buffer calls. If set to NO, the process does not run on the current caller thread. A new thread is created internally.
    configInfo.withContext = processBuffer ? YES : NO;        // Set to NO for texture calls and YES for buffer calls. If set to YES, an internal glContext is created.
    
    // A debugging API to enable the log feature of the retouching SDK.
    // configInfo.enableDebugLog = YES;

    // Initialize the engine.
    self.beautyEngine = [[QueenEngine alloc] initWithConfigInfo:configInfo];
    
    ...
    // The rest of the code remains the same.
}

4. Process frames

- (int)handleBeautyProcessTexture:(int)textureID withWidth:(int)width withHeight:(int)height {
    if (!self.beautyEngine) {
        [self initBeautyEngine:NO];
    }

    // Advanced retouching features require a face detection algorithm, so you must pass bufData.
    // If you enable only basic retouching, do not call the updateInputDataAndRunAlg method.
    uint8_t* bufData = [self extractI420DataFromPixelBuffer:videoSample.pixelBuffer];
    [self.beautyEngine updateInputDataAndRunAlg:bufData
                                            withImgFormat:kQueenImageFormatNV12
                                            withWidth:width
                                            withHeight:height
                                             withStride:0
                                         withInputAngle:0
                                        withOutputAngle:0
                                           withFlipAxis:0];
    free(bufData);
    
    QETextureData* textureData = [[QETextureData alloc] init];
    textureData.inputTextureID = textureID;
    textureData.width = width;
    textureData.height = height;
    kQueenResultCode result = [self.beautyEngine processTexture:textureData];
    if (result != kQueenResultCodeOK)
    {
        return textureID;
    }
    return textureData.outputTextureID;
}

When processing in texture mode, the application layer must extract the buffer and pass it to the retouching SDK to call the API. This method improves overall performance by avoiding the re-extraction of buffer data from the texture.

The following code shows how to extract the buffer:

-(uint8_t *)extractI420DataFromPixelBuffer:(CVPixelBufferRef) pixelBuffer {
    uint8_t *bufData = nil;
    // Get the format information.
    OSType pixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer);
    size_t bufWidth = CVPixelBufferGetWidth(pixelBuffer);
    size_t bufHeight = CVPixelBufferGetHeight(pixelBuffer);
        
    // Lock the buffer.
    CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
    
    if (CVPixelBufferIsPlanar(pixelBuffer)) {
        if (pixelFormat == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) {
            uint8_t *yPlane = (uint8_t *)CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0);
            uint8_t *uvPlane = (uint8_t *)CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1);

            size_t yStride = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0);
            size_t uvStride = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1);

            size_t width = CVPixelBufferGetWidth(pixelBuffer);
            size_t height = CVPixelBufferGetHeight(pixelBuffer);
            // Allocate the standard NV12 size (no padding).
            size_t bufDataSize = width * height * 3 / 2;
            bufData = (uint8_t *)malloc(bufDataSize);
            // Copy the Y plane row by row (remove stride padding).
            for (size_t i = 0; i < height; i++) {
                memcpy(bufData + i * width, yPlane + i * yStride, width);
            }
            // Copy the UV plane row by row.
            for (size_t i = 0; i < height / 2; i++) {
                memcpy(bufData + width * height + i * width,
                       uvPlane + i * uvStride,
                       width);
            }
        }
        
    } else {
        // Packed format, such as BGRA.
        void *baseAddress = CVPixelBufferGetBaseAddress(pixelBuffer);
        size_t dataSize = CVPixelBufferGetDataSize(pixelBuffer);
        // Process the baseAddress.
        bufData = (uint8_t *)malloc(dataSize);
        memcpy(bufData, baseAddress, dataSize);
    }
    CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);

    return bufData;
}