ApsaraVideo Live:Integrate Queen SDK for Web

Initialize QueenEngine.

const sdkLicenseKey = ""; // The license key of Queen SDK.
const queenEngine = new QueenEngine();
//const queenEngine = new QueenEnginePro(); // Initialize Queen SDK Pro Edition.
queenEngine.create({
  SdkLicenseKey: sdkLicenseKey,
  OnInit: function(result: Boolean){
      console.info("queen sdk initialized.", result)
  },
  Domain: '',// Domain. You can use the root domain, such as aliyun.com.
  OnProgress: function(progress: Number){
      console.info("queen sdk loading:", progress);
  }
});

Initialize a specified canvas.

Set the canvasElement type parameter to webgl2 or leave it empty.

const canvasElement = document.getElementById("canvas");
const queenEngine = new QueenEngine();
queenEngine.create({
  SdkLicenseKey: sdkLicenseKey,
  OnInit: function(result: Boolean){
      console.info("queen sdk initialized.", result)  
  },
  Domain: '',// Domain. You can use the root domain.
  OnProgress: function(progress: Number){
      console.info("queen sdk loading:", progress);
  },
  RenderCanvas: canvasElement
});

Initialize specified inference models.

const queenEngine = new QueenEnginePro();
queenEngine.create({
  SdkLicenseKey: sdkLicenseKey,
  OnInit: function(result: Boolean){
      console.info("queen sdk initialized.", result)     
  },
  Domain: '',// Domain. You can use the root domain.
  OnProgress: function(progress: Number){
      console.info("queen sdk loading:", progress);
  },
  RenderCanvas: canvasElement// Optional.
  SegmentModel: kQueenModelShapeType.Vertical,// For portrait-oriented video. If your application does not require image matting, pass kQueenModelShapeType.None.
  PoseModel: kQueenModelShapeType.None,// Set to `None` to avoid loading body shaping model at initialization.
  InferenceBackend: kBackendType.WebGL,// The backend for model inference. In this example, WebGL is used.
});

Initialize a QueenEngineWorker instance.

import {QueenEngineWorker, kQueenBeautyType, kQueenBeautyParams, kQueenBeautyMakeupType, kQueenBeautyFaceShapeType, kQueenBeautyBlend} from "aliyun-queen-engine"
const canvasElement = document.getElementById("canvas");
queenEngine = new QueenEngineWorker(kQueenVersion.Pro);
queenEngine.init(sdkLicenseKey, function(){
 // The initialization is complete.
}, function(progress){
 // progress: the loading progress.
}, canvasElement);

Configure basic face retouching.

// Enable basic retouching.
queenEngine.setQueenBeautyType(kQueenBeautyType.SkinBuffing, true);
// Set the level of skin smoothing.
queenEngine.setQueenBeautyParams(kQueenBeautyParams.SkinBuffing, 0.7);
// Set the level of image sharpening.
queenEngine.setQueenBeautyParams(kQueenBeautyParams.Sharpen, 0.5);
// Enable skin whitening.
queenEngine.setQueenBeautyType(kQueenBeautyType.SkinWhiting, true);
// Set the level of skin whitening.
queenEngine.setQueenBeautyParams(kQueenBeautyParams.Whitening, 0.6);
// Set the level of rosy cheeks.
queenEngine.setQueenBeautyParams(kQueenBeautyParams.SkinRed, 0.2);

Configure advanced face retouching.

// Enable advanced face retouching.
queenEngine.setQueenBeautyType(kQueenBeautyType.FaceBuffing, true);
// Set the level of eye bag removal.
queenEngine.setQueenBeautyParams(kQueenBeautyParams.Pouch, 0.9);
// Set the level of nasolabial fold removal.
queenEngine.setQueenBeautyParams(kQueenBeautyParams.NasolabialFolds, 0.9);
// Set the level of teeth whitening.
queenEngine.setQueenBeautyParams(kQueenBeautyParams.WhiteTeeth, 0.9);
// Set the level of lipstick.
queenEngine.setQueenBeautyParams(kQueenBeautyParams.Lipstick, 0.2);
// Set the level of blush.
queenEngine.setQueenBeautyParams(kQueenBeautyParams.Blush, 0.1);
// Set the color of lipstick.
queenEngine.setQueenBeautyParams(kQueenBeautyParams.LipstickColorParam, 0.1);
// Set the saturation of lipstick.
queenEngine.setQueenBeautyParams(kQueenBeautyParams.LipstickGlossParam, 0.1);
// Set the brightness of lipstick.
queenEngine.setQueenBeautyParams(kQueenBeautyParams.LipstickBrightnessParam, 0.1);
// Set the level of eye brightening.
queenEngine.setQueenBeautyParams(kQueenBeautyParams.BrightenEye, 0.9);
// Set the level of rosy cheeks.
queenEngine.setQueenBeautyParams(kQueenBeautyParams.SkinRed, 0.2);
// Set the level of wrinkle removal.
queenEngine.setQueenBeautyParams(kQueenBeautyParams.Wrinkles, 0.9);
// Set the level of skin brightening.
queenEngine.setQueenBeautyParams(kQueenBeautyParams.BrightenFace, 0.3);

Configure face shaping.

// Enable face shaping. The face shaping parameters are indicated by kQueenBeautyFaceShapeType.
queenEngine.setQueenBeautyType(kQueenBeautyType.FaceShape, true);
// Set the cheekbones. Valid values: [0,1]. Default value: 0.
queenEngine.setFaceShape(kQueenBeautyFaceShapeType.CutCheek, 0.6);
// Set cheekbone narrowing. Valid values: [0,1]. Default value: 0.
queenEngine.setFaceShape(kQueenBeautyFaceShapeType.CutFace, 0.7);
// Set face slimming. Valid values: [0,1]. Default value: 0.
queenEngine.setFaceShape(kQueenBeautyFaceShapeType.ThinFace, 0.8);
// Set the face length. Valid values: [0,1]. Default value: 0.
queenEngine.setFaceShape(kQueenBeautyFaceShapeType.LowerJaw, 0.8);
// Set chin lengthening. Valid values: [0,1]. Default value: 0.
queenEngine.setFaceShape(kQueenBeautyFaceShapeType.HigherJaw, 0.6); 

Configure makeup resources.

• Use the built-in makeup resources of the SDK.

  The SDK downloads retouching materials from Alibaba Cloud CDN when you use the built-in resources.

  // Enable makeup.
  queenEngine.setQueenBeautyType(kQueenBeautyType.Makeup, true);
  // Set the eyebrow effect.
  await queenEngine.setMakeupEyeBrow(Assets.kResMakeupEyeBrow.BiaoZhunMei, 0.6);
  // Set the eyelash effect.
  await queenEngine.setMackupEyeLash(Assets.kResMakeupEyeLash.ChenJing, 0.6);
  // Set the eye shadow effect.
  await queenEngine.setMakeupEyeShadow(Assets.kResMakeupEyeShadow.DaDiSe, 0.5);
  // Set the eyeliner effect.
  await queenEngine.setMakeupEyeLiner(Assets.kResMakeupEyeLiner.DaYan, 0.4);
  // Set the colored eye contacts effect.
  await queenEngine.setMakeupEyeBall(Assets.kResMakeupEyeBall.BiMuYu, 0.5);
  // Set the lipstick effect.
  await queenEngine.setMakeupMouth(Assets.kResMakeupMouth.AnYeZi, 0.3);
  // Set the blush effect.
  await queenEngine.setMakeupBlush(Assets.kResMakeupBlush.BlushWuGu, 0.2);
  // Set the highlight effect.
  await queenEngine.setMakeupHighlight(Assets.kResMakeupHighLight.Highlight, 0.1);
  // Remove the colored eye contacts effect.
  queenEngine.removeMakeupWithType(kQueenBeautyMakeupType.Eyeball);
  // For more information, see the QueenEngin.d.ts file.

• Use your local makeup resources.

  const makeupPackage = "./mouth.zip"
  const makeupName = "1.2.3.png";
  const band = kQueenBeautyBlend.LabMix;
  // Enable makeup. The makeup parameters are indicated by kQueenBeautyMakeupType.
  queenEngine.setQueenBeautyType(kQueenBeautyType.Makeup, true);
  // Set the transparency of the lipstick effect.
  queenEngine.setMakeupAlphaWithType(kQueenBeautyMakeupType.Mouth, true, 0.6);
  // Set the lipstick effect.
  queenEngine.setMakeupWithPackage(kQueenBeautyMakeupType.Mouth, makeupPackage, makeupName, band).then(() => {
  
  });

Configure filters.

• Use built-in filters.

  await queenEngine.setLutByType(Assets.kResLut.M1, 0.5);

• Use custom filters.

  const lutImageUrl = "./lut.png";
  queenEngine.setLutImageUrl(lutImageUrl).then(function () {
    queenEngine.setQueenBeautyType(kQueenBeautyType.LUT, true);
    queenEngine.setQueenBeautyParams(kQueenBeautyParams.LUT, 0.5);
  });

Configure stickers.

1. Use built-in stickers.

   queenEngine.addMaterialWithType(Assets.kResSticker.ILoveChina);
   // Specify a single sticker.
   queenEngine.addMaterialWithIndex(0);
   // Or, specify multiple stickers.
   queenEngine.addMaterialWithIndexs([0,1]);

2. Use custom stickers.

   const stickerZip = "./sticker.zip";
   queenEngine.addMaterialWithUrl(stickerZip).then(() => {
    });

Configure image matting.

const backgroundUrl = "./bg.png";
queenEngine.setSegmentBackgroundUrl(backgroundUrl).then(() => {
 });

Configure green-screen chroma keying.

const backgroundUrl = "./bg.png";
const isBlue = false;// Specify whether to use green-screen chroma keying.
queenEngine.setGreenScreenWithUrl(isBlue, backgroundUrl).then(() => {
 });

Configure background processing.

// Blur the background.
queenEngine.enableBokehBackground(true);
// Make the background transparent.
queenEngine.enableTransparentBackground(true);

Render the camera stream.

 queenEngine.openCameraAndRender().then((stream=>{
 const video = document.querySelector('video');
 video.srcObject = stream;
 video.play();
 })) 

Apply the rendered media to the canvas.

The canvas is the one specified during engine initialization.

 const sourceVideo = document.querySelector('video');
 queenEngine.renderMediaObjectToRenderCanvas(sourceVideo, sourceVideo.clientWidth, sourceVideo.clientHeight);

Render the video stream.

 navigator.mediaDevices.getUserMedia(constraints)
 .then(mediaStream => {
 let renderMediaStream = queenEngine.renderMediaStream(mediaStream);
 const video = document.querySelector('video');
 video.srcObject = renderMediaStream;
 video.play();
 });

Render an image.

fetch(Image URL)
.then(buffer => buffer.blob())
.then(createImageBitmap)
.then(img => {
 queenEngine.renderWithMediaObject(img, img.width, img.height, function(imageBufferData, imageWidth, imageHeight){
 const canvas = document.getElementById('playCanvas'); // The canvas. 
 const ctx = canvas.getContext("2d");
 const imageBuffer = new Uint8ClampedArray(imageBufferData);
 const imageData = new ImageData(imageBuffer, imageWidth, imageHeight);
 ctx.clearRect(0, 0, imageWidth, imageHeight);
 ctx.putImageData(imageData, 0, 0);
});
});

Render the texture.

const canvas = document.getElementById("sourceCanvas");
let outTexture = queenEngine.renderMediaObjectToTexture(canvas, canvas.width, canvas.height);
queenEngine.drawOutTexture(outTexture);// Draw the texture in the canvas that was specified during initialization.

Render the texture input.

let inputTexture = queenEngine.generateTextureId();
queenEngine.updateTexture(inputTexture, imageData);
let outTexture = queenEngine.renderTextureId(inputTexture, width, height);
queenEngine.drawOutTexture(outTexture)// Draw the texture in the canvas that was specified during initialization.

Render the pipeline stream.

const videoTrack = stream.getVideoTracks()[0];
const processor = new MediaStreamTrackProcessor({ track: videoTrack });
const readFrameStream = processor.readable;
const generator = new MediaStreamTrackGenerator({ kind: 'video' });
let writeFrameStream = generator.writable;
readFrameStream.pipeThrough(queenEngine.getTransformStream()).pipeTo(writeFrameStream);