Flutter

This document mainly introduces how to integrate beauty effects in TUICallKit.
To perform custom beauty processing in Flutter, it is necessary to use TRTC's custom video rendering. Due to Flutter's lack of proficiency in handling real-time transmission of large data volumes, work involving TRTC's custom video rendering needs to be completed in the Native section. The specific plan is as follows:
﻿
The integration plan is divided into 3 steps:
Step 1: Enable/Disable the TRTC custom rendering logic through MethodChannel;
Step 2: In the TRTC custom rendering processing logic onProcessVideoFrame(), use the beauty processing module to process the original video frames;
Step 3: On the user's beauty processing module, the beauty parameters also need to be set through the Dart interface. Users can set beauty parameters through MethodChannel based on their needs and the beauty effects they use.
Integrating Third-Party Beauty Effects
Step 1: Implement the control interface of enabling/disabling beauty effects from Dart to Native.
Implement the interface in Dart:
 final channel = MethodChannel('TUICallKitCustomBeauty');
  
 void enableTUICallKitCustomBeauty() async {
    await channel.invokeMethod('enableTUICallKitCustomBeauty');   
 }
﻿
 void disableTUICallKitCustomBeauty() async {
    await channel.invokeMethod('disableTUICallKitCustomBeauty');
 }
Implement the corresponding interface in Native:
Java
Swift
public class MainActivity extends FlutterActivity {
    private static final String channelName = "TUICallKitCustomBeauty";
    
    private MethodChannel channel;
    
    @Override
    public void configureFlutterEngine(@NonNull FlutterEngine flutterEngine) {
        super.configureFlutterEngine(flutterEngine);
        
        channel = new MethodChannel(flutterEngine.getDartExecutor().getBinaryMessenger(), channelName);
        channel.setMethodCallHandler(((call, result) -> {
            switch (call.method) {
                case "enableTUICallKitCustomBeauty":
                    enableTUICallKitCustomBeauty();
                    break;
                case "disableTUICallKitCustomBeauty":
                    disableTUICallKitCustomBeauty();
                    break;
                default:
                    break;
            }
            result.success("");
        }));
    }
    
    public void enableTUICallKitCustomBeauty() {
    
    }
    
    public void disableTUICallKitCustomBeauty() {
    
    }
}J
@UIApplicationMain
@objc class AppDelegate: FlutterAppDelegate {
    var channel: FlutterMethodChannel?
    
    override func application(_ application: UIApplication,
                              didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]?) -> Bool {
        GeneratedPluginRegistrant.register(with: self)
   
        guard let controller = window?.rootViewController as? FlutterViewController else {
            fatalError("Invalid root view controller")
        }
        channel = FlutterMethodChannel(name: "TUICallKitCustomBeauty", binaryMessenger: controller.binaryMessenger)
        channel?.setMethodCallHandler({ [weak self] call, result in
            guard let self = self else { return }
            switch (call.method) {
            case "enableTUICallKitCustomBeauty":
                self.enableTUICallKitCustomBeauty()
                break﻿
            case "disableTUICallKitCustomBeauty":
                self.disableTUICallKitCustomBeauty()
                break﻿
            default:
                break
            }
        })    
        result(nil)   
        return super.application(application, didFinishLaunchingWithOptions: launchOptions)
    }
    
    func enableTUICallKitCustomBeauty() {
    
    }
    
    func disableTUICallKitCustomBeauty() {
    
    }
}S
Step 2: Complete the beauty processing in the TRTC custom rendering logic in Native.
Note:
Android requires a dependency on LiteAVSDK_Professional during the beauty integration process. Add the following dependency in the Android project's app/build.gradle:
dependencies{
    api "com.tencent.liteav:LiteAVSDK_Professional:latest.release"
}
Java
Swift
Jvoid enableTUICallKitCustomBeauty() {
    TRTCCloud.sharedInstance(getApplicationContext()).
                setLocalVideoProcessListener(TRTC_VIDEO_PIXEL_FORMAT_Texture_2D,
                        TRTC_VIDEO_BUFFER_TYPE_TEXTURE, new VideoFrameListerer());
﻿
}
﻿
void disableTUICallKitCustomBeauty() {
    TRTCCloud.sharedInstance(getApplicationContext()).
                setLocalVideoProcessListener(TRTC_VIDEO_PIXEL_FORMAT_Texture_2D,
                        TRTC_VIDEO_BUFFER_TYPE_TEXTURE, null);
}
﻿
class VideoFrameListerer implements TRTCCloudListener.TRTCVideoFrameListener {
    private XXXBeautyModel  mBeautyModel = XXXBeautyModel.sharedInstance(); 
   
    @Override
    public int onProcessVideoFrame(TRTCCloudDef.TRTCVideoFrame trtcVideoFrame,
                                   TRTCCloudDef.TRTCVideoFrame trtcVideoFrame1) {
        //Beauty processing logic
        mBeautyModel.process(trtcVideoFrame, trtcVideoFrame1);
        …
        
        return 0;
    }

    @Override
    public void onGLContextCreated() {
    }
    
    @Override
    public void onGLContextDestory() {
    }
}
let videoFrameListener: TRTCVideoFrameListener = TRTCVideoFrameListener()
﻿
func enableTUICallKitCustomBeauty() {
    TRTCCloud.sharedInstance().setLocalVideoProcessDelegete(videoFrameListener, pixelFormat: ._Texture_2D, bufferType: .texture)
}
﻿
func disableTUICallKitCustomBeauty() {
    TRTCCloud.sharedInstance().setLocalVideoProcessDelegete(nil, pixelFormat: ._Texture_2D, bufferType: .texture)
}
﻿
class TRTCVideoFrameListener: NSObject, TRTCVideoFrameDelegate {
    let bueutyModel = XXXXBeautyModel.shareIntance()
    func onProcessVideoFrame(_ srcFrame: TRTCVideoFrame, dstFrame: TRTCVideoFrame) -> UInt32 {
        //Beauty processing logic
        bueutyModel.onProcessVideoFrame(srcFrame, dstFrame)
        …
        
        return 0
    }
}S
Step 3: Customize third-party beauty parameter control logic.
In this step, users can use a specific beauty module as needed. See Step 1 for the implementation of beauty parameter settings. The specific implementation depends on the actual use case.
Integrating Tencent Beauty Effects
The method of integrating Tencent beauty effects also follows the above-mentioned steps. The following description takes Tencent beauty effects as an example to illustrate the method of integration.
Step 1: Download and integrate beauty effect resources.
1. Based on the package you purchased, Download SDK.
2. Add the resource files to your own project:
Android
iOS
1. In the app module, find the build.gradle file and add the Maven reference address corresponding to your package. For example, if you have chosen the S1-04 package, add the following code:
dependencies {
   implementation 'com.tencent.mediacloud:TencentEffect_S1-04:latest.release'
}
For the Maven address corresponding to each package, refer to Documentation.
2. In the app module, find the src/main/assets folder, or create one if it does not exist. Check if the downloaded SDK package includes the MotionRes folder, and if so, copy this folder to the ../src/main/assets directory.
3. Find the AndroidManifest.xml file in the app module, and add the following tag in the application element.
 <uses-native-library
        android:name="libOpenCL.so"
        android:required="true" />
        //The "true" here indicates that if the library is unavailable, the application will not be able to run normally. The system does not allow installation of the application on devices without this library.
        //The "false" indicates that the application can utilize the library (if available), but is specifically designed to run without the library (if necessary). The system allows the application to be installed, even in the absence of this library. If you use "false", you need to properly handle the absence of the library.
        //Official Android website introduction: %!s(<nil>)
The tag is added, as shown in the following figure:
﻿
4. Aliasing  Configuration
If you enable compile optimizations while building the release package (by setting minifyEnabled to true), some codes that are not called at the Java level might be trimmed. However, these codes could potentially be called at the Native level, leading to a  no xxx method exception.
If you enable such compile optimizations, you must add these "keep" rules to prevent the codes of xmagic from being trimmed:
-keep class com.tencent.xmagic.** { *;}
-keep class org.light.** { *;}
-keep class org.libpag.** { *;}
-keep class org.extra.** { *;}
-keep class com.gyailib.**{ *;}
-keep class com.tencent.cloud.iai.lib.** { *;}
-keep class com.tencent.beacon.** { *;}
-keep class com.tencent.qimei.** { *;}
-keep class androidx.exifinterface.** { *;}
1. Add beauty effect resources to your project, as shown in the following figure (your resources may not exactly match those in the following figure):
﻿
2. In the demo, copy the following four classes from demo/lib/producer to your own Flutter project: BeautyDataManager, BeautyPropertyProducer, BeautyPropertyProducerAndroid, and BeautyPropertyProducerIOS. These four classes are used to configure beauty effect resources and display the beauty types on the beauty panel.
Step 2: Reference the Flutter SDK.
GitHub Reference: Add the following reference in your project's pubspec.yaml file:
 tencent_effect_flutter:
   git:
     url: https://github.com/TencentCloud/tencenteffect-sdk-flutter
Local Reference: Download the latest version of tencent_effect_flutter, then add the folders android, ios, lib, and the files pubspec.yaml, tencent_effect_flutter.iml to the project directory. Next, add the following reference in your project's pubspec.yaml file (you can refer to the demo) :
tencent_effect_flutter:
    path: ../
tencent_effect_flutter merely provides a bridge, and it is the XMagic that is has dependancy on, which is set to the latest version by default, that actually implements beauty effects.
To use the latest version of the beauty SDK, you can upgrade the SDK by following these steps:
Android
iOS
Execute the flutter pub upgrade command in the project directory or click Pub upgrade in the top right corner of the subspec.yaml page.
Execute the flutter pub upgrade command in the project directory, and then execute the pod update command in the iOS directory.
Step 3: Implement the control interface of enabling/disabling beauty effects from Dart to Native.
For this part, refer to Integrating Third-Party Beauty Effects - Step 1.
Step 4: Complete the beauty processing in the TRTC custom rendering logic in Native.
Android
iOS
Android requires a dependency on LiteAVSDK_Professional during the beauty integration process. Add the following dependency in the Android project's app/build.gradle:
dependencies{
    api "com.tencent.liteav:LiteAVSDK_Professional:latest.release"
}
void enableTUICallKitCustomBeauty() {
    TRTCCloud.sharedInstance(getApplicationContext()).
                setLocalVideoProcessListener(TRTC_VIDEO_PIXEL_FORMAT_Texture_2D,
                        TRTC_VIDEO_BUFFER_TYPE_TEXTURE, new VideoFrameListerer());
﻿
}
﻿
void disableTUICallKitCustomBeauty() {
    TRTCCloud.sharedInstance(getApplicationContext()).
                setLocalVideoProcessListener(TRTC_VIDEO_PIXEL_FORMAT_Texture_2D,
                        TRTC_VIDEO_BUFFER_TYPE_TEXTURE, null);
}
﻿
class VideoFrameListerer implements TRTCCloudListener.TRTCVideoFrameListener {
    private XXXBeautyModel  mBeautyModel = XXXBeautyModel.sharedInstance(); 
   
    @Override
    public int onProcessVideoFrame(TRTCCloudDef.TRTCVideoFrame trtcVideoFrame,
                                   TRTCCloudDef.TRTCVideoFrame trtcVideoFrame1) {
       trtcVideoFrame1.texture.textureId = XmagicApiManager.getInstance()
        .process(trtcVideoFrame.texture.textureId, trtcVideoFrame.width, trtcVideoFrame.height);
        return 0;
    }
﻿
    @Override
    public void onGLContextCreated() {
    }
﻿
    @Override
    public void onGLContextDestory() {
    }
}
import TXLiteAVSDK_Professional
import tencent_effect_flutter
﻿
let videoFrameListener: TRTCVideoFrameListener = TRTCVideoFrameListener()
﻿
func enableTUICallKitCustomBeauty() {
    TRTCCloud.sharedInstance().setLocalVideoProcessDelegete(videoFrameListener, pixelFormat: ._Texture_2D, bufferType: .texture)
}
﻿
func disableTUICallKitCustomBeauty() {
    TRTCCloud.sharedInstance().setLocalVideoProcessDelegete(nil, pixelFormat: ._Texture_2D, bufferType: .texture)
}
﻿
class TRTCVideoFrameListener: NSObject, TRTCVideoFrameDelegate {
    func onProcessVideoFrame(_ srcFrame: TRTCVideoFrame, dstFrame: TRTCVideoFrame) -> UInt32 {
        dstFrame.textureId = GLuint(XmagicApiManager.shareSingleton().getTextureId(ConvertBeautyFrame.convertTRTCVideoFrame(trtcVideoFrame: srcFrame)))
        return 0
    }
}
﻿
﻿
public class ConvertBeautyFrame: NSObject {
    public static func convertToTRTCPixelFormat(beautyPixelFormat: ITXCustomBeautyPixelFormat) -> TRTCVideoPixelFormat {
        switch beautyPixelFormat {
        case .Unknown:
            return ._Unknown
        case .I420:
            return ._I420
        case .Texture2D:
            return ._Texture_2D
        case .BGRA:
            return ._32BGRA
        case .NV12:
            return ._NV12
        }
    }
﻿
    public static func convertTRTCVideoFrame(trtcVideoFrame: TRTCVideoFrame) -> ITXCustomBeautyVideoFrame {
        let beautyVideoFrame = ITXCustomBeautyVideoFrame()
        beautyVideoFrame.data = trtcVideoFrame.data
        beautyVideoFrame.pixelBuffer = trtcVideoFrame.pixelBuffer
        beautyVideoFrame.width = UInt(trtcVideoFrame.width)
        beautyVideoFrame.height = UInt(trtcVideoFrame.height)
        beautyVideoFrame.textureId = trtcVideoFrame.textureId
        switch trtcVideoFrame.rotation {
        case ._0:
            beautyVideoFrame.rotation = .rotation_0
        case ._90:
            beautyVideoFrame.rotation = .rotation_90
        case ._180:
            beautyVideoFrame.rotation = .rotation_180
        case ._270:
            beautyVideoFrame.rotation = .rotation_270
        default:
            beautyVideoFrame.rotation = .rotation_0
        }
        switch trtcVideoFrame.pixelFormat {
        case ._Unknown:
            beautyVideoFrame.pixelFormat = .Unknown
        case ._I420:
            beautyVideoFrame.pixelFormat = .I420
        case ._Texture_2D:
            beautyVideoFrame.pixelFormat = .Texture2D
        case ._32BGRA:
            beautyVideoFrame.pixelFormat = .BGRA
        case ._NV12:
            beautyVideoFrame.pixelFormat = .NV12
        default:
            beautyVideoFrame.pixelFormat = .Unknown
        }
        beautyVideoFrame.bufferType = ITXCustomBeautyBufferType(rawValue: trtcVideoFrame.bufferType.rawValue) ?? .Unknown
        beautyVideoFrame.timestamp = trtcVideoFrame.timestamp
        return beautyVideoFrame
    }
}
Step 5: Enable beauty effects and set beauty parameters.
After completing the configurations above, you can enable/disable beauty effects by using enableTUICallKitCustomBeauty()/disableTUICallKitCustomBeauty() . You can set beauty parameters through the Tencent Effect beauty Flutter interface.
﻿
Integrating Third-Party Beauty Effects
Integrating Tencent Beauty Effects