Issue with Texture Format Conversion and Accessing GPU Data in Unity
Problem to solve
Texture Format Conversion and Accessing GPU Data in Unity
Intended users
For those who want to use VLC and add OpenCV processing
Proposal (Outline the problem you want to solve)
I'm encountering an issue when trying to perform face detection in Unity using the MinimalPlayback code provided below. The error message I receive is: Graphics.CopyTexture can only copy between same texture format groups (d3d11 base formats: src=0 dst=9). This error occurs at the line Graphics.CopyTexture(cpuAccessibleTexture, tex);.
Interestingly, when I check the format of the textures immediately before this line, both tex and cpuAccessibleTexture appear to have the same format: RGB24.
MinimalPlayback code goes here
using UnityEngine;
using System;
using LibVLCSharp;
using OpenCVForUnity;
using OpenCVForUnity.CoreModule;
using OpenCVForUnity.ImgprocModule;
using OpenCVForUnity.UnityUtils;
using OpenCVForUnityExample;
using DlibFaceLandmarkDetector;
using DlibFaceLandmarkDetectorExample;
using DlibFaceLandmarkDetector.UnityUtils;
using OpenCVForUnity.UnityUtils.Helper;
using System.Collections;
using System.Collections.Generic;
using System.Runtime.InteropServices;
/// this class serves as an example on how to configure playback in Unity with VLC for Unity using LibVLCSharp.
/// for libvlcsharp usage documentation, please visit https://code.videolan.org/videolan/LibVLCSharp/-/blob/master/docs/home.md
public class MinimalPlayback : MonoBehaviour
{
LibVLC _libVLC;
MediaPlayer _mediaPlayer;
const int seekTimeDelta = 5000;
Texture2D tex = null;
bool playing;
public string URL;
//def : http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4
FaceLandmarkDetector faceLandmarkDetector;
string dlibShapePredictorFileName = "DlibFaceLandmarkDetector/sp_human_face_68.dat";
string dlibShapePredictorFilePath;
//public TextureProcessingExample textureProcessor;
Texture2D cpuAccessibleTexture = null;
RenderTexture renderTexture = null;
void Awake()
{
LibVLCSharp.Core.Initialize(Application.dataPath);
_libVLC = new LibVLC("--no-osd", "--verbose=2");
dlibShapePredictorFilePath = DlibFaceLandmarkDetector.UnityUtils.Utils.getFilePath(dlibShapePredictorFileName);
faceLandmarkDetector = new FaceLandmarkDetector(dlibShapePredictorFilePath);
Application.SetStackTraceLogType(LogType.Log, StackTraceLogType.None);
//_libVLC.Log += (s, e) => UnityEngine.Debug.Log(e.FormattedLog); // enable this for logs in the editor
PlayPause();
}
public void SeekForward()
{
Debug.Log("[VLC] Seeking forward !");
_mediaPlayer.SetTime(_mediaPlayer.Time + seekTimeDelta);
}
public void SeekBackward()
{
Debug.Log("[VLC] Seeking backward !");
_mediaPlayer.SetTime(_mediaPlayer.Time - seekTimeDelta);
}
void OnDisable()
{
_mediaPlayer?.Stop();
_mediaPlayer?.Dispose();
_mediaPlayer = null;
_libVLC?.Dispose();
_libVLC = null;
}
public void PlayPause()
{
Debug.Log("[VLC] Toggling Play Pause !");
if (_mediaPlayer == null)
{
_mediaPlayer = new MediaPlayer(_libVLC);
}
if (_mediaPlayer.IsPlaying)
{
_mediaPlayer.Pause();
}
else
{
playing = true;
if (_mediaPlayer.Media == null)
{
// playing remote media
_mediaPlayer.Media = new Media(_libVLC, new Uri(URL));
}
_mediaPlayer.Play();
}
}
public void Stop()
{
Debug.Log("[VLC] Stopping Player !");
playing = false;
_mediaPlayer?.Stop();
// there is no need to dispose every time you stop, but you should do so when you're done using the mediaplayer and this is how:
// _mediaPlayer?.Dispose();
// _mediaPlayer = null;
GetComponent<Renderer>().material.mainTexture = null;
tex = null;
}
void OnDestroy()
{
if (renderTexture != null)
{
renderTexture.Release();
}
}
void Update()
{
if (!playing) return;
if (tex == null)
{
// If received size is not null, it and scale the texture
uint i_videoHeight = 0;
uint i_videoWidth = 0;
_mediaPlayer.Size(0, ref i_videoWidth, ref i_videoHeight);
var texptr = _mediaPlayer.GetTexture(out bool updated);
if (i_videoWidth != 0 && i_videoHeight != 0 && updated && texptr != IntPtr.Zero)
{
Debug.Log("Creating texture with height " + i_videoHeight + " and width " + i_videoWidth);
tex = Texture2D.CreateExternalTexture((int)i_videoWidth,
(int)i_videoHeight,
TextureFormat.RGB24,
false,
true,
texptr);
// RenderTextureの作成
renderTexture = new RenderTexture(tex.width, tex.height, 24, RenderTextureFormat.ARGB32);
// CPUアクセス可能なTexture2Dの作成
cpuAccessibleTexture = new Texture2D(tex.width, tex.height, TextureFormat.RGB24, false);
Debug.Log("cpuAccessibleTexture format1: " + cpuAccessibleTexture.format);
GetComponent<Renderer>().material.mainTexture = tex;
}
}
else if (tex != null)
{
var texptr = _mediaPlayer.GetTexture(out bool updated);
if (updated)
{
tex.UpdateExternalTexture(texptr);
// Blit the pixels on texture to the RenderTexture
Graphics.Blit(tex, renderTexture);
Debug.Log("cpuAccessibleTexture format2: " + cpuAccessibleTexture.format);
// Set the current RenderTexture to be active
RenderTexture.active = renderTexture;
// Read the active RenderTexture into the existing Texture2D
cpuAccessibleTexture.ReadPixels(new UnityEngine.Rect(0, 0, tex.width, tex.height), 0, 0);
cpuAccessibleTexture.Apply();
Debug.Log("cpuAccessibleTexture format3: " + cpuAccessibleTexture.format);
// Texture2Dからピクセルデータを取得します
Color32[] pixels = cpuAccessibleTexture.GetPixels32();
// Color32配列をbyte配列に変換します
GCHandle pinPixels = GCHandle.Alloc(pixels, GCHandleType.Pinned);
IntPtr ptrPixels = pinPixels.AddrOfPinnedObject();
// byte配列からMatを作成します
Mat rgbaMat = new Mat(tex.height, tex.width, CvType.CV_8UC3, ptrPixels);
Debug.Log("Success Mat conversion");
// GCHandleを解放します
pinPixels.Free();
// Clean up RenderTexture
RenderTexture.active = null;
if (renderTexture != null)
{
renderTexture.Release();
}
// Create a new Mat object
//Mat rgbaMat = new Mat(tex.height, tex.width, CvType.CV_8UC4);
// Convert the Texture2D to Mat ここでクラッシュ
//OpenCVForUnity.UnityUtils.Utils.texture2DToMat(tex, rgbaMat, true);
// ... (Your OpenCV code here)
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat);
Debug.Log("faceLandmarkDetector");
//detect face rects
List<UnityEngine.Rect> detectResult = faceLandmarkDetector.Detect();
foreach (var rect in detectResult)
{
//detect landmark points
List<Vector2> points = faceLandmarkDetector.DetectLandmark(rect);
//draw landmark points
OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, points, new Scalar(0, 255, 0), 2);
//draw face rect
OpenCVForUnityUtils.DrawFaceRect(rgbaMat, rect, new Scalar(255, 0, 0), 2);
}
// Convert the processed Mat back to Texture2D
OpenCVForUnity.UnityUtils.Utils.matToTexture2D(rgbaMat, cpuAccessibleTexture);
Debug.Log("Success matToTexture2D(rgbaMat, cpuAccessibleTexture);");
//OpenCVForUnity.UnityUtils.Utils.matToTexture2D(rgbaMat, tex);
Debug.Log("tex format: " + tex.format);
Debug.Log("cpuAccessibleTexture format4: " + cpuAccessibleTexture.format);
Graphics.CopyTexture(cpuAccessibleTexture, tex);
RenderTexture.active = null;
}
}
}
}
I would like to avoid data exchange with the GPU if possible, as it tends to degrade performance. However, when I attempt to retrieve a texture from an external video source (such as RTSP streaming) using the code provided below, I am unable to access the texture data. It seems clear that Texture2D.CreateExternalTexture is the appropriate method for retrieving a texture from an external video source.
void Update()
{
if(!playing) return;
if (tex == null)
{
// If received size is not null, it and scale the texture
uint i_videoHeight = 0;
uint i_videoWidth = 0;
_mediaPlayer.Size(0, ref i_videoWidth, ref i_videoHeight);
var texptr = _mediaPlayer.GetTexture(out bool updated);
if (i_videoWidth != 0 && i_videoHeight != 0 && updated && texptr != IntPtr.Zero)
{
Debug.Log("Creating texture with height " + i_videoHeight + " and width " + i_videoWidth);
tex = new Texture2D((int)i_videoWidth, (int)i_videoHeight, TextureFormat.RGB24, false);
GetComponent<Renderer>().material.mainTexture = tex;
}
}
else if (tex != null)
{
var texptr = _mediaPlayer.GetTexture(out bool updated);
if (updated)
{
//tex.UpdateExternalTexture(texptr);
tex.UpdateExternalTexture(texptr);
// Create a new Mat object
Mat rgbaMat = new Mat(tex.height, tex.width, CvType.CV_8UC4);
// Convert the Texture2D to Mat ここでクラッシュ
OpenCVForUnity.UnityUtils.Utils.texture2DToMat(tex, rgbaMat, true);
OpenCVForUnityUtils.SetImage(faceLandmarkDetector, rgbaMat);
Debug.Log("faceLandmarkDetector");
//detect face rects
List<UnityEngine.Rect> detectResult = faceLandmarkDetector.Detect();
foreach (var rect in detectResult)
{
//detect landmark points
List<Vector2> points = faceLandmarkDetector.DetectLandmark(rect);
//draw landmark points
OpenCVForUnityUtils.DrawFaceLandmark(rgbaMat, points, new Scalar(0, 255, 0), 2);
//draw face rect
OpenCVForUnityUtils.DrawFaceRect(rgbaMat, rect, new Scalar(255, 0, 0), 2);
}
// Convert the processed Mat back to Texture2D
OpenCVForUnity.UnityUtils.Utils.matToTexture2D(rgbaMat, tex);
Debug.Log("Success matToTexture2D(rgbaMat, cpuAccessibleTexture);");
}
}
}Documentation
Ultimately, I want to use Texture2D.CreateExternalTexture to convert GPU data to a Mat object. However, I can't retrieve RTSP video using a CPU-accessible new Texture2D, and I can't access the data using Texture2D.CreateExternalTexture. I'm wondering if it would be possible to achieve this using a programming language like CUDA.
Additionally, I noticed that ffmpeg provides a helper class for retrieving and converting a texture from an external RTSP source to a Mat object. However, this method introduces significant latency, so I decided to use VLC instead. I haven't been able to find a similar helper class for VLC. Is there a better way to accomplish this?
Version
- Windows 11
- Unity 2022.3.1f1
- OpenCV for Unity 2.5.4
- vlc-unity-windows-trial