UE5开发HoloLens遇到的坑和应对方法

第三方库

源码编译

对于第三方库最好是使用源码和UE工程代码一起编译，出问题的几率最小，但是需要把编译脚本自己改成UnrealBuildTools的C#，需要视工程大小来确定

vcpkg安装

HoloLens的运行环境是arm64-uwp，如果不想对于每个第三方库去编译，可以直接安装vcpkg包管理工具，安装一个库的时候分别安装x64-windows和arm64-uwp。如果需要编译为静态库可以到使用社区的编译方案x64-windows-static-md和arm64-uwp-static-md

具体使用MD还是MT可以保持和UE5的UnrealBuildTools一致的编译参数，可以查看源码[ROOT]\Epic_Games\UE_5.0\Engine\Source\Programs\UnrealBuildTool\Platform\。vcpkg中已经有的配置如果不满足要求可以在triplets中自己创建编译配置

静态库

对于一些普通的第三方库（没有涉及到Windows API）可以直接编译为arm64架构库，作为静态库链接

对于使用了系统API的库，由于Win32应用和UWP应用的权限设置和接口不一样，为了兼容Windows调试和HoloLens实机使用，尽可能的都使用WinRT接口

动态库

动态库除了需要注意编译问题，加载也需要注意。打包程序的时候可以把动态库和可执行程序放在同一目录，也可以使用延迟加载的方式把动态库打包到plugin里面加载，编译脚本写法不一样，同时延迟加载方案需动态库要在插件的模块初始化时候加载

以下是延迟加载FFmpeg的例子

if (Target.Platform == UnrealTargetPlatform.Win64)
{
  isLibrarySupported = true;
  prefixDir = "x64";
  string[] libs = { "avcodec.lib", "avdevice.lib", "avfilter.lib", "avformat.lib", "avutil.lib", "postproc.lib", "swresample.lib", "swscale.lib" };
  foreach (string lib in libs)
  {
    PublicAdditionalLibraries.Add(Path.Combine(ModuleDirectory, prefixDir, "lib", lib));
  }
  string[] dlls = { "avcodec-59.dll", "avdevice-59.dll", "avfilter-8.dll", "avformat-59.dll", "avutil-57.dll", "swresample-4.dll", "swscale-6.dll", "postproc-56.dll" };
  foreach (string dll in dlls)
  {
    PublicDelayLoadDLLs.Add(dll);
    RuntimeDependencies.Add("$(PluginDir)/Binaries/ThirdParty/FFmpegMediaLibrary/Win64/" + dll);
  }

}
else if (Target.Platform == UnrealTargetPlatform.HoloLens)
{
  isLibrarySupported = true;
  prefixDir = "arm64";
  string[] libs = { "avcodec.lib", "avdevice.lib", "avfilter.lib", "avformat.lib", "avutil.lib", "postproc.lib", "swresample.lib", "swscale.lib" };
  foreach (string lib in libs)
  {
    PublicAdditionalLibraries.Add(Path.Combine(ModuleDirectory, prefixDir, "lib", lib));
  }
  string[] dlls = { "avcodec-59.dll", "avdevice-59.dll", "avfilter-8.dll", "avformat-59.dll", "avutil-57.dll", "swresample-4.dll", "swscale-6.dll", "postproc-56.dll" };
  foreach (string dll in dlls)
  {
    PublicDelayLoadDLLs.Add(dll);
    RuntimeDependencies.Add("$(PluginDir)/Binaries/ThirdParty/FFmpegMediaLibrary/HoloLens/" + dll);
  }
}

virtual void StartupModule() override
{
    FString BaseDir = IPluginManager::Get().FindPlugin("FFmpegMedia")->GetBaseDir();
#if PLATFORM_WINDOWS || PLATFORM_HOLOLENS
#if PLATFORM_WINDOWS
  //开始d动态加载ffmpeg dll文件
  FString avcodeLibraryPath = FPaths::Combine(*BaseDir, TEXT("Binaries/ThirdParty/FFmpegMediaLibrary/Win64/avcodec-59.dll"));
  FString avdeviceLibraryPath = FPaths::Combine(*BaseDir, TEXT("Binaries/ThirdParty/FFmpegMediaLibrary/Win64/avdevice-59.dll"));
  FString avfilterLibraryPath = FPaths::Combine(*BaseDir, TEXT("Binaries/ThirdParty/FFmpegMediaLibrary/Win64/avfilter-8.dll"));
  FString avformatLibraryPath = FPaths::Combine(*BaseDir, TEXT("Binaries/ThirdParty/FFmpegMediaLibrary/Win64/avformat-59.dll"));
  FString avutilLibraryPath = FPaths::Combine(*BaseDir, TEXT("Binaries/ThirdParty/FFmpegMediaLibrary/Win64/avutil-57.dll"));
  FString postprocLibraryPath = FPaths::Combine(*BaseDir, TEXT("Binaries/ThirdParty/FFmpegMediaLibrary/Win64/postproc-56.dll"));
  FString swresampleLibraryPath = FPaths::Combine(*BaseDir, TEXT("Binaries/ThirdParty/FFmpegMediaLibrary/Win64/swresample-4.dll"));
  FString swscaleLibraryPath = FPaths::Combine(*BaseDir, TEXT("Binaries/ThirdParty/FFmpegMediaLibrary/Win64/swscale-6.dll"));
#else
  FString avcodeLibraryPath = FPaths::Combine(*BaseDir, TEXT("Binaries/ThirdParty/FFmpegMediaLibrary/HoloLens/avcodec-59.dll"));
  FString avdeviceLibraryPath = FPaths::Combine(*BaseDir, TEXT("Binaries/ThirdParty/FFmpegMediaLibrary/HoloLens/avdevice-59.dll"));
  FString avfilterLibraryPath = FPaths::Combine(*BaseDir, TEXT("Binaries/ThirdParty/FFmpegMediaLibrary/HoloLens/avfilter-8.dll"));
  FString avformatLibraryPath = FPaths::Combine(*BaseDir, TEXT("Binaries/ThirdParty/FFmpegMediaLibrary/HoloLens/avformat-59.dll"));
  FString avutilLibraryPath = FPaths::Combine(*BaseDir, TEXT("Binaries/ThirdParty/FFmpegMediaLibrary/HoloLens/avutil-57.dll"));
  FString postprocLibraryPath = FPaths::Combine(*BaseDir, TEXT("Binaries/ThirdParty/FFmpegMediaLibrary/HoloLens/postproc-56.dll"));
  FString swresampleLibraryPath = FPaths::Combine(*BaseDir, TEXT("Binaries/ThirdParty/FFmpegMediaLibrary/HoloLens/swresample-4.dll"));
  FString swscaleLibraryPath = FPaths::Combine(*BaseDir, TEXT("Binaries/ThirdParty/FFmpegMediaLibrary/HoloLens/swscale-6.dll"));
#endif
  AVUtilLibrary = !avutilLibraryPath.IsEmpty() ? FPlatformProcess::GetDllHandle(*avutilLibraryPath) : nullptr;
  SWResampleLibrary = !swresampleLibraryPath.IsEmpty() ? FPlatformProcess::GetDllHandle(*swresampleLibraryPath) : nullptr;
  PostProcLibrary = !postprocLibraryPath.IsEmpty() ? FPlatformProcess::GetDllHandle(*postprocLibraryPath) : nullptr;
  SWScaleLibrary = !swscaleLibraryPath.IsEmpty() ? FPlatformProcess::GetDllHandle(*swscaleLibraryPath) : nullptr;
  AVCodecLibrary = !avcodeLibraryPath.IsEmpty() ? FPlatformProcess::GetDllHandle(*avcodeLibraryPath) : nullptr;
  AVFormatLibrary = !avformatLibraryPath.IsEmpty() ? FPlatformProcess::GetDllHandle(*avformatLibraryPath) : nullptr;
  AVFilterLibrary = !avfilterLibraryPath.IsEmpty() ? FPlatformProcess::GetDllHandle(*avfilterLibraryPath) : nullptr;
  AVDeviceLibrary = !avdeviceLibraryPath.IsEmpty() ? FPlatformProcess::GetDllHandle(*avdeviceLibraryPath) : nullptr;

  if (!(AVUtilLibrary &&
      SWResampleLibrary &&
      PostProcLibrary &&
      SWScaleLibrary &&
      AVCodecLibrary &&
      AVFormatLibrary &&
      AVFilterLibrary &&
      AVDeviceLibrary)) {
    FMessageDialog::Open(EAppMsgType::Ok, LOCTEXT("LoadLibraryError", "Failed to load ffmpeg library"));
  }
#endif
  //av_register_all(); //ffmpeg注册组件，ffmpeg5中已经不存在
  avformat_network_init(); //初始化ffmpeg网络库

  UE_LOG(LogFFmpegMedia, Display, TEXT("FFmpeg AVCodec version: %d.%d.%d"), LIBAVFORMAT_VERSION_MAJOR, LIBAVFORMAT_VERSION_MINOR, LIBAVFORMAT_VERSION_MICRO);
  UE_LOG(LogFFmpegMedia, Display, TEXT("FFmpeg license: %s"), UTF8_TO_TCHAR(avformat_license()));
  av_log_set_level(AV_LOG_INFO);
  av_log_set_flags(AV_LOG_SKIP_REPEATED);
  av_log_set_callback(&log_callback);
  Initialized = true;
}

TTS接口

不要使用UE5中的TextToSpeech插件，测试不能使用，可以直接调用WinRT

FTTSSpeaker::FTTSSpeaker()
{
    auto voices = Synthesizer.AllVoices();
    // std::wstringstream ws;
    // auto v = Synthesizer.DefaultVoice();
  // 设置声音源
    for (const auto &v : voices) {
        if (v.DisplayName() == L"Microsoft Yaoyao") {
            UE_LOG(LogTemp, Log, TEXT("Set voice"));
            Synthesizer.Voice(v);
        }
        // ws.clear();
        // ws << L"|name:" << v.DisplayName().data()
        // << L"|gender:" << (int)v.Gender()
        // << L"|lang:" << v.Language().data()
        // << L"|lang:" << v.Id().data()
        // << L"|desc:" << v.Description().data()
        // << std::endl;
        // auto str = ws.str();
        // UE_LOG(LogTemp, Log, TEXT("%s"), *FString(str.c_str()));
    }
}

// 播放声音，如果之前一条没播放完毕会被打断并播放
void FTTSSpeaker::SpeakText(const std::wstring &str) const
{
    if (str.empty()) {
        return;
    }
    SpeechSynthesisStream speechStream = Synthesizer.SynthesizeTextToStreamAsync(str).get();
    Player.SetStreamSource(speechStream);
    Player.Play();
}

调试接口

使用ETW日志，同样是WinRT接口


std::shared_ptr<LoggingChannel> GChannel;

// GUID和UE5当前项目设置中的GUID一样就行
struct FChannelGuard
{
    FChannelGuard() {
        GChannel = std::make_shared<LoggingChannel>(L"HNHS_ROBOT", nullptr, winrt::guid{
                                                0x2725854Au, 0x490Du, 0x2E18u, {
                                                    0x67, 0x8C, 0xD2, 0xBE, 0xF9, 0xA0, 0xB4, 0xE8
                                                }
                                            });
    }
};

// 确保在main函数前调用
FChannelGuard GChannelGuard;

void FETWLoggerModule::StartupModule()
{
  // HoloLens2 网页管理处启用监听后调用
    GChannel->LoggingEnabled([](auto &channel, const auto &handler)
    {
        GChannel->LogEvent(TEXT("Enable HNHS_ROBOT Logging"));
    });
    GChannel->LogEvent(TEXT("Start HNHS_ROBOT App"));
}

void UETWLogger::WriteLog(const FString &Str, ELoggingLevel Level)
{
    GChannel->LogEvent(*Str, LoggingFields{}, static_cast<LoggingLevel>(Level));
    // UE_LOG(LogTemp, Log, TEXT("%s"), *Str)
}

然后在HoloLens2后台管理界面的ETW Logging中启用监听

视频流播放

VLC

libVLC不能在arm64-uwp下编译成功

FFmpeg

https://github.com/linkinlin/FFmpegMedia：可以在HoloLens2上播放本地视频，需要把视频一起打包。对于RTSP，测试只在PC能够播放
https://github.com/xiaoqi3278/FFmpegExtension：使用图片控件进行播放，可以播放本地视频和流

WinRT接口

头文件隔离

WinRT接口的头文件如果直接包含会和UE5的头文件冲突，需要用下面宏包起来

#if (PLATFORM_WINDOWS || PLATFORM_HOLOLENS)
//Before writing any code, you need to disable common warnings in WinRT headers
#pragma warning(disable : 5205 4265 4268 4946)

#include "Windows/AllowWindowsPlatformTypes.h"
#include "Windows/AllowWindowsPlatformAtomics.h"
#include "Windows/PreWindowsApi.h"

// 添加你要包含的头文件
#include <winrt/Windows.Foundation.h>
#include <winrt/Windows.Perception.Spatial.h>
#include <winrt/Windows.Foundation.Collections.h>

#include "Windows/PostWindowsApi.h"
#include "Windows/HideWindowsPlatformAtomics.h"
#include "Windows/HideWindowsPlatformTypes.h"
#endif

模型优化

使用了装配模型进行转化，需要对细节进行处理。采取的办法有几种：

凸壳。凸壳可以去掉集合体内部的网格，极大的减少面数。
融并顶点。把孔洞的顶点融并起来。
使用精度较低的模型，把精度高的模型细节烘焙成图片贴图。

模型消失

在镜头移动的时候，模型会突然的消失。是由于静态网格或者骨骼网格没有设置好或没有PhysicsAsset。可以在UE5编辑器中自己创建新的并进行调整

通信

gRPC

在这上面花费了很多时间，可以使用vcpkg来编译arm64-uwp架构库来使用。

rpclib

官方的rpclib库太久没更新，arm下编译会报错。需要更新高版本的boost.predef库。替换后并修改宏，代码地址：https://github.com/helywin/rpclib

参考学习

文章

UE5调用WinRT接口: https://learn.microsoft.com/en-us/windows/mixed-reality/develop/unreal/unreal-winrt?tabs=426

第三方库#

源码编译#

vcpkg安装#

静态库#

动态库#

TTS接口#

调试接口#

视频流播放#

VLC#

FFmpeg#

WinRT接口#

头文件隔离#

模型优化#

模型消失#

通信#

gRPC#

rpclib#

参考学习#

文章#

教程#