第三方库
源码编译
对于第三方库最好是使用源码和UE工程代码一起编译，出问题的几率最小，但是需要把编译脚本自己改成UnrealBuildTools的C#，需要视工程大小来确定
vcpkg安装
HoloLens的运行环境是arm64-uwp，如果不想对于每个第三方库去编译，可以直接安装vcpkg包管理工具，安装一个库的时候分别安装x64-windows和arm64-uwp。如果需要编译为静态库可以到使用社区的编译方案x64-windows-static-md和arm64-uwp-static-md
具体使用MD还是MT可以保持和UE5的UnrealBuildTools一致的编译参数，可以查看源码[ROOT]\Epic_Games\UE_5.0\Engine\Source\Programs\UnrealBuildTool\Platform\。vcpkg中已经有的配置如果不满足要求可以在triplets中自己创建编译配置
静态库
对于一些普通的第三方库（没有涉及到Windows API）可以直接编译为arm64架构库，作为静态库链接
对于使用了系统API的库，由于Win32应用和UWP应用的权限设置和接口不一样，为了兼容Windows调试和HoloLens实机使用，尽可能的都使用WinRT接口
动态库
动态库除了需要注意编译问题，加载也需要注意。打包程序的时候可以把动态库和可执行程序放在同一目录，也可以使用延迟加载的方式把动态库打包到plugin里面加载，编译脚本写法不一样，同时延迟加载方案需动态库要在插件的模块初始化时候加载
以下是延迟加载FFmpeg的例子
1if (Target.Platform == UnrealTargetPlatform.Win64)
2{
3 isLibrarySupported = true;
4 prefixDir = "x64";
5 string[] libs = { "avcodec.lib", "avdevice.lib", "avfilter.lib", "avformat.lib", "avutil.lib", "postproc.lib", "swresample.lib", "swscale.lib" };
6 foreach (string lib in libs)
7 {
8 PublicAdditionalLibraries.Add(Path.Combine(ModuleDirectory, prefixDir, "lib", lib));
9 }
10 string[] dlls = { "avcodec-59.dll", "avdevice-59.dll", "avfilter-8.dll", "avformat-59.dll", "avutil-57.dll", "swresample-4.dll", "swscale-6.dll", "postproc-56.dll" };
11 foreach (string dll in dlls)
12 {
13 PublicDelayLoadDLLs.Add(dll);
14 RuntimeDependencies.Add("$(PluginDir)/Binaries/ThirdParty/FFmpegMediaLibrary/Win64/" + dll);
15 }
16
17}
18else if (Target.Platform == UnrealTargetPlatform.HoloLens)
19{
20 isLibrarySupported = true;
21 prefixDir = "arm64";
22 string[] libs = { "avcodec.lib", "avdevice.lib", "avfilter.lib", "avformat.lib", "avutil.lib", "postproc.lib", "swresample.lib", "swscale.lib" };
23 foreach (string lib in libs)
24 {
25 PublicAdditionalLibraries.Add(Path.Combine(ModuleDirectory, prefixDir, "lib", lib));
26 }
27 string[] dlls = { "avcodec-59.dll", "avdevice-59.dll", "avfilter-8.dll", "avformat-59.dll", "avutil-57.dll", "swresample-4.dll", "swscale-6.dll", "postproc-56.dll" };
28 foreach (string dll in dlls)
29 {
30 PublicDelayLoadDLLs.Add(dll);
31 RuntimeDependencies.Add("$(PluginDir)/Binaries/ThirdParty/FFmpegMediaLibrary/HoloLens/" + dll);
32 }
33} 1virtual void StartupModule() override
2{
3 FString BaseDir = IPluginManager::Get().FindPlugin("FFmpegMedia")->GetBaseDir();
4#if PLATFORM_WINDOWS || PLATFORM_HOLOLENS
5#if PLATFORM_WINDOWS
6 //开始d动态加载ffmpeg dll文件
7 FString avcodeLibraryPath = FPaths::Combine(*BaseDir, TEXT("Binaries/ThirdParty/FFmpegMediaLibrary/Win64/avcodec-59.dll"));
8 FString avdeviceLibraryPath = FPaths::Combine(*BaseDir, TEXT("Binaries/ThirdParty/FFmpegMediaLibrary/Win64/avdevice-59.dll"));
9 FString avfilterLibraryPath = FPaths::Combine(*BaseDir, TEXT("Binaries/ThirdParty/FFmpegMediaLibrary/Win64/avfilter-8.dll"));
10 FString avformatLibraryPath = FPaths::Combine(*BaseDir, TEXT("Binaries/ThirdParty/FFmpegMediaLibrary/Win64/avformat-59.dll"));
11 FString avutilLibraryPath = FPaths::Combine(*BaseDir, TEXT("Binaries/ThirdParty/FFmpegMediaLibrary/Win64/avutil-57.dll"));
12 FString postprocLibraryPath = FPaths::Combine(*BaseDir, TEXT("Binaries/ThirdParty/FFmpegMediaLibrary/Win64/postproc-56.dll"));
13 FString swresampleLibraryPath = FPaths::Combine(*BaseDir, TEXT("Binaries/ThirdParty/FFmpegMediaLibrary/Win64/swresample-4.dll"));
14 FString swscaleLibraryPath = FPaths::Combine(*BaseDir, TEXT("Binaries/ThirdParty/FFmpegMediaLibrary/Win64/swscale-6.dll"));
15#else
16 FString avcodeLibraryPath = FPaths::Combine(*BaseDir, TEXT("Binaries/ThirdParty/FFmpegMediaLibrary/HoloLens/avcodec-59.dll"));
17 FString avdeviceLibraryPath = FPaths::Combine(*BaseDir, TEXT("Binaries/ThirdParty/FFmpegMediaLibrary/HoloLens/avdevice-59.dll"));
18 FString avfilterLibraryPath = FPaths::Combine(*BaseDir, TEXT("Binaries/ThirdParty/FFmpegMediaLibrary/HoloLens/avfilter-8.dll"));
19 FString avformatLibraryPath = FPaths::Combine(*BaseDir, TEXT("Binaries/ThirdParty/FFmpegMediaLibrary/HoloLens/avformat-59.dll"));
20 FString avutilLibraryPath = FPaths::Combine(*BaseDir, TEXT("Binaries/ThirdParty/FFmpegMediaLibrary/HoloLens/avutil-57.dll"));
21 FString postprocLibraryPath = FPaths::Combine(*BaseDir, TEXT("Binaries/ThirdParty/FFmpegMediaLibrary/HoloLens/postproc-56.dll"));
22 FString swresampleLibraryPath = FPaths::Combine(*BaseDir, TEXT("Binaries/ThirdParty/FFmpegMediaLibrary/HoloLens/swresample-4.dll"));
23 FString swscaleLibraryPath = FPaths::Combine(*BaseDir, TEXT("Binaries/ThirdParty/FFmpegMediaLibrary/HoloLens/swscale-6.dll"));
24#endif
25 AVUtilLibrary = !avutilLibraryPath.IsEmpty() ? FPlatformProcess::GetDllHandle(*avutilLibraryPath) : nullptr;
26 SWResampleLibrary = !swresampleLibraryPath.IsEmpty() ? FPlatformProcess::GetDllHandle(*swresampleLibraryPath) : nullptr;
27 PostProcLibrary = !postprocLibraryPath.IsEmpty() ? FPlatformProcess::GetDllHandle(*postprocLibraryPath) : nullptr;
28 SWScaleLibrary = !swscaleLibraryPath.IsEmpty() ? FPlatformProcess::GetDllHandle(*swscaleLibraryPath) : nullptr;
29 AVCodecLibrary = !avcodeLibraryPath.IsEmpty() ? FPlatformProcess::GetDllHandle(*avcodeLibraryPath) : nullptr;
30 AVFormatLibrary = !avformatLibraryPath.IsEmpty() ? FPlatformProcess::GetDllHandle(*avformatLibraryPath) : nullptr;
31 AVFilterLibrary = !avfilterLibraryPath.IsEmpty() ? FPlatformProcess::GetDllHandle(*avfilterLibraryPath) : nullptr;
32 AVDeviceLibrary = !avdeviceLibraryPath.IsEmpty() ? FPlatformProcess::GetDllHandle(*avdeviceLibraryPath) : nullptr;
33
34 if (!(AVUtilLibrary &&
35 SWResampleLibrary &&
36 PostProcLibrary &&
37 SWScaleLibrary &&
38 AVCodecLibrary &&
39 AVFormatLibrary &&
40 AVFilterLibrary &&
41 AVDeviceLibrary)) {
42 FMessageDialog::Open(EAppMsgType::Ok, LOCTEXT("LoadLibraryError", "Failed to load ffmpeg library"));
43 }
44#endif
45 //av_register_all(); //ffmpeg注册组件，ffmpeg5中已经不存在
46 avformat_network_init(); //初始化ffmpeg网络库
47
48 UE_LOG(LogFFmpegMedia, Display, TEXT("FFmpeg AVCodec version: %d.%d.%d"), LIBAVFORMAT_VERSION_MAJOR, LIBAVFORMAT_VERSION_MINOR, LIBAVFORMAT_VERSION_MICRO);
49 UE_LOG(LogFFmpegMedia, Display, TEXT("FFmpeg license: %s"), UTF8_TO_TCHAR(avformat_license()));
50 av_log_set_level(AV_LOG_INFO);
51 av_log_set_flags(AV_LOG_SKIP_REPEATED);
52 av_log_set_callback(&log_callback);
53 Initialized = true;
54}TTS接口
不要使用UE5中的TextToSpeech插件，测试不能使用，可以直接调用WinRT
1FTTSSpeaker::FTTSSpeaker()
2{
3 auto voices = Synthesizer.AllVoices();
4 // std::wstringstream ws;
5 // auto v = Synthesizer.DefaultVoice();
6 // 设置声音源
7 for (const auto &v : voices) {
8 if (v.DisplayName() == L"Microsoft Yaoyao") {
9 UE_LOG(LogTemp, Log, TEXT("Set voice"));
10 Synthesizer.Voice(v);
11 }
12 // ws.clear();
13 // ws << L"|name:" << v.DisplayName().data()
14 // << L"|gender:" << (int)v.Gender()
15 // << L"|lang:" << v.Language().data()
16 // << L"|lang:" << v.Id().data()
17 // << L"|desc:" << v.Description().data()
18 // << std::endl;
19 // auto str = ws.str();
20 // UE_LOG(LogTemp, Log, TEXT("%s"), *FString(str.c_str()));
21 }
22}
23
24// 播放声音，如果之前一条没播放完毕会被打断并播放
25void FTTSSpeaker::SpeakText(const std::wstring &str) const
26{
27 if (str.empty()) {
28 return;
29 }
30 SpeechSynthesisStream speechStream = Synthesizer.SynthesizeTextToStreamAsync(str).get();
31 Player.SetStreamSource(speechStream);
32 Player.Play();
33}调试接口
使用ETW日志，同样是WinRT接口
1
2std::shared_ptr<LoggingChannel> GChannel;
3
4// GUID和UE5当前项目设置中的GUID一样就行
5struct FChannelGuard
6{
7 FChannelGuard() {
8 GChannel = std::make_shared<LoggingChannel>(L"HNHS_ROBOT", nullptr, winrt::guid{
9 0x2725854Au, 0x490Du, 0x2E18u, {
10 0x67, 0x8C, 0xD2, 0xBE, 0xF9, 0xA0, 0xB4, 0xE8
11 }
12 });
13 }
14};
15
16// 确保在main函数前调用
17FChannelGuard GChannelGuard;
18
19void FETWLoggerModule::StartupModule()
20{
21 // HoloLens2 网页管理处启用监听后调用
22 GChannel->LoggingEnabled([](auto &channel, const auto &handler)
23 {
24 GChannel->LogEvent(TEXT("Enable HNHS_ROBOT Logging"));
25 });
26 GChannel->LogEvent(TEXT("Start HNHS_ROBOT App"));
27}
28
29void UETWLogger::WriteLog(const FString &Str, ELoggingLevel Level)
30{
31 GChannel->LogEvent(*Str, LoggingFields{}, static_cast<LoggingLevel>(Level));
32 // UE_LOG(LogTemp, Log, TEXT("%s"), *Str)
33}然后在HoloLens2后台管理界面的ETW Logging中启用监听
视频流播放
VLC
libVLC不能在arm64-uwp下编译成功
FFmpeg
- https://github.com/linkinlin/FFmpegMedia：可以在HoloLens2上播放本地视频，需要把视频一起打包。对于RTSP，测试只在PC能够播放
- https://github.com/xiaoqi3278/FFmpegExtension：使用图片控件进行播放，可以播放本地视频和流
WinRT接口
头文件隔离
WinRT接口的头文件如果直接包含会和UE5的头文件冲突，需要用下面宏包起来
1#if (PLATFORM_WINDOWS || PLATFORM_HOLOLENS)
2//Before writing any code, you need to disable common warnings in WinRT headers
3#pragma warning(disable : 5205 4265 4268 4946)
4
5#include "Windows/AllowWindowsPlatformTypes.h"
6#include "Windows/AllowWindowsPlatformAtomics.h"
7#include "Windows/PreWindowsApi.h"
8
9// 添加你要包含的头文件
10#include <winrt/Windows.Foundation.h>
11#include <winrt/Windows.Perception.Spatial.h>
12#include <winrt/Windows.Foundation.Collections.h>
13
14#include "Windows/PostWindowsApi.h"
15#include "Windows/HideWindowsPlatformAtomics.h"
16#include "Windows/HideWindowsPlatformTypes.h"
17#endif模型优化
使用了装配模型进行转化，需要对细节进行处理。采取的办法有几种：
- 凸壳。凸壳可以去掉集合体内部的网格，极大的减少面数。
- 融并顶点。把孔洞的顶点融并起来。
- 使用精度较低的模型，把精度高的模型细节烘焙成图片贴图。
模型消失
在镜头移动的时候，模型会突然的消失。是由于静态网格或者骨骼网格没有设置好或没有PhysicsAsset。可以在UE5编辑器中自己创建新的并进行调整
通信
gRPC
在这上面花费了很多时间，可以使用vcpkg来编译arm64-uwp架构库来使用。
rpclib
官方的rpclib库太久没更新，arm下编译会报错。需要更新高版本的boost.predef库。替换后并修改宏，代码地址：https://github.com/helywin/rpclib
参考学习
文章
- UE5调用WinRT接口: https://learn.microsoft.com/en-us/windows/mixed-reality/develop/unreal/unreal-winrt?tabs=426
教程
- 视频教程: https://dev.epicgames.com/community/learning/courses/M97/hololens-2-mixed-reality-production-for-unreal-engine/orLB/introduction-to-hololens-2-mixed-reality-production-for-unreal-engine
- 文字教程: https://learn.microsoft.com/zh-cn/windows/mixed-reality/develop/unreal/unreal-development-overview?tabs=ue427%2Cmrtk%2Casa%2CD365