Ronald Burns

EOS Voice Chat System

The following is a voice chat system used to convert the incoming voice stream into usable Sound Components. It parses the byte array and converts it into a stream usable by the USoundWaveProcedural component. These components are attached to "RemoteTalkers", which represent every player.

This system is designed to allow designers the ability to modify the sound using Unreal's MetaSounds, as well as position the sound in 3D space.

HolodeckVoiceChatSubsystem.h

// Copyright Hitbox Games, LLC. All Rights Reserved.
 
#pragma once
 
#include "VoiceChat.h"
#include "Components/PlayerStateComponent.h"
 
#include "HolodeckVoiceChatSubsystem.generated.h"
 
enum class ECommonUserOnlineContext : uint8;
enum class ECommonUserPrivilege : uint8;
class FEOSVoiceChatUser;
class IOnlineSubsystem;
class IOnlineSubsystemEOS;
class UCommonUserInfo;
class UCommonUserSubsystem;
 
DECLARE_DYNAMIC_MULTICAST_DELEGATE_SixParams(FOnVoiceChatBeforeRecvAudioRendered_Multicast, TArray<uint8>, PcmSamples, int, SampleRate, int, Channels, bool, bIsSilence, const FString&, ChannelName, const FString&, PlayerName);
 
UCLASS()
class HOLODECKGAME_API URemoteTalkerData : public UObject
{
	GENERATED_BODY()
 
public:
	URemoteTalkerData();
	virtual ~URemoteTalkerData();
 
	void Initialize(APlayerState* PlayerState, uint32 InSampleRate, int32 InNumChannels);
 
protected:
	UAudioComponent* CreateVoiceAudioComponent(uint32 InSampleRate, int32 InNumChannels);
 
	/** Maximum size of a single decoded packet */
	int32 MaxUncompressedDataSize;
	/** Maximum size of the outgoing playback queue */
	int32 MaxUncompressedDataQueueSize;
	/** Amount of data currently in the outgoing playback queue */
	int32 CurrentUncompressedDataQueueSize;
 
	/** Buffer for uncompressed audio data (valid during Tick only) */
	UPROPERTY()
	TArray<uint8> UncompressedData;
	/** Buffer for outgoing audio intended for procedural streaming */
	mutable FCriticalSection QueueLock;
	UPROPERTY()
	TArray<uint8> UncompressedDataQueue;
 
	UPROPERTY()
	TWeakObjectPtr<APlayerState> PlayerStatePtr;
 
	UPROPERTY()
	TObjectPtr<UAudioComponent> AudioComponent;
 
	double LastSeen;
 
	class UHolodeckVoiceChatSubsystem* GetSubsystem() const;
 
private:
	friend class UHolodeckVoiceChatSubsystem;
};
 
UCLASS(Blueprintable, BlueprintType, Within="PlayerState")
class UHolodeckVoiceTalkerComponent : public UPlayerStateComponent
{
	GENERATED_BODY()
 
public:
	UHolodeckVoiceTalkerComponent(const FObjectInitializer& ObjectInitializer = FObjectInitializer());
 
	virtual void BeginPlay() override;
	virtual void TickComponent(float DeltaTime, ELevelTick TickType, FActorComponentTickFunction* ThisTickFunction) override;
 
	UFUNCTION()
	void OnExperienceLoaded(const UHolodeckExperienceDefinition* Experience);
 
	UFUNCTION()
	void HandleVoiceChatBeforeRecvAudioRendered(TArray<uint8> PcmSamples, int SampleRate, int Channels, bool bIsSilence, const FString& ChannelName, const FString& PlayerName);
 
	UFUNCTION(BlueprintCallable)
	virtual UAudioComponent* GetAudioComponent() const;
 
protected:
	UPROPERTY()
	TObjectPtr<UAudioComponent> AudioComponent;
};
 
UCLASS()
class HOLODECKGAME_API UHolodeckVoiceChatSubsystem : public ULocalPlayerSubsystem
{
	GENERATED_BODY()
 
	UHolodeckVoiceChatSubsystem();
	~UHolodeckVoiceChatSubsystem();
 
	virtual void Initialize(FSubsystemCollectionBase& Collection) override;
	virtual void Deinitialize() override;
 
protected:
	UFUNCTION()
	void OnUserInitializeComplete(const UCommonUserInfo* UserInfo, bool bSuccess, FText Error, ECommonUserPrivilege RequestedPrivilege, ECommonUserOnlineContext OnlineContext);
 
	UPROPERTY(BlueprintAssignable, Transient)
	FOnVoiceChatBeforeRecvAudioRendered_Multicast OnVoiceChatBeforeRecvAudioRenderedDelegate_Multicast;
	FOnVoiceChatBeforeRecvAudioRenderedDelegate::FDelegate OnVoiceChatBeforeRecvAudioRenderedDelegate;
	void HandleVoiceChatBeforeRecvAudioRendered(TArrayView<int16> PcmSamples, int SampleRate, int Channels, bool bIsSilence, const FString& ChannelName, const FString& PlayerName);
 
	UFUNCTION()
	APlayerState* FindPlayerStateByVoiceChatPlayerName(const FString& PlayerName);
 
protected:
	/** Instance name of associated online subsystem */
	FName OnlineInstanceName;
 
	UPROPERTY()
	TWeakObjectPtr<UCommonUserSubsystem> CommonUserSubsystem;
 
	// UPROPERTY()
	// TMap<FUniqueNetIdWrapper, TObjectPtr<URemoteTalkerData>> RemoteTalkers;
 
#if COMMONUSER_OSSV1
	IOnlineSubsystemEOS* EOSOnlineSubsystem;
#endif
	FEOSVoiceChatUser* VoiceChatUser;
 
protected:
	virtual IOnlineSubsystem* GetOnlineSubSystem();
};

HolodeckVoiceChatSubsystem.cpp

// Copyright Hitbox Games, LLC. All Rights Reserved.
 
#include "HolodeckVoiceChatSubsystem.h"
 
#include "CommonUserSubsystem.h"
#include "EOSVoiceChatUser.h"
#include "HolodeckLogChannels.h"
#include "IOnlineSubsystemEOS.h"
#include "OnlineSubsystem.h"
 
#if COMMONUSER_OSSV1
#include "OnlineSubsystemNames.h"
#include "OnlineSubsystemUtils.h"
#endif
 
#include "AudioDevice.h"
#include "Experience/HolodeckExperienceManagerComponent.h"
#include "GameFramework/GameStateBase.h"
#include "GameFramework/PlayerState.h"
#include "Kismet/KismetSystemLibrary.h"
#include "Net/VoiceConfig.h"
 
#include UE_INLINE_GENERATED_CPP_BY_NAME(HolodeckVoiceChatSubsystem)
 
URemoteTalkerData::URemoteTalkerData()
	: MaxUncompressedDataSize(0),
	  CurrentUncompressedDataQueueSize(0),
	  LastSeen(0.0)
{
	// Approx 1 sec worth of data for a stereo microphone
	// NOTE - This might be half a second. I think EOSVoice samples every 0.1 seconds.
	MaxUncompressedDataSize = UVOIPStatics::GetMaxUncompressedVoiceDataSizePerChannel() * UVOIPStatics::GetVoiceNumChannels();
	MaxUncompressedDataQueueSize = MaxUncompressedDataSize * 5;
	{
		FScopeLock ScopeLock(&QueueLock);
		UncompressedDataQueue.Empty(MaxUncompressedDataQueueSize);
	}
}
 
URemoteTalkerData::~URemoteTalkerData()
{
	UncompressedData.Empty();
 
	{
		FScopeLock ScopeLock(&QueueLock);
		UncompressedDataQueue.Empty();
	}
}
 
void URemoteTalkerData::Initialize(APlayerState* PlayerState, uint32 InSampleRate, int32 InNumChannels)
{
	PlayerStatePtr = PlayerState;
 
	AudioComponent = CreateVoiceAudioComponent(InSampleRate, InNumChannels);
}
 
UAudioComponent* URemoteTalkerData::CreateVoiceAudioComponent(uint32 InSampleRate, int32 InNumChannels)
{
	UAudioComponent* NewAudioComponent = nullptr;
	if (GEngine != nullptr)
	{
		if (FAudioDeviceHandle AudioDevice = GEngine->GetMainAudioDevice())
		{
			USoundWaveProcedural* SoundStreaming = NewObject<USoundWaveProcedural>();
			SoundStreaming->SetSampleRate(InSampleRate);
			SoundStreaming->NumChannels = InNumChannels;
			SoundStreaming->Duration = INDEFINITELY_LOOPING_DURATION;
			SoundStreaming->SoundGroup = SOUNDGROUP_Voice;
			SoundStreaming->bLooping = false;
 
			SoundStreaming->bCanProcessAsync = true;
 
			FAudioDevice::FCreateComponentParams Params(GetWorld(), PlayerStatePtr.Get());
			Params.SetLocation(FVector::ZeroVector);
			Params.bStopWhenOwnerDestroyed = true;
 
			NewAudioComponent = AudioDevice->CreateComponent(SoundStreaming, Params);
			if (NewAudioComponent)
			{
				NewAudioComponent->bIsUISound = true;
				NewAudioComponent->bAllowSpatialization = true;
				// NewAudioComponent->SetVolumeMultiplier(1.5f);
 
				if (const auto Pawn = PlayerStatePtr->GetPawn())
				{
					if (const auto SceneRoot = Cast<USceneComponent>(Pawn->GetRootComponent()))
					{
						NewAudioComponent->AttachToComponent(SceneRoot, FAttachmentTransformRules::SnapToTargetNotIncludingScale);
					}
				}
 
				const FSoftObjectPath VoiPSoundClassName = GetDefault<UAudioSettings>()->VoiPSoundClass;
				if (VoiPSoundClassName.IsValid())
				{
					NewAudioComponent->SoundClassOverride = LoadObject<USoundClass>(nullptr, *VoiPSoundClassName.ToString());
				}
			}
			else
			{
				UE_LOG(LogHolodeckVoiceChat, Warning, TEXT("Unable to create voice audio component!"));
			}
		}
	}
 
	return NewAudioComponent;
}
 
UHolodeckVoiceChatSubsystem* URemoteTalkerData::GetSubsystem() const
{
	return Cast<UHolodeckVoiceChatSubsystem>(GetOuter());
}
 
/////////////////////////////////////////////////////////////
/// UHolodeckVoiceTalkerComponent
///
 
UHolodeckVoiceTalkerComponent::UHolodeckVoiceTalkerComponent(const FObjectInitializer& ObjectInitializer)
	: Super(ObjectInitializer)
{
	PrimaryComponentTick.bCanEverTick = true;
	PrimaryComponentTick.bStartWithTickEnabled = true;
	bAutoRegister = true;
	bAutoActivate = true;
}
 
void UHolodeckVoiceTalkerComponent::BeginPlay()
{
	Super::BeginPlay();
 
	const auto World = GetWorld();
	if (World && World->IsGameWorld() && World->GetNetMode() != NM_Client)
	{
		const auto GameState = GetWorld()->GetGameState();
		check(GameState);
		const auto ExperienceComponent = GameState->FindComponentByClass<UHolodeckExperienceManagerComponent>();
		check(ExperienceComponent);
		ExperienceComponent->CallOrRegister_OnExperienceLoaded(FOnHolodeckExperienceLoaded::FDelegate::CreateUObject(this, &ThisClass::OnExperienceLoaded));
	}
}
 
void UHolodeckVoiceTalkerComponent::TickComponent(float DeltaTime, ELevelTick TickType, FActorComponentTickFunction* ThisTickFunction)
{
	Super::TickComponent(DeltaTime, TickType, ThisTickFunction);
	// TODO
 
	if (const auto PlayerState = GetPlayerState<APlayerState>())
	{
		if (const auto Pawn = PlayerState->GetPawn())
		{
			FColor DebugColor = Pawn->IsLocallyControlled() ? FColor::Red : FColor::Green;
			UKismetSystemLibrary::DrawDebugSphere(GetWorld(), Pawn->GetActorLocation(), 40.0, 12, DebugColor, 0.0, 0.0);
		}
	}
}
 
void UHolodeckVoiceTalkerComponent::OnExperienceLoaded(const UHolodeckExperienceDefinition* Experience)
{
	if (GetNetMode() == NM_Standalone)
	{
		SetComponentTickEnabled(false);
		DestroyComponent();
	}
 
	if (const auto PlayerState = GetPlayerState<APlayerState>())
	{
		if (const auto PlayerController = PlayerState->GetPlayerController())
		{
			if (PlayerController->IsLocalController())
			{
				SetComponentTickEnabled(false);
				DestroyComponent();
			}
		}
	}
}
 
void UHolodeckVoiceTalkerComponent::HandleVoiceChatBeforeRecvAudioRendered(TArray<uint8> PcmSamples, int SampleRate, int Channels, bool bIsSilence, const FString& ChannelName, const FString& PlayerName)
{
	// TODO
}
 
/////////////////////////////////////////////////////////////
/// UHolodeckVoiceChatSubsystem
///
 
UAudioComponent* UHolodeckVoiceTalkerComponent::GetAudioComponent() const
{
	return AudioComponent;
}
 
UHolodeckVoiceChatSubsystem::UHolodeckVoiceChatSubsystem()
	: EOSOnlineSubsystem(nullptr)
	  , VoiceChatUser(nullptr)
{
}
 
UHolodeckVoiceChatSubsystem::~UHolodeckVoiceChatSubsystem()
{
	EOSOnlineSubsystem = nullptr;
	VoiceChatUser = nullptr;
}
 
void UHolodeckVoiceChatSubsystem::Initialize(FSubsystemCollectionBase& Collection)
{
	Super::Initialize(Collection);
 
#if COMMONUSER_OSSV1
	EOSOnlineSubsystem = static_cast<IOnlineSubsystemEOS*>(Online::GetSubsystem(GetWorld()));
	check(EOSOnlineSubsystem);
	OnlineInstanceName = EOSOnlineSubsystem->GetInstanceName();
#endif
 
	CommonUserSubsystem = GetWorld()->GetGameInstance()->GetSubsystem<UCommonUserSubsystem>();
	check(CommonUserSubsystem.IsValid());
 
	CommonUserSubsystem->OnUserInitializeComplete.AddDynamic(this, &ThisClass::OnUserInitializeComplete);
}
 
void UHolodeckVoiceChatSubsystem::Deinitialize()
{
	Super::Deinitialize();
 
	OnVoiceChatBeforeRecvAudioRenderedDelegate.Unbind();
}
 
void UHolodeckVoiceChatSubsystem::OnUserInitializeComplete(const UCommonUserInfo* UserInfo, bool bSuccess, FText Error, ECommonUserPrivilege RequestedPrivilege, ECommonUserOnlineContext OnlineContext)
{
	if (!bSuccess || !UserInfo) return;
 
	// If the user is not allowed to play online, ignore
	if (RequestedPrivilege != ECommonUserPrivilege::CanPlayOnline) return;
 
	// If a different local player was initialized, ignore
	if (UserInfo->LocalPlayerIndex != GetLocalPlayer()->GetLocalPlayerIndex()) return;
 
#if COMMONUSER_OSSV1
	// Initialize VoiceChat user
	VoiceChatUser = static_cast<FEOSVoiceChatUser*>(EOSOnlineSubsystem->GetVoiceChatUserInterface(*UserInfo->GetNetId()));
	check(VoiceChatUser);
#endif
 
	OnVoiceChatBeforeRecvAudioRenderedDelegate = FOnVoiceChatBeforeRecvAudioRenderedDelegate::FDelegate::CreateUObject(this, &ThisClass::HandleVoiceChatBeforeRecvAudioRendered);
	VoiceChatUser->RegisterOnVoiceChatBeforeRecvAudioRenderedDelegate(OnVoiceChatBeforeRecvAudioRenderedDelegate);
}
 
void UHolodeckVoiceChatSubsystem::HandleVoiceChatBeforeRecvAudioRendered(TArrayView<int16> PcmSamples, int SampleRate, int Channels, bool bIsSilence, const FString& ChannelName, const FString& PlayerName)
{
	if (OnVoiceChatBeforeRecvAudioRenderedDelegate_Multicast.IsBound())
	{
		TArray<uint8> NewPcmSamples;
		NewPcmSamples.Reserve(PcmSamples.Num());
		NewPcmSamples.Append(reinterpret_cast<uint8*>(PcmSamples.GetData()), PcmSamples.Num() * sizeof(int16_t));
 
		OnVoiceChatBeforeRecvAudioRenderedDelegate_Multicast.Broadcast(NewPcmSamples, SampleRate, Channels, bIsSilence, ChannelName, PlayerName);
	}
 
	// if (bIsSilence || ChannelName.IsEmpty() || PlayerName.IsEmpty()) return;
	//
	// const auto PlayerStateRecvFrom = FindPlayerStateByVoiceChatPlayerName(PlayerName);
	// if (!PlayerStateRecvFrom) return;
	//
	// const FUniqueNetIdWrapper RemoteTalkerId(PlayerStateRecvFrom->GetUniqueId()->AsShared());
	//
	// TObjectPtr<URemoteTalkerData> QueuedData = nullptr;
	// if (TObjectPtr<URemoteTalkerData>* FoundQueuedData = RemoteTalkers.Find(RemoteTalkerId))
	// {
	// 	QueuedData = *FoundQueuedData;
	// }
	// else
	// {
	// 	// Add new QueuedData
	// 	QueuedData = NewObject<URemoteTalkerData>(this);
	// 	QueuedData->Initialize(PlayerStateRecvFrom, SampleRate, Channels);
	// 	RemoteTalkers.Add(RemoteTalkerId, QueuedData);
	// }
	//
	// // new voice packet.
	// QueuedData->LastSeen = FPlatformTime::Seconds();
	//
	// uint32 BytesWritten = UVOIPStatics::GetMaxUncompressedVoiceDataSizePerChannel();
	// const int32 SampleSize = sizeof(uint16) * UVOIPStatics::GetVoiceNumChannels();
	//
	// {
	// 	FScopeLock ScopeLock(&QueuedData->QueueLock);
	// 	QueuedData->CurrentUncompressedDataQueueSize = QueuedData->UncompressedDataQueue.Num();
	// 	const int32 AvailableSamples = QueuedData->CurrentUncompressedDataQueueSize / Size;
	//
	// 	QueuedData->UncompressedDataQueue = 
	// }
}
 
APlayerState* UHolodeckVoiceChatSubsystem::FindPlayerStateByVoiceChatPlayerName(const FString& PlayerName)
{
	if (PlayerName.IsEmpty()) return nullptr;
 
	const UWorld* World = GetWorld();
	if (!World) return nullptr;
	const auto GameState = World->GetGameState();
	if (!GameState) return nullptr;
 
	const auto FoundPlayerState = GameState->PlayerArray.FindByPredicate([PlayerName](TObjectPtr<APlayerState> PlayerState)
	{
		return PlayerState->GetUniqueId().ToString().EndsWith(PlayerName, ESearchCase::CaseSensitive);
	});
 
	if (FoundPlayerState)
	{
		return *FoundPlayerState;
	}
 
	return nullptr;
}
 
IOnlineSubsystem* UHolodeckVoiceChatSubsystem::GetOnlineSubSystem()
{
	if (const UWorld* World = GetWorldForOnline(OnlineInstanceName))
	{
		return Online::GetSubsystem(World);
	}
 
	return nullptr;
}