1. 下载插件：motionverse官网地址：概述 · Motionverse 接口文档 (deepscience.cn)
2. 按照官方文档新建Unity工程：对接说明 · Motionverse 接口文档 (deepscience.cn)
3. 通过我们自己的ASR，将语音转换为文本，文本通过大语言模型（chatgpt、文心一言以及其他大语言模型）生成的结果，直接通过生成式AI技术，把文本转化为AI智能体的声音、动作和表情，和大语言模型完美连接，只需要在获取到文本的时候调用以下代码即可：
代码示例：
DriveTask task = new DriveTask();
task.player = player;
task.text = question;
NLPDrive.GetDrive(task);
其中，player即为挂载motionverse插件中Player脚本的对象，question即为大语言模型获取到的答案。
还可以自己生成语音，通过语音链接调用以下代码：
DriveTask task = new DriveTask();
task.player = player;
task.text = audioUrl;
AudioUrlDrive.GetDrive(task);其中，player即为挂载motionverse插件中Player脚本的对象，audioUrl即为语音链接。
4. 新建脚本AskManager，并挂载到场景中（可新建空物体），脚本代码如下：
using LitJson;
using Motionverse;
using MotionverseSDK;
using System;
using System.Collections;
using System.Text;
using UnityEngine;
using UnityEngine.Networking;
using UnityEngine.UI;
public class AskManager : MonoBehaviour
{
public Player player;
public Dropdown dropdown;
public InputField inputField;
public Button btnSend;
public string chatGptKey = "";
private string chatUrl = "https://chatgpt.kazava.io/v1/chat/completions";
public string wenXinAPI = "";
public string wenXinSECRET = "";
private string wenXinToken = "";
private int curSelectNLP = 0;
// Start is called before the first frame update
private void Awake()
{
for (int i = 0; i < Display.displays.Length; i++)
{
Display.displays[i].Activate();
}
}
void Start()
{
dropdown.onValueChanged.AddListener((value) =>
{
curSelectNLP = value;
});
StartCoroutine(GetWenxinToken());
btnSend.onClick.AddListener(() =>
{
if (string.IsNullOrEmpty(inputField.text))
{
Debug.Log("请输入内容！");
}
else
{
GetAnswer(inputField.text);
}
});
}
public void GetAnswer(string q)
{
StartCoroutine(RealAnswer(q));
}
public IEnumerator RealAnswer(string question)
{
switch (curSelectNLP)
{
case 0:
DriveTask task = new DriveTask();
task.player = player;
task.text = question;
NLPDrive.GetDrive(task);
break;
case 1:
StartCoroutine(RequestChat(question));
break;
case 2:
StartCoroutine(ChatCompletions(question));
break;
default:
break;
}
Invoke("restartRecording", 1);
yield return null;
}
IEnumerator GetWenxinToken()
{
string url =$"https://aip.baidubce.com/oauth/2.0/token?grant_type=client_credentials&client_id={wenXinAPI}&client_secret={wenXinSECRET}";
UnityWebRequest webRequest = UnityWebRequest.Get(url);
webRequest.timeout = 5000;
yield return webRequest.SendWebRequest();
if (webRequest.result == UnityWebRequest.Result.Success)
{
string response = webRequest.downloadHandler.text;
var result = JsonUtility.FromJson<AccessTokenResponse>(response);
wenXinToken = result.access_token;
}
else
{
Debug.LogError("Failed to get access token: " + webRequest.error);
}
}
IEnumerator ChatCompletions(string content)
{
string url = $"https://aip.baidubce.com/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/completions?access_token={wenXinToken}";
WenXinPostData postData = new WenXinPostData();
postData.messages = new WenXinMessage[1];
postData.messages[0] = new WenXinMessage();
postData.messages[0].content = content + "30字以内";
string data = JsonMapper.ToJson(postData);
UnityWebRequest webRequest = UnityWebRequest.Post(url, data, "application/json");
webRequest.timeout = 5000;
yield return webRequest.SendWebRequest();
if (webRequest.result == UnityWebRequest.Result.Success)
{
string response = webRequest.downloadHandler.text;
WenXinRequest requestData = JsonMapper.ToObject<WenXinRequest>(response);
DriveTask task = new DriveTask();
task.player = player;
task.text = requestData.result;
TextDrive.GetDrive(task);
}
else
{
Debug.LogError("Chat completions request failed: " + webRequest.error);
}
}
private IEnumerator RequestChat(string content)
{
using (UnityWebRequest webRequest = new UnityWebRequest(chatUrl, "POST"))
{
webRequest.SetRequestHeader("Content-Type", "application/json");
ChatGPTPostData postData = new ChatGPTPostData();
postData.key = chatGptKey;
postData.messages = new PostMessage[1];
postData.messages[0] = new PostMessage();
postData.messages[0].content = content + "30字以内"; ;
string data = JsonMapper.ToJson(postData);
byte[] jsonToSend = new UTF8Encoding().GetBytes(data);
webRequest.uploadHandler = new UploadHandlerRaw(jsonToSend);
webRequest.downloadHandler = new DownloadHandlerBuffer();
yield return webRequest.SendWebRequest();
if (webRequest.result != UnityWebRequest.Result.Success)
{
Debug.LogError("ChatGPT request error: " + content + webRequest.error);
}
else
{
string response = webRequest.downloadHandler.text;
ChatGPTRequestData requestData = JsonMapper.ToObject<ChatGPTRequestData>(response);
DriveTask task = new DriveTask();
task.player = player;
task.text = requestData.choices[0].message.content;
TextDrive.GetDrive(task);
}
}
}
}
5. 新建脚本RealtimeAsrManager，代码如下：
using System;
using System.Collections;
using UnityEngine;
using UnityEngine.UI;
using UnityWebSocket;
namespace Motionverse
{
public class RealtimeAsrManager : MonoBehaviour
{
private IWebSocket webSocket;
private Status status = Status.FirstFrame;
private bool lockReconnect = false;
[HideInInspector]
public static Action<string> Asr;
[SerializeField]
private GameObject TextBG;
//百度
public int AsrAppId;
public string AsrAppkey = "";
void Start()
{
CreateWebSocket();
RecorderManager.DataAvailable += OnDataAvailable;
}
private void OnDisable()
{
RecorderManager.DataAvailable -= OnDataAvailable;
}
void CreateWebSocket()
{
try
{
string uril = "wss://vop.baidu.com/realtime_asr?sn=" + Guid.NewGuid().ToString();
webSocket = new WebSocket(uril);
InitHandle();
webSocket.ConnectAsync();
}
catch (Exception e)
{
Debug.Log("websocket连接异常:" + e.Message);
ReConnect();
}
}
private void InitHandle()
{
RemoveHandle();
webSocket.OnOpen += OnOpen;
webSocket.OnMessage += OnMessage;
webSocket.OnClose += OnClose;
webSocket.OnError += OnError;
}
void RemoveHandle()
{
webSocket.OnOpen -= OnOpen;
webSocket.OnMessage -= OnMessage;
webSocket.OnClose -= OnClose;
webSocket.OnError -= OnError;
}
//请求开始
private void OnDataAvailable(byte[] data)
{
if (webSocket == null || (webSocket != null && webSocket.ReadyState != WebSocketState.Open))
return;
switch (status)
{
case Status.FirstFrame://握手
{
var firstFrame = new FirstFrame();
firstFrame.data.appid= AsrAppId;
firstFrame.data.appkey = AsrAppkey;
webSocket.SendAsync(JsonUtility.ToJson(firstFrame));
status = Status.ContinueFrame;
}
break;
case Status.ContinueFrame://开始发送
{
if (data.Length > 0)
{
webSocket.SendAsync(data);
}
}
break;
case Status.LastFrame://关闭
{
webSocket.SendAsync(JsonUtility.ToJson(new LastFrame()));
}
break;
default:
break;
}
}
void ReConnect()
{
if (this.lockReconnect)
return;
this.lockReconnect = true;
StartCoroutine(SetReConnect());
}
private IEnumerator SetReConnect()
{
yield return new WaitForSeconds(1);
CreateWebSocket();
lockReconnect = false;
}
#region WebSocket Event Handlers
private void OnOpen(object sender, OpenEventArgs e)
{
status = Status.FirstFrame;
}
private void OnMessage(object sender, MessageEventArgs e)
{
var err_msg = Utils.GetJsonValue(e.Data, "err_msg");
var type = Utils.GetJsonValue(e.Data, "type");
if (err_msg == "OK" && type == "MID_TEXT")
{
var result = Utils.GetJsonValue(e.Data, "result");
TextBG.GetComponentInChildren<Text>().text = result;
}
if (err_msg == "OK" && type == "FIN_TEXT")
{
var result = Utils.GetJsonValue(e.Data, "result");
TextBG.GetComponentInChildren<Text>().text = result;
if (result.Length > 1)
{
RecorderManager.Instance.EndRecording();
Asr?.Invoke(result);
}
}
}
private void OnClose(object sender, CloseEventArgs e)
{
Debug.Log("websocket关闭," + string.Format("Closed: StatusCode: {0}, Reason: {1}", e.StatusCode, e.Reason));
webSocket = null;
ReConnect();
}
private void OnError(object sender, ErrorEventArgs e)
{
if (e != null)
Debug.Log("websocket连接异常:" + e.Message);
webSocket = null;
ReConnect();
}
#endregion
}
}
6. 新建脚本RecorderManager，代码如下：
using UnityEngine;
using System;
using System.Collections;
using UnityEngine.UI;
using Unity.VisualScripting;
namespace Motionverse
{
public class RecorderManager : Singleton<RecorderManager>
{
//标记是否有麦克风
private bool isHaveMic = false;
//当前录音设备名称
private string currentDeviceName = string.Empty;
//表示录音的最大时长
int recordMaxLength = 600;
//录音频率,控制录音质量(16000)
int recordFrequency = 16000;
[HideInInspector]
public static Action<byte[]> DataAvailable;
[SerializeField]
private GameObject micON;
[SerializeField]
private GameObject micOFF;
[SerializeField]
private Image micAmount;
[SerializeField]
private GameObject TextBG;
private AudioClip saveAudioClip;
int offsetSamples = 0;
private void Start()
{
Debug.Log(Microphone.devices[0]);
if (Microphone.devices.Length > 0)
{
isHaveMic = true;
currentDeviceName = Microphone.devices[0];
StartCoroutine(GetAudioFrames());
StartRecording();
}
}
/// <summary>
/// 开始录音
/// </summary>
/// <returns></returns>
public void StartRecording() //16000
{
if (isHaveMic == false || Microphone.IsRecording(currentDeviceName))
{
return;
}
micOFF.gameObject.SetActive(false);
micON.gameObject.SetActive(true);
TextBG.GetComponentInChildren<Text>().text = null;
offsetSamples = 0;
saveAudioClip = Microphone.Start(currentDeviceName, true, recordMaxLength, recordFrequency);
}
public void OnButtonClick()
{
if (Microphone.IsRecording(currentDeviceName))
{
EndRecording();
}
else
{
StartRecording();
}
}
public void EndRecording()
{
if (isHaveMic == false || !Microphone.IsRecording(currentDeviceName))
{
return;
}
micOFF.gameObject.SetActive(true);
micON.gameObject.SetActive(false);
//结束录音
Microphone.End(currentDeviceName);
}
public bool IsRecording()
{
return Microphone.IsRecording(currentDeviceName);
}
IEnumerator GetAudioFrames()
{
while (true)
{
if (Microphone.IsRecording(currentDeviceName))
{
int lenght = Microphone.GetPosition(currentDeviceName) * saveAudioClip.channels - offsetSamples;
if (lenght > 0)
{
float[] samples = new float[lenght];
saveAudioClip.GetData(samples, offsetSamples);
var samplesShort = new short[samples.Length];
for (var index = 0; index < samples.Length; index++)
{
samplesShort[index] = (short)(samples[index] * short.MaxValue);
}
byte[] binaryData = new byte[samplesShort.Length * 2];
Buffer.BlockCopy(samplesShort, 0, binaryData, 0, binaryData.Length);
offsetSamples += lenght;
DataAvailable?.Invoke(binaryData);
}
yield return new WaitForSeconds(0.16f);
}
else
{
yield return new WaitForSeconds(0.16f);
byte[] binaryData = new byte[2];
DataAvailable?.Invoke(binaryData);
}
}
}
/// <summary>
/// 获取毫秒级别的时间戳,用于计算按下录音时长
/// </summary>
/// <returns></returns>
public double GetTimestampOfNowWithMillisecond()
{
return (DateTime.Now.ToUniversalTime().Ticks - 621355968000000000) / 10000;
}
private void LateUpdate()
{
if (isHaveMic == true && Microphone.IsRecording(currentDeviceName))
{
micAmount.fillAmount = Volume;
}
}
public float Volume
{
get
{
if (Microphone.IsRecording(currentDeviceName))
{
// 采样数
int sampleSize = 128;
float[] samples = new float[sampleSize];
int startPosition = Microphone.GetPosition(currentDeviceName) - (sampleSize + 1);
// 得到数据
if (startPosition < 0)
return 0;
saveAudioClip.GetData(samples, startPosition);
// Getting a peak on the last 128 samples
float levelMax = 0;
for (int i = 0; i < sampleSize; ++i)
{
float wavePeak = samples[i];
if (levelMax < wavePeak)
levelMax = wavePeak;
}
return levelMax;
}
return 0;
}
}
void OnGUI()
{
GUIStyle guiStyle = GUIStyle.none;
guiStyle.fontSize = 10;
guiStyle.normal.textColor = Color.white;
guiStyle.alignment = TextAnchor.UpperLeft;
Rect tr = new Rect(0, 0, 100, 100);
GUI.Label(tr, currentDeviceName, guiStyle);
}
}
}
7. 新建空物体，挂载脚本RecorderManager和RealtimeAsrManager
8. 输入百度asr的appId和secretKey：
9. 输入GPTkey、问心一眼appId和secretKey：
10. 根据需求选择相应的NLP方式：
注意：如果缺少Singleton文件，可使用如下代码：
using UnityEngine;
namespace Motionverse
{
public abstract class Singleton<T> : MonoBehaviour where T : Singleton<T>
{
private static T sInstance = null;
public static T Instance
{
get
{
if (sInstance == null)
{
GameObject gameObject = new(typeof(T).FullName);
sInstance = gameObject.AddComponent<T>();
}
return sInstance;
}
}
public static void Clear()
{
sInstance = null;
}
protected virtual void Awake()
{
if (sInstance != null) Debug.LogError(name + "error: already initialized", this);
sInstance = (T)this;
}
}
}若有收获，就点个赞吧