Commit 071a2401 authored by tyyin lan's avatar tyyin lan

feat: 使用应用场景增加数字人

parent 1a570b36
......@@ -4,4 +4,3 @@ VITE_APP_NAME = 'ModelLink'
VITE_APP_THEME_COLOR = '#000DFF'
VITE_PUBLIC_PATH = /fe
VITE_ROUTER_MODE = 'h5'
\ No newline at end of file
......@@ -16,6 +16,7 @@
"preinstall": "npx only-allow pnpm"
},
"dependencies": {
"@bddh/starling-dhiframe": "^2.1.9",
"@icon-park/vue-next": "^1.4.2",
"@iconify/vue": "^4.1.2",
"@microsoft/fetch-event-source": "^2.0.1",
......
......@@ -8,6 +8,9 @@ importers:
.:
dependencies:
'@bddh/starling-dhiframe':
specifier: ^2.1.9
version: 2.1.9(uuid@9.0.1)
'@icon-park/vue-next':
specifier: ^1.4.2
version: 1.4.2(vue@3.5.13(typescript@5.6.2))
......@@ -439,6 +442,11 @@ packages:
resolution: {integrity: sha512-+EzkxvLNfiUeKMgy/3luqfsCWFRXLb7U6wNQTk60tovuckwB15B191tJWvpp4HjiQWdJkCxO3Wbvc6jlk3Xb2Q==}
engines: {node: '>=6.9.0'}
'@bddh/starling-dhiframe@2.1.9':
resolution: {integrity: sha512-Y/RqKU0ggHEzszJPuEx0Hi2wVJwjVLiPBfHQrcY3VlrfKVNo3X+2ApqS8uk6BoBfgfE/GPy7XAqaNQJLWWersg==}
peerDependencies:
uuid: ^9.0.1
'@commitlint/cli@19.5.0':
resolution: {integrity: sha512-gaGqSliGwB86MDmAAKAtV9SV1SHdmN8pnGq4EJU4+hLisQ7IFfx4jvU4s+pk6tl0+9bv6yT+CaZkufOinkSJIQ==}
engines: {node: '>=v18'}
......@@ -3306,6 +3314,10 @@ packages:
util-deprecate@1.0.2:
resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==}
uuid@9.0.1:
resolution: {integrity: sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==}
hasBin: true
validator@13.12.0:
resolution: {integrity: sha512-c1Q0mCiPlgdTVVVIJIrBuxNicYE+t/7oKeI9MWLj3fh/uq2Pxh/3eeWbVZ4OcGW1TUf53At0njHw5SMdA3tmMg==}
engines: {node: '>= 0.10'}
......@@ -3776,6 +3788,10 @@ snapshots:
'@babel/helper-validator-identifier': 7.27.1
optional: true
'@bddh/starling-dhiframe@2.1.9(uuid@9.0.1)':
dependencies:
uuid: 9.0.1
'@commitlint/cli@19.5.0(@types/node@20.16.5)(typescript@5.6.2)':
dependencies:
'@commitlint/format': 19.5.0
......@@ -6690,6 +6706,8 @@ snapshots:
util-deprecate@1.0.2: {}
uuid@9.0.1: {}
validator@13.12.0: {}
vant@4.9.18(vue@3.5.13(typescript@5.6.2)):
......
......@@ -201,3 +201,11 @@ export function fetchGetAPIProfile<T>() {
export function fetchResetAPIProfile<T>() {
return request.post<T>('/agentApplicationRest/resetApiProfile.json')
}
/**
* @query agentId 应用Id
* @returns 数字人token
*/
export function fetchBaiduDigitalPeopleToken<T>(agentId: string) {
return request.post<T>(`/agentApplicationInfoRest/generalDigitalhumanToken.json?agentId=${agentId}`)
}
......@@ -879,3 +879,8 @@ smart_forms_module:
customer_visit: 'Customer visit'
meeting: 'Meeting'
training: 'Training'
digital_human_module:
digital_human_loading_tip: 'The digital human is loading'
more_users_tip: 'There are currently many online users. Please try again later'
timed_out: 'It has timed out. Please try again'
\ No newline at end of file
......@@ -877,3 +877,8 @@ smart_forms_module:
customer_visit: '客户拜访'
meeting: '会议'
training: '培训'
digital_human_module:
digital_human_loading_tip: '数字人加载中'
more_users_tip: '目前线上用户较多,请您稍后再试'
timed_out: '已超时,请重试'
\ No newline at end of file
......@@ -877,3 +877,8 @@ smart_forms_module:
customer_visit: '客戶拜訪'
meeting: '會議'
training: '培訓'
digital_human_module:
digital_human_loading_tip: '數字人加載中'
more_users_tip: '目前線上用戶較多,請您稍後再試'
timed_out: '已超時,請重試'
\ No newline at end of file
......@@ -57,4 +57,12 @@ export interface PersonalAppConfigState {
isCopy?: string
agentPublishId: number
unitIds: string[]
digitalhumanConfig: {
appId: string
appKey: string
enable: 'Y' | 'N'
figureId: string
timbreId: string
}
}
export function checkPlayUnMute() {
return new Promise<boolean>((resolve) => {
const audioElem = document.createElement('audio')
audioElem.src =
'data:audio/wav;base64,UklGRp4AAABXQVZFZm10IBAAAAABAAEARKwAAIhYAQACABAATElTVBoAAABJTkZPSVNGVA0AAABMYXZmNjEuNy4xMDAAAGRhdGFYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=='
audioElem.muted = false
const playPromise = audioElem.play()
if (playPromise !== undefined) {
playPromise
.then(() => {
// 自动播放成功
resolve(true)
})
.catch((error) => {
if (error.name === 'NotAllowedError' || error.name === 'AbortError') {
// 自动播放被禁止或中止
resolve(false)
} else {
// 其他错误
resolve(false)
}
})
// 防止playPromise为空对象,没有finially的情况
setTimeout(() => {
audioElem.remove()
resolve(true)
}, 300)
} else {
// 如果 play() 返回 undefined,可能是浏览器不支持 Promise 风格的 play()
audioElem.remove()
resolve(false)
}
})
}
<script setup lang="ts">
import { fetchBaiduDigitalPeopleToken } from '@/apis/agent-application'
import { checkPlayUnMute } from '@/utils/check-play-unmute'
import type DHIframe from '@bddh/starling-dhiframe'
import { computed, onMounted, onUnmounted, ref, useTemplateRef, watch, watchEffect } from 'vue'
import { useI18n } from 'vue-i18n'
interface Props {
agentId: string
digitalhumanConfig: {
appId: string
figureId: string
appKey: string
enable: string
timbreId: string
}
dhIframe: DHIframe
}
const props = defineProps<Props>()
const { t } = useI18n()
const digitalHumanIframeRef = useTemplateRef<HTMLIFrameElement>('digitalHumanIframeRef')
const token = ref('')
const digitalHumanLoading = ref(true)
const digitalHumanLoadError = ref(false)
const dhIframeMessageSubscription = ref(false)
const videoIsMuted = ref(false)
const serviceIsBusy = ref(false)
const serviceDisconnection = ref(false)
const showTimeoutTip = ref(false)
const iframeSrc = computed(() => {
return `https://open.xiling.baidu.com/cloud/react?mode=inline&token=${token.value}&figureId=${props.digitalhumanConfig.figureId}&initMode=noAudio&showLogo=false&autoChromaKey=true&cameraId=0&resolutionWidth=1080&resolutionHeight=1920&cp-inactiveDisconnectSec=600&cp-autoAnimoji=true&cp-preAlertSec=10&textAssist=false&showMessage=false&debug=false`
})
const onDigitalHumanMessage = (msg: any) => {
if (msg.origin === 'https://open.xiling.baidu.com') {
const { type, content } = msg.data
switch (type) {
case 'rtcState':
{
const { action, body } = content
if (action === 'remotevideoon' && body) {
digitalHumanLoading.value = false
}
if (action === 'localVideoMuted' && body) {
videoIsMuted.value = true
}
}
break
case 'msg': {
const { action, code } = content
if (action === 'DISCONNECT_ALERT') {
showTimeoutTip.value = true
} else if (action === 'TIMEOUT_EXIT') {
digitalHumanIframeRef.value && digitalHumanIframeRef.value.remove()
serviceDisconnection.value = true
digitalHumanLoadError.value = true
}
if (code === 1004 || code === 3001) {
serviceIsBusy.value = true
digitalHumanLoadError.value = true
}
break
}
default:
break
}
}
}
watchEffect(() => {
if (props.agentId) {
getBaiduDigitalPeopleToken(props.agentId)
}
})
watch(
() => props.dhIframe,
(newDhIframe) => {
if (newDhIframe && !dhIframeMessageSubscription.value) {
newDhIframe.registerMessageReceived(onDigitalHumanMessage)
dhIframeMessageSubscription.value = true
}
},
{ immediate: true },
)
onMounted(() => {
checkPlayUnMute().then((res) => {
videoIsMuted.value = !res
})
})
onUnmounted(() => {
props.dhIframe && props.dhIframe.removeMessageReceived(onDigitalHumanMessage)
})
function getBaiduDigitalPeopleToken(agentId: string) {
fetchBaiduDigitalPeopleToken<string>(agentId).then((res) => {
if (res.code !== 0) return null
token.value = res.data
})
}
// function handleUnmute() {
// if (videoIsMuted.value) {
// props.dhIframe?.sendCommand({
// subType: 'muteAudio',
// subContent: false,
// })
// }
// videoIsMuted.value = false
// }
</script>
<template>
<div class="relative flex h-full flex-1 items-end justify-center">
<div class="relative h-[80%] w-[400px]">
<div v-show="digitalHumanLoading" class="flex-center flex h-full w-full">
<div class="flex -translate-y-full flex-col justify-center">
<n-spin size="large" />
<div class="mt-[20px]">{{ t('digital_human_module.digital_human_loading_tip') }}...</div>
</div>
</div>
<iframe
v-show="!digitalHumanLoading"
id="digital-human-iframe"
ref="digitalHumanIframeRef"
class="h-full w-full"
:src="iframeSrc"
allow="autoplay"
/>
<!-- <div
v-if="!digitalHumanLoading && videoIsMuted && !digitalHumanLoadError"
class="absolute left-1/2 top-1/2 -translate-x-1/2 -translate-y-1/2 cursor-pointer select-none text-[16px] text-[#e74c3c]"
@click="handleUnmute"
>
取消静音
</div> -->
</div>
<template v-if="digitalHumanLoadError">
<div
v-if="serviceIsBusy"
class="absolute left-1/2 top-1/3 -translate-x-1/2 -translate-y-1/2 cursor-pointer select-none text-[16px] text-[#e74c3c]"
>
{{ t('digital_human_module.more_users_tip') }}
</div>
<div
v-else-if="serviceDisconnection"
class="absolute left-1/2 top-1/3 -translate-x-1/2 -translate-y-1/2 cursor-pointer select-none text-[16px] text-[#e74c3c]"
>
{{ t('digital_human_module.timed_out') }}
</div>
</template>
</div>
</template>
......@@ -16,6 +16,8 @@ import { ChannelType } from '@/enums/channel'
import { smartFormTypeConverter } from '@/views/personal-space/personal-app-setting/components/agent-config/agent-preview/components/smart-forms/utils/smart-forms'
import { SmartFormTypes } from '@/views/personal-space/personal-app-setting/components/agent-config/types'
import { SmartFormDisplayFormat } from '@/views/personal-space/personal-app-setting/components/agent-config/agent-preview/components/smart-forms/types/types'
import type DHIframe from '@bddh/starling-dhiframe'
import { PersonalAppConfigState } from '@/store/types/personal-app-config'
interface Props {
agentId: string
......@@ -26,7 +28,8 @@ interface Props {
isEnableVoice: boolean
answerAudioAutoPlay: boolean
answerAudioPlaying: boolean
timbreId: string
dhIframe: DHIframe | null
agentApplicationConfig: PersonalAppConfigState
}
const { t } = useI18n()
......@@ -489,13 +492,17 @@ function ttsSocketSendText(text: string, audioUrlSerialNo: number, messageId: st
const content = (text || '').replace(/\^\[[\d\\[\]-]+?\]\^/g, '')
if (content && props.timbreId) {
const timbreId = props.dhIframe
? props.agentApplicationConfig.digitalhumanConfig.timbreId
: props.agentApplicationConfig.voiceConfig.timbreId
if (content && timbreId) {
ttsSocketCtl.connect(() => {
ttsSocketCtl.send({
codec: 'wav',
sampleRate: 16000,
speed: 0,
voiceType: props.timbreId,
voiceType: timbreId,
volume: 0,
content,
})
......
......@@ -76,7 +76,9 @@ const messageAuthor = computed(() => {
})
const timbreEnabled = computed(() => {
return !!props.agentApplicationConfig.voiceConfig.timbreId
return (
!!props.agentApplicationConfig.voiceConfig.timbreId || !!props.agentApplicationConfig.digitalhumanConfig.timbreId
)
})
const isShowAudioControl = computed(() => {
......
<script setup lang="ts">
import { computed, onMounted, onUnmounted, ref, shallowRef } from 'vue'
import { computed, onMounted, onUnmounted, ref, shallowRef, watch } from 'vue'
import { useRouter } from 'vue-router'
import { useI18n } from 'vue-i18n'
import { Howl } from 'howler'
......@@ -23,6 +23,8 @@ import { fetchGetMemberInfoById } from '@/apis/user'
import { UserInfo } from '@/store/types/user'
import { validBrowser } from '@/utils/browser-detection'
import { SmartFormTypes } from '../personal-space/personal-app-setting/components/agent-config/types'
import DHIframe from '@bddh/starling-dhiframe'
import DigitalHuman from './components/digital-human.vue'
const { t } = useI18n()
......@@ -32,6 +34,9 @@ const userStore = useUserStore()
const { isMobile } = useLayoutConfig()
const dhIframe = shallowRef<DHIframe | null>(null)
const dhIframeMessageSubscription = ref(false)
const fullScreenLoading = ref(false)
const agentId = ref('')
const dialogsId = ref('')
......@@ -56,12 +61,88 @@ const isAnswerResponseLoading = ref(false)
const createContinueQuestionsException = ref(false)
const isAnswerResponseInterrupt = ref(false) // 回答响应被中断
const onDigitalHumanMessage = (msg: any) => {
if (msg.origin === 'https://open.xiling.baidu.com') {
const { type, content } = msg.data
switch (type) {
case 'rtcState':
break
case 'msg': {
const { action } = content
if (action === 'RENDER_START') {
if (currentPlayMessageItem.value) {
currentPlayMessageItem.value.isVoiceLoading = false
currentPlayMessageItem.value.isVoicePlaying = true
}
} else if (action === 'RENDER_COMPLETED') {
currentPlayAudioFragmentSerialNo.value += 1
if (
currentPlayMessageItem.value &&
currentPlayAudioFragmentSerialNo.value > currentPlayMessageItem.value.voiceFragmentUrlList.length - 1
) {
currentPlayMessageItem.value.isVoicePlaying = false
}
let audioFragmentUrl =
currentPlayMessageItem.value?.voiceFragmentUrlList[currentPlayAudioFragmentSerialNo.value]
if (audioFragmentUrl) {
dhIframe.value?.sendMessage({
action: 'TEXT_RENDER',
body: `<speak><audio src="${audioFragmentUrl}"/></speak>`,
// requestId: requestId || nanoid(),
})
} else if (
(currentPlayMessageItem.value?.voiceFragmentUrlList || []).length > currentPlayAudioFragmentSerialNo.value
) {
let timerId: NodeJS.Timeout | null = null
const audioFragmentCheck = () => {
audioFragmentUrl =
currentPlayMessageItem.value?.voiceFragmentUrlList[currentPlayAudioFragmentSerialNo.value]
if (audioFragmentUrl) {
timerId && clearInterval(timerId)
dhIframe.value?.sendMessage({
action: 'TEXT_RENDER',
body: `<speak><audio src="${audioFragmentUrl}"/></speak>`,
// requestId: requestId || nanoid(),
})
}
}
timerId = setInterval(audioFragmentCheck, 600)
}
}
break
}
default:
break
}
}
}
const isEnableDocumentParse = computed(() => {
return agentApplicationConfig.value.knowledgeConfig.isDocumentParsing === 'Y'
})
const isEnableVoice = computed(() => {
return !!agentApplicationConfig.value.voiceConfig.timbreId
return (
!!agentApplicationConfig.value.voiceConfig.timbreId || !!agentApplicationConfig.value.digitalhumanConfig?.timbreId
)
})
watch(dhIframe, (newDhIframe) => {
if (newDhIframe && !dhIframeMessageSubscription.value) {
newDhIframe.registerMessageReceived(onDigitalHumanMessage)
dhIframeMessageSubscription.value = true
}
})
onMounted(async () => {
......@@ -82,7 +163,7 @@ onMounted(async () => {
await handleGetApplicationDetail()
if (userStore.isLogin) {
await handleCreateDialogues()
await handleGetAutoPlayByAgentId()
await getAutoPlayByAgentId()
}
fullScreenLoading.value = false
......@@ -95,6 +176,8 @@ onMounted(async () => {
onUnmounted(() => {
handleAudioPause()
footerInputRef.value?.blockMessageResponse()
dhIframe.value?.removeMessageReceived(onDigitalHumanMessage)
})
async function handleGetApplicationDetail() {
......@@ -104,6 +187,10 @@ async function handleGetApplicationDetail() {
continuousQuestionStatus.value = res.data.commConfig.continuousQuestionStatus
document.title = agentApplicationConfig.value.baseInfo.agentTitle
handleGetMemberInfo()
if (agentApplicationConfig.value.digitalhumanConfig.enable === 'Y') {
dhIframe.value = new DHIframe('digital-human-iframe')
}
})
.catch(() => {
router.replace({ name: 'Home' })
......@@ -126,12 +213,12 @@ async function handleCreateDialogues() {
}
}
async function handleGetAutoPlayByAgentId() {
const res = await fetchGetAutoPlayByAgentId<'Y' | 'N'>(agentId.value)
function getAutoPlayByAgentId() {
fetchGetAutoPlayByAgentId<'Y' | 'N'>(agentId.value).then((res) => {
if (res.code === 0) {
answerAudioAutoPlay.value = res.data === 'Y'
}
})
}
function handleBack() {
......@@ -289,9 +376,18 @@ function handleAudioPlay(currentMessageItem: ConversationMessageItem, specificPl
const audioUrl = specificPlayUrl || currentMessageItem.voiceFragmentUrlList[0]
if (!dhIframe.value) {
howlSoundFactory(audioUrl)
isSoundCtlCreated.value = true
return
}
dhIframe.value?.sendMessage({
action: 'TEXT_RENDER',
body: `<speak><audio src="${audioUrl}"/></speak>`,
// requestId: requestId || nanoid(),
})
}
function handleAudioPause(isClearMessageList = false) {
......@@ -304,6 +400,11 @@ function handleAudioPause(isClearMessageList = false) {
isSoundCtlCreated.value = false
}
dhIframe.value?.sendMessage({
action: 'TEXT_RENDER',
body: '<interrupt></interrupt>',
})
currentPlayMessageItem.value && (currentPlayMessageItem.value.isVoicePlaying = false)
answerAudioPlaying.value = false
......@@ -327,7 +428,7 @@ function onSmartFormsStatusFreezeCheck(smartFormType: SmartFormTypes) {
<template>
<div v-loading="fullScreenLoading" class="flex h-screen min-h-[500px] w-full flex-col overflow-y-hidden">
<main class="h-full min-w-[1100px]">
<main class="relative h-full min-w-[1400px]">
<PageHeader
:agent-application-config="agentApplicationConfig"
:agent-member-info="agentMemberInfo"
......@@ -336,8 +437,15 @@ function onSmartFormsStatusFreezeCheck(smartFormType: SmartFormTypes) {
@to-create-application="handleCreateApplicationPage"
/>
<div id="share-agent-web-container" class="relative h-[calc(100%-68px)] w-full overflow-hidden bg-[#f2f5f9]">
<div class="relative mx-auto flex h-full w-[1000px] flex-col overflow-hidden">
<div
id="share-agent-web-container"
class="relative h-[calc(100%-68px)] w-full overflow-hidden bg-[#f2f5f9]"
:class="dhIframe ? 'flex' : ''"
>
<div
class="relative flex h-full flex-col overflow-hidden"
:class="dhIframe ? 'ml-[150px] mt-auto w-[900px]' : 'mx-auto w-[1000px]'"
>
<div v-show="isEnableVoice" class="absolute right-10 top-7 flex select-none items-center gap-2">
<span>{{ t('common_module.voice_auto_play') }}</span>
<n-switch v-model:value="answerAudioAutoPlay" size="small" @update:value="handleUpdateAutoPlaying">
......@@ -380,7 +488,8 @@ function onSmartFormsStatusFreezeCheck(smartFormType: SmartFormTypes) {
:is-enable-voice="isEnableVoice"
:answer-audio-auto-play="answerAudioAutoPlay"
:answer-audio-playing="answerAudioPlaying"
:timbre-id="agentApplicationConfig.voiceConfig.timbreId"
:dh-iframe="dhIframe"
:agent-application-config="agentApplicationConfig"
@add-message-item="handleAddMessageItem"
@update-specify-message-item="handleUpdateSpecifyMessageItem"
@delete-message-item="handleDeleteMessageItem"
......@@ -395,6 +504,13 @@ function onSmartFormsStatusFreezeCheck(smartFormType: SmartFormTypes) {
/>
</div>
</div>
<DigitalHuman
v-if="dhIframe"
:agent-id="agentApplicationConfig.baseInfo.agentId"
:digitalhuman-config="agentApplicationConfig.digitalhumanConfig"
:dh-iframe="dhIframe"
/>
</div>
</main>
</div>
......
......@@ -901,5 +901,11 @@ declare namespace I18n {
training: string
}
}
digital_human_module: {
digital_human_loading_tip: string
more_users_tip: string
timed_out: string
}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment