fix: 语音交互优化

src/locales/langs/en.yaml

@@ -100,6 +100,8 @@ common_module:
   voice_auto_play: 'Voice auto play'
   start_playing: 'play'
   stop_playing: 'stop'
+  unplayable: 'unplayable'
+  unplayable_tip: 'The voice setting do not match the model output language'
   response_error: 'Response error'
   agent_exception: 'Agent exception, please try again later!'
   equity: 'Equity'
@@ -116,6 +118,8 @@ common_module:
     cancel_associate_file_tip: 'No longer answer around this file'
     upload_file_limit: 'Only a single file can be uploaded in PDF, DOC, DOCX, MD, TXT format, up to 10MB'
     overwrite_file_tip: 'The newly uploaded file overwrites the original file, whether to continue uploading'
+    stop_playing_and_then_operate: 'When the audio is playing, stop playing and then perform the operation'
+    do_not_operate_until_the_reply_is_complete: 'Do not operate until the reply is complete'
 
   data_table_module:
     action: 'Controls'
@@ -294,9 +298,10 @@ personal_space_module:
         memory_fragment_delete_row_tip_content: 'After data deletion, it cannot be revoked. Are you sure you want to delete it?'
         add_knowledge_successfully: 'Data set {0} was added successfully'
         remove_knowledge_successfully: 'Data set {0} was removed successfully'
-        setting_timbre: 'Setting timbre'
-        setting_timbre_message: 'You can set the language and timbre'
-        setting_timbre_desc: 'You can customize the voice and timbre for voice broadcast, and you can set only one at a time.'
+        setting_voice: 'Setting voice'
+        setting_voice_message: 'You can set the language and tone. If the selected language is inconsistent with the model language, the speech cannot be played'
+        setting_voice_desc: 'You can customize the voice and timbre for voice broadcast, and you can set only one at a time.'
+        currently_only_one_voice_can_be_set: 'Currently, you can set only one voice'
 
         memory_variable_modal:
           edit_memory_variable: 'Edit memory variable'

src/locales/langs/zh-cn.yaml

@@ -99,6 +99,8 @@ common_module:
   voice_auto_play: '语音自动播放'
   start_playing: '开始播放'
   stop_playing: '停止播放'
+  unplayable: '不可播放'
+  unplayable_tip: '语音设置与模型输出语言不匹配'
   response_error: '响应错误'
   agent_exception: '应用异常，请稍后重试！'
   equity: '权益'
@@ -115,6 +117,8 @@ common_module:
     cancel_associate_file_tip: '不再围绕这个文件回答'
     upload_file_limit: '仅支持上传单个文件，支持PDF、DOC、DOCX、MD、TXT格式，最大10MB'
     overwrite_file_tip: '新上传的文件会覆盖原有文件，是否继续上传'
+    stop_playing_and_then_operate: '音频播放中，请停止播放后再操作'
+    do_not_operate_until_the_reply_is_complete: '回复完成后再操作'
 
   data_table_module:
     action: '操作'
@@ -292,9 +296,10 @@ personal_space_module:
         memory_fragment_delete_row_tip_content: '数据删除后不可撤销，确定要删除吗?'
         add_knowledge_successfully: '数据集 {0} 添加成功'
         remove_knowledge_successfully: '数据集 {0} 移除成功'
-        setting_timbre: '设置音色'
-        setting_timbre_message: '你可以设置语言和音色'
-        setting_timbre_desc: '您可自定义语音及音色，用于语音播报，且每次仅可设置一种。'
+        setting_voice: '设置语音'
+        setting_voice_message: '你可以设置语言和音色，若所选语言与模型语言不一致，则无法播放语音'
+        setting_voice_desc: '您可自定义语音及音色，用于语音播报，且每次仅可设置一种。'
+        currently_only_one_voice_can_be_set: '当前仅可设置一种声音'
 
         memory_variable_modal:
           edit_memory_variable: '编辑记忆变量'

src/locales/langs/zh-hk.yaml

@@ -99,6 +99,8 @@ common_module:
   voice_auto_play: '語音自動播放'
   start_playing: '開始播放'
   stop_playing: '停止播放'
+  unplayable: '不可播放'
+  unplayable_tip: '語音設置與模型輸出語言不匹配'
   response_error: '響應錯誤'
   agent_exception: '應用異常，請稍後重試！'
   equity: '权益'
@@ -115,6 +117,8 @@ common_module:
     cancel_associate_file_tip: '不再圍繞這個文件回答'
     upload_file_limit: '僅支持上傳單個文件，支持PDF、DOC、DOCX、MD、TXT格式，最大10MB'
     overwrite_file_tip: '新上傳的文件會覆蓋原有文件，是否繼續上傳'
+    stop_playing_and_then_operate: '音頻播放中，請停止播放後再操作'
+    do_not_operate_until_the_reply_is_complete: '回覆完成後再操作'
 
   data_table_module:
     action: '操作'
@@ -292,9 +296,10 @@ personal_space_module:
         memory_fragment_delete_row_tip_content: '數據删除後不可撤銷，確定要删除嗎?'
         add_knowledge_successfully: '數據集 {0} 添加成功'
         remove_knowledge_successfully: '數據集 {0} 移除成功'
-        setting_timbre: '設置音色'
-        setting_timbre_message: '你可以設置語言和音色'
-        setting_timbre_desc: '您可自定義語音及音色，用於語音播報，且每次僅可設置一種。'
+        setting_voice: '設置語音'
+        setting_voice_message: '你可以設置語言和音色，若所選語言與模型語言不一致，則無法播放語音'
+        setting_voice_desc: '您可自定義語音及音色，用於語音播報，且每次僅可設置一種。'
+        currently_only_one_voice_can_be_set: '當前僅可設置一種聲音'
 
         memory_variable_modal:
           edit_memory_variable: '編輯記憶變數'

src/utils/web-socket-ctr.ts

@@ -39,7 +39,6 @@ export default class WebSocketCtr {
           this.socket.close()
           this.socket = null
         }
-        window.$message.error(t('common_module.agent_exception'))
 
         this.onMessageError()
 

src/views/personal-space/personal-app-setting/components/agent-config/agent-preview/agent-preview.vue

@@ -5,6 +5,7 @@ import { useI18n } from 'vue-i18n'
 import { useRouter } from 'vue-router'
 import { Emitter } from 'mitt'
 import { Howl } from 'howler'
+import { ValueOf } from 'type-fest'
 import MessageList from './components/message-list.vue'
 import FooterInput from './components/footer-input.vue'
 import MemoryPreviewModal from './components/memory-preview-modal.vue'
@@ -26,15 +27,15 @@ const emitter = inject<Emitter<MittEvents>>('emitter')
 const messageListRef = ref<InstanceType<typeof MessageList> | null>(null)
 const footerInputRef = ref<InstanceType<typeof FooterInput> | null>(null)
 
-const messageList = ref<ConversationMessageItem[]>([])
+const messageList = ref(new Map<string, ConversationMessageItem>())
 
 const continuousQuestionStatus = ref<'default' | 'close'>(personalAppConfigStore.commConfig.continuousQuestionStatus)
 
 const continuousQuestionList = ref<string[]>([])
 const isShowMemoryPreviewModal = ref(false)
 const selectedMemoryTabName = ref('memoryVariable')
-const answerAudioAutoPlaying = ref(personalAppConfigStore.voiceConfig.defaultOpen === 'Y')
-
+const answerAudioAutoPlay = ref(personalAppConfigStore.voiceConfig.defaultOpen === 'Y')
+const answerAudioPlaying = ref(false) // 语音播放中
 const currentPlayMessageItem = ref<ConversationMessageItem | null>(null)
 const currentPlayAudioFragmentSerialNo = ref(0)
 const currentSoundCtl = shallowRef<Howl | null>(null)
@@ -48,7 +49,7 @@ onMounted(() => {
   emitter?.on('resetAgent', () => {
     handleAudioPause()
     footerInputRef.value?.blockMessageResponse()
-    messageList.value = []
+    messageList.value.clear()
   })
 })
 
@@ -56,23 +57,34 @@ onUnmounted(() => {
   emitter?.off('resetAgent')
   handleAudioPause()
   footerInputRef.value?.blockMessageResponse()
-  messageList.value = []
+  messageList.value.clear()
 })
 
-function handleAddMessageItem(messageItem: ConversationMessageItem) {
-  messageList.value.push(messageItem)
+function handleAddMessageItem(messageId: string, messageItem: ConversationMessageItem) {
+  messageList.value.set(messageId, messageItem)
 }
 
-function handleUpdateSpecifyMessageItem(messageItemIndex: number, newObj: Partial<ConversationMessageItem>) {
-  if (messageList.value[messageItemIndex]) {
-    Object.entries(newObj).forEach(([k, v]) => {
-      ;(messageList.value[messageItemIndex] as any)[k as keyof typeof newObj] = v
+function handleUpdateSpecifyMessageItem(messageId: string, newMessageItem: Partial<ConversationMessageItem>) {
+  const currentMessageItemInfo = messageList.value.get(messageId)
+
+  if (currentMessageItemInfo) {
+    const updatePropertyLength = Object.keys(newMessageItem).length
+
+    if (updatePropertyLength > 4) {
+      messageList.value.set(messageId, Object.assign({}, currentMessageItemInfo, newMessageItem))
+      return
+    }
+
+    Object.entries<ValueOf<typeof newMessageItem>>(newMessageItem).forEach(([key, value]) => {
+      if (Object.prototype.hasOwnProperty.call(currentMessageItemInfo, key)) {
+        ;(currentMessageItemInfo as any)[key as keyof ConversationMessageItem] = value
+      }
     })
   }
 }
 
-function handleDeleteLastMessageItem() {
-  messageList.value.pop()
+function handleDeleteLastMessageItem(messageId: string) {
+  messageList.value.delete(messageId)
 }
 
 function handleUpdatePageScroll() {
@@ -88,7 +100,8 @@ function handleClearAllMessage() {
     .then(() => {
       handleAudioPause()
       footerInputRef.value?.blockMessageResponse()
-      messageList.value = []
+      messageList.value.clear()
+      answerAudioPlaying.value = false
       window.$message.success(t('common_module.clear_success_message'))
     })
 }
@@ -135,6 +148,7 @@ function howlSoundFactory(url: string) {
     preload: true,
     autoplay: true,
     onplay: () => {
+      answerAudioPlaying.value = true
       currentSoundCtl.value = soundCtl
 
       if (currentPlayMessageItem.value) {
@@ -152,6 +166,7 @@ function howlSoundFactory(url: string) {
         currentPlayAudioFragmentSerialNo.value > currentPlayMessageItem.value.voiceFragmentUrlList.length - 1
       ) {
         currentPlayMessageItem.value.isVoicePlaying = false
+        answerAudioPlaying.value = false
       }
 
       let audioFragmentUrl = currentPlayMessageItem.value?.voiceFragmentUrlList[currentPlayAudioFragmentSerialNo.value]
@@ -205,9 +220,10 @@ function handleAudioPause(isClearMessageList = false) {
   }
 
   currentPlayMessageItem.value && (currentPlayMessageItem.value.isVoicePlaying = false)
+  answerAudioPlaying.value = false
 
   if (isClearMessageList) {
-    messageList.value = []
+    messageList.value.clear()
     footerInputRef.value?.blockMessageResponse()
   }
 }
@@ -231,11 +247,7 @@ function handleAudioPause(isClearMessageList = false) {
             </template>
             <div class="flex items-center gap-2.5">
               <span>{{ t('common_module.voice_auto_play') }}</span>
-              <n-switch
-                v-model:value="answerAudioAutoPlaying"
-                size="small"
-                @update:value="handleUpdateAudioAutoPlaying"
-              >
+              <n-switch v-model:value="answerAudioAutoPlay" size="small" @update:value="handleUpdateAudioAutoPlaying">
                 <template #checked> {{ t('common_module.open') }} </template>
                 <template #unchecked> {{ t('common_module.close') }} </template>
               </n-switch>
@@ -294,11 +306,11 @@ function handleAudioPause(isClearMessageList = false) {
     </div>
 
     <div class="flex w-full flex-1 overflow-hidden">
-      <div v-show="messageList.length === 0" class="flex w-full">
+      <div v-show="messageList.size === 0" class="flex w-full">
         <Preamble />
       </div>
 
-      <div v-show="messageList.length > 0" class="w-full">
+      <div v-show="messageList.size > 0" class="w-full">
         <MessageList
           ref="messageListRef"
           :message-list="messageList"
@@ -314,7 +326,8 @@ function handleAudioPause(isClearMessageList = false) {
       ref="footerInputRef"
       :continuous-question-status="continuousQuestionStatus"
       :message-list="messageList"
-      :answer-audio-auto-playing="answerAudioAutoPlaying"
+      :answer-audio-auto-play="answerAudioAutoPlay"
+      :answer-audio-playing="answerAudioPlaying"
       @add-message-item="handleAddMessageItem"
       @update-specify-message-item="handleUpdateSpecifyMessageItem"
       @delete-last-message-item="handleDeleteLastMessageItem"
@@ -323,6 +336,7 @@ function handleAudioPause(isClearMessageList = false) {
       @create-continue-questions="handleCreateContinueQuestions"
       @update-continuous-question-status="handleUpdateContinueQuestionStatus"
       @audio-play="handleAudioPlay"
+      @audio-pause="handleAudioPause"
     />
 
     <MemoryPreviewModal v-model="isShowMemoryPreviewModal" :data="selectedMemoryTabName" />

src/views/personal-space/personal-app-setting/components/agent-config/agent-preview/components/footer-input.vue

@@ -2,6 +2,7 @@
 import { computed, inject, onMounted, onUnmounted, ref, useTemplateRef, watch } from 'vue'
 import { Emitter } from 'mitt'
 import { useI18n } from 'vue-i18n'
+import { nanoid } from 'nanoid'
 import OverwriteMessageTipModal from './overwrite-message-tip-modal.vue'
 import { fetchCustomEventSource } from '@/composables/useEventSource'
 import { usePersonalAppConfigStore } from '@/store/modules/personal-app-config'
@@ -11,9 +12,10 @@ import { TEXTTOSPEECH_WS_URL } from '@/config/base-url'
 import WebSocketCtr from '@/utils/web-socket-ctr'
 
 interface Props {
-  messageList: ConversationMessageItem[]
+  messageList: Map<string, ConversationMessageItem>
   continuousQuestionStatus: 'default' | 'close'
-  answerAudioAutoPlaying: boolean
+  answerAudioAutoPlay: boolean
+  answerAudioPlaying: boolean
 }
 
 const { t } = useI18n()
@@ -21,14 +23,15 @@ const { t } = useI18n()
 const props = defineProps<Props>()
 
 const emit = defineEmits<{
-  addMessageItem: [value: ConversationMessageItem]
-  updateSpecifyMessageItem: [messageItemIndex: number, newObj: Partial<ConversationMessageItem>]
-  deleteLastMessageItem: []
+  addMessageItem: [messageId: string, value: ConversationMessageItem]
+  updateSpecifyMessageItem: [messageId: string, newObj: Partial<ConversationMessageItem>]
+  deleteLastMessageItem: [messageId: string]
   updatePageScroll: []
   clearAllMessage: []
   createContinueQuestions: [value: string]
   updateContinuousQuestionStatus: [value: 'default' | 'close']
   audioPlay: [messageItem: ConversationMessageItem, requestId?: string]
+  audioPause: []
 }>()
 
 const personalAppConfigStore = usePersonalAppConfigStore()
@@ -45,6 +48,10 @@ const currentReplyContentSentenceExtractIndex = ref(0)
 const sentenceFragmentSerialNo = ref(0)
 const sentenceExtractCheckEnabled = ref(false)
 const assistantFullAnswerContent = ref('')
+const currentAgentTimberId = ref('')
+const sentenceSpeechException = ref(false)
+const messageAudioLoading = ref(false)
+const currentLatestMessageItemKeyMap = ref(new Map<'assistant' | 'user', string>())
 
 let controller: AbortController | null = null
 
@@ -57,7 +64,7 @@ const isCreateContinueQuestions = computed(() => {
 })
 
 const isAllowClearMessage = computed(() => {
-  return props.messageList.length > 0
+  return props.messageList.size > 0
 })
 
 const isSendBtnDisabled = computed(() => {
@@ -121,7 +128,24 @@ function messageItemFactory(): ConversationMessageItem {
 }
 
 function handleMessageSend() {
-  if (!inputMessageContent.value.trim() || isAnswerResponseWait.value || isInputMessageDisabled.value) return ''
+  if (!inputMessageContent.value.trim() || isInputMessageDisabled.value) {
+    return
+  }
+
+  if (isAnswerResponseWait.value || messageAudioLoading.value) {
+    window.$message.warning(t('common_module.dialogue_module.do_not_operate_until_the_reply_is_complete'))
+    return
+  }
+
+  if (props.answerAudioPlaying) {
+    window.$message.warning(t('common_module.dialogue_module.stop_playing_and_then_operate'))
+    return
+  }
+
+  const latestUserMessageKey = nanoid()
+  const latestAssistantMessageKey = nanoid()
+  currentLatestMessageItemKeyMap.value.set('user', latestUserMessageKey)
+  currentLatestMessageItemKeyMap.value.set('assistant', latestAssistantMessageKey)
 
   const messages: {
     content: {
@@ -135,7 +159,7 @@ function handleMessageSend() {
   }[] = []
 
   emit('updateContinuousQuestionStatus', personalAppConfigStore.commConfig.continuousQuestionStatus)
-  emit('addMessageItem', { ...messageItemFactory(), textContent: inputMessageContent.value })
+  emit('addMessageItem', latestUserMessageKey, { ...messageItemFactory(), textContent: inputMessageContent.value })
   emit('updatePageScroll')
 
   props.messageList.forEach((messageItem) => {
@@ -160,8 +184,11 @@ function handleMessageSend() {
   sentenceFragmentSerialNo.value = 0
   sentenceExtractCheckEnabled.value = false
   assistantFullAnswerContent.value = ''
+  currentAgentTimberId.value = personalAppConfigStore.voiceConfig.timbreId
+  sentenceSpeechException.value = false
+  messageAudioLoading.value = false
 
-  emit('addMessageItem', {
+  emit('addMessageItem', latestAssistantMessageKey, {
     ...messageItemFactory(),
     role: 'assistant',
     isTextContentLoading: true,
@@ -170,7 +197,6 @@ function handleMessageSend() {
   })
   emit('updatePageScroll')
 
-  const currentMessageIndex = props.messageList.length - 1
   let replyTextContent = ''
 
   controller = new AbortController()
@@ -185,17 +211,12 @@ function handleMessageSend() {
     controller,
     onMessage: (data: any) => {
       if (data === '[DONE]') {
-        emit('updateSpecifyMessageItem', currentMessageIndex, {
+        emit('updateSpecifyMessageItem', latestAssistantMessageKey, {
           isEmptyContent: !replyTextContent,
           isTextContentLoading: false,
           isAnswerResponseLoading: false,
         })
 
-        if (!props.answerAudioAutoPlaying) {
-          emit('updateSpecifyMessageItem', currentMessageIndex, {
-            isVoiceLoading: false,
-          })
-        }
         isCreateContinueQuestions.value && emit('createContinueQuestions', replyTextContent)
         emit('updatePageScroll')
         blockMessageResponse()
@@ -210,9 +231,13 @@ function handleMessageSend() {
           '',
         )
 
-        !sentenceExtractCheckEnabled.value && isEnableVoice.value && sentenceExtract()
+        if (!sentenceExtractCheckEnabled.value && isEnableVoice.value) {
+          sentenceExtract(latestAssistantMessageKey)
+          sentenceExtractCheckEnabled.value = true
+          messageAudioLoading.value = true
+        }
 
-        emit('updateSpecifyMessageItem', currentMessageIndex, {
+        emit('updateSpecifyMessageItem', latestAssistantMessageKey, {
           textContent: replyTextContent,
           isTextContentLoading: false,
         })
@@ -232,12 +257,13 @@ function handleMessageSend() {
 }
 
 function errorMessageResponse() {
-  emit('updateSpecifyMessageItem', props.messageList.length - 1, {
+  emit('updateSpecifyMessageItem', currentLatestMessageItemKeyMap.value.get('assistant')!, {
     isTextContentLoading: false,
     textContent: '',
   })
-  emit('deleteLastMessageItem')
-  emit('deleteLastMessageItem')
+  emit('deleteLastMessageItem', currentLatestMessageItemKeyMap.value.get('assistant')!)
+  emit('deleteLastMessageItem', currentLatestMessageItemKeyMap.value.get('assistant')!)
+  emit('audioPause')
   blockMessageResponse()
 }
 
@@ -263,18 +289,16 @@ function handleSelectFile(cb: () => void) {
   cb()
 }
 
-function sentenceExtract() {
-  sentenceExtractCheckEnabled.value = true
-
+function sentenceExtract(messageId: string) {
   const symbolRegExp = /[。！？；.!?;]/g
   let sentenceDraft = assistantFullAnswerContent.value
-    .replace(/\s/gi, '')
+    .replace(/\s{5,}/gi, '')
     .slice(currentReplyContentSentenceExtractIndex.value)
 
   let matchResult = symbolRegExp.exec(sentenceDraft)
 
   function matchExtract() {
-    const article = assistantFullAnswerContent.value.replace(/\s/gi, '')
+    const article = assistantFullAnswerContent.value.replace(/\s{5,}/gi, '')
 
     if (matchResult && matchResult.index && matchResult.index > 60) {
       sentenceDraft = article.slice(
@@ -284,7 +308,7 @@ function sentenceExtract() {
 
       currentReplyContentSentenceExtractIndex.value += sentenceDraft.length
 
-      ttsSocketSendText(sentenceDraft, sentenceFragmentSerialNo.value)
+      ttsSocketSendText(sentenceDraft, sentenceFragmentSerialNo.value, messageId)
       sentenceFragmentSerialNo.value += 1
 
       if (article.length - currentReplyContentSentenceExtractIndex.value > 60) {
@@ -292,19 +316,19 @@ function sentenceExtract() {
         matchResult = symbolRegExp.exec(sentenceDraft)
         matchExtract()
       } else {
-        setTimeout(() => sentenceExtract(), 600)
+        setTimeout(() => sentenceExtract(messageId), 600)
       }
     } else if (!isAnswerResponseWait.value) {
       /* 延时避免最后回复内容没有更新全 */
       setTimeout(() => {
         sentenceDraft = article.slice(currentReplyContentSentenceExtractIndex.value)
 
-        ttsSocketSendText(sentenceDraft, sentenceFragmentSerialNo.value)
+        ttsSocketSendText(sentenceDraft, sentenceFragmentSerialNo.value, messageId)
         sentenceFragmentSerialNo.value += 1
       }, 700)
     } else {
       sentenceDraft = assistantFullAnswerContent.value
-        .replace(/\s/gi, '')
+        .replace(/\s{5,}/gi, '')
         .slice(currentReplyContentSentenceExtractIndex.value)
       matchResult = symbolRegExp.exec(sentenceDraft)
 
@@ -313,38 +337,52 @@ function sentenceExtract() {
   }
 
   if (matchResult) matchExtract()
-  else setTimeout(() => sentenceExtract(), 600)
+  else setTimeout(() => sentenceExtract(messageId), 600)
 }
 
-function ttsSocketSendText(text: string, audioUrlSerialNo: number) {
+function ttsSocketSendText(text: string, audioUrlSerialNo: number, messageId: string) {
+  if (sentenceSpeechException.value) {
+    return
+  }
+
   const ttsSocketCtl = new WebSocketCtr(TEXTTOSPEECH_WS_URL)
 
   ttsSocketCtl.onMessage = (data: { audio: string; replyVoiceUrl: string; final: boolean }) => {
     if (data.replyVoiceUrl) {
-      const currentMessageIndex = props.messageList.length - 1
-
-      if (props.messageList[currentMessageIndex]?.voiceFragmentUrlList) {
-        const voiceFragmentUrlListDraft = [...props.messageList[currentMessageIndex].voiceFragmentUrlList]
+      if (props.messageList.get(messageId)?.voiceFragmentUrlList) {
+        const voiceFragmentUrlListDraft = [...props.messageList.get(messageId)!.voiceFragmentUrlList]
         voiceFragmentUrlListDraft[audioUrlSerialNo] = data.replyVoiceUrl
+        messageAudioLoading.value = false
+
+        emit('updateSpecifyMessageItem', messageId, { voiceFragmentUrlList: voiceFragmentUrlListDraft })
 
-        emit('updateSpecifyMessageItem', currentMessageIndex, { voiceFragmentUrlList: voiceFragmentUrlListDraft })
+        if (props.answerAudioAutoPlay && audioUrlSerialNo === 0 && voiceFragmentUrlListDraft[audioUrlSerialNo]) {
+          emit('audioPlay', props.messageList.get(messageId)!)
+        }
 
-        if (props.answerAudioAutoPlaying && audioUrlSerialNo === 0 && voiceFragmentUrlListDraft[audioUrlSerialNo]) {
-          emit('audioPlay', props.messageList[currentMessageIndex])
+        if (!props.answerAudioAutoPlay) {
+          emit('updateSpecifyMessageItem', messageId, { isVoiceLoading: false })
         }
       }
     }
   }
 
+  ttsSocketCtl.onMessageError = () => {
+    emit('updateSpecifyMessageItem', messageId, { isVoiceLoading: false })
+    sentenceSpeechException.value = true
+    messageAudioLoading.value = false
+    window.$message.error(t('common_module.unplayable_tip'))
+  }
+
   const content = (text || '').replace(/\^\[[\d\\[\]-]+?\]\^/g, '')
 
-  if (content) {
+  if (content && currentAgentTimberId.value) {
     ttsSocketCtl.connect(() => {
       ttsSocketCtl.send({
         codec: 'wav',
         sampleRate: 16000,
         speed: 0,
-        voiceType: personalAppConfigStore.voiceConfig.timbreId,
+        voiceType: currentAgentTimberId.value,
         volume: 0,
         content,
       })
@@ -417,7 +455,13 @@ defineExpose({
           <div
             class="bg-px-send-png absolute bottom-2 right-[20px] h-[24px] w-[24px]"
             :class="
-              isSendBtnDisabled || isAnswerResponseWait || isInputMessageDisabled ? 'opacity-60' : 'cursor-pointer'
+              isSendBtnDisabled ||
+              isAnswerResponseWait ||
+              isInputMessageDisabled ||
+              answerAudioPlaying ||
+              messageAudioLoading
+                ? 'opacity-60'
+                : 'cursor-pointer'
             "
             @click="handleMessageSend"
           />

src/views/personal-space/personal-app-setting/components/agent-config/agent-preview/components/message-item.vue

@@ -31,13 +31,27 @@ const assistantAvatar = computed(() => {
   return personalAppConfigStore.baseInfo.agentAvatar
 })
 
+const timbreEnabled = computed(() => {
+  return !!personalAppConfigStore.voiceConfig.timbreId
+})
+
 const isShowAudioControl = computed(() => {
-  return (
-    props.role === 'assistant' && !props.messageItem.isVoiceLoading && !!props.messageItem.voiceFragmentUrlList.length
-  )
+  return props.role === 'assistant' && !props.messageItem.isVoiceLoading
+})
+
+const isPlayableAudio = computed(() => {
+  return isShowAudioControl.value && !!props.messageItem.voiceFragmentUrlList.length
+})
+
+const isShowVoiceLoading = computed(() => {
+  return props.role === 'assistant' && props.messageItem.isVoiceLoading && timbreEnabled.value
 })
 
 function handleAudioControl() {
+  if (!isPlayableAudio.value) {
+    return
+  }
+
   if (props.messageItem.isVoicePlaying) {
     emit('audioPause')
   } else {
@@ -86,14 +100,30 @@ function handleAudioControl() {
 
       <div
         v-show="isShowAudioControl"
-        class="hover:text-theme-color text-font-color flex cursor-pointer items-center gap-0.5 hover:opacity-80"
+        class="text-font-color flex items-center gap-0.5"
+        :class="isPlayableAudio ? 'hover:text-theme-color cursor-pointer hover:opacity-80' : 'cursor-not-allowed'"
         @click="handleAudioControl"
       >
         <i v-if="!messageItem.isVoicePlaying" class="iconfont icon-play text-[24px]" />
         <div v-else class="mx-1.5 my-3 h-[12px] w-[12px] bg-[url(@/assets/images/playing.gif)] bg-[length:100%_100%]" />
-        <span class="text-[12px]" :class="messageItem.isVoicePlaying ? 'text-theme-color' : ''">
+
+        <span
+          v-show="isPlayableAudio"
+          class="text-[12px]"
+          :class="messageItem.isVoicePlaying ? 'text-theme-color' : ''"
+        >
           {{ messageItem.isVoicePlaying ? t('common_module.stop_playing') : t('common_module.start_playing') }}
         </span>
+        <n-popover style="max-width: 310px">
+          <template #trigger>
+            <span v-show="!isPlayableAudio" class="text-[12px]"> {{ t('common_module.unplayable') }} </span>
+          </template>
+          {{ t('common_module.unplayable_tip') }}
+        </n-popover>
+      </div>
+
+      <div v-if="isShowVoiceLoading" class="py-3.5 pl-6">
+        <CustomLoading />
       </div>
     </div>
   </div>

src/views/personal-space/personal-app-setting/components/agent-config/agent-preview/components/message-list.vue

@@ -5,7 +5,7 @@ import ContinueQuestion from './continue-question.vue'
 import { useScroll } from '@/composables/useScroll'
 
 interface Props {
-  messageList: ConversationMessageItem[]
+  messageList: Map<string, ConversationMessageItem>
   continuousQuestionStatus: 'default' | 'close'
   continuousQuestionList: string[]
 }
@@ -22,8 +22,8 @@ const { scrollRef, scrollToBottom } = useScroll()
 const isShowContinueQuestion = computed(() => {
   return (
     props.continuousQuestionStatus === 'default' &&
-    props.messageList.length > 1 &&
-    !props.messageList[props.messageList.length - 1].isAnswerResponseLoading
+    props.messageList.size > 1 &&
+    !Array.from(props.messageList.entries()).pop()?.[1].isAnswerResponseLoading
   )
 })
 
@@ -36,8 +36,8 @@ defineExpose({
   <main ref="scrollRef" class="h-full overflow-y-auto px-5">
     <div>
       <MessageItem
-        v-for="messageItem in messageList"
-        :key="messageItem.timestamp"
+        v-for="[key, messageItem] in messageList"
+        :key="key"
         :role="messageItem.role"
         :message-item="messageItem"
         @audio-play="() => $emit('audioPlay', messageItem)"

src/views/personal-space/personal-app-setting/components/agent-config/agent-setting/components/agent-role-setting.vue

@@ -31,6 +31,8 @@ const timberFullName = computed(() => {
   return `${timbreInfoDetail.value?.timbreInfo?.[0]?.timbreName || '--'}(${t(currentLanguage?.label || 'common_module.sound')})`
 })
 
+const isHasTimbreId = computed(() => !!voiceConfig.value.timbreId)
+
 onMounted(() => {
   voiceConfig.value.timbreId && handleGetTimberInfoDetail()
 })
@@ -49,6 +51,10 @@ function handleUpdateRoleConfigExpandedNames(expandedNames: string[]) {
 }
 
 function handleShowAssociatedTimbreModel() {
+  if (voiceConfig.value.timbreId) {
+    return
+  }
+
   isShowTimbreSettingModal.value = true
   timbreInfo = { language: 0, matchLang: '', timbreInfo: [] }
 }
@@ -84,7 +90,7 @@ function handleEditAssociatedTimbreModel() {
           <RightOne theme="filled" size="17" fill="#333" :stroke-width="3" />
         </template>
 
-        <NCollapseItem :title="t('common_module.sound')" name="timbre" class="my-[13px]!">
+        <NCollapseItem :title="t('common_module.voice')" name="timbre" class="my-[13px]!">
           <template #header-extra>
             <NTooltip trigger="hover">
               <template #trigger>
@@ -92,16 +98,23 @@ function handleEditAssociatedTimbreModel() {
                   theme="outline"
                   size="22"
                   :stroke-width="3"
-                  class="text-theme-color cursor-pointer"
+                  class="text-theme-color"
+                  :class="isHasTimbreId ? 'cursor-not-allowed' : 'cursor-pointer'"
                   @click="handleShowAssociatedTimbreModel"
                 />
               </template>
-              {{ t('personal_space_module.agent_module.agent_setting_module.agent_config_module.setting_timbre') }}
+              {{
+                isHasTimbreId
+                  ? t(
+                      'personal_space_module.agent_module.agent_setting_module.agent_config_module.currently_only_one_voice_can_be_set',
+                    )
+                  : t('personal_space_module.agent_module.agent_setting_module.agent_config_module.setting_voice')
+              }}
             </NTooltip>
           </template>
 
           <span v-show="!voiceConfig.timbreId" class="text-xs text-[#84868c]">
-            {{ t('personal_space_module.agent_module.agent_setting_module.agent_config_module.setting_timbre_desc') }}
+            {{ t('personal_space_module.agent_module.agent_setting_module.agent_config_module.setting_voice_desc') }}
           </span>
 
           <div class="flex flex-1 flex-wrap items-center gap-[12px] overflow-hidden">
@@ -153,7 +166,7 @@ function handleEditAssociatedTimbreModel() {
   <TimbreSettingModal
     v-model:is-show-modal="isShowTimbreSettingModal"
     :btn-loading="false"
-    :modal-title="t('personal_space_module.agent_module.agent_setting_module.agent_config_module.setting_timbre')"
+    :modal-title="t('personal_space_module.agent_module.agent_setting_module.agent_config_module.setting_voice')"
     :timbre-info="timbreInfo"
     @confirm="handleUpdateTimbreId"
   />

src/views/personal-space/personal-app-setting/components/agent-config/agent-setting/components/timbre-setting-modal.vue

@@ -215,7 +215,7 @@ function handleAudioPause() {
   >
     <template #content>
       <div class="text-gray-font-color mb-2">
-        {{ t('personal_space_module.agent_module.agent_setting_module.agent_config_module.setting_timbre_message') }}
+        {{ t('personal_space_module.agent_module.agent_setting_module.agent_config_module.setting_voice_message') }}
       </div>
 
       <div v-show="!requestDataLoading" class="flex items-center gap-3">

src/views/share/components/footer-input.vue

@@ -2,6 +2,7 @@
 import { computed, inject, onMounted, onUnmounted, ref } from 'vue'
 import { Emitter } from 'mitt'
 import { useI18n } from 'vue-i18n'
+import { nanoid } from 'nanoid'
 import { fetchCustomEventSource } from '@/composables/useEventSource'
 import { useUserStore } from '@/store/modules/user'
 import { UploadStatus } from '@/enums/upload-status'
@@ -13,11 +14,12 @@ import WebSocketCtr from '@/utils/web-socket-ctr'
 interface Props {
   agentId: string
   dialogsId: string
-  messageList: ConversationMessageItem[]
+  messageList: Map<string, ConversationMessageItem>
   continuousQuestionStatus: 'default' | 'close'
   isEnableDocumentParse: boolean
   isEnableVoice: boolean
-  answerAudioAutoPlaying: boolean
+  answerAudioAutoPlay: boolean
+  answerAudioPlaying: boolean
   timbreId: string
 }
 
@@ -26,15 +28,16 @@ const { t } = useI18n()
 const props = defineProps<Props>()
 
 const emit = defineEmits<{
-  addMessageItem: [value: ConversationMessageItem]
-  updateSpecifyMessageItem: [messageItemIndex: number, newObj: Partial<ConversationMessageItem>]
-  deleteLastMessageItem: []
+  addMessageItem: [messageId: string, value: ConversationMessageItem]
+  updateSpecifyMessageItem: [messageId: string, newObj: Partial<ConversationMessageItem>]
+  deleteLastMessageItem: [messageId: string]
   updatePageScroll: []
   clearAllMessage: []
   toLogin: []
   createContinueQuestions: [value: string]
   resetContinueQuestionList: []
   audioPlay: [messageItem: ConversationMessageItem, requestId?: string]
+  audioPause: []
 }>()
 
 const { isMobile } = useLayoutConfig()
@@ -51,6 +54,9 @@ const currentReplyContentSentenceExtractIndex = ref(0)
 const sentenceFragmentSerialNo = ref(0)
 const sentenceExtractCheckEnabled = ref(false)
 const assistantFullAnswerContent = ref('')
+const sentenceSpeechException = ref(false)
+const messageAudioLoading = ref(false)
+const currentLatestMessageItemKeyMap = ref(new Map<'assistant' | 'user', string>())
 
 let controller: AbortController | null = null
 
@@ -59,7 +65,7 @@ const isLogin = computed(() => {
 })
 
 const isAllowClearMessage = computed(() => {
-  return props.messageList.length > 0
+  return props.messageList.size > 0
 })
 
 const isSendBtnDisabled = computed(() => {
@@ -123,13 +129,30 @@ function handleMessageSend() {
     return
   }
 
-  if (!inputMessageContent.value.trim() || isAnswerResponseWait.value || isInputMessageDisabled.value) return ''
+  if (!inputMessageContent.value.trim() || isInputMessageDisabled.value) {
+    return
+  }
+
+  if (isAnswerResponseWait.value || messageAudioLoading.value) {
+    window.$message.warning(t('common_module.dialogue_module.do_not_operate_until_the_reply_is_complete'))
+    return
+  }
+
+  if (props.answerAudioPlaying) {
+    window.$message.warning(t('common_module.dialogue_module.stop_playing_and_then_operate'))
+    return
+  }
+
+  const latestUserMessageKey = nanoid()
+  const latestAssistantMessageKey = nanoid()
+  currentLatestMessageItemKeyMap.value.set('user', latestUserMessageKey)
+  currentLatestMessageItemKeyMap.value.set('assistant', latestAssistantMessageKey)
 
   emit('resetContinueQuestionList')
-  emit('addMessageItem', { ...messageItemFactory(), textContent: inputMessageContent.value })
+  emit('addMessageItem', latestUserMessageKey, { ...messageItemFactory(), textContent: inputMessageContent.value })
   emit('updatePageScroll')
 
-  emit('addMessageItem', {
+  emit('addMessageItem', latestAssistantMessageKey, {
     ...messageItemFactory(),
     role: 'assistant',
     isTextContentLoading: true,
@@ -139,7 +162,7 @@ function handleMessageSend() {
   emit('updatePageScroll')
 
   const input = inputMessageContent.value
-  const currentMessageIndex = props.messageList.length - 1
+
   let replyTextContent = ''
   isAnswerResponseWait.value = true
   inputMessageContent.value = ''
@@ -147,6 +170,8 @@ function handleMessageSend() {
   sentenceFragmentSerialNo.value = 0
   sentenceExtractCheckEnabled.value = false
   assistantFullAnswerContent.value = ''
+  sentenceSpeechException.value = false
+  messageAudioLoading.value = false
 
   controller = new AbortController()
 
@@ -161,18 +186,12 @@ function handleMessageSend() {
     controller,
     onMessage: (data: any) => {
       if (data === '[DONE]') {
-        emit('updateSpecifyMessageItem', currentMessageIndex, {
+        emit('updateSpecifyMessageItem', latestAssistantMessageKey, {
           isEmptyContent: !replyTextContent,
           isTextContentLoading: false,
           isAnswerResponseLoading: false,
         })
 
-        if (!props.answerAudioAutoPlaying) {
-          emit('updateSpecifyMessageItem', currentMessageIndex, {
-            isVoiceLoading: false,
-          })
-        }
-
         isCreateContinueQuestions.value && emit('createContinueQuestions', replyTextContent)
         emit('updatePageScroll')
         blockMessageResponse()
@@ -187,9 +206,13 @@ function handleMessageSend() {
           '',
         )
 
-        !sentenceExtractCheckEnabled.value && props.isEnableVoice && sentenceExtract()
+        if (!sentenceExtractCheckEnabled.value && props.isEnableVoice) {
+          sentenceExtract(latestAssistantMessageKey)
+          sentenceExtractCheckEnabled.value = true
+          messageAudioLoading.value = true
+        }
 
-        emit('updateSpecifyMessageItem', currentMessageIndex, {
+        emit('updateSpecifyMessageItem', latestAssistantMessageKey, {
           textContent: replyTextContent,
           isTextContentLoading: false,
         })
@@ -209,12 +232,13 @@ function handleMessageSend() {
 }
 
 function errorMessageResponse() {
-  emit('updateSpecifyMessageItem', props.messageList.length - 1, {
+  emit('updateSpecifyMessageItem', currentLatestMessageItemKeyMap.value.get('assistant')!, {
     isTextContentLoading: false,
     textContent: '',
   })
-  emit('deleteLastMessageItem')
-  emit('deleteLastMessageItem')
+  emit('deleteLastMessageItem', currentLatestMessageItemKeyMap.value.get('assistant')!)
+  emit('deleteLastMessageItem', currentLatestMessageItemKeyMap.value.get('assistant')!)
+  emit('audioPause')
   blockMessageResponse()
 }
 
@@ -244,18 +268,16 @@ function handleSelectFile(cb: () => void) {
   cb()
 }
 
-function sentenceExtract() {
-  sentenceExtractCheckEnabled.value = true
-
+function sentenceExtract(messageId: string) {
   const symbolRegExp = /[。！？；.!?;]/g
   let sentenceDraft = assistantFullAnswerContent.value
-    .replace(/\s/gi, '')
+    .replace(/\s{5,}/gi, '')
     .slice(currentReplyContentSentenceExtractIndex.value)
 
   let matchResult = symbolRegExp.exec(sentenceDraft)
 
   function matchExtract() {
-    const article = assistantFullAnswerContent.value.replace(/\s/gi, '')
+    const article = assistantFullAnswerContent.value.replace(/\s{5,}/gi, '')
 
     if (matchResult && matchResult.index && matchResult.index > 60) {
       sentenceDraft = article.slice(
@@ -265,7 +287,7 @@ function sentenceExtract() {
 
       currentReplyContentSentenceExtractIndex.value += sentenceDraft.length
 
-      ttsSocketSendText(sentenceDraft, sentenceFragmentSerialNo.value)
+      ttsSocketSendText(sentenceDraft, sentenceFragmentSerialNo.value, messageId)
       sentenceFragmentSerialNo.value += 1
 
       if (article.length - currentReplyContentSentenceExtractIndex.value > 60) {
@@ -273,19 +295,19 @@ function sentenceExtract() {
         matchResult = symbolRegExp.exec(sentenceDraft)
         matchExtract()
       } else {
-        setTimeout(() => sentenceExtract(), 600)
+        setTimeout(() => sentenceExtract(messageId), 600)
       }
     } else if (!isAnswerResponseWait.value) {
       /* 延时避免最后回复内容没有更新全 */
       setTimeout(() => {
         sentenceDraft = article.slice(currentReplyContentSentenceExtractIndex.value)
 
-        ttsSocketSendText(sentenceDraft, sentenceFragmentSerialNo.value)
+        ttsSocketSendText(sentenceDraft, sentenceFragmentSerialNo.value, messageId)
         sentenceFragmentSerialNo.value += 1
       }, 700)
     } else {
       sentenceDraft = assistantFullAnswerContent.value
-        .replace(/\s/gi, '')
+        .replace(/\s{5,}/gi, '')
         .slice(currentReplyContentSentenceExtractIndex.value)
       matchResult = symbolRegExp.exec(sentenceDraft)
 
@@ -294,32 +316,46 @@ function sentenceExtract() {
   }
 
   if (matchResult) matchExtract()
-  else setTimeout(() => sentenceExtract(), 600)
+  else setTimeout(() => sentenceExtract(messageId), 600)
 }
 
-function ttsSocketSendText(text: string, audioUrlSerialNo: number) {
+function ttsSocketSendText(text: string, audioUrlSerialNo: number, messageId: string) {
+  if (sentenceSpeechException.value) {
+    return
+  }
+
   const ttsSocketCtl = new WebSocketCtr(TEXTTOSPEECH_WS_URL)
 
   ttsSocketCtl.onMessage = (data: { audio: string; replyVoiceUrl: string; final: boolean }) => {
     if (data.replyVoiceUrl) {
-      const currentMessageIndex = props.messageList.length - 1
-
-      if (props.messageList[currentMessageIndex]?.voiceFragmentUrlList) {
-        const voiceFragmentUrlListDraft = [...props.messageList[currentMessageIndex].voiceFragmentUrlList]
+      if (props.messageList.get(messageId)?.voiceFragmentUrlList) {
+        const voiceFragmentUrlListDraft = [...props.messageList.get(messageId)!.voiceFragmentUrlList]
         voiceFragmentUrlListDraft[audioUrlSerialNo] = data.replyVoiceUrl
+        messageAudioLoading.value = false
+
+        emit('updateSpecifyMessageItem', messageId, { voiceFragmentUrlList: voiceFragmentUrlListDraft })
 
-        emit('updateSpecifyMessageItem', currentMessageIndex, { voiceFragmentUrlList: voiceFragmentUrlListDraft })
+        if (props.answerAudioAutoPlay && audioUrlSerialNo === 0 && voiceFragmentUrlListDraft[audioUrlSerialNo]) {
+          emit('audioPlay', props.messageList.get(messageId)!)
+        }
 
-        if (props.answerAudioAutoPlaying && audioUrlSerialNo === 0 && voiceFragmentUrlListDraft[audioUrlSerialNo]) {
-          emit('audioPlay', props.messageList[currentMessageIndex])
+        if (!props.answerAudioAutoPlay) {
+          emit('updateSpecifyMessageItem', messageId, { isVoiceLoading: false })
         }
       }
     }
   }
 
+  ttsSocketCtl.onMessageError = () => {
+    emit('updateSpecifyMessageItem', messageId, { isVoiceLoading: false })
+    sentenceSpeechException.value = true
+    messageAudioLoading.value = false
+    window.$message.error(t('common_module.unplayable_tip'))
+  }
+
   const content = (text || '').replace(/\^\[[\d\\[\]-]+?\]\^/g, '')
 
-  if (content) {
+  if (content && props.timbreId) {
     ttsSocketCtl.connect(() => {
       ttsSocketCtl.send({
         codec: 'wav',
@@ -400,7 +436,12 @@ defineExpose({
           <div
             class="bg-px-send-png absolute bottom-2 right-[20px] h-[24px] w-[24px]"
             :class="
-              isSendBtnDisabled || isAnswerResponseWait || !isLogin || isInputMessageDisabled
+              isSendBtnDisabled ||
+              isAnswerResponseWait ||
+              !isLogin ||
+              isInputMessageDisabled ||
+              answerAudioPlaying ||
+              messageAudioLoading
                 ? 'opacity-60'
                 : 'cursor-pointer'
             "

src/views/share/components/message-item.vue

@@ -38,25 +38,27 @@ const assistantAvatar = computed(() => {
   )
 })
 
-const isShowWebAudioControl = computed(() => {
-  return (
-    props.role === 'assistant' &&
-    !props.messageItem.isVoiceLoading &&
-    !isMobile.value &&
-    !!props.messageItem.voiceFragmentUrlList.length
-  )
+const timbreEnabled = computed(() => {
+  return !!props.agentApplicationConfig.voiceConfig.timbreId
 })
 
-const isShowMobileAudioControl = computed(() => {
-  return (
-    props.role === 'assistant' &&
-    !props.messageItem.isVoiceLoading &&
-    isMobile.value &&
-    !!props.messageItem.voiceFragmentUrlList.length
-  )
+const isShowAudioControl = computed(() => {
+  return props.role === 'assistant' && !props.messageItem.isVoiceLoading && timbreEnabled.value
+})
+
+const isPlayableAudio = computed(() => {
+  return isShowAudioControl.value && !!props.messageItem.voiceFragmentUrlList.length
+})
+
+const isShowWebVoiceLoading = computed(() => {
+  return props.role === 'assistant' && !isMobile.value && props.messageItem.isVoiceLoading && timbreEnabled.value
 })
 
 function handleAudioControl() {
+  if (!isPlayableAudio.value) {
+    return
+  }
+
   if (props.messageItem.isVoicePlaying) {
     emit('audioPause')
   } else {
@@ -107,32 +109,60 @@ function handleAudioControl() {
             />
           </p>
 
-          <div v-show="role === 'assistant' && messageItem.isAnswerResponseLoading" class="mb-[5px] mt-4 px-4">
+          <div
+            v-show="
+              role === 'assistant' && (messageItem.isAnswerResponseLoading || (isMobile && messageItem.isVoiceLoading))
+            "
+            class="mb-[5px] mt-4 px-4"
+          >
             <CustomLoading />
           </div>
         </div>
 
-        <div v-show="isShowMobileAudioControl" class="mt-[13px] flex items-center gap-2">
+        <div v-show="isShowAudioControl && isMobile" class="mt-[13px] flex items-center gap-2">
           <div
             class="h-[18px] w-[18px] cursor-pointer"
             :class="messageItem.isVoicePlaying ? 'bg-svg-pause' : 'bg-svg-play'"
             @click="handleAudioControl"
           />
 
-          <MusicWavesLoading v-show="messageItem.isVoicePlaying" bar-bg-color="#333" />
+          <MusicWavesLoading v-show="messageItem.isVoicePlaying && isPlayableAudio" bar-bg-color="#333" />
+
+          <n-popover style="max-width: 310px">
+            <template #trigger>
+              <span v-show="!isPlayableAudio" class="text-[12px]"> {{ t('common_module.unplayable') }} </span>
+            </template>
+            {{ t('common_module.unplayable_tip') }}
+          </n-popover>
         </div>
       </div>
 
       <div
-        v-show="isShowWebAudioControl"
-        class="hover:text-theme-color text-font-color flex cursor-pointer items-center gap-0.5 hover:opacity-80"
+        v-show="isShowAudioControl && !isMobile"
+        class="text-font-color flex items-center gap-0.5"
+        :class="isPlayableAudio ? 'hover:text-theme-color cursor-pointer hover:opacity-80' : 'cursor-not-allowed'"
         @click="handleAudioControl"
       >
         <i v-if="!messageItem.isVoicePlaying" class="iconfont icon-play text-[24px]" />
         <div v-else class="mx-1.5 my-3 h-[12px] w-[12px] bg-[url(@/assets/images/playing.gif)] bg-[length:100%_100%]" />
-        <span class="text-[12px]" :class="messageItem.isVoicePlaying ? 'text-theme-color' : ''">
+
+        <span
+          v-show="isPlayableAudio"
+          class="text-[12px]"
+          :class="messageItem.isVoicePlaying ? 'text-theme-color' : ''"
+        >
           {{ messageItem.isVoicePlaying ? t('common_module.stop_playing') : t('common_module.start_playing') }}
         </span>
+        <n-popover style="max-width: 310px">
+          <template #trigger>
+            <span v-show="!isPlayableAudio" class="text-[12px]"> {{ t('common_module.unplayable') }} </span>
+          </template>
+          {{ t('common_module.unplayable_tip') }}
+        </n-popover>
+      </div>
+
+      <div v-if="isShowWebVoiceLoading" class="py-3.5 pl-5">
+        <CustomLoading />
       </div>
     </div>
   </div>

src/views/share/components/message-list.vue

@@ -6,7 +6,7 @@ import { PersonalAppConfigState } from '@/store/types/personal-app-config'
 import { computed } from 'vue'
 
 interface Props {
-  messageList: ConversationMessageItem[]
+  messageList: Map<string, ConversationMessageItem>
   agentApplicationConfig: PersonalAppConfigState
   continuousQuestionStatus: 'default' | 'close'
   continuousQuestionList: string[]
@@ -24,8 +24,8 @@ const { scrollRef, scrollToBottom } = useScroll()
 const isShowContinueQuestion = computed(() => {
   return (
     props.continuousQuestionStatus === 'default' &&
-    props.messageList.length > 1 &&
-    !props.messageList[props.messageList.length - 1].isAnswerResponseLoading
+    props.messageList.size > 1 &&
+    !Array.from(props.messageList.entries()).pop()?.[1].isAnswerResponseLoading
   )
 })
 
@@ -38,8 +38,8 @@ defineExpose({
   <main ref="scrollRef" class="h-full overflow-y-auto px-5">
     <div>
       <MessageItem
-        v-for="messageItem in messageList"
-        :key="messageItem.timestamp"
+        v-for="[key, messageItem] in messageList"
+        :key="key"
         :role="messageItem.role"
         :message-item="messageItem"
         :agent-application-config="agentApplicationConfig"

src/views/share/components/mobile-page-header.vue

@@ -38,7 +38,7 @@ function handleToLogin() {
       <NButton
         v-show="isLogin"
         type="primary"
-        class="rounded-md! h-[32px]! text-xs! w-[80px]!"
+        class="rounded-md! h-[32px]! text-xs! min-w-[80px]!"
         @click="handleToCreateApplication"
       >
         {{ t('common_module.create_agent_btn_text') }}

src/views/share/share-application-mobile.vue

@@ -3,6 +3,7 @@ import { computed, onMounted, onUnmounted, ref, shallowRef } from 'vue'
 import { useRouter } from 'vue-router'
 import { useI18n } from 'vue-i18n'
 import { Howl } from 'howler'
+import { ValueOf } from 'type-fest'
 import PageHeader from './components/mobile-page-header.vue'
 import Preamble from './components/preamble.vue'
 import MessageList from './components/message-list.vue'
@@ -37,11 +38,12 @@ const agentApplicationConfig = ref<PersonalAppConfigState>(defaultPersonalAppCon
 const messageListRef = ref<InstanceType<typeof MessageList> | null>(null)
 const footerInputRef = ref<InstanceType<typeof FooterInput> | null>(null)
 
-const messageList = ref<ConversationMessageItem[]>([])
+const messageList = ref(new Map<string, ConversationMessageItem>())
 
 const continuousQuestionStatus = ref<'default' | 'close'>('default')
 const continueQuestionList = ref<string[]>([])
-const answerAudioAutoPlaying = ref(false)
+const answerAudioAutoPlay = ref(false) // 语音是否自动播放
+const answerAudioPlaying = ref(false) // 语音播放中
 const currentPlayMessageItem = ref<ConversationMessageItem | null>(null)
 const currentPlayAudioFragmentSerialNo = ref(0)
 const currentSoundCtl = shallowRef<Howl | null>(null)
@@ -110,7 +112,7 @@ async function handleGetAutoPlayByAgentId() {
   const res = await fetchGetAutoPlayByAgentId<'Y' | 'N'>(agentId.value)
 
   if (res.code === 0) {
-    answerAudioAutoPlaying.value = res.data === 'Y'
+    answerAudioAutoPlay.value = res.data === 'Y'
   }
 }
 
@@ -134,20 +136,31 @@ async function handleUpdateAutoPlaying(isAutoPlaying: boolean) {
   await fetchUpdateAutoPlay(agentId.value, autoplay)
 }
 
-function handleAddMessageItem(messageItem: ConversationMessageItem) {
-  messageList.value.push(messageItem)
+function handleAddMessageItem(messageId: string, messageItem: ConversationMessageItem) {
+  messageList.value.set(messageId, messageItem)
 }
 
-function handleUpdateSpecifyMessageItem(messageItemIndex: number, newObj: Partial<ConversationMessageItem>) {
-  if (messageList.value[messageItemIndex]) {
-    Object.entries(newObj).forEach(([k, v]) => {
-      ;(messageList.value[messageItemIndex] as any)[k as keyof typeof newObj] = v
+function handleUpdateSpecifyMessageItem(messageId: string, newMessageItem: Partial<ConversationMessageItem>) {
+  const currentMessageItemInfo = messageList.value.get(messageId)
+
+  if (currentMessageItemInfo) {
+    const updatePropertyLength = Object.keys(newMessageItem).length
+
+    if (updatePropertyLength > 4) {
+      messageList.value.set(messageId, Object.assign({}, currentMessageItemInfo, newMessageItem))
+      return
+    }
+
+    Object.entries<ValueOf<typeof newMessageItem>>(newMessageItem).forEach(([key, value]) => {
+      if (Object.prototype.hasOwnProperty.call(currentMessageItemInfo, key)) {
+        ;(currentMessageItemInfo as any)[key as keyof ConversationMessageItem] = value
+      }
     })
   }
 }
 
-function handleDeleteLastMessageItem() {
-  messageList.value.pop()
+function handleDeleteLastMessageItem(messageId: string) {
+  messageList.value.delete(messageId)
 }
 
 function handleUpdatePageScroll() {
@@ -163,7 +176,8 @@ function handleClearAllMessage() {
     .then(() => {
       handleAudioPause()
       footerInputRef.value?.blockMessageResponse()
-      messageList.value = []
+      messageList.value.clear()
+      answerAudioPlaying.value = false
       window.$message.success(t('common_module.clear_success_message'))
     })
 }
@@ -189,6 +203,7 @@ function howlSoundFactory(url: string) {
     preload: true,
     autoplay: true,
     onplay: () => {
+      answerAudioPlaying.value = true
       currentSoundCtl.value = soundCtl
 
       if (currentPlayMessageItem.value) {
@@ -206,6 +221,7 @@ function howlSoundFactory(url: string) {
         currentPlayAudioFragmentSerialNo.value > currentPlayMessageItem.value.voiceFragmentUrlList.length - 1
       ) {
         currentPlayMessageItem.value.isVoicePlaying = false
+        answerAudioPlaying.value = false
       }
 
       let audioFragmentUrl = currentPlayMessageItem.value?.voiceFragmentUrlList[currentPlayAudioFragmentSerialNo.value]
@@ -259,9 +275,10 @@ function handleAudioPause(isClearMessageList = false) {
   }
 
   currentPlayMessageItem.value && (currentPlayMessageItem.value.isVoicePlaying = false)
+  answerAudioPlaying.value = false
 
   if (isClearMessageList) {
-    messageList.value = []
+    messageList.value.clear()
     footerInputRef.value?.blockMessageResponse()
   }
 }
@@ -279,18 +296,18 @@ function handleAudioPause(isClearMessageList = false) {
       <div class="mt-5 flex select-none justify-end px-4">
         <div v-show="isEnableVoice" class="flex items-center gap-2">
           <span>{{ t('common_module.voice_auto_play') }}</span>
-          <n-switch v-model:value="answerAudioAutoPlaying" size="small" @update:value="handleUpdateAutoPlaying">
+          <n-switch v-model:value="answerAudioAutoPlay" size="small" @update:value="handleUpdateAutoPlaying">
             <template #checked> {{ t('common_module.open') }} </template>
             <template #unchecked> {{ t('common_module.close') }} </template>
           </n-switch>
         </div>
       </div>
 
-      <div v-if="messageList.length === 0" class="w-full flex-1 overflow-auto px-4">
+      <div v-if="messageList.size === 0" class="w-full flex-1 overflow-auto px-4">
         <Preamble :agent-application-config="agentApplicationConfig" />
       </div>
 
-      <div v-if="messageList.length > 0" class="flex w-full flex-1 flex-col overflow-hidden pt-5">
+      <div v-if="messageList.size > 0" class="flex w-full flex-1 flex-col overflow-hidden pt-5">
         <div class="flex-1 overflow-auto">
           <MessageList
             ref="messageListRef"
@@ -313,7 +330,8 @@ function handleAudioPause(isClearMessageList = false) {
           :continuous-question-status="continuousQuestionStatus"
           :is-enable-document-parse="isEnableDocumentParse"
           :is-enable-voice="isEnableVoice"
-          :answer-audio-auto-playing="answerAudioAutoPlaying"
+          :answer-audio-auto-play="answerAudioAutoPlay"
+          :answer-audio-playing="answerAudioPlaying"
           :timbre-id="agentApplicationConfig.voiceConfig.timbreId"
           @add-message-item="handleAddMessageItem"
           @update-specify-message-item="handleUpdateSpecifyMessageItem"
@@ -324,6 +342,7 @@ function handleAudioPause(isClearMessageList = false) {
           @create-continue-questions="handleCreateContinueQuestions"
           @reset-continue-question-list="handleResetContinueQuestionList"
           @audio-play="handleAudioPlay"
+          @audio-pause="handleAudioPause"
         />
       </div>
     </div>

src/views/share/share-application-web.vue

@@ -3,6 +3,7 @@ import { computed, onMounted, onUnmounted, ref, shallowRef } from 'vue'
 import { useRouter } from 'vue-router'
 import { useI18n } from 'vue-i18n'
 import { Howl } from 'howler'
+import { ValueOf } from 'type-fest'
 import PageHeader from './components/web-page-header.vue'
 import Preamble from './components/preamble.vue'
 import MessageList from './components/message-list.vue'
@@ -40,11 +41,12 @@ const agentApplicationConfig = ref<PersonalAppConfigState>(defaultPersonalAppCon
 const messageListRef = ref<InstanceType<typeof MessageList> | null>(null)
 const footerInputRef = ref<InstanceType<typeof FooterInput> | null>(null)
 
-const messageList = ref<ConversationMessageItem[]>([])
+const messageList = ref(new Map<string, ConversationMessageItem>())
 
 const continuousQuestionStatus = ref<'default' | 'close'>('default')
 const continueQuestionList = ref<string[]>([])
-const answerAudioAutoPlaying = ref(false)
+const answerAudioAutoPlay = ref(false)
+const answerAudioPlaying = ref(false) // 语音播放中
 const currentPlayMessageItem = ref<ConversationMessageItem | null>(null)
 const currentPlayAudioFragmentSerialNo = ref(0)
 const currentSoundCtl = shallowRef<Howl | null>(null)
@@ -123,7 +125,7 @@ async function handleGetAutoPlayByAgentId() {
   const res = await fetchGetAutoPlayByAgentId<'Y' | 'N'>(agentId.value)
 
   if (res.code === 0) {
-    answerAudioAutoPlaying.value = res.data === 'Y'
+    answerAudioAutoPlay.value = res.data === 'Y'
   }
 }
 
@@ -155,20 +157,31 @@ async function handleUpdateAutoPlaying(isAutoPlaying: boolean) {
   await fetchUpdateAutoPlay(agentId.value, autoplay)
 }
 
-function handleAddMessageItem(messageItem: ConversationMessageItem) {
-  messageList.value.push(messageItem)
+function handleAddMessageItem(messageId: string, messageItem: ConversationMessageItem) {
+  messageList.value.set(messageId, messageItem)
 }
 
-function handleUpdateSpecifyMessageItem(messageItemIndex: number, newObj: Partial<ConversationMessageItem>) {
-  if (messageList.value[messageItemIndex]) {
-    Object.entries(newObj).forEach(([k, v]) => {
-      ;(messageList.value[messageItemIndex] as any)[k as keyof typeof newObj] = v
+function handleUpdateSpecifyMessageItem(messageId: string, newMessageItem: Partial<ConversationMessageItem>) {
+  const currentMessageItemInfo = messageList.value.get(messageId)
+
+  if (currentMessageItemInfo) {
+    const updatePropertyLength = Object.keys(newMessageItem).length
+
+    if (updatePropertyLength > 4) {
+      messageList.value.set(messageId, Object.assign({}, currentMessageItemInfo, newMessageItem))
+      return
+    }
+
+    Object.entries<ValueOf<typeof newMessageItem>>(newMessageItem).forEach(([key, value]) => {
+      if (Object.prototype.hasOwnProperty.call(currentMessageItemInfo, key)) {
+        ;(currentMessageItemInfo as any)[key as keyof ConversationMessageItem] = value
+      }
     })
   }
 }
 
-function handleDeleteLastMessageItem() {
-  messageList.value.pop()
+function handleDeleteLastMessageItem(messageId: string) {
+  messageList.value.delete(messageId)
 }
 
 function handleUpdatePageScroll() {
@@ -184,7 +197,8 @@ function handleClearAllMessage() {
     .then(() => {
       handleAudioPause()
       footerInputRef.value?.blockMessageResponse()
-      messageList.value = []
+      messageList.value.clear()
+      answerAudioPlaying.value = false
       window.$message.success(t('common_module.clear_success_message'))
     })
 }
@@ -210,6 +224,7 @@ function howlSoundFactory(url: string) {
     preload: true,
     autoplay: true,
     onplay: () => {
+      answerAudioPlaying.value = true
       currentSoundCtl.value = soundCtl
 
       if (currentPlayMessageItem.value) {
@@ -227,6 +242,7 @@ function howlSoundFactory(url: string) {
         currentPlayAudioFragmentSerialNo.value > currentPlayMessageItem.value.voiceFragmentUrlList.length - 1
       ) {
         currentPlayMessageItem.value.isVoicePlaying = false
+        answerAudioPlaying.value = false
       }
 
       let audioFragmentUrl = currentPlayMessageItem.value?.voiceFragmentUrlList[currentPlayAudioFragmentSerialNo.value]
@@ -280,9 +296,10 @@ function handleAudioPause(isClearMessageList = false) {
   }
 
   currentPlayMessageItem.value && (currentPlayMessageItem.value.isVoicePlaying = false)
+  answerAudioPlaying.value = false
 
   if (isClearMessageList) {
-    messageList.value = []
+    messageList.value.clear()
     footerInputRef.value?.blockMessageResponse()
   }
 }
@@ -303,17 +320,17 @@ function handleAudioPause(isClearMessageList = false) {
         <div class="relative mx-auto flex h-full w-[1000px] flex-col overflow-hidden">
           <div v-show="isEnableVoice" class="absolute right-10 top-7 flex select-none items-center gap-2">
             <span>{{ t('common_module.voice_auto_play') }}</span>
-            <n-switch v-model:value="answerAudioAutoPlaying" size="small" @update:value="handleUpdateAutoPlaying">
+            <n-switch v-model:value="answerAudioAutoPlay" size="small" @update:value="handleUpdateAutoPlaying">
               <template #checked> {{ t('common_module.open') }} </template>
               <template #unchecked> {{ t('common_module.close') }} </template>
             </n-switch>
           </div>
 
-          <div v-if="messageList.length === 0" class="w-full flex-1 overflow-auto px-5">
+          <div v-if="messageList.size === 0" class="w-full flex-1 overflow-auto px-5">
             <Preamble :agent-application-config="agentApplicationConfig" />
           </div>
 
-          <div v-if="messageList.length > 0" class="flex w-full flex-1 flex-col overflow-hidden">
+          <div v-if="messageList.size > 0" class="flex w-full flex-1 flex-col overflow-hidden">
             <div class="mt-20 flex-1 overflow-auto">
               <MessageList
                 ref="messageListRef"
@@ -336,7 +353,8 @@ function handleAudioPause(isClearMessageList = false) {
               :continuous-question-status="continuousQuestionStatus"
               :is-enable-document-parse="isEnableDocumentParse"
               :is-enable-voice="isEnableVoice"
-              :answer-audio-auto-playing="answerAudioAutoPlaying"
+              :answer-audio-auto-play="answerAudioAutoPlay"
+              :answer-audio-playing="answerAudioPlaying"
               :timbre-id="agentApplicationConfig.voiceConfig.timbreId"
               @add-message-item="handleAddMessageItem"
               @update-specify-message-item="handleUpdateSpecifyMessageItem"
@@ -347,6 +365,7 @@ function handleAudioPause(isClearMessageList = false) {
               @create-continue-questions="handleCreateContinueQuestions"
               @reset-continue-question-list="handleResetContinueQuestionList"
               @audio-play="handleAudioPlay"
+              @audio-pause="handleAudioPause"
             />
           </div>
         </div>

types/locales.d.ts

@@ -100,6 +100,8 @@ declare namespace I18n {
       voice_auto_play: string
       start_playing: string
       stop_playing: string
+      unplayable: string
+      unplayable_tip: string
       response_error: string
       agent_exception: string
       equity: string
@@ -116,6 +118,8 @@ declare namespace I18n {
         cancel_associate_file_tip: string
         upload_file_limit: string
         overwrite_file_tip: string
+        stop_playing_and_then_operate: string
+        do_not_operate_until_the_reply_is_complete: string
       }
 
       data_table_module: {
@@ -292,9 +296,10 @@ declare namespace I18n {
             memory_fragment_delete_row_tip_content: string
             add_knowledge_successfully: string
             remove_knowledge_successfully: string
-            setting_timbre: string
-            setting_timbre_message: string
-            setting_timbre_desc: string
+            setting_voice: string
+            setting_voice_message: string
+            setting_voice_desc: string
+            currently_only_one_voice_can_be_set: string
 
             memory_variable_modal: {
               edit_memory_variable: string