Skip to content

P
poc-api
Commit f8f0935e authored by alex yao's avatar alex yao
Browse files
Options

feat[长文阅读]:新增关键词/思维导图

parent 6935d327
Show whitespace changes
Inline Side-by-side
Showing with 450 additions and 92 deletions
src/main/java/cn/com/poc/long_document/aggregate/impl/LongTextDialoguesServiceImpl.java
View file @ f8f0935e
......@@ -22,6 +22,7 @@ import cn.com.poc.knowledge.query.KnowledgeDocumentRelationQueryItem;
import cn.com.poc.knowledge.service.BizKnowledgeDocumentService;
import cn.com.poc.long_document.aggregate.LongTextDialoguesService;
import cn.com.poc.long_document.domain.LongtextDialoguesResult;
import cn.com.poc.long_document.domain.MindMap;
import cn.com.poc.long_document.dto.LongTextExampleDto;
import cn.com.poc.long_document.dto.LongTextSummaryDto;
import cn.com.poc.long_document.entity.BizLongTextSummaryCacheEntity;
......@@ -47,6 +48,7 @@ import cn.com.yict.framemax.core.exception.BusinessException;
import cn.com.yict.framemax.frame.service.FmxParamConfigService;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.crypto.digest.MD5;
import com.fasterxml.jackson.core.type.TypeReference;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
......@@ -235,8 +237,16 @@ public class LongTextDialoguesServiceImpl implements LongTextDialoguesService {
throw new BusinessException("对话文件不存在，重新创建对话");
}
String summary;
String corePoint;
String summary = StringUtils.EMPTY;
String corePoint = StringUtils.EMPTY;
String mindMap = StringUtils.EMPTY;
String keyWord = StringUtils.EMPTY;
boolean summaryNeedGenerate = false;
boolean corePointNeedGenerate = false;
boolean mindMapNeedGenerate = false;
boolean keyWordNeedGenerate = false;
File file = DocumentLoad.downloadURLDocument(fileUrl);
String fileMD5 = MD5.create().digestHex(file);
......@@ -249,34 +259,117 @@ public class LongTextDialoguesServiceImpl implements LongTextDialoguesService {
BizLongTextSummaryCacheEntity longTextSummaryCacheEntity = bizLongTextSummaryCacheEntities.get(0);
summary = longTextSummaryCacheEntity.getSummary();
corePoint = longTextSummaryCacheEntity.getCorePoint();
} else {
keyWord = longTextSummaryCacheEntity.getKeyword();
mindMap = longTextSummaryCacheEntity.getMindmap();
}
summaryNeedGenerate = StringUtils.isBlank(summary);
corePointNeedGenerate = StringUtils.isBlank(corePoint);
mindMapNeedGenerate = StringUtils.isBlank(mindMap);
keyWordNeedGenerate = StringUtils.isBlank(keyWord);
String document = DocumentLoad.documentToText(file);
CompletableFuture<String> summaryFuture = CompletableFuture.supplyAsync(() -> createSummary(document)).exceptionally(throwable -> {
String finalSummary = summary;
boolean finalSummaryNeedGenerate = summaryNeedGenerate;
CompletableFuture<String> summaryFuture = CompletableFuture.supplyAsync(() -> {
if (finalSummaryNeedGenerate) {
return createSummary(document);
} else {
return finalSummary;
}
}).exceptionally(throwable -> {
logger.error("生成摘要失败", throwable);
throw new BusinessException(throwable);
});
CompletableFuture<String> corePointFuture = CompletableFuture.supplyAsync(() -> createCorePoint(document)).exceptionally(throwable -> {
String finalCorePoint = corePoint;
boolean finalCorePointNeedGenerate = corePointNeedGenerate;
CompletableFuture<String> corePointFuture = CompletableFuture.supplyAsync(() ->
finalCorePointNeedGenerate ? createCorePoint(document) : finalCorePoint
).exceptionally(throwable -> {
logger.error("生成核心观点失败", throwable);
throw new BusinessException(throwable);
});
CompletableFuture<Void> allOf = CompletableFuture.allOf(summaryFuture, corePointFuture);
boolean finalMindMapNeedGenerate = mindMapNeedGenerate;
String finalMindMap = mindMap;
CompletableFuture<String> mindMapFuture = CompletableFuture.supplyAsync(
() -> finalMindMapNeedGenerate ? createMindMap(document) : finalMindMap).exceptionally(throwable -> {
logger.error("生成思维导图失败", throwable);
throw new BusinessException(throwable);
});
boolean finalKeyWordNeedGenerate = keyWordNeedGenerate;
String finalKeyWord = keyWord;
CompletableFuture<String> keyWordFuture = CompletableFuture.supplyAsync(() ->
finalKeyWordNeedGenerate ? createKeyWord(document) : finalKeyWord).exceptionally(throwable -> {
logger.error("生成关键词失败", throwable);
throw new BusinessException(throwable);
});
CompletableFuture<Void> allOf = CompletableFuture.allOf(summaryFuture, corePointFuture, mindMapFuture, keyWordFuture);
allOf.join();
summary = summaryFuture.get();
corePoint = corePointFuture.get();
mindMap = mindMapFuture.get();
keyWord = keyWordFuture.get();
// 保存缓存
if (corePointNeedGenerate || summaryNeedGenerate || keyWordNeedGenerate || mindMapNeedGenerate) {
// 删除就缓存
if (CollectionUtils.isNotEmpty(bizLongTextSummaryCacheEntities)) {
BizLongTextSummaryCacheEntity updateCacheEntity = bizLongTextSummaryCacheEntities.get(0);
updateCacheEntity.setCorePoint(corePoint);
updateCacheEntity.setSummary(summary);
updateCacheEntity.setMindmap(mindMap);
updateCacheEntity.setKeyword(keyWord);
bizLongTextSummaryCacheService.update(updateCacheEntity);
} else {
BizLongTextSummaryCacheEntity saveCache = new BizLongTextSummaryCacheEntity();
saveCache.setCorePoint(corePoint);
saveCache.setSummary(summary);
saveCache.setFileUrl(fileUrl);
saveCache.setMindmap(mindMap);
saveCache.setKeyword(keyWord);
saveCache.setFileMd5(fileMD5);
saveCache.setIsDeleted(CommonConstant.IsDeleted.N);
bizLongTextSummaryCacheService.save(saveCache);
}
}
if (StringUtils.isBlank(summary)){
throw new BusinessException("获取/生成摘要失败");
}
if (StringUtils.isBlank(keyWord)){
throw new BusinessException("获取/生成关键词失败");
}
if (StringUtils.isBlank(corePoint)){
throw new BusinessException("获取/生成核心观点失败");
}
if (StringUtils.isBlank(mindMap)){
throw new BusinessException("获取/生成思维导图失败");
}
// 返回摘要和核心观点
// 转换格式 json to list
int startKeyWord = keyWord.lastIndexOf("[");
int endKeyWord = keyWord.lastIndexOf("]");
int startMindMap = mindMap.indexOf("{");
int endMindMap = mindMap.lastIndexOf("}");
LongTextSummaryDto longTextSummaryDto = new LongTextSummaryDto();
longTextSummaryDto.setSummary(summary);
longTextSummaryDto.setCorePoint(corePoint);
longTextSummaryDto.setKeyword(JsonUtils.deSerialize(keyWord.substring(startKeyWord, endKeyWord + 1), new TypeReference<List<String>>() {
}.getType()));
longTextSummaryDto.setMindMap(JsonUtils.deSerialize(mindMap.substring(startMindMap, endMindMap + 1), MindMap.class));
return longTextSummaryDto;
}
......@@ -606,6 +699,124 @@ public class LongTextDialoguesServiceImpl implements LongTextDialoguesService {
}
}
private String createKeyWord(String document) {
// 获取对话提示词
String promptCode = "CreateKeyWordPrompt";
BizAgentApplicationGcConfigEntity documentDialoguePrompt = bizAgentApplicationGcConfigService.getByConfigCode(promptCode);
if (documentDialoguePrompt == null || StringUtils.isBlank(documentDialoguePrompt.getConfigSystem())) {
logger.error("获取对话提示词失败 , configCode:{}", promptCode);
throw new BusinessException("获取对话提示词失败");
}
String prompt = documentDialoguePrompt.getConfigSystem();
String largeModel = documentDialoguePrompt.getLargeModel();
prompt = prompt.replace("{document}", document);
// 配置message
List<Message> messages = new ArrayList<>();
Message systemMessage = new Message();
systemMessage.setContent(prompt);
systemMessage.setRole(LLMRoleEnum.SYSTEM.getRole());
messages.add(systemMessage);
Message questionMessage = new Message();
questionMessage.setContent("获取关键词");
questionMessage.setRole(LLMRoleEnum.USER.getRole());
messages.add(questionMessage);
// 调用LLM
LargeModelResponse response = new LargeModelResponse();
response.setModel(largeModel);
response.setMessages(messages.toArray(new Message[0]));
response.setStream(true);
response.setUser("CreateKeyWord");
try {
StringBuilder summary = new StringBuilder();
BufferedReader bufferedReader = llmService.chatChunk(response);
String res;
while ((res = bufferedReader.readLine()) != null) {
if (StringUtils.isEmpty(res)) {
continue;
}
res = res.replace("data: ", StringUtils.EMPTY);
LargeModelDemandResult result = JsonUtils.deSerialize(res, LargeModelDemandResult.class);
if (ObjectUtil.isEmpty(result) || !result.getCode().equals("0")) {
logger.error("LLM Error,code:{}", result.getCode());
throw new BusinessException("获取关键词失败");
}
if (StringUtils.isNotBlank(result.getMessage())) {
String message = result.getMessage();
summary.append(message);
}
}
bufferedReader.close();
return summary.toString();
} catch (Exception e) {
logger.error("获取关键词失败", e);
throw new BusinessException("获取关键词失败");
}
}
private String createMindMap(String document) {
// 获取对话提示词
String promptCode = "CreateMindMapPrompt";
BizAgentApplicationGcConfigEntity documentDialoguePrompt = bizAgentApplicationGcConfigService.getByConfigCode(promptCode);
if (documentDialoguePrompt == null || StringUtils.isBlank(documentDialoguePrompt.getConfigSystem())) {
logger.error("获取对话提示词失败 , configCode:{}", promptCode);
throw new BusinessException("获取对话提示词失败");
}
String prompt = documentDialoguePrompt.getConfigSystem();
String largeModel = documentDialoguePrompt.getLargeModel();
prompt = prompt.replace("{document}", document);
// 配置message
List<Message> messages = new ArrayList<>();
Message systemMessage = new Message();
systemMessage.setContent(prompt);
systemMessage.setRole(LLMRoleEnum.SYSTEM.getRole());
messages.add(systemMessage);
Message questionMessage = new Message();
questionMessage.setContent("生成思维导图");
questionMessage.setRole(LLMRoleEnum.USER.getRole());
messages.add(questionMessage);
// 调用LLM
LargeModelResponse response = new LargeModelResponse();
response.setModel(largeModel);
response.setMessages(messages.toArray(new Message[0]));
response.setStream(true);
response.setUser("CreateMindMap");
try {
StringBuilder summary = new StringBuilder();
BufferedReader bufferedReader = llmService.chatChunk(response);
String res;
while ((res = bufferedReader.readLine()) != null) {
if (StringUtils.isEmpty(res)) {
continue;
}
res = res.replace("data: ", StringUtils.EMPTY);
LargeModelDemandResult result = JsonUtils.deSerialize(res, LargeModelDemandResult.class);
if (ObjectUtil.isEmpty(result) || !result.getCode().equals("0")) {
logger.error("LLM Error,code:{}", result.getCode());
throw new BusinessException("生成思维导图失败");
}
if (StringUtils.isNotBlank(result.getMessage())) {
String message = result.getMessage();
summary.append(message);
}
}
bufferedReader.close();
return summary.toString();
} catch (Exception e) {
logger.error("生成思维导图失败", e);
throw new BusinessException("生成思维导图失败");
}
}
private ToolFunction getToolFunction(String dialoguesId, String input, Boolean enableSearchEngine, SSEUtil sseUtil) throws IOException {
ToolFunction toolFunction = null;
if (enableSearchEngine) {
......
src/main/java/cn/com/poc/long_document/convert/BizLongTextSummaryCacheConvert.java
View file @ f8f0935e
......@@ -13,6 +13,8 @@ public class BizLongTextSummaryCacheConvert {
entity.setSummary(model.getSummary());
entity.setFileUrl(model.getFileUrl());
entity.setFileMd5(model.getFileMd5());
entity.setKeyword(model.getKeyword());
entity.setMindmap(model.getMindmap());
entity.setIsDeleted(model.getIsDeleted());
entity.setCreator(model.getCreator());
entity.setCreatedTime(model.getCreatedTime());
......@@ -27,6 +29,8 @@ public class BizLongTextSummaryCacheConvert {
model.setId(entity.getId());
model.setCorePoint(entity.getCorePoint());
model.setSummary(entity.getSummary());
model.setKeyword(entity.getKeyword());
model.setMindmap(entity.getMindmap());
model.setFileUrl(entity.getFileUrl());
model.setFileMd5(entity.getFileMd5());
model.setIsDeleted(entity.getIsDeleted());
......
src/main/java/cn/com/poc/long_document/domain/MindMap.java 0 → 100644
View file @ f8f0935e
package cn.com.poc.long_document.domain;
import java.util.List;
/**
* @author alex.yao
* @date 2025/8/29
*/
public class MindMap {
private String id;
private String topic;
private Boolean root;
private List<MindMap> children;
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public String getTopic() {
return topic;
}
public void setTopic(String topic) {
this.topic = topic;
}
public Boolean getRoot() {
return root;
}
public void setRoot(Boolean root) {
this.root = root;
}
public List<MindMap> getChildren() {
return children;
}
public void setChildren(List<MindMap> children) {
this.children = children;
}
}
src/main/java/cn/com/poc/long_document/dto/LongTextSummaryDto.java
View file @ f8f0935e
package cn.com.poc.long_document.dto;
import cn.com.poc.long_document.domain.MindMap;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import java.util.List;
/**
* @author alex.yao
* @date 2025/6/18
*/
@JsonInclude(JsonInclude.Include.NON_NULL)
@JsonIgnoreProperties(ignoreUnknown = true)
public class LongTextSummaryDto {
private String summary;
private String corePoint;
private List<String> keyword;
private MindMap mindMap;
public List<String> getKeyword() {
return keyword;
}
public void setKeyword(List<String> keyword) {
this.keyword = keyword;
}
public MindMap getMindMap() {
return mindMap;
}
public void setMindMap(MindMap mindMap) {
this.mindMap = mindMap;
}
public String getSummary() {
return summary;
}
......
src/main/java/cn/com/poc/long_document/entity/BizLongTextSummaryCacheEntity.java
View file @ f8f0935e
package cn.com.poc.long_document.entity;
import javax.persistence.Column;
public class BizLongTextSummaryCacheEntity {
private static final long serialVersionUID = 1L;
......@@ -40,6 +42,34 @@ public class BizLongTextSummaryCacheEntity {
public void setSummary(java.lang.String summary){
this.summary = summary;
}
/** keyword
*关键词
*/
private java.lang.String keyword;
public java.lang.String getKeyword(){
return this.keyword;
}
public void setKeyword(java.lang.String keyword){
this.keyword = keyword;
}
/** mindmap
*思维导图
*/
private java.lang.String mindmap;
public java.lang.String getMindmap(){
return this.mindmap;
}
public void setMindmap(java.lang.String mindmap){
this.mindmap = mindmap;
}
/** file_url
*文件地址
*/
......
src/main/java/cn/com/poc/long_document/model/BizLongTextSummaryCacheModel.java
View file @ f8f0935e
......@@ -74,6 +74,38 @@ public class BizLongTextSummaryCacheModel extends BaseModelClass implements Seri
}
/** keyword
*关键词
*/
private java.lang.String keyword;
@Column(name = "keyword",length = 2147483647)
public java.lang.String getKeyword(){
return this.keyword;
}
public void setKeyword(java.lang.String keyword){
this.keyword = keyword;
super.addValidField("keyword");
}
/** mindmap
*思维导图
*/
private java.lang.String mindmap;
@Column(name = "mindmap",length = 2147483647)
public java.lang.String getMindmap(){
return this.mindmap;
}
public void setMindmap(java.lang.String mindmap){
this.mindmap = mindmap;
super.addValidField("mindmap");
}
/** file_url
*文件地址
*/
......
src/main/java/cn/com/poc/long_document/service/impl/BizLongTextSummaryCacheServiceImpl.java
View file @ f8f0935e
......@@ -10,10 +10,12 @@ import cn.com.yict.framemax.data.model.PagingInfo;
import org.springframework.stereotype.Service;
import org.apache.commons.collections4.CollectionUtils;
import java.util.ArrayList;
import java.util.stream.Collectors;
import java.util.List;
import javax.annotation.Resource;
import org.springframework.util.Assert;
@Service
......@@ -23,34 +25,34 @@ public class BizLongTextSummaryCacheServiceImpl extends BaseServiceImpl
@Resource
private BizLongTextSummaryCacheRepository repository;
public BizLongTextSummaryCacheEntity get(java.lang.Long id) throws Exception{
public BizLongTextSummaryCacheEntity get(java.lang.Long id) throws Exception {
Assert.notNull(id);
BizLongTextSummaryCacheModel model = this.repository.get(id);
if (model == null){
if (model == null) {
return null;
}
if ("Y".equals(model.getIsDeleted())){
if ("Y".equals(model.getIsDeleted())) {
return null;
}
return BizLongTextSummaryCacheConvert.modelToEntity(model);
}
public List<BizLongTextSummaryCacheEntity> findByExample(BizLongTextSummaryCacheEntity example,PagingInfo pagingInfo) throws Exception{
public List<BizLongTextSummaryCacheEntity> findByExample(BizLongTextSummaryCacheEntity example, PagingInfo pagingInfo) throws Exception {
List<BizLongTextSummaryCacheEntity> result = new ArrayList<BizLongTextSummaryCacheEntity>();
BizLongTextSummaryCacheModel model = new BizLongTextSummaryCacheModel();
if (example != null){
if (example != null) {
model = BizLongTextSummaryCacheConvert.entityToModel(example);
}
model.setIsDeleted("N");
List<BizLongTextSummaryCacheModel> models = this.repository.findByExample(model,pagingInfo);
List<BizLongTextSummaryCacheModel> models = this.repository.findByExample(model, pagingInfo);
if (CollectionUtils.isNotEmpty(models)) {
result = models.stream().map(BizLongTextSummaryCacheConvert::modelToEntity).collect(Collectors.toList());
}
return result;
}
public BizLongTextSummaryCacheEntity save(BizLongTextSummaryCacheEntity entity) throws Exception{
public BizLongTextSummaryCacheEntity save(BizLongTextSummaryCacheEntity entity) throws Exception {
Assert.notNull(entity);
entity.setId(null);
entity.setIsDeleted("N");
......@@ -59,50 +61,56 @@ public class BizLongTextSummaryCacheServiceImpl extends BaseServiceImpl
return BizLongTextSummaryCacheConvert.modelToEntity(saveModel);
}
public BizLongTextSummaryCacheEntity update(BizLongTextSummaryCacheEntity entity) throws Exception{
public BizLongTextSummaryCacheEntity update(BizLongTextSummaryCacheEntity entity) throws Exception {
Assert.notNull(entity);
Assert.notNull(entity.getId(),"update pk can not be null");
Assert.notNull(entity.getId(), "update pk can not be null");
BizLongTextSummaryCacheModel model = this.repository.get(entity.getId());
if (entity.getCorePoint() != null){
if (entity.getCorePoint() != null) {
model.setCorePoint(entity.getCorePoint());
}
if (entity.getSummary() != null){
if (entity.getSummary() != null) {
model.setSummary(entity.getSummary());
}
if (entity.getFileUrl() != null){
if (entity.getFileUrl() != null) {
model.setFileUrl(entity.getFileUrl());
}
if (entity.getFileMd5() != null){
if (entity.getMindmap() != null) {
model.setMindmap(entity.getMindmap());
}
if (entity.getKeyword() != null) {
model.setKeyword(entity.getKeyword());
}
if (entity.getFileMd5() != null) {
model.setFileMd5(entity.getFileMd5());
}
if (entity.getIsDeleted() != null){
if (entity.getIsDeleted() != null) {
model.setIsDeleted(entity.getIsDeleted());
}
if (entity.getCreator() != null){
if (entity.getCreator() != null) {
model.setCreator(entity.getCreator());
}
if (entity.getCreatedTime() != null){
if (entity.getCreatedTime() != null) {
model.setCreatedTime(entity.getCreatedTime());
}
if (entity.getModifier() != null){
if (entity.getModifier() != null) {
model.setModifier(entity.getModifier());
}
if (entity.getModifiedTime() != null){
if (entity.getModifiedTime() != null) {
model.setModifiedTime(entity.getModifiedTime());
}
if (entity.getSysVersion() != null){
if (entity.getSysVersion() != null) {
model.setSysVersion(entity.getSysVersion());
}
BizLongTextSummaryCacheModel saveModel = this.repository.save(model);
return BizLongTextSummaryCacheConvert.modelToEntity(saveModel);
}
public void deletedById(java.lang.Long id) throws Exception{
public void deletedById(java.lang.Long id) throws Exception {
Assert.notNull(id);
BizLongTextSummaryCacheModel model = this.repository.get(id);
if (model != null){
if ("N".equals(model.getIsDeleted())){
if (model != null) {
if ("N".equals(model.getIsDeleted())) {
model.setIsDeleted("Y");
this.repository.save(model);
}
......
src/main/java/cn/com/poc/meeting/analysis/KeyWordAnalyzer.java
View file @ f8f0935e
......@@ -70,19 +70,15 @@ public class KeyWordAnalyzer extends MeetingContentAnalysisAbstract<List<String>
double progress = ((double) (i + 1) / segmentationResultList.size()) * 100D;
logger.info("-------------关键词提取进度 {} %--------------", progress);
}
} catch (Exception e) {
logger.error("关键词提取失败", e);
return Collections.emptyList();
}
if (StringUtils.isBlank(generateContent)) {
return Collections.emptyList();
}
// 转换格式 json to list
int start = generateContent.lastIndexOf("[");
int end = generateContent.lastIndexOf("]");
List<String> res = JsonUtils.deSerialize(generateContent.substring(start, end + 1), new TypeReference<List<String>>() {
}.getType());
return res;
} catch (Exception e) {
logger.error("关键词提取失败", e);
return Collections.emptyList();
}
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment