Skip to content

P
poc-api
Commit 9aa5dba8 authored by alex yao's avatar alex yao
Browse files
Options

feat: 知识库接口

parent f031f1e6
Show whitespace changes
Inline Side-by-side
Showing with 1954 additions and 81 deletions
src/main/java/cn/com/poc/agent_application/aggregate/impl/AgentApplicationInfoServiceImpl.java
View file @ 9aa5dba8
......@@ -96,22 +96,20 @@ public class AgentApplicationInfoServiceImpl implements AgentApplicationInfoServ
public String callAgentApplication(String largeModel, String[] unitIds, String agentSystem, String[] knowledgeIds, Integer communicationTurn, Float topP, List<Message> messages, HttpServletResponse httpServletResponse) throws Exception {
logger.info("--------- Call Agent Application large model:{},unitIds:{},agentSystem:{},knowledgeIds:{}" + " communicationTurn:{},topP:{},messages:{}--------------", largeModel, unitIds, agentSystem, knowledgeIds, communicationTurn, topP, messages);
BizAgentApplicationLargeModelListEntity largeModelEntity = bizAgentApplicationLargeModelListService.findByModelNickName(largeModel);
if (largeModelEntity == null) {
throw new BusinessException("模型不存在");
}
//todo 调用组件
String model = modelConvert(largeModel);
String promptTemplate = buildDialogsPrompt(messages, agentSystem, knowledgeIds);
Message[] messageArray = buildMessages(messages, communicationTurn, promptTemplate);
BufferedReader bufferedReader = invokeLLM(largeModelEntity.getModelName(), messageArray, topP);
//todo buildFunction 配置
BufferedReader bufferedReader = invokeLLM(model, messageArray, topP);
return textOutput(httpServletResponse, bufferedReader);
}
@Override
public void createAgentSystem(String input, HttpServletResponse httpServletResponse) throws Exception {
BizAgentApplicationGcConfigEntity configEntity = bizAgentApplicationGcConfigService.getByConfigCode(AgentApplicationGCConfigConstants.AGENT_SYSTEM);
......@@ -355,17 +353,39 @@ public class AgentApplicationInfoServiceImpl implements AgentApplicationInfoServ
* @throws Exception
*/
private BufferedReader invokeLLM(String largeModel, Message[] messageArray, Float topP) throws Exception {
// todo 配置tools
LargeModelResponse largeModelResponse = new LargeModelResponse();
largeModelResponse.setModel(largeModel);
largeModelResponse.setMessages(messageArray);
largeModelResponse.setTopP(topP);
largeModelResponse.setStream(true);
//todo 添加tools参数
largeModelResponse.setUser("POE");
return llmService.chatChunk(largeModelResponse);
BufferedReader bufferedReader = llmService.chatChunk(largeModelResponse);
String res = "";
bufferedReader.mark(0);
while ((res = bufferedReader.readLine()) != null) {
if (StringUtils.isBlank(res)) {
continue;
}
//todo 添加function 参数 LargeModelDemandResult 和中台接口一致
LargeModelDemandResult result = JsonUtils.deSerialize(res.replaceFirst(EVENT_STREAM_PREFIX, StringUtils.EMPTY), LargeModelDemandResult.class);
if (!"0".equals(result.getCode())) {
logger.error("LLM Error,code:{}", result.getCode());
BusinessException ex = new BusinessException("调用失败");
throw ex;
}
// todo 判断是否为function_call
// todo 如果是function_call，则处理function
//todo
// todo 处理function - 1. 获取function_call 2. 调用对应function 3. 返回function_call并且调用LLM 4. 获取bufferedReader 返回
// todo 若不为function_call, 则直接返回bufferedReader
bufferedReader.reset();
break;
}
return bufferedReader;
}
/**
......@@ -433,4 +453,19 @@ public class AgentApplicationInfoServiceImpl implements AgentApplicationInfoServ
}
return messages.toArray(new Message[0]);
}
/**
* 获取模型
*
* @param largeModel
* @return
*/
private String modelConvert(String largeModel) {
BizAgentApplicationLargeModelListEntity largeModelEntity = bizAgentApplicationLargeModelListService.findByModelNickName(largeModel);
if (largeModelEntity == null) {
throw new BusinessException("模型不存在");
}
return largeModelEntity.getModelName();
}
}
src/main/java/cn/com/poc/knowledge/aggregate/KnowledgeService.java
View file @ 9aa5dba8
package cn.com.poc.knowledge.aggregate;
import cn.com.poc.knowledge.entity.BizKnowledgeDocumentEntity;
import cn.com.poc.thirdparty.resource.demand.ai.entity.knowledge.GetKnowledgeChunkInfoResult;
import cn.com.poc.thirdparty.resource.demand.ai.entity.knowledge.SegmentationConfigRequest;
import cn.com.yict.framemax.data.model.PagingInfo;
import org.springframework.web.multipart.MultipartFile;
......@@ -42,7 +44,7 @@ public interface KnowledgeService {
/**
* 开启训练
*/
Boolean trainKnowledge(List<Integer> kdId) throws Exception;
Boolean trainKnowledge(Integer knowledgeInfoId, List<Integer> kdId, SegmentationConfigRequest segmentationConfig) throws Exception;
/**
* 获取用户知识库
......@@ -55,4 +57,45 @@ public interface KnowledgeService {
List<BizKnowledgeDocumentEntity> getListByKdIds(List<Integer> kdIds);
/**
* 获取知识库分片
*/
GetKnowledgeChunkInfoResult getChunkInfo(List<Integer> kdIds, PagingInfo pagingInfo);
/**
* 开关知识库分片
*
* @param kdId 知识库id
* @param chunkRelationId 分片关系id
* @param isOpen 是否开启 Y/N
*/
void openKnowledgeChunk(Integer kdId, String chunkRelationId, String isOpen);
/**
* 删除知识库分片
*
* @param kdId 知识库id
* @param chunkRelationId 分片关系id
*/
void deleteKnowledgeChunk(Integer kdId, String chunkRelationId);
/**
* 更新知识库分片内容
*
* @param kdId 知识库id
* @param chunkRelationId 分片关系id
* @param content 分片内容
*/
void updateKnowledgeChunkDoc(Integer kdId, String chunkRelationId, String content);
/**
* 新增知识库分片
*
* @param kdId 知识库id
* @param content 分片内容
* @param chunkSort 分片排序
*/
void addKnowledgeChunk(Integer kdId, String content, Integer chunkSort);
}
src/main/java/cn/com/poc/knowledge/aggregate/impl/KnowledgeServiceImpl.java
View file @ 9aa5dba8
......@@ -7,13 +7,18 @@ import cn.com.poc.common.utils.*;
import cn.com.poc.knowledge.aggregate.KnowledgeService;
import cn.com.poc.knowledge.constant.KnowledgeConstant;
import cn.com.poc.knowledge.entity.BizKnowledgeDocumentEntity;
import cn.com.poc.knowledge.entity.BizKnowledgeInfoEntity;
import cn.com.poc.knowledge.entity.BizMemberKnowledgeRelationEntity;
import cn.com.poc.knowledge.query.KnowledgeQueryItem;
import cn.com.poc.knowledge.service.BizKnowledgeDocumentService;
import cn.com.poc.knowledge.service.BizKnowledgeInfoService;
import cn.com.poc.knowledge.service.BizMemberKnowledgeRelationService;
import cn.com.poc.message.entity.TrainKnowledgeMessage;
import cn.com.poc.message.service.KnowledgeProducerService;
import cn.com.poc.support.security.oauth.entity.UserBaseEntity;
import cn.com.poc.thirdparty.resource.demand.ai.aggregate.DemandKnowledgeService;
import cn.com.poc.thirdparty.resource.demand.ai.entity.knowledge.GetKnowledgeChunkInfoResult;
import cn.com.poc.thirdparty.resource.demand.ai.entity.knowledge.SegmentationConfigRequest;
import cn.com.yict.framemax.core.exception.BusinessException;
import cn.com.yict.framemax.data.model.PagingInfo;
import cn.hutool.core.bean.BeanUtil;
......@@ -42,6 +47,11 @@ public class KnowledgeServiceImpl implements KnowledgeService {
@Resource
private KnowledgeProducerService knowledgeProducerService;
@Resource
private DemandKnowledgeService demandKnowledgeService;
@Resource
private BizKnowledgeInfoService bizKnowledgeInfoService;
@Resource
private BosConfigService bosConfigService;
......@@ -61,12 +71,6 @@ public class KnowledgeServiceImpl implements KnowledgeService {
//获取文件名
String documentName = documentFile.getOriginalFilename();
// 查看是否有重名
List<KnowledgeQueryItem> knowledgeQueryItems = bizKnowledgeDocumentService.searchKnowledge(null, null, memberId, documentName, null);
if (CollectionUtils.isNotEmpty(knowledgeQueryItems)) {
throw new BusinessException("上传文件存在重名,文件名:" + documentName);
}
// 统计文件字符数
String type = documentFile.getOriginalFilename().substring(documentFile.getOriginalFilename().lastIndexOf(".") + 1, documentFile.getOriginalFilename().length());
File file = File.createTempFile(UUIDTool.getUUID(), "." + type);
......@@ -90,7 +94,7 @@ public class KnowledgeServiceImpl implements KnowledgeService {
}
//文件上传
String documentUrl = bosConfigService.upload(Files.newInputStream(file.toPath()),type, documentFile.getContentType());
String documentUrl = bosConfigService.upload(Files.newInputStream(file.toPath()), type, documentFile.getContentType());
//保存记录
BizKnowledgeDocumentEntity bizKnowledgeDocumentEntity = new BizKnowledgeDocumentEntity();
......@@ -156,15 +160,11 @@ public class KnowledgeServiceImpl implements KnowledgeService {
}
@Override
public Boolean trainKnowledge(List<Integer> kdIds) throws Exception {
public Boolean trainKnowledge(Integer knowledgeInfoId, List<Integer> kdIds, SegmentationConfigRequest segmentationConfig) throws Exception {
Assert.notEmpty(kdIds);
Assert.isTrue(kdIds.size() <= 5, "Training cannot exceed five documents");
for (Integer kdId : kdIds) {
BizKnowledgeDocumentEntity bizKnowledgeDocumentEntity = bizKnowledgeDocumentService.get(kdId);
// 扣减次数
// AiGcMemberEntity currentMember = aiGcVipService.getCurrentMember(BlContext.getCurrentUserNotException());
// Map<Integer, Integer> reduced = bizAiGcMemberFreqService.reduceNumber(currentMember.getRelationId(), AiGcPaymentConfigurationConstant.DOMAIN.TRAIN, currentMember.getType(), bizKnowledgeDocumentEntity.getCharCount().intValue());
String currentStatus = bizKnowledgeDocumentEntity.getTrainStatus();
if (KnowledgeConstant.TrainStatus.COMPLETE.equals(currentStatus) ||
KnowledgeConstant.TrainStatus.LINE.equals(currentStatus) ||
......@@ -178,10 +178,15 @@ public class KnowledgeServiceImpl implements KnowledgeService {
TrainKnowledgeMessage message = new TrainKnowledgeMessage();
message.setKid(kdId);
message.setFileUrl(bizKnowledgeDocumentEntity.getDocumentUrl());
// message.setReduced(reduced);
message.setSegmentationConfig(segmentationConfig);
knowledgeProducerService.trainKnowledge(message);
}
BizKnowledgeInfoEntity bizKnowledgeInfoEntity = new BizKnowledgeInfoEntity();
bizKnowledgeInfoEntity.setId(knowledgeInfoId);
bizKnowledgeInfoEntity.setTrainStatus(KnowledgeConstant.TrainStatus.TRAINING);
bizKnowledgeInfoEntity.setKdIds(JsonUtils.serialize(kdIds));
bizKnowledgeInfoService.update(bizKnowledgeInfoEntity);
return true;
}
......@@ -213,4 +218,43 @@ public class KnowledgeServiceImpl implements KnowledgeService {
}
return res;
}
@Override
public GetKnowledgeChunkInfoResult getChunkInfo(List<Integer> kdIds, PagingInfo pagingInfo) {
List<String> knowledgeIds = new ArrayList<>();
for (Integer kdId : kdIds) {
BizKnowledgeDocumentEntity bizKnowledgeDocumentEntity = bizKnowledgeDocumentService.get(kdId);
String knowledgeId = bizKnowledgeDocumentEntity.getKnowledgeId();
knowledgeIds.add(knowledgeId);
}
return demandKnowledgeService.getKnowledgeChunkInfos(knowledgeIds, pagingInfo);
}
@Override
public void openKnowledgeChunk(Integer kdId, String chunkRelationId, String isOpen) {
BizKnowledgeDocumentEntity bizKnowledgeDocumentEntity = bizKnowledgeDocumentService.get(kdId);
String knowledgeId = bizKnowledgeDocumentEntity.getKnowledgeId();
demandKnowledgeService.openKnowledgeChunk(knowledgeId, chunkRelationId, isOpen);
}
@Override
public void deleteKnowledgeChunk(Integer kdId, String chunkRelationId) {
BizKnowledgeDocumentEntity bizKnowledgeDocumentEntity = bizKnowledgeDocumentService.get(kdId);
String knowledgeId = bizKnowledgeDocumentEntity.getKnowledgeId();
demandKnowledgeService.deleteKnowledgeChunk(knowledgeId, chunkRelationId);
}
@Override
public void updateKnowledgeChunkDoc(Integer kdId, String chunkRelationId, String content) {
BizKnowledgeDocumentEntity bizKnowledgeDocumentEntity = bizKnowledgeDocumentService.get(kdId);
String knowledgeId = bizKnowledgeDocumentEntity.getKnowledgeId();
demandKnowledgeService.updateKnowledgeChunkDoc(knowledgeId, chunkRelationId, content);
}
@Override
public void addKnowledgeChunk(Integer kdId, String content, Integer chunkSort) {
BizKnowledgeDocumentEntity bizKnowledgeDocumentEntity = bizKnowledgeDocumentService.get(kdId);
String knowledgeId = bizKnowledgeDocumentEntity.getKnowledgeId();
demandKnowledgeService.addKnowledgeChunk(knowledgeId, content, chunkSort);
}
}
src/main/java/cn/com/poc/knowledge/convert/BizKnowledgeInfoConvert.java 0 → 100644
View file @ 9aa5dba8
package cn.com.poc.knowledge.convert;
import cn.com.poc.knowledge.model.BizKnowledgeInfoModel;
import cn.com.poc.knowledge.entity.BizKnowledgeInfoEntity;
import cn.com.poc.knowledge.dto.BizKnowledgeInfoDto;
public class BizKnowledgeInfoConvert {
public static BizKnowledgeInfoEntity modelToEntity(BizKnowledgeInfoModel model){
BizKnowledgeInfoEntity entity = new BizKnowledgeInfoEntity();
entity.setId(model.getId());
entity.setKnowledgeName(model.getKnowledgeName());
entity.setMemberId(model.getMemberId());
entity.setKdIds(model.getKdIds());
entity.setTrainStatus(model.getTrainStatus());
entity.setDesc(model.getDesc());
entity.setIsDeleted(model.getIsDeleted());
entity.setCreator(model.getCreator());
entity.setCreatedTime(model.getCreatedTime());
entity.setModifier(model.getModifier());
entity.setModifiedTime(model.getModifiedTime());
entity.setSysVersion(model.getSysVersion());
return entity;
}
public static BizKnowledgeInfoModel entityToModel(BizKnowledgeInfoEntity entity){
BizKnowledgeInfoModel model = new BizKnowledgeInfoModel();
model.setId(entity.getId());
model.setKnowledgeName(entity.getKnowledgeName());
model.setMemberId(entity.getMemberId());
model.setKdIds(entity.getKdIds());
model.setTrainStatus(entity.getTrainStatus());
model.setDesc(entity.getDesc());
model.setIsDeleted(entity.getIsDeleted());
model.setCreator(entity.getCreator());
model.setCreatedTime(entity.getCreatedTime());
model.setModifier(entity.getModifier());
model.setModifiedTime(entity.getModifiedTime());
model.setSysVersion(entity.getSysVersion());
return model;
}
public static BizKnowledgeInfoDto entityToDto(BizKnowledgeInfoEntity entity){
BizKnowledgeInfoDto dto = new BizKnowledgeInfoDto();
dto.setId(entity.getId());
dto.setKnowledgeName(entity.getKnowledgeName());
dto.setMemberId(entity.getMemberId());
dto.setKdIds(entity.getKdIds());
dto.setTrainStatus(entity.getTrainStatus());
dto.setDesc(entity.getDesc());
return dto;
}
public static BizKnowledgeInfoEntity dtoToEntity(BizKnowledgeInfoDto dto){
BizKnowledgeInfoEntity entity = new BizKnowledgeInfoEntity();
entity.setId(dto.getId());
entity.setKnowledgeName(dto.getKnowledgeName());
entity.setMemberId(dto.getMemberId());
entity.setKdIds(dto.getKdIds());
entity.setTrainStatus(dto.getTrainStatus());
entity.setDesc(dto.getDesc());
return entity;
}
}
\ No newline at end of file
src/main/java/cn/com/poc/knowledge/dto/BizKnowledgeInfoDto.java 0 → 100644
View file @ 9aa5dba8
package cn.com.poc.knowledge.dto;
public class BizKnowledgeInfoDto {
/**
* id
*/
private java.lang.Integer id;
public java.lang.Integer getId() {
return this.id;
}
public void setId(java.lang.Integer id) {
this.id = id;
}
/**
* knowledge_name
* 知识库名字
*/
private java.lang.String knowledgeName;
public java.lang.String getKnowledgeName() {
return this.knowledgeName;
}
public void setKnowledgeName(java.lang.String knowledgeName) {
this.knowledgeName = knowledgeName;
}
/**
* member_id
* 用户ID
*/
private java.lang.String memberId;
public java.lang.String getMemberId() {
return this.memberId;
}
public void setMemberId(java.lang.String memberId) {
this.memberId = memberId;
}
/**
* kd_ids
* 关联的文件ID
*/
private java.lang.String kdIds;
public java.lang.String getKdIds() {
return this.kdIds;
}
public void setKdIds(java.lang.String kdIds) {
this.kdIds = kdIds;
}
/**
* train_status
* 状态：未执行-Unopened、 排队-Line 、训练中-Training、 完成-Complete、失败-Fail
*/
private java.lang.String trainStatus;
public java.lang.String getTrainStatus() {
return this.trainStatus;
}
public void setTrainStatus(java.lang.String trainStatus) {
this.trainStatus = trainStatus;
}
/**
* desc
*/
private java.lang.String desc;
public java.lang.String getDesc() {
return this.desc;
}
public void setDesc(java.lang.String desc) {
this.desc = desc;
}
}
\ No newline at end of file
src/main/java/cn/com/poc/knowledge/dto/TrainKnowledgeDto.java 0 → 100644
View file @ 9aa5dba8
package cn.com.poc.knowledge.dto;
import cn.com.poc.thirdparty.resource.demand.ai.entity.knowledge.SegmentationConfigRequest;
import java.io.Serializable;
import java.util.List;
public class TrainKnowledgeDto implements Serializable {
private Integer knowledgeInfoId;
private List<Integer> kdIds;
private SegmentationConfigRequest segmentationConfig;
public Integer getKnowledgeInfoId() {
return knowledgeInfoId;
}
public void setKnowledgeInfoId(Integer knowledgeInfoId) {
this.knowledgeInfoId = knowledgeInfoId;
}
public List<Integer> getKdIds() {
return kdIds;
}
public void setKdIds(List<Integer> kdIds) {
this.kdIds = kdIds;
}
public SegmentationConfigRequest getSegmentationConfig() {
return segmentationConfig;
}
public void setSegmentationConfig(SegmentationConfigRequest segmentationConfig) {
this.segmentationConfig = segmentationConfig;
}
}
src/main/java/cn/com/poc/knowledge/dto/UpsertChunkInfoDto.java 0 → 100644
View file @ 9aa5dba8
package cn.com.poc.knowledge.dto;
import cn.com.poc.support.dgTools.request.AbstractRequest;
import cn.com.poc.support.dgTools.result.AbstractResult;
import java.io.Serializable;
public class UpsertChunkInfoDto extends AbstractRequest<AbstractResult> implements Serializable {
/**
* 知识库ID
*/
private Integer kdId;
/**
* 大小块关联ID
*/
private String chunkRelationId;
/**
* 分片内容
*/
private String chunkContent;
/**
* 分片是否开启
*/
private String isOpen;
/**
* 分片排序
*/
private Integer chunkSort;
public Integer getKdId() {
return kdId;
}
public void setKdId(Integer kdId) {
this.kdId = kdId;
}
public String getChunkRelationId() {
return chunkRelationId;
}
public void setChunkRelationId(String chunkRelationId) {
this.chunkRelationId = chunkRelationId;
}
public String getChunkContent() {
return chunkContent;
}
public void setChunkContent(String chunkContent) {
this.chunkContent = chunkContent;
}
public String getIsOpen() {
return isOpen;
}
public void setIsOpen(String isOpen) {
this.isOpen = isOpen;
}
public Integer getChunkSort() {
return chunkSort;
}
public void setChunkSort(Integer chunkSort) {
this.chunkSort = chunkSort;
}
@Override
public String getMethod() throws Exception {
return null;
}
}
src/main/java/cn/com/poc/knowledge/entity/BizKnowledgeInfoEntity.java 0 → 100644
View file @ 9aa5dba8
package cn.com.poc.knowledge.entity;
public class BizKnowledgeInfoEntity {
private static final long serialVersionUID = 1L;
/** id
*
*/
private java.lang.Integer id;
public java.lang.Integer getId(){
return this.id;
}
public void setId(java.lang.Integer id){
this.id = id;
}
/** knowledge_name
*知识库名字
*/
private java.lang.String knowledgeName;
public java.lang.String getKnowledgeName(){
return this.knowledgeName;
}
public void setKnowledgeName(java.lang.String knowledgeName){
this.knowledgeName = knowledgeName;
}
/** member_id
*用户ID
*/
private java.lang.String memberId;
public java.lang.String getMemberId(){
return this.memberId;
}
public void setMemberId(java.lang.String memberId){
this.memberId = memberId;
}
/** kd_ids
*关联的文件ID
*/
private java.lang.String kdIds;
public java.lang.String getKdIds(){
return this.kdIds;
}
public void setKdIds(java.lang.String kdIds){
this.kdIds = kdIds;
}
/** train_status
*状态：未执行-Unopened、 排队-Line 、训练中-Training、 完成-Complete、失败-Fail
*/
private java.lang.String trainStatus;
public java.lang.String getTrainStatus(){
return this.trainStatus;
}
public void setTrainStatus(java.lang.String trainStatus){
this.trainStatus = trainStatus;
}
/** desc
*
*/
private java.lang.String desc;
public java.lang.String getDesc(){
return this.desc;
}
public void setDesc(java.lang.String desc){
this.desc = desc;
}
/** is_deleted
*是否删除 Y 是 N 否
*/
private java.lang.String isDeleted;
public java.lang.String getIsDeleted(){
return this.isDeleted;
}
public void setIsDeleted(java.lang.String isDeleted){
this.isDeleted = isDeleted;
}
/** CREATOR
*创建人
*/
private java.lang.String creator;
public java.lang.String getCreator(){
return this.creator;
}
public void setCreator(java.lang.String creator){
this.creator = creator;
}
/** CREATED_TIME
*创建时间
*/
private java.util.Date createdTime;
public java.util.Date getCreatedTime(){
return this.createdTime;
}
public void setCreatedTime(java.util.Date createdTime){
this.createdTime = createdTime;
}
/** MODIFIER
*修改人
*/
private java.lang.String modifier;
public java.lang.String getModifier(){
return this.modifier;
}
public void setModifier(java.lang.String modifier){
this.modifier = modifier;
}
/** MODIFIED_TIME
*修改时间
*/
private java.util.Date modifiedTime;
public java.util.Date getModifiedTime(){
return this.modifiedTime;
}
public void setModifiedTime(java.util.Date modifiedTime){
this.modifiedTime = modifiedTime;
}
/** SYS_VERSION
*乐观锁,版本号
*/
private java.lang.Integer sysVersion;
public java.lang.Integer getSysVersion(){
return this.sysVersion;
}
public void setSysVersion(java.lang.Integer sysVersion){
this.sysVersion = sysVersion;
}
}
\ No newline at end of file
src/main/java/cn/com/poc/knowledge/model/BizKnowledgeInfoModel.java 0 → 100644
View file @ 9aa5dba8
package cn.com.poc.knowledge.model;
import java.io.Serializable;
import cn.com.yict.framemax.data.model.BaseModelClass;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Table;
import javax.persistence.Id;
import org.hibernate.annotations.DynamicInsert;
import org.hibernate.annotations.DynamicUpdate;
import javax.persistence.Version;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
/**
* Model class for biz_knowledge_info
* 知识库信息配置
*/
@Entity
@Table(name = "biz_knowledge_info")
@DynamicInsert
@DynamicUpdate
public class BizKnowledgeInfoModel extends BaseModelClass implements Serializable {
private static final long serialVersionUID = 1L;
/** id
*
*/
private java.lang.Integer id;
@Column(name = "id",length = 10)
@Id @GeneratedValue(strategy = GenerationType.AUTO)
public java.lang.Integer getId(){
return this.id;
}
public void setId(java.lang.Integer id){
this.id = id;
super.addValidField("id");
}
/** knowledge_name
*知识库名字
*/
private java.lang.String knowledgeName;
@Column(name = "knowledge_name",length = 100)
public java.lang.String getKnowledgeName(){
return this.knowledgeName;
}
public void setKnowledgeName(java.lang.String knowledgeName){
this.knowledgeName = knowledgeName;
super.addValidField("knowledgeName");
}
/** member_id
*用户ID
*/
private java.lang.String memberId;
@Column(name = "member_id",length = 100)
public java.lang.String getMemberId(){
return this.memberId;
}
public void setMemberId(java.lang.String memberId){
this.memberId = memberId;
super.addValidField("memberId");
}
/** kd_ids
*关联的文件ID
*/
private java.lang.String kdIds;
@Column(name = "kd_ids",length = 1073741824)
public java.lang.String getKdIds(){
return this.kdIds;
}
public void setKdIds(java.lang.String kdIds){
this.kdIds = kdIds;
super.addValidField("kdIds");
}
/** train_status
*状态：未执行-Unopened、 排队-Line 、训练中-Training、 完成-Complete、失败-Fail
*/
private java.lang.String trainStatus;
@Column(name = "train_status",length = 15)
public java.lang.String getTrainStatus(){
return this.trainStatus;
}
public void setTrainStatus(java.lang.String trainStatus){
this.trainStatus = trainStatus;
super.addValidField("trainStatus");
}
/** desc
*
*/
private java.lang.String desc;
@Column(name = "desc",length = 500)
public java.lang.String getDesc(){
return this.desc;
}
public void setDesc(java.lang.String desc){
this.desc = desc;
super.addValidField("desc");
}
/** is_deleted
*是否删除 Y 是 N 否
*/
private java.lang.String isDeleted;
@Column(name = "is_deleted",length = 1)
public java.lang.String getIsDeleted(){
return this.isDeleted;
}
public void setIsDeleted(java.lang.String isDeleted){
this.isDeleted = isDeleted;
super.addValidField("isDeleted");
}
/** CREATOR
*创建人
*/
private java.lang.String creator;
@Column(name = "CREATOR",length = 50)
public java.lang.String getCreator(){
return this.creator;
}
public void setCreator(java.lang.String creator){
this.creator = creator;
super.addValidField("creator");
}
/** CREATED_TIME
*创建时间
*/
private java.util.Date createdTime;
@Column(name = "CREATED_TIME",length = 19)
public java.util.Date getCreatedTime(){
return this.createdTime;
}
public void setCreatedTime(java.util.Date createdTime){
this.createdTime = createdTime;
super.addValidField("createdTime");
}
/** MODIFIER
*修改人
*/
private java.lang.String modifier;
@Column(name = "MODIFIER",length = 50)
public java.lang.String getModifier(){
return this.modifier;
}
public void setModifier(java.lang.String modifier){
this.modifier = modifier;
super.addValidField("modifier");
}
/** MODIFIED_TIME
*修改时间
*/
private java.util.Date modifiedTime;
@Column(name = "MODIFIED_TIME",length = 19)
public java.util.Date getModifiedTime(){
return this.modifiedTime;
}
public void setModifiedTime(java.util.Date modifiedTime){
this.modifiedTime = modifiedTime;
super.addValidField("modifiedTime");
}
/** SYS_VERSION
*乐观锁,版本号
*/
private java.lang.Integer sysVersion;
@Column(name = "SYS_VERSION",length = 10)
@Version
public java.lang.Integer getSysVersion(){
return this.sysVersion;
}
public void setSysVersion(java.lang.Integer sysVersion){
this.sysVersion = sysVersion;
super.addValidField("sysVersion");
}
}
\ No newline at end of file
src/main/java/cn/com/poc/knowledge/query/KnowledgeInfosQuery.sql 0 → 100644
View file @ 9aa5dba8
select
id,
knowledge_name,
member_id,
kd_ids,
train_status,
`desc`,
is_deleted,
CREATOR,
CREATED_TIME,
MODIFIER,
MODIFIED_TIME
from
biz_knowledge_info
where
is_deleted = 'N'
<< and member_id = :memberId>>
<< and (LOCATE(:search ,knowledge_name) >>
<< and train_status = :trainStatus>>
order by MODIFIED_TIME
\ No newline at end of file
src/main/java/cn/com/poc/knowledge/query/KnowledgeInfosQueryCondition.java 0 → 100644
View file @ 9aa5dba8
package cn.com.poc.knowledge.query;
import java.io.Serializable;
/**
* Query Condition class for KnowledgeInfosQuery
*/
public class KnowledgeInfosQueryCondition implements Serializable{
private static final long serialVersionUID = 1L;
private java.lang.String memberId;
public java.lang.String getMemberId(){
return this.memberId;
}
public void setMemberId(java.lang.String memberId){
this.memberId = memberId;
}
private java.lang.String search;
public java.lang.String getSearch(){
return this.search;
}
public void setSearch(java.lang.String search){
this.search = search;
}
private java.lang.String trainStatus;
public java.lang.String getTrainStatus(){
return this.trainStatus;
}
public void setTrainStatus(java.lang.String trainStatus){
this.trainStatus = trainStatus;
}
}
\ No newline at end of file
src/main/java/cn/com/poc/knowledge/query/KnowledgeInfosQueryItem.java 0 → 100644
View file @ 9aa5dba8
package cn.com.poc.knowledge.query;
import java.io.Serializable;
import javax.persistence.Column;
import javax.persistence.Entity;
import cn.com.yict.framemax.data.model.BaseItemClass;
/**
* Query Item class for KnowledgeInfosQuery
*/
@Entity
public class KnowledgeInfosQueryItem extends BaseItemClass implements Serializable{
private static final long serialVersionUID = 1L;
/** id
*id
*/
private java.lang.Integer id;
@Column(name = "id")
public java.lang.Integer getId(){
return this.id;
}
public void setId(java.lang.Integer id){
this.id = id;
}
/** knowledge_name
*knowledge_name
*/
private java.lang.String knowledgeName;
@Column(name = "knowledge_name")
public java.lang.String getKnowledgeName(){
return this.knowledgeName;
}
public void setKnowledgeName(java.lang.String knowledgeName){
this.knowledgeName = knowledgeName;
}
/** member_id
*member_id
*/
private java.lang.String memberId;
@Column(name = "member_id")
public java.lang.String getMemberId(){
return this.memberId;
}
public void setMemberId(java.lang.String memberId){
this.memberId = memberId;
}
/** kd_ids
*kd_ids
*/
private java.lang.String kdIds;
@Column(name = "kd_ids")
public java.lang.String getKdIds(){
return this.kdIds;
}
public void setKdIds(java.lang.String kdIds){
this.kdIds = kdIds;
}
/** train_status
*train_status
*/
private java.lang.String trainStatus;
@Column(name = "train_status")
public java.lang.String getTrainStatus(){
return this.trainStatus;
}
public void setTrainStatus(java.lang.String trainStatus){
this.trainStatus = trainStatus;
}
/** desc
*desc
*/
private java.lang.String desc;
@Column(name = "desc")
public java.lang.String getDesc(){
return this.desc;
}
public void setDesc(java.lang.String desc){
this.desc = desc;
}
/** is_deleted
*is_deleted
*/
private java.lang.String isDeleted;
@Column(name = "is_deleted")
public java.lang.String getIsDeleted(){
return this.isDeleted;
}
public void setIsDeleted(java.lang.String isDeleted){
this.isDeleted = isDeleted;
}
/** CREATOR
*CREATOR
*/
private java.lang.String creator;
@Column(name = "CREATOR")
public java.lang.String getCreator(){
return this.creator;
}
public void setCreator(java.lang.String creator){
this.creator = creator;
}
/** CREATED_TIME
*CREATED_TIME
*/
private java.util.Date createdTime;
@Column(name = "CREATED_TIME")
public java.util.Date getCreatedTime(){
return this.createdTime;
}
public void setCreatedTime(java.util.Date createdTime){
this.createdTime = createdTime;
}
/** MODIFIER
*MODIFIER
*/
private java.lang.String modifier;
@Column(name = "MODIFIER")
public java.lang.String getModifier(){
return this.modifier;
}
public void setModifier(java.lang.String modifier){
this.modifier = modifier;
}
/** MODIFIED_TIME
*MODIFIED_TIME
*/
private java.util.Date modifiedTime;
@Column(name = "MODIFIED_TIME")
public java.util.Date getModifiedTime(){
return this.modifiedTime;
}
public void setModifiedTime(java.util.Date modifiedTime){
this.modifiedTime = modifiedTime;
}
}
\ No newline at end of file
src/main/java/cn/com/poc/knowledge/repository/BizKnowledgeInfoRepository.java 0 → 100644
View file @ 9aa5dba8
package cn.com.poc.knowledge.repository;
import cn.com.yict.framemax.data.repository.Repository;
import cn.com.poc.knowledge.model.BizKnowledgeInfoModel;
public interface BizKnowledgeInfoRepository extends Repository<BizKnowledgeInfoModel,java.lang.Integer> {
}
\ No newline at end of file
src/main/java/cn/com/poc/knowledge/rest/KnowledgeRest.java
View file @ 9aa5dba8
package cn.com.poc.knowledge.rest;
import cn.com.poc.knowledge.dto.BizKnowledgeDocumentDto;
import cn.com.poc.knowledge.dto.BizKnowledgeInfoDto;
import cn.com.poc.knowledge.dto.TrainKnowledgeDto;
import cn.com.poc.knowledge.dto.UpsertChunkInfoDto;
import cn.com.poc.thirdparty.resource.demand.ai.entity.knowledge.GetKnowledgeChunkInfoResult;
import cn.com.yict.framemax.core.rest.BaseRest;
import cn.com.yict.framemax.data.model.PagingInfo;
import cn.com.yict.framemax.web.permission.Access;
......@@ -14,7 +18,6 @@ import java.util.List;
@Permission(Access.Safety)
public interface KnowledgeRest extends BaseRest {
/**
* 上传文档
*
......@@ -39,15 +42,13 @@ public interface KnowledgeRest extends BaseRest {
/**
* 启用 / 关闭
*
* @
*/
String enableKnowledge(@RequestParam Integer kdId) throws Exception;
/**
* 开启训练
*/
Boolean trainKnowledge(@RequestBody List<Integer> kdIds) throws Exception;
Boolean trainKnowledge(@RequestBody TrainKnowledgeDto dto) throws Exception;
/**
* 获取用户知识库
......@@ -64,5 +65,41 @@ public interface KnowledgeRest extends BaseRest {
*/
List<BizKnowledgeDocumentDto> getListByKdIds(@RequestBody List<Integer> kdIds);
/**
* 获取用户知识库列表
*
* @param search 搜索条件
* @param trainStatus 训练状态
*/
List<BizKnowledgeInfoDto> getKnowledgeList(@RequestParam(required = false) String search, @RequestParam(required = false) String trainStatus, PagingInfo pagingInfo) throws Exception;
/**
* 创建知识库
*/
BizKnowledgeInfoDto createKnowledge(@RequestBody BizKnowledgeInfoDto dto) throws Exception;
/**
* 开关知识库分片
*/
void openKnowledgeChunk(@RequestBody UpsertChunkInfoDto dto);
/**
* 删除知识库分片
*/
void deleteKnowledgeChunk(@RequestBody UpsertChunkInfoDto dto);
/**
* 更新知识库分片内容
*/
void updateKnowledgeChunkDoc(@RequestBody UpsertChunkInfoDto dto);
/**
* 新增知识库分片
*/
void addKnowledgeChunk(@RequestBody UpsertChunkInfoDto dto);
/**
* 获取知识库分片信息
*/
GetKnowledgeChunkInfoResult getChunks(@RequestParam List<Integer> kdIds, PagingInfo pagingInfo);
}
src/main/java/cn/com/poc/knowledge/rest/impl/KnowledgeRestImpl.java
View file @ 9aa5dba8
package cn.com.poc.knowledge.rest.impl;
import cn.com.poc.common.utils.BlContext;
import cn.com.poc.knowledge.aggregate.KnowledgeService;
import cn.com.poc.knowledge.constant.KnowledgeConstant;
import cn.com.poc.knowledge.convert.BizKnowledgeInfoConvert;
import cn.com.poc.knowledge.dto.BizKnowledgeDocumentDto;
import cn.com.poc.knowledge.dto.BizKnowledgeInfoDto;
import cn.com.poc.knowledge.dto.TrainKnowledgeDto;
import cn.com.poc.knowledge.dto.UpsertChunkInfoDto;
import cn.com.poc.knowledge.entity.BizKnowledgeDocumentEntity;
import cn.com.poc.knowledge.entity.BizKnowledgeInfoEntity;
import cn.com.poc.knowledge.query.KnowledgeInfosQueryCondition;
import cn.com.poc.knowledge.query.KnowledgeInfosQueryItem;
import cn.com.poc.knowledge.rest.KnowledgeRest;
import cn.com.poc.knowledge.service.BizKnowledgeInfoService;
import cn.com.poc.support.security.oauth.entity.UserBaseEntity;
import cn.com.poc.thirdparty.resource.demand.ai.entity.knowledge.GetKnowledgeChunkInfoResult;
import cn.com.yict.framemax.data.model.PagingInfo;
import cn.hutool.core.bean.BeanUtil;
import cn.hutool.core.lang.Assert;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.pdfbox.pdmodel.graphics.blend.BlendComposite;
import org.beetl.ext.fn.StringUtil;
import org.springframework.stereotype.Component;
import org.springframework.web.multipart.MultipartFile;
......@@ -22,6 +37,9 @@ public class KnowledgeRestImpl implements KnowledgeRest {
@Resource
private KnowledgeService knowledgeService;
@Resource
private BizKnowledgeInfoService bizKnowledgeInfoService;
@Override
public List<BizKnowledgeDocumentDto> uploadDocument(MultipartFile[] documentFiles) throws Exception {
Assert.notEmpty(documentFiles);
......@@ -52,14 +70,14 @@ public class KnowledgeRestImpl implements KnowledgeRest {
}
@Override
public Boolean trainKnowledge(List<Integer> kdIds) throws Exception {
public Boolean trainKnowledge(TrainKnowledgeDto dto) throws Exception {
//执行任务
knowledgeService.trainKnowledge(kdIds);
knowledgeService.trainKnowledge(dto.getKnowledgeInfoId(), dto.getKdIds(), dto.getSegmentationConfig());
return true;
}
@Override
public List<BizKnowledgeDocumentDto> getList(String search,String trainStatus, PagingInfo pagingInfo) throws Exception {
public List<BizKnowledgeDocumentDto> getList(String search, String trainStatus, PagingInfo pagingInfo) throws Exception {
List<BizKnowledgeDocumentDto> result = new ArrayList<>();
List<BizKnowledgeDocumentEntity> entities = knowledgeService.getList(search, trainStatus, pagingInfo);
if (CollectionUtils.isNotEmpty(entities)) {
......@@ -86,4 +104,77 @@ public class KnowledgeRestImpl implements KnowledgeRest {
}
return res;
}
@Override
public List<BizKnowledgeInfoDto> getKnowledgeList(String search, String trainStatus, PagingInfo pagingInfo) throws Exception {
List<BizKnowledgeInfoDto> result = new ArrayList<>();
UserBaseEntity userBaseEntity = BlContext.getCurrentUserNotException();
KnowledgeInfosQueryCondition condition = new KnowledgeInfosQueryCondition();
condition.setMemberId(userBaseEntity.getUserId().toString());
if (StringUtils.isNoneBlank(search)) {
condition.setSearch(search);
}
if (StringUtils.isNoneBlank(trainStatus)) {
condition.setTrainStatus(trainStatus);
}
List<KnowledgeInfosQueryItem> items = bizKnowledgeInfoService.knowledgeInfos(condition, pagingInfo);
if (CollectionUtils.isNotEmpty(items)) {
result = items.stream().map(item -> {
BizKnowledgeInfoDto dto = new BizKnowledgeInfoDto();
BeanUtil.copyProperties(item, dto);
return dto;
}).collect(Collectors.toList());
}
return result;
}
@Override
public BizKnowledgeInfoDto createKnowledge(BizKnowledgeInfoDto dto) throws Exception {
Assert.notNull(dto.getKnowledgeName());
UserBaseEntity userBaseEntity = BlContext.getCurrentUserNotException();
BizKnowledgeInfoEntity bizKnowledgeInfoEntity = new BizKnowledgeInfoEntity();
bizKnowledgeInfoEntity.setKnowledgeName(dto.getKnowledgeName());
bizKnowledgeInfoEntity.setMemberId(userBaseEntity.getUserId().toString());
bizKnowledgeInfoEntity.setTrainStatus(KnowledgeConstant.TrainStatus.UNOPENED);
bizKnowledgeInfoEntity.setDesc(dto.getDesc());
BizKnowledgeInfoEntity entity = bizKnowledgeInfoService.save(bizKnowledgeInfoEntity);
return BizKnowledgeInfoConvert.entityToDto(entity);
}
@Override
public void openKnowledgeChunk(UpsertChunkInfoDto dto) {
Assert.notNull(dto.getIsOpen());
Assert.notNull(dto.getChunkRelationId());
Assert.notNull(dto.getKdId());
knowledgeService.openKnowledgeChunk(dto.getKdId(), dto.getChunkRelationId(), dto.getIsOpen());
}
@Override
public void deleteKnowledgeChunk(UpsertChunkInfoDto dto) {
Assert.notNull(dto.getKdId());
Assert.notNull(dto.getChunkRelationId());
knowledgeService.deleteKnowledgeChunk(dto.getKdId(), dto.getChunkRelationId());
}
@Override
public void updateKnowledgeChunkDoc(UpsertChunkInfoDto dto) {
Assert.notNull(dto.getChunkContent());
Assert.notNull(dto.getChunkRelationId());
Assert.notNull(dto.getKdId());
knowledgeService.updateKnowledgeChunkDoc(dto.getKdId(), dto.getChunkRelationId(), dto.getChunkContent());
}
@Override
public void addKnowledgeChunk(UpsertChunkInfoDto dto) {
Assert.notNull(dto.getChunkContent());
Assert.notNull(dto.getKdId());
Assert.notNull(dto.getChunkSort());
knowledgeService.addKnowledgeChunk(dto.getKdId(), dto.getChunkContent(), dto.getChunkSort());
}
@Override
public GetKnowledgeChunkInfoResult getChunks(List<Integer> kdIds, PagingInfo pagingInfo) {
Assert.notEmpty(kdIds);
return knowledgeService.getChunkInfo(kdIds, pagingInfo);
}
}
src/main/java/cn/com/poc/knowledge/scheduler/KnowledgeInfoScheduler.java 0 → 100644
View file @ 9aa5dba8
package cn.com.poc.knowledge.scheduler;
import cn.com.poc.knowledge.constant.KnowledgeConstant;
import cn.com.poc.knowledge.query.KnowledgeInfosQueryCondition;
import cn.com.poc.knowledge.query.KnowledgeInfosQueryItem;
import cn.com.poc.knowledge.service.BizKnowledgeInfoService;
import cn.com.poc.message.entity.KnowledgeTrainStatusMessage;
import cn.com.poc.message.service.KnowledgeProducerService;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import java.util.List;
@Component
public class KnowledgeInfoScheduler {
@Resource
private KnowledgeProducerService knowledgeProducerService;
@Resource
private BizKnowledgeInfoService bizKnowledgeInfoService;
@Scheduled(cron = "0 0/1 * * * ?")
public void knowledgeInfoStatusUpdateScheduler() throws Exception {
KnowledgeInfosQueryCondition condition = new KnowledgeInfosQueryCondition();
condition.setTrainStatus(KnowledgeConstant.TrainStatus.TRAINING);
List<KnowledgeInfosQueryItem> items = bizKnowledgeInfoService.knowledgeInfos(condition, null);
for (KnowledgeInfosQueryItem item : items) {
KnowledgeTrainStatusMessage knowledgeTrainStatusMessage = new KnowledgeTrainStatusMessage();
knowledgeTrainStatusMessage.setKnowledgeInfoId(item.getId());
knowledgeProducerService.knowledgeInfoStatusCheck(knowledgeTrainStatusMessage);
}
}
}
src/main/java/cn/com/poc/knowledge/service/BizKnowledgeInfoService.java 0 → 100644
View file @ 9aa5dba8
package cn.com.poc.knowledge.service;
import cn.com.poc.knowledge.query.KnowledgeInfosQueryCondition;
import cn.com.poc.knowledge.query.KnowledgeInfosQueryItem;
import cn.com.poc.knowledge.query.KnowledgeQueryItem;
import cn.com.yict.framemax.core.service.BaseService;
import cn.com.poc.knowledge.entity.BizKnowledgeInfoEntity;
import cn.com.yict.framemax.data.model.PagingInfo;
import java.util.Collection;
import java.util.List;
public interface BizKnowledgeInfoService extends BaseService {
BizKnowledgeInfoEntity get(java.lang.Integer id) throws Exception;
List<BizKnowledgeInfoEntity> findByExample(BizKnowledgeInfoEntity example, PagingInfo pagingInfo) throws Exception;
BizKnowledgeInfoEntity save(BizKnowledgeInfoEntity entity) throws Exception;
BizKnowledgeInfoEntity update(BizKnowledgeInfoEntity entity) throws Exception;
void deletedById(java.lang.Integer id) throws Exception;
List<KnowledgeInfosQueryItem> knowledgeInfos(KnowledgeInfosQueryCondition condition, PagingInfo pagingInfo) throws Exception;
}
\ No newline at end of file
src/main/java/cn/com/poc/knowledge/service/impl/BizKnowledgeInfoServiceImpl.java 0 → 100644
View file @ 9aa5dba8
package cn.com.poc.knowledge.service.impl;
import cn.com.poc.knowledge.query.KnowledgeInfosQueryCondition;
import cn.com.poc.knowledge.query.KnowledgeInfosQueryItem;
import cn.com.yict.framemax.core.service.impl.BaseServiceImpl;
import cn.com.poc.knowledge.service.BizKnowledgeInfoService;
import cn.com.poc.knowledge.model.BizKnowledgeInfoModel;
import cn.com.poc.knowledge.entity.BizKnowledgeInfoEntity;
import cn.com.poc.knowledge.convert.BizKnowledgeInfoConvert;
import cn.com.poc.knowledge.repository.BizKnowledgeInfoRepository;
import cn.com.yict.framemax.data.model.PagingInfo;
import org.springframework.stereotype.Service;
import org.apache.commons.collections4.CollectionUtils;
import java.util.ArrayList;
import java.util.stream.Collectors;
import java.util.List;
import javax.annotation.Resource;
import org.springframework.util.Assert;
@Service
public class BizKnowledgeInfoServiceImpl extends BaseServiceImpl
implements BizKnowledgeInfoService {
@Resource
private BizKnowledgeInfoRepository repository;
public BizKnowledgeInfoEntity get(java.lang.Integer id) throws Exception {
Assert.notNull(id);
BizKnowledgeInfoModel model = this.repository.get(id);
if (model == null) {
return null;
}
if ("Y".equals(model.getIsDeleted())) {
return null;
}
return BizKnowledgeInfoConvert.modelToEntity(model);
}
public List<BizKnowledgeInfoEntity> findByExample(BizKnowledgeInfoEntity example, PagingInfo pagingInfo) throws Exception {
List<BizKnowledgeInfoEntity> result = new ArrayList<BizKnowledgeInfoEntity>();
BizKnowledgeInfoModel model = new BizKnowledgeInfoModel();
if (example != null) {
model = BizKnowledgeInfoConvert.entityToModel(example);
}
model.setIsDeleted("N");
List<BizKnowledgeInfoModel> models = this.repository.findByExample(model, pagingInfo);
if (CollectionUtils.isNotEmpty(models)) {
result = models.stream().map(BizKnowledgeInfoConvert::modelToEntity).collect(Collectors.toList());
}
return result;
}
public BizKnowledgeInfoEntity save(BizKnowledgeInfoEntity entity) throws Exception {
Assert.notNull(entity);
entity.setId(null);
entity.setIsDeleted("N");
BizKnowledgeInfoModel model = BizKnowledgeInfoConvert.entityToModel(entity);
BizKnowledgeInfoModel saveModel = this.repository.save(model);
return BizKnowledgeInfoConvert.modelToEntity(saveModel);
}
public BizKnowledgeInfoEntity update(BizKnowledgeInfoEntity entity) throws Exception {
Assert.notNull(entity);
Assert.notNull(entity.getId(), "update pk can not be null");
BizKnowledgeInfoModel model = this.repository.get(entity.getId());
if (entity.getKnowledgeName() != null) {
model.setKnowledgeName(entity.getKnowledgeName());
}
if (entity.getMemberId() != null) {
model.setMemberId(entity.getMemberId());
}
if (entity.getKdIds() != null) {
model.setKdIds(entity.getKdIds());
}
if (entity.getTrainStatus() != null) {
model.setTrainStatus(entity.getTrainStatus());
}
if (entity.getDesc() != null) {
model.setDesc(entity.getDesc());
}
if (entity.getIsDeleted() != null) {
model.setIsDeleted(entity.getIsDeleted());
}
if (entity.getCreator() != null) {
model.setCreator(entity.getCreator());
}
if (entity.getCreatedTime() != null) {
model.setCreatedTime(entity.getCreatedTime());
}
if (entity.getModifier() != null) {
model.setModifier(entity.getModifier());
}
if (entity.getModifiedTime() != null) {
model.setModifiedTime(entity.getModifiedTime());
}
if (entity.getSysVersion() != null) {
model.setSysVersion(entity.getSysVersion());
}
BizKnowledgeInfoModel saveModel = this.repository.save(model);
return BizKnowledgeInfoConvert.modelToEntity(saveModel);
}
public void deletedById(java.lang.Integer id) throws Exception {
Assert.notNull(id);
BizKnowledgeInfoModel model = this.repository.get(id);
if (model != null) {
if ("N".equals(model.getIsDeleted())) {
model.setIsDeleted("Y");
this.repository.save(model);
}
}
}
@Override
public List<KnowledgeInfosQueryItem> knowledgeInfos(KnowledgeInfosQueryCondition condition, PagingInfo pagingInfo) throws Exception {
return this.sqlDao.query(condition, KnowledgeInfosQueryItem.class, pagingInfo);
}
}
\ No newline at end of file
src/main/java/cn/com/poc/message/entity/KnowledgeTrainStatusMessage.java
View file @ 9aa5dba8
......@@ -4,12 +4,22 @@ import java.io.Serializable;
public class KnowledgeTrainStatusMessage implements Serializable {
private Integer knowledgeInfoId;
private Integer kdId;
private String status;
private String knowledgeId;
public Integer getKnowledgeInfoId() {
return knowledgeInfoId;
}
public void setKnowledgeInfoId(Integer knowledgeInfoId) {
this.knowledgeInfoId = knowledgeInfoId;
}
public String getKnowledgeId() {
return knowledgeId;
}
......
src/main/java/cn/com/poc/message/entity/TrainKnowledgeMessage.java
View file @ 9aa5dba8
package cn.com.poc.message.entity;
import cn.com.poc.thirdparty.resource.demand.ai.entity.knowledge.SegmentationConfigRequest;
import java.io.Serializable;
import java.util.Map;
public class TrainKnowledgeMessage implements Serializable {
private Integer knowledgeInfoId;
private Integer kid;
......@@ -12,6 +15,24 @@ public class TrainKnowledgeMessage implements Serializable {
private Map<Integer, Integer> reduced;
private SegmentationConfigRequest segmentationConfig;
public Integer getKnowledgeInfoId() {
return knowledgeInfoId;
}
public void setKnowledgeInfoId(Integer knowledgeInfoId) {
this.knowledgeInfoId = knowledgeInfoId;
}
public SegmentationConfigRequest getSegmentationConfig() {
return segmentationConfig;
}
public void setSegmentationConfig(SegmentationConfigRequest segmentationConfig) {
this.segmentationConfig = segmentationConfig;
}
public Map<Integer, Integer> getReduced() {
return reduced;
}
......@@ -42,6 +63,7 @@ public class TrainKnowledgeMessage implements Serializable {
"kid=" + kid +
", fileUrl='" + fileUrl + '\'' +
", reduced=" + reduced +
", segmentationConfig=" + segmentationConfig +
'}';
}
}
src/main/java/cn/com/poc/message/service/KnowledgeConsumerService.java
View file @ 9aa5dba8
......@@ -19,4 +19,10 @@ public interface KnowledgeConsumerService extends BaseService {
* 训练状态更新
*/
void trainStatusUpdate(KnowledgeTrainStatusMessage message) throws Exception;
/**
* 检查知识库信息状态改变
*/
void knowledgeInfoStatusCheck(KnowledgeTrainStatusMessage knowledgeTrainStatusMessage) throws Exception;
}
src/main/java/cn/com/poc/message/service/KnowledgeProducerService.java
View file @ 9aa5dba8
......@@ -21,4 +21,10 @@ public interface KnowledgeProducerService extends BaseService {
*/
KnowledgeTrainStatusMessage trainStatusUpdate(KnowledgeTrainStatusMessage knowledgeTrainStatusMessage);
/**
* 检查知识库信息状态改变
*/
KnowledgeTrainStatusMessage knowledgeInfoStatusCheck(KnowledgeTrainStatusMessage knowledgeTrainStatusMessage);
}
src/main/java/cn/com/poc/message/service/impl/KnowledgeConsumerServiceImpl.java
View file @ 9aa5dba8
......@@ -2,13 +2,15 @@ package cn.com.poc.message.service.impl;
import cn.com.poc.common.constant.CommonConstant;
import cn.com.poc.common.utils.Assert;
import cn.com.poc.common.utils.JsonUtils;
import cn.com.poc.knowledge.constant.KnowledgeConstant;
import cn.com.poc.knowledge.entity.BizKnowledgeDocumentEntity;
import cn.com.poc.knowledge.entity.BizKnowledgeInfoEntity;
import cn.com.poc.knowledge.entity.BizKnowledgeTrainLogEntity;
import cn.com.poc.knowledge.entity.BizMemberKnowledgeRelationEntity;
import cn.com.poc.knowledge.service.BizKnowledgeDocumentService;
import cn.com.poc.knowledge.service.BizKnowledgeInfoService;
import cn.com.poc.knowledge.service.BizKnowledgeTrainLogService;
import cn.com.poc.knowledge.service.BizMemberKnowledgeRelationService;
import cn.com.poc.message.entity.KnowledgeTrainStatusMessage;
import cn.com.poc.message.entity.TrainKnowledgeMessage;
import cn.com.poc.message.service.KnowledgeConsumerService;
......@@ -17,13 +19,13 @@ import cn.com.poc.message.topic.KnowledgeTopic;
import cn.com.poc.thirdparty.resource.demand.ai.aggregate.DemandKnowledgeService;
import cn.com.yict.framemax.core.exception.BusinessException;
import cn.com.yict.framemax.tumbleweed.client.annotation.Consumer;
import org.apache.commons.collections4.MapUtils;
import com.alibaba.fastjson.TypeReference;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.security.core.parameters.P;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.Date;
import java.util.List;
......@@ -42,16 +44,10 @@ public class KnowledgeConsumerServiceImpl implements KnowledgeConsumerService {
private DemandKnowledgeService demandKnowledgeService;
@Resource
private BizMemberKnowledgeRelationService bizMemberKnowledgeRelationService;
// @Resource
// private BizAiGcMemberUsageDetailsService bizAiGcMemberUsageDetailsService;
//
// @Resource
// private AiGcMemberFreqProducerService aiGcMemberFreqProducerService;
private KnowledgeProducerService knowledgeProducerService;
@Resource
private KnowledgeProducerService knowledgeProducerService;
private BizKnowledgeInfoService bizKnowledgeInfoService;
/**
......@@ -61,7 +57,7 @@ public class KnowledgeConsumerServiceImpl implements KnowledgeConsumerService {
* @return
*/
@Override
@Consumer(topic = KnowledgeTopic.TRAIN_KNOWLEDGE, scale = 5, retry = false)
@Consumer(topic = KnowledgeTopic.TRAIN_KNOWLEDGE, scale = 3, retry = true)
public void trainKnowledge(TrainKnowledgeMessage message) throws Exception {
//修改训练状态
KnowledgeTrainStatusMessage trainStatusMessage = new KnowledgeTrainStatusMessage();
......@@ -69,7 +65,7 @@ public class KnowledgeConsumerServiceImpl implements KnowledgeConsumerService {
trainStatusMessage.setKdId(message.getKid());
knowledgeProducerService.trainStatusUpdate(trainStatusMessage);
try {
String knowledgeId = demandKnowledgeService.trainKnowledge(message.getFileUrl());
String knowledgeId = demandKnowledgeService.trainKnowledge(message.getFileUrl(), message.getSegmentationConfig());
//训练日志
BizKnowledgeTrainLogEntity bizKnowledgeTrainLogEntity = new BizKnowledgeTrainLogEntity();
......@@ -85,9 +81,8 @@ public class KnowledgeConsumerServiceImpl implements KnowledgeConsumerService {
completeMessage.setStatus(KnowledgeConstant.TrainStatus.COMPLETE);
completeMessage.setKdId(message.getKid());
completeMessage.setKnowledgeId(knowledgeId);
completeMessage.setKnowledgeInfoId(message.getKnowledgeInfoId());
knowledgeProducerService.trainStatusUpdate(completeMessage);
//保存明细
// saveUsageDetailByKnowledgeTrain(message.getKid());
} catch (BusinessException e) {
logger.warn("--------------message:{},知识库训练失败：{}------------", message, e.getMessage());
......@@ -107,14 +102,12 @@ public class KnowledgeConsumerServiceImpl implements KnowledgeConsumerService {
bizKnowledgeTrainLogService.save(bizKnowledgeTrainLogEntity);
logger.warn("-------保存知识库训练失败日志----------");
//次数回滚
// if (MapUtils.isNotEmpty(message.getReduced())) {
// logger.warn("----------知识库训练失败，训练次数回滚【开始】-------------");
// AiGcMemberFreqMessage aiGcMemberFreqMessage = new AiGcMemberFreqMessage();
// aiGcMemberFreqMessage.setReduceMap(message.getReduced());
// aiGcMemberFreqProducerService.rollbackNum(aiGcMemberFreqMessage);
// logger.warn("----------知识库训练失败，训练次数回滚【结束】-------------");
// }
BizKnowledgeInfoEntity bizKnowledgeInfoEntity = bizKnowledgeInfoService.get(message.getKnowledgeInfoId());
bizKnowledgeInfoEntity.setTrainStatus(KnowledgeConstant.TrainStatus.FAIL);
bizKnowledgeInfoService.update(bizKnowledgeInfoEntity);
logger.warn("-------保存知识库训练失败状态----------");
}
}
......@@ -126,28 +119,38 @@ public class KnowledgeConsumerServiceImpl implements KnowledgeConsumerService {
bizKnowledgeDocumentEntity.setTrainStatus(message.getStatus());
bizKnowledgeDocumentEntity.setKnowledgeId(message.getKnowledgeId());
bizKnowledgeDocumentService.update(message.getKdId(), bizKnowledgeDocumentEntity);
knowledgeProducerService.knowledgeInfoStatusCheck(message);
}
@Override
@Consumer(topic = KnowledgeTopic.TRAIN_STATUS, scale = 3, retry = true)
public void knowledgeInfoStatusCheck(KnowledgeTrainStatusMessage knowledgeTrainStatusMessage) throws Exception {
Assert.notNull(knowledgeTrainStatusMessage.getKnowledgeInfoId());
BizKnowledgeInfoEntity bizKnowledgeInfoEntity = bizKnowledgeInfoService.get(knowledgeTrainStatusMessage.getKnowledgeInfoId());
if (!KnowledgeConstant.TrainStatus.TRAINING.equals(bizKnowledgeInfoEntity.getTrainStatus())) {
return;
}
private void saveUsageDetailByKnowledgeTrain(Integer kdId) throws Exception {
BizMemberKnowledgeRelationEntity bizMemberKnowledgeRelationEntity = new BizMemberKnowledgeRelationEntity();
bizMemberKnowledgeRelationEntity.setMemberType("system");
bizMemberKnowledgeRelationEntity.setKdId(kdId);
bizMemberKnowledgeRelationEntity.setIsDeleted(CommonConstant.IsDeleted.N);
List<BizMemberKnowledgeRelationEntity> entities = bizMemberKnowledgeRelationService.findByExample(bizMemberKnowledgeRelationEntity, null);
// BizMemberKnowledgeRelationEntity memberKnowledgeRelationEntity = entities.get(0);
//
// BizKnowledgeDocumentEntity bizKnowledgeDocumentEntity = bizKnowledgeDocumentService.get(kdId);
// AiGcMemberUsageDetailsEntity aiGcMemberUsageDetailsEntity = new AiGcMemberUsageDetailsEntity();
// aiGcMemberUsageDetailsEntity.setRelationId(memberKnowledgeRelationEntity.getMemberId());
// aiGcMemberUsageDetailsEntity.setDomain(AiGcPaymentConfigurationConstant.DOMAIN.TRAIN);
// aiGcMemberUsageDetailsEntity.setTitile(bizKnowledgeDocumentEntity.getDocumentName());
// aiGcMemberUsageDetailsEntity.setDate(new Date());
// aiGcMemberUsageDetailsEntity.setScenes(null);
// aiGcMemberUsageDetailsEntity.setDetailType("知识库训练");
// aiGcMemberUsageDetailsEntity.setOperation(AiGcMemberUsageDetailsConstant.Operation.USE);
// aiGcMemberUsageDetailsEntity.setNum(bizKnowledgeDocumentEntity.getCharCount().intValue());
// aiGcMemberUsageDetailsEntity.setIsDeleted(MkpCommonConstant.IsDeleted.N);
// bizAiGcMemberUsageDetailsService.save(aiGcMemberUsageDetailsEntity);
String kdIds = bizKnowledgeInfoEntity.getKdIds();
List<Integer> kdIdList = JsonUtils.deSerialize(kdIds, new TypeReference<List<Integer>>() {
}.getType());
boolean isAllComplete = true;
logger.info("-------知识库训练状态检查，kdIds:{}-------", kdIds);
for (Integer kdId : kdIdList) {
BizKnowledgeDocumentEntity documentEntity = bizKnowledgeDocumentService.get(kdId);
if (!documentEntity.getTrainStatus().equals(KnowledgeConstant.TrainStatus.COMPLETE)) {
isAllComplete = false;
break;
}
}
if (isAllComplete) {
logger.info("-------知识库训练状态检查，全部完成, knowledgeInfoId:{}-------", knowledgeTrainStatusMessage.getKnowledgeInfoId());
bizKnowledgeInfoEntity.setTrainStatus(KnowledgeConstant.TrainStatus.COMPLETE);
bizKnowledgeInfoService.update(bizKnowledgeInfoEntity);
}
}
}
src/main/java/cn/com/poc/message/service/impl/KnowledgeProducerServiceImpl.java
View file @ 9aa5dba8
......@@ -31,4 +31,10 @@ public class KnowledgeProducerServiceImpl implements KnowledgeProducerService {
public KnowledgeTrainStatusMessage trainStatusUpdate(KnowledgeTrainStatusMessage knowledgeTrainStatusMessage) {
return knowledgeTrainStatusMessage;
}
@Override
@Producer(topic = KnowledgeTopic.KNOWLEDGE_INFO_STATUS_CHECK, publishPoint = PublishPoint.PUBLISH_REAL_TIME)
public KnowledgeTrainStatusMessage knowledgeInfoStatusCheck(KnowledgeTrainStatusMessage knowledgeTrainStatusMessage) {
return knowledgeTrainStatusMessage;
}
}
src/main/java/cn/com/poc/message/topic/KnowledgeTopic.java
View file @ 9aa5dba8
......@@ -7,5 +7,7 @@ public interface KnowledgeTopic {
String TRAIN_STATUS = "TRAIN_STATUS";
String KNOWLEDGE_INFO_STATUS_CHECK = "KNOWLEDGE_INFO_STATUS_CHECK";
}
src/main/java/cn/com/poc/support/dgTools/DgtoolsAbstractHttpClient.java
View file @ 9aa5dba8
......@@ -11,6 +11,7 @@ import cn.com.poc.support.dgTools.result.AbstractResult;
import cn.com.poc.support.dgTools.result.DgtoolsApiResult;
import cn.com.poc.support.dgTools.utils.AesEncodeUtil;
import cn.com.yict.framemax.core.exception.BusinessException;
import cn.com.yict.framemax.data.model.PagingInfo;
import cn.com.yict.framemax.frame.service.FmxParamConfigService;
import com.alibaba.fastjson.JSON;
import org.apache.commons.lang3.StringUtils;
......@@ -63,6 +64,19 @@ public class DgtoolsAbstractHttpClient {
* @param headers 请求头列表
*/
public <T extends AbstractResult> T doRequest(String url, AbstractRequest<T> abstractRequest, List<Header> headers) {
return doRequest(url, abstractRequest, headers, null);
}
/**
* 调用中台接口通用请求
* <br/>注：基于fastjson对JSON数据转换
*
* @param url 请求全路径
* @param abstractRequest 请求实体，需要继承响应体泛型的AbstractRequest
* @param headers 请求头列表
* @param pagingInfo 分页信息
*/
public <T extends AbstractResult> T doRequest(String url, AbstractRequest<T> abstractRequest, List<Header> headers, PagingInfo pagingInfo) {
Class<T> responseClass = abstractRequest.getResponseClass();
......@@ -97,6 +111,11 @@ public class DgtoolsAbstractHttpClient {
post.addHeader(header);
}
}
// 配置分页参数
if (pagingInfo != null) {
url = url + "?pageSize=" + pagingInfo.getPageSize() + "&pageNo=" + pagingInfo.getPageNo();
}
HttpUriRequest httpUriRequest = post
.setHeader(DgtoolsApiConstants.JSON_HEADER)
.setUri(dgtoolsDomainurl + DgtoolsApiConstants.BASE_URL + url)
......@@ -112,6 +131,10 @@ public class DgtoolsAbstractHttpClient {
//调用服务超时 2000毫秒
throw new BusinessException(dgtoolsApiResult.getMessage());
}
if (dgtoolsApiResult.getPagingInfo() != null) {
pagingInfo.setTotalRows(dgtoolsApiResult.getPagingInfo().getTotalRows());
pagingInfo.setPageSize(dgtoolsApiResult.getPagingInfo().getPageSize());
}
return JSON.parseObject(JSON.toJSONString(dgtoolsApiResult.getData()), responseClass);
}
......
src/main/java/cn/com/poc/support/dgTools/constants/DgtoolsApiConstants.java
View file @ 9aa5dba8
......@@ -236,11 +236,19 @@ public interface DgtoolsApiConstants {
String BAIDU_HANDWRITING = "aIRest/handWriting.json";
/**
* 知识库
*/
String TRAIN_KNOWLEDGE = "knowLedgeRest/trainKnowLedge.json";
String DEL_KNOWLEDGE = "knowLedgeRest/delKnowLedge.json";
String SEARCH_KNOWLEDGE = "knowLedgeRest/searchKnowledge.json";
String GET_KNOWLEDGE_CHUNK_INFOS = "/knowLedgeRest/getKnowledgeChunkInfos.json";
String OPEN_KNOWLEDGE_CHUNK = "/knowLedgeRest/openKnowledgeChunk.json";
String DELETE_KNOWLEDGE_CHUNK = "/knowLedgeRest/deleteKnowledgeChunk.json";
String UPDATE_KNOWLEDGE_CHUNK_DOC = "/knowLedgeRest/updateKnowledgeChunkDoc.json";
String ADD_KNOWLEDGE_CHUNK = "/knowLedgeRest/addKnowledgeChunk.json";
/**
* 大模型【通用】
......@@ -248,7 +256,7 @@ public interface DgtoolsApiConstants {
String LARGE_MODEL = "largeModelRest/completion.json";
}
interface ClickHouse{
interface ClickHouse {
String WEB_BROWSE_HARVEST_REPORT = "webBrowseHarvest/report.json";
}
......
src/main/java/cn/com/poc/support/dgTools/result/DgtoolsApiResult.java
View file @ 9aa5dba8
package cn.com.poc.support.dgTools.result;
import cn.com.poc.support.dgTools.constants.ErrorCodeEnum;
import cn.com.yict.framemax.data.model.PagingInfo;
/**
* @author Roger Wu
......@@ -13,6 +14,16 @@ public class DgtoolsApiResult {
private Object data;
private PagingInfo pagingInfo;
public PagingInfo getPagingInfo() {
return pagingInfo;
}
public void setPagingInfo(PagingInfo pagingInfo) {
this.pagingInfo = pagingInfo;
}
public String getCode() {
return code;
}
......
src/main/java/cn/com/poc/thirdparty/resource/demand/ai/aggregate/DemandKnowledgeService.java
View file @ 9aa5dba8
package cn.com.poc.thirdparty.resource.demand.ai.aggregate;
import cn.com.poc.thirdparty.resource.demand.ai.entity.knowledge.GetKnowledgeChunkInfoResult;
import cn.com.poc.thirdparty.resource.demand.ai.entity.knowledge.SegmentationConfigRequest;
import cn.com.yict.framemax.data.model.PagingInfo;
import java.util.List;
/**
......@@ -13,7 +17,7 @@ public interface DemandKnowledgeService {
* @param fileURL 训练文档
* @return 知识库id
*/
String trainKnowledge(String fileURL);
String trainKnowledge(String fileURL, SegmentationConfigRequest segmentationConfig);
/**
* 删除知识库
......@@ -32,4 +36,47 @@ public interface DemandKnowledgeService {
*/
List<String> searchKnowledge(String query, List<String> knowledgeIds, Integer topK);
/**
* 获取知识库分片
*
* @param knowledgeIds 知识库id
* @param pagingInfo 分页信息
*/
GetKnowledgeChunkInfoResult getKnowledgeChunkInfos(List<String> knowledgeIds, PagingInfo pagingInfo);
/**
* 开关知识库分片
*
* @param knowledgeId 知识库id
* @param chunkRelationId 分片关系id
* @param isOpen 是否开启 Y/N
*/
void openKnowledgeChunk(String knowledgeId, String chunkRelationId, String isOpen);
/**
* 删除知识库分片
*
* @param knowledgeId 知识库id
* @param chunkRelationId 分片关系id
*/
void deleteKnowledgeChunk(String knowledgeId, String chunkRelationId);
/**
* 更新知识库分片内容
*
* @param knowledgeId 知识库id
* @param chunkRelationId 分片关系id
* @param content 分片内容
*/
void updateKnowledgeChunkDoc(String knowledgeId, String chunkRelationId, String content);
/**
* 新增知识库分片
*
* @param knowledgeId 知识库id
* @param content 分片内容
* @param chunkSort 分片排序
*/
void addKnowledgeChunk(String knowledgeId, String content, Integer chunkSort);
}
src/main/java/cn/com/poc/thirdparty/resource/demand/ai/aggregate/impl/DemandKnowledgeServiceImpl.java
View file @ 9aa5dba8
package cn.com.poc.thirdparty.resource.demand.ai.aggregate.impl;
import cn.com.poc.common.utils.JsonUtils;
import cn.com.poc.common.utils.ListUtils;
import cn.com.poc.common.utils.http.LocalHttpClient;
import cn.com.poc.support.dgTools.result.DgtoolsApiResult;
import cn.com.poc.thirdparty.resource.demand.ai.aggregate.DemandKnowledgeService;
import cn.com.poc.thirdparty.resource.demand.ai.entity.knowledge.*;
import cn.com.poc.thirdparty.resource.demand.member.service.DemandAuthService;
import cn.com.poc.support.dgTools.DgtoolsAbstractHttpClient;
import cn.com.poc.support.dgTools.constants.DgtoolsApiConstants;
import cn.com.poc.support.dgTools.result.AbstractResult;
import cn.com.yict.framemax.core.config.Config;
import cn.com.yict.framemax.core.exception.BusinessException;
import cn.com.yict.framemax.data.model.PagingInfo;
import cn.hutool.core.lang.Assert;
import com.alibaba.fastjson.JSON;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest;
import org.apache.http.Header;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.entity.StringEntity;
import org.apache.http.message.BasicHeader;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
......@@ -27,10 +40,11 @@ public class DemandKnowledgeServiceImpl implements DemandKnowledgeService {
private DemandAuthService demandAuthService;
@Override
public String trainKnowledge(String fileURL) {
public String trainKnowledge(String fileURL, SegmentationConfigRequest segmentationConfig) {
Assert.notBlank(fileURL);
TrainKnowledgeRequest request = new TrainKnowledgeRequest();
request.setDocumentUrl(fileURL);
request.setSegmentationConfig(segmentationConfig);
TrainKnowledgeResult trainKnowledgeResult = dgToolsAbstractHttpClient.doRequest(DgtoolsApiConstants.DgtoolsAI.TRAIN_KNOWLEDGE, request, getHeaders());
if (null == trainKnowledgeResult) {
throw new BusinessException("train knowledge error");
......@@ -69,6 +83,48 @@ public class DemandKnowledgeServiceImpl implements DemandKnowledgeService {
return searchKnowledgeResult.getDocuments();
}
@Override
public GetKnowledgeChunkInfoResult getKnowledgeChunkInfos(List<String> knowledgeIds, PagingInfo pagingInfo) {
GetKnowledgeChunkInfoRequest request = new GetKnowledgeChunkInfoRequest();
request.setKnowledgeIds(knowledgeIds);
return dgToolsAbstractHttpClient.doRequest(DgtoolsApiConstants.DgtoolsAI.GET_KNOWLEDGE_CHUNK_INFOS, request, getHeaders(), pagingInfo);
}
@Override
public void openKnowledgeChunk(String knowledgeId, String chunkRelationId, String isOpen) {
UpsertChunkInfoRequest request = new UpsertChunkInfoRequest();
request.setKnowledgeId(knowledgeId);
request.setChunkRelationId(chunkRelationId);
request.setIsOpen(isOpen);
dgToolsAbstractHttpClient.doRequest(DgtoolsApiConstants.DgtoolsAI.OPEN_KNOWLEDGE_CHUNK, request, getHeaders());
}
@Override
public void deleteKnowledgeChunk(String knowledgeId, String chunkRelationId) {
UpsertChunkInfoRequest request = new UpsertChunkInfoRequest();
request.setKnowledgeId(knowledgeId);
request.setChunkRelationId(chunkRelationId);
dgToolsAbstractHttpClient.doRequest(DgtoolsApiConstants.DgtoolsAI.DELETE_KNOWLEDGE_CHUNK, request, getHeaders());
}
@Override
public void updateKnowledgeChunkDoc(String knowledgeId, String chunkRelationId, String content) {
UpsertChunkInfoRequest request = new UpsertChunkInfoRequest();
request.setKnowledgeId(knowledgeId);
request.setChunkRelationId(chunkRelationId);
request.setChunkContent(content);
dgToolsAbstractHttpClient.doRequest(DgtoolsApiConstants.DgtoolsAI.UPDATE_KNOWLEDGE_CHUNK_DOC, request, getHeaders());
}
@Override
public void addKnowledgeChunk(String knowledgeId, String content, Integer chunkSort) {
UpsertChunkInfoRequest request = new UpsertChunkInfoRequest();
request.setKnowledgeId(knowledgeId);
request.setChunkSort(chunkSort);
request.setChunkContent(content);
dgToolsAbstractHttpClient.doRequest(DgtoolsApiConstants.DgtoolsAI.ADD_KNOWLEDGE_CHUNK, request, getHeaders());
}
private List<Header> getHeaders() {
List<Header> headers = new ArrayList<>();
headers.add(DgtoolsApiConstants.JSON_HEADER);
......
src/main/java/cn/com/poc/thirdparty/resource/demand/ai/entity/knowledge/ChunkInfo.java 0 → 100644
View file @ 9aa5dba8
package cn.com.poc.thirdparty.resource.demand.ai.entity.knowledge;
public class ChunkInfo {
private String knowledgeId;
private String chunkContent;
private String chunkRelationId;
private Integer chunkSort;
public String getKnowledgeId() {
return knowledgeId;
}
public void setKnowledgeId(String knowledgeId) {
this.knowledgeId = knowledgeId;
}
public String getChunkContent() {
return chunkContent;
}
public void setChunkContent(String chunkContent) {
this.chunkContent = chunkContent;
}
public String getChunkRelationId() {
return chunkRelationId;
}
public void setChunkRelationId(String chunkRelationId) {
this.chunkRelationId = chunkRelationId;
}
public Integer getChunkSort() {
return chunkSort;
}
public void setChunkSort(Integer chunkSort) {
this.chunkSort = chunkSort;
}
@Override
public String toString() {
return "ChunkInfo{" +
"chunkContent='" + chunkContent + '\'' +
", chunkRelationId='" + chunkRelationId + '\'' +
", chunkSort=" + chunkSort +
'}';
}
}
src/main/java/cn/com/poc/thirdparty/resource/demand/ai/entity/knowledge/GetKnowledgeChunkInfoRequest.java 0 → 100644
View file @ 9aa5dba8
package cn.com.poc.thirdparty.resource.demand.ai.entity.knowledge;
import cn.com.poc.support.dgTools.request.AbstractRequest;
import java.io.Serializable;
import java.util.List;
public class GetKnowledgeChunkInfoRequest extends AbstractRequest<GetKnowledgeChunkInfoResult> implements Serializable {
private List<String> knowledgeIds;
public List<String> getKnowledgeIds() {
return knowledgeIds;
}
public void setKnowledgeIds(List<String> knowledgeIds) {
this.knowledgeIds = knowledgeIds;
}
@Override
public String getMethod() throws Exception {
return null;
}
}
src/main/java/cn/com/poc/thirdparty/resource/demand/ai/entity/knowledge/GetKnowledgeChunkInfoResult.java 0 → 100644
View file @ 9aa5dba8
package cn.com.poc.thirdparty.resource.demand.ai.entity.knowledge;
import cn.com.poc.support.dgTools.result.AbstractResult;
import java.io.Serializable;
import java.util.List;
public class GetKnowledgeChunkInfoResult extends AbstractResult implements Serializable {
private static final long serialVersionUID = 11243413567468L;
private List<ChunkInfo> chunkInfos;
public List<ChunkInfo> getChunkInfos() {
return chunkInfos;
}
public void setChunkInfos(List<ChunkInfo> chunkInfos) {
this.chunkInfos = chunkInfos;
}
@Override
public String toString() {
return "{" +
"chunkInfos:" + chunkInfos +
'}';
}
}
src/main/java/cn/com/poc/thirdparty/resource/demand/ai/entity/knowledge/SegmentationConfigRequest.java 0 → 100644
View file @ 9aa5dba8
package cn.com.poc.thirdparty.resource.demand.ai.entity.knowledge;
import java.util.List;
public class SegmentationConfigRequest {
/**
* 切分类型
* DEFAULT 默认-分片大小
* IDENTIFIER 标识符或正则表达式
* SEMANTIC 语义切分
*/
private String segmentationType;
/**
* 大分片大小
*/
private int chunkSize;
/**
* 小分片大小
*/
private int scrapSize;
/**
* 分片重复率
*/
private int repetitionRate;
/**
* 关联信息
*/
private List<String> relationInfo;
/**
* 自定义正则
*/
private String regex;
/**
* 标点符号
*/
private List<String> punctuations;
public String getSegmentationType() {
return segmentationType;
}
public void setSegmentationType(String segmentationType) {
this.segmentationType = segmentationType;
}
public int getChunkSize() {
return chunkSize;
}
public void setChunkSize(int chunkSize) {
this.chunkSize = chunkSize;
}
public int getScrapSize() {
return scrapSize;
}
public void setScrapSize(int scrapSize) {
this.scrapSize = scrapSize;
}
public int getRepetitionRate() {
return repetitionRate;
}
public void setRepetitionRate(int repetitionRate) {
this.repetitionRate = repetitionRate;
}
public List<String> getRelationInfo() {
return relationInfo;
}
public void setRelationInfo(List<String> relationInfo) {
this.relationInfo = relationInfo;
}
public String getRegex() {
return regex;
}
public void setRegex(String regex) {
this.regex = regex;
}
public List<String> getPunctuations() {
return punctuations;
}
public void setPunctuations(List<String> punctuations) {
this.punctuations = punctuations;
}
}
src/main/java/cn/com/poc/thirdparty/resource/demand/ai/entity/knowledge/TrainKnowledgeRequest.java
View file @ 9aa5dba8
......@@ -8,6 +8,16 @@ public class TrainKnowledgeRequest extends AbstractRequest<TrainKnowledgeResult>
private String documentUrl;
private SegmentationConfigRequest segmentationConfig;
public SegmentationConfigRequest getSegmentationConfig() {
return segmentationConfig;
}
public void setSegmentationConfig(SegmentationConfigRequest segmentationConfig) {
this.segmentationConfig = segmentationConfig;
}
public String getDocumentUrl() {
return documentUrl;
}
......
src/main/java/cn/com/poc/thirdparty/resource/demand/ai/entity/knowledge/UpsertChunkInfoRequest.java 0 → 100644
View file @ 9aa5dba8
package cn.com.poc.thirdparty.resource.demand.ai.entity.knowledge;
import cn.com.poc.support.dgTools.request.AbstractRequest;
import cn.com.poc.support.dgTools.result.AbstractResult;
import java.io.Serializable;
public class UpsertChunkInfoRequest extends AbstractRequest<AbstractResult> implements Serializable {
/**
* 知识库ID
*/
private String knowledgeId;
/**
* 大小块关联ID
*/
private String chunkRelationId;
/**
* 分片内容
*/
private String chunkContent;
/**
* 分片是否开启
*/
private String isOpen;
/**
* 分片排序
*/
private Integer chunkSort;
public String getKnowledgeId() {
return knowledgeId;
}
public void setKnowledgeId(String knowledgeId) {
this.knowledgeId = knowledgeId;
}
public String getChunkRelationId() {
return chunkRelationId;
}
public void setChunkRelationId(String chunkRelationId) {
this.chunkRelationId = chunkRelationId;
}
public String getChunkContent() {
return chunkContent;
}
public void setChunkContent(String chunkContent) {
this.chunkContent = chunkContent;
}
public String getIsOpen() {
return isOpen;
}
public void setIsOpen(String isOpen) {
this.isOpen = isOpen;
}
public Integer getChunkSort() {
return chunkSort;
}
public void setChunkSort(Integer chunkSort) {
this.chunkSort = chunkSort;
}
@Override
public String getMethod() throws Exception {
return null;
}
}
src/test/java/cn/com/poc/AgentApplicationInfoTest.java
View file @ 9aa5dba8
......@@ -2,7 +2,10 @@ package cn.com.poc;
import cn.com.poc.agent_application.aggregate.AgentApplicationInfoService;
import cn.com.poc.thirdparty.resource.demand.ai.aggregate.DemandKnowledgeService;
import cn.com.yict.framemax.core.spring.SingleContextInitializer;
import com.google.common.collect.Lists;
import org.apache.commons.collections4.ListUtils;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.test.context.ContextConfiguration;
......@@ -10,6 +13,10 @@ import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.springframework.test.context.web.WebAppConfiguration;
import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration(initializers = SingleContextInitializer.class)
......@@ -20,6 +27,9 @@ public class AgentApplicationInfoTest {
@Resource
private AgentApplicationInfoService applicationInfoService;
@Resource
private DemandKnowledgeService demandKnowledgeService;
/**
* Agent 应用标题，描述生成
*/
......@@ -28,4 +38,12 @@ public class AgentApplicationInfoTest {
String input = "测试标题";
System.out.println(applicationInfoService.createAgentTitleAndDesc(input));
}
@Test
public void knowledgeTest() {
ArrayList<String> list = new ArrayList<String>() {{
}};
System.out.println(list.toString());
}
}
src/test/java/cn/com/poc/knowledge/Demand.java 0 → 100644
View file @ 9aa5dba8
package cn.com.poc.knowledge;
import cn.com.poc.thirdparty.resource.demand.ai.aggregate.DemandKnowledgeService;
import cn.com.poc.thirdparty.resource.demand.ai.entity.knowledge.GetKnowledgeChunkInfoResult;
import cn.com.yict.framemax.core.spring.SingleContextInitializer;
import cn.com.yict.framemax.data.model.PagingInfo;
import cn.hutool.core.collection.ListUtil;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.springframework.test.context.web.WebAppConfiguration;
import javax.annotation.Resource;
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration(initializers = SingleContextInitializer.class)
@WebAppConfiguration
public class Demand {
@Resource
private DemandKnowledgeService demandKnowledgeService;
@Test
public void test() {
GetKnowledgeChunkInfoResult knowledgeChunkInfos = demandKnowledgeService.getKnowledgeChunkInfos(ListUtil.toList("KnowLedge-1050ef90-7232-487f-ac8b-501002c7b789","KnowLedge-315f8992-9aa2-4f0b-a06f-9298b1de7504"), new PagingInfo());
System.out.println(knowledgeChunkInfos);
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment