Skip to content

P
poc-api
Commit 75153ee9 authored by alex yao's avatar alex yao
Browse files
Options

feat: 知识库文档添加文件大小

parent 7f4d979d
Hide whitespace changes
Inline Side-by-side
Showing with 248 additions and 89 deletions
src/main/java/cn/com/poc/common/utils/FileUtils.java
View file @ 75153ee9
......@@ -13,6 +13,21 @@ import java.util.UUID;
*/
public class FileUtils {
/**
* 格式化文件大小
*/
public static String formatFileSize(long bytes) {
if (bytes < 1024) {
return bytes + " B";
} else if (bytes < 1024 * 1024) {
return String.format("%.2f KB", bytes / 1024.0);
} else if (bytes < 1024 * 1024 * 1024) {
return String.format("%.2f MB", bytes / (1024.0 * 1024));
} else {
return String.format("%.2f GB", bytes / (1024.0 * 1024 * 1024));
}
}
public static File multipartFileToFile(MultipartFile multiFile) {
// 获取文件名
......@@ -36,7 +51,7 @@ public class FileUtils {
* @param data
* @throws Exception
*/
public static File createTempFile(byte[] data,String fileSuffix) throws IOException {
public static File createTempFile(byte[] data, String fileSuffix) throws IOException {
if (data != null) {
File file = File.createTempFile(UUID.randomUUID().toString(), fileSuffix);
if (file.exists()) {
......
src/main/java/cn/com/poc/knowledge/aggregate/impl/KnowledgeServiceImpl.java
View file @ 75153ee9
......@@ -80,7 +80,7 @@ public class KnowledgeServiceImpl implements KnowledgeService {
// 文件大小不能超过10M
long fileSizeInBytes = file.length();
double fileSizeInMB = (double) fileSizeInBytes / (1024 * 1024);
if (fileSizeInMB > 5) {
if (fileSizeInMB > 10) {
throw new BusinessException("上传的文件不能超过5M,文件名:" + documentName);
}
......@@ -104,6 +104,7 @@ public class KnowledgeServiceImpl implements KnowledgeService {
bizKnowledgeDocumentEntity.setKnowledgeType(KnowledgeConstant.KnowledgeType.BASE);
bizKnowledgeDocumentEntity.setDocumentName(documentName);
bizKnowledgeDocumentEntity.setDocumentUrl(documentUrl);
bizKnowledgeDocumentEntity.setDocumentSize(fileSizeInBytes);
bizKnowledgeDocumentEntity.setCharCount(charCount);
bizKnowledgeDocumentEntity.setUploadTime(uploadTime);
bizKnowledgeDocumentEntity.setIsEnable(CommonConstant.YOrN.N);
......
src/main/java/cn/com/poc/knowledge/convert/KnowledgeDocumentConvert.java
View file @ 75153ee9
......@@ -24,6 +24,7 @@ public class KnowledgeDocumentConvert {
bizKnowledgeDocumentEntity.setSegmentationConfig(segmentationConfigRequest);
}
bizKnowledgeDocumentEntity.setKnowledgeId(model.getKnowledgeId());
bizKnowledgeDocumentEntity.setDocumentSize(model.getDocumentSize());
bizKnowledgeDocumentEntity.setTrainStatus(model.getTrainStatus());
bizKnowledgeDocumentEntity.setIsDeleted(model.getIsDeleted());
bizKnowledgeDocumentEntity.setCreator(model.getCreator());
......@@ -43,6 +44,7 @@ public class KnowledgeDocumentConvert {
bizKnowledgeDocumentModel.setDocumentUrl(entity.getDocumentUrl());
bizKnowledgeDocumentModel.setCharCount(entity.getCharCount());
bizKnowledgeDocumentModel.setUploadTime(entity.getUploadTime());
bizKnowledgeDocumentModel.setDocumentSize(entity.getDocumentSize());
bizKnowledgeDocumentModel.setIsEnable(entity.getIsEnable());
bizKnowledgeDocumentModel.setSegmentationConfig(JsonUtils.serialize(entity.getSegmentationConfig()));
bizKnowledgeDocumentModel.setKnowledgeId(entity.getKnowledgeId());
......
src/main/java/cn/com/poc/knowledge/dto/BizKnowledgeDocumentDto.java
View file @ 75153ee9
......@@ -9,10 +9,10 @@ import cn.com.poc.thirdparty.resource.demand.ai.entity.knowledge.SegmentationCon
public class BizKnowledgeDocumentDto {
/** kd_id
*主键
*/
/**
* kd_id
* 主键
*/
private Integer kdId;
public Integer getKdId() {
......@@ -23,128 +23,149 @@ public class BizKnowledgeDocumentDto {
this.kdId = kdId;
}
/** owner
*拥有者：System-系统 、User-用户
*/
/**
* owner
* 拥有者：System-系统 、User-用户
*/
private String owner;
public String getOwner(){
public String getOwner() {
return this.owner;
}
public void setOwner(String owner){
public void setOwner(String owner) {
this.owner = owner;
}
/** knowledge_type
*知识库类型 QA-问答、Base-基础
*/
/**
* knowledge_type
* 知识库类型 QA-问答、Base-基础
*/
private String knowledgeType;
public String getKnowledgeType(){
public String getKnowledgeType() {
return this.knowledgeType;
}
public void setKnowledgeType(String knowledgeType){
public void setKnowledgeType(String knowledgeType) {
this.knowledgeType = knowledgeType;
}
/** document_name
*文档名
*/
/**
* document_name
* 文档名
*/
private String documentName;
public String getDocumentName(){
public String getDocumentName() {
return this.documentName;
}
public void setDocumentName(String documentName){
public void setDocumentName(String documentName) {
this.documentName = documentName;
}
/** document_url
*文件下载地址
*/
/**
* document_url
* 文件下载地址
*/
private String documentUrl;
public String getDocumentUrl(){
public String getDocumentUrl() {
return this.documentUrl;
}
public void setDocumentUrl(String documentUrl){
public void setDocumentUrl(String documentUrl) {
this.documentUrl = documentUrl;
}
/** char_count
*文档字符数
*/
/**
* documentSize
*/
private String documentSize;
public String getDocumentSize() {
return documentSize;
}
public void setDocumentSize(String documentSize) {
this.documentSize = documentSize;
}
/**
* char_count
* 文档字符数
*/
private Long charCount;
public Long getCharCount(){
public Long getCharCount() {
return this.charCount;
}
public void setCharCount(Long charCount){
public void setCharCount(Long charCount) {
this.charCount = charCount;
}
/** upload_time
*上传时间
*/
/**
* upload_time
* 上传时间
*/
private java.util.Date uploadTime;
public java.util.Date getUploadTime(){
public java.util.Date getUploadTime() {
return this.uploadTime;
}
public void setUploadTime(java.util.Date uploadTime){
public void setUploadTime(java.util.Date uploadTime) {
this.uploadTime = uploadTime;
}
/** is_enable
*是否启用 N-不启用 Y-启用
*/
/**
* is_enable
* 是否启用 N-不启用 Y-启用
*/
private String isEnable;
public String getIsEnable(){
public String getIsEnable() {
return this.isEnable;
}
public void setIsEnable(String isEnable){
public void setIsEnable(String isEnable) {
this.isEnable = isEnable;
}
/** knowledge_id
*中台知识库ID
*/
/**
* knowledge_id
* 中台知识库ID
*/
private String knowledgeId;
public String getKnowledgeId(){
public String getKnowledgeId() {
return this.knowledgeId;
}
public void setKnowledgeId(String knowledgeId){
public void setKnowledgeId(String knowledgeId) {
this.knowledgeId = knowledgeId;
}
/** train_status
*状态：未执行-Unopened、 排队-Line 、训练中-Training、 完成-Complete、失败-Fail
*/
/**
* train_status
* 状态：未执行-Unopened、 排队-Line 、训练中-Training、 完成-Complete、失败-Fail
*/
private String trainStatus;
public String getTrainStatus(){
public String getTrainStatus() {
return this.trainStatus;
}
public void setTrainStatus(String trainStatus){
public void setTrainStatus(String trainStatus) {
this.trainStatus = trainStatus;
}
......
src/main/java/cn/com/poc/knowledge/entity/BizKnowledgeDocumentEntity.java
View file @ 75153ee9
......@@ -2,6 +2,8 @@ package cn.com.poc.knowledge.entity;
import cn.com.poc.thirdparty.resource.demand.ai.entity.knowledge.SegmentationConfigRequest;
import javax.persistence.Column;
/**
* Model class for biz_knowledge_document
* 知识库文件表
......@@ -148,6 +150,20 @@ public class BizKnowledgeDocumentEntity{
this.trainStatus = trainStatus;
}
/**
* 文档大小【单位：字节】
*/
private Long documentSize;
public Long getDocumentSize() {
return documentSize;
}
public void setDocumentSize(Long documentSize) {
this.documentSize = documentSize;
}
/**
* 分片规则
*/
......
src/main/java/cn/com/poc/knowledge/model/BizKnowledgeDocumentModel.java
View file @ 75153ee9
package cn.com.poc.knowledge.model;
import cn.com.poc.thirdparty.resource.demand.ai.entity.knowledge.SegmentationConfigRequest;
import cn.com.yict.framemax.data.model.BaseModelClass;
import org.hibernate.annotations.DynamicInsert;
import org.hibernate.annotations.DynamicUpdate;
......@@ -146,7 +145,7 @@ public class BizKnowledgeDocumentModel extends BaseModelClass implements Seriali
*/
private String segmentationConfig;
@Column(name = "segmentation_config",length = 1073741824)
@Column(name = "segmentation_config", length = 1073741824)
public String getSegmentationConfig() {
return segmentationConfig;
}
......@@ -189,6 +188,20 @@ public class BizKnowledgeDocumentModel extends BaseModelClass implements Seriali
super.addValidField("knowledgeId");
}
/**
* 文档大小【单位：字节】
*/
private Long documentSize;
@Column(name = "document_size", length = 19)
public Long getDocumentSize() {
return documentSize;
}
public void setDocumentSize(Long documentSize) {
this.documentSize = documentSize;
super.addValidField("documentSize");
}
/**
* train_status
......
src/main/java/cn/com/poc/knowledge/query/KnowledgeQuery.sql
View file @ 75153ee9
......@@ -6,6 +6,7 @@ select
bkd.document_url,
bkd.char_count,
bkd.segmentation_config,
bkd.document_size,
bkd.upload_time,
bkd.is_enable,
bkd.knowledge_id,
......
src/main/java/cn/com/poc/knowledge/query/KnowledgeQueryItem.java
View file @ 75153ee9
......@@ -104,6 +104,20 @@ public class KnowledgeQueryItem extends BaseItemClass implements Serializable {
this.charCount = charCount;
}
/**
* document_size
*/
private Long documentSize;
@Column(name = "document_size")
public Long getDocumentSize() {
return this.documentSize;
}
public void setDocumentSize(Long documentSize) {
this.documentSize = documentSize;
}
/**
* upload_time
* upload_time
......
src/main/java/cn/com/poc/knowledge/rest/impl/KnowledgeRestImpl.java
View file @ 75153ee9
......@@ -2,6 +2,7 @@ package cn.com.poc.knowledge.rest.impl;
import cn.com.poc.common.constant.CommonConstant;
import cn.com.poc.common.utils.BlContext;
import cn.com.poc.common.utils.FileUtils;
import cn.com.poc.common.utils.JsonUtils;
import cn.com.poc.knowledge.aggregate.KnowledgeService;
import cn.com.poc.knowledge.constant.KnowledgeConstant;
......@@ -50,6 +51,7 @@ public class KnowledgeRestImpl implements KnowledgeRest {
List<BizKnowledgeDocumentDto> result = entities.stream().map(entity -> {
BizKnowledgeDocumentDto dto = new BizKnowledgeDocumentDto();
BeanUtil.copyProperties(entity, dto);
dto.setDocumentSize(FileUtils.formatFileSize(entity.getDocumentSize()));
return dto;
}).collect(Collectors.toList());
return result;
......@@ -120,6 +122,7 @@ public class KnowledgeRestImpl implements KnowledgeRest {
result = entities.stream().map(entity -> {
BizKnowledgeDocumentDto dto = new BizKnowledgeDocumentDto();
BeanUtil.copyProperties(entity, dto);
dto.setDocumentSize(FileUtils.formatFileSize(entity.getDocumentSize()));
return dto;
}).collect(Collectors.toList());
}
......@@ -146,6 +149,7 @@ public class KnowledgeRestImpl implements KnowledgeRest {
res = entities.stream().map(entity -> {
BizKnowledgeDocumentDto dto = new BizKnowledgeDocumentDto();
BeanUtil.copyProperties(entity, dto);
dto.setDocumentSize(FileUtils.formatFileSize(entity.getDocumentSize()));
return dto;
}).collect(Collectors.toList());
}
......@@ -162,6 +166,7 @@ public class KnowledgeRestImpl implements KnowledgeRest {
res = entities.stream().map(entity -> {
BizKnowledgeDocumentDto dto = new BizKnowledgeDocumentDto();
BeanUtil.copyProperties(entity, dto);
dto.setDocumentSize(FileUtils.formatFileSize(entity.getDocumentSize()));
return dto;
}).collect(Collectors.toList());
}
......
src/test/java/cn/com/poc/knowledge/Demand.java
View file @ 75153ee9
package cn.com.poc.knowledge;
import cn.com.poc.common.utils.FileUtils;
import cn.com.poc.common.utils.JsonUtils;
import cn.com.poc.thirdparty.resource.demand.ai.aggregate.DemandKnowledgeService;
import cn.com.poc.thirdparty.resource.demand.ai.common.domain.Message;
import cn.com.poc.thirdparty.resource.demand.ai.common.domain.MultiContent;
import cn.com.poc.thirdparty.resource.demand.ai.entity.knowledge.GetKnowledgeChunkInfoResult;
import cn.com.poc.thirdparty.resource.demand.ai.entity.largemodel.LargeModelDemandResult;
import cn.com.poc.thirdparty.resource.demand.ai.entity.largemodel.LargeModelResponse;
import cn.com.poc.thirdparty.service.LLMService;
import cn.com.yict.framemax.core.exception.BusinessException;
import cn.com.yict.framemax.core.spring.SingleContextInitializer;
import cn.com.yict.framemax.data.model.PagingInfo;
import cn.hutool.core.collection.ListUtil;
import org.apache.commons.lang3.StringUtils;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.test.context.ContextConfiguration;
......@@ -13,19 +22,81 @@ import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.springframework.test.context.web.WebAppConfiguration;
import javax.annotation.Resource;
import java.io.BufferedReader;
import java.util.ArrayList;
import java.util.List;
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration(initializers = SingleContextInitializer.class)
@WebAppConfiguration
public class Demand {
final private String EVENT_STREAM_PREFIX = "data: ";
@Resource
private DemandKnowledgeService demandKnowledgeService;
@Resource
private LLMService llmService;
@Test
public void test() {
GetKnowledgeChunkInfoResult knowledgeChunkInfos = demandKnowledgeService.getKnowledgeChunkInfos(ListUtil.toList("KnowLedge-1050ef90-7232-487f-ac8b-501002c7b789","KnowLedge-315f8992-9aa2-4f0b-a06f-9298b1de7504"), new PagingInfo());
System.out.println(knowledgeChunkInfos);
String str = FileUtils.formatFileSize(1024);
System.out.println(str);
// GetKnowledgeChunkInfoResult knowledgeChunkInfos = demandKnowledgeService.getKnowledgeChunkInfos(ListUtil.toList("KnowLedge-1050ef90-7232-487f-ac8b-501002c7b789", "KnowLedge-315f8992-9aa2-4f0b-a06f-9298b1de7504"), new PagingInfo());
// System.out.println(knowledgeChunkInfos);
}
@Test
public void llmTest() throws Exception {
List<Message> messageList = new ArrayList<>();
List<MultiContent> multiContents = new ArrayList<>();
MultiContent multiContent = new MultiContent();
multiContents.add(multiContent);
multiContent.setType("text");
multiContent.setText("Hello, how are you?");
Message userMessage = new Message();
userMessage.setRole("user");
userMessage.setContent(multiContents);
messageList.add(userMessage);
String largeModel = "qwen-max";
LargeModelResponse largeModelResponse = new LargeModelResponse();
largeModelResponse.setModel(largeModel);
largeModelResponse.setMessages(messageList.toArray(new Message[0]));
largeModelResponse.setTopP(0F);
largeModelResponse.setStream(true);
largeModelResponse.setUser("POE");
BufferedReader bufferedReader = llmService.chatChunk(largeModelResponse);
String res = "";
bufferedReader.mark(200);
while ((res = bufferedReader.readLine()) != null) {
if (StringUtils.isBlank(res)) {
continue;
}
LargeModelDemandResult result = JsonUtils.deSerialize(res.replaceFirst(EVENT_STREAM_PREFIX, StringUtils.EMPTY), LargeModelDemandResult.class);
if (!"0".equals(result.getCode())) {
BusinessException ex = new BusinessException("ʧ");
throw ex;
}
bufferedReader.reset();
break;
}
while ((res = bufferedReader.readLine()) != null) {
if (StringUtils.isBlank(res)) {
continue;
}
LargeModelDemandResult result = JsonUtils.deSerialize(res.replaceFirst(EVENT_STREAM_PREFIX, StringUtils.EMPTY), LargeModelDemandResult.class);
if (!"0".equals(result.getCode())) {
BusinessException ex = new BusinessException("ʧ");
throw ex;
}
System.out.println("result:" + result);
}
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment