feat: 知识库文档添加文件大小

src/main/java/cn/com/poc/common/utils/FileUtils.java

@@ -13,6 +13,21 @@ import java.util.UUID;
  */
 public class FileUtils {
 
+    /**
+     * 格式化文件大小
+     */
+    public static String formatFileSize(long bytes) {
+        if (bytes < 1024) {
+            return bytes + " B";
+        } else if (bytes < 1024 * 1024) {
+            return String.format("%.2f KB", bytes / 1024.0);
+        } else if (bytes < 1024 * 1024 * 1024) {
+            return String.format("%.2f MB", bytes / (1024.0 * 1024));
+        } else {
+            return String.format("%.2f GB", bytes / (1024.0 * 1024 * 1024));
+        }
+    }
+
     public static File multipartFileToFile(MultipartFile multiFile) {
 
         // 获取文件名
@@ -36,7 +51,7 @@ public class FileUtils {
      * @param data
      * @throws Exception
      */
-    public static File createTempFile(byte[] data,String fileSuffix) throws IOException {
+    public static File createTempFile(byte[] data, String fileSuffix) throws IOException {
         if (data != null) {
             File file = File.createTempFile(UUID.randomUUID().toString(), fileSuffix);
             if (file.exists()) {

src/main/java/cn/com/poc/knowledge/aggregate/impl/KnowledgeServiceImpl.java

@@ -80,7 +80,7 @@ public class KnowledgeServiceImpl implements KnowledgeService {
             // 文件大小不能超过10M
             long fileSizeInBytes = file.length();
             double fileSizeInMB = (double) fileSizeInBytes / (1024 * 1024);
-            if (fileSizeInMB > 5) {
+            if (fileSizeInMB > 10) {
                 throw new BusinessException("上传的文件不能超过5M,文件名:" + documentName);
             }
 
@@ -104,6 +104,7 @@ public class KnowledgeServiceImpl implements KnowledgeService {
             bizKnowledgeDocumentEntity.setKnowledgeType(KnowledgeConstant.KnowledgeType.BASE);
             bizKnowledgeDocumentEntity.setDocumentName(documentName);
             bizKnowledgeDocumentEntity.setDocumentUrl(documentUrl);
+            bizKnowledgeDocumentEntity.setDocumentSize(fileSizeInBytes);
             bizKnowledgeDocumentEntity.setCharCount(charCount);
             bizKnowledgeDocumentEntity.setUploadTime(uploadTime);
             bizKnowledgeDocumentEntity.setIsEnable(CommonConstant.YOrN.N);

src/main/java/cn/com/poc/knowledge/convert/KnowledgeDocumentConvert.java

@@ -24,6 +24,7 @@ public class KnowledgeDocumentConvert {
             bizKnowledgeDocumentEntity.setSegmentationConfig(segmentationConfigRequest);
         }
         bizKnowledgeDocumentEntity.setKnowledgeId(model.getKnowledgeId());
+        bizKnowledgeDocumentEntity.setDocumentSize(model.getDocumentSize());
         bizKnowledgeDocumentEntity.setTrainStatus(model.getTrainStatus());
         bizKnowledgeDocumentEntity.setIsDeleted(model.getIsDeleted());
         bizKnowledgeDocumentEntity.setCreator(model.getCreator());
@@ -43,6 +44,7 @@ public class KnowledgeDocumentConvert {
         bizKnowledgeDocumentModel.setDocumentUrl(entity.getDocumentUrl());
         bizKnowledgeDocumentModel.setCharCount(entity.getCharCount());
         bizKnowledgeDocumentModel.setUploadTime(entity.getUploadTime());
+        bizKnowledgeDocumentModel.setDocumentSize(entity.getDocumentSize());
         bizKnowledgeDocumentModel.setIsEnable(entity.getIsEnable());
         bizKnowledgeDocumentModel.setSegmentationConfig(JsonUtils.serialize(entity.getSegmentationConfig()));
         bizKnowledgeDocumentModel.setKnowledgeId(entity.getKnowledgeId());

src/main/java/cn/com/poc/knowledge/dto/BizKnowledgeDocumentDto.java

@@ -9,10 +9,10 @@ import cn.com.poc.thirdparty.resource.demand.ai.entity.knowledge.SegmentationCon
 public class BizKnowledgeDocumentDto {
 
 
-  
-    /** kd_id
-    *主键
-    */ 
+    /**
+     * kd_id
+     * 主键
+     */
     private Integer kdId;
 
     public Integer getKdId() {
@@ -23,128 +23,149 @@ public class BizKnowledgeDocumentDto {
         this.kdId = kdId;
     }
 
-    /** owner
-    *拥有者：System-系统 、User-用户
-    */ 
+    /**
+     * owner
+     * 拥有者：System-系统 、User-用户
+     */
     private String owner;
-    
-    public String getOwner(){
+
+    public String getOwner() {
         return this.owner;
     }
-    
-    public void setOwner(String owner){
+
+    public void setOwner(String owner) {
         this.owner = owner;
     }
-    
-  
-    /** knowledge_type
-    *知识库类型   QA-问答、Base-基础
-    */ 
+
+
+    /**
+     * knowledge_type
+     * 知识库类型   QA-问答、Base-基础
+     */
     private String knowledgeType;
-    
-    public String getKnowledgeType(){
+
+    public String getKnowledgeType() {
         return this.knowledgeType;
     }
-    
-    public void setKnowledgeType(String knowledgeType){
+
+    public void setKnowledgeType(String knowledgeType) {
         this.knowledgeType = knowledgeType;
     }
-    
-  
-    /** document_name
-    *文档名
-    */ 
+
+
+    /**
+     * document_name
+     * 文档名
+     */
     private String documentName;
-    
-    public String getDocumentName(){
+
+    public String getDocumentName() {
         return this.documentName;
     }
-    
-    public void setDocumentName(String documentName){
+
+    public void setDocumentName(String documentName) {
         this.documentName = documentName;
     }
-    
-  
-    /** document_url
-    *文件下载地址
-    */ 
+
+
+    /**
+     * document_url
+     * 文件下载地址
+     */
     private String documentUrl;
-    
-    public String getDocumentUrl(){
+
+    public String getDocumentUrl() {
         return this.documentUrl;
     }
-    
-    public void setDocumentUrl(String documentUrl){
+
+    public void setDocumentUrl(String documentUrl) {
         this.documentUrl = documentUrl;
     }
-    
-  
-    /** char_count
-    *文档字符数
-    */ 
+
+    /**
+     * documentSize
+     */
+    private String documentSize;
+
+    public String getDocumentSize() {
+        return documentSize;
+    }
+
+    public void setDocumentSize(String documentSize) {
+        this.documentSize = documentSize;
+    }
+
+    /**
+     * char_count
+     * 文档字符数
+     */
     private Long charCount;
-    
-    public Long getCharCount(){
+
+    public Long getCharCount() {
         return this.charCount;
     }
-    
-    public void setCharCount(Long charCount){
+
+    public void setCharCount(Long charCount) {
         this.charCount = charCount;
     }
-    
-  
-    /** upload_time
-    *上传时间
-    */ 
+
+
+    /**
+     * upload_time
+     * 上传时间
+     */
     private java.util.Date uploadTime;
-    
-    public java.util.Date getUploadTime(){
+
+    public java.util.Date getUploadTime() {
         return this.uploadTime;
     }
-    
-    public void setUploadTime(java.util.Date uploadTime){
+
+    public void setUploadTime(java.util.Date uploadTime) {
         this.uploadTime = uploadTime;
     }
-    
-  
-    /** is_enable
-    *是否启用 N-不启用 Y-启用
-    */ 
+
+
+    /**
+     * is_enable
+     * 是否启用 N-不启用 Y-启用
+     */
     private String isEnable;
-    
-    public String getIsEnable(){
+
+    public String getIsEnable() {
         return this.isEnable;
     }
-    
-    public void setIsEnable(String isEnable){
+
+    public void setIsEnable(String isEnable) {
         this.isEnable = isEnable;
     }
-    
-  
-    /** knowledge_id
-    *中台知识库ID
-    */ 
+
+
+    /**
+     * knowledge_id
+     * 中台知识库ID
+     */
     private String knowledgeId;
-    
-    public String getKnowledgeId(){
+
+    public String getKnowledgeId() {
         return this.knowledgeId;
     }
-    
-    public void setKnowledgeId(String knowledgeId){
+
+    public void setKnowledgeId(String knowledgeId) {
         this.knowledgeId = knowledgeId;
     }
-    
-  
-    /** train_status
-    *状态：未执行-Unopened、 排队-Line 、训练中-Training、 完成-Complete、失败-Fail
-    */ 
+
+
+    /**
+     * train_status
+     * 状态：未执行-Unopened、 排队-Line 、训练中-Training、 完成-Complete、失败-Fail
+     */
     private String trainStatus;
-    
-    public String getTrainStatus(){
+
+    public String getTrainStatus() {
         return this.trainStatus;
     }
-    
-    public void setTrainStatus(String trainStatus){
+
+    public void setTrainStatus(String trainStatus) {
         this.trainStatus = trainStatus;
     }
 

src/main/java/cn/com/poc/knowledge/entity/BizKnowledgeDocumentEntity.java

@@ -2,6 +2,8 @@ package cn.com.poc.knowledge.entity;
 
 import cn.com.poc.thirdparty.resource.demand.ai.entity.knowledge.SegmentationConfigRequest;
 
+import javax.persistence.Column;
+
 /**
  * Model class for biz_knowledge_document
  * 知识库文件表
@@ -148,6 +150,20 @@ public class BizKnowledgeDocumentEntity{
         this.trainStatus = trainStatus;
     }
 
+
+    /**
+     * 文档大小【单位：字节】
+     */
+    private Long documentSize;
+
+    public Long getDocumentSize() {
+        return documentSize;
+    }
+
+    public void setDocumentSize(Long documentSize) {
+        this.documentSize = documentSize;
+    }
+
     /**
      * 分片规则
      */

src/main/java/cn/com/poc/knowledge/model/BizKnowledgeDocumentModel.java

 package cn.com.poc.knowledge.model;
 
-import cn.com.poc.thirdparty.resource.demand.ai.entity.knowledge.SegmentationConfigRequest;
 import cn.com.yict.framemax.data.model.BaseModelClass;
 import org.hibernate.annotations.DynamicInsert;
 import org.hibernate.annotations.DynamicUpdate;
@@ -146,7 +145,7 @@ public class BizKnowledgeDocumentModel extends BaseModelClass implements Seriali
      */
     private String segmentationConfig;
 
-    @Column(name = "segmentation_config",length = 1073741824)
+    @Column(name = "segmentation_config", length = 1073741824)
     public String getSegmentationConfig() {
         return segmentationConfig;
     }
@@ -189,6 +188,20 @@ public class BizKnowledgeDocumentModel extends BaseModelClass implements Seriali
         super.addValidField("knowledgeId");
     }
 
+    /**
+     * 文档大小【单位：字节】
+     */
+    private Long documentSize;
+
+    @Column(name = "document_size", length = 19)
+    public Long getDocumentSize() {
+        return documentSize;
+    }
+
+    public void setDocumentSize(Long documentSize) {
+        this.documentSize = documentSize;
+        super.addValidField("documentSize");
+    }
 
     /**
      * train_status

src/main/java/cn/com/poc/knowledge/query/KnowledgeQuery.sql

@@ -6,6 +6,7 @@ select
     bkd.document_url,
     bkd.char_count,
     bkd.segmentation_config,
+    bkd.document_size,
     bkd.upload_time,
     bkd.is_enable,
     bkd.knowledge_id,

src/main/java/cn/com/poc/knowledge/query/KnowledgeQueryItem.java

@@ -104,6 +104,20 @@ public class KnowledgeQueryItem extends BaseItemClass implements Serializable {
         this.charCount = charCount;
     }
 
+    /**
+     * document_size
+     */
+    private Long documentSize;
+
+    @Column(name = "document_size")
+    public Long getDocumentSize() {
+        return this.documentSize;
+    }
+
+    public void setDocumentSize(Long documentSize) {
+        this.documentSize = documentSize;
+    }
+
     /**
      * upload_time
      * upload_time

src/main/java/cn/com/poc/knowledge/rest/impl/KnowledgeRestImpl.java

@@ -2,6 +2,7 @@ package cn.com.poc.knowledge.rest.impl;
 
 import cn.com.poc.common.constant.CommonConstant;
 import cn.com.poc.common.utils.BlContext;
+import cn.com.poc.common.utils.FileUtils;
 import cn.com.poc.common.utils.JsonUtils;
 import cn.com.poc.knowledge.aggregate.KnowledgeService;
 import cn.com.poc.knowledge.constant.KnowledgeConstant;
@@ -50,6 +51,7 @@ public class KnowledgeRestImpl implements KnowledgeRest {
         List<BizKnowledgeDocumentDto> result = entities.stream().map(entity -> {
             BizKnowledgeDocumentDto dto = new BizKnowledgeDocumentDto();
             BeanUtil.copyProperties(entity, dto);
+            dto.setDocumentSize(FileUtils.formatFileSize(entity.getDocumentSize()));
             return dto;
         }).collect(Collectors.toList());
         return result;
@@ -120,6 +122,7 @@ public class KnowledgeRestImpl implements KnowledgeRest {
             result = entities.stream().map(entity -> {
                 BizKnowledgeDocumentDto dto = new BizKnowledgeDocumentDto();
                 BeanUtil.copyProperties(entity, dto);
+                dto.setDocumentSize(FileUtils.formatFileSize(entity.getDocumentSize()));
                 return dto;
             }).collect(Collectors.toList());
         }
@@ -146,6 +149,7 @@ public class KnowledgeRestImpl implements KnowledgeRest {
                 res = entities.stream().map(entity -> {
                     BizKnowledgeDocumentDto dto = new BizKnowledgeDocumentDto();
                     BeanUtil.copyProperties(entity, dto);
+                    dto.setDocumentSize(FileUtils.formatFileSize(entity.getDocumentSize()));
                     return dto;
                 }).collect(Collectors.toList());
             }
@@ -162,6 +166,7 @@ public class KnowledgeRestImpl implements KnowledgeRest {
             res = entities.stream().map(entity -> {
                 BizKnowledgeDocumentDto dto = new BizKnowledgeDocumentDto();
                 BeanUtil.copyProperties(entity, dto);
+                dto.setDocumentSize(FileUtils.formatFileSize(entity.getDocumentSize()));
                 return dto;
             }).collect(Collectors.toList());
         }

src/test/java/cn/com/poc/knowledge/Demand.java

 package cn.com.poc.knowledge;
 
 
+import cn.com.poc.common.utils.FileUtils;
+import cn.com.poc.common.utils.JsonUtils;
 import cn.com.poc.thirdparty.resource.demand.ai.aggregate.DemandKnowledgeService;
+import cn.com.poc.thirdparty.resource.demand.ai.common.domain.Message;
+import cn.com.poc.thirdparty.resource.demand.ai.common.domain.MultiContent;
 import cn.com.poc.thirdparty.resource.demand.ai.entity.knowledge.GetKnowledgeChunkInfoResult;
+import cn.com.poc.thirdparty.resource.demand.ai.entity.largemodel.LargeModelDemandResult;
+import cn.com.poc.thirdparty.resource.demand.ai.entity.largemodel.LargeModelResponse;
+import cn.com.poc.thirdparty.service.LLMService;
+import cn.com.yict.framemax.core.exception.BusinessException;
 import cn.com.yict.framemax.core.spring.SingleContextInitializer;
 import cn.com.yict.framemax.data.model.PagingInfo;
 import cn.hutool.core.collection.ListUtil;
+import org.apache.commons.lang3.StringUtils;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.springframework.test.context.ContextConfiguration;
@@ -13,19 +22,81 @@ import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
 import org.springframework.test.context.web.WebAppConfiguration;
 
 import javax.annotation.Resource;
+import java.io.BufferedReader;
+import java.util.ArrayList;
+import java.util.List;
 
 @RunWith(SpringJUnit4ClassRunner.class)
 @ContextConfiguration(initializers = SingleContextInitializer.class)
 @WebAppConfiguration
 public class Demand {
 
+    final private String EVENT_STREAM_PREFIX = "data: ";
+
     @Resource
     private DemandKnowledgeService demandKnowledgeService;
 
+    @Resource
+    private LLMService llmService;
+
     @Test
     public void test() {
-        GetKnowledgeChunkInfoResult knowledgeChunkInfos = demandKnowledgeService.getKnowledgeChunkInfos(ListUtil.toList("KnowLedge-1050ef90-7232-487f-ac8b-501002c7b789","KnowLedge-315f8992-9aa2-4f0b-a06f-9298b1de7504"), new PagingInfo());
-        System.out.println(knowledgeChunkInfos);
+
+        String str = FileUtils.formatFileSize(1024);
+        System.out.println(str);
+//        GetKnowledgeChunkInfoResult knowledgeChunkInfos = demandKnowledgeService.getKnowledgeChunkInfos(ListUtil.toList("KnowLedge-1050ef90-7232-487f-ac8b-501002c7b789", "KnowLedge-315f8992-9aa2-4f0b-a06f-9298b1de7504"), new PagingInfo());
+//        System.out.println(knowledgeChunkInfos);
+    }
+
+    @Test
+    public void llmTest() throws Exception {
+        List<Message> messageList = new ArrayList<>();
+        List<MultiContent> multiContents = new ArrayList<>();
+        MultiContent multiContent = new MultiContent();
+        multiContents.add(multiContent);
+        multiContent.setType("text");
+        multiContent.setText("Hello, how are you?");
+        Message userMessage = new Message();
+        userMessage.setRole("user");
+        userMessage.setContent(multiContents);
+        messageList.add(userMessage);
+
+
+        String largeModel = "qwen-max";
+
+        LargeModelResponse largeModelResponse = new LargeModelResponse();
+        largeModelResponse.setModel(largeModel);
+        largeModelResponse.setMessages(messageList.toArray(new Message[0]));
+        largeModelResponse.setTopP(0F);
+        largeModelResponse.setStream(true);
+        largeModelResponse.setUser("POE");
+        BufferedReader bufferedReader = llmService.chatChunk(largeModelResponse);
+        String res = "";
+        bufferedReader.mark(200);
+        while ((res = bufferedReader.readLine()) != null) {
+            if (StringUtils.isBlank(res)) {
+                continue;
+            }
+            LargeModelDemandResult result = JsonUtils.deSerialize(res.replaceFirst(EVENT_STREAM_PREFIX, StringUtils.EMPTY), LargeModelDemandResult.class);
+            if (!"0".equals(result.getCode())) {
+                BusinessException ex = new BusinessException("ʧ");
+                throw ex;
+            }
+            bufferedReader.reset();
+            break;
+        }
+
+        while ((res = bufferedReader.readLine()) != null) {
+            if (StringUtils.isBlank(res)) {
+                continue;
+            }
+            LargeModelDemandResult result = JsonUtils.deSerialize(res.replaceFirst(EVENT_STREAM_PREFIX, StringUtils.EMPTY), LargeModelDemandResult.class);
+            if (!"0".equals(result.getCode())) {
+                BusinessException ex = new BusinessException("ʧ");
+                throw ex;
+            }
+            System.out.println("result:" + result);
+        }
 
     }