fix:配置网页读取超时

src/main/java/cn/com/poc/agent_application/aggregate/impl/AgentApplicationInfoServiceImpl.java

@@ -768,7 +768,7 @@ public class AgentApplicationInfoServiceImpl implements AgentApplicationInfoServ
             if (!"0".equals(result.getCode())) {
                 logger.error("LLM Error,code:{}", result.getCode());
                 I18nMessageException ex = new I18nMessageException("exception/call.failure");
-                writer.write(EVENT_STREAM_PREFIX + JsonUtils.serialize(ex) + "\n\n");
+                writer.write(EVENT_STREAM_PREFIX + JsonUtils.serialize(ex.getMessage()) + "\n\n");
                 writer.flush();
                 throw ex;
             }

src/main/java/cn/com/poc/thirdparty/resource/demand/ai/function/web_seach/WebSearchFunction.java

@@ -12,7 +12,6 @@ import cn.com.yict.framemax.core.config.Config;
 import cn.com.yict.framemax.core.exception.BusinessException;
 import cn.hutool.core.collection.ListUtil;
 import com.alibaba.fastjson.JSONObject;
-import com.baidubce.services.bec.model.vm.NetworkConfig;
 import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
 import com.google.api.client.http.HttpTransport;
 import com.google.api.client.http.apache.v2.ApacheHttpTransport;
@@ -26,8 +25,8 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Component;
 
-import java.util.ArrayList;
 import java.util.List;
+import java.util.concurrent.*;
 
 /**
  * @author alex.yao
@@ -38,6 +37,7 @@ public class WebSearchFunction extends AbstractLargeModelFunction {
 
     private Logger logger = LoggerFactory.getLogger(WebSearchFunction.class);
 
+    private CustomSearchAPI customSearchAPI;
     private final HttpTransport transport = new ApacheHttpTransport();
 
     private final JsonFactory jsonFactory = new GsonFactory();
@@ -54,6 +54,20 @@ public class WebSearchFunction extends AbstractLargeModelFunction {
                     .addProperties("query", new Properties("string", "搜索关键词")))
             .build();
 
+    private CustomSearchAPI getCustomSearchAPI() {
+        try {
+            if (customSearchAPI == null) {
+                customSearchAPI = new CustomSearchAPI.Builder(transport, jsonFactory, GoogleNetHttpTransport.newTrustedTransport().createRequestFactory().getInitializer())
+                        .setRootUrl("https://google-api.gsstcloud.com")
+                        .setCustomSearchAPIRequestInitializer(new CustomSearchAPIRequestInitializer("AIzaSyCV8PTQ10rG5wo4E004dR3mcGD1RM_PrBw"))
+                        .build();
+            }
+            return customSearchAPI;
+        } catch (Exception e) {
+            throw new BusinessException("初始化google搜索服务失败");
+        }
+    }
+
     @Override
     public String doFunction(String content, String identifier) {
         if (StringUtils.isBlank(content)) {
@@ -62,33 +76,16 @@ public class WebSearchFunction extends AbstractLargeModelFunction {
         JSONObject jsonObject = JSONObject.parseObject(content);
         String query = jsonObject.getString("query");
         query = query.replaceAll(StringUtils.SPACE, StringUtils.EMPTY);
-        List<WebSearchFunctionResult> results = new ArrayList<>();
+        List<WebSearchFunctionResult> results = new CopyOnWriteArrayList<>();
         try {
-            CustomSearchAPI customSearchAPI = new CustomSearchAPI.Builder(transport, jsonFactory, GoogleNetHttpTransport.newTrustedTransport().createRequestFactory().getInitializer())
-                    .setRootUrl("https://google-api.gsstcloud.com")
-                    .setCustomSearchAPIRequestInitializer(new CustomSearchAPIRequestInitializer("AIzaSyCV8PTQ10rG5wo4E004dR3mcGD1RM_PrBw"))
-                    .build();
-            Search execute = customSearchAPI
+            Search execute = getCustomSearchAPI()
                     .cse().list().setCx(CX)
                     .setKey(KEY)
                     .setQ(query)
-                    .setStart(1L)
+                    .setStart(2L)
                     .setNum(3)
                     .execute();
-            List<Result> items = execute.getItems();
-            for (Result item : items) {
-                String link = item.getLink();
-                String htmlContent = DocumentLoad.htmlToMarkdown(link);
-                if (StringUtils.isNotBlank(htmlContent)) {
-                    htmlContent = htmlContent.replaceAll(StringUtils.SPACE, StringUtils.EMPTY);
-                    if (htmlContent.length() > 10000) {
-                        htmlContent = htmlContent.substring(0, 10000);
-                    }
-                }
-                WebSearchFunctionResult webSearchResult = new WebSearchFunctionResult();
-                webSearchResult.setContent(htmlContent);
-                results.add(webSearchResult);
-            }
+            loadHTML(execute, results);
             logger.info("web search result：{}", results.toString());
             return JsonUtils.serialize(results);
         } catch (Exception e) {
@@ -97,6 +94,35 @@ public class WebSearchFunction extends AbstractLargeModelFunction {
         }
     }
 
+    private void loadHTML(Search execute, List<WebSearchFunctionResult> results) {
+        List<Result> items = execute.getItems();
+        try {
+            ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutor(3, 3, 10, TimeUnit.SECONDS, new LinkedBlockingQueue<>(3));
+            CountDownLatch countDownLatch = new CountDownLatch(items.size());
+            for (Result item : items) {
+                threadPoolExecutor.submit(() -> {
+                    String link = item.getLink();
+                    String htmlContent = DocumentLoad.htmlToMarkdown(link);
+                    if (StringUtils.isNotBlank(htmlContent)) {
+                        htmlContent = htmlContent.replaceAll(StringUtils.SPACE, StringUtils.EMPTY);
+                        if (htmlContent.length() > 1500) {
+                            int index = htmlContent.length() / 2;
+                            htmlContent = htmlContent.substring(index - 725, index + 725);
+                        }
+                    }
+                    WebSearchFunctionResult webSearchResult = new WebSearchFunctionResult();
+                    webSearchResult.setContent(htmlContent);
+                    results.add(webSearchResult);
+                    countDownLatch.countDown();
+                });
+            }
+            countDownLatch.await(30, TimeUnit.SECONDS);
+            threadPoolExecutor.shutdown();
+        } catch (Exception e) {
+            logger.error("web search error:" + e.getMessage());
+        }
+    }
+
     @Override
     public String getDesc() {
         return DESC;