Merge branch 'feature' into 'release'

Feature

See merge request !574

pom.xml

@@ -269,7 +269,7 @@
         <dependency>
             <groupId>com.squareup.okhttp3</groupId>
             <artifactId>okhttp</artifactId>
-            <version>3.8.0</version>
+            <version>3.12.0</version>
         </dependency>
         <!-- dubbo -->
         <dependency>
@@ -324,6 +324,12 @@
             <artifactId>ansj_seg</artifactId>
             <version>5.0.2</version>
         </dependency>
+        <!--火山引擎 豆包大模型-->
+        <dependency>
+            <groupId>com.volcengine</groupId>
+            <artifactId>volcengine-java-sdk-ark-runtime</artifactId>
+            <version>0.1.121</version>
+        </dependency>
     </dependencies>
     <build>
         <plugins>

src/main/java/com/zhiwei/brandkbs2/aop/AopLogRecord.java

@@ -144,7 +144,7 @@ public class AopLogRecord {
         String[] value = method.getAnnotation(LogRecord.class).values();
         // 注解value为空字符串（value使用默认值未设置）
         if (1 == value.length && StringUtils.isEmpty(value[0])){
-            return new UserLogRecord(projectId, userInfo.getUserId(), userInfo.getNickname(), prefix, userInfo.getRoleId(), now, now);
+            return new UserLogRecord(projectId, userInfo.getUserId(), userInfo.getNickname(), prefix, userInfo.getRoleId(), now, now, null);
         }
         // 获取接口传参（value为获取传参的具体字段）并与操作描述description拼接返回，传参值为实体
         if (method.getAnnotation(LogRecord.class).arguments() && method.getAnnotation(LogRecord.class).entity()) {
@@ -176,7 +176,7 @@ public class AopLogRecord {
                 }
             }
             String suffix = CollectionUtils.isNotEmpty(res) ? "-" + Tools.concatWithMinus(res) : "";
-            return new UserLogRecord(projectId, userInfo.getUserId(), userInfo.getNickname(), prefix + suffix, userInfo.getRoleId(), now, now);
+            return new UserLogRecord(projectId, userInfo.getUserId(), userInfo.getNickname(), prefix + suffix, userInfo.getRoleId(), now, now, null);
         }
         // 获取接口传参（value为获取传参的具体字段）并与操作描述description拼接返回，传参值不为实体
         if (method.getAnnotation(LogRecord.class).arguments() && !method.getAnnotation(LogRecord.class).entity()) {
@@ -190,7 +190,7 @@ public class AopLogRecord {
                 }
             }
             String suffix = CollectionUtils.isNotEmpty(res) ? "-" + Tools.concatWithMinus(res) : "";
-            return new UserLogRecord(projectId, userInfo.getUserId(), userInfo.getNickname(), prefix + suffix, userInfo.getRoleId(), now, now);
+            return new UserLogRecord(projectId, userInfo.getUserId(), userInfo.getNickname(), prefix + suffix, userInfo.getRoleId(), now, now, null);
         }
         // 获取接口返回值（value为获取返回值的具体字段）并与操作描述description拼接返回，返回值为实体
         if (!method.getAnnotation(LogRecord.class).arguments() && method.getAnnotation(LogRecord.class).entity() && Objects.nonNull(responseResult)) {
@@ -204,9 +204,9 @@ public class AopLogRecord {
                 }
             }
             String suffix = CollectionUtils.isNotEmpty(res) ? "-" + Tools.concatWithMinus(res) : "";
-            return new UserLogRecord(projectId, userInfo.getUserId(), userInfo.getNickname(), prefix + suffix, userInfo.getRoleId(), now, now);
+            return new UserLogRecord(projectId, userInfo.getUserId(), userInfo.getNickname(), prefix + suffix, userInfo.getRoleId(), now, now, null);
         }
-        return new UserLogRecord(projectId, userInfo.getUserId(), userInfo.getNickname(), prefix, userInfo.getRoleId(), now, now);
+        return new UserLogRecord(projectId, userInfo.getUserId(), userInfo.getNickname(), prefix, userInfo.getRoleId(), now, now, null);
     }
 
     /**

src/main/java/com/zhiwei/brandkbs2/common/DoubaoAIAccountFactor.java

+package com.zhiwei.brandkbs2.common;
+
+import com.volcengine.ark.runtime.service.ArkService;
+import com.zhiwei.brandkbs2.pojo.ai.AccessModel;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @ClassName: DoubaoAIAccountFactor
+ * @Description DoubaoAIAccountFactor
+ * @author: sjj
+ * @date: 2024-07-24 14:01
+ */
+public class DoubaoAIAccountFactor {
+
+    public static Account getCompanyAccount() {
+        String apiKey = "607764fc-c9d9-47e4-a673-a310852917a0";
+        List<AccessModel> modelList = new ArrayList<>();
+        modelList.add(new AccessModel("ep-20240617061616-8d2ls", AccessModel.Model.DOUBAO_PRO_4K));
+        modelList.add(new AccessModel("ep-20240618021538-t6dpf", AccessModel.Model.DOUBAO_PRO_32K));
+        return new Account(apiKey, modelList);
+    }
+
+    @Data
+    @AllArgsConstructor
+    public static class Account {
+        String apiKey;
+        List<AccessModel> modelList;
+    }
+
+    public static ArkService arkService;
+
+    static {
+        arkService = new ArkService(getCompanyAccount().getApiKey());
+    }
+}

src/main/java/com/zhiwei/brandkbs2/common/RedisKeyPrefix.java

@@ -121,8 +121,13 @@ public class RedisKeyPrefix {
 
     public static final String CUSTOM_YUQING_ANALYZE_HIGH_WORD = "BRANDKBS:CUSTOM:YUQING:ANALYZE:HIGH:WORD:";
 
+    /**
+     * 搜索相关缓存
+     */
     public static final String SEARCH_KEYWORD = "BRANDKBS:SEARCH:KEYWORD:";
 
+    public static final String AI_SEARCH_QUESTION = "BRANDKBS:AI:SEARCH:QUESTION:";
+
     public static String projectWarnHotTopKeyAll(String projectId, String type) {
         return RedisKeyPrefix.generateRedisKey(RedisKeyPrefix.PROJECT_WARN_HOT_TOP, projectId, Tools.concat(type, "*"));
     }

src/main/java/com/zhiwei/brandkbs2/controller/app/AppDownloadController.java

@@ -396,9 +396,9 @@ public class AppDownloadController extends BaseController {
     @PostMapping(value = "/contend/mark")
     @DownloadTask(taskName = "竞品库竞品舆情下载", description = "竞品库竞品舆情")
     public ResponseResult exportContendMarkList(@RequestBody MarkSearchDTO markSearchDTO) {
-        if (StringUtils.isNotEmpty(markSearchDTO.getKeyword()) && Tools.checkUniteString(markSearchDTO.getKeyword())){
-            return ResponseResult.failure("不支持特殊符号字段查询");
-        }
+//        if (StringUtils.isNotEmpty(markSearchDTO.getKeyword()) && Tools.checkUniteString(markSearchDTO.getKeyword())){
+//            return ResponseResult.failure("不支持特殊符号字段查询");
+//        }
         Pair<String, List<ExportAppYuqingDTO>> stringListPair = markDataService.downloadContendMarkList(markSearchDTO);
         // excel写入至指定路径
         String projectName = projectService.getProjectById(UserThreadLocal.getProjectId()).getProjectName();
@@ -412,9 +412,9 @@ public class AppDownloadController extends BaseController {
     @LogRecord(description = "全网搜-舆情导出", values = {"startTime", "endTime", "fans", "filterType", "filterWords", "search", "keyword", "platforms", "sensitiveChannels", "sourceKeyword"}, entity = true, arguments = true)
     @DownloadTask(taskName = "全网搜舆情下载", description = "全网搜舆情")
     public ResponseResult exportSearchWhole(@RequestBody SearchFilterDTO dto) {
-        if (StringUtils.isNotEmpty(dto.getKeyword()) && Tools.checkUniteString(dto.getKeyword())){
-            return ResponseResult.failure("不支持特殊符号字段查询");
-        }
+//        if (StringUtils.isNotEmpty(dto.getKeyword()) && Tools.checkUniteString(dto.getKeyword())){
+//            return ResponseResult.failure("不支持特殊符号字段查询");
+//        }
         // 针对商业数据库做限制
         if (dto.isExternalDataSource()) {
             long time = DateUtils.addDays(Tools.truncDate(new Date(), Constant.DAY_PATTERN), -89).getTime();

src/main/java/com/zhiwei/brandkbs2/controller/app/AppSearchController.java

@@ -161,9 +161,9 @@ public class AppSearchController extends BaseController {
     @LogRecord(values = {"fans", "sensitiveChannels:father,son", "keyword", "search"}, description = "全网搜", arguments = true, entity = true)
     @PostMapping("/searchWhole")
     public ResponseResult searchWholeNetwork(@RequestBody SearchFilterDTO dto) {
-        if (StringUtils.isNotEmpty(dto.getKeyword()) && Tools.checkUniteString(dto.getKeyword())){
-            return ResponseResult.failure("不支持特殊符号字段查询");
-        }
+//        if (StringUtils.isNotEmpty(dto.getKeyword()) && Tools.checkUniteString(dto.getKeyword())){
+//            return ResponseResult.failure("不支持特殊符号字段查询");
+//        }
         cacheSearchKeyword(dto.getKeyword(), "whole");
         // 针对商业数据库做限制
         if (dto.isExternalDataSource()) {
@@ -200,9 +200,9 @@ public class AppSearchController extends BaseController {
     @PostMapping("/exportSearchWhole")
     @LogRecord(description = "全网搜-舆情导出", values = {"startTime", "endTime", "fans", "filterType", "filterWords", "search", "keyword", "platforms", "sensitiveChannels", "sourceKeyword"}, entity = true, arguments = true)
     public ResponseResult exportSearchWhole(@RequestBody SearchFilterDTO dto) {
-        if (StringUtils.isNotEmpty(dto.getKeyword()) && Tools.checkUniteString(dto.getKeyword())){
-            return ResponseResult.failure("不支持特殊符号字段查询");
-        }
+//        if (StringUtils.isNotEmpty(dto.getKeyword()) && Tools.checkUniteString(dto.getKeyword())){
+//            return ResponseResult.failure("不支持特殊符号字段查询");
+//        }
         // 针对商业数据库做限制
         if (dto.isExternalDataSource()) {
             long time = DateUtils.addDays(Tools.truncDate(new Date(), Constant.DAY_PATTERN), -89).getTime();
@@ -231,9 +231,9 @@ public class AppSearchController extends BaseController {
     @LogRecord(values = {"searchType", "keyword"}, description = "查舆情", arguments = true, entity = true)
     @PostMapping("/mark/list")
     public ResponseResult getYuqingMarkList(@RequestBody MarkSearchDTO markSearchDTO) {
-        if (StringUtils.isNotEmpty(markSearchDTO.getKeyword()) && Tools.checkUniteString(markSearchDTO.getKeyword())){
-            return ResponseResult.failure("不支持特殊符号字段查询");
-        }
+//        if (StringUtils.isNotEmpty(markSearchDTO.getKeyword()) && Tools.checkUniteString(markSearchDTO.getKeyword())){
+//            return ResponseResult.failure("不支持特殊符号字段查询");
+//        }
         cacheSearchKeyword(markSearchDTO.getKeyword(), "yuqing");
         PageVO<MarkFlowEntity> yuqingMarkList = markDataService.getYuqingMarkList(markSearchDTO);
         // 仅第一页增加平台进量（声量）统计
@@ -339,9 +339,9 @@ public class AppSearchController extends BaseController {
     @LogRecord(values = "keyword", description = "查竞品",arguments = true, entity = true)
     @PostMapping("/contend/list")
     public ResponseResult getContendSearchList(@RequestBody MarkSearchDTO markSearchDTO) {
-        if (StringUtils.isNotEmpty(markSearchDTO.getKeyword()) && Tools.checkUniteString(markSearchDTO.getKeyword())){
-            return ResponseResult.failure("不支持特殊符号字段查询");
-        }
+//        if (StringUtils.isNotEmpty(markSearchDTO.getKeyword()) && Tools.checkUniteString(markSearchDTO.getKeyword())){
+//            return ResponseResult.failure("不支持特殊符号字段查询");
+//        }
         cacheSearchKeyword(markSearchDTO.getKeyword(), "contend");
         return ResponseResult.success(markDataService.getContendSearchList(markSearchDTO));
     }
@@ -352,6 +352,34 @@ public class AppSearchController extends BaseController {
         return ResponseResult.success(markDataService.getContendSearchCriteria(contendId));
     }
 
+    @ApiOperation("搜索-AI搜索")
+    @GetMapping("/ai/answer")
+    public ResponseResult getAISearchResult(@RequestParam(value = "question") String question,
+                                            @RequestParam(value = "keyword", required = false) String keyword,
+                                            @RequestParam(value = "startTime", required = false) Long startTime,
+                                            @RequestParam(value = "endTime", required = false) Long endTime) {
+        return ResponseResult.success(markDataService.getAISearchResult(question, keyword, startTime, endTime));
+    }
+
+    @ApiOperation("搜索-AI搜索-联网搜索")
+    @GetMapping("/ai/answer/online")
+    public ResponseResult getAIOnlineSearchResult(@RequestParam(value = "question") String question) {
+        return ResponseResult.success(markDataService.getAIOnlineSearchResult(question));
+    }
+
+    @ApiOperation("搜索-AI推荐提问")
+    @GetMapping("/ai/question")
+    public ResponseResult getAIReferenceQuestion(@RequestParam(value = "question") String question,
+                                                 @RequestParam(value = "size") int size) {
+        return ResponseResult.success(markDataService.getAIReferenceQuestion(question, size));
+    }
+
+    @ApiOperation("搜索-AI参考提问")
+    @GetMapping("/ai/question-cache")
+    public ResponseResult getAIReferenceQuestion() {
+        return ResponseResult.success(markDataService.getAIReferenceQuestionCache(true));
+    }
+
     @ApiOperation("搜索-搜索关键词历史记录")
     @GetMapping("/keyword/cache")
     public ResponseResult getSearchKeywordCache(@ApiParam(name = "searchType",

src/main/java/com/zhiwei/brandkbs2/dao/AISearchQuestionRecordDao.java

+package com.zhiwei.brandkbs2.dao;
+
+import com.zhiwei.brandkbs2.pojo.AISearchQuestionRecord;
+
+import java.util.List;
+
+/**
+ * @ClassName: AISearchQuestionRecordDao
+ * @Description AISearchQuestionRecordDao
+ * @author: cjz
+ * @date: 2024-08-12 17:07
+ */
+public interface AISearchQuestionRecordDao extends BaseMongoDao<AISearchQuestionRecord>{
+
+    List<String> findDistinctQuestion(String projectId);
+}

src/main/java/com/zhiwei/brandkbs2/dao/impl/AISearchQuestionRecordDaoImpl.java

+package com.zhiwei.brandkbs2.dao.impl;
+
+import com.zhiwei.brandkbs2.dao.AISearchQuestionRecordDao;
+import com.zhiwei.brandkbs2.pojo.AISearchQuestionRecord;
+import org.springframework.data.domain.Sort;
+import org.springframework.data.mongodb.core.query.Criteria;
+import org.springframework.data.mongodb.core.query.Query;
+import org.springframework.stereotype.Component;
+
+import java.util.List;
+
+/**
+ * @ClassName: AISearchQuestionRecordDao
+ * @Description AISearchQuestionRecordDao
+ * @author: cjz
+ * @date: 2024-08-12 17:07
+ */
+@Component("aiSearchQuestionRecordDao")
+public class AISearchQuestionRecordDaoImpl extends BaseMongoDaoImpl<AISearchQuestionRecord> implements AISearchQuestionRecordDao {
+    private static final String COLLECTION_NAME = "brandkbs_ai_search_question_record";
+
+    public AISearchQuestionRecordDaoImpl() {
+        super(COLLECTION_NAME);
+    }
+
+    @Override
+    public List<String> findDistinctQuestion(String projectId) {
+        Query query = new Query().addCriteria(Criteria.where("projectId").is(projectId)).with(Sort.by(Sort.Order.desc("cTime"))).limit(10);
+        return mongoTemplate.findDistinct(query, "question", COLLECTION_NAME, String.class);
+    }
+}

src/main/java/com/zhiwei/brandkbs2/es/EsClientDao.java

 package com.zhiwei.brandkbs2.es;
 
 import com.alibaba.fastjson.JSONObject;
+import com.hankcs.hanlp.HanLP;
+import com.zhiwei.brandkbs2.auth.UserThreadLocal;
 import com.zhiwei.brandkbs2.common.GenericAttribute;
+import com.zhiwei.brandkbs2.common.GlobalPojo;
 import com.zhiwei.brandkbs2.config.Constant;
 import com.zhiwei.brandkbs2.pojo.ChannelIndex;
+import com.zhiwei.brandkbs2.pojo.Contend;
+import com.zhiwei.brandkbs2.pojo.Project;
+import com.zhiwei.brandkbs2.pojo.ai.FieldMapping;
+import com.zhiwei.brandkbs2.util.TextUtil;
 import com.zhiwei.brandkbs2.util.Tools;
 import lombok.Getter;
 import lombok.Setter;
@@ -33,6 +40,7 @@ import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
 import org.elasticsearch.search.sort.FieldSortBuilder;
+import org.elasticsearch.search.sort.SortOrder;
 import org.joda.time.Period;
 import org.joda.time.PeriodType;
 import org.springframework.beans.factory.annotation.Value;
@@ -331,6 +339,135 @@ public class EsClientDao {
         return Pair.of(new Long[]{startTime, endTime}, res);
     }
 
+    public List<JSONObject> findSearch(String question, String keyword, Long startTime, Long endTime) throws IOException {
+        List<JSONObject> list = new ArrayList<>();
+        String projectId = UserThreadLocal.getProjectId();
+        BoolQueryBuilder query = QueryBuilders.boolQuery();
+        // 默认一周
+        if (Objects.isNull(startTime) || Objects.isNull(endTime)){
+            endTime = System.currentTimeMillis();
+            startTime = System.currentTimeMillis() - Constant.ONE_WEEK;
+        }
+        // time
+        query.must(QueryBuilders.rangeQuery(GenericAttribute.ES_TIME).gte(startTime).lt(endTime));
+        // contend
+        Project project = GlobalPojo.PROJECT_MAP.get(projectId);
+        if (CollectionUtils.isNotEmpty(project.getContendList())) {
+            List<Contend> contendList = new ArrayList<>();
+            for (Contend contend : project.getContendList()) {
+                if (question.contains(contend.getBrandName())) {
+                    contendList.add(contend);
+                }
+            }
+            if (CollectionUtils.isNotEmpty(contendList)) {
+                BoolQueryBuilder contendQuery = QueryBuilders.boolQuery();
+                for (Contend contend : contendList) {
+                    contendQuery.should(EsQueryTools.assembleCacheMapsQuery(projectId, contend.getId()));
+                }
+                contendQuery.minimumShouldMatch(1);
+                query.must(contendQuery);
+            }else {
+                query.must(EsQueryTools.assembleCacheMapsQuery(projectId, Constant.PRIMARY_CONTEND_ID));
+            }
+        }
+        // keyword
+        query.must(EsQueryTools.assembleNormalKeywordQuery(keyword, new String[]{GenericAttribute.ES_IND_FULL_TEXT}));
+        String[] fetchSource = {"id", GenericAttribute.ES_TIME, GenericAttribute.ES_IND_TITLE};
+        // hit
+        SearchHit[] hits = searchHits(getIndexes(), query, null, fetchSource, null, 0, 10000, null).getHits();
+        Pair<Map<String, JSONObject>, Map<String, String>> searchProcess = findSearchResultProcess(hits);
+        Map<String, JSONObject> idBaseMap = searchProcess.getLeft();
+        Map<String, String> idTitle = searchProcess.getRight();
+        if (idTitle.isEmpty()){
+            return list;
+        }
+        // 按标题聚合，取聚合结果集前9，并取结果集中最新的文章的id
+        List<String> idList = TextUtil.getKResult(idTitle).stream()
+                .sorted(Comparator.comparing(List<String>::size, Comparator.reverseOrder()))
+                .limit(9)
+                .map(ids -> ids.stream().map(idBaseMap::get).max(Comparator.comparingLong(json -> json.getLongValue(GenericAttribute.ES_TIME))).orElse(null))
+                .filter(Objects::nonNull)
+                .map(json -> json.getString("id"))
+                .collect(Collectors.toList());
+        // 反查原数据
+        for (String id : idList) {
+            list.add(getTopTitleLatest(id));
+        }
+        return list;
+    }
+
+    public List<JSONObject> findSearch(List<FieldMapping> fieldMappings) throws IOException {
+        List<JSONObject> list = new ArrayList<>();
+        BoolQueryBuilder query = getBoolQueryBuilder(fieldMappings);
+        String[] fetchSource = {"id", GenericAttribute.ES_TIME, GenericAttribute.ES_IND_TITLE};
+        SearchHit[] hits = searchHits(getIndexes(), query, null, fetchSource, null, 0, 10000, null).getHits();
+        Pair<Map<String, JSONObject>, Map<String, String>> searchProcess = findSearchResultProcess(hits);
+        Map<String, JSONObject> idBaseMap = searchProcess.getLeft();
+        Map<String, String> idTitle = searchProcess.getRight();
+        // 搜索条件未找到结果，将搜索关键词分词处理，再次查询
+        if (idTitle.isEmpty()){
+            SearchHit[] searchHitHanLP = findSearchHanLP(fieldMappings, fetchSource);
+            Pair<Map<String, JSONObject>, Map<String, String>> searchProcessHanLP = findSearchResultProcess(searchHitHanLP);
+            idBaseMap = searchProcessHanLP.getLeft();
+            idTitle = searchProcessHanLP.getRight();
+        }
+        if (idTitle.isEmpty()){
+            return list;
+        }
+        // 按标题聚合，取聚合结果集前9，并取结果集中最新的文章的id
+        Map<String, JSONObject> finalIdBaseMap = idBaseMap;
+        List<String> idList = TextUtil.getKResult(idTitle).stream()
+                .sorted(Comparator.comparing(List<String>::size, Comparator.reverseOrder()))
+                .limit(9)
+                .map(ids -> ids.stream().map(finalIdBaseMap::get).max(Comparator.comparingLong(json -> json.getLongValue(GenericAttribute.ES_TIME))).orElse(null))
+                .filter(Objects::nonNull)
+                .map(json -> json.getString("id"))
+                .collect(Collectors.toList());
+        // 反查原数据
+        for (String id : idList) {
+            list.add(getTopTitleLatest(id));
+        }
+        return list;
+    }
+
+    private Pair<Map<String, JSONObject>, Map<String, String>> findSearchResultProcess(SearchHit[] hits){
+        Map<String, JSONObject> idBaseMap = Arrays.stream(hits).map(hit -> new JSONObject(hit.getSourceAsMap())).collect(Collectors.toMap(json -> json.getString("id"), o -> o));
+        Map<String, String> idTitle = Arrays.stream(hits)
+                .map(hit -> new JSONObject(hit.getSourceAsMap()))
+                .filter(json -> Objects.nonNull(json.getString(GenericAttribute.ES_IND_TITLE)) || Tools.filterUselessTitle(GenericAttribute.ES_IND_TITLE))
+                .collect(Collectors.toMap(json -> json.getString("id"), json -> json.getString(GenericAttribute.ES_IND_TITLE)));
+        return Pair.of(idBaseMap, idTitle);
+    }
+
+    private SearchHit[] findSearchHanLP(List<FieldMapping> fieldMappings, String[] fetchSource) throws IOException {
+        fieldMappings.stream().filter(fieldMapping -> Objects.equals(FieldMapping.FieldMap.IND_FULL_TEXT, fieldMapping.getFieldMap()))
+                .findFirst().ifPresent(fieldMapping -> {
+                    String value = String.valueOf(fieldMapping.getValue());
+                    String newValue = HanLP.segment(Tools.filterSpecialCharacter(value)).stream().map(s -> s.word).distinct().collect(Collectors.joining(" "));
+                    fieldMapping.setValue(newValue);
+                });
+        BoolQueryBuilder query = getBoolQueryBuilder(fieldMappings);
+        return searchHits(getIndexes(), query, null, fetchSource, null, 0, 10000, null).getHits();
+    }
+
+    private JSONObject getTopTitleLatest(String id) throws IOException {
+        BoolQueryBuilder query = QueryBuilders.boolQuery();
+        query.must(QueryBuilders.termQuery("id", id));
+        return searchById(id);
+    }
+
+    private BoolQueryBuilder getBoolQueryBuilder(List<FieldMapping> fieldMappings) {
+        BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
+        Map<String, List<FieldMapping>> groupMap = fieldMappings.stream().collect(Collectors.groupingBy(mapping -> mapping.getFieldMap().getFatherName()));
+        groupMap.forEach((fatherName, list) -> {
+            if (list.size() > 2) {
+                throw new IllegalStateException("构建搜索条件分组异常");
+            }
+            boolQueryBuilder.must(list.get(0).buildQuery(list.size() > 1 ? list.get(1) : null));
+        });
+        return boolQueryBuilder;
+    }
+
     public String[] getIndexes() {
         return getIndexList().toArray(new String[0]);
     }

src/main/java/com/zhiwei/brandkbs2/pojo/AISearchQuestionRecord.java

+package com.zhiwei.brandkbs2.pojo;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.Setter;
+
+/**
+ * @ClassName: AISearchQuestionRecord
+ * @Description ai搜索问题记录
+ * @author: cjz
+ * @date: 2024-8-12 14:58
+ */
+@Getter
+@Setter
+@AllArgsConstructor
+public class AISearchQuestionRecord extends AbstractBaseMongo {
+    /**
+     * 问题
+     */
+    private String question;
+
+    /**
+     * 项目id
+     */
+    private String projectId;
+
+    /**
+     * 创建时间
+     */
+    private Long cTime;
+}

src/main/java/com/zhiwei/brandkbs2/pojo/UserLogRecord.java

 package com.zhiwei.brandkbs2.pojo;
 
+import com.zhiwei.brandkbs2.auth.UserThreadLocal;
 import lombok.AllArgsConstructor;
 import lombok.Getter;
+import lombok.NoArgsConstructor;
 import lombok.Setter;
-import org.springframework.data.mongodb.core.mapping.Document;
 
 /**
  * @ClassName: UserLogRecord
@@ -14,6 +15,7 @@ import org.springframework.data.mongodb.core.mapping.Document;
 @Getter
 @Setter
 @AllArgsConstructor
+@NoArgsConstructor
 public class UserLogRecord extends AbstractBaseMongo{
     /**
      * 项目id
@@ -43,4 +45,33 @@ public class UserLogRecord extends AbstractBaseMongo{
      * 更新时间
      */
     private Long updateTime;
+    /**
+     * 本次调用费用（ai搜索）
+     */
+    private Double cost;
+
+    public static UserLogRecord userLogRecordCost(String description, double cost){
+        UserLogRecord record = new UserLogRecord();
+        record.setProjectId(UserThreadLocal.getProjectId());
+        record.setUserId(UserThreadLocal.getUserId());
+        record.setNickname(UserThreadLocal.getNickname());
+        record.setDescription(description);
+        record.setRoleId(UserThreadLocal.getRoleId());
+        record.setCTime(System.currentTimeMillis());
+        record.setUpdateTime(System.currentTimeMillis());
+        record.setCost(cost);
+        return record;
+    }
+
+    public static UserLogRecord defaultUserLogRecord(String description){
+        UserLogRecord record = new UserLogRecord();
+        record.setProjectId(UserThreadLocal.getProjectId());
+        record.setUserId(UserThreadLocal.getUserId());
+        record.setNickname(UserThreadLocal.getNickname());
+        record.setDescription(description);
+        record.setRoleId(UserThreadLocal.getRoleId());
+        record.setCTime(System.currentTimeMillis());
+        record.setUpdateTime(System.currentTimeMillis());
+        return record;
+    }
 }

src/main/java/com/zhiwei/brandkbs2/pojo/ai/AccessModel.java

+package com.zhiwei.brandkbs2.pojo.ai;
+
+import lombok.Data;
+import lombok.Getter;
+
+/**
+ * @ClassName: AccessModel
+ * @Description AccessModel
+ * @author: sjj
+ * @date: 2024-07-24 14:05
+ */
+@Data
+public class AccessModel {
+    // 模型名称
+    private String modelName;
+    // 接入点id
+    private String modelId;
+    // 输入单价(每千个token)
+    private Double inputPrice;
+    // 输出单价(每千个token)
+    private Double outputPrice;
+
+    public AccessModel(String modelId, Model model) {
+        this.modelId = modelId;
+        this.modelName = model.modelName;
+        this.inputPrice = model.inputPrice;
+        this.outputPrice = model.outputPrice;
+    }
+
+    public enum Model {
+        DOUBAO_PRO_4K("Doubao-pro-4k", 0.0008, 0.0020),
+        DOUBAO_PRO_32K("Doubao-pro-32k", 0.0008, 0.0020),
+        DOUBAO_PRO_128K("Doubao-pro-128k", 0.0090, 0.0500),
+        DOUBAO_LITE_4K("doubao-lite-4k", 0.0003, 0.0006),
+        DOUBAO_LITE_32K("Doubao-lite-32k", 0.0003, 0.0006),
+        DOUBAO_LITE_128K("Doubao-lite-128k", 0.0008, 0.0010);
+
+        @Getter
+        private final String modelName;
+        // 输入单价(每千个token)
+        @Getter
+        private final Double inputPrice;
+        // 输出单价(每千个token)
+        @Getter
+        private final Double outputPrice;
+
+        Model(String modelName, Double inputPrice, Double outputPrice) {
+            this.modelName = modelName;
+            this.inputPrice = inputPrice;
+            this.outputPrice = outputPrice;
+        }
+    }
+
+}

src/main/java/com/zhiwei/brandkbs2/pojo/ai/FieldMapping.java

+package com.zhiwei.brandkbs2.pojo.ai;
+
+import com.zhiwei.brandkbs2.es.EsQueryTools;
+import lombok.Data;
+import lombok.Getter;
+import org.apache.lucene.search.join.ScoreMode;
+import org.elasticsearch.index.query.*;
+
+import java.util.*;
+
+/**
+ * @ClassName: FieldMap
+ * @Description FieldMap
+ * @author: sjj
+ * @date: 2024-08-02 14:01
+ */
+@Data
+public class FieldMapping {
+
+    private FieldMap fieldMap;
+    private Object value;
+
+    public FieldMapping(FieldMap fieldMap, Object value) {
+        this.fieldMap = fieldMap;
+        this.value = value;
+    }
+
+    public QueryBuilder buildQuery(FieldMapping fieldMapping) {
+        boolean existsAnd = null != fieldMapping;
+        RangeQueryBuilder timeRangeBuilder;
+        String[] contendIds = {"0"};
+        // 项目组需绑定查询
+        switch (fieldMap) {
+            case START_TIME:
+                timeRangeBuilder = QueryBuilders.rangeQuery(fieldMap.databaseName).gte(value);
+                if (existsAnd && fieldMapping.fieldMap.equals(FieldMap.END_TIME)) {
+                    timeRangeBuilder.lt(fieldMapping.value);
+                }
+                return timeRangeBuilder;
+            case END_TIME:
+                timeRangeBuilder = QueryBuilders.rangeQuery(fieldMap.databaseName).lt(value);
+                if (existsAnd && fieldMapping.fieldMap.equals(FieldMap.START_TIME)) {
+                    timeRangeBuilder.gte(fieldMapping.value);
+                }
+                return timeRangeBuilder;
+            case PROJECT:
+                if (existsAnd && fieldMapping.fieldMap == FieldMap.BRAND) {
+                    contendIds = ((String) fieldMapping.value).split("\\|");
+                }
+                BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
+                for (String contendId : contendIds) {
+                    BoolQueryBuilder nestedBoolBuilder = QueryBuilders.boolQuery();
+                    // 必要条件
+                    nestedBoolBuilder.must(QueryBuilders.termQuery(fieldMap.databaseName, value + "_" + contendId));
+                    boolQueryBuilder.should(new NestedQueryBuilder("brandkbs_cache_maps", nestedBoolBuilder, ScoreMode.None));
+                }
+                boolQueryBuilder.minimumShouldMatch(1);
+                return boolQueryBuilder;
+            case BRAND:
+                if (!existsAnd || fieldMapping.fieldMap != FieldMap.PROJECT) {
+                    throw new IllegalStateException("项目条件缺失");
+                }
+                return fieldMapping.buildQuery(this);
+            case IND_FULL_TEXT:
+                return EsQueryTools.assembleNormalKeywordQuery(String.valueOf(value), new String[]{fieldMap.databaseName});
+            case SOURCE:
+            case MTAG:
+                return QueryBuilders.termQuery(fieldMap.databaseName, value);
+        }
+        return null;
+    }
+
+    public enum FieldMap {
+
+        START_TIME("起始时间", "时间", "time"),
+        END_TIME("结束时间", "时间", "time"),
+        PROJECT("项目", "项目", "brandkbs_cache_maps.key.keyword"),
+        BRAND("品牌", "项目", "brandkbs_cache_maps.key.keyword"),
+        SOURCE("渠道", "渠道", "source"),
+        MTAG("标签", "标签", "mark_cache_maps.name.keyword"),
+        IND_FULL_TEXT("搜索条件", "搜索条件", "ind_full_text");
+
+        @Getter
+        private final String name;
+        @Getter
+        private final String fatherName;
+        @Getter
+        private final String databaseName;
+
+        FieldMap(String name, String fatherName, String databaseName) {
+            this.name = name;
+            this.fatherName = fatherName;
+            this.databaseName = databaseName;
+        }
+    }
+
+    public static FieldMapping createFromNameAndValue(String name, Object value, String question) {
+        FieldMap fieldMap = null;
+//        String projectId = UserThreadLocal.getProjectId();
+        // TODO 字段转换待完善,引入数据库
+        for (FieldMap f : FieldMap.values()) {
+            if (name.equals(f.getName())) {
+//                // 项目名需要转成id
+//                if (FieldMap.PROJECT == f) {
+//                    Map<String, Project> projectMap = GlobalPojo.PROJECT_MAP.values().stream().collect(Collectors.toMap(AbstractProject::getProjectName, o -> o));
+//                    if (projectMap.containsKey(String.valueOf(value))) {
+//                        value = projectMap.get(String.valueOf(value)).getId();
+//                    }else {
+//                        value = projectId;
+//                    }
+//                }
+                // 品牌需要转换
+//                if (FieldMap.BRAND == f) {
+//                    Project project = GlobalPojo.PROJECT_MAP.get(projectId);
+//                    if (CollectionUtils.isNotEmpty(project.getContendList())){
+//                        List<String> contends = new ArrayList<>();
+//                        List<String> contendNames = project.getContendList().stream().map(AbstractProject::getBrandName).collect(Collectors.toList());
+//                        for (String contendName : contendNames) {
+//                            if (question.contains(contendName)) {
+//                                contends.add(contendName);
+//                            }
+//                        }
+//                        if (CollectionUtils.isNotEmpty(contends)){
+//                            value = String.join("|", contends);
+//                        }else {
+//                            value = Constant.PRIMARY_CONTEND_ID;
+//                        }
+//                    }else {
+//                        value = Constant.PRIMARY_CONTEND_ID;
+//                    }
+//                    if ("主品牌".equals(value)) {
+//                        value = Constant.PRIMARY_CONTEND_ID;
+//                    } else {
+//                        // 寻找对应的竞品id
+//                        Optional<FieldMapping> project = fieldMappings.stream().filter(field -> Objects.equals(FieldMap.PROJECT, field.getFieldMap())).findFirst();
+//                        if (project.isPresent()){
+//                            List<Contend> contendList = GlobalPojo.PROJECT_MAP.get(String.valueOf(project.get().getValue())).getContendList();
+//                            Object finalValue = value;
+//                            Optional<Contend> contendOptional = contendList.stream().filter(contend -> Objects.equals(contend.getBrandName(), finalValue)).findFirst();
+//                            if (contendOptional.isPresent()){
+//                                value = contendOptional.get().getId();
+//                            }else {
+//                                value = Constant.PRIMARY_CONTEND_ID;
+//                            }
+//                        }else {
+//                            value = Constant.PRIMARY_CONTEND_ID;
+//                        }
+//                    }
+//                }
+                // 标签只包含正负中
+                if (FieldMap.MTAG == f) {
+                    if (!Arrays.asList("正面", "中性", "负面").contains(String.valueOf(value))) {
+                        return null;
+                    }
+                }
+                fieldMap = f;
+                break;
+            }
+        }
+        if (null == fieldMap) {
+            return null;
+        }
+        return new FieldMapping(fieldMap, value);
+    }
+
+}

src/main/java/com/zhiwei/brandkbs2/service/MarkDataService.java

@@ -2,10 +2,7 @@ package com.zhiwei.brandkbs2.service;
 
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.brandkbs2.model.ResponseResult;
-import com.zhiwei.brandkbs2.pojo.BaseMap;
-import com.zhiwei.brandkbs2.pojo.DailyReport;
-import com.zhiwei.brandkbs2.pojo.Event;
-import com.zhiwei.brandkbs2.pojo.MarkFlowEntity;
+import com.zhiwei.brandkbs2.pojo.*;
 import com.zhiwei.brandkbs2.pojo.dto.*;
 import com.zhiwei.brandkbs2.pojo.vo.LineVO;
 import com.zhiwei.brandkbs2.pojo.vo.PageVO;
@@ -836,4 +833,32 @@ public interface MarkDataService {
      * @return
      */
     List<String> expandOriginRange(MarkSearchDTO dto);
+
+    /**
+     * AI搜索-推荐提问
+     * @param question
+     * @return
+     */
+    List<String> getAIReferenceQuestion(String question, int size);
+
+    /**
+     * AI搜索-参考提问
+     * @param cache
+     * @return
+     */
+    List<String> getAIReferenceQuestionCache(boolean cache);
+
+    /**
+     * AI搜索-搜索结果
+     * @param question
+     * @return
+     */
+    JSONObject getAISearchResult(String question, String keyword, Long startTime, Long endTime);
+
+    /**
+     * AI搜索-联网搜索
+     * @param question
+     * @return
+     */
+    JSONObject getAIOnlineSearchResult(String question);
 }

src/main/java/com/zhiwei/brandkbs2/service/TaskService.java

@@ -72,4 +72,9 @@ public interface TaskService{
      * 定时拉取并进行渠道库更新任务
      */
     void refreshChannelRecord();
+
+    /**
+     * 生成ai搜索参考提问缓存
+     */
+    void cacheAIQuestion();
 }

src/main/java/com/zhiwei/brandkbs2/service/impl/BehaviorServiceImpl.java

@@ -174,7 +174,7 @@ public class BehaviorServiceImpl implements BehaviorService {
             if (null == userInfo) {
                 return;
             }
-            UserLogRecord userLogRecord = new UserLogRecord(projectId, userInfo.getUserId(), userInfo.getNickname(), description, userInfo.getRoleId(), now, now);
+            UserLogRecord userLogRecord = new UserLogRecord(projectId, userInfo.getUserId(), userInfo.getNickname(), description, userInfo.getRoleId(), now, now, null);
             String collectionName = userLogRecordDao.generateCollectionName();
             userLogRecordDao.insertOne(userLogRecord, collectionName);
         }

src/main/java/com/zhiwei/brandkbs2/service/impl/MarkDataServiceImpl.java

@@ -6,13 +6,11 @@ import com.alibaba.fastjson.JSONObject;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
 import com.hankcs.hanlp.HanLP;
+import com.volcengine.ark.runtime.model.completion.chat.*;
 import com.zhiwei.base.category.ClassB;
 import com.zhiwei.base.entity.subclass.mark.MarkInfo;
 import com.zhiwei.brandkbs2.auth.UserThreadLocal;
-import com.zhiwei.brandkbs2.common.ChannelType;
-import com.zhiwei.brandkbs2.common.GenericAttribute;
-import com.zhiwei.brandkbs2.common.GlobalPojo;
-import com.zhiwei.brandkbs2.common.RedisKeyPrefix;
+import com.zhiwei.brandkbs2.common.*;
 import com.zhiwei.brandkbs2.config.Constant;
 import com.zhiwei.brandkbs2.dao.*;
 import com.zhiwei.brandkbs2.easyexcel.EasyExcelUtil;
@@ -29,6 +27,8 @@ import com.zhiwei.brandkbs2.listener.ApplicationProjectListener;
 import com.zhiwei.brandkbs2.model.CommonCodeEnum;
 import com.zhiwei.brandkbs2.model.ResponseResult;
 import com.zhiwei.brandkbs2.pojo.*;
+import com.zhiwei.brandkbs2.pojo.ai.AccessModel;
+import com.zhiwei.brandkbs2.pojo.ai.FieldMapping;
 import com.zhiwei.brandkbs2.pojo.dto.*;
 import com.zhiwei.brandkbs2.pojo.vo.*;
 import com.zhiwei.brandkbs2.service.*;
@@ -83,6 +83,7 @@ import org.springframework.web.client.RestTemplate;
 
 import javax.annotation.Resource;
 import java.io.IOException;
+import java.text.MessageFormat;
 import java.text.SimpleDateFormat;
 import java.util.*;
 import java.util.concurrent.CompletableFuture;
@@ -108,6 +109,42 @@ public class MarkDataServiceImpl implements MarkDataService {
 
     private static final String XHS_PLATFORM_ID = "6433c2251701316728003be4";
 
+    private static final String QUESTION_PROMPT = "###\n" +
+            "假如你是专业的问题提炼人员，你将根据用户提供的内容，来提炼问题要素和条件。根据以下规则一步步执行：\n" +
+            "1.提及到年、月、日、天、礼拜的定义为时间要素，未提及到默认条件算近一周，条件给到具体的起始时间和结束时间（结束时间为当前时间则不用返回）的时间戳。\n" +
+            "2.提及到 XX 渠道的定义为渠道要素，条件给到该渠道名，注意：{0}不可作为渠道名。\n" +
+            "3.提及到正面、中性、负面的定义为标签要素，条件给到该标签名。\n" +
+            "4.提及到针对 XX ，针对 XX 相关或 XX 相关的定义为搜索条件要素（包含 针对/相关 字样）。未提及时按照文义在文段内容中提取的一个或多个词作为该文段的关键词，将其定义为搜索条件要素。多个关键词时每个关键词之间严格用” “（空格）作为分隔符进行分隔，单个关键词则无需分隔，关键词必须在给到的文段中出现，条件给到具体值\n" +
+            "5.时间和关键词要素为必需要素，若不满足则返回“无法回答”。\n" +
+            "\n" +
+            "参考例子：\n" +
+            "示例 1：\n" +
+            "'{用户：今年7月腾讯项目张清相关的正面数据}\n" +
+            "输出：{\"时间\":{\"起始时间\":1719763200000,\"结束时间\":1722355200000},\"标签\":\"正面\",\"搜索条件\":\"张清\"}\n" +
+            "示例 2：\n" +
+            "{用户：近一个月老乡鸡竞品1品牌新浪网渠道数据}\n" +
+            "输出：{\"时间\":{\"起始时间\":1719763200000},\"渠道\":\"新浪网\"}\n" +
+            "示例 3：\n" +
+            "{用户：近一年数据}\n" +
+            "输出：无法回答\n" +
+            "\n" +
+            "要求：\n" +
+            "1 按照指定输出格式输出。\n" +
+            "2 严格按照规则进行提炼。\n" +
+            "###";
+    private static final String RESULT_PROMPT = "假如你是专业的分析报告人员，你将根据用户提供的内容(无关内容则无需引用)，贴合问题给出自己的一点或多点详细分析和见解，每个观点需同时提炼出一个贴合自己的详细分析和见解的小标题（无需注明“小标题”三个字）。并在每点详细分析和见解后用数字表示注明1-{0" +
+            "}的参考文章，分析结果和参考文章之间严格用”|“作为分隔符进行分隔（小标题与分析结果无需分隔），并且多个参考文章之间也严格用”|“作为分隔符进行分隔，若没有对应的参考文章则无需返回，示例：分析结果。|1|2|3" +
+            "请回答该问题：";
+
+    private static final String REFERENCE_QUESTION_PROMPT = "假如你是专业的问题提出人员，提出自己{0}个关于的{1}参考问题，每个问题无需给到对应的序号，问题必须包含 针对/相关 字样，每个问题之间严格用”|“作为分隔符进行分隔。" +
+            "请提出：";
+
+    private static final String CACHE_REFERENCE_QUESTION_PROMPT = "假如你是专业的问题提出人员，请参考给到的问题，提出自己5个类似的的参考问题，每个问题无需给到对应的序号，问题必须包含 针对/相关 字样，每个问题之间严格用”|“作为分隔符进行分隔。" +
+            "请提出：";
+
+    private static final String ONLINE_RESULT_PROMPT = "假如你是专业的分析报告人员，你将根据用户提供的问题，贴合问题给出自己的一点或多点详细分析和见解，每个观点需同时提炼出一个贴合自己的详细分析和见解的小标题（无需注明“小标题”三个字）。每个观点间需要严格换行" +
+            "请回答该问题：";
+
     @Value("${istarshine.addIStarShineKSData.url}")
     private String addIStarShineKSDataUrl;
 
@@ -150,6 +187,9 @@ public class MarkDataServiceImpl implements MarkDataService {
     @Resource(name = "xiaohongshuWordDao")
     private XiaohongshuWordDao xiaohongshuWordDao;
 
+    @Resource(name = "UserLogRecordDao")
+    private UserLogRecordDao userLogRecordDao;
+
     @Resource(name = "commonServiceImpl")
     private CommonService commonService;
 
@@ -195,6 +235,9 @@ public class MarkDataServiceImpl implements MarkDataService {
     @Resource(name = "dailyReportDao")
     DailyReportDao dailyReportDao;
 
+    @Resource(name = "aiSearchQuestionRecordDao")
+    AISearchQuestionRecordDao aiSearchQuestionRecordDao;
+
     @Resource(name = "toolsetServiceImpl")
     private ToolsetService toolsetService;
 
@@ -3919,6 +3962,290 @@ public class MarkDataServiceImpl implements MarkDataService {
         return Collections.emptyList();
     }
 
+    @Override
+    public List<String> getAIReferenceQuestion(String question, int size) {
+        try {
+            String projectId = UserThreadLocal.getProjectId();
+            // 选用的模型名称
+            AccessModel.Model model = AccessModel.Model.DOUBAO_PRO_32K;
+            String modelName = model.getModelName();
+            String projectName = GlobalPojo.PROJECT_MAP.get(projectId).getProjectName();
+            Pair<String, long[]> pair = standardRequest(question, modelName, MessageFormat.format(REFERENCE_QUESTION_PROMPT, size, projectName));
+            if (Objects.isNull(pair)){
+                return getAIReferenceQuestionTemplate(projectName);
+            }
+            String resultContent = pair.getLeft();
+            String[] splits = resultContent.split("\\|");
+            // 需记录本次耗费
+            double cost = calculateCost(pair.getRight(), model);
+            userLogRecordDao.insertOne(UserLogRecord.userLogRecordCost("AI搜索-生成推荐提问-" + question, cost), userLogRecordDao.generateCollectionName());
+            return new ArrayList<>(Arrays.asList(splits)).stream().filter(StringUtils::isNoneBlank).map(String::trim).collect(Collectors.toList());
+        }catch (Exception e){
+            ExceptionCast.cast(CommonCodeEnum.FAIL, "获取ai参考提问异常-", e);
+        }
+        return null;
+    }
+
+    @Override
+    public List<String> getAIReferenceQuestionCache(boolean cache) {
+        String projectId = UserThreadLocal.getProjectId();
+        String key = RedisUtil.getAISearchQuestionCacheKey(projectId);
+        String resultStr;
+        // 返回缓存
+        if (cache && StringUtils.isNotEmpty(resultStr = redisUtil.get(key))) {
+            return JSONObject.parseArray(resultStr).toJavaList(String.class);
+        }
+        List<String> questionList = aiSearchQuestionRecordDao.findDistinctQuestion(projectId);
+        String projectName = GlobalPojo.PROJECT_MAP.get(projectId).getProjectName();
+        if (CollectionUtils.isEmpty(questionList)){
+            return getAIReferenceQuestionTemplate(projectName);
+        }
+        // 选用的模型名称
+        String modelName = AccessModel.Model.DOUBAO_PRO_32K.getModelName();
+        StringBuilder sb = new StringBuilder();
+        int count = 1;
+        for (String question : questionList) {
+            sb.append(count++).append("、").append(question).append("；");
+        }
+        Pair<String, long[]> pair = standardRequest(sb.toString(), modelName, MessageFormat.format(CACHE_REFERENCE_QUESTION_PROMPT, projectName));
+        if (Objects.isNull(pair)){
+            return getAIReferenceQuestionTemplate(projectName);
+        }
+        // 需记录耗费
+        userLogRecordDao.insertOne(UserLogRecord.userLogRecordCost("AI搜索-生成参考提问",
+                calculateCost(pair.getRight(), AccessModel.Model.DOUBAO_PRO_32K)), userLogRecordDao.generateCollectionName());
+        String resultContent = pair.getLeft();
+        String[] splits = resultContent.split("\\|");
+        List<String> result = new ArrayList<>(Arrays.asList(splits)).stream().filter(StringUtils::isNoneBlank).map(String::trim).collect(Collectors.toList());
+        redisUtil.setExpire(key, JSONObject.toJSONString(result));
+        return result;
+    }
+
+    private List<String> getAIReferenceQuestionTemplate(String projectName){
+        String question1 = MessageFormat.format("今年 7 月{0}项目{1}相关的正面数据", projectName, projectName);
+        String question2 = MessageFormat.format("近一个周{0}项目有发生哪些重大舆情", projectName);
+        String question3 = MessageFormat.format("{0}项目的竞品舆情有哪些", projectName);
+        String question4 = MessageFormat.format("{0}项目最近发生了哪些事件", projectName);
+        String question5 = MessageFormat.format("{0}项目近期负面报道有哪些，主要是那几家集中涉及哪些事件", projectName);
+        return Arrays.asList(question1, question2, question3, question4, question5);
+    }
+
+    @Override
+    public JSONObject getAISearchResult(String question, String keyword, Long startTime, Long endTime) {
+        try {
+            // 选用的模型名称
+            String modelName = AccessModel.Model.DOUBAO_PRO_32K.getModelName();
+            List<JSONObject> list;
+            Pair<String, long[]> questionPair = null;
+            if (StringUtils.isNotBlank(keyword)){  // 已填辅助信息，则只用辅助信息
+                keyword = Tools.canonicalKeyword(keyword);
+                list = esClientDao.findSearch(question, keyword, startTime, endTime);
+            }else { // 未填辅助信息，则根据AI生成条件
+                Project project = GlobalPojo.PROJECT_MAP.get(UserThreadLocal.getProjectId());
+                StringBuilder brandStr = new StringBuilder(project.getProjectName());
+                if (CollectionUtils.isNotEmpty(project.getContendList())){
+                    project.getContendList().forEach(contend -> brandStr.append("、").append(contend.getBrandName()));
+                }
+                questionPair = standardRequest(question, modelName, MessageFormat.format(QUESTION_PROMPT, brandStr));
+                if (Objects.isNull(questionPair)) {
+                    return null;
+                }
+                JSONObject json = JSON.parseObject(questionPair.getLeft());
+                // 数据条件
+                List<FieldMapping> filedMapping = getFiledMapping(json, question);
+                addDefaultFiledMapping(filedMapping, question);
+                list = esClientDao.findSearch(filedMapping);
+            }
+            String collectionName = userLogRecordDao.generateCollectionName();
+            if (CollectionUtils.isEmpty(list)){
+                if (Objects.nonNull(questionPair)) {
+                    // 需记录耗费
+                    userLogRecordDao.insertOne(UserLogRecord.userLogRecordCost("AI搜索-无结果-提取搜索条件-" + question,
+                            calculateCost(questionPair.getRight(), AccessModel.Model.DOUBAO_PRO_32K)), collectionName);
+                }else {
+                    userLogRecordDao.insertOne(UserLogRecord.defaultUserLogRecord("AI搜索-无结果-"+ question +"-辅助信息："+ Tools.concatWithMinus(Arrays.asList(keyword, startTime, endTime))), collectionName);
+                }
+                return null;
+            }
+            // AI回答
+            StringBuilder sb = new StringBuilder();
+            List<BaseMap> articles = list.stream().map(Tools::getBaseFromEsMap).collect(Collectors.toList());
+            int count = 1;
+            for (BaseMap baseMap : articles) {
+                String text = baseMap.getContent();
+                sb.append(count++).append("、").append(text).append("；");
+            }
+            String sbContent = sb.toString();
+            Pair<String, long[]> answerPair = streamStandardRequest(sbContent, modelName, MessageFormat.format(RESULT_PROMPT, list.size()) + question);
+            // 结果处理
+            JSONObject res = aiResultDataProcess(answerPair, false);
+            res.put("articles", articles);
+            res.put("searchCriteria", Objects.isNull(questionPair) ? Tools.concat(keyword, startTime, endTime) : questionPair.getLeft());
+            // 记录返回成功的提问
+            aiSearchQuestionRecordDao.insertOne(new AISearchQuestionRecord(question, UserThreadLocal.getProjectId(), System.currentTimeMillis()));
+            // 需记录耗费
+            double cost = calculateCost(Objects.isNull(questionPair) ? null : questionPair.getRight(), AccessModel.Model.DOUBAO_PRO_32K) + calculateCost(answerPair.getRight(), AccessModel.Model.DOUBAO_PRO_32K);
+            String description = "AI搜索-结果-" + question;
+            String extraDescription = "-辅助信息：" + Tools.concatWithMinus(Arrays.asList(keyword, startTime, endTime));
+            userLogRecordDao.insertOne(UserLogRecord.userLogRecordCost(Objects.nonNull(keyword) ? description + extraDescription : description, cost), collectionName);
+            return res;
+        }catch (Exception e){
+            ExceptionCast.cast(CommonCodeEnum.FAIL, "ai搜索异常-", e);
+        }
+        return null;
+    }
+
+    private JSONObject aiResultDataProcess(Pair<String, long[]> answerPair, boolean isOnline){
+        JSONObject res = new JSONObject();
+        // 结果处理
+        String[] splits = answerPair.getLeft().split("\\r?\\n");
+        List<JSONObject> answers = new ArrayList<>();
+        for (int i = 0; i < splits.length; i++) {
+            JSONObject answer = new JSONObject();
+            String[] sonSplit = splits[i].split("\\|");
+            if (0 == sonSplit.length){
+                continue;
+            }
+            if (i == 0 || isOnline){
+                answer.put("answer", splits[i].trim());
+                answers.add(answer);
+                continue;
+            }
+            if (StringUtils.isNotBlank(sonSplit[0])) {
+                answer.put("answer", sonSplit[0]);
+                List<String> sonSplitList = new ArrayList<>(Arrays.asList(sonSplit)).stream().skip(1).collect(Collectors.toList());
+                answer.put("referenceArticles", sonSplitList);
+                answers.add(answer);
+            }
+        }
+        res.put("answers", answers);
+        return res;
+    }
+
+    @Override
+    public JSONObject getAIOnlineSearchResult(String question) {
+        String modelName = AccessModel.Model.DOUBAO_PRO_32K.getModelName();
+        Pair<String, long[]> answerPair = streamStandardRequest(question, modelName, ONLINE_RESULT_PROMPT);
+        // 需记录耗费
+        userLogRecordDao.insertOne(UserLogRecord.userLogRecordCost("AI搜索-联网搜索-" + question,
+                calculateCost(answerPair.getRight(), AccessModel.Model.DOUBAO_PRO_32K)), userLogRecordDao.generateCollectionName());
+        return aiResultDataProcess(answerPair, true);
+    }
+
+    private double calculateCost(long[] tokens, AccessModel.Model model){
+        if (Objects.isNull(tokens)){
+            return 0d;
+        }
+        double inputCost = tokens[0] / 1000d * model.getInputPrice();
+        double outputCost = tokens[1] / 1000d * model.getOutputPrice();
+        return inputCost + outputCost;
+    }
+
+    private Pair<String, long[]> streamStandardRequest(String content, String modelName, String prompt) {
+        AccessModel model = DoubaoAIAccountFactor.getCompanyAccount().getModelList().stream().collect(Collectors.toMap(AccessModel::getModelName, m -> m)).get(modelName);
+        StringBuilder result = new StringBuilder();
+        AtomicLong promptTokens = new AtomicLong();
+        AtomicLong completionTokens = new AtomicLong();
+        try {
+            final List<ChatMessage> streamMessages = new ArrayList<>();
+            final ChatMessage streamSystemMessage = ChatMessage.builder().role(ChatMessageRole.SYSTEM).content(prompt).build();
+            final ChatMessage streamUserMessage = ChatMessage.builder().role(ChatMessageRole.USER).content(content).build();
+            streamMessages.add(streamSystemMessage);
+            streamMessages.add(streamUserMessage);
+            ChatCompletionRequest streamChatCompletionRequest = ChatCompletionRequest.builder().stream(true)
+                    .streamOptions(ChatCompletionRequest.ChatCompletionRequestStreamOptions.of(true)).model(model.getModelId()).messages(streamMessages).build();
+            DoubaoAIAccountFactor.arkService.streamChatCompletion(streamChatCompletionRequest).doOnError(Throwable::printStackTrace).blockingForEach(choice -> {
+                if (Objects.nonNull(choice.getUsage())){
+                    // 本次调用输入、输出使用tokens量
+                    promptTokens.set(choice.getUsage().getPromptTokens());
+                    completionTokens.set(choice.getUsage().getCompletionTokens());
+                }
+                if (choice.getChoices().size() > 0) {
+                    result.append(choice.getChoices().get(0).getMessage().getContent());
+                }
+            });
+        } catch (Exception e) {
+            log.error("standardRequest,chatCompletion:{}", JSON.toJSONString(result), e);
+        }
+        long[] tokens = {promptTokens.get(), completionTokens.get()};
+        return Pair.of(result.toString(), tokens);
+    }
+
+    private Pair<String, long[]> standardRequest(String content, String modelName, String prompt) {
+        AccessModel model = DoubaoAIAccountFactor.getCompanyAccount().getModelList().stream().collect(Collectors.toMap(AccessModel::getModelName, m -> m)).get(modelName);
+        ChatCompletionResult chatCompletion = null;
+        try {
+            final List<ChatMessage> messages = new ArrayList<>();
+            final ChatMessage systemMessage = ChatMessage.builder().role(ChatMessageRole.SYSTEM).content(prompt).build();
+            final ChatMessage userMessage = ChatMessage.builder().role(ChatMessageRole.USER).content(content).build();
+            messages.add(systemMessage);
+            messages.add(userMessage);
+            ChatCompletionRequest chatCompletionRequest = ChatCompletionRequest.builder().model(model.getModelId()).messages(messages).build();
+            chatCompletion = DoubaoAIAccountFactor.arkService.createChatCompletion(chatCompletionRequest);
+            if (chatCompletion.getChoices().size() > 1) {
+                log.error("异常chatCompletion:{}", JSON.toJSONString(chatCompletion));
+                return null;
+            }
+            // 本次调用输入、输出使用tokens量
+            long[] tokens = {chatCompletion.getUsage().getPromptTokens(), chatCompletion.getUsage().getCompletionTokens()};
+            String resultContent = String.valueOf(chatCompletion.getChoices().get(0).getMessage().getContent());
+            return Pair.of(resultContent, tokens);
+        } catch (Exception e) {
+            log.error("standardRequest,chatCompletion:{}", JSON.toJSONString(chatCompletion), e);
+        }
+        return null;
+    }
+
+    /**
+     * 获取
+     * @param json
+     * @param content
+     * @return
+     */
+    private static List<FieldMapping> getFiledMapping(JSONObject json, String content) {
+        List<FieldMapping> res = new ArrayList<>();
+        for (Map.Entry<String, Object> entry : json.entrySet()) {
+            if (entry.getValue() instanceof JSONObject) {
+                res.addAll(getFiledMapping((JSONObject) entry.getValue(), content));
+            } else {
+                if (Objects.isNull(entry.getValue())){
+                    continue;
+                }
+                FieldMapping fieldMapping = FieldMapping.createFromNameAndValue(entry.getKey(), entry.getValue(), content);
+                if (null != fieldMapping && Objects.nonNull(fieldMapping.getValue())) {
+                    res.add(fieldMapping);
+                }
+            }
+        }
+        return res;
+    }
+
+    private void addDefaultFiledMapping(List<FieldMapping> fieldMappings, String question){
+        Project project = GlobalPojo.PROJECT_MAP.get(UserThreadLocal.getProjectId());
+        fieldMappings.add(new FieldMapping(FieldMapping.FieldMap.PROJECT, UserThreadLocal.getProjectId()));
+        List<String> projectBandNames = new ArrayList<>();
+        projectBandNames.add(project.getProjectName());
+        projectBandNames.add(project.getBrandName());
+        if (CollectionUtils.isNotEmpty(project.getContendList())){
+            List<String> contends = new ArrayList<>();
+            for (Contend contend : project.getContendList()) {
+                projectBandNames.add(contend.getBrandName());
+                if (question.contains(contend.getBrandName())) {
+                    contends.add(contend.getId());
+                }
+            }
+            if (CollectionUtils.isNotEmpty(contends)){
+                fieldMappings.add(new FieldMapping(FieldMapping.FieldMap.BRAND, String.join("|", contends)));
+            }else {
+                fieldMappings.add(new FieldMapping(FieldMapping.FieldMap.BRAND, Constant.PRIMARY_CONTEND_ID));
+            }
+        }else {
+            fieldMappings.add(new FieldMapping(FieldMapping.FieldMap.BRAND, Constant.PRIMARY_CONTEND_ID));
+        }
+        // 防止将项目/品牌作为渠道
+        fieldMappings.removeIf(fieldMapping -> Objects.equals(fieldMapping.getFieldMap(), FieldMapping.FieldMap.SOURCE) && projectBandNames.contains(String.valueOf(fieldMapping.getValue())));
+    }
+
     /**
      * 原发溯源大库es查询
      * @param dto

src/main/java/com/zhiwei/brandkbs2/service/impl/ProjectWarnServiceImpl.java

@@ -89,7 +89,7 @@ public class ProjectWarnServiceImpl implements ProjectWarnService {
 
     static {
         TYPE_SEARCH.put("微博热搜", "weibo");
-        TYPE_SEARCH.put("微博话题", "weibo-topic");
+//        TYPE_SEARCH.put("微博话题", "weibo-topic");
         TYPE_SEARCH.put("微博预热", "weibo-rise");
         TYPE_SEARCH.put("头条热搜", "toutiao");
         TYPE_SEARCH.put("抖音热搜", "douyin");
@@ -385,7 +385,8 @@ public class ProjectWarnServiceImpl implements ProjectWarnService {
         // key2
         String key2 = "";
         List<String> key2Element = new ArrayList<>();
-        config.getListType().forEach(type -> {
+        // 2024/8/19 微博话题榜采集下线，防止微博话题配置历史数据影响
+        config.getListType().stream().filter(type -> !Objects.equals("微博话题", type)).collect(Collectors.toList()).forEach(type -> {
             if (config.getFirstTop()) {
                 key2Element.add(type + "榜-首次上榜");
             }

src/main/java/com/zhiwei/brandkbs2/service/impl/TaskServiceImpl.java

@@ -89,6 +89,9 @@ public class TaskServiceImpl implements TaskService {
     @Resource(name = "channelRecordRefreshTaskDao")
     private ChannelRecordRefreshTaskDao channelRecordRefreshTaskDao;
 
+    @Resource(name = "aiSearchQuestionRecordDao")
+    private AISearchQuestionRecordDao aiSearchQuestionRecordDao;
+
     @Resource(name = "brandkbsTaskServiceImpl")
     BrandkbsTaskService brandkbsTaskService;
 
@@ -486,6 +489,21 @@ public class TaskServiceImpl implements TaskService {
         log.info("更新渠道库记录完成-taskId:{}", task.getId());
     }
 
+    @Override
+    public void cacheAIQuestion() {
+        AtomicInteger total = new AtomicInteger();
+        CompletableFuture.allOf(GlobalPojo.PROJECT_MAP.values().stream().map(project -> CompletableFuture.supplyAsync(() -> {
+            UserInfo userInfo = new UserInfo().setProjectId(project.getId());
+            userInfo.setUserId("0");
+            userInfo.setNickname("系统");
+            userInfo.setRoleId(1);
+            UserThreadLocal.set(userInfo);
+            markDataService.getAIReferenceQuestionCache(false);
+            log.info("项目:{}-{}-AI参考问题缓存完成:{}个", project.getProjectName(), project.getId(), total.incrementAndGet());
+            return null;
+        }, cacheServiceExecutor)).toArray(CompletableFuture[]::new)).join();
+    }
+
     private void updateRefreshTask(String id, String status){
         Update update = new Update();
         update.set("status", status);

src/main/java/com/zhiwei/brandkbs2/task/ControlCenter.java

@@ -47,6 +47,7 @@ public class ControlCenter {
 //            taskService.customEventCache();
             taskService.eventAggTitleCache();
             taskService.yuqingAnalyzeHighWordCache();
+            taskService.cacheAIQuestion();
         } catch (Exception e) {
             log.error("定时按天缓存数据-出错", e);
         } finally {

src/main/java/com/zhiwei/brandkbs2/util/RedisUtil.java

@@ -130,6 +130,10 @@ public class RedisUtil {
         return RedisKeyPrefix.SEARCH_KEYWORD + Tools.concat(projectId, userId, searchType);
     }
 
+    public static String getAISearchQuestionCacheKey(String projectId){
+        return RedisKeyPrefix.AI_SEARCH_QUESTION + projectId;
+    }
+
     public void setExpire(String key, String value, long timeout, TimeUnit unit) {
         stringRedisTemplate.opsForValue().set(key, value, timeout, unit);
     }

src/main/java/com/zhiwei/brandkbs2/util/Tools.java

@@ -554,10 +554,13 @@ public class Tools {
         String separator = "-";
         StringBuilder sb = new StringBuilder();
         for (Object obj : objects) {
+            if (Objects.isNull(obj)){
+                continue;
+            }
             sb.append(obj).append(separator);
         }
         String resultStr = sb.toString();
-        return resultStr.substring(0, resultStr.length() - 1);
+        return StringUtils.isBlank(resultStr) ? resultStr : resultStr.substring(0, resultStr.length() - 1);
     }
 
     public static String[] split(String concatStr) {