Merge branch 'release' into 'master'

Release

See merge request !575

pom.xml

@@ -269,7 +269,7 @@
         <dependency>
             <groupId>com.squareup.okhttp3</groupId>
             <artifactId>okhttp</artifactId>
-            <version>3.8.0</version>
+            <version>3.12.0</version>
         </dependency>
         <!-- dubbo -->
         <dependency>
@@ -324,6 +324,12 @@
             <artifactId>ansj_seg</artifactId>
             <version>5.0.2</version>
         </dependency>
+        <!--火山引擎 豆包大模型-->
+        <dependency>
+            <groupId>com.volcengine</groupId>
+            <artifactId>volcengine-java-sdk-ark-runtime</artifactId>
+            <version>0.1.121</version>
+        </dependency>
     </dependencies>
     <build>
         <plugins>

src/main/java/com/zhiwei/brandkbs2/aop/AopLogRecord.java

@@ -144,7 +144,7 @@ public class AopLogRecord {
         String[] value = method.getAnnotation(LogRecord.class).values();
         // 注解value为空字符串（value使用默认值未设置）
         if (1 == value.length && StringUtils.isEmpty(value[0])){
-            return new UserLogRecord(projectId, userInfo.getUserId(), userInfo.getNickname(), prefix, userInfo.getRoleId(), now, now);
+            return new UserLogRecord(projectId, userInfo.getUserId(), userInfo.getNickname(), prefix, userInfo.getRoleId(), now, now, null);
         }
         // 获取接口传参（value为获取传参的具体字段）并与操作描述description拼接返回，传参值为实体
         if (method.getAnnotation(LogRecord.class).arguments() && method.getAnnotation(LogRecord.class).entity()) {
@@ -176,7 +176,7 @@ public class AopLogRecord {
                 }
             }
             String suffix = CollectionUtils.isNotEmpty(res) ? "-" + Tools.concatWithMinus(res) : "";
-            return new UserLogRecord(projectId, userInfo.getUserId(), userInfo.getNickname(), prefix + suffix, userInfo.getRoleId(), now, now);
+            return new UserLogRecord(projectId, userInfo.getUserId(), userInfo.getNickname(), prefix + suffix, userInfo.getRoleId(), now, now, null);
         }
         // 获取接口传参（value为获取传参的具体字段）并与操作描述description拼接返回，传参值不为实体
         if (method.getAnnotation(LogRecord.class).arguments() && !method.getAnnotation(LogRecord.class).entity()) {
@@ -190,7 +190,7 @@ public class AopLogRecord {
                 }
             }
             String suffix = CollectionUtils.isNotEmpty(res) ? "-" + Tools.concatWithMinus(res) : "";
-            return new UserLogRecord(projectId, userInfo.getUserId(), userInfo.getNickname(), prefix + suffix, userInfo.getRoleId(), now, now);
+            return new UserLogRecord(projectId, userInfo.getUserId(), userInfo.getNickname(), prefix + suffix, userInfo.getRoleId(), now, now, null);
         }
         // 获取接口返回值（value为获取返回值的具体字段）并与操作描述description拼接返回，返回值为实体
         if (!method.getAnnotation(LogRecord.class).arguments() && method.getAnnotation(LogRecord.class).entity() && Objects.nonNull(responseResult)) {
@@ -204,9 +204,9 @@ public class AopLogRecord {
                 }
             }
             String suffix = CollectionUtils.isNotEmpty(res) ? "-" + Tools.concatWithMinus(res) : "";
-            return new UserLogRecord(projectId, userInfo.getUserId(), userInfo.getNickname(), prefix + suffix, userInfo.getRoleId(), now, now);
+            return new UserLogRecord(projectId, userInfo.getUserId(), userInfo.getNickname(), prefix + suffix, userInfo.getRoleId(), now, now, null);
         }
-        return new UserLogRecord(projectId, userInfo.getUserId(), userInfo.getNickname(), prefix, userInfo.getRoleId(), now, now);
+        return new UserLogRecord(projectId, userInfo.getUserId(), userInfo.getNickname(), prefix, userInfo.getRoleId(), now, now, null);
     }
 
     /**

src/main/java/com/zhiwei/brandkbs2/common/DoubaoAIAccountFactor.java

+package com.zhiwei.brandkbs2.common;
+
+import com.volcengine.ark.runtime.service.ArkService;
+import com.zhiwei.brandkbs2.pojo.ai.AccessModel;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @ClassName: DoubaoAIAccountFactor
+ * @Description DoubaoAIAccountFactor
+ * @author: sjj
+ * @date: 2024-07-24 14:01
+ */
+public class DoubaoAIAccountFactor {
+
+    public static Account getCompanyAccount() {
+        String apiKey = "607764fc-c9d9-47e4-a673-a310852917a0";
+        List<AccessModel> modelList = new ArrayList<>();
+        modelList.add(new AccessModel("ep-20240617061616-8d2ls", AccessModel.Model.DOUBAO_PRO_4K));
+        modelList.add(new AccessModel("ep-20240618021538-t6dpf", AccessModel.Model.DOUBAO_PRO_32K));
+        return new Account(apiKey, modelList);
+    }
+
+    @Data
+    @AllArgsConstructor
+    public static class Account {
+        String apiKey;
+        List<AccessModel> modelList;
+    }
+
+    public static ArkService arkService;
+
+    static {
+        arkService = new ArkService(getCompanyAccount().getApiKey());
+    }
+}

src/main/java/com/zhiwei/brandkbs2/common/RedisKeyPrefix.java

@@ -121,8 +121,13 @@ public class RedisKeyPrefix {
 
     public static final String CUSTOM_YUQING_ANALYZE_HIGH_WORD = "BRANDKBS:CUSTOM:YUQING:ANALYZE:HIGH:WORD:";
 
+    /**
+     * 搜索相关缓存
+     */
     public static final String SEARCH_KEYWORD = "BRANDKBS:SEARCH:KEYWORD:";
 
+    public static final String AI_SEARCH_QUESTION = "BRANDKBS:AI:SEARCH:QUESTION:";
+
     public static String projectWarnHotTopKeyAll(String projectId, String type) {
         return RedisKeyPrefix.generateRedisKey(RedisKeyPrefix.PROJECT_WARN_HOT_TOP, projectId, Tools.concat(type, "*"));
     }

src/main/java/com/zhiwei/brandkbs2/controller/app/AppDownloadController.java

@@ -396,9 +396,9 @@ public class AppDownloadController extends BaseController {
     @PostMapping(value = "/contend/mark")
     @DownloadTask(taskName = "竞品库竞品舆情下载", description = "竞品库竞品舆情")
     public ResponseResult exportContendMarkList(@RequestBody MarkSearchDTO markSearchDTO) {
-        if (StringUtils.isNotEmpty(markSearchDTO.getKeyword()) && Tools.checkUniteString(markSearchDTO.getKeyword())){
-            return ResponseResult.failure("不支持特殊符号字段查询");
-        }
+//        if (StringUtils.isNotEmpty(markSearchDTO.getKeyword()) && Tools.checkUniteString(markSearchDTO.getKeyword())){
+//            return ResponseResult.failure("不支持特殊符号字段查询");
+//        }
         Pair<String, List<ExportAppYuqingDTO>> stringListPair = markDataService.downloadContendMarkList(markSearchDTO);
         // excel写入至指定路径
         String projectName = projectService.getProjectById(UserThreadLocal.getProjectId()).getProjectName();
@@ -412,9 +412,9 @@ public class AppDownloadController extends BaseController {
     @LogRecord(description = "全网搜-舆情导出", values = {"startTime", "endTime", "fans", "filterType", "filterWords", "search", "keyword", "platforms", "sensitiveChannels", "sourceKeyword"}, entity = true, arguments = true)
     @DownloadTask(taskName = "全网搜舆情下载", description = "全网搜舆情")
     public ResponseResult exportSearchWhole(@RequestBody SearchFilterDTO dto) {
-        if (StringUtils.isNotEmpty(dto.getKeyword()) && Tools.checkUniteString(dto.getKeyword())){
-            return ResponseResult.failure("不支持特殊符号字段查询");
-        }
+//        if (StringUtils.isNotEmpty(dto.getKeyword()) && Tools.checkUniteString(dto.getKeyword())){
+//            return ResponseResult.failure("不支持特殊符号字段查询");
+//        }
         // 针对商业数据库做限制
         if (dto.isExternalDataSource()) {
             long time = DateUtils.addDays(Tools.truncDate(new Date(), Constant.DAY_PATTERN), -89).getTime();

src/main/java/com/zhiwei/brandkbs2/controller/app/AppSearchController.java

@@ -161,9 +161,9 @@ public class AppSearchController extends BaseController {
     @LogRecord(values = {"fans", "sensitiveChannels:father,son", "keyword", "search"}, description = "全网搜", arguments = true, entity = true)
     @PostMapping("/searchWhole")
     public ResponseResult searchWholeNetwork(@RequestBody SearchFilterDTO dto) {
-        if (StringUtils.isNotEmpty(dto.getKeyword()) && Tools.checkUniteString(dto.getKeyword())){
-            return ResponseResult.failure("不支持特殊符号字段查询");
-        }
+//        if (StringUtils.isNotEmpty(dto.getKeyword()) && Tools.checkUniteString(dto.getKeyword())){
+//            return ResponseResult.failure("不支持特殊符号字段查询");
+//        }
         cacheSearchKeyword(dto.getKeyword(), "whole");
         // 针对商业数据库做限制
         if (dto.isExternalDataSource()) {
@@ -200,9 +200,9 @@ public class AppSearchController extends BaseController {
     @PostMapping("/exportSearchWhole")
     @LogRecord(description = "全网搜-舆情导出", values = {"startTime", "endTime", "fans", "filterType", "filterWords", "search", "keyword", "platforms", "sensitiveChannels", "sourceKeyword"}, entity = true, arguments = true)
     public ResponseResult exportSearchWhole(@RequestBody SearchFilterDTO dto) {
-        if (StringUtils.isNotEmpty(dto.getKeyword()) && Tools.checkUniteString(dto.getKeyword())){
-            return ResponseResult.failure("不支持特殊符号字段查询");
-        }
+//        if (StringUtils.isNotEmpty(dto.getKeyword()) && Tools.checkUniteString(dto.getKeyword())){
+//            return ResponseResult.failure("不支持特殊符号字段查询");
+//        }
         // 针对商业数据库做限制
         if (dto.isExternalDataSource()) {
             long time = DateUtils.addDays(Tools.truncDate(new Date(), Constant.DAY_PATTERN), -89).getTime();
@@ -231,9 +231,9 @@ public class AppSearchController extends BaseController {
     @LogRecord(values = {"searchType", "keyword"}, description = "查舆情", arguments = true, entity = true)
     @PostMapping("/mark/list")
     public ResponseResult getYuqingMarkList(@RequestBody MarkSearchDTO markSearchDTO) {
-        if (StringUtils.isNotEmpty(markSearchDTO.getKeyword()) && Tools.checkUniteString(markSearchDTO.getKeyword())){
-            return ResponseResult.failure("不支持特殊符号字段查询");
-        }
+//        if (StringUtils.isNotEmpty(markSearchDTO.getKeyword()) && Tools.checkUniteString(markSearchDTO.getKeyword())){
+//            return ResponseResult.failure("不支持特殊符号字段查询");
+//        }
         cacheSearchKeyword(markSearchDTO.getKeyword(), "yuqing");
         PageVO<MarkFlowEntity> yuqingMarkList = markDataService.getYuqingMarkList(markSearchDTO);
         // 仅第一页增加平台进量（声量）统计
@@ -339,9 +339,9 @@ public class AppSearchController extends BaseController {
     @LogRecord(values = "keyword", description = "查竞品",arguments = true, entity = true)
     @PostMapping("/contend/list")
     public ResponseResult getContendSearchList(@RequestBody MarkSearchDTO markSearchDTO) {
-        if (StringUtils.isNotEmpty(markSearchDTO.getKeyword()) && Tools.checkUniteString(markSearchDTO.getKeyword())){
-            return ResponseResult.failure("不支持特殊符号字段查询");
-        }
+//        if (StringUtils.isNotEmpty(markSearchDTO.getKeyword()) && Tools.checkUniteString(markSearchDTO.getKeyword())){
+//            return ResponseResult.failure("不支持特殊符号字段查询");
+//        }
         cacheSearchKeyword(markSearchDTO.getKeyword(), "contend");
         return ResponseResult.success(markDataService.getContendSearchList(markSearchDTO));
     }
@@ -352,6 +352,34 @@ public class AppSearchController extends BaseController {
         return ResponseResult.success(markDataService.getContendSearchCriteria(contendId));
     }
 
+    @ApiOperation("搜索-AI搜索")
+    @GetMapping("/ai/answer")
+    public ResponseResult getAISearchResult(@RequestParam(value = "question") String question,
+                                            @RequestParam(value = "keyword", required = false) String keyword,
+                                            @RequestParam(value = "startTime", required = false) Long startTime,
+                                            @RequestParam(value = "endTime", required = false) Long endTime) {
+        return ResponseResult.success(markDataService.getAISearchResult(question, keyword, startTime, endTime));
+    }
+
+    @ApiOperation("搜索-AI搜索-联网搜索")
+    @GetMapping("/ai/answer/online")
+    public ResponseResult getAIOnlineSearchResult(@RequestParam(value = "question") String question) {
+        return ResponseResult.success(markDataService.getAIOnlineSearchResult(question));
+    }
+
+    @ApiOperation("搜索-AI推荐提问")
+    @GetMapping("/ai/question")
+    public ResponseResult getAIReferenceQuestion(@RequestParam(value = "question") String question,
+                                                 @RequestParam(value = "size") int size) {
+        return ResponseResult.success(markDataService.getAIReferenceQuestion(question, size));
+    }
+
+    @ApiOperation("搜索-AI参考提问")
+    @GetMapping("/ai/question-cache")
+    public ResponseResult getAIReferenceQuestion() {
+        return ResponseResult.success(markDataService.getAIReferenceQuestionCache(true));
+    }
+
     @ApiOperation("搜索-搜索关键词历史记录")
     @GetMapping("/keyword/cache")
     public ResponseResult getSearchKeywordCache(@ApiParam(name = "searchType",

src/main/java/com/zhiwei/brandkbs2/dao/AISearchQuestionRecordDao.java

+package com.zhiwei.brandkbs2.dao;
+
+import com.zhiwei.brandkbs2.pojo.AISearchQuestionRecord;
+
+import java.util.List;
+
+/**
+ * @ClassName: AISearchQuestionRecordDao
+ * @Description AISearchQuestionRecordDao
+ * @author: cjz
+ * @date: 2024-08-12 17:07
+ */
+public interface AISearchQuestionRecordDao extends BaseMongoDao<AISearchQuestionRecord>{
+
+    List<String> findDistinctQuestion(String projectId);
+}

src/main/java/com/zhiwei/brandkbs2/dao/impl/AISearchQuestionRecordDaoImpl.java

+package com.zhiwei.brandkbs2.dao.impl;
+
+import com.zhiwei.brandkbs2.dao.AISearchQuestionRecordDao;
+import com.zhiwei.brandkbs2.pojo.AISearchQuestionRecord;
+import org.springframework.data.domain.Sort;
+import org.springframework.data.mongodb.core.query.Criteria;
+import org.springframework.data.mongodb.core.query.Query;
+import org.springframework.stereotype.Component;
+
+import java.util.List;
+
+/**
+ * @ClassName: AISearchQuestionRecordDao
+ * @Description AISearchQuestionRecordDao
+ * @author: cjz
+ * @date: 2024-08-12 17:07
+ */
+@Component("aiSearchQuestionRecordDao")
+public class AISearchQuestionRecordDaoImpl extends BaseMongoDaoImpl<AISearchQuestionRecord> implements AISearchQuestionRecordDao {
+    private static final String COLLECTION_NAME = "brandkbs_ai_search_question_record";
+
+    public AISearchQuestionRecordDaoImpl() {
+        super(COLLECTION_NAME);
+    }
+
+    @Override
+    public List<String> findDistinctQuestion(String projectId) {
+        Query query = new Query().addCriteria(Criteria.where("projectId").is(projectId)).with(Sort.by(Sort.Order.desc("cTime"))).limit(10);
+        return mongoTemplate.findDistinct(query, "question", COLLECTION_NAME, String.class);
+    }
+}

src/main/java/com/zhiwei/brandkbs2/es/EsClientDao.java

 package com.zhiwei.brandkbs2.es;
 
 import com.alibaba.fastjson.JSONObject;
+import com.hankcs.hanlp.HanLP;
+import com.zhiwei.brandkbs2.auth.UserThreadLocal;
 import com.zhiwei.brandkbs2.common.GenericAttribute;
+import com.zhiwei.brandkbs2.common.GlobalPojo;
 import com.zhiwei.brandkbs2.config.Constant;
 import com.zhiwei.brandkbs2.pojo.ChannelIndex;
+import com.zhiwei.brandkbs2.pojo.Contend;
+import com.zhiwei.brandkbs2.pojo.Project;
+import com.zhiwei.brandkbs2.pojo.ai.FieldMapping;
+import com.zhiwei.brandkbs2.util.TextUtil;
 import com.zhiwei.brandkbs2.util.Tools;
 import lombok.Getter;
 import lombok.Setter;
@@ -33,6 +40,7 @@ import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
 import org.elasticsearch.search.sort.FieldSortBuilder;
+import org.elasticsearch.search.sort.SortOrder;
 import org.joda.time.Period;
 import org.joda.time.PeriodType;
 import org.springframework.beans.factory.annotation.Value;
@@ -331,6 +339,135 @@ public class EsClientDao {
         return Pair.of(new Long[]{startTime, endTime}, res);
     }
 
+    public List<JSONObject> findSearch(String question, String keyword, Long startTime, Long endTime) throws IOException {
+        List<JSONObject> list = new ArrayList<>();
+        String projectId = UserThreadLocal.getProjectId();
+        BoolQueryBuilder query = QueryBuilders.boolQuery();
+        // 默认一周
+        if (Objects.isNull(startTime) || Objects.isNull(endTime)){
+            endTime = System.currentTimeMillis();
+            startTime = System.currentTimeMillis() - Constant.ONE_WEEK;
+        }
+        // time
+        query.must(QueryBuilders.rangeQuery(GenericAttribute.ES_TIME).gte(startTime).lt(endTime));
+        // contend
+        Project project = GlobalPojo.PROJECT_MAP.get(projectId);
+        if (CollectionUtils.isNotEmpty(project.getContendList())) {
+            List<Contend> contendList = new ArrayList<>();
+            for (Contend contend : project.getContendList()) {
+                if (question.contains(contend.getBrandName())) {
+                    contendList.add(contend);
+                }
+            }
+            if (CollectionUtils.isNotEmpty(contendList)) {
+                BoolQueryBuilder contendQuery = QueryBuilders.boolQuery();
+                for (Contend contend : contendList) {
+                    contendQuery.should(EsQueryTools.assembleCacheMapsQuery(projectId, contend.getId()));
+                }
+                contendQuery.minimumShouldMatch(1);
+                query.must(contendQuery);
+            }else {
+                query.must(EsQueryTools.assembleCacheMapsQuery(projectId, Constant.PRIMARY_CONTEND_ID));
+            }
+        }
+        // keyword
+        query.must(EsQueryTools.assembleNormalKeywordQuery(keyword, new String[]{GenericAttribute.ES_IND_FULL_TEXT}));
+        String[] fetchSource = {"id", GenericAttribute.ES_TIME, GenericAttribute.ES_IND_TITLE};
+        // hit
+        SearchHit[] hits = searchHits(getIndexes(), query, null, fetchSource, null, 0, 10000, null).getHits();
+        Pair<Map<String, JSONObject>, Map<String, String>> searchProcess = findSearchResultProcess(hits);
+        Map<String, JSONObject> idBaseMap = searchProcess.getLeft();
+        Map<String, String> idTitle = searchProcess.getRight();
+        if (idTitle.isEmpty()){
+            return list;
+        }
+        // 按标题聚合，取聚合结果集前9，并取结果集中最新的文章的id
+        List<String> idList = TextUtil.getKResult(idTitle).stream()
+                .sorted(Comparator.comparing(List<String>::size, Comparator.reverseOrder()))
+                .limit(9)
+                .map(ids -> ids.stream().map(idBaseMap::get).max(Comparator.comparingLong(json -> json.getLongValue(GenericAttribute.ES_TIME))).orElse(null))
+                .filter(Objects::nonNull)
+                .map(json -> json.getString("id"))
+                .collect(Collectors.toList());
+        // 反查原数据
+        for (String id : idList) {
+            list.add(getTopTitleLatest(id));
+        }
+        return list;
+    }
+
+    public List<JSONObject> findSearch(List<FieldMapping> fieldMappings) throws IOException {
+        List<JSONObject> list = new ArrayList<>();
+        BoolQueryBuilder query = getBoolQueryBuilder(fieldMappings);
+        String[] fetchSource = {"id", GenericAttribute.ES_TIME, GenericAttribute.ES_IND_TITLE};
+        SearchHit[] hits = searchHits(getIndexes(), query, null, fetchSource, null, 0, 10000, null).getHits();
+        Pair<Map<String, JSONObject>, Map<String, String>> searchProcess = findSearchResultProcess(hits);
+        Map<String, JSONObject> idBaseMap = searchProcess.getLeft();
+        Map<String, String> idTitle = searchProcess.getRight();
+        // 搜索条件未找到结果，将搜索关键词分词处理，再次查询
+        if (idTitle.isEmpty()){
+            SearchHit[] searchHitHanLP = findSearchHanLP(fieldMappings, fetchSource);
+            Pair<Map<String, JSONObject>, Map<String, String>> searchProcessHanLP = findSearchResultProcess(searchHitHanLP);
+            idBaseMap = searchProcessHanLP.getLeft();
+            idTitle = searchProcessHanLP.getRight();
+        }
+        if (idTitle.isEmpty()){
+            return list;
+        }
+        // 按标题聚合，取聚合结果集前9，并取结果集中最新的文章的id
+        Map<String, JSONObject> finalIdBaseMap = idBaseMap;
+        List<String> idList = TextUtil.getKResult(idTitle).stream()
+                .sorted(Comparator.comparing(List<String>::size, Comparator.reverseOrder()))
+                .limit(9)
+                .map(ids -> ids.stream().map(finalIdBaseMap::get).max(Comparator.comparingLong(json -> json.getLongValue(GenericAttribute.ES_TIME))).orElse(null))
+                .filter(Objects::nonNull)
+                .map(json -> json.getString("id"))
+                .collect(Collectors.toList());
+        // 反查原数据
+        for (String id : idList) {
+            list.add(getTopTitleLatest(id));
+        }
+        return list;
+    }
+
+    private Pair<Map<String, JSONObject>, Map<String, String>> findSearchResultProcess(SearchHit[] hits){
+        Map<String, JSONObject> idBaseMap = Arrays.stream(hits).map(hit -> new JSONObject(hit.getSourceAsMap())).collect(Collectors.toMap(json -> json.getString("id"), o -> o));
+        Map<String, String> idTitle = Arrays.stream(hits)
+                .map(hit -> new JSONObject(hit.getSourceAsMap()))
+                .filter(json -> Objects.nonNull(json.getString(GenericAttribute.ES_IND_TITLE)) || Tools.filterUselessTitle(GenericAttribute.ES_IND_TITLE))
+                .collect(Collectors.toMap(json -> json.getString("id"), json -> json.getString(GenericAttribute.ES_IND_TITLE)));
+        return Pair.of(idBaseMap, idTitle);
+    }
+
+    private SearchHit[] findSearchHanLP(List<FieldMapping> fieldMappings, String[] fetchSource) throws IOException {
+        fieldMappings.stream().filter(fieldMapping -> Objects.equals(FieldMapping.FieldMap.IND_FULL_TEXT, fieldMapping.getFieldMap()))
+                .findFirst().ifPresent(fieldMapping -> {
+                    String value = String.valueOf(fieldMapping.getValue());
+                    String newValue = HanLP.segment(Tools.filterSpecialCharacter(value)).stream().map(s -> s.word).distinct().collect(Collectors.joining(" "));
+                    fieldMapping.setValue(newValue);
+                });
+        BoolQueryBuilder query = getBoolQueryBuilder(fieldMappings);
+        return searchHits(getIndexes(), query, null, fetchSource, null, 0, 10000, null).getHits();
+    }
+
+    private JSONObject getTopTitleLatest(String id) throws IOException {
+        BoolQueryBuilder query = QueryBuilders.boolQuery();
+        query.must(QueryBuilders.termQuery("id", id));
+        return searchById(id);
+    }
+
+    private BoolQueryBuilder getBoolQueryBuilder(List<FieldMapping> fieldMappings) {
+        BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
+        Map<String, List<FieldMapping>> groupMap = fieldMappings.stream().collect(Collectors.groupingBy(mapping -> mapping.getFieldMap().getFatherName()));
+        groupMap.forEach((fatherName, list) -> {
+            if (list.size() > 2) {
+                throw new IllegalStateException("构建搜索条件分组异常");
+            }
+            boolQueryBuilder.must(list.get(0).buildQuery(list.size() > 1 ? list.get(1) : null));
+        });
+        return boolQueryBuilder;
+    }
+
     public String[] getIndexes() {
         return getIndexList().toArray(new String[0]);
     }

src/main/java/com/zhiwei/brandkbs2/pojo/AISearchQuestionRecord.java

+package com.zhiwei.brandkbs2.pojo;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.Setter;
+
+/**
+ * @ClassName: AISearchQuestionRecord
+ * @Description ai搜索问题记录
+ * @author: cjz
+ * @date: 2024-8-12 14:58
+ */
+@Getter
+@Setter
+@AllArgsConstructor
+public class AISearchQuestionRecord extends AbstractBaseMongo {
+    /**
+     * 问题
+     */
+    private String question;
+
+    /**
+     * 项目id
+     */
+    private String projectId;
+
+    /**
+     * 创建时间
+     */
+    private Long cTime;
+}

src/main/java/com/zhiwei/brandkbs2/pojo/UserLogRecord.java

 package com.zhiwei.brandkbs2.pojo;
 
+import com.zhiwei.brandkbs2.auth.UserThreadLocal;
 import lombok.AllArgsConstructor;
 import lombok.Getter;
+import lombok.NoArgsConstructor;
 import lombok.Setter;
-import org.springframework.data.mongodb.core.mapping.Document;
 
 /**
  * @ClassName: UserLogRecord
@@ -14,6 +15,7 @@ import org.springframework.data.mongodb.core.mapping.Document;
 @Getter
 @Setter
 @AllArgsConstructor
+@NoArgsConstructor
 public class UserLogRecord extends AbstractBaseMongo{
     /**
      * 项目id
@@ -43,4 +45,33 @@ public class UserLogRecord extends AbstractBaseMongo{
      * 更新时间
      */
     private Long updateTime;
+    /**
+     * 本次调用费用（ai搜索）
+     */
+    private Double cost;
+
+    public static UserLogRecord userLogRecordCost(String description, double cost){
+        UserLogRecord record = new UserLogRecord();
+        record.setProjectId(UserThreadLocal.getProjectId());
+        record.setUserId(UserThreadLocal.getUserId());
+        record.setNickname(UserThreadLocal.getNickname());
+        record.setDescription(description);
+        record.setRoleId(UserThreadLocal.getRoleId());
+        record.setCTime(System.currentTimeMillis());
+        record.setUpdateTime(System.currentTimeMillis());
+        record.setCost(cost);
+        return record;
+    }
+
+    public static UserLogRecord defaultUserLogRecord(String description){
+        UserLogRecord record = new UserLogRecord();
+        record.setProjectId(UserThreadLocal.getProjectId());
+        record.setUserId(UserThreadLocal.getUserId());
+        record.setNickname(UserThreadLocal.getNickname());
+        record.setDescription(description);
+        record.setRoleId(UserThreadLocal.getRoleId());
+        record.setCTime(System.currentTimeMillis());
+        record.setUpdateTime(System.currentTimeMillis());
+        return record;
+    }
 }

src/main/java/com/zhiwei/brandkbs2/pojo/ai/AccessModel.java

+package com.zhiwei.brandkbs2.pojo.ai;
+
+import lombok.Data;
+import lombok.Getter;
+
+/**
+ * @ClassName: AccessModel
+ * @Description AccessModel
+ * @author: sjj
+ * @date: 2024-07-24 14:05
+ */
+@Data
+public class AccessModel {
+    // 模型名称
+    private String modelName;
+    // 接入点id
+    private String modelId;
+    // 输入单价(每千个token)
+    private Double inputPrice;
+    // 输出单价(每千个token)
+    private Double outputPrice;
+
+    public AccessModel(String modelId, Model model) {
+        this.modelId = modelId;
+        this.modelName = model.modelName;
+        this.inputPrice = model.inputPrice;
+        this.outputPrice = model.outputPrice;
+    }
+
+    public enum Model {
+        DOUBAO_PRO_4K("Doubao-pro-4k", 0.0008, 0.0020),
+        DOUBAO_PRO_32K("Doubao-pro-32k", 0.0008, 0.0020),
+        DOUBAO_PRO_128K("Doubao-pro-128k", 0.0090, 0.0500),
+        DOUBAO_LITE_4K("doubao-lite-4k", 0.0003, 0.0006),
+        DOUBAO_LITE_32K("Doubao-lite-32k", 0.0003, 0.0006),
+        DOUBAO_LITE_128K("Doubao-lite-128k", 0.0008, 0.0010);
+
+        @Getter
+        private final String modelName;
+        // 输入单价(每千个token)
+        @Getter
+        private final Double inputPrice;
+        // 输出单价(每千个token)
+        @Getter
+        private final Double outputPrice;
+
+        Model(String modelName, Double inputPrice, Double outputPrice) {
+            this.modelName = modelName;
+            this.inputPrice = inputPrice;
+            this.outputPrice = outputPrice;
+        }
+    }
+
+}

src/main/java/com/zhiwei/brandkbs2/pojo/ai/FieldMapping.java

+package com.zhiwei.brandkbs2.pojo.ai;
+
+import com.zhiwei.brandkbs2.es.EsQueryTools;
+import lombok.Data;
+import lombok.Getter;
+import org.apache.lucene.search.join.ScoreMode;
+import org.elasticsearch.index.query.*;
+
+import java.util.*;
+
+/**
+ * @ClassName: FieldMap
+ * @Description FieldMap
+ * @author: sjj
+ * @date: 2024-08-02 14:01
+ */
+@Data
+public class FieldMapping {
+
+    private FieldMap fieldMap;
+    private Object value;
+
+    public FieldMapping(FieldMap fieldMap, Object value) {
+        this.fieldMap = fieldMap;
+        this.value = value;
+    }
+
+    public QueryBuilder buildQuery(FieldMapping fieldMapping) {
+        boolean existsAnd = null != fieldMapping;
+        RangeQueryBuilder timeRangeBuilder;
+        String[] contendIds = {"0"};
+        // 项目组需绑定查询
+        switch (fieldMap) {
+            case START_TIME:
+                timeRangeBuilder = QueryBuilders.rangeQuery(fieldMap.databaseName).gte(value);
+                if (existsAnd && fieldMapping.fieldMap.equals(FieldMap.END_TIME)) {
+                    timeRangeBuilder.lt(fieldMapping.value);
+                }
+                return timeRangeBuilder;
+            case END_TIME:
+                timeRangeBuilder = QueryBuilders.rangeQuery(fieldMap.databaseName).lt(value);
+                if (existsAnd && fieldMapping.fieldMap.equals(FieldMap.START_TIME)) {
+                    timeRangeBuilder.gte(fieldMapping.value);
+                }
+                return timeRangeBuilder;
+            case PROJECT:
+                if (existsAnd && fieldMapping.fieldMap == FieldMap.BRAND) {
+                    contendIds = ((String) fieldMapping.value).split("\\|");
+                }
+                BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
+                for (String contendId : contendIds) {
+                    BoolQueryBuilder nestedBoolBuilder = QueryBuilders.boolQuery();
+                    // 必要条件
+                    nestedBoolBuilder.must(QueryBuilders.termQuery(fieldMap.databaseName, value + "_" + contendId));
+                    boolQueryBuilder.should(new NestedQueryBuilder("brandkbs_cache_maps", nestedBoolBuilder, ScoreMode.None));
+                }
+                boolQueryBuilder.minimumShouldMatch(1);
+                return boolQueryBuilder;
+            case BRAND:
+                if (!existsAnd || fieldMapping.fieldMap != FieldMap.PROJECT) {
+                    throw new IllegalStateException("项目条件缺失");
+                }
+                return fieldMapping.buildQuery(this);
+            case IND_FULL_TEXT:
+                return EsQueryTools.assembleNormalKeywordQuery(String.valueOf(value), new String[]{fieldMap.databaseName});
+            case SOURCE:
+            case MTAG:
+                return QueryBuilders.termQuery(fieldMap.databaseName, value);
+        }
+        return null;
+    }
+
+    public enum FieldMap {
+
+        START_TIME("起始时间", "时间", "time"),
+        END_TIME("结束时间", "时间", "time"),
+        PROJECT("项目", "项目", "brandkbs_cache_maps.key.keyword"),
+        BRAND("品牌", "项目", "brandkbs_cache_maps.key.keyword"),
+        SOURCE("渠道", "渠道", "source"),
+        MTAG("标签", "标签", "mark_cache_maps.name.keyword"),
+        IND_FULL_TEXT("搜索条件", "搜索条件", "ind_full_text");
+
+        @Getter
+        private final String name;
+        @Getter
+        private final String fatherName;
+        @Getter
+        private final String databaseName;
+
+        FieldMap(String name, String fatherName, String databaseName) {
+            this.name = name;
+            this.fatherName = fatherName;
+            this.databaseName = databaseName;
+        }
+    }
+
+    public static FieldMapping createFromNameAndValue(String name, Object value, String question) {
+        FieldMap fieldMap = null;
+//        String projectId = UserThreadLocal.getProjectId();
+        // TODO 字段转换待完善,引入数据库
+        for (FieldMap f : FieldMap.values()) {
+            if (name.equals(f.getName())) {
+//                // 项目名需要转成id
+//                if (FieldMap.PROJECT == f) {
+//                    Map<String, Project> projectMap = GlobalPojo.PROJECT_MAP.values().stream().collect(Collectors.toMap(AbstractProject::getProjectName, o -> o));
+//                    if (projectMap.containsKey(String.valueOf(value))) {
+//                        value = projectMap.get(String.valueOf(value)).getId();
+//                    }else {
+//                        value = projectId;
+//                    }
+//                }
+                // 品牌需要转换
+//                if (FieldMap.BRAND == f) {
+//                    Project project = GlobalPojo.PROJECT_MAP.get(projectId);
+//                    if (CollectionUtils.isNotEmpty(project.getContendList())){
+//                        List<String> contends = new ArrayList<>();
+//                        List<String> contendNames = project.getContendList().stream().map(AbstractProject::getBrandName).collect(Collectors.toList());
+//                        for (String contendName : contendNames) {
+//                            if (question.contains(contendName)) {
+//                                contends.add(contendName);
+//                            }
+//                        }
+//                        if (CollectionUtils.isNotEmpty(contends)){
+//                            value = String.join("|", contends);
+//                        }else {
+//                            value = Constant.PRIMARY_CONTEND_ID;
+//                        }
+//                    }else {
+//                        value = Constant.PRIMARY_CONTEND_ID;
+//                    }
+//                    if ("主品牌".equals(value)) {
+//                        value = Constant.PRIMARY_CONTEND_ID;
+//                    } else {
+//                        // 寻找对应的竞品id
+//                        Optional<FieldMapping> project = fieldMappings.stream().filter(field -> Objects.equals(FieldMap.PROJECT, field.getFieldMap())).findFirst();
+//                        if (project.isPresent()){
+//                            List<Contend> contendList = GlobalPojo.PROJECT_MAP.get(String.valueOf(project.get().getValue())).getContendList();
+//                            Object finalValue = value;
+//                            Optional<Contend> contendOptional = contendList.stream().filter(contend -> Objects.equals(contend.getBrandName(), finalValue)).findFirst();
+//                            if (contendOptional.isPresent()){
+//                                value = contendOptional.get().getId();
+//                            }else {
+//                                value = Constant.PRIMARY_CONTEND_ID;
+//                            }
+//                        }else {
+//                            value = Constant.PRIMARY_CONTEND_ID;
+//                        }
+//                    }
+//                }
+                // 标签只包含正负中
+                if (FieldMap.MTAG == f) {
+                    if (!Arrays.asList("正面", "中性", "负面").contains(String.valueOf(value))) {
+                        return null;
+                    }
+                }
+                fieldMap = f;
+                break;
+            }
+        }
+        if (null == fieldMap) {
+            return null;
+        }
+        return new FieldMapping(fieldMap, value);
+    }
+
+}

src/main/java/com/zhiwei/brandkbs2/service/MarkDataService.java

@@ -2,10 +2,7 @@ package com.zhiwei.brandkbs2.service;
 
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.brandkbs2.model.ResponseResult;
-import com.zhiwei.brandkbs2.pojo.BaseMap;
-import com.zhiwei.brandkbs2.pojo.DailyReport;
-import com.zhiwei.brandkbs2.pojo.Event;
-import com.zhiwei.brandkbs2.pojo.MarkFlowEntity;
+import com.zhiwei.brandkbs2.pojo.*;
 import com.zhiwei.brandkbs2.pojo.dto.*;
 import com.zhiwei.brandkbs2.pojo.vo.LineVO;
 import com.zhiwei.brandkbs2.pojo.vo.PageVO;
@@ -836,4 +833,32 @@ public interface MarkDataService {
      * @return
      */
     List<String> expandOriginRange(MarkSearchDTO dto);
+
+    /**
+     * AI搜索-推荐提问
+     * @param question
+     * @return
+     */
+    List<String> getAIReferenceQuestion(String question, int size);
+
+    /**
+     * AI搜索-参考提问
+     * @param cache
+     * @return
+     */
+    List<String> getAIReferenceQuestionCache(boolean cache);
+
+    /**
+     * AI搜索-搜索结果
+     * @param question
+     * @return
+     */
+    JSONObject getAISearchResult(String question, String keyword, Long startTime, Long endTime);
+
+    /**
+     * AI搜索-联网搜索
+     * @param question
+     * @return
+     */
+    JSONObject getAIOnlineSearchResult(String question);
 }

src/main/java/com/zhiwei/brandkbs2/service/TaskService.java

@@ -72,4 +72,9 @@ public interface TaskService{
      * 定时拉取并进行渠道库更新任务
      */
     void refreshChannelRecord();
+
+    /**
+     * 生成ai搜索参考提问缓存
+     */
+    void cacheAIQuestion();
 }

src/main/java/com/zhiwei/brandkbs2/service/impl/BehaviorServiceImpl.java

@@ -174,7 +174,7 @@ public class BehaviorServiceImpl implements BehaviorService {
             if (null == userInfo) {
                 return;
             }
-            UserLogRecord userLogRecord = new UserLogRecord(projectId, userInfo.getUserId(), userInfo.getNickname(), description, userInfo.getRoleId(), now, now);
+            UserLogRecord userLogRecord = new UserLogRecord(projectId, userInfo.getUserId(), userInfo.getNickname(), description, userInfo.getRoleId(), now, now, null);
             String collectionName = userLogRecordDao.generateCollectionName();
             userLogRecordDao.insertOne(userLogRecord, collectionName);
         }

src/main/java/com/zhiwei/brandkbs2/service/impl/ProjectWarnServiceImpl.java

@@ -89,7 +89,7 @@ public class ProjectWarnServiceImpl implements ProjectWarnService {
 
     static {
         TYPE_SEARCH.put("微博热搜", "weibo");
-        TYPE_SEARCH.put("微博话题", "weibo-topic");
+//        TYPE_SEARCH.put("微博话题", "weibo-topic");
         TYPE_SEARCH.put("微博预热", "weibo-rise");
         TYPE_SEARCH.put("头条热搜", "toutiao");
         TYPE_SEARCH.put("抖音热搜", "douyin");
@@ -385,7 +385,8 @@ public class ProjectWarnServiceImpl implements ProjectWarnService {
         // key2
         String key2 = "";
         List<String> key2Element = new ArrayList<>();
-        config.getListType().forEach(type -> {
+        // 2024/8/19 微博话题榜采集下线，防止微博话题配置历史数据影响
+        config.getListType().stream().filter(type -> !Objects.equals("微博话题", type)).collect(Collectors.toList()).forEach(type -> {
             if (config.getFirstTop()) {
                 key2Element.add(type + "榜-首次上榜");
             }

src/main/java/com/zhiwei/brandkbs2/service/impl/TaskServiceImpl.java

@@ -89,6 +89,9 @@ public class TaskServiceImpl implements TaskService {
     @Resource(name = "channelRecordRefreshTaskDao")
     private ChannelRecordRefreshTaskDao channelRecordRefreshTaskDao;
 
+    @Resource(name = "aiSearchQuestionRecordDao")
+    private AISearchQuestionRecordDao aiSearchQuestionRecordDao;
+
     @Resource(name = "brandkbsTaskServiceImpl")
     BrandkbsTaskService brandkbsTaskService;
 
@@ -486,6 +489,21 @@ public class TaskServiceImpl implements TaskService {
         log.info("更新渠道库记录完成-taskId:{}", task.getId());
     }
 
+    @Override
+    public void cacheAIQuestion() {
+        AtomicInteger total = new AtomicInteger();
+        CompletableFuture.allOf(GlobalPojo.PROJECT_MAP.values().stream().map(project -> CompletableFuture.supplyAsync(() -> {
+            UserInfo userInfo = new UserInfo().setProjectId(project.getId());
+            userInfo.setUserId("0");
+            userInfo.setNickname("系统");
+            userInfo.setRoleId(1);
+            UserThreadLocal.set(userInfo);
+            markDataService.getAIReferenceQuestionCache(false);
+            log.info("项目:{}-{}-AI参考问题缓存完成:{}个", project.getProjectName(), project.getId(), total.incrementAndGet());
+            return null;
+        }, cacheServiceExecutor)).toArray(CompletableFuture[]::new)).join();
+    }
+
     private void updateRefreshTask(String id, String status){
         Update update = new Update();
         update.set("status", status);

src/main/java/com/zhiwei/brandkbs2/task/ControlCenter.java

@@ -47,6 +47,7 @@ public class ControlCenter {
 //            taskService.customEventCache();
             taskService.eventAggTitleCache();
             taskService.yuqingAnalyzeHighWordCache();
+            taskService.cacheAIQuestion();
         } catch (Exception e) {
             log.error("定时按天缓存数据-出错", e);
         } finally {

src/main/java/com/zhiwei/brandkbs2/util/RedisUtil.java

@@ -130,6 +130,10 @@ public class RedisUtil {
         return RedisKeyPrefix.SEARCH_KEYWORD + Tools.concat(projectId, userId, searchType);
     }
 
+    public static String getAISearchQuestionCacheKey(String projectId){
+        return RedisKeyPrefix.AI_SEARCH_QUESTION + projectId;
+    }
+
     public void setExpire(String key, String value, long timeout, TimeUnit unit) {
         stringRedisTemplate.opsForValue().set(key, value, timeout, unit);
     }

src/main/java/com/zhiwei/brandkbs2/util/Tools.java

@@ -554,10 +554,13 @@ public class Tools {
         String separator = "-";
         StringBuilder sb = new StringBuilder();
         for (Object obj : objects) {
+            if (Objects.isNull(obj)){
+                continue;
+            }
             sb.append(obj).append(separator);
         }
         String resultStr = sb.toString();
-        return resultStr.substring(0, resultStr.length() - 1);
+        return StringUtils.isBlank(resultStr) ? resultStr : resultStr.substring(0, resultStr.length() - 1);
     }
 
     public static String[] split(String concatStr) {