ai搜索调整2

src/main/java/com/zhiwei/brandkbs2/common/RedisKeyPrefix.java

@@ -121,8 +121,13 @@ public class RedisKeyPrefix {
 
     public static final String CUSTOM_YUQING_ANALYZE_HIGH_WORD = "BRANDKBS:CUSTOM:YUQING:ANALYZE:HIGH:WORD:";
 
+    /**
+     * 搜索相关缓存
+     */
     public static final String SEARCH_KEYWORD = "BRANDKBS:SEARCH:KEYWORD:";
 
+    public static final String AI_SEARCH_QUESTION = "BRANDKBS:AI:SEARCH:QUESTION:";
+
     public static String projectWarnHotTopKeyAll(String projectId, String type) {
         return RedisKeyPrefix.generateRedisKey(RedisKeyPrefix.PROJECT_WARN_HOT_TOP, projectId, Tools.concat(type, "*"));
     }

src/main/java/com/zhiwei/brandkbs2/controller/app/AppSearchController.java

@@ -358,13 +358,19 @@ public class AppSearchController extends BaseController {
         return ResponseResult.success(markDataService.getAISearchResult(question));
     }
 
-    @ApiOperation("搜索-AI参考提问")
+    @ApiOperation("搜索-AI推荐提问")
     @GetMapping("/ai/question")
     public ResponseResult getAIReferenceQuestion(@RequestParam(value = "question") String question,
                                                  @RequestParam(value = "size") int size) {
         return ResponseResult.success(markDataService.getAIReferenceQuestion(question, size));
     }
 
+    @ApiOperation("搜索-AI参考提问")
+    @GetMapping("/ai/question-cache")
+    public ResponseResult getAIReferenceQuestion() {
+        return ResponseResult.success(markDataService.getAIReferenceQuestionCache(true));
+    }
+
     @ApiOperation("搜索-搜索关键词历史记录")
     @GetMapping("/keyword/cache")
     public ResponseResult getSearchKeywordCache(@ApiParam(name = "searchType",

src/main/java/com/zhiwei/brandkbs2/dao/AISearchQuestionRecordDao.java

+package com.zhiwei.brandkbs2.dao;
+
+import com.zhiwei.brandkbs2.pojo.AISearchQuestionRecord;
+
+import java.util.List;
+
+/**
+ * @ClassName: AISearchQuestionRecordDao
+ * @Description AISearchQuestionRecordDao
+ * @author: cjz
+ * @date: 2024-08-12 17:07
+ */
+public interface AISearchQuestionRecordDao extends BaseMongoDao<AISearchQuestionRecord>{
+
+    List<String> findDistinctQuestion(String projectId);
+}

src/main/java/com/zhiwei/brandkbs2/dao/impl/AISearchQuestionRecordDaoImpl.java

+package com.zhiwei.brandkbs2.dao.impl;
+
+import com.zhiwei.brandkbs2.dao.AISearchQuestionRecordDao;
+import com.zhiwei.brandkbs2.pojo.AISearchQuestionRecord;
+import org.springframework.data.domain.Sort;
+import org.springframework.data.mongodb.core.query.Criteria;
+import org.springframework.data.mongodb.core.query.Query;
+import org.springframework.stereotype.Component;
+
+import java.util.List;
+
+/**
+ * @ClassName: AISearchQuestionRecordDao
+ * @Description AISearchQuestionRecordDao
+ * @author: cjz
+ * @date: 2024-08-12 17:07
+ */
+@Component("aiSearchQuestionRecordDao")
+public class AISearchQuestionRecordDaoImpl extends BaseMongoDaoImpl<AISearchQuestionRecord> implements AISearchQuestionRecordDao {
+    private static final String COLLECTION_NAME = "brandkbs_ai_search_question_record";
+
+    public AISearchQuestionRecordDaoImpl() {
+        super(COLLECTION_NAME);
+    }
+
+    @Override
+    public List<String> findDistinctQuestion(String projectId) {
+        Query query = new Query().addCriteria(Criteria.where("projectId").is(projectId)).with(Sort.by(Sort.Order.desc("cTime"))).limit(10);
+        return mongoTemplate.findDistinct(query, "question", COLLECTION_NAME, String.class);
+    }
+}

src/main/java/com/zhiwei/brandkbs2/es/EsClientDao.java

 package com.zhiwei.brandkbs2.es;
 
 import com.alibaba.fastjson.JSONObject;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Maps;
+import com.hankcs.hanlp.HanLP;
 import com.zhiwei.brandkbs2.common.GenericAttribute;
 import com.zhiwei.brandkbs2.config.Constant;
 import com.zhiwei.brandkbs2.pojo.ChannelIndex;
@@ -34,9 +33,6 @@ import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
-import org.elasticsearch.search.aggregations.AggregationBuilders;
-import org.elasticsearch.search.aggregations.bucket.terms.Terms;
-import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
 import org.elasticsearch.search.sort.FieldSortBuilder;
@@ -342,17 +338,32 @@ public class EsClientDao {
     public List<JSONObject> findSearch(List<FieldMapping> fieldMappings) throws IOException {
         List<JSONObject> list = new ArrayList<>();
         BoolQueryBuilder query = getBoolQueryBuilder(fieldMappings);
-        List<JSONObject> searchHits = searchScroll(query, 10000, new String[]{"id", GenericAttribute.ES_TIME, GenericAttribute.ES_IND_TITLE});
-        ImmutableMap<String, JSONObject> idMap = Maps.uniqueIndex(searchHits, json -> json.getString("id"));
-        Map<String, String> idTitle = searchHits.stream().filter(json -> Objects.nonNull(json.getString("ind_title"))).collect(Collectors.toMap(json -> json.getString("id"), json -> json.getString("ind_title")));
+        String[] fetchSource = {"id", GenericAttribute.ES_TIME, GenericAttribute.ES_IND_TITLE};
+        SearchHit[] hits = searchHits(getIndexes(), query, null, fetchSource, null, 0, 10000, null).getHits();
+//        Map<String, JSONObject> idBaseMap = Arrays.stream(hits).map(hit -> new JSONObject(hit.getSourceAsMap())).collect(Collectors.toMap(json -> json.getString("id"), o -> o));
+//        Map<String, String> idTitle = Arrays.stream(hits)
+//                .map(hit -> new JSONObject(hit.getSourceAsMap()))
+//                .filter(json -> Objects.nonNull(json.getString(GenericAttribute.ES_IND_TITLE)))
+//                .collect(Collectors.toMap(json -> json.getString("id"), json -> json.getString(GenericAttribute.ES_IND_TITLE)));
+        Pair<Map<String, JSONObject>, Map<String, String>> searchProcess = findSearchResultProcess(hits);
+        Map<String, JSONObject> idBaseMap = searchProcess.getLeft();
+        Map<String, String> idTitle = searchProcess.getRight();
+        // 搜索条件未找到结果，将搜索关键词分词处理，再次查询
+        if (idTitle.isEmpty()){
+            SearchHit[] searchHitHanLP = findSearchHanLP(fieldMappings, fetchSource);
+            Pair<Map<String, JSONObject>, Map<String, String>> searchProcessHanLP = findSearchResultProcess(searchHitHanLP);
+            idBaseMap = searchProcessHanLP.getLeft();
+            idTitle = searchProcessHanLP.getRight();
+        }
         if (idTitle.isEmpty()){
             return list;
         }
         // 按标题聚合，取聚合结果集前9，并取结果集中最新的文章的id
+        Map<String, JSONObject> finalIdBaseMap = idBaseMap;
         List<String> idList = TextUtil.getKResult(idTitle).stream()
-                .sorted(Comparator.comparingInt(List<String>::size).reversed())
+                .sorted(Comparator.comparing(List<String>::size, Comparator.reverseOrder()))
                 .limit(9)
-                .map(ids -> ids.stream().map(idMap::get).max(Comparator.comparingInt(json -> (int) json.getLongValue(GenericAttribute.ES_TIME))).orElse(null))
+                .map(ids -> ids.stream().map(finalIdBaseMap::get).max(Comparator.comparingLong(json -> json.getLongValue(GenericAttribute.ES_TIME))).orElse(null))
                 .filter(Objects::nonNull)
                 .map(json -> json.getString("id"))
                 .collect(Collectors.toList());
@@ -363,6 +374,26 @@ public class EsClientDao {
         return list;
     }
 
+    private Pair<Map<String, JSONObject>, Map<String, String>> findSearchResultProcess(SearchHit[] hits){
+        Map<String, JSONObject> idBaseMap = Arrays.stream(hits).map(hit -> new JSONObject(hit.getSourceAsMap())).collect(Collectors.toMap(json -> json.getString("id"), o -> o));
+        Map<String, String> idTitle = Arrays.stream(hits)
+                .map(hit -> new JSONObject(hit.getSourceAsMap()))
+                .filter(json -> Objects.nonNull(json.getString(GenericAttribute.ES_IND_TITLE)))
+                .collect(Collectors.toMap(json -> json.getString("id"), json -> json.getString(GenericAttribute.ES_IND_TITLE)));
+        return Pair.of(idBaseMap, idTitle);
+    }
+
+    private SearchHit[] findSearchHanLP(List<FieldMapping> fieldMappings, String[] fetchSource) throws IOException {
+        fieldMappings.stream().filter(fieldMapping -> Objects.equals(FieldMapping.FieldMap.IND_FULL_TEXT, fieldMapping.getFieldMap()))
+                .findFirst().ifPresent(fieldMapping -> {
+                    String value = String.valueOf(fieldMapping.getValue());
+                    String newValue = HanLP.segment(Tools.filterSpecialCharacter(value)).stream().map(s -> s.word).distinct().collect(Collectors.joining("|"));
+                    fieldMapping.setValue(newValue);
+                });
+        BoolQueryBuilder query = getBoolQueryBuilder(fieldMappings);
+        return searchHits(getIndexes(), query, null, fetchSource, null, 0, 10000, null).getHits();
+    }
+
     private JSONObject getTopTitleLatest(BoolQueryBuilder query, String id) throws IOException {
         query.must(QueryBuilders.termQuery("id", id));
         FieldSortBuilder sort = new FieldSortBuilder(GenericAttribute.ES_TIME).order(SortOrder.DESC);
@@ -370,13 +401,6 @@ public class EsClientDao {
         return new JSONObject(searchHits.getAt(0).getSourceAsMap());
     }
 
-//    private JSONObject getTopTitleLatest(BoolQueryBuilder query, String title) throws IOException {
-//        query.must(QueryBuilders.termQuery("agg_title.keyword", title));
-//        FieldSortBuilder sort = new FieldSortBuilder(GenericAttribute.ES_TIME).order(SortOrder.DESC);
-//        SearchHits searchHits = searchHits(getIndexes(), query, null, null, sort, 0, 1, null);
-//        return new JSONObject(searchHits.getAt(0).getSourceAsMap());
-//    }
-
     private BoolQueryBuilder getBoolQueryBuilder(List<FieldMapping> fieldMappings) {
         BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
         Map<String, List<FieldMapping>> groupMap = fieldMappings.stream().collect(Collectors.groupingBy(mapping -> mapping.getFieldMap().getFatherName()));

src/main/java/com/zhiwei/brandkbs2/pojo/AISearchQuestionRecord.java

+package com.zhiwei.brandkbs2.pojo;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.Setter;
+
+/**
+ * @ClassName: AISearchQuestionRecord
+ * @Description ai搜索问题记录
+ * @author: cjz
+ * @date: 2024-8-12 14:58
+ */
+@Getter
+@Setter
+@AllArgsConstructor
+public class AISearchQuestionRecord extends AbstractBaseMongo {
+    /**
+     * 问题
+     */
+    private String question;
+
+    /**
+     * 项目id
+     */
+    private String projectId;
+
+    /**
+     * 创建时间
+     */
+    private Long cTime;
+}

src/main/java/com/zhiwei/brandkbs2/pojo/ai/FieldMapping.java

@@ -2,6 +2,7 @@ package com.zhiwei.brandkbs2.pojo.ai;
 
 import com.zhiwei.brandkbs2.common.GlobalPojo;
 import com.zhiwei.brandkbs2.config.Constant;
+import com.zhiwei.brandkbs2.es.EsQueryTools;
 import com.zhiwei.brandkbs2.pojo.AbstractProject;
 import com.zhiwei.brandkbs2.pojo.Contend;
 import com.zhiwei.brandkbs2.pojo.Project;
@@ -62,7 +63,8 @@ public class FieldMapping {
                 }
                 return fieldMapping.buildQuery(this);
             case IND_FULL_TEXT:
-                return QueryBuilders.matchPhraseQuery(fieldMap.databaseName, value);
+                return EsQueryTools.assembleNormalKeywordQuery(String.valueOf(value), new String[]{fieldMap.databaseName});
+//                return QueryBuilders.matchPhraseQuery(fieldMap.databaseName, value);
             case SOURCE:
             case MTAG:
                 return QueryBuilders.termQuery(fieldMap.databaseName, value);

src/main/java/com/zhiwei/brandkbs2/service/MarkDataService.java

@@ -2,10 +2,7 @@ package com.zhiwei.brandkbs2.service;
 
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.brandkbs2.model.ResponseResult;
-import com.zhiwei.brandkbs2.pojo.BaseMap;
-import com.zhiwei.brandkbs2.pojo.DailyReport;
-import com.zhiwei.brandkbs2.pojo.Event;
-import com.zhiwei.brandkbs2.pojo.MarkFlowEntity;
+import com.zhiwei.brandkbs2.pojo.*;
 import com.zhiwei.brandkbs2.pojo.dto.*;
 import com.zhiwei.brandkbs2.pojo.vo.LineVO;
 import com.zhiwei.brandkbs2.pojo.vo.PageVO;
@@ -838,14 +835,21 @@ public interface MarkDataService {
     List<String> expandOriginRange(MarkSearchDTO dto);
 
     /**
-     * ai搜索-ai参考提问
+     * AI搜索-推荐提问
      * @param question
      * @return
      */
     List<String> getAIReferenceQuestion(String question, int size);
 
     /**
-     * ai搜索-搜索结果
+     * AI搜索-参考提问
+     * @param cache
+     * @return
+     */
+    List<String> getAIReferenceQuestionCache(boolean cache);
+
+    /**
+     * AI搜索-搜索结果
      * @param question
      * @return
      */

src/main/java/com/zhiwei/brandkbs2/service/TaskService.java

@@ -72,4 +72,9 @@ public interface TaskService{
      * 定时拉取并进行渠道库更新任务
      */
     void refreshChannelRecord();
+
+    /**
+     * 生成ai搜索参考提问缓存
+     */
+    void cacheAIQuestion();
 }

src/main/java/com/zhiwei/brandkbs2/service/impl/TaskServiceImpl.java

@@ -89,6 +89,9 @@ public class TaskServiceImpl implements TaskService {
     @Resource(name = "channelRecordRefreshTaskDao")
     private ChannelRecordRefreshTaskDao channelRecordRefreshTaskDao;
 
+    @Resource(name = "aiSearchQuestionRecordDao")
+    private AISearchQuestionRecordDao aiSearchQuestionRecordDao;
+
     @Resource(name = "brandkbsTaskServiceImpl")
     BrandkbsTaskService brandkbsTaskService;
 
@@ -486,6 +489,17 @@ public class TaskServiceImpl implements TaskService {
         log.info("更新渠道库记录完成-taskId:{}", task.getId());
     }
 
+    @Override
+    public void cacheAIQuestion() {
+        AtomicInteger total = new AtomicInteger();
+        CompletableFuture.allOf(GlobalPojo.PROJECT_MAP.values().stream().map(project -> CompletableFuture.supplyAsync(() -> {
+            UserThreadLocal.set(new UserInfo().setProjectId(project.getId()));
+            markDataService.getAIReferenceQuestionCache(false);
+            log.info("项目:{}-{}-AI参考问题缓存完成:{}个", project.getProjectName(), project.getId(), total.incrementAndGet());
+            return null;
+        }, cacheServiceExecutor)).toArray(CompletableFuture[]::new)).join();
+    }
+
     private void updateRefreshTask(String id, String status){
         Update update = new Update();
         update.set("status", status);

src/main/java/com/zhiwei/brandkbs2/task/ControlCenter.java

@@ -47,6 +47,7 @@ public class ControlCenter {
 //            taskService.customEventCache();
             taskService.eventAggTitleCache();
             taskService.yuqingAnalyzeHighWordCache();
+            taskService.cacheAIQuestion();
         } catch (Exception e) {
             log.error("定时按天缓存数据-出错", e);
         } finally {

src/main/java/com/zhiwei/brandkbs2/util/RedisUtil.java

@@ -130,6 +130,10 @@ public class RedisUtil {
         return RedisKeyPrefix.SEARCH_KEYWORD + Tools.concat(projectId, userId, searchType);
     }
 
+    public static String getAISearchQuestionCacheKey(String projectId){
+        return RedisKeyPrefix.AI_SEARCH_QUESTION + projectId;
+    }
+
     public void setExpire(String key, String value, long timeout, TimeUnit unit) {
         stringRedisTemplate.opsForValue().set(key, value, timeout, unit);
     }