Merge branch 'feature' into 'release'

Feature

See merge request !588

src/main/java/com/zhiwei/brandkbs2/controller/app/AppDownloadController.java

@@ -332,11 +332,11 @@ public class AppDownloadController extends BaseController {
     @LogRecord(description = "舆情库-有效舆情导出", values = {"startTime", "endTime", "customTags", "field", "keyword", "politicsLevel", "mainBodyType", "platforms", "region", "tags"}, entity = true, arguments = true)
     @DownloadTask(taskName = "舆情库有效舆情数据下载", description = "舆情库有效舆情数据")
     public ResponseResult exportYuqingMarkList(@RequestBody MarkSearchDTO markSearchDTO) {
-        Pair<String, List<ExportAppYuqingDTO>> stringListPair = markDataService.downloadYuqingMarkList(markSearchDTO);
+        Pair<String, Pair<List<List<String>>, List<List<Object>>>> stringPairPair = markDataService.downloadMarkListDynamicHead(markSearchDTO);
         // excel写入至指定路径
         String projectName = projectService.getProjectById(UserThreadLocal.getProjectId()).getProjectName();
-        String filePath = EasyExcelUtil.generateExcelFilePath(brandkbsFilePath, projectName, UserThreadLocal.getNickname(), stringListPair.getLeft() + "_舆情列表数据");
-        EasyExcelUtil.write(filePath, "sheet1", ExportAppYuqingDTO.class, stringListPair.getRight());
+        String filePath = EasyExcelUtil.generateExcelFilePath(brandkbsFilePath, projectName, UserThreadLocal.getNickname(), stringPairPair.getLeft() + "_舆情列表数据");
+        EasyExcelUtil.dynamicHeadWrite(filePath, "sheet1", stringPairPair.getRight().getLeft(), stringPairPair.getRight().getRight());
         return ResponseResult.success(filePath);
     }
 

src/main/java/com/zhiwei/brandkbs2/pojo/ai/FieldMapping.java

@@ -7,6 +7,7 @@ import org.apache.lucene.search.join.ScoreMode;
 import org.elasticsearch.index.query.*;
 
 import java.util.*;
+import java.util.stream.Collectors;
 
 /**
  * @ClassName: FieldMap
@@ -94,65 +95,26 @@ public class FieldMapping {
         }
     }
 
-    public static FieldMapping createFromNameAndValue(String name, Object value, String question) {
+    public static FieldMapping createFromNameAndValue(String name, Object value, String brandStr) {
         FieldMap fieldMap = null;
-//        String projectId = UserThreadLocal.getProjectId();
-        // TODO 字段转换待完善,引入数据库
         for (FieldMap f : FieldMap.values()) {
             if (name.equals(f.getName())) {
-//                // 项目名需要转成id
-//                if (FieldMap.PROJECT == f) {
-//                    Map<String, Project> projectMap = GlobalPojo.PROJECT_MAP.values().stream().collect(Collectors.toMap(AbstractProject::getProjectName, o -> o));
-//                    if (projectMap.containsKey(String.valueOf(value))) {
-//                        value = projectMap.get(String.valueOf(value)).getId();
-//                    }else {
-//                        value = projectId;
-//                    }
-//                }
-                // 品牌需要转换
-//                if (FieldMap.BRAND == f) {
-//                    Project project = GlobalPojo.PROJECT_MAP.get(projectId);
-//                    if (CollectionUtils.isNotEmpty(project.getContendList())){
-//                        List<String> contends = new ArrayList<>();
-//                        List<String> contendNames = project.getContendList().stream().map(AbstractProject::getBrandName).collect(Collectors.toList());
-//                        for (String contendName : contendNames) {
-//                            if (question.contains(contendName)) {
-//                                contends.add(contendName);
-//                            }
-//                        }
-//                        if (CollectionUtils.isNotEmpty(contends)){
-//                            value = String.join("|", contends);
-//                        }else {
-//                            value = Constant.PRIMARY_CONTEND_ID;
-//                        }
-//                    }else {
-//                        value = Constant.PRIMARY_CONTEND_ID;
-//                    }
-//                    if ("主品牌".equals(value)) {
-//                        value = Constant.PRIMARY_CONTEND_ID;
-//                    } else {
-//                        // 寻找对应的竞品id
-//                        Optional<FieldMapping> project = fieldMappings.stream().filter(field -> Objects.equals(FieldMap.PROJECT, field.getFieldMap())).findFirst();
-//                        if (project.isPresent()){
-//                            List<Contend> contendList = GlobalPojo.PROJECT_MAP.get(String.valueOf(project.get().getValue())).getContendList();
-//                            Object finalValue = value;
-//                            Optional<Contend> contendOptional = contendList.stream().filter(contend -> Objects.equals(contend.getBrandName(), finalValue)).findFirst();
-//                            if (contendOptional.isPresent()){
-//                                value = contendOptional.get().getId();
-//                            }else {
-//                                value = Constant.PRIMARY_CONTEND_ID;
-//                            }
-//                        }else {
-//                            value = Constant.PRIMARY_CONTEND_ID;
-//                        }
-//                    }
-//                }
                 // 标签只包含正负中
                 if (FieldMap.MTAG == f) {
                     if (!Arrays.asList("正面", "中性", "负面").contains(String.valueOf(value))) {
                         return null;
                     }
                 }
+                // 去除关键词中的项目、品牌名干扰
+                if (FieldMap.IND_FULL_TEXT == f){
+                    String keyword = String.valueOf(value);
+                    List<String> brands = Arrays.stream(brandStr.split("、")).collect(Collectors.toList());
+                    for (String brand : brands) {
+                        if (String.valueOf(value).contains(brand)){
+                            value = keyword.replace(brand, "").trim();
+                        }
+                    }
+                }
                 fieldMap = f;
                 break;
             }

src/main/java/com/zhiwei/brandkbs2/service/MarkDataService.java

@@ -79,6 +79,12 @@ public interface MarkDataService {
     Pair<String, List<ExportAppYuqingDTO>> downloadYuqingMarkList(MarkSearchDTO markSearchDTO);
 
     /**
+     * @param markSearchDTO 标注数据搜索传输类
+     * @return 返回结果
+     */
+    Pair<String, Pair<List<List<String>>, List<List<Object>>>> downloadMarkListDynamicHead(MarkSearchDTO markSearchDTO);
+
+    /**
      * 生成聚合列表并返回id
      *
      * @return 聚合id

src/main/java/com/zhiwei/brandkbs2/service/impl/MarkDataServiceImpl.java

@@ -111,8 +111,9 @@ public class MarkDataServiceImpl implements MarkDataService {
             "1.提及到年、月、日、天、礼拜的定义为时间要素，未提及到默认条件算近一周，条件给到具体的起始时间和结束时间（结束时间为当前时间则不用返回）的时间戳。\n" +
             "2.提及到 XX 渠道的定义为渠道要素，条件给到该渠道名，注意：{0}不可作为渠道名。\n" +
             "3.提及到正面、中性、负面的定义为标签要素，条件给到该标签名。\n" +
-            "4.提及到针对 XX ，针对 XX 相关或 XX 相关的定义为搜索条件要素（包含 针对/相关 字样）。未提及时按照文义在文段内容中提取的一个或多个词作为该文段的关键词，将其定义为搜索条件要素。多个关键词时每个关键词之间严格用” “（空格）作为分隔符进行分隔，单个关键词则无需分隔，关键词必须在给到的文段中出现，条件给到具体值\n" +
-            "5.时间和关键词要素为必需要素，若不满足则返回“无法回答”。\n" +
+            "4.提及到针对 XX ，针对 XX 相关或 XX 相关的定义为搜索条件要素（包含 针对/相关 字样）。未提及时请对内容进行分析，提取出最具代表性和区分性的关键词，这些关键词在搜索过程中能够让搜索引擎精确地找到与该句子含义、所涉及的对象、事件、观点等方面完全契合的数据，尽可能减少无关信息的干扰。\n" +
+            "要确保这些关键词能够作为精准的搜索线索，使得搜索结果紧密围绕句子所表达的核心信息，无论是实体信息、行为信息还是抽象概念信息，都能准确无误地被搜索出来。将这些关键词定义为搜索条件要素。多个关键词时每个关键词之间严格用” “（空格）作为分隔符进行分隔，单个关键词则无需分隔，关键词必须在给到的文段中出现，条件给到具体值\n" +
+            "5.时间要素为必需要素，若不满足则返回“无法回答”。\n" +
             "\n" +
             "参考例子：\n" +
             "示例 1：\n" +
@@ -129,9 +130,9 @@ public class MarkDataServiceImpl implements MarkDataService {
             "1 按照指定输出格式输出。\n" +
             "2 严格按照规则进行提炼。\n" +
             "###";
-    private static final String RESULT_PROMPT = "假如你是专业的分析报告人员，你将根据用户提供的一篇或多篇文章(无关内容则无需引用)，贴合问题给出自己的一点或多点详细分析和见解，每个观点需同时提炼出一个贴合自己的详细分析和见解的小标题（无需注明“小标题”三个字）。并在每点详细分析和见解后用数字表示注明1-{0" +
+    private static final String RESULT_PROMPT = "假如你是专业的文章分析提炼人员，你将根据用户提供的一篇或多篇文章(无关内容则无需引用)，给出自己对于这些文章的一点或多点详细分析和见解，每个观点需同时提炼出一个贴合自己的详细分析和见解的小标题（无需注明“小标题”三个字）。并在每点详细分析和见解后用数字表示注明1-{0" +
             "}的参考文章，分析结果和参考文章之间严格用”|“作为分隔符进行分隔（小标题与分析结果无需分隔），并且多个参考文章之间也严格用”|“作为分隔符进行分隔，若没有对应的参考文章则无需返回，示例：分析结果。|1|2|3。同时注意：无论什么原因无法回答该问题请只需要返回”无法回答“，示例：无法回答" +
-            "请回答该问题：";
+            "文章如下：";
 
     private static final String REFERENCE_QUESTION_PROMPT = "假如你是专业的问题提出人员，提出自己{0}个关于的{1}参考问题，每个问题无需给到对应的序号，问题必须包含 针对/相关 字样，每个问题之间严格用”|“作为分隔符进行分隔。" +
             "请提出：";
@@ -371,6 +372,11 @@ public class MarkDataServiceImpl implements MarkDataService {
     }
 
     @Override
+    public Pair<String, Pair<List<List<String>>, List<List<Object>>>> downloadMarkListDynamicHead(MarkSearchDTO markSearchDTO) {
+        return downloadMarkListDynamicHead(markSearchDTO, esSearchService::searchMarkHitsAndCount);
+    }
+
+    @Override
     public String generateYuqingMarkAggreeList(Long startTime, Long endTime) {
         return generateAggreeList(UserThreadLocal.getProjectId(), startTime, endTime, esSearchService::searchMarkHitsAndCount);
     }
@@ -3105,7 +3111,13 @@ public class MarkDataServiceImpl implements MarkDataService {
         String resultStr;
         // 返回缓存
         if (cache && StringUtils.isNotEmpty(resultStr = redisUtil.get(redisKey))) {
-            return JSONArray.parseArray(resultStr, JSONObject.class);
+            List<JSONObject> res = JSONArray.parseArray(resultStr, JSONObject.class);
+            // 缓存为空，尝试重新生成
+            if (CollectionUtils.isEmpty(res)){
+                res = generateHighWord(dto);
+                redisUtil.setExpire(redisKey, JSONArray.toJSONString(res));
+            }
+            return res;
         }
         List<JSONObject> res = generateHighWord(dto);
         redisUtil.setExpire(redisKey, JSONArray.toJSONString(res));
@@ -3120,9 +3132,9 @@ public class MarkDataServiceImpl implements MarkDataService {
             long sTime = System.currentTimeMillis();
             if (Objects.nonNull(existTask)) {
                 while (!existTask.getFinish()) {
-                    existTask = markSearchAnalyzeTaskDao.findOne("searchFilterDTO", dto);
+                    existTask = markSearchAnalyzeTaskDao.findOne("markSearchDTO", dto);
                     long eTime = System.currentTimeMillis();
-                    if (existTask.getFinish() || eTime - sTime >= 3 * 60 * 1000) {
+                    if (Objects.isNull(existTask) || existTask.getFinish() || eTime - sTime >= 3 * 60 * 1000) {
                         break;
                     }
                     Thread.sleep(10000L);
@@ -4081,7 +4093,7 @@ public class MarkDataServiceImpl implements MarkDataService {
                 }
                 JSONObject json = JSON.parseObject(questionPair.getLeft());
                 // 数据条件
-                List<FieldMapping> filedMapping = getFiledMapping(json, question);
+                List<FieldMapping> filedMapping = getFiledMapping(json, brandStr.toString());
                 addDefaultFiledMapping(filedMapping, question);
                 list = esClientDao.findSearch(filedMapping);
             }
@@ -4242,19 +4254,19 @@ public class MarkDataServiceImpl implements MarkDataService {
     /**
      * 获取
      * @param json
-     * @param content
+     * @param brandStr
      * @return
      */
-    private static List<FieldMapping> getFiledMapping(JSONObject json, String content) {
+    private static List<FieldMapping> getFiledMapping(JSONObject json, String brandStr) {
         List<FieldMapping> res = new ArrayList<>();
         for (Map.Entry<String, Object> entry : json.entrySet()) {
             if (entry.getValue() instanceof JSONObject) {
-                res.addAll(getFiledMapping((JSONObject) entry.getValue(), content));
+                res.addAll(getFiledMapping((JSONObject) entry.getValue(), brandStr));
             } else {
                 if (Objects.isNull(entry.getValue())){
                     continue;
                 }
-                FieldMapping fieldMapping = FieldMapping.createFromNameAndValue(entry.getKey(), entry.getValue(), content);
+                FieldMapping fieldMapping = FieldMapping.createFromNameAndValue(entry.getKey(), entry.getValue(), brandStr);
                 if (null != fieldMapping && Objects.nonNull(fieldMapping.getValue())) {
                     res.add(fieldMapping);
                 }
@@ -4269,6 +4281,7 @@ public class MarkDataServiceImpl implements MarkDataService {
         List<String> projectBandNames = new ArrayList<>();
         projectBandNames.add(project.getProjectName());
         projectBandNames.add(project.getBrandName());
+
         if (CollectionUtils.isNotEmpty(project.getContendList())){
             List<String> contends = new ArrayList<>();
             for (Contend contend : project.getContendList()) {
@@ -5110,4 +5123,88 @@ public class MarkDataServiceImpl implements MarkDataService {
         }
         return null;
     }
+
+    private Pair<String, Pair<List<List<String>>, List<List<Object>>>> downloadMarkListDynamicHead(MarkSearchDTO markSearchDTO, ArticleFunction.SearchHitsAndCount searchHitsAndCount) {
+        try {
+            List<TagFilterDTO> tagFilters = tagFilterService.getTagFilter();
+            List<List<String>> heads = getMarkDownloadHead(tagFilters);
+            List<List<Object>> datas = new ArrayList<>();
+            Project project = projectService.getProjectById(UserThreadLocal.getProjectId());
+            defaultMarkSearch(markSearchDTO);
+            markSearchDTO.setPageSize(null);
+            for (SearchHits searchHits : searchHitsAndCount.search(markSearchDTO, false).getLeft()) {
+                 datas.addAll(createMarkDownloadData(searchHits, tagFilters, heads));
+            }
+            return Pair.of(project.getBrandName() + "_" + markSearchDTO.getStartTime() + "_" + markSearchDTO.getEndTime(), Pair.of(heads, datas));
+        } catch (IOException e) {
+            ExceptionCast.cast(CommonCodeEnum.FAIL, "es查询异常");
+        }
+        return null;
+    }
+
+    /**
+     * 组装excel下载舆情数据
+     * @param searchHits
+     * @param tagFilters
+     * @param heads
+     * @return
+     */
+    private List<List<Object>> createMarkDownloadData(SearchHits searchHits, List<TagFilterDTO> tagFilters, List<List<String>> heads){
+        List<JSONObject> hits = Arrays.stream(searchHits.getHits()).map(SearchHit::getSourceAsMap).map(JSONObject::new).collect(Collectors.toList());
+        List<String> uniqueIds = new ArrayList<>();
+        // 自定义标签uniqueIds
+        tagFilters.forEach(tagFilter -> tagFilter.getTagList().forEach(tag -> uniqueIds.add(tag.getUniqueId())));
+        List<String> groupNames = tagFilters.stream().map(TagFilterDTO::getGroupName).collect(Collectors.toList());
+        List<List<Object>> res = new ArrayList<>();
+        for (JSONObject hit : hits) {
+            BaseMap baseMap = Tools.getBaseFromEsMap(hit);
+            List<Object> data = new ArrayList<>();
+            data.add(new Date(baseMap.getTime()));
+            data.add(baseMap.getPlatform());
+            data.add(baseMap.getSource());
+            data.add(baseMap.getTitle());
+            data.add(baseMap.getContent());
+            data.add(baseMap.getUrl());
+            data.add(baseMap.getEmotion());
+            List<JSONObject> markCacheMaps = hit.getJSONArray(GenericAttribute.ES_MARK_CACHE_MAPS).toJavaList(JSONObject.class);
+            if (CollectionUtils.isEmpty(markCacheMaps)){
+                res.add(data);
+                continue;
+            }
+            // 过滤非自定义标签
+            Map<String, JSONObject> map = markCacheMaps.stream()
+                    .filter(json -> uniqueIds.contains(json.getString("unique_id")))
+                    .collect(Collectors.toMap(json -> json.getString("group_name"), o -> o, (v1, v2) -> v1));
+            for (String name : groupNames) {
+                JSONObject jsonObject = map.get(name);
+                if (Objects.isNull(jsonObject)){
+                    data.add(null);
+                    continue;
+                }
+                data.add(jsonObject.getString("name"));
+            }
+            res.add(data);
+        }
+        return res;
+    }
+
+    /**
+     * 获取舆情数据下载表头
+     * @param tagFilter
+     * @return
+     */
+    private List<List<String>> getMarkDownloadHead(List<TagFilterDTO> tagFilter){
+        List<List<String>> heads = new ArrayList<>();
+        heads.add(Collections.singletonList("时间"));
+        heads.add(Collections.singletonList("平台"));
+        heads.add(Collections.singletonList("渠道"));
+        heads.add(Collections.singletonList("标题"));
+        heads.add(Collections.singletonList("文本"));
+        heads.add(Collections.singletonList("地址"));
+        heads.add(Collections.singletonList("情感调性"));
+        for (TagFilterDTO tagFilterDTO : tagFilter) {
+            heads.add(Collections.singletonList(tagFilterDTO.getGroupName()));
+        }
+        return heads;
+    }
 }
\ No newline at end of file

src/main/java/com/zhiwei/brandkbs2/service/impl/SearchWholeServiceImpl.java

@@ -699,7 +699,7 @@ public class SearchWholeServiceImpl implements SearchWholeService {
                 while (!existTask.getFinish()) {
                     existTask = markSearchAnalyzeTaskDao.findOne("searchFilterDTO", dto);
                     long eTime = System.currentTimeMillis();
-                    if (existTask.getFinish() || eTime - sTime >= 3 * 60 * 1000) {
+                    if (Objects.isNull(existTask) || existTask.getFinish() || eTime - sTime >= 3 * 60 * 1000) {
                         break;
                     }
                     Thread.sleep(10000L);