Merge branch 'feature' into 'release'

Feature

See merge request !592

src/main/java/com/zhiwei/brandkbs2/config/Constant.java

@@ -26,10 +26,14 @@ public class Constant {
     public static final String MONTH_PATTERN = "yyyy-MM";
     public static final String YEAR_PATTERN = "yyyy";
 
+    public static final String SECOND_PATTERN = "yyyy-MM-dd HH:mm:ss";
+
     public static final FastDateFormat SPEC_MINUTE_FORMAT = FastDateFormat.getInstance(SPEC_MINUTE_PATTERN);
     public static final FastDateFormat SPEC_DAY_POINT_FORMAT = FastDateFormat.getInstance(SPEC_DAY_PATTERN);
     public static final FastDateFormat HOUR_FORMAT = FastDateFormat.getInstance(HOUR_PATTERN);
     public static final FastDateFormat DAY_FORMAT = FastDateFormat.getInstance(DAY_PATTERN);
+
+    public static final FastDateFormat EXCEL_TIME_FORMAT = FastDateFormat.getInstance(SECOND_PATTERN);
     public static final FastDateFormat SPEC_DAY_FORMAT = FastDateFormat.getInstance("yyyy/MM/dd");
     public static final FastDateFormat ONLY_MONTH_FORMAT = FastDateFormat.getInstance("M月");
     public static final FastDateFormat DF_yyyyMMdd = FastDateFormat.getInstance("yyyyMMdd");

src/main/java/com/zhiwei/brandkbs2/easyexcel/EasyExcelUtil.java

@@ -9,6 +9,7 @@ import com.alibaba.excel.write.metadata.WriteSheet;
 import com.zhiwei.brandkbs2.easyexcel.config.ReadExcelDTO;
 import com.zhiwei.brandkbs2.easyexcel.config.WriteExcelDTO;
 import com.zhiwei.brandkbs2.util.Tools;
+import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -26,6 +27,7 @@ import java.nio.channels.ReadableByteChannel;
 import java.nio.channels.WritableByteChannel;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Objects;
 
 /**
  * @author sjj
@@ -112,7 +114,7 @@ public class EasyExcelUtil {
      */
     public static <T> void write(String filePath, String sheetName, Class<T> clazz, List<T> datas) {
         try {
-            formatExcelExports(clazz, datas);
+            formatExcelExports(clazz, datas, null);
             EasyExcel.write(filePath, clazz).sheet(sheetName).doWrite(datas);
         } catch (Exception e) {
             log.error("file:{}，write error：", filePath, e);
@@ -172,6 +174,7 @@ public class EasyExcelUtil {
      */
     public static void dynamicHeadWrite(String filePath, String sheetName, List<List<String>> head, List<List<Object>> datas) {
         try {
+            formatExcelExports(null, null, datas);
             EasyExcel.write(filePath).head(head).sheet(sheetName).doWrite(datas);
         } catch (Exception e) {
             log.error("file:{}，write error：", filePath, e);
@@ -190,7 +193,7 @@ public class EasyExcelUtil {
      */
     public static <T> void download(String fileName, String sheetName, Class<T> clazz, List<T> datas, HttpServletResponse response) {
         try {
-            formatExcelExports(clazz, datas);
+            formatExcelExports(clazz, datas, null);
             response.setContentType("application/vnd.ms-excel");
             response.setCharacterEncoding("utf-8");
             // 这里URLEncoder.encode可以防止中文乱码
@@ -238,26 +241,37 @@ public class EasyExcelUtil {
         return filePath + Tools.concat(projectName, nickName, System.currentTimeMillis(), fileName) + ".xlsx";
     }
 
-    private static <T> void formatExcelExports(Class<T> clazz, List<T> datas){
-        List<Field> stringFields = new ArrayList<>();
-        // 记录需要设置的部分
-        for (Field field : clazz.getDeclaredFields()) {
-            // String格式截取前32767位
-            if (field.getGenericType().toString().equals(String.class.toString())) {
-                field.setAccessible(true);
-                stringFields.add(field);
+    private static <T> void formatExcelExports(Class<T> clazz, List<T> datas, List<List<Object>> dynamicDatas){
+        if (Objects.nonNull(clazz)) {
+            List<Field> stringFields = new ArrayList<>();
+            // 记录需要设置的部分
+            for (Field field : clazz.getDeclaredFields()) {
+                // String格式截取前5000位
+                if (field.getGenericType().toString().equals(String.class.toString())) {
+                    field.setAccessible(true);
+                    stringFields.add(field);
+                }
             }
-        }
-        datas.forEach(data -> {
-            for (Field field : stringFields) {
-                try {
-                    // 截取为excel单元格允许的最大长度
-                    field.set(data, StringUtils.substring((String) (field.get(data)), 0, 32767));
-                } catch (IllegalAccessException e) {
-                    log.error(e);
+            datas.forEach(data -> {
+                for (Field field : stringFields) {
+                    try {
+                        // 截取为excel单元格允许的最大长度
+                        field.set(data, StringUtils.substring((String) (field.get(data)), 0, 5000));
+                    } catch (IllegalAccessException e) {
+                        log.error(e);
+                    }
                 }
+            });
+        }
+        if (CollectionUtils.isNotEmpty(dynamicDatas)){
+            for (List<Object> dynamicData : dynamicDatas) {
+                dynamicData.replaceAll(obj -> {
+                    if (Objects.nonNull(obj)) {
+                        obj = StringUtils.substring(String.valueOf(obj), 0, 5000);
+                    }
+                    return obj;
+                });
             }
-        });
+        }
     }
-
 }

src/main/java/com/zhiwei/brandkbs2/pojo/dto/SearchFilterDTO.java

@@ -177,13 +177,13 @@ public class SearchFilterDTO {
         map.put("source", sourceKeyword);
         map.put("url", hostKeyword);
         if (null == sorter || sorter.isEmpty()) {
-            map.put("sort", "DESC");
+            map.put("sort", "desc");
         } else {
             for (Map.Entry<String, Object> entry : sorter.entrySet()) {
                 if (entry.getValue().toString().contains("desc")) {
-                    map.put("sort", "DESC");
+                    map.put("sort", "desc");
                 } else {
-                    map.put("sort", "ASC");
+                    map.put("sort", "asc");
                 }
             }
         }

src/main/java/com/zhiwei/brandkbs2/service/impl/MarkDataServiceImpl.java

@@ -87,6 +87,7 @@ import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -934,7 +935,7 @@ public class MarkDataServiceImpl implements MarkDataService {
         TermsAggregationBuilder sourceAggregationBuilder = AggregationBuilders.terms("source").field("source").size(10000);
         // query
         BoolQueryBuilder query = yuqingAnalyzeQuery(dto);
-                // 过滤微博
+        // 过滤微博
         query.mustNot(QueryBuilders.termQuery("platform_id", "5d02236e6395002a7c380b79"));
         SearchResponse searchResponse;
         // 单独处理buckets超过上限异常，出错时按时间分段查询
@@ -1098,7 +1099,13 @@ public class MarkDataServiceImpl implements MarkDataService {
         if (0 == hits.getTotalHits().value) {
             return new BaseMap();
         }
-        return Tools.getBaseFromEsMap(hits.getAt(0).getSourceAsMap());
+        JSONObject jsonObject = new JSONObject(hits.getAt(0).getSourceAsMap());
+        BaseMap baseMap = Tools.getBaseFromEsMap(jsonObject);
+        if (StringUtils.isBlank(baseMap.getContent()) && Objects.nonNull(jsonObject.getString("ocr_text")) && !jsonObject.getJSONArray("ocr_text").isEmpty()){
+            List<String> ocrText = jsonObject.getJSONArray("ocr_text").toJavaList(String.class);
+            baseMap.setContent(ocrText.get(0));
+        }
+        return baseMap;
     }
 
     @Override
@@ -2870,11 +2877,11 @@ public class MarkDataServiceImpl implements MarkDataService {
             if (Objects.nonNull(negativeSpreadTendency.getLeft()) || Objects.nonNull(spreadTendency.getLeft()) || Objects.nonNull(positiveSpreadTendency.getLeft())) {
                 CompletableFuture.allOf(Stream.of(spreadTendency.getLeft(), negativeSpreadTendency.getLeft(), positiveSpreadTendency.getLeft())
                         .filter(Objects::nonNull).map(aggTitle -> CompletableFuture.runAsync(() -> {
-                    try {
-                        baseMaps.put(aggTitle, getAnalyzeFirstArticle(dto, aggTitle));
-                    } catch (IOException ignored) {
-                    }
-                }, executor)).toArray(CompletableFuture[]::new)).join();
+                            try {
+                                baseMaps.put(aggTitle, getAnalyzeFirstArticle(dto, aggTitle));
+                            } catch (IOException ignored) {
+                            }
+                        }, executor)).toArray(CompletableFuture[]::new)).join();
             }
             // 标题为空，但趋势图不为空，此时说明最高点的时间段发文平台全为微博，此时取最高点时间段内时间最早的微博发文
             if (Objects.isNull(spreadTendency.getLeft()) && CollectionUtils.isNotEmpty(spreadTendency.getRight())){
@@ -2935,7 +2942,10 @@ public class MarkDataServiceImpl implements MarkDataService {
             return null;
         }
         // 最高点的时间为开始时间
-        LineVO startLine = lines.stream().max(Comparator.comparingLong(LineVO::getCount)).get();
+        LineVO startLine = lines.stream().max(Comparator.comparingLong(LineVO::getCount)).orElse(null);
+        if (Objects.isNull(startLine)){
+            return null;
+        }
         Long startTime = startLine.getDate();
         int startIndex = lines.indexOf(startLine);
         // 最高点后一个点的时间为结束时间
@@ -4076,9 +4086,19 @@ public class MarkDataServiceImpl implements MarkDataService {
                 calculateCost(pair.getRight(), AccessModel.Model.DOUBAO_PRO_32K)), userLogRecordDao.generateCollectionName());
         String resultContent = pair.getLeft();
         String[] splits = resultContent.split("\\|");
-        List<String> result = new ArrayList<>(Arrays.asList(splits)).stream().filter(StringUtils::isNoneBlank).map(String::trim).collect(Collectors.toList());
-        redisUtil.setExpire(key, JSONObject.toJSONString(result));
-        return result;
+        List<String> result = new ArrayList<>(Arrays.asList(splits)).stream().filter(StringUtils::isNoneBlank).map(String::trim)
+                .limit(5).collect(Collectors.toList());
+        List<String> res = new ArrayList<>();
+        for (int i = 0; i < result.size(); i++) {
+            if (result.get(i).startsWith(i + 1 + "、") || result.get(i).startsWith(i + 1 + ".")){
+                String substring = result.get(i).substring(2);
+                res.add(substring);
+            }else {
+                res.add(result.get(i));
+            }
+        }
+        redisUtil.setExpire(key, JSONObject.toJSONString(res));
+        return res;
     }
 
     private List<String> getAIReferenceQuestionTemplate(String projectName){
@@ -4107,7 +4127,7 @@ public class MarkDataServiceImpl implements MarkDataService {
                     project.getContendList().forEach(contend -> brandStr.append("、").append(contend.getBrandName()));
                 }
                 questionPair = standardRequest(question, modelName, MessageFormat.format(QUESTION_PROMPT, brandStr));
-                if (Objects.isNull(questionPair) || Objects.equals("无法回答", questionPair.getLeft())) {
+                if (Objects.isNull(questionPair) || Objects.equals("无法回答", questionPair.getLeft().trim())) {
                     return null;
                 }
                 JSONObject json = JSON.parseObject(questionPair.getLeft());
@@ -5183,7 +5203,7 @@ public class MarkDataServiceImpl implements MarkDataService {
         for (JSONObject hit : hits) {
             BaseMap baseMap = Tools.getBaseFromEsMap(hit);
             List<Object> data = new ArrayList<>();
-            data.add(new Date(baseMap.getTime()));
+            data.add(Constant.EXCEL_TIME_FORMAT.format(baseMap.getTime()));
             data.add(baseMap.getPlatform());
             data.add(baseMap.getSource());
             data.add(baseMap.getTitle());

src/main/java/com/zhiwei/brandkbs2/service/impl/SearchWholeServiceImpl.java

@@ -826,7 +826,10 @@ public class SearchWholeServiceImpl implements SearchWholeService {
      */
     private BaseMap getFirstArticle(SearchFilterDTO dto, List<LineVO> totalSpreadTendency) throws IOException {
         // 最高点的时间为开始时间
-        LineVO startLine = totalSpreadTendency.stream().max(Comparator.comparingLong(LineVO::getCount)).get();
+        LineVO startLine = totalSpreadTendency.stream().max(Comparator.comparingLong(LineVO::getCount)).orElse(null);
+        if (Objects.isNull(startLine)){
+            return null;
+        }
         Long startTime = startLine.getDate();
         int startIndex = totalSpreadTendency.indexOf(startLine);
         // 最高点后一个点的时间为结束时间

src/main/java/com/zhiwei/brandkbs2/util/Tools.java

@@ -80,7 +80,7 @@ public class Tools {
     /**
      * 须过滤的标题
      */
-    public static final List<String> FILTER_TITLE = Arrays.asList("Notitle", "分享一篇文章", "分享一篇文章。", "暂无标题", "");
+    public static final List<String> FILTER_TITLE = Arrays.asList("Notitle", "分享一篇文章", "分享一篇文章。", "暂无标题", "", "No title");
 
     /**
      * 须过滤的渠道