Merge branch 'feature' into 'dev'

2023/10/12 工具库-词云提取功能

See merge request !402

src/main/java/com/zhiwei/brandkbs2/aop/AopDownloadTask.java

@@ -6,7 +6,6 @@ import com.zhiwei.brandkbs2.model.CommonCodeEnum;
 import com.zhiwei.brandkbs2.model.ResponseResult;
 import com.zhiwei.brandkbs2.pojo.DownloadTask;
 import com.zhiwei.brandkbs2.service.DownloadTaskService;
-import io.swagger.annotations.ApiOperation;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.aspectj.lang.ProceedingJoinPoint;
@@ -19,8 +18,6 @@ import org.springframework.stereotype.Component;
 
 import javax.annotation.Resource;
 import java.lang.reflect.Method;
-import java.util.Arrays;
-import java.util.List;
 
 /**
  * @author cjz
@@ -37,18 +34,20 @@ public class AopDownloadTask {
     @Resource(name = "downloadTaskServiceImpl")
     DownloadTaskService downloadTaskService;
 
-    private static final List<String> METHOD = Arrays.asList("downloadBatchArticleSummary", "downloadUrlInteractionUpdate", "getMarkInteractionUpdate");
-
-    @Around(value = "execution(public * com..controller..app..AppDownloadController.*(..)) || execution(* com..controller..app..AppToolsetController.download*(..)) || execution(* com..controller..app..AppArticleController.getMarkInteractionUpdate(..))")
+    @Around(value = "execution(public * com..controller..admin..*Controller.*(..)) || execution(* com..controller..app..*Controller.*(..))")
     public Object around(ProceedingJoinPoint joinPoint) throws Throwable{
         Signature signature = joinPoint.getSignature();
         Method method = ((MethodSignature) signature).getMethod();
+        // 方法上无此注解，无需生成下载任务
+        if (!method.isAnnotationPresent(com.zhiwei.brandkbs2.aop.DownloadTask.class)){
+            return joinPoint.proceed();
+        }
         // 生成下载任务
-        String taskName = method.getAnnotation(ApiOperation.class).value() + "下载";
-        String description = method.getAnnotation(ApiOperation.class).value();
+        String taskName = method.getAnnotation(com.zhiwei.brandkbs2.aop.DownloadTask.class).taskName();
+        String description = method.getAnnotation(com.zhiwei.brandkbs2.aop.DownloadTask.class).description();
         String taskId = downloadTaskService.createDownloadTask(taskName, description);
-        Object proceed = null;
         String fileAddress;
+        Object proceed = null;
         // 执行目标方法
         try {
             proceed = joinPoint.proceed();
@@ -58,7 +57,7 @@ public class AopDownloadTask {
             ExceptionCast.cast(CommonCodeEnum.FAIL, "下载异常", e);
         }
         // 更新下载任务
-        if (METHOD.contains(method.getName())){
+        if (method.getAnnotation(com.zhiwei.brandkbs2.aop.DownloadTask.class).entity()){
             fileAddress = JSONObject.parseObject(((ResponseResult) proceed).getData().toString()).getString("filePath");
         }else {
             fileAddress = ((ResponseResult) proceed).getData().toString();

src/main/java/com/zhiwei/brandkbs2/aop/DownloadTask.java

+package com.zhiwei.brandkbs2.aop;
+
+import java.lang.annotation.*;
+
+@Target(ElementType.METHOD)
+@Retention(RetentionPolicy.RUNTIME)
+@Documented
+public @interface DownloadTask {
+
+    String taskName();
+
+    String description();
+
+    boolean entity() default false;
+}

src/main/java/com/zhiwei/brandkbs2/controller/app/AppArticleController.java

 package com.zhiwei.brandkbs2.controller.app;
 
 import com.alibaba.fastjson.JSONObject;
+import com.zhiwei.brandkbs2.aop.DownloadTask;
 import com.zhiwei.brandkbs2.aop.LogRecord;
 import com.zhiwei.brandkbs2.auth.Auth;
 import com.zhiwei.brandkbs2.auth.UserThreadLocal;
@@ -306,9 +307,10 @@ public class AppArticleController extends BaseController {
         return reportService.getReportSchedule(idList);
     }
 
-    @ApiOperation("有效舆情互动量更新")
+    @ApiOperation("舆情库-有效舆情-互动量更新")
     @PostMapping("/mark/interaction-update")
     @LogRecord(description = "舆情库-有效舆情互动量更新")
+    @DownloadTask(taskName = "有效舆情互动量更新结果下载", description = "有效舆情互动量更新结果", entity = true)
     public ResponseResult getMarkInteractionUpdate(@RequestBody MarkSearchDTO markSearchDTO){
         return markDataService.markInteractionUpdate(markSearchDTO);
     }

src/main/java/com/zhiwei/brandkbs2/controller/app/AppSearchController.java

@@ -133,9 +133,8 @@ public class AppSearchController extends BaseController {
 
     @ApiOperation("搜索-全网搜条件")
     @GetMapping("/getWholeSearchCriteria")
-    public ResponseResult getWholeSearchCriteria() {
-        JSONObject wholeSearchCriteria = markDataService.getWholeNetworkSearchCriteria();
-        wholeSearchCriteria.putAll(systemInfoService.getExtraParam());
+    public ResponseResult getWholeSearchCriteria(@RequestParam(value = "externalDataSource", defaultValue = "false") boolean externalDataSource) {
+        JSONObject wholeSearchCriteria = markDataService.getWholeNetworkSearchCriteria(externalDataSource);
         return ResponseResult.success(wholeSearchCriteria);
     }
 

src/main/java/com/zhiwei/brandkbs2/controller/app/AppToolsetController.java

 package com.zhiwei.brandkbs2.controller.app;
 
+import com.zhiwei.brandkbs2.aop.DownloadTask;
 import com.zhiwei.brandkbs2.aop.LogRecord;
 import com.zhiwei.brandkbs2.auth.Auth;
 import com.zhiwei.brandkbs2.enmus.RoleEnum;
@@ -7,6 +8,7 @@ import com.zhiwei.brandkbs2.model.ResponseResult;
 import com.zhiwei.brandkbs2.service.ToolsetService;
 import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiOperation;
+import io.swagger.annotations.ApiParam;
 import org.springframework.web.bind.annotation.*;
 import org.springframework.web.multipart.MultipartFile;
 
@@ -27,36 +29,46 @@ public class AppToolsetController {
     @Resource(name = "toolsetServiceImpl")
     private ToolsetService toolsetService;
 
-    @ApiOperation("摘要提取-单条")
+    @ApiOperation("工具库-摘要提取-单条")
     @GetMapping("/article-summary/single")
     @LogRecord(description = "工具库-摘要提取-单条")
     public ResponseResult getSingleArticleSummary(@RequestParam(value = "url") String url) {
         return toolsetService.getSingleArticleSummary(url);
     }
 
-    @ApiOperation("摘要提取")
+    @ApiOperation("工具库-摘要提取-批量")
     @PostMapping("/article-summary/batch")
     @LogRecord(description = "工具库-摘要提取-批量")
+    @DownloadTask(taskName = "批量摘要提取结果下载", description = "批量摘要提取结果", entity = true)
     public ResponseResult downloadBatchArticleSummary(@RequestParam(value = "file") MultipartFile file){
         return toolsetService.getBatchArticleSummary(file);
     }
 
-    @ApiOperation("摘要提取-剩余可用次数")
+    @ApiOperation("工具库-摘要提取-剩余可用次数")
     @GetMapping("/article-summary/remaining")
     public ResponseResult getArticleSummaryRemainingCount(){
         return toolsetService.getArticleSummaryRemainingCount();
     }
 
-    @ApiOperation("链接互动量更新")
+    @ApiOperation("工具库-互动量更新-链接互动量更新")
     @PostMapping("/interaction-update/url")
     @LogRecord(description = "工具库-链接互动量更新")
+    @DownloadTask(taskName = "链接互动量更新结果下载", description = "链接互动量更新结果", entity = true)
     public ResponseResult downloadUrlInteractionUpdate(@RequestParam(value = "file") MultipartFile file){
         return toolsetService.urlInteractionUpdate(file);
     }
 
-    @ApiOperation("互动量更新-剩余可用次数")
+    @ApiOperation("工具库-互动量更新-剩余可用次数")
     @GetMapping("/interaction-update/remaining")
     public ResponseResult getInteractionRemainingCount(){
         return ResponseResult.success(toolsetService.getInteractionRemainingCount());
     }
+
+    @ApiOperation("工具库-词云提取")
+    @PostMapping("/high-word")
+    @LogRecord(description = "工具库-词云提取")
+    public ResponseResult getHighWord(@RequestParam MultipartFile file,
+                                      @ApiParam(name = "type", value = "关键词|文章") @RequestParam String type){
+        return ResponseResult.success(toolsetService.getHighWord(file, type));
+    }
 }

src/main/java/com/zhiwei/brandkbs2/easyexcel/dto/UploadHighWordArticleDTO.java

+package com.zhiwei.brandkbs2.easyexcel.dto;
+
+import com.alibaba.excel.annotation.ExcelProperty;
+import lombok.Data;
+
+/**
+ * @author cjz
+ * @version 1.0
+ * @description 解析文章词云提取上传文件
+ * @date 2023/10/10 10:22
+ */
+@Data
+public class UploadHighWordArticleDTO {
+    @ExcelProperty("序号")
+    private String id;
+
+    @ExcelProperty("标题")
+    private String title;
+
+    @ExcelProperty("文本")
+    private String content;
+
+    @ExcelProperty("链接")
+    private String url;
+}

src/main/java/com/zhiwei/brandkbs2/easyexcel/dto/UploadHighWordKeywordDTO.java

+package com.zhiwei.brandkbs2.easyexcel.dto;
+
+import com.alibaba.excel.annotation.ExcelProperty;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+/**
+ * @author cjz
+ * @version 1.0
+ * @description 解析关键词词云提取上传文件
+ * @date 2023/10/10 10:22
+ */
+@Data
+@NoArgsConstructor
+public class UploadHighWordKeywordDTO {
+    @ExcelProperty("序号")
+    private String id;
+
+    @ExcelProperty("关键词")
+    private String keyword;
+
+    @ExcelProperty("词频")
+    private Integer frequency;
+}

src/main/java/com/zhiwei/brandkbs2/easyexcel/listener/HighWordArticleListener.java

+package com.zhiwei.brandkbs2.easyexcel.listener;
+
+import com.alibaba.excel.context.AnalysisContext;
+import com.alibaba.excel.event.AnalysisEventListener;
+import com.zhiwei.brandkbs2.easyexcel.dto.UploadHighWordArticleDTO;
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.List;
+
+/**
+ * @author cjz
+ * @version 1.0
+ * @description 解析文章词云提取上传文件
+ * @date 2023/10/10 10:23
+ */
+public class HighWordArticleListener extends AnalysisEventListener<UploadHighWordArticleDTO> {
+
+    List<String> contents;
+
+    public HighWordArticleListener(List<String> contents){
+        this.contents = contents;
+    }
+
+    @Override
+    public void invoke(UploadHighWordArticleDTO data, AnalysisContext context) {
+        if (StringUtils.isNotBlank(data.getContent())) {
+            contents.add(data.getContent());
+        }
+    }
+
+    @Override
+    public void doAfterAllAnalysed(AnalysisContext context) {
+
+    }
+}

src/main/java/com/zhiwei/brandkbs2/easyexcel/listener/HighWordKeywordListener.java

+package com.zhiwei.brandkbs2.easyexcel.listener;
+
+import com.alibaba.excel.context.AnalysisContext;
+import com.alibaba.excel.event.AnalysisEventListener;
+import com.zhiwei.brandkbs2.easyexcel.dto.UploadHighWordKeywordDTO;
+
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * @author lxj
+ * @version 1.0
+ * @description 解析词云关键词文件监听器
+ * @date 2023/10/10 10:30
+ */
+public class HighWordKeywordListener extends AnalysisEventListener<UploadHighWordKeywordDTO> {
+
+    private Map<String, Integer> map;
+
+    public HighWordKeywordListener(Map<String, Integer> map){
+        this.map = map;
+    }
+
+    @Override
+    public void invoke(UploadHighWordKeywordDTO data, AnalysisContext context) {
+        map.compute(data.getKeyword(), (k, v) ->{
+            if (Objects.isNull(v)){
+                return data.getFrequency();
+            }
+            if (v < data.getFrequency()){
+                return data.getFrequency();
+            }
+            return v;
+        });
+    }
+
+    @Override
+    public void doAfterAllAnalysed(AnalysisContext context) {
+    }
+}

src/main/java/com/zhiwei/brandkbs2/service/MarkDataService.java

@@ -296,7 +296,7 @@ public interface MarkDataService {
      * 搜索-全网搜条件
      * @return JSONObject
      */
-    JSONObject getWholeNetworkSearchCriteria();
+    JSONObject getWholeNetworkSearchCriteria(boolean externalDataSource);
 
     /**
      * 搜索-全网搜

src/main/java/com/zhiwei/brandkbs2/service/ToolsetService.java

@@ -62,4 +62,11 @@ public interface ToolsetService {
      */
     List<JSONObject> getInteractionResult(List<String> taskId);
 
+    /**
+     * 词云提取
+     * @param file  excel文件
+     * @param type  类型 关键词|文章
+     * @return
+     */
+    List<JSONObject> getHighWord(MultipartFile file, String type);
 }

src/main/java/com/zhiwei/brandkbs2/service/impl/MarkDataServiceImpl.java

@@ -163,6 +163,9 @@ public class MarkDataServiceImpl implements MarkDataService {
     @Resource(name = "extraServiceImpl")
     private ExtraService extraService;
 
+    @Resource(name = "systemInfoServiceImpl")
+    SystemInfoService systemInfoService;
+
     @Value("${brandkbs.file.url}")
     private String brandkbsFilePath;
 
@@ -1128,14 +1131,19 @@ public class MarkDataServiceImpl implements MarkDataService {
     }
 
     @Override
-    public JSONObject getWholeNetworkSearchCriteria() {
-        String projectId = UserThreadLocal.getProjectId();
-        Project project = projectService.getProjectById(projectId);
+    public JSONObject getWholeNetworkSearchCriteria(boolean externalDataSource) {
         JSONObject result = new JSONObject();
         // 搜索时间
         result.put("times", Arrays.asList("今天", "24小时", "三天", "七天", "近30天"));
         result.put("ninetyDays", DateUtils.addDays(Tools.truncDate(new Date(), Constant.DAY_PATTERN), -89).getTime());
-        result.put("platformList", commonService.getQbjcPlatform("id", "name").stream().filter(s -> !s.get("name").equals("脉脉")).collect(Collectors.toList()));
+        // 外部数据源（实时采集）条件
+        if (externalDataSource) {
+            result.put("platformList", Arrays.asList("App", "博客", "平媒", "新闻", "微博", "微信", "论坛", "视频", "短视频"));
+        } else {
+            result.put("platformList", commonService.getQbjcPlatform("id", "name")
+                    .stream().filter(s -> !s.get("name").equals("脉脉")).collect(Collectors.toList()));
+            result.putAll(systemInfoService.getExtraParam());
+        }
         return result;
     }
 
@@ -1144,9 +1152,6 @@ public class MarkDataServiceImpl implements MarkDataService {
         if (dto.isExternalDataSource()) { // 查商业数据库
             try {
                 JSONObject params = externalTransform(dto);
-//                JSONObject params = IstarShineUtil.getKSSearchParam(dto.getSearch(), dto.getFilterWords(), dto.getStartTime(), dto.getEndTime(), dto.getPlatformsStr(),
-//                        dto.getFilterForward(), dto.getSortOf(), dto.getPage(), dto.getPageSize(), dto.getMatchFields(), null, null);
-
                 JSONObject taskEntity = new JSONObject();
                 taskEntity.put("appId", "6183571e0d710000f6003a12");   // 应用id, 由张志伟提供给使用者
                 taskEntity.put("taskType", null);                      // 可为null, 任务类型
@@ -1154,7 +1159,6 @@ public class MarkDataServiceImpl implements MarkDataService {
                 taskEntity.put("taskName", UserThreadLocal.getProjectId() + "_" + UserThreadLocal.getUserId());
                 taskEntity.put("userName", "品见");                     // 提交任务的人
                 taskEntity.put("searchParams", params);                // 任务详情参数
-
                 ResponseEntity<JSONObject> responseEntity = restTemplate.postForEntity(addIStarShineKSDataUrl, taskEntity, JSONObject.class);
                 JSONObject result = Objects.requireNonNull(responseEntity.getBody()).getJSONObject("data");
                 JSONObject json = searchWholeBackUp(result.get("id").toString());
@@ -1249,8 +1253,12 @@ public class MarkDataServiceImpl implements MarkDataService {
                 //pageSize
                 Integer receiveCount = task.getInteger("receiveCount");
                 Integer searchCount = task.getInteger("searchCount");
-                int totalPage = searchCount % receiveCount == 0 ? searchCount / receiveCount : (searchCount / receiveCount + 1);
-
+                int totalPage;
+                if (0 == receiveCount || 0 == searchCount){
+                    totalPage = 0;
+                }else {
+                    totalPage = searchCount % receiveCount == 0 ? searchCount / receiveCount : (searchCount / receiveCount + 1);
+                }
                 List<JSONObject> listArray = data.getJSONArray("data").toJavaList(JSONObject.class);
                 for (JSONObject json : listArray) {
                     JSONObject result = new JSONObject();

src/main/java/com/zhiwei/brandkbs2/service/impl/ToolsetServiceImpl.java

@@ -5,11 +5,10 @@ import com.zhiwei.brandkbs2.auth.UserThreadLocal;
 import com.zhiwei.brandkbs2.controller.app.AppToolsetController;
 import com.zhiwei.brandkbs2.easyexcel.EasyExcelUtil;
 import com.zhiwei.brandkbs2.easyexcel.config.ReadExcelDTO;
-import com.zhiwei.brandkbs2.easyexcel.dto.ExportArticleSummaryDTO;
-import com.zhiwei.brandkbs2.easyexcel.dto.ExportInteractionUpdateDTO;
-import com.zhiwei.brandkbs2.easyexcel.dto.UploadArticleSummaryDTO;
-import com.zhiwei.brandkbs2.easyexcel.dto.UploadInteractionUpdateDTO;
+import com.zhiwei.brandkbs2.easyexcel.dto.*;
 import com.zhiwei.brandkbs2.easyexcel.listener.ArticleSummaryListener;
+import com.zhiwei.brandkbs2.easyexcel.listener.HighWordArticleListener;
+import com.zhiwei.brandkbs2.easyexcel.listener.HighWordKeywordListener;
 import com.zhiwei.brandkbs2.easyexcel.listener.InteractionUpdateListener;
 import com.zhiwei.brandkbs2.enmus.InteractionEnum;
 import com.zhiwei.brandkbs2.exception.ExceptionCast;
@@ -21,6 +20,7 @@ import com.zhiwei.brandkbs2.service.ExtraService;
 import com.zhiwei.brandkbs2.service.ProjectService;
 import com.zhiwei.brandkbs2.service.ToolsetService;
 import com.zhiwei.brandkbs2.util.RedisUtil;
+import com.zhiwei.brandkbs2.util.TextUtil;
 import com.zhiwei.brandkbs2.util.Tools;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.collections4.ListUtils;
@@ -62,6 +62,9 @@ public class ToolsetServiceImpl implements ToolsetService {
     @Autowired
     private RestTemplate restTemplate;
 
+    @Autowired
+    private TextUtil textUtil;
+
     @Value("${toolset.articleSummary.url}")
     private String articleSummaryUrl;
 
@@ -287,6 +290,56 @@ public class ToolsetServiceImpl implements ToolsetService {
         return res;
     }
 
+    @Override
+    public List<JSONObject> getHighWord(MultipartFile file, String type) {
+        if (Objects.equals("关键词", type)){
+            return getKeywordHighWord(file);
+        }
+        return getArticleHighWord(file);
+    }
+
+    /**
+     * 关键词词云提取
+     * @param file  excel文件
+     * @return
+     */
+    private List<JSONObject> getKeywordHighWord(MultipartFile file){
+        Map<String, Integer> map = new HashMap<>();
+        // excel信息提取
+        ReadExcelDTO<UploadHighWordKeywordDTO> readExcel = new ReadExcelDTO<>();
+        readExcel.setClazz(UploadHighWordKeywordDTO.class);
+        readExcel.setAnalysisEventListener(new HighWordKeywordListener(map));
+        EasyExcelUtil.read(file, readExcel);
+        Map<String, Integer> sortMap = Tools.sortMap(map, 200);
+        List<JSONObject> res = new ArrayList<>();
+        for (Map.Entry<String, Integer> entry : sortMap.entrySet()) {
+            JSONObject jsonObject = new JSONObject();
+            jsonObject.put("name", entry.getKey());
+            jsonObject.put("value", entry.getValue());
+            res.add(jsonObject);
+        }
+        return res;
+    }
+
+    /**
+     * 文章词云提取
+     * @param file excel文件
+     * @return
+     */
+    private List<JSONObject> getArticleHighWord(MultipartFile file){
+        List<String> contents = new ArrayList<>();
+        // excel信息提取
+        ReadExcelDTO<UploadHighWordArticleDTO> readExcel = new ReadExcelDTO<>();
+        readExcel.setClazz(UploadHighWordArticleDTO.class);
+        readExcel.setAnalysisEventListener(new HighWordArticleListener(contents));
+        EasyExcelUtil.read(file, readExcel);
+        return textUtil.getHighWordsJsonDifferentFieldName(contents, 200);
+    }
+
+    /**
+     * 获取互动量请求头
+     * @return
+     */
     private HttpHeaders getInteractionRequestHeader(){
         // 请求头参数
         Long timeStamp = System.currentTimeMillis();

src/main/java/com/zhiwei/brandkbs2/util/TextUtil.java

@@ -120,4 +120,16 @@ public class TextUtil {
         return result;
     }
 
+    public List<JSONObject> getHighWordsJsonDifferentFieldName(List<String> texts, Integer maxSize) {
+//        Map<String, Integer> highWords = getHighWords(texts, maxSize);
+        List<Map.Entry<String, Integer>> wordRate = ansjSeg.getFenCi(texts, maxSize);
+        List<JSONObject> result = new ArrayList<>(wordRate.size());
+        for (Map.Entry<String, Integer> entry : wordRate) {
+            JSONObject jsonObject = new JSONObject();
+            jsonObject.put("name", entry.getKey());
+            jsonObject.put("value", entry.getValue());
+            result.add(jsonObject);
+        }
+        return result;
+    }
 }