Merge branch 'feature' into 'release'

2023/10/19 工具库-词云提取上线

See merge request !403

src/main/java/com/zhiwei/brandkbs2/aop/AopDownloadTask.java

@@ -6,7 +6,6 @@ import com.zhiwei.brandkbs2.model.CommonCodeEnum;
 import com.zhiwei.brandkbs2.model.ResponseResult;
 import com.zhiwei.brandkbs2.pojo.DownloadTask;
 import com.zhiwei.brandkbs2.service.DownloadTaskService;
-import io.swagger.annotations.ApiOperation;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.aspectj.lang.ProceedingJoinPoint;
@@ -19,8 +18,6 @@ import org.springframework.stereotype.Component;
 
 import javax.annotation.Resource;
 import java.lang.reflect.Method;
-import java.util.Arrays;
-import java.util.List;
 
 /**
  * @author cjz
@@ -37,18 +34,20 @@ public class AopDownloadTask {
     @Resource(name = "downloadTaskServiceImpl")
     DownloadTaskService downloadTaskService;
 
-    private static final List<String> METHOD = Arrays.asList("downloadBatchArticleSummary", "downloadUrlInteractionUpdate", "getMarkInteractionUpdate");
-
-    @Around(value = "execution(public * com..controller..app..AppDownloadController.*(..)) || execution(* com..controller..app..AppToolsetController.download*(..)) || execution(* com..controller..app..AppArticleController.getMarkInteractionUpdate(..))")
+    @Around(value = "execution(public * com..controller..admin..*Controller.*(..)) || execution(* com..controller..app..*Controller.*(..))")
     public Object around(ProceedingJoinPoint joinPoint) throws Throwable{
         Signature signature = joinPoint.getSignature();
         Method method = ((MethodSignature) signature).getMethod();
+        // 方法上无此注解，无需生成下载任务
+        if (!method.isAnnotationPresent(com.zhiwei.brandkbs2.aop.DownloadTask.class)){
+            return joinPoint.proceed();
+        }
         // 生成下载任务
-        String taskName = method.getAnnotation(ApiOperation.class).value() + "下载";
-        String description = method.getAnnotation(ApiOperation.class).value();
+        String taskName = method.getAnnotation(com.zhiwei.brandkbs2.aop.DownloadTask.class).taskName();
+        String description = method.getAnnotation(com.zhiwei.brandkbs2.aop.DownloadTask.class).description();
         String taskId = downloadTaskService.createDownloadTask(taskName, description);
-        Object proceed = null;
         String fileAddress;
+        Object proceed = null;
         // 执行目标方法
         try {
             proceed = joinPoint.proceed();
@@ -58,7 +57,7 @@ public class AopDownloadTask {
             ExceptionCast.cast(CommonCodeEnum.FAIL, "下载异常", e);
         }
         // 更新下载任务
-        if (METHOD.contains(method.getName())){
+        if (method.getAnnotation(com.zhiwei.brandkbs2.aop.DownloadTask.class).entity()){
             fileAddress = JSONObject.parseObject(((ResponseResult) proceed).getData().toString()).getString("filePath");
         }else {
             fileAddress = ((ResponseResult) proceed).getData().toString();

src/main/java/com/zhiwei/brandkbs2/aop/DownloadTask.java

+package com.zhiwei.brandkbs2.aop;
+
+import java.lang.annotation.*;
+
+@Target(ElementType.METHOD)
+@Retention(RetentionPolicy.RUNTIME)
+@Documented
+public @interface DownloadTask {
+
+    String taskName();
+
+    String description();
+
+    boolean entity() default false;
+}

src/main/java/com/zhiwei/brandkbs2/controller/app/AppArticleController.java

 package com.zhiwei.brandkbs2.controller.app;
 
 import com.alibaba.fastjson.JSONObject;
+import com.zhiwei.brandkbs2.aop.DownloadTask;
 import com.zhiwei.brandkbs2.aop.LogRecord;
 import com.zhiwei.brandkbs2.auth.Auth;
 import com.zhiwei.brandkbs2.auth.UserThreadLocal;
@@ -306,9 +307,10 @@ public class AppArticleController extends BaseController {
         return reportService.getReportSchedule(idList);
     }
 
-    @ApiOperation("有效舆情互动量更新")
+    @ApiOperation("舆情库-有效舆情-互动量更新")
     @PostMapping("/mark/interaction-update")
     @LogRecord(description = "舆情库-有效舆情互动量更新")
+    @DownloadTask(taskName = "有效舆情互动量更新结果下载", description = "有效舆情互动量更新结果", entity = true)
     public ResponseResult getMarkInteractionUpdate(@RequestBody MarkSearchDTO markSearchDTO){
         return markDataService.markInteractionUpdate(markSearchDTO);
     }

src/main/java/com/zhiwei/brandkbs2/controller/app/AppToolsetController.java

 package com.zhiwei.brandkbs2.controller.app;
 
+import com.zhiwei.brandkbs2.aop.DownloadTask;
 import com.zhiwei.brandkbs2.aop.LogRecord;
 import com.zhiwei.brandkbs2.auth.Auth;
 import com.zhiwei.brandkbs2.enmus.RoleEnum;
@@ -7,6 +8,7 @@ import com.zhiwei.brandkbs2.model.ResponseResult;
 import com.zhiwei.brandkbs2.service.ToolsetService;
 import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiOperation;
+import io.swagger.annotations.ApiParam;
 import org.springframework.web.bind.annotation.*;
 import org.springframework.web.multipart.MultipartFile;
 
@@ -27,36 +29,46 @@ public class AppToolsetController {
     @Resource(name = "toolsetServiceImpl")
     private ToolsetService toolsetService;
 
-    @ApiOperation("摘要提取-单条")
+    @ApiOperation("工具库-摘要提取-单条")
     @GetMapping("/article-summary/single")
     @LogRecord(description = "工具库-摘要提取-单条")
     public ResponseResult getSingleArticleSummary(@RequestParam(value = "url") String url) {
         return toolsetService.getSingleArticleSummary(url);
     }
 
-    @ApiOperation("摘要提取")
+    @ApiOperation("工具库-摘要提取-批量")
     @PostMapping("/article-summary/batch")
     @LogRecord(description = "工具库-摘要提取-批量")
+    @DownloadTask(taskName = "批量摘要提取结果下载", description = "批量摘要提取结果", entity = true)
     public ResponseResult downloadBatchArticleSummary(@RequestParam(value = "file") MultipartFile file){
         return toolsetService.getBatchArticleSummary(file);
     }
 
-    @ApiOperation("摘要提取-剩余可用次数")
+    @ApiOperation("工具库-摘要提取-剩余可用次数")
     @GetMapping("/article-summary/remaining")
     public ResponseResult getArticleSummaryRemainingCount(){
         return toolsetService.getArticleSummaryRemainingCount();
     }
 
-    @ApiOperation("链接互动量更新")
+    @ApiOperation("工具库-互动量更新-链接互动量更新")
     @PostMapping("/interaction-update/url")
     @LogRecord(description = "工具库-链接互动量更新")
+    @DownloadTask(taskName = "链接互动量更新结果下载", description = "链接互动量更新结果", entity = true)
     public ResponseResult downloadUrlInteractionUpdate(@RequestParam(value = "file") MultipartFile file){
         return toolsetService.urlInteractionUpdate(file);
     }
 
-    @ApiOperation("互动量更新-剩余可用次数")
+    @ApiOperation("工具库-互动量更新-剩余可用次数")
     @GetMapping("/interaction-update/remaining")
     public ResponseResult getInteractionRemainingCount(){
         return ResponseResult.success(toolsetService.getInteractionRemainingCount());
     }
+
+    @ApiOperation("工具库-词云提取")
+    @PostMapping("/high-word")
+    @LogRecord(description = "工具库-词云提取")
+    public ResponseResult getHighWord(@RequestParam MultipartFile file,
+                                      @ApiParam(name = "type", value = "关键词|文章") @RequestParam String type){
+        return toolsetService.getHighWord(file, type);
+    }
 }

src/main/java/com/zhiwei/brandkbs2/easyexcel/EasyExcelUtil.java

@@ -16,6 +16,7 @@ import org.springframework.web.multipart.MultipartFile;
 
 import javax.servlet.http.HttpServletResponse;
 import java.io.FileOutputStream;
+import java.io.IOException;
 import java.io.InputStream;
 import java.lang.reflect.Field;
 import java.net.URLEncoder;
@@ -66,6 +67,10 @@ public class EasyExcelUtil {
         }
     }
 
+    public static void readWithoutTryCatch(MultipartFile file, ReadExcelDTO readExcelDTO) throws IOException {
+        EasyExcel.read(file.getInputStream(), readExcelDTO.getClazz(), readExcelDTO.getAnalysisEventListener()).sheet().doRead();
+    }
+
     /**
      * 读取多个sheet
      *

src/main/java/com/zhiwei/brandkbs2/easyexcel/dto/UploadHighWordArticleDTO.java

+package com.zhiwei.brandkbs2.easyexcel.dto;
+
+import com.alibaba.excel.annotation.ExcelProperty;
+import lombok.Data;
+
+/**
+ * @author cjz
+ * @version 1.0
+ * @description 解析文章词云提取上传文件
+ * @date 2023/10/10 10:22
+ */
+@Data
+public class UploadHighWordArticleDTO {
+    @ExcelProperty("序号")
+    private String id;
+
+    @ExcelProperty("标题")
+    private String title;
+
+    @ExcelProperty("文本")
+    private String content;
+
+    @ExcelProperty("链接")
+    private String url;
+}

src/main/java/com/zhiwei/brandkbs2/easyexcel/dto/UploadHighWordKeywordDTO.java

+package com.zhiwei.brandkbs2.easyexcel.dto;
+
+import com.alibaba.excel.annotation.ExcelProperty;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+/**
+ * @author cjz
+ * @version 1.0
+ * @description 解析关键词词云提取上传文件
+ * @date 2023/10/10 10:22
+ */
+@Data
+@NoArgsConstructor
+public class UploadHighWordKeywordDTO {
+    @ExcelProperty("序号")
+    private String id;
+
+    @ExcelProperty("关键词")
+    private String keyword;
+
+    @ExcelProperty("词频")
+    private Integer frequency;
+}

src/main/java/com/zhiwei/brandkbs2/easyexcel/listener/HighWordArticleListener.java

+package com.zhiwei.brandkbs2.easyexcel.listener;
+
+import com.alibaba.excel.context.AnalysisContext;
+import com.alibaba.excel.event.AnalysisEventListener;
+import com.zhiwei.brandkbs2.easyexcel.dto.UploadHighWordArticleDTO;
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author cjz
+ * @version 1.0
+ * @description 解析文章词云提取上传文件
+ * @date 2023/10/10 10:23
+ */
+public class HighWordArticleListener extends AnalysisEventListener<UploadHighWordArticleDTO> {
+
+    List<String> contents;
+
+    public HighWordArticleListener(List<String> contents){
+        this.contents = contents;
+    }
+
+    @Override
+    public void invokeHeadMap(Map<Integer, String> headMap, AnalysisContext context){
+        if (!"序号".equals(headMap.get(0)) || !"标题".equals(headMap.get(1)) || !"文本".equals(headMap.get(2)) || !"链接".equals(headMap.get(3))) {
+            throw new  IllegalArgumentException("表格格式有误");
+        }
+    }
+
+    @Override
+    public void invoke(UploadHighWordArticleDTO data, AnalysisContext context) {
+        if (StringUtils.isNotBlank(data.getContent())) {
+            contents.add(data.getContent());
+        }
+    }
+
+    @Override
+    public void doAfterAllAnalysed(AnalysisContext context) {
+
+    }
+}

src/main/java/com/zhiwei/brandkbs2/easyexcel/listener/HighWordKeywordListener.java

+package com.zhiwei.brandkbs2.easyexcel.listener;
+
+import com.alibaba.excel.context.AnalysisContext;
+import com.alibaba.excel.event.AnalysisEventListener;
+import com.zhiwei.brandkbs2.easyexcel.dto.UploadHighWordKeywordDTO;
+
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * @author lxj
+ * @version 1.0
+ * @description 解析词云关键词文件监听器
+ * @date 2023/10/10 10:30
+ */
+public class HighWordKeywordListener extends AnalysisEventListener<UploadHighWordKeywordDTO> {
+
+    private Map<String, Integer> map;
+
+    public HighWordKeywordListener(Map<String, Integer> map){
+        this.map = map;
+    }
+
+    @Override
+    public void invokeHeadMap(Map<Integer, String> headMap, AnalysisContext context){
+        if (!"序号".equals(headMap.get(0)) || !"关键词".equals(headMap.get(1)) || !"词频".equals(headMap.get(2))) {
+            throw new IllegalArgumentException("表格格式有误");
+        }
+    }
+
+    @Override
+    public void invoke(UploadHighWordKeywordDTO data, AnalysisContext context) {
+        map.compute(data.getKeyword(), (k, v) ->{
+            if (Objects.isNull(v)){
+                return data.getFrequency();
+            }
+            if (v < data.getFrequency()){
+                return data.getFrequency();
+            }
+            return v;
+        });
+    }
+
+    @Override
+    public void doAfterAllAnalysed(AnalysisContext context) {
+    }
+}

src/main/java/com/zhiwei/brandkbs2/service/ToolsetService.java

@@ -62,4 +62,11 @@ public interface ToolsetService {
      */
     List<JSONObject> getInteractionResult(List<String> taskId);
 
+    /**
+     * 词云提取
+     * @param file  excel文件
+     * @param type  类型 关键词|文章
+     * @return
+     */
+    ResponseResult getHighWord(MultipartFile file, String type);
 }

src/main/java/com/zhiwei/brandkbs2/service/impl/ToolsetServiceImpl.java

@@ -5,11 +5,10 @@ import com.zhiwei.brandkbs2.auth.UserThreadLocal;
 import com.zhiwei.brandkbs2.controller.app.AppToolsetController;
 import com.zhiwei.brandkbs2.easyexcel.EasyExcelUtil;
 import com.zhiwei.brandkbs2.easyexcel.config.ReadExcelDTO;
-import com.zhiwei.brandkbs2.easyexcel.dto.ExportArticleSummaryDTO;
-import com.zhiwei.brandkbs2.easyexcel.dto.ExportInteractionUpdateDTO;
-import com.zhiwei.brandkbs2.easyexcel.dto.UploadArticleSummaryDTO;
-import com.zhiwei.brandkbs2.easyexcel.dto.UploadInteractionUpdateDTO;
+import com.zhiwei.brandkbs2.easyexcel.dto.*;
 import com.zhiwei.brandkbs2.easyexcel.listener.ArticleSummaryListener;
+import com.zhiwei.brandkbs2.easyexcel.listener.HighWordArticleListener;
+import com.zhiwei.brandkbs2.easyexcel.listener.HighWordKeywordListener;
 import com.zhiwei.brandkbs2.easyexcel.listener.InteractionUpdateListener;
 import com.zhiwei.brandkbs2.enmus.InteractionEnum;
 import com.zhiwei.brandkbs2.exception.ExceptionCast;
@@ -21,6 +20,7 @@ import com.zhiwei.brandkbs2.service.ExtraService;
 import com.zhiwei.brandkbs2.service.ProjectService;
 import com.zhiwei.brandkbs2.service.ToolsetService;
 import com.zhiwei.brandkbs2.util.RedisUtil;
+import com.zhiwei.brandkbs2.util.TextUtil;
 import com.zhiwei.brandkbs2.util.Tools;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.collections4.ListUtils;
@@ -62,6 +62,9 @@ public class ToolsetServiceImpl implements ToolsetService {
     @Autowired
     private RestTemplate restTemplate;
 
+    @Autowired
+    private TextUtil textUtil;
+
     @Value("${toolset.articleSummary.url}")
     private String articleSummaryUrl;
 
@@ -287,6 +290,70 @@ public class ToolsetServiceImpl implements ToolsetService {
         return res;
     }
 
+    @Override
+    public ResponseResult getHighWord(MultipartFile file, String type) {
+        if (Objects.equals("关键词", type)){
+            return getKeywordHighWord(file);
+        }
+        return getArticleHighWord(file);
+    }
+
+    /**
+     * 关键词词云提取
+     * @param file  excel文件
+     * @return
+     */
+    private ResponseResult getKeywordHighWord(MultipartFile file){
+        Map<String, Integer> map = new HashMap<>();
+        // excel信息提取
+        try {
+            ReadExcelDTO<UploadHighWordKeywordDTO> readExcel = new ReadExcelDTO<>();
+            readExcel.setClazz(UploadHighWordKeywordDTO.class);
+            readExcel.setAnalysisEventListener(new HighWordKeywordListener(map));
+            EasyExcelUtil.readWithoutTryCatch(file, readExcel);
+        }catch (Exception e){
+            return ResponseResult.failure("表格格式有误");
+        }
+        if (map.size() > 500){
+            return ResponseResult.failure("表格关键词数量超过上限500");
+        }
+        Map<String, Integer> sortMap = Tools.sortMap(map, 500);
+        List<JSONObject> res = new ArrayList<>();
+        for (Map.Entry<String, Integer> entry : sortMap.entrySet()) {
+            JSONObject jsonObject = new JSONObject();
+            jsonObject.put("name", entry.getKey());
+            jsonObject.put("value", entry.getValue());
+            res.add(jsonObject);
+        }
+        return ResponseResult.success(res);
+    }
+
+    /**
+     * 文章词云提取
+     * @param file excel文件
+     * @return
+     */
+    private ResponseResult getArticleHighWord(MultipartFile file){
+        List<String> contents = new ArrayList<>();
+        // excel信息提取
+        try {
+            ReadExcelDTO<UploadHighWordArticleDTO> readExcel = new ReadExcelDTO<>();
+            readExcel.setClazz(UploadHighWordArticleDTO.class);
+            readExcel.setAnalysisEventListener(new HighWordArticleListener(contents));
+            EasyExcelUtil.readWithoutTryCatch(file, readExcel);
+        }catch (Exception e){
+            return ResponseResult.failure("表格格式有误");
+        }
+        if (contents.size() > 500){
+            return ResponseResult.failure("表格文章数量超过上限500");
+        }
+        return ResponseResult.success(textUtil.getHighWordsJsonDifferentFieldName(contents, 500));
+    }
+
+    /**
+     * 获取互动量请求头
+     * @return
+     */
     private HttpHeaders getInteractionRequestHeader(){
         // 请求头参数
         Long timeStamp = System.currentTimeMillis();

src/main/java/com/zhiwei/brandkbs2/util/TextUtil.java

@@ -120,4 +120,16 @@ public class TextUtil {
         return result;
     }
 
+    public List<JSONObject> getHighWordsJsonDifferentFieldName(List<String> texts, Integer maxSize) {
+//        Map<String, Integer> highWords = getHighWords(texts, maxSize);
+        List<Map.Entry<String, Integer>> wordRate = ansjSeg.getFenCi(texts, maxSize);
+        List<JSONObject> result = new ArrayList<>(wordRate.size());
+        for (Map.Entry<String, Integer> entry : wordRate) {
+            JSONObject jsonObject = new JSONObject();
+            jsonObject.put("name", entry.getKey());
+            jsonObject.put("value", entry.getValue());
+            result.add(jsonObject);
+        }
+        return result;
+    }
 }