Merge branch 'feature' into 'dev'

2023/10/08 互动量更新调整

See merge request !399

src/main/java/com/zhiwei/brandkbs2/service/impl/MarkDataServiceImpl.java

@@ -1139,19 +1139,6 @@ public class MarkDataServiceImpl implements MarkDataService {
         return result;
     }
 
-    private JSONObject getBackUpPlatform() {
-        JSONObject jsonObject = new JSONObject();
-        jsonObject.put("微博", "微博");
-        jsonObject.put("微信", "微信");
-        jsonObject.put("网媒", "新闻");
-        jsonObject.put("平媒", "平媒");
-        jsonObject.put("自媒体", "自媒体");
-        jsonObject.put("贴吧论坛", "论坛");
-        jsonObject.put("视频", "视频");
-        jsonObject.put("短视频", "短视频");
-        return jsonObject;
-    }
-
     @Override
     public Pair<JSONObject, Integer> searchWholeNetworkWithBalance(SearchFilterDTO dto) {
         if (dto.isExternalDataSource()) { // 查商业数据库
@@ -1523,33 +1510,32 @@ public class MarkDataServiceImpl implements MarkDataService {
         if (interactionBalance - list.size() < 0){
             return ResponseResult.failure("剩余可用次数不足");
         }
-        List<ExportInteractionUpdateDTO> exportInteractionUpdateDTOList = new ArrayList<>();
-        Map<String, String> map = list.stream().collect(Collectors.toMap(ExportAppYuqingDTO::getUrl, ExportAppYuqingDTO::getPlatform));
+        Map<String, String> map = list.stream().collect(Collectors.toMap(ExportAppYuqingDTO::getUrl, ExportAppYuqingDTO::getPlatform, (v1, v2) -> v1));
         // 分割map，list里每个map.size为100，目前品见最大链接限制为100，见知微tapd-wiki《互动量更新中间件使用》说明
         List<Map<String, String>> mapList = Tools.splitMap(map, 100);
         AtomicInteger id = new AtomicInteger(0);
+        List<JSONObject> result = new ArrayList<>();
         for (Map<String, String> urlPlatformMap : mapList) {
             List<String> taskIdList = new ArrayList<>(100);
-            List<String> urlList = new ArrayList<>(100);
             for (Map.Entry<String, String> entry : urlPlatformMap.entrySet()) {
                 // 提交互动量更新任务
                 String type = InteractionEnum.getTypeByPlatform(entry.getValue());
                 List<String> taskIds = toolsetService.requestInteractionUrl(type, Collections.singletonList(entry.getKey()));
-                urlList.add(entry.getKey());
                 taskIdList.addAll(taskIds);
             }
-            // 每100条作为一批提交完时，获取这一批的互动量任务更新结果
-            List<JSONObject> result = toolsetService.getInteractionResult(taskIdList);
+            // 每100条作为一批提交完时，获取这一批的互动量任务更新结果，单次获取不超50
+            for (List<String> ids : ListUtils.partition(taskIdList, 50)) {
+                result.addAll(toolsetService.getInteractionResult(ids));
+            }
+        }
         Map<Object, JSONObject> urlMap = result.stream().collect(Collectors.toMap(jsonObject -> jsonObject.get("url"), o -> o));
-            List<ExportInteractionUpdateDTO> exportList = urlList.stream()
-                    .map(url -> ExportInteractionUpdateDTO.createWithPlatform(String.valueOf(id.incrementAndGet()), urlPlatformMap, urlMap, url))
+        List<ExportInteractionUpdateDTO> exportList = list.stream().map(ExportAppYuqingDTO::getUrl)
+                .map(url -> ExportInteractionUpdateDTO.createWithPlatform(String.valueOf(id.incrementAndGet()), map, urlMap, url))
                 .collect(Collectors.toList());
-            exportInteractionUpdateDTOList.addAll(exportList);
-        }
         // excel输出到指定路径
         String projectName = projectService.getProjectById(UserThreadLocal.getProjectId()).getProjectName();
         String filePath = EasyExcelUtil.generateExcelFilePath(brandkbsFilePath, projectName, UserThreadLocal.getNickname(), "舆情库互动量更新结果");
-        EasyExcelUtil.write(filePath, "sheet1", ExportInteractionUpdateDTO.class, exportInteractionUpdateDTOList);
+        EasyExcelUtil.write(filePath, "sheet1", ExportInteractionUpdateDTO.class, exportList);
         JSONObject res = new JSONObject();
         // 记录使用情况
         extraService.decreaseInteractionRecord(InteractionUpdateRecord.UsedType.yuqing, list.size());

src/main/java/com/zhiwei/brandkbs2/service/impl/ToolsetServiceImpl.java

 package com.zhiwei.brandkbs2.service.impl;
 
-import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.brandkbs2.auth.UserThreadLocal;
 import com.zhiwei.brandkbs2.controller.app.AppToolsetController;
@@ -24,10 +23,8 @@ import com.zhiwei.brandkbs2.service.ToolsetService;
 import com.zhiwei.brandkbs2.util.RedisUtil;
 import com.zhiwei.brandkbs2.util.Tools;
 import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.collections4.ListUtils;
 import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.tuple.Pair;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -184,7 +181,6 @@ public class ToolsetServiceImpl implements ToolsetService {
 
     @Override
     public ResponseResult urlInteractionUpdate(MultipartFile file) {
-        List<ExportInteractionUpdateDTO> exportInteractionUpdateDTOList = new ArrayList<>();
         // excel信息提取
         List<String> urls = new ArrayList<>();
         ReadExcelDTO<UploadInteractionUpdateDTO> readExcel = new ReadExcelDTO<>();
@@ -200,35 +196,37 @@ public class ToolsetServiceImpl implements ToolsetService {
             return ResponseResult.failure("剩余可用次数不足");
         }
         // 通过url获取域名进而获取任务类型
-        Map<String, String> map = new LinkedHashMap<>();
-        urls.forEach(url -> {
+        Map<String, String> map = new HashMap<>();
+        urls.forEach(url -> map.compute(url, (key, value) -> {
+            if (Objects.isNull(value)) {
                 String domain = Tools.topDomainOfDomain(url);
-            String type = InteractionEnum.getTypeByDomain(domain);
-            map.put(url, type);
-        });
+                return InteractionEnum.getTypeByDomain(domain);
+            }
+            return value;
+        }));
         // 分割map，list里每个map.size为100，目前品见最大链接限制为100，见知微tapd-wiki《互动量更新中间件使用》说明
         List<Map<String, String>> mapList = Tools.splitMap(map, 100);
         AtomicInteger id = new AtomicInteger(0);
+        List<JSONObject> result = new ArrayList<>(300);
         for (Map<String, String> urlTypeMap : mapList) {
             List<String> taskIdList = new ArrayList<>(100);
-            List<String> urlList = new ArrayList<>(100);
             for (Map.Entry<String, String> entry : urlTypeMap.entrySet()) {
                 // 提交互动量更新任务
                 List<String> taskIds = requestInteractionUrl(entry.getValue(), Collections.singletonList(entry.getKey()));
-                urlList.add(entry.getKey());
                 taskIdList.addAll(taskIds);
             }
-            // 每100条作为一批提交完时，获取互动量任务更新结果
-            List<JSONObject> result = getInteractionResult(taskIdList);
+            // 每100条作为一批提交完时，获取互动量任务更新结果，单次获取不超50
+            for (List<String> ids : ListUtils.partition(taskIdList, 50)) {
+                result.addAll(getInteractionResult(ids));
+            }
+        }
         Map<Object, JSONObject> urlMap = result.stream().collect(Collectors.toMap(jsonObject -> jsonObject.get("url"), o -> o));
         List<ExportInteractionUpdateDTO> exportList =
-                    urlList.stream().map(url -> new ExportInteractionUpdateDTO(String.valueOf(id.incrementAndGet()), urlMap, url)).collect(Collectors.toList());
-            exportInteractionUpdateDTOList.addAll(exportList);
-        }
+                urls.stream().map(url -> new ExportInteractionUpdateDTO(String.valueOf(id.incrementAndGet()), urlMap, url)).collect(Collectors.toList());
         // excel输出到指定路径
         String projectName = projectService.getProjectById(UserThreadLocal.getProjectId()).getProjectName();
         String filePath = EasyExcelUtil.generateExcelFilePath(brandkbsFilePath, projectName, UserThreadLocal.getNickname(), "链接互动量更新结果");
-        EasyExcelUtil.write(filePath, "sheet1", ExportInteractionUpdateDTO.class, exportInteractionUpdateDTOList);
+        EasyExcelUtil.write(filePath, "sheet1", ExportInteractionUpdateDTO.class, exportList);
         JSONObject res = new JSONObject();
         // 记录使用情况
         extraService.decreaseInteractionRecord(InteractionUpdateRecord.UsedType.url, urls.size());

src/main/java/com/zhiwei/brandkbs2/util/Tools.java

@@ -621,6 +621,12 @@ public class Tools {
         return result;
     }
 
+    /**
+     * 分割map
+     * @param map
+     * @param chunkSize
+     * @return
+     */
     public static List<Map<String, String>> splitMap(Map<String, String> map, int chunkSize) {
         List<Map<String, String>> result = new ArrayList<>();
         int count = 0;