Skip to content

B
brandkbs2
Commit 7634e2c8 by 陈健智
Options

工具库-互动量更新模块

parent 42cc8455
Showing with 1271 additions and 204 deletions
src/main/java/com/zhiwei/brandkbs2/aop/AopDownloadTask.java
...@@ -19,7 +19,8 @@ import org.springframework.stereotype.Component; ...@@ -19,7 +19,8 @@ import org.springframework.stereotype.Component;
import javax.annotation.Resource; import javax.annotation.Resource;
import java.lang.reflect.Method; import java.lang.reflect.Method;
import java.util.Objects; import java.util.Arrays;
import java.util.List;
/** /**
* @author cjz * @author cjz
...@@ -36,7 +37,9 @@ public class AopDownloadTask { ...@@ -36,7 +37,9 @@ public class AopDownloadTask {
@Resource(name = "downloadTaskServiceImpl") @Resource(name = "downloadTaskServiceImpl")
DownloadTaskService downloadTaskService; DownloadTaskService downloadTaskService;
@Around(value = "execution(public * com..controller..app..AppDownloadController.*(..)) || execution(public * com..controller..app..AppToolsetController.getBatchArticleSummary(..)))") private static final List<String> METHOD = Arrays.asList("downloadBatchArticleSummary", "downloadUrlInteractionUpdate", "getMarkInteractionUpdate");
@Around(value = "execution(public * com..controller..app..AppDownloadController.*(..)) || execution(* com..controller..app..AppToolsetController.download*(..)) || execution(* com..controller..app..AppArticleController.getMarkInteractionUpdate(..))")
public Object around(ProceedingJoinPoint joinPoint) throws Throwable{ public Object around(ProceedingJoinPoint joinPoint) throws Throwable{
Signature signature = joinPoint.getSignature(); Signature signature = joinPoint.getSignature();
Method method = ((MethodSignature) signature).getMethod(); Method method = ((MethodSignature) signature).getMethod();
...@@ -55,7 +58,7 @@ public class AopDownloadTask { ...@@ -55,7 +58,7 @@ public class AopDownloadTask {
ExceptionCast.cast(CommonCodeEnum.FAIL, "下载异常", e); ExceptionCast.cast(CommonCodeEnum.FAIL, "下载异常", e);
} }
// 更新下载任务 // 更新下载任务
if (Objects.equals(method.getName(), "getBatchArticleSummary")){ if (METHOD.contains(method.getName())){
fileAddress = JSONObject.parseObject(((ResponseResult) proceed).getData().toString()).getString("filePath"); fileAddress = JSONObject.parseObject(((ResponseResult) proceed).getData().toString()).getString("filePath");
}else { }else {
fileAddress = ((ResponseResult) proceed).getData().toString(); fileAddress = ((ResponseResult) proceed).getData().toString();
......
src/main/java/com/zhiwei/brandkbs2/controller/admin/ExtraController.java
...@@ -8,7 +8,7 @@ import com.zhiwei.brandkbs2.easyexcel.dto.ExportLineDTO; ...@@ -8,7 +8,7 @@ import com.zhiwei.brandkbs2.easyexcel.dto.ExportLineDTO;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportWholeSearchRecordDTO; import com.zhiwei.brandkbs2.easyexcel.dto.ExportWholeSearchRecordDTO;
import com.zhiwei.brandkbs2.enmus.RoleEnum; import com.zhiwei.brandkbs2.enmus.RoleEnum;
import com.zhiwei.brandkbs2.model.ResponseResult; import com.zhiwei.brandkbs2.model.ResponseResult;
import com.zhiwei.brandkbs2.service.WholeSearchService; import com.zhiwei.brandkbs2.service.ExtraService;
import io.swagger.annotations.*; import io.swagger.annotations.*;
import org.springframework.web.bind.annotation.*; import org.springframework.web.bind.annotation.*;
...@@ -28,8 +28,8 @@ import java.util.stream.Collectors; ...@@ -28,8 +28,8 @@ import java.util.stream.Collectors;
@Auth(role = RoleEnum.COMMON_ADMIN) @Auth(role = RoleEnum.COMMON_ADMIN)
public class ExtraController extends BaseController { public class ExtraController extends BaseController {
@Resource(name = "wholeSearchServiceImpl") @Resource(name = "extraServiceImpl")
private WholeSearchService wholeSearchService; private ExtraService extraService;
@ApiOperation("全网搜-使用记录-查询") @ApiOperation("全网搜-使用记录-查询")
@ApiImplicitParams({@ApiImplicitParam(name = "page", value = "页码", defaultValue = "1", paramType = "query", dataType = "int"), @ApiImplicitParams({@ApiImplicitParam(name = "page", value = "页码", defaultValue = "1", paramType = "query", dataType = "int"),
...@@ -40,7 +40,7 @@ public class ExtraController extends BaseController { ...@@ -40,7 +40,7 @@ public class ExtraController extends BaseController {
public ResponseResult findUseList(@RequestParam(value = "page", defaultValue = "1") int page, public ResponseResult findUseList(@RequestParam(value = "page", defaultValue = "1") int page,
@RequestParam(value = "size", defaultValue = "5") int pageSize, @RequestParam(value = "size", defaultValue = "5") int pageSize,
@RequestParam(value = "sorter", defaultValue = "{\"usedCount\":\"ascend\"}") String sorter) { @RequestParam(value = "sorter", defaultValue = "{\"usedCount\":\"ascend\"}") String sorter) {
return ResponseResult.success(wholeSearchService.findUsedList(page, pageSize, sorter)); return ResponseResult.success(extraService.findUsedList(page, pageSize, sorter));
} }
@ApiOperation("全网搜-维护记录-查询") @ApiOperation("全网搜-维护记录-查询")
...@@ -49,13 +49,13 @@ public class ExtraController extends BaseController { ...@@ -49,13 +49,13 @@ public class ExtraController extends BaseController {
@GetMapping("/maintain/list") @GetMapping("/maintain/list")
public ResponseResult findMaintainList(@RequestParam(value = "page", defaultValue = "1") int page, public ResponseResult findMaintainList(@RequestParam(value = "page", defaultValue = "1") int page,
@RequestParam(value = "size", defaultValue = "5") int pageSize) { @RequestParam(value = "size", defaultValue = "5") int pageSize) {
return ResponseResult.success(wholeSearchService.findMaintainList(page, pageSize)); return ResponseResult.success(extraService.findMaintainList(page, pageSize));
} }
@ApiOperation("全网搜-数据总览") @ApiOperation("全网搜-数据总览")
@GetMapping("/overview") @GetMapping("/overview")
public ResponseResult overview() { public ResponseResult overview() {
return ResponseResult.success(wholeSearchService.overview()); return ResponseResult.success(extraService.overview());
} }
...@@ -67,7 +67,7 @@ public class ExtraController extends BaseController { ...@@ -67,7 +67,7 @@ public class ExtraController extends BaseController {
Long startTime = json.getLong("startTime"); Long startTime = json.getLong("startTime");
Long endTime = json.getLong("endTime"); Long endTime = json.getLong("endTime");
boolean day = json.getBooleanValue("day"); boolean day = json.getBooleanValue("day");
List<JSONObject> collect = wholeSearchService.outputUsedList(personal, startTime, endTime, day); List<JSONObject> collect = extraService.outputUsedList(personal, startTime, endTime, day);
String fileName = startTime + "_" + endTime + "使用记录"; String fileName = startTime + "_" + endTime + "使用记录";
if (personal) { if (personal) {
EasyExcelUtil.download(fileName, "sheet", ExportWholeSearchRecordDTO.class, collect.stream().map(ExportWholeSearchRecordDTO::createFromJSONObject).collect(Collectors.toList()), response); EasyExcelUtil.download(fileName, "sheet", ExportWholeSearchRecordDTO.class, collect.stream().map(ExportWholeSearchRecordDTO::createFromJSONObject).collect(Collectors.toList()), response);
...@@ -81,8 +81,42 @@ public class ExtraController extends BaseController { ...@@ -81,8 +81,42 @@ public class ExtraController extends BaseController {
@PutMapping("/maintain/adjust") @PutMapping("/maintain/adjust")
@Auth(role = RoleEnum.SUPER_ADMIN) @Auth(role = RoleEnum.SUPER_ADMIN)
public ResponseResult adjustRecord(@ApiParam(name = "json:{count:调整数值)") @RequestBody JSONObject json) { public ResponseResult adjustRecord(@ApiParam(name = "json:{count:调整数值)") @RequestBody JSONObject json) {
wholeSearchService.adjustRecord(json.getInteger("count")); extraService.adjustRecord(json.getInteger("count"));
return ResponseResult.success(); return ResponseResult.success();
} }
@ApiOperation("互动量更新-使用记录-查询")
@ApiImplicitParams({@ApiImplicitParam(name = "page", value = "页码", defaultValue = "1", paramType = "query", dataType = "int"),
@ApiImplicitParam(name = "pageSize", value = "每页记录数", defaultValue = "5", paramType = "query", dataType = "int"),
@ApiImplicitParam(name = "sorter", defaultValue = "{\"usedCount\":\"ascend\"}", value = "排序字段", paramType = "query", dataType = "string")}
)
@GetMapping("/interaction/used/list")
public ResponseResult findInteractionUsedList(@RequestParam(value = "page", defaultValue = "1") int page,
@RequestParam(value = "size", defaultValue = "5") int pageSize,
@RequestParam(value = "sorter", defaultValue = "{\"usedCount\":\"ascend\"}") String sorter) {
return ResponseResult.success(extraService.findInteractionUsedList(page, pageSize, sorter));
}
@ApiOperation("互动量更新-维护记录-查询")
@ApiImplicitParams({@ApiImplicitParam(name = "page", value = "页码", defaultValue = "1", paramType = "query", dataType = "int"),
@ApiImplicitParam(name = "pageSize", value = "每页记录数", defaultValue = "5", paramType = "query", dataType = "int")})
@GetMapping("/interaction/maintain/list")
public ResponseResult findInteractionMaintainList(@RequestParam(value = "page", defaultValue = "1") int page,
@RequestParam(value = "size", defaultValue = "5") int pageSize) {
return ResponseResult.success(extraService.findInteractionMaintainList(page, pageSize));
}
@ApiOperation("互动量更新-数据总览")
@GetMapping("/interaction/overview")
public ResponseResult interactionOverview() {
return ResponseResult.success(extraService.interactionOverview());
}
@ApiOperation("互动量更新-维护记录-调整")
@PutMapping("/interaction/maintain/adjust")
@Auth(role = RoleEnum.SUPER_ADMIN)
public ResponseResult adjustInteractionRecord(@ApiParam(name = "json:{count:调整数值)") @RequestBody JSONObject json) {
extraService.adjustInteractionRecord(json.getInteger("count"));
return ResponseResult.success();
}
} }
src/main/java/com/zhiwei/brandkbs2/controller/app/AppArticleController.java
...@@ -306,6 +306,19 @@ public class AppArticleController extends BaseController { ...@@ -306,6 +306,19 @@ public class AppArticleController extends BaseController {
return reportService.getReportSchedule(idList); return reportService.getReportSchedule(idList);
} }
@ApiOperation("有效舆情互动量更新")
@PostMapping("/mark/interaction-update")
@LogRecord(description = "舆情库-有效舆情互动量更新")
public ResponseResult getMarkInteractionUpdate(@RequestBody MarkSearchDTO markSearchDTO){
return markDataService.markInteractionUpdate(markSearchDTO);
}
@ApiOperation("有效舆情互动量更新-数据量预估")
@PostMapping("/mark/interaction-prediction")
public ResponseResult getMarkInteractionPrediction(@RequestBody MarkSearchDTO markSearchDTO){
return ResponseResult.success(markDataService.interactionUpdatePrediction(markSearchDTO));
}
private boolean checkMTagIllegal(StringBuilder mtag) { private boolean checkMTagIllegal(StringBuilder mtag) {
List<MarkerTag> hitTags = projectService.getProjectById(UserThreadLocal.getProjectId()).getHitTags(); List<MarkerTag> hitTags = projectService.getProjectById(UserThreadLocal.getProjectId()).getHitTags();
if (!Tools.isEmpty(hitTags)) { if (!Tools.isEmpty(hitTags)) {
......
src/main/java/com/zhiwei/brandkbs2/controller/app/AppDownloadController.java
...@@ -58,8 +58,8 @@ public class AppDownloadController extends BaseController { ...@@ -58,8 +58,8 @@ public class AppDownloadController extends BaseController {
@Resource(name = "channelServiceImpl") @Resource(name = "channelServiceImpl")
ChannelService channelService; ChannelService channelService;
@Resource(name = "wholeSearchServiceImpl") @Resource(name = "extraServiceImpl")
private WholeSearchService wholeSearchService; private ExtraService extraService;
@Resource(name = "markDataServiceImpl") @Resource(name = "markDataServiceImpl")
MarkDataService markDataService; MarkDataService markDataService;
...@@ -247,10 +247,10 @@ public class AppDownloadController extends BaseController { ...@@ -247,10 +247,10 @@ public class AppDownloadController extends BaseController {
Long startTime = json.getLong("startTime"); Long startTime = json.getLong("startTime");
Long endTime = json.getLong("endTime"); Long endTime = json.getLong("endTime");
boolean day = json.getBooleanValue("day"); boolean day = json.getBooleanValue("day");
List<JSONObject> collect = wholeSearchService.outputUsedList(personal, startTime, endTime, day); List<JSONObject> collect = extraService.outputUsedList(personal, startTime, endTime, day);
// excel写入至指定路径 // excel写入至指定路径
String projectName = projectService.getProjectById(UserThreadLocal.getProjectId()).getProjectName(); String projectName = projectService.getProjectById(UserThreadLocal.getProjectId()).getProjectName();
String filePath = EasyExcelUtil.generateExcelFilePath(brandkbsFilePath, projectName, UserThreadLocal.getNickname(), startTime + "_" + endTime + "使用记录"); String filePath = EasyExcelUtil.generateExcelFilePath(brandkbsFilePath, projectName, UserThreadLocal.getNickname(), "全网搜使用记录");
if (personal) { if (personal) {
List<ExportWholeSearchRecordDTO> list = collect.stream().map(ExportWholeSearchRecordDTO::createFromJSONObject).collect(Collectors.toList()); List<ExportWholeSearchRecordDTO> list = collect.stream().map(ExportWholeSearchRecordDTO::createFromJSONObject).collect(Collectors.toList());
EasyExcelUtil.write(filePath, "sheet1", ExportWholeSearchRecordDTO.class, list); EasyExcelUtil.write(filePath, "sheet1", ExportWholeSearchRecordDTO.class, list);
...@@ -261,6 +261,29 @@ public class AppDownloadController extends BaseController { ...@@ -261,6 +261,29 @@ public class AppDownloadController extends BaseController {
return ResponseResult.success(filePath); return ResponseResult.success(filePath);
} }
@ApiOperation("互动量更新使用记录")
@PostMapping("/back/interaction/used")
@Auth(role = RoleEnum.ADMIN)
public ResponseResult outputInteractionUsedList(@ApiParam(name = "json:{personal:个人明细=true,startTime:起始时间,endTime:结束时间,day:颗粒度天级=true}")
@RequestBody JSONObject json) {
boolean personal = json.getBooleanValue("personal");
Long startTime = json.getLong("startTime");
Long endTime = json.getLong("endTime");
boolean day = json.getBooleanValue("day");
List<JSONObject> collect = extraService.outputInteractionUsedList(personal, startTime, endTime, day);
// excel写入至指定路径
String projectName = projectService.getProjectById(UserThreadLocal.getProjectId()).getProjectName();
String filePath = EasyExcelUtil.generateExcelFilePath(brandkbsFilePath, projectName, UserThreadLocal.getNickname(), "互动量更新使用记录");
if (personal) {
List<ExportInteractionUpdateRecordDTO> list = collect.stream().map(ExportInteractionUpdateRecordDTO::createFromJSONObject).collect(Collectors.toList());
EasyExcelUtil.write(filePath, "sheet", ExportInteractionUpdateRecordDTO.class, list);
} else {
List<ExportLineDTO> list = collect.stream().map(ExportLineDTO::createFromJSONObject).collect(Collectors.toList());
EasyExcelUtil.write(filePath, "sheet", ExportLineDTO.class, list);
}
return ResponseResult.success();
}
@ApiOperation("项目关键词") @ApiOperation("项目关键词")
@ApiImplicitParam(name = "pid", value = "项目ID", required = true, paramType = "path", dataType = "string") @ApiImplicitParam(name = "pid", value = "项目ID", required = true, paramType = "path", dataType = "string")
@GetMapping("/back/project/keyword/{pid}") @GetMapping("/back/project/keyword/{pid}")
...@@ -374,7 +397,7 @@ public class AppDownloadController extends BaseController { ...@@ -374,7 +397,7 @@ public class AppDownloadController extends BaseController {
String filePath = EasyExcelUtil.generateExcelFilePath(brandkbsFilePath, projectName, UserThreadLocal.getNickname(), "全网搜舆情列表数据"); String filePath = EasyExcelUtil.generateExcelFilePath(brandkbsFilePath, projectName, UserThreadLocal.getNickname(), "全网搜舆情列表数据");
EasyExcelUtil.write(filePath, "sheet1", ExportSearchWholeDTO.class, exportList); EasyExcelUtil.write(filePath, "sheet1", ExportSearchWholeDTO.class, exportList);
if (dto.isExternalDataSource()) { if (dto.isExternalDataSource()) {
wholeSearchService.decreaseRecord(dto.getSearch(), WholeSearchRecord.UsedType.output, exportList.size()); extraService.decreaseRecord(dto.getSearch(), WholeSearchRecord.UsedType.output, exportList.size());
} }
return ResponseResult.success(filePath); return ResponseResult.success(filePath);
} }
...@@ -393,6 +416,19 @@ public class AppDownloadController extends BaseController { ...@@ -393,6 +416,19 @@ public class AppDownloadController extends BaseController {
return ResponseResult.success(filePath); return ResponseResult.success(filePath);
} }
@ApiOperation("链接互动量更新模板")
@GetMapping(value = "/interaction-update/template")
public ResponseResult downloadInteractionUpdateTemplate() {
List<List<String>> head = new ArrayList<>();
head.add(Collections.singletonList("序号"));
head.add(Collections.singletonList("链接"));
// excel写入至指定路径
String projectName = projectService.getProjectById(UserThreadLocal.getProjectId()).getProjectName();
String filePath = EasyExcelUtil.generateExcelFilePath(brandkbsFilePath, projectName, UserThreadLocal.getNickname(), "链接互动量更新模板");
EasyExcelUtil.dynamicHeadWrite(filePath, "模板", head, Collections.emptyList());
return ResponseResult.success(filePath);
}
private HttpHeaders getHeaders() { private HttpHeaders getHeaders() {
HttpHeaders httpHeaders = new HttpHeaders(); HttpHeaders httpHeaders = new HttpHeaders();
httpHeaders.set("token", token); httpHeaders.set("token", token);
......
src/main/java/com/zhiwei/brandkbs2/controller/app/AppSearchController.java
...@@ -78,8 +78,8 @@ public class AppSearchController extends BaseController { ...@@ -78,8 +78,8 @@ public class AppSearchController extends BaseController {
@Resource(name = "projectServiceImpl") @Resource(name = "projectServiceImpl")
ProjectService projectService; ProjectService projectService;
@Resource(name = "wholeSearchServiceImpl") @Resource(name = "extraServiceImpl")
WholeSearchService wholeSearchService; ExtraService extraService;
@ApiOperation("搜索-查热点") @ApiOperation("搜索-查热点")
@LogRecord(values = "keyword", description = "查热点", arguments = true, entity = false) @LogRecord(values = "keyword", description = "查热点", arguments = true, entity = false)
...@@ -164,7 +164,7 @@ public class AppSearchController extends BaseController { ...@@ -164,7 +164,7 @@ public class AppSearchController extends BaseController {
List<JSONObject> list = pair.getLeft().getJSONArray("list").toJavaList(JSONObject.class); List<JSONObject> list = pair.getLeft().getJSONArray("list").toJavaList(JSONObject.class);
commonService.sentimentAnalysis(list); commonService.sentimentAnalysis(list);
} }
wholeSearchService.decreaseRecord(dto.getSearch(), WholeSearchRecord.UsedType.query, pair.getRight()); extraService.decreaseRecord(dto.getSearch(), WholeSearchRecord.UsedType.query, pair.getRight());
return ResponseResult.success(pair.getLeft()); return ResponseResult.success(pair.getLeft());
} }
...@@ -191,7 +191,7 @@ public class AppSearchController extends BaseController { ...@@ -191,7 +191,7 @@ public class AppSearchController extends BaseController {
List<ExportSearchWholeDTO> exportList = markDataService.exportSearchWhole(dto); List<ExportSearchWholeDTO> exportList = markDataService.exportSearchWhole(dto);
EasyExcelUtil.download("全网搜舆情列表数据", "sheet1", ExportSearchWholeDTO.class, exportList, response); EasyExcelUtil.download("全网搜舆情列表数据", "sheet1", ExportSearchWholeDTO.class, exportList, response);
if (dto.isExternalDataSource()) { if (dto.isExternalDataSource()) {
wholeSearchService.decreaseRecord(dto.getSearch(), WholeSearchRecord.UsedType.output, exportList.size()); extraService.decreaseRecord(dto.getSearch(), WholeSearchRecord.UsedType.output, exportList.size());
} }
return ResponseResult.success(); return ResponseResult.success();
} }
......
src/main/java/com/zhiwei/brandkbs2/controller/app/AppToolsetController.java
package com.zhiwei.brandkbs2.controller.app; package com.zhiwei.brandkbs2.controller.app;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.aop.LogRecord; import com.zhiwei.brandkbs2.aop.LogRecord;
import com.zhiwei.brandkbs2.auth.Auth; import com.zhiwei.brandkbs2.auth.Auth;
import com.zhiwei.brandkbs2.auth.UserThreadLocal;
import com.zhiwei.brandkbs2.easyexcel.EasyExcelUtil;
import com.zhiwei.brandkbs2.easyexcel.config.ReadExcelDTO;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportArticleSummaryDTO;
import com.zhiwei.brandkbs2.easyexcel.dto.UploadArticleSummaryDTO;
import com.zhiwei.brandkbs2.easyexcel.listener.ArticleSummaryListener;
import com.zhiwei.brandkbs2.enmus.RoleEnum; import com.zhiwei.brandkbs2.enmus.RoleEnum;
import com.zhiwei.brandkbs2.model.ResponseResult; import com.zhiwei.brandkbs2.model.ResponseResult;
import com.zhiwei.brandkbs2.pojo.Project; import com.zhiwei.brandkbs2.service.ToolsetService;
import com.zhiwei.brandkbs2.service.ProjectService;
import com.zhiwei.brandkbs2.util.RedisUtil;
import com.zhiwei.brandkbs2.util.Tools;
import io.swagger.annotations.Api; import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiOperation;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*; import org.springframework.web.bind.annotation.*;
import org.springframework.web.client.RestTemplate;
import org.springframework.web.multipart.MultipartFile; import org.springframework.web.multipart.MultipartFile;
import javax.annotation.Resource; import javax.annotation.Resource;
import java.util.*;
import java.util.concurrent.TimeUnit;
/** /**
* @author cjz * @author cjz
...@@ -45,171 +23,40 @@ import java.util.concurrent.TimeUnit; ...@@ -45,171 +23,40 @@ import java.util.concurrent.TimeUnit;
@Api(tags = "工具集", description = "工具集") @Api(tags = "工具集", description = "工具集")
@Auth(role = RoleEnum.CUSTOMER) @Auth(role = RoleEnum.CUSTOMER)
public class AppToolsetController { public class AppToolsetController {
public static final Logger log = LogManager.getLogger(AppToolsetController.class);
@Resource(name = "redisUtil") @Resource(name = "toolsetServiceImpl")
private RedisUtil redisUtil; private ToolsetService toolsetService;
@Resource(name = "projectServiceImpl")
private ProjectService projectService;
@Autowired
private RestTemplate restTemplate;
@Value("${toolset.articleSummary.url}")
private String articleSummaryUrl;
@Value("${toolset.articleInfo.url}")
private String articleInfoUrl;
@Value("${brandkbs.file.url}")
private String brandkbsFilePath;
private static final String TEXT_SUMMARY_PREVIOUS = "作为一名公关人员，将文章进行简短的中文摘要，摘要文字中需包括日期、地点等核心要素，其中日期不用包括年份，摘要起始必须为日期，字数少于150个字\n文章：\n";
private static final int ARTICLE_SUMMARY_LIMIT = 1000;
@ApiOperation("摘要提取-单条") @ApiOperation("摘要提取-单条")
@GetMapping("/article-summary/single") @GetMapping("/article-summary/single")
@LogRecord(description = "工具库-摘要提取-单条") @LogRecord(description = "工具库-摘要提取-单条")
public ResponseResult getSingleArticleSummary(@RequestParam(value = "url") String url) { public ResponseResult getSingleArticleSummary(@RequestParam(value = "url") String url) {
JSONObject res = new JSONObject(); return toolsetService.getSingleArticleSummary(url);
JSONObject info = getUrlInfo(url, UserThreadLocal.getProjectId());
if (Objects.isNull(info)){
return ResponseResult.failure("链接解析异常");
}
String text = info.getString("content");
String articleSummaryResult = getArticleSummaryResult(text);
// 剩余次数限制
String redisKey = RedisUtil.getToolsetArticleSummaryLimitKey(UserThreadLocal.getProjectId());
String redisResult = redisUtil.get(redisKey);
int usedCount = 1;
if (Objects.nonNull(redisResult)){
int redisCount = Integer.parseInt(redisResult);
if (redisCount >= ARTICLE_SUMMARY_LIMIT){
return ResponseResult.failure("本日摘要提取次数已达上限");
}
usedCount = redisCount + 1;
redisUtil.setExpire(redisKey, String.valueOf(usedCount), Tools.getMillSecondNextDay(), TimeUnit.MILLISECONDS);
}else {
redisUtil.setExpire(redisKey, String.valueOf(usedCount), Tools.getMillSecondNextDay(), TimeUnit.MILLISECONDS);
}
res.put("source", info.get("source"));
res.put("platform", info.getString("platform"));
res.put("time", info.getLong("time"));
res.put("text", articleSummaryResult);
res.put("remainingCount", ARTICLE_SUMMARY_LIMIT - usedCount);
return ResponseResult.success(res);
} }
@ApiOperation("摘要提取") @ApiOperation("摘要提取")
@PostMapping("/article-summary/batch") @PostMapping("/article-summary/batch")
@LogRecord(description = "工具库-摘要提取-批量") @LogRecord(description = "工具库-摘要提取-批量")
public ResponseResult getBatchArticleSummary(@RequestParam(value = "file") MultipartFile file){ public ResponseResult downloadBatchArticleSummary(@RequestParam(value = "file") MultipartFile file){
JSONObject res = new JSONObject(); return toolsetService.getBatchArticleSummary(file);
Project project = projectService.getProjectById(UserThreadLocal.getProjectId());
// 调用前剩余可用次数
String redisKey = RedisUtil.getToolsetArticleSummaryLimitKey(UserThreadLocal.getProjectId());
String redisResult = redisUtil.get(redisKey);
int remainingCount = ARTICLE_SUMMARY_LIMIT;
if (Objects.nonNull(redisResult)){
remainingCount = ARTICLE_SUMMARY_LIMIT - Integer.parseInt(redisResult);
}
if (remainingCount <= 0){
return ResponseResult.failure("本日摘要提取次数已达上限");
}
// excel信息提取
Map<String, String> map = new LinkedHashMap<>();
ReadExcelDTO<UploadArticleSummaryDTO> readExcel = new ReadExcelDTO<>();
readExcel.setClazz(UploadArticleSummaryDTO.class);
readExcel.setAnalysisEventListener(new ArticleSummaryListener(map));
EasyExcelUtil.read(file, readExcel);
if (map.size() > 50){
return ResponseResult.failure("超过每次批量提取上限50");
}
int usedCount = 0; // 本次批量调用次数
List<ExportArticleSummaryDTO> datas = new ArrayList<>(50);
// 摘要提取
for (Map.Entry<String, String> entry : map.entrySet()) {
String text = entry.getValue();
String url = entry.getKey();
// 文章内容空时，通过链接提取文章内容
if (Objects.isNull(text) && Objects.nonNull(url)){
JSONObject json = getUrlInfo(entry.getKey(), UserThreadLocal.getProjectId());
text = Objects.nonNull(json) ? json.getString("content") : null;
}
String articleSummaryResult = getArticleSummaryResult(text);
usedCount = usedCount + 1;
datas.add(new ExportArticleSummaryDTO(String.valueOf(usedCount), entry.getKey(), entry.getValue(), articleSummaryResult));
}
// 本次批量调用后剩余次数
remainingCount = remainingCount - usedCount;
// 更新已用次数
redisUtil.setExpire(redisKey, String.valueOf(ARTICLE_SUMMARY_LIMIT - remainingCount), Tools.getMillSecondNextDay(), TimeUnit.MILLISECONDS);
// excel输出到指定路径
String filePath = EasyExcelUtil.generateExcelFilePath(brandkbsFilePath, project.getProjectName(), UserThreadLocal.getNickname(), "摘要提取结果");
EasyExcelUtil.write(filePath, "sheet1", ExportArticleSummaryDTO.class, datas);
res.put("filePath", filePath);
res.put("remainingCount", remainingCount);
return ResponseResult.success(res);
} }
@ApiOperation("摘要提取-剩余可用次数") @ApiOperation("摘要提取-剩余可用次数")
@GetMapping("/article-summary/remaining") @GetMapping("/article-summary/remaining")
public ResponseResult getArticleSummaryRemainingCount(){ public ResponseResult getArticleSummaryRemainingCount(){
String redisKey = RedisUtil.getToolsetArticleSummaryLimitKey(UserThreadLocal.getProjectId()); return toolsetService.getArticleSummaryRemainingCount();
String redisResult = redisUtil.get(redisKey);
if (Objects.nonNull(redisResult)){
return ResponseResult.success(ARTICLE_SUMMARY_LIMIT - Integer.parseInt(redisResult));
}
return ResponseResult.success(ARTICLE_SUMMARY_LIMIT);
} }
/** @ApiOperation("链接互动量更新")
* 链接信息提取 @PostMapping("/interaction-update/url")
* @param url 链接 @LogRecord(description = "工具库-链接互动量更新")
* @param projectId 项目id public ResponseResult downloadUrlInteractionUpdate(@RequestParam(value = "file") MultipartFile file){
* @return return toolsetService.urlInteractionUpdate(file);
*/
private JSONObject getUrlInfo(String url, String projectId) {
JSONObject jsonObject;
// 即使抛出异常(大概率会是timeout)，也要保证批量时其他链接正常执行
try {
String linkedGroupId = projectService.getProjectVOById(projectId).getBrandLinkedGroupId();
jsonObject = restTemplate.getForEntity(articleInfoUrl, JSONObject.class, url, linkedGroupId, UserThreadLocal.getNickname()).getBody();
} catch (Exception e) {
log.info("url:{},访问链接信息提取接口异常-", url, e);
return null;
}
if (Objects.isNull(jsonObject) || !jsonObject.getBoolean("status")) {
return null;
}
return jsonObject.getJSONObject("data");
} }
/** @ApiOperation("互动量更新-剩余可用次数")
* 获取摘要提取结果 @GetMapping("/interaction-update/remaining")
* @param text 文本 public ResponseResult getInteractionRemainingCount(){
* @return return ResponseResult.success(toolsetService.getInteractionRemainingCount());
*/
private String getArticleSummaryResult(String text){
// 即使抛出异常(大概率会是timeout)，也要保证批量时其他链接正常执行
String errorString = "访问超时,请稍后重试此条数据";
if (Objects.isNull(text)){
return errorString;
}
try {
// 拼接提示词模板
String resultText = TEXT_SUMMARY_PREVIOUS + StringUtils.substring(text, 0, 5000);
// 请求参数 请求头
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.TEXT_PLAIN);
HttpEntity<String> request = new HttpEntity<>(resultText, headers);
ResponseEntity<String> response = restTemplate.postForEntity(articleSummaryUrl, request, String.class);
return response.getBody();
}catch (Exception e){
log.info("访问摘要提取接口异常-", e);
return errorString;
}
} }
} }
src/main/java/com/zhiwei/brandkbs2/dao/InteractionUpdateRecordDao.java 0 → 100644
package com.zhiwei.brandkbs2.dao;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.pojo.InteractionUpdateRecord;
import com.zhiwei.brandkbs2.pojo.vo.LineVO;
import java.util.List;
/**
* @ClassName: InteractionUpdateRecordDao
* @Description InteractionUpdateRecordDao
* @author: cjz
* @date: 2023-09-21 15:57
*/
public interface InteractionUpdateRecordDao extends BaseMongoDao<InteractionUpdateRecord>, ShardingMongo{
/**
*
* @param startTime 起始时间
* @param endTime 结束时间
* @param day 颗粒度
* @return list
*/
List<JSONObject> aggregateProjectUsedRecord(long startTime, long endTime, boolean day);
/**
* 数据总览
* @param startTime 起始时间
* @param endTime 结束时间
* @return list
*/
List<LineVO> aggregateDayLastRecord(long startTime, long endTime);
}
src/main/java/com/zhiwei/brandkbs2/dao/impl/InteractionUpdateRecordDaoImpl.java 0 → 100644
package com.zhiwei.brandkbs2.dao.impl;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.auth.UserThreadLocal;
import com.zhiwei.brandkbs2.dao.InteractionUpdateRecordDao;
import com.zhiwei.brandkbs2.pojo.InteractionUpdateRecord;
import com.zhiwei.brandkbs2.pojo.vo.LineVO;
import org.springframework.data.domain.Sort;
import org.springframework.data.mongodb.core.aggregation.Aggregation;
import org.springframework.data.mongodb.core.aggregation.AggregationOperation;
import org.springframework.data.mongodb.core.aggregation.AggregationResults;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
@Component("interactionUpdateRecordDao")
public class InteractionUpdateRecordDaoImpl extends BaseMongoDaoImpl<InteractionUpdateRecord> implements InteractionUpdateRecordDao {
private static final String COLLECTION_PREFIX = "brandkbs_interaction_update_record";
private static final String TIME_PATTERN = "yyyy";
public InteractionUpdateRecordDaoImpl() {
super(null);
}
@Override
public String collectionPrefix() {
return COLLECTION_PREFIX;
}
@Override
public String timePattern() {
return TIME_PATTERN;
}
@Override
public List<JSONObject> aggregateProjectUsedRecord(long startTime, long endTime, boolean day) {
List<JSONObject> res = new ArrayList<>();
String[] collectionNames = generateCollectionNames(new Date(startTime), new Date(endTime));
Criteria criteria = Criteria.where("isUsed").is(true).and("projectId").is(UserThreadLocal.getProjectId());
int nrOfChars = day ? 10 : 7;
List<AggregationOperation> operations = new ArrayList<>();
// operations区分先后顺序
operations.add(Aggregation.match(criteria));
operations.add(Aggregation.project( "usedCount", "cTime").and("cTime").substring(0, nrOfChars).as("cTime"));
operations.add(Aggregation.group("cTime").sum("usedCount").as("count"));
operations.add(Aggregation.sort(Sort.by(Sort.Order.desc("_id"))));
Aggregation aggregation = Aggregation.newAggregation(operations);
for (String collectionName : collectionNames) {
AggregationResults<JSONObject> aggregateResult = mongoTemplate.aggregate(aggregation, collectionName, JSONObject.class);
res.addAll(aggregateResult.getMappedResults());
}
return res;
}
@Override
public List<LineVO> aggregateDayLastRecord(long startTime, long endTime) {
List<LineVO> res = new ArrayList<>();
String[] collectionNames = generateCollectionNames(new Date(startTime), new Date(endTime));
Criteria criteria = Criteria.where("projectId").is(UserThreadLocal.getProjectId());
List<AggregationOperation> operations = new ArrayList<>();
// operations区分先后顺序
operations.add(Aggregation.match(criteria));
operations.add(Aggregation.project("currentCount").and("cTime").substring(0, 10).as("cTime"));
operations.add(Aggregation.group("cTime").last("currentCount").as("count"));
operations.add(Aggregation.sort(Sort.by(Sort.Order.asc("_id"))));
Aggregation aggregation = Aggregation.newAggregation(operations);
// 优先小的年份
for (int i = collectionNames.length - 1; i >= 0; i--) {
AggregationResults<JSONObject> aggregateResult = mongoTemplate.aggregate(aggregation, collectionNames[i], JSONObject.class);
List<JSONObject> mappedResults = aggregateResult.getMappedResults();
for (JSONObject mappedResult : mappedResults) {
res.add(new LineVO(mappedResult.getLong("count"), mappedResult.getDate("_id").getTime()));
}
}
return res;
}
}
src/main/java/com/zhiwei/brandkbs2/easyexcel/dto/ExportInteractionUpdateDTO.java 0 → 100644
package com.zhiwei.brandkbs2.easyexcel.dto;
import com.alibaba.excel.annotation.ExcelProperty;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.enmus.InteractionEnum;
import com.zhiwei.brandkbs2.util.Tools;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Objects;
/**
* @author cjz
* @version 1.0
* @description 导出互动量更新结果数据实体类
* @date 2023年9月22日 11:13:08
*/
@Data
@ToString
@NoArgsConstructor
public class ExportInteractionUpdateDTO {
@ExcelProperty("序号")
private String id;
@ExcelProperty("平台")
private String platform;
@ExcelProperty("地址")
private String url;
@ExcelProperty("更新时间")
private Date time;
@ExcelProperty("评论数")
private int commentCount;
@ExcelProperty("点赞数")
private int likeCount;
@ExcelProperty("转发数")
private int repostCount;
@ExcelProperty("阅读数")
private int readCount;
public ExportInteractionUpdateDTO(String id, Map<Object, JSONObject> urlMap, String url){
this.id = id;
this.platform = InteractionEnum.getPlatformByDomain(Tools.topDomainOfDomain(url));
this.url = url;
if (Objects.nonNull(urlMap.get(url))){
JSONObject jsonObject = urlMap.get(url);
this.commentCount = jsonObject.getIntValue("commentCount");
this.likeCount = jsonObject.getIntValue("likeCount");
this.repostCount = jsonObject.getIntValue("repostCount");
this.readCount = jsonObject.getIntValue("readCount");
}
this.time = new Date();
}
public static ExportInteractionUpdateDTO createWithPlatform(String id, Map<String, String> urlPlatformMap, Map<Object, JSONObject> urlMap, String url){
ExportInteractionUpdateDTO dto = new ExportInteractionUpdateDTO();
dto.setId(id);
dto.setPlatform(urlPlatformMap.get(url));
dto.setUrl(url);
if (Objects.nonNull(urlMap.get(url))){
JSONObject jsonObject = urlMap.get(url);
dto.setCommentCount(jsonObject.getIntValue("commentCount"));
dto.setLikeCount(jsonObject.getIntValue("likeCount"));
dto.setRepostCount(jsonObject.getIntValue("repostCount"));
dto.setReadCount(jsonObject.getIntValue("readCount"));
}
dto.setTime(new Date());
return dto;
}
}
src/main/java/com/zhiwei/brandkbs2/easyexcel/dto/ExportInteractionUpdateRecordDTO.java 0 → 100644
package com.zhiwei.brandkbs2.easyexcel.dto;
import com.alibaba.excel.annotation.ExcelProperty;
import com.alibaba.fastjson.JSONObject;
import lombok.Data;
import lombok.ToString;
import java.util.Date;
/**
* @ClassName: ExportInteractionUpdateRecordDTO
* @Description ExportInteractionUpdateRecordDTO
* @author: cjz
* @date: 2023-09-25 11:02
*/
@Data
@ToString
public class ExportInteractionUpdateRecordDTO {
@ExcelProperty("类型")
private String usedType;
@ExcelProperty("数据量")
private Integer usedCount;
@ExcelProperty("操作时间")
private Date cTime;
@ExcelProperty("操作人")
private String submitter;
public static ExportInteractionUpdateRecordDTO createFromJSONObject(JSONObject json) {
ExportInteractionUpdateRecordDTO dto = new ExportInteractionUpdateRecordDTO();
dto.setUsedType(json.getString("usedType"));
dto.setUsedCount(json.getInteger("usedCount"));
dto.setCTime(new Date(json.getLong("cTime")));
dto.setSubmitter(json.getString("submitter"));
return dto;
}
}
src/main/java/com/zhiwei/brandkbs2/easyexcel/dto/UploadArticleSummaryDTO.java
...@@ -6,7 +6,7 @@ import lombok.Data; ...@@ -6,7 +6,7 @@ import lombok.Data;
/** /**
* @author cjz * @author cjz
* @version 1.0 * @version 1.0
* @description 解析文章摘要提前上传文件 * @description 解析文章摘要上传文件
* @date 2023/9/14 14:22 * @date 2023/9/14 14:22
*/ */
@Data @Data
......
src/main/java/com/zhiwei/brandkbs2/easyexcel/dto/UploadInteractionUpdateDTO.java 0 → 100644
package com.zhiwei.brandkbs2.easyexcel.dto;
import com.alibaba.excel.annotation.ExcelProperty;
import lombok.Data;
/**
* @author cjz
* @version 1.0
* @description 解析互动量更新上传文件
* @date 2023/9/14 14:22
*/
@Data
public class UploadInteractionUpdateDTO {
@ExcelProperty("序号")
private String id;
@ExcelProperty("链接")
private String url;
}
src/main/java/com/zhiwei/brandkbs2/easyexcel/listener/InteractionUpdateListener.java 0 → 100644
package com.zhiwei.brandkbs2.easyexcel.listener;
import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.event.AnalysisEventListener;
import com.zhiwei.brandkbs2.easyexcel.dto.UploadInteractionUpdateDTO;
import java.util.List;
/**
* @ClassName: ArticleSummaryListener
* @Description 链接互动量更新上传监听类
* @author: cjz
* @date: 2023-09-22 13:44
*/
public class InteractionUpdateListener extends AnalysisEventListener<UploadInteractionUpdateDTO> {
private List<String> urls;
public InteractionUpdateListener(List<String> urls){
this.urls = urls;
}
@Override
public void invoke(UploadInteractionUpdateDTO data, AnalysisContext context) {
urls.add(data.getUrl());
}
@Override
public void doAfterAllAnalysed(AnalysisContext context) {
}
}
src/main/java/com/zhiwei/brandkbs2/enmus/InteractionEnum.java 0 → 100644
package com.zhiwei.brandkbs2.enmus;
import lombok.Getter;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
/**
* @ClassName: InteractionEnum
* @Description 互动量更新枚举类
* @author: cjz
* @date: 2023-09-21 17:56
*/
public enum InteractionEnum {
WEIBO("微博", "微博互动量", Arrays.asList("www.weibo.com", "weibo.com", "video.weibo.com")),
WECHAT("微信", "微信互动量", Arrays.asList("mp.weixin.qq.com", "qq.com")),
WEB_MEDIA("网媒", "链接信息更新", null),
PING_MEDIA("平媒", "链接信息更新", null),
TOUTIAO("今日头条", "链接信息更新", Arrays.asList("toutiao.com", "www.toutiao.com")),
TIKTOK("抖音", "抖音互动量", Arrays.asList("douyin.com", "iesdouyin.com", "vdouyin.com", "v.vdouyin.com")),
XIAOHONGSHU("小红书", "小红书互动量", Collections.singletonList("xiaohongshu.com")),
SELF_MEDIA("自媒体", "链接信息更新", null),
ZHIHU("知乎", "知乎互动量", Arrays.asList("www.zhihu.com", "zhihu.com")),
MAIMAI("脉脉", "链接信息更新", null),
QA("问答", "链接信息更新", null),
TIABA("贴吧论坛", "链接信息更新", null),
VIDEO("视频", "链接信息更新", null),
SHORT_VIDEO("短视频", "链接信息更新", null);
@Getter
private final String platform;
@Getter
private final String type;
@Getter
private final List<String> topDomain;
InteractionEnum(String platform, String type, List<String> topDomain){
this.platform = platform;
this.type = type;
this.topDomain = topDomain;
}
public static String getTypeByPlatform(String platform){
if (Objects.equals(platform, "其他")){
return "链接信息更新";
}
for (InteractionEnum value : InteractionEnum.values()) {
if (Objects.equals(value.getPlatform(), platform)){
return value.getType();
}
}
return "链接信息更新";
}
public static String getTypeByDomain(String topDomain){
if (Objects.isNull(topDomain)) {
return "链接信息更新";
}
for (InteractionEnum value : InteractionEnum.values()) {
if (Objects.nonNull(value.getTopDomain()) && value.getTopDomain().contains(topDomain)){
return value.getType();
}
}
return "链接信息更新";
}
public static String getPlatformByDomain(String topDomain){
if (Objects.isNull(topDomain)) {
return "其他";
}
for (InteractionEnum value : InteractionEnum.values()) {
if (Objects.nonNull(value.getTopDomain()) && value.getTopDomain().contains(topDomain)){
return value.getPlatform();
}
}
return "其他";
}
}
src/main/java/com/zhiwei/brandkbs2/pojo/InteractionUpdateRecord.java 0 → 100644
package com.zhiwei.brandkbs2.pojo;
import lombok.Getter;
import lombok.Setter;
import java.util.Date;
/**
* @ClassName: VolumeUpdateRecord
* @Description 互动量更新记录实体类
* @author: cjz
* @date: 2023-09-21 15:21
*/
@Getter
@Setter
public class InteractionUpdateRecord extends AbstractBaseMongo{
/**
* 项目id
*/
private String projectId;
/**
* 使用记录（互动量可用次数调整记录/互动量更新使用记录）
*/
private boolean isUsed;
/**
* 使用类型(导出/查询)
*/
private String usedType;
/**
* 上一次记录数(本次使用前剩余次数)
*/
private int lastCount;
/**
* 该次操作使用(减少)数值 (本次使用的次数)
*/
private int usedCount;
/**
* 本次记录数(本次使用后剩余次数)
*/
private int currentCount;
/**
* 提交人
*/
private String submitter;
/**
* 创建时间(Date类型方便聚合查询)
*/
private Date cTime;
public static InteractionUpdateRecord createUsedRecord(String projectId, String usedType, int lastCount, int usedCount, String submitter) {
InteractionUpdateRecord record = new InteractionUpdateRecord();
record.projectId = projectId;
record.isUsed = true;
record.usedType = usedType;
record.lastCount = lastCount;
record.usedCount = usedCount;
record.currentCount = lastCount - usedCount;
record.submitter = submitter;
record.cTime = new Date();
return record;
}
public static InteractionUpdateRecord createMaintainRecord(String projectId, int lastCount, int currentCount, String submitter) {
InteractionUpdateRecord record = new InteractionUpdateRecord();
record.projectId = projectId;
record.isUsed = false;
record.lastCount = lastCount;
record.currentCount = currentCount;
record.submitter = submitter;
record.cTime = new Date();
return record;
}
public enum UsedType {
url("链接"),
yuqing("舆情库");
@Getter
private final String value;
UsedType(String value) {
this.value = value;
}
}
}
src/main/java/com/zhiwei/brandkbs2/pojo/Project.java
...@@ -92,6 +92,11 @@ public class Project extends AbstractProject { ...@@ -92,6 +92,11 @@ public class Project extends AbstractProject {
private int wholeSearchBalance; private int wholeSearchBalance;
/** /**
* 互动量更新余额
*/
private int interactionBalance;
/**
* 项目主品牌配置信息转换 * 项目主品牌配置信息转换
* *
* @return 项目对象 * @return 项目对象
......
src/main/java/com/zhiwei/brandkbs2/service/WholeSearchService.java → src/main/java/com/zhiwei/brandkbs2/service/ExtraService.java
package com.zhiwei.brandkbs2.service; package com.zhiwei.brandkbs2.service;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.pojo.InteractionUpdateRecord;
import com.zhiwei.brandkbs2.pojo.WholeSearchRecord; import com.zhiwei.brandkbs2.pojo.WholeSearchRecord;
import com.zhiwei.brandkbs2.pojo.vo.PageVO; import com.zhiwei.brandkbs2.pojo.vo.PageVO;
...@@ -12,7 +13,7 @@ import java.util.List; ...@@ -12,7 +13,7 @@ import java.util.List;
* @author: sjj * @author: sjj
* @date: 2023-07-19 11:43 * @date: 2023-07-19 11:43
*/ */
public interface WholeSearchService { public interface ExtraService {
/** /**
* 查询使用列表 * 查询使用列表
...@@ -22,7 +23,7 @@ public interface WholeSearchService { ...@@ -22,7 +23,7 @@ public interface WholeSearchService {
* @param sorter 排序字段 * @param sorter 排序字段
* @return PageVO * @return PageVO
*/ */
PageVO<JSONObject> findUsedList(int page,int pageSize,String sorter); PageVO<JSONObject> findUsedList(int page, int pageSize, String sorter);
/** /**
* 查询维护列表 * 查询维护列表
...@@ -31,7 +32,7 @@ public interface WholeSearchService { ...@@ -31,7 +32,7 @@ public interface WholeSearchService {
* @param pageSize 页码大小 * @param pageSize 页码大小
* @return PageVO * @return PageVO
*/ */
PageVO<JSONObject> findMaintainList(int page,int pageSize); PageVO<JSONObject> findMaintainList(int page, int pageSize);
/** /**
* 导出使用列表 * 导出使用列表
...@@ -41,7 +42,7 @@ public interface WholeSearchService { ...@@ -41,7 +42,7 @@ public interface WholeSearchService {
* @param day 颗粒度(天级/月级) * @param day 颗粒度(天级/月级)
* @return List * @return List
*/ */
List<JSONObject> outputUsedList(boolean personal,long startTime,long endTime,boolean day); List<JSONObject> outputUsedList(boolean personal, long startTime, long endTime, boolean day);
/** /**
* @param currentCount 调整后数值 * @param currentCount 调整后数值
...@@ -61,4 +62,51 @@ public interface WholeSearchService { ...@@ -61,4 +62,51 @@ public interface WholeSearchService {
*/ */
void decreaseRecord(String keyword, WholeSearchRecord.UsedType usedType, Integer usedCount); void decreaseRecord(String keyword, WholeSearchRecord.UsedType usedType, Integer usedCount);
/**
* 查询互动量更新使用列表
*
* @param page 页码
* @param pageSize 页码大小
* @param sorter 排序字段
* @return PageVO
*/
PageVO<JSONObject> findInteractionUsedList(int page, int pageSize, String sorter);
/**
* 查询互动量更新维护列表
*
* @param page 页码
* @param pageSize 页码大小
* @return PageVO
*/
PageVO<JSONObject> findInteractionMaintainList(int page, int pageSize);
/**
* 导出使用列表
* @param personal 个人明细/项目明细
* @param startTime 开始时间
* @param endTime 结束时间
* @param day 颗粒度(天级/月级)
* @return List
*/
List<JSONObject> outputInteractionUsedList(boolean personal, long startTime, long endTime, boolean day);
/**
* 互动量更新可用次数调整
* @param currentCount 调整后数值
*/
void adjustInteractionRecord(Integer currentCount);
/**
* 数据总览
* @return json
*/
JSONObject interactionOverview();
/**
* 记录互动量更新使用
* @param usedType 使用类型
* @param usedCount 使用次数
*/
void decreaseInteractionRecord(InteractionUpdateRecord.UsedType usedType, Integer usedCount);
} }
src/main/java/com/zhiwei/brandkbs2/service/MarkDataService.java
...@@ -365,4 +365,17 @@ public interface MarkDataService { ...@@ -365,4 +365,17 @@ public interface MarkDataService {
BoolQueryBuilder projectContendIdQuery(String projectId, String contendId); BoolQueryBuilder projectContendIdQuery(String projectId, String contendId);
/**
* 舆情库-互动量更新
* @param dto 标注数据搜索传输类
* @return
*/
ResponseResult markInteractionUpdate(MarkSearchDTO dto);
/**
* 工具库-舆情库互动量更新预估
* @param dto 标注数据搜索传输类
* @return
*/
Long interactionUpdatePrediction(MarkSearchDTO dto);
} }
src/main/java/com/zhiwei/brandkbs2/service/ToolsetService.java 0 → 100644
package com.zhiwei.brandkbs2.service;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.model.ResponseResult;
import org.springframework.web.multipart.MultipartFile;
import java.util.List;
/**
* @ClassName: ToolsetService
* @Description: 工具库服务抽象类
* @author: cjz
* @date: 2023-09-22 09:36
*/
public interface ToolsetService {
/**
* 摘要提取-单条
* @param url
* @return
*/
ResponseResult getSingleArticleSummary(String url);
/**
* 摘要提取-批量
* @param file 文件
* @return
*/
ResponseResult getBatchArticleSummary(MultipartFile file);
/**
* 获取摘要提取剩余可用次数
* @return
*/
ResponseResult getArticleSummaryRemainingCount();
/**
* 链接互动量更新
* @param file excel文件
* @return
*/
ResponseResult urlInteractionUpdate(MultipartFile file);
/**
* 获取互动量更新剩余可用次数
* @return
*/
int getInteractionRemainingCount();
/**
* 提交互动量更新任务
* @param type 任务类型
* @param urls 链接
* @return 任务id
*/
List<String> requestInteractionUrl(String type, List<String> urls);
/**
* 获取互动量更新结果
* @param taskId 任务id
* @return
*/
List<JSONObject> getInteractionResult(List<String> taskId);
}
src/main/java/com/zhiwei/brandkbs2/service/impl/WholeSearchServiceImpl.java → src/main/java/com/zhiwei/brandkbs2/service/impl/ExtraServiceImpl.java
...@@ -3,12 +3,14 @@ package com.zhiwei.brandkbs2.service.impl; ...@@ -3,12 +3,14 @@ package com.zhiwei.brandkbs2.service.impl;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.auth.UserThreadLocal; import com.zhiwei.brandkbs2.auth.UserThreadLocal;
import com.zhiwei.brandkbs2.config.Constant; import com.zhiwei.brandkbs2.config.Constant;
import com.zhiwei.brandkbs2.dao.InteractionUpdateRecordDao;
import com.zhiwei.brandkbs2.dao.ProjectDao; import com.zhiwei.brandkbs2.dao.ProjectDao;
import com.zhiwei.brandkbs2.dao.WholeSearchRecordDao; import com.zhiwei.brandkbs2.dao.WholeSearchRecordDao;
import com.zhiwei.brandkbs2.pojo.Project; import com.zhiwei.brandkbs2.pojo.Project;
import com.zhiwei.brandkbs2.pojo.InteractionUpdateRecord;
import com.zhiwei.brandkbs2.pojo.WholeSearchRecord; import com.zhiwei.brandkbs2.pojo.WholeSearchRecord;
import com.zhiwei.brandkbs2.pojo.vo.PageVO; import com.zhiwei.brandkbs2.pojo.vo.PageVO;
import com.zhiwei.brandkbs2.service.WholeSearchService; import com.zhiwei.brandkbs2.service.ExtraService;
import com.zhiwei.brandkbs2.util.MongoUtil; import com.zhiwei.brandkbs2.util.MongoUtil;
import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Query;
...@@ -18,6 +20,7 @@ import org.springframework.stereotype.Service; ...@@ -18,6 +20,7 @@ import org.springframework.stereotype.Service;
import javax.annotation.Resource; import javax.annotation.Resource;
import java.util.Date; import java.util.Date;
import java.util.List; import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors; import java.util.stream.Collectors;
/** /**
...@@ -26,12 +29,15 @@ import java.util.stream.Collectors; ...@@ -26,12 +29,15 @@ import java.util.stream.Collectors;
* @author: sjj * @author: sjj
* @date: 2023-07-19 11:55 * @date: 2023-07-19 11:55
*/ */
@Service("wholeSearchServiceImpl") @Service("extraServiceImpl")
public class WholeSearchServiceImpl implements WholeSearchService { public class ExtraServiceImpl implements ExtraService {
@Resource(name = "wholeSearchRecordDao") @Resource(name = "wholeSearchRecordDao")
private WholeSearchRecordDao wholeSearchRecordDao; private WholeSearchRecordDao wholeSearchRecordDao;
@Resource(name = "interactionUpdateRecordDao")
private InteractionUpdateRecordDao interactionUpdateRecordDao;
@Resource(name = "projectDao") @Resource(name = "projectDao")
private ProjectDao projectDao; private ProjectDao projectDao;
...@@ -115,6 +121,95 @@ public class WholeSearchServiceImpl implements WholeSearchService { ...@@ -115,6 +121,95 @@ public class WholeSearchServiceImpl implements WholeSearchService {
wholeSearchRecordDao.insertOne(WholeSearchRecord.createUsedRecord(projectId, keyword, usedType.getValue(), project.getWholeSearchBalance(), usedCount, UserThreadLocal.getNickname()), wholeSearchRecordDao.generateCollectionName()); wholeSearchRecordDao.insertOne(WholeSearchRecord.createUsedRecord(projectId, keyword, usedType.getValue(), project.getWholeSearchBalance(), usedCount, UserThreadLocal.getNickname()), wholeSearchRecordDao.generateCollectionName());
} }
@Override
public PageVO<JSONObject> findInteractionUsedList(int page, int pageSize, String sorter) {
String collectionName = interactionUpdateRecordDao.generateCollectionName();
Query query = Query.query(Criteria.where("projectId").is(UserThreadLocal.getProjectId()).and("isUsed").is(true));
interactionUpdateRecordDao.addSort(query, sorter);
long total = interactionUpdateRecordDao.count(query, collectionName);
mongoUtil.start(page, pageSize, query);
return PageVO.createPageVo(total, page, pageSize, interactionUsedList(query, collectionName));
}
@Override
public PageVO<JSONObject> findInteractionMaintainList(int page, int pageSize) {
String collectionName = interactionUpdateRecordDao.generateCollectionName();
Query query = Query.query(Criteria.where("projectId").is(UserThreadLocal.getProjectId()).and("isUsed").is(false));
interactionUpdateRecordDao.addSort(query, "{\"cTime\":\"descend\"}");
long total = interactionUpdateRecordDao.count(query, collectionName);
mongoUtil.start(page, pageSize, query);
List<JSONObject> collect = interactionUpdateRecordDao.findList(query, collectionName).stream().map(record -> {
JSONObject json = new JSONObject();
json.put("lastCount", record.getLastCount());
json.put("currentCount", record.getCurrentCount());
json.put("cTime", record.getCTime().getTime());
json.put("submitter", record.getSubmitter());
return json;
}).collect(Collectors.toList());
return PageVO.createPageVo(total, page, pageSize, collect);
}
@Override
public List<JSONObject> outputInteractionUsedList(boolean personal, long startTime, long endTime, boolean day) {
// 个人明细
if (personal) {
Criteria criteria = Criteria.where("isUsed").is(true);
criteria.and("submitter").is(UserThreadLocal.getNickname());
Query query = Query.query(criteria);
interactionUpdateRecordDao.addSort(query, "{\"cTime\":\"descend\"}");
return interactionUsedList(query, interactionUpdateRecordDao.generateCollectionNames(new Date(startTime), new Date(endTime)));
} else {// 项目明细,需要细分颗粒度
return interactionUpdateRecordDao.aggregateProjectUsedRecord(startTime, endTime, day);
}
}
@Override
public void adjustInteractionRecord(Integer currentCount) {
if (null == currentCount) {
return;
}
String projectId = UserThreadLocal.getProjectId();
Update update = Update.update("interactionBalance", currentCount);
Project project = projectDao.findOneById(projectId);
projectDao.updateOneByIdWithField(projectId, update);
InteractionUpdateRecord record =
InteractionUpdateRecord.createMaintainRecord(projectId, project.getInteractionBalance(), currentCount, UserThreadLocal.getNickname());
interactionUpdateRecordDao.insertOne(record, interactionUpdateRecordDao.generateCollectionName());
}
@Override
public JSONObject interactionOverview() {
JSONObject json = new JSONObject();
long endTime = System.currentTimeMillis();
long startTime = endTime - Constant.ONE_MONTH;
json.put("balance", projectDao.findOneById(UserThreadLocal.getProjectId()).getInteractionBalance());
json.put("trendList", interactionUpdateRecordDao.aggregateDayLastRecord(startTime, endTime));
return json;
}
@Override
public void decreaseInteractionRecord(InteractionUpdateRecord.UsedType usedType, Integer usedCount) {
String projectId = UserThreadLocal.getProjectId();
Project project = projectDao.findOneById(projectId);
Update update = new Update();
update.inc("interactionBalance", -usedCount);
projectDao.updateOneByIdWithField(projectId, update);
InteractionUpdateRecord record =
InteractionUpdateRecord.createUsedRecord(projectId, usedType.getValue(), project.getInteractionBalance(), usedCount, UserThreadLocal.getNickname());
interactionUpdateRecordDao.insertOne(record, interactionUpdateRecordDao.generateCollectionName());
}
private List<JSONObject> interactionUsedList(Query query, String... collectionNames) {
return interactionUpdateRecordDao.findList(query, collectionNames).stream().map(record -> {
JSONObject json = new JSONObject();
json.put("usedType", record.getUsedType());
json.put("usedCount", record.getUsedCount());
json.put("cTime", record.getCTime().getTime());
json.put("submitter", record.getSubmitter());
return json;
}).collect(Collectors.toList());
}
private List<JSONObject> usedList(Query query, String... collectionNames) { private List<JSONObject> usedList(Query query, String... collectionNames) {
return wholeSearchRecordDao.findList(query, collectionNames).stream().map(record -> { return wholeSearchRecordDao.findList(query, collectionNames).stream().map(record -> {
JSONObject json = new JSONObject(); JSONObject json = new JSONObject();
......
src/main/java/com/zhiwei/brandkbs2/service/impl/MarkDataServiceImpl.java
...@@ -17,9 +17,12 @@ import com.zhiwei.brandkbs2.config.Constant; ...@@ -17,9 +17,12 @@ import com.zhiwei.brandkbs2.config.Constant;
import com.zhiwei.brandkbs2.dao.AggreeResultDao; import com.zhiwei.brandkbs2.dao.AggreeResultDao;
import com.zhiwei.brandkbs2.dao.ChannelDao; import com.zhiwei.brandkbs2.dao.ChannelDao;
import com.zhiwei.brandkbs2.dao.HighlightWordDao; import com.zhiwei.brandkbs2.dao.HighlightWordDao;
import com.zhiwei.brandkbs2.easyexcel.EasyExcelUtil;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportInteractionUpdateDTO;
import com.zhiwei.brandkbs2.enmus.ChannelEmotion; import com.zhiwei.brandkbs2.enmus.ChannelEmotion;
import com.zhiwei.brandkbs2.enmus.EmotionEnum; import com.zhiwei.brandkbs2.enmus.EmotionEnum;
import com.zhiwei.brandkbs2.enmus.ImportantChannelEnum; import com.zhiwei.brandkbs2.enmus.ImportantChannelEnum;
import com.zhiwei.brandkbs2.enmus.InteractionEnum;
import com.zhiwei.brandkbs2.es.EsClientDao; import com.zhiwei.brandkbs2.es.EsClientDao;
import com.zhiwei.brandkbs2.es.EsQueryTools; import com.zhiwei.brandkbs2.es.EsQueryTools;
import com.zhiwei.brandkbs2.exception.ExceptionCast; import com.zhiwei.brandkbs2.exception.ExceptionCast;
...@@ -42,6 +45,7 @@ import com.zhiwei.brandkbs2.util.RedisUtil; ...@@ -42,6 +45,7 @@ import com.zhiwei.brandkbs2.util.RedisUtil;
import com.zhiwei.brandkbs2.util.TextUtil; import com.zhiwei.brandkbs2.util.TextUtil;
import com.zhiwei.brandkbs2.util.Tools; import com.zhiwei.brandkbs2.util.Tools;
import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.ListUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils; import org.apache.commons.lang3.time.DateUtils;
import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.lang3.tuple.Pair;
...@@ -153,6 +157,15 @@ public class MarkDataServiceImpl implements MarkDataService { ...@@ -153,6 +157,15 @@ public class MarkDataServiceImpl implements MarkDataService {
@Resource(name = "highlightWordDao") @Resource(name = "highlightWordDao")
HighlightWordDao highlightWordDao; HighlightWordDao highlightWordDao;
@Resource(name = "toolsetServiceImpl")
private ToolsetService toolsetService;
@Resource(name = "extraServiceImpl")
private ExtraService extraService;
@Value("${brandkbs.file.url}")
private String brandkbsFilePath;
@Override @Override
public PageVO<MarkFlowEntity> getOriginList(MarkSearchDTO markSearchDTO) { public PageVO<MarkFlowEntity> getOriginList(MarkSearchDTO markSearchDTO) {
try { try {
...@@ -1500,6 +1513,65 @@ public class MarkDataServiceImpl implements MarkDataService { ...@@ -1500,6 +1513,65 @@ public class MarkDataServiceImpl implements MarkDataService {
return EsQueryTools.assembleCacheMapsQuery(projectId, contendId); return EsQueryTools.assembleCacheMapsQuery(projectId, contendId);
} }
@Override
public ResponseResult markInteractionUpdate(MarkSearchDTO dto) {
Pair<String, List<ExportAppYuqingDTO>> pair = downloadList(dto, esSearchService::searchMarkHitsAndCount);
assert pair != null;
List<ExportAppYuqingDTO> list = pair.getRight();
// 互动量更新剩余可用次数
int interactionBalance = projectService.getProjectById(UserThreadLocal.getProjectId()).getInteractionBalance();
if (interactionBalance - list.size() < 0){
return ResponseResult.failure("剩余可用次数不足");
}
List<ExportInteractionUpdateDTO> exportInteractionUpdateDTOList = new ArrayList<>();
Map<String, String> map = list.stream().collect(Collectors.toMap(ExportAppYuqingDTO::getUrl, ExportAppYuqingDTO::getPlatform));
// 分割map，list里每个map.size为100，目前品见最大链接限制为100，见知微tapd-wiki《互动量更新中间件使用》说明
List<Map<String, String>> mapList = Tools.splitMap(map, 100);
AtomicInteger id = new AtomicInteger(0);
for (Map<String, String> urlPlatformMap : mapList) {
List<String> taskIdList = new ArrayList<>(100);
List<String> urlList = new ArrayList<>(100);
for (Map.Entry<String, String> entry : urlPlatformMap.entrySet()) {
// 提交互动量更新任务
String type = InteractionEnum.getTypeByPlatform(entry.getValue());
List<String> taskIds = toolsetService.requestInteractionUrl(type, Collections.singletonList(entry.getKey()));
urlList.add(entry.getKey());
taskIdList.addAll(taskIds);
}
// 每100条作为一批提交完时，获取这一批的互动量任务更新结果
List<JSONObject> result = toolsetService.getInteractionResult(taskIdList);
Map<Object, JSONObject> urlMap = result.stream().collect(Collectors.toMap(jsonObject -> jsonObject.get("url"), o -> o));
List<ExportInteractionUpdateDTO> exportList = urlList.stream()
.map(url -> ExportInteractionUpdateDTO.createWithPlatform(String.valueOf(id.incrementAndGet()), urlPlatformMap, urlMap, url))
.collect(Collectors.toList());
exportInteractionUpdateDTOList.addAll(exportList);
}
// excel输出到指定路径
String projectName = projectService.getProjectById(UserThreadLocal.getProjectId()).getProjectName();
String filePath = EasyExcelUtil.generateExcelFilePath(brandkbsFilePath, projectName, UserThreadLocal.getNickname(), "舆情库互动量更新结果");
EasyExcelUtil.write(filePath, "sheet1", ExportInteractionUpdateDTO.class, exportInteractionUpdateDTOList);
JSONObject res = new JSONObject();
// 记录使用情况
extraService.decreaseInteractionRecord(InteractionUpdateRecord.UsedType.yuqing, list.size());
res.put("filePath", filePath);
res.put("remainingCount", interactionBalance - list.size());
// 记录使用
return ResponseResult.success(res);
}
@Override
public Long interactionUpdatePrediction(MarkSearchDTO dto) {
try {
defaultMarkSearch(dto);
dto.setPageSize(null);
Pair<SearchHits[], Map<String, Long>> hitsAndCounts = esSearchService.searchMarkHitsAndCount(dto, false);
return hitsAndCounts.getLeft()[0].getTotalHits().value;
}catch (Exception e){
ExceptionCast.cast(CommonCodeEnum.FAIL, "es查询异常");
}
return null;
}
private static JSONObject getTitleAndUrl(Map<String, Object> map) { private static JSONObject getTitleAndUrl(Map<String, Object> map) {
JSONObject json = new JSONObject(); JSONObject json = new JSONObject();
BaseMap baseMap = Tools.getBaseFromEsMap(map); BaseMap baseMap = Tools.getBaseFromEsMap(map);
......
src/main/java/com/zhiwei/brandkbs2/service/impl/ToolsetServiceImpl.java 0 → 100644
package com.zhiwei.brandkbs2.service.impl;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.auth.UserThreadLocal;
import com.zhiwei.brandkbs2.controller.app.AppToolsetController;
import com.zhiwei.brandkbs2.easyexcel.EasyExcelUtil;
import com.zhiwei.brandkbs2.easyexcel.config.ReadExcelDTO;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportArticleSummaryDTO;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportInteractionUpdateDTO;
import com.zhiwei.brandkbs2.easyexcel.dto.UploadArticleSummaryDTO;
import com.zhiwei.brandkbs2.easyexcel.dto.UploadInteractionUpdateDTO;
import com.zhiwei.brandkbs2.easyexcel.listener.ArticleSummaryListener;
import com.zhiwei.brandkbs2.easyexcel.listener.InteractionUpdateListener;
import com.zhiwei.brandkbs2.enmus.InteractionEnum;
import com.zhiwei.brandkbs2.exception.ExceptionCast;
import com.zhiwei.brandkbs2.model.CommonCodeEnum;
import com.zhiwei.brandkbs2.model.ResponseResult;
import com.zhiwei.brandkbs2.pojo.InteractionUpdateRecord;
import com.zhiwei.brandkbs2.pojo.Project;
import com.zhiwei.brandkbs2.service.ExtraService;
import com.zhiwei.brandkbs2.service.ProjectService;
import com.zhiwei.brandkbs2.service.ToolsetService;
import com.zhiwei.brandkbs2.util.RedisUtil;
import com.zhiwei.brandkbs2.util.Tools;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.ListUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service;
import org.springframework.web.client.RestTemplate;
import org.springframework.web.multipart.MultipartFile;
import javax.annotation.Resource;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
/**
* @ClassName: ToolsetServiceImpl
* @Description ToolsetServiceImpl
* @author: cjz
* @date: 2023-09-22 09:37
*/
@Service("toolsetServiceImpl")
public class ToolsetServiceImpl implements ToolsetService {
public static final Logger log = LogManager.getLogger(AppToolsetController.class);
@Resource(name = "redisUtil")
private RedisUtil redisUtil;
@Resource(name = "projectServiceImpl")
private ProjectService projectService;
@Autowired
private RestTemplate restTemplate;
@Value("${toolset.articleSummary.url}")
private String articleSummaryUrl;
@Value("${toolset.articleInfo.url}")
private String articleInfoUrl;
@Value("${toolset.interactionUpdate.url}")
private String interactionUpdateUrl;
@Value("${toolset.interactionResult.url}")
private String interactionResultUrl;
@Resource(name = "extraServiceImpl")
private ExtraService extraService;
@Value("${brandkbs.file.url}")
private String brandkbsFilePath;
private static final String TEXT_SUMMARY_PREVIOUS = "作为一名公关人员，将文章进行简短的中文摘要，摘要文字中需包括日期、地点等核心要素，其中日期不用包括年份，摘要起始必须为日期，字数少于150个字\n文章：\n";
private static final int ARTICLE_SUMMARY_LIMIT = 1000;
private static final String BRANDKBS_INTERACTION_SALT = "650bee0d393b382938003695";
private static final String USER = "brandkbs";
@Override
public ResponseResult getSingleArticleSummary(String url) {
JSONObject res = new JSONObject();
JSONObject info = getUrlInfo(url, UserThreadLocal.getProjectId());
if (Objects.isNull(info)){
return ResponseResult.failure("链接解析异常");
}
String text = info.getString("content");
String articleSummaryResult = getArticleSummaryResult(text);
// 剩余次数限制
String redisKey = RedisUtil.getToolsetArticleSummaryLimitKey(UserThreadLocal.getProjectId());
String redisResult = redisUtil.get(redisKey);
int usedCount = 1;
if (Objects.nonNull(redisResult)){
int redisCount = Integer.parseInt(redisResult);
if (redisCount >= ARTICLE_SUMMARY_LIMIT){
return ResponseResult.failure("本日摘要提取次数已达上限");
}
usedCount = redisCount + 1;
redisUtil.setExpire(redisKey, String.valueOf(usedCount), Tools.getMillSecondNextDay(), TimeUnit.MILLISECONDS);
}else {
redisUtil.setExpire(redisKey, String.valueOf(usedCount), Tools.getMillSecondNextDay(), TimeUnit.MILLISECONDS);
}
res.put("source", info.get("source"));
res.put("platform", info.getString("platform"));
res.put("time", info.getLong("time"));
res.put("text", articleSummaryResult);
res.put("remainingCount", ARTICLE_SUMMARY_LIMIT - usedCount);
return ResponseResult.success(res);
}
@Override
public ResponseResult getBatchArticleSummary(MultipartFile file) {
JSONObject res = new JSONObject();
Project project = projectService.getProjectById(UserThreadLocal.getProjectId());
// 调用前剩余可用次数
String redisKey = RedisUtil.getToolsetArticleSummaryLimitKey(UserThreadLocal.getProjectId());
String redisResult = redisUtil.get(redisKey);
int remainingCount = ARTICLE_SUMMARY_LIMIT;
if (Objects.nonNull(redisResult)){
remainingCount = ARTICLE_SUMMARY_LIMIT - Integer.parseInt(redisResult);
}
if (remainingCount <= 0){
return ResponseResult.failure("本日摘要提取次数已达上限");
}
// excel信息提取
Map<String, String> map = new LinkedHashMap<>();
ReadExcelDTO<UploadArticleSummaryDTO> readExcel = new ReadExcelDTO<>();
readExcel.setClazz(UploadArticleSummaryDTO.class);
readExcel.setAnalysisEventListener(new ArticleSummaryListener(map));
EasyExcelUtil.read(file, readExcel);
if (map.size() > 50){
return ResponseResult.failure("超过每次批量提取上限50");
}
int usedCount = 0; // 本次批量调用次数
List<ExportArticleSummaryDTO> datas = new ArrayList<>(50);
// 摘要提取
for (Map.Entry<String, String> entry : map.entrySet()) {
String text = entry.getValue();
String url = entry.getKey();
// 文章内容空时，通过链接提取文章内容
if (Objects.isNull(text) && Objects.nonNull(url)){
JSONObject json = getUrlInfo(entry.getKey(), UserThreadLocal.getProjectId());
text = Objects.nonNull(json) ? json.getString("content") : null;
}
String articleSummaryResult = getArticleSummaryResult(text);
usedCount = usedCount + 1;
datas.add(new ExportArticleSummaryDTO(String.valueOf(usedCount), entry.getKey(), entry.getValue(), articleSummaryResult));
}
// 本次批量调用后剩余次数
remainingCount = remainingCount - usedCount;
// 更新已用次数
redisUtil.setExpire(redisKey, String.valueOf(ARTICLE_SUMMARY_LIMIT - remainingCount), Tools.getMillSecondNextDay(), TimeUnit.MILLISECONDS);
// excel输出到指定路径
String filePath = EasyExcelUtil.generateExcelFilePath(brandkbsFilePath, project.getProjectName(), UserThreadLocal.getNickname(), "摘要提取结果");
EasyExcelUtil.write(filePath, "sheet1", ExportArticleSummaryDTO.class, datas);
res.put("filePath", filePath);
res.put("remainingCount", remainingCount);
return ResponseResult.success(res);
}
@Override
public ResponseResult getArticleSummaryRemainingCount() {
String redisKey = RedisUtil.getToolsetArticleSummaryLimitKey(UserThreadLocal.getProjectId());
String redisResult = redisUtil.get(redisKey);
if (Objects.nonNull(redisResult)){
return ResponseResult.success(ARTICLE_SUMMARY_LIMIT - Integer.parseInt(redisResult));
}
return ResponseResult.success(ARTICLE_SUMMARY_LIMIT);
}
@Override
public ResponseResult urlInteractionUpdate(MultipartFile file) {
List<ExportInteractionUpdateDTO> exportInteractionUpdateDTOList = new ArrayList<>();
// excel信息提取
List<String> urls = new ArrayList<>();
ReadExcelDTO<UploadInteractionUpdateDTO> readExcel = new ReadExcelDTO<>();
readExcel.setClazz(UploadInteractionUpdateDTO.class);
readExcel.setAnalysisEventListener(new InteractionUpdateListener(urls));
EasyExcelUtil.read(file, readExcel);
if (urls.size() > 300){
return ResponseResult.failure("超过每次更新链接数量上限300");
}
// 互动量更新剩余可用次数
int interactionBalance = projectService.getProjectById(UserThreadLocal.getProjectId()).getInteractionBalance();
if (interactionBalance - urls.size() < 0){
return ResponseResult.failure("剩余可用次数不足");
}
// 通过url获取域名进而获取任务类型
Map<String, String> map = new LinkedHashMap<>();
urls.forEach(url -> {
String domain = Tools.topDomainOfDomain(url);
String type = InteractionEnum.getTypeByDomain(domain);
map.put(url, type);
});
// 分割map，list里每个map.size为100，目前品见最大链接限制为100，见知微tapd-wiki《互动量更新中间件使用》说明
List<Map<String, String>> mapList = Tools.splitMap(map, 100);
AtomicInteger id = new AtomicInteger(0);
for (Map<String, String> urlTypeMap : mapList) {
List<String> taskIdList = new ArrayList<>(100);
List<String> urlList = new ArrayList<>(100);
for (Map.Entry<String, String> entry : urlTypeMap.entrySet()) {
// 提交互动量更新任务
List<String> taskIds = requestInteractionUrl(entry.getValue(), Collections.singletonList(entry.getKey()));
urlList.add(entry.getKey());
taskIdList.addAll(taskIds);
}
// 每100条作为一批提交完时，获取互动量任务更新结果
List<JSONObject> result = getInteractionResult(taskIdList);
Map<Object, JSONObject> urlMap = result.stream().collect(Collectors.toMap(jsonObject -> jsonObject.get("url"), o -> o));
List<ExportInteractionUpdateDTO> exportList =
urlList.stream().map(url -> new ExportInteractionUpdateDTO(String.valueOf(id.incrementAndGet()), urlMap, url)).collect(Collectors.toList());
exportInteractionUpdateDTOList.addAll(exportList);
}
// excel输出到指定路径
String projectName = projectService.getProjectById(UserThreadLocal.getProjectId()).getProjectName();
String filePath = EasyExcelUtil.generateExcelFilePath(brandkbsFilePath, projectName, UserThreadLocal.getNickname(), "链接互动量更新结果");
EasyExcelUtil.write(filePath, "sheet1", ExportInteractionUpdateDTO.class, exportInteractionUpdateDTOList);
JSONObject res = new JSONObject();
// 记录使用情况
extraService.decreaseInteractionRecord(InteractionUpdateRecord.UsedType.url, urls.size());
res.put("filePath", filePath);
res.put("remainingCount", interactionBalance - urls.size());
return ResponseResult.success(res);
}
@Override
public int getInteractionRemainingCount() {
return projectService.getProjectById(UserThreadLocal.getProjectId()).getInteractionBalance();
}
@Override
public List<String> requestInteractionUrl(String type, List<String> urls) {
List<String> taskIds = new ArrayList<>(2);
// 请求头参数
HttpHeaders headers = getInteractionRequestHeader();
// 请求参数，一个任务最多只支持50条链接，见知微tapd-wiki《互动量更新中间件使用》说明
List<List<String>> partition = ListUtils.partition(urls, 50);
for (List<String> urlList : partition) {
JSONObject requestJson = new JSONObject();
requestJson.put("type", type);
requestJson.put("urlList", urlList);
HttpEntity<String> request = new HttpEntity<>(requestJson.toJSONString(), headers);
JSONObject body = restTemplate.postForEntity(interactionUpdateUrl, request, JSONObject.class).getBody();
if (Objects.nonNull(body) && body.getBoolean("success")){
taskIds.add(body.getString("data"));
}
}
return taskIds;
}
@Override
public List<JSONObject> getInteractionResult(List<String> taskIds) {
HttpHeaders headers = getInteractionRequestHeader();
// 请求参数
JSONObject requestJson = new JSONObject();
requestJson.put("uuidList", taskIds);
HttpEntity<String> request = new HttpEntity<>(requestJson.toJSONString(), headers);
List<JSONObject> res = new ArrayList<>();
// 轮询任务结果
for (int i = 0; i < 30; i++) {
try {
ResponseEntity<JSONObject> response = restTemplate.postForEntity(interactionResultUrl, request, JSONObject.class);
JSONObject body = response.getBody();
if (Objects.nonNull(body) && 200 == body.getIntValue("code") && !body.isEmpty() && !body.getJSONArray("data").toJavaList(JSONObject.class).isEmpty() &&
body.getJSONArray("data").toJavaList(JSONObject.class).stream().map(m -> m.getIntValue("code")).allMatch(code -> 200 == code)) {
res = body.getJSONArray("data").toJavaList(JSONObject.class);
break;
} else {
Thread.sleep(10000L);
}
}catch (Exception e){
ExceptionCast.cast(CommonCodeEnum.FAIL, "轮询互动量更新任务异常,taskId:" + taskIds, e);
}
}
return res;
}
private HttpHeaders getInteractionRequestHeader(){
// 请求头参数
Long timeStamp = System.currentTimeMillis();
HttpHeaders headers = new HttpHeaders();
headers.add("user", USER);
headers.add("token", DigestUtils.md5Hex(timeStamp + "_" + BRANDKBS_INTERACTION_SALT));
headers.add("timestamp", String.valueOf(timeStamp));
headers.setContentType(MediaType.APPLICATION_JSON);
return headers;
}
/**
* 链接信息提取
* @param url 链接
* @param projectId 项目id
* @return
*/
private JSONObject getUrlInfo(String url, String projectId) {
JSONObject jsonObject;
// 即使抛出异常(大概率会是timeout)，也要保证批量时其他链接正常执行
try {
String linkedGroupId = projectService.getProjectVOById(projectId).getBrandLinkedGroupId();
jsonObject = restTemplate.getForEntity(articleInfoUrl, JSONObject.class, url, linkedGroupId, UserThreadLocal.getNickname()).getBody();
} catch (Exception e) {
log.info("url:{},访问链接信息提取接口异常-", url, e);
return null;
}
if (Objects.isNull(jsonObject) || !jsonObject.getBoolean("status")) {
return null;
}
return jsonObject.getJSONObject("data");
}
/**
* 获取摘要提取结果
* @param text 文本
* @return
*/
private String getArticleSummaryResult(String text){
// 即使抛出异常(大概率会是timeout)，也要保证批量时其他链接正常执行
String errorString = "访问超时,请稍后重试此条数据";
if (Objects.isNull(text)){
return errorString;
}
try {
// 拼接提示词模板
String resultText = TEXT_SUMMARY_PREVIOUS + StringUtils.substring(text, 0, 5000);
// 请求参数 请求头
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.TEXT_PLAIN);
HttpEntity<String> request = new HttpEntity<>(resultText, headers);
ResponseEntity<String> response = restTemplate.postForEntity(articleSummaryUrl, request, String.class);
return response.getBody();
}catch (Exception e){
log.info("访问摘要提取接口异常-", e);
return errorString;
}
}
}
src/main/java/com/zhiwei/brandkbs2/util/Tools.java
...@@ -621,6 +621,25 @@ public class Tools { ...@@ -621,6 +621,25 @@ public class Tools {
return result; return result;
} }
public static List<Map<String, String>> splitMap(Map<String, String> map, int chunkSize) {
List<Map<String, String>> result = new ArrayList<>();
int count = 0;
Map<String, String> chunk = new LinkedHashMap<>();
for (Map.Entry<String, String> entry : map.entrySet()) {
chunk.put(entry.getKey(), entry.getValue());
count++;
if (count == chunkSize) {
result.add(chunk);
chunk = new LinkedHashMap<>();
count = 0;
}
}
if (!chunk.isEmpty()) {
result.add(chunk);
}
return result;
}
public static Long[] formatTimeRange(Long startTime, Long endTime) { public static Long[] formatTimeRange(Long startTime, Long endTime) {
if (Objects.isNull(startTime) || Objects.isNull(endTime)) { if (Objects.isNull(startTime) || Objects.isNull(endTime)) {
endTime = DateUtils.addDays(Tools.truncDate(new Date(), DAY_PATTERN), 1).getTime(); endTime = DateUtils.addDays(Tools.truncDate(new Date(), DAY_PATTERN), 1).getTime();
......
src/main/resources/application-dev.properties
...@@ -120,4 +120,6 @@ wx.accesstoken.url=https://ef.zhiweidata.com/smallprogram/api/codeToken/getToken ...@@ -120,4 +120,6 @@ wx.accesstoken.url=https://ef.zhiweidata.com/smallprogram/api/codeToken/getToken
wx.getuserphonenumber=https://api.weixin.qq.com/wxa/business/getuserphonenumber?access_token= wx.getuserphonenumber=https://api.weixin.qq.com/wxa/business/getuserphonenumber?access_token=
#\u5DE5\u5177\u5E93\u76F8\u5173\u5916\u90E8\u63A5\u53E3 #\u5DE5\u5177\u5E93\u76F8\u5173\u5916\u90E8\u63A5\u53E3
toolset.articleSummary.url=https://zhiweidata.xyz/api/front/chat-swagger toolset.articleSummary.url=https://zhiweidata.xyz/api/front/chat-swagger
toolset.articleInfo.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/match?url={1}&projectId={2}&submitter={3} toolset.articleInfo.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/match?url={1}&projectId={2}&submitter={3}
\ No newline at end of file toolset.interactionUpdate.url=https://zhiwei-middleware.zhiweidata.com/interact/insertData
toolset.interactionResult.url=https://zhiwei-middleware.zhiweidata.com/interact/findDataBatch
\ No newline at end of file
src/main/resources/application-local.properties
...@@ -55,7 +55,7 @@ channel.index.application.name=brandkbs2 ...@@ -55,7 +55,7 @@ channel.index.application.name=brandkbs2
hqd.groupAll.url= https://sensitive.zhiweidata.com/sensitive/planA/groupAll hqd.groupAll.url= https://sensitive.zhiweidata.com/sensitive/planA/groupAll
#\u6807\u6CE8\u4E2D\u95F4\u4EF6 #\u6807\u6CE8\u4E2D\u95F4\u4EF6
mark.registry.address=zookeeper://192.168.0.11:2181?backup=192.168.0.30:2181,192.168.0.35:2181 mark.registry.address=zookeeper://192.168.0.11:2181?backup=192.168.0.30:2181,192.168.0.35:2181
mark.provider.group=zhiwei-mark-test_liuyu mark.provider.group=zhiwei-mark-local-liuyu
#\u4E8B\u4EF6\u4E2D\u95F4\u4EF6 #\u4E8B\u4EF6\u4E2D\u95F4\u4EF6
event.registry.address=zookeeper://192.168.0.11:2181?backup=192.168.0.30:2181,192.168.0.35:2181 event.registry.address=zookeeper://192.168.0.11:2181?backup=192.168.0.30:2181,192.168.0.35:2181
event.provider.group=zhiwei-event-ygd event.provider.group=zhiwei-event-ygd
...@@ -124,4 +124,6 @@ wx.accesstoken.url=https://ef.zhiweidata.com/smallprogram/api/codeToken/getToken ...@@ -124,4 +124,6 @@ wx.accesstoken.url=https://ef.zhiweidata.com/smallprogram/api/codeToken/getToken
wx.getuserphonenumber=https://api.weixin.qq.com/wxa/business/getuserphonenumber?access_token= wx.getuserphonenumber=https://api.weixin.qq.com/wxa/business/getuserphonenumber?access_token=
#\u5DE5\u5177\u5E93\u76F8\u5173\u5916\u90E8\u63A5\u53E3 #\u5DE5\u5177\u5E93\u76F8\u5173\u5916\u90E8\u63A5\u53E3
toolset.articleSummary.url=https://zhiweidata.xyz/api/front/chat-swagger toolset.articleSummary.url=https://zhiweidata.xyz/api/front/chat-swagger
toolset.articleInfo.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/match?url={1}&projectId={2}&submitter={3} toolset.articleInfo.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/match?url={1}&projectId={2}&submitter={3}
\ No newline at end of file toolset.interactionUpdate.url=https://zhiwei-middleware.zhiweidata.com/interact/insertData
toolset.interactionResult.url=https://zhiwei-middleware.zhiweidata.com/interact/findDataBatch
\ No newline at end of file
src/main/resources/application-prod.properties
...@@ -120,4 +120,6 @@ wx.accesstoken.url=https://ef.zhiweidata.com/smallprogram/api/codeToken/getToken ...@@ -120,4 +120,6 @@ wx.accesstoken.url=https://ef.zhiweidata.com/smallprogram/api/codeToken/getToken
wx.getuserphonenumber=https://api.weixin.qq.com/wxa/business/getuserphonenumber?access_token= wx.getuserphonenumber=https://api.weixin.qq.com/wxa/business/getuserphonenumber?access_token=
#\u5DE5\u5177\u5E93\u76F8\u5173\u5916\u90E8\u63A5\u53E3 #\u5DE5\u5177\u5E93\u76F8\u5173\u5916\u90E8\u63A5\u53E3
toolset.articleSummary.url=https://zhiweidata.xyz/api/front/chat-swagger toolset.articleSummary.url=https://zhiweidata.xyz/api/front/chat-swagger
toolset.articleInfo.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/match?url={1}&projectId={2}&submitter={3} toolset.articleInfo.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/match?url={1}&projectId={2}&submitter={3}
\ No newline at end of file toolset.interactionUpdate.url=https://zhiwei-middleware.zhiweidata.com/interact/insertData
toolset.interactionResult.url=https://zhiwei-middleware.zhiweidata.com/interact/findDataBatch
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment