Skip to content

B
brandkbs2
Commit 506f4ae3 by shentao
Options

Merge branch 'feature' into 'release' 

2023/10/10 互动量更新调整、全网搜-实时采集筛选条件调整、摘要提取次数修复、摘要批量上传监听类提交

See merge request !400
parents 2482e874 810414be
Showing with 1334 additions and 233 deletions
src/main/java/com/zhiwei/brandkbs2/aop/AopDownloadTask.java
......@@ -19,7 +19,8 @@ import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import java.lang.reflect.Method;
import java.util.Objects;
import java.util.Arrays;
import java.util.List;
/**
* @author cjz
......@@ -36,7 +37,9 @@ public class AopDownloadTask {
@Resource(name = "downloadTaskServiceImpl")
DownloadTaskService downloadTaskService;
@Around(value = "execution(public * com..controller..app..AppDownloadController.*(..)) || execution(public * com..controller..app..AppToolsetController.getBatchArticleSummary(..)))")
private static final List<String> METHOD = Arrays.asList("downloadBatchArticleSummary", "downloadUrlInteractionUpdate", "getMarkInteractionUpdate");
@Around(value = "execution(public * com..controller..app..AppDownloadController.*(..)) || execution(* com..controller..app..AppToolsetController.download*(..)) || execution(* com..controller..app..AppArticleController.getMarkInteractionUpdate(..))")
public Object around(ProceedingJoinPoint joinPoint) throws Throwable{
Signature signature = joinPoint.getSignature();
Method method = ((MethodSignature) signature).getMethod();
......@@ -55,7 +58,7 @@ public class AopDownloadTask {
ExceptionCast.cast(CommonCodeEnum.FAIL, "下载异常", e);
}
// 更新下载任务
if (Objects.equals(method.getName(), "getBatchArticleSummary")){
if (METHOD.contains(method.getName())){
fileAddress = JSONObject.parseObject(((ResponseResult) proceed).getData().toString()).getString("filePath");
}else {
fileAddress = ((ResponseResult) proceed).getData().toString();
......
src/main/java/com/zhiwei/brandkbs2/controller/admin/ExtraController.java
......@@ -8,7 +8,7 @@ import com.zhiwei.brandkbs2.easyexcel.dto.ExportLineDTO;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportWholeSearchRecordDTO;
import com.zhiwei.brandkbs2.enmus.RoleEnum;
import com.zhiwei.brandkbs2.model.ResponseResult;
import com.zhiwei.brandkbs2.service.WholeSearchService;
import com.zhiwei.brandkbs2.service.ExtraService;
import io.swagger.annotations.*;
import org.springframework.web.bind.annotation.*;
......@@ -28,8 +28,8 @@ import java.util.stream.Collectors;
@Auth(role = RoleEnum.COMMON_ADMIN)
public class ExtraController extends BaseController {
@Resource(name = "wholeSearchServiceImpl")
private WholeSearchService wholeSearchService;
@Resource(name = "extraServiceImpl")
private ExtraService extraService;
@ApiOperation("全网搜-使用记录-查询")
@ApiImplicitParams({@ApiImplicitParam(name = "page", value = "页码", defaultValue = "1", paramType = "query", dataType = "int"),
......@@ -40,7 +40,7 @@ public class ExtraController extends BaseController {
public ResponseResult findUseList(@RequestParam(value = "page", defaultValue = "1") int page,
@RequestParam(value = "size", defaultValue = "5") int pageSize,
@RequestParam(value = "sorter", defaultValue = "{\"usedCount\":\"ascend\"}") String sorter) {
return ResponseResult.success(wholeSearchService.findUsedList(page, pageSize, sorter));
return ResponseResult.success(extraService.findUsedList(page, pageSize, sorter));
}
@ApiOperation("全网搜-维护记录-查询")
......@@ -49,13 +49,13 @@ public class ExtraController extends BaseController {
@GetMapping("/maintain/list")
public ResponseResult findMaintainList(@RequestParam(value = "page", defaultValue = "1") int page,
@RequestParam(value = "size", defaultValue = "5") int pageSize) {
return ResponseResult.success(wholeSearchService.findMaintainList(page, pageSize));
return ResponseResult.success(extraService.findMaintainList(page, pageSize));
}
@ApiOperation("全网搜-数据总览")
@GetMapping("/overview")
public ResponseResult overview() {
return ResponseResult.success(wholeSearchService.overview());
return ResponseResult.success(extraService.overview());
}
......@@ -67,7 +67,7 @@ public class ExtraController extends BaseController {
Long startTime = json.getLong("startTime");
Long endTime = json.getLong("endTime");
boolean day = json.getBooleanValue("day");
List<JSONObject> collect = wholeSearchService.outputUsedList(personal, startTime, endTime, day);
List<JSONObject> collect = extraService.outputUsedList(personal, startTime, endTime, day);
String fileName = startTime + "_" + endTime + "使用记录";
if (personal) {
EasyExcelUtil.download(fileName, "sheet", ExportWholeSearchRecordDTO.class, collect.stream().map(ExportWholeSearchRecordDTO::createFromJSONObject).collect(Collectors.toList()), response);
......@@ -81,8 +81,42 @@ public class ExtraController extends BaseController {
@PutMapping("/maintain/adjust")
@Auth(role = RoleEnum.SUPER_ADMIN)
public ResponseResult adjustRecord(@ApiParam(name = "json:{count:调整数值)") @RequestBody JSONObject json) {
wholeSearchService.adjustRecord(json.getInteger("count"));
extraService.adjustRecord(json.getInteger("count"));
return ResponseResult.success();
}
@ApiOperation("互动量更新-使用记录-查询")
@ApiImplicitParams({@ApiImplicitParam(name = "page", value = "页码", defaultValue = "1", paramType = "query", dataType = "int"),
@ApiImplicitParam(name = "pageSize", value = "每页记录数", defaultValue = "5", paramType = "query", dataType = "int"),
@ApiImplicitParam(name = "sorter", defaultValue = "{\"usedCount\":\"ascend\"}", value = "排序字段", paramType = "query", dataType = "string")}
)
@GetMapping("/interaction/used/list")
public ResponseResult findInteractionUsedList(@RequestParam(value = "page", defaultValue = "1") int page,
@RequestParam(value = "size", defaultValue = "5") int pageSize,
@RequestParam(value = "sorter", defaultValue = "{\"usedCount\":\"ascend\"}") String sorter) {
return ResponseResult.success(extraService.findInteractionUsedList(page, pageSize, sorter));
}
@ApiOperation("互动量更新-维护记录-查询")
@ApiImplicitParams({@ApiImplicitParam(name = "page", value = "页码", defaultValue = "1", paramType = "query", dataType = "int"),
@ApiImplicitParam(name = "pageSize", value = "每页记录数", defaultValue = "5", paramType = "query", dataType = "int")})
@GetMapping("/interaction/maintain/list")
public ResponseResult findInteractionMaintainList(@RequestParam(value = "page", defaultValue = "1") int page,
@RequestParam(value = "size", defaultValue = "5") int pageSize) {
return ResponseResult.success(extraService.findInteractionMaintainList(page, pageSize));
}
@ApiOperation("互动量更新-数据总览")
@GetMapping("/interaction/overview")
public ResponseResult interactionOverview() {
return ResponseResult.success(extraService.interactionOverview());
}
@ApiOperation("互动量更新-维护记录-调整")
@PutMapping("/interaction/maintain/adjust")
@Auth(role = RoleEnum.SUPER_ADMIN)
public ResponseResult adjustInteractionRecord(@ApiParam(name = "json:{count:调整数值)") @RequestBody JSONObject json) {
extraService.adjustInteractionRecord(json.getInteger("count"));
return ResponseResult.success();
}
}
src/main/java/com/zhiwei/brandkbs2/controller/app/AppArticleController.java
......@@ -306,6 +306,19 @@ public class AppArticleController extends BaseController {
return reportService.getReportSchedule(idList);
}
@ApiOperation("有效舆情互动量更新")
@PostMapping("/mark/interaction-update")
@LogRecord(description = "舆情库-有效舆情互动量更新")
public ResponseResult getMarkInteractionUpdate(@RequestBody MarkSearchDTO markSearchDTO){
return markDataService.markInteractionUpdate(markSearchDTO);
}
@ApiOperation("有效舆情互动量更新-数据量预估")
@PostMapping("/mark/interaction-prediction")
public ResponseResult getMarkInteractionPrediction(@RequestBody MarkSearchDTO markSearchDTO){
return ResponseResult.success(markDataService.interactionUpdatePrediction(markSearchDTO));
}
private boolean checkMTagIllegal(StringBuilder mtag) {
List<MarkerTag> hitTags = projectService.getProjectById(UserThreadLocal.getProjectId()).getHitTags();
if (!Tools.isEmpty(hitTags)) {
......
src/main/java/com/zhiwei/brandkbs2/controller/app/AppDownloadController.java
......@@ -58,8 +58,8 @@ public class AppDownloadController extends BaseController {
@Resource(name = "channelServiceImpl")
ChannelService channelService;
@Resource(name = "wholeSearchServiceImpl")
private WholeSearchService wholeSearchService;
@Resource(name = "extraServiceImpl")
private ExtraService extraService;
@Resource(name = "markDataServiceImpl")
MarkDataService markDataService;
......@@ -247,10 +247,10 @@ public class AppDownloadController extends BaseController {
Long startTime = json.getLong("startTime");
Long endTime = json.getLong("endTime");
boolean day = json.getBooleanValue("day");
List<JSONObject> collect = wholeSearchService.outputUsedList(personal, startTime, endTime, day);
List<JSONObject> collect = extraService.outputUsedList(personal, startTime, endTime, day);
// excel写入至指定路径
String projectName = projectService.getProjectById(UserThreadLocal.getProjectId()).getProjectName();
String filePath = EasyExcelUtil.generateExcelFilePath(brandkbsFilePath, projectName, UserThreadLocal.getNickname(), startTime + "_" + endTime + "使用记录");
String filePath = EasyExcelUtil.generateExcelFilePath(brandkbsFilePath, projectName, UserThreadLocal.getNickname(), "全网搜使用记录");
if (personal) {
List<ExportWholeSearchRecordDTO> list = collect.stream().map(ExportWholeSearchRecordDTO::createFromJSONObject).collect(Collectors.toList());
EasyExcelUtil.write(filePath, "sheet1", ExportWholeSearchRecordDTO.class, list);
......@@ -261,6 +261,29 @@ public class AppDownloadController extends BaseController {
return ResponseResult.success(filePath);
}
@ApiOperation("互动量更新使用记录")
@PostMapping("/back/interaction/used")
@Auth(role = RoleEnum.ADMIN)
public ResponseResult outputInteractionUsedList(@ApiParam(name = "json:{personal:个人明细=true,startTime:起始时间,endTime:结束时间,day:颗粒度天级=true}")
@RequestBody JSONObject json) {
boolean personal = json.getBooleanValue("personal");
Long startTime = json.getLong("startTime");
Long endTime = json.getLong("endTime");
boolean day = json.getBooleanValue("day");
List<JSONObject> collect = extraService.outputInteractionUsedList(personal, startTime, endTime, day);
// excel写入至指定路径
String projectName = projectService.getProjectById(UserThreadLocal.getProjectId()).getProjectName();
String filePath = EasyExcelUtil.generateExcelFilePath(brandkbsFilePath, projectName, UserThreadLocal.getNickname(), "互动量更新使用记录");
if (personal) {
List<ExportInteractionUpdateRecordDTO> list = collect.stream().map(ExportInteractionUpdateRecordDTO::createFromJSONObject).collect(Collectors.toList());
EasyExcelUtil.write(filePath, "sheet", ExportInteractionUpdateRecordDTO.class, list);
} else {
List<ExportLineDTO> list = collect.stream().map(ExportLineDTO::createFromJSONObject).collect(Collectors.toList());
EasyExcelUtil.write(filePath, "sheet", ExportLineDTO.class, list);
}
return ResponseResult.success();
}
@ApiOperation("项目关键词")
@ApiImplicitParam(name = "pid", value = "项目ID", required = true, paramType = "path", dataType = "string")
@GetMapping("/back/project/keyword/{pid}")
......@@ -374,7 +397,7 @@ public class AppDownloadController extends BaseController {
String filePath = EasyExcelUtil.generateExcelFilePath(brandkbsFilePath, projectName, UserThreadLocal.getNickname(), "全网搜舆情列表数据");
EasyExcelUtil.write(filePath, "sheet1", ExportSearchWholeDTO.class, exportList);
if (dto.isExternalDataSource()) {
wholeSearchService.decreaseRecord(dto.getSearch(), WholeSearchRecord.UsedType.output, exportList.size());
extraService.decreaseRecord(dto.getSearch(), WholeSearchRecord.UsedType.output, exportList.size());
}
return ResponseResult.success(filePath);
}
......@@ -393,6 +416,19 @@ public class AppDownloadController extends BaseController {
return ResponseResult.success(filePath);
}
@ApiOperation("链接互动量更新模板")
@GetMapping(value = "/interaction-update/template")
public ResponseResult downloadInteractionUpdateTemplate() {
List<List<String>> head = new ArrayList<>();
head.add(Collections.singletonList("序号"));
head.add(Collections.singletonList("链接"));
// excel写入至指定路径
String projectName = projectService.getProjectById(UserThreadLocal.getProjectId()).getProjectName();
String filePath = EasyExcelUtil.generateExcelFilePath(brandkbsFilePath, projectName, UserThreadLocal.getNickname(), "链接互动量更新模板");
EasyExcelUtil.dynamicHeadWrite(filePath, "模板", head, Collections.emptyList());
return ResponseResult.success(filePath);
}
private HttpHeaders getHeaders() {
HttpHeaders httpHeaders = new HttpHeaders();
httpHeaders.set("token", token);
......
src/main/java/com/zhiwei/brandkbs2/controller/app/AppSearchController.java
......@@ -78,8 +78,8 @@ public class AppSearchController extends BaseController {
@Resource(name = "projectServiceImpl")
ProjectService projectService;
@Resource(name = "wholeSearchServiceImpl")
WholeSearchService wholeSearchService;
@Resource(name = "extraServiceImpl")
ExtraService extraService;
@ApiOperation("搜索-查热点")
@LogRecord(values = "keyword", description = "查热点", arguments = true, entity = false)
......@@ -133,9 +133,8 @@ public class AppSearchController extends BaseController {
@ApiOperation("搜索-全网搜条件")
@GetMapping("/getWholeSearchCriteria")
public ResponseResult getWholeSearchCriteria() {
JSONObject wholeSearchCriteria = markDataService.getWholeNetworkSearchCriteria();
wholeSearchCriteria.putAll(systemInfoService.getExtraParam());
public ResponseResult getWholeSearchCriteria(@RequestParam(value = "externalDataSource", defaultValue = "false") boolean externalDataSource) {
JSONObject wholeSearchCriteria = markDataService.getWholeNetworkSearchCriteria(externalDataSource);
return ResponseResult.success(wholeSearchCriteria);
}
......@@ -164,7 +163,7 @@ public class AppSearchController extends BaseController {
List<JSONObject> list = pair.getLeft().getJSONArray("list").toJavaList(JSONObject.class);
commonService.sentimentAnalysis(list);
}
wholeSearchService.decreaseRecord(dto.getSearch(), WholeSearchRecord.UsedType.query, pair.getRight());
extraService.decreaseRecord(dto.getSearch(), WholeSearchRecord.UsedType.query, pair.getRight());
return ResponseResult.success(pair.getLeft());
}
......@@ -191,7 +190,7 @@ public class AppSearchController extends BaseController {
List<ExportSearchWholeDTO> exportList = markDataService.exportSearchWhole(dto);
EasyExcelUtil.download("全网搜舆情列表数据", "sheet1", ExportSearchWholeDTO.class, exportList, response);
if (dto.isExternalDataSource()) {
wholeSearchService.decreaseRecord(dto.getSearch(), WholeSearchRecord.UsedType.output, exportList.size());
extraService.decreaseRecord(dto.getSearch(), WholeSearchRecord.UsedType.output, exportList.size());
}
return ResponseResult.success();
}
......
src/main/java/com/zhiwei/brandkbs2/controller/app/AppToolsetController.java
package com.zhiwei.brandkbs2.controller.app;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.aop.LogRecord;
import com.zhiwei.brandkbs2.auth.Auth;
import com.zhiwei.brandkbs2.auth.UserThreadLocal;
import com.zhiwei.brandkbs2.easyexcel.EasyExcelUtil;
import com.zhiwei.brandkbs2.easyexcel.config.ReadExcelDTO;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportArticleSummaryDTO;
import com.zhiwei.brandkbs2.easyexcel.dto.UploadArticleSummaryDTO;
import com.zhiwei.brandkbs2.easyexcel.listener.ArticleSummaryListener;
import com.zhiwei.brandkbs2.enmus.RoleEnum;
import com.zhiwei.brandkbs2.model.ResponseResult;
import com.zhiwei.brandkbs2.pojo.Project;
import com.zhiwei.brandkbs2.service.ProjectService;
import com.zhiwei.brandkbs2.util.RedisUtil;
import com.zhiwei.brandkbs2.service.ToolsetService;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.client.RestTemplate;
import org.springframework.web.multipart.MultipartFile;
import javax.annotation.Resource;
import java.util.*;
/**
* @author cjz
......@@ -43,168 +23,40 @@ import java.util.*;
@Api(tags = "工具集", description = "工具集")
@Auth(role = RoleEnum.CUSTOMER)
public class AppToolsetController {
public static final Logger log = LogManager.getLogger(AppToolsetController.class);
@Resource(name = "redisUtil")
private RedisUtil redisUtil;
@Resource(name = "projectServiceImpl")
private ProjectService projectService;
@Autowired
private RestTemplate restTemplate;
@Value("${toolset.articleSummary.url}")
private String articleSummaryUrl;
@Value("${toolset.articleInfo.url}")
private String articleInfoUrl;
@Value("${brandkbs.file.url}")
private String brandkbsFilePath;
private static final String TEXT_SUMMARY_PREVIOUS = "作为一名公关人员，将文章进行简短的中文摘要，摘要文字中需包括日期、地点等核心要素，其中日期不用包括年份，摘要起始必须为日期，字数少于150个字\n文章：\n";
private static final int ARTICLE_SUMMARY_LIMIT = 1000;
@Resource(name = "toolsetServiceImpl")
private ToolsetService toolsetService;
@ApiOperation("摘要提取-单条")
@GetMapping("/article-summary/single")
@LogRecord(description = "工具库-摘要提取-单条")
public ResponseResult getSingleArticleSummary(@RequestParam(value = "url") String url) {
JSONObject res = new JSONObject();
JSONObject info = getUrlInfo(url, UserThreadLocal.getProjectId());
if (Objects.isNull(info)){
return ResponseResult.failure("链接解析异常");
}
String text = info.getString("content");
String articleSummaryResult = getArticleSummaryResult(text);
// 剩余次数限制
String redisKey = RedisUtil.getToolsetArticleSummaryLimitKey(UserThreadLocal.getProjectId(), UserThreadLocal.getUserId());
String redisResult = redisUtil.get(redisKey);
int usedCount = 1;
if (Objects.nonNull(redisResult)){
int redisCount = Integer.parseInt(redisResult);
if (redisCount >= ARTICLE_SUMMARY_LIMIT){
return ResponseResult.failure("本日摘要提取次数已达上限");
}
usedCount = redisCount + 1;
redisUtil.getAndSet(redisKey, String.valueOf(usedCount));
}else {
redisUtil.setExpire(redisKey, String.valueOf(usedCount));
}
res.put("source", info.get("source"));
res.put("platform", info.getString("platform"));
res.put("time", info.getLong("time"));
res.put("text", articleSummaryResult);
res.put("remainingCount", ARTICLE_SUMMARY_LIMIT - usedCount);
return ResponseResult.success(res);
return toolsetService.getSingleArticleSummary(url);
}
@ApiOperation("摘要提取")
@PostMapping("/article-summary/batch")
@LogRecord(description = "工具库-摘要提取-批量")
public ResponseResult getBatchArticleSummary(@RequestParam(value = "file") MultipartFile file){
JSONObject res = new JSONObject();
Project project = projectService.getProjectById(UserThreadLocal.getProjectId());
// 调用前剩余可用次数
String redisKey = RedisUtil.getToolsetArticleSummaryLimitKey(UserThreadLocal.getProjectId(), UserThreadLocal.getUserId());
String redisResult = redisUtil.get(redisKey);
int remainingCount = ARTICLE_SUMMARY_LIMIT;
if (Objects.nonNull(redisResult)){
remainingCount = ARTICLE_SUMMARY_LIMIT - Integer.parseInt(redisResult);
}
if (remainingCount <= 0){
return ResponseResult.failure("本日摘要提取次数已达上限");
}
// excel信息提取
Map<String, String> map = new LinkedHashMap<>(50);
ReadExcelDTO<UploadArticleSummaryDTO> readExcel = new ReadExcelDTO<>();
readExcel.setClazz(UploadArticleSummaryDTO.class);
readExcel.setAnalysisEventListener(new ArticleSummaryListener(map, remainingCount));
EasyExcelUtil.read(file, readExcel);
int usedCount = 0; // 本次批量调用次数
List<ExportArticleSummaryDTO> datas = new ArrayList<>(50);
// 摘要提取
for (Map.Entry<String, String> entry : map.entrySet()) {
String text = entry.getValue();
String url = entry.getKey();
// 文章内容空时，通过链接提取文章内容
if (Objects.isNull(text) && Objects.nonNull(url)){
JSONObject json = getUrlInfo(entry.getKey(), UserThreadLocal.getProjectId());
text = Objects.nonNull(json) ? json.getString("content") : null;
}
String articleSummaryResult = getArticleSummaryResult(text);
usedCount = usedCount + 1;
datas.add(new ExportArticleSummaryDTO(String.valueOf(usedCount), entry.getKey(), entry.getValue(), articleSummaryResult));
}
// 本次批量调用后剩余次数
remainingCount = remainingCount - usedCount;
// 更新已用次数
redisUtil.getAndSet(redisKey, String.valueOf(ARTICLE_SUMMARY_LIMIT - remainingCount));
// excel输出到指定路径
String filePath = EasyExcelUtil.generateExcelFilePath(brandkbsFilePath, project.getProjectName(), UserThreadLocal.getNickname(), "摘要提取结果");
EasyExcelUtil.write(filePath, "sheet1", ExportArticleSummaryDTO.class, datas);
res.put("filePath", filePath);
res.put("remainingCount", remainingCount);
return ResponseResult.success(res);
public ResponseResult downloadBatchArticleSummary(@RequestParam(value = "file") MultipartFile file){
return toolsetService.getBatchArticleSummary(file);
}
@ApiOperation("摘要提取-剩余可用次数")
@GetMapping("/article-summary/remaining")
public ResponseResult getArticleSummaryRemainingCount(){
String redisKey = RedisUtil.getToolsetArticleSummaryLimitKey(UserThreadLocal.getProjectId(), UserThreadLocal.getUserId());
String redisResult = redisUtil.get(redisKey);
if (Objects.nonNull(redisResult)){
return ResponseResult.success(ARTICLE_SUMMARY_LIMIT - Integer.parseInt(redisResult));
}
return ResponseResult.success(ARTICLE_SUMMARY_LIMIT);
return toolsetService.getArticleSummaryRemainingCount();
}
/**
* 链接信息提取
* @param url 链接
* @param projectId 项目id
* @return
*/
private JSONObject getUrlInfo(String url, String projectId) {
JSONObject jsonObject;
// 即使抛出异常(大概率会是timeout)，也要保证批量时其他链接正常执行
try {
String linkedGroupId = projectService.getProjectVOById(projectId).getBrandLinkedGroupId();
jsonObject = restTemplate.getForEntity(articleInfoUrl, JSONObject.class, url, linkedGroupId, UserThreadLocal.getNickname()).getBody();
} catch (Exception e) {
log.info("url:{},访问链接信息提取接口异常-", url, e);
return null;
}
if (Objects.isNull(jsonObject) || !jsonObject.getBoolean("status")) {
return null;
}
return jsonObject.getJSONObject("data");
@ApiOperation("链接互动量更新")
@PostMapping("/interaction-update/url")
@LogRecord(description = "工具库-链接互动量更新")
public ResponseResult downloadUrlInteractionUpdate(@RequestParam(value = "file") MultipartFile file){
return toolsetService.urlInteractionUpdate(file);
}
/**
* 获取摘要提取结果
* @param text 文本
* @return
*/
private String getArticleSummaryResult(String text){
// 即使抛出异常(大概率会是timeout)，也要保证批量时其他链接正常执行
String errorString = "访问超时,请稍后重试此条数据";
if (Objects.isNull(text)){
return errorString;
}
try {
// 拼接提示词模板
String resultText = TEXT_SUMMARY_PREVIOUS + StringUtils.substring(text, 0, 5000);
// 请求参数 请求头
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.TEXT_PLAIN);
HttpEntity<String> request = new HttpEntity<>(resultText, headers);
ResponseEntity<String> response = restTemplate.postForEntity(articleSummaryUrl, request, String.class);
return response.getBody();
}catch (Exception e){
log.info("访问摘要提取接口异常-", e);
return errorString;
}
@ApiOperation("互动量更新-剩余可用次数")
@GetMapping("/interaction-update/remaining")
public ResponseResult getInteractionRemainingCount(){
return ResponseResult.success(toolsetService.getInteractionRemainingCount());
}
}
src/main/java/com/zhiwei/brandkbs2/dao/InteractionUpdateRecordDao.java 0 → 100644
package com.zhiwei.brandkbs2.dao;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.pojo.InteractionUpdateRecord;
import com.zhiwei.brandkbs2.pojo.vo.LineVO;
import java.util.List;
/**
* @ClassName: InteractionUpdateRecordDao
* @Description InteractionUpdateRecordDao
* @author: cjz
* @date: 2023-09-21 15:57
*/
public interface InteractionUpdateRecordDao extends BaseMongoDao<InteractionUpdateRecord>, ShardingMongo{
/**
*
* @param startTime 起始时间
* @param endTime 结束时间
* @param day 颗粒度
* @return list
*/
List<JSONObject> aggregateProjectUsedRecord(long startTime, long endTime, boolean day);
/**
* 数据总览
* @param startTime 起始时间
* @param endTime 结束时间
* @return list
*/
List<LineVO> aggregateDayLastRecord(long startTime, long endTime);
}
src/main/java/com/zhiwei/brandkbs2/dao/impl/InteractionUpdateRecordDaoImpl.java 0 → 100644
package com.zhiwei.brandkbs2.dao.impl;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.auth.UserThreadLocal;
import com.zhiwei.brandkbs2.dao.InteractionUpdateRecordDao;
import com.zhiwei.brandkbs2.pojo.InteractionUpdateRecord;
import com.zhiwei.brandkbs2.pojo.vo.LineVO;
import org.springframework.data.domain.Sort;
import org.springframework.data.mongodb.core.aggregation.Aggregation;
import org.springframework.data.mongodb.core.aggregation.AggregationOperation;
import org.springframework.data.mongodb.core.aggregation.AggregationResults;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
@Component("interactionUpdateRecordDao")
public class InteractionUpdateRecordDaoImpl extends BaseMongoDaoImpl<InteractionUpdateRecord> implements InteractionUpdateRecordDao {
private static final String COLLECTION_PREFIX = "brandkbs_interaction_update_record";
private static final String TIME_PATTERN = "yyyy";
public InteractionUpdateRecordDaoImpl() {
super(null);
}
@Override
public String collectionPrefix() {
return COLLECTION_PREFIX;
}
@Override
public String timePattern() {
return TIME_PATTERN;
}
@Override
public List<JSONObject> aggregateProjectUsedRecord(long startTime, long endTime, boolean day) {
List<JSONObject> res = new ArrayList<>();
String[] collectionNames = generateCollectionNames(new Date(startTime), new Date(endTime));
Criteria criteria = Criteria.where("isUsed").is(true).and("projectId").is(UserThreadLocal.getProjectId());
int nrOfChars = day ? 10 : 7;
List<AggregationOperation> operations = new ArrayList<>();
// operations区分先后顺序
operations.add(Aggregation.match(criteria));
operations.add(Aggregation.project( "usedCount", "cTime").and("cTime").substring(0, nrOfChars).as("cTime"));
operations.add(Aggregation.group("cTime").sum("usedCount").as("count"));
operations.add(Aggregation.sort(Sort.by(Sort.Order.desc("_id"))));
Aggregation aggregation = Aggregation.newAggregation(operations);
for (String collectionName : collectionNames) {
AggregationResults<JSONObject> aggregateResult = mongoTemplate.aggregate(aggregation, collectionName, JSONObject.class);
res.addAll(aggregateResult.getMappedResults());
}
return res;
}
@Override
public List<LineVO> aggregateDayLastRecord(long startTime, long endTime) {
List<LineVO> res = new ArrayList<>();
String[] collectionNames = generateCollectionNames(new Date(startTime), new Date(endTime));
Criteria criteria = Criteria.where("projectId").is(UserThreadLocal.getProjectId());
List<AggregationOperation> operations = new ArrayList<>();
// operations区分先后顺序
operations.add(Aggregation.match(criteria));
operations.add(Aggregation.project("currentCount").and("cTime").substring(0, 10).as("cTime"));
operations.add(Aggregation.group("cTime").last("currentCount").as("count"));
operations.add(Aggregation.sort(Sort.by(Sort.Order.asc("_id"))));
Aggregation aggregation = Aggregation.newAggregation(operations);
// 优先小的年份
for (int i = collectionNames.length - 1; i >= 0; i--) {
AggregationResults<JSONObject> aggregateResult = mongoTemplate.aggregate(aggregation, collectionNames[i], JSONObject.class);
List<JSONObject> mappedResults = aggregateResult.getMappedResults();
for (JSONObject mappedResult : mappedResults) {
res.add(new LineVO(mappedResult.getLong("count"), mappedResult.getDate("_id").getTime()));
}
}
return res;
}
}
src/main/java/com/zhiwei/brandkbs2/easyexcel/dto/ExportInteractionUpdateDTO.java 0 → 100644
package com.zhiwei.brandkbs2.easyexcel.dto;
import com.alibaba.excel.annotation.ExcelProperty;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.enmus.InteractionEnum;
import com.zhiwei.brandkbs2.util.Tools;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Objects;
/**
* @author cjz
* @version 1.0
* @description 导出互动量更新结果数据实体类
* @date 2023年9月22日 11:13:08
*/
@Data
@ToString
@NoArgsConstructor
public class ExportInteractionUpdateDTO {
@ExcelProperty("序号")
private String id;
@ExcelProperty("平台")
private String platform;
@ExcelProperty("地址")
private String url;
@ExcelProperty("更新时间")
private Date time;
@ExcelProperty("评论数")
private int commentCount;
@ExcelProperty("点赞数")
private int likeCount;
@ExcelProperty("转发数")
private int repostCount;
@ExcelProperty("阅读数")
private int readCount;
public ExportInteractionUpdateDTO(String id, Map<Object, JSONObject> urlMap, String url){
this.id = id;
this.platform = InteractionEnum.getPlatformByDomain(Tools.topDomainOfDomain(url));
this.url = url;
if (Objects.nonNull(urlMap.get(url))){
JSONObject jsonObject = urlMap.get(url);
this.commentCount = jsonObject.getIntValue("commentCount");
this.likeCount = jsonObject.getIntValue("likeCount");
this.repostCount = jsonObject.getIntValue("repostCount");
this.readCount = jsonObject.getIntValue("readCount");
}
this.time = new Date();
}
public static ExportInteractionUpdateDTO createWithPlatform(String id, Map<String, String> urlPlatformMap, Map<Object, JSONObject> urlMap, String url){
ExportInteractionUpdateDTO dto = new ExportInteractionUpdateDTO();
dto.setId(id);
dto.setPlatform(urlPlatformMap.get(url));
dto.setUrl(url);
if (Objects.nonNull(urlMap.get(url))){
JSONObject jsonObject = urlMap.get(url);
dto.setCommentCount(jsonObject.getIntValue("commentCount"));
dto.setLikeCount(jsonObject.getIntValue("likeCount"));
dto.setRepostCount(jsonObject.getIntValue("repostCount"));
dto.setReadCount(jsonObject.getIntValue("readCount"));
}
dto.setTime(new Date());
return dto;
}
}
src/main/java/com/zhiwei/brandkbs2/easyexcel/dto/ExportInteractionUpdateRecordDTO.java 0 → 100644
package com.zhiwei.brandkbs2.easyexcel.dto;
import com.alibaba.excel.annotation.ExcelProperty;
import com.alibaba.fastjson.JSONObject;
import lombok.Data;
import lombok.ToString;
import java.util.Date;
/**
* @ClassName: ExportInteractionUpdateRecordDTO
* @Description ExportInteractionUpdateRecordDTO
* @author: cjz
* @date: 2023-09-25 11:02
*/
@Data
@ToString
public class ExportInteractionUpdateRecordDTO {
@ExcelProperty("类型")
private String usedType;
@ExcelProperty("数据量")
private Integer usedCount;
@ExcelProperty("操作时间")
private Date cTime;
@ExcelProperty("操作人")
private String submitter;
public static ExportInteractionUpdateRecordDTO createFromJSONObject(JSONObject json) {
ExportInteractionUpdateRecordDTO dto = new ExportInteractionUpdateRecordDTO();
dto.setUsedType(json.getString("usedType"));
dto.setUsedCount(json.getInteger("usedCount"));
dto.setCTime(new Date(json.getLong("cTime")));
dto.setSubmitter(json.getString("submitter"));
return dto;
}
}
src/main/java/com/zhiwei/brandkbs2/easyexcel/dto/UploadArticleSummaryDTO.java
......@@ -6,7 +6,7 @@ import lombok.Data;
/**
* @author cjz
* @version 1.0
* @description 解析文章摘要提前上传文件
* @description 解析文章摘要上传文件
* @date 2023/9/14 14:22
*/
@Data
......
src/main/java/com/zhiwei/brandkbs2/easyexcel/dto/UploadInteractionUpdateDTO.java 0 → 100644
package com.zhiwei.brandkbs2.easyexcel.dto;
import com.alibaba.excel.annotation.ExcelProperty;
import lombok.Data;
/**
* @author cjz
* @version 1.0
* @description 解析互动量更新上传文件
* @date 2023/9/14 14:22
*/
@Data
public class UploadInteractionUpdateDTO {
@ExcelProperty("序号")
private String id;
@ExcelProperty("链接")
private String url;
}
src/main/java/com/zhiwei/brandkbs2/easyexcel/listener/ArticleSummaryListener.java
......@@ -8,7 +8,7 @@ import java.util.Map;
/**
* @ClassName: ArticleSummaryListener
* @Description 文章摘要提前文件批量上传监听类
* @Description 文章摘要文件批量上传监听类
* @author: cjz
* @date: 2023-09-14 14:44
*/
......@@ -16,19 +16,14 @@ public class ArticleSummaryListener extends AnalysisEventListener<UploadArticleS
private Map<String, String> map;
private int remainingCount;
public ArticleSummaryListener(Map<String, String> map, int remainingCount){
public ArticleSummaryListener(Map<String, String> map){
this.map = map;
this.remainingCount = remainingCount;
}
@Override
public void invoke(UploadArticleSummaryDTO data, AnalysisContext context) {
if (map.size() < remainingCount && map.size() <= 50) {
map.put(data.getUrl(), data.getText());
}
}
@Override
public void doAfterAllAnalysed(AnalysisContext context) {
......
src/main/java/com/zhiwei/brandkbs2/easyexcel/listener/InteractionUpdateListener.java 0 → 100644
package com.zhiwei.brandkbs2.easyexcel.listener;
import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.event.AnalysisEventListener;
import com.zhiwei.brandkbs2.easyexcel.dto.UploadInteractionUpdateDTO;
import java.util.List;
/**
* @ClassName: ArticleSummaryListener
* @Description 链接互动量更新上传监听类
* @author: cjz
* @date: 2023-09-22 13:44
*/
public class InteractionUpdateListener extends AnalysisEventListener<UploadInteractionUpdateDTO> {
private List<String> urls;
public InteractionUpdateListener(List<String> urls){
this.urls = urls;
}
@Override
public void invoke(UploadInteractionUpdateDTO data, AnalysisContext context) {
urls.add(data.getUrl());
}
@Override
public void doAfterAllAnalysed(AnalysisContext context) {
}
}
src/main/java/com/zhiwei/brandkbs2/enmus/InteractionEnum.java 0 → 100644
package com.zhiwei.brandkbs2.enmus;
import lombok.Getter;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
/**
* @ClassName: InteractionEnum
* @Description 互动量更新枚举类
* @author: cjz
* @date: 2023-09-21 17:56
*/
public enum InteractionEnum {
WEIBO("微博", "微博互动量", Arrays.asList("www.weibo.com", "weibo.com", "video.weibo.com")),
WECHAT("微信", "微信互动量", Arrays.asList("mp.weixin.qq.com", "qq.com")),
WEB_MEDIA("网媒", "链接信息更新", null),
PING_MEDIA("平媒", "链接信息更新", null),
TOUTIAO("今日头条", "链接信息更新", Arrays.asList("toutiao.com", "www.toutiao.com")),
TIKTOK("抖音", "抖音互动量", Arrays.asList("douyin.com", "iesdouyin.com", "vdouyin.com", "v.vdouyin.com")),
XIAOHONGSHU("小红书", "小红书互动量", Collections.singletonList("xiaohongshu.com")),
SELF_MEDIA("自媒体", "链接信息更新", null),
ZHIHU("知乎", "知乎互动量", Arrays.asList("www.zhihu.com", "zhihu.com")),
MAIMAI("脉脉", "链接信息更新", null),
QA("问答", "链接信息更新", null),
TIABA("贴吧论坛", "链接信息更新", null),
VIDEO("视频", "链接信息更新", null),
SHORT_VIDEO("短视频", "链接信息更新", null);
@Getter
private final String platform;
@Getter
private final String type;
@Getter
private final List<String> topDomain;
InteractionEnum(String platform, String type, List<String> topDomain){
this.platform = platform;
this.type = type;
this.topDomain = topDomain;
}
public static String getTypeByPlatform(String platform){
if (Objects.equals(platform, "其他")){
return "链接信息更新";
}
for (InteractionEnum value : InteractionEnum.values()) {
if (Objects.equals(value.getPlatform(), platform)){
return value.getType();
}
}
return "链接信息更新";
}
public static String getTypeByDomain(String topDomain){
if (Objects.isNull(topDomain)) {
return "链接信息更新";
}
for (InteractionEnum value : InteractionEnum.values()) {
if (Objects.nonNull(value.getTopDomain()) && value.getTopDomain().contains(topDomain)){
return value.getType();
}
}
return "链接信息更新";
}
public static String getPlatformByDomain(String topDomain){
if (Objects.isNull(topDomain)) {
return "其他";
}
for (InteractionEnum value : InteractionEnum.values()) {
if (Objects.nonNull(value.getTopDomain()) && value.getTopDomain().contains(topDomain)){
return value.getPlatform();
}
}
return "其他";
}
}
src/main/java/com/zhiwei/brandkbs2/pojo/InteractionUpdateRecord.java 0 → 100644
package com.zhiwei.brandkbs2.pojo;
import lombok.Getter;
import lombok.Setter;
import java.util.Date;
/**
* @ClassName: VolumeUpdateRecord
* @Description 互动量更新记录实体类
* @author: cjz
* @date: 2023-09-21 15:21
*/
@Getter
@Setter
public class InteractionUpdateRecord extends AbstractBaseMongo{
/**
* 项目id
*/
private String projectId;
/**
* 使用记录（互动量可用次数调整记录/互动量更新使用记录）
*/
private boolean isUsed;
/**
* 使用类型(导出/查询)
*/
private String usedType;
/**
* 上一次记录数(本次使用前剩余次数)
*/
private int lastCount;
/**
* 该次操作使用(减少)数值 (本次使用的次数)
*/
private int usedCount;
/**
* 本次记录数(本次使用后剩余次数)
*/
private int currentCount;
/**
* 提交人
*/
private String submitter;
/**
* 创建时间(Date类型方便聚合查询)
*/
private Date cTime;
public static InteractionUpdateRecord createUsedRecord(String projectId, String usedType, int lastCount, int usedCount, String submitter) {
InteractionUpdateRecord record = new InteractionUpdateRecord();
record.projectId = projectId;
record.isUsed = true;
record.usedType = usedType;
record.lastCount = lastCount;
record.usedCount = usedCount;
record.currentCount = lastCount - usedCount;
record.submitter = submitter;
record.cTime = new Date();
return record;
}
public static InteractionUpdateRecord createMaintainRecord(String projectId, int lastCount, int currentCount, String submitter) {
InteractionUpdateRecord record = new InteractionUpdateRecord();
record.projectId = projectId;
record.isUsed = false;
record.lastCount = lastCount;
record.currentCount = currentCount;
record.submitter = submitter;
record.cTime = new Date();
return record;
}
public enum UsedType {
url("链接"),
yuqing("舆情库");
@Getter
private final String value;
UsedType(String value) {
this.value = value;
}
}
}
src/main/java/com/zhiwei/brandkbs2/pojo/Project.java
......@@ -92,6 +92,11 @@ public class Project extends AbstractProject {
private int wholeSearchBalance;
/**
* 互动量更新余额
*/
private int interactionBalance;
/**
* 项目主品牌配置信息转换
*
* @return 项目对象
......
src/main/java/com/zhiwei/brandkbs2/service/WholeSearchService.java → src/main/java/com/zhiwei/brandkbs2/service/ExtraService.java
package com.zhiwei.brandkbs2.service;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.pojo.InteractionUpdateRecord;
import com.zhiwei.brandkbs2.pojo.WholeSearchRecord;
import com.zhiwei.brandkbs2.pojo.vo.PageVO;
......@@ -12,7 +13,7 @@ import java.util.List;
* @author: sjj
* @date: 2023-07-19 11:43
*/
public interface WholeSearchService {
public interface ExtraService {
/**
* 查询使用列表
......@@ -22,7 +23,7 @@ public interface WholeSearchService {
* @param sorter 排序字段
* @return PageVO
*/
PageVO<JSONObject> findUsedList(int page,int pageSize,String sorter);
PageVO<JSONObject> findUsedList(int page, int pageSize, String sorter);
/**
* 查询维护列表
......@@ -31,7 +32,7 @@ public interface WholeSearchService {
* @param pageSize 页码大小
* @return PageVO
*/
PageVO<JSONObject> findMaintainList(int page,int pageSize);
PageVO<JSONObject> findMaintainList(int page, int pageSize);
/**
* 导出使用列表
......@@ -41,7 +42,7 @@ public interface WholeSearchService {
* @param day 颗粒度(天级/月级)
* @return List
*/
List<JSONObject> outputUsedList(boolean personal,long startTime,long endTime,boolean day);
List<JSONObject> outputUsedList(boolean personal, long startTime, long endTime, boolean day);
/**
* @param currentCount 调整后数值
......@@ -61,4 +62,51 @@ public interface WholeSearchService {
*/
void decreaseRecord(String keyword, WholeSearchRecord.UsedType usedType, Integer usedCount);
/**
* 查询互动量更新使用列表
*
* @param page 页码
* @param pageSize 页码大小
* @param sorter 排序字段
* @return PageVO
*/
PageVO<JSONObject> findInteractionUsedList(int page, int pageSize, String sorter);
/**
* 查询互动量更新维护列表
*
* @param page 页码
* @param pageSize 页码大小
* @return PageVO
*/
PageVO<JSONObject> findInteractionMaintainList(int page, int pageSize);
/**
* 导出使用列表
* @param personal 个人明细/项目明细
* @param startTime 开始时间
* @param endTime 结束时间
* @param day 颗粒度(天级/月级)
* @return List
*/
List<JSONObject> outputInteractionUsedList(boolean personal, long startTime, long endTime, boolean day);
/**
* 互动量更新可用次数调整
* @param currentCount 调整后数值
*/
void adjustInteractionRecord(Integer currentCount);
/**
* 数据总览
* @return json
*/
JSONObject interactionOverview();
/**
* 记录互动量更新使用
* @param usedType 使用类型
* @param usedCount 使用次数
*/
void decreaseInteractionRecord(InteractionUpdateRecord.UsedType usedType, Integer usedCount);
}
src/main/java/com/zhiwei/brandkbs2/service/MarkDataService.java
......@@ -296,7 +296,7 @@ public interface MarkDataService {
* 搜索-全网搜条件
* @return JSONObject
*/
JSONObject getWholeNetworkSearchCriteria();
JSONObject getWholeNetworkSearchCriteria(boolean externalDataSource);
/**
* 搜索-全网搜
......@@ -365,4 +365,17 @@ public interface MarkDataService {
BoolQueryBuilder projectContendIdQuery(String projectId, String contendId);
/**
* 舆情库-互动量更新
* @param dto 标注数据搜索传输类
* @return
*/
ResponseResult markInteractionUpdate(MarkSearchDTO dto);
/**
* 工具库-舆情库互动量更新预估
* @param dto 标注数据搜索传输类
* @return
*/
Long interactionUpdatePrediction(MarkSearchDTO dto);
}
src/main/java/com/zhiwei/brandkbs2/service/ToolsetService.java 0 → 100644
package com.zhiwei.brandkbs2.service;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.model.ResponseResult;
import org.springframework.web.multipart.MultipartFile;
import java.util.List;
/**
* @ClassName: ToolsetService
* @Description: 工具库服务抽象类
* @author: cjz
* @date: 2023-09-22 09:36
*/
public interface ToolsetService {
/**
* 摘要提取-单条
* @param url
* @return
*/
ResponseResult getSingleArticleSummary(String url);
/**
* 摘要提取-批量
* @param file 文件
* @return
*/
ResponseResult getBatchArticleSummary(MultipartFile file);
/**
* 获取摘要提取剩余可用次数
* @return
*/
ResponseResult getArticleSummaryRemainingCount();
/**
* 链接互动量更新
* @param file excel文件
* @return
*/
ResponseResult urlInteractionUpdate(MultipartFile file);
/**
* 获取互动量更新剩余可用次数
* @return
*/
int getInteractionRemainingCount();
/**
* 提交互动量更新任务
* @param type 任务类型
* @param urls 链接
* @return 任务id
*/
List<String> requestInteractionUrl(String type, List<String> urls);
/**
* 获取互动量更新结果
* @param taskId 任务id
* @return
*/
List<JSONObject> getInteractionResult(List<String> taskId);
}
src/main/java/com/zhiwei/brandkbs2/service/impl/WholeSearchServiceImpl.java → src/main/java/com/zhiwei/brandkbs2/service/impl/ExtraServiceImpl.java
......@@ -3,12 +3,14 @@ package com.zhiwei.brandkbs2.service.impl;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.auth.UserThreadLocal;
import com.zhiwei.brandkbs2.config.Constant;
import com.zhiwei.brandkbs2.dao.InteractionUpdateRecordDao;
import com.zhiwei.brandkbs2.dao.ProjectDao;
import com.zhiwei.brandkbs2.dao.WholeSearchRecordDao;
import com.zhiwei.brandkbs2.pojo.Project;
import com.zhiwei.brandkbs2.pojo.InteractionUpdateRecord;
import com.zhiwei.brandkbs2.pojo.WholeSearchRecord;
import com.zhiwei.brandkbs2.pojo.vo.PageVO;
import com.zhiwei.brandkbs2.service.WholeSearchService;
import com.zhiwei.brandkbs2.service.ExtraService;
import com.zhiwei.brandkbs2.util.MongoUtil;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
......@@ -18,6 +20,7 @@ import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.Date;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
/**
......@@ -26,12 +29,15 @@ import java.util.stream.Collectors;
* @author: sjj
* @date: 2023-07-19 11:55
*/
@Service("wholeSearchServiceImpl")
public class WholeSearchServiceImpl implements WholeSearchService {
@Service("extraServiceImpl")
public class ExtraServiceImpl implements ExtraService {
@Resource(name = "wholeSearchRecordDao")
private WholeSearchRecordDao wholeSearchRecordDao;
@Resource(name = "interactionUpdateRecordDao")
private InteractionUpdateRecordDao interactionUpdateRecordDao;
@Resource(name = "projectDao")
private ProjectDao projectDao;
......@@ -115,6 +121,95 @@ public class WholeSearchServiceImpl implements WholeSearchService {
wholeSearchRecordDao.insertOne(WholeSearchRecord.createUsedRecord(projectId, keyword, usedType.getValue(), project.getWholeSearchBalance(), usedCount, UserThreadLocal.getNickname()), wholeSearchRecordDao.generateCollectionName());
}
@Override
public PageVO<JSONObject> findInteractionUsedList(int page, int pageSize, String sorter) {
String collectionName = interactionUpdateRecordDao.generateCollectionName();
Query query = Query.query(Criteria.where("projectId").is(UserThreadLocal.getProjectId()).and("isUsed").is(true));
interactionUpdateRecordDao.addSort(query, sorter);
long total = interactionUpdateRecordDao.count(query, collectionName);
mongoUtil.start(page, pageSize, query);
return PageVO.createPageVo(total, page, pageSize, interactionUsedList(query, collectionName));
}
@Override
public PageVO<JSONObject> findInteractionMaintainList(int page, int pageSize) {
String collectionName = interactionUpdateRecordDao.generateCollectionName();
Query query = Query.query(Criteria.where("projectId").is(UserThreadLocal.getProjectId()).and("isUsed").is(false));
interactionUpdateRecordDao.addSort(query, "{\"cTime\":\"descend\"}");
long total = interactionUpdateRecordDao.count(query, collectionName);
mongoUtil.start(page, pageSize, query);
List<JSONObject> collect = interactionUpdateRecordDao.findList(query, collectionName).stream().map(record -> {
JSONObject json = new JSONObject();
json.put("lastCount", record.getLastCount());
json.put("currentCount", record.getCurrentCount());
json.put("cTime", record.getCTime().getTime());
json.put("submitter", record.getSubmitter());
return json;
}).collect(Collectors.toList());
return PageVO.createPageVo(total, page, pageSize, collect);
}
@Override
public List<JSONObject> outputInteractionUsedList(boolean personal, long startTime, long endTime, boolean day) {
// 个人明细
if (personal) {
Criteria criteria = Criteria.where("isUsed").is(true);
criteria.and("submitter").is(UserThreadLocal.getNickname());
Query query = Query.query(criteria);
interactionUpdateRecordDao.addSort(query, "{\"cTime\":\"descend\"}");
return interactionUsedList(query, interactionUpdateRecordDao.generateCollectionNames(new Date(startTime), new Date(endTime)));
} else {// 项目明细,需要细分颗粒度
return interactionUpdateRecordDao.aggregateProjectUsedRecord(startTime, endTime, day);
}
}
@Override
public void adjustInteractionRecord(Integer currentCount) {
if (null == currentCount) {
return;
}
String projectId = UserThreadLocal.getProjectId();
Update update = Update.update("interactionBalance", currentCount);
Project project = projectDao.findOneById(projectId);
projectDao.updateOneByIdWithField(projectId, update);
InteractionUpdateRecord record =
InteractionUpdateRecord.createMaintainRecord(projectId, project.getInteractionBalance(), currentCount, UserThreadLocal.getNickname());
interactionUpdateRecordDao.insertOne(record, interactionUpdateRecordDao.generateCollectionName());
}
@Override
public JSONObject interactionOverview() {
JSONObject json = new JSONObject();
long endTime = System.currentTimeMillis();
long startTime = endTime - Constant.ONE_MONTH;
json.put("balance", projectDao.findOneById(UserThreadLocal.getProjectId()).getInteractionBalance());
json.put("trendList", interactionUpdateRecordDao.aggregateDayLastRecord(startTime, endTime));
return json;
}
@Override
public void decreaseInteractionRecord(InteractionUpdateRecord.UsedType usedType, Integer usedCount) {
String projectId = UserThreadLocal.getProjectId();
Project project = projectDao.findOneById(projectId);
Update update = new Update();
update.inc("interactionBalance", -usedCount);
projectDao.updateOneByIdWithField(projectId, update);
InteractionUpdateRecord record =
InteractionUpdateRecord.createUsedRecord(projectId, usedType.getValue(), project.getInteractionBalance(), usedCount, UserThreadLocal.getNickname());
interactionUpdateRecordDao.insertOne(record, interactionUpdateRecordDao.generateCollectionName());
}
private List<JSONObject> interactionUsedList(Query query, String... collectionNames) {
return interactionUpdateRecordDao.findList(query, collectionNames).stream().map(record -> {
JSONObject json = new JSONObject();
json.put("usedType", record.getUsedType());
json.put("usedCount", record.getUsedCount());
json.put("cTime", record.getCTime().getTime());
json.put("submitter", record.getSubmitter());
return json;
}).collect(Collectors.toList());
}
private List<JSONObject> usedList(Query query, String... collectionNames) {
return wholeSearchRecordDao.findList(query, collectionNames).stream().map(record -> {
JSONObject json = new JSONObject();
......
src/main/java/com/zhiwei/brandkbs2/service/impl/MarkDataServiceImpl.java
......@@ -17,9 +17,12 @@ import com.zhiwei.brandkbs2.config.Constant;
import com.zhiwei.brandkbs2.dao.AggreeResultDao;
import com.zhiwei.brandkbs2.dao.ChannelDao;
import com.zhiwei.brandkbs2.dao.HighlightWordDao;
import com.zhiwei.brandkbs2.easyexcel.EasyExcelUtil;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportInteractionUpdateDTO;
import com.zhiwei.brandkbs2.enmus.ChannelEmotion;
import com.zhiwei.brandkbs2.enmus.EmotionEnum;
import com.zhiwei.brandkbs2.enmus.ImportantChannelEnum;
import com.zhiwei.brandkbs2.enmus.InteractionEnum;
import com.zhiwei.brandkbs2.es.EsClientDao;
import com.zhiwei.brandkbs2.es.EsQueryTools;
import com.zhiwei.brandkbs2.exception.ExceptionCast;
......@@ -42,6 +45,7 @@ import com.zhiwei.brandkbs2.util.RedisUtil;
import com.zhiwei.brandkbs2.util.TextUtil;
import com.zhiwei.brandkbs2.util.Tools;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.ListUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.commons.lang3.tuple.Pair;
......@@ -153,6 +157,18 @@ public class MarkDataServiceImpl implements MarkDataService {
@Resource(name = "highlightWordDao")
HighlightWordDao highlightWordDao;
@Resource(name = "toolsetServiceImpl")
private ToolsetService toolsetService;
@Resource(name = "extraServiceImpl")
private ExtraService extraService;
@Resource(name = "systemInfoServiceImpl")
SystemInfoService systemInfoService;
@Value("${brandkbs.file.url}")
private String brandkbsFilePath;
@Override
public PageVO<MarkFlowEntity> getOriginList(MarkSearchDTO markSearchDTO) {
try {
......@@ -1115,28 +1131,20 @@ public class MarkDataServiceImpl implements MarkDataService {
}
@Override
public JSONObject getWholeNetworkSearchCriteria() {
String projectId = UserThreadLocal.getProjectId();
Project project = projectService.getProjectById(projectId);
public JSONObject getWholeNetworkSearchCriteria(boolean externalDataSource) {
JSONObject result = new JSONObject();
// 搜索时间
result.put("times", Arrays.asList("今天", "24小时", "三天", "七天", "近30天"));
result.put("ninetyDays", DateUtils.addDays(Tools.truncDate(new Date(), Constant.DAY_PATTERN), -89).getTime());
result.put("platformList", commonService.getQbjcPlatform("id", "name").stream().filter(s -> !s.get("name").equals("脉脉")).collect(Collectors.toList()));
return result;
// 外部数据源（实时采集）条件
if (externalDataSource) {
result.put("platformList", Arrays.asList("App", "博客", "平媒", "新闻", "微博", "微信", "论坛", "视频", "短视频"));
} else {
result.put("platformList", commonService.getQbjcPlatform("id", "name")
.stream().filter(s -> !s.get("name").equals("脉脉")).collect(Collectors.toList()));
result.putAll(systemInfoService.getExtraParam());
}
private JSONObject getBackUpPlatform() {
JSONObject jsonObject = new JSONObject();
jsonObject.put("微博", "微博");
jsonObject.put("微信", "微信");
jsonObject.put("网媒", "新闻");
jsonObject.put("平媒", "平媒");
jsonObject.put("自媒体", "自媒体");
jsonObject.put("贴吧论坛", "论坛");
jsonObject.put("视频", "视频");
jsonObject.put("短视频", "短视频");
return jsonObject;
return result;
}
@Override
......@@ -1144,9 +1152,6 @@ public class MarkDataServiceImpl implements MarkDataService {
if (dto.isExternalDataSource()) { // 查商业数据库
try {
JSONObject params = externalTransform(dto);
// JSONObject params = IstarShineUtil.getKSSearchParam(dto.getSearch(), dto.getFilterWords(), dto.getStartTime(), dto.getEndTime(), dto.getPlatformsStr(),
// dto.getFilterForward(), dto.getSortOf(), dto.getPage(), dto.getPageSize(), dto.getMatchFields(), null, null);
JSONObject taskEntity = new JSONObject();
taskEntity.put("appId", "6183571e0d710000f6003a12"); // 应用id, 由张志伟提供给使用者
taskEntity.put("taskType", null); // 可为null, 任务类型
......@@ -1154,7 +1159,6 @@ public class MarkDataServiceImpl implements MarkDataService {
taskEntity.put("taskName", UserThreadLocal.getProjectId() + "_" + UserThreadLocal.getUserId());
taskEntity.put("userName", "品见"); // 提交任务的人
taskEntity.put("searchParams", params); // 任务详情参数
ResponseEntity<JSONObject> responseEntity = restTemplate.postForEntity(addIStarShineKSDataUrl, taskEntity, JSONObject.class);
JSONObject result = Objects.requireNonNull(responseEntity.getBody()).getJSONObject("data");
JSONObject json = searchWholeBackUp(result.get("id").toString());
......@@ -1249,8 +1253,12 @@ public class MarkDataServiceImpl implements MarkDataService {
//pageSize
Integer receiveCount = task.getInteger("receiveCount");
Integer searchCount = task.getInteger("searchCount");
int totalPage = searchCount % receiveCount == 0 ? searchCount / receiveCount : (searchCount / receiveCount + 1);
int totalPage;
if (0 == receiveCount || 0 == searchCount){
totalPage = 0;
}else {
totalPage = searchCount % receiveCount == 0 ? searchCount / receiveCount : (searchCount / receiveCount + 1);
}
List<JSONObject> listArray = data.getJSONArray("data").toJavaList(JSONObject.class);
for (JSONObject json : listArray) {
JSONObject result = new JSONObject();
......@@ -1500,6 +1508,64 @@ public class MarkDataServiceImpl implements MarkDataService {
return EsQueryTools.assembleCacheMapsQuery(projectId, contendId);
}
@Override
public ResponseResult markInteractionUpdate(MarkSearchDTO dto) {
Pair<String, List<ExportAppYuqingDTO>> pair = downloadList(dto, esSearchService::searchMarkHitsAndCount);
assert pair != null;
List<ExportAppYuqingDTO> list = pair.getRight();
// 互动量更新剩余可用次数
int interactionBalance = projectService.getProjectById(UserThreadLocal.getProjectId()).getInteractionBalance();
if (interactionBalance - list.size() < 0){
return ResponseResult.failure("剩余可用次数不足");
}
Map<String, String> map = list.stream().collect(Collectors.toMap(ExportAppYuqingDTO::getUrl, ExportAppYuqingDTO::getPlatform, (v1, v2) -> v1));
// 分割map，list里每个map.size为100，目前品见最大链接限制为100，见知微tapd-wiki《互动量更新中间件使用》说明
List<Map<String, String>> mapList = Tools.splitMap(map, 100);
AtomicInteger id = new AtomicInteger(0);
List<JSONObject> result = new ArrayList<>();
for (Map<String, String> urlPlatformMap : mapList) {
List<String> taskIdList = new ArrayList<>(100);
for (Map.Entry<String, String> entry : urlPlatformMap.entrySet()) {
// 提交互动量更新任务
String type = InteractionEnum.getTypeByPlatform(entry.getValue());
List<String> taskIds = toolsetService.requestInteractionUrl(type, Collections.singletonList(entry.getKey()));
taskIdList.addAll(taskIds);
}
// 每100条作为一批提交完时，获取这一批的互动量任务更新结果，单次获取不超50
for (List<String> ids : ListUtils.partition(taskIdList, 50)) {
result.addAll(toolsetService.getInteractionResult(ids));
}
}
Map<Object, JSONObject> urlMap = result.stream().collect(Collectors.toMap(jsonObject -> jsonObject.get("url"), o -> o));
List<ExportInteractionUpdateDTO> exportList = list.stream().map(ExportAppYuqingDTO::getUrl)
.map(url -> ExportInteractionUpdateDTO.createWithPlatform(String.valueOf(id.incrementAndGet()), map, urlMap, url))
.collect(Collectors.toList());
// excel输出到指定路径
String projectName = projectService.getProjectById(UserThreadLocal.getProjectId()).getProjectName();
String filePath = EasyExcelUtil.generateExcelFilePath(brandkbsFilePath, projectName, UserThreadLocal.getNickname(), "舆情库互动量更新结果");
EasyExcelUtil.write(filePath, "sheet1", ExportInteractionUpdateDTO.class, exportList);
JSONObject res = new JSONObject();
// 记录使用情况
extraService.decreaseInteractionRecord(InteractionUpdateRecord.UsedType.yuqing, list.size());
res.put("filePath", filePath);
res.put("remainingCount", interactionBalance - list.size());
// 记录使用
return ResponseResult.success(res);
}
@Override
public Long interactionUpdatePrediction(MarkSearchDTO dto) {
try {
defaultMarkSearch(dto);
dto.setPageSize(null);
Pair<SearchHits[], Map<String, Long>> hitsAndCounts = esSearchService.searchMarkHitsAndCount(dto, false);
return hitsAndCounts.getLeft()[0].getTotalHits().value;
}catch (Exception e){
ExceptionCast.cast(CommonCodeEnum.FAIL, "es查询异常");
}
return null;
}
private static JSONObject getTitleAndUrl(Map<String, Object> map) {
JSONObject json = new JSONObject();
BaseMap baseMap = Tools.getBaseFromEsMap(map);
......
src/main/java/com/zhiwei/brandkbs2/service/impl/ToolsetServiceImpl.java 0 → 100644
package com.zhiwei.brandkbs2.service.impl;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.auth.UserThreadLocal;
import com.zhiwei.brandkbs2.controller.app.AppToolsetController;
import com.zhiwei.brandkbs2.easyexcel.EasyExcelUtil;
import com.zhiwei.brandkbs2.easyexcel.config.ReadExcelDTO;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportArticleSummaryDTO;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportInteractionUpdateDTO;
import com.zhiwei.brandkbs2.easyexcel.dto.UploadArticleSummaryDTO;
import com.zhiwei.brandkbs2.easyexcel.dto.UploadInteractionUpdateDTO;
import com.zhiwei.brandkbs2.easyexcel.listener.ArticleSummaryListener;
import com.zhiwei.brandkbs2.easyexcel.listener.InteractionUpdateListener;
import com.zhiwei.brandkbs2.enmus.InteractionEnum;
import com.zhiwei.brandkbs2.exception.ExceptionCast;
import com.zhiwei.brandkbs2.model.CommonCodeEnum;
import com.zhiwei.brandkbs2.model.ResponseResult;
import com.zhiwei.brandkbs2.pojo.InteractionUpdateRecord;
import com.zhiwei.brandkbs2.pojo.Project;
import com.zhiwei.brandkbs2.service.ExtraService;
import com.zhiwei.brandkbs2.service.ProjectService;
import com.zhiwei.brandkbs2.service.ToolsetService;
import com.zhiwei.brandkbs2.util.RedisUtil;
import com.zhiwei.brandkbs2.util.Tools;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.collections4.ListUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service;
import org.springframework.web.client.RestTemplate;
import org.springframework.web.multipart.MultipartFile;
import javax.annotation.Resource;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
/**
* @ClassName: ToolsetServiceImpl
* @Description ToolsetServiceImpl
* @author: cjz
* @date: 2023-09-22 09:37
*/
@Service("toolsetServiceImpl")
public class ToolsetServiceImpl implements ToolsetService {
public static final Logger log = LogManager.getLogger(AppToolsetController.class);
@Resource(name = "redisUtil")
private RedisUtil redisUtil;
@Resource(name = "projectServiceImpl")
private ProjectService projectService;
@Autowired
private RestTemplate restTemplate;
@Value("${toolset.articleSummary.url}")
private String articleSummaryUrl;
@Value("${toolset.articleInfo.url}")
private String articleInfoUrl;
@Value("${toolset.interactionUpdate.url}")
private String interactionUpdateUrl;
@Value("${toolset.interactionResult.url}")
private String interactionResultUrl;
@Resource(name = "extraServiceImpl")
private ExtraService extraService;
@Value("${brandkbs.file.url}")
private String brandkbsFilePath;
private static final String TEXT_SUMMARY_PREVIOUS = "作为一名公关人员，将文章进行简短的中文摘要，摘要文字中需包括日期、地点等核心要素，其中日期不用包括年份，摘要起始必须为日期，字数少于150个字\n文章：\n";
private static final int ARTICLE_SUMMARY_LIMIT = 1000;
private static final String BRANDKBS_INTERACTION_SALT = "650bee0d393b382938003695";
private static final String USER = "brandkbs";
@Override
public ResponseResult getSingleArticleSummary(String url) {
JSONObject res = new JSONObject();
JSONObject info = getUrlInfo(url, UserThreadLocal.getProjectId());
if (Objects.isNull(info)){
return ResponseResult.failure("链接解析异常");
}
String text = info.getString("content");
String articleSummaryResult = getArticleSummaryResult(text);
// 剩余次数限制
String redisKey = RedisUtil.getToolsetArticleSummaryLimitKey(UserThreadLocal.getProjectId());
String redisResult = redisUtil.get(redisKey);
int usedCount = 1;
if (Objects.nonNull(redisResult)){
int redisCount = Integer.parseInt(redisResult);
if (redisCount >= ARTICLE_SUMMARY_LIMIT){
return ResponseResult.failure("本日摘要提取次数已达上限");
}
usedCount = redisCount + 1;
redisUtil.setExpire(redisKey, String.valueOf(usedCount), Tools.getMillSecondNextDay(), TimeUnit.MILLISECONDS);
}else {
redisUtil.setExpire(redisKey, String.valueOf(usedCount), Tools.getMillSecondNextDay(), TimeUnit.MILLISECONDS);
}
res.put("source", info.get("source"));
res.put("platform", info.getString("platform"));
res.put("time", info.getLong("time"));
res.put("text", articleSummaryResult);
res.put("remainingCount", ARTICLE_SUMMARY_LIMIT - usedCount);
return ResponseResult.success(res);
}
@Override
public ResponseResult getBatchArticleSummary(MultipartFile file) {
JSONObject res = new JSONObject();
Project project = projectService.getProjectById(UserThreadLocal.getProjectId());
// 调用前剩余可用次数
String redisKey = RedisUtil.getToolsetArticleSummaryLimitKey(UserThreadLocal.getProjectId());
String redisResult = redisUtil.get(redisKey);
int remainingCount = ARTICLE_SUMMARY_LIMIT;
if (Objects.nonNull(redisResult)){
remainingCount = ARTICLE_SUMMARY_LIMIT - Integer.parseInt(redisResult);
}
if (remainingCount <= 0){
return ResponseResult.failure("本日摘要提取次数已达上限");
}
// excel信息提取
Map<String, String> map = new LinkedHashMap<>();
ReadExcelDTO<UploadArticleSummaryDTO> readExcel = new ReadExcelDTO<>();
readExcel.setClazz(UploadArticleSummaryDTO.class);
readExcel.setAnalysisEventListener(new ArticleSummaryListener(map));
EasyExcelUtil.read(file, readExcel);
if (map.size() > 50){
return ResponseResult.failure("超过每次批量提取上限50");
}
int usedCount = 0; // 本次批量调用次数
List<ExportArticleSummaryDTO> datas = new ArrayList<>(50);
// 摘要提取
for (Map.Entry<String, String> entry : map.entrySet()) {
String text = entry.getValue();
String url = entry.getKey();
// 文章内容空时，通过链接提取文章内容
if (Objects.isNull(text) && Objects.nonNull(url)){
JSONObject json = getUrlInfo(entry.getKey(), UserThreadLocal.getProjectId());
text = Objects.nonNull(json) ? json.getString("content") : null;
}
String articleSummaryResult = getArticleSummaryResult(text);
usedCount = usedCount + 1;
datas.add(new ExportArticleSummaryDTO(String.valueOf(usedCount), entry.getKey(), entry.getValue(), articleSummaryResult));
}
// 本次批量调用后剩余次数
remainingCount = remainingCount - usedCount;
// 更新已用次数
redisUtil.setExpire(redisKey, String.valueOf(ARTICLE_SUMMARY_LIMIT - remainingCount), Tools.getMillSecondNextDay(), TimeUnit.MILLISECONDS);
// excel输出到指定路径
String filePath = EasyExcelUtil.generateExcelFilePath(brandkbsFilePath, project.getProjectName(), UserThreadLocal.getNickname(), "摘要提取结果");
EasyExcelUtil.write(filePath, "sheet1", ExportArticleSummaryDTO.class, datas);
res.put("filePath", filePath);
res.put("remainingCount", remainingCount);
return ResponseResult.success(res);
}
@Override
public ResponseResult getArticleSummaryRemainingCount() {
String redisKey = RedisUtil.getToolsetArticleSummaryLimitKey(UserThreadLocal.getProjectId());
String redisResult = redisUtil.get(redisKey);
if (Objects.nonNull(redisResult)){
return ResponseResult.success(ARTICLE_SUMMARY_LIMIT - Integer.parseInt(redisResult));
}
return ResponseResult.success(ARTICLE_SUMMARY_LIMIT);
}
@Override
public ResponseResult urlInteractionUpdate(MultipartFile file) {
// excel信息提取
List<String> urls = new ArrayList<>();
ReadExcelDTO<UploadInteractionUpdateDTO> readExcel = new ReadExcelDTO<>();
readExcel.setClazz(UploadInteractionUpdateDTO.class);
readExcel.setAnalysisEventListener(new InteractionUpdateListener(urls));
EasyExcelUtil.read(file, readExcel);
if (urls.size() > 300){
return ResponseResult.failure("超过每次更新链接数量上限300");
}
// 互动量更新剩余可用次数
int interactionBalance = projectService.getProjectById(UserThreadLocal.getProjectId()).getInteractionBalance();
if (interactionBalance - urls.size() < 0){
return ResponseResult.failure("剩余可用次数不足");
}
// 通过url获取域名进而获取任务类型
Map<String, String> map = new HashMap<>();
urls.forEach(url -> map.compute(url, (key, value) -> {
if (Objects.isNull(value)) {
String domain = Tools.topDomainOfDomain(url);
return InteractionEnum.getTypeByDomain(domain);
}
return value;
}));
// 分割map，list里每个map.size为100，目前品见最大链接限制为100，见知微tapd-wiki《互动量更新中间件使用》说明
List<Map<String, String>> mapList = Tools.splitMap(map, 100);
AtomicInteger id = new AtomicInteger(0);
List<JSONObject> result = new ArrayList<>(300);
for (Map<String, String> urlTypeMap : mapList) {
List<String> taskIdList = new ArrayList<>(100);
for (Map.Entry<String, String> entry : urlTypeMap.entrySet()) {
// 提交互动量更新任务
List<String> taskIds = requestInteractionUrl(entry.getValue(), Collections.singletonList(entry.getKey()));
taskIdList.addAll(taskIds);
}
// 每100条作为一批提交完时，获取互动量任务更新结果，单次获取不超50
for (List<String> ids : ListUtils.partition(taskIdList, 50)) {
result.addAll(getInteractionResult(ids));
}
}
Map<Object, JSONObject> urlMap = result.stream().collect(Collectors.toMap(jsonObject -> jsonObject.get("url"), o -> o));
List<ExportInteractionUpdateDTO> exportList =
urls.stream().map(url -> new ExportInteractionUpdateDTO(String.valueOf(id.incrementAndGet()), urlMap, url)).collect(Collectors.toList());
// excel输出到指定路径
String projectName = projectService.getProjectById(UserThreadLocal.getProjectId()).getProjectName();
String filePath = EasyExcelUtil.generateExcelFilePath(brandkbsFilePath, projectName, UserThreadLocal.getNickname(), "链接互动量更新结果");
EasyExcelUtil.write(filePath, "sheet1", ExportInteractionUpdateDTO.class, exportList);
JSONObject res = new JSONObject();
// 记录使用情况
extraService.decreaseInteractionRecord(InteractionUpdateRecord.UsedType.url, urls.size());
res.put("filePath", filePath);
res.put("remainingCount", interactionBalance - urls.size());
return ResponseResult.success(res);
}
@Override
public int getInteractionRemainingCount() {
return projectService.getProjectById(UserThreadLocal.getProjectId()).getInteractionBalance();
}
@Override
public List<String> requestInteractionUrl(String type, List<String> urls) {
List<String> taskIds = new ArrayList<>(2);
// 请求头参数
HttpHeaders headers = getInteractionRequestHeader();
// 请求参数，一个任务最多只支持50条链接，见知微tapd-wiki《互动量更新中间件使用》说明
List<List<String>> partition = ListUtils.partition(urls, 50);
for (List<String> urlList : partition) {
JSONObject requestJson = new JSONObject();
requestJson.put("type", type);
requestJson.put("urlList", urlList);
HttpEntity<String> request = new HttpEntity<>(requestJson.toJSONString(), headers);
JSONObject body = restTemplate.postForEntity(interactionUpdateUrl, request, JSONObject.class).getBody();
if (Objects.nonNull(body) && body.getBoolean("success")){
taskIds.add(body.getString("data"));
}
}
return taskIds;
}
@Override
public List<JSONObject> getInteractionResult(List<String> taskIds) {
HttpHeaders headers = getInteractionRequestHeader();
// 请求参数
JSONObject requestJson = new JSONObject();
requestJson.put("uuidList", taskIds);
HttpEntity<String> request = new HttpEntity<>(requestJson.toJSONString(), headers);
List<JSONObject> res = new ArrayList<>();
// 轮询任务结果
for (int i = 0; i < 30; i++) {
try {
ResponseEntity<JSONObject> response = restTemplate.postForEntity(interactionResultUrl, request, JSONObject.class);
JSONObject body = response.getBody();
if (Objects.nonNull(body) && 200 == body.getIntValue("code") && !body.isEmpty() && !body.getJSONArray("data").toJavaList(JSONObject.class).isEmpty() &&
body.getJSONArray("data").toJavaList(JSONObject.class).stream().map(m -> m.getIntValue("code")).allMatch(code -> 200 == code)) {
res = body.getJSONArray("data").toJavaList(JSONObject.class);
break;
} else {
Thread.sleep(10000L);
}
}catch (Exception e){
ExceptionCast.cast(CommonCodeEnum.FAIL, "轮询互动量更新任务异常,taskId:" + taskIds, e);
}
}
return res;
}
private HttpHeaders getInteractionRequestHeader(){
// 请求头参数
Long timeStamp = System.currentTimeMillis();
HttpHeaders headers = new HttpHeaders();
headers.add("user", USER);
headers.add("token", DigestUtils.md5Hex(timeStamp + "_" + BRANDKBS_INTERACTION_SALT));
headers.add("timestamp", String.valueOf(timeStamp));
headers.setContentType(MediaType.APPLICATION_JSON);
return headers;
}
/**
* 链接信息提取
* @param url 链接
* @param projectId 项目id
* @return
*/
private JSONObject getUrlInfo(String url, String projectId) {
JSONObject jsonObject;
// 即使抛出异常(大概率会是timeout)，也要保证批量时其他链接正常执行
try {
String linkedGroupId = projectService.getProjectVOById(projectId).getBrandLinkedGroupId();
jsonObject = restTemplate.getForEntity(articleInfoUrl, JSONObject.class, url, linkedGroupId, UserThreadLocal.getNickname()).getBody();
} catch (Exception e) {
log.info("url:{},访问链接信息提取接口异常-", url, e);
return null;
}
if (Objects.isNull(jsonObject) || !jsonObject.getBoolean("status")) {
return null;
}
return jsonObject.getJSONObject("data");
}
/**
* 获取摘要提取结果
* @param text 文本
* @return
*/
private String getArticleSummaryResult(String text){
// 即使抛出异常(大概率会是timeout)，也要保证批量时其他链接正常执行
String errorString = "访问超时,请稍后重试此条数据";
if (Objects.isNull(text)){
return errorString;
}
try {
// 拼接提示词模板
String resultText = TEXT_SUMMARY_PREVIOUS + StringUtils.substring(text, 0, 5000);
// 请求参数 请求头
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.TEXT_PLAIN);
HttpEntity<String> request = new HttpEntity<>(resultText, headers);
ResponseEntity<String> response = restTemplate.postForEntity(articleSummaryUrl, request, String.class);
return response.getBody();
}catch (Exception e){
log.info("访问摘要提取接口异常-", e);
return errorString;
}
}
}
src/main/java/com/zhiwei/brandkbs2/util/RedisUtil.java
......@@ -74,8 +74,8 @@ public class RedisUtil {
return RedisKeyPrefix.PROJECT_WARN_NEW_CASE_CURSOR + projectId;
}
public static String getToolsetArticleSummaryLimitKey(String projectId, String userId){
return RedisKeyPrefix.TOOLSET_ARTICLE_SUMMARY_LIMIT + Tools.concat(projectId, userId);
public static String getToolsetArticleSummaryLimitKey(String projectId){
return RedisKeyPrefix.TOOLSET_ARTICLE_SUMMARY_LIMIT + projectId;
}
public void setExpire(String key, String value, long timeout, TimeUnit unit) {
......@@ -97,10 +97,6 @@ public class RedisUtil {
return stringRedisTemplate.opsForValue().get(key);
}
public void getAndSet(String key, String newValue){
stringRedisTemplate.opsForValue().getAndSet(key, newValue);
}
public Set<String> keys(String key) {
return stringRedisTemplate.keys(key);
}
......
src/main/java/com/zhiwei/brandkbs2/util/Tools.java
......@@ -4,6 +4,7 @@ import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.read.listener.PageReadListener;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.google.common.net.InternetDomainName;
import com.zhiwei.base.category.ClassCodec;
import com.zhiwei.base.entity.subclass.CompleteText;
import com.zhiwei.base.entity.subclass.IncompleteText;
......@@ -19,6 +20,7 @@ import com.zhiwei.brandkbs2.pojo.BaseMap;
import com.zhiwei.qbjc.bean.pojo.common.MessagePlatform;
import com.zhiwei.qbjc.bean.pojo.common.Tag;
import com.zhiwei.qbjc.bean.tools.BeanTools;
import okhttp3.HttpUrl;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.MapUtils;
......@@ -619,6 +621,31 @@ public class Tools {
return result;
}
/**
* 分割map
* @param map
* @param chunkSize
* @return
*/
public static List<Map<String, String>> splitMap(Map<String, String> map, int chunkSize) {
List<Map<String, String>> result = new ArrayList<>();
int count = 0;
Map<String, String> chunk = new LinkedHashMap<>();
for (Map.Entry<String, String> entry : map.entrySet()) {
chunk.put(entry.getKey(), entry.getValue());
count++;
if (count == chunkSize) {
result.add(chunk);
chunk = new LinkedHashMap<>();
count = 0;
}
}
if (!chunk.isEmpty()) {
result.add(chunk);
}
return result;
}
public static Long[] formatTimeRange(Long startTime, Long endTime) {
if (Objects.isNull(startTime) || Objects.isNull(endTime)) {
endTime = DateUtils.addDays(Tools.truncDate(new Date(), DAY_PATTERN), 1).getTime();
......@@ -1183,4 +1210,39 @@ public class Tools {
return count != mtags.size();
}
/**
* 获取当前时间到第二天零点的毫秒数
* @return
*/
public static long getMillSecondNextDay(){
Calendar calendar = Calendar.getInstance();
calendar.add(Calendar.DAY_OF_YEAR, 1);
calendar.set(Calendar.HOUR_OF_DAY, 0);
calendar.set(Calendar.SECOND, 0);
calendar.set(Calendar.MINUTE, 0);
calendar.set(Calendar.MILLISECOND, 0);
return calendar.getTimeInMillis() - System.currentTimeMillis();
}
/**
* 获取url的域名
* @param url
* @return
*/
public static String topDomainOfDomain(String url) {
try {
String host = getHost(url);
return InternetDomainName.from(host).topPrivateDomain().toString();
} catch (Exception e) {
return null;
}
}
public static String getHost(String url) {
try {
return HttpUrl.parse(url).host();
} catch (Exception e) {
return null;
}
}
}
\ No newline at end of file
src/main/resources/application-dev.properties
......@@ -121,3 +121,5 @@ wx.getuserphonenumber=https://api.weixin.qq.com/wxa/business/getuserphonenumber?
#\u5DE5\u5177\u5E93\u76F8\u5173\u5916\u90E8\u63A5\u53E3
toolset.articleSummary.url=https://zhiweidata.xyz/api/front/chat-swagger
toolset.articleInfo.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/match?url={1}&projectId={2}&submitter={3}
toolset.interactionUpdate.url=https://zhiwei-middleware.zhiweidata.com/interact/insertData
toolset.interactionResult.url=https://zhiwei-middleware.zhiweidata.com/interact/findDataBatch
\ No newline at end of file
src/main/resources/application-local.properties
......@@ -55,7 +55,7 @@ channel.index.application.name=brandkbs2
hqd.groupAll.url= https://sensitive.zhiweidata.com/sensitive/planA/groupAll
#\u6807\u6CE8\u4E2D\u95F4\u4EF6
mark.registry.address=zookeeper://192.168.0.11:2181?backup=192.168.0.30:2181,192.168.0.35:2181
mark.provider.group=zhiwei-mark-test_liuyu
mark.provider.group=zhiwei-mark-local-liuyu
#\u4E8B\u4EF6\u4E2D\u95F4\u4EF6
event.registry.address=zookeeper://192.168.0.11:2181?backup=192.168.0.30:2181,192.168.0.35:2181
event.provider.group=zhiwei-event-ygd
......@@ -125,3 +125,5 @@ wx.getuserphonenumber=https://api.weixin.qq.com/wxa/business/getuserphonenumber?
#\u5DE5\u5177\u5E93\u76F8\u5173\u5916\u90E8\u63A5\u53E3
toolset.articleSummary.url=https://zhiweidata.xyz/api/front/chat-swagger
toolset.articleInfo.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/match?url={1}&projectId={2}&submitter={3}
toolset.interactionUpdate.url=https://zhiwei-middleware.zhiweidata.com/interact/insertData
toolset.interactionResult.url=https://zhiwei-middleware.zhiweidata.com/interact/findDataBatch
\ No newline at end of file
src/main/resources/application-prod.properties
......@@ -121,3 +121,5 @@ wx.getuserphonenumber=https://api.weixin.qq.com/wxa/business/getuserphonenumber?
#\u5DE5\u5177\u5E93\u76F8\u5173\u5916\u90E8\u63A5\u53E3
toolset.articleSummary.url=https://zhiweidata.xyz/api/front/chat-swagger
toolset.articleInfo.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/match?url={1}&projectId={2}&submitter={3}
toolset.interactionUpdate.url=https://zhiwei-middleware.zhiweidata.com/interact/insertData
toolset.interactionResult.url=https://zhiwei-middleware.zhiweidata.com/interact/findDataBatch
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment