Skip to content

B
brandkbs2
Commit 52b07c90 by shentao
Options

Merge branch 'release' into 'master' 

Release

See merge request !391
parents d7e63d47 d4e39c43
Showing with 379 additions and 56 deletions
src/main/java/com/zhiwei/brandkbs2/aop/AopDownloadTask.java
package com.zhiwei.brandkbs2.aop;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.exception.ExceptionCast;
import com.zhiwei.brandkbs2.model.CommonCodeEnum;
import com.zhiwei.brandkbs2.model.ResponseResult;
......@@ -18,6 +19,7 @@ import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import java.lang.reflect.Method;
import java.util.Objects;
/**
* @author cjz
......@@ -34,7 +36,7 @@ public class AopDownloadTask {
@Resource(name = "downloadTaskServiceImpl")
DownloadTaskService downloadTaskService;
@Around(value = "execution(public * com..controller..app..AppDownloadController.*(..)))")
@Around(value = "execution(public * com..controller..app..AppDownloadController.*(..)) || execution(public * com..controller..app..AppToolsetController.getBatchArticleSummary(..)))")
public Object around(ProceedingJoinPoint joinPoint) throws Throwable{
Signature signature = joinPoint.getSignature();
Method method = ((MethodSignature) signature).getMethod();
......@@ -53,7 +55,11 @@ public class AopDownloadTask {
ExceptionCast.cast(CommonCodeEnum.FAIL, "下载异常", e);
}
// 更新下载任务
fileAddress = ((ResponseResult) proceed).getData().toString();
if (Objects.equals(method.getName(), "getBatchArticleSummary")){
fileAddress = JSONObject.parseObject(((ResponseResult) proceed).getData().toString()).getString("filePath");
}else {
fileAddress = ((ResponseResult) proceed).getData().toString();
}
downloadTaskService.updateDownloadTask(taskId, 100, DownloadTask.Status.FINISH.getName(), fileAddress);
return proceed;
}
......
src/main/java/com/zhiwei/brandkbs2/common/CommonConfig.java
......@@ -11,8 +11,14 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.http.client.OkHttp3ClientHttpRequestFactory;
import org.springframework.http.converter.StringHttpMessageConverter;
import org.springframework.web.client.RestTemplate;
import javax.net.ssl.*;
import java.nio.charset.StandardCharsets;
import java.security.SecureRandom;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.concurrent.TimeUnit;
/**
......@@ -41,12 +47,37 @@ public class CommonConfig {
private String filterGroup;
@Bean
public RestTemplate restTemplate() {
public RestTemplate restTemplate() throws Exception {
final OkHttpClient client = new OkHttpClient().newBuilder()
.connectTimeout(60, TimeUnit.SECONDS)
.connectTimeout(120, TimeUnit.SECONDS)
.readTimeout(30, TimeUnit.SECONDS)
// 忽略SSL证书验证
.sslSocketFactory(createSslSocketFactory(), new SkipX509TrustManager())
.hostnameVerifier((s, sslSession) -> true)
.build();
return new RestTemplate(new OkHttp3ClientHttpRequestFactory(client));
RestTemplate restTemplate = new RestTemplate(new OkHttp3ClientHttpRequestFactory(client));
// 设置字符串编码
restTemplate.getMessageConverters().set(1, new StringHttpMessageConverter(StandardCharsets.UTF_8));
return restTemplate;
}
private SSLSocketFactory createSslSocketFactory() throws Exception {
SSLContext context = SSLContext.getInstance("TLS");
context.init(null, new TrustManager[]{new SkipX509TrustManager()}, new SecureRandom());
return context.getSocketFactory();
}
private static class SkipX509TrustManager implements X509TrustManager {
@Override
public void checkClientTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
}
@Override
public void checkServerTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
}
@Override
public X509Certificate[] getAcceptedIssuers() {
return new X509Certificate[0];
}
}
/**
......
src/main/java/com/zhiwei/brandkbs2/common/RedisKeyPrefix.java
......@@ -95,6 +95,11 @@ public class RedisKeyPrefix {
public static final String PROJECT_WARN_NEW_CASE_LIST = "BRANDKBS:NEW_CASE:LIST_";
public static final String PROJECT_WARN_NEW_CASE_CURSOR = "BRANDKBS:NEW_CASE:CURSOR:";
/**
* 工具集相关缓存
*/
public static final String TOOLSET_ARTICLE_SUMMARY_LIMIT = "BRANDKBS:TOOLSET:ARTICLE_SUMMARY:LIMIT:";
public static String projectWarnHotTopKeyAll(String projectId, String type) {
return RedisKeyPrefix.generateRedisKey(RedisKeyPrefix.PROJECT_WARN_HOT_TOP, projectId, Tools.concat(type, "*"));
}
......
src/main/java/com/zhiwei/brandkbs2/controller/app/AppToolsetController.java 0 → 100644
package com.zhiwei.brandkbs2.controller.app;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.aop.LogRecord;
import com.zhiwei.brandkbs2.auth.Auth;
import com.zhiwei.brandkbs2.auth.UserThreadLocal;
import com.zhiwei.brandkbs2.easyexcel.EasyExcelUtil;
import com.zhiwei.brandkbs2.easyexcel.config.ReadExcelDTO;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportArticleSummaryDTO;
import com.zhiwei.brandkbs2.easyexcel.dto.UploadArticleSummaryDTO;
import com.zhiwei.brandkbs2.easyexcel.listener.ArticleSummaryListener;
import com.zhiwei.brandkbs2.enmus.RoleEnum;
import com.zhiwei.brandkbs2.model.ResponseResult;
import com.zhiwei.brandkbs2.pojo.Project;
import com.zhiwei.brandkbs2.service.ProjectService;
import com.zhiwei.brandkbs2.util.RedisUtil;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.client.RestTemplate;
import org.springframework.web.multipart.MultipartFile;
import javax.annotation.Resource;
import java.util.*;
/**
* @author cjz
* @ClassName AppToolsetController
* @Description 提供前台工具集相关接口
* @date 2023-09-14 10:27
*/
@RestController
@RequestMapping("/app/toolset")
@Api(tags = "工具集", description = "工具集")
@Auth(role = RoleEnum.CUSTOMER)
public class AppToolsetController {
@Resource(name = "redisUtil")
private RedisUtil redisUtil;
@Resource(name = "projectServiceImpl")
private ProjectService projectService;
@Autowired
private RestTemplate restTemplate;
@Value("${toolset.articleSummary.url}")
private String articleSummaryUrl;
@Value("${toolset.articleInfo.url}")
private String articleInfoUrl;
@Value("${brandkbs.file.url}")
private String brandkbsFilePath;
private static final String TEXT_SUMMARY_PREVIOUS = "作为一名公关人员，将文章进行简短的中文摘要，摘要文字中需包括日期、地点等核心要素，其中日期不用包括年份，摘要起始必须为日期，字数少于150个字\n文章：\n";
private static final int ARTICLE_SUMMARY_LIMIT = 1000;
@ApiOperation("摘要提取-单条")
@GetMapping("/article-summary/single")
@LogRecord(description = "工具库-摘要提取-单条")
public ResponseResult getSingleArticleSummary(@RequestParam(value = "url") String url) {
JSONObject res = new JSONObject();
JSONObject info = getUrlInfo(url, UserThreadLocal.getProjectId());
if (Objects.isNull(info)){
return ResponseResult.failure("链接解析异常");
}
String text = info.getString("content");
String articleSummaryResult = getArticleSummaryResult(text);
// 剩余次数限制
String redisKey = RedisUtil.getToolsetArticleSummaryLimitKey(UserThreadLocal.getProjectId(), UserThreadLocal.getUserId());
String redisResult = redisUtil.get(redisKey);
int usedCount = 1;
if (Objects.nonNull(redisResult)){
int redisCount = Integer.parseInt(redisResult);
if (redisCount >= ARTICLE_SUMMARY_LIMIT){
return ResponseResult.failure("本日摘要提取次数已达上限");
}
usedCount = redisCount + 1;
redisUtil.getAndSet(redisKey, String.valueOf(usedCount));
}else {
redisUtil.setExpire(redisKey, String.valueOf(usedCount));
}
res.put("source", info.get("source"));
res.put("platform", info.getString("platform"));
res.put("time", info.getLong("time"));
res.put("text", articleSummaryResult);
res.put("remainingCount", ARTICLE_SUMMARY_LIMIT - usedCount);
return ResponseResult.success(res);
}
@ApiOperation("摘要提取")
@PostMapping("/article-summary/batch")
@LogRecord(description = "工具库-摘要提取-批量")
public ResponseResult getBatchArticleSummary(@RequestParam(value = "file") MultipartFile file){
JSONObject res = new JSONObject();
Project project = projectService.getProjectById(UserThreadLocal.getProjectId());
// 调用前剩余可用次数
String redisKey = RedisUtil.getToolsetArticleSummaryLimitKey(UserThreadLocal.getProjectId(), UserThreadLocal.getUserId());
String redisResult = redisUtil.get(redisKey);
int remainingCount = ARTICLE_SUMMARY_LIMIT;
if (Objects.nonNull(redisResult)){
remainingCount = ARTICLE_SUMMARY_LIMIT - Integer.parseInt(redisResult);
}
if (remainingCount <= 0){
return ResponseResult.failure("本日摘要提取次数已达上限");
}
// excel信息提取
Map<String, String> map = new LinkedHashMap<>(50);
ReadExcelDTO<UploadArticleSummaryDTO> readExcel = new ReadExcelDTO<>();
readExcel.setClazz(UploadArticleSummaryDTO.class);
readExcel.setAnalysisEventListener(new ArticleSummaryListener(map, remainingCount));
EasyExcelUtil.read(file, readExcel);
int usedCount = 0; // 本次批量调用次数
List<ExportArticleSummaryDTO> datas = new ArrayList<>(50);
// 摘要提取
for (Map.Entry<String, String> entry : map.entrySet()) {
String text = entry.getValue();
String url = entry.getKey();
// 文章内容空时，通过链接提取文章内容
if (Objects.isNull(text) && Objects.nonNull(url)){
JSONObject json = getUrlInfo(entry.getKey(), UserThreadLocal.getProjectId());
text = Objects.nonNull(json) ? json.getString("content") : null;
}
String articleSummaryResult = getArticleSummaryResult(text);
usedCount = usedCount + 1;
datas.add(new ExportArticleSummaryDTO(String.valueOf(usedCount), entry.getKey(), entry.getValue(), articleSummaryResult));
}
// 本次批量调用后剩余次数
remainingCount = remainingCount - usedCount;
// 更新已用次数
redisUtil.getAndSet(redisKey, String.valueOf(ARTICLE_SUMMARY_LIMIT - remainingCount));
// excel输出到指定路径
String filePath = EasyExcelUtil.generateExcelFilePath(brandkbsFilePath, project.getProjectName(), UserThreadLocal.getNickname(), "摘要提取结果");
EasyExcelUtil.write(filePath, "sheet1", ExportArticleSummaryDTO.class, datas);
res.put("filePath", filePath);
res.put("remainingCount", remainingCount);
return ResponseResult.success(res);
}
@ApiOperation("摘要提取-剩余可用次数")
@GetMapping("/article-summary/remaining")
public ResponseResult getArticleSummaryRemainingCount(){
String redisKey = RedisUtil.getToolsetArticleSummaryLimitKey(UserThreadLocal.getProjectId(), UserThreadLocal.getUserId());
String redisResult = redisUtil.get(redisKey);
if (Objects.nonNull(redisResult)){
return ResponseResult.success(ARTICLE_SUMMARY_LIMIT - Integer.parseInt(redisResult));
}
return ResponseResult.success(ARTICLE_SUMMARY_LIMIT);
}
/**
* 链接信息提取
* @param url 链接
* @param projectId 项目id
* @return
*/
private JSONObject getUrlInfo(String url, String projectId){
String linkedGroupId = projectService.getProjectVOById(projectId).getBrandLinkedGroupId();
JSONObject jsonObject = restTemplate.getForEntity(articleInfoUrl, JSONObject.class, url, linkedGroupId, UserThreadLocal.getNickname()).getBody();
if (Objects.isNull(jsonObject) || !jsonObject.getBoolean("status")){
return null;
}
return jsonObject.getJSONObject("data");
}
/**
* 获取摘要提取结果
* @param text 文本
* @return
*/
private String getArticleSummaryResult(String text){
if (Objects.isNull(text)){
return null;
}
// 拼接提示词模板
String resultText = TEXT_SUMMARY_PREVIOUS + StringUtils.substring(text, 0, 5000);
// 请求参数 请求头
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.TEXT_PLAIN);
HttpEntity<String> request = new HttpEntity<>(resultText, headers);
ResponseEntity<String> response = restTemplate.postForEntity(articleSummaryUrl, request, String.class);
return response.getBody();
}
}
src/main/java/com/zhiwei/brandkbs2/easyexcel/EasyExcelUtil.java
......@@ -8,6 +8,7 @@ import com.alibaba.excel.read.metadata.ReadSheet;
import com.alibaba.excel.write.metadata.WriteSheet;
import com.zhiwei.brandkbs2.easyexcel.config.ReadExcelDTO;
import com.zhiwei.brandkbs2.easyexcel.config.WriteExcelDTO;
import com.zhiwei.brandkbs2.util.Tools;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
......@@ -113,6 +114,28 @@ public class EasyExcelUtil {
}
/**
* 写单个sheet
* @param filePath 文件路径
* @param input 文件输入流
*/
public static void write(String filePath, InputStream input){
try {
FileOutputStream output = new FileOutputStream(filePath);
ReadableByteChannel inputChannel = Channels.newChannel(input);
WritableByteChannel outputChannel = Channels.newChannel(output);
ByteBuffer buffer = ByteBuffer.allocateDirect(10240);
long size = 0;
while (inputChannel.read(buffer) != -1) {
buffer.flip();
size += outputChannel.write(buffer);
buffer.clear();
}
}catch (Exception e){
log.error("file:{}，write error：", filePath, e);
}
}
/**
* 写多个sheet
*
* @param filePath 文件路径
......@@ -198,52 +221,15 @@ public class EasyExcelUtil {
}
/**
* 将excel文件保存至指定路径
* @param filePath
* @param fileName
* @param sheetName
* @param clazz
* @param datas
* @param <T>
*/
public static <T> String saveExcelWithPath(String filePath, String fileName, String sheetName, Class<T> clazz, List<T> datas){
try {
formatExcelExports(clazz, datas);
String fileAddress = filePath + fileName + ".xlsx";
FileOutputStream out = new FileOutputStream(fileAddress);
EasyExcel.write(out, clazz).sheet(sheetName).doWrite(datas);
return fileAddress;
}catch (Exception e){
log.error("file:{}，saveExcelWithPath error：", fileName, e);
return null;
}
}
/**
* InputStream转换excel文件保存至指定路径
* @param filePath
* @param fileName
* @param input
* 生成excel路径
* @param filePath 文件保存路径
* @param projectName 项目名
* @param nickName 昵称
* @param fileName 文件名
* @return
*/
public static String saveExcelWithPath(String filePath, String fileName, InputStream input){
try {
String fileAddress = filePath + fileName + ".xlsx";
FileOutputStream output = new FileOutputStream(fileAddress);
ReadableByteChannel inputChannel = Channels.newChannel(input);
WritableByteChannel outputChannel = Channels.newChannel(output);
ByteBuffer buffer = ByteBuffer.allocateDirect(10240);
long size = 0;
while (inputChannel.read(buffer) != -1) {
buffer.flip();
size += outputChannel.write(buffer);
buffer.clear();
}
return fileAddress;
}catch (Exception e){
log.error("file:{}，saveExcelWithPath error：", fileName, e);
return null;
}
public static String generateExcelFilePath(String filePath, String projectName, String nickName, String fileName){
return filePath + Tools.concat(projectName, nickName, System.currentTimeMillis(), fileName) + ".xlsx";
}
private static <T> void formatExcelExports(Class<T> clazz, List<T> datas){
......
src/main/java/com/zhiwei/brandkbs2/easyexcel/dto/ExportArticleSummaryDTO.java 0 → 100644
package com.zhiwei.brandkbs2.easyexcel.dto;
import com.alibaba.excel.annotation.ExcelProperty;
import lombok.AllArgsConstructor;
import lombok.Data;
/**
* @author cjz
* @version 1.0
* @description 前台导出摘要提取结果实体类
* @date 2023/9/14 15:31
*/
@Data
@AllArgsConstructor
public class ExportArticleSummaryDTO {
@ExcelProperty("序号")
private String id;
@ExcelProperty("链接")
private String url;
@ExcelProperty("文章内容")
private String text;
@ExcelProperty("摘要提取结果")
private String result;
}
src/main/java/com/zhiwei/brandkbs2/easyexcel/dto/UploadArticleSummaryDTO.java 0 → 100644
package com.zhiwei.brandkbs2.easyexcel.dto;
import com.alibaba.excel.annotation.ExcelProperty;
import lombok.Data;
/**
* @author cjz
* @version 1.0
* @description 解析文章摘要提前上传文件
* @date 2023/9/14 14:22
*/
@Data
public class UploadArticleSummaryDTO {
@ExcelProperty("序号")
private String id;
@ExcelProperty("链接")
private String url;
@ExcelProperty("文章内容")
private String text;
}
src/main/java/com/zhiwei/brandkbs2/easyexcel/listener/ArticleSummaryListener.java 0 → 100644
package com.zhiwei.brandkbs2.easyexcel.listener;
import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.event.AnalysisEventListener;
import com.zhiwei.brandkbs2.easyexcel.dto.UploadArticleSummaryDTO;
import java.util.Map;
/**
* @ClassName: ArticleSummaryListener
* @Description 文章摘要提前文件批量上传监听类
* @author: cjz
* @date: 2023-09-14 14:44
*/
public class ArticleSummaryListener extends AnalysisEventListener<UploadArticleSummaryDTO> {
private Map<String, String> map;
private int remainingCount;
public ArticleSummaryListener(Map<String, String> map, int remainingCount){
this.map = map;
this.remainingCount = remainingCount;
}
@Override
public void invoke(UploadArticleSummaryDTO data, AnalysisContext context) {
if (map.size() < remainingCount && map.size() <= 50) {
map.put(data.getUrl(), data.getText());
}
}
@Override
public void doAfterAllAnalysed(AnalysisContext context) {
}
}
src/main/java/com/zhiwei/brandkbs2/util/RedisUtil.java
......@@ -74,6 +74,10 @@ public class RedisUtil {
return RedisKeyPrefix.PROJECT_WARN_NEW_CASE_CURSOR + projectId;
}
public static String getToolsetArticleSummaryLimitKey(String projectId, String userId){
return RedisKeyPrefix.TOOLSET_ARTICLE_SUMMARY_LIMIT + Tools.concat(projectId, userId);
}
public void setExpire(String key, String value, long timeout, TimeUnit unit) {
stringRedisTemplate.opsForValue().set(key, value, timeout, unit);
}
......@@ -93,6 +97,10 @@ public class RedisUtil {
return stringRedisTemplate.opsForValue().get(key);
}
public void getAndSet(String key, String newValue){
stringRedisTemplate.opsForValue().getAndSet(key, newValue);
}
public Set<String> keys(String key) {
return stringRedisTemplate.keys(key);
}
......@@ -131,5 +139,4 @@ public class RedisUtil {
public void remove(String key) {
setExpire(key, "-1", 1, TimeUnit.SECONDS);
}
}
src/main/resources/application-dev.properties
......@@ -117,4 +117,7 @@ ef.external.filterNew.url=https://ef.zhiweidata.com/external/filterNew.do?firstT
hot.search.url=https://hotsearch-manage.zhiweidata.com/hotsearch/hotSearch/findNewHotSearch?type={1}
#\u5FAE\u4FE1\u76F8\u5173\u63A5\u53E3
wx.accesstoken.url=https://ef.zhiweidata.com/smallprogram/api/codeToken/getToken?appId=7FFBB9B377D0D28FBCF9FA481D6FF77546718A121E4BD0EA1AAB28011C53E7EE
wx.getuserphonenumber=https://api.weixin.qq.com/wxa/business/getuserphonenumber?access_token=
\ No newline at end of file
wx.getuserphonenumber=https://api.weixin.qq.com/wxa/business/getuserphonenumber?access_token=
#\u5DE5\u5177\u5E93\u76F8\u5173\u5916\u90E8\u63A5\u53E3
toolset.articleSummary.url=https://zhiweidata.xyz/api/front/chat-swagger
toolset.articleInfo.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/match?url={1}&projectId={2}&submitter={3}
\ No newline at end of file
src/main/resources/application-local.properties
......@@ -121,4 +121,7 @@ ef.external.filterNew.url=https://ef.zhiweidata.com/external/filterNew.do?firstT
hot.search.url=https://hotsearch-manage.zhiweidata.com/hotsearch/hotSearch/findNewHotSearch?type={1}
#\u5FAE\u4FE1\u76F8\u5173\u63A5\u53E3
wx.accesstoken.url=https://ef.zhiweidata.com/smallprogram/api/codeToken/getToken?appId=7FFBB9B377D0D28FBCF9FA481D6FF77546718A121E4BD0EA1AAB28011C53E7EE
wx.getuserphonenumber=https://api.weixin.qq.com/wxa/business/getuserphonenumber?access_token=
\ No newline at end of file
wx.getuserphonenumber=https://api.weixin.qq.com/wxa/business/getuserphonenumber?access_token=
#\u5DE5\u5177\u5E93\u76F8\u5173\u5916\u90E8\u63A5\u53E3
toolset.articleSummary.url=https://zhiweidata.xyz/api/front/chat-swagger
toolset.articleInfo.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/match?url={1}&projectId={2}&submitter={3}
\ No newline at end of file
src/main/resources/application-prod.properties
......@@ -117,4 +117,7 @@ ef.external.filterNew.url=https://ef.zhiweidata.com/external/filterNew.do?firstT
hot.search.url=https://hotsearch-manage.zhiweidata.com/hotsearch/hotSearch/findNewHotSearch?type={1}
#\u5FAE\u4FE1\u76F8\u5173\u63A5\u53E3
wx.accesstoken.url=https://ef.zhiweidata.com/smallprogram/api/codeToken/getToken?appId=7FFBB9B377D0D28FBCF9FA481D6FF77546718A121E4BD0EA1AAB28011C53E7EE
wx.getuserphonenumber=https://api.weixin.qq.com/wxa/business/getuserphonenumber?access_token=
\ No newline at end of file
wx.getuserphonenumber=https://api.weixin.qq.com/wxa/business/getuserphonenumber?access_token=
#\u5DE5\u5177\u5E93\u76F8\u5173\u5916\u90E8\u63A5\u53E3
toolset.articleSummary.url=https://zhiweidata.xyz/api/front/chat-swagger
toolset.articleInfo.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/match?url={1}&projectId={2}&submitter={3}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment