Skip to content

B
brandkbs2
Commit 60c28946 by shenjunjie
Options

Merge branch 'release' into 'master' 

Release

See merge request !593
parents edef09fe 8081d07c
Showing with 79 additions and 38 deletions
src/main/java/com/zhiwei/brandkbs2/config/Constant.java
......@@ -26,10 +26,14 @@ public class Constant {
public static final String MONTH_PATTERN = "yyyy-MM";
public static final String YEAR_PATTERN = "yyyy";
public static final String SECOND_PATTERN = "yyyy-MM-dd HH:mm:ss";
public static final FastDateFormat SPEC_MINUTE_FORMAT = FastDateFormat.getInstance(SPEC_MINUTE_PATTERN);
public static final FastDateFormat SPEC_DAY_POINT_FORMAT = FastDateFormat.getInstance(SPEC_DAY_PATTERN);
public static final FastDateFormat HOUR_FORMAT = FastDateFormat.getInstance(HOUR_PATTERN);
public static final FastDateFormat DAY_FORMAT = FastDateFormat.getInstance(DAY_PATTERN);
public static final FastDateFormat EXCEL_TIME_FORMAT = FastDateFormat.getInstance(SECOND_PATTERN);
public static final FastDateFormat SPEC_DAY_FORMAT = FastDateFormat.getInstance("yyyy/MM/dd");
public static final FastDateFormat ONLY_MONTH_FORMAT = FastDateFormat.getInstance("M月");
public static final FastDateFormat DF_yyyyMMdd = FastDateFormat.getInstance("yyyyMMdd");
......
src/main/java/com/zhiwei/brandkbs2/easyexcel/EasyExcelUtil.java
......@@ -9,6 +9,7 @@ import com.alibaba.excel.write.metadata.WriteSheet;
import com.zhiwei.brandkbs2.easyexcel.config.ReadExcelDTO;
import com.zhiwei.brandkbs2.easyexcel.config.WriteExcelDTO;
import com.zhiwei.brandkbs2.util.Tools;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
......@@ -26,6 +27,7 @@ import java.nio.channels.ReadableByteChannel;
import java.nio.channels.WritableByteChannel;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
/**
* @author sjj
......@@ -112,7 +114,7 @@ public class EasyExcelUtil {
*/
public static <T> void write(String filePath, String sheetName, Class<T> clazz, List<T> datas) {
try {
formatExcelExports(clazz, datas);
formatExcelExports(clazz, datas, null);
EasyExcel.write(filePath, clazz).sheet(sheetName).doWrite(datas);
} catch (Exception e) {
log.error("file:{}，write error：", filePath, e);
......@@ -172,6 +174,7 @@ public class EasyExcelUtil {
*/
public static void dynamicHeadWrite(String filePath, String sheetName, List<List<String>> head, List<List<Object>> datas) {
try {
formatExcelExports(null, null, datas);
EasyExcel.write(filePath).head(head).sheet(sheetName).doWrite(datas);
} catch (Exception e) {
log.error("file:{}，write error：", filePath, e);
......@@ -190,7 +193,7 @@ public class EasyExcelUtil {
*/
public static <T> void download(String fileName, String sheetName, Class<T> clazz, List<T> datas, HttpServletResponse response) {
try {
formatExcelExports(clazz, datas);
formatExcelExports(clazz, datas, null);
response.setContentType("application/vnd.ms-excel");
response.setCharacterEncoding("utf-8");
// 这里URLEncoder.encode可以防止中文乱码
......@@ -238,26 +241,37 @@ public class EasyExcelUtil {
return filePath + Tools.concat(projectName, nickName, System.currentTimeMillis(), fileName) + ".xlsx";
}
private static <T> void formatExcelExports(Class<T> clazz, List<T> datas){
List<Field> stringFields = new ArrayList<>();
// 记录需要设置的部分
for (Field field : clazz.getDeclaredFields()) {
// String格式截取前32767位
if (field.getGenericType().toString().equals(String.class.toString())) {
field.setAccessible(true);
stringFields.add(field);
private static <T> void formatExcelExports(Class<T> clazz, List<T> datas, List<List<Object>> dynamicDatas){
if (Objects.nonNull(clazz)) {
List<Field> stringFields = new ArrayList<>();
// 记录需要设置的部分
for (Field field : clazz.getDeclaredFields()) {
// String格式截取前5000位
if (field.getGenericType().toString().equals(String.class.toString())) {
field.setAccessible(true);
stringFields.add(field);
}
}
}
datas.forEach(data -> {
for (Field field : stringFields) {
try {
// 截取为excel单元格允许的最大长度
field.set(data, StringUtils.substring((String) (field.get(data)), 0, 32767));
} catch (IllegalAccessException e) {
log.error(e);
datas.forEach(data -> {
for (Field field : stringFields) {
try {
// 截取为excel单元格允许的最大长度
field.set(data, StringUtils.substring((String) (field.get(data)), 0, 5000));
} catch (IllegalAccessException e) {
log.error(e);
}
}
});
}
if (CollectionUtils.isNotEmpty(dynamicDatas)){
for (List<Object> dynamicData : dynamicDatas) {
dynamicData.replaceAll(obj -> {
if (Objects.nonNull(obj)) {
obj = StringUtils.substring(String.valueOf(obj), 0, 5000);
}
return obj;
});
}
});
}
}
}
src/main/java/com/zhiwei/brandkbs2/pojo/dto/SearchFilterDTO.java
......@@ -177,13 +177,13 @@ public class SearchFilterDTO {
map.put("source", sourceKeyword);
map.put("url", hostKeyword);
if (null == sorter || sorter.isEmpty()) {
map.put("sort", "DESC");
map.put("sort", "desc");
} else {
for (Map.Entry<String, Object> entry : sorter.entrySet()) {
if (entry.getValue().toString().contains("desc")) {
map.put("sort", "DESC");
map.put("sort", "desc");
} else {
map.put("sort", "ASC");
map.put("sort", "asc");
}
}
}
......
src/main/java/com/zhiwei/brandkbs2/service/impl/MarkDataServiceImpl.java
......@@ -87,6 +87,7 @@ import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
......@@ -934,7 +935,7 @@ public class MarkDataServiceImpl implements MarkDataService {
TermsAggregationBuilder sourceAggregationBuilder = AggregationBuilders.terms("source").field("source").size(10000);
// query
BoolQueryBuilder query = yuqingAnalyzeQuery(dto);
// 过滤微博
// 过滤微博
query.mustNot(QueryBuilders.termQuery("platform_id", "5d02236e6395002a7c380b79"));
SearchResponse searchResponse;
// 单独处理buckets超过上限异常，出错时按时间分段查询
......@@ -1098,7 +1099,13 @@ public class MarkDataServiceImpl implements MarkDataService {
if (0 == hits.getTotalHits().value) {
return new BaseMap();
}
return Tools.getBaseFromEsMap(hits.getAt(0).getSourceAsMap());
JSONObject jsonObject = new JSONObject(hits.getAt(0).getSourceAsMap());
BaseMap baseMap = Tools.getBaseFromEsMap(jsonObject);
if (StringUtils.isBlank(baseMap.getContent()) && Objects.nonNull(jsonObject.getString("ocr_text")) && !jsonObject.getJSONArray("ocr_text").isEmpty()){
List<String> ocrText = jsonObject.getJSONArray("ocr_text").toJavaList(String.class);
baseMap.setContent(ocrText.get(0));
}
return baseMap;
}
@Override
......@@ -2870,11 +2877,11 @@ public class MarkDataServiceImpl implements MarkDataService {
if (Objects.nonNull(negativeSpreadTendency.getLeft()) || Objects.nonNull(spreadTendency.getLeft()) || Objects.nonNull(positiveSpreadTendency.getLeft())) {
CompletableFuture.allOf(Stream.of(spreadTendency.getLeft(), negativeSpreadTendency.getLeft(), positiveSpreadTendency.getLeft())
.filter(Objects::nonNull).map(aggTitle -> CompletableFuture.runAsync(() -> {
try {
baseMaps.put(aggTitle, getAnalyzeFirstArticle(dto, aggTitle));
} catch (IOException ignored) {
}
}, executor)).toArray(CompletableFuture[]::new)).join();
try {
baseMaps.put(aggTitle, getAnalyzeFirstArticle(dto, aggTitle));
} catch (IOException ignored) {
}
}, executor)).toArray(CompletableFuture[]::new)).join();
}
// 标题为空，但趋势图不为空，此时说明最高点的时间段发文平台全为微博，此时取最高点时间段内时间最早的微博发文
if (Objects.isNull(spreadTendency.getLeft()) && CollectionUtils.isNotEmpty(spreadTendency.getRight())){
......@@ -2935,7 +2942,10 @@ public class MarkDataServiceImpl implements MarkDataService {
return null;
}
// 最高点的时间为开始时间
LineVO startLine = lines.stream().max(Comparator.comparingLong(LineVO::getCount)).get();
LineVO startLine = lines.stream().max(Comparator.comparingLong(LineVO::getCount)).orElse(null);
if (Objects.isNull(startLine)){
return null;
}
Long startTime = startLine.getDate();
int startIndex = lines.indexOf(startLine);
// 最高点后一个点的时间为结束时间
......@@ -4076,9 +4086,19 @@ public class MarkDataServiceImpl implements MarkDataService {
calculateCost(pair.getRight(), AccessModel.Model.DOUBAO_PRO_32K)), userLogRecordDao.generateCollectionName());
String resultContent = pair.getLeft();
String[] splits = resultContent.split("\\|");
List<String> result = new ArrayList<>(Arrays.asList(splits)).stream().filter(StringUtils::isNoneBlank).map(String::trim).collect(Collectors.toList());
redisUtil.setExpire(key, JSONObject.toJSONString(result));
return result;
List<String> result = new ArrayList<>(Arrays.asList(splits)).stream().filter(StringUtils::isNoneBlank).map(String::trim)
.limit(5).collect(Collectors.toList());
List<String> res = new ArrayList<>();
for (int i = 0; i < result.size(); i++) {
if (result.get(i).startsWith(i + 1 + "、") || result.get(i).startsWith(i + 1 + ".")){
String substring = result.get(i).substring(2);
res.add(substring);
}else {
res.add(result.get(i));
}
}
redisUtil.setExpire(key, JSONObject.toJSONString(res));
return res;
}
private List<String> getAIReferenceQuestionTemplate(String projectName){
......@@ -4107,7 +4127,7 @@ public class MarkDataServiceImpl implements MarkDataService {
project.getContendList().forEach(contend -> brandStr.append("、").append(contend.getBrandName()));
}
questionPair = standardRequest(question, modelName, MessageFormat.format(QUESTION_PROMPT, brandStr));
if (Objects.isNull(questionPair) || Objects.equals("无法回答", questionPair.getLeft())) {
if (Objects.isNull(questionPair) || Objects.equals("无法回答", questionPair.getLeft().trim())) {
return null;
}
JSONObject json = JSON.parseObject(questionPair.getLeft());
......@@ -5183,7 +5203,7 @@ public class MarkDataServiceImpl implements MarkDataService {
for (JSONObject hit : hits) {
BaseMap baseMap = Tools.getBaseFromEsMap(hit);
List<Object> data = new ArrayList<>();
data.add(new Date(baseMap.getTime()));
data.add(Constant.EXCEL_TIME_FORMAT.format(baseMap.getTime()));
data.add(baseMap.getPlatform());
data.add(baseMap.getSource());
data.add(baseMap.getTitle());
......
src/main/java/com/zhiwei/brandkbs2/service/impl/SearchWholeServiceImpl.java
......@@ -826,7 +826,10 @@ public class SearchWholeServiceImpl implements SearchWholeService {
*/
private BaseMap getFirstArticle(SearchFilterDTO dto, List<LineVO> totalSpreadTendency) throws IOException {
// 最高点的时间为开始时间
LineVO startLine = totalSpreadTendency.stream().max(Comparator.comparingLong(LineVO::getCount)).get();
LineVO startLine = totalSpreadTendency.stream().max(Comparator.comparingLong(LineVO::getCount)).orElse(null);
if (Objects.isNull(startLine)){
return null;
}
Long startTime = startLine.getDate();
int startIndex = totalSpreadTendency.indexOf(startLine);
// 最高点后一个点的时间为结束时间
......
src/main/java/com/zhiwei/brandkbs2/util/Tools.java
......@@ -80,7 +80,7 @@ public class Tools {
/**
* 须过滤的标题
*/
public static final List<String> FILTER_TITLE = Arrays.asList("Notitle", "分享一篇文章", "分享一篇文章。", "暂无标题", "");
public static final List<String> FILTER_TITLE = Arrays.asList("Notitle", "分享一篇文章", "分享一篇文章。", "暂无标题", "", "No title");
/**
* 须过滤的渠道
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment