Skip to content

B
brandkbs2
Commit 8d01723a by shenjunjie
Options

渠道库部分完结

parent 34d25fdf
Showing with 852 additions and 371 deletions
src/main/java/com/zhiwei/brandkbs2/common/GenericAttribute.java
......@@ -16,7 +16,7 @@ public class GenericAttribute {
*/
public static final String ES_INDEX_PRE = "brandkbs2";
public static final String ES_INDEX_TEST = "brandkbs2_test";
public static final String ES_CHANNEL_INDEX_TEST = "brandkbs2_channel_record_test2";
public static final String ES_CHANNEL_INDEX_TEST = "brandkbs2_channel_record_test7";
/**
* es ind_title
**/
......
src/main/java/com/zhiwei/brandkbs2/config/Constant.java
......@@ -11,9 +11,11 @@ import org.apache.commons.lang3.time.FastDateFormat;
public class Constant {
public static final Long ONE_DAY = 24 * 60 * 60 * 1000L;
public static final Long ONE_MONTH = 30 * 24 * 60 * 60 * 1000L;
public static final String HOUR_PATTERN = "yyyy-MM-dd HH";
public static final String DAY_PATTERN = "yyyy-MM-dd";
public static final String MONTH_PATTERN = "yyyy-MM";
public static final FastDateFormat HOUR_FORMAT = FastDateFormat.getInstance(HOUR_PATTERN);
public static final FastDateFormat DAY_FORMAT = FastDateFormat.getInstance(DAY_PATTERN);
......@@ -28,7 +30,7 @@ public class Constant {
*/
public static final String EMOTION_LABEL_KEY = "情感倾向";
public static final String EMOTION_SENSITIVE = "敏感";
public static final String EMOTION_NEGATIVE= "负面";
public static final String EMOTION_NEGATIVE = "负面";
public static final String BRAND_LABEL_KEY = "品牌归属";
/**
......
src/main/java/com/zhiwei/brandkbs2/config/TaskPoolConfig.java
......@@ -61,4 +61,22 @@ public class TaskPoolConfig {
return executor;
}
@Bean
public ThreadPoolTaskExecutor mongoQueryExecutor() {
log.info("start mongoQueryExecutor");
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
// 配置核心线程数
executor.setCorePoolSize(8);
// 配置最大线程数
executor.setMaxPoolSize(16);
// 配置线程池中的线程的名称前缀
executor.setThreadNamePrefix("mongoQuery-");
// rejection-policy：当pool已经达到max size的时候，如何处理新任务
// CALLER_RUNS：不在新线程中执行任务，而是有调用者所在的线程来执行
executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
// 执行初始化
executor.initialize();
return executor;
}
}
src/main/java/com/zhiwei/brandkbs2/controller/app/AppChannelController.java
......@@ -2,6 +2,8 @@ package com.zhiwei.brandkbs2.controller.app;
import com.zhiwei.brandkbs2.auth.Auth;
import com.zhiwei.brandkbs2.controller.BaseController;
import com.zhiwei.brandkbs2.easyexcel.EasyExcelUtil;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportAppChannelArticleDTO;
import com.zhiwei.brandkbs2.enmus.RoleEnum;
import com.zhiwei.brandkbs2.model.ResponseResult;
import com.zhiwei.brandkbs2.service.ChannelService;
......@@ -12,6 +14,7 @@ import io.swagger.annotations.ApiOperation;
import org.springframework.web.bind.annotation.*;
import javax.annotation.Resource;
import java.util.List;
import java.util.Set;
/**
......@@ -30,56 +33,64 @@ public class AppChannelController extends BaseController {
ChannelService channelService;
@ApiImplicitParams({@ApiImplicitParam(name = "contendId", value = "品牌id", paramType = "query", dataType = "string"),
@ApiImplicitParams({@ApiImplicitParam(name = "contendId", value = "品牌id", defaultValue = "0", paramType = "query", dataType = "string"),
@ApiImplicitParam(name = "platform", value = "平台", paramType = "query", dataType = "string"),
@ApiImplicitParam(name = "keyword", value = "关键字搜索", paramType = "query", dataType = "string"),
@ApiImplicitParam(name = "startTime", value = "起始时间", paramType = "query", dataType = "long"),
@ApiImplicitParam(name = "endTime", value = "结束时间", paramType = "query", dataType = "long"),
@ApiImplicitParam(name = "size", value = "选取前几", defaultValue = "50", paramType = "query", dataType = "int"),
@ApiImplicitParam(name = "pageSize", value = "选取前几", defaultValue = "50", paramType = "query", dataType = "int"),
})
@ApiOperation("渠道库-活跃渠道榜")
@GetMapping("/list/active")
public ResponseResult getActiveChannelList(@RequestParam(value = "contendId", required = false, defaultValue = "0") String contendId,
public ResponseResult getActiveChannelList(@RequestParam(value = "contendId", defaultValue = "0") String contendId,
@RequestParam(value = "platform", required = false) String platform,
@RequestParam(value = "keyword", required = false) String keyword,
@RequestParam(value = "startTime", required = false) Long startTime,
@RequestParam(value = "endTime", required = false) Long endTime,
@RequestParam(value = "size", defaultValue = "50") int size) {
@RequestParam(value = "pageSize", defaultValue = "50") int size) {
return ResponseResult.success(channelService.getActiveChannelList(contendId, platform, keyword, startTime, endTime, size));
}
@ApiOperation("渠道库-友好渠道榜")
@ApiImplicitParams({
@ApiImplicitParam(name = "contendId", value = "品牌ID", paramType = "query", dataType = "string"),
@ApiImplicitParam(name = "contendId", value = "品牌ID", defaultValue = "0", paramType = "query", dataType = "string"),
@ApiImplicitParam(name = "platform", value = "平台筛选", paramType = "query", dataType = "string"),
@ApiImplicitParam(name = "keyword", value = "关键字搜索", paramType = "query", dataType = "string"),
@ApiImplicitParam(name = "sorter", defaultValue = "{\"index\":\"descend\"}", value = "排序字段", paramType = "query", dataType = "string"),
@ApiImplicitParam(name = "size", value = "选取前几", defaultValue = "50", paramType = "query", dataType = "int")
@ApiImplicitParam(name = "startTime", value = "起始时间", paramType = "query", dataType = "long"),
@ApiImplicitParam(name = "endTime", value = "结束时间", paramType = "query", dataType = "long"),
@ApiImplicitParam(name = "pageSize", value = "选取前几", defaultValue = "50", paramType = "query", dataType = "int")
})
@GetMapping("/list/positive")
public ResponseResult getPositiveList(@RequestParam(value = "contendId", required = false) String contendId,
public ResponseResult getPositiveList(@RequestParam(value = "contendId", defaultValue = "0") String contendId,
@RequestParam(value = "platform", required = false) String platform,
@RequestParam(value = "keyword", required = false) String keyword,
@RequestParam(value = "sorter", required = false) String sorter,
@RequestParam(value = "size", defaultValue = "50") int size) {
return ResponseResult.success(channelService.getPositiveList(contendId, platform, keyword, sorter, size));
@RequestParam(value = "sorter", defaultValue = "{\"index\":\"descend\"}") String sorter,
@RequestParam(value = "startTime", required = false) Long startTime,
@RequestParam(value = "endTime", required = false) Long endTime,
@RequestParam(value = "pageSize", defaultValue = "50") int size) {
return ResponseResult.success(channelService.getPositiveList(contendId, platform, keyword, sorter, startTime, endTime, size));
}
@ApiOperation("渠道库-敏感渠道榜")
@ApiImplicitParams({
@ApiImplicitParam(name = "contendId", value = "品牌ID", paramType = "query", dataType = "string"),
@ApiImplicitParam(name = "contendId", value = "品牌ID", defaultValue = "0", paramType = "query", dataType = "string"),
@ApiImplicitParam(name = "platform", value = "平台筛选", paramType = "query", dataType = "string"),
@ApiImplicitParam(name = "keyword", value = "关键字搜索", paramType = "query", dataType = "string"),
@ApiImplicitParam(name = "sorter", defaultValue = "{\"index\":\"descend\"}", value = "排序字段", paramType = "query", dataType = "string"),
@ApiImplicitParam(name = "size", value = "选取前几", defaultValue = "50", paramType = "query", dataType = "int")
@ApiImplicitParam(name = "startTime", value = "起始时间", paramType = "query", dataType = "long"),
@ApiImplicitParam(name = "endTime", value = "结束时间", paramType = "query", dataType = "long"),
@ApiImplicitParam(name = "pageSize", value = "选取前几", defaultValue = "50", paramType = "query", dataType = "int")
})
@GetMapping("/list/negative")
public ResponseResult getNegativeList(@RequestParam(value = "contendId", required = false) String contendId,
public ResponseResult getNegativeList(@RequestParam(value = "contendId", defaultValue = "0") String contendId,
@RequestParam(value = "platform", required = false) String platform,
@RequestParam(value = "keyword", required = false) String keyword,
@RequestParam(value = "sorter", required = false) String sorter,
@RequestParam(value = "size", defaultValue = "50") int size) {
return ResponseResult.success(channelService.getNegativeList(contendId, platform, keyword, sorter, size));
@RequestParam(value = "sorter", defaultValue = "{\"index\":\"descend\"}") String sorter,
@RequestParam(value = "startTime", required = false) Long startTime,
@RequestParam(value = "endTime", required = false) Long endTime,
@RequestParam(value = "pageSize", defaultValue = "50") int size) {
return ResponseResult.success(channelService.getNegativeList(contendId, platform, keyword, sorter, startTime, endTime, size));
}
@ApiOperation("渠道库-收藏渠道")
......@@ -91,7 +102,7 @@ public class AppChannelController extends BaseController {
@ApiOperation("渠道库-取消收藏渠道")
@ApiImplicitParam(name = "channelId", value = "渠道ID", required = true, paramType = "path", dataType = "string")
@PostMapping("/removeCollect/{channelId}}")
@PostMapping("/removeCollect/{channelId}")
public ResponseResult removeCollectChannel(@PathVariable String channelId) {
return ResponseResult.success(channelService.removeCollectChannel(channelId));
}
......@@ -121,10 +132,61 @@ public class AppChannelController extends BaseController {
@GetMapping("/spreadingTend")
public ResponseResult getSpreadingTend(@RequestParam("channelId") String channelId,
@RequestParam(value = "type", defaultValue = "文章") String type,
@RequestParam(value = "brandIds", required = false) Set<String> contends,
@RequestParam(value = "contends", required = false) Set<String> contends,
@RequestParam(value = "startTime", required = false) Long startTime,
@RequestParam(value = "endTime", required = false) Long endTime) {
return ResponseResult.success(channelService.getSpreadingTend(channelId, type, contends, startTime, endTime));
}
@ApiOperation("渠道库-渠道动向-摘要")
@ApiImplicitParams({
@ApiImplicitParam(name = "channelId", value = "渠道ID", required = true, paramType = "query", dataType = "string"),
@ApiImplicitParam(name = "contends", value = "品牌ID集合", paramType = "query", dataType = "string"),
@ApiImplicitParam(name = "startTime", value = "开始时间", paramType = "query", dataType = "long"),
@ApiImplicitParam(name = "endTime", value = "结束时间", paramType = "query", dataType = "long")
})
@GetMapping("/spreadingTend/summary")
public ResponseResult getSpreadingTendSummary(@RequestParam("channelId") String channelId,
@RequestParam(value = "contends", required = false) Set<String> contends,
@RequestParam(value = "startTime", required = false) Long startTime,
@RequestParam(value = "endTime", required = false) Long endTime) {
return ResponseResult.success(channelService.getSpreadingTendSummary(channelId, contends, startTime, endTime));
}
@ApiOperation("渠道-文章列表")
@ApiImplicitParams({
@ApiImplicitParam(name = "startTime", value = "开始时间", defaultValue = "0", paramType = "query", dataType = "long"),
@ApiImplicitParam(name = "endTime", value = "结束时间", defaultValue = "0", paramType = "query", dataType = "long"),
@ApiImplicitParam(name = "page", value = "页码", defaultValue = "1", paramType = "query", dataType = "int"),
@ApiImplicitParam(name = "pageSize", value = "页码大小", defaultValue = "10", paramType = "query", dataType = "int"),
@ApiImplicitParam(name = "channelId", value = "渠道ID", required = true, paramType = "query", dataType = "int"),
@ApiImplicitParam(name = "contendId", value = "竞品ID", defaultValue = "0", paramType = "query", dataType = "int")
})
@GetMapping("/articles")
public ResponseResult getArticles(@RequestParam(value = "startTime") Long startTime,
@RequestParam(value = "endTime") Long endTime,
@RequestParam(value = "page", defaultValue = "1") int page,
@RequestParam(value = "pageSize", defaultValue = "10") int pageSize,
@RequestParam("channelId") String channelId,
@RequestParam(value = "contendId", defaultValue = "0") String contendId) {
return ResponseResult.success(channelService.getArticlesByTime(startTime, endTime, page, pageSize, channelId, contendId));
}
@ApiOperation("渠道库-下载文章列表")
@ApiImplicitParams({
@ApiImplicitParam(name = "startTime", value = "开始时间", required = true, paramType = "query", dataType = "long"),
@ApiImplicitParam(name = "endTime", value = "结束时间", required = true, paramType = "query", dataType = "long"),
@ApiImplicitParam(name = "channelId", value = "渠道ID", required = true, paramType = "query", dataType = "string"),
@ApiImplicitParam(name = "contendId", value = "品牌ID", defaultValue = "0", paramType = "query", dataType = "string")
})
@GetMapping("/articles/download")
public ResponseResult downloadArticles(@RequestParam(value = "startTime") long startTime,
@RequestParam(value = "endTime") long endTime,
@RequestParam("channelId") String channelId,
@RequestParam(value = "contendId", defaultValue = "0") String contendId) {
List<ExportAppChannelArticleDTO> exportAppChannelArticleDTOS = channelService.downloadArticlesByTime(startTime, endTime, channelId, contendId);
EasyExcelUtil.download(channelId + "稿件列表数据", "sheet1", ExportAppChannelArticleDTO.class, exportAppChannelArticleDTOS, response);
return ResponseResult.success();
}
}
src/main/java/com/zhiwei/brandkbs2/dao/ChannelDao.java
......@@ -3,16 +3,21 @@ package com.zhiwei.brandkbs2.dao;
import com.zhiwei.brandkbs2.pojo.Channel;
import com.zhiwei.brandkbs2.pojo.ChannelIndex;
import java.util.Collection;
import java.util.Map;
/**
* @ClassName: ChannelDao
* @Description ChannelDao
* @author: sjj
* @date: 2022-06-16 15:30
*/
public interface ChannelDao extends BaseMongoDao<Channel>{
public interface ChannelDao extends BaseMongoDao<Channel> {
Channel queryUnique(String channelFid);
Channel queryUnique(ChannelIndex channelIndex);
Map<String, Channel> queryUniqueAsync(Collection<String> channelFids);
}
src/main/java/com/zhiwei/brandkbs2/dao/EventDao.java
......@@ -4,6 +4,7 @@ import com.zhiwei.brandkbs2.pojo.ChannelIndex;
import com.zhiwei.brandkbs2.pojo.Event;
import java.util.List;
import java.util.Map;
/**
* @ClassName: EventDao
......@@ -11,13 +12,13 @@ import java.util.List;
* @author: sjj
* @date: 2022-05-18 14:35
*/
public interface EventDao extends BaseMongoDao<Event>{
public interface EventDao extends BaseMongoDao<Event> {
/**
* 是否已存在事件
*
* @param yqEventId 舆情事件id
* @param projectId 项目id
* @param yqEventId 舆情事件id
* @param projectId 项目id
* @param linkedGroupId 关联项目组id
* @return 是否存在
*/
......@@ -26,12 +27,12 @@ public interface EventDao extends BaseMongoDao<Event>{
/**
* 根据联合id查询事件
*
* @param yqEventId 舆情事件id
* @param projectId 项目id
* @param yqEventId 舆情事件id
* @param projectId 项目id
* @param linkedGroupId 关联项目id
* @return 事件
*/
Event getEventByUniqueIds(String yqEventId,String projectId,String linkedGroupId);
Event getEventByUniqueIds(String yqEventId, String projectId, String linkedGroupId);
/**
......@@ -42,4 +43,21 @@ public interface EventDao extends BaseMongoDao<Event>{
*/
List<String> getEventCount(ChannelIndex channelIndex);
/**
* 获取参与事件数
*
* @param channelIndex 渠道标识
* @return 参与事件数
*/
Map<Long, List<Event>> getEventDay(ChannelIndex channelIndex, Long startTime, Long endTime);
/**
* 获取参与事件数
*
* @param channelIndex 渠道标识
* @return 参与事件数
*/
Map<Long, List<Event>> getEventMonth(ChannelIndex channelIndex, Long startTime, Long endTime);
}
src/main/java/com/zhiwei/brandkbs2/dao/impl/ChannelDaoImpl.java
......@@ -3,10 +3,18 @@ package com.zhiwei.brandkbs2.dao.impl;
import com.zhiwei.brandkbs2.dao.ChannelDao;
import com.zhiwei.brandkbs2.pojo.Channel;
import com.zhiwei.brandkbs2.pojo.ChannelIndex;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.stereotype.Component;
import java.util.*;
import java.util.concurrent.CompletableFuture;
/**
* @ClassName: ChannelDaoImpl
* @Description ChannelDaoImpl
......@@ -15,11 +23,14 @@ import org.springframework.stereotype.Component;
*/
@Component("channelDao")
public class ChannelDaoImpl extends BaseMongoDaoImpl<Channel> implements ChannelDao {
private static final Logger log = LogManager.getLogger(ChannelDaoImpl.class);
private static final String COLLECTION_PREFIX = "brandkbs_channel";
private final ThreadPoolTaskExecutor executor;
public ChannelDaoImpl() {
public ChannelDaoImpl(@Qualifier("mongoQueryExecutor") ThreadPoolTaskExecutor executor) {
super(COLLECTION_PREFIX);
this.executor = executor;
}
@Override
......@@ -41,4 +52,24 @@ public class ChannelDaoImpl extends BaseMongoDaoImpl<Channel> implements Channel
return queryUnique(channelIndex.getFid());
}
@Override
public Map<String, Channel> queryUniqueAsync(Collection<String> channelFids) {
Map<String, Channel> res = new HashMap<>();
List<CompletableFuture<Pair<String, Channel>>> futureList = new ArrayList<>();
for (String fid : channelFids) {
futureList.add(CompletableFuture.supplyAsync(() -> Pair.of(fid, queryUnique(fid)), executor));
}
CompletableFuture.allOf(futureList.toArray(new CompletableFuture[0])).whenComplete((r, e) -> {
for (CompletableFuture<Pair<String, Channel>> future : futureList) {
try {
Pair<String, Channel> channelPair = future.get();
res.put(channelPair.getLeft(), channelPair.getRight());
} catch (Exception e1) {
log.error("queryUniqueAsync", e1);
}
}
}).join();
return res;
}
}
src/main/java/com/zhiwei/brandkbs2/dao/impl/EventDaoImpl.java
......@@ -11,7 +11,9 @@ import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Component;
import java.util.Calendar;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import static com.zhiwei.brandkbs2.dao.impl.EventDataDaoImpl.COLLECTION_PREFIX;
......@@ -54,6 +56,29 @@ public class EventDaoImpl extends BaseMongoDaoImpl<Event> implements EventDao {
return mappedResults.stream().map(json -> json.getString("_id")).collect(Collectors.toList());
}
@Override
public Map<Long, List<Event>> getEventDay(ChannelIndex channelIndex, Long startTime, Long endTime) {
return getEventTimePattern(channelIndex, startTime, endTime, 8);
}
@Override
public Map<Long, List<Event>> getEventMonth(ChannelIndex channelIndex, Long startTime, Long endTime) {
return getEventTimePattern(channelIndex, startTime, endTime, 6);
}
private Map<Long, List<Event>> getEventTimePattern(ChannelIndex channelIndex, Long startTime, Long endTime, int nrOfChars) {
// 添加渠道唯一标识
Criteria criteria = addChannelIndex(channelIndex);
// 分组
Aggregation agg = Aggregation.newAggregation(Aggregation.match(criteria),
// 截取前8位,按日分组
Aggregation.project("startTime", "eventId", "emotion").andExpression("add(new java.util.Date(0l),startTime)").substring(0, nrOfChars).as("patternDate"),
Aggregation.group("patternDate").count().as("eventCount"));
AggregationResults<JSONObject> aggregate = mongoTemplate.aggregate(agg, getAggreeCollection(), JSONObject.class);
List<JSONObject> mappedResults = aggregate.getMappedResults();
return new HashMap<>();
}
private String getAggreeCollection() {
Calendar date = Calendar.getInstance();
int year = date.get(Calendar.YEAR);
......
src/main/java/com/zhiwei/brandkbs2/easyexcel/dto/ExportAppChannelArticleDTO.java 0 → 100644
package com.zhiwei.brandkbs2.easyexcel.dto;
import com.alibaba.excel.annotation.ExcelProperty;
import com.zhiwei.brandkbs2.enmus.EmotionEnum;
import com.zhiwei.brandkbs2.pojo.ChannelIndex;
import lombok.Data;
import lombok.ToString;
import java.util.Date;
/**
* @author lxj
* @version 1.0
* @description 前台导出渠道稿件实体类
* @date 2020/3/30 16:31
*/
@Data
@ToString
public class ExportAppChannelArticleDTO {
@ExcelProperty("时间")
private Date time;
@ExcelProperty("链接")
private String url;
@ExcelProperty("标题")
private String title;
@ExcelProperty("正文")
private String content;
@ExcelProperty("情感倾向")
private String emotion;
public static ExportAppChannelArticleDTO createFromArticle(ChannelIndex.Article article, String url, String title) {
ExportAppChannelArticleDTO dto = new ExportAppChannelArticleDTO();
dto.setTime(new Date(article.getTime()));
dto.setUrl(url);
dto.setTitle(title);
dto.setEmotion(EmotionEnum.state2Name(article.getEmotion()));
return dto;
}
}
src/main/java/com/zhiwei/brandkbs2/enmus/EmotionEnum.java
......@@ -121,4 +121,13 @@ public enum EmotionEnum {
return 0;
}
public static String state2Name(int state) {
for (EmotionEnum value : EmotionEnum.values()) {
if (value.getState() == state) {
return value.getName();
}
}
return EmotionEnum.UNDEFINED.getName();
}
}
src/main/java/com/zhiwei/brandkbs2/es/EsClientDao.java
......@@ -2,7 +2,6 @@ package com.zhiwei.brandkbs2.es;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.common.GenericAttribute;
import com.zhiwei.brandkbs2.enmus.EmotionEnum;
import com.zhiwei.brandkbs2.pojo.ChannelIndex;
import com.zhiwei.brandkbs2.pojo.Event;
import com.zhiwei.brandkbs2.util.Tools;
......@@ -52,7 +51,8 @@ import java.util.stream.Collectors;
public class EsClientDao {
private static final Logger log = LogManager.getLogger(EsClientDao.class);
protected static final FastDateFormat DF = FastDateFormat.getInstance("yyyy-MM-dd HH:mm:ss");
private static final String[] CHANNEL_RECORD_FETCH_SOURCE = new String[]{"id", "c5", "foreign", "real_source", "source", "mtime", "time", "brandkbs_cache_maps"};
private static final String[] CHANNEL_RECORD_FETCH_SOURCE = new String[]{"id", "c5", "foreign", "real_source", "source", "mtime", "time",
"brandkbs_cache_maps", "brandkbs_mark_cache_maps"};
private static final String[] EVENT_FETCH_SOURCE = new String[]{"ind_full_text", "c5", "real_source", "source", "mtime", "time", "url", "mtag"};
private static final Long ONE_HOUR = 60 * 60 * 1000L;
......@@ -71,6 +71,10 @@ public class EsClientDao {
@Resource(name = "retryTemplate")
RetryTemplate retryTemplate;
public JSONObject searchById(String queryId) throws IOException {
return searchByIds(Collections.singleton(queryId)).get(queryId);
}
public Map<String, JSONObject> searchByIds(Collection<String> queryIds) throws IOException {
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
QueryBuilder queryBuilder = QueryBuilders.idsQuery().addIds(queryIds.toArray(new String[0]));
......@@ -180,8 +184,7 @@ public class EsClientDao {
v = new ChannelIndex.Record();
}
try {
return v.mergeRecord(new ChannelIndex.Record((long) result.get(GenericAttribute.ES_TIME), (long) result.get(GenericAttribute.ES_MTIME),
String.valueOf(result.get("id")), EmotionEnum.parseFromName(Tools.getEmotion(result))));
return v.mergeRecord(new ChannelIndex.Record(result));
} catch (Exception e) {
log.error("searchRecord-error-id:{}", result.get("id"), e);
return null;
......
src/main/java/com/zhiwei/brandkbs2/es/EsQueryTools.java
......@@ -124,12 +124,12 @@ public class EsQueryTools {
public static void assembleContendsQuery(BoolQueryBuilder query, Collection<String> contends) {
BoolQueryBuilder contendQuery = QueryBuilders.boolQuery();
// 主品牌一定参与
contendQuery.should(QueryBuilders.termQuery("contend_id", "0"));
contendQuery.should(QueryBuilders.termQuery("contend_id.keyword", "0"));
if (null == contends) {
return;
}
for (String contendId : contends) {
contendQuery.should((QueryBuilders.termQuery("contend_id", contendId)));
contendQuery.should((QueryBuilders.termQuery("contend_id.keyword", contendId)));
}
query.must(contendQuery);
}
......
src/main/java/com/zhiwei/brandkbs2/pojo/Channel.java
......@@ -4,6 +4,7 @@ import com.zhiwei.brandkbs2.enmus.ChannelEmotion;
import com.zhiwei.brandkbs2.util.Tools;
import lombok.Getter;
import lombok.Setter;
import org.bson.types.ObjectId;
import java.util.Date;
......@@ -43,16 +44,6 @@ public class Channel extends ChannelIndex {
private long eventCount;
/**
* 渠道倾向
*/
private int emotion;
/**
* 渠道指数
*/
private Double emotionIndex = 0d;
/**
* 经验评级
*/
private String experienceLevel;
......@@ -97,6 +88,7 @@ public class Channel extends ChannelIndex {
public static Channel createFromChannelIndexRecord(ChannelIndex channelIndex, Record record) {
Channel channel = new Channel();
channel.setId(ObjectId.get().toString());
channel.setCTime(new Date().getTime());
channel.setProjectId(channelIndex.getProjectId());
channel.setLinkedGroupId(channelIndex.getLinkedGroupId());
......@@ -117,6 +109,8 @@ public class Channel extends ChannelIndex {
} else {
channel.setEmotion(ChannelEmotion.NEUTRAL.getState());
}
// TODO 情感指数随机分配
channel.setEmotionIndex(Math.random() * 100);
return channel;
}
......
src/main/java/com/zhiwei/brandkbs2/pojo/ChannelIndex.java
......@@ -2,6 +2,7 @@ package com.zhiwei.brandkbs2.pojo;
import com.zhiwei.brandkbs2.common.GenericAttribute;
import com.zhiwei.brandkbs2.common.GlobalPojo;
import com.zhiwei.brandkbs2.enmus.EmotionEnum;
import com.zhiwei.brandkbs2.util.Tools;
import com.zhiwei.qbjc.bean.pojo.common.MessagePlatform;
import com.zhiwei.qbjc.bean.tools.BeanTools;
......@@ -54,6 +55,17 @@ public class ChannelIndex extends AbstractBaseMongo {
private String fid;
private int emotion;
/**
* 情感指数
*/
private Double emotionIndex;
public ChannelIndex(String projectId, String linkedGroupId, Channel channel) {
this(projectId, linkedGroupId, channel.getPlatform(), channel.getRealSource(), channel.getSource());
}
public ChannelIndex(String projectId, String linkedGroupId, String platform, String realSource, String source) {
this.projectId = projectId;
this.linkedGroupId = linkedGroupId;
......@@ -117,6 +129,12 @@ public class ChannelIndex extends AbstractBaseMongo {
return mergeRecord;
}
public void setChannelInfo(Channel channel) {
this.id = channel.getId();
this.emotion = channel.getEmotion();
this.emotionIndex = channel.getEmotionIndex();
}
@Setter
@Getter
@AllArgsConstructor
......@@ -128,9 +146,9 @@ public class ChannelIndex extends AbstractBaseMongo {
public Record() {
}
public Record(Long lastTime, Long mtime, String articleId, int emotion) {
this.lastTime = lastTime;
this.articles.add(new Article(lastTime, mtime, articleId, emotion));
public Record(Map<String, Object> esMap) {
this.lastTime = Long.parseLong(esMap.get("time") + "");
this.articles.add(Article.fromEsMap(esMap));
}
public void setLastTime(Long lastTime) {
......@@ -145,6 +163,33 @@ public class ChannelIndex extends AbstractBaseMongo {
return this;
}
public static List<Article> sortArticles(List<Article> articles) {
articles.sort(Comparator.comparingLong(ChannelIndex.Article::getTime));
return articles;
}
public static List<Article> filterArticles(Long startTime, Long endTime, List<Article> articles) {
// 去除不符合时间段数据
articles = articles.stream().filter(article -> Tools.hitTimeRange(startTime, endTime, article.getTime())).collect(Collectors.toList());
// 去重并保留最近标注时间
Map<String, ChannelIndex.Article> setMap = new HashMap<>();
for (ChannelIndex.Article article : articles) {
setMap.compute(article.getId(), (k, v) -> {
// 旧值为null或标注时间更新
if (null == v || article.getMtime() > v.getMtime()) {
return article;
}
return v;
});
}
return new ArrayList<>(setMap.values());
}
public static List<Article> filterSortArticles(Long startTime, Long endTime, List<Article> articles) {
return sortArticles(filterArticles(startTime, endTime, articles));
}
public Map<String, Object> toEsMap() {
Map<String, Object> map = new HashMap<>();
map.put("last_time", lastTime);
......@@ -160,11 +205,29 @@ public class ChannelIndex extends AbstractBaseMongo {
Long time;
Long mtime;
String id;
// String url;
// String title;
int emotion;
public static Article fromRecordMap(Map<String, Object> recordMap) {
Long time = Long.parseLong(recordMap.get("time") + "");
Long mtime = Long.parseLong(recordMap.get("mtime") + "");
String id = String.valueOf(recordMap.get("id"));
// String url = recordMap.get("url") + "";
// String title = recordMap.get("title") + "";
int emotion = Integer.parseInt(recordMap.get("emotion") + "");
return new Article(time, mtime, id, emotion);
}
public static Article fromEsMap(Map<String, Object> esMap) {
return new Article(Long.parseLong(esMap.get("time") + ""), Long.parseLong(esMap.get("mtime") + ""), String.valueOf(esMap.get("id")), Integer.parseInt(
esMap.get("emotion") + ""));
BaseMap baseMap = Tools.getBaseFromEsMap(esMap);
Long time = baseMap.getTime();
Long mtime = Long.parseLong(esMap.get("mtime") + "");
String id = String.valueOf(esMap.get("id"));
// String url = baseMap.getUrl();
// String title = baseMap.getTitle();
String emotionStr = baseMap.getEmotion();
return new Article(time, mtime, id, EmotionEnum.parseFromName(emotionStr));
}
public Map<String, Object> toEsMap() {
......@@ -172,6 +235,8 @@ public class ChannelIndex extends AbstractBaseMongo {
map.put("time", time);
map.put("mtime", mtime);
map.put("id", id);
// map.put("url", url);
// map.put("title", title);
map.put("emotion", emotion);
return map;
}
......
src/main/java/com/zhiwei/brandkbs2/pojo/ChannelRecord.java
......@@ -34,7 +34,10 @@ public class ChannelRecord {
public static final String CHANNEL_FID = "channel_fid";
public static final String RECORD = "record";
public static final String ARTICLE_COUNT = "article_count";
public static final String EMOTION = "emotion";
public static final String EMOTION_INDEX = "emotion_index";
public static final String SHARDS = "shards";
private static final Integer SHARDS_LIMIT = 50;
/**
......@@ -82,6 +85,21 @@ public class ChannelRecord {
private String channelFid;
/**
* 渠道id
*/
private String channelId;
/**
* 情感倾向
*/
private int emotion;
/**
* 情感指数
*/
private Double emotionIndex;
/**
* 时间段内唯一id
*/
private String key;
......@@ -89,7 +107,7 @@ public class ChannelRecord {
/**
* articleCount
*/
private int articleCount;
private long articleCount;
/**
* 记录信息
*/
......@@ -110,6 +128,9 @@ public class ChannelRecord {
this.realSource = channelIndex.getRealSource();
this.source = channelIndex.getSource();
this.channelFid = channelIndex.getFid();
this.emotion = channelIndex.getEmotion();
this.emotionIndex = channelIndex.getEmotionIndex();
this.channelId = channelIndex.getId();
this.key = Tools.concat(channelFid, contendId);
this.articleCount = record.getArticles().size();
this.record = record;
......@@ -125,7 +146,7 @@ public class ChannelRecord {
List<ChannelIndex.Article> articles = templateRecord.getArticles();
int shards = 0;
// 防止数据量过大无法存储,故按100 拆分
for (List<ChannelIndex.Article> partList : Lists.partition(articles, 100)) {
for (List<ChannelIndex.Article> partList : Lists.partition(articles, SHARDS_LIMIT)) {
ChannelRecord channelRecord = new ChannelRecord(rangeStartTime, rangeEndTime, entry.getKey(), entry.getValue());
// 保留最近发文时间,更新partList,articleCount和shards
channelRecord.setRecord(new ChannelIndex.Record(lastTime, partList));
......@@ -149,6 +170,9 @@ public class ChannelRecord {
this.contendId = json.getString("contend_id");
this.channelFid = json.getString("channel_fid");
this.key = json.getString("key");
this.emotion = json.getIntValue("emotion");
this.emotionIndex = json.getDoubleValue("emotion_index");
this.channelId = json.getString("channel_id");
this.articleCount = json.getIntValue("article_count");
this.shards = json.getIntValue("shards");
......@@ -157,7 +181,7 @@ public class ChannelRecord {
ChannelIndex.Record record = new ChannelIndex.Record();
record.setLastTime(Long.valueOf(map.get("last_time") + ""));
List<Map<String, Object>> list = (List<Map<String, Object>>) map.get("articles");
record.setArticles(list.stream().map(ChannelIndex.Article::fromEsMap).collect(Collectors.toList()));
record.setArticles(list.stream().map(ChannelIndex.Article::fromRecordMap).collect(Collectors.toList()));
this.record = record;
}
......@@ -180,10 +204,19 @@ public class ChannelRecord {
esMap.put(CHANNEL_FID, channelFid);
esMap.put(RECORD, record.toEsMap());
esMap.put(ARTICLE_COUNT, articleCount);
esMap.put(EMOTION, emotion);
esMap.put(EMOTION_INDEX, emotionIndex);
esMap.put("channel_id", channelId);
esMap.put(SHARDS, shards);
return esMap;
}
public void setChannelInfo(ChannelRecord channelRecord) {
this.channelId = channelRecord.getChannelId();
this.emotion = channelRecord.getEmotion();
this.emotionIndex = channelRecord.getEmotionIndex();
}
@Deprecated
public static List<ChannelRecord> createChannelRecords(long rangeStartTime, long rangeEndTime, @NonNull Map<ChannelIndex, ChannelIndex.Record> records) {
// 通过platform+realSource+source 合并数据
......
src/main/java/com/zhiwei/brandkbs2/pojo/vo/ChannelListVO.java
......@@ -2,6 +2,7 @@ package com.zhiwei.brandkbs2.pojo.vo;
import com.zhiwei.brandkbs2.enmus.ChannelEmotion;
import com.zhiwei.brandkbs2.pojo.Channel;
import com.zhiwei.brandkbs2.pojo.ChannelRecord;
import com.zhiwei.brandkbs2.util.Tools;
import lombok.Data;
import lombok.ToString;
......@@ -36,6 +37,11 @@ public class ChannelListVO {
* 发文数
*/
private long articleCount;
/**
* 参与事件数
*/
private long eventCount;
/**
* 情感倾向
*/
......@@ -45,6 +51,8 @@ public class ChannelListVO {
*/
private String avatarUrl;
private Boolean isCollect;
public static ChannelListVO createFromChannel(String channelFid, long articleCount) {
ChannelListVO channelListVO = new ChannelListVO();
// projectId, linkedGroupId, platform, realSource, source
......@@ -64,4 +72,13 @@ public class ChannelListVO {
channelListVO.setArticleCount(articleCount);
return channelListVO;
}
public static ChannelListVO createFromChannel(ChannelRecord channelRecord, long articleCount) {
ChannelListVO channelListVO = Tools.convertMap(channelRecord, ChannelListVO.class);
String name = ChannelEmotion.getNameFromState(channelRecord.getEmotion());
channelListVO.setShowEmotion(name);
channelListVO.setArticleCount(articleCount);
return channelListVO;
}
}
src/main/java/com/zhiwei/brandkbs2/service/ChannelService.java
......@@ -3,6 +3,7 @@ package com.zhiwei.brandkbs2.service;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportAdminChannelArticleDTO;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportAdminChannelEventDTO;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportAppChannelArticleDTO;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportChannelDTO;
import com.zhiwei.brandkbs2.pojo.dto.ChannelDTO;
import com.zhiwei.brandkbs2.pojo.vo.ChannelListVO;
......@@ -131,7 +132,7 @@ public interface ChannelService {
* @param size
* @return
*/
List<ChannelListVO> getPositiveList(String contendId, String platform, String keyword, String sorter, int size);
List<ChannelListVO> getPositiveList(String contendId, String platform, String keyword, String sorter, Long startTime, Long endTime, int size);
/**
* 获取敏感渠道榜
......@@ -143,7 +144,7 @@ public interface ChannelService {
* @param size
* @return
*/
List<ChannelListVO> getNegativeList(String contendId, String platform, String keyword, String sorter, int size);
List<ChannelListVO> getNegativeList(String contendId, String platform, String keyword, String sorter, Long startTime, Long endTime, int size);
/**
* 收藏渠道
......@@ -183,4 +184,42 @@ public interface ChannelService {
* @return 渠道动向
*/
JSONObject getSpreadingTend(String channelId, String type, Set<String> contends, Long startTime, Long endTime);
/**
* 获取渠道动向-摘要
*
* @param channelId 渠道ID
* @param contends 品牌ID集合
* @param startTime 开始时间
* @param endTime 结束时间
* @return 渠道动向-摘要
*/
JSONObject getSpreadingTendSummary(String channelId, Set<String> contends, Long startTime, Long endTime);
/**
* 获取时间段稿件信息
*
* @param startTime 开始时间时间戳
* @param endTime 结束时间
* @param page 页码
* @param pageSize 页码大小
* @param channelId 渠道ID
* @param contendId 竞品ID
* @return 稿件信息
*/
JSONObject getArticlesByTime(Long startTime, Long endTime, int page, int pageSize, String channelId, String contendId);
/**
* 下载时间段稿件信息
*
* @param startTime 开始时间时间戳
* @param endTime 结束时间
* @param page 页码
* @param pageSize 页码大小
* @param channelId 渠道ID
* @param contendId 竞品ID
* @return 稿件信息
*/
List<ExportAppChannelArticleDTO> downloadArticlesByTime(Long startTime, Long endTime,String channelId, String contendId);
}
src/main/java/com/zhiwei/brandkbs2/service/impl/ChannelServiceImpl.java
......@@ -2,6 +2,7 @@ package com.zhiwei.brandkbs2.service.impl;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.zhiwei.brandkbs2.auth.UserThreadLocal;
import com.zhiwei.brandkbs2.common.GenericAttribute;
import com.zhiwei.brandkbs2.common.GlobalPojo;
......@@ -9,6 +10,7 @@ import com.zhiwei.brandkbs2.config.Constant;
import com.zhiwei.brandkbs2.dao.*;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportAdminChannelArticleDTO;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportAdminChannelEventDTO;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportAppChannelArticleDTO;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportChannelDTO;
import com.zhiwei.brandkbs2.enmus.ChannelEmotion;
import com.zhiwei.brandkbs2.enmus.EmotionEnum;
......@@ -28,8 +30,10 @@ import com.zhiwei.brandkbs2.service.CommonService;
import com.zhiwei.brandkbs2.service.ProjectService;
import com.zhiwei.brandkbs2.util.MongoUtil;
import com.zhiwei.brandkbs2.util.Tools;
import io.netty.util.concurrent.CompleteFuture;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.search.SearchResponse;
......@@ -51,6 +55,7 @@ import java.io.IOException;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
/**
......@@ -284,7 +289,7 @@ public class ChannelServiceImpl implements ChannelService {
List<ChannelListVO> resList = new ArrayList<>();
try {
String projectId = UserThreadLocal.getProjectId();
EsClientDao.SearchHelper searchHelper = createSearchHelperByChannelCriteria(projectId, contendId, platform, keyword, startTime, endTime);
EsClientDao.SearchHelper searchHelper = createSearchHelperByChannelCriteria(projectId, null, Collections.singleton(contendId), platform, keyword, startTime, endTime);
// 只需要聚合结果,不需返回hit
searchHelper.setSize(0);
// aggregation
......@@ -296,34 +301,125 @@ public class ChannelServiceImpl implements ChannelService {
searchHelper.setAggregationBuilder(aggregationBuilder);
SearchResponse searchResponse = channelEsDao.searchResponse(searchHelper);
Terms keyTerms = searchResponse.getAggregations().get("keyAgg");
Map<String, Long> fidCount = new HashMap<>();
for (Terms.Bucket bucket : keyTerms.getBuckets()) {
String channelFid = Tools.getFidFromChannelKey(bucket.getKeyAsString());
Sum countSum = bucket.getAggregations().get("countAgg");
Channel channel = channelDao.queryUnique(channelFid);
fidCount.put(channelFid, (long) countSum.getValue());
// Channel channel = channelDao.queryUnique(channelFid);
// if (null != channel) {
// resList.add(ChannelListVO.createFromChannel(channel, (long) countSum.getValue()));
// } else {
// log.error("getActiveChannelList-未收录渠道:{}", bucket.getKeyAsString());
// resList.add(ChannelListVO.createFromChannel(bucket.getKeyAsString(), (long) countSum.getValue()));
// }
}
Map<String, Channel> fidChannel = channelDao.queryUniqueAsync(fidCount.keySet());
fidCount.forEach((fid, count) -> {
Channel channel = fidChannel.get(fid);
if (null != channel) {
resList.add(ChannelListVO.createFromChannel(channel, (long) countSum.getValue()));
resList.add(ChannelListVO.createFromChannel(channel, count));
} else {
log.error("getActiveChannelList-未收录渠道:{}", bucket.getKeyAsString());
resList.add(ChannelListVO.createFromChannel(bucket.getKeyAsString(), (long) countSum.getValue()));
log.error("getActiveChannelList-未收录渠道:{}", fid);
resList.add(ChannelListVO.createFromChannel(fid, count));
}
}
return resList;
});
} catch (IOException e) {
ExceptionCast.cast(CommonCodeEnum.FAIL.message("es查询异常"));
}
return null;
return resList;
}
@Override
public List<ChannelListVO> getPositiveList(String contendId, String platform, String keyword, String sorter, int size) {
// TODO
return null;
public List<ChannelListVO> getPositiveList(String contendId, String platform, String keyword, String sorter, Long startTime, Long endTime, int size) {
return getEmotionList(contendId, platform, keyword, sorter, startTime, endTime, size, EmotionEnum.POSITIVE.getState());
}
@Override
public List<ChannelListVO> getNegativeList(String contendId, String platform, String keyword, String sorter, int size) {
// TODO
return null;
public List<ChannelListVO> getNegativeList(String contendId, String platform, String keyword, String sorter, Long startTime, Long endTime, int size) {
return getEmotionList(contendId, platform, keyword, sorter, startTime, endTime, size, EmotionEnum.NEGATIVE.getState());
}
private List<ChannelListVO> getEmotionList(String contendId, String platform, String keyword, String sorter, Long startTime, Long endTime, int size,
int emotion) {
List<ChannelListVO> resList = new ArrayList<>();
try {
Map<String, Pair<Long, ChannelRecord>> keyMap = new HashMap<>();
String projectId = UserThreadLocal.getProjectId();
EsClientDao.SearchHelper searchHelper = createSearchHelperByChannelCriteria(projectId, null, Collections.singleton(contendId), platform, keyword, startTime, endTime);
// 分页查询所有结果
List<SearchResponse> searchResponses = channelEsDao.searchScrollResponse(searchHelper);
for (SearchResponse searchResponse : searchResponses) {
for (SearchHit hit : searchResponse.getHits().getHits()) {
ChannelRecord channelRecord = new ChannelRecord(hit.getSourceAsMap());
keyMap.compute(channelRecord.getKey(), (k, v) -> {
if (null == v) {
return Pair.of(channelRecord.getRangeEndTime(), channelRecord);
} else {
// 比较数据时间
if (channelRecord.getRangeEndTime() > v.getLeft()) {
v.getRight().setChannelInfo(channelRecord);
}
// 合并文章数据
v.getRight().getRecord().mergeRecord(channelRecord.getRecord());
return v;
}
});
}
}
// 过滤掉不符合时间条件的数据并排序
List<ChannelRecord> channelRecords = keyMap.values().stream().map(pair -> {
ChannelRecord channelRecord = pair.getRight();
// 情感过滤
if (emotion == channelRecord.getEmotion()) {
List<ChannelIndex.Article> articles = ChannelIndex.Record.filterArticles(startTime, endTime, channelRecord.getRecord().getArticles());
articles.sort(Comparator.comparingLong(ChannelIndex.Article::getTime).reversed());
channelRecord.getRecord().setArticles(articles);
return channelRecord;
}
return null;
}).filter(Objects::nonNull).sorted((x, y) -> {
// emotionIndex降序
if (sorter.contains("index")) {
if (x.getEmotionIndex() > y.getEmotionIndex()) {
return -1;
} else if (Objects.equals(x.getEmotionIndex(), y.getEmotionIndex())) {
return 0;
}
return 1;
}
// 稿件数排序
if (x.getRecord().getArticles().size() > y.getRecord().getArticles().size()) {
return -1;
} else if (Objects.equals(x.getRecord().getArticles().size(), y.getRecord().getArticles().size())) {
return 0;
}
return 1;
}).limit(size)
// .map(channelRecord -> {
// Channel channel = channelDao.queryUnique(channelRecord.getChannelFid());
// if (null != channel) {
// return ChannelListVO.createFromChannel(channel, channelRecord.getRecord().getArticles().size());
// } else {
// log.error("getActiveChannelList-未收录渠道:{}", channelRecord.getChannelFid());
// return ChannelListVO.createFromChannel(channelRecord, channelRecord.getRecord().getArticles().size());
// }
// })
.collect(Collectors.toList());
Map<String, ChannelRecord> collect = Maps.uniqueIndex(channelRecords, ChannelRecord::getChannelFid);
Map<String, Channel> fidChannel = channelDao.queryUniqueAsync(collect.keySet());
collect.forEach((fid, record) -> {
Channel channel = fidChannel.get(fid);
if (null != channel) {
resList.add(ChannelListVO.createFromChannel(channel, record.getRecord().getArticles().size()));
} else {
resList.add(ChannelListVO.createFromChannel(record, record.getRecord().getArticles().size()));
}
});
} catch (IOException e) {
ExceptionCast.cast(CommonCodeEnum.FAIL.message("es查询异常"));
}
return resList;
}
@Override
......@@ -384,74 +480,219 @@ public class ChannelServiceImpl implements ChannelService {
@Override
public JSONObject getSpreadingTend(String channelId, String type, Set<String> contends, Long startTime, Long endTime) {
try {
JSONObject res = new JSONObject();
// 默认搜索一周
if (null == startTime || null == endTime) {
Long[] timeRangeWeek = commonService.getTimeRangeWeek();
startTime = timeRangeWeek[0];
endTime = timeRangeWeek[1];
JSONObject res = new JSONObject();
// 默认搜索一周
if (null == startTime || null == endTime) {
Long[] timeRangeWeek = commonService.getTimeRangeWeek();
startTime = timeRangeWeek[0];
endTime = timeRangeWeek[1];
}
List<JSONObject> spreadDatas = new ArrayList<>();
if ("事件".equals(type)) {
// 根据品牌分类
for (String contend : contends) {
JSONObject json = new JSONObject();
Channel channel = channelDao.findOneById(channelId);
json.put("spreadingTend", spreadingTendEvent(startTime, endTime, channel, UserThreadLocal.getProjectId(), contend));
if ("0".equals(contend)) {
spreadDatas.add(0, json);
} else {
spreadDatas.add(json);
}
}
Channel channel = channelDao.findOneById(channelId);
EsClientDao.SearchHelper searchHelper = createSearchHelperByChannelCriteria(UserThreadLocal.getProjectId(), null, null, null, startTime, endTime);
// 添加contends集合 查询
EsQueryTools.assembleContendsQuery(searchHelper.getQuery(), contends);
// 添加渠道限制
searchHelper.getQuery().must(QueryBuilders.termQuery("channel_fid.keyword", channel.getFid()));
} else {
for (Map.Entry<String, List<ChannelIndex.Article>> entry : getSourceContendMap(channelId, contends, startTime, endTime).entrySet()) {
JSONObject json = new JSONObject();
json.put("spreadingTend", spreadingTendData(startTime, endTime, entry.getValue()));
if ("0".equals(entry.getKey())) {
spreadDatas.add(0, json);
} else {
spreadDatas.add(json);
}
}
}
res.put("spreadDatas", spreadDatas);
return res;
}
@Override
public JSONObject getSpreadingTendSummary(String channelId, Set<String> contends, Long startTime, Long endTime) {
JSONObject res = new JSONObject();
String projectId = UserThreadLocal.getProjectId();
// 默认搜索一周
if (null == startTime || null == endTime) {
Long[] timeRangeWeek = commonService.getTimeRangeWeek();
startTime = timeRangeWeek[0];
endTime = timeRangeWeek[1];
}
// 根据品牌分类
Map<String, List<ChannelIndex.Article>> contendMap = getSourceContendMap(channelId, contends, startTime, endTime);
Channel channel = channelDao.findOneById(channelId);
long articleTotal = 0;
long eventTotal = 0;
List<JSONObject> spreadDatas = new ArrayList<>();
for (Map.Entry<String, List<ChannelIndex.Article>> entry : contendMap.entrySet()) {
String contendId = entry.getKey();
Pair<Long, JSONObject> dataCount = getDataCount(entry.getValue());
Pair<Long, JSONObject> eventCount = getEventCount(startTime, endTime, projectId, contendId, channel);
articleTotal += dataCount.getLeft();
eventTotal += eventCount.getLeft();
// 合并统计
dataCount.getRight().putAll(eventCount.getRight());
if ("0".equals(contendId)) {
spreadDatas.add(0, dataCount.getRight());
} else {
spreadDatas.add(dataCount.getRight());
}
}
res.put("startTime", startTime);
res.put("endTime", endTime);
res.put("articleTotal", articleTotal);
res.put("eventTotal", eventTotal);
res.put("spreadDatas", spreadDatas);
return res;
}
@Override
public JSONObject getArticlesByTime(Long startTime, Long endTime, int page, int pageSize, String channelId, String contendId) {
JSONObject res = new JSONObject();
Map<String, List<ChannelIndex.Article>> sourceContendMap = getSourceContendMap(channelId, Collections.singleton(contendId), startTime, endTime);
Map<Long, List<ChannelIndex.Article>> timeRangeList = partition(startTime, endTime, Constant.DAY_PATTERN, sourceContendMap.get(contendId));
List<JSONObject> dayList = new ArrayList<>(timeRangeList.size());
timeRangeList.forEach((time, list) -> {
JSONObject dayResult = new JSONObject();
// 每天返回前10条
List<CompletableFuture<JSONObject>> futureList = list.stream().limit(10).map(article -> CompletableFuture.supplyAsync(() -> {
JSONObject json = new JSONObject();
json.put("emotion", article.getEmotion());
json.put("time", article.getTime());
String[] titleAndUrl = getTitleAndUrlById(article.getId());
json.put("title", titleAndUrl[0]);
json.put("url", titleAndUrl[1]);
return json;
})).collect(Collectors.toList());
CompletableFuture.allOf(futureList.toArray(new CompletableFuture[0])).join();
dayResult.put("articles", futureList.stream().map(CompletableFuture::join).collect(Collectors.toList()));
dayResult.put("articleCount", list.size());
dayResult.put("time", time);
dayList.add(dayResult);
});
res.put("list", dayList);
return res;
}
@Override
public List<ExportAppChannelArticleDTO> downloadArticlesByTime(Long startTime, Long endTime, String channelId, String contendId) {
List<ExportAppChannelArticleDTO> res = new ArrayList<>();
List<CompleteFuture<ExportAppChannelArticleDTO>> futureList = new ArrayList<>();
Map<String, List<ChannelIndex.Article>> sourceContendMap = getSourceContendMap(channelId, Collections.singleton(contendId), startTime, endTime);
for (ChannelIndex.Article article : sourceContendMap.get(contendId)) {
// res.add(ExportAppChannelArticleDTO.createFromArticle(article));
}
return res;
}
private Map<String, List<ChannelIndex.Article>> getSourceContendMap(String channelId, Set<String> contendIds, String platform, String keyword, Long startTime,
Long endTime) {
try {
String projectId = UserThreadLocal.getProjectId();
EsClientDao.SearchHelper searchHelper = createSearchHelperByChannelCriteria(projectId, channelId, contendIds, platform, keyword, startTime, endTime);
// 分页查询所有结果
List<SearchResponse> searchResponses = channelEsDao.searchScrollResponse(searchHelper);
// 根据品牌分类
Map<String, List<ChannelIndex.Article>> contendMap = convert2ContendMap(searchResponses, startTime, endTime);
long articleTotal = 0;
List<JSONObject> spreadDatas = new ArrayList<>();
for (Map.Entry<String, List<ChannelIndex.Article>> entry : contendMap.entrySet()) {
JSONObject spreadData = new JSONObject();
// 按日分组并根据id去重保留最近标注时间
Map<Long, List<ChannelIndex.Article>> dateListMap = partitionWithDuplicate(startTime, endTime, entry.getValue());
List<JSONObject> spread = dateListMap.entrySet().stream().map(e -> {
JSONObject spreadJson = new JSONObject();
spreadJson.put("time", e.getKey());
spreadJson.put("sum", e.getValue().size());
return spreadJson;
}).collect(Collectors.toList());
spreadData.put("spreadingTend", spread);
// 各稿件数统计
long positiveCount = 0;
long negativeCount = 0;
long total = 0;
for (List<ChannelIndex.Article> articles : dateListMap.values()) {
for (ChannelIndex.Article article : articles) {
if (article.getEmotion() == EmotionEnum.POSITIVE.getState()) {
positiveCount++;
} else if (article.getEmotion() == EmotionEnum.NEGATIVE.getState()) {
negativeCount++;
}
total++;
}
}
spreadData.put("positiveCount", positiveCount);
spreadData.put("negativeCount", negativeCount);
spreadData.put("articleTotal", total);
spreadData.put("positivePercent", new BigDecimal((double) positiveCount * 100 / total).setScale(1, RoundingMode.UP));
spreadData.put("negativePercent", new BigDecimal((double) negativeCount * 100 / total).setScale(1, RoundingMode.UP));
articleTotal += dateListMap.values().stream().mapToInt(List::size).sum();
spreadDatas.add(spreadData);
}
res.put("startTime", startTime);
res.put("endTime", endTime);
res.put("articleTotal", articleTotal);
res.put("eventTotal", 0);
res.put("spreadDatas", spreadDatas);
return res;
return convert2ContendMap(searchResponses, startTime, endTime);
} catch (IOException e) {
ExceptionCast.cast(CommonCodeEnum.FAIL.message("es查询异常"));
}
return null;
return Collections.emptyMap();
}
private Map<String, List<ChannelIndex.Article>> getSourceContendMap(String channelId, Set<String> contendIds, Long startTime, Long endTime) {
return getSourceContendMap(channelId, contendIds, null, null, startTime, endTime);
}
private Pair<Long, JSONObject> getDataCount(List<ChannelIndex.Article> articles) {
JSONObject res = new JSONObject();
// 各稿件数统计
long positiveCount = 0;
long negativeCount = 0;
long total = 0;
for (ChannelIndex.Article article : articles) {
if (article.getEmotion() == EmotionEnum.POSITIVE.getState()) {
positiveCount++;
} else if (article.getEmotion() == EmotionEnum.NEGATIVE.getState()) {
negativeCount++;
}
total++;
}
res.put("positiveCount", positiveCount);
res.put("negativeCount", negativeCount);
res.put("articleTotal", total);
res.put("positivePercent", new BigDecimal((double) positiveCount * 100 / total).setScale(1, RoundingMode.UP));
res.put("negativePercent", new BigDecimal((double) negativeCount * 100 / total).setScale(1, RoundingMode.UP));
return Pair.of(total, res);
}
private Pair<Long, JSONObject> getEventCount(Long startTime, Long endTime, String projectId, String contendId, Channel channel) {
JSONObject res = new JSONObject();
String linkedGroupId = projectService.getProjectByContendId(contendId).getBrandLinkedGroupId();
Map<Long, List<Event>> eventCount;
if (endTime - startTime > Constant.ONE_MONTH) {
eventCount = eventDao.getEventMonth(new ChannelIndex(projectId, linkedGroupId, channel), startTime, endTime);
} else {
eventCount = eventDao.getEventDay(new ChannelIndex(projectId, linkedGroupId, channel), startTime, endTime);
}
// 事件部分
long positiveEventCount = 0;
long negativeEventCount = 0;
long totalEvent = 0;
// 统计参与事件
for (List<Event> events : eventCount.values()) {
for (Event event : events) {
if (EmotionEnum.NEGATIVE.getName().equals(event.getEmotion())) {
negativeEventCount++;
} else if (EmotionEnum.POSITIVE.getName().equals(event.getEmotion())) {
positiveEventCount++;
}
totalEvent++;
}
}
res.put("positiveEventCount", positiveEventCount);
res.put("negativeEventCount", negativeEventCount);
res.put("totalEvent", totalEvent);
return Pair.of(totalEvent, res);
}
// private List<JSONObject> getSpreadData(){
//
// }
private List<JSONObject> spreadingTendData(Long startTime, Long endTime, List<ChannelIndex.Article> articleList) {
String timePattern = Constant.DAY_PATTERN;
if (endTime - startTime > Constant.ONE_MONTH) {
timePattern = Constant.MONTH_PATTERN;
}
// 按日分组并根据id去重保留最近标注时间
Map<Long, List<ChannelIndex.Article>> dateListMap = partition(startTime, endTime, timePattern, articleList);
return dateListMap.entrySet().stream().map(e -> {
JSONObject spreadJson = new JSONObject();
spreadJson.put("time", e.getKey());
spreadJson.put("sum", e.getValue().size());
return spreadJson;
}).collect(Collectors.toList());
}
private List<JSONObject> spreadingTendEvent(Long startTime, Long endTime, Channel channel, String projectId, String contendId) {
String linkedGroupId = projectService.getProjectByContendId(contendId).getBrandLinkedGroupId();
Map<Long, List<Event>> eventCount;
if (endTime - startTime > Constant.ONE_MONTH) {
eventCount = eventDao.getEventMonth(new ChannelIndex(projectId, linkedGroupId, channel), startTime, endTime);
} else {
eventCount = eventDao.getEventDay(new ChannelIndex(projectId, linkedGroupId, channel), startTime, endTime);
}
return eventCount.entrySet().stream().map(e -> {
JSONObject spreadJson = new JSONObject();
spreadJson.put("time", e.getKey());
spreadJson.put("sum", e.getValue().size());
return spreadJson;
}).collect(Collectors.toList());
}
private Map<String, List<ChannelIndex.Article>> convert2ContendMap(List<SearchResponse> searchResponses, Long startTime, Long endTime) {
Map<String, List<ChannelIndex.Article>> contendMap = new HashMap<>();
......@@ -459,16 +700,20 @@ public class ChannelServiceImpl implements ChannelService {
for (SearchHit hit : searchResponse.getHits().getHits()) {
ChannelRecord channelRecord = new ChannelRecord(hit.getSourceAsMap());
// 过滤掉不符合时间条件的数据
List<ChannelIndex.Article> articles = channelRecord.getRecord().getArticles().stream().filter(article -> Tools.hitTimeRange(startTime, endTime, article.getTime())).collect(Collectors.toList());
contendMap.compute(channelRecord.getContendId(), (k, v) -> {
if (null == v) {
return articles;
}
v.addAll(articles);
return v;
});
// List<ChannelIndex.Article> articles = channelRecord.getRecord().getArticles().stream().filter(article -> Tools.hitTimeRange(startTime, endTime, article.getTime())).collect(Collectors.toList());
// contendMap.compute(channelRecord.getContendId(), (k, v) -> {
// if (null == v) {
// return articles;
// }
// v.addAll(articles);
// return v;
// });
contendMap.putIfAbsent(channelRecord.getContendId(), new ArrayList<>());
contendMap.get(channelRecord.getChannelId()).addAll(channelRecord.getRecord().getArticles());
}
}
// contendMap.forEach((k, v) -> v.sort(Comparator.comparingLong(ChannelIndex.Article::getTime)));
contendMap.replaceAll((k, v) -> ChannelIndex.Record.filterSortArticles(startTime, endTime, v));
return contendMap;
}
......@@ -477,27 +722,15 @@ public class ChannelServiceImpl implements ChannelService {
*
* @return
*/
private Map<Long, List<ChannelIndex.Article>> partitionWithDuplicate(Long startTime, Long endTime, List<ChannelIndex.Article> articles) {
private Map<Long, List<ChannelIndex.Article>> partition(Long startTime, Long endTime, String timePattern, List<ChannelIndex.Article> articles) {
Map<Long, List<ChannelIndex.Article>> res = new HashMap<>();
// 自动补全时间段
for (Long timeKey : Tools.parseToDays(startTime, endTime)) {
res.put(timeKey, Lists.newArrayList());
}
Map<String, ChannelIndex.Article> setMap = new HashMap<>();
// 去重并保留最近标注时间
for (ChannelIndex.Article article : articles) {
setMap.compute(article.getId(), (k, v) -> {
// 旧值为null或标注时间更新
if (null == v || article.getMtime() > v.getMtime()) {
return article;
}
return v;
});
}
for (ChannelIndex.Article article : setMap.values()) {
// 按日分组
long key = Tools.truncDate(new Date(article.getTime()), Constant.DAY_PATTERN).getTime();
long key = Tools.truncDate(new Date(article.getTime()), timePattern).getTime();
res.compute(key, (k, v) -> {
if (null == v) {
v = new ArrayList<>();
......@@ -510,7 +743,7 @@ public class ChannelServiceImpl implements ChannelService {
}
private EsClientDao.SearchHelper createSearchHelperByChannelCriteria(String projectId, String contendId, String platform, String keyword, Long startTime, Long endTime) {
private EsClientDao.SearchHelper createSearchHelperByChannelCriteria(String projectId, String channelId, Set<String> contendIds, String platform, String keyword, Long startTime, Long endTime) {
EsClientDao.SearchHelper helper = EsClientDao.createSearchHelper();
// 默认搜索一周
if (null == startTime || null == endTime) {
......@@ -522,8 +755,14 @@ public class ChannelServiceImpl implements ChannelService {
BoolQueryBuilder query = QueryBuilders.boolQuery();
// project和contendId
query.must(QueryBuilders.termQuery("project_id.keyword", projectId));
if (null != contendId) {
query.must(QueryBuilders.termQuery("contend_id.keyword", contendId));
// 添加contends集合 查询
if (null != contendIds) {
EsQueryTools.assembleContendsQuery(query, contendIds);
}
// 添加渠道限制
if (null != channelId) {
Channel channel = channelDao.findOneById(channelId);
query.must(QueryBuilders.termQuery("channel_fid.keyword", channel.getFid()));
}
// platformId
String platformId = GlobalPojo.getPlatformIdByName(platform);
......@@ -562,4 +801,13 @@ public class ChannelServiceImpl implements ChannelService {
return query;
}
private String[] getTitleAndUrlById(String id) {
try {
BaseMap baseMap = Tools.getBaseFromEsMap(esClientDao.searchById(id));
return new String[]{baseMap.getTitle(), baseMap.getUrl()};
} catch (IOException ignored) {
}
return new String[]{null, null};
}
}
src/main/java/com/zhiwei/brandkbs2/service/impl/TaskServiceImpl.java
......@@ -20,6 +20,7 @@ import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
......@@ -70,6 +71,64 @@ public class TaskServiceImpl implements TaskService {
@Override
public void messageFlowCount(int day) {
List<Pair<Long[], Map<ChannelIndex, ChannelIndex.Record>>> rangeTimeRecords = esClientDao.searchRecordRecentDay(day);
// 结果合并
List<Map<ChannelIndex, ChannelIndex.Record>> channelList = rangeTimeRecords.stream().map(Pair::getRight).collect(Collectors.toList());
// 合并渠道记录
Map<ChannelIndex, ChannelIndex.Record> channelIndexRecordMap = ChannelIndex.mergeRecord(channelList);
log.info("渠道统计-搜索到近{}天的受影响渠道数{}条", day, channelIndexRecordMap.size());
long handleSize = 0;
List<Channel> insertList = new ArrayList<>();
List<Map.Entry<ChannelIndex, ChannelIndex.Record>> batchList = new ArrayList<>();
// 新recordMap
Map<ChannelIndex, ChannelIndex.Record> newRecordMap = new HashMap<>();
for (Map.Entry<ChannelIndex, ChannelIndex.Record> entry : channelIndexRecordMap.entrySet()) {
batchList.add(entry);
// 每100条做一次清算
if (++handleSize % 100 == 0 || handleSize == channelIndexRecordMap.size()) {
insertList.addAll(batchHandle(batchList, newRecordMap));
batchList = new ArrayList<>();
}
if (handleSize % 10000 == 0) {
log.info("渠道统计-渠道总计-查询更新已完成{}/{}", handleSize, channelIndexRecordMap.size());
}
}
// 替换成新的记录map
channelIndexRecordMap = newRecordMap;
ListUtils.partition(insertList, 1000).forEach(list -> {
channelDao.insertMany(list);
});
log.info("渠道统计-渠道总计-录入完毕,新增渠道{}条,更新渠道{}条", insertList.size(), channelIndexRecordMap.size() - insertList.size());
// 获得单位时间内最小最大时间戳
Long[] timeMinMax = Tools.timeMinMax(rangeTimeRecords.stream().map(Pair::getLeft).collect(Collectors.toList()));
List<ChannelRecord> channelRecords = ChannelRecord.createChannelRecords(timeMinMax[0], timeMinMax[1], channelIndexRecordMap);
channelEsDao.upsertChannelRecord(channelRecords);
log.info("渠道统计-渠道记录-统计结束");
}
private List<Channel> batchHandle(List<Map.Entry<ChannelIndex, ChannelIndex.Record>> batchList, Map<ChannelIndex, ChannelIndex.Record> newRecordMap) {
List<Channel> insertList = new ArrayList<>();
List<String> fids = batchList.stream().map(channelIndexRecordEntry -> channelIndexRecordEntry.getKey().getFid()).collect(Collectors.toList());
Map<String, Channel> fidChannel = channelDao.queryUniqueAsync(fids);
for (Map.Entry<ChannelIndex, ChannelIndex.Record> entry : batchList) {
String fid = entry.getKey().getFid();
Channel channel = fidChannel.get(fid);
if (null == channel) {
channel = Channel.createFromChannelIndexRecord(entry.getKey(), entry.getValue());
insertList.add(channel);
} else {
channel.setRecord(entry.getValue());
channelDao.updateOne(channel);
}
// 设置查询数值
entry.getKey().setChannelInfo(channel);
newRecordMap.put(entry.getKey(), entry.getValue());
}
return insertList;
}
@Deprecated
public void messageFlowCount2(int day) {
List<Pair<Long[], Map<ChannelIndex, ChannelIndex.Record>>> rangeTimeRecords = esClientDao.searchRecordRecentDay(day);
int total = rangeTimeRecords.stream().mapToInt(pair -> pair.getRight().values().size()).sum();
log.info("渠道统计-搜索到近{}天的受影响渠道数{}条", day, total);
// 结果合并
......@@ -79,9 +138,11 @@ public class TaskServiceImpl implements TaskService {
// 获得单位时间内最小最大时间戳
Long[] timeMinMax = Tools.timeMinMax(rangeTimeRecords.stream().map(Pair::getLeft).collect(Collectors.toList()));
List<ChannelRecord> channelRecords = ChannelRecord.createChannelRecords(timeMinMax[0], timeMinMax[1], channelIndexRecordMap);
// List<ChannelRecord> channelRecords = rangeTimeRecords.stream().map(pair -> ChannelRecord.createChannelRecords(pair.getLeft()[0], pair.getLeft()[1], pair.getRight())).flatMap(Collection::stream).collect(Collectors.toList());
channelEsDao.upsertChannelRecord(channelRecords);
log.info("渠道统计-小时级渠道记录-统计结束");
log.info("渠道统计-渠道记录-统计结束");
// List<ChannelRecord> channelRecords = rangeTimeRecords.stream().map(pair -> ChannelRecord.createChannelRecords(pair.getLeft()[0], pair.getLeft()[1], pair.getRight())).flatMap(Collection::stream).collect(Collectors.toList());
long handleSize = 0;
List<Channel> insertList = new ArrayList<>();
for (Map.Entry<ChannelIndex, ChannelIndex.Record> entry : channelIndexRecordMap.entrySet()) {
// 是否已存在
......@@ -93,8 +154,13 @@ public class TaskServiceImpl implements TaskService {
channel.setRecord(entry.getValue());
channelDao.updateOne(channel);
}
// 设置查询数值
entry.getKey().setChannelInfo(channel);
if (++handleSize % 10000 == 0) {
log.info("渠道统计-渠道总计-查询更新已完成{}/{}", handleSize, channelIndexRecordMap.size());
}
}
log.info("渠道统计-渠道总计-查询并更新结束,开始批量入库");
log.info("渠道统计-渠道总计-查询更新结束,开始批量入库");
ListUtils.partition(insertList, 1000).forEach(list -> {
channelDao.insertMany(list);
});
......
src/main/webapp/WEB-INF/applicationContext.xml deleted 100644 → 0
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!-- @version $Id: applicationContext.xml 561608 2007-08-01 00:33:12Z vgritsenko $ -->
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:util="http://www.springframework.org/schema/util"
xmlns:configurator="http://cocoon.apache.org/schema/configurator"
xmlns:avalon="http://cocoon.apache.org/schema/avalon"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-2.0.xsd
http://cocoon.apache.org/schema/configurator http://cocoon.apache.org/schema/configurator/cocoon-configurator-1.0.1.xsd
http://cocoon.apache.org/schema/avalon http://cocoon.apache.org/schema/avalon/cocoon-avalon-1.0.xsd">
<!-- Activate Cocoon Spring Configurator -->
<configurator:settings/>
<!-- Configure Log4j -->
<bean name="org.apache.cocoon.spring.configurator.log4j"
class="org.apache.cocoon.spring.configurator.log4j.Log4JConfigurator"
scope="singleton">
<property name="settings" ref="org.apache.cocoon.configuration.Settings"/>
<property name="resource" value="/WEB-INF/log4j.xml"/>
</bean>
<!-- Activate Avalon Bridge -->
<avalon:bridge/>
</beans>
src/main/webapp/WEB-INF/log4j.xml deleted 100644 → 0
<?xml version="1.0" encoding="UTF-8" ?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
<!--
- This is a sample configuration for log4j.
- It simply just logs everything into a single log file.
- Note, that you can use properties for value substitution.
-->
<appender name="CORE" class="org.apache.log4j.FileAppender">
<param name="File" value="${org.apache.cocoon.work.directory}/cocoon-logs/log4j.log" />
<param name="Append" value="false" />
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%d %-5p %t %c - %m%n"/>
</layout>
</appender>
<root>
<priority value="${org.apache.cocoon.log4j.loglevel}"/>
<appender-ref ref="CORE"/>
</root>
</log4j:configuration>
src/main/webapp/WEB-INF/web.xml deleted 100644 → 0
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!--
- This is the Cocoon web-app configurations file
-
- $Id$
-->
<web-app version="2.4"
xmlns="http://java.sun.com/xml/ns/j2ee"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://java.sun.com/xml/ns/j2ee http://java.sun.com/xml/ns/j2ee/web-app_2_4.xsd">
<!-- Servlet Filters ================================================ -->
<!--
- Declare a filter for multipart MIME handling
-->
<filter>
<description>Multipart MIME handling filter for Cocoon</description>
<display-name>Cocoon multipart filter</display-name>
<filter-name>CocoonMultipartFilter</filter-name>
<filter-class>org.apache.cocoon.servlet.multipart.MultipartFilter</filter-class>
</filter>
<!--
- Declare a filter for debugging incoming request
-->
<filter>
<description>Log debug information about each request</description>
<display-name>Cocoon debug filter</display-name>
<filter-name>CocoonDebugFilter</filter-name>
<filter-class>org.apache.cocoon.servlet.DebugFilter</filter-class>
</filter>
<!-- Filter mappings ================================================ -->
<!--
- Use the Cocoon multipart filter together with the Cocoon demo webapp
-->
<filter-mapping>
<filter-name>CocoonMultipartFilter</filter-name>
<servlet-name>Cocoon</servlet-name>
</filter-mapping>
<filter-mapping>
<filter-name>CocoonMultipartFilter</filter-name>
<servlet-name>DispatcherServlet</servlet-name>
</filter-mapping>
<!--
- Use the Cocoon debug filter together with the Cocoon demo webapp
<filter-mapping>
<filter-name>CocoonDebugFilter</filter-name>
<servlet-name>Cocoon</servlet-name>
</filter-mapping>
-->
<!-- Servlet Context Listener ======================================= -->
<!--
- Declare Spring context listener which sets up the Spring Application Context
- containing all Cocoon components (and user defined beans as well).
-->
<listener>
<listener-class>org.springframework.web.context.ContextLoaderListener</listener-class>
</listener>
<!--
- Declare Spring request listener which sets up the required RequestAttributes
- to support Springs and Cocoon custom bean scopes like the request scope or the
- session scope.
-->
<listener>
<listener-class>org.springframework.web.context.request.RequestContextListener</listener-class>
</listener>
<!-- Servlet Configuration ========================================== -->
<!--
- Servlet that dispatches requests to the Spring managed block servlets
-->
<servlet>
<description>Cocoon blocks dispatcher</description>
<display-name>DispatcherServlet</display-name>
<servlet-name>DispatcherServlet</servlet-name>
<servlet-class>org.apache.cocoon.servletservice.DispatcherServlet</servlet-class>
<load-on-startup>1</load-on-startup>
</servlet>
<!-- URL space mappings ============================================= -->
<!--
- Cocoon handles all the URL space assigned to the webapp using its sitemap.
- It is recommended to leave it unchanged. Under some circumstances though
- (like integration with proprietary webapps or servlets) you might have
- to change this parameter.
-->
<servlet-mapping>
<servlet-name>DispatcherServlet</servlet-name>
<url-pattern>/*</url-pattern>
</servlet-mapping>
</web-app>
\ No newline at end of file
src/test/java/com/zhiwei/brandkbs2/Test.java
package com.zhiwei.brandkbs2;
import com.hankcs.hanlp.HanLP;
import com.alibaba.fastjson.JSONObject;
import com.hankcs.hanlp.dictionary.CustomDictionary;
import com.hankcs.hanlp.dictionary.stopword.CoreStopWordDictionary;
import com.hankcs.hanlp.seg.common.Term;
import com.zhiwei.brandkbs2.util.Tools;
import java.io.UnsupportedEncodingException;
import java.util.List;
/**
* @ClassName: Test
......@@ -22,15 +21,19 @@ public class Test {
}
public static void main(String[] args) throws UnsupportedEncodingException {
List<Term> results= HanLP.segment("国家发改委：猪肉供应有保障 猪肉价格不具备大幅上涨基础");
for(Term term: CoreStopWordDictionary.apply(results)){
// if(CoreStopWordDictionary.shouldInclude(term)){
// System.out.println(term);
// }else{
System.err.println(term);
// }
}
System.out.println(JSONObject.toJSONString(Tools.parseToDays(1658512800000L,1659031200000L)));
// List<Term> results= HanLP.segment("国家发改委：猪肉供应有保障 猪肉价格不具备大幅上涨基础");
// for(Term term: CoreStopWordDictionary.apply(results)){
//// if(CoreStopWordDictionary.shouldInclude(term)){
//// System.out.println(term);
//// }else{
// System.err.println(term);
//// }
//
// }
// String token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJwYXlsb2FkIjoie1widXNlcklkXCI6XCIyMFwiLFwia2V5XCI6XCIyMFwiLFwibmlja05hbWVcIjpcIuayiOWQm-adsFwiLFwic2VydmljZVwiOlwiXCJ9IiwiZXhwIjoxNjUzMDExNjcwLCJpYXQiOjE2NTI0MDY4MTB9.jcVXxeZkayc6-Aiq8cyYc1uyq4ugji6FdWQXCCp4M2o";
// String token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJwYXlsb2FkIjoiXCLlk4Hop4FcIiIsImV4cCI6NDc2MjgyMjEzMiwiaWF0IjoxNjUyNDIyMDcyfQ.DXQ8yKgfsCMjhT0xniZeWCMv4syqIoDvztU4QWsd-Fg";
......
src/test/java/com/zhiwei/brandkbs2/TestRunWith.java
......@@ -62,14 +62,15 @@ public class TestRunWith {
// UserThreadLocal.set(userInfo);
// ResponseResult result = appArticleController.getMarkSpread(1657468800000L, 1657555200000L);
// System.out.println(JSONObject.toJSONString(result));
// taskService.messageFlowCount(1);
taskService.messageFlowCount(2);
reportService.getReportsAggCount();
// reportService.getReportsAggCount();
}
@Test
public void test2() {
taskService.generateReportAndSend();
taskService.messageFlowCount(1);
// taskService.generateReportAndSend();
// ChannelIndex channelIndex = new ChannelIndex();
// channelIndex.setPlatform("微信");
// channelIndex.setRealSource("微信公众号");
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment