Skip to content

B
brandkbs2
Commit 8d01723a by shenjunjie
Options

渠道库部分完结

parent 34d25fdf
Showing with 826 additions and 345 deletions
src/main/java/com/zhiwei/brandkbs2/common/GenericAttribute.java
...@@ -16,7 +16,7 @@ public class GenericAttribute { ...@@ -16,7 +16,7 @@ public class GenericAttribute {
*/ */
public static final String ES_INDEX_PRE = "brandkbs2"; public static final String ES_INDEX_PRE = "brandkbs2";
public static final String ES_INDEX_TEST = "brandkbs2_test"; public static final String ES_INDEX_TEST = "brandkbs2_test";
public static final String ES_CHANNEL_INDEX_TEST = "brandkbs2_channel_record_test2"; public static final String ES_CHANNEL_INDEX_TEST = "brandkbs2_channel_record_test7";
/** /**
* es ind_title * es ind_title
**/ **/
......
src/main/java/com/zhiwei/brandkbs2/config/Constant.java
...@@ -11,9 +11,11 @@ import org.apache.commons.lang3.time.FastDateFormat; ...@@ -11,9 +11,11 @@ import org.apache.commons.lang3.time.FastDateFormat;
public class Constant { public class Constant {
public static final Long ONE_DAY = 24 * 60 * 60 * 1000L; public static final Long ONE_DAY = 24 * 60 * 60 * 1000L;
public static final Long ONE_MONTH = 30 * 24 * 60 * 60 * 1000L;
public static final String HOUR_PATTERN = "yyyy-MM-dd HH"; public static final String HOUR_PATTERN = "yyyy-MM-dd HH";
public static final String DAY_PATTERN = "yyyy-MM-dd"; public static final String DAY_PATTERN = "yyyy-MM-dd";
public static final String MONTH_PATTERN = "yyyy-MM";
public static final FastDateFormat HOUR_FORMAT = FastDateFormat.getInstance(HOUR_PATTERN); public static final FastDateFormat HOUR_FORMAT = FastDateFormat.getInstance(HOUR_PATTERN);
public static final FastDateFormat DAY_FORMAT = FastDateFormat.getInstance(DAY_PATTERN); public static final FastDateFormat DAY_FORMAT = FastDateFormat.getInstance(DAY_PATTERN);
...@@ -28,7 +30,7 @@ public class Constant { ...@@ -28,7 +30,7 @@ public class Constant {
*/ */
public static final String EMOTION_LABEL_KEY = "情感倾向"; public static final String EMOTION_LABEL_KEY = "情感倾向";
public static final String EMOTION_SENSITIVE = "敏感"; public static final String EMOTION_SENSITIVE = "敏感";
public static final String EMOTION_NEGATIVE= "负面"; public static final String EMOTION_NEGATIVE = "负面";
public static final String BRAND_LABEL_KEY = "品牌归属"; public static final String BRAND_LABEL_KEY = "品牌归属";
/** /**
......
src/main/java/com/zhiwei/brandkbs2/config/TaskPoolConfig.java
...@@ -61,4 +61,22 @@ public class TaskPoolConfig { ...@@ -61,4 +61,22 @@ public class TaskPoolConfig {
return executor; return executor;
} }
@Bean
public ThreadPoolTaskExecutor mongoQueryExecutor() {
log.info("start mongoQueryExecutor");
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
// 配置核心线程数
executor.setCorePoolSize(8);
// 配置最大线程数
executor.setMaxPoolSize(16);
// 配置线程池中的线程的名称前缀
executor.setThreadNamePrefix("mongoQuery-");
// rejection-policy：当pool已经达到max size的时候，如何处理新任务
// CALLER_RUNS：不在新线程中执行任务，而是有调用者所在的线程来执行
executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
// 执行初始化
executor.initialize();
return executor;
}
} }
src/main/java/com/zhiwei/brandkbs2/controller/app/AppChannelController.java
...@@ -2,6 +2,8 @@ package com.zhiwei.brandkbs2.controller.app; ...@@ -2,6 +2,8 @@ package com.zhiwei.brandkbs2.controller.app;
import com.zhiwei.brandkbs2.auth.Auth; import com.zhiwei.brandkbs2.auth.Auth;
import com.zhiwei.brandkbs2.controller.BaseController; import com.zhiwei.brandkbs2.controller.BaseController;
import com.zhiwei.brandkbs2.easyexcel.EasyExcelUtil;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportAppChannelArticleDTO;
import com.zhiwei.brandkbs2.enmus.RoleEnum; import com.zhiwei.brandkbs2.enmus.RoleEnum;
import com.zhiwei.brandkbs2.model.ResponseResult; import com.zhiwei.brandkbs2.model.ResponseResult;
import com.zhiwei.brandkbs2.service.ChannelService; import com.zhiwei.brandkbs2.service.ChannelService;
...@@ -12,6 +14,7 @@ import io.swagger.annotations.ApiOperation; ...@@ -12,6 +14,7 @@ import io.swagger.annotations.ApiOperation;
import org.springframework.web.bind.annotation.*; import org.springframework.web.bind.annotation.*;
import javax.annotation.Resource; import javax.annotation.Resource;
import java.util.List;
import java.util.Set; import java.util.Set;
/** /**
...@@ -30,56 +33,64 @@ public class AppChannelController extends BaseController { ...@@ -30,56 +33,64 @@ public class AppChannelController extends BaseController {
ChannelService channelService; ChannelService channelService;
@ApiImplicitParams({@ApiImplicitParam(name = "contendId", value = "品牌id", paramType = "query", dataType = "string"), @ApiImplicitParams({@ApiImplicitParam(name = "contendId", value = "品牌id", defaultValue = "0", paramType = "query", dataType = "string"),
@ApiImplicitParam(name = "platform", value = "平台", paramType = "query", dataType = "string"), @ApiImplicitParam(name = "platform", value = "平台", paramType = "query", dataType = "string"),
@ApiImplicitParam(name = "keyword", value = "关键字搜索", paramType = "query", dataType = "string"), @ApiImplicitParam(name = "keyword", value = "关键字搜索", paramType = "query", dataType = "string"),
@ApiImplicitParam(name = "startTime", value = "起始时间", paramType = "query", dataType = "long"), @ApiImplicitParam(name = "startTime", value = "起始时间", paramType = "query", dataType = "long"),
@ApiImplicitParam(name = "endTime", value = "结束时间", paramType = "query", dataType = "long"), @ApiImplicitParam(name = "endTime", value = "结束时间", paramType = "query", dataType = "long"),
@ApiImplicitParam(name = "size", value = "选取前几", defaultValue = "50", paramType = "query", dataType = "int"), @ApiImplicitParam(name = "pageSize", value = "选取前几", defaultValue = "50", paramType = "query", dataType = "int"),
}) })
@ApiOperation("渠道库-活跃渠道榜") @ApiOperation("渠道库-活跃渠道榜")
@GetMapping("/list/active") @GetMapping("/list/active")
public ResponseResult getActiveChannelList(@RequestParam(value = "contendId", required = false, defaultValue = "0") String contendId, public ResponseResult getActiveChannelList(@RequestParam(value = "contendId", defaultValue = "0") String contendId,
@RequestParam(value = "platform", required = false) String platform, @RequestParam(value = "platform", required = false) String platform,
@RequestParam(value = "keyword", required = false) String keyword, @RequestParam(value = "keyword", required = false) String keyword,
@RequestParam(value = "startTime", required = false) Long startTime, @RequestParam(value = "startTime", required = false) Long startTime,
@RequestParam(value = "endTime", required = false) Long endTime, @RequestParam(value = "endTime", required = false) Long endTime,
@RequestParam(value = "size", defaultValue = "50") int size) { @RequestParam(value = "pageSize", defaultValue = "50") int size) {
return ResponseResult.success(channelService.getActiveChannelList(contendId, platform, keyword, startTime, endTime, size)); return ResponseResult.success(channelService.getActiveChannelList(contendId, platform, keyword, startTime, endTime, size));
} }
@ApiOperation("渠道库-友好渠道榜") @ApiOperation("渠道库-友好渠道榜")
@ApiImplicitParams({ @ApiImplicitParams({
@ApiImplicitParam(name = "contendId", value = "品牌ID", paramType = "query", dataType = "string"), @ApiImplicitParam(name = "contendId", value = "品牌ID", defaultValue = "0", paramType = "query", dataType = "string"),
@ApiImplicitParam(name = "platform", value = "平台筛选", paramType = "query", dataType = "string"), @ApiImplicitParam(name = "platform", value = "平台筛选", paramType = "query", dataType = "string"),
@ApiImplicitParam(name = "keyword", value = "关键字搜索", paramType = "query", dataType = "string"), @ApiImplicitParam(name = "keyword", value = "关键字搜索", paramType = "query", dataType = "string"),
@ApiImplicitParam(name = "sorter", defaultValue = "{\"index\":\"descend\"}", value = "排序字段", paramType = "query", dataType = "string"), @ApiImplicitParam(name = "sorter", defaultValue = "{\"index\":\"descend\"}", value = "排序字段", paramType = "query", dataType = "string"),
@ApiImplicitParam(name = "size", value = "选取前几", defaultValue = "50", paramType = "query", dataType = "int") @ApiImplicitParam(name = "startTime", value = "起始时间", paramType = "query", dataType = "long"),
@ApiImplicitParam(name = "endTime", value = "结束时间", paramType = "query", dataType = "long"),
@ApiImplicitParam(name = "pageSize", value = "选取前几", defaultValue = "50", paramType = "query", dataType = "int")
}) })
@GetMapping("/list/positive") @GetMapping("/list/positive")
public ResponseResult getPositiveList(@RequestParam(value = "contendId", required = false) String contendId, public ResponseResult getPositiveList(@RequestParam(value = "contendId", defaultValue = "0") String contendId,
@RequestParam(value = "platform", required = false) String platform, @RequestParam(value = "platform", required = false) String platform,
@RequestParam(value = "keyword", required = false) String keyword, @RequestParam(value = "keyword", required = false) String keyword,
@RequestParam(value = "sorter", required = false) String sorter, @RequestParam(value = "sorter", defaultValue = "{\"index\":\"descend\"}") String sorter,
@RequestParam(value = "size", defaultValue = "50") int size) { @RequestParam(value = "startTime", required = false) Long startTime,
return ResponseResult.success(channelService.getPositiveList(contendId, platform, keyword, sorter, size)); @RequestParam(value = "endTime", required = false) Long endTime,
@RequestParam(value = "pageSize", defaultValue = "50") int size) {
return ResponseResult.success(channelService.getPositiveList(contendId, platform, keyword, sorter, startTime, endTime, size));
} }
@ApiOperation("渠道库-敏感渠道榜") @ApiOperation("渠道库-敏感渠道榜")
@ApiImplicitParams({ @ApiImplicitParams({
@ApiImplicitParam(name = "contendId", value = "品牌ID", paramType = "query", dataType = "string"), @ApiImplicitParam(name = "contendId", value = "品牌ID", defaultValue = "0", paramType = "query", dataType = "string"),
@ApiImplicitParam(name = "platform", value = "平台筛选", paramType = "query", dataType = "string"), @ApiImplicitParam(name = "platform", value = "平台筛选", paramType = "query", dataType = "string"),
@ApiImplicitParam(name = "keyword", value = "关键字搜索", paramType = "query", dataType = "string"), @ApiImplicitParam(name = "keyword", value = "关键字搜索", paramType = "query", dataType = "string"),
@ApiImplicitParam(name = "sorter", defaultValue = "{\"index\":\"descend\"}", value = "排序字段", paramType = "query", dataType = "string"), @ApiImplicitParam(name = "sorter", defaultValue = "{\"index\":\"descend\"}", value = "排序字段", paramType = "query", dataType = "string"),
@ApiImplicitParam(name = "size", value = "选取前几", defaultValue = "50", paramType = "query", dataType = "int") @ApiImplicitParam(name = "startTime", value = "起始时间", paramType = "query", dataType = "long"),
@ApiImplicitParam(name = "endTime", value = "结束时间", paramType = "query", dataType = "long"),
@ApiImplicitParam(name = "pageSize", value = "选取前几", defaultValue = "50", paramType = "query", dataType = "int")
}) })
@GetMapping("/list/negative") @GetMapping("/list/negative")
public ResponseResult getNegativeList(@RequestParam(value = "contendId", required = false) String contendId, public ResponseResult getNegativeList(@RequestParam(value = "contendId", defaultValue = "0") String contendId,
@RequestParam(value = "platform", required = false) String platform, @RequestParam(value = "platform", required = false) String platform,
@RequestParam(value = "keyword", required = false) String keyword, @RequestParam(value = "keyword", required = false) String keyword,
@RequestParam(value = "sorter", required = false) String sorter, @RequestParam(value = "sorter", defaultValue = "{\"index\":\"descend\"}") String sorter,
@RequestParam(value = "size", defaultValue = "50") int size) { @RequestParam(value = "startTime", required = false) Long startTime,
return ResponseResult.success(channelService.getNegativeList(contendId, platform, keyword, sorter, size)); @RequestParam(value = "endTime", required = false) Long endTime,
@RequestParam(value = "pageSize", defaultValue = "50") int size) {
return ResponseResult.success(channelService.getNegativeList(contendId, platform, keyword, sorter, startTime, endTime, size));
} }
@ApiOperation("渠道库-收藏渠道") @ApiOperation("渠道库-收藏渠道")
...@@ -91,7 +102,7 @@ public class AppChannelController extends BaseController { ...@@ -91,7 +102,7 @@ public class AppChannelController extends BaseController {
@ApiOperation("渠道库-取消收藏渠道") @ApiOperation("渠道库-取消收藏渠道")
@ApiImplicitParam(name = "channelId", value = "渠道ID", required = true, paramType = "path", dataType = "string") @ApiImplicitParam(name = "channelId", value = "渠道ID", required = true, paramType = "path", dataType = "string")
@PostMapping("/removeCollect/{channelId}}") @PostMapping("/removeCollect/{channelId}")
public ResponseResult removeCollectChannel(@PathVariable String channelId) { public ResponseResult removeCollectChannel(@PathVariable String channelId) {
return ResponseResult.success(channelService.removeCollectChannel(channelId)); return ResponseResult.success(channelService.removeCollectChannel(channelId));
} }
...@@ -121,10 +132,61 @@ public class AppChannelController extends BaseController { ...@@ -121,10 +132,61 @@ public class AppChannelController extends BaseController {
@GetMapping("/spreadingTend") @GetMapping("/spreadingTend")
public ResponseResult getSpreadingTend(@RequestParam("channelId") String channelId, public ResponseResult getSpreadingTend(@RequestParam("channelId") String channelId,
@RequestParam(value = "type", defaultValue = "文章") String type, @RequestParam(value = "type", defaultValue = "文章") String type,
@RequestParam(value = "brandIds", required = false) Set<String> contends, @RequestParam(value = "contends", required = false) Set<String> contends,
@RequestParam(value = "startTime", required = false) Long startTime, @RequestParam(value = "startTime", required = false) Long startTime,
@RequestParam(value = "endTime", required = false) Long endTime) { @RequestParam(value = "endTime", required = false) Long endTime) {
return ResponseResult.success(channelService.getSpreadingTend(channelId, type, contends, startTime, endTime)); return ResponseResult.success(channelService.getSpreadingTend(channelId, type, contends, startTime, endTime));
} }
@ApiOperation("渠道库-渠道动向-摘要")
@ApiImplicitParams({
@ApiImplicitParam(name = "channelId", value = "渠道ID", required = true, paramType = "query", dataType = "string"),
@ApiImplicitParam(name = "contends", value = "品牌ID集合", paramType = "query", dataType = "string"),
@ApiImplicitParam(name = "startTime", value = "开始时间", paramType = "query", dataType = "long"),
@ApiImplicitParam(name = "endTime", value = "结束时间", paramType = "query", dataType = "long")
})
@GetMapping("/spreadingTend/summary")
public ResponseResult getSpreadingTendSummary(@RequestParam("channelId") String channelId,
@RequestParam(value = "contends", required = false) Set<String> contends,
@RequestParam(value = "startTime", required = false) Long startTime,
@RequestParam(value = "endTime", required = false) Long endTime) {
return ResponseResult.success(channelService.getSpreadingTendSummary(channelId, contends, startTime, endTime));
}
@ApiOperation("渠道-文章列表")
@ApiImplicitParams({
@ApiImplicitParam(name = "startTime", value = "开始时间", defaultValue = "0", paramType = "query", dataType = "long"),
@ApiImplicitParam(name = "endTime", value = "结束时间", defaultValue = "0", paramType = "query", dataType = "long"),
@ApiImplicitParam(name = "page", value = "页码", defaultValue = "1", paramType = "query", dataType = "int"),
@ApiImplicitParam(name = "pageSize", value = "页码大小", defaultValue = "10", paramType = "query", dataType = "int"),
@ApiImplicitParam(name = "channelId", value = "渠道ID", required = true, paramType = "query", dataType = "int"),
@ApiImplicitParam(name = "contendId", value = "竞品ID", defaultValue = "0", paramType = "query", dataType = "int")
})
@GetMapping("/articles")
public ResponseResult getArticles(@RequestParam(value = "startTime") Long startTime,
@RequestParam(value = "endTime") Long endTime,
@RequestParam(value = "page", defaultValue = "1") int page,
@RequestParam(value = "pageSize", defaultValue = "10") int pageSize,
@RequestParam("channelId") String channelId,
@RequestParam(value = "contendId", defaultValue = "0") String contendId) {
return ResponseResult.success(channelService.getArticlesByTime(startTime, endTime, page, pageSize, channelId, contendId));
}
@ApiOperation("渠道库-下载文章列表")
@ApiImplicitParams({
@ApiImplicitParam(name = "startTime", value = "开始时间", required = true, paramType = "query", dataType = "long"),
@ApiImplicitParam(name = "endTime", value = "结束时间", required = true, paramType = "query", dataType = "long"),
@ApiImplicitParam(name = "channelId", value = "渠道ID", required = true, paramType = "query", dataType = "string"),
@ApiImplicitParam(name = "contendId", value = "品牌ID", defaultValue = "0", paramType = "query", dataType = "string")
})
@GetMapping("/articles/download")
public ResponseResult downloadArticles(@RequestParam(value = "startTime") long startTime,
@RequestParam(value = "endTime") long endTime,
@RequestParam("channelId") String channelId,
@RequestParam(value = "contendId", defaultValue = "0") String contendId) {
List<ExportAppChannelArticleDTO> exportAppChannelArticleDTOS = channelService.downloadArticlesByTime(startTime, endTime, channelId, contendId);
EasyExcelUtil.download(channelId + "稿件列表数据", "sheet1", ExportAppChannelArticleDTO.class, exportAppChannelArticleDTOS, response);
return ResponseResult.success();
}
} }
src/main/java/com/zhiwei/brandkbs2/dao/ChannelDao.java
...@@ -3,16 +3,21 @@ package com.zhiwei.brandkbs2.dao; ...@@ -3,16 +3,21 @@ package com.zhiwei.brandkbs2.dao;
import com.zhiwei.brandkbs2.pojo.Channel; import com.zhiwei.brandkbs2.pojo.Channel;
import com.zhiwei.brandkbs2.pojo.ChannelIndex; import com.zhiwei.brandkbs2.pojo.ChannelIndex;
import java.util.Collection;
import java.util.Map;
/** /**
* @ClassName: ChannelDao * @ClassName: ChannelDao
* @Description ChannelDao * @Description ChannelDao
* @author: sjj * @author: sjj
* @date: 2022-06-16 15:30 * @date: 2022-06-16 15:30
*/ */
public interface ChannelDao extends BaseMongoDao<Channel>{ public interface ChannelDao extends BaseMongoDao<Channel> {
Channel queryUnique(String channelFid); Channel queryUnique(String channelFid);
Channel queryUnique(ChannelIndex channelIndex); Channel queryUnique(ChannelIndex channelIndex);
Map<String, Channel> queryUniqueAsync(Collection<String> channelFids);
} }
src/main/java/com/zhiwei/brandkbs2/dao/EventDao.java
...@@ -4,6 +4,7 @@ import com.zhiwei.brandkbs2.pojo.ChannelIndex; ...@@ -4,6 +4,7 @@ import com.zhiwei.brandkbs2.pojo.ChannelIndex;
import com.zhiwei.brandkbs2.pojo.Event; import com.zhiwei.brandkbs2.pojo.Event;
import java.util.List; import java.util.List;
import java.util.Map;
/** /**
* @ClassName: EventDao * @ClassName: EventDao
...@@ -11,7 +12,7 @@ import java.util.List; ...@@ -11,7 +12,7 @@ import java.util.List;
* @author: sjj * @author: sjj
* @date: 2022-05-18 14:35 * @date: 2022-05-18 14:35
*/ */
public interface EventDao extends BaseMongoDao<Event>{ public interface EventDao extends BaseMongoDao<Event> {
/** /**
* 是否已存在事件 * 是否已存在事件
...@@ -31,7 +32,7 @@ public interface EventDao extends BaseMongoDao<Event>{ ...@@ -31,7 +32,7 @@ public interface EventDao extends BaseMongoDao<Event>{
* @param linkedGroupId 关联项目id * @param linkedGroupId 关联项目id
* @return 事件 * @return 事件
*/ */
Event getEventByUniqueIds(String yqEventId,String projectId,String linkedGroupId); Event getEventByUniqueIds(String yqEventId, String projectId, String linkedGroupId);
/** /**
...@@ -42,4 +43,21 @@ public interface EventDao extends BaseMongoDao<Event>{ ...@@ -42,4 +43,21 @@ public interface EventDao extends BaseMongoDao<Event>{
*/ */
List<String> getEventCount(ChannelIndex channelIndex); List<String> getEventCount(ChannelIndex channelIndex);
/**
* 获取参与事件数
*
* @param channelIndex 渠道标识
* @return 参与事件数
*/
Map<Long, List<Event>> getEventDay(ChannelIndex channelIndex, Long startTime, Long endTime);
/**
* 获取参与事件数
*
* @param channelIndex 渠道标识
* @return 参与事件数
*/
Map<Long, List<Event>> getEventMonth(ChannelIndex channelIndex, Long startTime, Long endTime);
} }
src/main/java/com/zhiwei/brandkbs2/dao/impl/ChannelDaoImpl.java
...@@ -3,10 +3,18 @@ package com.zhiwei.brandkbs2.dao.impl; ...@@ -3,10 +3,18 @@ package com.zhiwei.brandkbs2.dao.impl;
import com.zhiwei.brandkbs2.dao.ChannelDao; import com.zhiwei.brandkbs2.dao.ChannelDao;
import com.zhiwei.brandkbs2.pojo.Channel; import com.zhiwei.brandkbs2.pojo.Channel;
import com.zhiwei.brandkbs2.pojo.ChannelIndex; import com.zhiwei.brandkbs2.pojo.ChannelIndex;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Query;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import java.util.*;
import java.util.concurrent.CompletableFuture;
/** /**
* @ClassName: ChannelDaoImpl * @ClassName: ChannelDaoImpl
* @Description ChannelDaoImpl * @Description ChannelDaoImpl
...@@ -15,11 +23,14 @@ import org.springframework.stereotype.Component; ...@@ -15,11 +23,14 @@ import org.springframework.stereotype.Component;
*/ */
@Component("channelDao") @Component("channelDao")
public class ChannelDaoImpl extends BaseMongoDaoImpl<Channel> implements ChannelDao { public class ChannelDaoImpl extends BaseMongoDaoImpl<Channel> implements ChannelDao {
private static final Logger log = LogManager.getLogger(ChannelDaoImpl.class);
private static final String COLLECTION_PREFIX = "brandkbs_channel"; private static final String COLLECTION_PREFIX = "brandkbs_channel";
private final ThreadPoolTaskExecutor executor;
public ChannelDaoImpl() { public ChannelDaoImpl(@Qualifier("mongoQueryExecutor") ThreadPoolTaskExecutor executor) {
super(COLLECTION_PREFIX); super(COLLECTION_PREFIX);
this.executor = executor;
} }
@Override @Override
...@@ -41,4 +52,24 @@ public class ChannelDaoImpl extends BaseMongoDaoImpl<Channel> implements Channel ...@@ -41,4 +52,24 @@ public class ChannelDaoImpl extends BaseMongoDaoImpl<Channel> implements Channel
return queryUnique(channelIndex.getFid()); return queryUnique(channelIndex.getFid());
} }
@Override
public Map<String, Channel> queryUniqueAsync(Collection<String> channelFids) {
Map<String, Channel> res = new HashMap<>();
List<CompletableFuture<Pair<String, Channel>>> futureList = new ArrayList<>();
for (String fid : channelFids) {
futureList.add(CompletableFuture.supplyAsync(() -> Pair.of(fid, queryUnique(fid)), executor));
}
CompletableFuture.allOf(futureList.toArray(new CompletableFuture[0])).whenComplete((r, e) -> {
for (CompletableFuture<Pair<String, Channel>> future : futureList) {
try {
Pair<String, Channel> channelPair = future.get();
res.put(channelPair.getLeft(), channelPair.getRight());
} catch (Exception e1) {
log.error("queryUniqueAsync", e1);
}
}
}).join();
return res;
}
} }
src/main/java/com/zhiwei/brandkbs2/dao/impl/EventDaoImpl.java
...@@ -11,7 +11,9 @@ import org.springframework.data.mongodb.core.query.Query; ...@@ -11,7 +11,9 @@ import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import java.util.Calendar; import java.util.Calendar;
import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import static com.zhiwei.brandkbs2.dao.impl.EventDataDaoImpl.COLLECTION_PREFIX; import static com.zhiwei.brandkbs2.dao.impl.EventDataDaoImpl.COLLECTION_PREFIX;
...@@ -54,6 +56,29 @@ public class EventDaoImpl extends BaseMongoDaoImpl<Event> implements EventDao { ...@@ -54,6 +56,29 @@ public class EventDaoImpl extends BaseMongoDaoImpl<Event> implements EventDao {
return mappedResults.stream().map(json -> json.getString("_id")).collect(Collectors.toList()); return mappedResults.stream().map(json -> json.getString("_id")).collect(Collectors.toList());
} }
@Override
public Map<Long, List<Event>> getEventDay(ChannelIndex channelIndex, Long startTime, Long endTime) {
return getEventTimePattern(channelIndex, startTime, endTime, 8);
}
@Override
public Map<Long, List<Event>> getEventMonth(ChannelIndex channelIndex, Long startTime, Long endTime) {
return getEventTimePattern(channelIndex, startTime, endTime, 6);
}
private Map<Long, List<Event>> getEventTimePattern(ChannelIndex channelIndex, Long startTime, Long endTime, int nrOfChars) {
// 添加渠道唯一标识
Criteria criteria = addChannelIndex(channelIndex);
// 分组
Aggregation agg = Aggregation.newAggregation(Aggregation.match(criteria),
// 截取前8位,按日分组
Aggregation.project("startTime", "eventId", "emotion").andExpression("add(new java.util.Date(0l),startTime)").substring(0, nrOfChars).as("patternDate"),
Aggregation.group("patternDate").count().as("eventCount"));
AggregationResults<JSONObject> aggregate = mongoTemplate.aggregate(agg, getAggreeCollection(), JSONObject.class);
List<JSONObject> mappedResults = aggregate.getMappedResults();
return new HashMap<>();
}
private String getAggreeCollection() { private String getAggreeCollection() {
Calendar date = Calendar.getInstance(); Calendar date = Calendar.getInstance();
int year = date.get(Calendar.YEAR); int year = date.get(Calendar.YEAR);
......
src/main/java/com/zhiwei/brandkbs2/easyexcel/dto/ExportAppChannelArticleDTO.java 0 → 100644
package com.zhiwei.brandkbs2.easyexcel.dto;
import com.alibaba.excel.annotation.ExcelProperty;
import com.zhiwei.brandkbs2.enmus.EmotionEnum;
import com.zhiwei.brandkbs2.pojo.ChannelIndex;
import lombok.Data;
import lombok.ToString;
import java.util.Date;
/**
* @author lxj
* @version 1.0
* @description 前台导出渠道稿件实体类
* @date 2020/3/30 16:31
*/
@Data
@ToString
public class ExportAppChannelArticleDTO {
@ExcelProperty("时间")
private Date time;
@ExcelProperty("链接")
private String url;
@ExcelProperty("标题")
private String title;
@ExcelProperty("正文")
private String content;
@ExcelProperty("情感倾向")
private String emotion;
public static ExportAppChannelArticleDTO createFromArticle(ChannelIndex.Article article, String url, String title) {
ExportAppChannelArticleDTO dto = new ExportAppChannelArticleDTO();
dto.setTime(new Date(article.getTime()));
dto.setUrl(url);
dto.setTitle(title);
dto.setEmotion(EmotionEnum.state2Name(article.getEmotion()));
return dto;
}
}
src/main/java/com/zhiwei/brandkbs2/enmus/EmotionEnum.java
...@@ -121,4 +121,13 @@ public enum EmotionEnum { ...@@ -121,4 +121,13 @@ public enum EmotionEnum {
return 0; return 0;
} }
public static String state2Name(int state) {
for (EmotionEnum value : EmotionEnum.values()) {
if (value.getState() == state) {
return value.getName();
}
}
return EmotionEnum.UNDEFINED.getName();
}
} }
src/main/java/com/zhiwei/brandkbs2/es/EsClientDao.java
...@@ -2,7 +2,6 @@ package com.zhiwei.brandkbs2.es; ...@@ -2,7 +2,6 @@ package com.zhiwei.brandkbs2.es;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.common.GenericAttribute; import com.zhiwei.brandkbs2.common.GenericAttribute;
import com.zhiwei.brandkbs2.enmus.EmotionEnum;
import com.zhiwei.brandkbs2.pojo.ChannelIndex; import com.zhiwei.brandkbs2.pojo.ChannelIndex;
import com.zhiwei.brandkbs2.pojo.Event; import com.zhiwei.brandkbs2.pojo.Event;
import com.zhiwei.brandkbs2.util.Tools; import com.zhiwei.brandkbs2.util.Tools;
...@@ -52,7 +51,8 @@ import java.util.stream.Collectors; ...@@ -52,7 +51,8 @@ import java.util.stream.Collectors;
public class EsClientDao { public class EsClientDao {
private static final Logger log = LogManager.getLogger(EsClientDao.class); private static final Logger log = LogManager.getLogger(EsClientDao.class);
protected static final FastDateFormat DF = FastDateFormat.getInstance("yyyy-MM-dd HH:mm:ss"); protected static final FastDateFormat DF = FastDateFormat.getInstance("yyyy-MM-dd HH:mm:ss");
private static final String[] CHANNEL_RECORD_FETCH_SOURCE = new String[]{"id", "c5", "foreign", "real_source", "source", "mtime", "time", "brandkbs_cache_maps"}; private static final String[] CHANNEL_RECORD_FETCH_SOURCE = new String[]{"id", "c5", "foreign", "real_source", "source", "mtime", "time",
"brandkbs_cache_maps", "brandkbs_mark_cache_maps"};
private static final String[] EVENT_FETCH_SOURCE = new String[]{"ind_full_text", "c5", "real_source", "source", "mtime", "time", "url", "mtag"}; private static final String[] EVENT_FETCH_SOURCE = new String[]{"ind_full_text", "c5", "real_source", "source", "mtime", "time", "url", "mtag"};
private static final Long ONE_HOUR = 60 * 60 * 1000L; private static final Long ONE_HOUR = 60 * 60 * 1000L;
...@@ -71,6 +71,10 @@ public class EsClientDao { ...@@ -71,6 +71,10 @@ public class EsClientDao {
@Resource(name = "retryTemplate") @Resource(name = "retryTemplate")
RetryTemplate retryTemplate; RetryTemplate retryTemplate;
public JSONObject searchById(String queryId) throws IOException {
return searchByIds(Collections.singleton(queryId)).get(queryId);
}
public Map<String, JSONObject> searchByIds(Collection<String> queryIds) throws IOException { public Map<String, JSONObject> searchByIds(Collection<String> queryIds) throws IOException {
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
QueryBuilder queryBuilder = QueryBuilders.idsQuery().addIds(queryIds.toArray(new String[0])); QueryBuilder queryBuilder = QueryBuilders.idsQuery().addIds(queryIds.toArray(new String[0]));
...@@ -180,8 +184,7 @@ public class EsClientDao { ...@@ -180,8 +184,7 @@ public class EsClientDao {
v = new ChannelIndex.Record(); v = new ChannelIndex.Record();
} }
try { try {
return v.mergeRecord(new ChannelIndex.Record((long) result.get(GenericAttribute.ES_TIME), (long) result.get(GenericAttribute.ES_MTIME), return v.mergeRecord(new ChannelIndex.Record(result));
String.valueOf(result.get("id")), EmotionEnum.parseFromName(Tools.getEmotion(result))));
} catch (Exception e) { } catch (Exception e) {
log.error("searchRecord-error-id:{}", result.get("id"), e); log.error("searchRecord-error-id:{}", result.get("id"), e);
return null; return null;
......
src/main/java/com/zhiwei/brandkbs2/es/EsQueryTools.java
...@@ -124,12 +124,12 @@ public class EsQueryTools { ...@@ -124,12 +124,12 @@ public class EsQueryTools {
public static void assembleContendsQuery(BoolQueryBuilder query, Collection<String> contends) { public static void assembleContendsQuery(BoolQueryBuilder query, Collection<String> contends) {
BoolQueryBuilder contendQuery = QueryBuilders.boolQuery(); BoolQueryBuilder contendQuery = QueryBuilders.boolQuery();
// 主品牌一定参与 // 主品牌一定参与
contendQuery.should(QueryBuilders.termQuery("contend_id", "0")); contendQuery.should(QueryBuilders.termQuery("contend_id.keyword", "0"));
if (null == contends) { if (null == contends) {
return; return;
} }
for (String contendId : contends) { for (String contendId : contends) {
contendQuery.should((QueryBuilders.termQuery("contend_id", contendId))); contendQuery.should((QueryBuilders.termQuery("contend_id.keyword", contendId)));
} }
query.must(contendQuery); query.must(contendQuery);
} }
......
src/main/java/com/zhiwei/brandkbs2/pojo/Channel.java
...@@ -4,6 +4,7 @@ import com.zhiwei.brandkbs2.enmus.ChannelEmotion; ...@@ -4,6 +4,7 @@ import com.zhiwei.brandkbs2.enmus.ChannelEmotion;
import com.zhiwei.brandkbs2.util.Tools; import com.zhiwei.brandkbs2.util.Tools;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
import org.bson.types.ObjectId;
import java.util.Date; import java.util.Date;
...@@ -43,16 +44,6 @@ public class Channel extends ChannelIndex { ...@@ -43,16 +44,6 @@ public class Channel extends ChannelIndex {
private long eventCount; private long eventCount;
/** /**
* 渠道倾向
*/
private int emotion;
/**
* 渠道指数
*/
private Double emotionIndex = 0d;
/**
* 经验评级 * 经验评级
*/ */
private String experienceLevel; private String experienceLevel;
...@@ -97,6 +88,7 @@ public class Channel extends ChannelIndex { ...@@ -97,6 +88,7 @@ public class Channel extends ChannelIndex {
public static Channel createFromChannelIndexRecord(ChannelIndex channelIndex, Record record) { public static Channel createFromChannelIndexRecord(ChannelIndex channelIndex, Record record) {
Channel channel = new Channel(); Channel channel = new Channel();
channel.setId(ObjectId.get().toString());
channel.setCTime(new Date().getTime()); channel.setCTime(new Date().getTime());
channel.setProjectId(channelIndex.getProjectId()); channel.setProjectId(channelIndex.getProjectId());
channel.setLinkedGroupId(channelIndex.getLinkedGroupId()); channel.setLinkedGroupId(channelIndex.getLinkedGroupId());
...@@ -117,6 +109,8 @@ public class Channel extends ChannelIndex { ...@@ -117,6 +109,8 @@ public class Channel extends ChannelIndex {
} else { } else {
channel.setEmotion(ChannelEmotion.NEUTRAL.getState()); channel.setEmotion(ChannelEmotion.NEUTRAL.getState());
} }
// TODO 情感指数随机分配
channel.setEmotionIndex(Math.random() * 100);
return channel; return channel;
} }
......
src/main/java/com/zhiwei/brandkbs2/pojo/ChannelIndex.java
...@@ -2,6 +2,7 @@ package com.zhiwei.brandkbs2.pojo; ...@@ -2,6 +2,7 @@ package com.zhiwei.brandkbs2.pojo;
import com.zhiwei.brandkbs2.common.GenericAttribute; import com.zhiwei.brandkbs2.common.GenericAttribute;
import com.zhiwei.brandkbs2.common.GlobalPojo; import com.zhiwei.brandkbs2.common.GlobalPojo;
import com.zhiwei.brandkbs2.enmus.EmotionEnum;
import com.zhiwei.brandkbs2.util.Tools; import com.zhiwei.brandkbs2.util.Tools;
import com.zhiwei.qbjc.bean.pojo.common.MessagePlatform; import com.zhiwei.qbjc.bean.pojo.common.MessagePlatform;
import com.zhiwei.qbjc.bean.tools.BeanTools; import com.zhiwei.qbjc.bean.tools.BeanTools;
...@@ -54,6 +55,17 @@ public class ChannelIndex extends AbstractBaseMongo { ...@@ -54,6 +55,17 @@ public class ChannelIndex extends AbstractBaseMongo {
private String fid; private String fid;
private int emotion;
/**
* 情感指数
*/
private Double emotionIndex;
public ChannelIndex(String projectId, String linkedGroupId, Channel channel) {
this(projectId, linkedGroupId, channel.getPlatform(), channel.getRealSource(), channel.getSource());
}
public ChannelIndex(String projectId, String linkedGroupId, String platform, String realSource, String source) { public ChannelIndex(String projectId, String linkedGroupId, String platform, String realSource, String source) {
this.projectId = projectId; this.projectId = projectId;
this.linkedGroupId = linkedGroupId; this.linkedGroupId = linkedGroupId;
...@@ -117,6 +129,12 @@ public class ChannelIndex extends AbstractBaseMongo { ...@@ -117,6 +129,12 @@ public class ChannelIndex extends AbstractBaseMongo {
return mergeRecord; return mergeRecord;
} }
public void setChannelInfo(Channel channel) {
this.id = channel.getId();
this.emotion = channel.getEmotion();
this.emotionIndex = channel.getEmotionIndex();
}
@Setter @Setter
@Getter @Getter
@AllArgsConstructor @AllArgsConstructor
...@@ -128,9 +146,9 @@ public class ChannelIndex extends AbstractBaseMongo { ...@@ -128,9 +146,9 @@ public class ChannelIndex extends AbstractBaseMongo {
public Record() { public Record() {
} }
public Record(Long lastTime, Long mtime, String articleId, int emotion) { public Record(Map<String, Object> esMap) {
this.lastTime = lastTime; this.lastTime = Long.parseLong(esMap.get("time") + "");
this.articles.add(new Article(lastTime, mtime, articleId, emotion)); this.articles.add(Article.fromEsMap(esMap));
} }
public void setLastTime(Long lastTime) { public void setLastTime(Long lastTime) {
...@@ -145,6 +163,33 @@ public class ChannelIndex extends AbstractBaseMongo { ...@@ -145,6 +163,33 @@ public class ChannelIndex extends AbstractBaseMongo {
return this; return this;
} }
public static List<Article> sortArticles(List<Article> articles) {
articles.sort(Comparator.comparingLong(ChannelIndex.Article::getTime));
return articles;
}
public static List<Article> filterArticles(Long startTime, Long endTime, List<Article> articles) {
// 去除不符合时间段数据
articles = articles.stream().filter(article -> Tools.hitTimeRange(startTime, endTime, article.getTime())).collect(Collectors.toList());
// 去重并保留最近标注时间
Map<String, ChannelIndex.Article> setMap = new HashMap<>();
for (ChannelIndex.Article article : articles) {
setMap.compute(article.getId(), (k, v) -> {
// 旧值为null或标注时间更新
if (null == v || article.getMtime() > v.getMtime()) {
return article;
}
return v;
});
}
return new ArrayList<>(setMap.values());
}
public static List<Article> filterSortArticles(Long startTime, Long endTime, List<Article> articles) {
return sortArticles(filterArticles(startTime, endTime, articles));
}
public Map<String, Object> toEsMap() { public Map<String, Object> toEsMap() {
Map<String, Object> map = new HashMap<>(); Map<String, Object> map = new HashMap<>();
map.put("last_time", lastTime); map.put("last_time", lastTime);
...@@ -160,11 +205,29 @@ public class ChannelIndex extends AbstractBaseMongo { ...@@ -160,11 +205,29 @@ public class ChannelIndex extends AbstractBaseMongo {
Long time; Long time;
Long mtime; Long mtime;
String id; String id;
// String url;
// String title;
int emotion; int emotion;
public static Article fromRecordMap(Map<String, Object> recordMap) {
Long time = Long.parseLong(recordMap.get("time") + "");
Long mtime = Long.parseLong(recordMap.get("mtime") + "");
String id = String.valueOf(recordMap.get("id"));
// String url = recordMap.get("url") + "";
// String title = recordMap.get("title") + "";
int emotion = Integer.parseInt(recordMap.get("emotion") + "");
return new Article(time, mtime, id, emotion);
}
public static Article fromEsMap(Map<String, Object> esMap) { public static Article fromEsMap(Map<String, Object> esMap) {
return new Article(Long.parseLong(esMap.get("time") + ""), Long.parseLong(esMap.get("mtime") + ""), String.valueOf(esMap.get("id")), Integer.parseInt( BaseMap baseMap = Tools.getBaseFromEsMap(esMap);
esMap.get("emotion") + "")); Long time = baseMap.getTime();
Long mtime = Long.parseLong(esMap.get("mtime") + "");
String id = String.valueOf(esMap.get("id"));
// String url = baseMap.getUrl();
// String title = baseMap.getTitle();
String emotionStr = baseMap.getEmotion();
return new Article(time, mtime, id, EmotionEnum.parseFromName(emotionStr));
} }
public Map<String, Object> toEsMap() { public Map<String, Object> toEsMap() {
...@@ -172,6 +235,8 @@ public class ChannelIndex extends AbstractBaseMongo { ...@@ -172,6 +235,8 @@ public class ChannelIndex extends AbstractBaseMongo {
map.put("time", time); map.put("time", time);
map.put("mtime", mtime); map.put("mtime", mtime);
map.put("id", id); map.put("id", id);
// map.put("url", url);
// map.put("title", title);
map.put("emotion", emotion); map.put("emotion", emotion);
return map; return map;
} }
......
src/main/java/com/zhiwei/brandkbs2/pojo/ChannelRecord.java
...@@ -34,7 +34,10 @@ public class ChannelRecord { ...@@ -34,7 +34,10 @@ public class ChannelRecord {
public static final String CHANNEL_FID = "channel_fid"; public static final String CHANNEL_FID = "channel_fid";
public static final String RECORD = "record"; public static final String RECORD = "record";
public static final String ARTICLE_COUNT = "article_count"; public static final String ARTICLE_COUNT = "article_count";
public static final String EMOTION = "emotion";
public static final String EMOTION_INDEX = "emotion_index";
public static final String SHARDS = "shards"; public static final String SHARDS = "shards";
private static final Integer SHARDS_LIMIT = 50;
/** /**
...@@ -82,6 +85,21 @@ public class ChannelRecord { ...@@ -82,6 +85,21 @@ public class ChannelRecord {
private String channelFid; private String channelFid;
/** /**
* 渠道id
*/
private String channelId;
/**
* 情感倾向
*/
private int emotion;
/**
* 情感指数
*/
private Double emotionIndex;
/**
* 时间段内唯一id * 时间段内唯一id
*/ */
private String key; private String key;
...@@ -89,7 +107,7 @@ public class ChannelRecord { ...@@ -89,7 +107,7 @@ public class ChannelRecord {
/** /**
* articleCount * articleCount
*/ */
private int articleCount; private long articleCount;
/** /**
* 记录信息 * 记录信息
*/ */
...@@ -110,6 +128,9 @@ public class ChannelRecord { ...@@ -110,6 +128,9 @@ public class ChannelRecord {
this.realSource = channelIndex.getRealSource(); this.realSource = channelIndex.getRealSource();
this.source = channelIndex.getSource(); this.source = channelIndex.getSource();
this.channelFid = channelIndex.getFid(); this.channelFid = channelIndex.getFid();
this.emotion = channelIndex.getEmotion();
this.emotionIndex = channelIndex.getEmotionIndex();
this.channelId = channelIndex.getId();
this.key = Tools.concat(channelFid, contendId); this.key = Tools.concat(channelFid, contendId);
this.articleCount = record.getArticles().size(); this.articleCount = record.getArticles().size();
this.record = record; this.record = record;
...@@ -125,7 +146,7 @@ public class ChannelRecord { ...@@ -125,7 +146,7 @@ public class ChannelRecord {
List<ChannelIndex.Article> articles = templateRecord.getArticles(); List<ChannelIndex.Article> articles = templateRecord.getArticles();
int shards = 0; int shards = 0;
// 防止数据量过大无法存储,故按100 拆分 // 防止数据量过大无法存储,故按100 拆分
for (List<ChannelIndex.Article> partList : Lists.partition(articles, 100)) { for (List<ChannelIndex.Article> partList : Lists.partition(articles, SHARDS_LIMIT)) {
ChannelRecord channelRecord = new ChannelRecord(rangeStartTime, rangeEndTime, entry.getKey(), entry.getValue()); ChannelRecord channelRecord = new ChannelRecord(rangeStartTime, rangeEndTime, entry.getKey(), entry.getValue());
// 保留最近发文时间,更新partList,articleCount和shards // 保留最近发文时间,更新partList,articleCount和shards
channelRecord.setRecord(new ChannelIndex.Record(lastTime, partList)); channelRecord.setRecord(new ChannelIndex.Record(lastTime, partList));
...@@ -149,6 +170,9 @@ public class ChannelRecord { ...@@ -149,6 +170,9 @@ public class ChannelRecord {
this.contendId = json.getString("contend_id"); this.contendId = json.getString("contend_id");
this.channelFid = json.getString("channel_fid"); this.channelFid = json.getString("channel_fid");
this.key = json.getString("key"); this.key = json.getString("key");
this.emotion = json.getIntValue("emotion");
this.emotionIndex = json.getDoubleValue("emotion_index");
this.channelId = json.getString("channel_id");
this.articleCount = json.getIntValue("article_count"); this.articleCount = json.getIntValue("article_count");
this.shards = json.getIntValue("shards"); this.shards = json.getIntValue("shards");
...@@ -157,7 +181,7 @@ public class ChannelRecord { ...@@ -157,7 +181,7 @@ public class ChannelRecord {
ChannelIndex.Record record = new ChannelIndex.Record(); ChannelIndex.Record record = new ChannelIndex.Record();
record.setLastTime(Long.valueOf(map.get("last_time") + "")); record.setLastTime(Long.valueOf(map.get("last_time") + ""));
List<Map<String, Object>> list = (List<Map<String, Object>>) map.get("articles"); List<Map<String, Object>> list = (List<Map<String, Object>>) map.get("articles");
record.setArticles(list.stream().map(ChannelIndex.Article::fromEsMap).collect(Collectors.toList())); record.setArticles(list.stream().map(ChannelIndex.Article::fromRecordMap).collect(Collectors.toList()));
this.record = record; this.record = record;
} }
...@@ -180,10 +204,19 @@ public class ChannelRecord { ...@@ -180,10 +204,19 @@ public class ChannelRecord {
esMap.put(CHANNEL_FID, channelFid); esMap.put(CHANNEL_FID, channelFid);
esMap.put(RECORD, record.toEsMap()); esMap.put(RECORD, record.toEsMap());
esMap.put(ARTICLE_COUNT, articleCount); esMap.put(ARTICLE_COUNT, articleCount);
esMap.put(EMOTION, emotion);
esMap.put(EMOTION_INDEX, emotionIndex);
esMap.put("channel_id", channelId);
esMap.put(SHARDS, shards); esMap.put(SHARDS, shards);
return esMap; return esMap;
} }
public void setChannelInfo(ChannelRecord channelRecord) {
this.channelId = channelRecord.getChannelId();
this.emotion = channelRecord.getEmotion();
this.emotionIndex = channelRecord.getEmotionIndex();
}
@Deprecated @Deprecated
public static List<ChannelRecord> createChannelRecords(long rangeStartTime, long rangeEndTime, @NonNull Map<ChannelIndex, ChannelIndex.Record> records) { public static List<ChannelRecord> createChannelRecords(long rangeStartTime, long rangeEndTime, @NonNull Map<ChannelIndex, ChannelIndex.Record> records) {
// 通过platform+realSource+source 合并数据 // 通过platform+realSource+source 合并数据
......
src/main/java/com/zhiwei/brandkbs2/pojo/vo/ChannelListVO.java
...@@ -2,6 +2,7 @@ package com.zhiwei.brandkbs2.pojo.vo; ...@@ -2,6 +2,7 @@ package com.zhiwei.brandkbs2.pojo.vo;
import com.zhiwei.brandkbs2.enmus.ChannelEmotion; import com.zhiwei.brandkbs2.enmus.ChannelEmotion;
import com.zhiwei.brandkbs2.pojo.Channel; import com.zhiwei.brandkbs2.pojo.Channel;
import com.zhiwei.brandkbs2.pojo.ChannelRecord;
import com.zhiwei.brandkbs2.util.Tools; import com.zhiwei.brandkbs2.util.Tools;
import lombok.Data; import lombok.Data;
import lombok.ToString; import lombok.ToString;
...@@ -36,6 +37,11 @@ public class ChannelListVO { ...@@ -36,6 +37,11 @@ public class ChannelListVO {
* 发文数 * 发文数
*/ */
private long articleCount; private long articleCount;
/**
* 参与事件数
*/
private long eventCount;
/** /**
* 情感倾向 * 情感倾向
*/ */
...@@ -45,6 +51,8 @@ public class ChannelListVO { ...@@ -45,6 +51,8 @@ public class ChannelListVO {
*/ */
private String avatarUrl; private String avatarUrl;
private Boolean isCollect;
public static ChannelListVO createFromChannel(String channelFid, long articleCount) { public static ChannelListVO createFromChannel(String channelFid, long articleCount) {
ChannelListVO channelListVO = new ChannelListVO(); ChannelListVO channelListVO = new ChannelListVO();
// projectId, linkedGroupId, platform, realSource, source // projectId, linkedGroupId, platform, realSource, source
...@@ -64,4 +72,13 @@ public class ChannelListVO { ...@@ -64,4 +72,13 @@ public class ChannelListVO {
channelListVO.setArticleCount(articleCount); channelListVO.setArticleCount(articleCount);
return channelListVO; return channelListVO;
} }
public static ChannelListVO createFromChannel(ChannelRecord channelRecord, long articleCount) {
ChannelListVO channelListVO = Tools.convertMap(channelRecord, ChannelListVO.class);
String name = ChannelEmotion.getNameFromState(channelRecord.getEmotion());
channelListVO.setShowEmotion(name);
channelListVO.setArticleCount(articleCount);
return channelListVO;
}
} }
src/main/java/com/zhiwei/brandkbs2/service/ChannelService.java
...@@ -3,6 +3,7 @@ package com.zhiwei.brandkbs2.service; ...@@ -3,6 +3,7 @@ package com.zhiwei.brandkbs2.service;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportAdminChannelArticleDTO; import com.zhiwei.brandkbs2.easyexcel.dto.ExportAdminChannelArticleDTO;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportAdminChannelEventDTO; import com.zhiwei.brandkbs2.easyexcel.dto.ExportAdminChannelEventDTO;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportAppChannelArticleDTO;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportChannelDTO; import com.zhiwei.brandkbs2.easyexcel.dto.ExportChannelDTO;
import com.zhiwei.brandkbs2.pojo.dto.ChannelDTO; import com.zhiwei.brandkbs2.pojo.dto.ChannelDTO;
import com.zhiwei.brandkbs2.pojo.vo.ChannelListVO; import com.zhiwei.brandkbs2.pojo.vo.ChannelListVO;
...@@ -131,7 +132,7 @@ public interface ChannelService { ...@@ -131,7 +132,7 @@ public interface ChannelService {
* @param size * @param size
* @return * @return
*/ */
List<ChannelListVO> getPositiveList(String contendId, String platform, String keyword, String sorter, int size); List<ChannelListVO> getPositiveList(String contendId, String platform, String keyword, String sorter, Long startTime, Long endTime, int size);
/** /**
* 获取敏感渠道榜 * 获取敏感渠道榜
...@@ -143,7 +144,7 @@ public interface ChannelService { ...@@ -143,7 +144,7 @@ public interface ChannelService {
* @param size * @param size
* @return * @return
*/ */
List<ChannelListVO> getNegativeList(String contendId, String platform, String keyword, String sorter, int size); List<ChannelListVO> getNegativeList(String contendId, String platform, String keyword, String sorter, Long startTime, Long endTime, int size);
/** /**
* 收藏渠道 * 收藏渠道
...@@ -183,4 +184,42 @@ public interface ChannelService { ...@@ -183,4 +184,42 @@ public interface ChannelService {
* @return 渠道动向 * @return 渠道动向
*/ */
JSONObject getSpreadingTend(String channelId, String type, Set<String> contends, Long startTime, Long endTime); JSONObject getSpreadingTend(String channelId, String type, Set<String> contends, Long startTime, Long endTime);
/**
* 获取渠道动向-摘要
*
* @param channelId 渠道ID
* @param contends 品牌ID集合
* @param startTime 开始时间
* @param endTime 结束时间
* @return 渠道动向-摘要
*/
JSONObject getSpreadingTendSummary(String channelId, Set<String> contends, Long startTime, Long endTime);
/**
* 获取时间段稿件信息
*
* @param startTime 开始时间时间戳
* @param endTime 结束时间
* @param page 页码
* @param pageSize 页码大小
* @param channelId 渠道ID
* @param contendId 竞品ID
* @return 稿件信息
*/
JSONObject getArticlesByTime(Long startTime, Long endTime, int page, int pageSize, String channelId, String contendId);
/**
* 下载时间段稿件信息
*
* @param startTime 开始时间时间戳
* @param endTime 结束时间
* @param page 页码
* @param pageSize 页码大小
* @param channelId 渠道ID
* @param contendId 竞品ID
* @return 稿件信息
*/
List<ExportAppChannelArticleDTO> downloadArticlesByTime(Long startTime, Long endTime,String channelId, String contendId);
} }
src/main/java/com/zhiwei/brandkbs2/service/impl/ChannelServiceImpl.java
...@@ -2,6 +2,7 @@ package com.zhiwei.brandkbs2.service.impl; ...@@ -2,6 +2,7 @@ package com.zhiwei.brandkbs2.service.impl;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.zhiwei.brandkbs2.auth.UserThreadLocal; import com.zhiwei.brandkbs2.auth.UserThreadLocal;
import com.zhiwei.brandkbs2.common.GenericAttribute; import com.zhiwei.brandkbs2.common.GenericAttribute;
import com.zhiwei.brandkbs2.common.GlobalPojo; import com.zhiwei.brandkbs2.common.GlobalPojo;
...@@ -9,6 +10,7 @@ import com.zhiwei.brandkbs2.config.Constant; ...@@ -9,6 +10,7 @@ import com.zhiwei.brandkbs2.config.Constant;
import com.zhiwei.brandkbs2.dao.*; import com.zhiwei.brandkbs2.dao.*;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportAdminChannelArticleDTO; import com.zhiwei.brandkbs2.easyexcel.dto.ExportAdminChannelArticleDTO;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportAdminChannelEventDTO; import com.zhiwei.brandkbs2.easyexcel.dto.ExportAdminChannelEventDTO;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportAppChannelArticleDTO;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportChannelDTO; import com.zhiwei.brandkbs2.easyexcel.dto.ExportChannelDTO;
import com.zhiwei.brandkbs2.enmus.ChannelEmotion; import com.zhiwei.brandkbs2.enmus.ChannelEmotion;
import com.zhiwei.brandkbs2.enmus.EmotionEnum; import com.zhiwei.brandkbs2.enmus.EmotionEnum;
...@@ -28,8 +30,10 @@ import com.zhiwei.brandkbs2.service.CommonService; ...@@ -28,8 +30,10 @@ import com.zhiwei.brandkbs2.service.CommonService;
import com.zhiwei.brandkbs2.service.ProjectService; import com.zhiwei.brandkbs2.service.ProjectService;
import com.zhiwei.brandkbs2.util.MongoUtil; import com.zhiwei.brandkbs2.util.MongoUtil;
import com.zhiwei.brandkbs2.util.Tools; import com.zhiwei.brandkbs2.util.Tools;
import io.netty.util.concurrent.CompleteFuture;
import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchResponse;
...@@ -51,6 +55,7 @@ import java.io.IOException; ...@@ -51,6 +55,7 @@ import java.io.IOException;
import java.math.BigDecimal; import java.math.BigDecimal;
import java.math.RoundingMode; import java.math.RoundingMode;
import java.util.*; import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors; import java.util.stream.Collectors;
/** /**
...@@ -284,7 +289,7 @@ public class ChannelServiceImpl implements ChannelService { ...@@ -284,7 +289,7 @@ public class ChannelServiceImpl implements ChannelService {
List<ChannelListVO> resList = new ArrayList<>(); List<ChannelListVO> resList = new ArrayList<>();
try { try {
String projectId = UserThreadLocal.getProjectId(); String projectId = UserThreadLocal.getProjectId();
EsClientDao.SearchHelper searchHelper = createSearchHelperByChannelCriteria(projectId, contendId, platform, keyword, startTime, endTime); EsClientDao.SearchHelper searchHelper = createSearchHelperByChannelCriteria(projectId, null, Collections.singleton(contendId), platform, keyword, startTime, endTime);
// 只需要聚合结果,不需返回hit // 只需要聚合结果,不需返回hit
searchHelper.setSize(0); searchHelper.setSize(0);
// aggregation // aggregation
...@@ -296,34 +301,125 @@ public class ChannelServiceImpl implements ChannelService { ...@@ -296,34 +301,125 @@ public class ChannelServiceImpl implements ChannelService {
searchHelper.setAggregationBuilder(aggregationBuilder); searchHelper.setAggregationBuilder(aggregationBuilder);
SearchResponse searchResponse = channelEsDao.searchResponse(searchHelper); SearchResponse searchResponse = channelEsDao.searchResponse(searchHelper);
Terms keyTerms = searchResponse.getAggregations().get("keyAgg"); Terms keyTerms = searchResponse.getAggregations().get("keyAgg");
Map<String, Long> fidCount = new HashMap<>();
for (Terms.Bucket bucket : keyTerms.getBuckets()) { for (Terms.Bucket bucket : keyTerms.getBuckets()) {
String channelFid = Tools.getFidFromChannelKey(bucket.getKeyAsString()); String channelFid = Tools.getFidFromChannelKey(bucket.getKeyAsString());
Sum countSum = bucket.getAggregations().get("countAgg"); Sum countSum = bucket.getAggregations().get("countAgg");
Channel channel = channelDao.queryUnique(channelFid); fidCount.put(channelFid, (long) countSum.getValue());
// Channel channel = channelDao.queryUnique(channelFid);
// if (null != channel) {
// resList.add(ChannelListVO.createFromChannel(channel, (long) countSum.getValue()));
// } else {
// log.error("getActiveChannelList-未收录渠道:{}", bucket.getKeyAsString());
// resList.add(ChannelListVO.createFromChannel(bucket.getKeyAsString(), (long) countSum.getValue()));
// }
}
Map<String, Channel> fidChannel = channelDao.queryUniqueAsync(fidCount.keySet());
fidCount.forEach((fid, count) -> {
Channel channel = fidChannel.get(fid);
if (null != channel) { if (null != channel) {
resList.add(ChannelListVO.createFromChannel(channel, (long) countSum.getValue())); resList.add(ChannelListVO.createFromChannel(channel, count));
} else { } else {
log.error("getActiveChannelList-未收录渠道:{}", bucket.getKeyAsString()); log.error("getActiveChannelList-未收录渠道:{}", fid);
resList.add(ChannelListVO.createFromChannel(bucket.getKeyAsString(), (long) countSum.getValue())); resList.add(ChannelListVO.createFromChannel(fid, count));
}
} }
return resList; });
} catch (IOException e) { } catch (IOException e) {
ExceptionCast.cast(CommonCodeEnum.FAIL.message("es查询异常")); ExceptionCast.cast(CommonCodeEnum.FAIL.message("es查询异常"));
} }
return null; return resList;
} }
@Override @Override
public List<ChannelListVO> getPositiveList(String contendId, String platform, String keyword, String sorter, int size) { public List<ChannelListVO> getPositiveList(String contendId, String platform, String keyword, String sorter, Long startTime, Long endTime, int size) {
// TODO return getEmotionList(contendId, platform, keyword, sorter, startTime, endTime, size, EmotionEnum.POSITIVE.getState());
return null;
} }
@Override @Override
public List<ChannelListVO> getNegativeList(String contendId, String platform, String keyword, String sorter, int size) { public List<ChannelListVO> getNegativeList(String contendId, String platform, String keyword, String sorter, Long startTime, Long endTime, int size) {
// TODO return getEmotionList(contendId, platform, keyword, sorter, startTime, endTime, size, EmotionEnum.NEGATIVE.getState());
}
private List<ChannelListVO> getEmotionList(String contendId, String platform, String keyword, String sorter, Long startTime, Long endTime, int size,
int emotion) {
List<ChannelListVO> resList = new ArrayList<>();
try {
Map<String, Pair<Long, ChannelRecord>> keyMap = new HashMap<>();
String projectId = UserThreadLocal.getProjectId();
EsClientDao.SearchHelper searchHelper = createSearchHelperByChannelCriteria(projectId, null, Collections.singleton(contendId), platform, keyword, startTime, endTime);
// 分页查询所有结果
List<SearchResponse> searchResponses = channelEsDao.searchScrollResponse(searchHelper);
for (SearchResponse searchResponse : searchResponses) {
for (SearchHit hit : searchResponse.getHits().getHits()) {
ChannelRecord channelRecord = new ChannelRecord(hit.getSourceAsMap());
keyMap.compute(channelRecord.getKey(), (k, v) -> {
if (null == v) {
return Pair.of(channelRecord.getRangeEndTime(), channelRecord);
} else {
// 比较数据时间
if (channelRecord.getRangeEndTime() > v.getLeft()) {
v.getRight().setChannelInfo(channelRecord);
}
// 合并文章数据
v.getRight().getRecord().mergeRecord(channelRecord.getRecord());
return v;
}
});
}
}
// 过滤掉不符合时间条件的数据并排序
List<ChannelRecord> channelRecords = keyMap.values().stream().map(pair -> {
ChannelRecord channelRecord = pair.getRight();
// 情感过滤
if (emotion == channelRecord.getEmotion()) {
List<ChannelIndex.Article> articles = ChannelIndex.Record.filterArticles(startTime, endTime, channelRecord.getRecord().getArticles());
articles.sort(Comparator.comparingLong(ChannelIndex.Article::getTime).reversed());
channelRecord.getRecord().setArticles(articles);
return channelRecord;
}
return null; return null;
}).filter(Objects::nonNull).sorted((x, y) -> {
// emotionIndex降序
if (sorter.contains("index")) {
if (x.getEmotionIndex() > y.getEmotionIndex()) {
return -1;
} else if (Objects.equals(x.getEmotionIndex(), y.getEmotionIndex())) {
return 0;
}
return 1;
}
// 稿件数排序
if (x.getRecord().getArticles().size() > y.getRecord().getArticles().size()) {
return -1;
} else if (Objects.equals(x.getRecord().getArticles().size(), y.getRecord().getArticles().size())) {
return 0;
}
return 1;
}).limit(size)
// .map(channelRecord -> {
// Channel channel = channelDao.queryUnique(channelRecord.getChannelFid());
// if (null != channel) {
// return ChannelListVO.createFromChannel(channel, channelRecord.getRecord().getArticles().size());
// } else {
// log.error("getActiveChannelList-未收录渠道:{}", channelRecord.getChannelFid());
// return ChannelListVO.createFromChannel(channelRecord, channelRecord.getRecord().getArticles().size());
// }
// })
.collect(Collectors.toList());
Map<String, ChannelRecord> collect = Maps.uniqueIndex(channelRecords, ChannelRecord::getChannelFid);
Map<String, Channel> fidChannel = channelDao.queryUniqueAsync(collect.keySet());
collect.forEach((fid, record) -> {
Channel channel = fidChannel.get(fid);
if (null != channel) {
resList.add(ChannelListVO.createFromChannel(channel, record.getRecord().getArticles().size()));
} else {
resList.add(ChannelListVO.createFromChannel(record, record.getRecord().getArticles().size()));
}
});
} catch (IOException e) {
ExceptionCast.cast(CommonCodeEnum.FAIL.message("es查询异常"));
}
return resList;
} }
@Override @Override
...@@ -384,7 +480,6 @@ public class ChannelServiceImpl implements ChannelService { ...@@ -384,7 +480,6 @@ public class ChannelServiceImpl implements ChannelService {
@Override @Override
public JSONObject getSpreadingTend(String channelId, String type, Set<String> contends, Long startTime, Long endTime) { public JSONObject getSpreadingTend(String channelId, String type, Set<String> contends, Long startTime, Long endTime) {
try {
JSONObject res = new JSONObject(); JSONObject res = new JSONObject();
// 默认搜索一周 // 默认搜索一周
if (null == startTime || null == endTime) { if (null == startTime || null == endTime) {
...@@ -392,34 +487,136 @@ public class ChannelServiceImpl implements ChannelService { ...@@ -392,34 +487,136 @@ public class ChannelServiceImpl implements ChannelService {
startTime = timeRangeWeek[0]; startTime = timeRangeWeek[0];
endTime = timeRangeWeek[1]; endTime = timeRangeWeek[1];
} }
List<JSONObject> spreadDatas = new ArrayList<>();
if ("事件".equals(type)) {
// 根据品牌分类
for (String contend : contends) {
JSONObject json = new JSONObject();
Channel channel = channelDao.findOneById(channelId); Channel channel = channelDao.findOneById(channelId);
EsClientDao.SearchHelper searchHelper = createSearchHelperByChannelCriteria(UserThreadLocal.getProjectId(), null, null, null, startTime, endTime); json.put("spreadingTend", spreadingTendEvent(startTime, endTime, channel, UserThreadLocal.getProjectId(), contend));
// 添加contends集合 查询 if ("0".equals(contend)) {
EsQueryTools.assembleContendsQuery(searchHelper.getQuery(), contends); spreadDatas.add(0, json);
// 添加渠道限制 } else {
searchHelper.getQuery().must(QueryBuilders.termQuery("channel_fid.keyword", channel.getFid())); spreadDatas.add(json);
// 分页查询所有结果 }
List<SearchResponse> searchResponses = channelEsDao.searchScrollResponse(searchHelper); }
} else {
for (Map.Entry<String, List<ChannelIndex.Article>> entry : getSourceContendMap(channelId, contends, startTime, endTime).entrySet()) {
JSONObject json = new JSONObject();
json.put("spreadingTend", spreadingTendData(startTime, endTime, entry.getValue()));
if ("0".equals(entry.getKey())) {
spreadDatas.add(0, json);
} else {
spreadDatas.add(json);
}
}
}
res.put("spreadDatas", spreadDatas);
return res;
}
@Override
public JSONObject getSpreadingTendSummary(String channelId, Set<String> contends, Long startTime, Long endTime) {
JSONObject res = new JSONObject();
String projectId = UserThreadLocal.getProjectId();
// 默认搜索一周
if (null == startTime || null == endTime) {
Long[] timeRangeWeek = commonService.getTimeRangeWeek();
startTime = timeRangeWeek[0];
endTime = timeRangeWeek[1];
}
// 根据品牌分类 // 根据品牌分类
Map<String, List<ChannelIndex.Article>> contendMap = convert2ContendMap(searchResponses, startTime, endTime); Map<String, List<ChannelIndex.Article>> contendMap = getSourceContendMap(channelId, contends, startTime, endTime);
Channel channel = channelDao.findOneById(channelId);
long articleTotal = 0; long articleTotal = 0;
long eventTotal = 0;
List<JSONObject> spreadDatas = new ArrayList<>(); List<JSONObject> spreadDatas = new ArrayList<>();
for (Map.Entry<String, List<ChannelIndex.Article>> entry : contendMap.entrySet()) { for (Map.Entry<String, List<ChannelIndex.Article>> entry : contendMap.entrySet()) {
JSONObject spreadData = new JSONObject(); String contendId = entry.getKey();
// 按日分组并根据id去重保留最近标注时间 Pair<Long, JSONObject> dataCount = getDataCount(entry.getValue());
Map<Long, List<ChannelIndex.Article>> dateListMap = partitionWithDuplicate(startTime, endTime, entry.getValue()); Pair<Long, JSONObject> eventCount = getEventCount(startTime, endTime, projectId, contendId, channel);
List<JSONObject> spread = dateListMap.entrySet().stream().map(e -> { articleTotal += dataCount.getLeft();
JSONObject spreadJson = new JSONObject(); eventTotal += eventCount.getLeft();
spreadJson.put("time", e.getKey()); // 合并统计
spreadJson.put("sum", e.getValue().size()); dataCount.getRight().putAll(eventCount.getRight());
return spreadJson; if ("0".equals(contendId)) {
}).collect(Collectors.toList()); spreadDatas.add(0, dataCount.getRight());
spreadData.put("spreadingTend", spread); } else {
spreadDatas.add(dataCount.getRight());
}
}
res.put("startTime", startTime);
res.put("endTime", endTime);
res.put("articleTotal", articleTotal);
res.put("eventTotal", eventTotal);
res.put("spreadDatas", spreadDatas);
return res;
}
@Override
public JSONObject getArticlesByTime(Long startTime, Long endTime, int page, int pageSize, String channelId, String contendId) {
JSONObject res = new JSONObject();
Map<String, List<ChannelIndex.Article>> sourceContendMap = getSourceContendMap(channelId, Collections.singleton(contendId), startTime, endTime);
Map<Long, List<ChannelIndex.Article>> timeRangeList = partition(startTime, endTime, Constant.DAY_PATTERN, sourceContendMap.get(contendId));
List<JSONObject> dayList = new ArrayList<>(timeRangeList.size());
timeRangeList.forEach((time, list) -> {
JSONObject dayResult = new JSONObject();
// 每天返回前10条
List<CompletableFuture<JSONObject>> futureList = list.stream().limit(10).map(article -> CompletableFuture.supplyAsync(() -> {
JSONObject json = new JSONObject();
json.put("emotion", article.getEmotion());
json.put("time", article.getTime());
String[] titleAndUrl = getTitleAndUrlById(article.getId());
json.put("title", titleAndUrl[0]);
json.put("url", titleAndUrl[1]);
return json;
})).collect(Collectors.toList());
CompletableFuture.allOf(futureList.toArray(new CompletableFuture[0])).join();
dayResult.put("articles", futureList.stream().map(CompletableFuture::join).collect(Collectors.toList()));
dayResult.put("articleCount", list.size());
dayResult.put("time", time);
dayList.add(dayResult);
});
res.put("list", dayList);
return res;
}
@Override
public List<ExportAppChannelArticleDTO> downloadArticlesByTime(Long startTime, Long endTime, String channelId, String contendId) {
List<ExportAppChannelArticleDTO> res = new ArrayList<>();
List<CompleteFuture<ExportAppChannelArticleDTO>> futureList = new ArrayList<>();
Map<String, List<ChannelIndex.Article>> sourceContendMap = getSourceContendMap(channelId, Collections.singleton(contendId), startTime, endTime);
for (ChannelIndex.Article article : sourceContendMap.get(contendId)) {
// res.add(ExportAppChannelArticleDTO.createFromArticle(article));
}
return res;
}
private Map<String, List<ChannelIndex.Article>> getSourceContendMap(String channelId, Set<String> contendIds, String platform, String keyword, Long startTime,
Long endTime) {
try {
String projectId = UserThreadLocal.getProjectId();
EsClientDao.SearchHelper searchHelper = createSearchHelperByChannelCriteria(projectId, channelId, contendIds, platform, keyword, startTime, endTime);
// 分页查询所有结果
List<SearchResponse> searchResponses = channelEsDao.searchScrollResponse(searchHelper);
// 根据品牌分类
return convert2ContendMap(searchResponses, startTime, endTime);
} catch (IOException e) {
ExceptionCast.cast(CommonCodeEnum.FAIL.message("es查询异常"));
}
return Collections.emptyMap();
}
private Map<String, List<ChannelIndex.Article>> getSourceContendMap(String channelId, Set<String> contendIds, Long startTime, Long endTime) {
return getSourceContendMap(channelId, contendIds, null, null, startTime, endTime);
}
private Pair<Long, JSONObject> getDataCount(List<ChannelIndex.Article> articles) {
JSONObject res = new JSONObject();
// 各稿件数统计 // 各稿件数统计
long positiveCount = 0; long positiveCount = 0;
long negativeCount = 0; long negativeCount = 0;
long total = 0; long total = 0;
for (List<ChannelIndex.Article> articles : dateListMap.values()) {
for (ChannelIndex.Article article : articles) { for (ChannelIndex.Article article : articles) {
if (article.getEmotion() == EmotionEnum.POSITIVE.getState()) { if (article.getEmotion() == EmotionEnum.POSITIVE.getState()) {
positiveCount++; positiveCount++;
...@@ -428,30 +625,74 @@ public class ChannelServiceImpl implements ChannelService { ...@@ -428,30 +625,74 @@ public class ChannelServiceImpl implements ChannelService {
} }
total++; total++;
} }
res.put("positiveCount", positiveCount);
res.put("negativeCount", negativeCount);
res.put("articleTotal", total);
res.put("positivePercent", new BigDecimal((double) positiveCount * 100 / total).setScale(1, RoundingMode.UP));
res.put("negativePercent", new BigDecimal((double) negativeCount * 100 / total).setScale(1, RoundingMode.UP));
return Pair.of(total, res);
} }
spreadData.put("positiveCount", positiveCount);
spreadData.put("negativeCount", negativeCount); private Pair<Long, JSONObject> getEventCount(Long startTime, Long endTime, String projectId, String contendId, Channel channel) {
spreadData.put("articleTotal", total); JSONObject res = new JSONObject();
spreadData.put("positivePercent", new BigDecimal((double) positiveCount * 100 / total).setScale(1, RoundingMode.UP)); String linkedGroupId = projectService.getProjectByContendId(contendId).getBrandLinkedGroupId();
spreadData.put("negativePercent", new BigDecimal((double) negativeCount * 100 / total).setScale(1, RoundingMode.UP)); Map<Long, List<Event>> eventCount;
articleTotal += dateListMap.values().stream().mapToInt(List::size).sum(); if (endTime - startTime > Constant.ONE_MONTH) {
spreadDatas.add(spreadData); eventCount = eventDao.getEventMonth(new ChannelIndex(projectId, linkedGroupId, channel), startTime, endTime);
} else {
eventCount = eventDao.getEventDay(new ChannelIndex(projectId, linkedGroupId, channel), startTime, endTime);
} }
res.put("startTime", startTime); // 事件部分
res.put("endTime", endTime); long positiveEventCount = 0;
res.put("articleTotal", articleTotal); long negativeEventCount = 0;
res.put("eventTotal", 0); long totalEvent = 0;
res.put("spreadDatas", spreadDatas); // 统计参与事件
return res; for (List<Event> events : eventCount.values()) {
} catch (IOException e) { for (Event event : events) {
ExceptionCast.cast(CommonCodeEnum.FAIL.message("es查询异常")); if (EmotionEnum.NEGATIVE.getName().equals(event.getEmotion())) {
negativeEventCount++;
} else if (EmotionEnum.POSITIVE.getName().equals(event.getEmotion())) {
positiveEventCount++;
} }
return null; totalEvent++;
}
}
res.put("positiveEventCount", positiveEventCount);
res.put("negativeEventCount", negativeEventCount);
res.put("totalEvent", totalEvent);
return Pair.of(totalEvent, res);
} }
// private List<JSONObject> getSpreadData(){ private List<JSONObject> spreadingTendData(Long startTime, Long endTime, List<ChannelIndex.Article> articleList) {
// String timePattern = Constant.DAY_PATTERN;
// } if (endTime - startTime > Constant.ONE_MONTH) {
timePattern = Constant.MONTH_PATTERN;
}
// 按日分组并根据id去重保留最近标注时间
Map<Long, List<ChannelIndex.Article>> dateListMap = partition(startTime, endTime, timePattern, articleList);
return dateListMap.entrySet().stream().map(e -> {
JSONObject spreadJson = new JSONObject();
spreadJson.put("time", e.getKey());
spreadJson.put("sum", e.getValue().size());
return spreadJson;
}).collect(Collectors.toList());
}
private List<JSONObject> spreadingTendEvent(Long startTime, Long endTime, Channel channel, String projectId, String contendId) {
String linkedGroupId = projectService.getProjectByContendId(contendId).getBrandLinkedGroupId();
Map<Long, List<Event>> eventCount;
if (endTime - startTime > Constant.ONE_MONTH) {
eventCount = eventDao.getEventMonth(new ChannelIndex(projectId, linkedGroupId, channel), startTime, endTime);
} else {
eventCount = eventDao.getEventDay(new ChannelIndex(projectId, linkedGroupId, channel), startTime, endTime);
}
return eventCount.entrySet().stream().map(e -> {
JSONObject spreadJson = new JSONObject();
spreadJson.put("time", e.getKey());
spreadJson.put("sum", e.getValue().size());
return spreadJson;
}).collect(Collectors.toList());
}
private Map<String, List<ChannelIndex.Article>> convert2ContendMap(List<SearchResponse> searchResponses, Long startTime, Long endTime) { private Map<String, List<ChannelIndex.Article>> convert2ContendMap(List<SearchResponse> searchResponses, Long startTime, Long endTime) {
Map<String, List<ChannelIndex.Article>> contendMap = new HashMap<>(); Map<String, List<ChannelIndex.Article>> contendMap = new HashMap<>();
...@@ -459,16 +700,20 @@ public class ChannelServiceImpl implements ChannelService { ...@@ -459,16 +700,20 @@ public class ChannelServiceImpl implements ChannelService {
for (SearchHit hit : searchResponse.getHits().getHits()) { for (SearchHit hit : searchResponse.getHits().getHits()) {
ChannelRecord channelRecord = new ChannelRecord(hit.getSourceAsMap()); ChannelRecord channelRecord = new ChannelRecord(hit.getSourceAsMap());
// 过滤掉不符合时间条件的数据 // 过滤掉不符合时间条件的数据
List<ChannelIndex.Article> articles = channelRecord.getRecord().getArticles().stream().filter(article -> Tools.hitTimeRange(startTime, endTime, article.getTime())).collect(Collectors.toList()); // List<ChannelIndex.Article> articles = channelRecord.getRecord().getArticles().stream().filter(article -> Tools.hitTimeRange(startTime, endTime, article.getTime())).collect(Collectors.toList());
contendMap.compute(channelRecord.getContendId(), (k, v) -> { // contendMap.compute(channelRecord.getContendId(), (k, v) -> {
if (null == v) { // if (null == v) {
return articles; // return articles;
} // }
v.addAll(articles); // v.addAll(articles);
return v; // return v;
}); // });
contendMap.putIfAbsent(channelRecord.getContendId(), new ArrayList<>());
contendMap.get(channelRecord.getChannelId()).addAll(channelRecord.getRecord().getArticles());
} }
} }
// contendMap.forEach((k, v) -> v.sort(Comparator.comparingLong(ChannelIndex.Article::getTime)));
contendMap.replaceAll((k, v) -> ChannelIndex.Record.filterSortArticles(startTime, endTime, v));
return contendMap; return contendMap;
} }
...@@ -477,27 +722,15 @@ public class ChannelServiceImpl implements ChannelService { ...@@ -477,27 +722,15 @@ public class ChannelServiceImpl implements ChannelService {
* *
* @return * @return
*/ */
private Map<Long, List<ChannelIndex.Article>> partitionWithDuplicate(Long startTime, Long endTime, List<ChannelIndex.Article> articles) { private Map<Long, List<ChannelIndex.Article>> partition(Long startTime, Long endTime, String timePattern, List<ChannelIndex.Article> articles) {
Map<Long, List<ChannelIndex.Article>> res = new HashMap<>(); Map<Long, List<ChannelIndex.Article>> res = new HashMap<>();
// 自动补全时间段 // 自动补全时间段
for (Long timeKey : Tools.parseToDays(startTime, endTime)) { for (Long timeKey : Tools.parseToDays(startTime, endTime)) {
res.put(timeKey, Lists.newArrayList()); res.put(timeKey, Lists.newArrayList());
} }
Map<String, ChannelIndex.Article> setMap = new HashMap<>();
// 去重并保留最近标注时间
for (ChannelIndex.Article article : articles) { for (ChannelIndex.Article article : articles) {
setMap.compute(article.getId(), (k, v) -> {
// 旧值为null或标注时间更新
if (null == v || article.getMtime() > v.getMtime()) {
return article;
}
return v;
});
}
for (ChannelIndex.Article article : setMap.values()) {
// 按日分组 // 按日分组
long key = Tools.truncDate(new Date(article.getTime()), Constant.DAY_PATTERN).getTime(); long key = Tools.truncDate(new Date(article.getTime()), timePattern).getTime();
res.compute(key, (k, v) -> { res.compute(key, (k, v) -> {
if (null == v) { if (null == v) {
v = new ArrayList<>(); v = new ArrayList<>();
...@@ -510,7 +743,7 @@ public class ChannelServiceImpl implements ChannelService { ...@@ -510,7 +743,7 @@ public class ChannelServiceImpl implements ChannelService {
} }
private EsClientDao.SearchHelper createSearchHelperByChannelCriteria(String projectId, String contendId, String platform, String keyword, Long startTime, Long endTime) { private EsClientDao.SearchHelper createSearchHelperByChannelCriteria(String projectId, String channelId, Set<String> contendIds, String platform, String keyword, Long startTime, Long endTime) {
EsClientDao.SearchHelper helper = EsClientDao.createSearchHelper(); EsClientDao.SearchHelper helper = EsClientDao.createSearchHelper();
// 默认搜索一周 // 默认搜索一周
if (null == startTime || null == endTime) { if (null == startTime || null == endTime) {
...@@ -522,8 +755,14 @@ public class ChannelServiceImpl implements ChannelService { ...@@ -522,8 +755,14 @@ public class ChannelServiceImpl implements ChannelService {
BoolQueryBuilder query = QueryBuilders.boolQuery(); BoolQueryBuilder query = QueryBuilders.boolQuery();
// project和contendId // project和contendId
query.must(QueryBuilders.termQuery("project_id.keyword", projectId)); query.must(QueryBuilders.termQuery("project_id.keyword", projectId));
if (null != contendId) { // 添加contends集合 查询
query.must(QueryBuilders.termQuery("contend_id.keyword", contendId)); if (null != contendIds) {
EsQueryTools.assembleContendsQuery(query, contendIds);
}
// 添加渠道限制
if (null != channelId) {
Channel channel = channelDao.findOneById(channelId);
query.must(QueryBuilders.termQuery("channel_fid.keyword", channel.getFid()));
} }
// platformId // platformId
String platformId = GlobalPojo.getPlatformIdByName(platform); String platformId = GlobalPojo.getPlatformIdByName(platform);
...@@ -562,4 +801,13 @@ public class ChannelServiceImpl implements ChannelService { ...@@ -562,4 +801,13 @@ public class ChannelServiceImpl implements ChannelService {
return query; return query;
} }
private String[] getTitleAndUrlById(String id) {
try {
BaseMap baseMap = Tools.getBaseFromEsMap(esClientDao.searchById(id));
return new String[]{baseMap.getTitle(), baseMap.getUrl()};
} catch (IOException ignored) {
}
return new String[]{null, null};
}
} }
src/main/java/com/zhiwei/brandkbs2/service/impl/TaskServiceImpl.java
...@@ -20,6 +20,7 @@ import org.springframework.stereotype.Service; ...@@ -20,6 +20,7 @@ import org.springframework.stereotype.Service;
import javax.annotation.Resource; import javax.annotation.Resource;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletableFuture;
...@@ -70,6 +71,64 @@ public class TaskServiceImpl implements TaskService { ...@@ -70,6 +71,64 @@ public class TaskServiceImpl implements TaskService {
@Override @Override
public void messageFlowCount(int day) { public void messageFlowCount(int day) {
List<Pair<Long[], Map<ChannelIndex, ChannelIndex.Record>>> rangeTimeRecords = esClientDao.searchRecordRecentDay(day); List<Pair<Long[], Map<ChannelIndex, ChannelIndex.Record>>> rangeTimeRecords = esClientDao.searchRecordRecentDay(day);
// 结果合并
List<Map<ChannelIndex, ChannelIndex.Record>> channelList = rangeTimeRecords.stream().map(Pair::getRight).collect(Collectors.toList());
// 合并渠道记录
Map<ChannelIndex, ChannelIndex.Record> channelIndexRecordMap = ChannelIndex.mergeRecord(channelList);
log.info("渠道统计-搜索到近{}天的受影响渠道数{}条", day, channelIndexRecordMap.size());
long handleSize = 0;
List<Channel> insertList = new ArrayList<>();
List<Map.Entry<ChannelIndex, ChannelIndex.Record>> batchList = new ArrayList<>();
// 新recordMap
Map<ChannelIndex, ChannelIndex.Record> newRecordMap = new HashMap<>();
for (Map.Entry<ChannelIndex, ChannelIndex.Record> entry : channelIndexRecordMap.entrySet()) {
batchList.add(entry);
// 每100条做一次清算
if (++handleSize % 100 == 0 || handleSize == channelIndexRecordMap.size()) {
insertList.addAll(batchHandle(batchList, newRecordMap));
batchList = new ArrayList<>();
}
if (handleSize % 10000 == 0) {
log.info("渠道统计-渠道总计-查询更新已完成{}/{}", handleSize, channelIndexRecordMap.size());
}
}
// 替换成新的记录map
channelIndexRecordMap = newRecordMap;
ListUtils.partition(insertList, 1000).forEach(list -> {
channelDao.insertMany(list);
});
log.info("渠道统计-渠道总计-录入完毕,新增渠道{}条,更新渠道{}条", insertList.size(), channelIndexRecordMap.size() - insertList.size());
// 获得单位时间内最小最大时间戳
Long[] timeMinMax = Tools.timeMinMax(rangeTimeRecords.stream().map(Pair::getLeft).collect(Collectors.toList()));
List<ChannelRecord> channelRecords = ChannelRecord.createChannelRecords(timeMinMax[0], timeMinMax[1], channelIndexRecordMap);
channelEsDao.upsertChannelRecord(channelRecords);
log.info("渠道统计-渠道记录-统计结束");
}
private List<Channel> batchHandle(List<Map.Entry<ChannelIndex, ChannelIndex.Record>> batchList, Map<ChannelIndex, ChannelIndex.Record> newRecordMap) {
List<Channel> insertList = new ArrayList<>();
List<String> fids = batchList.stream().map(channelIndexRecordEntry -> channelIndexRecordEntry.getKey().getFid()).collect(Collectors.toList());
Map<String, Channel> fidChannel = channelDao.queryUniqueAsync(fids);
for (Map.Entry<ChannelIndex, ChannelIndex.Record> entry : batchList) {
String fid = entry.getKey().getFid();
Channel channel = fidChannel.get(fid);
if (null == channel) {
channel = Channel.createFromChannelIndexRecord(entry.getKey(), entry.getValue());
insertList.add(channel);
} else {
channel.setRecord(entry.getValue());
channelDao.updateOne(channel);
}
// 设置查询数值
entry.getKey().setChannelInfo(channel);
newRecordMap.put(entry.getKey(), entry.getValue());
}
return insertList;
}
@Deprecated
public void messageFlowCount2(int day) {
List<Pair<Long[], Map<ChannelIndex, ChannelIndex.Record>>> rangeTimeRecords = esClientDao.searchRecordRecentDay(day);
int total = rangeTimeRecords.stream().mapToInt(pair -> pair.getRight().values().size()).sum(); int total = rangeTimeRecords.stream().mapToInt(pair -> pair.getRight().values().size()).sum();
log.info("渠道统计-搜索到近{}天的受影响渠道数{}条", day, total); log.info("渠道统计-搜索到近{}天的受影响渠道数{}条", day, total);
// 结果合并 // 结果合并
...@@ -79,9 +138,11 @@ public class TaskServiceImpl implements TaskService { ...@@ -79,9 +138,11 @@ public class TaskServiceImpl implements TaskService {
// 获得单位时间内最小最大时间戳 // 获得单位时间内最小最大时间戳
Long[] timeMinMax = Tools.timeMinMax(rangeTimeRecords.stream().map(Pair::getLeft).collect(Collectors.toList())); Long[] timeMinMax = Tools.timeMinMax(rangeTimeRecords.stream().map(Pair::getLeft).collect(Collectors.toList()));
List<ChannelRecord> channelRecords = ChannelRecord.createChannelRecords(timeMinMax[0], timeMinMax[1], channelIndexRecordMap); List<ChannelRecord> channelRecords = ChannelRecord.createChannelRecords(timeMinMax[0], timeMinMax[1], channelIndexRecordMap);
// List<ChannelRecord> channelRecords = rangeTimeRecords.stream().map(pair -> ChannelRecord.createChannelRecords(pair.getLeft()[0], pair.getLeft()[1], pair.getRight())).flatMap(Collection::stream).collect(Collectors.toList());
channelEsDao.upsertChannelRecord(channelRecords); channelEsDao.upsertChannelRecord(channelRecords);
log.info("渠道统计-小时级渠道记录-统计结束"); log.info("渠道统计-渠道记录-统计结束");
// List<ChannelRecord> channelRecords = rangeTimeRecords.stream().map(pair -> ChannelRecord.createChannelRecords(pair.getLeft()[0], pair.getLeft()[1], pair.getRight())).flatMap(Collection::stream).collect(Collectors.toList());
long handleSize = 0;
List<Channel> insertList = new ArrayList<>(); List<Channel> insertList = new ArrayList<>();
for (Map.Entry<ChannelIndex, ChannelIndex.Record> entry : channelIndexRecordMap.entrySet()) { for (Map.Entry<ChannelIndex, ChannelIndex.Record> entry : channelIndexRecordMap.entrySet()) {
// 是否已存在 // 是否已存在
...@@ -93,8 +154,13 @@ public class TaskServiceImpl implements TaskService { ...@@ -93,8 +154,13 @@ public class TaskServiceImpl implements TaskService {
channel.setRecord(entry.getValue()); channel.setRecord(entry.getValue());
channelDao.updateOne(channel); channelDao.updateOne(channel);
} }
// 设置查询数值
entry.getKey().setChannelInfo(channel);
if (++handleSize % 10000 == 0) {
log.info("渠道统计-渠道总计-查询更新已完成{}/{}", handleSize, channelIndexRecordMap.size());
}
} }
log.info("渠道统计-渠道总计-查询并更新结束,开始批量入库"); log.info("渠道统计-渠道总计-查询更新结束,开始批量入库");
ListUtils.partition(insertList, 1000).forEach(list -> { ListUtils.partition(insertList, 1000).forEach(list -> {
channelDao.insertMany(list); channelDao.insertMany(list);
}); });
......
src/main/webapp/WEB-INF/applicationContext.xml deleted 100644 → 0
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!-- @version $Id: applicationContext.xml 561608 2007-08-01 00:33:12Z vgritsenko $ -->
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:util="http://www.springframework.org/schema/util"
xmlns:configurator="http://cocoon.apache.org/schema/configurator"
xmlns:avalon="http://cocoon.apache.org/schema/avalon"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd
http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-2.0.xsd
http://cocoon.apache.org/schema/configurator http://cocoon.apache.org/schema/configurator/cocoon-configurator-1.0.1.xsd
http://cocoon.apache.org/schema/avalon http://cocoon.apache.org/schema/avalon/cocoon-avalon-1.0.xsd">
<!-- Activate Cocoon Spring Configurator -->
<configurator:settings/>
<!-- Configure Log4j -->
<bean name="org.apache.cocoon.spring.configurator.log4j"
class="org.apache.cocoon.spring.configurator.log4j.Log4JConfigurator"
scope="singleton">
<property name="settings" ref="org.apache.cocoon.configuration.Settings"/>
<property name="resource" value="/WEB-INF/log4j.xml"/>
</bean>
<!-- Activate Avalon Bridge -->
<avalon:bridge/>
</beans>
src/main/webapp/WEB-INF/log4j.xml deleted 100644 → 0
<?xml version="1.0" encoding="UTF-8" ?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
<!--
- This is a sample configuration for log4j.
- It simply just logs everything into a single log file.
- Note, that you can use properties for value substitution.
-->
<appender name="CORE" class="org.apache.log4j.FileAppender">
<param name="File" value="${org.apache.cocoon.work.directory}/cocoon-logs/log4j.log" />
<param name="Append" value="false" />
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%d %-5p %t %c - %m%n"/>
</layout>
</appender>
<root>
<priority value="${org.apache.cocoon.log4j.loglevel}"/>
<appender-ref ref="CORE"/>
</root>
</log4j:configuration>
src/main/webapp/WEB-INF/web.xml deleted 100644 → 0
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!--
- This is the Cocoon web-app configurations file
-
- $Id$
-->
<web-app version="2.4"
xmlns="http://java.sun.com/xml/ns/j2ee"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://java.sun.com/xml/ns/j2ee http://java.sun.com/xml/ns/j2ee/web-app_2_4.xsd">
<!-- Servlet Filters ================================================ -->
<!--
- Declare a filter for multipart MIME handling
-->
<filter>
<description>Multipart MIME handling filter for Cocoon</description>
<display-name>Cocoon multipart filter</display-name>
<filter-name>CocoonMultipartFilter</filter-name>
<filter-class>org.apache.cocoon.servlet.multipart.MultipartFilter</filter-class>
</filter>
<!--
- Declare a filter for debugging incoming request
-->
<filter>
<description>Log debug information about each request</description>
<display-name>Cocoon debug filter</display-name>
<filter-name>CocoonDebugFilter</filter-name>
<filter-class>org.apache.cocoon.servlet.DebugFilter</filter-class>
</filter>
<!-- Filter mappings ================================================ -->
<!--
- Use the Cocoon multipart filter together with the Cocoon demo webapp
-->
<filter-mapping>
<filter-name>CocoonMultipartFilter</filter-name>
<servlet-name>Cocoon</servlet-name>
</filter-mapping>
<filter-mapping>
<filter-name>CocoonMultipartFilter</filter-name>
<servlet-name>DispatcherServlet</servlet-name>
</filter-mapping>
<!--
- Use the Cocoon debug filter together with the Cocoon demo webapp
<filter-mapping>
<filter-name>CocoonDebugFilter</filter-name>
<servlet-name>Cocoon</servlet-name>
</filter-mapping>
-->
<!-- Servlet Context Listener ======================================= -->
<!--
- Declare Spring context listener which sets up the Spring Application Context
- containing all Cocoon components (and user defined beans as well).
-->
<listener>
<listener-class>org.springframework.web.context.ContextLoaderListener</listener-class>
</listener>
<!--
- Declare Spring request listener which sets up the required RequestAttributes
- to support Springs and Cocoon custom bean scopes like the request scope or the
- session scope.
-->
<listener>
<listener-class>org.springframework.web.context.request.RequestContextListener</listener-class>
</listener>
<!-- Servlet Configuration ========================================== -->
<!--
- Servlet that dispatches requests to the Spring managed block servlets
-->
<servlet>
<description>Cocoon blocks dispatcher</description>
<display-name>DispatcherServlet</display-name>
<servlet-name>DispatcherServlet</servlet-name>
<servlet-class>org.apache.cocoon.servletservice.DispatcherServlet</servlet-class>
<load-on-startup>1</load-on-startup>
</servlet>
<!-- URL space mappings ============================================= -->
<!--
- Cocoon handles all the URL space assigned to the webapp using its sitemap.
- It is recommended to leave it unchanged. Under some circumstances though
- (like integration with proprietary webapps or servlets) you might have
- to change this parameter.
-->
<servlet-mapping>
<servlet-name>DispatcherServlet</servlet-name>
<url-pattern>/*</url-pattern>
</servlet-mapping>
</web-app>
\ No newline at end of file
src/test/java/com/zhiwei/brandkbs2/Test.java
package com.zhiwei.brandkbs2; package com.zhiwei.brandkbs2;
import com.hankcs.hanlp.HanLP; import com.alibaba.fastjson.JSONObject;
import com.hankcs.hanlp.dictionary.CustomDictionary; import com.hankcs.hanlp.dictionary.CustomDictionary;
import com.hankcs.hanlp.dictionary.stopword.CoreStopWordDictionary; import com.hankcs.hanlp.dictionary.stopword.CoreStopWordDictionary;
import com.hankcs.hanlp.seg.common.Term; import com.zhiwei.brandkbs2.util.Tools;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.util.List;
/** /**
* @ClassName: Test * @ClassName: Test
...@@ -22,15 +21,19 @@ public class Test { ...@@ -22,15 +21,19 @@ public class Test {
} }
public static void main(String[] args) throws UnsupportedEncodingException { public static void main(String[] args) throws UnsupportedEncodingException {
List<Term> results= HanLP.segment("国家发改委：猪肉供应有保障 猪肉价格不具备大幅上涨基础");
for(Term term: CoreStopWordDictionary.apply(results)){
// if(CoreStopWordDictionary.shouldInclude(term)){
// System.out.println(term);
// }else{
System.err.println(term);
// }
}
System.out.println(JSONObject.toJSONString(Tools.parseToDays(1658512800000L,1659031200000L)));
// List<Term> results= HanLP.segment("国家发改委：猪肉供应有保障 猪肉价格不具备大幅上涨基础");
// for(Term term: CoreStopWordDictionary.apply(results)){
//// if(CoreStopWordDictionary.shouldInclude(term)){
//// System.out.println(term);
//// }else{
// System.err.println(term);
//// }
//
// }
// String token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJwYXlsb2FkIjoie1widXNlcklkXCI6XCIyMFwiLFwia2V5XCI6XCIyMFwiLFwibmlja05hbWVcIjpcIuayiOWQm-adsFwiLFwic2VydmljZVwiOlwiXCJ9IiwiZXhwIjoxNjUzMDExNjcwLCJpYXQiOjE2NTI0MDY4MTB9.jcVXxeZkayc6-Aiq8cyYc1uyq4ugji6FdWQXCCp4M2o"; // String token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJwYXlsb2FkIjoie1widXNlcklkXCI6XCIyMFwiLFwia2V5XCI6XCIyMFwiLFwibmlja05hbWVcIjpcIuayiOWQm-adsFwiLFwic2VydmljZVwiOlwiXCJ9IiwiZXhwIjoxNjUzMDExNjcwLCJpYXQiOjE2NTI0MDY4MTB9.jcVXxeZkayc6-Aiq8cyYc1uyq4ugji6FdWQXCCp4M2o";
// String token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJwYXlsb2FkIjoiXCLlk4Hop4FcIiIsImV4cCI6NDc2MjgyMjEzMiwiaWF0IjoxNjUyNDIyMDcyfQ.DXQ8yKgfsCMjhT0xniZeWCMv4syqIoDvztU4QWsd-Fg"; // String token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJwYXlsb2FkIjoiXCLlk4Hop4FcIiIsImV4cCI6NDc2MjgyMjEzMiwiaWF0IjoxNjUyNDIyMDcyfQ.DXQ8yKgfsCMjhT0xniZeWCMv4syqIoDvztU4QWsd-Fg";
......
src/test/java/com/zhiwei/brandkbs2/TestRunWith.java
...@@ -62,14 +62,15 @@ public class TestRunWith { ...@@ -62,14 +62,15 @@ public class TestRunWith {
// UserThreadLocal.set(userInfo); // UserThreadLocal.set(userInfo);
// ResponseResult result = appArticleController.getMarkSpread(1657468800000L, 1657555200000L); // ResponseResult result = appArticleController.getMarkSpread(1657468800000L, 1657555200000L);
// System.out.println(JSONObject.toJSONString(result)); // System.out.println(JSONObject.toJSONString(result));
// taskService.messageFlowCount(1); taskService.messageFlowCount(2);
reportService.getReportsAggCount(); // reportService.getReportsAggCount();
} }
@Test @Test
public void test2() { public void test2() {
taskService.generateReportAndSend(); taskService.messageFlowCount(1);
// taskService.generateReportAndSend();
// ChannelIndex channelIndex = new ChannelIndex(); // ChannelIndex channelIndex = new ChannelIndex();
// channelIndex.setPlatform("微信"); // channelIndex.setPlatform("微信");
// channelIndex.setRealSource("微信公众号"); // channelIndex.setRealSource("微信公众号");
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment