Skip to content

M
mongo-train
Commit 861c3990 by liang
Options

完整实现知微事件分析单个事件的指定分析结果与各项数据输出

parent ab74b2c3
Showing with 1081 additions and 40 deletions
mongo-event/src/main/java/com/liang/mongoevent/controller/MediaController.java 0 → 100644
package com.liang.mongoevent.controller;
import com.liang.mongoevent.pojo.Media;
import com.liang.mongoevent.service.MediaService;
import com.liang.mongoevent.utils.RespBean;
import io.swagger.annotations.ApiOperation;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import java.util.List;
import java.util.Map;
@RestController
@RequestMapping("/media")
public class MediaController {
@Autowired
private MongoTemplate mongoTemplate;
@Autowired
private MediaService mediaService;
@ApiOperation("数据相同标题聚合并记录渠道及单渠道发布此标题报道数目")
@GetMapping("countTitleAndSingle")
public RespBean countTitleAndSingle(@RequestParam(defaultValue = "1") Integer page, @RequestParam(defaultValue = "20") Integer size){
return mediaService.countTitleAndSingle(page,size);
}
@ApiOperation("百度新闻每小时数据趋势")
@GetMapping("trendPerHour")
public RespBean trendPerHour(@RequestParam(defaultValue = "1") Integer page, @RequestParam(defaultValue = "20") Integer size){
return mediaService.trendPerHour(page,size);
}
@ApiOperation("各级媒体占比计算")
@GetMapping("proportion")
public RespBean proportion(@RequestParam(defaultValue = "1") Integer page, @RequestParam(defaultValue = "20") Integer size){
return mediaService.proportion(page,size);
}
@ApiOperation("事件微信数据每小时趋势统计")
@GetMapping("wechatTrendPerHour")
public RespBean wechatTrendPerHour(@RequestParam(defaultValue = "1") Integer page, @RequestParam(defaultValue = "20") Integer size){
return mediaService.wechatTrendPerHour(page,size);
}
@ApiOperation("总数据趋势")
@GetMapping("totalDataTrend")
public RespBean totalDataTrend(@RequestParam(defaultValue = "1") Integer page, @RequestParam(defaultValue = "20") Integer size){
return mediaService.totalDataTrend(page,size);
}
@ApiOperation("各级媒体占比计算")
@GetMapping("singleProportion")
public Map<String,Long> singleProportion(String eventId){
return mediaService.singleProportion(eventId);
}
}
mongo-event/src/main/java/com/liang/mongoevent/controller/SingleController.java 0 → 100644
package com.liang.mongoevent.controller;
import com.liang.mongoevent.pojo.Media;
import com.liang.mongoevent.pojo.Weibo;
import com.liang.mongoevent.service.MediaService;
import com.liang.mongoevent.service.WeiboService;
import com.liang.mongoevent.utils.RespBean;
import com.liang.mongoevent.vo.Trend2;
import com.liang.mongoevent.vo.Trend3;
import io.swagger.annotations.ApiOperation;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Sort;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.aggregation.Aggregation;
import org.springframework.data.mongodb.core.aggregation.AggregationResults;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@RestController
@RequestMapping("/single")
public class SingleController {
@Autowired
private MongoTemplate mongoTemplate;
@Autowired
private MediaService mediaService;
@Autowired
private WeiboService weiboService;
/**
*
* @param eventId 40d0e2e2e7ca4c7d10007560
* @return
*/
@ApiOperation("分析单个事件")
@GetMapping("/singleEvent")
public RespBean singleEvent(String eventId){
Map<String, Object> map=new LinkedHashMap<>();
Map<String, Integer> map1=new LinkedHashMap<>();
List<Map> list=new ArrayList<>();
DateTimeFormatter df = DateTimeFormatter.ofPattern("yyyy-MM-dd HH");
Aggregation agg = Aggregation.newAggregation(
Aggregation.match(Criteria.where("eventId").is(eventId)),
Aggregation.group("time").first("time").as("time"),
Aggregation.sort(Sort.Direction.DESC,"time")
);
AggregationResults<Trend3> aggregate = mongoTemplate.aggregate(agg, Media.class, Trend3.class);
for (Trend3 trend3 : aggregate) {
String time = df.format(trend3.getTime().minusHours(8));
if (map1.containsKey(time)) {
int i = map1.get(time) + 1;
map1.put(time, i);
} else {
map1.put(time, 1);
}
}
AggregationResults<Trend2> aggregate1 = mongoTemplate.aggregate(agg, Weibo.class, Trend2.class);
for (Trend2 trend2 : aggregate1) {
String time = trend2.getTime().substring(0,13);
if (map1.containsKey(time)) {
int i = map1.get(time) + 1;
map1.put(time, i);
} else {
map1.put(time, 1);
}
}
map.put("eventId",eventId);
map.put("性别统计",weiboService.singleCountSex(eventId));
map.put("地域统计",weiboService.singleCountLocation(eventId));
map.put("用户发博数",weiboService.singleCountWeibo(eventId));
map.put("用户粉丝数前十",weiboService.singleTopTenFensiNum(eventId));
map.put("各级媒体占比",mediaService.singleProportion(eventId));
map.put("trend",map1);
list.add(map);
return RespBean.success("成功",list);
}
}
mongo-event/src/main/java/com/liang/mongoevent/controller/WeiboController.java
......@@ -10,6 +10,8 @@ import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import java.util.Map;
@RestController
@RequestMapping("/weibo")
public class WeiboController {
......@@ -46,4 +48,27 @@ public class WeiboController {
return weiboService.trendPerHour(page,size);
}
@ApiOperation("单个性别统计")
@GetMapping("singleCountSex")
public Map<String,Long> singleCountSex(String eventId){
return weiboService.singleCountSex(eventId);
}
@ApiOperation("单个地域统计")
@GetMapping("singleCountLocation")
public Map<String,Long> singleCountLocation(String eventId){
return weiboService.singleCountLocation(eventId);
}
@ApiOperation("当前用户发博数统计")
@GetMapping("singleCountWeibo")
public Map<String,Long> singleCountWeibo(String eventId){
return weiboService.singleCountWeibo(eventId);
}
@ApiOperation("当前参与用户粉丝数前10")
@GetMapping("singleTopTenFensiNum")
public Map<String,Long> singleTopTenFensiNum(String eventId){
return weiboService.singleTopTenFensiNum(eventId);
}
}
mongo-event/src/main/java/com/liang/mongoevent/pojo/Media.java
......@@ -2,6 +2,7 @@ package com.liang.mongoevent.pojo;
import cn.afterturn.easypoi.excel.annotation.Excel;
import cn.afterturn.easypoi.excel.annotation.ExcelTarget;
import com.fasterxml.jackson.annotation.JsonFormat;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
......@@ -10,6 +11,7 @@ import org.springframework.data.mongodb.core.index.CompoundIndex;
import org.springframework.data.mongodb.core.index.Indexed;
import org.springframework.data.mongodb.core.mapping.Document;
import java.time.LocalDateTime;
import java.util.Date;
@Data
......@@ -30,7 +32,7 @@ public class Media {
private String title;
@Excel(name = "创建时间",width = 20.0,orderNum = "3",format = "yyyy-MM-dd HH:mm:ss")
private Date time;
private LocalDateTime time;
@Excel(name = "来源",width = 30.0,orderNum = "4")
private String source;
@Excel(name = "类型",width = 20.0,orderNum = "5")
......
mongo-event/src/main/java/com/liang/mongoevent/pojo/StaNews.java 0 → 100644
package com.liang.mongoevent.pojo;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.index.Indexed;
@Data
@AllArgsConstructor
@NoArgsConstructor
public class StaNews {
@Id
private String id;
@Indexed
private String eventId;
private Long wechat;
private Long baidu;
}
mongo-event/src/main/java/com/liang/mongoevent/pojo/Weibo.java
......@@ -7,12 +7,13 @@ import org.springframework.data.mongodb.core.index.CompoundIndex;
import org.springframework.data.mongodb.core.index.Indexed;
import org.springframework.data.mongodb.core.mapping.Document;
import java.time.LocalDateTime;
import java.util.Date;
@Data
@AllArgsConstructor
@NoArgsConstructor
@CompoundIndex(def = "{'eventId':1,'time':1}")
@CompoundIndex(def = "{'eventId':1,'time':-1}")
@Document
public class Weibo {
......
mongo-event/src/main/java/com/liang/mongoevent/service/MediaService.java 0 → 100644
package com.liang.mongoevent.service;
import com.liang.mongoevent.pojo.Media;
import com.liang.mongoevent.utils.RespBean;
import java.util.List;
import java.util.Map;
public interface MediaService {
RespBean countTitleAndSingle(Integer page, Integer size);
RespBean trendPerHour(Integer page, Integer size);
RespBean proportion(Integer page, Integer size);
RespBean wechatTrendPerHour(Integer page, Integer size);
RespBean totalDataTrend(Integer page, Integer size);
Map<String, Long> singleProportion(String eventId);
}
mongo-event/src/main/java/com/liang/mongoevent/service/MediaServiceImpl.java 0 → 100644
package com.liang.mongoevent.service;
import com.liang.mongoevent.pojo.Media;
import com.liang.mongoevent.pojo.StaNews;
import com.liang.mongoevent.pojo.Weibo;
import com.liang.mongoevent.utils.RespBean;
import com.liang.mongoevent.vo.*;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Sort;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.aggregation.Aggregation;
import org.springframework.data.mongodb.core.aggregation.AggregationOptions;
import org.springframework.data.mongodb.core.aggregation.AggregationResults;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.Duration;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.*;
@Service
public class MediaServiceImpl implements MediaService {
@Autowired
private MongoTemplate mongoTemplate;
@Override
public RespBean countTitleAndSingle(Integer page, Integer size) {
Aggregation agg = Aggregation.newAggregation(
Aggregation.project("title","source").andExclude("_id"),
Aggregation.group("title").first("title").as("title").addToSet("source").as("sourceList").count().as("count"),
Aggregation.limit(20)
).withOptions(AggregationOptions.builder().allowDiskUse(true).build());
List<Map> mappedResults = mongoTemplate.aggregate(agg, Media.class, Map.class).getMappedResults();
return RespBean.success("成功",mappedResults);
}
@Override
public RespBean trendPerHour(Integer page, Integer size) {
List<StaNews> all = mongoTemplate.find(new Query().skip(size*(page-1)).limit(size).with(Sort.by(Sort.Direction.DESC,"time")),StaNews.class);
DateTimeFormatter df = DateTimeFormatter.ofPattern("yyyy-MM-dd HH");
Map<String, Object> map=new LinkedHashMap<>();
for (StaNews staNews : all) {
Map<String, Integer> map1=new LinkedHashMap<>();
if(staNews.getBaidu()!=0){
Aggregation agg = Aggregation.newAggregation(
Aggregation.match(Criteria.where("eventId").is(staNews.getEventId()).and("pt").is("百度新闻")),
Aggregation.group("time").first("time").as("time"),
Aggregation.sort(Sort.Direction.DESC,"time")
);
AggregationResults<Trend3> aggregate = mongoTemplate.aggregate(agg, Media.class, Trend3.class);
for (Trend3 trend3 : aggregate) {
String time = df.format(trend3.getTime().minusHours(8));
if (map1.containsKey(time)) {
int i = map1.get(time) + 1;
map1.put(time, i);
} else {
map1.put(time, 1);
}
}
}
map.put(staNews.getEventId(),map1);
}
return RespBean.success("成功",map);
}
@Override
public RespBean proportion(Integer page, Integer size) {
Aggregation agg = Aggregation.newAggregation(
Aggregation.group("source").first("source").as("source").count().as("count"),
Aggregation.sort(Sort.Direction.DESC,"count"),
Aggregation.skip(size*(page-1)),
Aggregation.limit(size)
);
HashMap<String, Long> map = new LinkedHashMap<>();
AggregationResults<SourceProp> aggregate = mongoTemplate.aggregate(agg, Media.class, SourceProp.class);
for (SourceProp sourceProp : aggregate) {
map.put(sourceProp.getSource(),sourceProp.getCount());
}
return RespBean.success("成功",map);
}
@Override
public RespBean wechatTrendPerHour(Integer page, Integer size) {
List<StaNews> all = mongoTemplate.find(new Query().skip(size*(page-1)).limit(size).with(Sort.by(Sort.Direction.DESC,"time")),StaNews.class);
DateTimeFormatter df = DateTimeFormatter.ofPattern("yyyy-MM-dd HH");
Map<String, Object> map=new LinkedHashMap<>();
for (StaNews staNews : all) {
Map<String, Integer> map1=new LinkedHashMap<>();
if(staNews.getBaidu()!=0){
Aggregation agg = Aggregation.newAggregation(
Aggregation.match(Criteria.where("eventId").is(staNews.getEventId()).and("pt").is("微信")),
Aggregation.group("time").first("time").as("time"),
Aggregation.sort(Sort.Direction.DESC,"time")
);
AggregationResults<Trend3> aggregate = mongoTemplate.aggregate(agg, Media.class, Trend3.class);
for (Trend3 trend3 : aggregate) {
String time = df.format(trend3.getTime().minusHours(8));
if (map1.containsKey(time)) {
int i = map1.get(time) + 1;
map1.put(time, i);
} else {
map1.put(time, 1);
}
}
}
map.put(staNews.getEventId(),map1);
}
return RespBean.success("成功",map);
}
@Override
public RespBean totalDataTrend(Integer page, Integer size) {
List<AllEventId> eventIds = mongoTemplate.find(new Query().skip(size * (page - 1)).limit(size)
.with(Sort.by(Sort.Direction.DESC, "time")), AllEventId.class);
DateTimeFormatter df = DateTimeFormatter.ofPattern("yyyy-MM-dd HH");
Map<String, Object> map=new LinkedHashMap<>();
for (AllEventId allEventId : eventIds) {
Map<String, Integer> map1 = new LinkedHashMap<>();
Aggregation agg = Aggregation.newAggregation(
Aggregation.match(Criteria.where("eventId").is(allEventId.getEventId())),
Aggregation.group("time").first("time").as("time"),
Aggregation.sort(Sort.Direction.DESC, "time")
);
AggregationResults<Trend3> aggregate = mongoTemplate.aggregate(agg, Media.class, Trend3.class);
for (Trend3 meida : aggregate) {
String time = df.format(meida.getTime().minusHours(8));
if (map1.containsKey(time)) {
int i = map1.get(time) + 1;
map1.put(time, i);
} else {
map1.put(time, 1);
}
}
AggregationResults<Trend2> aggregate1 = mongoTemplate.aggregate(agg, Weibo.class, Trend2.class);
for (Trend2 weibo : aggregate1) {
String time = weibo.getTime().substring(0,13);
if (map1.containsKey(time)) {
int i = map1.get(time) + 1;
map1.put(time, i);
} else {
map1.put(time, 1);
}
}
map.put(allEventId.getEventId(), map1);
}
return RespBean.success("成功",map);
}
@Override
public Map<String, Long> singleProportion(String eventId) {
Aggregation agg = Aggregation.newAggregation(
Aggregation.match(Criteria.where("eventId").is(eventId)),
Aggregation.group("source").first("source").as("source").count().as("count"),
Aggregation.sort(Sort.Direction.DESC,"count")
);
HashMap<String, Long> map = new LinkedHashMap<>();
AggregationResults<SourceProp> aggregate = mongoTemplate.aggregate(agg, Media.class, SourceProp.class);
long total=0;
for (SourceProp sourceProp : aggregate) {
map.put(sourceProp.getSource(),sourceProp.getCount());
total+=sourceProp.getCount();
}
map.put("total",total);
return map;
}
}
mongo-event/src/main/java/com/liang/mongoevent/service/WeiboService.java
......@@ -2,6 +2,8 @@ package com.liang.mongoevent.service;
import com.liang.mongoevent.utils.RespBean;
import java.util.Map;
public interface WeiboService {
RespBean countSex();
......@@ -13,4 +15,12 @@ public interface WeiboService {
RespBean topTenFensiNum();
RespBean trendPerHour(Integer page,Integer size);
Map<String,Long> singleCountSex(String eventId);
Map<String,Long> singleCountLocation(String eventId);
Map<String, Long> singleCountWeibo(String eventId);
Map<String, Long> singleTopTenFensiNum(String eventId);
}
mongo-event/src/main/java/com/liang/mongoevent/service/WeiboServiceImpl.java
......@@ -2,9 +2,7 @@ package com.liang.mongoevent.service;
import com.liang.mongoevent.pojo.Weibo;
import com.liang.mongoevent.utils.RespBean;
import com.liang.mongoevent.vo.Location;
import com.liang.mongoevent.vo.TimeCount;
import com.liang.mongoevent.vo.Trend;
import com.liang.mongoevent.vo.*;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Sort;
import org.springframework.data.mongodb.core.MongoTemplate;
......@@ -81,35 +79,121 @@ public class WeiboServiceImpl implements WeiboService {
@Override
public RespBean trendPerHour(Integer page,Integer size) {
Aggregation agg = Aggregation.newAggregation(
Aggregation.group("eventId").first("eventId").as("eventId").first("time").as("etime").last("time").as("stime"),
Aggregation.sort(Sort.Direction.DESC,"stime"),
Aggregation.skip(size*(page-1)),
Aggregation.limit(size)
);
ArrayList<Object> list = new ArrayList<>();
Map<String, Object> map=new LinkedHashMap<>();
AggregationResults<Trend> aggregate = mongoTemplate.aggregate(agg, Weibo.class, Trend.class);
List<String> eventIds = mongoTemplate.findDistinct(new Query().skip(size*(page-1)).limit(size).with(Sort.by(Sort.Direction.DESC,"time")),"eventId", Weibo.class, String.class);
DateTimeFormatter df = DateTimeFormatter.ofPattern("yyyy-MM-dd HH");
for (Trend trend : aggregate) {
LocalDateTime stime = LocalDateTime.parse(trend.getStime().substring(0,13), df);
LocalDateTime etime = LocalDateTime.parse(trend.getEtime().substring(0,13), df);
Duration duration = Duration.between(stime,etime);
long l = duration.toHours();
Map<String, Object> map1=new LinkedHashMap<>();
for (int i = 0; i <=l; i++) {
Query query = new Query(Criteria.where("eventId").is(trend.getEventId()).and("time").regex("^" + df.format(stime)));
long count = mongoTemplate.count(query, Weibo.class);
map1.put(df.format(stime),count);
stime=stime.plusHours(1);
Map<String, Object> map=new LinkedHashMap<>();
for (String eventId : eventIds) {
Aggregation agg = Aggregation.newAggregation(
Aggregation.match(Criteria.where("eventId").is(eventId)),
Aggregation.group("time").first("time").as("time"),
Aggregation.sort(Sort.Direction.DESC,"time")
);
Map<String, Integer> map1=new LinkedHashMap<>();
AggregationResults<Trend2> aggregate = mongoTemplate.aggregate(agg, Weibo.class, Trend2.class);
for (Trend2 trend2 : aggregate) {
String time = trend2.getTime().substring(0,13);
if (map1.containsKey(time)) {
int i = map1.get(time) + 1;
map1.put(time, i);
} else {
map1.put(time, 1);
}
}
map.put(trend.getEventId(),map1);
map.put(eventId,map1);
}
//Aggregation agg = Aggregation.newAggregation(
// Aggregation.group("eventId").first("eventId").as("eventId").first("time").as("etime").last("time").as("stime"),
// Aggregation.sort(Sort.Direction.DESC,"stime"),
// Aggregation.skip(size*(page-1)),
// Aggregation.limit(size)
//);
//
//Map<String, Object> map=new LinkedHashMap<>();
//AggregationResults<Trend> aggregate = mongoTemplate.aggregate(agg, Weibo.class, Trend.class);
//DateTimeFormatter df = DateTimeFormatter.ofPattern("yyyy-MM-dd HH");
//for (Trend trend : aggregate) {
// LocalDateTime stime = LocalDateTime.parse(trend.getStime().substring(0,13), df);
// LocalDateTime etime = LocalDateTime.parse(trend.getEtime().substring(0,13), df);
// Duration duration = Duration.between(stime,etime);
// long l = duration.toHours();
// Map<String, Object> map1=new LinkedHashMap<>();
//
// for (int i = 0; i <=l; i++) {
// Query query = new Query(Criteria.where("eventId").is(trend.getEventId()).and("time").regex("^" + df.format(stime)));
// long count = mongoTemplate.count(query, Weibo.class);
// map1.put(df.format(stime),count);
// stime=stime.plusHours(1);
// }
// map.put(trend.getEventId(),map1);
//
//}
return RespBean.success("成功",map);
}
@Override
public Map<String,Long> singleCountSex(String eventId) {
Query query1 = new Query(Criteria.where("eventId").is(eventId).and("sex").is("m"));
Query query2 = new Query(Criteria.where("eventId").is(eventId).and("sex").is("f"));
long male = mongoTemplate.count(query1, Weibo.class);
long female = mongoTemplate.count(query2, Weibo.class);
Map<String,Long> sexMap=new HashMap<>();
sexMap.put("男",male);
sexMap.put("女",female);
return sexMap;
}
@Override
public Map<String,Long> singleCountLocation(String eventId) {
Aggregation agg = Aggregation.newAggregation(
Aggregation.match(Criteria.where("eventId").is(eventId)),
Aggregation.group("location").first("location").as("location").count().as("count"),
Aggregation.sort(Sort.Direction.DESC,"count")
);
AggregationResults<Location> aggregate = mongoTemplate.aggregate(agg, Weibo.class, Location.class);
Map<String,Long> locationMap=new LinkedHashMap<>();
for (Location location : aggregate) {
locationMap.put(location.getLocation(),location.getCount());
}
return locationMap;
}
@Override
public Map<String, Long> singleCountWeibo(String eventId) {
Aggregation agg = Aggregation.newAggregation(
Aggregation.match(Criteria.where("eventId").is(eventId)),
Aggregation.group("user_id").first("user_id").as("user_id").first("username").as("username").first("weibo").as("weibo"),
Aggregation.sort(Sort.Direction.DESC,"weibo")
);
AggregationResults<Weibo> aggregate = mongoTemplate.aggregate(agg, Weibo.class, Weibo.class);
Map<String,Long> weiboMap=new LinkedHashMap<>();
for (Weibo weibo : aggregate) {
weiboMap.put(weibo.getUsername(),weibo.getWeibo());
}
return weiboMap;
}
@Override
public Map<String, Long> singleTopTenFensiNum(String eventId) {
Aggregation agg = Aggregation.newAggregation(
Aggregation.match(Criteria.where("eventId").is(eventId)),
Aggregation.group("user_id").first("user_id").as("user_id").first("username").as("username").first("fensi").as("fensi"),
Aggregation.sort(Sort.Direction.DESC,"fensi"),
Aggregation.limit(10)
);
AggregationResults<Weibo> aggregate = mongoTemplate.aggregate(agg, Weibo.class, Weibo.class);
Map<String,Long> fensiMap=new LinkedHashMap<>();
for (Weibo weibo : aggregate) {
fensiMap.put(weibo.getUsername(), weibo.getFensi());
}
return fensiMap;
}
}
mongo-event/src/main/java/com/liang/mongoevent/vo/AllEventId.java 0 → 100644
package com.liang.mongoevent.vo;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.springframework.data.mongodb.core.index.Indexed;
@Data
@AllArgsConstructor
@NoArgsConstructor
public class AllEventId {
private String id;
@Indexed
private String eventId;
}
mongo-event/src/main/java/com/liang/mongoevent/vo/Location.java
......@@ -8,7 +8,7 @@ import lombok.NoArgsConstructor;
@AllArgsConstructor
@NoArgsConstructor
public class Location {
private String id;
private String location;
private Long count;
}
mongo-event/src/main/java/com/liang/mongoevent/vo/SourceProp.java 0 → 100644
package com.liang.mongoevent.vo;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@AllArgsConstructor
@NoArgsConstructor
public class SourceProp {
private String id;
private String source;
private Long count;
private Long total;
}
mongo-event/src/main/java/com/liang/mongoevent/vo/TimeCount.java
......@@ -8,6 +8,7 @@ import lombok.NoArgsConstructor;
@AllArgsConstructor
@NoArgsConstructor
public class TimeCount {
private String id;
private String time;
private long count;
}
mongo-event/src/main/java/com/liang/mongoevent/vo/Trend.java
......@@ -9,6 +9,7 @@ import lombok.NoArgsConstructor;
@AllArgsConstructor
@NoArgsConstructor
public class Trend {
private String id;
private String eventId;
private String stime;
private String etime;
......
mongo-event/src/main/java/com/liang/mongoevent/vo/Trend2.java 0 → 100644
package com.liang.mongoevent.vo;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.time.LocalDateTime;
@Data
@AllArgsConstructor
@NoArgsConstructor
public class Trend2 {
private String id;
private String eventId;
private String time;
}
mongo-event/src/main/java/com/liang/mongoevent/vo/Trend3.java 0 → 100644
package com.liang.mongoevent.vo;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.time.LocalDateTime;
@Data
@AllArgsConstructor
@NoArgsConstructor
public class Trend3 {
private String id;
private String eventId;
private LocalDateTime time;
}
mongo-event/src/test/java/com/liang/mongoevent/MongoEventApplicationTests.java
package com.liang.mongoevent;
import com.liang.mongoevent.pojo.Media;
import com.liang.mongoevent.pojo.StaNews;
import com.liang.mongoevent.pojo.Weibo;
import com.liang.mongoevent.service.WeiboService;
import com.liang.mongoevent.utils.RespBean;
import com.liang.mongoevent.vo.Location;
import com.liang.mongoevent.vo.UserCount;
import com.liang.mongoevent.vo.*;
import com.mongodb.client.DistinctIterable;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.data.domain.Sort;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.aggregation.Aggregation;
import org.springframework.data.mongodb.core.aggregation.AggregationResults;
import org.springframework.data.mongodb.core.aggregation.*;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.Duration;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.*;
@SpringBootTest
class MongoEventApplicationTests {
......@@ -34,7 +38,6 @@ class MongoEventApplicationTests {
@Test
public void testCountsex(){
mongoTemplate.findDistinct("sex", Weibo.class, String.class);
Query query1 = new Query(Criteria.where("sex").is("m"));
Query query2 = new Query(Criteria.where("sex").is("f"));
long male = mongoTemplate.count(query1, Weibo.class);
......@@ -59,12 +62,485 @@ class MongoEventApplicationTests {
Aggregation agg = Aggregation.newAggregation(
Aggregation.group("user_id").first("user_id").as("user_id").first("fensi").as("count"),
Aggregation.sort(Sort.Direction.DESC,"count"),
Aggregation.limit(10));
AggregationResults<UserCount> aggregate = mongoTemplate.aggregate(agg, Weibo.class, UserCount.class);
for (UserCount userCount : aggregate) {
Aggregation.limit(10)
);
AggregationResults<Trend> aggregate = mongoTemplate.aggregate(agg, Weibo.class, Trend.class);
for (Trend userCount : aggregate) {
System.out.println(userCount);
}
}
@Test
public void testTrend(){
//List<String> eventIds = mongoTemplate.findDistinct("eventId", Weibo.class, String.class);
//Map<String,String> map=new HashMap<>();
//for (String eventId : eventIds) {
// Query query = new Query(Criteria.where("eventId").is(eventId));
// List<Weibo> weibos = mongoTemplate.find(query, Weibo.class);
// for (Weibo weibo : weibos) {
// System.out.println("eventId:"+eventId+"time:"+weibo.getTime());
// }
//}
List<Weibo> all = mongoTemplate.findAll(Weibo.class);
for (Weibo weibo : all) {
System.out.println(weibo);
}
//Aggregation agg = Aggregation.newAggregation(
// Aggregation.group("time").first("time").as("time"),
// Aggregation.sort(Sort.Direction.DESC,"time"),
// Aggregation.limit(10)
//);
Aggregation agg = Aggregation.newAggregation(
Aggregation.group("eventId").first("eventId").as("eventId").first("time").as("time")
);
AggregationResults<Weibo> aggregate = mongoTemplate.aggregate(agg, Weibo.class, Weibo.class);
for (Weibo weibo : aggregate) {
//System.out.println("eventId:"+weibo.getEventId()+"time:"+weibo.getTime());
System.out.println(weibo);
}
}
@Test
public void testPage(){
//Aggregation agg = Aggregation.newAggregation(
// Aggregation.group("eventId").first("eventId").as("eventId").first("time").as("etime").last("time").as("stime")
//);
//AggregationResults<Trend> aggregate = mongoTemplate.aggregate(agg, Weibo.class, Trend.class);
//Map<String, Object> map=new HashMap<>();
//DateTimeFormatter df = DateTimeFormatter.ofPattern("yyyy-MM-dd HH");
//for (Trend trend : aggregate) {
// LocalDateTime stime = LocalDateTime.parse(trend.getStime().substring(0,13), df);
// LocalDateTime etime = LocalDateTime.parse(trend.getEtime().substring(0,13), df);
// LocalDateTime time=null;
// Duration duration = Duration.between(stime,etime);
// long l = duration.toHours();
// //System.out.println(duration.toHours());
// for (int i = 0; i <=l; i++) {
// TimeCount timeCount = new TimeCount();
// timeCount.setTime(df.format(stime.plusHours(1)));
// timeCount.setCount(1);
// map.put(trend.getEventId(),timeCount);
// }
// //Query query = new Query(Criteria.where("eventId").is(trend.getEventId())
// // .andOperator(Criteria.where("time").lte(trend.getEtime()),Criteria.where("time").gte(trend.getStime())));
// //long count = mongoTemplate.count(query, Weibo.class);
// //map.put(trend.getEventId(),count);
//
//}
//System.out.println(map);
}
@Test
public void testf(){
int size=20;
int page=2;
Aggregation agg = Aggregation.newAggregation(
Aggregation.group("eventId").first("eventId").as("eventId").first("time").as("etime").last("time").as("stime"),
Aggregation.sort(Sort.Direction.DESC,"stime"),
Aggregation.skip(size*(page-1)),
Aggregation.limit(size)
);
AggregationResults<Trend> aggregate = mongoTemplate.aggregate(agg, Weibo.class, Trend.class);
//DateTimeFormatter df = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm");
//for (Trend trend : aggregate) {
//
// System.out.println(trend.getStime());
// //LocalDateTime stime = LocalDateTime.parse(trend.getStime().substring(0,13), df);
// //LocalDateTime etime = LocalDateTime.parse(trend.getEtime().substring(0,13), df);
// //Duration duration = Duration.between(stime,etime);
// //long l = duration.toHours();
// //Map<String, Object> map1=new HashMap<>();
// //
// //for (int i = 0; i <=l; i++) {
// // Query query = new Query(Criteria.where("eventId").is(trend.getEventId()).and("time").regex("^" + df.format(stime)));
// // long count = mongoTemplate.count(query, Weibo.class);
// // //System.out.println(count);
// // map1.put(df.format(stime),count);
// // stime=stime.plusHours(1);
// //}
// //System.out.println(map1);
//
//}
}
@Test
public void getTime(){
//Aggregation agg = Aggregation.newAggregation(
// Aggregation.group("eventId").first("eventId").as("eventId").first("time").as("etime").last("time").as("stime"),
// Aggregation.sort(Sort.Direction.DESC,"stime")
//);
//AggregationResults<Trend> aggregate = mongoTemplate.aggregate(agg, Weibo.class, Trend.class);
//for (Trend trend : aggregate) {
// System.out.println(trend);
//}
}
@Test
public void trendPerHour() {
int size=10;
int page=1;
Aggregation agg = Aggregation.newAggregation(
Aggregation.match(Criteria.where("pt").is("百度新闻")),
Aggregation.group("eventId").first("eventId").as("eventId"),
Aggregation.skip(size*(page-1)),
Aggregation.limit(size)
);
ArrayList<Object> list = new ArrayList<>();
Map<String, Object> map=new LinkedHashMap<>();
AggregationResults<Trend2> aggregate = mongoTemplate.aggregate(agg, Media.class, Trend2.class);
DateTimeFormatter df = DateTimeFormatter.ofPattern("yyyy-MM-dd HH");
for (Trend2 trend2 : aggregate) {
Aggregation agg1 = Aggregation.newAggregation(
Aggregation.match(Criteria.where("eventId").is(trend2.getEventId())),
Aggregation.group("time").first("time").as("time"),
Aggregation.sort(Sort.Direction.DESC,"time")
);
AggregationResults<Media> aggregate1 = mongoTemplate.aggregate(agg, Media.class, Media.class);
for (Media media : aggregate1) {
System.out.println(media);
}
//LocalDateTime stime = LocalDateTime.parse(trend2.getStime().format(df).substring(0,13), df);
//LocalDateTime etime = LocalDateTime.parse(trend2.getEtime().format(df).substring(0,13), df);
//
//Duration duration = Duration.between(stime,etime);
//long l = duration.toHours();
//Map<String, Object> map1=new LinkedHashMap<>();
//
//for (int i = 0; i <=l; i++) {
// Query query = new Query(Criteria.where("eventId").is(trend2.getEventId()).and("time").regex("^" + stime));
// long count = mongoTemplate.count(query, Media.class);
// map1.put(df.format(stime),count);
// stime=stime.plusHours(1);
//}
//map.put(trend2.getEventId(),map1);
//System.out.println(map);
}
}
@Test
public void trendPerHour2() {
//int size=20;
//int page=1;
//Aggregation agg = Aggregation.newAggregation(
// Aggregation.match(Criteria.where("pt").is("百度新闻")),
// Aggregation.group("eventId").first("eventId").as("eventId").first("time").as("etime").last("time").as("stime"),
// Aggregation.sort(Sort.Direction.DESC,"stime"),
// Aggregation.skip(size*(page-1)),
// Aggregation.limit(size)
//);
//AggregationResults<Trend2> aggregate = mongoTemplate.aggregate(agg, Media.class, Trend2.class);
//for (Trend2 trend2 : aggregate) {
// System.out.println(trend2);
//}
//
List<Media> eventId = mongoTemplate.findDistinct("eventId", Media.class, Media.class);
for (Media media : eventId) {
Query query = new Query(Criteria.where("pt").is("百度新闻"));
}
}
@Test
public void testpt(){
Aggregation agg = Aggregation.newAggregation(
Aggregation.match(Criteria.where("pt").is("百度新闻")),
Aggregation.group("eventId").first("eventId").as("eventId").first("time").as("etime").last("time").as("stime"),
Aggregation.sort(Sort.Direction.DESC,"stime")
);
AggregationResults<Trend2> aggregate = mongoTemplate.aggregate(agg, Media.class, Trend2.class);
for (Trend2 trend2 : aggregate) {
System.out.println(trend2);
}
}
@Test
public void testGetEventId(){
List<StaNews> all = mongoTemplate.findAll(StaNews.class);
DateTimeFormatter df = DateTimeFormatter.ofPattern("yyyy-MM-dd HH");
Map<String, Object> map=new LinkedHashMap<>();
Map<String, Integer> map1=new LinkedHashMap<>();
for (StaNews staNews : all) {
if(staNews.getBaidu()!=0){
Aggregation agg = Aggregation.newAggregation(
Aggregation.match(Criteria.where("eventId").is(staNews.getEventId())),
Aggregation.group("time").first("time").as("time").first("eventId").as("eventId"),
Aggregation.sort(Sort.Direction.DESC,"time")
);
AggregationResults<Trend3> aggregate = mongoTemplate.aggregate(agg, Media.class, Trend3.class);
for (Trend3 trend3 : aggregate) {
String time = df.format(trend3.getTime());
if(map1.containsKey(time)){
int i = map1.get(time) + 1;
map1.put(time,i);
}else{
map1.put(time,1);
}
}
map.put(staNews.getEventId(),map1);
System.out.println(map);
}
}
}
@Test
public void testFinally(){
Aggregation agg = Aggregation.newAggregation(
Aggregation.group("eventId").first("eventId").as("eventId").first("time").as("timefirst").addToSet("time").as("times"),
Aggregation.sort(Sort.Direction.DESC,"timefirst"),
Aggregation.limit(10)
);
AggregationResults<Trend3> aggregate = mongoTemplate.aggregate(agg, Media.class, Trend3.class);
for (Trend3 trend3 : aggregate) {
System.out.println(trend3);
}
}
@Test
public void testHaha(){
Query query = new Query(Criteria.where("eventId").is("03845e5965ba6a6c10007635"));
List<Media> media = mongoTemplate.find(query, Media.class);
for (Media media1 : media) {
System.out.println("eventId:"+media1.getEventId()+" time:"+media1.getTime()+" pt:"+media1.getPt()+" title:"+media1.getTitle());
System.out.println("---------------");
}
}
@Test
public void testLala(){
int size=20;
int page=1;
List<StaNews> all = mongoTemplate.find(new Query().skip(size*(page-1)).limit(size).with(Sort.by(Sort.Direction.DESC,"time")),StaNews.class);
DateTimeFormatter df = DateTimeFormatter.ofPattern("yyyy-MM-dd HH");
Map<String, Object> map=new LinkedHashMap<>();
for (StaNews staNews : all) {
Map<String, Integer> map1=new LinkedHashMap<>();
if(staNews.getBaidu()!=0){
Aggregation agg = Aggregation.newAggregation(
Aggregation.match(Criteria.where("eventId").is(staNews.getEventId()).and("pt").is("微信")),
Aggregation.sort(Sort.Direction.DESC,"time")
);
AggregationResults<Trend3> aggregate = mongoTemplate.aggregate(agg, Media.class, Trend3.class);
for (Trend3 trend3 : aggregate) {
String time = df.format(trend3.getTime());
if (map1.containsKey(time)) {
int i = map1.get(time) + 1;
map1.put(time, i);
} else {
map1.put(time, 1);
}
System.out.println(map1);
}
}
map.put(staNews.getEventId(),map1);
}
}
@Test
public void testLo(){
int size=20;
int page=1;
Aggregation agg = Aggregation.newAggregation(
Aggregation.group("source").first("source").as("source").count().as("count"),
Aggregation.sort(Sort.Direction.DESC,"count"),
Aggregation.skip(size*(page-1)),
Aggregation.limit(size)
);
AggregationResults<SourceProp> aggregate = mongoTemplate.aggregate(agg, Media.class, SourceProp.class);
for (SourceProp sourceProp : aggregate) {
System.out.println(sourceProp);
}
//AggregationResults<Media> aggregate = mongoTemplate.aggregate(agg, Media.class, Media.class);
//for (Media media : aggregate) {
//
// System.out.println(media);
//}
}
@Test
public void testTitle(){
Aggregation agg = Aggregation.newAggregation(
Aggregation.group("title").first("title").as("title")
);
AggregationResults<Media> aggregate = mongoTemplate.aggregate(agg, Media.class, Media.class);
for (Media media : aggregate) {
System.out.println(media.getTitle());
}
}
@Test
public void testDemo(){
//LookupOperation lookupToLots = LookupOperation.newLookup().
// from("Weibo").//关联表名 lots
// localField("eventId").//关联字段
// foreignField("eventId").//主表关联字段对应的次表字段
// as("groups");//查询结果集合名
//
////主表
//Criteria ordercri = Criteria.where("eccDocNo").ne(null).ne("");
//AggregationOperation match = Aggregation.match(ordercri);
////次表
//Criteria ordercri1 = Criteria.where("groups.success").is(true);
//AggregationOperation match1 = Aggregation.match(ordercri);
//Aggregation agg = Aggregation.newAggregation(
// Aggregation.lookup("Weibo", "eventId", "eventId", "news"),
// Aggregation.unwind("news"),
// Aggregation.group("Weibo.eventId").first("Weibo.time").as("time"),
// Aggregation.limit(10)
//);
//List<Map> mappedResults = mongoTemplate.aggregate(agg, Media.class, Map.class).getMappedResults();
//for (Map mappedResult : mappedResults) {
// System.out.println(mappedResult);
//}
}
@Test
public void testDemo2(){
LookupOperation lookupOperation = LookupOperation.newLookup()
.from("weibo")
.localField("eventId")
.foreignField("eventId")
.as("weibos");
Aggregation agg = Aggregation.newAggregation(
Aggregation.project("eventId","time","title"),
lookupOperation,
Aggregation.match(Criteria.where("eventId").is("70e153396066ac8610007408")),
Aggregation.unwind("weibos"),
Aggregation.limit(10)
);
List<Map> mappedResults = mongoTemplate.aggregate(agg, Media.class, Map.class).getMappedResults();
for (Map mappedResult : mappedResults) {
System.out.println(mappedResult);
}
}
@Test
public void testDemo3(){
Aggregation agg = Aggregation.newAggregation(
Aggregation.group("title").first("title").as("title").first("source").as("source"),
Aggregation.limit(10)
).withOptions(AggregationOptions.builder().allowDiskUse(true).build());
AggregationResults<Media> aggregate = mongoTemplate.aggregate(agg, Media.class, Media.class);
HashMap<Object, Object> map = new HashMap<>();
for (Media media : aggregate) {
map.put(media.getTitle(),media.getSource());
System.out.println("title:"+media.getTitle()+"Source:"+media.getSource()+"_id:"+media.getId());
//System.out.println(map);
}
}
@Test
public void testDemo04(){
//int size=10;
//int page=1;
//List<StaNews> all = mongoTemplate.find(new Query().skip(size*(page-1)).limit(size).with(Sort.by(Sort.Direction.DESC,"time")),StaNews.class);
//DateTimeFormatter df = DateTimeFormatter.ofPattern("yyyy-MM-dd HH");
//Map<String, Object> map=new LinkedHashMap<>();
//for (StaNews staNews : all) {
//
//}
//LookupOperation lookupOperation = LookupOperation.newLookup()
// .from("weibo")
// .localField("eventId")
// .foreignField("eventId")
// .as("weibos");
//
//Aggregation agg = Aggregation.newAggregation(
// Aggregation.project("eventId","time","title"),
// lookupOperation,
// Aggregation.match(Criteria.where("eventId").is("b7057b2464cbe86c10013609")),
// Aggregation.unwind("weibos")
//);
//AggregationResults<Map> aggregate = mongoTemplate.aggregate(agg, Media.class, Map.class);
//for (Map map : aggregate) {
// System.out.println(map.entrySet());
//}
List<String> eventIdWeibo = mongoTemplate.findDistinct("eventId", Weibo.class, String.class);
HashSet<String> set = new HashSet<>();
set.addAll(eventIdWeibo);
List<String> eventIdMedia = mongoTemplate.findDistinct("eventId", Media.class, String.class);
set.addAll(eventIdMedia);
DateTimeFormatter df = DateTimeFormatter.ofPattern("yyyy-MM-dd HH");
Map<String, Object> map=new LinkedHashMap<>();
for (String eventId : set) {
Map<String, Integer> map1=new LinkedHashMap<>();
Aggregation agg = Aggregation.newAggregation(
Aggregation.match(Criteria.where("eventId").is(eventId)),
Aggregation.group("time").first("time").as("time"),
Aggregation.sort(Sort.Direction.DESC,"time")
);
AggregationResults<Trend3> aggregate = mongoTemplate.aggregate(agg, Media.class, Trend3.class);
for (Trend3 meida : aggregate) {
String time = df.format(meida.getTime());
if (map1.containsKey(time)) {
int i = map1.get(time) + 1;
map1.put(time, i);
} else {
map1.put(time, 1);
}
}
AggregationResults<Trend2> aggregate1 = mongoTemplate.aggregate(agg, Weibo.class, Trend2.class);
for (Trend2 weibo : aggregate1) {
String time=weibo.getTime();
if (map1.containsKey(time)) {
int i = map1.get(time) + 1;
map1.put(time, i);
} else {
map1.put(time, 1);
}
}
map.put(eventId,map1);
System.out.println(map);
}
}
@Test
public void testAdd(){
List<String> eventIdWeibo = mongoTemplate.findDistinct("eventId", Weibo.class, String.class);
HashSet<String> set = new HashSet<>();
set.addAll(eventIdWeibo);
List<String> eventIdMedia = mongoTemplate.findDistinct("eventId", Media.class, String.class);
set.addAll(eventIdMedia);
for (String eventId : set) {
AllEventId allEventId = new AllEventId();
allEventId.setEventId(eventId);
mongoTemplate.save(allEventId);
}
}
@Test
public void testTime(){
List<Media> eventId = mongoTemplate.find(new Query(Criteria.where("eventId").is("219adc4e564872e910007300")), Media.class);
DateTimeFormatter df = DateTimeFormatter.ofPattern("yyyy-MM-dd HH");
for (Media media : eventId) {
System.out.println(df.format(media.getTime().minusHours(8)));
}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment