Skip to content

B
brandkbs2
Commit 881e33a0 by 陈健智
Options

全网搜

parent 40629424
Showing with 249 additions and 37 deletions
src/main/java/com/zhiwei/brandkbs2/controller/app/AppSearchController.java
......@@ -4,19 +4,25 @@ package com.zhiwei.brandkbs2.controller.app;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.auth.Auth;
import com.zhiwei.brandkbs2.config.Constant;
import com.zhiwei.brandkbs2.enmus.RoleEnum;
import com.zhiwei.brandkbs2.model.ResponseResult;
import com.zhiwei.brandkbs2.pojo.dto.SearchFilterDTO;
import com.zhiwei.brandkbs2.service.MarkDataService;
import com.zhiwei.brandkbs2.util.Tools;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.apache.commons.lang3.time.DateUtils;
import org.joda.time.Period;
import org.joda.time.PeriodType;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.client.RestTemplate;
import javax.annotation.Resource;
import java.util.Date;
import java.util.Objects;
/**
......@@ -42,6 +48,9 @@ public class AppSearchController {
@Value("${ef.search.url}")
private String getEfSearchUrl;
@Resource(name = "markDataServiceImpl")
MarkDataService markDataService;
@ApiOperation("搜索-查热点")
@GetMapping("/hot/list")
public ResponseResult searchHotList(@RequestParam(value = "pageSize",defaultValue ="10") Integer limit,
......@@ -76,4 +85,18 @@ public class AppSearchController {
JSONObject result = JSON.parseObject(responseEntity.getBody());
return ResponseResult.success(result);
}
@ApiOperation("搜索-全网搜")
@GetMapping("/searchWhole")
public ResponseResult searchWholeNetwork(@RequestBody SearchFilterDTO dto){
long time = DateUtils.addDays(Tools.truncDate(new Date(), Constant.DAY_PATTERN), -89).getTime();
if (time > dto.getStartTime()){
return ResponseResult.failure("仅能搜索近3个月内信息");
}
Period periodDay = new Period(dto.getStartTime(), dto.getEndTime(), PeriodType.days());
if (periodDay.getDays() > 30) {
return ResponseResult.failure("时间跨度不能超过30天");
}
return ResponseResult.success(markDataService.searchWholeNetwork(dto));
}
}
src/main/java/com/zhiwei/brandkbs2/pojo/dto/SearchFilterDTO.java
package com.zhiwei.brandkbs2.pojo.dto;
import com.alibaba.fastjson.JSONObject;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import java.util.List;
import java.util.*;
@Data
@ApiModel("全网搜搜索传输类")
......@@ -44,7 +45,7 @@ public class SearchFilterDTO {
* 关键字
*/
@ApiModelProperty(value = "关键字")
private String keyword;
private String search;
/**
* 平台
......@@ -53,13 +54,15 @@ public class SearchFilterDTO {
private List<String> platforms;
/**
* 平台(真正用于搜索的)
* 平台(查商业数据库用)
*/
@ApiModelProperty(value = "平台(查商业数据库用)")
private String platformsStr;
/**
* 数据类型 1:原创, 2:转发, 7:评论
* 数据类型 1:原创, 2:转发, 7:评论 (查商业数据库用)
*/
@ApiModelProperty(value = "数据类型(查商业数据库用)")
private String filterForward = "1,2";
/**
......@@ -69,13 +72,27 @@ public class SearchFilterDTO {
private String sorter;
/**
* 排序字段(真正用于搜索的)
* 排序字段(查商业数据库用)
*/
@ApiModelProperty(value = "排序字段(查商业数据库用)")
private String sortOf;
/**
* 匹配域名 默认标题+内容
* 来源
*/
@ApiModelProperty(value = "来源")
private String source;
/**
* 过滤关键字
*/
@ApiModelProperty(value = "过滤关键字")
private String filterWords;
/**
* 匹配域名 默认标题+内容(查商业数据库用)
*/
@ApiModelProperty(value = "匹配域名(查商业数据库用)")
private String matchFields = null;
/**
......@@ -84,4 +101,48 @@ public class SearchFilterDTO {
@ApiModelProperty(value = "查询类型")
private String matchType;
/**
* 重要渠道，{father:"区域",son:"北京"}，{father:"全部"}
*/
@ApiModelProperty(value = "重要渠道")
private Map<String, String> sensitiveChannels;
/**
* 粉丝数
*/
@ApiModelProperty(value = "粉丝数")
private String fans;
/**
* 过滤类型 标题/全文
*/
@ApiModelProperty(value = "过滤类型 标题/全文")
private String filterType;
public Map<String, Object> toMap() {
Map<String, Object> map = new HashMap<>();
map.put("timeType","time");
map.put("pt", getPt());
map.put("matchType", matchType);
map.put("search", search);
map.put("startTime", startTime);
map.put("endTime", endTime);
map.put("source", source);
map.put("sort", Objects.nonNull(sorter) ? sorter : "DESC");
map.put("page", page);
map.put("pageSize", pageSize);
map.put("notWords", filterWords);
map.put("sensitiveChannels", sensitiveChannels);
map.put("fans", fans);
map.put("filterType", filterType);
return map;
}
public List<String> getPt() {
return Objects.nonNull(platforms) ? this.platforms : Collections.emptyList();
}
public String getMatchType() {
return Objects.nonNull(this.matchType) ? this.matchType : "标题";
}
}
src/main/java/com/zhiwei/brandkbs2/service/impl/MarkDataServiceImpl.java
......@@ -2,6 +2,7 @@ package com.zhiwei.brandkbs2.service.impl;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.ImmutableMap;
import com.zhiwei.brandkbs2.auth.UserThreadLocal;
......@@ -32,6 +33,7 @@ import com.zhiwei.brandkbs2.util.RedisUtil;
import com.zhiwei.brandkbs2.util.TextUtil;
import com.zhiwei.brandkbs2.util.Tools;
import com.zhiwei.qbjc.bean.pojo.common.MessagePlatform;
import com.zhiwei.tools.istarshine.IstarShineUtil;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
......@@ -60,9 +62,13 @@ import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.joda.time.Period;
import org.joda.time.PeriodType;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service;
import org.springframework.web.client.RestTemplate;
import javax.annotation.Resource;
import java.io.IOException;
......@@ -84,6 +90,18 @@ public class MarkDataServiceImpl implements MarkDataService {
private static final String[] EVENT_FETCH_SOURCE = new String[]{"id", "ind_full_text", "c5", "real_source", "source", "mtime", "time", "url", "mtag", "foreign", "brandkbs_mark_cache_maps"};
@Value("${istarshine.addIStarShineKSData.url}")
private String addIStarShineKSDataUrl;
@Value("${istarshine.getIStarShineKSInfoData.url}")
private String getIStarShineKSInfoDataUrl;
@Value("${whole.search.url}")
private String wholeSearchUrl;
@Autowired
private RestTemplate restTemplate;
@Resource(name = "projectServiceImpl")
private ProjectService projectService;
......@@ -1220,38 +1238,141 @@ public class MarkDataServiceImpl implements MarkDataService {
@Override
public JSONObject searchWholeNetwork(SearchFilterDTO dto) {
// Project project = projectService.getProjectById(dto.getProjectId());
// if (1 == project.getWholeSearchDataSource()){
// defaultSearchFilter(dto);
// try {
// JSONObject params = IstarShineUtil.getKSSearchParam(keyword, null, dto.getStartTime(), dto.getEndTime(), platforms,
// filterForward, sortOf, dto.getPage(), dto.getPageSize(), matchFields,null,null);
//
// JSONObject taskEntity = new JSONObject();
//
// taskEntity.put("appId", "6183571e0d710000f6003a12"); //应用id,由张志伟提供给使用者
// taskEntity.put("taskType", ""); //可为null, 任务类型
// taskEntity.put("taskDescription", ""); //可为null， 任务描述
// taskEntity.put("userName", "品见"); //提交任务的人
// taskEntity.put("searchParams", params); //任务详情参数
//
// ResponseEntity<JSONObject> entity = restTemplate.postForEntity(url, taskEntity, JSONObject.class);
// JSONObject result = entity.getBody().getJSONObject("data");
// return searchWholeBackUp(result.get("id").toString());
// }
// }
return null; //TODO
Project project = projectService.getProjectById(dto.getProjectId());
JSONObject jsonObject = new JSONObject();
if (2 == project.getWholeSearchDataSource()){ // 查商业数据库
defaultSearchFilter(dto);
try {
JSONObject params = IstarShineUtil.getKSSearchParam(dto.getSearch(), null, dto.getStartTime(), dto.getEndTime(), dto.getPlatformsStr(),
dto.getFilterForward(), dto.getSortOf(), dto.getPage(), dto.getPageSize(), dto.getMatchFields(), null, null);
JSONObject taskEntity = new JSONObject();
taskEntity.put("appId", "6183571e0d710000f6003a12"); // 应用id, 由张志伟提供给使用者
taskEntity.put("taskType", ""); // 可为null, 任务类型
taskEntity.put("taskDescription", ""); // 可为null, 任务描述
taskEntity.put("userName", "品见"); // 提交任务的人
taskEntity.put("searchParams", params); // 任务详情参数
ResponseEntity<JSONObject> responseEntity = restTemplate.postForEntity(addIStarShineKSDataUrl, taskEntity, JSONObject.class);
JSONObject result = Objects.requireNonNull(responseEntity.getBody()).getJSONObject("data");
return searchWholeBackUp(result.get("id").toString());
}catch (Exception e){
log.error("error searchWholeNetwork - ", e);
}
}else { // 查舆情库
ResponseEntity<JSONObject> responseEntity = restTemplate.postForEntity(wholeSearchUrl, dto.toMap(), JSONObject.class);
return searchWholeOpinion(responseEntity, jsonObject);
}
return jsonObject;
}
private JSONObject searchWholeOpinion(ResponseEntity<JSONObject> responseEntity, JSONObject jsonObject){
JSONObject bodyData = Objects.requireNonNull(responseEntity.getBody()).getJSONObject("data");
JSONObject data = bodyData.getJSONObject("data");
JSONArray listArray = data.getJSONArray("list");
List<JSONObject> resultList = new ArrayList<>();
for (Object o : listArray) {
JSONObject list = (JSONObject) o;
JSONObject result = new JSONObject();
result.put("platform", list.get("real_source"));
JSONArray contentArray = list.getJSONArray("ind_full_text");
assert contentArray == null;
result.put("title", contentArray.get(0));
result.put("content", contentArray.get(1));
if (3 == contentArray.size()) {
result.put("content", result.getString("content") + contentArray.get(2));
}
result.put("channel", list.get("source"));
String url = list.getString("url");
// 如果url为空,则查询question_url字段
if (null == url) {
url = list.getString("question_url");
}
result.put("url", url);
result.put("time", list.get("time"));
resultList.add(result);
}
jsonObject.put("list", resultList);
jsonObject.put("page", data.get("page"));
jsonObject.put("pageLimit", data.get("pageLimit"));
jsonObject.put("total", data.get("total"));
jsonObject.put("totalPage", data.get("totalPage"));
jsonObject.put("isBackUp", false);
return jsonObject;
}
public JSONObject searchWholeBackUp(String taskId) {
JSONObject jsonObject = new JSONObject();
List<JSONObject> resultList = new ArrayList<>();
for (int i = 0; i < 30; i++) {
ResponseEntity<JSONObject> responseEntity = restTemplate.getForEntity(getIStarShineKSInfoDataUrl, JSONObject.class, taskId);
JSONObject data = Objects.requireNonNull(responseEntity.getBody()).getJSONObject("data");
JSONObject task = data.getJSONObject("task");
if(1 == task.getInteger("crawlerFinish")){
//pageSize
Integer receiveCount = task.getInteger("receiveCount");
Integer searchCount = task.getInteger("searchCount");
int totalPage = searchCount % receiveCount == 0 ? searchCount / receiveCount : (searchCount / receiveCount + 1);
JSONArray listArray = data.getJSONArray("data");
for (Object o : listArray) {
JSONObject list = (JSONObject) o;
JSONObject result = new JSONObject();
result.put("platform", list.getJSONObject("publisher").get("platform"));
result.put("title", list.get("title"));
result.put("content", list.get("content"));
result.put("channel", list.get("channel"));
result.put("url", getUrl(list));
result.put("time", list.get("ctime"));
resultList.add(result);
}
jsonObject.put("list",resultList);
//jsonObject.put("page",data1.get("page"));
jsonObject.put("pageLimit",receiveCount);
jsonObject.put("total",searchCount);
jsonObject.put("totalPage",totalPage);
jsonObject.put("isBackUp",true);
return jsonObject;
}
Tools.sleep(500L);
}
return jsonObject;
}
/**
* 获取全网搜接口信息url
*
* @param json
* @return
*/
private static String getUrl(JSONObject json) {
String url = json.getString("url");
// 如果url为空,则查询QA类型url字段
if (null == url) {
url = json.getString("answer_url");
}
if (null == url) {
url = json.getString("question_url");
}
return url;
}
private void defaultSearchFilter(SearchFilterDTO dto){
String projectId = UserThreadLocal.getProjectId();
dto.setProjectId(projectId);
if (Objects.isNull(dto.getStartTime()) || Objects.isNull(dto.getEndTime())) {
Date now = new Date();
dto.setEndTime(now.getTime());
dto.setStartTime(DateUtils.addDays(now, -1).getTime());
}
// 平台为空时，传全部
if (dto.getPlatforms().size() == 0 ){
dto.setPlatforms(Arrays.asList("微博","微信","新闻","平媒","自媒体","论坛","视频","短视频"));
}
// 平台
dto.setPlatformsStr(StringUtils.join(dto.getPlatforms(), ",")); //平台
dto.setPlatformsStr(StringUtils.join(dto.getPlatforms(), ","));
// sort
dto.setSortOf("ctime+" + dto.getSorter());
// matchFields
......@@ -1372,9 +1493,12 @@ public class MarkDataServiceImpl implements MarkDataService {
*/
private void defaultContendDTO(MarkSearchDTO dto) {
if (Objects.isNull(dto.getStartTime()) || Objects.isNull(dto.getEndTime())) {
Date now = new Date();
dto.setEndTime(now.getTime());
dto.setStartTime(DateUtils.addDays(now, -29).getTime());
Long[] timeRangeMonth = commonService.getTimeRangeMonth();
dto.setEndTime(timeRangeMonth[1]);
dto.setStartTime(timeRangeMonth[0]);
}else{
dto.setStartTime(Tools.truncDate(dto.getStartTime(), Constant.DAY_PATTERN));
dto.setEndTime(Tools.truncDate(dto.getEndTime(), Constant.DAY_PATTERN));
}
String projectId = UserThreadLocal.getProjectId();
dto.setProjectId(projectId);
......
src/main/resources/application-local.properties
......@@ -75,4 +75,8 @@ ef.search.url=https://ef.zhiweidata.com/external/search.do?name={1}&page={2}
ef.searchCriteria.url=https://ef.zhiweidata.com/index/getSearchKey.do
ef.filterNew.url=https://ef.zhiweidata.com/filterNew.do?firstType={1}&start={2}&end={3}&page={4}
ef.captcha.url=https://ef.zhiweidata.com/captchaApi/getNewCaptcha.do.do?type=test
ef.checkCaptcha.url=https://ef.zhiweidata.com/captchaApi/checkeCaptcha.do.do?id={1}&captcha={2}&type=test
\ No newline at end of file
ef.checkCaptcha.url=https://ef.zhiweidata.com/captchaApi/checkeCaptcha.do.do?id={1}&captcha={2}&type=test
#\u5168\u7F51\u641C\u5916\u90E8\u63A5\u53E3
istarshine.addIStarShineKSData.url=https://istarshine-service.zhiweidata.com/istarshine/ks/addIStarShineKSData
istarshine.getIStarShineKSInfoData.url=https://istarshine-service.zhiweidata.com/istarshine/ks/getIStarShineKSInfoData?taskId={1}&appId=6183571e0d710000f6003a12
whole.search.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/search/yuqing
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment