全网搜

src/main/java/com/zhiwei/brandkbs2/controller/app/AppSearchController.java

@@ -4,19 +4,25 @@ package com.zhiwei.brandkbs2.controller.app;
 import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.brandkbs2.auth.Auth;
+import com.zhiwei.brandkbs2.config.Constant;
 import com.zhiwei.brandkbs2.enmus.RoleEnum;
 import com.zhiwei.brandkbs2.model.ResponseResult;
+import com.zhiwei.brandkbs2.pojo.dto.SearchFilterDTO;
+import com.zhiwei.brandkbs2.service.MarkDataService;
+import com.zhiwei.brandkbs2.util.Tools;
 import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiOperation;
+import org.apache.commons.lang3.time.DateUtils;
+import org.joda.time.Period;
+import org.joda.time.PeriodType;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.http.ResponseEntity;
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestParam;
-import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.bind.annotation.*;
 import org.springframework.web.client.RestTemplate;
 
+import javax.annotation.Resource;
+import java.util.Date;
 import java.util.Objects;
 
 /**
@@ -42,6 +48,9 @@ public class AppSearchController {
     @Value("${ef.search.url}")
     private String getEfSearchUrl;
 
+    @Resource(name = "markDataServiceImpl")
+    MarkDataService markDataService;
+
     @ApiOperation("搜索-查热点")
     @GetMapping("/hot/list")
     public ResponseResult searchHotList(@RequestParam(value = "pageSize",defaultValue ="10") Integer limit,
@@ -76,4 +85,18 @@ public class AppSearchController {
         JSONObject result = JSON.parseObject(responseEntity.getBody());
         return ResponseResult.success(result);
     }
+
+    @ApiOperation("搜索-全网搜")
+    @GetMapping("/searchWhole")
+    public ResponseResult searchWholeNetwork(@RequestBody SearchFilterDTO dto){
+        long time = DateUtils.addDays(Tools.truncDate(new Date(), Constant.DAY_PATTERN), -89).getTime();
+        if (time > dto.getStartTime()){
+            return ResponseResult.failure("仅能搜索近3个月内信息");
+        }
+        Period periodDay = new Period(dto.getStartTime(), dto.getEndTime(), PeriodType.days());
+        if (periodDay.getDays() > 30) {
+            return ResponseResult.failure("时间跨度不能超过30天");
+        }
+        return ResponseResult.success(markDataService.searchWholeNetwork(dto));
+    }
 }

src/main/java/com/zhiwei/brandkbs2/pojo/dto/SearchFilterDTO.java

 package com.zhiwei.brandkbs2.pojo.dto;
 
+import com.alibaba.fastjson.JSONObject;
 import io.swagger.annotations.ApiModel;
 import io.swagger.annotations.ApiModelProperty;
 import lombok.Data;
 
-import java.util.List;
+import java.util.*;
 
 @Data
 @ApiModel("全网搜搜索传输类")
@@ -44,7 +45,7 @@ public class SearchFilterDTO {
      * 关键字
      */
     @ApiModelProperty(value = "关键字")
-    private String keyword;
+    private String search;
 
     /**
      * 平台
@@ -53,13 +54,15 @@ public class SearchFilterDTO {
     private List<String> platforms;
 
     /**
-     * 平台(真正用于搜索的)
+     * 平台(查商业数据库用)
      */
+    @ApiModelProperty(value = "平台(查商业数据库用)")
     private String platformsStr;
 
     /**
-     * 数据类型 1:原创, 2:转发, 7:评论
+     * 数据类型 1:原创, 2:转发, 7:评论 (查商业数据库用)
      */
+    @ApiModelProperty(value = "数据类型(查商业数据库用)")
     private String filterForward = "1,2";
 
     /**
@@ -69,13 +72,27 @@ public class SearchFilterDTO {
     private String sorter;
 
     /**
-     * 排序字段(真正用于搜索的)
+     * 排序字段(查商业数据库用)
      */
+    @ApiModelProperty(value = "排序字段(查商业数据库用)")
     private String sortOf;
 
     /**
-     * 匹配域名 默认标题+内容
+     * 来源
      */
+    @ApiModelProperty(value = "来源")
+    private String source;
+
+    /**
+     * 过滤关键字
+     */
+    @ApiModelProperty(value = "过滤关键字")
+    private String filterWords;
+
+    /**
+     * 匹配域名 默认标题+内容(查商业数据库用)
+     */
+    @ApiModelProperty(value = "匹配域名(查商业数据库用)")
     private String matchFields = null;
 
     /**
@@ -84,4 +101,48 @@ public class SearchFilterDTO {
     @ApiModelProperty(value = "查询类型")
     private String matchType;
 
+    /**
+     * 重要渠道，{father:"区域",son:"北京"}，{father:"全部"}
+     */
+    @ApiModelProperty(value = "重要渠道")
+    private Map<String, String> sensitiveChannels;
+
+    /**
+     * 粉丝数
+     */
+    @ApiModelProperty(value = "粉丝数")
+    private String fans;
+
+    /**
+     * 过滤类型 标题/全文
+     */
+    @ApiModelProperty(value = "过滤类型 标题/全文")
+    private String filterType;
+
+    public Map<String, Object> toMap() {
+        Map<String, Object> map = new HashMap<>();
+        map.put("timeType","time");
+        map.put("pt", getPt());
+        map.put("matchType", matchType);
+        map.put("search", search);
+        map.put("startTime", startTime);
+        map.put("endTime", endTime);
+        map.put("source", source);
+        map.put("sort", Objects.nonNull(sorter) ? sorter : "DESC");
+        map.put("page", page);
+        map.put("pageSize", pageSize);
+        map.put("notWords", filterWords);
+        map.put("sensitiveChannels", sensitiveChannels);
+        map.put("fans", fans);
+        map.put("filterType", filterType);
+        return map;
+    }
+
+    public List<String> getPt() {
+        return Objects.nonNull(platforms) ? this.platforms : Collections.emptyList();
+    }
+
+    public String getMatchType() {
+        return Objects.nonNull(this.matchType) ? this.matchType : "标题";
+    }
 }

src/main/java/com/zhiwei/brandkbs2/service/impl/MarkDataServiceImpl.java

@@ -2,6 +2,7 @@ package com.zhiwei.brandkbs2.service.impl;
 
 
 import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.google.common.collect.ImmutableMap;
 import com.zhiwei.brandkbs2.auth.UserThreadLocal;
@@ -32,6 +33,7 @@ import com.zhiwei.brandkbs2.util.RedisUtil;
 import com.zhiwei.brandkbs2.util.TextUtil;
 import com.zhiwei.brandkbs2.util.Tools;
 import com.zhiwei.qbjc.bean.pojo.common.MessagePlatform;
+import com.zhiwei.tools.istarshine.IstarShineUtil;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.time.DateUtils;
@@ -60,9 +62,13 @@ import org.elasticsearch.search.sort.SortBuilders;
 import org.elasticsearch.search.sort.SortOrder;
 import org.joda.time.Period;
 import org.joda.time.PeriodType;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.data.mongodb.core.query.Criteria;
 import org.springframework.data.mongodb.core.query.Query;
+import org.springframework.http.ResponseEntity;
 import org.springframework.stereotype.Service;
+import org.springframework.web.client.RestTemplate;
 
 import javax.annotation.Resource;
 import java.io.IOException;
@@ -84,6 +90,18 @@ public class MarkDataServiceImpl implements MarkDataService {
 
     private static final String[] EVENT_FETCH_SOURCE = new String[]{"id", "ind_full_text", "c5", "real_source", "source", "mtime", "time", "url", "mtag", "foreign", "brandkbs_mark_cache_maps"};
 
+    @Value("${istarshine.addIStarShineKSData.url}")
+    private String addIStarShineKSDataUrl;
+
+    @Value("${istarshine.getIStarShineKSInfoData.url}")
+    private String getIStarShineKSInfoDataUrl;
+
+    @Value("${whole.search.url}")
+    private String wholeSearchUrl;
+
+    @Autowired
+    private RestTemplate restTemplate;
+
     @Resource(name = "projectServiceImpl")
     private ProjectService projectService;
 
@@ -1220,38 +1238,141 @@ public class MarkDataServiceImpl implements MarkDataService {
 
     @Override
     public JSONObject searchWholeNetwork(SearchFilterDTO dto) {
-//        Project project = projectService.getProjectById(dto.getProjectId());
-//        if (1 == project.getWholeSearchDataSource()){
-//            defaultSearchFilter(dto);
-//            try {
-//                JSONObject params = IstarShineUtil.getKSSearchParam(keyword, null, dto.getStartTime(), dto.getEndTime(), platforms,
-//                                filterForward, sortOf, dto.getPage(), dto.getPageSize(), matchFields,null,null);
-//
-//                JSONObject taskEntity = new JSONObject();
-//
-//                taskEntity.put("appId", "6183571e0d710000f6003a12"); //应用id,由张志伟提供给使用者
-//                taskEntity.put("taskType", ""); //可为null, 任务类型
-//                taskEntity.put("taskDescription", ""); //可为null， 任务描述
-//                taskEntity.put("userName", "品见");      //提交任务的人
-//                taskEntity.put("searchParams", params);   //任务详情参数
-//
-//                ResponseEntity<JSONObject> entity = restTemplate.postForEntity(url, taskEntity, JSONObject.class);
-//                JSONObject result = entity.getBody().getJSONObject("data");
-//                return searchWholeBackUp(result.get("id").toString());
-//            }
-//        }
-        return null; //TODO
+        Project project = projectService.getProjectById(dto.getProjectId());
+        JSONObject jsonObject = new JSONObject();
+        if (2 == project.getWholeSearchDataSource()){ // 查商业数据库
+            defaultSearchFilter(dto);
+            try {
+                JSONObject params = IstarShineUtil.getKSSearchParam(dto.getSearch(), null, dto.getStartTime(), dto.getEndTime(), dto.getPlatformsStr(),
+                                dto.getFilterForward(), dto.getSortOf(), dto.getPage(), dto.getPageSize(), dto.getMatchFields(), null, null);
+
+                JSONObject taskEntity = new JSONObject();
+                taskEntity.put("appId", "6183571e0d710000f6003a12");    // 应用id, 由张志伟提供给使用者
+                taskEntity.put("taskType", "");                         // 可为null, 任务类型
+                taskEntity.put("taskDescription", "");                  // 可为null, 任务描述
+                taskEntity.put("userName", "品见");                      // 提交任务的人
+                taskEntity.put("searchParams", params);                 // 任务详情参数
+
+                ResponseEntity<JSONObject> responseEntity = restTemplate.postForEntity(addIStarShineKSDataUrl, taskEntity, JSONObject.class);
+                JSONObject result = Objects.requireNonNull(responseEntity.getBody()).getJSONObject("data");
+                return searchWholeBackUp(result.get("id").toString());
+            }catch (Exception e){
+                log.error("error searchWholeNetwork - ", e);
+            }
+        }else { // 查舆情库
+            ResponseEntity<JSONObject> responseEntity = restTemplate.postForEntity(wholeSearchUrl, dto.toMap(), JSONObject.class);
+            return searchWholeOpinion(responseEntity, jsonObject);
+        }
+        return jsonObject;
+    }
+
+    private JSONObject searchWholeOpinion(ResponseEntity<JSONObject> responseEntity, JSONObject jsonObject){
+        JSONObject bodyData = Objects.requireNonNull(responseEntity.getBody()).getJSONObject("data");
+        JSONObject data = bodyData.getJSONObject("data");
+        JSONArray listArray = data.getJSONArray("list");
+        List<JSONObject> resultList = new ArrayList<>();
+
+        for (Object o : listArray) {
+            JSONObject list = (JSONObject) o;
+            JSONObject result = new JSONObject();
+            result.put("platform", list.get("real_source"));
+            JSONArray contentArray = list.getJSONArray("ind_full_text");
+            assert contentArray == null;
+            result.put("title", contentArray.get(0));
+            result.put("content", contentArray.get(1));
+            if (3 == contentArray.size()) {
+                result.put("content", result.getString("content") + contentArray.get(2));
+            }
+            result.put("channel", list.get("source"));
+            String url = list.getString("url");
+            // 如果url为空,则查询question_url字段
+            if (null == url) {
+                url = list.getString("question_url");
+            }
+            result.put("url", url);
+            result.put("time", list.get("time"));
+            resultList.add(result);
+        }
+        jsonObject.put("list", resultList);
+        jsonObject.put("page", data.get("page"));
+        jsonObject.put("pageLimit", data.get("pageLimit"));
+        jsonObject.put("total", data.get("total"));
+        jsonObject.put("totalPage", data.get("totalPage"));
+        jsonObject.put("isBackUp", false);
+        return jsonObject;
+    }
+
+    public JSONObject searchWholeBackUp(String taskId) {
+        JSONObject jsonObject = new JSONObject();
+        List<JSONObject> resultList = new ArrayList<>();
+
+        for (int i = 0; i < 30; i++) {
+            ResponseEntity<JSONObject> responseEntity = restTemplate.getForEntity(getIStarShineKSInfoDataUrl, JSONObject.class, taskId);
+            JSONObject data = Objects.requireNonNull(responseEntity.getBody()).getJSONObject("data");
+            JSONObject task = data.getJSONObject("task");
+            if(1 == task.getInteger("crawlerFinish")){
+                //pageSize
+                Integer receiveCount = task.getInteger("receiveCount");
+                Integer searchCount = task.getInteger("searchCount");
+                int totalPage = searchCount % receiveCount == 0 ? searchCount / receiveCount : (searchCount / receiveCount + 1);
+
+                JSONArray listArray = data.getJSONArray("data");
+                for (Object o : listArray) {
+                    JSONObject list = (JSONObject) o;
+                    JSONObject result = new JSONObject();
+                    result.put("platform", list.getJSONObject("publisher").get("platform"));
+                    result.put("title", list.get("title"));
+                    result.put("content", list.get("content"));
+                    result.put("channel", list.get("channel"));
+                    result.put("url", getUrl(list));
+                    result.put("time", list.get("ctime"));
+                    resultList.add(result);
+                }
+                jsonObject.put("list",resultList);
+                //jsonObject.put("page",data1.get("page"));
+                jsonObject.put("pageLimit",receiveCount);
+                jsonObject.put("total",searchCount);
+                jsonObject.put("totalPage",totalPage);
+                jsonObject.put("isBackUp",true);
+                return jsonObject;
+            }
+            Tools.sleep(500L);
+        }
+        return jsonObject;
+    }
+
+    /**
+     * 获取全网搜接口信息url
+     *
+     * @param json
+     * @return
+     */
+    private static String getUrl(JSONObject json) {
+        String url = json.getString("url");
+        // 如果url为空,则查询QA类型url字段
+        if (null == url) {
+            url = json.getString("answer_url");
+        }
+        if (null == url) {
+            url = json.getString("question_url");
+        }
+        return url;
     }
 
     private void defaultSearchFilter(SearchFilterDTO dto){
         String projectId = UserThreadLocal.getProjectId();
         dto.setProjectId(projectId);
+        if (Objects.isNull(dto.getStartTime()) || Objects.isNull(dto.getEndTime())) {
+            Date now = new Date();
+            dto.setEndTime(now.getTime());
+            dto.setStartTime(DateUtils.addDays(now, -1).getTime());
+        }
         // 平台为空时，传全部
         if (dto.getPlatforms().size() == 0 ){
             dto.setPlatforms(Arrays.asList("微博","微信","新闻","平媒","自媒体","论坛","视频","短视频"));
         }
         // 平台
-        dto.setPlatformsStr(StringUtils.join(dto.getPlatforms(), ",")); //平台
+        dto.setPlatformsStr(StringUtils.join(dto.getPlatforms(), ","));
         // sort
         dto.setSortOf("ctime+" + dto.getSorter());
         // matchFields
@@ -1372,9 +1493,12 @@ public class MarkDataServiceImpl implements MarkDataService {
      */
     private void defaultContendDTO(MarkSearchDTO dto) {
         if (Objects.isNull(dto.getStartTime()) || Objects.isNull(dto.getEndTime())) {
-            Date now = new Date();
-            dto.setEndTime(now.getTime());
-            dto.setStartTime(DateUtils.addDays(now, -29).getTime());
+            Long[] timeRangeMonth = commonService.getTimeRangeMonth();
+            dto.setEndTime(timeRangeMonth[1]);
+            dto.setStartTime(timeRangeMonth[0]);
+        }else{
+            dto.setStartTime(Tools.truncDate(dto.getStartTime(), Constant.DAY_PATTERN));
+            dto.setEndTime(Tools.truncDate(dto.getEndTime(), Constant.DAY_PATTERN));
         }
         String projectId = UserThreadLocal.getProjectId();
         dto.setProjectId(projectId);

src/main/resources/application-local.properties

@@ -75,4 +75,8 @@ ef.search.url=https://ef.zhiweidata.com/external/search.do?name={1}&page={2}
 ef.searchCriteria.url=https://ef.zhiweidata.com/index/getSearchKey.do
 ef.filterNew.url=https://ef.zhiweidata.com/filterNew.do?firstType={1}&start={2}&end={3}&page={4}
 ef.captcha.url=https://ef.zhiweidata.com/captchaApi/getNewCaptcha.do.do?type=test
-ef.checkCaptcha.url=https://ef.zhiweidata.com/captchaApi/checkeCaptcha.do.do?id={1}&captcha={2}&type=test
+ef.checkCaptcha.url=https://ef.zhiweidata.com/captchaApi/checkeCaptcha.do.do?id={1}&captcha={2}&type=test
\ No newline at end of file
+#\u5168\u7F51\u641C\u5916\u90E8\u63A5\u53E3
+istarshine.addIStarShineKSData.url=https://istarshine-service.zhiweidata.com/istarshine/ks/addIStarShineKSData
+istarshine.getIStarShineKSInfoData.url=https://istarshine-service.zhiweidata.com/istarshine/ks/getIStarShineKSInfoData?taskId={1}&appId=6183571e0d710000f6003a12
+whole.search.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/search/yuqing
\ No newline at end of file