豆包模型实体类相关

src/main/java/com/zhiwei/brandkbs2/controller/admin/ProjectController.java

@@ -320,4 +320,11 @@ public class ProjectController extends BaseController {
     public ResponseResult switchExternalDataSource(@PathVariable String id){
         return ProjectService.switchExternalDataSource(id);
     }
+
+    @ApiOperation("非人工项目-方案查看")
+    @GetMapping("/non-manual/plan/{id}")
+    public ResponseResult findNonManualProjectPlan(@PathVariable String id){
+        return ProjectService.findNonManualProjectPlan(id);
+    }
+
 }

src/main/java/com/zhiwei/brandkbs2/es/EsClientDao.java

@@ -40,7 +40,6 @@ import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
 import org.elasticsearch.search.sort.FieldSortBuilder;
-import org.elasticsearch.search.sort.SortOrder;
 import org.joda.time.Period;
 import org.joda.time.PeriodType;
 import org.springframework.beans.factory.annotation.Value;
@@ -102,13 +101,24 @@ public class EsClientDao {
      * @return 渠道记录结果
      */
     public List<Pair<Long[], Map<ChannelIndex, ChannelIndex.Record>>> searchRecordRecentDay(int day) {
-        List<Pair<Long[], Map<ChannelIndex, ChannelIndex.Record>>> res = new ArrayList<>();
         Calendar calendar = Calendar.getInstance();
         // 设置为当天0:00
         calendar.setTime(Tools.truncDate(calendar.getTime(), Constant.DAY_PATTERN));
         long endTime = calendar.getTime().getTime();
         calendar.add(Calendar.DAY_OF_MONTH, -day);
         long startTime = calendar.getTime().getTime();
+        return searchRecordRecent(startTime, endTime);
+    }
+
+    /**
+     * 搜索近几天的数据
+     *
+     * @param startTime 开始时间
+     * @param endTime   结束时间
+     * @return 渠道记录结果
+     */
+    public List<Pair<Long[], Map<ChannelIndex, ChannelIndex.Record>>> searchRecordRecent(long startTime, long endTime) {
+        List<Pair<Long[], Map<ChannelIndex, ChannelIndex.Record>>> res = new ArrayList<>();
         List<Long[]> cutTimes = Tools.cutTimeRange(startTime, endTime, ONE_HOUR);
         List<CompletableFuture<Pair<Long[], Map<ChannelIndex, ChannelIndex.Record>>>> futures = new ArrayList<>(cutTimes.size());
         cutTimes.forEach(times -> futures.add(CompletableFuture.supplyAsync(() -> searchRecord(times[0], times[1]), executor)));

src/main/java/com/zhiwei/brandkbs2/pojo/NonManualProjectPlan.java

@@ -18,7 +18,7 @@ import java.util.List;
 @Getter
 @Setter
 @NoArgsConstructor
-public class NonManualProjectPlan extends AbstractBaseMongo{
+public class NonManualProjectPlan extends AbstractBaseMongo {
     /**
      * 项目id
      */
@@ -65,7 +65,12 @@ public class NonManualProjectPlan extends AbstractBaseMongo{
      */
     private String IStarShineSubjectTaskId;
 
-    public NonManualProjectPlan(NonManualProjectPlanDTO dto){
+    /**
+     * 情感倾向模型个性化配置
+     */
+    private ModelCustomConfig modelCustomConfig;
+
+    public NonManualProjectPlan(NonManualProjectPlanDTO dto) {
         this.id = dto.getId();
         this.projectId = UserThreadLocal.getProjectId();
         this.name = dto.getName();
@@ -78,4 +83,34 @@ public class NonManualProjectPlan extends AbstractBaseMongo{
         this.uTime = System.currentTimeMillis();
         this.submitter = UserThreadLocal.getNickname();
     }
+
+    @Getter
+    @Setter
+    public static class ModelCustomConfig {
+        /**
+         * 核心词类型
+         */
+        private String coreWordType;
+        /**
+         * 核心词(最多5个不能组合)
+         */
+        private String coreWord;
+        /**
+         * 负面词(最多20个不能组合)
+         */
+        private String negativeWord;
+        /**
+         * 负面观点(最多5个)
+         */
+        private List<String> negativePoints;
+        /**
+         * 正面词(最多20个不能组合)
+         */
+        private String positiveWord;
+        /**
+         * 正面观点(最多5个)
+         */
+        private List<String> positivePoints;
+    }
+
 }

src/main/java/com/zhiwei/brandkbs2/pojo/Project.java

@@ -119,6 +119,11 @@ public class Project extends AbstractProject {
     private Boolean externalDataSource;
 
     /**
+     * 标注模型（默认为0:nlp语料库,1:豆包模型）
+     */
+    private int markModel;
+
+    /**
      * 项目主品牌配置信息转换
      *
      * @return 项目对象

src/main/java/com/zhiwei/brandkbs2/pojo/dto/NonManualProjectPlanDTO.java

@@ -50,6 +50,11 @@ public class NonManualProjectPlanDTO {
      */
     private List<String> type;
 
+    /**
+     * 情感模型个性化配置
+     */
+    private NonManualProjectPlan.ModelCustomConfig modelCustomConfig;
+
     public NonManualProjectPlanDTO(NonManualProjectPlan plan){
         this.id = plan.getId();
         this.name = plan.getName();
@@ -58,5 +63,6 @@ public class NonManualProjectPlanDTO {
         this.lasting = plan.getLasting();
         this.endTime = plan.getEndTime();
         this.type = plan.getType();
+        this.modelCustomConfig = plan.getModelCustomConfig();
     }
 }

src/main/java/com/zhiwei/brandkbs2/service/ProjectService.java

@@ -216,4 +216,11 @@ public interface ProjectService {
      * @param projectId
      */
     ResponseResult switchExternalDataSource(String projectId);
+
+    /**
+     * 查看非人工项目方案列表
+     *
+     * @param projectId
+     */
+    ResponseResult findNonManualProjectPlan(String projectId);
 }

src/main/java/com/zhiwei/brandkbs2/service/impl/ProjectServiceImpl.java

@@ -767,6 +767,12 @@ public class ProjectServiceImpl implements ProjectService {
         return ResponseResult.success();
     }
 
+    @Override
+    public ResponseResult findNonManualProjectPlan(String projectId) {
+        List<NonManualProjectPlan> list = nonManualProjectPlanDao.findList(Query.query(Criteria.where("projectId").is(projectId)));
+        return ResponseResult.success(list);
+    }
+
     /**
      * 获取舆情对应项目的情感标签
      * @param brandName

src/main/java/com/zhiwei/brandkbs2/service/impl/TaskServiceImpl.java

@@ -192,6 +192,59 @@ public class TaskServiceImpl implements TaskService {
         log.info("渠道统计-渠道记录-统计结束");
     }
 
+    public void messageFlowCount2(long startTime,long endTime) {
+        List<Pair<Long[], Map<ChannelIndex, ChannelIndex.Record>>> rangeTimeRecords = esClientDao.searchRecordRecent(startTime,endTime);
+        // 结果合并
+        List<Map<ChannelIndex, ChannelIndex.Record>> channelList = rangeTimeRecords.stream().map(Pair::getRight).collect(Collectors.toList());
+        // 合并渠道记录
+        Map<ChannelIndex, ChannelIndex.Record> channelIndexRecordMap = ChannelIndex.mergeRecord(channelList);
+        log.info("渠道统计-搜索到的受影响渠道数{}条", channelIndexRecordMap.size());
+        long handleSize = 0;
+        List<Channel> insertList = new ArrayList<>();
+        List<Channel> updateList = new ArrayList<>();
+        List<Map.Entry<ChannelIndex, ChannelIndex.Record>> batchList = new ArrayList<>();
+        // 新recordMap
+        Map<ChannelIndex, ChannelIndex.Record> newRecordMap = new ConcurrentHashMap<>();
+        for (Map.Entry<ChannelIndex, ChannelIndex.Record> entry : channelIndexRecordMap.entrySet()) {
+            batchList.add(entry);
+            // 每100条做一次清算
+            if (++handleSize % 100 == 0 || handleSize == channelIndexRecordMap.size()) {
+                Pair<List<Channel>, List<Channel>> listPair = batchHandle(batchList, newRecordMap);
+                insertList.addAll(listPair.getLeft());
+                updateList.addAll(listPair.getRight());
+                batchList = new ArrayList<>();
+            }
+            if (handleSize % 10000 == 0) {
+                log.info("渠道统计-渠道总计-查询更新已完成{}/{}", handleSize, channelIndexRecordMap.size());
+            }
+        }
+        // 替换成新的记录map
+        channelIndexRecordMap = newRecordMap;
+        ListUtils.partition(insertList, 1000).forEach(list -> {
+            channelDao.insertMany(list);
+        });
+        log.info("渠道统计-渠道总计-录入完毕,新增渠道{}条,更新渠道{}条", insertList.size(), channelIndexRecordMap.size() - insertList.size());
+        // 获得单位时间内最小最大时间戳
+        Long[] timeMinMax = Tools.timeMinMax(rangeTimeRecords.stream().map(Pair::getLeft).collect(Collectors.toList()));
+        List<ChannelRecord> channelRecords = ChannelRecord.createChannelRecords(timeMinMax[0], timeMinMax[1], channelIndexRecordMap);
+        channelEsDao.upsertChannelRecord(channelRecords);
+        // 同步ES-channelCopy,区分insertList和updateList
+        for (int i = 1; i <= 3; i++) {
+            try {
+                ListUtils.partition(insertList, 1000).forEach(list -> {
+                    channelEsDao.batchInsert(list.stream().map(Channel::createChannelCopyMap).collect(Collectors.toList()));
+                });
+                ListUtils.partition(updateList, 1000).forEach(list -> {
+                    channelEsDao.batchInsert(list.stream().map(Channel::createChannelCopyMap).collect(Collectors.toList()));
+                });
+                break;
+            } catch (ElasticsearchStatusException e) {
+                log.info("渠道统计-es入库解析异常,正在重试第{}次", i);
+            }
+        }
+        log.info("渠道统计-渠道记录-统计结束");
+    }
+
     @Override
     public void channelRecordListCache() {
         AtomicInteger total = new AtomicInteger();

src/main/resources/application-local.properties

@@ -36,10 +36,10 @@ mongo.autoConnectRetry=true
 mongo.socketKeepAlive=true
 mongo.socketTimeout=120000
 mongo.slaveOk=true
-# \u6D4B\u8BD5
-#primary.uri=mongodb://rsync:rsync1q2w3e4r@115.236.59.88:30001/brandkbs2?authSource=admin
-# \u7EBF\u4E0A
-primary.uri=mongodb://brandkbs2:3vh65l$i6qQA@202.107.192.94:17152/brandkbs2?authSource=admin
+# 测试
+primary.uri=mongodb://rsync:rsync1q2w3e4r@115.236.59.88:30001/brandkbs2?authSource=admin
+# 线上
+#primary.uri=mongodb://brandkbs2:3vh65l$i6qQA@202.107.192.94:17152/brandkbs2?authSource=admin
 secondary.uri=mongodb://qbjcuser:qbjc1q2w3e4r@202.107.192.94:17152/qbjc?authSource=admin
 
 #es

src/test/java/com/zhiwei/brandkbs2/BrandkbsIndexCreateTest.java

 package com.zhiwei.brandkbs2;
 
-import com.zhiwei.brandkbs2.es.ChannelEsDao;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.RestHighLevelClient;
 import org.elasticsearch.client.indices.CreateIndexRequest;
+import org.elasticsearch.client.indices.GetIndexRequest;
+import org.elasticsearch.client.indices.GetIndexResponse;
+import org.elasticsearch.cluster.metadata.AliasMetadata;
+import org.elasticsearch.cluster.metadata.MappingMetadata;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.junit.Test;
@@ -16,6 +19,7 @@ import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
 import javax.annotation.Resource;
 import java.io.IOException;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 
@@ -29,12 +33,24 @@ public class BrandkbsIndexCreateTest {
 
     @Test
     public void test1() throws IOException {
-        String channelIndex = "brandkbs2_channel_record_2023";
+//        String channelIndex = "brandkbs2_channel_record_2023";
 //        String brandkbsIndex = "brandkbs2_2022";
-//          putChannelIndex(channelIndex);
+
 //        putBrandkbsIndex("brandkbs2_2018","brandkbs2_2019","brandkbs2_2020","brandkbs2_2021");
-//          putBrandkbsIndex("brandkbs2_2017");
-        putChannelCopyIndex(ChannelEsDao.CHANNEL_COPY_ES_INDEX_NAME);
+        putChannelIndex("brandkbs2_channel_record_2024");
+        putBrandkbsIndex("brandkbs2_2024");
+//        putChannelCopyIndex(ChannelEsDao.CHANNEL_COPY_ES_INDEX_NAME);
+    }
+
+    private void getIndexes(String... indexes) throws IOException {
+        GetIndexRequest request = new GetIndexRequest(indexes);
+        GetIndexResponse response = esClient.indices().get(request, RequestOptions.DEFAULT);
+        Map<String, List<AliasMetadata>> aliases = response.getAliases();
+        Map<String, MappingMetadata> mappings = response.getMappings();
+        Map<String, Settings> settings = response.getSettings();
+        System.out.println(aliases);
+        System.out.println(mappings);
+        System.out.println(settings);
     }
 
     public void putChannelCopyIndex(String... indexes) throws IOException {
@@ -307,6 +323,8 @@ public class BrandkbsIndexCreateTest {
 
         builder(obj2, "text", "key", true);
 
+        builder(obj2, "text", "plan_key", true);
+
         builder(obj2, "text", "linked_group_id", true);
 
         builder(obj2, "text", "project_id", true);

src/test/java/com/zhiwei/brandkbs2/MarkDataServiceTest.java

@@ -179,7 +179,7 @@ public class MarkDataServiceTest {
     @Test
     public void searchWholeNetworkTest(){
         SearchFilterDTO dto = new SearchFilterDTO();
-        dto.setSearch("阿里");
+        dto.setKeyword("阿里");
         JSONObject jsonObject = markDataService.searchWholeNetworkWithBalance(dto).getLeft();
         System.out.println(jsonObject);
     }

src/test/java/com/zhiwei/brandkbs2/ProjectServiceTest.java

@@ -24,7 +24,7 @@ public class ProjectServiceTest {
 
     @Test
     public void findProjectListTest(){
-        PageVO<JSONObject> projectList = projectService.findProjectList(1, 10, null);
+        PageVO<JSONObject> projectList = projectService.findProjectList(1, 10, null,null);
         for (JSONObject jsonObject : projectList.getList()) {
             System.out.println(jsonObject);
         }