Skip to content

B
brandkbs2
Commit e184bb40 by shenjunjie
Options

Merge branch 'release' into 'master' 

Release

See merge request !575
parents f3063184 6e7daab6
Showing with 634 additions and 33 deletions
pom.xml
......@@ -269,7 +269,7 @@
<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp</artifactId>
<version>3.8.0</version>
<version>3.12.0</version>
</dependency>
<!-- dubbo -->
<dependency>
......@@ -324,6 +324,12 @@
<artifactId>ansj_seg</artifactId>
<version>5.0.2</version>
</dependency>
<!--火山引擎 豆包大模型-->
<dependency>
<groupId>com.volcengine</groupId>
<artifactId>volcengine-java-sdk-ark-runtime</artifactId>
<version>0.1.121</version>
</dependency>
</dependencies>
<build>
<plugins>
......
src/main/java/com/zhiwei/brandkbs2/aop/AopLogRecord.java
......@@ -144,7 +144,7 @@ public class AopLogRecord {
String[] value = method.getAnnotation(LogRecord.class).values();
// 注解value为空字符串（value使用默认值未设置）
if (1 == value.length && StringUtils.isEmpty(value[0])){
return new UserLogRecord(projectId, userInfo.getUserId(), userInfo.getNickname(), prefix, userInfo.getRoleId(), now, now);
return new UserLogRecord(projectId, userInfo.getUserId(), userInfo.getNickname(), prefix, userInfo.getRoleId(), now, now, null);
}
// 获取接口传参（value为获取传参的具体字段）并与操作描述description拼接返回，传参值为实体
if (method.getAnnotation(LogRecord.class).arguments() && method.getAnnotation(LogRecord.class).entity()) {
......@@ -176,7 +176,7 @@ public class AopLogRecord {
}
}
String suffix = CollectionUtils.isNotEmpty(res) ? "-" + Tools.concatWithMinus(res) : "";
return new UserLogRecord(projectId, userInfo.getUserId(), userInfo.getNickname(), prefix + suffix, userInfo.getRoleId(), now, now);
return new UserLogRecord(projectId, userInfo.getUserId(), userInfo.getNickname(), prefix + suffix, userInfo.getRoleId(), now, now, null);
}
// 获取接口传参（value为获取传参的具体字段）并与操作描述description拼接返回，传参值不为实体
if (method.getAnnotation(LogRecord.class).arguments() && !method.getAnnotation(LogRecord.class).entity()) {
......@@ -190,7 +190,7 @@ public class AopLogRecord {
}
}
String suffix = CollectionUtils.isNotEmpty(res) ? "-" + Tools.concatWithMinus(res) : "";
return new UserLogRecord(projectId, userInfo.getUserId(), userInfo.getNickname(), prefix + suffix, userInfo.getRoleId(), now, now);
return new UserLogRecord(projectId, userInfo.getUserId(), userInfo.getNickname(), prefix + suffix, userInfo.getRoleId(), now, now, null);
}
// 获取接口返回值（value为获取返回值的具体字段）并与操作描述description拼接返回，返回值为实体
if (!method.getAnnotation(LogRecord.class).arguments() && method.getAnnotation(LogRecord.class).entity() && Objects.nonNull(responseResult)) {
......@@ -204,9 +204,9 @@ public class AopLogRecord {
}
}
String suffix = CollectionUtils.isNotEmpty(res) ? "-" + Tools.concatWithMinus(res) : "";
return new UserLogRecord(projectId, userInfo.getUserId(), userInfo.getNickname(), prefix + suffix, userInfo.getRoleId(), now, now);
return new UserLogRecord(projectId, userInfo.getUserId(), userInfo.getNickname(), prefix + suffix, userInfo.getRoleId(), now, now, null);
}
return new UserLogRecord(projectId, userInfo.getUserId(), userInfo.getNickname(), prefix, userInfo.getRoleId(), now, now);
return new UserLogRecord(projectId, userInfo.getUserId(), userInfo.getNickname(), prefix, userInfo.getRoleId(), now, now, null);
}
/**
......
src/main/java/com/zhiwei/brandkbs2/common/DoubaoAIAccountFactor.java 0 → 100644
package com.zhiwei.brandkbs2.common;
import com.volcengine.ark.runtime.service.ArkService;
import com.zhiwei.brandkbs2.pojo.ai.AccessModel;
import lombok.AllArgsConstructor;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
/**
* @ClassName: DoubaoAIAccountFactor
* @Description DoubaoAIAccountFactor
* @author: sjj
* @date: 2024-07-24 14:01
*/
public class DoubaoAIAccountFactor {
public static Account getCompanyAccount() {
String apiKey = "607764fc-c9d9-47e4-a673-a310852917a0";
List<AccessModel> modelList = new ArrayList<>();
modelList.add(new AccessModel("ep-20240617061616-8d2ls", AccessModel.Model.DOUBAO_PRO_4K));
modelList.add(new AccessModel("ep-20240618021538-t6dpf", AccessModel.Model.DOUBAO_PRO_32K));
return new Account(apiKey, modelList);
}
@Data
@AllArgsConstructor
public static class Account {
String apiKey;
List<AccessModel> modelList;
}
public static ArkService arkService;
static {
arkService = new ArkService(getCompanyAccount().getApiKey());
}
}
src/main/java/com/zhiwei/brandkbs2/common/RedisKeyPrefix.java
......@@ -121,8 +121,13 @@ public class RedisKeyPrefix {
public static final String CUSTOM_YUQING_ANALYZE_HIGH_WORD = "BRANDKBS:CUSTOM:YUQING:ANALYZE:HIGH:WORD:";
/**
* 搜索相关缓存
*/
public static final String SEARCH_KEYWORD = "BRANDKBS:SEARCH:KEYWORD:";
public static final String AI_SEARCH_QUESTION = "BRANDKBS:AI:SEARCH:QUESTION:";
public static String projectWarnHotTopKeyAll(String projectId, String type) {
return RedisKeyPrefix.generateRedisKey(RedisKeyPrefix.PROJECT_WARN_HOT_TOP, projectId, Tools.concat(type, "*"));
}
......
src/main/java/com/zhiwei/brandkbs2/controller/app/AppDownloadController.java
......@@ -396,9 +396,9 @@ public class AppDownloadController extends BaseController {
@PostMapping(value = "/contend/mark")
@DownloadTask(taskName = "竞品库竞品舆情下载", description = "竞品库竞品舆情")
public ResponseResult exportContendMarkList(@RequestBody MarkSearchDTO markSearchDTO) {
if (StringUtils.isNotEmpty(markSearchDTO.getKeyword()) && Tools.checkUniteString(markSearchDTO.getKeyword())){
return ResponseResult.failure("不支持特殊符号字段查询");
}
// if (StringUtils.isNotEmpty(markSearchDTO.getKeyword()) && Tools.checkUniteString(markSearchDTO.getKeyword())){
// return ResponseResult.failure("不支持特殊符号字段查询");
// }
Pair<String, List<ExportAppYuqingDTO>> stringListPair = markDataService.downloadContendMarkList(markSearchDTO);
// excel写入至指定路径
String projectName = projectService.getProjectById(UserThreadLocal.getProjectId()).getProjectName();
......@@ -412,9 +412,9 @@ public class AppDownloadController extends BaseController {
@LogRecord(description = "全网搜-舆情导出", values = {"startTime", "endTime", "fans", "filterType", "filterWords", "search", "keyword", "platforms", "sensitiveChannels", "sourceKeyword"}, entity = true, arguments = true)
@DownloadTask(taskName = "全网搜舆情下载", description = "全网搜舆情")
public ResponseResult exportSearchWhole(@RequestBody SearchFilterDTO dto) {
if (StringUtils.isNotEmpty(dto.getKeyword()) && Tools.checkUniteString(dto.getKeyword())){
return ResponseResult.failure("不支持特殊符号字段查询");
}
// if (StringUtils.isNotEmpty(dto.getKeyword()) && Tools.checkUniteString(dto.getKeyword())){
// return ResponseResult.failure("不支持特殊符号字段查询");
// }
// 针对商业数据库做限制
if (dto.isExternalDataSource()) {
long time = DateUtils.addDays(Tools.truncDate(new Date(), Constant.DAY_PATTERN), -89).getTime();
......
src/main/java/com/zhiwei/brandkbs2/controller/app/AppSearchController.java
......@@ -161,9 +161,9 @@ public class AppSearchController extends BaseController {
@LogRecord(values = {"fans", "sensitiveChannels:father,son", "keyword", "search"}, description = "全网搜", arguments = true, entity = true)
@PostMapping("/searchWhole")
public ResponseResult searchWholeNetwork(@RequestBody SearchFilterDTO dto) {
if (StringUtils.isNotEmpty(dto.getKeyword()) && Tools.checkUniteString(dto.getKeyword())){
return ResponseResult.failure("不支持特殊符号字段查询");
}
// if (StringUtils.isNotEmpty(dto.getKeyword()) && Tools.checkUniteString(dto.getKeyword())){
// return ResponseResult.failure("不支持特殊符号字段查询");
// }
cacheSearchKeyword(dto.getKeyword(), "whole");
// 针对商业数据库做限制
if (dto.isExternalDataSource()) {
......@@ -200,9 +200,9 @@ public class AppSearchController extends BaseController {
@PostMapping("/exportSearchWhole")
@LogRecord(description = "全网搜-舆情导出", values = {"startTime", "endTime", "fans", "filterType", "filterWords", "search", "keyword", "platforms", "sensitiveChannels", "sourceKeyword"}, entity = true, arguments = true)
public ResponseResult exportSearchWhole(@RequestBody SearchFilterDTO dto) {
if (StringUtils.isNotEmpty(dto.getKeyword()) && Tools.checkUniteString(dto.getKeyword())){
return ResponseResult.failure("不支持特殊符号字段查询");
}
// if (StringUtils.isNotEmpty(dto.getKeyword()) && Tools.checkUniteString(dto.getKeyword())){
// return ResponseResult.failure("不支持特殊符号字段查询");
// }
// 针对商业数据库做限制
if (dto.isExternalDataSource()) {
long time = DateUtils.addDays(Tools.truncDate(new Date(), Constant.DAY_PATTERN), -89).getTime();
......@@ -231,9 +231,9 @@ public class AppSearchController extends BaseController {
@LogRecord(values = {"searchType", "keyword"}, description = "查舆情", arguments = true, entity = true)
@PostMapping("/mark/list")
public ResponseResult getYuqingMarkList(@RequestBody MarkSearchDTO markSearchDTO) {
if (StringUtils.isNotEmpty(markSearchDTO.getKeyword()) && Tools.checkUniteString(markSearchDTO.getKeyword())){
return ResponseResult.failure("不支持特殊符号字段查询");
}
// if (StringUtils.isNotEmpty(markSearchDTO.getKeyword()) && Tools.checkUniteString(markSearchDTO.getKeyword())){
// return ResponseResult.failure("不支持特殊符号字段查询");
// }
cacheSearchKeyword(markSearchDTO.getKeyword(), "yuqing");
PageVO<MarkFlowEntity> yuqingMarkList = markDataService.getYuqingMarkList(markSearchDTO);
// 仅第一页增加平台进量（声量）统计
......@@ -339,9 +339,9 @@ public class AppSearchController extends BaseController {
@LogRecord(values = "keyword", description = "查竞品",arguments = true, entity = true)
@PostMapping("/contend/list")
public ResponseResult getContendSearchList(@RequestBody MarkSearchDTO markSearchDTO) {
if (StringUtils.isNotEmpty(markSearchDTO.getKeyword()) && Tools.checkUniteString(markSearchDTO.getKeyword())){
return ResponseResult.failure("不支持特殊符号字段查询");
}
// if (StringUtils.isNotEmpty(markSearchDTO.getKeyword()) && Tools.checkUniteString(markSearchDTO.getKeyword())){
// return ResponseResult.failure("不支持特殊符号字段查询");
// }
cacheSearchKeyword(markSearchDTO.getKeyword(), "contend");
return ResponseResult.success(markDataService.getContendSearchList(markSearchDTO));
}
......@@ -352,6 +352,34 @@ public class AppSearchController extends BaseController {
return ResponseResult.success(markDataService.getContendSearchCriteria(contendId));
}
@ApiOperation("搜索-AI搜索")
@GetMapping("/ai/answer")
public ResponseResult getAISearchResult(@RequestParam(value = "question") String question,
@RequestParam(value = "keyword", required = false) String keyword,
@RequestParam(value = "startTime", required = false) Long startTime,
@RequestParam(value = "endTime", required = false) Long endTime) {
return ResponseResult.success(markDataService.getAISearchResult(question, keyword, startTime, endTime));
}
@ApiOperation("搜索-AI搜索-联网搜索")
@GetMapping("/ai/answer/online")
public ResponseResult getAIOnlineSearchResult(@RequestParam(value = "question") String question) {
return ResponseResult.success(markDataService.getAIOnlineSearchResult(question));
}
@ApiOperation("搜索-AI推荐提问")
@GetMapping("/ai/question")
public ResponseResult getAIReferenceQuestion(@RequestParam(value = "question") String question,
@RequestParam(value = "size") int size) {
return ResponseResult.success(markDataService.getAIReferenceQuestion(question, size));
}
@ApiOperation("搜索-AI参考提问")
@GetMapping("/ai/question-cache")
public ResponseResult getAIReferenceQuestion() {
return ResponseResult.success(markDataService.getAIReferenceQuestionCache(true));
}
@ApiOperation("搜索-搜索关键词历史记录")
@GetMapping("/keyword/cache")
public ResponseResult getSearchKeywordCache(@ApiParam(name = "searchType",
......
src/main/java/com/zhiwei/brandkbs2/dao/AISearchQuestionRecordDao.java 0 → 100644
package com.zhiwei.brandkbs2.dao;
import com.zhiwei.brandkbs2.pojo.AISearchQuestionRecord;
import java.util.List;
/**
* @ClassName: AISearchQuestionRecordDao
* @Description AISearchQuestionRecordDao
* @author: cjz
* @date: 2024-08-12 17:07
*/
public interface AISearchQuestionRecordDao extends BaseMongoDao<AISearchQuestionRecord>{
List<String> findDistinctQuestion(String projectId);
}
src/main/java/com/zhiwei/brandkbs2/dao/impl/AISearchQuestionRecordDaoImpl.java 0 → 100644
package com.zhiwei.brandkbs2.dao.impl;
import com.zhiwei.brandkbs2.dao.AISearchQuestionRecordDao;
import com.zhiwei.brandkbs2.pojo.AISearchQuestionRecord;
import org.springframework.data.domain.Sort;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Component;
import java.util.List;
/**
* @ClassName: AISearchQuestionRecordDao
* @Description AISearchQuestionRecordDao
* @author: cjz
* @date: 2024-08-12 17:07
*/
@Component("aiSearchQuestionRecordDao")
public class AISearchQuestionRecordDaoImpl extends BaseMongoDaoImpl<AISearchQuestionRecord> implements AISearchQuestionRecordDao {
private static final String COLLECTION_NAME = "brandkbs_ai_search_question_record";
public AISearchQuestionRecordDaoImpl() {
super(COLLECTION_NAME);
}
@Override
public List<String> findDistinctQuestion(String projectId) {
Query query = new Query().addCriteria(Criteria.where("projectId").is(projectId)).with(Sort.by(Sort.Order.desc("cTime"))).limit(10);
return mongoTemplate.findDistinct(query, "question", COLLECTION_NAME, String.class);
}
}
src/main/java/com/zhiwei/brandkbs2/es/EsClientDao.java
package com.zhiwei.brandkbs2.es;
import com.alibaba.fastjson.JSONObject;
import com.hankcs.hanlp.HanLP;
import com.zhiwei.brandkbs2.auth.UserThreadLocal;
import com.zhiwei.brandkbs2.common.GenericAttribute;
import com.zhiwei.brandkbs2.common.GlobalPojo;
import com.zhiwei.brandkbs2.config.Constant;
import com.zhiwei.brandkbs2.pojo.ChannelIndex;
import com.zhiwei.brandkbs2.pojo.Contend;
import com.zhiwei.brandkbs2.pojo.Project;
import com.zhiwei.brandkbs2.pojo.ai.FieldMapping;
import com.zhiwei.brandkbs2.util.TextUtil;
import com.zhiwei.brandkbs2.util.Tools;
import lombok.Getter;
import lombok.Setter;
......@@ -33,6 +40,7 @@ import org.elasticsearch.search.aggregations.AggregationBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.sort.FieldSortBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.joda.time.Period;
import org.joda.time.PeriodType;
import org.springframework.beans.factory.annotation.Value;
......@@ -331,6 +339,135 @@ public class EsClientDao {
return Pair.of(new Long[]{startTime, endTime}, res);
}
public List<JSONObject> findSearch(String question, String keyword, Long startTime, Long endTime) throws IOException {
List<JSONObject> list = new ArrayList<>();
String projectId = UserThreadLocal.getProjectId();
BoolQueryBuilder query = QueryBuilders.boolQuery();
// 默认一周
if (Objects.isNull(startTime) || Objects.isNull(endTime)){
endTime = System.currentTimeMillis();
startTime = System.currentTimeMillis() - Constant.ONE_WEEK;
}
// time
query.must(QueryBuilders.rangeQuery(GenericAttribute.ES_TIME).gte(startTime).lt(endTime));
// contend
Project project = GlobalPojo.PROJECT_MAP.get(projectId);
if (CollectionUtils.isNotEmpty(project.getContendList())) {
List<Contend> contendList = new ArrayList<>();
for (Contend contend : project.getContendList()) {
if (question.contains(contend.getBrandName())) {
contendList.add(contend);
}
}
if (CollectionUtils.isNotEmpty(contendList)) {
BoolQueryBuilder contendQuery = QueryBuilders.boolQuery();
for (Contend contend : contendList) {
contendQuery.should(EsQueryTools.assembleCacheMapsQuery(projectId, contend.getId()));
}
contendQuery.minimumShouldMatch(1);
query.must(contendQuery);
}else {
query.must(EsQueryTools.assembleCacheMapsQuery(projectId, Constant.PRIMARY_CONTEND_ID));
}
}
// keyword
query.must(EsQueryTools.assembleNormalKeywordQuery(keyword, new String[]{GenericAttribute.ES_IND_FULL_TEXT}));
String[] fetchSource = {"id", GenericAttribute.ES_TIME, GenericAttribute.ES_IND_TITLE};
// hit
SearchHit[] hits = searchHits(getIndexes(), query, null, fetchSource, null, 0, 10000, null).getHits();
Pair<Map<String, JSONObject>, Map<String, String>> searchProcess = findSearchResultProcess(hits);
Map<String, JSONObject> idBaseMap = searchProcess.getLeft();
Map<String, String> idTitle = searchProcess.getRight();
if (idTitle.isEmpty()){
return list;
}
// 按标题聚合，取聚合结果集前9，并取结果集中最新的文章的id
List<String> idList = TextUtil.getKResult(idTitle).stream()
.sorted(Comparator.comparing(List<String>::size, Comparator.reverseOrder()))
.limit(9)
.map(ids -> ids.stream().map(idBaseMap::get).max(Comparator.comparingLong(json -> json.getLongValue(GenericAttribute.ES_TIME))).orElse(null))
.filter(Objects::nonNull)
.map(json -> json.getString("id"))
.collect(Collectors.toList());
// 反查原数据
for (String id : idList) {
list.add(getTopTitleLatest(id));
}
return list;
}
public List<JSONObject> findSearch(List<FieldMapping> fieldMappings) throws IOException {
List<JSONObject> list = new ArrayList<>();
BoolQueryBuilder query = getBoolQueryBuilder(fieldMappings);
String[] fetchSource = {"id", GenericAttribute.ES_TIME, GenericAttribute.ES_IND_TITLE};
SearchHit[] hits = searchHits(getIndexes(), query, null, fetchSource, null, 0, 10000, null).getHits();
Pair<Map<String, JSONObject>, Map<String, String>> searchProcess = findSearchResultProcess(hits);
Map<String, JSONObject> idBaseMap = searchProcess.getLeft();
Map<String, String> idTitle = searchProcess.getRight();
// 搜索条件未找到结果，将搜索关键词分词处理，再次查询
if (idTitle.isEmpty()){
SearchHit[] searchHitHanLP = findSearchHanLP(fieldMappings, fetchSource);
Pair<Map<String, JSONObject>, Map<String, String>> searchProcessHanLP = findSearchResultProcess(searchHitHanLP);
idBaseMap = searchProcessHanLP.getLeft();
idTitle = searchProcessHanLP.getRight();
}
if (idTitle.isEmpty()){
return list;
}
// 按标题聚合，取聚合结果集前9，并取结果集中最新的文章的id
Map<String, JSONObject> finalIdBaseMap = idBaseMap;
List<String> idList = TextUtil.getKResult(idTitle).stream()
.sorted(Comparator.comparing(List<String>::size, Comparator.reverseOrder()))
.limit(9)
.map(ids -> ids.stream().map(finalIdBaseMap::get).max(Comparator.comparingLong(json -> json.getLongValue(GenericAttribute.ES_TIME))).orElse(null))
.filter(Objects::nonNull)
.map(json -> json.getString("id"))
.collect(Collectors.toList());
// 反查原数据
for (String id : idList) {
list.add(getTopTitleLatest(id));
}
return list;
}
private Pair<Map<String, JSONObject>, Map<String, String>> findSearchResultProcess(SearchHit[] hits){
Map<String, JSONObject> idBaseMap = Arrays.stream(hits).map(hit -> new JSONObject(hit.getSourceAsMap())).collect(Collectors.toMap(json -> json.getString("id"), o -> o));
Map<String, String> idTitle = Arrays.stream(hits)
.map(hit -> new JSONObject(hit.getSourceAsMap()))
.filter(json -> Objects.nonNull(json.getString(GenericAttribute.ES_IND_TITLE)) || Tools.filterUselessTitle(GenericAttribute.ES_IND_TITLE))
.collect(Collectors.toMap(json -> json.getString("id"), json -> json.getString(GenericAttribute.ES_IND_TITLE)));
return Pair.of(idBaseMap, idTitle);
}
private SearchHit[] findSearchHanLP(List<FieldMapping> fieldMappings, String[] fetchSource) throws IOException {
fieldMappings.stream().filter(fieldMapping -> Objects.equals(FieldMapping.FieldMap.IND_FULL_TEXT, fieldMapping.getFieldMap()))
.findFirst().ifPresent(fieldMapping -> {
String value = String.valueOf(fieldMapping.getValue());
String newValue = HanLP.segment(Tools.filterSpecialCharacter(value)).stream().map(s -> s.word).distinct().collect(Collectors.joining(" "));
fieldMapping.setValue(newValue);
});
BoolQueryBuilder query = getBoolQueryBuilder(fieldMappings);
return searchHits(getIndexes(), query, null, fetchSource, null, 0, 10000, null).getHits();
}
private JSONObject getTopTitleLatest(String id) throws IOException {
BoolQueryBuilder query = QueryBuilders.boolQuery();
query.must(QueryBuilders.termQuery("id", id));
return searchById(id);
}
private BoolQueryBuilder getBoolQueryBuilder(List<FieldMapping> fieldMappings) {
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
Map<String, List<FieldMapping>> groupMap = fieldMappings.stream().collect(Collectors.groupingBy(mapping -> mapping.getFieldMap().getFatherName()));
groupMap.forEach((fatherName, list) -> {
if (list.size() > 2) {
throw new IllegalStateException("构建搜索条件分组异常");
}
boolQueryBuilder.must(list.get(0).buildQuery(list.size() > 1 ? list.get(1) : null));
});
return boolQueryBuilder;
}
public String[] getIndexes() {
return getIndexList().toArray(new String[0]);
}
......
src/main/java/com/zhiwei/brandkbs2/pojo/AISearchQuestionRecord.java 0 → 100644
package com.zhiwei.brandkbs2.pojo;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.Setter;
/**
* @ClassName: AISearchQuestionRecord
* @Description ai搜索问题记录
* @author: cjz
* @date: 2024-8-12 14:58
*/
@Getter
@Setter
@AllArgsConstructor
public class AISearchQuestionRecord extends AbstractBaseMongo {
/**
* 问题
*/
private String question;
/**
* 项目id
*/
private String projectId;
/**
* 创建时间
*/
private Long cTime;
}
src/main/java/com/zhiwei/brandkbs2/pojo/UserLogRecord.java
package com.zhiwei.brandkbs2.pojo;
import com.zhiwei.brandkbs2.auth.UserThreadLocal;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import org.springframework.data.mongodb.core.mapping.Document;
/**
* @ClassName: UserLogRecord
......@@ -14,6 +15,7 @@ import org.springframework.data.mongodb.core.mapping.Document;
@Getter
@Setter
@AllArgsConstructor
@NoArgsConstructor
public class UserLogRecord extends AbstractBaseMongo{
/**
* 项目id
......@@ -43,4 +45,33 @@ public class UserLogRecord extends AbstractBaseMongo{
* 更新时间
*/
private Long updateTime;
/**
* 本次调用费用（ai搜索）
*/
private Double cost;
public static UserLogRecord userLogRecordCost(String description, double cost){
UserLogRecord record = new UserLogRecord();
record.setProjectId(UserThreadLocal.getProjectId());
record.setUserId(UserThreadLocal.getUserId());
record.setNickname(UserThreadLocal.getNickname());
record.setDescription(description);
record.setRoleId(UserThreadLocal.getRoleId());
record.setCTime(System.currentTimeMillis());
record.setUpdateTime(System.currentTimeMillis());
record.setCost(cost);
return record;
}
public static UserLogRecord defaultUserLogRecord(String description){
UserLogRecord record = new UserLogRecord();
record.setProjectId(UserThreadLocal.getProjectId());
record.setUserId(UserThreadLocal.getUserId());
record.setNickname(UserThreadLocal.getNickname());
record.setDescription(description);
record.setRoleId(UserThreadLocal.getRoleId());
record.setCTime(System.currentTimeMillis());
record.setUpdateTime(System.currentTimeMillis());
return record;
}
}
src/main/java/com/zhiwei/brandkbs2/pojo/ai/AccessModel.java 0 → 100644
package com.zhiwei.brandkbs2.pojo.ai;
import lombok.Data;
import lombok.Getter;
/**
* @ClassName: AccessModel
* @Description AccessModel
* @author: sjj
* @date: 2024-07-24 14:05
*/
@Data
public class AccessModel {
// 模型名称
private String modelName;
// 接入点id
private String modelId;
// 输入单价(每千个token)
private Double inputPrice;
// 输出单价(每千个token)
private Double outputPrice;
public AccessModel(String modelId, Model model) {
this.modelId = modelId;
this.modelName = model.modelName;
this.inputPrice = model.inputPrice;
this.outputPrice = model.outputPrice;
}
public enum Model {
DOUBAO_PRO_4K("Doubao-pro-4k", 0.0008, 0.0020),
DOUBAO_PRO_32K("Doubao-pro-32k", 0.0008, 0.0020),
DOUBAO_PRO_128K("Doubao-pro-128k", 0.0090, 0.0500),
DOUBAO_LITE_4K("doubao-lite-4k", 0.0003, 0.0006),
DOUBAO_LITE_32K("Doubao-lite-32k", 0.0003, 0.0006),
DOUBAO_LITE_128K("Doubao-lite-128k", 0.0008, 0.0010);
@Getter
private final String modelName;
// 输入单价(每千个token)
@Getter
private final Double inputPrice;
// 输出单价(每千个token)
@Getter
private final Double outputPrice;
Model(String modelName, Double inputPrice, Double outputPrice) {
this.modelName = modelName;
this.inputPrice = inputPrice;
this.outputPrice = outputPrice;
}
}
}
src/main/java/com/zhiwei/brandkbs2/pojo/ai/FieldMapping.java 0 → 100644
package com.zhiwei.brandkbs2.pojo.ai;
import com.zhiwei.brandkbs2.es.EsQueryTools;
import lombok.Data;
import lombok.Getter;
import org.apache.lucene.search.join.ScoreMode;
import org.elasticsearch.index.query.*;
import java.util.*;
/**
* @ClassName: FieldMap
* @Description FieldMap
* @author: sjj
* @date: 2024-08-02 14:01
*/
@Data
public class FieldMapping {
private FieldMap fieldMap;
private Object value;
public FieldMapping(FieldMap fieldMap, Object value) {
this.fieldMap = fieldMap;
this.value = value;
}
public QueryBuilder buildQuery(FieldMapping fieldMapping) {
boolean existsAnd = null != fieldMapping;
RangeQueryBuilder timeRangeBuilder;
String[] contendIds = {"0"};
// 项目组需绑定查询
switch (fieldMap) {
case START_TIME:
timeRangeBuilder = QueryBuilders.rangeQuery(fieldMap.databaseName).gte(value);
if (existsAnd && fieldMapping.fieldMap.equals(FieldMap.END_TIME)) {
timeRangeBuilder.lt(fieldMapping.value);
}
return timeRangeBuilder;
case END_TIME:
timeRangeBuilder = QueryBuilders.rangeQuery(fieldMap.databaseName).lt(value);
if (existsAnd && fieldMapping.fieldMap.equals(FieldMap.START_TIME)) {
timeRangeBuilder.gte(fieldMapping.value);
}
return timeRangeBuilder;
case PROJECT:
if (existsAnd && fieldMapping.fieldMap == FieldMap.BRAND) {
contendIds = ((String) fieldMapping.value).split("\\|");
}
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
for (String contendId : contendIds) {
BoolQueryBuilder nestedBoolBuilder = QueryBuilders.boolQuery();
// 必要条件
nestedBoolBuilder.must(QueryBuilders.termQuery(fieldMap.databaseName, value + "_" + contendId));
boolQueryBuilder.should(new NestedQueryBuilder("brandkbs_cache_maps", nestedBoolBuilder, ScoreMode.None));
}
boolQueryBuilder.minimumShouldMatch(1);
return boolQueryBuilder;
case BRAND:
if (!existsAnd || fieldMapping.fieldMap != FieldMap.PROJECT) {
throw new IllegalStateException("项目条件缺失");
}
return fieldMapping.buildQuery(this);
case IND_FULL_TEXT:
return EsQueryTools.assembleNormalKeywordQuery(String.valueOf(value), new String[]{fieldMap.databaseName});
case SOURCE:
case MTAG:
return QueryBuilders.termQuery(fieldMap.databaseName, value);
}
return null;
}
public enum FieldMap {
START_TIME("起始时间", "时间", "time"),
END_TIME("结束时间", "时间", "time"),
PROJECT("项目", "项目", "brandkbs_cache_maps.key.keyword"),
BRAND("品牌", "项目", "brandkbs_cache_maps.key.keyword"),
SOURCE("渠道", "渠道", "source"),
MTAG("标签", "标签", "mark_cache_maps.name.keyword"),
IND_FULL_TEXT("搜索条件", "搜索条件", "ind_full_text");
@Getter
private final String name;
@Getter
private final String fatherName;
@Getter
private final String databaseName;
FieldMap(String name, String fatherName, String databaseName) {
this.name = name;
this.fatherName = fatherName;
this.databaseName = databaseName;
}
}
public static FieldMapping createFromNameAndValue(String name, Object value, String question) {
FieldMap fieldMap = null;
// String projectId = UserThreadLocal.getProjectId();
// TODO 字段转换待完善,引入数据库
for (FieldMap f : FieldMap.values()) {
if (name.equals(f.getName())) {
// // 项目名需要转成id
// if (FieldMap.PROJECT == f) {
// Map<String, Project> projectMap = GlobalPojo.PROJECT_MAP.values().stream().collect(Collectors.toMap(AbstractProject::getProjectName, o -> o));
// if (projectMap.containsKey(String.valueOf(value))) {
// value = projectMap.get(String.valueOf(value)).getId();
// }else {
// value = projectId;
// }
// }
// 品牌需要转换
// if (FieldMap.BRAND == f) {
// Project project = GlobalPojo.PROJECT_MAP.get(projectId);
// if (CollectionUtils.isNotEmpty(project.getContendList())){
// List<String> contends = new ArrayList<>();
// List<String> contendNames = project.getContendList().stream().map(AbstractProject::getBrandName).collect(Collectors.toList());
// for (String contendName : contendNames) {
// if (question.contains(contendName)) {
// contends.add(contendName);
// }
// }
// if (CollectionUtils.isNotEmpty(contends)){
// value = String.join("|", contends);
// }else {
// value = Constant.PRIMARY_CONTEND_ID;
// }
// }else {
// value = Constant.PRIMARY_CONTEND_ID;
// }
// if ("主品牌".equals(value)) {
// value = Constant.PRIMARY_CONTEND_ID;
// } else {
// // 寻找对应的竞品id
// Optional<FieldMapping> project = fieldMappings.stream().filter(field -> Objects.equals(FieldMap.PROJECT, field.getFieldMap())).findFirst();
// if (project.isPresent()){
// List<Contend> contendList = GlobalPojo.PROJECT_MAP.get(String.valueOf(project.get().getValue())).getContendList();
// Object finalValue = value;
// Optional<Contend> contendOptional = contendList.stream().filter(contend -> Objects.equals(contend.getBrandName(), finalValue)).findFirst();
// if (contendOptional.isPresent()){
// value = contendOptional.get().getId();
// }else {
// value = Constant.PRIMARY_CONTEND_ID;
// }
// }else {
// value = Constant.PRIMARY_CONTEND_ID;
// }
// }
// }
// 标签只包含正负中
if (FieldMap.MTAG == f) {
if (!Arrays.asList("正面", "中性", "负面").contains(String.valueOf(value))) {
return null;
}
}
fieldMap = f;
break;
}
}
if (null == fieldMap) {
return null;
}
return new FieldMapping(fieldMap, value);
}
}
src/main/java/com/zhiwei/brandkbs2/service/MarkDataService.java
......@@ -2,10 +2,7 @@ package com.zhiwei.brandkbs2.service;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.model.ResponseResult;
import com.zhiwei.brandkbs2.pojo.BaseMap;
import com.zhiwei.brandkbs2.pojo.DailyReport;
import com.zhiwei.brandkbs2.pojo.Event;
import com.zhiwei.brandkbs2.pojo.MarkFlowEntity;
import com.zhiwei.brandkbs2.pojo.*;
import com.zhiwei.brandkbs2.pojo.dto.*;
import com.zhiwei.brandkbs2.pojo.vo.LineVO;
import com.zhiwei.brandkbs2.pojo.vo.PageVO;
......@@ -836,4 +833,32 @@ public interface MarkDataService {
* @return
*/
List<String> expandOriginRange(MarkSearchDTO dto);
/**
* AI搜索-推荐提问
* @param question
* @return
*/
List<String> getAIReferenceQuestion(String question, int size);
/**
* AI搜索-参考提问
* @param cache
* @return
*/
List<String> getAIReferenceQuestionCache(boolean cache);
/**
* AI搜索-搜索结果
* @param question
* @return
*/
JSONObject getAISearchResult(String question, String keyword, Long startTime, Long endTime);
/**
* AI搜索-联网搜索
* @param question
* @return
*/
JSONObject getAIOnlineSearchResult(String question);
}
src/main/java/com/zhiwei/brandkbs2/service/TaskService.java
......@@ -72,4 +72,9 @@ public interface TaskService{
* 定时拉取并进行渠道库更新任务
*/
void refreshChannelRecord();
/**
* 生成ai搜索参考提问缓存
*/
void cacheAIQuestion();
}
src/main/java/com/zhiwei/brandkbs2/service/impl/BehaviorServiceImpl.java
......@@ -174,7 +174,7 @@ public class BehaviorServiceImpl implements BehaviorService {
if (null == userInfo) {
return;
}
UserLogRecord userLogRecord = new UserLogRecord(projectId, userInfo.getUserId(), userInfo.getNickname(), description, userInfo.getRoleId(), now, now);
UserLogRecord userLogRecord = new UserLogRecord(projectId, userInfo.getUserId(), userInfo.getNickname(), description, userInfo.getRoleId(), now, now, null);
String collectionName = userLogRecordDao.generateCollectionName();
userLogRecordDao.insertOne(userLogRecord, collectionName);
}
......
src/main/java/com/zhiwei/brandkbs2/service/impl/ProjectWarnServiceImpl.java
......@@ -89,7 +89,7 @@ public class ProjectWarnServiceImpl implements ProjectWarnService {
static {
TYPE_SEARCH.put("微博热搜", "weibo");
TYPE_SEARCH.put("微博话题", "weibo-topic");
// TYPE_SEARCH.put("微博话题", "weibo-topic");
TYPE_SEARCH.put("微博预热", "weibo-rise");
TYPE_SEARCH.put("头条热搜", "toutiao");
TYPE_SEARCH.put("抖音热搜", "douyin");
......@@ -385,7 +385,8 @@ public class ProjectWarnServiceImpl implements ProjectWarnService {
// key2
String key2 = "";
List<String> key2Element = new ArrayList<>();
config.getListType().forEach(type -> {
// 2024/8/19 微博话题榜采集下线，防止微博话题配置历史数据影响
config.getListType().stream().filter(type -> !Objects.equals("微博话题", type)).collect(Collectors.toList()).forEach(type -> {
if (config.getFirstTop()) {
key2Element.add(type + "榜-首次上榜");
}
......
src/main/java/com/zhiwei/brandkbs2/service/impl/TaskServiceImpl.java
......@@ -89,6 +89,9 @@ public class TaskServiceImpl implements TaskService {
@Resource(name = "channelRecordRefreshTaskDao")
private ChannelRecordRefreshTaskDao channelRecordRefreshTaskDao;
@Resource(name = "aiSearchQuestionRecordDao")
private AISearchQuestionRecordDao aiSearchQuestionRecordDao;
@Resource(name = "brandkbsTaskServiceImpl")
BrandkbsTaskService brandkbsTaskService;
......@@ -486,6 +489,21 @@ public class TaskServiceImpl implements TaskService {
log.info("更新渠道库记录完成-taskId:{}", task.getId());
}
@Override
public void cacheAIQuestion() {
AtomicInteger total = new AtomicInteger();
CompletableFuture.allOf(GlobalPojo.PROJECT_MAP.values().stream().map(project -> CompletableFuture.supplyAsync(() -> {
UserInfo userInfo = new UserInfo().setProjectId(project.getId());
userInfo.setUserId("0");
userInfo.setNickname("系统");
userInfo.setRoleId(1);
UserThreadLocal.set(userInfo);
markDataService.getAIReferenceQuestionCache(false);
log.info("项目:{}-{}-AI参考问题缓存完成:{}个", project.getProjectName(), project.getId(), total.incrementAndGet());
return null;
}, cacheServiceExecutor)).toArray(CompletableFuture[]::new)).join();
}
private void updateRefreshTask(String id, String status){
Update update = new Update();
update.set("status", status);
......
src/main/java/com/zhiwei/brandkbs2/task/ControlCenter.java
......@@ -47,6 +47,7 @@ public class ControlCenter {
// taskService.customEventCache();
taskService.eventAggTitleCache();
taskService.yuqingAnalyzeHighWordCache();
taskService.cacheAIQuestion();
} catch (Exception e) {
log.error("定时按天缓存数据-出错", e);
} finally {
......
src/main/java/com/zhiwei/brandkbs2/util/RedisUtil.java
......@@ -130,6 +130,10 @@ public class RedisUtil {
return RedisKeyPrefix.SEARCH_KEYWORD + Tools.concat(projectId, userId, searchType);
}
public static String getAISearchQuestionCacheKey(String projectId){
return RedisKeyPrefix.AI_SEARCH_QUESTION + projectId;
}
public void setExpire(String key, String value, long timeout, TimeUnit unit) {
stringRedisTemplate.opsForValue().set(key, value, timeout, unit);
}
......
src/main/java/com/zhiwei/brandkbs2/util/Tools.java
......@@ -554,10 +554,13 @@ public class Tools {
String separator = "-";
StringBuilder sb = new StringBuilder();
for (Object obj : objects) {
if (Objects.isNull(obj)){
continue;
}
sb.append(obj).append(separator);
}
String resultStr = sb.toString();
return resultStr.substring(0, resultStr.length() - 1);
return StringUtils.isBlank(resultStr) ? resultStr : resultStr.substring(0, resultStr.length() - 1);
}
public static String[] split(String concatStr) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment