Commit d300091b by shenjunjie

Merge branch 'feature' into 'dev'

词云图由hanLp调整为ansj 2

See merge request !141
parents 9928b97f 53c0e739
...@@ -39,13 +39,11 @@ public class MyDic { ...@@ -39,13 +39,11 @@ public class MyDic {
public void init() { public void init() {
try { try {
// 读取词典 // 读取词典
List<String> customDics = Tools.readListFile(customDic.getInputStream()); customWords = Tools.readListFile(customDic.getInputStream());
List<String> stopDics = Tools.readListFile(stopDic.getInputStream()); stopWords = Tools.readListFile(stopDic.getInputStream());
InputStream inputStream = negativeDic.getInputStream(); negativeWords = Tools.readListFile(negativeDic.getInputStream());
List<String> negativeDic = Tools.readListFile(inputStream); positiveWords= Tools.readListFile(positiveDic.getInputStream());
InputStream inputStream2 = positiveDic.getInputStream(); log.info("ansj自定义词典加载:{}条,停用词加载:{}条,负面词典加载:{}条,正面词典加载:{}条", customWords.size(), stopWords.size(), negativeWords.size(), positiveWords.size());
List<String> positiveDic = Tools.readListFile(inputStream2);
log.info("ansj自定义词典加载:{}条,停用词加载:{}条,负面词典加载:{}条,正面词典加载:{}条", customDics.size(), stopDics.size(), negativeDic.size(), positiveDic.size());
} catch (Exception e) { } catch (Exception e) {
log.info("MyDic-init 异常", e); log.info("MyDic-init 异常", e);
} }
...@@ -78,8 +76,8 @@ public class MyDic { ...@@ -78,8 +76,8 @@ public class MyDic {
* @Description: TODO(获取正面词) * @Description: TODO(获取正面词)
* 设定文件 * 设定文件
*/ */
public static List<String> getPosivtiveWords() { public static List<String> getPositiveWords() {
return negativeWords; return positiveWords;
} }
/** /**
......
...@@ -33,6 +33,7 @@ import org.apache.logging.log4j.LogManager; ...@@ -33,6 +33,7 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
import org.joda.time.Period; import org.joda.time.Period;
import org.joda.time.PeriodType; import org.joda.time.PeriodType;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Query;
import org.springframework.data.mongodb.core.query.Update; import org.springframework.data.mongodb.core.query.Update;
...@@ -81,6 +82,9 @@ public class CustomEventServiceImpl implements CustomEventService { ...@@ -81,6 +82,9 @@ public class CustomEventServiceImpl implements CustomEventService {
@Resource @Resource
MongoUtil mongoUtil; MongoUtil mongoUtil;
@Autowired
TextUtil textUtil;
@Override @Override
public List<JSONObject> getCustomEventRankList(Long startTime, Long endTime) { public List<JSONObject> getCustomEventRankList(Long startTime, Long endTime) {
List<JSONObject> resultList = null; List<JSONObject> resultList = null;
...@@ -375,7 +379,7 @@ public class CustomEventServiceImpl implements CustomEventService { ...@@ -375,7 +379,7 @@ public class CustomEventServiceImpl implements CustomEventService {
private List<JSONObject> getHotKeyword(List<BaseMap> articleList) { private List<JSONObject> getHotKeyword(List<BaseMap> articleList) {
List<String> texts = articleList.stream().map(article -> article.getTitle() + article.getContent()).collect(Collectors.toList()); List<String> texts = articleList.stream().map(article -> article.getTitle() + article.getContent()).collect(Collectors.toList());
//分析热评词 //分析热评词
return TextUtil.getHighWordsJson(texts, 30); return textUtil.getHighWordsJson(texts, 30);
} }
/** /**
......
...@@ -137,6 +137,9 @@ public class MarkDataServiceImpl implements MarkDataService { ...@@ -137,6 +137,9 @@ public class MarkDataServiceImpl implements MarkDataService {
@Resource(name = "mongoUtil") @Resource(name = "mongoUtil")
MongoUtil mongoUtil; MongoUtil mongoUtil;
@Autowired
TextUtil textUtil;
@Override @Override
public PageVO<MarkFlowEntity> getYuqingMarkList(MarkSearchDTO markSearchDTO) { public PageVO<MarkFlowEntity> getYuqingMarkList(MarkSearchDTO markSearchDTO) {
try { try {
...@@ -504,12 +507,13 @@ public class MarkDataServiceImpl implements MarkDataService { ...@@ -504,12 +507,13 @@ public class MarkDataServiceImpl implements MarkDataService {
log.info("es查询size:{},耗时:{}", texts.size(), System.currentTimeMillis() - s); log.info("es查询size:{},耗时:{}", texts.size(), System.currentTimeMillis() - s);
long s1 = System.currentTimeMillis(); long s1 = System.currentTimeMillis();
// 分析高频词 // 分析高频词
List<JSONObject> highWords = TextUtil.getHighWordsJson(texts, 30); List<JSONObject> highWords = textUtil.getHighWordsJson(texts, 30);
log.info("分析高频词耗时:{}", (System.currentTimeMillis() - s1)); log.info("分析高频词耗时:{}", (System.currentTimeMillis() - s1));
redisUtil.setExpire(redisKey, JSON.toJSONString(highWords)); redisUtil.setExpire(redisKey, JSON.toJSONString(highWords));
return highWords; return highWords;
} catch (IOException e) { } catch (IOException e) {
ExceptionCast.cast(CommonCodeEnum.FAIL, "es查询异常", e); log.error("getMarkHighWord-",e);
ExceptionCast.cast(CommonCodeEnum.FAIL);
} }
return null; return null;
} }
......
...@@ -15,6 +15,7 @@ import com.zhiwei.nlp.vo.KResult; ...@@ -15,6 +15,7 @@ import com.zhiwei.nlp.vo.KResult;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHit;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.Resource; import org.springframework.core.io.Resource;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
...@@ -39,7 +40,8 @@ public class TextUtil { ...@@ -39,7 +40,8 @@ public class TextUtil {
private Resource customDic; private Resource customDic;
@Value("classpath:wordDic/stopWordDictionary.txt") @Value("classpath:wordDic/stopWordDictionary.txt")
private Resource stopDic; private Resource stopDic;
private static AnsjSeg ansjSeg; @Autowired
private AnsjSeg ansjSeg;
@PostConstruct @PostConstruct
public void init() { public void init() {
...@@ -105,7 +107,7 @@ public class TextUtil { ...@@ -105,7 +107,7 @@ public class TextUtil {
return Tools.sortMap(result, maxSize); return Tools.sortMap(result, maxSize);
} }
public static List<JSONObject> getHighWordsJson(List<String> texts, Integer maxSize) { public List<JSONObject> getHighWordsJson(List<String> texts, Integer maxSize) {
// Map<String, Integer> highWords = getHighWords(texts, maxSize); // Map<String, Integer> highWords = getHighWords(texts, maxSize);
List<Map.Entry<String, Integer>> wordRate = ansjSeg.getFenCi(texts, maxSize); List<Map.Entry<String, Integer>> wordRate = ansjSeg.getFenCi(texts, maxSize);
List<JSONObject> result = new ArrayList<>(wordRate.size()); List<JSONObject> result = new ArrayList<>(wordRate.size());
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment