Skip to content

M
messageflow
Commit 3594cf63 by win7
Options

监测系统消息流模块

parents
Showing with 9929 additions and 0 deletions
.gitignore 0 → 100644
target/
!.mvn/wrapper/maven-wrapper.jar
### STS ###
.apt_generated
.classpath
.factorypath
.project
.settings
.springBeans
.sts4-cache
### IntelliJ IDEA ###
.idea
*.iws
*.iml
*.ipr
### NetBeans ###
nbproject/private/
build/
nbbuild/
dist/
nbdist/
.nb-gradle/
\ No newline at end of file
.mvn/wrapper/maven-wrapper.properties 0 → 100644
distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.5.3/apache-maven-3.5.3-bin.zip
mvnw 0 → 100644
#!/bin/sh
# ----------------------------------------------------------------------------
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# ----------------------------------------------------------------------------
# ----------------------------------------------------------------------------
# Maven2 Start Up Batch script
#
# Required ENV vars:
# ------------------
# JAVA_HOME - location of a JDK home dir
#
# Optional ENV vars
# -----------------
# M2_HOME - location of maven2's installed home dir
# MAVEN_OPTS - parameters passed to the Java VM when running Maven
# e.g. to debug Maven itself, use
# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
# MAVEN_SKIP_RC - flag to disable loading of mavenrc files
# ----------------------------------------------------------------------------
if [ -z "$MAVEN_SKIP_RC" ] ; then
if [ -f /etc/mavenrc ] ; then
. /etc/mavenrc
fi
if [ -f "$HOME/.mavenrc" ] ; then
. "$HOME/.mavenrc"
fi
fi
# OS specific support. $var _must_ be set to either true or false.
cygwin=false;
darwin=false;
mingw=false
case "`uname`" in
CYGWIN*) cygwin=true ;;
MINGW*) mingw=true;;
Darwin*) darwin=true
# Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home
# See https://developer.apple.com/library/mac/qa/qa1170/_index.html
if [ -z "$JAVA_HOME" ]; then
if [ -x "/usr/libexec/java_home" ]; then
export JAVA_HOME="`/usr/libexec/java_home`"
else
export JAVA_HOME="/Library/Java/Home"
fi
fi
;;
esac
if [ -z "$JAVA_HOME" ] ; then
if [ -r /etc/gentoo-release ] ; then
JAVA_HOME=`java-config --jre-home`
fi
fi
if [ -z "$M2_HOME" ] ; then
## resolve links - $0 may be a link to maven's home
PRG="$0"
# need this for relative symlinks
while [ -h "$PRG" ] ; do
ls=`ls -ld "$PRG"`
link=`expr "$ls" : '.*-> \(.*\)$'`
if expr "$link" : '/.*' > /dev/null; then
PRG="$link"
else
PRG="`dirname "$PRG"`/$link"
fi
done
saveddir=`pwd`
M2_HOME=`dirname "$PRG"`/..
# make it fully qualified
M2_HOME=`cd "$M2_HOME" && pwd`
cd "$saveddir"
# echo Using m2 at $M2_HOME
fi
# For Cygwin, ensure paths are in UNIX format before anything is touched
if $cygwin ; then
[ -n "$M2_HOME" ] &&
M2_HOME=`cygpath --unix "$M2_HOME"`
[ -n "$JAVA_HOME" ] &&
JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
[ -n "$CLASSPATH" ] &&
CLASSPATH=`cygpath --path --unix "$CLASSPATH"`
fi
# For Migwn, ensure paths are in UNIX format before anything is touched
if $mingw ; then
[ -n "$M2_HOME" ] &&
M2_HOME="`(cd "$M2_HOME"; pwd)`"
[ -n "$JAVA_HOME" ] &&
JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`"
# TODO classpath?
fi
if [ -z "$JAVA_HOME" ]; then
javaExecutable="`which javac`"
if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then
# readlink(1) is not available as standard on Solaris 10.
readLink=`which readlink`
if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then
if $darwin ; then
javaHome="`dirname \"$javaExecutable\"`"
javaExecutable="`cd \"$javaHome\" && pwd -P`/javac"
else
javaExecutable="`readlink -f \"$javaExecutable\"`"
fi
javaHome="`dirname \"$javaExecutable\"`"
javaHome=`expr "$javaHome" : '\(.*\)/bin'`
JAVA_HOME="$javaHome"
export JAVA_HOME
fi
fi
fi
if [ -z "$JAVACMD" ] ; then
if [ -n "$JAVA_HOME" ] ; then
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
# IBM's JDK on AIX uses strange locations for the executables
JAVACMD="$JAVA_HOME/jre/sh/java"
else
JAVACMD="$JAVA_HOME/bin/java"
fi
else
JAVACMD="`which java`"
fi
fi
if [ ! -x "$JAVACMD" ] ; then
echo "Error: JAVA_HOME is not defined correctly." >&2
echo " We cannot execute $JAVACMD" >&2
exit 1
fi
if [ -z "$JAVA_HOME" ] ; then
echo "Warning: JAVA_HOME environment variable is not set."
fi
CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher
# traverses directory structure from process work directory to filesystem root
# first directory with .mvn subdirectory is considered project base directory
find_maven_basedir() {
if [ -z "$1" ]
then
echo "Path not specified to find_maven_basedir"
return 1
fi
basedir="$1"
wdir="$1"
while [ "$wdir" != '/' ] ; do
if [ -d "$wdir"/.mvn ] ; then
basedir=$wdir
break
fi
# workaround for JBEAP-8937 (on Solaris 10/Sparc)
if [ -d "${wdir}" ]; then
wdir=`cd "$wdir/.."; pwd`
fi
# end of workaround
done
echo "${basedir}"
}
# concatenates all lines of a file
concat_lines() {
if [ -f "$1" ]; then
echo "$(tr -s '\n' ' ' < "$1")"
fi
}
BASE_DIR=`find_maven_basedir "$(pwd)"`
if [ -z "$BASE_DIR" ]; then
exit 1;
fi
export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"}
echo $MAVEN_PROJECTBASEDIR
MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS"
# For Cygwin, switch paths to Windows format before running java
if $cygwin; then
[ -n "$M2_HOME" ] &&
M2_HOME=`cygpath --path --windows "$M2_HOME"`
[ -n "$JAVA_HOME" ] &&
JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"`
[ -n "$CLASSPATH" ] &&
CLASSPATH=`cygpath --path --windows "$CLASSPATH"`
[ -n "$MAVEN_PROJECTBASEDIR" ] &&
MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"`
fi
WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
exec "$JAVACMD" \
$MAVEN_OPTS \
-classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \
"-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \
${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@"
mvnw.cmd 0 → 100644
@REM ----------------------------------------------------------------------------
@REM Licensed to the Apache Software Foundation (ASF) under one
@REM or more contributor license agreements. See the NOTICE file
@REM distributed with this work for additional information
@REM regarding copyright ownership. The ASF licenses this file
@REM to you under the Apache License, Version 2.0 (the
@REM "License"); you may not use this file except in compliance
@REM with the License. You may obtain a copy of the License at
@REM
@REM http://www.apache.org/licenses/LICENSE-2.0
@REM
@REM Unless required by applicable law or agreed to in writing,
@REM software distributed under the License is distributed on an
@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@REM KIND, either express or implied. See the License for the
@REM specific language governing permissions and limitations
@REM under the License.
@REM ----------------------------------------------------------------------------
@REM ----------------------------------------------------------------------------
@REM Maven2 Start Up Batch script
@REM
@REM Required ENV vars:
@REM JAVA_HOME - location of a JDK home dir
@REM
@REM Optional ENV vars
@REM M2_HOME - location of maven2's installed home dir
@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands
@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending
@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven
@REM e.g. to debug Maven itself, use
@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files
@REM ----------------------------------------------------------------------------
@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on'
@echo off
@REM enable echoing my setting MAVEN_BATCH_ECHO to 'on'
@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO%
@REM set %HOME% to equivalent of $HOME
if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%")
@REM Execute a user defined script before this one
if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre
@REM check for pre script, once with legacy .bat ending and once with .cmd ending
if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat"
if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd"
:skipRcPre
@setlocal
set ERROR_CODE=0
@REM To isolate internal variables from possible post scripts, we use another setlocal
@setlocal
@REM ==== START VALIDATION ====
if not "%JAVA_HOME%" == "" goto OkJHome
echo.
echo Error: JAVA_HOME not found in your environment. >&2
echo Please set the JAVA_HOME variable in your environment to match the >&2
echo location of your Java installation. >&2
echo.
goto error
:OkJHome
if exist "%JAVA_HOME%\bin\java.exe" goto init
echo.
echo Error: JAVA_HOME is set to an invalid directory. >&2
echo JAVA_HOME = "%JAVA_HOME%" >&2
echo Please set the JAVA_HOME variable in your environment to match the >&2
echo location of your Java installation. >&2
echo.
goto error
@REM ==== END VALIDATION ====
:init
@REM Find the project base dir, i.e. the directory that contains the folder ".mvn".
@REM Fallback to current working directory if not found.
set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR%
IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir
set EXEC_DIR=%CD%
set WDIR=%EXEC_DIR%
:findBaseDir
IF EXIST "%WDIR%"\.mvn goto baseDirFound
cd ..
IF "%WDIR%"=="%CD%" goto baseDirNotFound
set WDIR=%CD%
goto findBaseDir
:baseDirFound
set MAVEN_PROJECTBASEDIR=%WDIR%
cd "%EXEC_DIR%"
goto endDetectBaseDir
:baseDirNotFound
set MAVEN_PROJECTBASEDIR=%EXEC_DIR%
cd "%EXEC_DIR%"
:endDetectBaseDir
IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig
@setlocal EnableExtensions EnableDelayedExpansion
for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a
@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS%
:endReadAdditionalConfig
SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe"
set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar"
set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %*
if ERRORLEVEL 1 goto error
goto end
:error
set ERROR_CODE=1
:end
@endlocal & set ERROR_CODE=%ERROR_CODE%
if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost
@REM check for post script, once with legacy .bat ending and once with .cmd ending
if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat"
if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd"
:skipRcPost
@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on'
if "%MAVEN_BATCH_PAUSE%" == "on" pause
if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE%
exit /B %ERROR_CODE%
pom.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.zhiwei.messageflow</groupId>
<artifactId>messageflow</artifactId>
<version>0.0.1-SNAPSHOT</version>
<packaging>jar</packaging>
<name>messageflow</name>
<description>Demo project for Spring Boot</description>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.0.0.RELEASE</version>
<relativePath /> <!-- lookup parent from repository -->
</parent>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<java.version>1.8</java.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-mongodb</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-redis</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<!-- es -->
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>transport</artifactId>
<version>5.2.2</version><!--$NO-MVN-MAN-VER$ -->
</dependency>
<dependency>
<groupId>com.zhiwei</groupId>
<artifactId>es-client</artifactId>
<version>1.2</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>2.7</version><!--$NO-MVN-MAN-VER$ -->
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>2.7</version><!--$NO-MVN-MAN-VER$ -->
</dependency>
<!-- jackson -->
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
<!-- net.sf.json -->
<dependency>
<groupId>net.sf.json-lib</groupId>
<artifactId>json-lib</artifactId>
<version>2.4</version>
<classifier>jdk15</classifier>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-configuration-processor</artifactId>
<optional>true</optional>
</dependency>
<!-- Jedis -->
<dependency>
<groupId>redis.clients</groupId>
<artifactId>jedis</artifactId>
</dependency>
<!-- lombok -->
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
</dependency>
<!-- POI -->
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml</artifactId>
<version>3.9</version>
</dependency>
<!-- mongodb-plus -->
<dependency>
<groupId>com.spring4all</groupId>
<artifactId>mongodb-plus-spring-boot-starter</artifactId>
<version>1.0.0.RELEASE</version>
</dependency>
<!-- mail-send -->
<dependency>
<groupId>javax.mail</groupId>
<artifactId>mail</artifactId>
<version>1.4.7</version>
</dependency>
<!-- solr -->
<dependency>
<groupId>org.apache.solr</groupId>
<artifactId>solr-solrj</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>
src/main/java/com/zhiwei/messageflow/ES4RedisRunner.java 0 → 100644
package com.zhiwei.messageflow;
import java.io.IOException;
import java.util.Timer;
import java.util.TimerTask;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.ApplicationArguments;
import org.springframework.boot.ApplicationRunner;
import org.springframework.stereotype.Component;
import com.zhiwei.messageflow.listener.ApplicationContextProvider;
/**
* 消息流模块入口
*/
@Component
public class ES4RedisRunner implements ApplicationRunner {
public final static Logger log = LoggerFactory.getLogger(ES4RedisRunner.class);
@Override
public void run(ApplicationArguments args) throws Exception {
init();
}
public void init() {
try {
// /**
// * redis消息获取测试方法
// */
// getSet gs = ApplicationContextProvider.getBean("getSet", getSet.class);
// gs.getDataByeRedis("", "", 0, 0, 0);
// 手动注入bean ES4RedisStart
ES4RedisStart start = ApplicationContextProvider.getBean("ES4RedisStart", ES4RedisStart.class);
// 定时器 项目启动后100ms运行 每次间隔8min
Timer timer = new Timer();
timer.schedule(
new TimerTask() {
public void run() {
try {
start.startThread();
} catch (IOException | InterruptedException e) {
e.printStackTrace();
}
}
}, 100L, 30 * 1000L);
// /**
// * ES库消息输出Excel并分析关键词重复数据
// */
// GetMessage gm = ApplicationContextProvider.getBean("getMessage",
// GetMessage.class);
// gm.getSame();
// gm.getmessage();
} catch (Exception e) {
e.printStackTrace();
}
}
}
\ No newline at end of file
src/main/java/com/zhiwei/messageflow/ES4RedisStart.java 0 → 100644
package com.zhiwei.messageflow;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.zhiwei.messageflow.mongo.bean.Project;
import com.zhiwei.messageflow.mongo.dao.PlatformDao;
import com.zhiwei.messageflow.mongo.dao.ProjectDao;
@Component
public class ES4RedisStart {
private final static Logger log = LoggerFactory.getLogger(ES4RedisStart.class);
@Autowired
private ProjectDao projectDao;
@Autowired
private PlatformDao platformDao;
@Autowired
private ES4RedisTask es4RedisTask;
public void startThread() throws JsonParseException, JsonMappingException, IOException, InterruptedException {
// 项目列表
List<Project> projects = projectDao.getAllProjects();
// 公共平台名列表
List<String> platformNames = platformDao.getPublicPlatformName();
// 遍历项目
for (Project project : projects) {
/**
* 项目全部平台(公共+私有)
*/
List<String> allplatformNames = new ArrayList<>();
allplatformNames.addAll(platformNames);
allplatformNames.addAll(project.getDataPt());
// 获取线程
ES4RedisThread es4RedisThread = ES4RedisThread.getThread(project.getProjectName(), project,
allplatformNames, es4RedisTask);
// 获取线程失败
if (es4RedisThread == null) {
log.warn("{}项目获取线程失败", project.getProjectName());
continue;
}
// 线程启动
es4RedisThread.start();
}
}
}
src/main/java/com/zhiwei/messageflow/ES4RedisTask.java 0 → 100644
package com.zhiwei.messageflow;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.zhiwei.messageflow.bean.MediaMessage;
import com.zhiwei.messageflow.bean.VideoMessage;
import com.zhiwei.messageflow.bean.WeiboMessage;
import com.zhiwei.messageflow.bean.ZhihuMessage;
import com.zhiwei.messageflow.es.service.EarlyWarningService;
import com.zhiwei.messageflow.mongo.bean.KeywordNew;
import com.zhiwei.messageflow.mongo.bean.NoiseRule;
import com.zhiwei.messageflow.mongo.bean.Project;
import com.zhiwei.messageflow.mongo.bean.TrackRule;
import com.zhiwei.messageflow.mongo.dao.KeywordNewDao;
import com.zhiwei.messageflow.mongo.dao.NoiseRuleDao;
import com.zhiwei.messageflow.mongo.dao.TrackRuleDao;
import com.zhiwei.messageflow.redis.bean.RsidAndMessages;
import com.zhiwei.messageflow.redis.service.RedisService;
import com.zhiwei.messageflow.service.DisposeMessageService;
@Component
public class ES4RedisTask {
private final static Logger log = LoggerFactory.getLogger(ES4RedisTask.class);
@Autowired
private DisposeMessageService disposeMessageService;
@Autowired
private EarlyWarningService earlyWarningService;
@Autowired
private KeywordNewDao keywordNewDao;
@Autowired
private NoiseRuleDao noiseRuleDao;
@Autowired
private TrackRuleDao trackRuleDao;
@Autowired
private RedisService redisService;
@Autowired
private ObjectMapper mapper;
private static final int allkeywordcount = 5000;
private static final int keywordscount = 2000;
@SuppressWarnings("unchecked")
public boolean ES4Redis(Project project, List<String> platformNames, int count)
throws JsonParseException, JsonMappingException, IOException {
try {
/**
* 获取项目对应的rsid Map
*/
String rsidjson = redisService.getRsid(project.getProjectName());
Map<String, Integer> rsidMap = new HashMap<>();
if (rsidjson != null) {
rsidMap = mapper.readValue(rsidjson, Map.class);
} else {
log.info("{}项目RSID列表过期", project.getProjectName());
}
// 更新rsid用的rsid Map
Map<String, Integer> newRsidMap = new HashMap<>();
newRsidMap.putAll(rsidMap);
// 统计项目消息总数
int num = 0;
// 项目关键词组列表
List<KeywordNew> keywordNews = keywordNewDao.getKeywordNewByProject(project.getProjectName());
// 项目噪音规则列表
List<NoiseRule> noiseRules = noiseRuleDao.getNoiseRuleByProject(project.getProjectName());
// 项目预警规则列表
List<TrackRule> trackRules = trackRuleDao.getTrackRuleByProject(project.getProjectName());
/*
* //测试预警规则 List<TrackRule> trackRules=new ArrayList<>(); TrackRule tr1=new
* TrackRule(); TrackRule tr2=new TrackRule(); TrackRule tr3=new TrackRule();
* Date date = new Date(); Long createAt=date.getTime();
* tr1.setCreateAt(createAt); tr1.setAndOr(""); tr1.setWarn(false);
* tr1.setRuleType("keyWords"); tr1.setSubmitter("虞诚毅");
* tr1.setKeyWordsInputOne("知乎"); tr1.setRuleName("测试规则1");
* tr1.setRuleExplain("测试说明1"); tr1.setEarlyWarningTime("三天");
* tr1.setProject("腾讯"); tr1.setEarlyWarning("email");
* tr2.setCreateAt(createAt); tr2.setAndOr("或"); tr2.setWarn(false);
* tr2.setRuleType("keyWords"); tr2.setSubmitter("虞诚毅");
* tr2.setKeyWordsInputOne("知乎"); tr2.setKeyWordsInputTwo("微博");
* tr2.setRuleName("测试规则2"); tr2.setRuleExplain("测试说明3");
* tr2.setEarlyWarningTime("三天"); tr2.setProject("腾讯");
* tr2.setEarlyWarning("email"); tr3.setCreateAt(createAt); tr3.setAndOr("且");
* tr3.setWarn(false); tr3.setRuleType("keyWords"); tr3.setSubmitter("虞诚毅");
* tr3.setKeyWordsInputOne("知乎"); tr3.setKeyWordsInputTwo("问题");
* tr3.setRuleName("测试规则2"); tr3.setRuleExplain("测试说明3");
* tr3.setEarlyWarningTime("三天"); tr3.setProject("腾讯");
* tr3.setEarlyWarning("wechat"); trackRules.add(tr1); trackRules.add(tr2);
* trackRules.add(tr3);
*/
// 遍历平台
for (String platformName : platformNames) {
/**
* 平台全关键词查询
*/
String allkeytitle = "全部";
// 全关键词redis库中的key
String allRedisKey = platformName + "-" + project.getProjectName() + "-" + allkeytitle;
// 获取 平台-项目-关键词组 对应的rsid作为查询范围的起始点
long allstartrsid = rsidMap.get(allRedisKey) == null ? -1L : Long.valueOf(rsidMap.get(allRedisKey));
// 用于存储数据获取后新的rsid
Long allrsid = -1L;
/**
* 将项目下的所有关键词组合成全关键词组
*/
List<String> allkeywords = new ArrayList<>();
for (KeywordNew kwn : keywordNews) {
if (null == kwn.getKeyWords() || kwn.getKeyWords().isEmpty()) {
continue;
}
allkeywords.addAll(kwn.getKeyWords());
}
// 根据不同平台获取数据
if (platformName.equals("微博")) {
// 获取新的rsid和信息实体
RsidAndMessages ram = disposeMessageService.getFilteredWeiboMessage(trackRules, noiseRules,
allkeywords, count, allstartrsid, -1L, platformName, project.getProjectName());
// 获取查询到的信息
List<WeiboMessage> messages = ram.getWlist();
// 数据量为0
if (messages == null) {
// log.info("{}平台{}关键字词组无消息", platformName, allkeytitle);
continue;
}
// log.info("{}平台{}关键词数据获取{}条", platformName, "全部", messages.size());
// 记录新的rsid
allrsid = ram.getRsid();
num += messages.size();
// 向redis写入数据
redisService.setWeiboMessageMessage(allRedisKey, messages, allkeywordcount);
} else if (platformName.equals("知乎")) {
// 获取新的rsid和信息实体
RsidAndMessages ram = disposeMessageService.getFilteredZhihuMessage(trackRules, noiseRules,
allkeywords, count, allstartrsid, -1L, platformName, project.getProjectName());
// 获取查询到的信息
List<ZhihuMessage> messages = ram.getZlist();
// 查询到数据量为0
if (messages == null) {
// log.info("{}平台{}关键字词组无消息", platformName, allkeytitle);
continue;
}
// log.info("{}平台{}关键词数据获取{}条", platformName, "全部", messages.size());
// 记录新的rsid
allrsid = ram.getRsid();
num += messages.size();
// 向redis写入数据
redisService.setZhihuMessageMessage(allRedisKey, messages, allkeywordcount);
} else if (platformName.equals("视频")) {
// 获取新的rsid和信息实体
RsidAndMessages ram = disposeMessageService.getFilteredVideoMessage(trackRules, noiseRules,
allkeywords, count, allstartrsid, -1L, platformName, project.getProjectName());
// 获取查询到的信息
List<VideoMessage> messages = ram.getVlist();
// 查询到数据量为0
if (messages == null) {
// log.info("{}平台{}关键字词组无消息", platformName, allkeytitle);
continue;
}
// log.info("{}平台{}关键词数据获取{}条", platformName, "全部", messages.size());
// 记录新的rsid
allrsid = ram.getRsid();
num += messages.size();
// 向redis写入数据
redisService.setVideoMessageMessage(allRedisKey, messages, allkeywordcount);
} else {
// 获取新的rsid和信息实体
RsidAndMessages ram = disposeMessageService.getFilteredMediaMessage(trackRules, noiseRules,
allkeywords, count, allstartrsid, -1L, platformName, project.getProjectName());
// 获取查询到的信息
List<MediaMessage> messages = ram.getMlist();
// 查询到数据量为0
if (messages == null) {
// log.info("{}平台{}关键字词组无消息", platformName, allkeytitle);
continue;
}
// log.info("{}平台{}关键词数据获取{}条", platformName, "全部", messages.size());
// 记录新的rsid
allrsid = ram.getRsid();
num += messages.size();
// 向redis写入数据
redisService.setMediaMessageMessage(allRedisKey, messages, allkeywordcount);
}
newRsidMap.put(allRedisKey, Integer.valueOf(allrsid.toString()));
// 遍历关键词组
for (KeywordNew kwn : keywordNews) {
// 滤过空关键词组
if (null == kwn.getKeyWords() || kwn.getKeyWords().isEmpty()) {
continue;
}
// 关键词组在redis库中的key
String redisKey = platformName + "-" + project.getProjectName() + "-" + kwn.getKeyTitle();
// 获取 平台-项目-关键词组 对应的rsid作为查询范围的起始点
Long startrsid = rsidMap.get(redisKey) == null ? -1L : Long.valueOf(rsidMap.get(redisKey));
// 用于存储数据获取后新的rsid
Long keyrsid = -1L;
// System.out.println(kwn.toString());
// System.out.println(l++ + ":" + platformName);
// System.out.println("=============这是分界线=============");
// 根据平台查询数据
if (platformName.equals("微博")) {
// 获取新的rsid和信息实体
RsidAndMessages ram = disposeMessageService.getFilteredWeiboMessage(trackRules, noiseRules,
kwn.getKeyWords(), count, startrsid, -1L, platformName, project.getProjectName());
// 获取查询到的信息
List<WeiboMessage> messages = ram.getWlist();
// 查询到数据量为0
if (messages == null) {
newRsidMap.put(redisKey, Integer.valueOf(startrsid.toString()));
// log.info("{}平台{}关键字词组无消息", platformName, kwn.getKeyTitle());
continue;
}
// log.info("{}平台{}关键词数据获取{}条", platformName, kwn.getKeyTitle(),
// messages.size());
num += messages.size();
// 记录新的rsid
keyrsid = ram.getRsid();
// 向redis写入数据
redisService.setWeiboMessageMessage(redisKey, messages, keywordscount);
} else if (platformName.equals("知乎")) {
// 获取新的rsid和信息实体
RsidAndMessages ram = disposeMessageService.getFilteredZhihuMessage(trackRules, noiseRules,
kwn.getKeyWords(), count, startrsid, -1L, platformName, project.getProjectName());
// 获取查询到的信息
List<ZhihuMessage> messages = ram.getZlist();
// 查询到数据量为0
if (messages == null) {
newRsidMap.put(redisKey, Integer.valueOf(startrsid.toString()));
// log.info("{}平台{}关键字词组无消息", platformName, kwn.getKeyTitle());
continue;
}
// log.info("{}平台{}关键词数据获取{}条", platformName, kwn.getKeyTitle(),
// messages.size());
num += messages.size();
// 记录新的rsid
keyrsid = ram.getRsid();
// 向redis写入数据
redisService.setZhihuMessageMessage(redisKey, messages, keywordscount);
} else if (platformName.equals("视频")) {
// 获取新的rsid和信息实体
RsidAndMessages ram = disposeMessageService.getFilteredVideoMessage(trackRules, noiseRules,
kwn.getKeyWords(), count, startrsid, -1L, platformName, project.getProjectName());
// 获取查询到的信息
List<VideoMessage> messages = ram.getVlist();
// 查询到数据量为0
if (messages == null) {
newRsidMap.put(redisKey, Integer.valueOf(startrsid.toString()));
// log.info("{}平台{}关键字词组无消息", platformName, kwn.getKeyTitle());
continue;
}
// log.info("{}平台{}关键词数据获取{}条", platformName, kwn.getKeyTitle(),
// messages.size());
num += messages.size();
// 记录新的rsid
keyrsid = ram.getRsid();
// 向redis写入数据
redisService.setVideoMessageMessage(redisKey, messages, keywordscount);
} else {
// 获取新的rsid和信息实体
RsidAndMessages ram = disposeMessageService.getFilteredMediaMessage(trackRules, noiseRules,
kwn.getKeyWords(), count, startrsid, -1L, platformName, project.getProjectName());
// 获取查询到的信息
List<MediaMessage> messages = ram.getMlist();
// 查询到数据量为0
if (messages == null) {
newRsidMap.put(redisKey, Integer.valueOf(startrsid.toString()));
// log.info("{}平台{}关键字词组无消息", platformName, kwn.getKeyTitle());
continue;
}
// log.info("{}平台{}关键词数据获取{}条", platformName, kwn.getKeyTitle(),
// messages.size());
num += messages.size();
// 记录新的rsid
keyrsid = ram.getRsid();
// 向redis写入数据
redisService.setMediaMessageMessage(redisKey, messages, keywordscount);
} // 数据写入完成
// 预警
earlyWarningService.earlyWarning(trackRules, count, startrsid, keyrsid, platformName,
project.getProjectName());
// 向更新rsid用的rsid Map中存入rsid
newRsidMap.put(redisKey, Integer.valueOf(keyrsid.toString()));
} // 遍历关键词组
// 向redis库中存储新的rsid Map,覆盖原有数据
redisService.setRsid(newRsidMap, project.getProjectName());
} // 遍历平台
log.info("{}项目本次获取消息数:{}", project.getProjectName(), num);
} catch (Exception e) {
log.error("{} {}", e.getMessage(), e.getStackTrace());
e.printStackTrace();
return false;
}
return true;
}
}
\ No newline at end of file
src/main/java/com/zhiwei/messageflow/ES4RedisThread.java 0 → 100644
package com.zhiwei.messageflow;
import java.io.IOException;
import java.util.List;
import java.util.Timer;
import java.util.TimerTask;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.zhiwei.messageflow.mongo.bean.Project;
public class ES4RedisThread extends Thread {
private final static Logger log = LoggerFactory.getLogger(ES4RedisThread.class);
// 线程
private Thread t;
// 线程名
private String threadName;
// 项目
private Project project;
// 平台列表
private List<String> platformNames;
private ES4RedisTask es4RedisTask;
// 单个平台单个关键词组每次查询数量
private static final int count = 300;
// private static final int max_Thread_num = 40;
// private static int Thread_num = 0;
// private static final int max_Running_num = 3;
// private static Integer Running_num = 0;
// private static List<String> ThreadList = new ArrayList<>();
public ES4RedisThread(String name, Project project, List<String> platformNames, ES4RedisTask es4RedisTask) {
threadName = name;
this.project = project;
this.platformNames = platformNames;
this.es4RedisTask = es4RedisTask;
}
public ES4RedisThread() {
}
public static ES4RedisThread getThread(String name, Project project, List<String> platformNames,
ES4RedisTask es4RedisTask) {
ES4RedisThread es4RedisThread = new ES4RedisThread(name, project, platformNames, es4RedisTask);
return es4RedisThread;
}
public void start() {
// 线程开始
log.info("Starting {}", threadName);
if (t == null) {
t = new Thread(this, threadName);
// 通知执行run方法
t.start();
}
}
@SuppressWarnings("static-access")
@Override
public void run() {
try {
Thread.sleep(10L);
// 超时控制器
Timer timer = new Timer();
timer.schedule(new TimerTask() {
@Override
public void run() {
// 超时则线程中止
Thread.currentThread().interrupted();
log.warn("{}项目超时", project.getProjectName());
}
}, 29 * 1000L);
// 程序运行
log.info("Running {}", threadName);
// 该项目执行消息流获取
boolean flag = es4RedisTask.ES4Redis(project, platformNames, count);
if (!flag) {
// 程序执行出现异常则线程中止
timer.cancel();
Thread.currentThread().interrupted();
log.error("{}项目出现异常", project.getProjectName());
} else
// 程序正常执行完毕,关闭超时控制器
timer.cancel();
} catch (JsonParseException e) {
e.printStackTrace();
} catch (JsonMappingException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
} finally {
// 线程退出
log.info("Thread {} exiting.", threadName);
}
}
}
\ No newline at end of file
src/main/java/com/zhiwei/messageflow/MessageflowApplication.java 0 → 100644
package com.zhiwei.messageflow;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication
public class MessageflowApplication {
public static void main(String[] args) {
SpringApplication.run(MessageflowApplication.class, args);
}
}
src/main/java/com/zhiwei/messageflow/bean/MediaMessage.java 0 → 100644
package com.zhiwei.messageflow.bean;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.ToString;
@Data
@ToString
@EqualsAndHashCode(callSuper = false)
public class MediaMessage {
private String _id;
private String pt;
private Long rsid;
private String id;
private String source;
private String time;
private String title;
private String type;
private String content;
private String savetime;
private String spyderInfoId;
private String column;
private String url;
private String rootSource;
private String userid;
private String name;
private String company;
private String position;
private String coulmn;
private String author;
private String user;
private String colum;
private String rootSourcce;
private Double channelIndex;
}
\ No newline at end of file
src/main/java/com/zhiwei/messageflow/bean/VideoMessage.java 0 → 100644
package com.zhiwei.messageflow.bean;
import lombok.Data;
import lombok.ToString;
@Data
@ToString
public class VideoMessage {
private String _id;
private String videoLength;
private String pt;
private String spyderInfoId;
private String browseCount;
private Long rsid;
private String source;
private String title;
private String url;
private String content;
private String savetime;
private String id;
private String time;
private String uper;
private String type;
}
\ No newline at end of file
src/main/java/com/zhiwei/messageflow/bean/WeiboMessage.java 0 → 100644
package com.zhiwei.messageflow.bean;
import lombok.Data;
import lombok.ToString;
@Data
@ToString
public class WeiboMessage {
private String _id;
private Integer vtype;
private String sex;
private Integer fensi;
private String description;
private String pic_urls;
private String source;
private Integer reply_count;
private Integer retweet_count;
private String url;
private String ins;
private Integer isForward;
private String weibo_img;
private Integer weibo;
private String user_id;
private String img_url;
private String location;
private String id;
private String time;
private String text;
private Long rstime;
private String username;
private Integer guanzhu;
private String rooturl;
private String roottext;
private String root_time;
private String rootsource;
private String retweet_status_id;
private Integer root_reply;
private String rootuid;
private Integer root_repost;
private String rootname;
private String rootLocation;
private Double channelIndex;
}
src/main/java/com/zhiwei/messageflow/bean/ZhihuMessage.java 0 → 100644
package com.zhiwei.messageflow.bean;
import lombok.Data;
import lombok.ToString;
@Data
@ToString
public class ZhihuMessage {
private String _id;
private String pt;
private String spyderInfoId;
private String user_name;
private String created_at;
private Long rsid;
private String answers_updateTime;
private String source;
private Integer attitudes_count;
private String type;
private String insert_at;
private String question_title;
private Integer answer_count;
private String question_id;
private String url;
private String answer_content;
private String img_url;
private String update_at;
private String user_url;
private String id;
private String comments_count;
private String question_content;
private String createTime;
private String serialVersionUID;
private String question_url;
private String answer_url;
private String img;
private String follow;
private String browse;
}
\ No newline at end of file
src/main/java/com/zhiwei/messageflow/config/ESConfig.java 0 → 100644
package com.zhiwei.messageflow.config;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.PropertySource;
import org.springframework.stereotype.Component;
//ES配置类
@Component
@Configuration
@ConfigurationProperties(prefix = "es")
@PropertySource(value = "classpath:es.properties")
public class ESConfig {
public static String ip;
public static int esPort;
public static String clusterName;
public static String zhihuIndexName;
public static String zhihuType;
public static String mediaIndexName;
public static String mediaType;
public static String weiboIndexName;
public static String weiboType;
public static String videoIndexName;
public static String videoType;
public static String mediaMarkIndexName;
public static String mediaMarkType;
public static String weiboMarkIndexName;
public static String weiboMarkType;
public static String getIp() {
return ip;
}
public static void setIp(String ip) {
ESConfig.ip = ip;
}
public static int getEsPort() {
return esPort;
}
public static void setEsPort(int esPort) {
ESConfig.esPort = esPort;
}
public static String getClusterName() {
return clusterName;
}
public static void setClusterName(String clusterName) {
ESConfig.clusterName = clusterName;
}
public static String getZhihuIndexName() {
return zhihuIndexName;
}
public static void setZhihuIndexName(String zhihuIndexName) {
ESConfig.zhihuIndexName = zhihuIndexName;
}
public static String getZhihuType() {
return zhihuType;
}
public static void setZhihuType(String zhihuType) {
ESConfig.zhihuType = zhihuType;
}
public static String getMediaIndexName() {
return mediaIndexName;
}
public static void setMediaIndexName(String mediaIndexName) {
ESConfig.mediaIndexName = mediaIndexName;
}
public static String getMediaType() {
return mediaType;
}
public static void setMediaType(String mediaType) {
ESConfig.mediaType = mediaType;
}
public static String getWeiboIndexName() {
return weiboIndexName;
}
public static void setWeiboIndexName(String weiboIndexName) {
ESConfig.weiboIndexName = weiboIndexName;
}
public static String getWeiboType() {
return weiboType;
}
public static void setWeiboType(String weiboType) {
ESConfig.weiboType = weiboType;
}
public static String getVideoIndexName() {
return videoIndexName;
}
public static void setVideoIndexName(String videoIndexName) {
ESConfig.videoIndexName = videoIndexName;
}
public static String getVideoType() {
return videoType;
}
public static void setVideoType(String videoType) {
ESConfig.videoType = videoType;
}
public static String getMediaMarkIndexName() {
return mediaMarkIndexName;
}
public static void setMediaMarkIndexName(String mediaMarkIndexName) {
ESConfig.mediaMarkIndexName = mediaMarkIndexName;
}
public static String getMediaMarkType() {
return mediaMarkType;
}
public static void setMediaMarkType(String mediaMarkType) {
ESConfig.mediaMarkType = mediaMarkType;
}
public static String getWeiboMarkIndexName() {
return weiboMarkIndexName;
}
public static void setWeiboMarkIndexName(String weiboMarkIndexName) {
ESConfig.weiboMarkIndexName = weiboMarkIndexName;
}
public static String getWeiboMarkType() {
return weiboMarkType;
}
public static void setWeiboMarkType(String weiboMarkType) {
ESConfig.weiboMarkType = weiboMarkType;
}
}
src/main/java/com/zhiwei/messageflow/config/MongoConfig.java 0 → 100644
package com.zhiwei.messageflow.config;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.PropertySource;
import org.springframework.stereotype.Component;
import lombok.Data;
import lombok.ToString;
//Mongo配置类
@Data
@ToString
@Component
@Configuration
@ConfigurationProperties(prefix = "mongo")
@PropertySource(value = "classpath:mongo.properties")
public class MongoConfig {
private String LocalIP;
private int Localport;
private String LocalDBName;
private String ServerIP;
private int Serverport;
private String ServerDBName;
private String ServerUsername;
private String ServerPassword;
private String authenticationDatabase;
private String EventDBName;
private String WechatDBName;
private String TestDBName;
private String weibotagDBName;
private String mediatagDBName;
private String TagIP;
private int Tagport;
private int connectionsPerHost;
private int threadsAllowedToBlockForConnectionMultiplier;
private int connectTimeout;
private int maxWaitTime;
private boolean autoConnectRetry;
private boolean socketKeepAlive;
private int socketTimeout;
private boolean slaveOk;
}
src/main/java/com/zhiwei/messageflow/config/RedisConfig.java 0 → 100644
package com.zhiwei.messageflow.config;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.PropertySource;
import org.springframework.stereotype.Component;
//redus配置类
@Component
@Configuration
@ConfigurationProperties(prefix = "redis")
@PropertySource(value = "classpath:redis.properties")
public class RedisConfig {
private int maxTotal;
private int maxIdle;
private int maxWaitMillis;
private boolean testOnBorrow;
private boolean testOnReturn;
private String ip;
private int port;
private int keyMaxSize;
private int selectDB;
private int user_keyMaxSize;
private int cacheSize;
private int intitCount;
public int getMaxTotal() {
return maxTotal;
}
public void setMaxTotal(int maxTotal) {
this.maxTotal = maxTotal;
}
public int getMaxIdle() {
return maxIdle;
}
public void setMaxIdle(int maxIdle) {
this.maxIdle = maxIdle;
}
public int getMaxWaitMillis() {
return maxWaitMillis;
}
public void setMaxWaitMillis(int maxWaitMillis) {
this.maxWaitMillis = maxWaitMillis;
}
public boolean isTestOnBorrow() {
return testOnBorrow;
}
public void setTestOnBorrow(boolean testOnBorrow) {
this.testOnBorrow = testOnBorrow;
}
public boolean isTestOnReturn() {
return testOnReturn;
}
public void setTestOnReturn(boolean testOnReturn) {
this.testOnReturn = testOnReturn;
}
public String getIp() {
return ip;
}
public void setIp(String ip) {
this.ip = ip;
}
public int getPort() {
return port;
}
public void setPort(int port) {
this.port = port;
}
public int getKeyMaxSize() {
return keyMaxSize;
}
public void setKeyMaxSize(int keyMaxSize) {
this.keyMaxSize = keyMaxSize;
}
public int getSelectDB() {
return selectDB;
}
public void setSelectDB(int selectDB) {
this.selectDB = selectDB;
}
public int getUser_keyMaxSize() {
return user_keyMaxSize;
}
public void setUser_keyMaxSize(int user_keyMaxSize) {
this.user_keyMaxSize = user_keyMaxSize;
}
public int getCacheSize() {
return cacheSize;
}
public void setCacheSize(int cacheSize) {
this.cacheSize = cacheSize;
}
public int getIntitCount() {
return intitCount;
}
public void setIntitCount(int intitCount) {
this.intitCount = intitCount;
}
}
src/main/java/com/zhiwei/messageflow/es/ESClient.java 0 → 100644
package com.zhiwei.messageflow.es;
import java.net.InetAddress;
import java.net.UnknownHostException;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.transport.client.PreBuiltTransportClient;
import org.springframework.stereotype.Component;
import com.zhiwei.messageflow.config.ESConfig;
@Component
public class ESClient {
public static TransportClient client = ESClientHolder.getESClient();
private static class ESClientHolder {
private static TransportClient getESClient() {
Settings esSettings = Settings.builder().put("cluster.name", ESConfig.clusterName) // 设置ES实例的名称
.put("client.transport.sniff", false) // 自动嗅探整个集群的状态，把集群中其他ES节点的ip添加到本地的客户端列表中
// .put("client.transport.ping_timeout", "15s")
.build();
TransportClient client = new PreBuiltTransportClient(esSettings);
// 此步骤添加IP，至少一个，其实一个就够了，因为添加了自动嗅探配置
try {
client.addTransportAddress(
new InetSocketTransportAddress(InetAddress.getByName(ESConfig.ip), ESConfig.esPort));
} catch (UnknownHostException e) {
e.printStackTrace();
}
return client;
}
}
}
src/main/java/com/zhiwei/messageflow/es/bean/Template.java 0 → 100644
/**
* @Title: Template.java
* @Package com.zhiwei.weixin.pojo
* @Description: TODO(用一句话描述该文件做什么)
* @author hero
* @date 2016年1月27日 上午11:40:25
* @version V1.0
*/
package com.zhiwei.messageflow.es.bean;
import java.util.Map;
import lombok.Data;
import lombok.ToString;
/**
*
* @ClassName: Template
* @Description: TODO(模版消息类)
* @author 陈炜涛
* @date 2017年11月4日 上午11:31:49
*/
@Data
@ToString
public class Template {
/**
* 要发送的的用户
*/
private String touser;
/**
* 模版消息长id
*/
private String template_id;
/**
* 点击通知的跳转地址
*/
private String url;
/**
* 需要发送的模版消息内容
*/
private Map<String, Object> data;
}
src/main/java/com/zhiwei/messageflow/es/bean/TemplateData.java 0 → 100644
package com.zhiwei.messageflow.es.bean;
import java.util.Date;
import lombok.Data;
import lombok.ToString;
/**
*
* @ClassName: TemplateData
* @Description: TODO(消息模板数据)
* @author 陈炜涛
* @date 2017年11月4日 上午11:02:13
*/
@Data
@ToString
public class TemplateData {
/**
* 地址
*/
private String url;
/**
* 标题 若平台为知乎，则这为知乎问题 若平台为论坛/贴吧，则这为贴子名
*/
private String title;
/**
* 内容 若平台为知乎，则为回答内容 若平台为论坛/贴吧，则为用户回复内容
*/
private String content;
/**
* 来源 若平台为知乎，则为回答用户 若平台为论坛/贴吧，则为用户
*/
private String source;
/**
* 时间
*/
private Date time;
/**
* 平台
*/
private String pt;
/**
* 若为微博平台则有
*/
private int fensi;
/**
* 关键词规则
*/
private String wordRule;
/**
* 渠道规则
*/
private String channelRule;
public TemplateData(String url, String title, String content, String source, Date time, String pt) {
super();
this.url = url;
this.title = title;
this.content = content;
this.source = source;
this.time = time;
this.pt = pt;
}
public TemplateData() {
super();
}
}
src/main/java/com/zhiwei/messageflow/es/bean/WechatConstant.java 0 → 100644
package com.zhiwei.messageflow.es.bean;
/**
*
* @ClassName: WechatConstant
* @Description: TODO(放置微信中的一些常量)
* @author 陈炜涛
* @date 2017年11月4日 上午10:22:39
*/
public class WechatConstant {
/**
* 使用的微信号
*/
public static final String WECHAT_APPID = "wx2f555218d66e5948";
/**
* 预警模板id IT
*/
public static final String WECHAT_TEMPLATEID_EARLY_IT = "trBtKi7YyE_xbeH_xMtWYNGqAVgOZe4NlmPrH6mO-aw";
/**
* 获取access_token的url
*/
public static final String ACCESS_TOKEN_FETCH_URL = "https://api.weixin.qq.com/cgi-bin/token?grant_type=client_credential&appid=APPID&secret=APPSECRET";
/**
* 获取用户基本信息的url
*/
public static final String WECHAT_USER_FETCH_URL = "https://api.weixin.qq.com/cgi-bin/user/info?access_token=ACCESS_TOKEN&openid=OPENID&lang=LANG";
/**
* 批量获取用户基本信息的url
*/
public static final String WECHAT_USER_BATCH_FETCH_URL = "https://api.weixin.qq.com/cgi-bin/user/info/batchget?access_token=ACCESS_TOKEN";
/**
* 创建标签url
*/
public static final String WECHAT_CREATE_TAG_URL = "https://api.weixin.qq.com/cgi-bin/tags/create?access_token=ACCESS_TOKEN";
/**
* 获取已创建的标签url
*/
public static final String WECHAT_GET_TAG_URL = "https://api.weixin.qq.com/cgi-bin/tags/get?access_token=ACCESS_TOKEN";
/**
* 编辑标签url
*/
public static final String WECHAT_EDIT_TAG_URL = "https://api.weixin.qq.com/cgi-bin/tags/update?access_token=ACCESS_TOKEN";
/**
* 删除标签url
*/
public static final String WECHAT_DELETE_TAG_URL = "https://api.weixin.qq.com/cgi-bin/tags/delete?access_token=ACCESS_TOKEN";
/**
* 获取该标签下的用户列表url
*/
public static final String WECHAT_GET_USER_OF_TAG_URL = "https://api.weixin.qq.com/cgi-bin/user/tag/get?access_token=ACCESS_TOKEN";
/**
* 批量为用户打标签url
*/
public static final String WECHAT_BATCH_TAG_USER_URL = "https://api.weixin.qq.com/cgi-bin/tags/members/batchtagging?access_token=ACCESS_TOKEN";
/**
* 批量为用户取消标签url
*/
public static final String WECHAT_BATCH_UNTAG_USER_URL = "https://api.weixin.qq.com/cgi-bin/tags/members/batchuntagging?access_token=ACCESS_TOKEN";
/**
* 获取用户身上标注的标签列表url
*/
public static final String WECHAT_GET_TAGS_OF_USER_URL = "https://api.weixin.qq.com/cgi-bin/tags/getidlist?access_token=ACCESS_TOKEN";
/**
* 给用户备注的url
*/
public static final String WECHAT_REMARK_USER_URL = "https://api.weixin.qq.com/cgi-bin/user/info/updateremark?access_token=ACCESS_TOKEN";
/**
* 获取公众号的黑名单列表
*/
public static final String WECHAT_GET_BLACK_LIST_URL = "https://api.weixin.qq.com/cgi-bin/tags/members/getblacklist?access_token=ACCESS_TOKEN";
/**
* 拉黑用户url
*/
public static final String WECHAT_BLACK_USER_URL = "https://api.weixin.qq.com/cgi-bin/tags/members/batchblacklist?access_token=ACCESS_TOKEN";
/**
* 取消拉黑用户url
*/
public static final String WECHAT_UNBLACK_USER_URL = "https://api.weixin.qq.com/cgi-bin/tags/members/batchunblacklist?access_token=ACCESS_TOKEN";
/**
* 发送模版消息
*/
public static final String WECHAT_TEMPLET_SEND_URL = "https://api.weixin.qq.com/cgi-bin/message/template/send?access_token=ACCESS_TOKEN";
/**
* accesstoken的过期时间
*/
public static final long ACCESS_TOKEN_EXPIRED_TIME = 7200L;
/**
* accesstoken过期提前量（提前200秒过期）
*/
public static final long ACCESS_TOKEN_EXPIRED_delta = 200L;
/**
* 默认编码
*/
public static final String DEFAULT_CHARSET = "UTF-8";
/**
* 最多一次拉黑人数
*/
public static final int WECHAT_BLACK_USER_MAX_SIZE = 20;
/**
* 给用户备注的最大长度
*/
public static final int WECHAT_USER_REMARK_MAX_LENGTH = 30;
/**
* 每日笑话菜单的key
*/
public static final String MENU_JOKE_CLICK_KEY = "joke";
/**
* 用户标签的最大长度
*/
public static final int WECHAT_USER_TAG_LENGTH = 30;
/**
* url中的accessToken占位符
*/
public static final String ACCESS_TOKEN = "ACCESS_TOKEN";
/**
* [获取验证码]菜单对应的eventKey
*/
public static final String MENU_VER_CODE = "ver_code";
/**
* [我的信息]菜单对应的eventKey
*/
public static final String MENU_MY_CLICK_KEY = "my";
/**
* [发送位置]菜单对应的eventKey
*/
public static final String MENU_LOCATION_SELECT_KEY = "location_select";
/**
* [系统拍照发图]菜单对应的eventKey
*/
public static final String MENU_PIC_SYS_PHOTO = "pic_sysphoto";
/**
* [拍照或相册发图]菜单对应的eventKey
*/
public static final String MENU_PIC_PHOTO_OR_ALBUM = "pic_photo_or_album";
/**
* [微信相册发图]菜单对应的eventKey
*/
public static final String MENU_PIC_WEIXIN = "pic_weixin";
public static final String MENU_SCAN_CODE_WAIT_MSG = "scancode_waitmsg";
}
src/main/java/com/zhiwei/messageflow/es/dao/ESDao.java 0 → 100644
package com.zhiwei.messageflow.es.dao;
import java.util.List;
import org.elasticsearch.search.SearchHits;
//第一层 从ES获取原始数据
public interface ESDao {
/**
* 获取微博消息
*
* @param keywords
* 关键词组
* @param count
* 消息数量
* @param start
* 开始rsid
* @param end
* 结束rsid
* @param platform
* 平台
* @param project
* 平台
* @return
*/
public SearchHits getWeiboDataFromEs(List<String> keywords, int count, long start, long end, String platform,
String project);
/**
* 获取知乎消息
*
* @param keywords
* 关键词组
* @param count
* 消息数量
* @param start
* 开始rsid
* @param end
* 结束rsid
* @param platform
* 平台
* @param project
* 平台
* @return
*/
public SearchHits getZhihuDataFromEs(List<String> keywords, int count, long start, long end, String platform,
String project);
/**
* 获取视频消息
*
* @param keywords
* 关键词组
* @param count
* 消息数量
* @param start
* 开始rsid
* @param end
* 结束rsid
* @param platform
* 平台
* @param project
* 平台
* @return
*/
public SearchHits getVideoDataFromEs(List<String> keywords, int count, long start, long end, String platform,
String project);
/**
* 获取网媒消息
*
* @param keywords
* 关键词组
* @param count
* 消息数量
* @param start
* 开始rsid
* @param end
* 结束rsid
* @param platform
* 平台
* @param project
* 平台
* @return
*/
public SearchHits getMediaDataFromEs(List<String> keywords, int count, long start, long end, String platform,
String project);
public SearchHits getWeiboDataycy(long date, List<String> keywords, int count, String platform, String project);
public SearchHits getZhihuDataycy(long date, List<String> keywords, int count, String platform, String project);
public SearchHits getVideoDataycy(long date, List<String> keywords, int count, String platform, String project);
public SearchHits getMediaDataycy(long date, List<String> keywords, int count, String platform, String project);
}
src/main/java/com/zhiwei/messageflow/es/dao/TrackESDao.java 0 → 100644
package com.zhiwei.messageflow.es.dao;
import org.elasticsearch.search.SearchHits;
public interface TrackESDao {
/**
* 微博关键词预警
*
* @param anyWord
* 或关系关键词
* @param allWords
* 且关系关键词
* @param color
* 高亮颜色
* @param count
* 消息数量
* @param start
* 开始rsid
* @param end
* 结束rsid
* @return
*/
public SearchHits getkeyWordsTrackWeiboFromEs(String anyWord, String allWords, String color, int count, long start,
long end);
/**
* 知乎关键词预警
*
* @param anyWord
* 或关系关键词
* @param allWords
* 且关系关键词
* @param color
* 高亮颜色
* @param count
* 消息数量
* @param start
* 开始rsid
* @param end
* 结束rsid
* @return
*/
public SearchHits getkeyWordsTrackZhihuFromEs(String anyWord, String allWords, String color, int count, long start,
long end);
/**
* 视频关键词预警
*
* @param anyWord
* 或关系关键词
* @param allWords
* 且关系关键词
* @param color
* 高亮颜色
* @param count
* 消息数量
* @param start
* 开始rsid
* @param end
* 结束rsid
* @return
*/
public SearchHits getkeyWordsTrackVideoFromEs(String anyWord, String allWords, String color, int count, long start,
long end);
/**
* 网媒关键词预警
*
* @param anyWord
* 或关系关键词
* @param allWords
* 且关系关键词
* @param color
* 高亮颜色
* @param count
* 消息数量
* @param start
* 开始rsid
* @param end
* 结束rsid
* @param platformName
* 平台名
* @return
*/
public SearchHits getkeyWordsTrackMediaFromEs(String anyWord, String allWords, String color, int count, long start,
long end, String platformName);
/**
* 微博渠道预警
*
* @param channel
* 渠道名
* @param color
* 高亮颜色
* @param count
* 消息数
* @param start
* 开始rsid
* @param end
* 结束rsid
* @return
*/
public SearchHits getchannelTrackWeiboFromEs(String channel, String color, int count, long start, long end);
/**
* 知乎渠道预警
*
* @param channel
* 渠道名
* @param color
* 高亮颜色
* @param count
* 消息数
* @param start
* 开始rsid
* @param end
* 结束rsid
* @return
*/
public SearchHits getchannelTrackZhihuFromEs(String channel, String color, int count, long start, long end);
/**
* 视频渠道预警
*
* @param channel
* 渠道名
* @param color
* 高亮颜色
* @param count
* 消息数
* @param start
* 开始rsid
* @param end
* 结束rsid
* @return
*/
public SearchHits getchannelTrackVideoFromEs(String channel, String color, int count, long start, long end);
/**
* 网媒渠道预警
*
* @param channel
* 渠道名
* @param color
* 高亮颜色
* @param count
* 消息数
* @param start
* 开始rsid
* @param end
* 结束rsid
* @param platformName
* 平台名
* @return
*/
public SearchHits getchannelTrackMediaFromEs(String channel, String color, int count, long start, long end,
String platformName);
/**
* 微博相似消息数预警
*
* @param articleTitle
* 标题
* @param color
* 高亮颜色
* @param count
* 消息数
* @param start
* 开始时间
* @param end
* 结束时间
* @return
*/
public SearchHits getarticleTrackWeiboFromEs(String articleTitle, String color, int count, long start, long end);
/**
* 视频相似消息数预警
*
* @param articleTitle
* 标题
* @param color
* 高亮颜色
* @param count
* 消息数
* @param start
* 开始rsid
* @param end
* 结束rsid
* @return
*/
public SearchHits getarticleTrackVideoFromEs(String articleTitle, String color, int count, long start, long end);
/**
* 微博相似消息数预警
*
* @param articleTitle
* 标题
* @param color
* 高亮颜色
* @param count
* 消息数
* @param start
* 开始时间
* @param end
* 结束时间
* @param platformName
* 平台名
* @return
*/
public SearchHits getarticleTrackMediaFromEs(String articleTitle, String color, int count, long start, long end,
String platformName);
}
src/main/java/com/zhiwei/messageflow/es/dao/impl/ESDaoImpl.java 0 → 100644
package com.zhiwei.messageflow.es.dao.impl;
import java.util.Date;
import java.util.List;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.springframework.stereotype.Component;
import com.zhiwei.es.util.ESIndexesUtil;
import com.zhiwei.messageflow.config.ESConfig;
import com.zhiwei.messageflow.es.ESClient;
import com.zhiwei.messageflow.es.dao.ESDao;
import com.zhiwei.messageflow.util.ESQueryUtil;
import com.zhiwei.messageflow.util.TimeUtil;
@Component
public class ESDaoImpl implements ESDao {
TransportClient transportClient = ESClient.client;
protected static ESIndexesUtil.ESIndexes indexes = ESIndexesUtil.create("1.119.44.201", 9200);
@Override
public SearchHits getWeiboDataFromEs(List<String> keywords, int count, long startid, long endid, String platform,
String project) {
// 查询封装
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
BoolQueryBuilder boolQueryBuilder1 = QueryBuilders.boolQuery();
if (keywords.isEmpty()) {
return null;
}
for (int i = 0; i < keywords.size(); i++) {
String word = keywords.get(i).replace("serach---", "");
ESQueryUtil.matchPhraseQueryFields(boolQueryBuilder1, "should", word, "text", "roottext");
}
// 去除插件标注
boolQueryBuilder.mustNot(QueryBuilders.matchQuery("source", "插件标注"));
boolQueryBuilder.must(boolQueryBuilder1);
HighlightBuilder highlightBuilder = ESQueryUtil.getHighlightBuilder("text", "roottext");
List<String> list = indexes.getLastIndexes("network", 2);
String[] indexlist = list.toArray(new String[list.size()]);
SearchRequestBuilder searchRequestBuilder = transportClient.prepareSearch(indexlist)
.setTypes(ESConfig.weiboType).highlighter(highlightBuilder).setQuery(boolQueryBuilder)
.addSort("rstime", SortOrder.DESC).setFrom(0).setSize(count);
if (startid > 0 && endid <= 0) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("rstime").from(startid));
} else if (endid > 0 && startid <= 0) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("rstime").to(endid));
} else if (endid > 0 && startid > 0) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("rstime").from(startid).to(endid));
} else {
long startidlong = System.currentTimeMillis() - 5 * 24 * 60 * 60 * 1000L;
String startidTime = TimeUtil.formatDateToMinute(new Date(startidlong));
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("time").from(startidTime));
}
SearchResponse response = searchRequestBuilder.execute().actionGet();
SearchHits searchHits = response.getHits();
return searchHits;
}
@Override
public SearchHits getZhihuDataFromEs(List<String> keywords, int count, long startid, long endid, String platform,
String project) {
SearchHits searchHits = null;
try {
// 查询封装
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
if (keywords.isEmpty()) {
return null;
}
for (int i = 0; i < keywords.size(); i++) {
String word = keywords.get(i).replace("serach---", "");
ESQueryUtil.matchPhraseQueryFields(boolQueryBuilder, "should", word, "question_title",
"question_content", "answer_content");
}
HighlightBuilder highlightBuilder = ESQueryUtil.getHighlightBuilder("question_title", "question_content",
"answer_content");// 高亮设置
SearchRequestBuilder searchRequestBuilder = transportClient.prepareSearch(ESConfig.zhihuIndexName)
.setTypes(ESConfig.zhihuType).highlighter(highlightBuilder).setQuery(boolQueryBuilder)
.addSort("rsid", SortOrder.DESC).setFrom(0).setSize(count);
if (startid > 0 && endid <= 0) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("rsid").from(startid));
} else if (endid > 0 && startid <= 0) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("rsid").to(endid));
} else if (endid > 0 && startid > 0) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("rsid").from(startid).to(endid));
}
SearchResponse response = searchRequestBuilder.execute().actionGet();
searchHits = response.getHits();
} catch (Exception e) {
e.printStackTrace();
}
return searchHits;
}
@Override
public SearchHits getVideoDataFromEs(List<String> keywords, int count, long startid, long endid, String platform,
String project) {
// 查询封装
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
BoolQueryBuilder boolQueryBuilder1 = QueryBuilders.boolQuery();
if (keywords.isEmpty()) {
return null;
}
for (int i = 0; i < keywords.size(); i++) {
String word = keywords.get(i).replace("serach---", "");
ESQueryUtil.matchPhraseQueryFields(boolQueryBuilder1, "should", word, "title");
}
boolQueryBuilder.must(boolQueryBuilder1);
HighlightBuilder highlightBuilder = ESQueryUtil.getHighlightBuilder("title");// 高亮设置
List<String> list = indexes.getLastIndexes("videos", 2);
String[] indexlist = list.toArray(new String[list.size()]);
SearchRequestBuilder searchRequestBuilder = transportClient.prepareSearch(indexlist)
.setTypes(ESConfig.videoType).highlighter(highlightBuilder).setQuery(boolQueryBuilder)
.addSort("rsid", SortOrder.DESC).setFrom(0).setSize(count);
if (startid > 0 && endid <= 0) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("rsid").from(startid));
} else if (endid > 0 && startid <= 0) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("rsid").to(endid));
} else if (endid > 0 && startid > 0) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("rsid").from(startid).to(endid));
}
SearchResponse response = searchRequestBuilder.execute().actionGet();
SearchHits searchHits = response.getHits();
return searchHits;
}
@Override
public SearchHits getMediaDataFromEs(List<String> keywords, int count, long startid, long endid, String platform,
String project) {
// 查询封装
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
BoolQueryBuilder boolQueryBuilder1 = QueryBuilders.boolQuery();
if (keywords.isEmpty()) {
return null;
}
for (int i = 0; i < keywords.size(); i++) {
String word = keywords.get(i).replace("serach---", "");
ESQueryUtil.matchPhraseQueryFields(boolQueryBuilder1, "should", word, "title", "content");
}
HighlightBuilder highlightBuilder = ESQueryUtil.getHighlightBuilder("title", "content");// 高亮设置
// 根据platform选消息:(现有platform:网络媒体、微信公众平台、报刊、今日头条、客户端)(可能有论坛、贴吧)
if (platform.equals("网络媒体")) {
String[] platforms = { "网媒", "百度新闻", "360新闻", "google新闻", "一点资讯" };
boolQueryBuilder.must(ESQueryUtil.matchPhraseQueryWords(QueryBuilders.boolQuery(), ESQueryUtil.ES_QUERY_OR,
"type", platforms));
boolQueryBuilder.mustNot(QueryBuilders.matchQuery("type", "插件标注"));
} else if (platform.equals("微信公众平台")) {
boolQueryBuilder.must(QueryBuilders.wildcardQuery("type", "微信*"));
} else if (platform.equals("报刊")) {
boolQueryBuilder.must(QueryBuilders.matchPhraseQuery("type", "平媒"));
} else if (platform.equals("今日头条")) {
boolQueryBuilder.must(QueryBuilders.boolQuery().must(QueryBuilders.matchPhraseQuery("type", "今日头条"))
.must(QueryBuilders.existsQuery("spyderInfoId")));
} else if (platform.equals("客户端")) {
boolQueryBuilder.must(QueryBuilders.boolQuery().should(QueryBuilders.wildcardQuery("type", "*客户端"))
.should(QueryBuilders.matchPhraseQuery("type", "脉脉")));
} else if (platform.equals("论坛")) {
boolQueryBuilder.must(QueryBuilders.matchPhraseQuery("type", "论坛"));
} else if (platform.equals("贴吧")) {
boolQueryBuilder.must(QueryBuilders.matchPhraseQuery("type", "贴吧"));
}
boolQueryBuilder.must(boolQueryBuilder1);
List<String> list = indexes.getLastIndexes("mediaspider", 2);
String[] indexlist = list.toArray(new String[list.size()]);
SearchRequestBuilder searchRequestBuilder = transportClient.prepareSearch(indexlist)
.highlighter(highlightBuilder).setQuery(boolQueryBuilder).addSort("rsid", SortOrder.DESC).setFrom(0)
.setSize(count);
if (startid > 0 && endid <= 0) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("rsid").from(startid));
} else if (endid > 0 && startid <= 0) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("rsid").to(endid));
} else if (endid > 0 && startid > 0) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("rsid").from(startid).to(endid));
} else if (!platform.equals("报刊") && !platform.equals("论坛") && !platform.equals("贴吧")) {
long startidlong = System.currentTimeMillis() - 5 * 24 * 60 * 60 * 1000L;
String startidTime = TimeUtil.formatEsDate(new Date(startidlong));
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("time").from(startidTime));
}
SearchResponse response = searchRequestBuilder.execute().actionGet();
SearchHits searchHits = response.getHits();
return searchHits;
}
@Override
public SearchHits getWeiboDataycy(long date, List<String> keywords, int count, String platform, String project) {
String stime = null;
String etime = null;
stime = TimeUtil.formatDateToMinute(new Date(date));
etime = TimeUtil.formatDateToMinute(new Date(date + 6 * 3600 * 1000L - 60 * 1000L));
// 查询封装
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
if (keywords.isEmpty()) {
return null;
}
for (String keyword : keywords) {
BoolQueryBuilder boolQueryBuilder1 = QueryBuilders.boolQuery();
BoolQueryBuilder boolQueryBuilder2 = QueryBuilders.boolQuery();
String word = keyword.replace("serach---", "");
String[] andwords = word.split("==");
for (String andword : andwords) {
boolQueryBuilder1.must(QueryBuilders.matchPhraseQuery("text", andword));
boolQueryBuilder2.must(QueryBuilders.matchPhraseQuery("roottext", andword));
}
boolQueryBuilder.should(boolQueryBuilder1);
boolQueryBuilder.should(boolQueryBuilder2);
}
// 去除插件标注
boolQueryBuilder.mustNot(QueryBuilders.matchQuery("source", "插件标注"));
HighlightBuilder highlightBuilder = ESQueryUtil.getHighlightBuilder("text", "roottext");
List<String> list = indexes.getLastIndexes("network", 2);
String[] indexlist = list.toArray(new String[list.size()]);
SearchRequestBuilder searchRequestBuilder = transportClient.prepareSearch(indexlist)
.setTypes(ESConfig.weiboType).highlighter(highlightBuilder).setQuery(boolQueryBuilder)
.addSort("rstime", SortOrder.DESC).setFrom(0).setSize(count);
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("time").from(stime).to(etime));
SearchResponse response = searchRequestBuilder.execute().actionGet();
SearchHits searchHits = response.getHits();
return searchHits;
}
@Override
public SearchHits getZhihuDataycy(long date, List<String> keywords, int count, String platform, String project) {
String stime = null;
String etime = null;
stime = TimeUtil.formatEsDate(new Date(date)) + "Z";
etime = TimeUtil.formatEsDate(new Date(date + 6 * 3600 * 1000L - 1L)) + "Z";
// 查询封装
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
if (keywords.isEmpty()) {
return null;
}
for (String keyword : keywords) {
BoolQueryBuilder boolQueryBuilder1 = QueryBuilders.boolQuery();
BoolQueryBuilder boolQueryBuilder2 = QueryBuilders.boolQuery();
String word = keyword.replace("serach---", "");
String[] andwords = word.split("==");
for (String andword : andwords) {
boolQueryBuilder1.must(QueryBuilders.matchPhraseQuery("question_title", andword));
boolQueryBuilder2.must(QueryBuilders.matchPhraseQuery("question_content", andword));
}
boolQueryBuilder.should(boolQueryBuilder1);
boolQueryBuilder.should(boolQueryBuilder2);
}
HighlightBuilder highlightBuilder = ESQueryUtil.getHighlightBuilder("question_title", "question_content",
"answer_content");// 高亮设置
SearchRequestBuilder searchRequestBuilder = transportClient.prepareSearch(ESConfig.zhihuIndexName)
.setTypes(ESConfig.zhihuType).highlighter(highlightBuilder).setQuery(boolQueryBuilder)
.addSort("rsid", SortOrder.DESC).setFrom(0).setSize(count);
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("insert_at").from(stime).to(etime));
SearchResponse response = searchRequestBuilder.execute().actionGet();
SearchHits searchHits = response.getHits();
return searchHits;
}
@Override
public SearchHits getVideoDataycy(long date, List<String> keywords, int count, String platform, String project) {
String stime = null;
String etime = null;
stime = TimeUtil.formatEsDate(new Date(date));
etime = TimeUtil.formatEsDate(new Date(date + 6 * 3600 * 1000L - 1L));
// 查询封装
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
if (keywords.isEmpty()) {
return null;
}
for (String keyword : keywords) {
BoolQueryBuilder boolQueryBuilder1 = QueryBuilders.boolQuery();
String word = keyword.replace("serach---", "");
String[] andwords = word.split("==");
for (String andword : andwords) {
boolQueryBuilder1.must(QueryBuilders.matchPhraseQuery("title", andword));
}
boolQueryBuilder.should(boolQueryBuilder1);
}
HighlightBuilder highlightBuilder = ESQueryUtil.getHighlightBuilder("title");// 高亮设置
List<String> list = indexes.getLastIndexes("videos", 2);
String[] indexlist = list.toArray(new String[list.size()]);
SearchRequestBuilder searchRequestBuilder = transportClient.prepareSearch(indexlist)
.setTypes(ESConfig.videoType).highlighter(highlightBuilder).setQuery(boolQueryBuilder)
.addSort("rsid", SortOrder.DESC).setFrom(0).setSize(count);
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("time").from(stime).to(etime));
SearchResponse response = searchRequestBuilder.execute().actionGet();
SearchHits searchHits = response.getHits();
return searchHits;
}
@Override
public SearchHits getMediaDataycy(long date, List<String> keywords, int count, String platform, String project) {
String stime = null;
String etime = null;
stime = TimeUtil.formatEsDate(new Date(date));
etime = TimeUtil.formatEsDate(new Date(date + 6 * 3600 * 1000L - 1L));
// 查询封装
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
if (keywords.isEmpty()) {
return null;
}
BoolQueryBuilder boolQueryBuilder3 = new BoolQueryBuilder();
for (String keyword : keywords) {
BoolQueryBuilder boolQueryBuilder1 = QueryBuilders.boolQuery();
BoolQueryBuilder boolQueryBuilder2 = QueryBuilders.boolQuery();
String word = keyword.replace("serach---", "");
String[] andwords = word.split("==");
for (String andword : andwords) {
boolQueryBuilder1.must(QueryBuilders.matchPhraseQuery("title", andword));
boolQueryBuilder2.must(QueryBuilders.matchPhraseQuery("content", andword));
}
boolQueryBuilder3.should(boolQueryBuilder1);
boolQueryBuilder3.should(boolQueryBuilder2);
}
boolQueryBuilder.must(boolQueryBuilder3);
HighlightBuilder highlightBuilder = ESQueryUtil.getHighlightBuilder("title", "content");// 高亮设置
if (platform.equals("网络媒体")) {
String[] platforms = { "网媒", "百度新闻", "360新闻", "google新闻", "一点资讯" };
boolQueryBuilder.must(ESQueryUtil.matchPhraseQueryWords(QueryBuilders.boolQuery(), ESQueryUtil.ES_QUERY_OR,
"type", platforms));
boolQueryBuilder.mustNot(QueryBuilders.matchQuery("type", "插件标注"));
} else if (platform.equals("微信公众平台")) {
boolQueryBuilder.must(QueryBuilders.wildcardQuery("type", "微信*"));
} else if (platform.equals("报刊")) {
boolQueryBuilder.must(QueryBuilders.matchPhraseQuery("type", "平媒"));
} else if (platform.equals("今日头条")) {
boolQueryBuilder.must(QueryBuilders.boolQuery().must(QueryBuilders.matchPhraseQuery("type", "今日头条"))
.must(QueryBuilders.existsQuery("spyderInfoId")));
} else if (platform.equals("客户端")) {
boolQueryBuilder.must(QueryBuilders.boolQuery().should(QueryBuilders.wildcardQuery("type", "*客户端"))
.should(QueryBuilders.matchPhraseQuery("type", "脉脉")));
} else if (platform.equals("论坛")) {
boolQueryBuilder.must(QueryBuilders.matchPhraseQuery("type", "论坛"));
} else if (platform.equals("贴吧")) {
boolQueryBuilder.must(QueryBuilders.matchPhraseQuery("type", "贴吧"));
}
List<String> list = indexes.getLastIndexes("mediaspider", 2);
String[] indexlist = list.toArray(new String[list.size()]);
SearchRequestBuilder searchRequestBuilder = transportClient.prepareSearch(indexlist)
.highlighter(highlightBuilder).setQuery(boolQueryBuilder).addSort("rsid", SortOrder.DESC).setFrom(0)
.setSize(count).setPostFilter(QueryBuilders.rangeQuery("time").from(stime).to(etime));
SearchResponse response = searchRequestBuilder.execute().actionGet();
SearchHits searchHits = response.getHits();
return searchHits;
}
}
\ No newline at end of file
src/main/java/com/zhiwei/messageflow/es/dao/impl/TrackESDaoImpl.java 0 → 100644
package com.zhiwei.messageflow.es.dao.impl;
import java.util.Date;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.springframework.stereotype.Component;
import com.zhiwei.es.util.ESIndexesUtil;
import com.zhiwei.messageflow.config.ESConfig;
import com.zhiwei.messageflow.es.ESClient;
import com.zhiwei.messageflow.es.dao.TrackESDao;
import com.zhiwei.messageflow.util.ESQueryUtil;
import com.zhiwei.messageflow.util.TimeUtil;
@Component
public class TrackESDaoImpl implements TrackESDao {
TransportClient transportClient = ESClient.client;
protected static ESIndexesUtil.ESIndexes indexes = ESIndexesUtil.create("1.119.44.201", 9200);
@Override
public SearchHits getkeyWordsTrackWeiboFromEs(String anyWord, String allWords, String color, int count, long start,
long end) {
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();// 查询封装
// 或关系
BoolQueryBuilder anyboolQueryBuilder = QueryBuilders.boolQuery();// 查询封装
if (!anyWord.equals("")) {
String[] anywords = anyWord.split("，");
ESQueryUtil.matchPhraseQueryWords(anyboolQueryBuilder, "should", "text", anywords);
ESQueryUtil.matchPhraseQueryWords(anyboolQueryBuilder, "should", "roottext", anywords);
boolQueryBuilder.must(anyboolQueryBuilder);
}
// 且关系
BoolQueryBuilder allboolQueryBuilder = QueryBuilders.boolQuery();// 查询封装
if (!allWords.equals("")) {
String[] allwords = allWords.split("，");
allboolQueryBuilder
.should(ESQueryUtil.matchPhraseQueryWords(QueryBuilders.boolQuery(), "must", "text", allwords));
allboolQueryBuilder
.should(ESQueryUtil.matchPhraseQueryWords(QueryBuilders.boolQuery(), "must", "roottext", allwords));
boolQueryBuilder.must(allboolQueryBuilder);
}
String[] fields = { "text", "roottext" };
HighlightBuilder highlightBuilder = ESQueryUtil.getTrackHighlightBuilder(fields, color);
SearchRequestBuilder searchRequestBuilder = transportClient.prepareSearch(ESConfig.weiboIndexName)
.setTypes(ESConfig.weiboType).highlighter(highlightBuilder).setQuery(boolQueryBuilder)
.addSort("rstime", SortOrder.DESC).setFrom(0).setSize(count);
if (start != -1 && end == -1) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("rstime").from(start));
} else if (end != -1 && start == -1) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("rstime").to(end));
} else if (end != -1 && start != -1) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("rstime").from(start).to(end));
}
SearchResponse response = searchRequestBuilder.execute().actionGet();
SearchHits searchHits = response.getHits();
return searchHits;
}
@Override
public SearchHits getkeyWordsTrackZhihuFromEs(String anyWord, String allWords, String color, int count, long start,
long end) {
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();// 查询封装
// 或关系
BoolQueryBuilder anyboolQueryBuilder = QueryBuilders.boolQuery();// 查询封装
if (anyWord != "") {
String[] anywords = anyWord.split("，");
for (int i = 0; i < anywords.length; i++) {
String anyword = anywords[i];
anyboolQueryBuilder.should(ESQueryUtil.matchPhraseQueryFields(QueryBuilders.boolQuery(), "should",
anyword, "question_title", "question_content", "answer_content"));
}
boolQueryBuilder.must(anyboolQueryBuilder);
}
// 且关系
BoolQueryBuilder allboolQueryBuilder = QueryBuilders.boolQuery();// 查询封装
if (allWords != "") {
String[] allwords = allWords.split("，");
for (int i = 0; i < allwords.length; i++) {
String allword = allwords[i];
allboolQueryBuilder.must(ESQueryUtil.matchPhraseQueryFields(QueryBuilders.boolQuery(), "should",
allword, "question_title", "question_content", "answer_content"));
}
boolQueryBuilder.must(allboolQueryBuilder);
}
String[] fields = { "question_title", "question_content", "answer_content" };
HighlightBuilder highlightBuilder = ESQueryUtil.getTrackHighlightBuilder(fields, color);
SearchRequestBuilder searchRequestBuilder = transportClient.prepareSearch(ESConfig.zhihuIndexName)
.setTypes(ESConfig.zhihuType).highlighter(highlightBuilder).setQuery(boolQueryBuilder)
.addSort("rsid", SortOrder.DESC).setFrom(0).setSize(count);
if (start != -1 && end == -1) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("rsid").from(start));
} else if (end != -1 && start == -1) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("rsid").to(end));
} else if (end != -1 && start != -1) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("rsid").from(start).to(end));
}
SearchResponse response = searchRequestBuilder.execute().actionGet();
SearchHits searchHits = response.getHits();
return searchHits;
}
@Override
public SearchHits getkeyWordsTrackVideoFromEs(String anyWord, String allWords, String color, int count, long start,
long end) {
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();// 查询封装
// 或关系
BoolQueryBuilder anyboolQueryBuilder = QueryBuilders.boolQuery();// 查询封装
if (!anyWord.equals("")) {
String[] anywords = anyWord.split("，");
ESQueryUtil.matchPhraseQueryWords(anyboolQueryBuilder, "should", "title", anywords);
boolQueryBuilder.must(anyboolQueryBuilder);
}
// 且关系
BoolQueryBuilder allboolQueryBuilder = QueryBuilders.boolQuery();// 查询封装
if (!allWords.equals("")) {
String[] allwords = allWords.split("，");
allboolQueryBuilder
.should(ESQueryUtil.matchPhraseQueryWords(QueryBuilders.boolQuery(), "must", "title", allwords));
boolQueryBuilder.must(allboolQueryBuilder);
}
String[] fields = { "title" };
HighlightBuilder highlightBuilder = ESQueryUtil.getTrackHighlightBuilder(fields, color);
SearchRequestBuilder searchRequestBuilder = transportClient.prepareSearch(ESConfig.videoIndexName)
.setTypes(ESConfig.videoType).highlighter(highlightBuilder).setQuery(boolQueryBuilder)
.addSort("rsid", SortOrder.DESC).setFrom(0).setSize(count);
if (start != -1 && end == -1) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("rsid").from(start));
} else if (end != -1 && start == -1) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("rsid").to(end));
} else if (end != -1 && start != -1) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("rsid").from(start).to(end));
}
SearchResponse response = searchRequestBuilder.execute().actionGet();
SearchHits searchHits = response.getHits();
return searchHits;
}
@Override
public SearchHits getkeyWordsTrackMediaFromEs(String anyWord, String allWords, String color, int count, long start,
long end, String platformName) {
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();// 查询封装
// 或关系
BoolQueryBuilder anyboolQueryBuilder = QueryBuilders.boolQuery();// 查询封装
if (!anyWord.equals("")) {
String[] anywords = anyWord.split("，");
ESQueryUtil.matchPhraseQueryWords(anyboolQueryBuilder, "should", "title", anywords);
ESQueryUtil.matchPhraseQueryWords(anyboolQueryBuilder, "should", "content", anywords);
boolQueryBuilder.must(anyboolQueryBuilder);
}
// 且关系
BoolQueryBuilder allboolQueryBuilder = QueryBuilders.boolQuery();// 查询封装
if (!allWords.equals("")) {
String[] allwords = allWords.split("，");
allboolQueryBuilder
.should(ESQueryUtil.matchPhraseQueryWords(QueryBuilders.boolQuery(), "must", "title", allwords));
allboolQueryBuilder
.should(ESQueryUtil.matchPhraseQueryWords(QueryBuilders.boolQuery(), "must", "content", allwords));
boolQueryBuilder.must(allboolQueryBuilder);
}
if (platformName.equals("网络媒体")) {
String[] types = { "网媒", "百度新闻", "360新闻", "google新闻", "插件标注", "一点资讯" };
boolQueryBuilder.must(ESQueryUtil.matchPhraseQueryWords(QueryBuilders.boolQuery(), ESQueryUtil.ES_QUERY_OR,
"type", types));
boolQueryBuilder.mustNot(QueryBuilders.matchQuery("type", "插件标注"));
} else if (platformName.equals("微信公众平台")) {
boolQueryBuilder.must(QueryBuilders.wildcardQuery("type", "微信*"));
} else if (platformName.equals("报刊")) {
boolQueryBuilder.must(QueryBuilders.matchPhraseQuery("type", "平媒"));
} else if (platformName.equals("今日头条")) {
boolQueryBuilder.must(QueryBuilders.matchPhraseQuery("type", "今日头条"));
} else if (platformName.equals("客户端")) {
boolQueryBuilder.must(QueryBuilders.boolQuery().should(QueryBuilders.wildcardQuery("type", "*客户端"))
.should(QueryBuilders.matchPhraseQuery("type", "脉脉")));
} else if (platformName.equals("论坛")) {
boolQueryBuilder.must(QueryBuilders.matchPhraseQuery("type", "论坛"));
} else if (platformName.equals("贴吧")) {
boolQueryBuilder.must(QueryBuilders.matchPhraseQuery("type", "贴吧"));
}
String[] fields = { "title", "content" };
HighlightBuilder highlightBuilder = ESQueryUtil.getTrackHighlightBuilder(fields, color);
SearchRequestBuilder searchRequestBuilder = transportClient.prepareSearch(ESConfig.mediaIndexName)
.highlighter(highlightBuilder).setQuery(boolQueryBuilder).addSort("rsid", SortOrder.DESC).setFrom(0)
.setSize(count);
if (start != -1 && end == -1) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("rsid").from(start));
} else if (end != -1 && start == -1) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("rsid").to(end));
} else if (end != -1 && start != -1) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("rsid").from(start).to(end));
}
SearchResponse response = searchRequestBuilder.execute().actionGet();
SearchHits searchHits = response.getHits();
return searchHits;
}
@Override
public SearchHits getchannelTrackWeiboFromEs(String channel, String color, int count, long start, long end) {
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();// 查询封装
// 渠道条件
BoolQueryBuilder channelboolQueryBuilder = QueryBuilders.boolQuery();// 查询封装
channelboolQueryBuilder.must(QueryBuilders.matchPhraseQuery("username", channel));
boolQueryBuilder.must(channelboolQueryBuilder);
String[] fields = { "username" };
HighlightBuilder highlightBuilder = ESQueryUtil.getTrackHighlightBuilder(fields, color);
SearchRequestBuilder searchRequestBuilder = transportClient.prepareSearch(ESConfig.weiboIndexName)
.setTypes(ESConfig.weiboType).highlighter(highlightBuilder).setQuery(boolQueryBuilder)
.addSort("rstime", SortOrder.DESC).setFrom(0).setSize(count);
if (start != -1 && end == -1) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("rstime").from(start));
} else if (end != -1 && start == -1) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("rstime").to(end));
} else if (end != -1 && start != -1) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("rstime").from(start).to(end));
}
SearchResponse response = searchRequestBuilder.execute().actionGet();
SearchHits searchHits = response.getHits();
return searchHits;
}
@Override
public SearchHits getchannelTrackZhihuFromEs(String channel, String color, int count, long start, long end) {
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();// 查询封装
// 渠道条件
BoolQueryBuilder channelboolQueryBuilder = QueryBuilders.boolQuery();// 查询封装
channelboolQueryBuilder.must(QueryBuilders.matchPhraseQuery("user_name", channel));
boolQueryBuilder.must(channelboolQueryBuilder);
String[] fields = { "user_name" };
HighlightBuilder highlightBuilder = ESQueryUtil.getTrackHighlightBuilder(fields, color);
SearchRequestBuilder searchRequestBuilder = transportClient.prepareSearch(ESConfig.zhihuIndexName)
.setTypes(ESConfig.zhihuType).highlighter(highlightBuilder).setQuery(boolQueryBuilder)
.addSort("rsid", SortOrder.DESC).setFrom(0).setSize(count);
if (start != -1 && end == -1) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("rsid").from(start));
} else if (end != -1 && start == -1) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("rsid").to(end));
} else if (end != -1 && start != -1) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("rsid").from(start).to(end));
}
SearchResponse response = searchRequestBuilder.execute().actionGet();
SearchHits searchHits = response.getHits();
return searchHits;
}
@Override
public SearchHits getchannelTrackVideoFromEs(String channel, String color, int count, long start, long end) {
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();// 查询封装
// 渠道条件
BoolQueryBuilder channelboolQueryBuilder = QueryBuilders.boolQuery();// 查询封装
channelboolQueryBuilder.must(QueryBuilders.matchPhraseQuery("uper", channel));
boolQueryBuilder.must(channelboolQueryBuilder);
String[] fields = { "uper" };
HighlightBuilder highlightBuilder = ESQueryUtil.getTrackHighlightBuilder(fields, color);
SearchRequestBuilder searchRequestBuilder = transportClient.prepareSearch(ESConfig.videoIndexName)
.setTypes(ESConfig.videoType).highlighter(highlightBuilder).setQuery(boolQueryBuilder)
.addSort("rsid", SortOrder.DESC).setFrom(0).setSize(count);
if (start != -1 && end == -1) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("rsid").from(start));
} else if (end != -1 && start == -1) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("rsid").to(end));
} else if (end != -1 && start != -1) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("rsid").from(start).to(end));
}
SearchResponse response = searchRequestBuilder.execute().actionGet();
SearchHits searchHits = response.getHits();
return searchHits;
}
@Override
public SearchHits getchannelTrackMediaFromEs(String channel, String color, int count, long start, long end,
String platformName) {
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();// 查询封装
// 渠道条件
BoolQueryBuilder channelboolQueryBuilder = QueryBuilders.boolQuery();// 查询封装
channelboolQueryBuilder.must(QueryBuilders.matchPhraseQuery("source", channel));
boolQueryBuilder.must(channelboolQueryBuilder);
String[] fields = { "source" };
HighlightBuilder highlightBuilder = ESQueryUtil.getTrackHighlightBuilder(fields, color);
if (platformName.equals("网络媒体")) {
String[] types = { "网媒", "百度新闻", "360新闻", "google新闻", "插件标注", "一点资讯" };
boolQueryBuilder.must(ESQueryUtil.matchPhraseQueryWords(QueryBuilders.boolQuery(), ESQueryUtil.ES_QUERY_OR,
"type", types));
boolQueryBuilder.mustNot(QueryBuilders.matchQuery("type", "插件标注"));
} else if (platformName.equals("微信公众平台")) {
boolQueryBuilder.must(QueryBuilders.wildcardQuery("type", "微信*"));
} else if (platformName.equals("报刊")) {
boolQueryBuilder.must(QueryBuilders.matchPhraseQuery("type", "平媒"));
} else if (platformName.equals("今日头条")) {
boolQueryBuilder.must(QueryBuilders.matchPhraseQuery("type", "今日头条"));
} else if (platformName.equals("客户端")) {
boolQueryBuilder.must(QueryBuilders.boolQuery().should(QueryBuilders.wildcardQuery("type", "*客户端"))
.should(QueryBuilders.matchPhraseQuery("type", "脉脉")));
} else if (platformName.equals("论坛")) {
boolQueryBuilder.must(QueryBuilders.matchPhraseQuery("type", "论坛"));
} else if (platformName.equals("贴吧")) {
boolQueryBuilder.must(QueryBuilders.matchPhraseQuery("type", "贴吧"));
}
SearchRequestBuilder searchRequestBuilder = transportClient.prepareSearch(ESConfig.mediaIndexName)
.highlighter(highlightBuilder).setQuery(boolQueryBuilder).addSort("rsid", SortOrder.DESC).setFrom(0)
.setSize(count);
if (start != -1 && end == -1) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("rsid").from(start));
} else if (end != -1 && start == -1) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("rsid").to(end));
} else if (end != -1 && start != -1) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("rsid").from(start).to(end));
}
SearchResponse response = searchRequestBuilder.execute().actionGet();
SearchHits searchHits = response.getHits();
return searchHits;
}
@Override
public SearchHits getarticleTrackWeiboFromEs(String articleTitle, String color, int count, long start, long end) {
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();// 查询封装
// 文章数条件
BoolQueryBuilder articleboolQueryBuilder = QueryBuilders.boolQuery();// 查询封装
articleboolQueryBuilder.should(QueryBuilders.matchPhraseQuery("text", articleTitle));
articleboolQueryBuilder.should(QueryBuilders.matchPhraseQuery("roottext", articleTitle));
boolQueryBuilder.must(articleboolQueryBuilder);
String[] fields = { "text", "roottext" };
HighlightBuilder highlightBuilder = ESQueryUtil.getTrackHighlightBuilder(fields, color);
SearchRequestBuilder searchRequestBuilder = transportClient.prepareSearch(ESConfig.weiboIndexName)
.setTypes(ESConfig.weiboType).highlighter(highlightBuilder).setQuery(boolQueryBuilder)
.addSort("time", SortOrder.DESC).setFrom(0);
try {
String startTime = TimeUtil.formatDateToMinute(new Date(start));
String endTime = TimeUtil.formatDateToMinute(new Date(end));
if (start != -1 && end == -1) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("time").from(startTime));
} else if (end != -1 && start == -1) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("time").to(endTime));
} else if (end != -1 && start != -1) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("time").from(startTime).to(endTime));
}
} catch (Exception e) {
e.printStackTrace();
}
SearchResponse response = searchRequestBuilder.execute().actionGet();
SearchHits searchHits = response.getHits();// 查询结果集
return searchHits;
}
@Override
public SearchHits getarticleTrackVideoFromEs(String articleTitle, String color, int count, long start, long end) {
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();// 查询封装
// 文章数条件
BoolQueryBuilder articleboolQueryBuilder = QueryBuilders.boolQuery();// 查询封装
articleboolQueryBuilder.should(QueryBuilders.matchPhraseQuery("title", articleTitle));
boolQueryBuilder.must(articleboolQueryBuilder);
String[] fields = { "title" };
HighlightBuilder highlightBuilder = ESQueryUtil.getTrackHighlightBuilder(fields, color);
SearchRequestBuilder searchRequestBuilder = transportClient.prepareSearch(ESConfig.videoIndexName)
.setTypes(ESConfig.videoType).highlighter(highlightBuilder).setQuery(boolQueryBuilder)
.addSort("rsid", SortOrder.DESC).setFrom(0).setSize(count);
if (start != -1 && end == -1) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("rsid").from(start));
} else if (end != -1 && start == -1) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("rsid").to(end));
} else if (end != -1 && start != -1) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("rsid").from(start).to(end));
}
SearchResponse response = searchRequestBuilder.execute().actionGet();
SearchHits searchHits = response.getHits();
return searchHits;
}
@Override
public SearchHits getarticleTrackMediaFromEs(String articleTitle, String color, int count, long start, long end,
String platformName) {
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();// 查询封装
// 文章数条件
BoolQueryBuilder articleboolQueryBuilder = QueryBuilders.boolQuery();// 查询封装
articleboolQueryBuilder.should(QueryBuilders.matchPhraseQuery("title", articleTitle));
articleboolQueryBuilder.should(QueryBuilders.matchPhraseQuery("content", articleTitle));
boolQueryBuilder.must(articleboolQueryBuilder);
String[] fields = { "title", "content" };
HighlightBuilder highlightBuilder = ESQueryUtil.getTrackHighlightBuilder(fields, color);
if (platformName.equals("网络媒体")) {
String[] types = { "网媒", "百度新闻", "360新闻", "google新闻", "插件标注", "一点资讯" };
boolQueryBuilder.must(ESQueryUtil.matchPhraseQueryWords(QueryBuilders.boolQuery(), ESQueryUtil.ES_QUERY_OR,
"type", types));
boolQueryBuilder.mustNot(QueryBuilders.matchQuery("type", "插件标注"));
} else if (platformName.equals("微信公众平台")) {
boolQueryBuilder.must(QueryBuilders.wildcardQuery("type", "微信*"));
} else if (platformName.equals("报刊")) {
boolQueryBuilder.must(QueryBuilders.matchPhraseQuery("type", "平媒"));
} else if (platformName.equals("今日头条")) {
boolQueryBuilder.must(QueryBuilders.matchPhraseQuery("type", "今日头条"));
} else if (platformName.equals("客户端")) {
boolQueryBuilder.must(QueryBuilders.boolQuery().should(QueryBuilders.wildcardQuery("type", "*客户端"))
.should(QueryBuilders.matchPhraseQuery("type", "脉脉")));
} else if (platformName.equals("论坛")) {
boolQueryBuilder.must(QueryBuilders.matchPhraseQuery("type", "论坛"));
} else if (platformName.equals("贴吧")) {
boolQueryBuilder.must(QueryBuilders.matchPhraseQuery("type", "贴吧"));
}
SearchRequestBuilder searchRequestBuilder = transportClient.prepareSearch(ESConfig.mediaIndexName)
.highlighter(highlightBuilder).setQuery(boolQueryBuilder).addSort("time", SortOrder.DESC).setFrom(0);
try {
String startTime = TimeUtil.formatEsDate(new Date(start));
String endTime = TimeUtil.formatEsDate(new Date(end));
if (start != -1 && end == -1) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("time").from(startTime));
} else if (end != -1 && start == -1) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("time").to(endTime));
} else if (end != -1 && start != -1) {
searchRequestBuilder.setPostFilter(QueryBuilders.rangeQuery("time").from(startTime).to(endTime));
}
} catch (Exception e) {
e.printStackTrace();
}
SearchResponse response = searchRequestBuilder.execute().actionGet();
SearchHits searchHits = response.getHits();// 查询结果集
return searchHits;
}
}
src/main/java/com/zhiwei/messageflow/es/service/ES4BeanService.java 0 → 100644
package com.zhiwei.messageflow.es.service;
import java.util.List;
import com.zhiwei.messageflow.bean.MediaMessage;
import com.zhiwei.messageflow.bean.VideoMessage;
import com.zhiwei.messageflow.bean.WeiboMessage;
import com.zhiwei.messageflow.bean.ZhihuMessage;
import com.zhiwei.messageflow.mongo.bean.NoiseRule;
//第二层 将SearchHits封装成List<Message>
public interface ES4BeanService {
/**
* 获取微博消息
*
* @param keys
* 关键词
* @param count
* 消息数量
* @param startid
* 开始id
* @param endid
* 结束id
* @param platform
* 平台
* @param project
* 项目
* @return
*/
List<WeiboMessage> getWeiboMessage(List<NoiseRule> noiseRules,List<String> keywords, int count, long startid, long endid, String platform,
String project);
/**
* 获取知乎消息
*
* @param keys
* 关键词
* @param count
* 消息数量
* @param startid
* 开始id
* @param endid
* 结束id
* @param platform
* 平台
* @param project
* 项目
* @return
*/
List<ZhihuMessage> getZhihuMessage(List<NoiseRule> noiseRules,List<String> keywords, int count, long startid, long endid, String platform,
String project);
/**
* 获取视频消息
*
* @param keys
* 关键词
* @param count
* 消息数量
* @param startid
* 开始id
* @param endid
* 结束id
* @param platform
* 平台
* @param project
* 项目
* @return
*/
List<VideoMessage> getVideoMessage(List<NoiseRule> noiseRules,List<String> keywords, int count, long startid, long endid, String platform,
String project);
/**
* 获取网媒消息
*
* @param keys
* 关键词
* @param count
* 消息数量
* @param startid
* 开始id
* @param endid
* 结束id
* @param platform
* 平台
* @param project
* 项目
* @return
*/
List<MediaMessage> getMediaMessage(List<NoiseRule> noiseRules,List<String> keywords, int count, long startid, long endid, String platform,
String project);
}
src/main/java/com/zhiwei/messageflow/es/service/EarlyWarningService.java 0 → 100644
package com.zhiwei.messageflow.es.service;
import java.util.List;
import com.zhiwei.messageflow.mongo.bean.TrackRule;
public interface EarlyWarningService {
/**
* 预警
*
* @param trackRules
* 预警规则
* @param count
* 预警消息数
* @param startid
* 开始rsid
* @param endid
* 结束rsid
* @param platformName
* 平台名
* @param projectName
* 项目名
*/
void earlyWarning(List<TrackRule> trackRules, int count, long startid, long endid, String platformName,
String projectName);
}
src/main/java/com/zhiwei/messageflow/es/service/HighLightFillingService.java 0 → 100644
package com.zhiwei.messageflow.es.service;
import org.elasticsearch.search.SearchHit;
import com.zhiwei.messageflow.bean.MediaMessage;
import com.zhiwei.messageflow.bean.VideoMessage;
import com.zhiwei.messageflow.bean.WeiboMessage;
import com.zhiwei.messageflow.bean.ZhihuMessage;
public interface HighLightFillingService {
/**
* 微博平台ES数据封装+高亮处理+渠道影响力
*
* @param searchHit
* @return
*/
WeiboMessage getWeiboBean(SearchHit searchHit);
/**
* 知乎平台ES数据封装+高亮处理+渠道影响力
*
* @param searchHit
* @return
*/
ZhihuMessage getZhihuBean(SearchHit searchHit);
/**
* 视频ES数据封装+高亮处理+渠道影响力
*
* @param searchHit
* @return
*/
VideoMessage getVideoBean(SearchHit searchHit);
/**
* 网媒平台ES数据封装+高亮处理+渠道影响力
*
* @param searchHit
* @return
*/
MediaMessage getMediaBean(SearchHit searchHit);
}
src/main/java/com/zhiwei/messageflow/es/service/NoiseProcessingService.java 0 → 100644
package com.zhiwei.messageflow.es.service;
import java.util.List;
import org.elasticsearch.search.SearchHits;
import com.zhiwei.messageflow.bean.MediaMessage;
import com.zhiwei.messageflow.bean.VideoMessage;
import com.zhiwei.messageflow.bean.WeiboMessage;
import com.zhiwei.messageflow.bean.ZhihuMessage;
import com.zhiwei.messageflow.mongo.bean.NoiseRule;
public interface NoiseProcessingService {
List<WeiboMessage> weiboDenoising(List<NoiseRule> noiseRules,SearchHits searchHits, String platform, String project);
List<ZhihuMessage> zhihuDenoising(List<NoiseRule> noiseRules,SearchHits searchHits, String platform, String project);
List<VideoMessage> videoDenoising(List<NoiseRule> noiseRules,SearchHits searchHits, String platform, String project);
List<MediaMessage> mediaDenoising(List<NoiseRule> noiseRules,SearchHits searchHits, String platform, String project);
}
src/main/java/com/zhiwei/messageflow/es/service/TrackHitAndWarnService.java 0 → 100644
package com.zhiwei.messageflow.es.service;
import org.elasticsearch.search.SearchHits;
import com.zhiwei.messageflow.mongo.bean.TrackRule;
public interface TrackHitAndWarnService {
/**
* 关键词预警数据获取
*
* @param truckRule
* @param count
* @param startid
* @param endid
* @param platformName
* @return
*/
SearchHits keyWordsTrackHit(TrackRule truckRule, int count, long startid, long endid, String platformName);
/**
* 渠道预警数据获取
*
* @param truckRule
* @param count
* @param startid
* @param endid
* @param platformName
* @return
*/
SearchHits channelTrackHit(TrackRule truckRule, int count, long startid, long endid, String platformName);
/**
* 相似新闻数预警数据获取
*
* @param truckRule
* @param count
* @param startid
* @param endid
* @param platformName
* @return
*/
SearchHits articleTrackHit(TrackRule truckRule, int count, long startid, long endid, String platformName);
/**
* 邮件预警
*
* @param trackHits
* @param platformName
* @param trackRule
*/
void WarnEmail(SearchHits trackHit, String platformName, TrackRule trackRule, String projectName);
/**
* 微信预警
*
* @param trackHits
* @param platformName
* @param trackRule
*/
void WarnWechat(SearchHits trackHit, String platformName, TrackRule trackRule, String projectName);
}
src/main/java/com/zhiwei/messageflow/es/service/impl/ES4BeanServiceImpl.java 0 → 100644
package com.zhiwei.messageflow.es.service.impl;
import java.util.List;
import org.elasticsearch.search.SearchHits;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import com.zhiwei.messageflow.bean.MediaMessage;
import com.zhiwei.messageflow.bean.VideoMessage;
import com.zhiwei.messageflow.bean.WeiboMessage;
import com.zhiwei.messageflow.bean.ZhihuMessage;
import com.zhiwei.messageflow.es.dao.ESDao;
import com.zhiwei.messageflow.es.service.ES4BeanService;
import com.zhiwei.messageflow.es.service.NoiseProcessingService;
import com.zhiwei.messageflow.mongo.bean.NoiseRule;
@Component
public class ES4BeanServiceImpl implements ES4BeanService {
public final static Logger log = LoggerFactory.getLogger(ES4BeanServiceImpl.class);
@Autowired
private ESDao esDao;
@Autowired
private NoiseProcessingService noiseProcessingService;
@Override
public List<WeiboMessage> getWeiboMessage(List<NoiseRule> noiseRules, List<String> keywords, int count,
long startid, long endid, String platform, String project) {
List<WeiboMessage> messages = null;
try {
// 查询数据库 获得searchHits
SearchHits searchHits = esDao.getWeiboDataFromEs(keywords, count, startid, endid, platform, project);
if (searchHits == null) {
return null;
}
// 去噪并封装
messages = noiseProcessingService.weiboDenoising(noiseRules, searchHits, platform, project);
} catch (Exception e) {
log.error(e.getStackTrace() + " " + e.getMessage());
}
return messages;
}
@Override
public List<ZhihuMessage> getZhihuMessage(List<NoiseRule> noiseRules, List<String> keywords, int count,
long startid, long endid, String platform, String project) {
List<ZhihuMessage> messages = null;
// 查询数据库 获得searchHits
SearchHits searchHits = esDao.getZhihuDataFromEs(keywords, count, startid, endid, platform, project);
if (searchHits == null) {
return null;
}
// 去噪并封装
messages = noiseProcessingService.zhihuDenoising(noiseRules, searchHits, platform, project);
return messages;
}
@Override
public List<VideoMessage> getVideoMessage(List<NoiseRule> noiseRules, List<String> keywords, int count,
long startid, long endid, String platform, String project) {
List<VideoMessage> messages = null;
// 查询数据库 获得searchHits
SearchHits searchHits = esDao.getVideoDataFromEs(keywords, count, startid, endid, platform, project);
if (searchHits == null) {
return null;
}
// 去噪并封装
messages = noiseProcessingService.videoDenoising(noiseRules, searchHits, platform, project);
return messages;
}
@Override
public List<MediaMessage> getMediaMessage(List<NoiseRule> noiseRules, List<String> keywords, int count,
long startid, long endid, String platform, String project) {
List<MediaMessage> messages = null;
// 查询数据库 获得searchHits
SearchHits searchHits = esDao.getMediaDataFromEs(keywords, count, startid, endid, platform, project);
if (searchHits == null) {
return null;
}
// 去噪并封装
messages = noiseProcessingService.mediaDenoising(noiseRules, searchHits, platform, project);
return messages;
}
}
src/main/java/com/zhiwei/messageflow/es/service/impl/EarlyWarningServiceImpl.java 0 → 100644
package com.zhiwei.messageflow.es.service.impl;
import java.util.List;
import org.elasticsearch.search.SearchHits;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import com.zhiwei.messageflow.es.service.EarlyWarningService;
import com.zhiwei.messageflow.es.service.TrackHitAndWarnService;
import com.zhiwei.messageflow.mongo.bean.TrackRule;
import com.zhiwei.messageflow.mongo.dao.TrackRuleDao;
@Component
public class EarlyWarningServiceImpl implements EarlyWarningService {
@Autowired
private TrackRuleDao trackRuleDao;
@Autowired
private TrackHitAndWarnService trackHitAndWarnService;
@Override
public void earlyWarning(List<TrackRule> trackRules, int count, long startid, long endid, String platformName,
String projectName) {
for (TrackRule trackRule : trackRules) {
// 判断是否预警
if (trackRule.isWarn()) {
continue;
}
/**
* 判断是否过期
*/
long createAt = trackRule.getCreateAt();
long addDay = 0;
if (trackRule.getEarlyWarningTime().equals("一天")) {
addDay = 24 * 60 * 60 * 1000L;
} else if (trackRule.getEarlyWarningTime().equals("二天")) {
addDay = 2 * 24 * 60 * 60 * 1000L;
} else if (trackRule.getEarlyWarningTime().equals("三天")) {
addDay = 3 * 24 * 60 * 60 * 1000L;
}
long confirm = createAt + addDay;
if (confirm <= System.currentTimeMillis()) {
// 过期
trackRule.setWarn(true);
trackRuleDao.updateTrackrule(trackRule.get_id());
} else {
// 判断规则
// 首先判断预警方式
if (trackRule.getEarlyWarning().equals("no")) {
} else if (trackRule.getEarlyWarning().equals("wechat")) {
// 微信预警
// 处理规则
SearchHits TrackHit = null;
if (trackRule.getRuleType().equals("keyWords")) {
// 关键词追踪
TrackHit = trackHitAndWarnService.keyWordsTrackHit(trackRule, count, startid, endid,
platformName);
} else if (trackRule.getRuleType().equals("channel")) {
// 渠道追踪
TrackHit = trackHitAndWarnService.channelTrackHit(trackRule, count, startid, endid,
platformName);
} else if (trackRule.getRuleType().equals("article")) {
// 相似新闻数追踪
TrackHit = trackHitAndWarnService.articleTrackHit(trackRule, count, startid, endid,
platformName);
}
trackHitAndWarnService.WarnWechat(TrackHit, platformName, trackRule, projectName);
} else if (trackRule.getEarlyWarning().equals("email")) {
// 邮箱预警
// 处理规则
SearchHits TrackHit = null;
if (trackRule.getRuleType().equals("keyWords")) {
// 关键词追踪
TrackHit = trackHitAndWarnService.keyWordsTrackHit(trackRule, count, startid, endid,
platformName);
} else if (trackRule.getRuleType().equals("channel")) {
// 渠道追踪
TrackHit = trackHitAndWarnService.channelTrackHit(trackRule, count, startid, endid,
platformName);
} else if (trackRule.getRuleType().equals("article")) {
// 相似新闻数追踪
TrackHit = trackHitAndWarnService.articleTrackHit(trackRule, count, startid, endid,
platformName);
}
trackHitAndWarnService.WarnEmail(TrackHit, platformName, trackRule, projectName);
}
}
}
}
}
src/main/java/com/zhiwei/messageflow/es/service/impl/HighLightFillingServiceImpl.java 0 → 100644
package com.zhiwei.messageflow.es.service.impl;
import java.io.IOException;
import java.util.Date;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.zhiwei.messageflow.bean.MediaMessage;
import com.zhiwei.messageflow.bean.VideoMessage;
import com.zhiwei.messageflow.bean.WeiboMessage;
import com.zhiwei.messageflow.bean.ZhihuMessage;
import com.zhiwei.messageflow.es.service.HighLightFillingService;
import com.zhiwei.messageflow.mongo.bean.HuserInfoMedia;
import com.zhiwei.messageflow.mongo.bean.HuserInfoWeibo;
import com.zhiwei.messageflow.mongo.bean.HuserInfoWeixin;
import com.zhiwei.messageflow.mongo.dao.HuserInfoDao;
import com.zhiwei.messageflow.util.TimeUtil;
@Component
public class HighLightFillingServiceImpl implements HighLightFillingService {
private final static Logger log = LoggerFactory.getLogger(HighLightFillingServiceImpl.class);
@Autowired
private HuserInfoDao huserInfoDao;
@Autowired
private ObjectMapper mapper;
@Override
public WeiboMessage getWeiboBean(SearchHit searchHit) {
/**
* ES数据读取
*/
Map<String, Object> sourceHitMap = searchHit.getSource();
Map<String, HighlightField> highlightFieldsHitMap = searchHit.getHighlightFields();
String message;
WeiboMessage wm = new WeiboMessage();
try {
message = mapper.writeValueAsString(sourceHitMap);
wm = mapper.readValue(message, WeiboMessage.class);
} catch (IOException e) {
e.printStackTrace();
}
String text = "";
String roottext = "";
Text[] textlist = highlightFieldsHitMap.containsKey("text") ? highlightFieldsHitMap.get("text").getFragments()
: null;
Text[] roottextlist = highlightFieldsHitMap.containsKey("roottext")
? highlightFieldsHitMap.get("roottext").getFragments()
: null;
if (textlist != null && textlist.length > 0) {
for (Text string : textlist) {
text = text + string.string();
}
}
if (roottextlist != null && roottextlist.length > 0) {
for (Text string : roottextlist) {
roottext = roottext + string.string();
}
}
if (text.isEmpty()) {
text = String.valueOf(sourceHitMap.get("text"));
}
if (roottext.isEmpty()) {
roottext = String.valueOf(sourceHitMap.get("roottext"));
}
wm.set_id(searchHit.getId());
wm.setText(text);
wm.setRoottext(roottext);
// 处理渠道影响力
HuserInfoWeibo huserInfoWeibo = huserInfoDao.getHuserInfoWeibo(String.valueOf(sourceHitMap.get("user_id")));
wm.setChannelIndex(huserInfoWeibo == null ? 1.0 : huserInfoWeibo.getChannelIndex());
return wm;
}
@Override
public ZhihuMessage getZhihuBean(SearchHit searchHit) {
Map<String, Object> sourceHitMap = searchHit.getSource();
Map<String, HighlightField> highlightFieldsHitMap = searchHit.getHighlightFields();
String message;
ZhihuMessage zm = new ZhihuMessage();
try {
message = mapper.writeValueAsString(sourceHitMap);
zm = mapper.readValue(message, ZhihuMessage.class);
} catch (IOException e) {
e.printStackTrace();
}
zm.set_id(searchHit.getId());
String insert_at = zm.getInsert_at();
if (insert_at != null && !insert_at.equals("") && !insert_at.replaceFirst("000", "").equals("")) {
Date insertDate = TimeUtil.parseTime(zm.getInsert_at().replaceFirst("000", ""),
"yyyy-MM-dd'T'HH:mm:ss.SSS");
if (insertDate != null)
zm.setInsert_at(insertDate.toString());
}
String update_at = zm.getUpdate_at();
if (update_at != null && !update_at.equals("") && !update_at.replaceFirst("000", "").equals("")) {
Date updateDate = TimeUtil.parseTime(zm.getUpdate_at().replaceFirst("000", ""),
"yyyy-MM-dd'T'HH:mm:ss.SSS");
if (updateDate != null)
zm.setUpdate_at(updateDate.toString());
}
String questionTitle = "";
String questionContent = "";
String answerContent = "";
String img = "";
String content1 = zm.getQuestion_content() == null ? "" : zm.getQuestion_content().toString();
String content2 = zm.getAnswer_content() == null ? "" : zm.getAnswer_content().toString();
if (!"".equals(content1)) {
img = regxString(content1);
}
if (!"".equals(content2)) {
img = regxString(content2);
}
Text[] titlelist = highlightFieldsHitMap.containsKey("question_title")
? highlightFieldsHitMap.get("question_title").getFragments()
: null;
Text[] questionContentlist = highlightFieldsHitMap.containsKey("question_content")
? highlightFieldsHitMap.get("question_content").getFragments()
: null;
Text[] answerContentList = highlightFieldsHitMap.containsKey("answer_content")
? highlightFieldsHitMap.get("answer_content").getFragments()
: null;
if (titlelist != null && titlelist.length > 0) {
for (Text string : titlelist) {
questionTitle = questionTitle + string.string();
}
}
if (questionContentlist != null && questionContentlist.length > 0) {
for (Text string : questionContentlist) {
// 通过高亮位置截取过长字符串
String cutStr = string.string();
if (cutStr.length() > 150) {
int i = cutStr.indexOf("<font");
if (i > 10) {
cutStr = cutStr.substring(i - 10, cutStr.length());
cutStr = "……" + cutStr;
}
}
questionContent = questionContent + cutStr;
}
}
if (answerContentList != null && answerContentList.length > 0) {
for (Text string : answerContentList) {
// 通过高亮位置截取过长字符串
String cutStr = string.string();
if (cutStr.length() > 150) {
int i = cutStr.indexOf("<font");
if (i > 10) {
cutStr = cutStr.substring(i - 10, cutStr.length());
cutStr = "……" + cutStr;
}
}
answerContent = answerContent + cutStr;
}
}
if (questionTitle.isEmpty()) {
questionTitle = String.valueOf(zm.getQuestion_title());
if (questionTitle.length() > 300) {
questionTitle = questionTitle.substring(0, 300);
}
}
if (questionContent.isEmpty()) {
questionContent = String.valueOf(zm.getQuestion_content());
if (questionContent.length() > 300) {
questionContent = questionContent.substring(0, 300);
}
}
if (answerContent.isEmpty()) {
answerContent = String.valueOf(zm.getAnswer_content());
if (answerContent.length() > 300) {
answerContent = answerContent.substring(0, 300);
}
}
zm.setQuestion_title(questionTitle.replaceAll("<img[^>]*>", ""));
zm.setQuestion_content(questionContent.replaceAll("<img[^>]*>", ""));
zm.setAnswer_content(answerContent.replaceAll("<img[^>]*>", ""));
zm.setImg(img);
if (zm.getImg_url() != null) {
String imgUrl = zm.getImg_url().toString();
if (null != imgUrl && "".equals(imgUrl)) {
zm.setImg(imgUrl);
}
}
return zm;
}
@Override
public VideoMessage getVideoBean(SearchHit searchHit) {
Map<String, Object> sourceHitMap = searchHit.getSource();
Map<String, HighlightField> highlightFieldsHitMap = searchHit.getHighlightFields();
String message;
VideoMessage vm = new VideoMessage();
try {
message = mapper.writeValueAsString(sourceHitMap);
vm = mapper.readValue(message, VideoMessage.class);
} catch (IOException e) {
e.printStackTrace();
}
vm.set_id(searchHit.getId());
try {
vm.setTime(TimeUtil
.parseTime(sourceHitMap.get("time").toString().replaceFirst("000", ""), "yyyy-MM-dd'T'HH:mm:ss")
.toString());
} catch (Exception e) {
log.error("videotime为空" + searchHit.getId());
}
String title = "";
Text[] titlelist = highlightFieldsHitMap.containsKey("title")
? highlightFieldsHitMap.get("title").getFragments()
: null;
if (titlelist != null && titlelist.length > 0) {
for (Text string : titlelist) {
// 通过高亮位置截取过长字符串
String cutStr = string.string();
title = title + cutStr;
}
}
if (title.isEmpty()) {
title = vm.getTitle();
}
vm.setTitle(title);
return vm;
}
@Override
public MediaMessage getMediaBean(SearchHit searchHit) {
Map<String, Object> sourceHitMap = searchHit.getSource();
Map<String, HighlightField> highlightFieldsHitMap = searchHit.getHighlightFields();
String message;
MediaMessage mm = new MediaMessage();
try {
message = mapper.writeValueAsString(sourceHitMap);
mm = mapper.readValue(message, MediaMessage.class);
} catch (IOException e) {
e.printStackTrace();
}
mm.set_id(searchHit.getId());
try {
mm.setTime(TimeUtil
.parseTime(sourceHitMap.get("time").toString().replaceFirst("000", ""), "yyyy-MM-dd'T'HH:mm:ss")
.toString());
} catch (Exception e) {
log.error("mediatime为空" + searchHit.getId());
return null;
}
String content = "";
String title = "";
Text[] titlelist = highlightFieldsHitMap.containsKey("title")
? highlightFieldsHitMap.get("title").getFragments()
: null;
Text[] contentlist = highlightFieldsHitMap.containsKey("content")
? highlightFieldsHitMap.get("content").getFragments()
: null;
if (titlelist != null && titlelist.length > 0) {
for (Text string : titlelist) {
// 通过高亮位置截取过长字符串
String cutStr = string.string();
title = title + cutStr;
}
}
if (contentlist != null && contentlist.length > 0) {
for (Text string : contentlist) {
// 通过高亮位置截取过长字符串
String cutStr = string.string();
if (cutStr.length() > 150) {
int i = cutStr.indexOf("<font");
if (i > 10) {
cutStr = cutStr.substring(i - 10, cutStr.length());
cutStr = "……" + cutStr;
}
}
content = content + cutStr;
}
}
if (content.isEmpty()) {
content = sourceHitMap.containsKey("content") ? sourceHitMap.get("content") + "" : "";
if (content.length() > 300) {
content = content.substring(0, 300);
}
}
if (title.isEmpty()) {
title = mm.getTitle();
}
mm.setContent(content);
mm.setTitle(title);
if ((sourceHitMap.get("pt") + "").equals("微信")) {
HuserInfoWeixin huserInfoWeixin = huserInfoDao.getHuserInfoWeixin(mm.getSource());
if (huserInfoWeixin != null) {
mm.setChannelIndex(huserInfoWeixin.getChannelIndex());
} else {
mm.setChannelIndex(1.0);
}
} else {
HuserInfoMedia huserInfoMedia = huserInfoDao.getHuserInfoMedia(mm.getSource());
if (huserInfoMedia != null) {
mm.setChannelIndex(huserInfoMedia.getChannelIndex());
} else {
mm.setChannelIndex(1.0);
}
}
return mm;
}
private String regxString(String str) {
Pattern p = Pattern.compile("[http]{4}[:0-9a-zA-Z_/.]+.png");
Matcher m = p.matcher(str);
while (m.find()) {
return m.group();
}
return "";
}
}
src/main/java/com/zhiwei/messageflow/es/service/impl/NoiseProcessingServiceImpl.java 0 → 100644
package com.zhiwei.messageflow.es.service.impl;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import com.zhiwei.messageflow.bean.MediaMessage;
import com.zhiwei.messageflow.bean.VideoMessage;
import com.zhiwei.messageflow.bean.WeiboMessage;
import com.zhiwei.messageflow.bean.ZhihuMessage;
import com.zhiwei.messageflow.es.service.HighLightFillingService;
import com.zhiwei.messageflow.es.service.NoiseProcessingService;
import com.zhiwei.messageflow.mongo.bean.NoiseRule;
@Component
public class NoiseProcessingServiceImpl implements NoiseProcessingService {
@Autowired
HighLightFillingService highLightFillingService;
@Override
public List<WeiboMessage> weiboDenoising(List<NoiseRule> noiseRules, SearchHits searchHits, String platform,
String project) {
List<WeiboMessage> wmessages = new ArrayList<>();
SearchHit[] sh = searchHits.getHits();
for (SearchHit searchHit : sh) {
Map<String, Object> map = searchHit.getSource();
if (map == null) {
continue;
}
String text = map.get("text") != null ? map.get("text").toString() : null;
String roottext = map.get("roottext") != null ? map.get("roottext").toString() : null;
// 噪音判断参数
boolean isnoise = false;
if (noiseRules != null) {
for (NoiseRule n : noiseRules) {
/**
* 判断噪音规则
*/
if (n.getRuleType().equals("keyWords")
&& (n.getKeyWordPt().equals(platform) || n.getKeyWordPt().equals("全部"))) {
/**
* 关键词去噪
*/
String kone = n.getKeyWordsInputOne();
String ktwo = n.getKeyWordsInputTwo();
if (n.getAndOr().equals("且")) {
if (text != null && roottext != null
&& (text.indexOf(kone) != -1 || roottext.indexOf(kone) != -1)
&& (text.indexOf(ktwo) != -1 || roottext.indexOf(ktwo) != -1)) {
isnoise = true;
break;
} else if (text != null && roottext == null && (text.indexOf(kone) != -1)
&& (text.indexOf(ktwo) != -1)) {
isnoise = true;
break;
} else if (text == null && roottext == null) {
isnoise = true;
break;
}
} else if (n.getAndOr().equals("或")) {
if (text != null && roottext != null
&& ((text.indexOf(kone) != -1 || roottext.indexOf(kone) != -1)
|| (text.indexOf(ktwo) != -1 || roottext.indexOf(ktwo) != -1))) {
isnoise = true;
break;
} else if (text != null && roottext == null
&& ((text.indexOf(kone) != -1) || (text.indexOf(ktwo) != -1))) {
isnoise = true;
break;
} else if (text == null && roottext == null) {
isnoise = true;
break;
}
} else {
if (text != null && roottext != null
&& (text.indexOf(kone) != -1 || roottext.indexOf(kone) != -1)) {
isnoise = true;
break;
} else if (text != null && roottext == null && (text.indexOf(kone) != -1)) {
isnoise = true;
break;
} else if (text == null && roottext == null) {
isnoise = true;
break;
}
}
} else if ((n.getRuleType().equals("channel"))
&& ((n.getChannelPt().equals("微博") || n.getChannelPt().equals("全部")))) {
/**
* 渠道去噪
*/
String channelQd = n.getChannelQd();
String source = map.get("username").toString();
source = source.trim();
source = source.replaceAll(" ", "");
if (channelQd.equals(source)) {
isnoise = true;
break;
}
}
}
} // 噪音判断完成
if (isnoise) {
continue;
}
// 数据转换并高亮处理
WeiboMessage wm = highLightFillingService.getWeiboBean(searchHit);
if (wm != null) {
wmessages.add(wm);
}
}
return wmessages;
}
@Override
public List<ZhihuMessage> zhihuDenoising(List<NoiseRule> noiseRules, SearchHits searchHits, String platform,
String project) {
List<ZhihuMessage> zmessages = new ArrayList<>();
SearchHit[] sh = searchHits.getHits();
for (SearchHit searchHit : sh) {
Map<String, Object> map = searchHit.getSource();
if (map == null) {
continue;
}
String text = map.get("question_title") != null ? map.get("question_title").toString() : null;
String roottext = map.get("answer_content") != null ? map.get("answer_content").toString() : null;
// 噪音判断参数
boolean isnoise = false;
if (noiseRules != null) {
for (NoiseRule n : noiseRules) {
/**
* 判断噪音规则
*/
if (n.getRuleType().equals("keyWords")
&& (n.getKeyWordPt().equals(platform) || n.getKeyWordPt().equals("全部"))) {
/**
* 关键词去噪
*/
// searchHit.getSource().get("text").toString().indexOf(str);
String kone = n.getKeyWordsInputOne();
String ktwo = n.getKeyWordsInputTwo();
if (n.getAndOr().equals("且")) {
if (text != null && roottext != null
&& (text.indexOf(kone) != -1 || roottext.indexOf(kone) != -1)
&& (text.indexOf(ktwo) != -1 || roottext.indexOf(ktwo) != -1)) {
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
isnoise = true;
break;
} else if (text != null && roottext == null && (text.indexOf(kone) != -1)
&& (text.indexOf(ktwo) != -1)) {
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
isnoise = true;
break;
} else if (text == null && roottext == null) {
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
isnoise = true;
break;
}
} else if (n.getAndOr().equals("或")) {
if (text != null && roottext != null
&& ((text.indexOf(kone) != -1 || roottext.indexOf(kone) != -1)
|| (text.indexOf(ktwo) != -1 || roottext.indexOf(ktwo) != -1))) {
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
isnoise = true;
break;
} else if (text != null && roottext == null
&& ((text.indexOf(kone) != -1) || (text.indexOf(ktwo) != -1))) {
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
isnoise = true;
break;
} else if (text == null && roottext == null) {
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
isnoise = true;
break;
}
} else {
if (text != null && roottext != null
&& (text.indexOf(kone) != -1 || roottext.indexOf(kone) != -1)) {
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
isnoise = true;
break;
} else if (text != null && roottext == null && (text.indexOf(kone) != -1)) {
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
isnoise = true;
break;
} else if (text == null && roottext == null) {
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
isnoise = true;
break;
}
}
} else if ((n.getRuleType().equals("channel"))
&& ((n.getChannelPt().equals("知乎") || n.getChannelPt().equals("全部")))) {
/**
* 渠道去噪
*/
String channelQd = n.getChannelQd();
String source = map.get("user_name") == null ? "知乎提问" : map.get("user_name").toString();
source = source.trim();
source = source.replaceAll(" ", "");
if (channelQd.equals(source)) {
isnoise = true;
break;
}
}
}
} // 噪音判断完成
if (isnoise) {
continue;
}
// 数据转换并高亮处理
ZhihuMessage zm = highLightFillingService.getZhihuBean(searchHit);
if (zm != null) {
zmessages.add(zm);
}
}
return zmessages;
}
@Override
public List<VideoMessage> videoDenoising(List<NoiseRule> noiseRules, SearchHits searchHits, String platform,
String project) {
List<VideoMessage> vmessages = new ArrayList<>();
SearchHit[] sh = searchHits.getHits();
for (SearchHit searchHit : sh) {
Map<String, Object> map = searchHit.getSource();
if (map == null) {
continue;
}
String text = map.get("title") != null ? map.get("title").toString() : null;
String roottext = null;
// 噪音判断参数
boolean isnoise = false;
if (noiseRules != null) {
for (NoiseRule n : noiseRules) {
/**
* 判断噪音规则
*/
if (n.getRuleType().equals("keyWords")
&& (n.getKeyWordPt().equals(platform) || n.getKeyWordPt().equals("全部"))) {
/**
* 关键词去噪
*/
// searchHit.getSource().get("text").toString().indexOf(str);
String kone = n.getKeyWordsInputOne();
String ktwo = n.getKeyWordsInputTwo();
if (n.getAndOr().equals("且")) {
if (text != null && roottext == null && (text.indexOf(kone) != -1)
&& (text.indexOf(ktwo) != -1)) {
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
isnoise = true;
break;
} else if (text == null && roottext == null) {
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
isnoise = true;
break;
}
} else if (n.getAndOr().equals("或")) {
if (text != null && roottext != null
&& ((text.indexOf(kone) != -1 || roottext.indexOf(kone) != -1)
|| (text.indexOf(ktwo) != -1 || roottext.indexOf(ktwo) != -1))) {
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
isnoise = true;
break;
} else if (text != null && roottext == null
&& ((text.indexOf(kone) != -1) || (text.indexOf(ktwo) != -1))) {
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
isnoise = true;
break;
} else if (text == null && roottext == null) {
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
isnoise = true;
break;
}
} else {
if (text != null && roottext != null
&& (text.indexOf(kone) != -1 || roottext.indexOf(kone) != -1)) {
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
isnoise = true;
break;
} else if (text != null && roottext == null && (text.indexOf(kone) != -1)) {
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
isnoise = true;
break;
} else if (text == null && roottext == null) {
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
isnoise = true;
break;
}
}
} else if ((n.getRuleType().equals("channel"))
&& ((n.getChannelPt().equals("视频") || n.getChannelPt().equals("全部")))) {
/**
* 渠道去噪
*/
String channelQd = n.getChannelQd();
String source = map.get("uper") != null ? map.get("uper").toString() : "";
source = source.trim();
source = source.replaceAll(" ", "");
if (channelQd.equals(source)) {
isnoise = true;
break;
}
}
}
} // 噪音判断完成
if (isnoise) {
continue;
}
// 数据转换并高亮处理
VideoMessage vm = highLightFillingService.getVideoBean(searchHit);
if (vm != null) {
vmessages.add(vm);
}
}
return vmessages;
}
@Override
public List<MediaMessage> mediaDenoising(List<NoiseRule> noiseRules, SearchHits searchHits, String platform,
String project) {
List<MediaMessage> mmessages = new ArrayList<>();
SearchHit[] sh = searchHits.getHits();
for (SearchHit searchHit : sh) {
Map<String, Object> map = searchHit.getSource();
if (map == null) {
continue;
}
String text = map.get("title") != null ? map.get("title").toString() : null;
String roottext = map.get("content") != null ? map.get("content").toString() : null;
boolean isnoise = false;
if (noiseRules != null) {
for (NoiseRule n : noiseRules) {
/**
* 判断噪音规则
*/
if (n.getRuleType().equals("keyWords")
&& (n.getKeyWordPt().equals(platform) || n.getKeyWordPt().equals("全部"))) {
/**
* 关键词去噪
*/
// searchHit.getSource().get("text").toString().indexOf(str);
String kone = n.getKeyWordsInputOne();
String ktwo = n.getKeyWordsInputTwo();
if (n.getAndOr().equals("且")) {
if (text != null && roottext != null
&& (text.indexOf(kone) != -1 || roottext.indexOf(kone) != -1)
&& (text.indexOf(ktwo) != -1 || roottext.indexOf(ktwo) != -1)) {
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
isnoise = true;
break;
} else if (text != null && roottext == null && (text.indexOf(kone) != -1)
&& (text.indexOf(ktwo) != -1)) {
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
isnoise = true;
break;
} else if (text == null && roottext == null) {
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
isnoise = true;
break;
}
} else if (n.getAndOr().equals("或")) {
if (text != null && roottext != null
&& ((text.indexOf(kone) != -1 || roottext.indexOf(kone) != -1)
|| (text.indexOf(ktwo) != -1 || roottext.indexOf(ktwo) != -1))) {
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
isnoise = true;
break;
} else if (text != null && roottext == null
&& ((text.indexOf(kone) != -1) || (text.indexOf(ktwo) != -1))) {
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
isnoise = true;
break;
} else if (text == null && roottext == null) {
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
isnoise = true;
break;
}
} else {
if (text != null && roottext != null
&& (text.indexOf(kone) != -1 || roottext.indexOf(kone) != -1)) {
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
isnoise = true;
break;
} else if (text != null && roottext == null && (text.indexOf(kone) != -1)) {
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
isnoise = true;
break;
} else if (text == null && roottext == null) {
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
isnoise = true;
break;
}
}
} else if ((n.getRuleType().equals("channel"))
&& ((n.getChannelPt() == platform) || n.getChannelPt().equals("全部"))) {
/**
* 渠道去噪
*/
String channelQd = n.getChannelQd();
String source = map.get("source").toString();
source = source.trim();
source = source.replaceAll(" ", "");
if (channelQd.equals(source)) {
isnoise = true;
break;
}
}
}
} // 噪音判断完成
if (isnoise) {
continue;
}
// 数据转换并高亮处理
MediaMessage mm = highLightFillingService.getMediaBean(searchHit);
if (mm != null) {
mmessages.add(mm);
}
}
return mmessages;
}
}
src/main/java/com/zhiwei/messageflow/es/service/impl/TrackHitAndWarnServiceImpl.java 0 → 100644
package com.zhiwei.messageflow.es.service.impl;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import com.zhiwei.messageflow.es.bean.TemplateData;
import com.zhiwei.messageflow.es.dao.TrackESDao;
import com.zhiwei.messageflow.es.service.TrackHitAndWarnService;
import com.zhiwei.messageflow.mongo.bean.TrackRule;
import com.zhiwei.messageflow.mongo.dao.TrackRuleDao;
import com.zhiwei.messageflow.mongo.service.UserMailService;
import com.zhiwei.messageflow.mongo.service.WechatSendService;
import com.zhiwei.messageflow.util.TimeUtil;
@Component
public class TrackHitAndWarnServiceImpl implements TrackHitAndWarnService {
@Autowired
private TrackESDao trackESDao;
@Autowired
TrackRuleDao trackRuleDao;
@Autowired
private UserMailService userMailService;
@Autowired
private WechatSendService wechatSendService;
@Override
public SearchHits keyWordsTrackHit(TrackRule truckRule, int count, long startid, long endid, String platformName) {
SearchHits searchHits = null;
String anyWord = "";
String allWords = "";
// 取关键词
if (truckRule.getAndOr().equals("或")) {
anyWord = truckRule.getKeyWordsInputOne() + "，" + truckRule.getKeyWordsInputTwo();
} else if (truckRule.getAndOr().equals("且")) {
allWords = truckRule.getKeyWordsInputOne() + "，" + truckRule.getKeyWordsInputTwo();
} else {
allWords = truckRule.getKeyWordsInputOne();
}
// 取高亮颜色
String color = "#ff330D";
if (null != truckRule.getHighlighted() && "" != truckRule.getHighlighted()) {
color = truckRule.getHighlighted();
}
// 根据平台获取数据
if (platformName.equals("微博")) {
searchHits = trackESDao.getkeyWordsTrackWeiboFromEs(anyWord, allWords, color, count, startid, endid);
} else if (platformName.equals("知乎")) {
searchHits = trackESDao.getkeyWordsTrackZhihuFromEs(anyWord, allWords, color, count, startid, endid);
} else if (platformName.equals("视频")) {
searchHits = trackESDao.getkeyWordsTrackVideoFromEs(anyWord, allWords, color, count, startid, endid);
} else {
searchHits = trackESDao.getkeyWordsTrackMediaFromEs(anyWord, allWords, color, count, startid, endid,
platformName);
}
return searchHits;
}
@Override
public SearchHits channelTrackHit(TrackRule truckRule, int count, long startid, long endid, String platformName) {
SearchHits searchHits = null;
// 取渠道平台
String pt = truckRule.getChannelPt();
String channel = truckRule.getChannelQd();
// 取高亮颜色
String color = "#ff330D";
if (null != truckRule.getHighlighted() && "" != truckRule.getHighlighted()) {
color = truckRule.getHighlighted();
}
if (pt.equals("微博")) {
searchHits = trackESDao.getchannelTrackWeiboFromEs(channel, color, count, startid, endid);
} else if (pt.equals("知乎")) {
searchHits = trackESDao.getchannelTrackZhihuFromEs(channel, color, count, startid, endid);
} else if (pt.equals("视频")) {
searchHits = trackESDao.getchannelTrackVideoFromEs(channel, color, count, startid, endid);
} else {
searchHits = trackESDao.getchannelTrackMediaFromEs(channel, color, count, startid, endid, platformName);
}
return searchHits;
}
@Override
public SearchHits articleTrackHit(TrackRule truckRule, int count, long startid, long endid, String platformName) {
SearchHits searchHits = null;
// 取相似新闻数信息
String articleTitle = truckRule.getArticleTitle();
String timeRange = truckRule.getTimeRange();
long createAt = truckRule.getCreateAt();
// 算时间区段
if (timeRange.equals("3小时内")) {
startid = createAt;
endid = createAt + 3 * 60 * 60 * 1000L;
} else if (timeRange.equals("6小时内")) {
startid = createAt;
endid = createAt + 6 * 60 * 60 * 1000L;
} else if (timeRange.equals("12小时内")) {
startid = createAt;
endid = createAt + 12 * 60 * 60 * 1000L;
} else if (timeRange.equals("24小时内")) {
startid = createAt;
endid = createAt + 24 * 60 * 60 * 1000L;
}
// 取高亮颜色
String color = "#ff330D";
if (null != truckRule.getHighlighted() && "" != truckRule.getHighlighted()) {
color = truckRule.getHighlighted();
}
if (platformName.equals("微博")) {
searchHits = trackESDao.getarticleTrackWeiboFromEs(articleTitle, color, count, startid, endid);
} else if (platformName.equals("知乎")) {
searchHits = null;
} else if (platformName.equals("视频")) {
searchHits = trackESDao.getarticleTrackVideoFromEs(articleTitle, color, count, startid, endid);
;
} else {
searchHits = trackESDao.getarticleTrackMediaFromEs(articleTitle, color, count, startid, endid,
platformName);
}
return searchHits;
}
@Override
public void WarnEmail(SearchHits trackHit, String platformName, TrackRule trackRule, String projectName) {
for (SearchHit hit : trackHit) {
String url = "";
String title = "";
String content = "";
String source = "";
String time = "";
String pt = "";
int fensi = -1;
if (platformName.equals("微博")) {
url = hit.getSource().get("url") + "";
title = "";
content = hit.getSource().get("text") + "";
source = hit.getSource().get("username") + "";
time = hit.getSource().get("time") + "";
pt = "微博";
fensi = Integer.valueOf(hit.getSource().get("fensi") + "");
} else if (platformName.equals("知乎")) {
url = hit.getId() + "";
title = hit.getSource().get("question_title") + "";
content = hit.getSource().get("answer_content") + "";
source = hit.getSource().get("user_name") + "";
time = hit.getSource().get("created_at") + "";
pt = "知乎";
} else if (platformName.equals("视频")) {
url = hit.getId() + "";
title = hit.getSource().get("title") + "";
content = "";
source = hit.getSource().get("uper") + "";
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
String timein = String.valueOf(hit.getSource().get("time"));
String timeout = timein.replaceAll("T", " ");
try {
Date date = sdf.parse(timeout);
try {
// "2017-09-28 20:35:33"
Date mediadate = TimeUtil.parseTime(TimeUtil.formatTime(TimeUtil.addTimeHour(date, 8)),
"yyyy-MM-dd HH:mm:ss");
time = sdf.format(mediadate);
} catch (Exception e) {
time = sdf.format(System.currentTimeMillis());
}
} catch (ParseException e) {
e.printStackTrace();
}
pt = "视频";
} else {
String trueid = hit.getSource().get("id") != null ? String.valueOf(hit.getSource().get("id"))
: String.valueOf(hit.getId());
url = trueid;
title = hit.getSource().get("title") + "";
content = hit.getSource().get("content") + "";
source = hit.getSource().get("source") + "";
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
String timein = String.valueOf(hit.getSource().get("time"));
String timeout = timein.replaceAll("T", " ");
try {
Date date = sdf.parse(timeout);
try {
// "2017-09-28 20:35:33"
Date mediadate = TimeUtil.parseTime(TimeUtil.formatTime(TimeUtil.addTimeHour(date, 8)),
"yyyy-MM-dd HH:mm:ss");
time = sdf.format(mediadate);
} catch (Exception e) {
time = sdf.format(System.currentTimeMillis());
}
} catch (ParseException e) {
e.printStackTrace();
}
pt = platformName;
}
TemplateData templateData = new TemplateData();
if (trackRule.getRuleType().equals("keyWords")) {
// 关键词追踪
templateData.setWordRule(trackRule.getRuleExplain());
} else if (trackRule.getRuleType().equals("channel")) {
// 渠道追踪
templateData.setChannelRule(trackRule.getRuleExplain());
}
templateData.setUrl(url);
templateData.setTitle(title);
templateData.setContent(content);
templateData.setSource(source);
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm");
Date datetime;
try {
datetime = sdf.parse(time);
templateData.setTime(datetime);
} catch (ParseException e) {
e.printStackTrace();
templateData.setTime(new Date());
}
templateData.setPt(pt);
if (fensi != -1) {
templateData.setFensi(fensi);
}
Map<String, Object> result = new HashMap<String, Object>();
result = userMailService.sendMailByProject(projectName, templateData);
if ((result.get("message") + "").equals("用户组邮件发送成功")) {
System.out.println("发送邮件预警成功");
trackRule.setWarn(true);
trackRuleDao.updateTrackrule(trackRule.get_id());
break;
} else if ((result.get("message") + "").equals("用户组邮箱未获取到")) {
System.out.println("发送邮件预警失败，用户获取问题");
} else {
System.out.println("本次部分条发送失败");
trackRule.setWarn(true);
trackRuleDao.updateTrackrule(trackRule.get_id());
break;
}
}
}
@Override
public void WarnWechat(SearchHits trackHit, String platformName, TrackRule trackRule, String projectName) {
for (SearchHit searchHit : trackHit) {
String url = "";
String title = "";
String content = "";
String source = "";
String time = "";
String pt = "";
int fensi = -1;
if (platformName.equals("微博")) {
url = searchHit.getSource().get("url") + "";
title = "";
content = searchHit.getSource().get("text") + "";
source = searchHit.getSource().get("username") + "";
time = searchHit.getSource().get("time") + "";
pt = "微博";
fensi = Integer.valueOf(searchHit.getSource().get("fensi") + "");
} else if (platformName.equals("知乎")) {
url = searchHit.getId() + "";
title = searchHit.getSource().get("question_title") + "";
content = searchHit.getSource().get("answer_content") + "";
source = searchHit.getSource().get("user_name") + "";
time = searchHit.getSource().get("created_at") + "";
pt = "知乎";
} else if (platformName.equals("视频")) {
url = searchHit.getId() + "";
title = searchHit.getSource().get("title") + "";
content = "";
source = searchHit.getSource().get("uper") + "";
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
String timein = String.valueOf(searchHit.getSource().get("time"));
String timeout = timein.replaceAll("T", " ");
try {
Date date = sdf.parse(timeout);
try {
// "2017-09-28 20:35:33"
Date mediadate = TimeUtil.parseTime(TimeUtil.formatTime(TimeUtil.addTimeHour(date, 8)),
"yyyy-MM-dd HH:mm:ss");
time = sdf.format(mediadate);
} catch (Exception e) {
time = sdf.format(System.currentTimeMillis());
}
} catch (ParseException e) {
e.printStackTrace();
}
pt = "视频";
} else {
String trueid = searchHit.getSource().get("id") != null
? String.valueOf(searchHit.getSource().get("id"))
: String.valueOf(searchHit.getId());
url = trueid;
title = searchHit.getSource().get("title") + "";
content = searchHit.getSource().get("content") + "";
source = searchHit.getSource().get("source") + "";
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
String timein = String.valueOf(searchHit.getSource().get("time"));
String timeout = timein.replaceAll("T", " ");
try {
Date date = sdf.parse(timeout);
try {
// "2017-09-28 20:35:33"
Date mediadate = TimeUtil.parseTime(TimeUtil.formatTime(TimeUtil.addTimeHour(date, 8)),
"yyyy-MM-dd HH:mm:ss");
time = sdf.format(mediadate);
} catch (Exception e) {
time = sdf.format(System.currentTimeMillis());
}
} catch (ParseException e) {
e.printStackTrace();
}
pt = platformName;
}
TemplateData templateData = new TemplateData();
if (trackRule.getRuleType().equals("keyWords")) {
// 关键词追踪
templateData.setWordRule(trackRule.getRuleExplain());
} else if (trackRule.getRuleType().equals("channel")) {
// 渠道追踪
templateData.setChannelRule(trackRule.getRuleExplain());
}
templateData.setUrl(url);
templateData.setTitle(title);
templateData.setContent(content);
templateData.setSource(source);
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm");
Date datetime;
try {
datetime = sdf.parse(time);
templateData.setTime(datetime);
} catch (ParseException e) {
e.printStackTrace();
templateData.setTime(new Date());
}
templateData.setPt(pt);
if (fensi != -1) {
templateData.setFensi(fensi);
}
int size = wechatSendService.sendTemplateByProject(projectName, templateData);
if (size == -1) {
// result.put("message", "发送微信预警失败，用户获取问题");
// result.put("status", false);
System.out.println("发送微信预警失败，用户获取问题");
} else if (size > 0) {
// result.put("message", "本次微信预警有"+size+"条发送失败");
// result.put("status", false);
System.out.println("本次微信预警有" + size + "条发送失败");
} else {
// result.put("message", "发送微信预警成功");
// result.put("status", true);
System.out.println("发送微信预警成功");
trackRule.setWarn(true);
trackRuleDao.updateTrackrule(trackRule.get_id());
break;
}
}
}
}
src/main/java/com/zhiwei/messageflow/getmessage/MatchingInfo.java 0 → 100644
package com.zhiwei.messageflow.getmessage;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import org.springframework.stereotype.Component;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.zhiwei.messageflow.getmessage.bean.Media;
import com.zhiwei.messageflow.getmessage.bean.Video;
import com.zhiwei.messageflow.getmessage.bean.Weibo;
import com.zhiwei.messageflow.getmessage.bean.ZhihuInfo;
import redis.clients.jedis.Tuple;
@Component
@SuppressWarnings("unchecked")
public class MatchingInfo {
private ObjectMapper mapper = new ObjectMapper();
SimpleDateFormat sdf = new SimpleDateFormat("E MMM dd HH:mm:ss z yyyy", Locale.US);
public List<Weibo> weiboMatchingInfo(Set<Tuple> set) {
List<Weibo> weibos = new ArrayList<>();
Map<String, Object> map = new HashMap<>();
for (Tuple t : set) {
Weibo weibo = new Weibo();
try {
map = mapper.readValue(t.getElement(), Map.class);
if (map.get("username") != null)
weibo.setUsername(map.get("username").toString());
if (map.get("roottext") != null)
weibo.setRoottext(map.get("roottext").toString());
if (map.get("rootname") != null)
weibo.setRootname(map.get("rootname").toString());
if (map.get("root_time") != null)
weibo.setRoot_time(map.get("root_time").toString());
if (map.get("rooturl") != null)
weibo.setRooturl(map.get("rooturl").toString());
if (map.get("img_url") != null)
weibo.setImg_url(map.get("img_url").toString());
if (map.get("channelIndex") != null)
weibo.setChannelIndex(Double.parseDouble(map.get("channelIndex").toString()));
if (map.get("retweet_status_id") != null)
weibo.setRetweet_status_id(map.get("retweet_status_id").toString());
if (map.get("weibo") != null)
weibo.setWeibo(Integer.parseInt(map.get("weibo").toString()));
if (map.get("isForward") != null)
weibo.setIsForward(Integer.parseInt(map.get("isForward").toString()));
if (map.get("fensi") != null)
weibo.setFensi(Integer.parseInt(map.get("fensi").toString()));
if (map.get("vtype") != null)
weibo.setVtype((Integer.parseInt(map.get("vtype").toString())));
if (map.get("_id") != null)
weibo.setId(map.get("_id").toString());
if (map.get("rstime") != null)
weibo.setRstime(Long.parseLong(map.get("rstime").toString()));
if (map.get("text") != null)
weibo.setText(map.get("text").toString());
if (map.get("user_id") != null)
weibo.setUser_id((map.get("user_id").toString()));
if (map.get("url") != null)
weibo.setUrl(map.get("url").toString());
if (map.get("time") != null)
weibo.setTime(map.get("time").toString());
System.out.println(weibo.toString());
weibos.add(weibo);
} catch (IOException e) {
e.printStackTrace();
}
}
return weibos;
}
public List<ZhihuInfo> zhihuMatchingInfo(Set<Tuple> set) {
List<ZhihuInfo> zhihuInfos = new ArrayList<>();
Map<String, Object> map = new HashMap<>();
for (Tuple t : set) {
ZhihuInfo zhihuInfo = new ZhihuInfo();
try {
map = mapper.readValue(t.getElement(), Map.class);
if (map.get("_id") != null)
zhihuInfo.setId(map.get("_id").toString());
if (map.get("rsid") != null)
zhihuInfo.setRsid(Long.parseLong(map.get("rsid").toString()));
if (map.get("type") != null)
zhihuInfo.setType(map.get("type").toString());
if (map.get("user_name") != null)
zhihuInfo.setUser_name(map.get("user_name").toString());
if (map.get("user_url") != null)
zhihuInfo.setUser_url(map.get("user_url").toString());
if (map.get("answer_count") != null)
zhihuInfo.setAnswer_count(Integer.parseInt(map.get("answer_count").toString()));
if (map.get("question_url") != null)
zhihuInfo.setQuestion_url(map.get("question_url").toString());
if (map.get("question_title") != null)
zhihuInfo.setQuestion_title(map.get("question_title").toString());
if (map.get("answer_content") != null)
zhihuInfo.setAnswer_content(map.get("answer_content").toString());
if (map.get("answer_url") != null)
zhihuInfo.setAnswer_url(map.get("answer_url").toString());
if (map.get("img") != null)
zhihuInfo.setImg(map.get("img").toString());
if (map.get("follow") != null)
zhihuInfo.setFollow(map.get("follow").toString());
if (map.get("browse") != null)
zhihuInfo.setBrowse(map.get("browse").toString());
if (map.get("insert_at") != null) {
Date d = sdf.parse(map.get("insert_at").toString());
zhihuInfo.setInsert_at(d);
}
if (map.get("created_at") != null) {
Date d = sdf.parse(map.get("created_at").toString());
zhihuInfo.setInsert_at(d);
}
System.out.println(zhihuInfo.toString());
zhihuInfos.add(zhihuInfo);
} catch (IOException | ParseException e) {
e.printStackTrace();
}
}
return zhihuInfos;
}
public List<Video> videoMatchingInfo(Set<Tuple> set) {
List<Video> videos = new ArrayList<>();
Map<String, Object> map = new HashMap<>();
for (Tuple t : set) {
Video video = new Video();
try {
map = mapper.readValue(t.getElement(), Map.class);
if (map.get("pt") != null)
video.setPt(map.get("pt").toString());
if (map.get("_id") != null)
video.setId(map.get("_id").toString());
if (map.get("rsid") != null)
video.setRsid(Long.parseLong(map.get("rsid").toString()));
if (map.get("title") != null)
video.setTitle(map.get("title").toString());
if (map.get("source") != null)
video.setSource(map.get("source").toString());
if (map.get("url") != null)
video.setUrl(map.get("url").toString());
if (map.get("uper") != null)
video.setUper(map.get("uper").toString());
if (map.get("time") != null) {
Date d = sdf.parse(map.get("time").toString());
video.setTime(d);
}
System.out.println(video.toString());
videos.add(video);
} catch (IOException | ParseException e) {
e.printStackTrace();
}
}
return videos;
}
public List<Media> mediaMatchingInfo(Set<Tuple> set) {
List<Media> medias = new ArrayList<>();
Map<String, Object> map = new HashMap<>();
for (Tuple t : set) {
Media media = new Media();
try {
map = mapper.readValue(t.getElement(), Map.class);
if (map.get("type") != null)
media.setType(map.get("type").toString());
if (map.get("pt") != null)
media.setPt(map.get("pt").toString());
if (map.get("_id") != null)
media.setId(map.get("_id").toString());
if (map.get("rsid") != null)
media.setRsid(Long.parseLong(map.get("rsid").toString()));
if (map.get("title") != null)
media.setTitle(map.get("title").toString());
if (map.get("source") != null)
media.setSource(map.get("source").toString());
if (map.get("content") != null)
media.setContent(map.get("content").toString());
if (map.get("channelIndex") != null)
media.setChannelIndex(Double.parseDouble(map.get("channelIndex").toString()));
if (map.get("time") != null) {
Date d = sdf.parse(map.get("time").toString());
media.setTime(d);
}
System.out.println(media.toString());
medias.add(media);
} catch (IOException | ParseException e) {
e.printStackTrace();
}
}
return medias;
}
}
src/main/java/com/zhiwei/messageflow/getmessage/bean/Media.java 0 → 100644
package com.zhiwei.messageflow.getmessage.bean;
import java.util.Date;
import org.springframework.data.mongodb.core.mapping.Document;
import lombok.Data;
import lombok.ToString;
@Data
@ToString
@Document(collection = "net_media")
public class Media {
private String id;// 标志id,在数据库中指的是url
private String title;// 标题
private String content;// 内容
private String source;// 来源
private String rootSource;// 来源
private Date time;// 时间
private String savetime;// 保存时间
private String type;// 类别
private String pt;// 所属平台
private long rsid;// 自增id
private Double channelIndex;
private String emotionType;// 情感分类
private String eventType;// 事件分类
private String dangerLevel;// 危险等级
private String markGroup;// 标记组
private String markPerson;// 标记人
}
src/main/java/com/zhiwei/messageflow/getmessage/bean/Video.java 0 → 100644
package com.zhiwei.messageflow.getmessage.bean;
import java.util.Date;
import lombok.Data;
import lombok.ToString;
@Data
@ToString
public class Video {
private String id;// 视频
private String videoLength;// videoLength 视频时长
private Date time;// time date 发布时间
private String title;// title 视频标题
private String url;// url 视频链接
private String content;// content 视频简介
private String uper;// uper 视频上传者
private String pt;// pt 平台
private String browseCount;// browseCount 视频观看数
private long rsid;// rsid 视频rsid
private String source;// source 视频来源
private String savetime;// 保存时间
}
src/main/java/com/zhiwei/messageflow/getmessage/bean/Weibo.java 0 → 100644
package com.zhiwei.messageflow.getmessage.bean;
import lombok.Data;
import lombok.ToString;
@Data
@ToString
public class Weibo {
private String id;// 微博
private String time;// 发微博时间
private String day;
private String month;
private String hour;
private String text;// 微博内容
private long rstime;
private String user_id;// 用户id
private String username;// 微博用户昵称
private String source;// 微博来源
private int retweet_count;// 转发数
private int reply_count;// 评论数
private String url;// 微博地址
private String img_url;// 用户头像图片地址
private String weibo_img;// weibo含有的图片
private int vtype;// 认证类型
private int fensi;// 粉丝数
private int guanzhu;// 关注数
private int weibo;// 微博数
private String description;// 微博简介
private int isForward;// 原创还是转发
private String roottext;// 原创微博内容
private String retweet_status_id;
private String rootuid;// root用户id
private String rootname;// root用户的昵称
private String root_time;// root用户的发微博时间
private String rooturl;// root微博地址
private String rootsource;// 来源
private int root_repost;
private int root_reply;
private Double channelIndex;
private String emotionType;// 情感分类
private String eventType;// 事件分类
private String dangerLevel;// 危险等级
private String markTag;// 新版标签信息
private String markGroup;// 标记组
private String markPerson;// 标记人
private String markDate;// 标记日期
private int isMarked;// 是否标记
private String sex;// 性别
private String location;// 地点
public Weibo() {
}
}
src/main/java/com/zhiwei/messageflow/getmessage/bean/ZhihuInfo.java 0 → 100644
package com.zhiwei.messageflow.getmessage.bean;
import java.util.Date;
import org.springframework.data.mongodb.core.mapping.Document;
import lombok.Data;
import lombok.ToString;
@Data
@ToString
@Document(collection = "zhihu")
public class ZhihuInfo {
private String id;// 主键 问题地址或答案地址
private String question_title;// 问题标题
private String question_content;// 问题标题
private String question_url;// 问题地址
private String answer_content;// 回答内容
private String answer_url;// 答案地址
private String user_name;// 回答者昵称
private String user_url;// 回答者主页
private int answer_count;// 问题回答数
private int attitudes_count;// 回答点赞数
private int comments_count;// 评论数
private String created_at;// 问题或回答创建时间
private Date insert_at;// 问题或答案第一次采集到的时间
private Date update_at;// 答案点赞书最后更新时间
private String type;// 消息类型 answer or question
private long rsid;// 信息自增键
private long question_id;// 问题id
private String img;// 图片
private String follow;
private String browse;
}
src/main/java/com/zhiwei/messageflow/getmessage/getSet.java 0 → 100644
package com.zhiwei.messageflow.getmessage;
import java.util.List;
import java.util.Set;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import com.zhiwei.messageflow.getmessage.bean.ZhihuInfo;
import com.zhiwei.messageflow.redis.RedisPoolAndTools;
import redis.clients.jedis.Tuple;
@Component
public class getSet {
@Autowired
private RedisPoolAndTools redisPoolAndTools;
@Autowired
private MatchingInfo matchingInfo;
@SuppressWarnings("unused")
public Set<Tuple> getDataByeRedis(String redisTools,String keyValue,int page,int pageSize,int newDataCount) {
newDataCount=0;
page=1;
pageSize=20;
keyValue="知乎-美团-全部";
long startIndex = ((page - 1) * pageSize) + newDataCount;
long endIndex = startIndex + pageSize - 1;
Set<Tuple> set = null;
try {
set = redisPoolAndTools.getRevDataListByIndex(keyValue, startIndex, endIndex);
} catch (Exception e) {
e.printStackTrace();
}
// List<Media> medias= matchingInfo.mediaMatchingInfo(set);
// List<Weibo> weibos= matchingInfo.weiboMatchingInfo(set);
// List<Video> videos= matchingInfo.videoMatchingInfo(set);
List<ZhihuInfo> zhihuInfos= matchingInfo.zhihuMatchingInfo(set);
return set;
}
}
src/main/java/com/zhiwei/messageflow/listener/ApplicationContextProvider.java 0 → 100644
package com.zhiwei.messageflow.listener;
import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.stereotype.Component;
@Component
public class ApplicationContextProvider implements ApplicationContextAware{
private static ApplicationContext context;
private ApplicationContextProvider(){}
@Override
public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
context = applicationContext;
}
public static <T> T getBean(String name,Class<T> aClass){
return context.getBean(name,aClass);
}
}
src/main/java/com/zhiwei/messageflow/mail/MailSenderInfo.java 0 → 100644
package com.zhiwei.messageflow.mail;
import java.util.Properties;
public class MailSenderInfo
{
// 发送服务器的IP和端口
private String mailServerHost;
private String mailServerPort = "25";
// 邮件发送者地址
private String fromAddress;
private String nickName;
// 邮件接受者的地址
private String toAddress;
// 登录邮件发送服务器的用户名和密码
private String userName;
private String password;
// 是否需要身份验证
private boolean validate = false;
// 邮件主题
private String subject;
// 邮件的文本内容
private String content;
// 邮件附件的文件名
private String[] attachFileNames;
/*
* 获得邮件会话属性
*/
public Properties getProperties()
{
Properties p = new Properties();
p.put("mail.smtp.host", this.mailServerHost);
p.put("mail.smtp.port", this.mailServerPort);
p.put("mail.smtp.auth", validate ? "true" : "false");
return p;
}
public String getMailServerHost()
{
return mailServerHost;
}
public void setMailServerHost(String mailServerHost)
{
this.mailServerHost = mailServerHost;
}
public String getMailServerPort()
{
return mailServerPort;
}
public void setMailServerPort(String mailServerPort)
{
this.mailServerPort = mailServerPort;
}
public String getFromAddress()
{
return fromAddress;
}
public void setFromAddress(String fromAddress)
{
this.fromAddress = fromAddress;
}
public String getToAddress()
{
return toAddress;
}
public void setToAddress(String toAddress)
{
this.toAddress = toAddress;
}
public String getUserName()
{
return userName;
}
public void setUserName(String userName)
{
this.userName = userName;
}
public String getPassword()
{
return password;
}
public void setPassword(String password)
{
this.password = password;
}
public boolean isValidate()
{
return validate;
}
public void setValidate(boolean validate)
{
this.validate = validate;
}
public String getSubject()
{
return subject;
}
public void setSubject(String subject)
{
this.subject = subject;
}
public String getContent()
{
return content;
}
public void setContent(String content)
{
this.content = content;
}
public String[] getAttachFileNames()
{
return attachFileNames;
}
public void setAttachFileNames(String[] attachFileNames)
{
this.attachFileNames = attachFileNames;
}
public String getNickName() {
return nickName;
}
public void setNickName(String nickName) {
this.nickName = nickName;
}
}
src/main/java/com/zhiwei/messageflow/mail/MyAuthenticator.java 0 → 100644
package com.zhiwei.messageflow.mail;
import javax.mail.Authenticator;
import javax.mail.PasswordAuthentication;
public class MyAuthenticator extends Authenticator{
String userName=null;//用户名
String password=null;//密码
/*
* 无参构造
*/
public MyAuthenticator(){}
/*
* 代参构造
*/
public MyAuthenticator(String username,String cs){
this.userName=username;
this.password=cs;
}
/*
*
*/
protected PasswordAuthentication getPasswordAuthentication(){
return new PasswordAuthentication(userName,password);
}
}
src/main/java/com/zhiwei/messageflow/mail/SendMail.java 0 → 100644
package com.zhiwei.messageflow.mail;
import com.zhiwei.messageflow.es.bean.TemplateData;
import com.zhiwei.messageflow.util.TimeUtil;
public class SendMail {
/**
* @Title: sendMail
* @Description: TODO(发送邮件)
* @param mailContent
* @param email
* 邮件目标用户 设定文件
* @return boolean 返回类型
*/
public static boolean sendMailWarning(String email, TemplateData templateData) {
// 这个类主要是设置邮件
MailSenderInfo mailInfo = new MailSenderInfo();
mailInfo.setMailServerHost("smtp.ym.163.com");
// mailInfo.setMailServerHost("smtp.163.com");
mailInfo.setMailServerPort("25");
mailInfo.setValidate(true);
mailInfo.setUserName("qbjcyj@zhiweidata.com");
mailInfo.setPassword("1q2w3e4r");// 您的邮箱密码
mailInfo.setFromAddress("qbjcyj@zhiweidata.com");
mailInfo.setNickName("知微情报监测服务");
boolean f = false;
mailInfo.setToAddress(email);
mailInfo.setSubject("知微情报监测预警");
mailInfo.setContent(mailContent(templateData));
// 这个类主要来发送邮件
f = SimpleMailSender.sendHtmlMail(mailInfo);// 发送文体格式
return f;
}
// /**
// * @Title: sendMailBriefing
// * @Description: TODO(发送舆情简报)
// * @param email
// * @param mailTitle 邮件标题
// * @param briefs
// * @return 设定文件
// * @date 2017年11月16日 下午5:48:14
// * @return boolean 返回类型
// */
// public static boolean sendMailBriefing(String email, String
// mailTitle,List<Brief> briefs) {
// // 这个类主要是设置邮件
// MailSenderInfo mailInfo = new MailSenderInfo();
// mailInfo.setMailServerHost("smtp.ym.163.com");
// // mailInfo.setMailServerHost("smtp.163.com");
// mailInfo.setMailServerPort("25");
// mailInfo.setValidate(true);
// mailInfo.setUserName("qbjcyj@zhiweidata.com");
// mailInfo.setPassword("1q2w3e4r");// 您的邮箱密码
// mailInfo.setFromAddress("qbjcyj@zhiweidata.com");
// mailInfo.setNickName("知微情报监测服务");
//
// boolean f = false;
// mailInfo.setToAddress(email);
// mailInfo.setSubject(mailTitle);
//
// mailInfo.setContent(mailContent(briefs));
// // 这个类主要来发送邮件
// f = SimpleMailSender.sendHtmlMail(mailInfo);// 发送文体格式
// return f;
// }
/**
*
* @Title: mailContent
* @Description: TODO(拼接所发邮件内容)
* @param templateData
* @return 设定文件
* @date 2017年11月8日 下午4:29:41
* @return String 返回类型
*/
private static String mailContent(TemplateData templateData) {
StringBuffer sb = new StringBuffer();
sb.append("<style> </style> ");
sb.append("<div>你好，收到一条" + templateData.getPt() + "的预警通知，预警内容如下：</div>");
if ("新闻客户端".equals(templateData.getPt()) || "网媒".equals(templateData.getPt())
|| "微信".equals(templateData.getPt()) || "报刊".equals(templateData.getPt())
|| "今日头条".equals(templateData.getPt())) {
sb.append("<div>渠道：" + templateData.getSource() + "</div><br>");
sb.append("<div>平台：网络媒体</div><br>");
sb.append("<div>发布时间："
+ getHotLink(TimeUtil.formatDate(templateData.getTime(), "yyyy-MM-dd HH:mm"), templateData.getUrl())
+ "</div><br>");
sb.append("<div>标题：" + templateData.getTitle() + "</div><br>");
sb.append("<div>摘要：" + templateData.getContent() + "</div><br>");
} else if ("微博".equals(templateData.getPt())) {
sb.append("<div>微博账号：" + templateData.getSource() + "(粉丝量：" + dealFensi(templateData.getFensi())
+ ")</div><br>");
sb.append("<div>发布时间："
+ getHotLink(TimeUtil.formatDate(templateData.getTime(), "yyyy-MM-dd HH:mm"), templateData.getUrl())
+ "</div><br>");
sb.append("<div>发布内容：" + templateData.getContent() + "</div><br>");
} else if ("知乎".equals(templateData.getPt()) || "论坛".equals(templateData.getPt())
|| "贴吧".equals(templateData.getPt())) {
sb.append("<div>问题：" + templateData.getTitle() + "</div><br>");
sb.append("<div>回答用户：" + templateData.getSource() + "</div><br>");
sb.append("<div>回答时间："
+ getHotLink(TimeUtil.formatDate(templateData.getTime(), "yyyy-MM-dd HH:mm"), templateData.getUrl())
+ "</div><br>");
sb.append("<div>答案：" + templateData.getContent() + "</div><br>");
} else {
sb.append("<div>来源：" + templateData.getSource() + "</div><br>");
sb.append("<div>平台：" + templateData.getPt() + "</div><br>");
sb.append("<div>发布时间："
+ getHotLink(TimeUtil.formatDate(templateData.getTime(), "yyyy-MM-dd HH:mm"), templateData.getUrl())
+ "</div><br>");
sb.append("<div>标题：" + templateData.getTitle() + "</div><br>");
sb.append("<div>摘要：" + templateData.getContent() + "</div><br>");
}
if (null != templateData.getChannelRule()) {
sb.append("<div>命中渠道规则：" + templateData.getChannelRule() + "</div><br>");
}
if (null != templateData.getWordRule()) {
sb.append("<div>命中关键词规则：" + templateData.getWordRule() + "</div><br>");
}
sb.append("<div>Best Regards，</div>");
sb.append("<div>知微——情报监测</div>");
sb.append("<style>*{font-family:Microsoft YaHei;}</style>");
return sb.toString();
}
// /**
// *
// * @Title: mailContent
// * @Description: TODO(拼接所发邮件内容)
// * @param templateData
// * @return 设定文件
// * @date 2017年11月8日 下午4:29:41
// * @return String 返回类型
// */
// private static String mailContent(List<Brief> briefs) {
// StringBuffer sb = new StringBuffer();
//
// Brief brief;
// for (int i = 0; i < briefs.size(); i++) {
// brief = briefs.get(i);
// sb.append("<div><B>" +getHotLink(brief.getTitle(), brief.getUrl())
// +"</B></div><br>");
// sb.append("<div>" + brief.getDetails() + "</div><br>");
// sb.append("<br>");
// }
// sb.append("<div>Best Regards，</div>");
// sb.append("<div>知微——情报监测</div>");
// sb.append("<style>*{ font-size:18px} *{font-family:Microsoft
// YaHei;}</style>");
//
// return sb.toString();
// }
/**
* @Title: getHotLink
* @Description: TODO(这里用一句话描述这个方法的作用)
* @param relation2
* 设定文件
* @return String 返回类型
*/
private static String getHotLink(String name, String url) {
return "<a target='_blank' href='" + url + "'>" + name + "</a>";
}
/**
*
* @Title: dealFensi
* @Description: TODO(处理粉丝量)
* @param fensi
* @return 设定文件
* @date 2017年11月8日 下午4:10:12
* @return String 返回类型
*/
private static String dealFensi(int fensi) {
String fensiStr = "";
if (fensi < 10000) {
fensiStr = fensi + "";
} else {
fensiStr = fensi / 10000 + "万";
}
return fensiStr;
}
}
src/main/java/com/zhiwei/messageflow/mail/SimpleMailSender.java 0 → 100644
package com.zhiwei.messageflow.mail;
import java.io.UnsupportedEncodingException;
import java.util.Date;
import java.util.Properties;
import javax.mail.Address;
import javax.mail.BodyPart;
import javax.mail.Message;
import javax.mail.MessagingException;
import javax.mail.Multipart;
import javax.mail.Session;
import javax.mail.Transport;
import javax.mail.internet.InternetAddress;
import javax.mail.internet.MimeBodyPart;
import javax.mail.internet.MimeMessage;
import javax.mail.internet.MimeMultipart;
public class SimpleMailSender
{
/*
* 以文本的形式发送邮件 param mailInfo待发送的邮件信息
*/
public boolean sendTextMail(MailSenderInfo mailInfo)
{
// 判断是否需要身份验证֤
MyAuthenticator authenticator = null;
Properties pro = mailInfo.getProperties();
if (mailInfo.isValidate())
{
// 如果需要身份验证，则创建一个密码验证器
authenticator = new MyAuthenticator(mailInfo.getUserName(),
mailInfo.getPassword());
}
// 根据邮件会话属性和密码验证器构造一个发送邮件的session
Session sendMailSession = Session.getInstance(pro, authenticator);
try
{
// 根据Session创建一个邮件消息
Message mailMessage = new MimeMessage(sendMailSession);
// 创建邮件发送者地址ߵ�ַ
Address from = new InternetAddress(mailInfo.getFromAddress());
//设置自定义发件人昵称
if (mailInfo.getNickName()!=null) {
String nick="";
try {
nick=javax.mail.internet.MimeUtility.encodeText(mailInfo.getNickName());
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
}
mailMessage.setFrom(new InternetAddress(nick+" <"+mailInfo.getFromAddress()+">"));
}else {
// 设置邮件消息的发送者
mailMessage.setFrom(from);
}
// 创建邮件的接收者地址，并设置到邮件消息中
Address to = new InternetAddress(mailInfo.getToAddress());
mailMessage.setRecipient(Message.RecipientType.TO, to);
// 设置邮件消息主题
mailMessage.setSubject(mailInfo.getSubject());
// 设置邮件消息发送的时间
mailMessage.setSentDate(new Date());
// 设置邮件消息发送的时间
String mailContent = mailInfo.getContent();
mailMessage.setText(mailContent);
// 发送邮件
Transport.send(mailMessage);
return true;
}
catch (Exception e)
{
e.printStackTrace();
}
return false;
}
/*
* 以HTML的格式发送邮件
*/
public static boolean sendHtmlMail(MailSenderInfo mailInfo)
{
// 判断是否需要身份认证
MyAuthenticator authenticator = null;
Properties pro = mailInfo.getProperties();
// 如果需要身份认证，则创建一个密码验证器
if (mailInfo.isValidate())
{
authenticator = new MyAuthenticator(mailInfo.getUserName(),
mailInfo.getPassword());
}
// 根据邮件会话属性和密码验证器构造一个发送邮件的session
Session sendMailSession = Session
.getDefaultInstance(pro, authenticator);
try
{
// 根据session创建一个邮件消息
Message mailMessage = new MimeMessage(sendMailSession);
// 创建邮件发送者地址
Address from = new InternetAddress(mailInfo.getFromAddress());
//设置自定义发件人昵称
if (mailInfo.getNickName()!=null) {
String nick="";
try {
nick=javax.mail.internet.MimeUtility.encodeText(mailInfo.getNickName());
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
}
mailMessage.setFrom(new InternetAddress(nick+" <"+mailInfo.getFromAddress()+">"));
}else {
// 设置邮件消息的发送者
mailMessage.setFrom(from);
}
// 创建邮件的接收者地址，并设置到邮件消息中
Address to = new InternetAddress(mailInfo.getToAddress());
// Message.RecipientType.TO属性表示接收者的类型为TO
mailMessage.setRecipient(Message.RecipientType.TO, to);
// 设置邮件消息的主题
mailMessage.setSubject(mailInfo.getSubject());
// 设置邮件消息发送的时间
mailMessage.setSentDate(new Date());
// MiniMultipart类是一个容器类，包含MimeBodyPart类型的对象
Multipart mainPart = new MimeMultipart();
// 创建一个包含HTML内容的MimeBodyPart
BodyPart html = new MimeBodyPart();
// 设置HTML内容
html.setContent(mailInfo.getContent(), "text/html; charset=utf-8");
mainPart.addBodyPart(html);
// 将MiniMultipart对象设置为邮件内容
mailMessage.setContent(mainPart);
// 发送邮件
Transport.send(mailMessage);
return true;
}
catch (MessagingException ex)
{
ex.printStackTrace();
}
return false;
}
}
src/main/java/com/zhiwei/messageflow/mongo/MongoFactoryBean.java 0 → 100644
//package com.zhiwei.messageflow.mongo;
//
//import org.springframework.beans.factory.annotation.Autowired;
//import org.springframework.data.mongodb.MongoDbFactory;
//import org.springframework.data.mongodb.core.MongoTemplate;
//import org.springframework.data.mongodb.core.SimpleMongoDbFactory;
//import org.springframework.stereotype.Component;
//
//import com.mongodb.MongoClient;
//import com.mongodb.MongoClientOptions;
//import com.mongodb.MongoCredential;
//import com.mongodb.ServerAddress;
//import com.zhiwei.messageflow.config.MongoConfig;
//
//@Component
//public class MongoFactoryBean {
//
// @Autowired
// private static MongoConfig mongoConfig;
//
// public MongoDbFactory mongoDbFactory (){
//
// /**
// * 客户端配置（连接数、副本集群验证）
// */
// MongoClientOptions.Builder builder = new MongoClientOptions.Builder();
// System.out.println(mongoConfig.toString());
// builder.connectionsPerHost(mongoConfig.getConnectionsPerHost());
// builder.connectTimeout(mongoConfig.getConnectTimeout());
// builder.threadsAllowedToBlockForConnectionMultiplier(
// mongoConfig.getThreadsAllowedToBlockForConnectionMultiplier());
// builder.maxWaitTime(mongoConfig.getMaxWaitTime());
// MongoClientOptions mongoClientOptions = builder.build();
//
// // MongoDB地址列表
// ServerAddress serverAddress = new ServerAddress(mongoConfig.getServerIP(), mongoConfig.getServerport());
//
// // 连接认证
// MongoCredential mongoCredential = MongoCredential.createScramSha1Credential(mongoConfig.getServerUsername(),
// mongoConfig.getAuthenticationDatabase(), mongoConfig.getServerPassword().toCharArray());
//
// //创建客户端和Factory
// MongoClient mongoClient = new MongoClient(serverAddress, mongoCredential, mongoClientOptions);
// MongoDbFactory mongoDbFactory = new SimpleMongoDbFactory(mongoClient, mongoConfig.getServerDBName());
//
// return mongoDbFactory;
// }
//
// public MongoTemplate mongoTemplate() {
// return new MongoTemplate(mongoDbFactory());
// }
//}
\ No newline at end of file
src/main/java/com/zhiwei/messageflow/mongo/bean/HuserInfoMedia.java 0 → 100644
package com.zhiwei.messageflow.mongo.bean;
import org.springframework.data.mongodb.core.mapping.Document;
import lombok.Data;
import lombok.ToString;
@Data
@ToString
@Document(collection = "HuserInfoMedia")
public class HuserInfoMedia {
private String _id;
private Integer pr;
private Integer aleax;
private Integer monthAVGAleax;
private Long monthPV;
private Integer H;
private String source;
private Double pv;
private Double normalization;
private Integer mediaType;
private Double channelIndex;
}
src/main/java/com/zhiwei/messageflow/mongo/bean/HuserInfoWeibo.java 0 → 100644
package com.zhiwei.messageflow.mongo.bean;
import org.springframework.data.mongodb.core.mapping.Document;
import lombok.Data;
import lombok.ToString;
@Data
@ToString
@Document(collection = "HuserInfoWeibo")
public class HuserInfoWeibo {
private String _id;
private String username;
private Integer followsCount;
private Integer weiboCount;
private Integer friendsCound;
private String vType;
private Integer H;
private Double channelIndex;
}
src/main/java/com/zhiwei/messageflow/mongo/bean/HuserInfoWeixin.java 0 → 100644
package com.zhiwei.messageflow.mongo.bean;
import org.springframework.data.mongodb.core.mapping.Document;
import lombok.Data;
import lombok.ToString;
@Data
@ToString
@Document(collection = "HuserInfoWeixin")
public class HuserInfoWeixin {
private String _id;
private String uid;
private String username;
private String levelOne;
private String levelTwo;
private String openid;
private Integer avgReadCount;
private Integer avgLikeCount;
private Integer H;
private Double channelIndex;
}
src/main/java/com/zhiwei/messageflow/mongo/bean/KeywordNew.java 0 → 100644
package com.zhiwei.messageflow.mongo.bean;
import java.util.List;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.mapping.Document;
import lombok.Data;
import lombok.ToString;
@Data
@ToString
@Document(collection="qbjc_keywordNew")
public class KeywordNew {
@Id
private String _id;
private String keyTitle;// 关键词组
private List<String> keyWords;//关键词
private String project;// 项目
private String submitter;
private long createAt;
}
src/main/java/com/zhiwei/messageflow/mongo/bean/NoiseRule.java 0 → 100644
package com.zhiwei.messageflow.mongo.bean;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.mapping.Document;
import lombok.Data;
import lombok.ToString;
@Data
@ToString
@Document(collection = "qbjc_noiserule")
public class NoiseRule {
@Id
private long _id;
private String ruleName;
private String keyWordPt;
private String ruleExplain;
private String ruleType;
private String keyWordsInputOne;
private String andOr;
private String keyWordsInputTwo;
private String project;
private long createAt;
private String submitter;
private String channelPt;
private String channelQd;
}
src/main/java/com/zhiwei/messageflow/mongo/bean/Platform.java 0 → 100644
package com.zhiwei.messageflow.mongo.bean;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.mapping.Document;
import lombok.Data;
import lombok.ToString;
@Data
@ToString
@Document(collection = "qbjc_platform")
public class Platform{
@Id
private long _id;
private String Platform;
private String PlatformName;
private String Type;
private long createAt;
}
src/main/java/com/zhiwei/messageflow/mongo/bean/Project.java 0 → 100644
package com.zhiwei.messageflow.mongo.bean;
import java.util.List;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.mapping.Document;
import lombok.Data;
import lombok.ToString;
@Data
@ToString
@Document(collection = "qbjc_project")
public class Project {
/**
* id
*/
@Id
private long _id;
/**
* 项目名
*/
private String projectName;
private int keyGroupLimit;
private int keywordsLimit;
private int traceLimit;
private int noiseLimit;
private int userLimit;
private int tagLimit;
private long createAt;
private boolean search;
private int searchLimit;
/**
* 私有平台列表
*/
private List<String> dataPt;
private boolean weiboHotSearch;
private int gatherKeyWord;
private boolean download;
}
src/main/java/com/zhiwei/messageflow/mongo/bean/TrackRule.java 0 → 100644
package com.zhiwei.messageflow.mongo.bean;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.mapping.Document;
import lombok.Data;
import lombok.ToString;
@Data
@ToString
@Document(collection = "qbjc_trackrule")
public class TrackRule {
@Id
private long _id;
private String ruleName;
private String ruleExplain;
private String ruleType;
private String KeyWordsInputOne;
private String andOr;
private String KeyWordsInputTwo;
private String earlyWarning;
private String highlighted;
private String earlyWarningTime;
private boolean isWarn;
private long createAt;
private String submitter;
private String project;
private String channelPt;
private String channelQd;
private String articleTitle;
private String articleNum;
private String timeRange;
}
src/main/java/com/zhiwei/messageflow/mongo/bean/UserMail.java 0 → 100644
package com.zhiwei.messageflow.mongo.bean;
import java.util.Date;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.mapping.Document;
import lombok.Data;
import lombok.ToString;
@Data
@ToString
@Document(collection = "qbjc_user_mail")
public class UserMail {
@Id
private String _id;
private String address;
private String project;
private Date createAt;
}
src/main/java/com/zhiwei/messageflow/mongo/bean/UserWechat.java 0 → 100644
package com.zhiwei.messageflow.mongo.bean;
import java.util.Date;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.mapping.Document;
import lombok.Data;
import lombok.ToString;
@Data
@ToString
@Document(collection = "qbjc_user_wechat")
public class UserWechat {
@Id
private String _id;
private String nickname;
private Long uid;
private String username;
private String project;
private String appId;
private String openid;
private Date createAt;
}
src/main/java/com/zhiwei/messageflow/mongo/bean/WechatCode.java 0 → 100644
package com.zhiwei.messageflow.mongo.bean;
import java.util.Date;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.mapping.Document;
import lombok.Data;
import lombok.ToString;
@Data
@ToString
@Document(collection = "wechatCode")
public class WechatCode {
@Id
private String _id;
private String appsecret;
private String token;
private String encodingAESKey;
private String accessToken;
private Date expiresTime;
private Date updateTime;
}
src/main/java/com/zhiwei/messageflow/mongo/config/AbstractMongoConfig.java 0 → 100644
package com.zhiwei.messageflow.mongo.config;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.MongoDbFactory;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.SimpleMongoDbFactory;
import com.mongodb.MongoClient;
import com.mongodb.MongoClientOptions;
import com.mongodb.MongoCredential;
import com.mongodb.ServerAddress;
import com.zhiwei.messageflow.config.MongoConfig;
import lombok.Data;
@Data
public abstract class AbstractMongoConfig {
private String database;
private String username;
private String password;
private String host;
private String authenticationDatabase;
private Integer port;
@Autowired
private MongoConfig mongoConfig;
/**
* 创建mongoDbFactory
*
* @return
* @throws Exception
*/
public MongoDbFactory mongoDbFactory() throws Exception {
ServerAddress serverAddress = new ServerAddress(host, port);
MongoCredential mongoCredential = MongoCredential.createCredential(username, authenticationDatabase,
password.toCharArray());
MongoClientOptions.Builder builder = new MongoClientOptions.Builder();
builder.connectionsPerHost(mongoConfig.getConnectionsPerHost());
builder.connectTimeout(mongoConfig.getConnectTimeout());
builder.threadsAllowedToBlockForConnectionMultiplier(
mongoConfig.getThreadsAllowedToBlockForConnectionMultiplier());
builder.maxWaitTime(mongoConfig.getMaxWaitTime());
MongoClientOptions mongoClientOptions = builder.build();
return new SimpleMongoDbFactory(new MongoClient(serverAddress, mongoCredential, mongoClientOptions), database);
}
/**
* 获取mongoTemplate
*
* @return
* @throws Exception
*/
abstract public MongoTemplate getMongoTemplate() throws Exception;
}
src/main/java/com/zhiwei/messageflow/mongo/config/PrimaryMongoConfig.java 0 → 100644
package com.zhiwei.messageflow.mongo.config;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import org.springframework.data.mongodb.core.MongoTemplate;
@Configuration
@ConfigurationProperties(prefix = "spring.data.mongodb.primary")
public class PrimaryMongoConfig extends AbstractMongoConfig{
@Primary
@Override
public @Bean(name = "primaryMongoTemplate") MongoTemplate getMongoTemplate() throws Exception {
return new MongoTemplate(mongoDbFactory());
}
}
src/main/java/com/zhiwei/messageflow/mongo/config/SecondaryMongoConfig.java 0 → 100644
package com.zhiwei.messageflow.mongo.config;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.mongodb.core.MongoTemplate;
@Configuration
@ConfigurationProperties(prefix = "spring.data.mongodb.secondary")
public class SecondaryMongoConfig extends AbstractMongoConfig {
@Override
public @Bean(name = "secondaryMongoTemplate") MongoTemplate getMongoTemplate() throws Exception {
return new MongoTemplate(mongoDbFactory());
}
}
src/main/java/com/zhiwei/messageflow/mongo/config/ThirdaryMongoConfig.java 0 → 100644
package com.zhiwei.messageflow.mongo.config;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.mongodb.core.MongoTemplate;
@Configuration
@ConfigurationProperties(prefix = "spring.data.mongodb.thirdary")
public class ThirdaryMongoConfig extends AbstractMongoConfig{
@Override
public @Bean(name = "thirdaryMongoTemplate") MongoTemplate getMongoTemplate() throws Exception {
return new MongoTemplate(mongoDbFactory());
}
}
src/main/java/com/zhiwei/messageflow/mongo/dao/HuserInfoDao.java 0 → 100644
package com.zhiwei.messageflow.mongo.dao;
import com.zhiwei.messageflow.mongo.bean.HuserInfoMedia;
import com.zhiwei.messageflow.mongo.bean.HuserInfoWeibo;
import com.zhiwei.messageflow.mongo.bean.HuserInfoWeixin;
public interface HuserInfoDao {
/**
* 微博渠道信息获取
*
* @param user_id
* @return
*/
HuserInfoWeibo getHuserInfoWeibo(String user_id);
/**
* 微信渠道信息获取
*
* @param username
* @return
*/
HuserInfoWeixin getHuserInfoWeixin(String username);
/**
* 网媒渠道信息获取
*
* @param source
* @return
*/
HuserInfoMedia getHuserInfoMedia(String source);
}
src/main/java/com/zhiwei/messageflow/mongo/dao/KeywordNewDao.java 0 → 100644
package com.zhiwei.messageflow.mongo.dao;
import java.util.List;
import com.zhiwei.messageflow.mongo.bean.KeywordNew;
public interface KeywordNewDao {
/**
* 获取项目关键词组列表
*
* @param projectName
* 项目名
* @return
*/
List<KeywordNew> getKeywordNewByProject(String projectName);
}
src/main/java/com/zhiwei/messageflow/mongo/dao/NoiseRuleDao.java 0 → 100644
package com.zhiwei.messageflow.mongo.dao;
import java.util.List;
import com.zhiwei.messageflow.mongo.bean.NoiseRule;
public interface NoiseRuleDao {
/**
* 根据项目获取噪音规则
*
* @param projectName
* @return
*/
List<NoiseRule> getNoiseRuleByProject(String projectName);
}
src/main/java/com/zhiwei/messageflow/mongo/dao/PlatformDao.java 0 → 100644
package com.zhiwei.messageflow.mongo.dao;
import java.util.List;
public interface PlatformDao {
/**
* 获取公共平台名列表
*
* @return
*/
List<String> getPublicPlatformName();
/**
* 获取全部平台名列表
*
* @return
*/
List<String> getAllPlatformName();
}
src/main/java/com/zhiwei/messageflow/mongo/dao/ProjectDao.java 0 → 100644
package com.zhiwei.messageflow.mongo.dao;
import java.util.List;
import com.zhiwei.messageflow.mongo.bean.Project;
public interface ProjectDao {
/**
* 获取项目列表
*
* @return
*/
List<Project> getAllProjects();
}
src/main/java/com/zhiwei/messageflow/mongo/dao/TrackRuleDao.java 0 → 100644
package com.zhiwei.messageflow.mongo.dao;
import java.util.List;
import com.zhiwei.messageflow.mongo.bean.TrackRule;
public interface TrackRuleDao {
/**
* 根据项目查询预警规则
*
* @param projectName
* @return
*/
List<TrackRule> getTrackRuleByProject(String projectName);
/**
* 更新项目预警情况
*
* @param trackRule_id
*/
void updateTrackrule(Long trackRule_id);
}
src/main/java/com/zhiwei/messageflow/mongo/dao/UserMailDao.java 0 → 100644
package com.zhiwei.messageflow.mongo.dao;
import java.util.List;
import com.zhiwei.messageflow.mongo.bean.UserMail;
public interface UserMailDao {
/**
* 获取预警邮箱
*
* @param projectName
* @return
*/
List<UserMail> findUserMailByProject(String projectName);
}
src/main/java/com/zhiwei/messageflow/mongo/dao/UserWechatDao.java 0 → 100644
package com.zhiwei.messageflow.mongo.dao;
import java.util.List;
import com.zhiwei.messageflow.mongo.bean.UserWechat;
public interface UserWechatDao {
/**
* 获取预警微信
*
* @param projectName
* @return
*/
List<UserWechat> getUserWechatByProject(String projectName);
}
src/main/java/com/zhiwei/messageflow/mongo/dao/WechatCodeDao.java 0 → 100644
package com.zhiwei.messageflow.mongo.dao;
import com.zhiwei.messageflow.mongo.bean.WechatCode;
public interface WechatCodeDao {
/**
* 获取预警微信公众号权限
*
* @param AppId
* @return
*/
WechatCode getWechatCodeByAppId(String AppId);
}
src/main/java/com/zhiwei/messageflow/mongo/dao/impl/HuserInfoDaoImpl.java 0 → 100644
package com.zhiwei.messageflow.mongo.dao.impl;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Component;
import com.zhiwei.messageflow.mongo.bean.HuserInfoMedia;
import com.zhiwei.messageflow.mongo.bean.HuserInfoWeibo;
import com.zhiwei.messageflow.mongo.bean.HuserInfoWeixin;
import com.zhiwei.messageflow.mongo.dao.HuserInfoDao;
@Component
public class HuserInfoDaoImpl implements HuserInfoDao {
@Autowired
@Qualifier(value = "secondaryMongoTemplate")
protected MongoTemplate secondaryMongoTemplate;
@Override
public HuserInfoWeibo getHuserInfoWeibo(String user_id) {
Query query = new Query().addCriteria(Criteria.where("_id").is(user_id));
return secondaryMongoTemplate.findOne(query, HuserInfoWeibo.class);
}
@Override
public HuserInfoWeixin getHuserInfoWeixin(String username) {
Query query = new Query().addCriteria(Criteria.where("username").is(username));
return secondaryMongoTemplate.findOne(query, HuserInfoWeixin.class);
}
@Override
public HuserInfoMedia getHuserInfoMedia(String source) {
Query query = new Query().addCriteria(Criteria.where("source").is(source));
return secondaryMongoTemplate.findOne(query, HuserInfoMedia.class);
}
}
src/main/java/com/zhiwei/messageflow/mongo/dao/impl/KeywordNewDaoImpl.java 0 → 100644
package com.zhiwei.messageflow.mongo.dao.impl;
import java.util.List;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Component;
import com.zhiwei.messageflow.mongo.bean.KeywordNew;
import com.zhiwei.messageflow.mongo.dao.KeywordNewDao;
@Component
public class KeywordNewDaoImpl implements KeywordNewDao {
@Autowired
@Qualifier(value = "primaryMongoTemplate")
protected MongoTemplate primaryMongoTemplate;
@Override
public List<KeywordNew> getKeywordNewByProject(String projectName) {
Query query = new Query();
query.addCriteria(Criteria.where("project").is(projectName));
return primaryMongoTemplate.find(query, KeywordNew.class);
}
}
src/main/java/com/zhiwei/messageflow/mongo/dao/impl/NoiseRuleDaoImpl.java 0 → 100644
package com.zhiwei.messageflow.mongo.dao.impl;
import java.util.List;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Component;
import com.zhiwei.messageflow.mongo.bean.NoiseRule;
import com.zhiwei.messageflow.mongo.dao.NoiseRuleDao;
@Component
public class NoiseRuleDaoImpl implements NoiseRuleDao {
@Autowired
@Qualifier(value = "primaryMongoTemplate")
protected MongoTemplate primaryMongoTemplate;
@Override
public List<NoiseRule> getNoiseRuleByProject(String projectName) {
Query query = new Query();
query.addCriteria(Criteria.where("project").is(projectName));
return primaryMongoTemplate.find(query, NoiseRule.class);
}
}
src/main/java/com/zhiwei/messageflow/mongo/dao/impl/PlatformDaoImpl.java 0 → 100644
package com.zhiwei.messageflow.mongo.dao.impl;
import java.util.ArrayList;
import java.util.List;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.stereotype.Component;
import com.zhiwei.messageflow.mongo.bean.Platform;
import com.zhiwei.messageflow.mongo.dao.PlatformDao;
@Component
public class PlatformDaoImpl implements PlatformDao {
@Autowired
@Qualifier(value = "primaryMongoTemplate")
protected MongoTemplate primaryMongoTemplate;
@Override
public List<String> getPublicPlatformName() {
// mongoTemplate.getCollectionName(Platform.class);
// Query query = new Query();
// query.addCriteria(Criteria.where("Type").is("public"));
// List<Platform> platforms = mongoTemplate.find(query, Platform.class);
List<Platform> platforms = primaryMongoTemplate.findAll(Platform.class);
List<String> platformNames = new ArrayList<>();
for (Platform p : platforms) {
if (p.getType().equals("public")) {
platformNames.add(p.getPlatformName());
}
}
return platformNames;
}
@Override
public List<String> getAllPlatformName() {
List<Platform> platforms = primaryMongoTemplate.findAll(Platform.class);
List<String> platformNames = new ArrayList<>();
for (Platform p : platforms) {
platformNames.add(p.getPlatformName());
}
return platformNames;
}
}
src/main/java/com/zhiwei/messageflow/mongo/dao/impl/ProjectDaoImpl.java 0 → 100644
package com.zhiwei.messageflow.mongo.dao.impl;
import java.util.List;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.stereotype.Component;
import com.zhiwei.messageflow.mongo.bean.Project;
import com.zhiwei.messageflow.mongo.dao.ProjectDao;
@Component
public class ProjectDaoImpl implements ProjectDao {
@Autowired
@Qualifier(value = "primaryMongoTemplate")
protected MongoTemplate primaryMongoTemplate;
@Override
public List<Project> getAllProjects() {
return primaryMongoTemplate.findAll(Project.class);
}
}
src/main/java/com/zhiwei/messageflow/mongo/dao/impl/TrackRuleDaoImpl.java 0 → 100644
package com.zhiwei.messageflow.mongo.dao.impl;
import java.util.List;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Component;
import com.zhiwei.messageflow.mongo.bean.TrackRule;
import com.zhiwei.messageflow.mongo.dao.TrackRuleDao;
@Component
public class TrackRuleDaoImpl implements TrackRuleDao {
@Autowired
@Qualifier(value = "primaryMongoTemplate")
protected MongoTemplate primaryMongoTemplate;
@Override
public List<TrackRule> getTrackRuleByProject(String projectName) {
Query query = new Query();
query.addCriteria(Criteria.where("project").is(projectName));
primaryMongoTemplate.getCollectionName(TrackRule.class);
List<TrackRule> trackRules = primaryMongoTemplate.find(query, TrackRule.class);
return trackRules;
}
@Override
public void updateTrackrule(Long trackRule_id) {
System.out.println("预警状态更新");
// primaryMongoTemplate.updateFirst(Query.query(Criteria.where("_id").is(trackRule_id)),
// new Update().set("isWarn", true), TrackRule.class);
}
}
src/main/java/com/zhiwei/messageflow/mongo/dao/impl/UserMailDaoImal.java 0 → 100644
package com.zhiwei.messageflow.mongo.dao.impl;
import java.util.List;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Component;
import com.zhiwei.messageflow.mongo.bean.UserMail;
import com.zhiwei.messageflow.mongo.dao.UserMailDao;
@Component
public class UserMailDaoImal implements UserMailDao {
@Autowired
@Qualifier(value = "primaryMongoTemplate")
protected MongoTemplate primaryMongoTemplate;
@Override
public List<UserMail> findUserMailByProject(String projectName) {
Query query = new Query();
query.addCriteria(Criteria.where("project").is(projectName));
return primaryMongoTemplate.find(query, UserMail.class);
}
}
src/main/java/com/zhiwei/messageflow/mongo/dao/impl/UserWechatDaoImpl.java 0 → 100644
package com.zhiwei.messageflow.mongo.dao.impl;
import java.util.List;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Component;
import com.zhiwei.messageflow.mongo.bean.UserWechat;
import com.zhiwei.messageflow.mongo.dao.UserWechatDao;
@Component
public class UserWechatDaoImpl implements UserWechatDao {
@Autowired
@Qualifier(value = "primaryMongoTemplate")
protected MongoTemplate primaryMongoTemplate;
@Override
public List<UserWechat> getUserWechatByProject(String projectName) {
Query query=new Query();
query.addCriteria(Criteria.where("project").is(projectName));
return primaryMongoTemplate.find(query, UserWechat.class);
}
}
src/main/java/com/zhiwei/messageflow/mongo/dao/impl/WechatCodeDaoImpl.java 0 → 100644
package com.zhiwei.messageflow.mongo.dao.impl;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.stereotype.Component;
import com.zhiwei.messageflow.mongo.bean.WechatCode;
import com.zhiwei.messageflow.mongo.dao.WechatCodeDao;
@Component
public class WechatCodeDaoImpl implements WechatCodeDao {
@Autowired
@Qualifier(value = "thirdaryMongoTemplate")
protected MongoTemplate thirdaryMongoTemplate;
@Override
public WechatCode getWechatCodeByAppId(String AppId) {
return thirdaryMongoTemplate.findById(AppId, WechatCode.class);
}
}
src/main/java/com/zhiwei/messageflow/mongo/service/UserMailService.java 0 → 100644
package com.zhiwei.messageflow.mongo.service;
import java.util.Map;
import com.zhiwei.messageflow.es.bean.TemplateData;
public interface UserMailService {
Map<String,Object> sendMailByProject(String project, TemplateData templateData);
}
src/main/java/com/zhiwei/messageflow/mongo/service/WechatSendService.java 0 → 100644
package com.zhiwei.messageflow.mongo.service;
import com.zhiwei.messageflow.es.bean.TemplateData;
import com.zhiwei.messageflow.mongo.bean.WechatCode;
import net.sf.json.JSONObject;
public interface WechatSendService {
int sendTemplateByProject(String projectName, TemplateData templateData);
int sendDataJson(WechatCode wechatCode,JSONObject dataJson);
}
src/main/java/com/zhiwei/messageflow/mongo/service/impl/UserMailServiceImpl.java 0 → 100644
package com.zhiwei.messageflow.mongo.service.impl;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import com.zhiwei.messageflow.es.bean.TemplateData;
import com.zhiwei.messageflow.mail.SendMail;
import com.zhiwei.messageflow.mongo.bean.UserMail;
import com.zhiwei.messageflow.mongo.dao.UserMailDao;
import com.zhiwei.messageflow.mongo.service.UserMailService;
@Component
public class UserMailServiceImpl implements UserMailService {
@SuppressWarnings("unused")
@Autowired
private UserMailDao userMailDao;
@Override
public Map<String, Object> sendMailByProject(String project, TemplateData templateData) {
Map<String, Object> result = new HashMap<>();
// List<UserMail> mails = userMailDao.findUserMailByProject(project);
List<UserMail> mails = new ArrayList<>();
UserMail mail = new UserMail();
mail.setAddress("1454551152@qq.com");
mail.setProject("腾讯");
mails.add(mail);
List<String> failMails = new ArrayList<>();
int success = 0;
for (UserMail userMail : mails) {
if (SendMail.sendMailWarning(userMail.getAddress(), templateData)) {
success++;
} else {
failMails.add(userMail.getAddress());
}
}
if (mails.size() == success) {
result.put("status", true);
result.put("message", "用户组邮件发送成功");
} else {
result.put("status", false);
String failStr = "";
for (String string : failMails) {
failStr += string + ";";
}
result.put("message", "用户组邮箱发送成功【" + success + "】个，发送失败邮箱：【" + failStr + "】");
}
return result;
}
}
src/main/java/com/zhiwei/messageflow/mongo/service/impl/WechatSendServiceImpl.java 0 → 100644
package com.zhiwei.messageflow.mongo.service.impl;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import com.zhiwei.messageflow.es.bean.Template;
import com.zhiwei.messageflow.es.bean.TemplateData;
import com.zhiwei.messageflow.es.bean.WechatConstant;
import com.zhiwei.messageflow.mongo.bean.UserWechat;
import com.zhiwei.messageflow.mongo.bean.WechatCode;
import com.zhiwei.messageflow.mongo.dao.UserWechatDao;
import com.zhiwei.messageflow.mongo.dao.WechatCodeDao;
import com.zhiwei.messageflow.mongo.service.WechatSendService;
import com.zhiwei.messageflow.util.HttpRequest;
import com.zhiwei.messageflow.util.TimeUtil;
import net.sf.json.JSONObject;
@Component
public class WechatSendServiceImpl implements WechatSendService {
private final static Logger log = LoggerFactory.getLogger(WechatSendServiceImpl.class);
@Autowired
private WechatCodeDao wechatCodeDao;
@SuppressWarnings("unused")
@Autowired
private UserWechatDao userWechatDao;
@SuppressWarnings("unused")
@Override
public int sendTemplateByProject(String projectName, TemplateData templateData) {
WechatCode wechatCode=wechatCodeDao.getWechatCodeByAppId(WechatConstant.WECHAT_APPID);
// List<UserWechat> userWechats=userWechatDao.getUserWechatByProject(projectName);
List<UserWechat> userWechats=new ArrayList<>();
UserWechat userWechat1=new UserWechat();
userWechat1.setOpenid("o_J5m0b9Un5rRfUXZS5S3kKjkzeo");
userWechats.add(userWechat1);
if (null == userWechats) {
log.error("未获取到UserWechat，查询依据【{}】", projectName);
return -1;
}
int size = userWechats.size();
int k = 0;
if (wechatCode != null) {
Map<String, Object> dataMap = new HashMap<>();
JSONObject first = new JSONObject();
first.put("value", "知微情报监测服务：有一条来自" + templateData.getPt() + "平台的预警通知。");
dataMap.put("first", first);
JSONObject keyword1 = new JSONObject();
keyword1.put("value", templateData.getTitle());
keyword1.put("color", "#173177");
dataMap.put("keyword1", keyword1);
JSONObject keyword2 = new JSONObject();
keyword2.put("value", templateData.getSource());
keyword2.put("color", "#173177");
dataMap.put("keyword2", keyword2);
JSONObject keyword3 = new JSONObject();
keyword3.put("value", TimeUtil.formatTime(templateData.getTime()));
keyword3.put("color", "#173177");
dataMap.put("keyword3", keyword3);
JSONObject keyword4 = new JSONObject();
keyword4.put("value", templateData.getContent());
keyword4.put("color", "#173177");
dataMap.put("keyword4", keyword4);
JSONObject remark = new JSONObject();
if (null != templateData.getWordRule()) {
remark.put("value", "关键词规则：" + templateData.getWordRule());
} else if (null != templateData.getChannelRule()) {
remark.put("value", "渠道规则：" + templateData.getChannelRule());
} else {
remark.put("value", "知微情报监测服务");
}
dataMap.put("remark", remark);
for (UserWechat userWechat : userWechats) {
Template template = new Template();
template.setTouser(userWechat.getOpenid());
template.setUrl(templateData.getUrl());
template.setTemplate_id(WechatConstant.WECHAT_TEMPLATEID_EARLY_IT);
template.setData(dataMap);
JSONObject templateJson = JSONObject.fromObject(template);
k = sendDataJson(wechatCode, templateJson);
if (0 == k) {
k = -1;
log.error("发送模版消息【{}】时出现错误.....", template);
} else {
size--;
}
}
} else {
log.error("未获取到appId【{}】的wechatCode.....", WechatConstant.WECHAT_APPID);
}
return size;
}
@Override
public int sendDataJson(WechatCode wechatCode, JSONObject dataJson) {
int msgid = 0;
String url = WechatConstant.WECHAT_TEMPLET_SEND_URL.replace("ACCESS_TOKEN", wechatCode.getAccessToken());
try {
JSONObject jsonObject = HttpRequest.httpRequest(url, "POST", dataJson.toString());
if (null != jsonObject) {
if ("ok".equals(jsonObject.getString("errmsg"))) {
msgid = jsonObject.getInt("msgid");
}
}
} catch (Exception e) {
e.printStackTrace();
log.error("发送创建模版消息请求时出现错误", e.getMessage());
msgid = 0;
}
return msgid;
}
}
src/main/java/com/zhiwei/messageflow/redis/RedisPoolAndTools.java 0 → 100644
package com.zhiwei.messageflow.redis;
import java.util.Set;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Configuration;
import com.zhiwei.messageflow.config.RedisConfig;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPoolConfig;
import redis.clients.jedis.Tuple;
import redis.clients.jedis.exceptions.JedisConnectionException;
@Configuration
public class RedisPoolAndTools {
public final static Logger log = LoggerFactory.getLogger(RedisPoolAndTools.class);
@Autowired
private RedisConfig redisConfig;
private JedisPool pool;
/**
* jedis连接池创建
*/
private void initializePool() {
JedisPoolConfig config = new JedisPoolConfig();
config.setMaxTotal(redisConfig.getMaxTotal());
config.setMaxIdle(redisConfig.getMaxIdle());
config.setMaxWaitMillis(redisConfig.getMaxWaitMillis());
config.setTestOnBorrow(redisConfig.isTestOnBorrow());
config.setTestOnReturn(redisConfig.isTestOnReturn());
pool = new JedisPool(config, redisConfig.getIp(), redisConfig.getPort(), 0, null, redisConfig.getSelectDB());
}
/**
* 获取jedis
*
* @return jedis连接
*/
private Jedis getJedis() {
// 连接请求次数
int timeoutCount = 0;
if (null == pool) {
initializePool();
}
while (true) {
try {
if (null != pool) {
return pool.getResource();
}
} catch (Exception e) {
if (e instanceof JedisConnectionException) {
timeoutCount++;
log.warn("jedis连接请求次数{}", timeoutCount);
if (timeoutCount > 3) {
break;
}
} else {
log.warn("jedisInfo ... NumActive=" + pool.getNumActive() + ", NumIdle=" + pool.getNumIdle()
+ ", NumWaiters=" + pool.getNumWaiters() + ", isClosed=" + pool.isClosed());
log.error("获取jedis连接错误:{}", e.getMessage());
break;
}
}
break;
}//请求jedis完成
return null;
}
/**
* 关闭jedis连接
*
* @param jedis
*/
public static void returnResource(Jedis jedis) {
if (null != jedis) {
jedis.close();
}
}
/**
* 获取RSID
*
* @param rsidkey
* 键值为platform-project-keywords
* @return
*/
public String getRSID(String rsidkey) {
Jedis jedis = getJedis();
while (true) {
if (null != jedis) {
break;
} else {
jedis = getJedis();
}
}
String rsid = jedis.get(rsidkey);
returnResource(jedis);
return rsid;
}
/**
* 写入RSID,有效时间为12小时
*
* @param rsidkey
* @param rsid
*/
public void setRSID(String rsidkey, String rsid) {
Jedis jedis = getJedis();
while (true) {
if (null != jedis) {
break;
} else {
jedis = getJedis();
}
}
jedis.setex(rsidkey, 60 * 60 * 12, rsid);
returnResource(jedis);
}
/**
* 消息流数据写入
*
* @param redisKey
* @param rsid
* @param message
*/
public void sortedSetZadd(String redisKey, double rsid, String message) {
Jedis jedis = getJedis();
while (true) {
if (null != jedis) {
break;
} else {
jedis = getJedis();
}
}
jedis.zadd(redisKey, rsid, message);
returnResource(jedis);
}
/**
* 删除超出上限的数据
*
* @param key
* @param removeIndex
*/
public void removeDataByName(byte[] key, int removeIndex) {
Jedis jedis = getJedis();
while (true) {
if (null != jedis) {
break;
} else {
jedis = getJedis();
}
}
jedis.zremrangeByRank(key, 0, removeIndex);
returnResource(jedis);
}
/**
* 获取有序集合消息数量
*
* @param key
* @return
*/
public Long getNowCount(byte[] key) {
Jedis jedis = getJedis();
while (true) {
if (null != jedis) {
break;
} else {
jedis = getJedis();
}
}
Long nowCount = jedis.zcard(key);
returnResource(jedis);
return nowCount;
}
/**
* 分页获取redis数据
*
* @param key
* @param start
* @param end
* @return
*/
public Set<Tuple> getRevDataListByIndex(String key, long start, long end) {
Set<Tuple> set = null;
Jedis jedis = getJedis();
while (true) {
if (null != jedis) {
break;
} else {
jedis = getJedis();
}
}
set = jedis.zrevrangeWithScores(key.getBytes(), start, end);
returnResource(jedis);
return set;
}
}
\ No newline at end of file
src/main/java/com/zhiwei/messageflow/redis/bean/RsidAndMessages.java 0 → 100644
package com.zhiwei.messageflow.redis.bean;
import java.util.List;
import com.zhiwei.messageflow.bean.MediaMessage;
import com.zhiwei.messageflow.bean.VideoMessage;
import com.zhiwei.messageflow.bean.WeiboMessage;
import com.zhiwei.messageflow.bean.ZhihuMessage;
import lombok.Data;
import lombok.ToString;
@Data
@ToString
public class RsidAndMessages {
/**
* 微博消息列表
*/
private List<WeiboMessage> wlist;
/**
* 知乎消息列表
*/
private List<ZhihuMessage> zlist;
/**
* 视频消息列表
*/
private List<VideoMessage> vlist;
/**
* 网媒消息列表
*/
private List<MediaMessage> mlist;
/**
* 本次读取最后信息的rsid
*/
private long rsid;
}
src/main/java/com/zhiwei/messageflow/redis/service/RedisService.java 0 → 100644
package com.zhiwei.messageflow.redis.service;
import java.util.List;
import java.util.Map;
import com.zhiwei.messageflow.bean.MediaMessage;
import com.zhiwei.messageflow.bean.VideoMessage;
import com.zhiwei.messageflow.bean.WeiboMessage;
import com.zhiwei.messageflow.bean.ZhihuMessage;
public interface RedisService {
/**
* 获取项目Rsid列表
*
* @param project
* 项目名
* @return
*/
String getRsid(String project);
/**
* 更新项目Rsid列表
*
* @param newRsidMap
* 存入redis的数据
* @param project
* 项目名
*/
void setRsid(Map<String, Integer> newRsidMap, String project);
/**
* 向redis写入微博数据
*
* @param redisKey
* 数据键
* @param messages
* 数据列表
* @param maxSize
* redis数据存储上限
*/
void setWeiboMessageMessage(String redisKey, List<WeiboMessage> messages, int maxSize);
/**
* 向redis写入知乎数据
*
* @param redisKey
* 数据键
* @param messages
* 数据列表
* @param maxSize
* redis数据存储上限
*/
void setZhihuMessageMessage(String redisKey, List<ZhihuMessage> messages, int maxSize);
/**
* 向redis写入视频数据
*
* @param redisKey
* 数据键
* @param messages
* 数据列表
* @param maxSize
* redis数据存储上限
*/
void setVideoMessageMessage(String redisKey, List<VideoMessage> messages, int maxSize);
/**
* 向redis写入网媒数据
*
* @param redisKey
* 数据键
* @param messages
* 数据列表
* @param maxSize
* redis数据存储上限
*/
void setMediaMessageMessage(String redisKey, List<MediaMessage> messages, int maxSize);
}
src/main/java/com/zhiwei/messageflow/redis/service/impl/RedisServiceImpl.java 0 → 100644
package com.zhiwei.messageflow.redis.service.impl;
import java.util.List;
import java.util.Map;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.zhiwei.messageflow.bean.MediaMessage;
import com.zhiwei.messageflow.bean.VideoMessage;
import com.zhiwei.messageflow.bean.WeiboMessage;
import com.zhiwei.messageflow.bean.ZhihuMessage;
import com.zhiwei.messageflow.redis.RedisPoolAndTools;
import com.zhiwei.messageflow.redis.service.RedisService;
@Component
public class RedisServiceImpl implements RedisService {
@Autowired
private RedisPoolAndTools redisPoolAndTools;
static ObjectMapper mapper=new ObjectMapper();
static {
mapper.setSerializationInclusion(Include.NON_EMPTY);
}
@Override
public String getRsid(String project) {
return redisPoolAndTools.getRSID(project);
}
@Override
public void setRsid(Map<String, Integer> rsidmap, String project) {
try {
redisPoolAndTools.setRSID(project, mapper.writeValueAsString(rsidmap));
} catch (JsonProcessingException e) {
e.printStackTrace();
}
}
@Override
public void setWeiboMessageMessage(String redisKey, List<WeiboMessage> messages, int maxSize) {
for (WeiboMessage wm : messages) {
try {
// 写入数据
redisPoolAndTools.sortedSetZadd(redisKey, (double) wm.getRstime(), mapper.writeValueAsString(wm));
} catch (JsonProcessingException e) {
e.printStackTrace();
}
}
/**
* 删除超出存储上限的数据
*/
long nowCount = redisPoolAndTools.getNowCount(redisKey.getBytes());// 当前数据量
int removeIndex = (int) (nowCount - maxSize);// 需移除数据数量
if (maxSize > 0 && nowCount > maxSize) {
redisPoolAndTools.removeDataByName(redisKey.getBytes(), removeIndex);
}
}
@Override
public void setZhihuMessageMessage(String redisKey, List<ZhihuMessage> messages, int maxSize) {
for (ZhihuMessage zm : messages) {
try {
// 写入数据
redisPoolAndTools.sortedSetZadd(redisKey, (double) zm.getRsid(), mapper.writeValueAsString(zm));
} catch (JsonProcessingException e) {
e.printStackTrace();
}
}
/**
* 删除超出存储上限的数据
*/
long nowCount = redisPoolAndTools.getNowCount(redisKey.getBytes());// 当前数据量
int removeIndex = (int) (nowCount - maxSize);// 需移除数据数量
if (maxSize > 0 && nowCount > maxSize) {
redisPoolAndTools.removeDataByName(redisKey.getBytes(), removeIndex);
}
}
@Override
public void setVideoMessageMessage(String redisKey, List<VideoMessage> messages, int maxSize) {
for (VideoMessage vm : messages) {
try {
// 写入数据
redisPoolAndTools.sortedSetZadd(redisKey, (double) vm.getRsid(), mapper.writeValueAsString(vm));
} catch (JsonProcessingException e) {
e.printStackTrace();
}
}
/**
* 删除超出存储上限的数据
*/
long nowCount = redisPoolAndTools.getNowCount(redisKey.getBytes());// 当前数据量
int removeIndex = (int) (nowCount - maxSize);// 需移除数据数量
if (maxSize > 0 && nowCount > maxSize) {
redisPoolAndTools.removeDataByName(redisKey.getBytes(), removeIndex);
}
}
@Override
public void setMediaMessageMessage(String redisKey, List<MediaMessage> messages, int maxSize) {
for (MediaMessage mm : messages) {
try {
// 写入数据
redisPoolAndTools.sortedSetZadd(redisKey, (double) mm.getRsid(), mapper.writeValueAsString(mm));
} catch (JsonProcessingException e) {
e.printStackTrace();
}
}
/**
* 删除超出存储上限的数据
*/
long nowCount = redisPoolAndTools.getNowCount(redisKey.getBytes());// 当前数据量
int removeIndex = (int) (nowCount - maxSize);// 需移除数据数量
if (maxSize > 0 && nowCount > maxSize) {
redisPoolAndTools.removeDataByName(redisKey.getBytes(), removeIndex);
}
}
}
src/main/java/com/zhiwei/messageflow/service/DisposeMessageService.java 0 → 100644
package com.zhiwei.messageflow.service;
import java.util.List;
import com.zhiwei.messageflow.mongo.bean.NoiseRule;
import com.zhiwei.messageflow.mongo.bean.TrackRule;
import com.zhiwei.messageflow.redis.bean.RsidAndMessages;
public interface DisposeMessageService {
/**
* 获取微博平台RsidAndMessages
*
* @param trackRules
* 预警规则
* @param noiseRules
* 去噪规则
* @param keywords
* 关键词组
* @param count
* 每次查询数量
* @param startid
* 开始rsid
* @param endid
* 结束rsid
* @param platform
* 平台
* @param project
* 项目
* @return
*/
RsidAndMessages getFilteredWeiboMessage(List<TrackRule> trackRules, List<NoiseRule> noiseRules,
List<String> keywords, int count, Long startid, Long endid, String platform, String project);
/**
* 获取知乎平台RsidAndMessages
*
* @param trackRules
* 预警规则
* @param noiseRules
* 去噪规则
* @param keywords
* 关键词组
* @param count
* 每次查询数量
* @param startid
* 开始rsid
* @param endid
* 结束rsid
* @param platform
* 平台
* @param project
* 项目
* @return
*/
RsidAndMessages getFilteredZhihuMessage(List<TrackRule> trackRules, List<NoiseRule> noiseRules,
List<String> keywords, int count, Long startid, Long endid, String platform, String project);
/**
* 获取视频平台RsidAndMessages
*
* @param trackRules
* 预警规则
* @param noiseRules
* 去噪规则
* @param keywords
* 关键词组
* @param count
* 每次查询数量
* @param startid
* 开始rsid
* @param endid
* 结束rsid
* @param platform
* 平台
* @param project
* 项目
* @return
*/
RsidAndMessages getFilteredVideoMessage(List<TrackRule> trackRules, List<NoiseRule> noiseRules,
List<String> keywords, int count, Long startid, Long endid, String platform, String project);
/**
* 获取网媒平台RsidAndMessages
*
* @param trackRules
* 预警规则
* @param noiseRules
* 去噪规则
* @param keywords
* 关键词组
* @param count
* 每次查询数量
* @param startid
* 开始rsid
* @param endid
* 结束rsid
* @param platform
* 平台
* @param project
* 项目
* @return
*/
RsidAndMessages getFilteredMediaMessage(List<TrackRule> trackRules, List<NoiseRule> noiseRules,
List<String> keywords, int count, Long startid, Long endid, String platform, String project);
}
src/main/java/com/zhiwei/messageflow/service/impl/DisposeMessageServiceImpl.java 0 → 100644
package com.zhiwei.messageflow.service.impl;
import java.util.ArrayList;
import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import com.zhiwei.messageflow.bean.MediaMessage;
import com.zhiwei.messageflow.bean.VideoMessage;
import com.zhiwei.messageflow.bean.WeiboMessage;
import com.zhiwei.messageflow.bean.ZhihuMessage;
import com.zhiwei.messageflow.es.service.ES4BeanService;
import com.zhiwei.messageflow.mongo.bean.NoiseRule;
import com.zhiwei.messageflow.mongo.bean.TrackRule;
import com.zhiwei.messageflow.redis.bean.RsidAndMessages;
import com.zhiwei.messageflow.service.DisposeMessageService;
@Component
public class DisposeMessageServiceImpl implements DisposeMessageService {
public final static Logger log = LoggerFactory.getLogger(DisposeMessageServiceImpl.class);
@Autowired
private ES4BeanService es4BeanDao;
@Override
public RsidAndMessages getFilteredWeiboMessage(List<TrackRule> trackRules, List<NoiseRule> noiseRules,
List<String> keywords, int count, Long startid, Long endid, String platform, String project) {
RsidAndMessages ram = new RsidAndMessages();
// 消息列表
List<WeiboMessage> messages = new ArrayList<WeiboMessage>();
messages = es4BeanDao.getWeiboMessage(noiseRules, keywords, count, startid + 1L, -1L, platform, project);
/**
* 获取最后消息的rsid
*/
if (messages!=null&&!messages.isEmpty()) {
WeiboMessage wmf = messages.get(0);
WeiboMessage wml = messages.get(messages.size() - 1);
ram.setWlist(messages);
ram.setRsid(wmf.getRstime() > wml.getRstime() ? wmf.getRstime() : wml.getRstime());
}
return ram;
}
@Override
public RsidAndMessages getFilteredZhihuMessage(List<TrackRule> trackRules, List<NoiseRule> noiseRules,
List<String> keywords, int count, Long startid, Long endid, String platform, String project) {
RsidAndMessages ram = new RsidAndMessages();
// 消息列表
List<ZhihuMessage> messages = new ArrayList<ZhihuMessage>();
messages = es4BeanDao.getZhihuMessage(noiseRules, keywords, count, startid + 1L, -1L, platform, project);
/**
* 获取最后消息的rsid
*/
if (messages!=null&&!messages.isEmpty()) {
ZhihuMessage zmf = messages.get(0);
ZhihuMessage zml = messages.get(messages.size() - 1);
ram.setZlist(messages);
ram.setRsid(zmf.getRsid() > zml.getRsid() ? zmf.getRsid() : zml.getRsid());
}
return ram;
}
@Override
public RsidAndMessages getFilteredVideoMessage(List<TrackRule> trackRules, List<NoiseRule> noiseRules,
List<String> keywords, int count, Long startid, Long endid, String platform, String project) {
RsidAndMessages ram = new RsidAndMessages();
// 消息列表
List<VideoMessage> messages = new ArrayList<VideoMessage>();
messages = es4BeanDao.getVideoMessage(noiseRules, keywords, count, startid + 1L, -1L, platform, project);
/**
* 获取最后消息的rsid
*/
if (messages!=null&&!messages.isEmpty()) {
VideoMessage vmf = messages.get(0);
VideoMessage vml = messages.get(messages.size() - 1);
ram.setVlist(messages);
ram.setRsid(vmf.getRsid() > vml.getRsid() ? vmf.getRsid() : vml.getRsid());
}
return ram;
}
@Override
public RsidAndMessages getFilteredMediaMessage(List<TrackRule> trackRules, List<NoiseRule> noiseRules,
List<String> keywords, int count, Long startid, Long endid, String platform, String project) {
RsidAndMessages ram = new RsidAndMessages();
// 消息列表
List<MediaMessage> messages = new ArrayList<MediaMessage>();
messages = es4BeanDao.getMediaMessage(noiseRules, keywords, count, startid + 1L, -1L, platform, project);
/**
* 获取最后消息的rsid
*/
if (messages!=null&&!messages.isEmpty()) {
MediaMessage mmf = messages.get(0);
MediaMessage mml = messages.get(messages.size() - 1);
ram.setMlist(messages);
ram.setRsid(mmf.getRsid() > mml.getRsid() ? mmf.getRsid() : mml.getRsid());
}
return ram;
}
}
src/main/java/com/zhiwei/messageflow/test/GetMessage.java 0 → 100644
package com.zhiwei.messageflow.test;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import org.apache.poi.hssf.usermodel.HSSFCell;
import org.apache.poi.hssf.usermodel.HSSFRichTextString;
import org.apache.poi.hssf.usermodel.HSSFRow;
import org.apache.poi.hssf.usermodel.HSSFSheet;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.zhiwei.messageflow.bean.MediaMessage;
import com.zhiwei.messageflow.bean.VideoMessage;
import com.zhiwei.messageflow.bean.WeiboMessage;
import com.zhiwei.messageflow.bean.ZhihuMessage;
import com.zhiwei.messageflow.es.dao.impl.ESDaoImpl;
import com.zhiwei.messageflow.mongo.dao.PlatformDao;
import com.zhiwei.messageflow.mongo.dao.ProjectDao;
@Component
public class GetMessage {
@Autowired
ObjectMapper mapper;
@Autowired
ESDaoImpl esDao;
@Autowired
PlatformDao platformDao;
@Autowired
ProjectDao projectDao;
public void getmessage() throws FileNotFoundException, ParseException {
/*
* 3月21日0点——3月22日0点 京东 京东+假货 京东+刷单 京东+二手 京东+合作 京东+战略
*
* 微信 微信+支付 微信+红包
*/
List<WeiboMessage> wmessages = new ArrayList<>();
List<ZhihuMessage> zmessages = new ArrayList<>();
List<VideoMessage> vmessages = new ArrayList<>();
List<MediaMessage> mmessages = new ArrayList<>();
DateFormat df = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");
Date date = df.parse("2018-03-01 00:00:00");
int n = 4 * 30;
HSSFWorkbook workbook = new HSSFWorkbook();
OutputStream out = new FileOutputStream("E://微信+红包.xls");
List<String> platformNames = platformDao.getAllPlatformName();
List<String> keywords = new ArrayList<>();
keywords.add("微信+红包");
// keywords.add("微信+支付");
// keywords.add("京东,假货");
// keywords.add("京东,刷单");
// keywords.add("京东,二手");
// keywords.add("京东,合作");
// keywords.add("京东,战略");
// keywords.add("京东,");
for (String platformName : platformNames) {
System.out.println(platformName);
if (platformName.equals("微博")) {
for (long i = 0; i < n; i++) {
SearchHits searchHits = esDao.getWeiboDataycy(date.getTime() + i * 6 * 3000 * 1000L, keywords,
10000, platformName, "京东");
SearchHit[] sh = searchHits.getHits();
for (SearchHit s : sh) {
Map<String, Object> source = s.getSource();
if (!source.isEmpty()) {
try {
String message = mapper.writeValueAsString(source);
WeiboMessage wm = mapper.readValue(message, WeiboMessage.class);
wmessages.add(wm);
System.out.println(wm.getText());
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
if (!wmessages.isEmpty())
workbook = exportExcelWeibo(out, workbook, null, wmessages, platformName);
} else if (platformName.equals("知乎")) {
for (long i = 0; i < n; i++) {
SearchHits searchHits = esDao.getZhihuDataycy(date.getTime() + i * 6 * 3000 * 1000L, keywords,
10000, platformName, "京东");
SearchHit[] sh = searchHits.getHits();
for (SearchHit s : sh) {
Map<String, Object> source = s.getSource();
if (!source.isEmpty()) {
try {
String message = mapper.writeValueAsString(source);
ZhihuMessage zm = mapper.readValue(message, ZhihuMessage.class);
zmessages.add(zm);
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
if (!zmessages.isEmpty())
workbook = exportExcelZhihu(out, workbook, null, zmessages, platformName);
} else if (platformName.equals("视频")) {
for (long i = 0; i < n; i++) {
SearchHits searchHits = esDao.getVideoDataycy(date.getTime() + i * 6 * 3000 * 1000L, keywords,
10000, platformName, "京东");
SearchHit[] sh = searchHits.getHits();
for (SearchHit s : sh) {
Map<String, Object> source = s.getSource();
if (!source.isEmpty()) {
try {
String message = mapper.writeValueAsString(source);
VideoMessage vm = mapper.readValue(message, VideoMessage.class);
vmessages.add(vm);
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
if (!vmessages.isEmpty())
workbook = exportExcelVideo(out, workbook, null, vmessages, platformName);
} else {
for (long i = 0; i < n; i++) {
SearchHits searchHits = esDao.getMediaDataycy(date.getTime() + i * 6 * 3000 * 1000L, keywords,
10000, platformName, "京东");
SearchHit[] sh = searchHits.getHits();
for (SearchHit s : sh) {
Map<String, Object> source = s.getSource();
if (!source.isEmpty()) {
try {
String message = mapper.writeValueAsString(source);
MediaMessage mm = mapper.readValue(message, MediaMessage.class);
mmessages.add(mm);
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
if (!mmessages.isEmpty())
workbook = exportExcelMedia(out, workbook, null, mmessages, platformName);
}
} // 获取完成
try {
workbook.write(out);
} catch (IOException e) {
e.printStackTrace();
}
}
public void getSame() throws FileNotFoundException, ParseException {
/*
* 3月21日0点——3月22日0点 京东 京东+假货 京东+刷单 京东+二手 京东+合作 京东+战略
*
* 微信 微信+支付 微信+红包
*/
List<WeiboMessage> wmessages = new ArrayList<>();
List<ZhihuMessage> zmessages = new ArrayList<>();
List<VideoMessage> vmessages = new ArrayList<>();
List<MediaMessage> mmessages = new ArrayList<>();
List<WeiboMessage> fwmessages = new ArrayList<>();
List<ZhihuMessage> fzmessages = new ArrayList<>();
List<VideoMessage> fvmessages = new ArrayList<>();
List<MediaMessage> fmmessages = new ArrayList<>();
List<String> platformNames = platformDao.getAllPlatformName();
// List<String> platformNames = new ArrayList<>();
// platformNames.add("微博");
List<String> keywords = new ArrayList<>();
List<String> fkeywords = new ArrayList<>();
keywords.add("京东==刷单");
keywords.add("京东==二手");
keywords.add("京东==合作");
keywords.add("京东==战略");
// keywords.add("微信==红包");
// keywords.add("微信==支付");
fkeywords.add("京东");
// fkeywords.add("微信");
DateFormat df = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");
Date date = df.parse("2018-02-25 00:00:00");
int n = 4 * 30;
System.out.println("超集:" + fkeywords.toString());
System.out.println("子集:" + keywords.toString());
for (String platformName : platformNames) {
System.out.println(platformName);
if (platformName.equals("微博")) {
for (long i = 0; i < n; i++) {
SearchHits searchHits = esDao.getWeiboDataycy(date.getTime() + i * 6 * 3600 * 1000L, keywords,
10000, platformName, "京东");
SearchHit[] sh = searchHits.getHits();
for (SearchHit s : sh) {
Map<String, Object> source = s.getSource();
if (!source.isEmpty()) {
try {
String message = mapper.writeValueAsString(source);
WeiboMessage wm = mapper.readValue(message, WeiboMessage.class);
wmessages.add(wm);
} catch (Exception e) {
e.printStackTrace();
}
}
}
SearchHits fsearchHits = esDao.getWeiboDataycy(date.getTime() + i * 6 * 3600 * 1000L, fkeywords,
10000, platformName, "京东");
SearchHit[] fsh = fsearchHits.getHits();
for (SearchHit s : fsh) {
Map<String, Object> fsource = s.getSource();
if (!fsource.isEmpty()) {
try {
String fmessage = mapper.writeValueAsString(fsource);
WeiboMessage fwm = mapper.readValue(fmessage, WeiboMessage.class);
fwmessages.add(fwm);
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
float fnum = fwmessages.size();
float samenum = 0;
float snum = wmessages.size();
for (WeiboMessage fwm : fwmessages) {
for (WeiboMessage wm : wmessages) {
if (wm.getRstime() == fwm.getRstime()) {
samenum++;
}
}
}
System.out.println(platformName + "平台:");
System.out.println("子集数据量:" + snum);
System.out.println("超集数据量:" + fnum);
System.out.println("包含率:" + samenum / snum * 100 + "%");
System.out.println("未包含条数:" + (snum - samenum));
} else if (platformName.equals("知乎")) {
for (long i = 0; i < n; i++) {
SearchHits searchHits = esDao.getZhihuDataycy(date.getTime() + i * 6 * 3600 * 1000L, keywords,
10000, platformName, "京东");
SearchHit[] sh = searchHits.getHits();
for (SearchHit s : sh) {
Map<String, Object> source = s.getSource();
if (!source.isEmpty()) {
try {
String message = mapper.writeValueAsString(source);
ZhihuMessage zm = mapper.readValue(message, ZhihuMessage.class);
zmessages.add(zm);
} catch (Exception e) {
e.printStackTrace();
}
}
}
SearchHits fsearchHits = esDao.getZhihuDataycy(date.getTime() + i * 6 * 3600 * 1000L, fkeywords,
10000, platformName, "京东");
SearchHit[] fsh = fsearchHits.getHits();
for (SearchHit s : fsh) {
Map<String, Object> fsource = s.getSource();
if (!fsource.isEmpty()) {
try {
String fmessage = mapper.writeValueAsString(fsource);
ZhihuMessage fzm = mapper.readValue(fmessage, ZhihuMessage.class);
fzmessages.add(fzm);
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
float fnum = fzmessages.size();
float samenum = 0;
float snum = zmessages.size();
for (ZhihuMessage fzm : fzmessages) {
for (ZhihuMessage zm : zmessages) {
if (zm.getRsid() == fzm.getRsid()) {
samenum++;
}
}
}
System.out.println(platformName + "平台:");
System.out.println("子集数据量:" + snum);
System.out.println("超集数据量:" + fnum);
System.out.println("包含率:" + samenum / snum * 100 + "%");
System.out.println("未包含条数:" + (snum - samenum));
} else if (platformName.equals("视频")) {
for (long i = 0; i < n; i++) {
SearchHits searchHits = esDao.getVideoDataycy(date.getTime() + i * 6 * 3600 * 1000L, keywords,
10000, platformName, "京东");
SearchHit[] sh = searchHits.getHits();
for (SearchHit s : sh) {
Map<String, Object> source = s.getSource();
if (!source.isEmpty()) {
try {
String message = mapper.writeValueAsString(source);
VideoMessage vm = mapper.readValue(message, VideoMessage.class);
vmessages.add(vm);
} catch (Exception e) {
e.printStackTrace();
}
}
}
SearchHits fsearchHits = esDao.getVideoDataycy(date.getTime() + i * 6 * 3600 * 1000L, fkeywords,
10000, platformName, "京东");
SearchHit[] fsh = fsearchHits.getHits();
for (SearchHit s : fsh) {
Map<String, Object> fsource = s.getSource();
if (!fsource.isEmpty()) {
try {
String fmessage = mapper.writeValueAsString(fsource);
VideoMessage fvm = mapper.readValue(fmessage, VideoMessage.class);
fvmessages.add(fvm);
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
float fnum = fvmessages.size();
float samenum = 0;
float snum = vmessages.size();
for (VideoMessage fvm : fvmessages) {
for (VideoMessage vm : vmessages) {
if (vm.getRsid() == fvm.getRsid()) {
samenum++;
}
}
}
System.out.println(platformName + "平台:");
System.out.println("子集数据量:" + snum);
System.out.println("超集数据量:" + fnum);
System.out.println("包含率:" + (samenum / snum) * 100 + "%");
System.out.println("未包含条数:" + (snum - samenum));
} else {
for (long i = 0; i < n; i++) {
SearchHits searchHits = esDao.getMediaDataycy(date.getTime() + i * 6 * 3600 * 1000L, keywords,
10000, platformName, "京东");
SearchHit[] sh = searchHits.getHits();
for (SearchHit s : sh) {
Map<String, Object> source = s.getSource();
if (!source.isEmpty()) {
try {
String message = mapper.writeValueAsString(source);
MediaMessage mm = mapper.readValue(message, MediaMessage.class);
mmessages.add(mm);
} catch (Exception e) {
e.printStackTrace();
}
}
}
SearchHits fsearchHits = esDao.getMediaDataycy(date.getTime() + i * 6 * 3600 * 1000L, fkeywords,
10000, platformName, "京东");
SearchHit[] fsh = fsearchHits.getHits();
for (SearchHit s : fsh) {
Map<String, Object> fsource = s.getSource();
if (!fsource.isEmpty()) {
try {
String fmessage = mapper.writeValueAsString(fsource);
MediaMessage fm = mapper.readValue(fmessage, MediaMessage.class);
fmmessages.add(fm);
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
float fnum = fmmessages.size();
float samenum = 0;
float snum = mmessages.size();
for (MediaMessage fmm : fmmessages) {
for (MediaMessage mm : mmessages) {
if (mm.getRsid() == fmm.getRsid()) {
samenum++;
}
}
}
System.out.println(platformName + "平台:");
System.out.println("子集数据量:" + snum);
System.out.println("超集数据量:" + fnum);
System.out.println("包含率:" + samenum / snum * 100 + "%");
System.out.println("未包含条数:" + (snum - samenum));
fmmessages = new ArrayList<>();
mmessages = new ArrayList<>();
} // 网媒
} // 获取完成
}
@SuppressWarnings({ "rawtypes", "unchecked" })
public HSSFWorkbook exportExcelWeibo(OutputStream out, HSSFWorkbook workbook, String keyword,
List<WeiboMessage> messages, String platformName) {
// 生成一个表格
HSSFSheet sheet = workbook.createSheet("京东" + "-" + platformName);
HSSFRow row = sheet.createRow(0);
try {
// 创建标题行
Class c = messages.iterator().next().getClass();
Field[] fields1 = c.getDeclaredFields();
List<String> headers = new ArrayList<>();
for (Field f : fields1) {
f.setAccessible(true);
headers.add(f.getName());
}
for (int i = 0; i < headers.size(); i++) {
HSSFCell cell = row.createCell(i);
HSSFRichTextString text = new HSSFRichTextString(headers.get(i));
cell.setCellValue(text);
}
// 遍历数据
Iterator<WeiboMessage> it = messages.iterator();
int index = 0;
while (it.hasNext()) {
index++;
row = sheet.createRow(index);
WeiboMessage t = it.next();
// 利用反射，根据javabean属性的先后顺序，动态调用getXxx()方法得到属性值
Field[] fields = t.getClass().getDeclaredFields();
for (int i = 0; i < fields.length; i++) {
HSSFCell cell = row.createCell(i);
Field field = fields[i];
String fieldName = field.getName();
String getMethodName = "get" + fieldName.substring(0, 1).toUpperCase() + fieldName.substring(1);
Class tCls = t.getClass();
Method getMethod = tCls.getMethod(getMethodName, new Class[] {});
Object value = getMethod.invoke(t, new Object[] {});
// 判断值的类型后进行强制类型转换
String textValue = null;
// if (value instanceof Integer) {
// int intValue = (Integer) value;
// cell.setCellValue(intValue);
// } else if (value instanceof Float) {
// float fValue = (Float) value;
// textValue = new HSSFRichTextString(String.valueOf(fValue));
// cell.setCellValue(textValue);
// } else if (value instanceof Double) {
// double dValue = (Double) value;
// textValue = new HSSFRichTextString(String.valueOf(dValue));
// cell.setCellValue(textValue);
// } else if (value instanceof Long) {
// long longValue = (Long) value;
// cell.setCellValue(longValue);
// } else {
textValue = value == null ? "" : value.toString();
cell.setCellValue(textValue);
// }
}
}
} catch (IllegalAccessException e) {
e.printStackTrace();
} catch (IllegalArgumentException e) {
e.printStackTrace();
} catch (InvocationTargetException e) {
e.printStackTrace();
} catch (NoSuchMethodException e) {
e.printStackTrace();
} catch (SecurityException e) {
e.printStackTrace();
}
return workbook;
}
@SuppressWarnings({ "unchecked", "rawtypes" })
public HSSFWorkbook exportExcelZhihu(OutputStream out, HSSFWorkbook workbook, String keyword,
List<ZhihuMessage> messages, String platformName) {
// 生成一个表格
HSSFSheet sheet = workbook.createSheet("京东" + "-" + platformName);
HSSFRow row = sheet.createRow(0);
try {
// 创建标题行
Class c = messages.iterator().next().getClass();
Field[] fields1 = c.getDeclaredFields();
List<String> headers = new ArrayList<>();
for (Field f : fields1) {
f.setAccessible(true);
headers.add(f.getName());
}
for (int i = 0; i < headers.size(); i++) {
HSSFCell cell = row.createCell(i);
HSSFRichTextString text = new HSSFRichTextString(headers.get(i));
cell.setCellValue(text);
}
// 遍历数据
Iterator<ZhihuMessage> it = messages.iterator();
int index = 0;
while (it.hasNext()) {
index++;
row = sheet.createRow(index);
ZhihuMessage t = it.next();
// 利用反射，根据javabean属性的先后顺序，动态调用getXxx()方法得到属性值
Field[] fields = t.getClass().getDeclaredFields();
for (int i = 0; i < fields.length; i++) {
HSSFCell cell = row.createCell(i);
Field field = fields[i];
String fieldName = field.getName();
String getMethodName = "get" + fieldName.substring(0, 1).toUpperCase() + fieldName.substring(1);
Class tCls = t.getClass();
Method getMethod = tCls.getMethod(getMethodName, new Class[] {});
Object value = getMethod.invoke(t, new Object[] {});
// 判断值的类型后进行强制类型转换
String textValue = null;
// if (value instanceof Integer) {
// int intValue = (Integer) value;
// cell.setCellValue(intValue);
// } else if (value instanceof Float) {
// float fValue = (Float) value;
// textValue = new HSSFRichTextString(String.valueOf(fValue));
// cell.setCellValue(textValue);
// } else if (value instanceof Double) {
// double dValue = (Double) value;
// textValue = new HSSFRichTextString(String.valueOf(dValue));
// cell.setCellValue(textValue);
// } else if (value instanceof Long) {
// long longValue = (Long) value;
// cell.setCellValue(longValue);
// } else {
textValue = value == null ? "" : value.toString();
if (textValue.length() > 10000) {
cell.setCellValue(textValue.substring(0, 9999));
continue;
}
cell.setCellValue(textValue);
// }
}
}
} catch (IllegalAccessException e) {
e.printStackTrace();
} catch (IllegalArgumentException e) {
e.printStackTrace();
} catch (InvocationTargetException e) {
e.printStackTrace();
} catch (NoSuchMethodException e) {
e.printStackTrace();
} catch (SecurityException e) {
e.printStackTrace();
}
return workbook;
}
@SuppressWarnings({ "unchecked", "rawtypes" })
public HSSFWorkbook exportExcelVideo(OutputStream out, HSSFWorkbook workbook, String keyword,
List<VideoMessage> messages, String platformName) {
// 生成一个表格
HSSFSheet sheet = workbook.createSheet("京东" + "-" + platformName);
HSSFRow row = sheet.createRow(0);
try {
// 创建标题行
Class c = messages.iterator().next().getClass();
Field[] fields1 = c.getDeclaredFields();
List<String> headers = new ArrayList<>();
for (Field f : fields1) {
f.setAccessible(true);
headers.add(f.getName());
}
for (int i = 0; i < headers.size(); i++) {
HSSFCell cell = row.createCell(i);
HSSFRichTextString text = new HSSFRichTextString(headers.get(i));
cell.setCellValue(text);
}
// 遍历数据
Iterator<VideoMessage> it = messages.iterator();
int index = 0;
while (it.hasNext()) {
index++;
row = sheet.createRow(index);
VideoMessage t = it.next();
// 利用反射，根据javabean属性的先后顺序，动态调用getXxx()方法得到属性值
Field[] fields = t.getClass().getDeclaredFields();
for (int i = 0; i < fields.length; i++) {
HSSFCell cell = row.createCell(i);
Field field = fields[i];
String fieldName = field.getName();
String getMethodName = "get" + fieldName.substring(0, 1).toUpperCase() + fieldName.substring(1);
Class tCls = t.getClass();
Method getMethod = tCls.getMethod(getMethodName, new Class[] {});
Object value = getMethod.invoke(t, new Object[] {});
// 判断值的类型后进行强制类型转换
String textValue = null;
// if (value instanceof Integer) {
// int intValue = (Integer) value;
// cell.setCellValue(intValue);
// } else if (value instanceof Float) {
// float fValue = (Float) value;
// textValue = new HSSFRichTextString(String.valueOf(fValue));
// cell.setCellValue(textValue);
// } else if (value instanceof Double) {
// double dValue = (Double) value;
// textValue = new HSSFRichTextString(String.valueOf(dValue));
// cell.setCellValue(textValue);
// } else if (value instanceof Long) {
// long longValue = (Long) value;
// cell.setCellValue(longValue);
// } else {
textValue = value == null ? "" : value.toString();
cell.setCellValue(textValue);
// }
}
}
} catch (IllegalAccessException e) {
e.printStackTrace();
} catch (IllegalArgumentException e) {
e.printStackTrace();
} catch (InvocationTargetException e) {
e.printStackTrace();
} catch (NoSuchMethodException e) {
e.printStackTrace();
} catch (SecurityException e) {
e.printStackTrace();
}
return workbook;
}
@SuppressWarnings({ "unchecked", "rawtypes" })
public HSSFWorkbook exportExcelMedia(OutputStream out, HSSFWorkbook workbook, String keyword,
List<MediaMessage> messages, String platformName) {
// 生成一个表格
HSSFSheet sheet = workbook.createSheet("京东" + "-" + platformName);
HSSFRow row = sheet.createRow(0);
try {
// 创建标题行
Class c = messages.iterator().next().getClass();
Field[] fields1 = c.getDeclaredFields();
List<String> headers = new ArrayList<>();
for (Field f : fields1) {
f.setAccessible(true);
headers.add(f.getName());
}
for (int i = 0; i < headers.size(); i++) {
HSSFCell cell = row.createCell(i);
HSSFRichTextString text = new HSSFRichTextString(headers.get(i));
cell.setCellValue(text);
}
// 遍历数据
Iterator<MediaMessage> it = messages.iterator();
int index = 0;
while (it.hasNext()) {
index++;
row = sheet.createRow(index);
MediaMessage t = it.next();
// 利用反射，根据javabean属性的先后顺序，动态调用getXxx()方法得到属性值
Field[] fields = t.getClass().getDeclaredFields();
for (int i = 0; i < fields.length; i++) {
HSSFCell cell = row.createCell(i);
Field field = fields[i];
String fieldName = field.getName();
String getMethodName = "get" + fieldName.substring(0, 1).toUpperCase() + fieldName.substring(1);
Class tCls = t.getClass();
Method getMethod = tCls.getMethod(getMethodName, new Class[] {});
Object value = getMethod.invoke(t, new Object[] {});
// 判断值的类型后进行强制类型转换
String textValue = null;
// if (value instanceof Integer) {
// int intValue = (Integer) value;
// cell.setCellValue(intValue);
// } else if (value instanceof Float) {
// float fValue = (Float) value;
// textValue = new HSSFRichTextString(String.valueOf(fValue));
// cell.setCellValue(textValue);
// } else if (value instanceof Double) {
// double dValue = (Double) value;
// textValue = new HSSFRichTextString(String.valueOf(dValue));
// cell.setCellValue(textValue);
// } else if (value instanceof Long) {
// long longValue = (Long) value;
// cell.setCellValue(longValue);
// } else {
textValue = value == null ? "" : value.toString();
if (textValue.length() > 10000) {
cell.setCellValue(textValue.substring(0, 9999));
continue;
}
cell.setCellValue(textValue);
// }
}
}
} catch (IllegalAccessException e) {
e.printStackTrace();
} catch (IllegalArgumentException e) {
e.printStackTrace();
} catch (InvocationTargetException e) {
e.printStackTrace();
} catch (NoSuchMethodException e) {
e.printStackTrace();
} catch (SecurityException e) {
e.printStackTrace();
}
return workbook;
}
}
src/main/java/com/zhiwei/messageflow/util/CommonUtils.java 0 → 100644
package com.zhiwei.messageflow.util;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
public final class CommonUtils
{
private final static Log log = LogFactory.getLog(CommonUtils.class);
/**
* 字符串转化为整形，若字符串非法，则返回0
*
* @param str 字符串
* @return 整形结果
*/
public static int strToInt(String str)
{
return strToIntDef(str, 0);
}
/**
* 字符串转化为整形
*
* @param str 字符串
* @param def 默认值(若为非法字符串是提供的返回值)
* @return 整形结果
*/
public static int strToIntDef(String str, int def)
{
try
{
return Integer.parseInt(str);
}
catch (NumberFormatException e)
{
if (log.isDebugEnabled())
log.error("[CommonUtils][strToIntDef(String, int)] NumberFormatException: " + str, e);
return def;
}
}
/**
* 字符串转化为长整形，若字符串非法，则返回0
*
* @param str 字符串
* @return 长整形结果
*/
public static long strToLong(String str)
{
return strToLongDef(str, 0);
}
/**
* 字符串转化为长整形，若字符串非法，则返回0
*
* @param str 字符串
* @param def 默认值(若为非法字符串是提供的返回值)
* @return 长整形结果
*/
public static long strToLongDef(String str, long def)
{
try
{
return Long.parseLong(str);
}
catch (NumberFormatException e)
{
if (log.isDebugEnabled())
log.error("[CommonUtils][strToLong(String, long)] NumberFormatException: " + str, e);
return def;
}
}
/**
* 判断是否是大整形格式(@see ava.math.BigDecimal)
*
* @param input 数字格式字符串
* @return
*/
public static boolean isBigDecimal(String input)
{
try
{
new java.math.BigDecimal(input);
return true;
}
catch (NumberFormatException e)
{
if (log.isDebugEnabled())
log.error("[CommonUtils][isBigDecimal(String)] Exception: " + input, e);
return false;
}
}
/**
* 判断是否是整形格式
*
* @param input 数字字符
* @return
*/
public static boolean isInteger(String input)
{
try
{
Integer.parseInt(input);
return true;
}
catch (NumberFormatException e)
{
if (log.isDebugEnabled())
log.error("[CommonUtils][strToLong(String)] NumberFormatException: " + input, e);
return false;
}
}
public static void main(String[] args)
{
}
}
src/main/java/com/zhiwei/messageflow/util/ESQueryUtil.java 0 → 100644
package com.zhiwei.messageflow.util;
import java.util.List;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
/**
*
* @TODO(z es 的查询工具类)
* @name ESQueryUtil
* @author 陈炜涛
* @time 2017年7月13日下午3:08:28
*/
public class ESQueryUtil {
public static final String ES_QUERY_AND = "must";
public static final String ES_QUERY_OR = "should";
public static final String ES_QUERY_NOT = "not";
public static HighlightBuilder getTrackHighlightBuilder(String[] fields, String color) {
HighlightBuilder hiBuilder = new HighlightBuilder();
hiBuilder.preTags("<font color='" + color + "'>");
hiBuilder.postTags("</font>");
for (String field : fields) {
hiBuilder.field(field, 300);
}
return hiBuilder;
}
/**
*
* @TODO(TODO 获取高亮设置)
* @author 陈炜涛
* @param fields
* @return
* @time 2017年7月13日下午3:44:39
* @return HighlightBuilder
*/
public static HighlightBuilder getHighlightBuilder(String... fields) {
HighlightBuilder hiBuilder = new HighlightBuilder();
hiBuilder.preTags("<font style='font-weight:bold;' color='#ff6c60'>");
hiBuilder.postTags("</font>");
for (String field : fields) {
hiBuilder.field(field, 300);
}
return hiBuilder;
}
/**
*
* @TODO(TODO 字段不拆封,多字段 同关键词)
* @author 陈炜涛
* @param boolQueryBuilder
* @param esQuery
* 是must 还是should
* @param word
* @param fields
* @return
* @time 2017年7月13日下午3:38:14
* @return BoolQueryBuilder
*/
public static BoolQueryBuilder matchPhraseQueryFields(BoolQueryBuilder boolQueryBuilder, String esQuery,
String word, String... fields) {
if (ES_QUERY_AND.equals(esQuery)) {
for (String field : fields) {
boolQueryBuilder.must(QueryBuilders.matchPhraseQuery(field, word));
}
} else if (ES_QUERY_OR.equals(esQuery)) {
for (String field : fields) {
boolQueryBuilder.should(QueryBuilders.matchPhraseQuery(field, word));
}
} else if (ES_QUERY_NOT.equals(esQuery)) {
for (String field : fields) {
boolQueryBuilder.mustNot(QueryBuilders.matchPhraseQuery(field, word));
}
}
return boolQueryBuilder;
}
/**
*
* @TODO(TODO 字段不拆封,多字段 同关键词)
* @author 陈炜涛
* @param boolQueryBuilder
* @param esQuery
* 是must 还是should
* @param word
* @param fields
* @return
* @time 2017年7月13日下午3:38:14
* @return BoolQueryBuilder
*/
public static BoolQueryBuilder matchPhraseQueryFields(BoolQueryBuilder boolQueryBuilder, String esQuery,
String word, List<String> fields) {
if (ES_QUERY_AND.equals(esQuery)) {
for (String field : fields) {
boolQueryBuilder.must(QueryBuilders.matchPhraseQuery(field, word));
}
} else if (ES_QUERY_OR.equals(esQuery)) {
for (String field : fields) {
boolQueryBuilder.should(QueryBuilders.matchPhraseQuery(field, word));
}
} else if (ES_QUERY_NOT.equals(esQuery)) {
for (String field : fields) {
boolQueryBuilder.mustNot(QueryBuilders.matchPhraseQuery(field, word));
}
}
return boolQueryBuilder;
}
/**
*
* @TODO(TODO 字段不拆封,单字段 多关键词)
* @author 陈炜涛
* @param boolQueryBuilder
* @param field
* @param words
* @return
* @time 2017年7月13日下午3:39:43
* @return BoolQueryBuilder
*/
public static BoolQueryBuilder matchPhraseQueryWords(BoolQueryBuilder boolQueryBuilder, String esQuery,
String field, List<String> words) {
if (ES_QUERY_AND.equals(esQuery)) {
for (String word : words) {
boolQueryBuilder.must(QueryBuilders.matchPhraseQuery(field, word));
}
} else if (ES_QUERY_OR.equals(esQuery)) {
for (String word : words) {
boolQueryBuilder.should(QueryBuilders.matchPhraseQuery(field, word));
}
} else if (ES_QUERY_NOT.equals(esQuery)) {
for (String word : words) {
boolQueryBuilder.mustNot(QueryBuilders.matchPhraseQuery(field, word));
}
}
return boolQueryBuilder;
}
/**
*
* @TODO(TODO 字段不拆封,单字段 多关键词)
* @author 陈炜涛
* @param boolQueryBuilder
* @param field
* @param words
* @return
* @time 2017年7月13日下午3:39:43
* @return BoolQueryBuilder
*/
public static BoolQueryBuilder matchPhraseQueryWords(BoolQueryBuilder boolQueryBuilder, String esQuery,
String field, String... words) {
if (ES_QUERY_AND.equals(esQuery)) {
for (String word : words) {
boolQueryBuilder.must(QueryBuilders.matchPhraseQuery(field, word));
}
} else if (ES_QUERY_OR.equals(esQuery)) {
for (String word : words) {
boolQueryBuilder.should(QueryBuilders.matchPhraseQuery(field, word));
}
} else if (ES_QUERY_NOT.equals(esQuery)) {
for (String word : words) {
boolQueryBuilder.mustNot(QueryBuilders.matchPhraseQuery(field, word));
}
}
return boolQueryBuilder;
}
}
src/main/java/com/zhiwei/messageflow/util/ESTransport.java 0 → 100644
package com.zhiwei.messageflow.util;
import com.zhiwei.es.util.ESIndexesUtil;
public class ESTransport {
protected static ESIndexesUtil.ESIndexes indexes = ESIndexesUtil.create("1.119.44.201", 9200);
}
src/main/java/com/zhiwei/messageflow/util/HttpRequest.java 0 → 100644
package com.zhiwei.messageflow.util;
import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.net.ConnectException;
import java.net.URL;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLSocketFactory;
import javax.net.ssl.TrustManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import net.sf.json.JSONObject;
/**
* 菜单管理器类
*
* @author liufeng
* @date 2013-08-08
*/
public class HttpRequest {
private static Logger log = LoggerFactory.getLogger(HttpRequest.class);
/**
* 发起https请求并获取结果
*
* @param requestUrl
* 请求地址
* @param requestMethod
* 请求方式（GET、POST）
* @param outputStr
* 提交的数据
* @return JSONObject(通过JSONObject.get(key)的方式获取json对象的属性值)
*/
public static JSONObject httpRequest(String requestUrl,
String requestMethod, String outputStr) {
JSONObject jsonObject = null;
StringBuffer buffer = new StringBuffer();
try {
// 创建SSLContext对象，并使用我们指定的信任管理器初始化
TrustManager[] tm = { new MyX509TrustManager() };
SSLContext sslContext = SSLContext.getInstance("SSL", "SunJSSE");
sslContext.init(null, tm, new java.security.SecureRandom());
// 从上述SSLContext对象中得到SSLSocketFactory对象
SSLSocketFactory ssf = sslContext.getSocketFactory();
URL url = new URL(requestUrl);
HttpsURLConnection httpUrlConn = (HttpsURLConnection) url
.openConnection();
httpUrlConn.setSSLSocketFactory(ssf);
httpUrlConn.setDoOutput(true);
httpUrlConn.setDoInput(true);
httpUrlConn.setUseCaches(false);
// 设置请求方式（GET/POST）
httpUrlConn.setRequestMethod(requestMethod);
if ("GET".equalsIgnoreCase(requestMethod))
httpUrlConn.connect();
// 当有数据需要提交时
if (null != outputStr) {
OutputStream outputStream = httpUrlConn.getOutputStream();
// 注意编码格式，防止中文乱码
outputStream.write(outputStr.getBytes("UTF-8"));
outputStream.close();
}
// 将返回的输入流转换成字符串
InputStream inputStream = httpUrlConn.getInputStream();
InputStreamReader inputStreamReader = new InputStreamReader(
inputStream, "utf-8");
BufferedReader bufferedReader = new BufferedReader(
inputStreamReader);
String str = null;
while ((str = bufferedReader.readLine()) != null) {
buffer.append(str);
}
bufferedReader.close();
inputStreamReader.close();
// 释放资源
inputStream.close();
inputStream = null;
httpUrlConn.disconnect();
jsonObject = JSONObject.fromObject(buffer.toString());
} catch (ConnectException ce) {
log.error("Weixin server connection timed out.");
} catch (Exception e) {
log.error("https request error:{}", e);
}
return jsonObject;
}
}
src/main/java/com/zhiwei/messageflow/util/MyX509TrustManager.java 0 → 100644
package com.zhiwei.messageflow.util;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import javax.net.ssl.X509TrustManager;
/**
* 证书信任管理器（用于https请求）
*
* @author liufeng
* @date 2013-08-08
*/
public class MyX509TrustManager implements X509TrustManager {
public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {
}
public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {
}
public X509Certificate[] getAcceptedIssuers() {
return null;
}
}
\ No newline at end of file
src/main/java/com/zhiwei/messageflow/util/SerializeUtil.java 0 → 100644
package com.zhiwei.messageflow.util;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
/**
* @deprecated:序列化 & 反序列化
* @author TangTou
*
*/
public class SerializeUtil
{
/**
* @deprecated:JDK序列化
* @param obj
* @return
*/
public static byte[] JDKserialize(Object obj)
{
ObjectOutputStream oos = null;
ByteArrayOutputStream baos = null;
try
{
baos = new ByteArrayOutputStream();
oos = new ObjectOutputStream(baos);
oos.writeObject(obj);
return baos.toByteArray();
}
catch (IOException e)
{
return baos.toByteArray();
}
finally
{
if (oos != null)
try
{
oos.close();
baos.close();
}
catch (IOException e)
{
}
}
}
/**
* @deprecated: JDK反序列化
* @param bytes
* @return
*/
public static Object JDKunserialize(byte[] bytes)
throws Exception
{
ObjectInputStream ois = null;
ByteArrayInputStream bais = null;
try
{
bais = new ByteArrayInputStream(bytes);
ois = new ObjectInputStream(bais);
return ois.readObject();
}
catch (Exception e)
{
return bais;
}
finally
{
if (ois != null)
try
{
ois.close();
bais.close();
}
catch (Exception e)
{
e.printStackTrace();
}
}
}
}
src/main/java/com/zhiwei/messageflow/util/ThreadEndbyTimerTask.java 0 → 100644
package com.zhiwei.messageflow.util;
import java.util.TimerTask;
public class ThreadEndbyTimerTask extends TimerTask{
@Override
public void run() {
}
}
src/main/java/com/zhiwei/messageflow/util/TimeUtil.java 0 → 100644
package com.zhiwei.messageflow.util;
import java.text.DateFormat;
/*
* 创建日期 2006-1-12
*
* 时间运算函数
*
*/
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
/**
* @author zhengqinyong <br>
* 常用时间函数， 统一格式:日期 yyyy-MM-dd, 时间 yyyy-MM-dd HH:mm:ss
*/
public class TimeUtil {
private final static Log logger = LogFactory.getLog(TimeUtil.class);
static final SimpleDateFormat DATE_FORMATE = new SimpleDateFormat("yyyy-MM-dd");
static final SimpleDateFormat TIME_FORMATE = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
static final SimpleDateFormat ES_TIME_FORMATE = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS");
static final SimpleDateFormat HOUR_FORMATE = new SimpleDateFormat("yyyy-MM-dd HH");
static final SimpleDateFormat MINUTE_FORMATE = new SimpleDateFormat("yyyy-MM-dd HH:mm");
static final SimpleDateFormat Month_FORMATE = new SimpleDateFormat("yyyy-MM");
static final SimpleDateFormat Year = new SimpleDateFormat("yyyy");
static final SimpleDateFormat Month = new SimpleDateFormat("MM");
static final SimpleDateFormat TIME_PART = new SimpleDateFormat("HH:mm:ss SSS");
static final Calendar G_CALENDAR = new java.util.GregorianCalendar();
/**
* 判断是否是日期格式(yyyy-mm-dd)
*
* @param input
* 日期字符串
* @return
*/
public static boolean isDate(String input) {
if (input != null && input.matches("\\d\\d\\d\\d\\-\\d\\d\\-\\d\\d")) {
int year = CommonUtils.strToInt(input.substring(0, 4));
if (1900 > year || year > 3000)
return false;
int month = CommonUtils.strToInt(input.substring(5, 7));
if (1 > month || month > 12)
return false;
int day = CommonUtils.strToInt(input.substring(8, 10));
if (1 > day || day > 31)
return false;
return true;
} else {
return false;
}
}
/**
* 判断是否是时间格式(yy-MM-dd HH:mm:ss)
*
* @param input
* 时间字符串
* @return
*/
public static boolean isDatetime(String input) {
if (input != null && input.matches("\\d\\d\\d\\d-\\d\\d-\\d\\d \\d\\d:\\d\\d:\\d\\d")) {
int year = CommonUtils.strToInt(input.substring(0, 4));
if (1900 > year || year > 3000)
return false;
int month = CommonUtils.strToInt(input.substring(5, 7));
if (1 > month || month > 12)
return false;
int day = CommonUtils.strToInt(input.substring(8, 10));
if (1 > day || day > 31)
return false;
int hour = CommonUtils.strToInt(input.substring(11, 13));
if (0 > hour || hour > 23)
return false;
int minute = CommonUtils.strToInt(input.substring(14, 16));
if (0 > minute || minute > 59)
return false;
int second = CommonUtils.strToInt(input.substring(17, 19));
if (0 > second && second > 59)
return false;
return true;
} else {
return false;
}
}
/**
* @return 当前时间
*/
public static Date getCurrentTime() {
return new Date();
}
/**
* @return 获取当前时间字符串
*/
public static String getCurrentTimeStr() {
return formatTime(getCurrentTime());
}
/**
* @return 获取当前日期字符串
*/
public static String getCurrentDateStr() {
return formatDate(getCurrentTime());
}
/**
* 格式化时间当前月
*
* @param Date
* @return String
*/
public static synchronized String formatTimeMonth(Date date) {
if (null == date) {
return null;
}
return Month.format(date);
}
/**
* 格式化时间当前年份
*
* @param Date
* @return String
*/
public static synchronized String formatTimeYear(Date date) {
if (null == date) {
return null;
}
return Year.format(date);
}
/**
* 格式化时间
*
* @param Date
* @return String
*/
public static synchronized String formatTime(Date date) {
if (null == date) {
return null;
}
return TIME_FORMATE.format(date);
}
/**
* 格式化时间
*
* @param DateString
* @return String
*/
public static String formatTime(String timeStr) {
if (null == timeStr) {
return null;
}
Date date = parseTime(timeStr);
return formatTime(date);
}
/**
* 格式化日期
*
* @param dateString
* @return String
*/
public static String formatDate(String dateStr) {
if (null == dateStr) {
return null;
}
Date date = parseDate(dateStr);
return formatDate(date);
}
/**
* 格式化日期
*
* @param Date
* @return String
*/
public static synchronized String formatDate(Date date) {
if (null == date) {
return null;
}
return DATE_FORMATE.format(date);
}
/**
* 格式化日期
*
* @param Date
* @return String
*/
public static synchronized String formatDateToHour(Date date) {
if (null == date) {
return null;
}
return HOUR_FORMATE.format(date);
}
/**
* 格式化日期
*
* @param Date
* @return String
*/
public static synchronized String formatDateToMonth(Date date) {
if (null == date) {
return null;
}
return Month_FORMATE.format(date);
}
/**
* 格式化日期到分
*
* @param Date
* @return String
*/
public static synchronized String formatDateToMinute(Date date) {
if (null == date) {
return null;
}
return MINUTE_FORMATE.format(date);
}
/**
*
* @Title: getTimePointToNowOrEnd
* @Description: 获取开始到结束时间的时间点，按小时
* @param @return 设定文件
* @return List<String> 返回类型
* @throws ParseException
*/
public static List<String> getTimePointToNowOrEnd(Date startTime,
Date endTime,String timeType)
{
Date start = startTime;
Long end = null;
if (null == endTime)
{
end = System.currentTimeMillis();
}
else
{
end = endTime.getTime();
}
Calendar car = Calendar.getInstance();
car.setTime(start);
boolean flag = true;
// 为完善表现层，填充无数据时间点
// 创建完整时间点
List<String> times = new ArrayList<String>();
if ("day".equals(timeType)) {
while (flag)
{
times.add(DATE_FORMATE.format(car.getTime()));
if (end <= car.getTimeInMillis())
{
flag = false;
}
car.add(Calendar.DAY_OF_MONTH, 1);
}
}else {
while (flag)
{
times.add(HOUR_FORMATE.format(car.getTime()));
if (end <= car.getTimeInMillis())
{
flag = false;
}
car.add(Calendar.HOUR, 1);
}
}
return times;
}
/**
* 根据格式化字符串，格式化日期
*
* @param Date
* 时间
* @param partern
* 格式
* @return
*/
public static String formatDate(Date date, String pattern) {
if (null == date || pattern == null) {
return null;
}
return new SimpleDateFormat(pattern).format(date);
}
/**
* 解析日期字符串
*
* @param dateString
* @return Date
*/
public static synchronized Date parseEsDate(String dateStr) {
if (null == dateStr) {
return null;
}
Date date = null;
try {
date = ES_TIME_FORMATE.parse(dateStr.replaceFirst("000", ""));
} catch (ParseException e) {
e.printStackTrace();
}
return date;
}
/**
* 根据格式化字符串，格式化日期
*
* @param Date
* 时间
* @param partern
* 格式
* @return
*/
public static synchronized String formatEsDate(Date date) {
if (null == date) {
return null;
}
return ES_TIME_FORMATE.format(date);
}
/**
* 解析日期字符串
*
* @param dateString
* @return Date
*/
public static synchronized Date parseDate(String dateStr) {
if (null == dateStr) {
return null;
}
Date date = null;
try {
date = DATE_FORMATE.parse(dateStr);
} catch (ParseException e) {
}
return date;
}
/**
* 解析日期字符串
*
* @param dateString
* @return Date
*/
public static synchronized Date parseHour(String hourStr) {
if (null == hourStr) {
return null;
}
Date date = null;
try {
date = HOUR_FORMATE.parse(hourStr);
} catch (ParseException e) {
}
return date;
}
/**
* 解析时间字符串
*
* @param timeString
* @return Date
*/
public static synchronized Date parseTime(String timeStr) {
if (null == timeStr) {
return null;
}
Date date = null;
try {
date = TIME_FORMATE.parse(timeStr);
} catch (ParseException e) {
}
return date;
}
/**
* 解析时间字符串
*
* @param timeString
* @param 时间格式
* @return Date
*/
public static synchronized Date parseTime(String timeStr, String pattern) {
if (null == timeStr || pattern == null) {
return null;
}
try {
return new SimpleDateFormat(pattern).parse(timeStr);
} catch (ParseException e) {
}
return null;
}
/**
* 当前日期偏移运算(增、减几日）
*
* @param skipDate
* 年偏移量
* @return Date 偏移日期
*/
public static Date getSkipYear(Date date, int skipYear) {
G_CALENDAR.setTime(date);
G_CALENDAR.add(Calendar.YEAR, skipYear);
return G_CALENDAR.getTime();
}
/**
* 当前日期偏移运算(增、减几日）
*
* @param skipDate
* 日偏移量
* @return Date 偏移日期
*/
public static Date getSkipTime(int skipDay) {
return getSkipTime(getCurrentTime(), skipDay);
}
/**
* 某一时间的偏移运算(增、减几日）
*
* @param Date
* @param skipDate
* 日偏移量
* @return Date 偏移日期
*/
public static synchronized Date getSkipTime(Date date, int skipDay) {
G_CALENDAR.setTime(date);
G_CALENDAR.add(Calendar.DAY_OF_MONTH, skipDay);
return G_CALENDAR.getTime();
}
/**
* 当前时间的偏移运算(增、减几日、几小时、几分、几秒）
*
* @param skipDate
* 日偏移量
* @param skipHour
* 偏移时
* @param skipMinute
* 偏移分
* @param skipSecond
* 偏移秒
* @return Date 偏移日期
*/
public static Date getSkipTime(int skipDay, int skipHour, int skipMinute, int skipSecond) {
return getSkipTime(getCurrentTime(), skipDay, skipHour, skipMinute, skipSecond);
}
/**
* 某一时间的偏移运算(增、减几日、几小时、几分）
*
* @param date
* 原时间
* @param skipDate
* 日偏移量
* @param skipHour
* 偏移时
* @param skipMinute
* 偏移分
* @param skipSecond
* 偏移秒
* @return Date 偏移时间
*/
public static Date getSkipTime(Date date, int skipDay, int skipHour, int skipMinute, int skipSecond) {
if (null == date) {
return null;
}
return getSkipTime(date, 0, 0, skipDay, skipHour, skipMinute, skipSecond);
}
/**
* 某一时间的偏移运算
*
* @param date
* 原时间
* @param skipYear
* 年偏移量
* @param skipMonth
* 月偏移量
* @param skipDay
* 日偏移量
* @param skipHour
* 小时偏移量
* @param skipMinute
* 分偏移量
* @return 偏移时间
*/
public static synchronized Date getSkipTime(Date date, int skipYear, int skipMonth, int skipDay, int skipHour,
int skipMinute, int skipSecond) {
if (null == date) {
return null;
}
G_CALENDAR.setTime(date);
G_CALENDAR.add(Calendar.YEAR, skipYear);
G_CALENDAR.add(Calendar.MONTH, skipMonth);
G_CALENDAR.add(Calendar.DAY_OF_MONTH, skipDay);
G_CALENDAR.add(Calendar.HOUR_OF_DAY, skipHour);
G_CALENDAR.add(Calendar.MINUTE, skipMinute);
G_CALENDAR.add(Calendar.SECOND, skipSecond);
return G_CALENDAR.getTime();
}
private static final int MSEL_OF_HOUR = 1000 * 3600;
private static final int MSEL_OF_DAY = MSEL_OF_HOUR * 24;
/**
* 计算日期相差小时数
*
* @param base
* 比较基准时间
* @param compare
* 比较时间
* @return long 时偏差
*/
public static long getSubhour(Date base, Date compare) {// subtrahend
long daterange = base.getTime() - compare.getTime();
long time = MSEL_OF_HOUR;
return (daterange % time == 0) ? (daterange / time) : (daterange / time + 1);
}
/**
* 计算日期相差几天
*
* @param base
* 比较基准时间
* @param compare
* 比较时间
* @return long 日偏差
*/
public static long getSubday(Date base, Date compare) {// subtrahend
long daterange = base.getTime() - compare.getTime();
long time = MSEL_OF_DAY;
return (daterange % time == 0) ? (daterange / time) : (daterange / time + 1);
}
/**
* @param date
* @return 所属星期
*/
public static int dayOfWeek(Date date) {
return getFieldOfDate(date, Calendar.DAY_OF_WEEK);
}
/**
* 返回日期属性字段值，字段定义见 com.util.Calendar，
*
* @param date
* 当前日期
* @param field
* @see com.util.Calendar
* @return
*/
public static synchronized int getFieldOfDate(Date date, int field) {
G_CALENDAR.setTime(date);
return G_CALENDAR.get(field);
}
/**
* @return 取日期部分，截去时间
*/
public static synchronized Date getDatePart(Date date) {
String d = formatDate(date);
return parseDate(d);
}
/**
* @return 取日期到小时部分，截去分钟和秒
*/
public static synchronized Date getDatePartToHour(Date date) {
String d = formatDateToHour(date);
return parseHour(d);
}
/**
* @return 取时间部分，截去日期
*/
public static synchronized Date getTimePart(Date date) {
if (null == date)
return null;
String d = TIME_PART.format(date);
try {
return TIME_PART.parse(d);
} catch (ParseException e) {
return null;
}
}
public static synchronized Date addTimeHour(Date date, int hour) {
long ms = date.getTime()+hour*60*60*1000;
return new Date(ms);
}
/**
* 爬虫专用处理（xxxx年xx月xx日 xx:xx:xx 、xx月xx日 xx:xx:xx、xx小时前、xx分钟前、xx秒前等不规则时间）
*
* @param String
* date
* @return Date
*/
public static String conversionTime(String time) {
if (time.contains("小时")) {
time = TimeUtil.formatTime(TimeUtil.getSkipTime(0, -Integer.valueOf(time.split("小时")[0]), 0, 0));
} else if (time.contains("分")) {
time = TimeUtil.formatTime(TimeUtil.getSkipTime(0, 0, -Integer.valueOf(time.split("分")[0]), 0));
} else if (time.contains("秒")) {
time = TimeUtil.formatTime(TimeUtil.getSkipTime(0, 0, 0, -Integer.valueOf(time.split("秒")[0])));
} else if (time.contains("今天")) {
time = TimeUtil.getCurrentDateStr() + " " + time.split(" ")[1] + ":00";
return time;
} else if (time.contains("昨天")) {
time = TimeUtil.formatDate(TimeUtil.getSkipTime(-1)) + " " + time.split(" ")[1] + ":00";
return time;
} else {
time = conversionTimeYMD(time);
}
return time;
}
// 处理带有年月日字符串的
private static String conversionTimeYMD(String time) {
String year = "";
String month = "";
String day = "";
String second = "";
String[] timeArray = null;
if (time.contains("年")) {
timeArray = time.split("年");
year = timeArray[0];
timeArray = timeArray[1].split("月");
} else {
year = TimeUtil.getCurrentDateStr().split("-")[0];
timeArray = time.split("月");
}
if (time.contains("月")) {
month = Integer.valueOf(timeArray[0]) < 10 ? "0" + Integer.valueOf(timeArray[0]) : timeArray[0];
timeArray = timeArray[1].split("日");
}
if (time.contains("日")) {
day = Integer.valueOf(timeArray[0]) < 10 ? "0" + Integer.valueOf(timeArray[0]) : timeArray[0];
}
if (time.split(" ").length < 2) {
second = "00:00:00";
} else {
second = time.split(" ")[1].replace("：", ":");
switch (second.split(":|：").length) {
case 1:
second += ":00:00";
break;
case 2:
second += ":00";
break;
}
}
return year + "-" + month + "-" + day + " " + second;
}
/**
* @Title: isNum
* @Description: 验证是否为时间戳
* @param @param
* time
* @param @return
* 设定文件
* @return boolean 返回类型
*/
public static boolean isNum(String time) {
String regex = "^[0-9]*$";
Pattern pattern = Pattern.compile(regex);
Matcher matcher = pattern.matcher(time);
if (matcher.find()) {
return true;
}
return false;
}
/**
* @Title: FormatDate
* @Description: 处理时间格式
* @param @param
* dateStr
* @param @return
* 设定文件
* @return String 返回类型
*/
@SuppressWarnings("finally")
public static Date stringFormatDate(String dateStr){
HashMap<String, String> dateRegFormat = new HashMap<String, String>();
dateRegFormat.put("^\\d{4}\\D+\\d{1,2}\\D+\\d{1,2}\\D+\\d{1,2}\\D+\\d{1,2}\\D+\\d{1,2}\\D*$",
"yyyy-MM-dd-HH-mm-ss");// 2014年3月12日 13时5分34秒，2014-03-12
// 12:05:34，2014/3/12 12:5:34
dateRegFormat.put("^\\d{4}\\D+\\d{1,2}\\D+\\d{1,2}\\D+\\d{1,2}\\D+\\d{1,2}$", "yyyy-MM-dd-HH-mm");// 2014-03-12
// 12:05
dateRegFormat.put("^\\d{4}\\D+\\d{1,2}\\D+\\d{1,2}\\D+\\d{1,2}$", "yyyy-MM-dd-HH");// 2014-03-12
// 12
dateRegFormat.put("^\\d{4}\\D+\\d{1,2}\\D+\\d{1,2}$", "yyyy-MM-dd");// 2014-03-12
dateRegFormat.put("^\\d{4}\\D+\\d{1,2}$", "yyyy-MM");// 2014-03
dateRegFormat.put("^\\d{4}$", "yyyy");// 2014
dateRegFormat.put("^\\d{14}$", "yyyyMMddHHmmss");// 20140312120534
dateRegFormat.put("^\\d{12}$", "yyyyMMddHHmm");// 201403121205
dateRegFormat.put("^\\d{10}$", "yyyyMMddHH");// 2014031212
dateRegFormat.put("^\\d{8}$", "yyyyMMdd");// 20140312
dateRegFormat.put("^\\d{6}$", "yyyyMM");// 201403
dateRegFormat.put("^\\d{2}\\s*:\\s*\\d{2}\\s*:\\s*\\d{2}$", "yyyy-MM-dd-HH-mm-ss");// 13:05:34
// 拼接当前日期
dateRegFormat.put("^\\d{2}\\s*:\\s*\\d{2}$", "yyyy-MM-dd-HH-mm");// 13:05
// 拼接当前日期
dateRegFormat.put("^\\d{2}\\D+\\d{1,2}\\D+\\d{1,2}$", "yy-MM-dd");// 14.10.18(年.月.日)
dateRegFormat.put("^\\d{1,2}\\D+\\d{1,2}$", "yyyy-dd-MM");// 30.12(日.月)
// 拼接当前年份
dateRegFormat.put("^\\d{1,2}\\D+\\d{1,2}\\D+\\d{4}$", "dd-MM-yyyy");// 12.21.2013(日.月.年)
String curDate = new SimpleDateFormat("yyyy-MM-dd").format(new Date());
DateFormat formatter2;
String dateReplace;
Date date = null;
try {
for (String key : dateRegFormat.keySet()) {
if (Pattern.compile(key).matcher(dateStr).matches()) {
formatter2 = new SimpleDateFormat(dateRegFormat.get(key));
if (key.equals("^\\d{2}\\s*:\\s*\\d{2}\\s*:\\s*\\d{2}$") || key.equals("^\\d{2}\\s*:\\s*\\d{2}$")) {// 13:05:34
dateStr = curDate + "-" + dateStr;
} else if (key.equals("^\\d{1,2}\\D+\\d{1,2}$")) {// 21.1
dateStr = curDate.substring(0, 4) + "-" + dateStr;
}
dateReplace = dateStr.replaceAll("\\D+", "-");
date = formatter2.parse(dateReplace);
break;
}
}
} catch (Exception e) {
logger.info("日期格式无效",e.fillInStackTrace());
} finally {
return date;
}
}
}
src/main/resources/META-INF/additional-spring-configuration-metadata.json 0 → 100644
{"properties": [
{
"name": "keyMaxSize",
"type": "java.lang.String",
"description": "A description for 'keyMaxSize'"
},
{
"name": "redis.maxWaitMills",
"type": "java.lang.String",
"description": "A description for 'redis.maxWaitMills'"
}
]}
\ No newline at end of file
src/main/resources/application.properties 0 → 100644
##服务端uri
#spring.data.mongodb.uri=115.236.59.91:27017
#服务端数据库
spring.data.mongodb.primary.database=qbjcPhoenix
#服务ip
spring.data.mongodb.primary.host=115.236.59.91
#服务port
spring.data.mongodb.primary.port=27017
spring.data.mongodb.primary.username=stno
spring.data.mongodb.primary.password=stno1q2w3e4r
spring.data.mongodb.primary.authenticationDatabase=admin
#服务端数据库
spring.data.mongodb.secondary.database=eventMuseum
#服务ip
spring.data.mongodb.secondary.host=115.236.59.91
#服务port
spring.data.mongodb.secondary.port=27017
spring.data.mongodb.secondary.username=stno
spring.data.mongodb.secondary.password=stno1q2w3e4r
spring.data.mongodb.secondary.authenticationDatabase=admin
#服务端数据库
spring.data.mongodb.thirdary.database=WechatPublic
#服务ip
spring.data.mongodb.thirdary.host=115.236.59.91
#服务port
spring.data.mongodb.thirdary.port=27017
spring.data.mongodb.thirdary.username=stno
spring.data.mongodb.thirdary.password=stno1q2w3e4r
spring.data.mongodb.thirdary.authenticationDatabase=admin
#spring.data.mongodb.option.min-connection-per-host=0
#spring.data.mongodb.option.max-connection-per-host=100
#spring.data.mongodb.option.threads-allowed-to-block-for-connection-multiplier=5
#spring.data.mongodb.option.server-selection-timeout=30000
#spring.data.mongodb.option.max-wait-time=120000
#spring.data.mongodb.option.max-connection-idle-time=0
#spring.data.mongodb.option.max-connection-life-time=0
#spring.data.mongodb.option.connect-timeout=10000
#spring.data.mongodb.option.socket-timeout=0
#
#spring.data.mongodb.option.socket-keep-alive=false
#spring.data.mongodb.option.ssl-enabled=false
#spring.data.mongodb.option.ssl-invalid-host-name-allowed=false
#spring.data.mongodb.option.always-use-m-beans=false
#
#spring.data.mongodb.option.heartbeat-socket-timeout=20000
#spring.data.mongodb.option.heartbeat-connect-timeout=20000
#spring.data.mongodb.option.min-heartbeat-frequency=500
#spring.data.mongodb.option.heartbeat-frequency=10000
#spring.data.mongodb.option.local-threshold=15
##本地ip
#spring.data.mongodb.host=192.168.0.241
##本地port
#spring.data.mongodb.port=27017
##本地数据库
#spring.data.mongodb.database=qbjcPhoenix
#其他数据库
#spring.data.mongodb.database=eventMuseum
#spring.data.mongodb.database=WechatPublic
#tag用uri
#spring.data.mongodb.uri=1.119.44.206:30000
#spring.data.mongodb.uri=192.168.0.245:27017
#tag数据库
#spring.data.mongodb.database=Testqbjc
#spring.data.mongodb.database=weibotag
#mongo.connectionsPerHost=200
#mongo.threadsAllowedToBlockForConnectionMultiplier=10
#
#mongo.connectTimeout=30000
#
#mongo.maxWaitTime=50000
#mongo.autoConnectRetry=true
#mongo.socketKeepAlive=true
#
#mongo.socketTimeout=120000
#mongo.slaveOk=true
\ No newline at end of file
src/main/resources/es.properties 0 → 100644
es.ip=1.119.44.201
#es.ip=192.168.0.234
#es.ip=115.231.214.234
#es.ip=192.168.0.11
es.esPort=9300
es.clusterName=Media-University
#es.clusterName=elasticsearch
#es.clusterName=MyEsCluster
es.zhihuIndexName=zhihu*
es.zhihuType=zhihu
#es.mediaIndexName=media*
es.mediaIndexName=mediaspider*
es.mediaType=media
es.weiboIndexName=network*
es.weiboType=status
es.videoIndexName=videos*
es.videoType=video
#es.mediaMarkIndexName=network*
#es.mediaMarkType=qbjc_media_mark
es.mediaMarkIndexName=mediatag
es.mediaMarkType=mediatag
es.weiboMarkIndexName=weibotag
es.weiboMarkType=weibotag
\ No newline at end of file
src/main/resources/logback-spring.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<!-- scan 配置文件如果发生改变，将会被重新加载 scanPeriod 检测间隔时间-->
<configuration scan="true" scanPeriod="60 seconds" debug="false">
<contextName>spring-boot-log</contextName>
<include resource="org/springframework/boot/logging/logback/base.xml"/>
<!-- 普通日志 -->
<appender name="INFO_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>log/spring-boot-log-info.log</file>
<!-- 循环政策：基于时间创建日志文件 -->
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<!-- 日志命名:单个文件大于128MB 按照时间+自增i 生成log文件 -->
<fileNamePattern>log/spring-boot-log-info-%d{yyyy-MM-dd}.%i.log</fileNamePattern>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<maxFileSize>128MB</maxFileSize>
</timeBasedFileNamingAndTriggeringPolicy>
<!-- 最大保存时间：30天-->
<maxHistory>30</maxHistory>
</rollingPolicy>
<append>true</append>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level %logger Line:%-3L - %msg%n</pattern>
<charset>utf-8</charset>
</encoder>
<filter class="ch.qos.logback.classic.filter.LevelFilter">
<level>info</level>
<onMatch>ACCEPT</onMatch>
<onMismatch>DENY</onMismatch>
</filter>
</appender>
<!-- 错误日志 -->
<appender name="ERROR_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>log/spring-boot-log-error.log</file>
<!-- 循环政策：基于时间创建日志文件 -->
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<!-- 日志命名:单个文件大于2MB 按照时间+自增i 生成log文件 -->
<fileNamePattern>log/spring-boot-log-error-%d{yyyy-MM-dd}.%i.log</fileNamePattern>
<timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
<maxFileSize>10MB</maxFileSize>
</timeBasedFileNamingAndTriggeringPolicy>
<!-- 最大保存时间：180天-->
<maxHistory>180</maxHistory>
</rollingPolicy>
<append>true</append>
<!-- 日志格式 -->
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level %logger Line:%-3L - %msg%n</pattern>
<charset>utf-8</charset>
</encoder>
<!-- 日志级别过滤器 -->
<filter class="ch.qos.logback.classic.filter.LevelFilter">
<!-- 过滤的级别 -->
<level>ERROR</level>
<!-- 匹配时的操作：接收（记录） -->
<onMatch>ACCEPT</onMatch>
<!-- 不匹配时的操作：拒绝（不记录） -->
<onMismatch>DENY</onMismatch>
</filter>
</appender>
<!-- 控制台 -->
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<!-- 日志格式 -->
<encoder>
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level %logger Line:%-3L - %msg%n</pattern>
<charset>utf-8</charset>
</encoder>
<!--此日志appender是为开发使用，只配置最底级别，控制台输出的日志级别是大于或等于此级别的日志信息-->
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>INFO</level>
</filter>
</appender>
<!-- additivity 避免执行2次 -->
<logger name="com.itstyle" level="INFO" additivity="false">
<appender-ref ref="STDOUT"/>
<appender-ref ref="INFO_FILE"/>
<appender-ref ref="ERROR_FILE"/>
</logger>
<root level="INFO">
<appender-ref ref="STDOUT" />
<appender-ref ref="INFO_FILE" />
<appender-ref ref="ERROR_FILE" />
</root>
</configuration>
\ No newline at end of file
src/main/resources/mongo.properties 0 → 100644
mongo.LocalIP = 192.168.0.241
#mongo.LocalDBName = qbjcPhoenix
#mongo.Localport = 27017
#mongo.LocalIP = 115.236.59.91
#mongo.LocalIP = 192.168.0.101
mongo.LocalDBName = qbjcPhoenix
mongo.Localport = 27017
#mongo.ServerIP = 192.168.0.101
mongo.ServerIP = 115.236.59.91
mongo.ServerDBName = qbjcPhoenix
mongo.Serverport = 27017
mongo.ServerUsername = stno
mongo.ServerPassword = stno1q2w3e4r
mongo.authenticationDatabase=admin
mongo.EventDBName = eventMuseum
mongo.WechatDBName = WechatPublic
mongo.TestDBName = Testqbjc
mongo.weibotagDBName = weibotag
mongo.mediatagDBName = mediatag
#mongo.TagIP = 1.119.44.206
mongo.TagIP = 192.168.0.245
#mongo.Tagport = 30000
mongo.Tagport = 27017
mongo.connectionsPerHost=200
mongo.threadsAllowedToBlockForConnectionMultiplier=10
mongo.connectTimeout=30000
mongo.maxWaitTime=50000
mongo.autoConnectRetry=true
mongo.socketKeepAlive=true
mongo.socketTimeout=120000
mongo.slaveOk=true
\ No newline at end of file
src/main/resources/redis.properties 0 → 100644
redis.maxTotal=2048
redis.maxIdle=200
redis.maxWaitMillis=1000
redis.testOnBorrow=true
redis.testOnReturn=true
#redis.ip=127.0.0.1
#redis.ip = 192.168.0.202
#redis.ip = 202.107.192.94
#redis.port=6379
redis.ip=192.168.1.74
redis.port=6388
redis.keyMaxSize=5000
redis.selectDB=2
#redis.selectDB=2
redis.user_keyMaxSize=1000
redis.cacheSize=1000
redis.intitCount=3000
\ No newline at end of file
src/test/java/com/zhiwei/messageflow/MessageflowApplicationTests.java 0 → 100644
package com.zhiwei.messageflow;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
@RunWith(SpringRunner.class)
@SpringBootTest
public class MessageflowApplicationTests {
@Test
public void contextLoads() {
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment