Skip to content

Q
qbjc-utils
Commit 427d91bb by liuminghuan
Options

2020/08/21 初版提交

parents
Showing with 4676 additions and 0 deletions
.gitignore 0 → 100644
HELP.md
/target/
!.mvn/wrapper/maven-wrapper.jar
.mvn/
mvnw
mvnw.cmd
# maven ignore
target/
*.jar
*.war
*.zip
*.tar
*.tar.gz
dependency-reduced-pom.xml
# eclipse ignore
.settings/
.project
.classpath
# idea ignore
.idea/
*.ipr
*.iml
*.iws
# git ignore
*/.gitignore
# temp ignore
*.log
*.cache
*.diff
*.patch
*.tmp
# test ignore
test-output/
# system ignore
.DS_Store
Thumbs.db
# directory ignore
log/
backup/
*/src/test/
# jekins ignore
Jenkinsfile
pom.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.zhiwei</groupId>
<artifactId>qbjc-utils</artifactId>
<version>1.0.0-SNAPSHOT</version>
<name>qbjc-utils</name>
<description>utils for qbjc system</description>
<properties>
<java.version>1.8</java.version>
<log4j.version>2.11.2</log4j.version>
<slf4j.version>1.8.0-beta4</slf4j.version>
<disruptor.version>3.4.2</disruptor.version>
<base-objects.version>1.2.8-SNAPSHOT</base-objects.version>
<es.version>5.5.2</es.version>
<es-client.version>1.9</es-client.version>
<springboot.version>2.1.6.RELEASE</springboot.version>
<lombok.version>1.18.12</lombok.version>
<redisson.version>3.10.7</redisson.version>
<json.version>1.2.47</json.version>
<commons-lang3.version>3.9</commons-lang3.version>
<commons-collections4.version>4.3</commons-collections4.version>
<easy-excel.version>2.1.0-beta4</easy-excel.version>
<poi.version>4.0.1</poi.version>
<dubbo.version>2.7.4.1</dubbo.version>
<marker-client.version>1.1.0-SNAPSHOT</marker-client.version>
<cleaner-unified-filter.version>1.1.4.RELEASE</cleaner-unified-filter.version>
<automaticmark.version>2.1.0-SNAPSHOT</automaticmark.version>
<channel-index.version>0.4.2-SNAPSHOT</channel-index.version>
<crawler.version>0.6.0.1-RELEASE</crawler.version>
</properties>
<dependencies>
<dependency>
<groupId>com.zhiwei</groupId>
<artifactId>push-log</artifactId>
<version>2.11.2</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<version>${springboot.version}</version>
<exclusions><!-- 去掉默认配置 -->
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-logging</artifactId>
</exclusion>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
</exclusion>
</exclusions>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-mongodb</artifactId>
<version>${springboot.version}</version>
<exclusions>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-logging</artifactId>
</exclusion>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-aop</artifactId>
<version>${springboot.version}</version>
<exclusions>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-logging</artifactId>
</exclusion>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- https://mvnrepository.com/artifact/org.redisson/redisson-spring-boot-starter -->
<dependency>
<groupId>org.redisson</groupId>
<artifactId>redisson-spring-boot-starter</artifactId>
<version>${redisson.version}</version>
<exclusions>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
</exclusion>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- base-object -->
<dependency>
<groupId>com.zhiwei.base</groupId>
<artifactId>base-objects</artifactId>
<version>${base-objects.version}</version>
</dependency>
<dependency>
<groupId>com.lmax</groupId>
<artifactId>disruptor</artifactId>
<version>${disruptor.version}</version>
</dependency>
<!-- elasticsearch.client -->
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>transport</artifactId>
<version>${es.version}</version>
</dependency>
<dependency>
<groupId>com.zhiwei</groupId>
<artifactId>es-client</artifactId>
<version>${es-client.version}</version>
</dependency>
<!--lombok-->
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>${lombok.version}</version>
</dependency>
<!-- fastjson -->
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>${json.version}</version>
</dependency>
<!-- apache 常用工具类 -->
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>${commons-lang3.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-collections4 -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-collections4</artifactId>
<version>${commons-collections4.version}</version>
</dependency>
<!-- Excel -->
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>easyexcel</artifactId>
<version>${easy-excel.version}</version>
</dependency>
<!-- poi -->
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi</artifactId>
<version>${poi.version}</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml</artifactId>
<version>${poi.version}</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-scratchpad</artifactId>
<version>${poi.version}</version>
</dependency>
<!--dubbo 导入 -->
<dependency>
<groupId>org.apache.dubbo</groupId>
<artifactId>dubbo</artifactId>
<version>${dubbo.version}</version>
<scope>provided</scope>
<exclusions>
<exclusion>
<groupId>org.springframework</groupId>
<artifactId>spring-context</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- 去重 -->
<dependency>
<groupId>com.zhiwei.middleware</groupId>
<artifactId>cleaner-unified-filter</artifactId>
<version>${cleaner-unified-filter.version}</version>
</dependency>
<!-- 自动 -->
<dependency>
<groupId>com.zhiwei.middleware</groupId>
<artifactId>automaticmark-client</artifactId>
<version>${automaticmark.version}</version>
</dependency>
<!--channel-index-->
<dependency>
<groupId>com.zhiwei</groupId>
<artifactId>channel-index-client</artifactId>
<version>${channel-index.version}</version>
</dependency>
<dependency>
<groupId>com.zhiwei.crawler</groupId>
<artifactId>crawler-core</artifactId>
<version>${crawler.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.testng/testng -->
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
<version>7.1.0</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<version>${springboot.version}</version>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
\ No newline at end of file
src/main/java/com/zhiwei/qbjcutils/QbjcUtilsApplication.java 0 → 100644
package com.zhiwei.qbjcutils;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration;
import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration;
/**
* @author lmh
*/
@SpringBootApplication(exclude = { MongoAutoConfiguration.class, MongoDataAutoConfiguration.class })
public class QbjcUtilsApplication {
public static void main(String[] args) {
SpringApplication.run(QbjcUtilsApplication.class, args);
}
}
src/main/java/com/zhiwei/qbjcutils/config/EsProperties.java 0 → 100644
package com.zhiwei.qbjcutils.config;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
/**
* @Description:
* @Author: shentao
* @Date: 2020/4/27 13:47
*/
@Component
@Data
@ConfigurationProperties(prefix = "es")
public class EsProperties {
/**
* httpHost httpClient端口
*/
private int httpHost;
/**
* httpIp httpClientIp
*/
private String httpIp;
/**
* clusterName 集群名
*/
private String clusterName;
/**
* 自动嗅探
*/
private boolean transportSniff;
/**
* 集群节点s
*/
private String clusterNodes;
}
src/main/java/com/zhiwei/qbjcutils/config/GenericAttribute.java 0 → 100644
package com.zhiwei.qbjcutils.config;
/**
* Description:
*
* @author LiuMingHuan
* @classname GenericAttribute
* @date 2020/4/2
*/
public class GenericAttribute {
/**文件地址**/
public static final String FILE_PATH = "D:\\stWork\\download\\excel\\";
/**时间常量**/
public static final long AN_HOUR = 60 * 60 * 1000L;
/**es字段**/
public static final String ES_ORIGIN = "origin";
public static final String ES_FOREIGN = "foreign";
public static final String ES_C5 = "c5";
public static final String ES_IND_FULL_TEXT = "ind_full_text";
public static final String ES_IND_TITLE = "ind_title";
public static final String ES_SOURCE = "source";
public static final String ES_MGROUP = "mgroup";
public static final String ES_MTIME = "mtime";
public static final String ES_MTAG = "mtag";
public static final String ES_MUPDATE = "mupdate";
public static final String ES_MUPDATE_TWO = "mupdate_two";
public static final int SEARCH_TOTAL_LIMIT =10000 ;
/** redisKey **/
public static final String SYSTEM_KEY="YUQING:System:";
/** URL **/
public static final String WARN_URL = "https://login.zhiweidata.com/wechatwork/sendMessage";
}
src/main/java/com/zhiwei/qbjcutils/config/MiddlewareConfig.java 0 → 100644
package com.zhiwei.qbjcutils.config;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
/**
* @ClassName: MiddlewareConfig
* @Description: 中间件配置
* @author shentao
* @date 2019年7月16日 下午7:08:07
*/
@Component
public class MiddlewareConfig {
public static String REGISTRY;
public static String CHANNEL_INDEX_REGISTRY;
public static String CLEAN_FILTER_GROUP;
public static String AUTOMARK_GROUP;
public static String CHANNEL_GROUP;
@Value("${zookeeper.registry}")
public void setRegistry(String registry) {
REGISTRY = registry;
System.out.println("REGISTRY:" + REGISTRY);
}
@Value("${channel-index.zookeeper.registry}")
public void setChannelIndexRegistry(String registry) {
CHANNEL_INDEX_REGISTRY = registry;
System.out.println("CHANNEL_INDEX_REGISTRY:" + CHANNEL_INDEX_REGISTRY);
}
@Value("${dubbo.filter.group}")
public void setCleanFilterGroup(String group) {
CLEAN_FILTER_GROUP = group;
System.out.println("CLEAN_FILTER_GROUP:" + CLEAN_FILTER_GROUP);
}
@Value("${dubbo.automark.group}")
public void setAutoMarkGroup(String group) {
AUTOMARK_GROUP = group;
System.out.println("AUTOMARK_GROUP:" + AUTOMARK_GROUP);
}
@Value("${dubbo.channel.group}")
public void setChannelGroup(String group) {
CHANNEL_GROUP = group;
System.out.println("CHANNEL_GROUP:" + CHANNEL_GROUP);
}
}
src/main/java/com/zhiwei/qbjcutils/config/MongoConfig.java 0 → 100644
package com.zhiwei.qbjcutils.config;
import com.mongodb.MongoClientOptions;
import com.mongodb.MongoClientURI;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import org.springframework.data.mongodb.MongoDbFactory;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.SimpleMongoDbFactory;
import org.springframework.data.mongodb.core.convert.DbRefResolver;
import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver;
import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper;
import org.springframework.data.mongodb.core.convert.MappingMongoConverter;
import org.springframework.data.mongodb.core.mapping.MongoMappingContext;
/**
* MongoConfig
*
* @author shentao
* @ClassName: MongoConfig
* @Description: MongoConfig
* @date 2019年7月8日 上午10:32:02
*/
@Configuration
public class MongoConfig {
private final static Logger log = LogManager.getLogger(MongoConfig.class);
@Value("${mongo.connectionsPerHost}")
private int connectionsPerHost;
@Value("${mongo.threadsAllowedToBlockForConnectionMultiplier}")
private int threadsAllowedToBlockForConnectionMultiplier;
@Value("${mongo.connectTimeout}")
private int connectTimeout;
@Value("${mongo.maxWaitTime}")
private int maxWaitTime;
@Value("${mongo.autoConnectRetry}")
private boolean autoConnectRetry;
@Value("${mongo.socketKeepAlive}")
private boolean socketKeepAlive;
@Value("${mongo.socketTimeout}")
private int socketTimeout;
@Value("${mongo.slaveOk}")
private boolean slaveOk;
@Value("${primary.uri}")
private String uri;
private MongoDbFactory mongoDbFactory() {
MongoClientOptions.Builder builder = new MongoClientOptions.Builder();
builder.connectionsPerHost(connectionsPerHost);
builder.connectTimeout(connectTimeout);
builder.threadsAllowedToBlockForConnectionMultiplier(threadsAllowedToBlockForConnectionMultiplier);
builder.maxWaitTime(maxWaitTime);
return new SimpleMongoDbFactory(new MongoClientURI(uri, builder));
}
@Primary
@Bean(name = "primaryMongoTemplate")
public MongoTemplate getMongoTemplate() {
log.info("@Primary");
MongoDbFactory mongoDbFactory = mongoDbFactory();
DbRefResolver dbRefResolver = new DefaultDbRefResolver(mongoDbFactory);
MappingMongoConverter converter = new MappingMongoConverter(dbRefResolver, new MongoMappingContext());
// 不插入_class
converter.setTypeMapper(new DefaultMongoTypeMapper(null));
return new MongoTemplate(mongoDbFactory(), converter);
}
private MongoDbFactory mongoDbFactory2() {
MongoClientOptions.Builder builder = new MongoClientOptions.Builder();
builder.connectionsPerHost(connectionsPerHost);
builder.connectTimeout(connectTimeout);
builder.threadsAllowedToBlockForConnectionMultiplier(threadsAllowedToBlockForConnectionMultiplier);
builder.maxWaitTime(maxWaitTime);
return new SimpleMongoDbFactory(new MongoClientURI(secondUri, builder));
}
@Value("${secondary.uri}")
private String secondUri;
@Bean(name = "secondaryMongoTemplate")
public MongoTemplate getSecondaryMongoTemplate() {
log.info("@Secondary");
MongoDbFactory mongoDbFactory = mongoDbFactory2();
DbRefResolver dbRefResolver = new DefaultDbRefResolver(mongoDbFactory);
MappingMongoConverter converter = new MappingMongoConverter(dbRefResolver, new MongoMappingContext());
// 不插入_class
converter.setTypeMapper(new DefaultMongoTypeMapper(null));
return new MongoTemplate(mongoDbFactory2(), converter);
}
}
src/main/java/com/zhiwei/qbjcutils/config/TaskConfig.java 0 → 100644
package com.zhiwei.qbjcutils.config;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import java.util.concurrent.Executor;
import java.util.concurrent.ThreadPoolExecutor;
/**
* Description:
*
* @author LiuMingHuan
* @classname TaskConfig
* @date 2020/3/10
*/
@Configuration
@EnableAsync
public class TaskConfig {
@Bean
public Executor scheduleExecutor(){
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
executor.setCorePoolSize(2);
executor.setMaxPoolSize(2);
executor.setThreadNamePrefix("schedule-");
// rejection-policy：当pool已经达到max size的时候，如何处理新任务
// CALLER_RUNS：不在新线程中执行任务，而是有调用者所在的线程来执行
executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
executor.initialize();
return executor;
}
}
src/main/java/com/zhiwei/qbjcutils/dao/DirectGroupDao.java 0 → 100644
package com.zhiwei.qbjcutils.dao;
import com.zhiwei.qbjcutils.pojo.entity.DirectGroup;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.data.mongodb.core.query.Update;
/**
* Description:
*
* @author LiuMingHuan
* @classname DirectGroupDao
* @date 2020/4/16
*/
public interface DirectGroupDao {
/**
* updateFirst
* @param query
* @param update
*/
void updateFirst(Query query, Update update);
/**
* insert
* @param group
*/
void insert(DirectGroup group);
/**
*
* @param query
* @return
*/
DirectGroup findFirst(Query query);
}
src/main/java/com/zhiwei/qbjcutils/dao/EventDao.java 0 → 100644
package com.zhiwei.qbjcutils.dao;
import com.zhiwei.qbjcutils.pojo.entity.Event;
import org.springframework.data.mongodb.core.aggregation.Aggregation;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.data.mongodb.core.query.Update;
import java.util.List;
/**
* Description:
*
* @author LiuMingHuan
* @classname EventDao
* @date 2020/4/8
*/
public interface EventDao {
List<Event> aggregate(Aggregation aggregation);
List<Event> find(Query query);
void update(Query query, Update update);
}
src/main/java/com/zhiwei/qbjcutils/dao/PlatformDao.java 0 → 100644
package com.zhiwei.qbjcutils.dao;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.data.mongodb.core.query.Update;
/**
* Description:
*
* @author LiuMingHuan
* @classname PlatformDao
* @date 2020/4/3
*/
public interface PlatformDao {
/**
* 更新平台
* @param query
* @param update
*/
void update(Query query, Update update);
}
src/main/java/com/zhiwei/qbjcutils/dao/TestController.java 0 → 100644
package com.zhiwei.qbjcutils.dao;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.slf4j.log4j12.Log4jLoggerAdapter;
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RestController;
/**
* Description:
*
* @author LiuMingHuan
* @classname TestController
* @date 2020/6/4
*/
@RestController
public class TestController {
private Logger log = LogManager.getLogger(TestController.class);
@RequestMapping(method = RequestMethod.GET, value = "/get")
public void test(){
AnnotationConfigApplicationContext ann = new AnnotationConfigApplicationContext(Log4jLoggerAdapter.class);
ann.getBeanFactory().registerSingleton("log", log);
log.error("测试出错{}", "user");
log.error("测试出错{},{}", "user1", "user2");
}
@RequestMapping(method = RequestMethod.GET, value = "/get1")
public void test1(){
try {
int i = 10/0;
}catch (Exception e){
System.out.println(e.getCause());
}
}
}
src/main/java/com/zhiwei/qbjcutils/dao/impl/DirectGroupDaoImpl.java 0 → 100644
package com.zhiwei.qbjcutils.dao.impl;
import com.zhiwei.qbjcutils.dao.DirectGroupDao;
import com.zhiwei.qbjcutils.pojo.entity.DirectGroup;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.data.mongodb.core.query.Update;
import org.springframework.stereotype.Component;
/**
* Description:
*
* @author LiuMingHuan
* @classname DirectGroupDaoImpl
* @date 2020/4/16
*/
@Component
public class DirectGroupDaoImpl implements DirectGroupDao {
private final MongoTemplate mongoTemplate;
public DirectGroupDaoImpl(@Qualifier("primaryMongoTemplate") MongoTemplate mongoTemplate) {
this.mongoTemplate = mongoTemplate;
}
@Override
public void updateFirst(Query query, Update update) {
mongoTemplate.updateFirst(query, update, DirectGroup.class);
}
@Override
public void insert(DirectGroup group) {
mongoTemplate.insert(group);
}
@Override
public DirectGroup findFirst(Query query) {
return mongoTemplate.findOne(query, DirectGroup.class);
}
}
src/main/java/com/zhiwei/qbjcutils/dao/impl/EventDaoImpl.java 0 → 100644
package com.zhiwei.qbjcutils.dao.impl;
import com.zhiwei.qbjcutils.dao.EventDao;
import com.zhiwei.qbjcutils.pojo.entity.Event;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.aggregation.Aggregation;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.data.mongodb.core.query.Update;
import org.springframework.stereotype.Component;
import java.util.List;
/**
* Description:
*
* @author LiuMingHuan
* @classname EventDaoImpl
* @date 2020/4/8
*/
@Component
public class EventDaoImpl implements EventDao {
private final MongoTemplate mongoTemplate;
public EventDaoImpl(MongoTemplate mongoTemplate) {
this.mongoTemplate = mongoTemplate;
}
@Override
public List<Event> aggregate(Aggregation aggregation) {
return mongoTemplate.aggregate(aggregation, Event.class, Event.class).getMappedResults();
}
@Override
public List<Event> find(Query query) {
return mongoTemplate.find(query, Event.class);
}
@Override
public void update(Query query, Update update) {
mongoTemplate.updateFirst(query, update, Event.class);
}
}
src/main/java/com/zhiwei/qbjcutils/dao/impl/PlatformDaoImpl.java 0 → 100644
package com.zhiwei.qbjcutils.dao.impl;
import com.zhiwei.qbjcutils.dao.PlatformDao;
import com.zhiwei.qbjcutils.pojo.entity.MessagePlatform;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.data.mongodb.core.query.Update;
import org.springframework.stereotype.Component;
/**
* Description:
*
* @author LiuMingHuan
* @classname PlatformDaoImpl
* @date 2020/4/3
*/
@Component
public class PlatformDaoImpl implements PlatformDao {
private final MongoTemplate mongoTemplate;
public PlatformDaoImpl(@Qualifier("primaryMongoTemplate") MongoTemplate mongoTemplate) {
this.mongoTemplate = mongoTemplate;
}
@Override
public void update(Query query, Update update) {
mongoTemplate.updateFirst(query, update, MessagePlatform.class);
}
}
src/main/java/com/zhiwei/qbjcutils/dubbo/DubboClient.java 0 → 100644
package com.zhiwei.qbjcutils.dubbo;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.base.filter.FilterInfo;
import com.zhiwei.channel.index.middleware.core.client.ChannelDataClient;
import com.zhiwei.channel.index.middleware.core.client.ChannelIndexClientFactory;
import com.zhiwei.channel.index.middleware.core.client.ChannelStatisticsClient;
import com.zhiwei.middleware.automaticmark.dubbo.client.AutomaticClientFactory;
import com.zhiwei.middleware.automaticmark.dubbo.client.AutomaticMarkClient;
import com.zhiwei.middleware.cleaner.filter.UnifiedFilterClient;
import com.zhiwei.qbjcutils.config.MiddlewareConfig;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* @author shentao
* @ClassName: DubboClient
* @Description: 中间件接口类
* @date 2019年7月16日 下午6:44:56
*/
@Component
public class DubboClient {
private static final Logger log = LogManager.getLogger(DubboClient.class);
private static final String APP_NAME = "qbjc-utils";
public final MiddlewareConfig config;
private static AutomaticMarkClient AUTO_MATICMARK_CLIENT;
private static UnifiedFilterClient FILTER_CLIENT;
private static ChannelDataClient CHANNEL_DATA_CLIENT;
private static ChannelStatisticsClient CHANNEL_STATICS_CLIENT;
public DubboClient(MiddlewareConfig config) {
this.config = config;
}
@PostConstruct
public void init() {
try {
AUTO_MATICMARK_CLIENT = AutomaticClientFactory.automaticMarkClient(MiddlewareConfig.REGISTRY, MiddlewareConfig.AUTOMARK_GROUP, APP_NAME);
FILTER_CLIENT = UnifiedFilterClient.getClient(APP_NAME, MiddlewareConfig.REGISTRY, MiddlewareConfig.CLEAN_FILTER_GROUP);
CHANNEL_DATA_CLIENT = ChannelIndexClientFactory.channelDataClient(MiddlewareConfig.CHANNEL_INDEX_REGISTRY, MiddlewareConfig.CHANNEL_GROUP, APP_NAME);
CHANNEL_STATICS_CLIENT = ChannelIndexClientFactory.channelStatisticsClient(MiddlewareConfig.CHANNEL_INDEX_REGISTRY, MiddlewareConfig.CHANNEL_GROUP, APP_NAME);
} catch (Exception e) {
log.error("初始化中间件Client失败", e);
}
}
/**
* contains 去重 true重复（库里存在）/false不重复（不存在）
*
* @param filterInfo
* @return boolean
*/
public boolean contains(FilterInfo filterInfo) {
return FILTER_CLIENT.contains(filterInfo, false);
}
/**
* 返回去重条件
*
* @param filterInfo
* @return String []
*/
public String[] getUpdateInfo(FilterInfo filterInfo) {
return FILTER_CLIENT.getUpdateInfo(filterInfo);
}
/**
* 渠道索引获取对应账号
* @param mediaId
* @param keyword
* @param isCollected
* @param page
* @param pageSize
* @return
*/
public JSONObject searchCollectedAccountList(int mediaId, String keyword, Boolean isCollected, int page, int pageSize){
return CHANNEL_DATA_CLIENT.pageSearchAccount(mediaId, keyword, isCollected, page, pageSize);
}
/**
* searchCollectedAccountList
* @param queryTags
* @param mediaId
* @param keyword
* @param isCollected
* @param page
* @param pageSize
* @return
*/
public JSONObject searchCollectedAccountList(List<Long> queryTags, int mediaId, String keyword, Boolean isCollected, int page, int pageSize){
return CHANNEL_DATA_CLIENT.pageSearchAccount(queryTags, keyword, isCollected, mediaId, page, pageSize, null, null);
}
/**
* getPlatformsPermutation
* @return
*/
public Set<Map<String,Object>> getPlatformsPermutation(){
return CHANNEL_STATICS_CLIENT.channelArticleDataTypes();
}
}
src/main/java/com/zhiwei/qbjcutils/es/EsClient.java 0 → 100644
package com.zhiwei.qbjcutils.es;
import com.zhiwei.es.util.ESIndexesUtil;
import com.zhiwei.qbjcutils.config.EsProperties;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.transport.client.PreBuiltTransportClient;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.util.Assert;
import org.springframework.util.StringUtils;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.Arrays;
/**
* es连接客户端
*
* @ClassName: EsClient
* @Description: es连接客户端
* @author shentao
* @date 2019年7月8日 下午3:08:04
*/
@Configuration
public class EsClient {
public static final Logger log = LogManager.getLogger(EsClient.class);
private static final String COLON = ":";
private static final String COMMA = ",";
private final EsProperties esProperties;
public EsClient(EsProperties esProperties) {
this.esProperties = esProperties;
}
@Bean(name = "esIndexes")
public ESIndexesUtil.ESIndexes esIndexes(){
return ESIndexesUtil.create(esProperties.getHttpIp(), esProperties.getHttpHost());
}
@Bean(name = "esTransportClient")
public TransportClient esTransportClient() {
System.setProperty("es.set.netty.runtime.available.processors", "false");
Settings esSettings = Settings.builder()
// 设置ES实例的名称
.put("cluster.name", esProperties.getClusterName())
// 自动嗅探整个集群的状态，把集群中其他ES节点的ip添加到本地的客户端列表中
.put("client.transport.sniff", esProperties.isTransportSniff())
// .put("client.transport.ping_timeout", "15s")
.build();
TransportClient client = new PreBuiltTransportClient(esSettings);
// 此步骤添加IP，至少一个，其实一个就够了，因为添加了自动嗅探配置
try {
Assert.hasText(esProperties.getClusterNodes(), "Cluster nodes source must not be null or empty!");
String[] nodes = StringUtils.delimitedListToStringArray(esProperties.getClusterNodes(), COMMA);
Arrays.stream(nodes).forEach(node -> {
String[] segments = StringUtils.delimitedListToStringArray(node, COLON);
Assert.isTrue(segments.length == 2,
() -> String.format("Invalid cluster node %s in %s! Must be in the format host:port!", node,
esProperties.getClusterNodes()));
String host = segments[0].trim();
String port = segments[1].trim();
Assert.hasText(host, () -> String.format("No host name given cluster node %s!", node));
Assert.hasText(port, () -> String.format("No port given in cluster node %s!", node));
client.addTransportAddress(new InetSocketTransportAddress(toInetAddress(host), Integer.parseInt(port)));
});
} catch (Exception e) {
log.error("es client初始化异常",e);
}
return client;
}
private static InetAddress toInetAddress(String host) {
try {
return InetAddress.getByName(host);
} catch (UnknownHostException e) {
throw new IllegalArgumentException(e);
}
}
}
src/main/java/com/zhiwei/qbjcutils/es/EsIndexTools.java 0 → 100644
package com.zhiwei.qbjcutils.es;
import com.zhiwei.es.index.Index;
import com.zhiwei.es.util.ESIndexesUtil;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
/**
* es索引辅助类
*
* @author shentao
* @ClassName: EsIndexTools
* @Description: es索引辅助类
* @date 2019年7月8日 下午6:05:22
*/
@Service
public class EsIndexTools {
@Resource(name ="esIndexes")
private ESIndexesUtil.ESIndexes indexes;
/**
* 获取标注indexs-近三年
*
* @return String []
*/
public String[] getMarkIndexs() {
List<String> list = indexes.getMarkIndexes();
// List<String> list = new ArrayList<>();
// list.add("mark_test");
return list.toArray(new String[list.size()]);
}
/**
* 获取四大库indexs
*
* @param start
* @param end
* @return String []
*/
public String[] getAllIndex(Long start, Long end) {
List<String> list = new ArrayList<>();
list.addAll(indexes.getIndexes(Index.complete_text.name(), new Date(start), new Date(end)));
list.addAll(indexes.getIndexes(Index.incomplete_text.name(), new Date(start), new Date(end)));
list.addAll(indexes.getIndexes(Index.qa_text.name(), new Date(start), new Date(end)));
list.addAll(indexes.getIndexes(Index.video.name(), new Date(start), new Date(end)));
list = new ArrayList<>(new HashSet<>(list));
return list.toArray(new String[list.size()]);
}
}
src/main/java/com/zhiwei/qbjcutils/es/dao/EsDao.java 0 → 100644
package com.zhiwei.qbjcutils.es.dao;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.sort.FieldSortBuilder;
/**
* EsDao-interface
*
* @ClassName: EsDao
* @Description: EsDao-interface
* @author shentao
* @date 2019年7月8日 下午6:05:54
*/
public interface EsDao {
/**
* search-es检索
*
* @param indexs
* @param types
* @param postFilter
* @param query
* @param sort
* @param from
* @param size
* @param highlightBuilder
* @return SearchHits
*/
SearchHits search(String[] indexs, String[] types, QueryBuilder postFilter, QueryBuilder query,
FieldSortBuilder sort, int from, int size, HighlightBuilder highlightBuilder);
/**
* search-es检索限制返回值
*
* @param indexs
* @param types
* @param postFilter
* @param query
* @param sort
* @param from
* @param size
* @param includes
* @param excludes
* @param highlightBuilder
* @return SearchHits
*/
SearchHits searchFatchSource(String[] indexs, String[] types, QueryBuilder postFilter, QueryBuilder query,
FieldSortBuilder sort, int from, int size, String[] includes, String[] excludes,
HighlightBuilder highlightBuilder);
/**
* es统计量
*
* @param indexes
* @param types
* @param postFilter
* @param query
* @return
*/
Long count(String[] indexes, String[] types, QueryBuilder postFilter, QueryBuilder query);
/**
* scroll query
*
* @param indexs
* @param types
* @param postFilter
* @param query
* @param sort
* @param from
* @param size
* @param includes
* @param excludes
* @param highlightBuilder
* @param timeValue
* @return
*/
SearchResponse scroll(String[] indexs, String[] types, QueryBuilder postFilter, QueryBuilder query,
FieldSortBuilder sort, int from, int size, String[] includes, String[] excludes, HighlightBuilder highlightBuilder, TimeValue timeValue);
/**
* scroll query
*
* @param indexs
* @param types
* @param postFilter
* @param query
* @param sort
* @param from
* @param size
* @param highlightBuilder
* @param timeValue
* @return
*/
SearchResponse scroll(String[] indexs, String[] types, QueryBuilder postFilter, QueryBuilder query,
FieldSortBuilder sort, int from, int size, HighlightBuilder highlightBuilder, TimeValue timeValue);
/**
* es scroll query
*
* @param scrollId
* @param timeValue
*/
SearchResponse scroll(String scrollId, TimeValue timeValue);
}
src/main/java/com/zhiwei/qbjcutils/es/dao/impl/EsDaoImpl.java 0 → 100644
package com.zhiwei.qbjcutils.es.dao.impl;
import com.zhiwei.qbjcutils.es.dao.EsDao;
import com.zhiwei.qbjcutils.utils.Tools;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchScrollRequestBuilder;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.sort.FieldSortBuilder;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
/**
* EsDaoImpl
*
* @ClassName: EsDaoImpl
* @Description: EsDaoImpl
* @author shentao
* @date 2019年7月8日 上午11:31:41
*/
@Component
public final class EsDaoImpl implements EsDao {
@Resource(name ="esTransportClient")
private TransportClient client;
@Override
public SearchHits search(String[] indexs, String[] types, QueryBuilder postFilter, QueryBuilder query,
FieldSortBuilder sort, int from, int size, HighlightBuilder highlighter) {
SearchRequestBuilder searchRequestBuilder = client.prepareSearch(indexs);
if (!Tools.isNull(types)) {
searchRequestBuilder.setTypes(types);
}
if (!Tools.isNull(postFilter)) {
searchRequestBuilder.setPostFilter(postFilter);
}
if (!Tools.isNull(query)) {
searchRequestBuilder.setQuery(query);
}
if (!Tools.isNull(sort)) {
searchRequestBuilder.addSort(sort);
}
if (!Tools.isNull(from)) {
searchRequestBuilder.setFrom(from);
}
if (!Tools.isNull(size)) {
searchRequestBuilder.setSize(size);
}
if (!Tools.isNull(highlighter)) {
searchRequestBuilder.highlighter(highlighter);
}
SearchResponse response = searchRequestBuilder.execute().actionGet();
return response.getHits();
}
@Override
public SearchHits searchFatchSource(String[] indexs, String[] types, QueryBuilder postFilter, QueryBuilder query,
FieldSortBuilder sort, int from, int size, String[] includes, String[] excludes,
HighlightBuilder highlightBuilder) {
SearchRequestBuilder searchRequestBuilder = client.prepareSearch(indexs);
if (!Tools.isNull(types)) {
searchRequestBuilder.setTypes(types);
}
if (!Tools.isNull(postFilter)) {
searchRequestBuilder.setPostFilter(postFilter);
}
if (!Tools.isNull(query)) {
searchRequestBuilder.setQuery(query);
}
if (!Tools.isNull(sort)) {
searchRequestBuilder.addSort(sort);
}
if (!Tools.isNull(from)) {
searchRequestBuilder.setFrom(from);
}
if (!Tools.isNull(size)) {
searchRequestBuilder.setSize(size);
}
if (!Tools.isNull(highlightBuilder)) {
searchRequestBuilder.highlighter(highlightBuilder);
}
searchRequestBuilder.setFetchSource(includes, excludes);
SearchResponse response = searchRequestBuilder.execute().actionGet();
return response.getHits();
}
@Override
public Long count(String[] indexes, String[] types, QueryBuilder postFilter, QueryBuilder query) {
SearchRequestBuilder searchRequestBuilder = client.prepareSearch(indexes);
if (!Tools.isNull(types)) {
searchRequestBuilder.setTypes(types);
}
if (!Tools.isNull(postFilter)) {
searchRequestBuilder.setPostFilter(postFilter);
}
if (!Tools.isNull(query)) {
searchRequestBuilder.setQuery(query);
}
SearchResponse response = searchRequestBuilder.setFrom(0).setSize(0).execute().actionGet();
return response.getHits().getTotalHits();
}
@Override
public SearchResponse scroll(String[] indexs, String[] types, QueryBuilder postFilter, QueryBuilder query, FieldSortBuilder sort, int from, int size, String[] includes, String[] excludes, HighlightBuilder highlightBuilder, TimeValue timeValue) {
SearchRequestBuilder searchRequestBuilder = client.prepareSearch(indexs);
if (!Tools.isNull(types)) {
searchRequestBuilder.setTypes(types);
}
if (!Tools.isNull(postFilter)) {
searchRequestBuilder.setPostFilter(postFilter);
}
if (!Tools.isNull(query)) {
searchRequestBuilder.setQuery(query);
}
if (!Tools.isNull(sort)) {
searchRequestBuilder.addSort(sort);
}
if (!Tools.isNull(from)) {
searchRequestBuilder.setFrom(from);
}
if (!Tools.isNull(size)) {
searchRequestBuilder.setSize(size);
}
if(!Tools.isNull(includes) || !Tools.isNull(excludes)){
searchRequestBuilder.setFetchSource(includes, excludes);
}
if (!Tools.isNull(highlightBuilder)) {
searchRequestBuilder.highlighter(highlightBuilder);
}
if(!Tools.isNull(timeValue)){
searchRequestBuilder.setScroll(timeValue);
}
return searchRequestBuilder.execute().actionGet();
}
@Override
public SearchResponse scroll(String[] indexs, String[] types, QueryBuilder postFilter, QueryBuilder query, FieldSortBuilder sort, int from, int size, HighlightBuilder highlightBuilder, TimeValue timeValue) {
SearchRequestBuilder searchRequestBuilder = client.prepareSearch(indexs);
if (!Tools.isNull(types)) {
searchRequestBuilder.setTypes(types);
}
if (!Tools.isNull(postFilter)) {
searchRequestBuilder.setPostFilter(postFilter);
}
if (!Tools.isNull(query)) {
searchRequestBuilder.setQuery(query);
}
if (!Tools.isNull(sort)) {
searchRequestBuilder.addSort(sort);
}
if (!Tools.isNull(from)) {
searchRequestBuilder.setFrom(from);
}
if (!Tools.isNull(size)) {
searchRequestBuilder.setSize(size);
}
if (!Tools.isNull(highlightBuilder)) {
searchRequestBuilder.highlighter(highlightBuilder);
}
if(!Tools.isNull(timeValue)){
searchRequestBuilder.setScroll(timeValue);
}
return searchRequestBuilder.execute().actionGet();
}
@Override
public SearchResponse scroll(String scrollId, TimeValue timeValue) {
SearchScrollRequestBuilder searchScrollRequestBuilder = client.prepareSearchScroll(scrollId);
if(!Tools.isNull(timeValue)) {
searchScrollRequestBuilder.setScroll(timeValue);
}
return searchScrollRequestBuilder.execute().actionGet();
}
}
src/main/java/com/zhiwei/qbjcutils/pojo/dto/BaseObjectCacheDTO.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.dto;
import com.zhiwei.qbjcutils.pojo.entity.BaseObjectCache;
import lombok.*;
/**
* Description:
*
* @author LiuMingHuan
* @classname BaseObjectCacheDTO
* @date 2020/4/9
*/
@Getter
@Setter
@NoArgsConstructor
@AllArgsConstructor
public class BaseObjectCacheDTO extends BaseObjectCache {
private String mtag;
@Override
public String toString() {
return "BaseObjectCacheDTO{" +
"mtag='" + mtag + '\'' +
"} " + super.toString();
}
}
src/main/java/com/zhiwei/qbjcutils/pojo/emums/TimeUnit.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.emums;
import java.util.Calendar;
/**
* Description:
*
* @author LiuMingHuan
* @classname TimeUnit
* @date 2020/4/17
*/
public enum TimeUnit {
/**Second**/
SECOND(1000L),
/**MINUTE**/
MINUTE(60 * 1000L),
/**Hour**/
HOUR(60 * 60 * 1000L),
/**Day**/
DAY(24 * 60 * 60 * 1000L);
private Long timeMillSeconds;
private String name;
TimeUnit(long timeMillSeconds) {
this.name = this.toString();
this.timeMillSeconds = timeMillSeconds;
}
public long timeMillSeconds() {
return this.timeMillSeconds;
}
/**
* currentTimeMillSeconds
* @param timeType
* @param targetTime
* @return
*/
public static long currentTimeMillSeconds(String timeType, long targetTime) {
Calendar calendar = Calendar.getInstance();
calendar.setTimeInMillis(targetTime);
switch (timeType){
case "SECOND":
return SECOND.timeMillSeconds;
case "MINUTE":
return MINUTE.timeMillSeconds;
case "HOUR":
return HOUR.timeMillSeconds;
case "DAY":
return DAY.timeMillSeconds;
case "MONTH":
return calendar.getActualMaximum(Calendar.DAY_OF_MONTH) * DAY.timeMillSeconds;
case "YEAR":
return calendar.getActualMaximum(Calendar.DAY_OF_YEAR) * DAY.timeMillSeconds;
default:
return 0L;
}
}
}
src/main/java/com/zhiwei/qbjcutils/pojo/entity/BaseObjectCache.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.entity;
import lombok.AllArgsConstructor;
import lombok.NoArgsConstructor;
import lombok.ToString;
/**
* Description:
*
* @author LiuMingHuan
* @classname BaseObjectCache
* @date 2020/3/29
*/
@NoArgsConstructor
@AllArgsConstructor
@ToString
public class BaseObjectCache {
/**
* comment_num : -1
* like_num : -1
* real_source : 微信公众号
* gid : 1027174022
* origin : PC
* cname : 微信账号采集
* ind_full_text : ["长沙人注意!今年清明祭扫需先预约,这些你必须知道!","春暖花开芳草萋，又是一年清明时 按照传统 人们会在这天祭祀扫墓、踏青出游 受疫情影响 今年的清明注定与往年不同 文明祭扫 同舟共济，携手抗疫 在全国人民的共同努力下 疫情防控形势持续向好 但“革命”尚未结束，我们仍不能掉以轻心 清明时节，适逢全国上下共抗疫情关键时刻 为避免清明期间人员扎堆聚集 减少交叉感染风险 近期，全国多地发布倡议 提倡利用网络“云上祭扫”缅怀先人 为进一步弘扬优良传统、倡导文明新风 确保2020年清明节祭扫绿色、文明、低碳、安全 近日，湖南发布 《关于做好清明期间疫情防控和安全防范工作的通知》 湘疫防【2020】26号 倡导广大市民绿色低碳祭扫，推广网络祭扫 降低因人员聚集产生的疫情防控风险 对确有实地祭扫需求的 实行按日分时段限流预约祭扫 清明期间，如需现场祭扫 可通过以下方式预约： 01 点击下方二维码进入“云上清明”提前预约 02 拨打长沙市公墓电话提前预约 团团倡议 团子们： 　　今年，清明时节正值新冠肺炎疫情防控进入“下半场”的特殊时刻，为切实做好清明祭扫期间的疫情防控工作，营造安全、有序的祭祀环境，大力倡导绿色、文明、节俭的祭祀新风尚，弘扬移风易俗的文明新风，我们倡议： 安全祭祀，守护健康 提倡通过网络祭祀、撰写缅怀文章、选择通过公墓地提供的“代客祭扫”服务等方式，实现“云祭扫”和“云哀悼”；提倡外出祭扫人员做好个人防护，预约或错峰祭扫，不搞组团祭扫，简化祭扫流程，压缩祭扫时间。 文明祭祀，移风易俗 提倡以鲜花祭奠、植树纪念、踏青遥祭、家庭追思等文明、安全、低碳的方式，来替代烧纸钱、放爆竹等祭扫陋习，抵制迷信行为和低俗祭祀用品，将中华民族慎终追远的情感用现代文明的方式来表达。 绿色祭祀，爱护环境 倡导以献花、鞠躬等简约环保方式进行祭扫，不在路口、公园、小区周边绿化带内焚烧纸钱、燃放鞭炮，不影响他人正常生活，不影响公共环境卫生；不在林区焚烧纸钱、燃放鞭炮，防止造成火灾隐患。 以身作则，践行文明 人人争做文明新风的倡导者和传播者，自觉抵制和劝导不文明祭扫行为，以实际行动影响和带动身边的团员青年安全祭扫、践行文明、摒弃陋习。 团子们，参与疫情防控、践行文明祭祀、倡导移风易俗是你我共同的责任。保护好自身健康就是对家庭、对社会的最大负责，更是对逝者的最好告慰。让我们携起手来，从现在做起，从自身做起，从点滴做起，以文明祭祀的实际行动，弘扬时代新风，共度一个安全、文明、环保的清明节！ 本期小编 小巷 来源：长沙共青团新媒体中心 审稿：羅、格格 ☎ 0731-81859810 抖音号：1189978172 ✉ xcbtsw@163.com 团小青手机&微信：18674395454 爱 民 路 51 号 有态度 有深度 有温度 文明祭扫，从你我做起！"]
* stime : 1585296042222
* ind_title : ["长沙人注意!今年清明祭扫需先预约,这些你必须知道!"]
* source : 长沙共青团
* c1 : 134217728
* url : http://mp.weixin.qq.com/s?__biz=MzA5MjIxNDkxMQ==&mid=2651558191&idx=1&sn=50e5ad1a5fe372921b159569c1bf62ee&chksm=8b8fb23fbcf83b294d33b803e77af455456f747b3dff46f8d9f579423f5b3e07208df190d991&scene=0&xtrack=1#rd
* foreign : 0
* c2 : 8388608
* c3 : 524288
* c4 : 1000
* read_num : -1
* c5 : 143131624
* user_id : MzA5MjIxNDkxMQ==
* ctime : 1585296040212
* time : 1585295803000
* id : 20969db4040769f4b3d1f4182ba50672
* cid : 100030001
*/
private String real_source;
private int gid;
private String origin;
private String cname;
private String ind_full_text;
private long stime;
private String ind_title;
private String source;
private int c1;
private String url;
private int foreign;
private int c2;
private int c3;
private int c4;
private int c5;
private long ctime;
private long time;
private String id;
private int cid;
public String getReal_source() {
return real_source;
}
public void setReal_source(String real_source) {
this.real_source = real_source;
}
public int getGid() {
return gid;
}
public void setGid(int gid) {
this.gid = gid;
}
public String getOrigin() {
return origin;
}
public void setOrigin(String origin) {
this.origin = origin;
}
public String getCname() {
return cname;
}
public void setCname(String cname) {
this.cname = cname;
}
public String getInd_full_text() {
return ind_full_text;
}
public void setInd_full_text(String ind_full_text) {
this.ind_full_text = ind_full_text;
}
public long getStime() {
return stime;
}
public void setStime(long stime) {
this.stime = stime;
}
public String getInd_title() {
return ind_title;
}
public void setInd_title(String ind_title) {
this.ind_title = ind_title;
}
public String getSource() {
return source;
}
public void setSource(String source) {
this.source = source;
}
public int getC1() {
return c1;
}
public void setC1(int c1) {
this.c1 = c1;
}
public String getUrl() {
return url;
}
public void setUrl(String url) {
this.url = url;
}
public int getForeign() {
return foreign;
}
public void setForeign(int foreign) {
this.foreign = foreign;
}
public int getC2() {
return c2;
}
public void setC2(int c2) {
this.c2 = c2;
}
public int getC3() {
return c3;
}
public void setC3(int c3) {
this.c3 = c3;
}
public int getC4() {
return c4;
}
public void setC4(int c4) {
this.c4 = c4;
}
public int getC5() {
return c5;
}
public void setC5(int c5) {
this.c5 = c5;
}
public long getCtime() {
return ctime;
}
public void setCtime(long ctime) {
this.ctime = ctime;
}
public long getTime() {
return time;
}
public void setTime(long time) {
this.time = time;
}
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public int getCid() {
return cid;
}
public void setCid(int cid) {
this.cid = cid;
}
}
src/main/java/com/zhiwei/qbjcutils/pojo/entity/BaseObjectCacheKeyword.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.entity;
/**
* Description:
*
* @author LiuMingHuan
* @classname BaseObjectCacheKeyword
* @date 2020/5/7
*/
public class BaseObjectCacheKeyword extends BaseObjectCache {
private int wordFrequency;
public int getWordFrequency() {
return wordFrequency;
}
public void setWordFrequency(int wordFrequency) {
this.wordFrequency = wordFrequency;
}
}
src/main/java/com/zhiwei/qbjcutils/pojo/entity/CommonKeyword.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.entity;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.List;
/**
* Description:
*
* @author LiuMingHuan
* @classname KeywordCommon
* @date 2020/5/22
*/
@AllArgsConstructor
@NoArgsConstructor
@Data
public class CommonKeyword {
/**
* id
*/
private String id;
/**
* groupName
*/
private String groupName;
/**
* usedKeywords
*/
private List<String> usedKeywords;
/**
* keywordInfos all
*/
private List<KeywordInfo> keywordInfos;
/**
* platforms
*/
private List<String> platforms;
/**
* directGroups
*/
private List<String> directGroups;
/**
* groupUsed groupUsed
*/
private Boolean groupUsed;
/**
* show show
*/
private Boolean show;
/**
* project project
*/
private String project;
/**
* submitter submitter
*/
private String submitter;
/**
* createAt createAt
*/
private Long createAt;
/**
* updateAt updateAt
*/
private Long updateAt;
}
src/main/java/com/zhiwei/qbjcutils/pojo/entity/DirectGroup.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.entity;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import org.springframework.data.mongodb.core.mapping.Document;
import java.util.HashMap;
import java.util.List;
/**
* 定向监测组
*
* @ClassName: DirectGroup
* @Description: 定向监测组
* @author shentao
* @date 2019年7月8日 下午6:08:50
*/
@Getter
@Setter
@AllArgsConstructor
@NoArgsConstructor
@Document(collection = "qbjc_direct_group")
public class DirectGroup {
/**
* "id"
*/
private String id;
/**
* "name": 渠道组名
*/
private String name;
/**
* project 所属项目
*/
private String project;
/**
* submitter提交人员
*/
private String submitter;
/**
* 创建时间createAt
*/
private Long createAt;
/**
* 是否使用used
*/
private Boolean used;
/**
* 是否标注tag
*/
private Boolean tag;
/**
* memberList member使用中的list
*/
private List<String> memberList;
/**
* memberMaps member全信息list
*/
private List<HashMap<String, Object>> memberMaps;
}
src/main/java/com/zhiwei/qbjcutils/pojo/entity/Event.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.entity;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import org.springframework.data.mongodb.core.mapping.Document;
import java.util.List;
/**
* @ClassName: Event
* @Description: 事件
* @author shentao
* @date 2019年7月17日 上午10:27:48
*/
@Getter
@Setter
@AllArgsConstructor
@NoArgsConstructor
@Document(collection = "qbjc_event")
public class Event {
/**
* 事件id
*/
private String id;
/**
* 事件采集id
*/
private String collectionId;
/**
* 事件Name
*/
private String name;
/**
* 事件开始时间
*/
private Long startTime;
/**
* 事件结束时间
*/
private Long endTime;
/**
* 事件关键词
*/
private String keyword;
/**
* 事件提交人
*/
private String submitter;
/**
* 事件项目
*/
private String project;
/**
* 事件创建时间
*/
private Long createAt;
/**
* 事件更新时间
*/
private Long updateAt;
/**
* 事件传播状态spreadStatus
*/
private String spreadStatus;
/**
* 事件状态status（）
*/
private String status;
/**
* 事件标签markTag
*/
private String markTag;
/**
* 事件结果下载地址fileAddress
*/
private List<String> fileAddress;
/**
* 事件热度hotPoint
*/
private Double hotPoint;
/**
* 事件传播量
*/
private String spreadAmount;
/**
* 事件weibo互动量
*/
private Integer weiboInteraction;
/**
* 事件weibo评论量
*/
private Integer weiboReplyCount;
/**
* 事件weibo转发量
*/
private Integer weiboRetweetCount;
/**
* 网媒影响力
*/
private Integer mediaInfluence;
/**
* 网媒总条数
*/
private Integer mediaCount;
/**
* 平媒数据量
*/
private Integer printCount;
/**
* printH 平媒H因子量
*/
private Integer printInfluence;
/**
* 自媒体数据量
*/
private Integer selfMediaCount;
/**
* selfMediaH 自媒体H因子量
*/
private Integer selfMediaInfluence;
/**
* 微博影响力
*/
private Integer weiboInfluence;
/**
* 微博总条数
*/
private Integer weiboCount;
/**
* 微信影响力
*/
private Integer wechatInfluence;
/**
* 微信总条数
*/
private Integer wechatCount;
/**
* 头条影响力
*/
private Integer toutiaoInfluence;
/**
* 头条总条数
*/
private Integer toutiaoCount;
/**
* 是否为有上传文件
*/
private Integer fileSourceStatus;
/**
* 表格解析出错
*/
private Boolean errorStatus;
/**
* 事件数据源
*/
private String dataSource;
}
src/main/java/com/zhiwei/qbjcutils/pojo/entity/EventCollection.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.entity;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import org.springframework.data.mongodb.core.mapping.Document;
import java.util.List;
import java.util.Map;
/**
* @ClassName: EventCollection
* @Description: 事件采集
* @author shentao
* @date 2019年7月17日 上午10:28:40
*/
@Getter
@Setter
@AllArgsConstructor
@NoArgsConstructor
@Document(collection = "qbjc_event_collection")
public class EventCollection {
/**
* 事件采集id
*/
private String id;
/**
* 事件id
*/
private String eventId;
/**
* 事件type 0 普通|1 危机|2 带微博危机
*/
private Integer eventType;
/**
* 事件采集Name
*/
private String name;
/**
* 事件采集开始时间
*/
private Long startTime;
/**
* 事件采集结束时间
*/
private Long endTime;
/**
* 事件采集关键词
*/
private String keyword;
/**
* 已选关键词
*/
private String selectKeyword;
/**
* 事件采集提交人
*/
private String submitter;
/**
* 事件采集项目
*/
private String project;
/**
* 事件采集创建时间
*/
private Long createAt;
/**
* 事件采集更新时间
*/
private Long updateAt;
/**
* 事件采集状态status
*/
private String status;
/**
* 事件采集细节结果detials
*/
private String detials;
/**
* 事件采集结果下载地址fileAddress
*/
private List<String> fileAddress;
/**
* 事件历史采集记录
*/
private List<String> records;
/**
* 下载频率(下载人，/下载时间)
*/
private List<Map<String, Object>> downloadFrequency;
}
\ No newline at end of file
src/main/java/com/zhiwei/qbjcutils/pojo/entity/Keyword.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.entity;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.mapping.Document;
import java.util.HashMap;
import java.util.List;
/**
* Keyword
*
* @ClassName: Keyword
* @Description: Keyword
* @author shentao
* @date 2019年7月8日 下午4:06:52
*/
@Getter
@Setter
@AllArgsConstructor
@NoArgsConstructor
@Document(collection = "qbjc_keyword_back")
public class Keyword {
/**
* "id": keyTitle_project
*/
@Id
private String id;
/**
* "title": "小米产品",关键词主题名
*/
private String keyTitle;
/**
* "keyWords":关键词词组
*/
private List<String> keyWords;
/**
* "kwtimes":关键词词组
*/
private HashMap<String, Object> kwtimes;
/**
* project 所属项目
*/
private String project;
/**
* submitter提交人员
*/
private String submitter;
/**
* 创建时间createAt
*/
private Long createAt;
/**
* 创建时间是否使用used
*/
private Boolean used;
/**
* 定向监测是否使用dxIsUsed
*/
private Boolean dxIsUsed;
/**
* 平台列表
*/
private List<String> ptList;
/**
* 渠道列表
*/
private List<String> qdList;
}
src/main/java/com/zhiwei/qbjcutils/pojo/entity/KeywordInfo.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.entity;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* Description:
*
* @author LiuMingHuan
* @classname KeywordInfo
* @date 2020/5/21
*/
@AllArgsConstructor
@NoArgsConstructor
@Data
public class KeywordInfo {
/**
* name
*/
private String name;
/**
* used
*/
private Boolean used;
/**
* updateAt
*/
private Long updateAt;
/**
* submitter
*/
private String submitter;
}
src/main/java/com/zhiwei/qbjcutils/pojo/entity/MessagePlatform.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.entity;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import org.springframework.data.mongodb.core.mapping.Document;
import java.util.List;
/**
* @ClassName: MessagePlatform
* @Description: 最新消息流-平台实体
* @author shentao
* @date 2019年8月27日 下午7:14:28
*/
@Getter
@Setter
@AllArgsConstructor
@NoArgsConstructor
@Document(collection = "qbjc_message_platform")
public class MessagePlatform {
/**
* id 平台id
*/
private String id;
/**
* name 平台name
*/
private String name;
/**
* templateHeaders 模板头部id
*/
private List<String> templateHeaders;
/**
* createAt 创建时间
*/
private Long createAt;
/**
* submitter 提交人
*/
private String submitter;
}
src/main/java/com/zhiwei/qbjcutils/pojo/entity/NoiseRule.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.entity;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import org.springframework.data.mongodb.core.mapping.Document;
import java.util.List;
/**
* 去噪规则
*
* @ClassName: NoiseRule
* @Description: 去噪规则
* @author shentao
* @date 2019年7月8日 下午4:52:18
*/
@Getter
@Setter
@NoArgsConstructor
@Document(collection = "qbjc_noise_rule")
public class NoiseRule {
/**
* id
*/
private Long id;
/**
* ruleName 规则名
*/
private String ruleName;
/**
* ruleExplain 规则说明
*/
private String ruleExplain;
/**
* "ruleType": " (keyWords, channel)",
*/
private String ruleType;
/**
* keyWordPt 关键词规则适用平台
*/
private List<String> keyWordPt;
/**
* keyWordsInputOne 关键词规则第一个input输入框
*/
private String keyWordsInputOne;
/**
* andOr 且或者或
*/
private String andOr;
/**
* keyWordsInputTwo 关键词规则第二个input输入框
*/
private String keyWordsInputTwo;
/**
* 关键词汇总
*/
private String keyWords;
/**
* channelQd 渠道
*/
private String channelQd;
/**
* channelPt 渠道平台
*/
private List<String> channelPt;
/**
* project 所属项目
*/
private String project;
/**
* createAt 创建时间
*/
private Long createAt;
/**
* submitter 创建人
*/
private String submitter;
/**
* kwSearch 检索范围：全文、标题
*/
private String kwSearch;
/**
* used 是否被使用
*/
private Boolean used;
/**
* effectNum 生效次数
*/
private Long effectNum;
/**
* effectNumWeek 历史一周生效次数
*/
private List<Long> effectNumWeek;
}
src/main/java/com/zhiwei/qbjcutils/pojo/entity/NoiseRuleEntity.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.entity;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.springframework.data.mongodb.core.mapping.Document;
import java.util.List;
/**
* Description:
*
* @author LiuMingHuan
* @classname NoiseRuleEntity
* @date 2020/5/20
*/
@AllArgsConstructor
@NoArgsConstructor
@Data
@Document("qbjc_noise_rule_entity")
public class NoiseRuleEntity {
/**
* id
*/
private String id;
/**
* ruleName 规则名
*/
private String ruleName;
/**
* ruleContent 规则内容
*/
private String ruleContent;
/**
* ruleType 去噪类型
*/
private String ruleType;
/**
* kwSearch keyword 检索范围
*/
private String kwSearch;
/**
* currentNoiseNum 当日
*/
private Long currentNoiseNum;
/**
* totalNoiseNum 历史累计
*/
private Long totalNoiseNum;
/**
* noiseRuleExplain
*/
private String noiseRuleExplain;
/**
* platforms
*/
private List<String> platforms;
/**
* project
*/
private String project;
/**
* createAt
*/
private Long createAt;
/**
* updateAt
*/
private Long updateAt;
/**
* submitter
*/
private String submitter;
}
src/main/java/com/zhiwei/qbjcutils/pojo/entity/SentimentResult.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.entity;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.List;
import java.util.Map;
/**
* Description:
*
* @author LiuMingHuan
* @classname SentimentResult
* @date 2020/6/5
*/
@AllArgsConstructor
@NoArgsConstructor
@Data
public class SentimentResult {
private Long id;
private String text;
private List<Map<String, Object>> result;
}
src/main/java/com/zhiwei/qbjcutils/pojo/entity/Tag.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.entity;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import org.springframework.data.mongodb.core.mapping.Document;
/**
* Tag
*
* @ClassName: Tag
* @Description: Tag
* @author shentao
* @date 2019年7月8日 下午4:50:26
*/
@Getter
@Setter
@AllArgsConstructor
@NoArgsConstructor
@Document(collection = "qbjc_tag")
public class Tag {
/**
* id
*/
private Long id;
/**
* groupId 标签组id 删除时删除
*/
private Long groupId;
/**
* uniqueId
*/
private String uniqueId;
/**
* name 标签名称
*/
private String name;
/**
* groupName 标签组名称
*/
private String groupName;
/**
* createAt 标签创建时间
*/
private Long createAt;
/**
* submitter
*/
private String submitter;
/**
* project
*/
private String project;
/**
* show 是否呈现在事件列表
*/
private Boolean show;
}
src/main/java/com/zhiwei/qbjcutils/pojo/excelentity/BaseObjectExcelEntity.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.excelentity;
import com.alibaba.excel.annotation.ExcelProperty;
import com.alibaba.excel.annotation.write.style.ColumnWidth;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.springframework.data.mongodb.core.mapping.Document;
/**
* Description:
*
* @author LiuMingHuan
* @classname BaseObjectExcelEntity
* @date 2020/3/31
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
@Document(collection = "base_object_excel_entity")
public class BaseObjectExcelEntity {
@ExcelProperty(value = "id",index = 0)
@ColumnWidth(5)
private String id;
@ExcelProperty(value = "时间",index = 1)
@ColumnWidth(5)
private long time;
@ExcelProperty(value = "url",index = 2)
@ColumnWidth(8)
private String url;
@ExcelProperty(value = "标题",index = 3)
@ColumnWidth(12)
private String title;
@ExcelProperty(value = "内容",index = 4)
@ColumnWidth(12)
private String content;
@ExcelProperty(value = "平台",index = 5)
@ColumnWidth(7)
private String platform;
@ExcelProperty(value = "来源",index = 6)
@ColumnWidth(7)
private String source;
@ExcelProperty(value = "信源",index = 7)
@ColumnWidth(7)
private String origin;
@ExcelProperty(value = "foreign",index = 8)
@ColumnWidth(7)
private Integer foreign;
@ExcelProperty(value = "c1",index = 9)
@ColumnWidth(5)
private int c1;
@ExcelProperty(value = "c2",index = 10)
@ColumnWidth(5)
private int c2;
@ExcelProperty(value = "c3",index = 11)
@ColumnWidth(5)
private int c3;
@ExcelProperty(value = "c4",index = 12)
@ColumnWidth(5)
private int c4;
@ExcelProperty(value = "c5",index = 13)
@ColumnWidth(5)
private int c5;
@ExcelProperty(value = "采集时间",index = 14)
@ColumnWidth(7)
private long ctime;
@ExcelProperty(value = "cid",index = 15)
@ColumnWidth(7)
private int cid;
}
src/main/java/com/zhiwei/qbjcutils/pojo/excelentity/DirectChannel.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.excelentity;
import com.alibaba.excel.annotation.ExcelProperty;
import com.alibaba.excel.annotation.write.style.ColumnWidth;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
/**
* Description:
*
* @author LiuMingHuan
* @classname DirecChannel
* @date 2020/4/16
*/
@Data
@ToString
@AllArgsConstructor
@NoArgsConstructor
public class DirectChannel {
@ExcelProperty(value = "渠道组", index = 0)
@ColumnWidth(8)
private String directGroup;
@ExcelProperty(value = "渠道", index = 1)
@ColumnWidth(12)
private String channel;
@ColumnWidth(3)
@ExcelProperty(value = "平台", index = 2)
private String platform;
}
src/main/java/com/zhiwei/qbjcutils/pojo/excelentity/KeywordReader.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.excelentity;
import lombok.*;
/**
* Description:
*
* @author LiuMingHuan
* @classname KeywordReader
* @date 2020/4/22
*/
@Data
@AllArgsConstructor
public class KeywordReader {
}
src/main/java/com/zhiwei/qbjcutils/pojo/excelentity/MarkDownloadInfo.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.excelentity;
import com.alibaba.excel.annotation.ExcelProperty;
import com.alibaba.excel.annotation.write.style.ColumnWidth;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
/**
* @author shentao
* @ClassName: MarkDownloadInfo
* @Description: 标注下载表实体[序号 时间 标题 文本 地址 来源 平台 渠道 认证信息 采编权 粉丝数 认证类型 级别 领域 是否原发
* 原创作者 原创文本 命中词 标注时间 标注人 标签一 标签二 标签三 标签四 标签五]
* @date 2019年7月23日 上午9:38:21
*/
@Getter
@Setter
@AllArgsConstructor
@NoArgsConstructor
public class MarkDownloadInfo {
@ExcelProperty(value = "序号", index = 0)
@ColumnWidth(5)
private String id;
@ExcelProperty(value = "时间", index = 1)
@ColumnWidth(16)
private String time;
@ExcelProperty(value = "标题", index = 2)
@ColumnWidth(33)
private String title;
@ExcelProperty(value = "文本", index = 3)
@ColumnWidth(5)
private String content;
@ExcelProperty(value = "地址", index = 4)
@ColumnWidth(5)
private String url;
/**
* 固定平台
*/
@ExcelProperty(value = "平台", index = 5)
@ColumnWidth(10)
private String platform;
/**
* 信源 APP\PC\MOBI
*/
@ExcelProperty(value = "信源", index = 6)
@ColumnWidth(5)
private String origin;
/**
* c4 name realSource
*/
@ExcelProperty(value = "来源", index = 7)
@ColumnWidth(5)
private String clientFrom;
/**
* 来源 source
*/
@ExcelProperty(value = "渠道", index = 8)
@ColumnWidth(8)
private String source;
@ExcelProperty(value = "级别", index = 9)
@ColumnWidth(5)
private String level;
@ExcelProperty(value = "采编权", index = 10)
@ColumnWidth(5)
private String acquisitionRights;
@ExcelProperty(value = "领域", index = 11)
@ColumnWidth(5)
private String field;
@ExcelProperty(value = "认证信息", index = 12)
@ColumnWidth(10)
private String authenticationInformation;
@ExcelProperty(value = "粉丝数", index = 13)
@ColumnWidth(9)
private String fans;
@ExcelProperty(value = "认证类型", index = 14)
@ColumnWidth(5)
private String authenticationType;
@ExcelProperty(value = "是否原发", index = 15)
@ColumnWidth(8)
private String primary;
@ExcelProperty(value = "原创作者", index = 16)
@ColumnWidth(7)
private String rootSource;
@ExcelProperty(value = "原创文本", index = 17)
@ColumnWidth(7)
private String rootContent;
@ExcelProperty(value = "命中词", index = 18)
@ColumnWidth(7)
private String keywords;
@ExcelProperty(value = "UID", index = 19)
@ColumnWidth(5)
private String uid;
@ExcelProperty(value = "MID", index = 20)
@ColumnWidth(5)
private String mid;
@ExcelProperty(value = "标注时间", index = 21)
@ColumnWidth(8)
private String mtime;
@ExcelProperty(value = "标注人", index = 22)
@ColumnWidth(7)
private String mperson;
@ExcelProperty(value = "标签一", index = 23)
@ColumnWidth(7)
private String mtagOne;
@ExcelProperty(value = "标签二", index = 24)
@ColumnWidth(7)
private String mtagTwo;
@ExcelProperty(value = "标签三", index = 25)
@ColumnWidth(7)
private String mtagThree;
@ExcelProperty(value = "标签四", index = 26)
@ColumnWidth(7)
private String mtagFour;
@ExcelProperty(value = "标签五", index = 27)
@ColumnWidth(7)
private String mtagFive;
public void setCompleteTextMark(String time, String title, String content, String url, String platform, String origin,
String clientFrom, String source, String authenticationInformation, String acquisitionRights, String level,
String field, String primary, String rootSource, String mtime, String mperson) {
this.time = time;
this.title = title;
this.content = content;
this.url = url;
this.platform = platform;
this.origin = origin;
this.clientFrom = clientFrom;
this.source = source;
this.authenticationInformation = authenticationInformation;
this.acquisitionRights = acquisitionRights;
this.level = level;
this.field = field;
this.primary = primary;
this.rootSource = rootSource;
this.mtime = mtime;
this.mperson = mperson;
}
public void setIncompleteTextMark(String time, String content, String url, String platform, String origin, String clientFrom,
String source, String fans, String authenticationType, String level, String field, String primary,
String rootSource, String rootContent, String uid, String mid, String mtime, String mperson) {
this.time = time;
this.title = content;
this.content = content;
this.url = url;
this.platform = platform;
this.origin = origin;
this.clientFrom = clientFrom;
this.source = source;
this.fans = fans;
this.authenticationType = authenticationType;
this.level = level;
this.field = field;
this.primary = primary;
this.rootSource = rootSource;
this.rootContent = rootContent;
this.uid = uid;
this.mid = mid;
this.mtime = mtime;
this.mperson = mperson;
}
public void setQaTextMark(String time, String title, String content, String url, String platform,
String origin, String clientFrom,
String source, String mtime, String mperson) {
this.time = time;
this.title = title;
this.content = content;
this.url = url;
this.platform = platform;
this.origin = origin;
this.clientFrom = clientFrom;
this.source = source;
this.mtime = mtime;
this.mperson = mperson;
}
public void setVideoMark(String time, String title, String content, String url, String platform, String origin,
String clientFrom, String source, String mtime, String mperson) {
this.time = time;
this.title = title;
this.content = content;
this.url = url;
this.platform = platform;
this.origin = origin;
this.clientFrom = clientFrom;
this.source = source;
this.mtime = mtime;
this.mperson = mperson;
}
}
src/main/java/com/zhiwei/qbjcutils/pojo/excelentity/SentimentInfo.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.excelentity;
import com.alibaba.excel.annotation.ExcelProperty;
import lombok.Data;
/**
* Description:
*
* @author LiuMingHuan
* @classname SentimentInfo
* @date 2020/6/11
*/
@Data
public class SentimentInfo {
@ExcelProperty(value = "id", index = 0)
private String id;
@ExcelProperty(value = "url", index = 1)
private String url;
@ExcelProperty(value = "情感倾向", index = 10)
private String sentiment;
@ExcelProperty(value = "情感倾向2", index = 11)
private String sentimentBack;
}
src/main/java/com/zhiwei/qbjcutils/pojo/excelentity/UploadData.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.excelentity;
import com.alibaba.excel.annotation.ExcelProperty;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
/**
* Description:
*
* @author LiuMingHuan
* @classname UploadData
* @date 2020/4/2
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
@ToString
public class UploadData {
@ExcelProperty(value = "平台",index = 0)
private String platform;
@ExcelProperty(value = "来源",index = 1)
private String source;
@ExcelProperty(value = "时间",index = 2)
private String time;
@ExcelProperty(value = "标题",index = 3)
private String title;
@ExcelProperty(value = "地址",index = 4)
private String url;
@ExcelProperty(value = "文本",index = 5)
private String content;
@ExcelProperty(value = "UID",index = 6)
private String uid;
@ExcelProperty(value = "MID",index = 7)
private String mid;
}
src/main/java/com/zhiwei/qbjcutils/pojo/excelhadler/PlatformCountHandler.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.excelhadler;
import com.alibaba.excel.metadata.CellData;
import com.alibaba.excel.metadata.Head;
import com.alibaba.excel.write.handler.CellWriteHandler;
import com.alibaba.excel.write.metadata.holder.WriteSheetHolder;
import com.alibaba.excel.write.metadata.holder.WriteTableHolder;
import org.apache.poi.hssf.util.HSSFColor;
import org.apache.poi.ss.usermodel.*;
import java.util.List;
/**
* Description:
*
* @author LiuMingHuan
* @classname PlatformCountHandler
* @date 2020/4/20
*/
public class PlatformCountHandler implements CellWriteHandler {
@Override
public void beforeCellCreate(WriteSheetHolder writeSheetHolder, WriteTableHolder writeTableHolder, Row row, Head head, Integer integer, Integer integer1, Boolean aBoolean) {
}
@Override
public void afterCellCreate(WriteSheetHolder writeSheetHolder, WriteTableHolder writeTableHolder, Cell cell, Head head, Integer integer, Boolean aBoolean) {
}
@Override
public void afterCellDispose(WriteSheetHolder writeSheetHolder, WriteTableHolder writeTableHolder, List<CellData> list, Cell cell, Head head, Integer integer, Boolean aBoolean) {
Workbook workbook = writeSheetHolder.getSheet().getWorkbook();
CellStyle cellStyle = workbook.createCellStyle();
cellStyle.setAlignment(HorizontalAlignment.CENTER);
if (cell.getRowIndex() == 0) {
cellStyle.setFillBackgroundColor(HSSFColor.HSSFColorPredefined.PALE_BLUE.getIndex());
}
cell.setCellStyle(cellStyle);
}
}
src/main/java/com/zhiwei/qbjcutils/pojo/excellistener/DirectChannelListener.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.excellistener;
import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.event.AnalysisEventListener;
import com.zhiwei.qbjcutils.pojo.excelentity.DirectChannel;
import lombok.Getter;
import java.util.ArrayList;
import java.util.List;
/**
* Description:
*
* @author LiuMingHuan
* @classname DirectChannelListener
* @date 2020/4/16
*/
@Getter
public class DirectChannelListener extends AnalysisEventListener<DirectChannel> {
private List<DirectChannel> directChannels= new ArrayList<>();
@Override
public void invoke(DirectChannel directChannel, AnalysisContext analysisContext) {
directChannels.add(directChannel);
}
@Override
public void doAfterAllAnalysed(AnalysisContext analysisContext) {
}
}
src/main/java/com/zhiwei/qbjcutils/pojo/excellistener/Sentimentistener.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.excellistener;
import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.event.AnalysisEventListener;
import com.zhiwei.qbjcutils.pojo.excelentity.SentimentInfo;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
/**
* Description:
*
* @author LiuMingHuan
* @classname SentimenListener
* @date 2020/6/11
*/
@Data
public class Sentimentistener extends AnalysisEventListener<SentimentInfo> {
private List<SentimentInfo> infos = new ArrayList<>();
@Override
public void invoke(SentimentInfo sentimentInfo, AnalysisContext analysisContext) {
infos.add(sentimentInfo);
}
@Override
public void doAfterAllAnalysed(AnalysisContext analysisContext) {
}
}
src/main/java/com/zhiwei/qbjcutils/pojo/excellistener/UploadDataListener.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.excellistener;
import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.event.AnalysisEventListener;
import com.zhiwei.qbjcutils.pojo.excelentity.UploadData;
import lombok.Data;
import lombok.EqualsAndHashCode;
import java.util.ArrayList;
import java.util.List;
/**
* Description:
*
* @author LiuMingHuan
* @classname UploadDataListener
* @date 2020/4/2
*/
@EqualsAndHashCode(callSuper = true)
@Data
public class UploadDataListener extends AnalysisEventListener<UploadData> {
private List<UploadData> infos =new ArrayList<>();
@Override
public void invoke(UploadData uploadData, AnalysisContext analysisContext) {
uploadData.setUid("无".equals(uploadData.getUid()) ? "" : uploadData.getUid());
uploadData.setMid("无".equals(uploadData.getMid()) ? "" : uploadData.getMid());
infos.add(uploadData);
}
@Override
public void doAfterAllAnalysed(AnalysisContext analysisContext) {
}
}
src/main/java/com/zhiwei/qbjcutils/redis/RedisTools.java 0 → 100644
package com.zhiwei.qbjcutils.redis;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.qbjcutils.config.GenericAttribute;
import com.zhiwei.qbjcutils.pojo.entity.MessagePlatform;
import com.zhiwei.qbjcutils.pojo.entity.Tag;
import org.redisson.api.RBucket;
import org.redisson.api.RMap;
import org.redisson.api.RScoredSortedSet;
import org.redisson.api.RedissonClient;
import org.redisson.client.codec.StringCodec;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
/**
* es 方法类
*
* @author shentao
* @ClassName: RedisTools
* @Description: es 方法类
* @date 2019年7月3日 上午9:00:10
*/
@Service
public class RedisTools {
private static final String KEY_INFO = "YUQING:Info:";
private final RedissonClient redissonClient;
public RedisTools(RedissonClient redissonClient) {
this.redissonClient = redissonClient;
}
/**
* key是否存在
*
* @param @param key
* @param @return 设定文件
* @return boolean 返回类型
* @Title: exists
* @Description: key是否存在
*/
public boolean exists(String key) {
RBucket<?> bucket = redissonClient.getBucket(key, StringCodec.INSTANCE);
return bucket.isExists();
}
/**
* 获取Score降序 info
*
* @param @param key
* @param @return 设定文件
* @return List<String> 返回类型
* @Title: zrevrange
* @Description: 获取Score降序 info
*/
public List<String> zrevrange(String key, int start, int end) {
RScoredSortedSet<String> set = redissonClient.getScoredSortedSet(key, StringCodec.INSTANCE);
return new ArrayList<>(set.valueRangeReversed(start, end));
}
/**
* 计算在有序集合中指定区间分数的成员数
*
* @param @param key
* @param @param startScore
* @param @param startScoreInclusive
* @param @param endScore
* @param @param endScoreInclusive
* @param @return 设定文件
* @return int 返回类型
* @Title: zcount
* @Description: 计算在有序集合中指定区间分数的成员数
*/
public int zcount(String key, double startScore, boolean startScoreInclusive, double endScore,
boolean endScoreInclusive) {
RScoredSortedSet<String> set = redissonClient.getScoredSortedSet(key, StringCodec.INSTANCE);
return set.count(startScore, startScoreInclusive, endScore, endScoreInclusive);
}
/**
* redis get String 方法
*
* @param @param key
* @param @return 设定文件
* @return String 返回类型
* @Title: get
* @Description: redis get String 方法
*/
public String get(String key) {
RBucket<String> bucket = redissonClient.getBucket(key, StringCodec.INSTANCE);
return bucket.get();
}
/**
* redis get RMap方法
*
* @param key
* @return
*/
public RMap<String, Object> getRMap(String key) {
return redissonClient.getMap(key, StringCodec.INSTANCE);
}
/**
* redis set String 方法
*
* @param @param key
* @param @param value 设定文件
* @return void 返回类型
* @Title: set
* @Description: redis set String 方法
*/
public void set(String key, String value) {
RBucket<String> bucket = redissonClient.getBucket(key, StringCodec.INSTANCE);
bucket.set(value);
}
/**
* redis setex String 方法
*
* @param @param key
* @param @param value 设定文件
* @return void 返回类型
* @Title: set
* @Description: redis setex String 方法
*/
public void setex(String key, String value, long timeToLive, TimeUnit timeUnit) {
RBucket<String> bucket = redissonClient.getBucket(key, StringCodec.INSTANCE);
bucket.set(value, timeToLive, timeUnit);
}
/**
* 统计SortSet总量
*
* @param @param key
* @param @return 设定文件
* @return long 返回类型
* @Title: countSortSet
* @Description: 统计SortSet总量
*/
public long countSortSet(String key) {
RScoredSortedSet<String> set = redissonClient.getScoredSortedSet(key, StringCodec.INSTANCE);
return set.size();
}
/**
* 获取info-Key
*
* @param @param pt
* @param @param keyWord
* @param @param project
* @param @return 设定文件
* @return String 返回类型
* @Title: getInfoKey
* @Description: 获取info-Key
*/
public String getInfoKey(String pt, String keyWord, String project) {
return new StringBuilder().append(KEY_INFO).append(project).append(":").append(pt).append(":")
.append(keyWord).toString();
}
/**
* 获取对应key的全部消息流集合
* @param infoKey
* @return
*/
public RScoredSortedSet<String> getAllMessageFlowEntity(String infoKey) {
return redissonClient.getScoredSortedSet(infoKey, StringCodec.INSTANCE);
}
/**
* 获取redis中保存的原有平台信息
* @param key
* @return
*/
public List<MessagePlatform> getAllMessagePlatform(String key) {
RMap<String, String> map = redissonClient.getMap(key, StringCodec.INSTANCE);
return new ArrayList<>(map.values()).stream().map(m -> JSONObject.parseObject(m, MessagePlatform.class)).collect(Collectors.toList());
}
/**
* 获取指定时段消息流缓存
* @param infoKey
* @param startIndex
* @param endIndex
* @return
*/
public List<JSONObject> getMessageFlowEntityWithTime(String infoKey, long startIndex, long endIndex) {
RScoredSortedSet<String> scoredSortedSet = redissonClient.getScoredSortedSet(infoKey, StringCodec.INSTANCE);
return new ArrayList<>(scoredSortedSet.valueRange(startIndex, false, endIndex, true))
.stream().map(JSONObject::parseObject).collect(Collectors.toList());
}
/**
* 获取全部平台信息
*/
public Map<String, String> getAllPlatForms() {
String key = GenericAttribute.SYSTEM_KEY+"platform";
return redissonClient.getMap(key, StringCodec.INSTANCE);
}
/**
*
* @param res
*/
public void cachePlatForms(Map<String, String> res) {
String key = GenericAttribute.SYSTEM_KEY+"platform";
RMap<String, String> map = redissonClient.getMap(key, StringCodec.INSTANCE);
map.putAll(res);
}
/**
* 更新缓存中的tags
* @param res
*/
public void setAllTags(List<Tag> res) {
String key = GenericAttribute.SYSTEM_KEY + "tag";
RBucket<String> bucket = redissonClient.getBucket(key, StringCodec.INSTANCE);
bucket.set(JSONObject.toJSONString(res));
}
}
src/main/java/com/zhiwei/qbjcutils/service/ClassifyUploadDataService.java 0 → 100644
package com.zhiwei.qbjcutils.service;
/**
* Description:
*
* @author LiuMingHuan
* @classname ClassfyUploadDataService
* @date 2020/4/2
*/
public interface ClassifyUploadDataService {
/**
* 分类上传数据
* @param path 文件路径
* @param project 项目
*/
void classifyUploadData(String path, String project);
}
src/main/java/com/zhiwei/qbjcutils/service/DirectChannelService.java 0 → 100644
package com.zhiwei.qbjcutils.service;
/**
* Description:
*
* @author LiuMingHuan
* @classname DirectChannelService
* @date 2020/4/16
*/
public interface DirectChannelService {
/**
* 定向渠道表格上传-新增逻辑（监测系统已实现该功能）
* @param project
* @param path
*/
@Deprecated
void readAndAddDirectChannel(String project, String path) throws Exception;
/**
* 定向渠道表格上传-补充逻辑（监测系统已实现该功能）
* @param project
* @param path
*/
@Deprecated
void read(String project, String path) throws Exception;
}
src/main/java/com/zhiwei/qbjcutils/service/MarkDataCountService.java 0 → 100644
package com.zhiwei.qbjcutils.service;
/**
* Description:
*
* @author LiuMingHuan
* @classname MarkDataCountService
* @date 2020/8/18
*/
public interface MarkDataCountService {
/**
* 按需统计标注库指定时段各平台数据量(excel导出)
* @param startTime
* @param endTime
* @param timeType
* @param timeUnit
* @param project
*/
void outputMarkDataCountByPlatforms(long startTime, long endTime, String timeType, String timeUnit, String project);
}
src/main/java/com/zhiwei/qbjcutils/service/MarkDataOutputService.java 0 → 100644
package com.zhiwei.qbjcutils.service;
/**
* Description:
*
* @author LiuMingHuan
* @classname MarkDataCountOutputService
* @date 2020/3/31
*/
public interface MarkDataOutputService {
}
src/main/java/com/zhiwei/qbjcutils/service/MessageFlowService.java 0 → 100644
package com.zhiwei.qbjcutils.service;
import com.zhiwei.qbjcutils.pojo.dto.BaseObjectCacheDTO;
import com.zhiwei.qbjcutils.pojo.entity.BaseObjectCache;
import com.zhiwei.qbjcutils.pojo.entity.BaseObjectCacheKeyword;
import com.zhiwei.qbjcutils.pojo.entity.MessagePlatform;
import com.zhiwei.qbjcutils.pojo.excelentity.BaseObjectExcelEntity;
import java.util.List;
/**
* Description:
*
* @author LiuMingHuan
* @classname CacheBaseObjectService
* @date 2020/3/30
*/
public interface MessageFlowService {
/**
* redis指定平台关键词项目数据导出 - mongo
* @param platform
* @param keyword
* @param project
* @return
*/
List<BaseObjectCache> getBaseObjectCacheFromMessageFlow(String platform,String keyword,String project);
/**
* redis指定平台关键词项目数据导出（附带词频）- mongo
* @param platform
* @param keyword
* @param project
* @return
*/
List<BaseObjectCacheKeyword> getBaseObjectCacheKeywordFromMessageFlow(String platform, String keyword, String project);
/**
* redis指定平台关键词项目数据导出 - excel
* @param platform
* @param keyword
* @param project
* @return
*/
List<BaseObjectExcelEntity> getBaseObjectCacheExcelFromMessageFlow(String platform,String keyword,String project);
/**
* redis指定平台关键词项目数据导出（附带标签） - mongo
* @param startTime
* @param endTime
* @param platform
* @param keyword
* @param project
* @return
*/
List<BaseObjectCacheDTO> getMessageFlowEntityWithTime(Long startTime, Long endTime, MessagePlatform platform, String keyword, String project);
/**
* 读取对应集合生成语料库（情感倾向匹配使用）
* @param collectionName
*/
void generateText(String collectionName);
}
src/main/java/com/zhiwei/qbjcutils/service/PlatformService.java 0 → 100644
package com.zhiwei.qbjcutils.service;
import com.zhiwei.qbjcutils.pojo.entity.MessagePlatform;
import java.util.List;
/**
* Description:
*
* @author LiuMingHuan
* @classname PlatformService
* @date 2020/4/3
*/
public interface PlatformService {
/**
* 获取全部平台（缓存日更新）
* @return
*/
List<MessagePlatform> getAllPlatforms();
/**
* 获取对应平台具体信息
* @param name
* @return
*/
MessagePlatform getPlatformByName(String name);
/**
* 按 c5_origin_foreign 组装header匹配 未匹配返回null
* @param c5
* @param origin
* @param foreign
* @return
*/
MessagePlatform getPlatform(int c5, String origin, int foreign);
/**
* 平台自动更新
*/
void autoUpdate();
/**
* 查询全量平台（mongo）
* @return
*/
List<MessagePlatform> getPlatforms();
}
src/main/java/com/zhiwei/qbjcutils/service/impl/ClassifyUploadDataServiceImpl.java 0 → 100644
package com.zhiwei.qbjcutils.service.impl;
import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.ExcelWriter;
import com.alibaba.excel.support.ExcelTypeEnum;
import com.alibaba.excel.write.metadata.WriteSheet;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.base.category.ClassB;
import com.zhiwei.qbjcutils.config.GenericAttribute;
import com.zhiwei.qbjcutils.es.EsIndexTools;
import com.zhiwei.qbjcutils.es.dao.EsDao;
import com.zhiwei.qbjcutils.pojo.excelentity.UploadData;
import com.zhiwei.qbjcutils.pojo.excellistener.UploadDataListener;
import com.zhiwei.qbjcutils.service.ClassifyUploadDataService;
import com.zhiwei.qbjcutils.utils.TimeUtils;
import com.zhiwei.qbjcutils.utils.Tools;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.springframework.stereotype.Service;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
/**
* Description:
*
* @author LiuMingHuan
* @classname ClassifyUploadDataServiceImpl
* @date 2020/4/2
*/
@Service
public class ClassifyUploadDataServiceImpl implements ClassifyUploadDataService {
private final EsDao esDao;
private final EsIndexTools esIndexTools;
public ClassifyUploadDataServiceImpl(EsDao esDao, EsIndexTools esIndexTools) {
this.esDao = esDao;
this.esIndexTools = esIndexTools;
}
@Override
public void classifyUploadData(String path, String project) {
try {
List<UploadData> markData = new ArrayList<>();
List<UploadData> dwData = new ArrayList<>();
List<UploadData> unchecked = new ArrayList<>();
UploadDataListener listener = new UploadDataListener();
try (InputStream in = new FileInputStream(path)) {
EasyExcel.read(in).sheet(0).registerReadListener(listener).head(UploadData.class).doRead();
} catch (Exception e) {
e.printStackTrace();
}
int i = 0;
for (UploadData info : listener.getInfos()) {
if (i >= 5000) {
break;
}
if (checkMarkExist(info, project)) {
markData.add(info);
} else if (checkDwExist(info)) {
dwData.add(info);
} else {
unchecked.add(info);
}
i++;
}
transportForm(Arrays.asList(markData, dwData, unchecked));
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* 标注库是否存在该条消息
* @param info
* @return
*/
private boolean checkMarkExist(UploadData info, String project) throws Exception {
if (Tools.isEmpty(info.getTitle()) || Tools.isEmpty(info.getSource()) || Tools.isEmpty(info.getUrl())) {
return false;
}
String[] markIndexes = esIndexTools.getMarkIndexs();
String[] types = new String[]{ClassB.TypeB.COMPLETE.name(), ClassB.TypeB.INCOMPLETE.name(), ClassB.TypeB.QA.name(), ClassB.TypeB.VIDEO.name()};
//优先指定url查询
BoolQueryBuilder urlQuery = QueryBuilders.boolQuery()
.must(QueryBuilders.boolQuery()
.should(QueryBuilders.termQuery("url", info.getUrl()))
.should(QueryBuilders.boolQuery()
.must(QueryBuilders.matchPhraseQuery("ind_full_text", info.getTitle()))
.must(QueryBuilders.termQuery("source", info.getSource()))))
.must(QueryBuilders.termQuery("mgroup", project));
SearchHits searchHits = esDao.searchFatchSource(markIndexes, types, null, urlQuery, null, 0, 10, new String[]{"id", "url"}, null, null);
if (searchHits.getTotalHits() >= 1) {
for (SearchHit hit : searchHits.getHits()) {
JSONObject source = JSONObject.parseObject(hit.getSourceAsString());
if (new URL(source.getString("url")).getHost().equals(new URL(info.getUrl()).getHost())) {
return true;
}
}
}
return false;
}
/**
* 舆情库是否存在该条消息
* @param info
* @return
*/
private boolean checkDwExist(UploadData info) throws Exception {
if(Tools.isEmpty(info.getTitle())|| Tools.isEmpty(info.getSource()) || Tools.isEmpty(info.getUrl())){
return false;
}
long lastMonth = TimeUtils.getFirstDayOfLastMonth(TimeUtils.TIME_FORMATE.parse(info.getTime()).getTime());
long nextMonth = TimeUtils.getFirstDayOfNextMonth(TimeUtils.TIME_FORMATE.parse(info.getTime()).getTime());
String[] indexes = esIndexTools.getAllIndex(lastMonth, nextMonth);
String[] types = new String[]{"status"};
//优先指定url查询
BoolQueryBuilder urlQuery = QueryBuilders.boolQuery()
.must(QueryBuilders.boolQuery()
.should(QueryBuilders.termQuery("url", info.getUrl()))
.should(QueryBuilders.boolQuery()
.must(QueryBuilders.matchPhraseQuery("ind_full_text", info.getTitle()))
.must(QueryBuilders.termQuery("source", info.getSource()))));
SearchHits searchHits = esDao.searchFatchSource(indexes, types, null, urlQuery, null, 0, 10, new String[]{"id", "url"}, null, null);
if (searchHits.getTotalHits() >= 1) {
for (SearchHit hit : searchHits.getHits()) {
JSONObject source = JSONObject.parseObject(hit.getSourceAsString());
if (new URL(source.getString("url")).getHost().equals(new URL(info.getUrl()).getHost())) {
return true;
}
}
}
return false;
}
/**
* 表格划分导出
* @param data
*/
private void transportForm(List<List<UploadData>> data) {
for (int i = 0; i < data.size(); i++) {
String filename = GenericAttribute.FILE_PATH + (i == 0 ? "mark" : (i == 1 ? "dw" : "unchecked"));
try (OutputStream out = new FileOutputStream(filename)) {
ExcelWriter writer = EasyExcel.write(out).needHead(true).head(UploadData.class).excelType(ExcelTypeEnum.XLSX).build();
WriteSheet writeSheet = EasyExcel.writerSheet(0, "总表").needHead(true).head(UploadData.class).build();
writer.write(data.get(i), writeSheet);
writer.finish();
out.flush();
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
src/main/java/com/zhiwei/qbjcutils/service/impl/DirectChannelServiceImpl.java 0 → 100644
package com.zhiwei.qbjcutils.service.impl;
import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.ExcelWriter;
import com.alibaba.excel.support.ExcelTypeEnum;
import com.alibaba.excel.write.metadata.WriteSheet;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.qbjcutils.config.GenericAttribute;
import com.zhiwei.qbjcutils.dao.DirectGroupDao;
import com.zhiwei.qbjcutils.dubbo.DubboClient;
import com.zhiwei.qbjcutils.pojo.entity.DirectGroup;
import com.zhiwei.qbjcutils.pojo.excelentity.DirectChannel;
import com.zhiwei.qbjcutils.pojo.excellistener.DirectChannelListener;
import com.zhiwei.qbjcutils.service.DirectChannelService;
import com.zhiwei.qbjcutils.utils.Tools;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/**
* Description:
*
* @author LiuMingHuan
* @classname DirectChannelServiceImpl
* @date 2020/4/16
*/
@Service
public class DirectChannelServiceImpl implements DirectChannelService {
private static final String SYSTEM_MANAGER = "系统管理员";
private final DubboClient client;
private final DirectGroupDao directGroupDao;
public DirectChannelServiceImpl(DubboClient client, DirectGroupDao directGroupDao) {
this.client = client;
this.directGroupDao = directGroupDao;
}
@Override
public void readAndAddDirectChannel(String project, String path) throws Exception {
operate(project, path, "insert");
}
@Override
public void read(String project, String path) throws Exception {
operate(project, path, "update");
}
private void operate(String project, String path,String type) throws Exception {
DirectChannelListener channelListener = new DirectChannelListener();
try (InputStream in = new FileInputStream(path)) {
EasyExcel.read(in).sheet(0).head(DirectChannel.class).registerReadListener(channelListener).doRead();
} catch (Exception e) {
e.printStackTrace();
}
List<DirectChannel> directChannels = channelListener.getDirectChannels();
Map<String, List<DirectChannel>> groups = directChannels.stream().collect(Collectors.groupingBy(DirectChannel::getDirectGroup));
List<DirectChannel> unMatched=new ArrayList<>();
for (Map.Entry<String, List<DirectChannel>> entry : groups.entrySet()) {
int groupLimit = 0;
DirectGroup group = "insert".equals(type) ? new DirectGroup(null, entry.getKey(), project, SYSTEM_MANAGER, System.currentTimeMillis(), true, true,
new ArrayList<>(), new ArrayList<>())
: directGroupDao.findFirst(new Query(Criteria.where("name").is(entry.getKey())));
for (DirectChannel directChannel : entry.getValue()) {
JSONObject res = client.searchCollectedAccountList(getMediaId(directChannel.getPlatform()), directChannel.getChannel(), true, 1, 1);
List<JSONObject> rows = res.getJSONArray("rows").toJavaList(JSONObject.class);
if (!Tools.isEmpty(rows)) {
JSONObject info = rows.get(0);
groupLimit = getDirectGroup(group, directChannel.getPlatform(), directChannel.getChannel(), info.getString("userId"), groupLimit);
} else {
unMatched.add(directChannel);
}
}
if("insert".equals(type)){
directGroupDao.insert(group);
}else {
directGroupDao.updateFirst(new Query(Criteria.where("_id").is(group.getId())), Tools.bean2Update(group));
}
}
getUnMatched(unMatched, project);
}
/**
* containsDirectMember
* @param mid
* @param directGroup
* @return
*/
public boolean containsDirectMember(String mid, DirectGroup directGroup) {
if (!Tools.isEmpty(directGroup.getMemberMaps())) {
List<String> mids = directGroup.getMemberMaps().stream()
.filter(a -> Boolean.parseBoolean(String.valueOf(a.get("used")))).map(a -> a.get("id") + "").collect(Collectors.toList());
return mids.contains(mid);
}
return false;
}
/**
* getDirectGroup
* @param directGroup
* @param platform
* @param name
* @param url
* @param groupLimit
* @return
* @throws Exception
*/
public int getDirectGroup(DirectGroup directGroup, String platform, String name, String url, int groupLimit) {
String mid = platform + "_" + name + "_" + url;
if (!containsDirectMember(mid, directGroup)) {
groupLimit++;
boolean isUsed = groupLimit <= 500;
HashMap<String, Object> memberHashMap = new HashMap<>(8);
memberHashMap.put("createAt", System.currentTimeMillis());
memberHashMap.put("id", mid);
memberHashMap.put("pt", platform);
memberHashMap.put("name", name);
memberHashMap.put("url", url);
memberHashMap.put("used", isUsed);
List<String> memberList = !Tools.isNull(directGroup.getMemberList()) ? directGroup.getMemberList()
: new ArrayList<>();
List<HashMap<String, Object>> existDirectGroupMapList = !Tools.isNull(directGroup.getMemberMaps())
? directGroup.getMemberMaps()
: new ArrayList<>();
if (isUsed) {
memberList.add(mid);
}
existDirectGroupMapList.add(memberHashMap);
directGroup.setMemberList(memberList);
directGroup.setMemberMaps(existDirectGroupMapList);
}
return groupLimit;
}
/**
*
* @param unMatched
*/
private void getUnMatched(List<DirectChannel> unMatched, String project) {
String fileName = project + "_" + System.currentTimeMillis() + "_UnMatched.xlsx";
try (OutputStream out = new FileOutputStream(GenericAttribute.FILE_PATH + fileName)) {
ExcelWriter writer = EasyExcel.write(out).needHead(true).head(DirectChannel.class).excelType(ExcelTypeEnum.XLSX).build();
WriteSheet writeSheet = EasyExcel.writerSheet(0, "总表").head(DirectChannel.class).build();
writer.write(unMatched, writeSheet);
writer.finish();
out.flush();
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* 获取mediaId
* @param pt
* @return
*/
private int getMediaId(String pt) {
int mediaId = 0;
switch (pt) {
case "微博":
mediaId = 4;
break;
case "微信":
mediaId = 3;
break;
case "网媒":
mediaId = 1;
break;
case "今日头条":
mediaId = 5;
break;
case "平媒":
mediaId = 2;
break;
case "抖音":
mediaId = 36;
break;
default:
break;
}
return mediaId;
}
}
src/main/java/com/zhiwei/qbjcutils/service/impl/MarkDataContServiceImpl.java 0 → 100644
package com.zhiwei.qbjcutils.service.impl;
import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.ExcelWriter;
import com.alibaba.excel.support.ExcelTypeEnum;
import com.alibaba.excel.write.metadata.WriteSheet;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.base.category.ClassB;
import com.zhiwei.qbjcutils.config.GenericAttribute;
import com.zhiwei.qbjcutils.es.EsIndexTools;
import com.zhiwei.qbjcutils.es.dao.EsDao;
import com.zhiwei.qbjcutils.pojo.emums.TimeUnit;
import com.zhiwei.qbjcutils.pojo.entity.MessagePlatform;
import com.zhiwei.qbjcutils.pojo.excelhadler.PlatformCountHandler;
import com.zhiwei.qbjcutils.service.MarkDataCountService;
import com.zhiwei.qbjcutils.service.PlatformService;
import com.zhiwei.qbjcutils.utils.TimeUtils;
import com.zhiwei.qbjcutils.utils.Tools;
import org.apache.commons.lang3.time.FastDateFormat;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.springframework.stereotype.Service;
import java.io.FileOutputStream;
import java.io.OutputStream;
import java.util.*;
import java.util.stream.Collectors;
/**
* Description:
*
* @author LiuMingHuan
* @classname MarkDataContServiceImpl
* @date 2020/8/18
*/
@Service
public class MarkDataContServiceImpl implements MarkDataCountService {
private final EsDao esDao;
private final EsIndexTools esIndexTools;
private final PlatformService platformService;
public MarkDataContServiceImpl(EsDao esDao, EsIndexTools esIndexTools, PlatformService platformService) {
this.esDao = esDao;
this.esIndexTools = esIndexTools;
this.platformService = platformService;
}
@Override
public void outputMarkDataCountByPlatforms(long startTime, long endTime, String timeType, String timeUnit, String project) {
List<List<Object>> data = divideTime(startTime, endTime, timeType, timeUnit, project);
String fileName = TimeUtils.DATE_FORMATE.format(startTime) + "-" + TimeUtils.DATE_FORMATE.format(endTime) + "-" + timeUnit + ".xlsx";
List<List<String>> header = platformService.getPlatforms().stream().map(MessagePlatform::getName).map(Arrays::asList).collect(Collectors.toList());
header.add(0, Collections.singletonList("时间"));
header.add(Collections.singletonList("未知"));
try (OutputStream out = new FileOutputStream(GenericAttribute.FILE_PATH + fileName)) {
ExcelWriter writer = EasyExcel.write(out).registerWriteHandler(new PlatformCountHandler()).excelType(ExcelTypeEnum.XLSX).build();
WriteSheet sheet = EasyExcel.writerSheet(0, "总表").needHead(Boolean.TRUE).head(header).build();
writer.write(data, sheet);
writer.finish();
out.flush();
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* 时间划分
* @param startTime
* @param endTime
* @param timeUnit
* @return
*/
private List<List<Object>> divideTime(long startTime, long endTime, String timeType, String timeUnit, String project) {
Map<String, Map<String, Long>> result = new LinkedHashMap<>(4);
List<List<Object>> data = new ArrayList<>();
List<String> header = platformService.getPlatforms().stream().map(MessagePlatform::getName).collect(Collectors.toList());
header.add("未知");
FastDateFormat format = getFormat(timeUnit);
for (long currentTime = startTime; currentTime < endTime; currentTime += TimeUnit.currentTimeMillSeconds(timeUnit, currentTime)) {
long currentEnd = Math.min(currentTime + TimeUnit.currentTimeMillSeconds(timeUnit, currentTime), endTime);
Map<String, Long> platformCount = dividePlatform(currentTime, currentEnd, timeType, project);
result.put(format.format(currentTime) , platformCount);
}
for (Map.Entry<String, Map<String, Long>> entry : result.entrySet()) {
List<Object> list = new ArrayList<>();
Map<String, Long> platformCount = entry.getValue();
list.add(entry.getKey());
for (String platform : header) {
list.add(platformCount.containsKey(platform) ? platformCount.get(platform) : "");
}
data.add(list);
}
return data;
}
/**
* getFormat
* @param timeUnit
*/
private FastDateFormat getFormat(String timeUnit) {
switch (timeUnit){
case "SECOND":
return TimeUtils.MINUTE_FORMATE;
case "HOUR":
return TimeUtils.HOUR_FORMATE;
case "DAY":
default:
return TimeUtils.DATE_FORMATE;
}
}
/**
* TODO （平台策略更新后需调整）
* dividePlatform
* @param currentTime
* @param currentEnd
* @param timeType
* @param project
* @return
*/
private Map<String, Long> dividePlatform(long currentTime, long currentEnd, String timeType, String project) {
Map<String, Long> result = new HashMap<>(4);
String[] indexes = esIndexTools.getMarkIndexs();
String[] types = new String[]{ClassB.TypeB.COMPLETE.name(), ClassB.TypeB.INCOMPLETE.name(), ClassB.TypeB.QA.name(), ClassB.TypeB.VIDEO.name()};
String[] includes = new String[]{GenericAttribute.ES_C5, GenericAttribute.ES_ORIGIN, GenericAttribute.ES_FOREIGN};
BoolQueryBuilder query = QueryBuilders.boolQuery()
.must(QueryBuilders.termQuery(GenericAttribute.ES_MGROUP, project))
.must(QueryBuilders.rangeQuery(timeType).gte(currentTime).lte(currentEnd));
SearchHits searchHits = esDao.searchFatchSource(indexes, types, null, query, null, 0, GenericAttribute.SEARCH_TOTAL_LIMIT, includes, null, null);
if (searchHits.getTotalHits() > GenericAttribute.SEARCH_TOTAL_LIMIT) {
int totalPage = Tools.totalPage(searchHits.getTotalHits(), 1000);
SearchResponse searchResponse = esDao.scroll(indexes, types, null, query, null, 0, 1000, null, TimeValue.timeValueMinutes(2));
for (int i = 0; i < totalPage; i++) {
SearchHit[] hits = searchResponse.getHits().getHits();
getHits(result, hits);
searchResponse = esDao.scroll(searchResponse.getScrollId(), TimeValue.timeValueMinutes(2));
}
return result;
} else if (searchHits.getTotalHits() > 0) {
getHits(result, searchHits.getHits());
}
return result;
}
/**
* getHits
* @param result
* @param hits
*/
private void getHits(Map<String, Long> result, SearchHit[] hits) {
Arrays.stream(hits).map(SearchHit::getSourceAsString).collect(Collectors.groupingBy(this::searchHit2Platform))
.forEach((m, n) -> {
long count = result.containsKey(m) ? result.get(m) : 0;
count += n.size();
result.put(m, count);
});
}
/**
* searchHit2Platform
* @param sourceAsString
* @return
*/
private String searchHit2Platform(String sourceAsString) {
JSONObject jsonObject = JSONObject.parseObject(sourceAsString);
MessagePlatform messagePlatform = platformService.getPlatform(jsonObject.getIntValue(GenericAttribute.ES_C5), jsonObject.getString(GenericAttribute.ES_ORIGIN), jsonObject.getIntValue(GenericAttribute.ES_FOREIGN));
return !Tools.isEmpty(messagePlatform) ? messagePlatform.getName() : "未知";
}
}
src/main/java/com/zhiwei/qbjcutils/service/impl/MarkDataOutputServiceImpl.java 0 → 100644
package com.zhiwei.qbjcutils.service.impl;
import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.ExcelWriter;
import com.alibaba.excel.support.ExcelTypeEnum;
import com.alibaba.excel.write.metadata.WriteSheet;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.base.category.ClassB;
import com.zhiwei.qbjcutils.config.GenericAttribute;
import com.zhiwei.qbjcutils.es.EsIndexTools;
import com.zhiwei.qbjcutils.es.dao.EsDao;
import com.zhiwei.qbjcutils.pojo.emums.TimeUnit;
import com.zhiwei.qbjcutils.pojo.entity.MessagePlatform;
import com.zhiwei.qbjcutils.pojo.excelhadler.PlatformCountHandler;
import com.zhiwei.qbjcutils.service.MarkDataOutputService;
import com.zhiwei.qbjcutils.service.PlatformService;
import com.zhiwei.qbjcutils.utils.TimeUtils;
import com.zhiwei.qbjcutils.utils.Tools;
import org.apache.commons.lang3.time.FastDateFormat;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.springframework.stereotype.Service;
import java.io.FileOutputStream;
import java.io.OutputStream;
import java.util.*;
import java.util.stream.Collectors;
/**
* Description:
*
* @author LiuMingHuan
* @classname MarkDataCountOutputServiceImpl
* @date 2020/3/31
*/
@Service
public class MarkDataOutputServiceImpl implements MarkDataOutputService {
}
src/main/java/com/zhiwei/qbjcutils/task/AsyncTask.java 0 → 100644
package com.zhiwei.qbjcutils.task;
import com.zhiwei.qbjcutils.redis.RedisTools;
import org.springframework.stereotype.Component;
/**
* Description:
*
* @author LiuMingHuan
* @classname AsyncTask
* @date 2020/4/3
*/
@Component
public class AsyncTask {
private final RedisTools redisTools;
public AsyncTask(RedisTools redisTools) {
this.redisTools = redisTools;
}
public void initMessagePlatforms(){
// GenericAttribute.
// redisTools.getRMap()
}
}
src/main/java/com/zhiwei/qbjcutils/task/InitTask.java 0 → 100644
package com.zhiwei.qbjcutils.task;
import org.springframework.stereotype.Component;
/**
* Description:
*
* @author LiuMingHuan
* @classname initTask
* @date 2020/4/3
*/
@Component
public class InitTask {
public void init() {
}
}
src/main/java/com/zhiwei/qbjcutils/utils/QYWeChatSendUtils.java 0 → 100644
package com.zhiwei.qbjcutils.utils;
import com.zhiwei.qbjcutils.config.GenericAttribute;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/**
* Description:
*
* @author LiuMingHuan
* @classname QYWeChatSendUtils
* @date 2019/12/27
*/
public class QYWeChatSendUtils {
private static final Logger log = LogManager.getLogger(QYWeChatSendUtils.class);
private static long LAST_WARN;
private static final long WARN_LIMIT = 1L;
/**
* 发送预警消息
*
* @param s
*/
public static void sendWarnMessage(String s) {
try {
if ((LAST_WARN + WARN_LIMIT) < System.currentTimeMillis()) {
LAST_WARN = System.currentTimeMillis();
StringBuilder msgBuilder = new StringBuilder();
msgBuilder.append("{\"touser\": \"YangYongDeYu\",\"msgtype\": \"text\",\"text\": {\"content\": \"");
msgBuilder.append(s);
msgBuilder.append("\"},\"safe\": 0}");
log.info("预警发送结果：" + Tools.httpPostRequestBody(GenericAttribute.WARN_URL, msgBuilder.toString()));
}
} catch (Exception e) {
log.error("预警发送失败", e);
}
}
}
src/main/java/com/zhiwei/qbjcutils/utils/TimeUtils.java 0 → 100644
package com.zhiwei.qbjcutils.utils;
import org.apache.commons.lang3.time.FastDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.List;
/**
* @ClassName: TimeUtil
* @Description: 时间工具类
* @author shentao
* @date 2019年7月9日 上午9:31:37
*/
public class TimeUtils {
/** day **/
public static final String TIMETYPE_DAY = "day";
/** yyyy FastDateFormat **/
public static final FastDateFormat YEAR_FORMAT = FastDateFormat.getInstance("yyyy");
/** MM FastDateFormat **/
public static final FastDateFormat MONTH_FORMAT = FastDateFormat.getInstance("MM");
/** dd FastDateFormat **/
public static final FastDateFormat DAY_FORMAT = FastDateFormat.getInstance("dd");
/** yyyy-MM-dd HH:mm:ss FastDateFormat **/
public static final FastDateFormat TIME_FORMATE = FastDateFormat.getInstance("yyyy-MM-dd HH:mm:ss");
/** yyyy-MM-dd HH:mm FastDateFormat **/
public static final FastDateFormat MINUTE_FORMATE = FastDateFormat.getInstance("yyyy-MM-dd HH:mm");
/** yyyy-MM-dd HH FastDateFormat **/
public static final FastDateFormat HOUR_FORMATE = FastDateFormat.getInstance("yyyy-MM-dd HH");
/** yyyy-MM-dd FastDateFormat **/
public static final FastDateFormat DATE_FORMATE = FastDateFormat.getInstance("yyyy-MM-dd");
/** file yyyyMMddHHmmss FastDateFormat **/
public static final FastDateFormat FILE_FORMAT = FastDateFormat.getInstance("yyyyMMddHHmmss");
/** file yyyyMMdd FastDateFormat **/
public static final FastDateFormat FILE_DATE_FORMAT = FastDateFormat.getInstance("yyyyMMdd");
/**
* 按年月获取 当前月份第一天
*
* @param year
* @param month
* @return long
*/
public static long getStartTimeByYearAndMonth(int year, int month) {
Calendar calendar = Calendar.getInstance();
calendar.clear();
calendar.set(Calendar.YEAR, year);
calendar.set(Calendar.MONTH, month - 1);
calendar.set(Calendar.DAY_OF_MONTH, 1);
return calendar.getTimeInMillis();
}
/**
* 按年月获取 当前月份最后一天
*
* @param year
* @param month
* @return long
*/
public static long getEndTimeByYearAndMonth(int year, int month) {
Calendar calendar = Calendar.getInstance();
calendar.clear();
calendar.set(Calendar.YEAR, year);
calendar.set(Calendar.MONTH, month - 1);
calendar.set(Calendar.DAY_OF_MONTH, calendar.getActualMaximum(Calendar.DAY_OF_MONTH));
return calendar.getTimeInMillis();
}
/**
* 取时间点 day hour两钟
*
* @param start
* @param end
* @param timeType
* @return
* @return List<String>
*/
public static List<String> getTimePointToNowOrEnd(long start, long end, String timeType) {
Calendar car = Calendar.getInstance();
car.setTimeInMillis(start);
boolean flag = true;
// 为完善表现层，填充无数据时间点
// 创建完整时间点
List<String> times = new ArrayList<>();
if (TIMETYPE_DAY.equals(timeType)) {
while (flag) {
times.add(DATE_FORMATE.format(car.getTime()));
if (end <= car.getTimeInMillis()) {
flag = false;
}
car.add(Calendar.DAY_OF_MONTH, 1);
}
} else {
while (flag) {
times.add(HOUR_FORMATE.format(car.getTime()));
if (end <= car.getTimeInMillis()) {
flag = false;
}
car.add(Calendar.HOUR, 1);
}
}
return times;
}
/**
* 获取上个月第一天
* @param time
* @return
*/
public static long getFirstDayOfLastMonth(long time) {
Calendar calendar = Calendar.getInstance();
calendar.setTimeInMillis(time);
calendar.add(Calendar.MONTH, -1);
calendar.set(Calendar.DAY_OF_MONTH, 1);
return calendar.getTimeInMillis();
}
/**
* 获取下个月第一天
* @param time
* @return
*/
public static long getFirstDayOfNextMonth(long time) {
Calendar calendar = Calendar.getInstance();
calendar.setTimeInMillis(time);
calendar.add(Calendar.MONTH, 1);
calendar.set(Calendar.DAY_OF_MONTH, 1);
return calendar.getTimeInMillis();
}
}
src/main/resources/application-dev.properties 0 → 100644
#primary-mongo
primary.uri=mongodb://qbjc:asSADf5ffs@115.236.59.88:30001/qbjc?authSource=admin
#second-mongo
secondary.uri=mongodb://192.168.0.246:27017/channel-platform
#mongo 连接池配置
mongo.connectionsPerHost=50
mongo.threadsAllowedToBlockForConnectionMultiplier=10
mongo.connectTimeout=30000
mongo.maxWaitTime=50000
mongo.autoConnectRetry=true
mongo.socketKeepAlive=true
mongo.socketTimeout=120000
mongo.slaveOk=true
#es
es.httpHost=9400
es.httpIp=202.107.192.94
es.clusterNodes=202.107.192.94:9500
es.clusterName=zhiweidata-es
es.transportSniff=false
#path to redisson.yaml or redisson-local.json
spring.redis.redisson.config=classpath:redisson-dev.json
#zookeeper
zookeeper.registry=zookeeper://192.168.0.11:2181?backup=192.168.0.30:2181,192.168.0.35:2181
channel-index.zookeeper.registry=zookeeper://192.168.0.11:2182?backup=192.168.0.30:2182,192.168.0.35:2182
#dubbo
dubbo.mark.group=local-marker
dubbo.filter.group=local-filter
dubbo.automark.group=automatic-test_sjj
dubbo.channel.group=channel-index-local
#baiduMap
baiduMap.ak=2rkFM5ofHuhLvfQnEuuv5GkTXnSPEdiV
src/main/resources/application-prod.properties 0 → 100644
#primary-mongo
primary.uri=mongodb://qbjc:asSADf5ffs@192.168.0.101:30000/qbjc?authSource=admin
#second-mongo
secondary.uri=secondary.uri=mongodb://qbjcuser:qbjc1q2w3e4r@202.107.192.94:30000/eventMuseum?authSource=admin
#mongo 连接池配置
mongo.connectionsPerHost=200
mongo.threadsAllowedToBlockForConnectionMultiplier=10
mongo.connectTimeout=30000
mongo.maxWaitTime=50000
mongo.autoConnectRetry=true
mongo.socketKeepAlive=true
mongo.socketTimeout=120000
mongo.slaveOk=true
#es
es.httpHost=9200
es.httpIp=192.168.0.25
es.clusterNodes=192.168.0.25:9300,192.168.0.26:9300,192.168.0.27:9300
es.clusterName=zhiweidata-es
es.transportSniff=true
#path to redisson.yaml or redisson-local.json
spring.redis.redisson.config=classpath:redisson-prod.json
#zookeeper
zookeeper.registry=zookeeper://192.168.0.11:2181?backup=192.168.0.30:2181,192.168.0.35:2181
channel-index.zookeeper.registry=zookeeper://192.168.0.11:2182?backup=192.168.0.30:2182,192.168.0.35:2182
#dubbo
dubbo.mark.group=local-marker
dubbo.filter.group=local-filter
dubbo.automark.group=automatic-test_sjj
dubbo.channel.group=channel-index-local
\ No newline at end of file
src/main/resources/application.properties 0 → 100644
server.port=5788
spring.profiles.active=prod
\ No newline at end of file
src/main/resources/log4j2.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE xml>
<!-- log4j2 自身的日志级别 -->
<Configuration status="WARN">
<Properties> <!-- 配置日志文件输出目录 -->
<Property name="LOG_HOME">./log/</Property>
<property name="APP_NAME">qbjc-back</property>
</Properties>
<Appenders>
<!-- 定义日志输出地 -->
<Console name="Console" target="SYSTEM_OUT">
<PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level %logger{36} %L [%t] - %msg%n"/>
</Console>
<!--[%t]-->
<!-- INFO级别日志 -->
<RollingFile name="info_appender" fileName="${LOG_HOME}/${APP_NAME}.log"
filePattern="${LOG_HOME}/${APP_NAME}-%d{yyyy-MM-dd}-%i.log">
<Filters>     
<PushLogFilter onMatch="NEUTRAL" onMismatch="DENY"/>   
</Filters>
<PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level %logger{36} %L [%t] - %msg%n"/>
<Policies>
<TimeBasedTriggeringPolicy/>
<SizeBasedTriggeringPolicy size="20 MB"/>
</Policies>
</RollingFile>
</Appenders>
<Loggers>
<AsyncRoot level="info">
<AppenderRef ref="Console"/>
<AppenderRef ref="info_appender"/>
</AsyncRoot>
<!--<AsyncLogger level="info">-->
<!--<AppenderRef ref="Console"/>-->
<!--<AppenderRef ref="info_appender"/>-->
<!--</AsyncLogger>-->
<!-- 所有的 logger 均继承 Root
当 additivity 为 true 时, 父子 logger 均会打印
当 additivity 为 false 时, 仅子 logger 会打印 -->
<Logger name="mylog" level="error" additivity="false">
<AppenderRef ref="Console"/>
</Logger>
<Logger name="com.alibaba.dubbo.monitor.dubbo.DubboMonitor" level="OFF" >
</Logger>
<Logger name="org.apache.dubbo.monitor.dubbo.DubboMonitor" level="OFF" >
</Logger>
</Loggers>
</Configuration>
src/main/resources/pushlog.properties 0 → 100644
robot.push.address=https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=e2218c6e-af6a-4296-9d75-7178b941a3b5
robot.push.enable=true
robot.push.filterclass=org.apache.dubbo.common.Version,com.alibaba.dubbo.common.Version,org.apache.dubbo.monitor.dubbo.DubboMonitor,com.alibaba.dubbo.monitor.dubbo.DubboMonitor
robot.push.level=warn
\ No newline at end of file
src/main/resources/redisson-dev.json 0 → 100644
{
"singleServerConfig": {
"idleConnectionTimeout": 60000,
"pingTimeout": 1000,
"connectTimeout": 10000,
"timeout": 100000,
"retryAttempts": 3,
"retryInterval": 1500,
"address": "redis://115.236.59.91:7373",
"connectionMinimumIdleSize": 32,
"connectionPoolSize": 64,
"database": 4
},
"codec": {
"class": "org.redisson.client.codec.StringCodec"
},
"transportMode": "NIO"
}
\ No newline at end of file
src/main/resources/redisson-local.json 0 → 100644
{
"singleServerConfig": {
"idleConnectionTimeout": 60000,
"pingTimeout": 1000,
"connectTimeout": 10000,
"timeout": 3000,
"retryAttempts": 3,
"retryInterval": 1500,
"address": "redis://192.168.0.41:6379",
"connectionMinimumIdleSize": 64,
"connectionPoolSize": 128,
"database": 12
},
"codec": {
"class": "org.redisson.client.codec.ByteArrayCodec"
},
"transportMode": "NIO"
}
\ No newline at end of file
src/main/resources/redisson-prod.json 0 → 100644
{
"singleServerConfig": {
"idleConnectionTimeout": 60000,
"pingTimeout": 1000,
"connectTimeout": 10000,
"timeout": 3000,
"retryAttempts": 3,
"retryInterval": 1500,
"address": "redis://192.168.0.2:7373",
"connectionMinimumIdleSize": 128,
"connectionPoolSize": 256,
"database": 4
},
"codec": {
"class": "org.redisson.client.codec.StringCodec"
},
"transportMode": "NIO"
}
\ No newline at end of file
src/test/java/com/zhiwei/qbjcutils/Tests.java 0 → 100644
package com.zhiwei.qbjcutils;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.qbjcutils.pojo.excelentity.SentimentInfo;
import com.zhiwei.qbjcutils.utils.Tools;
import org.apache.logging.log4j.Level;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.junit.Test;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.text.NumberFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/**
* Description:
*
* @author LiuMingHuan
* @classname Test
* @date 2020/5/18
*/
public class Tests {
private final static String GEO_CODING = "http://api.map.baidu.com/geocoding/v3/?&output=json&ak=";
private final static String DRIVING = "http://api.map.baidu.com/routematrix/v2/driving?output=json&tactics=13&ak=2rkFM5ofHuhLvfQnEuuv5GkTXnSPEdiV&origins=&destinations=";
@Test
public void run(){
String laiLngUrl = new StringBuilder(GEO_CODING).append("2rkFM5ofHuhLvfQnEuuv5GkTXnSPEdiV").append("&address=").append("广州").toString();
JSONObject latLngInfo = JSONObject.parseObject(Tools.httpGet(laiLngUrl));
System.out.println(latLngInfo.toJSONString());
}
@Test
public void aopTest() {
List<String> excelNames = Arrays.asList("地摊经济引关注", "广州市方圆实验小学教师涉嫌体罚学生", "十三届全国人大三次会议闭幕", "微软中国被列为被执行人", "中共中央、国务院印发《海南自由贸易港建设总体方案》");
for (String excelName : excelNames) {
System.out.println(excelName + "分析：");
String predictPath = "C:\\Users\\pread\\Desktop\\新建文件夹\\predict\\";
String zhiweiPath = "C:\\Users\\pread\\Desktop\\新建文件夹\\result\\";
String baiduPath = "C:\\Users\\pread\\Desktop\\新建文件夹\\baidu\\";
Map<String, SentimentInfo> local = readExcel(new File(zhiweiPath + excelName + ".xlsx")).stream().collect(Collectors.toMap(SentimentInfo::getId, m -> m));
Map<String, SentimentInfo> baidu = readExcel(new File(baiduPath + excelName + ".xlsx")).stream().collect(Collectors.toMap(SentimentInfo::getId, m -> m));
Map<String, SentimentInfo> module = readExcel(new File(predictPath + excelName + ".xlsx")).stream().collect(Collectors.toMap(SentimentInfo::getId, m -> m));
System.out.println(str(excelName, permunate(local, baidu), "本地", "百度"));
System.out.println(str(excelName, permunate(local, module), "本地", "模型"));
System.out.println(str(excelName, permunate(module, baidu), "模型", "百度"));
System.out.println("--------------------");
}
}
public List<SentimentInfo> readExcel(File file) {
try {
//create input stream ,get data
InputStream stream = new FileInputStream(file);
Workbook wb = new XSSFWorkbook(stream);
int sheetSize = wb.getNumberOfSheets();
List<SentimentInfo> sheetList = new ArrayList<>();
for (int i = 0; i < sheetSize; i++) {
List<SentimentInfo> outerList = new ArrayList<>();
Sheet sheet = wb.getSheetAt(i);
column:for (int j = 1; j <= sheet.getLastRowNum(); j++) {
SentimentInfo sentimentInfo = new SentimentInfo();
sentimentInfo.setId(sheet.getRow(j).getCell(0).getStringCellValue());
for (int c = 0; c < sheet.getRow(j).getLastCellNum(); c++) {
Cell cell = sheet.getRow(j).getCell(c);
String head = sheet.getRow(0).getCell(c, Row.MissingCellPolicy.CREATE_NULL_AS_BLANK).toString();
if("分类结果".equals(head)){
Double numericCellValue = cell.getNumericCellValue();
sentimentInfo.setSentiment(switchSentiment(numericCellValue.intValue()));
break;
}else if("".equals(head)){
String str = cell.getStringCellValue();
sentimentInfo.setSentiment(str);
break;
}
}
outerList.add(sentimentInfo);
}
sheetList.addAll(outerList);
}
return sheetList;
} catch (Exception e) {
e.printStackTrace();
}
return null;
}
public String switchSentiment(int sentiment){
switch (sentiment){
case 0:
return "负面";
case 1:
return "中性";
case 2:
return "正面";
default:
return "";
}
}
public JSONObject permunate(Map<String, SentimentInfo> matched, Map<String, SentimentInfo> bematched){
JSONObject jsonObject = new JSONObject();
int count =0;
int matchedCount = 0;
for (Map.Entry<String, SentimentInfo> entry : matched.entrySet()) {
SentimentInfo value = entry.getValue();
SentimentInfo sentimentInfo = bematched.get(entry.getKey());
if(!Tools.isNull(sentimentInfo)){
count++;
if(value.getSentiment().equals(sentimentInfo.getSentiment())){
matchedCount++;
}
}
}
jsonObject.put("matchCount", matchedCount);
jsonObject.put("unMatchCount", count - matchedCount);
jsonObject.put("hasId", count);
jsonObject.put("matchMapSize", matched.size());
jsonObject.put("beMatchMapSize", bematched.size());
return jsonObject;
}
public String str(String fileName, JSONObject jsonObject, String matched, String bematched){
StringBuilder stringBuilder = new StringBuilder(fileName);
float v = (float) jsonObject.getIntValue("matchCount") / jsonObject.getInteger("hasId");
float v1 = (float) jsonObject.getIntValue("unMatchCount") / jsonObject.getInteger("hasId");
NumberFormat instance = NumberFormat.getInstance();
instance.setMaximumFractionDigits(2);
stringBuilder.append(" ").append(matched).append("-").append(bematched).append(" 判断一致: ").append(instance.format(v)).append(" 判断不一致：").append(instance.format(v1));
return stringBuilder.toString();
}
@Test
public void normalTest(){
System.out.println(Arrays.toString(Level.values()));
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment