Skip to content

Q
qbjc-utils
Commit 427d91bb by liuminghuan
Options

2020/08/21 初版提交

parents
Showing with 7024 additions and 0 deletions
.gitignore 0 → 100644
HELP.md
/target/
!.mvn/wrapper/maven-wrapper.jar
.mvn/
mvnw
mvnw.cmd
# maven ignore
target/
*.jar
*.war
*.zip
*.tar
*.tar.gz
dependency-reduced-pom.xml
# eclipse ignore
.settings/
.project
.classpath
# idea ignore
.idea/
*.ipr
*.iml
*.iws
# git ignore
*/.gitignore
# temp ignore
*.log
*.cache
*.diff
*.patch
*.tmp
# test ignore
test-output/
# system ignore
.DS_Store
Thumbs.db
# directory ignore
log/
backup/
*/src/test/
# jekins ignore
Jenkinsfile
pom.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.zhiwei</groupId>
<artifactId>qbjc-utils</artifactId>
<version>1.0.0-SNAPSHOT</version>
<name>qbjc-utils</name>
<description>utils for qbjc system</description>
<properties>
<java.version>1.8</java.version>
<log4j.version>2.11.2</log4j.version>
<slf4j.version>1.8.0-beta4</slf4j.version>
<disruptor.version>3.4.2</disruptor.version>
<base-objects.version>1.2.8-SNAPSHOT</base-objects.version>
<es.version>5.5.2</es.version>
<es-client.version>1.9</es-client.version>
<springboot.version>2.1.6.RELEASE</springboot.version>
<lombok.version>1.18.12</lombok.version>
<redisson.version>3.10.7</redisson.version>
<json.version>1.2.47</json.version>
<commons-lang3.version>3.9</commons-lang3.version>
<commons-collections4.version>4.3</commons-collections4.version>
<easy-excel.version>2.1.0-beta4</easy-excel.version>
<poi.version>4.0.1</poi.version>
<dubbo.version>2.7.4.1</dubbo.version>
<marker-client.version>1.1.0-SNAPSHOT</marker-client.version>
<cleaner-unified-filter.version>1.1.4.RELEASE</cleaner-unified-filter.version>
<automaticmark.version>2.1.0-SNAPSHOT</automaticmark.version>
<channel-index.version>0.4.2-SNAPSHOT</channel-index.version>
<crawler.version>0.6.0.1-RELEASE</crawler.version>
</properties>
<dependencies>
<dependency>
<groupId>com.zhiwei</groupId>
<artifactId>push-log</artifactId>
<version>2.11.2</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<version>${springboot.version}</version>
<exclusions><!-- 去掉默认配置 -->
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-logging</artifactId>
</exclusion>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
</exclusion>
</exclusions>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-mongodb</artifactId>
<version>${springboot.version}</version>
<exclusions>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-logging</artifactId>
</exclusion>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-aop</artifactId>
<version>${springboot.version}</version>
<exclusions>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-logging</artifactId>
</exclusion>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- https://mvnrepository.com/artifact/org.redisson/redisson-spring-boot-starter -->
<dependency>
<groupId>org.redisson</groupId>
<artifactId>redisson-spring-boot-starter</artifactId>
<version>${redisson.version}</version>
<exclusions>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
</exclusion>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- base-object -->
<dependency>
<groupId>com.zhiwei.base</groupId>
<artifactId>base-objects</artifactId>
<version>${base-objects.version}</version>
</dependency>
<dependency>
<groupId>com.lmax</groupId>
<artifactId>disruptor</artifactId>
<version>${disruptor.version}</version>
</dependency>
<!-- elasticsearch.client -->
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>transport</artifactId>
<version>${es.version}</version>
</dependency>
<dependency>
<groupId>com.zhiwei</groupId>
<artifactId>es-client</artifactId>
<version>${es-client.version}</version>
</dependency>
<!--lombok-->
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>${lombok.version}</version>
</dependency>
<!-- fastjson -->
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>${json.version}</version>
</dependency>
<!-- apache 常用工具类 -->
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>${commons-lang3.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-collections4 -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-collections4</artifactId>
<version>${commons-collections4.version}</version>
</dependency>
<!-- Excel -->
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>easyexcel</artifactId>
<version>${easy-excel.version}</version>
</dependency>
<!-- poi -->
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi</artifactId>
<version>${poi.version}</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml</artifactId>
<version>${poi.version}</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-scratchpad</artifactId>
<version>${poi.version}</version>
</dependency>
<!--dubbo 导入 -->
<dependency>
<groupId>org.apache.dubbo</groupId>
<artifactId>dubbo</artifactId>
<version>${dubbo.version}</version>
<scope>provided</scope>
<exclusions>
<exclusion>
<groupId>org.springframework</groupId>
<artifactId>spring-context</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- 去重 -->
<dependency>
<groupId>com.zhiwei.middleware</groupId>
<artifactId>cleaner-unified-filter</artifactId>
<version>${cleaner-unified-filter.version}</version>
</dependency>
<!-- 自动 -->
<dependency>
<groupId>com.zhiwei.middleware</groupId>
<artifactId>automaticmark-client</artifactId>
<version>${automaticmark.version}</version>
</dependency>
<!--channel-index-->
<dependency>
<groupId>com.zhiwei</groupId>
<artifactId>channel-index-client</artifactId>
<version>${channel-index.version}</version>
</dependency>
<dependency>
<groupId>com.zhiwei.crawler</groupId>
<artifactId>crawler-core</artifactId>
<version>${crawler.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.testng/testng -->
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
<version>7.1.0</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<version>${springboot.version}</version>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
\ No newline at end of file
src/main/java/com/zhiwei/qbjcutils/QbjcUtilsApplication.java 0 → 100644
package com.zhiwei.qbjcutils;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration;
import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration;
/**
* @author lmh
*/
@SpringBootApplication(exclude = { MongoAutoConfiguration.class, MongoDataAutoConfiguration.class })
public class QbjcUtilsApplication {
public static void main(String[] args) {
SpringApplication.run(QbjcUtilsApplication.class, args);
}
}
src/main/java/com/zhiwei/qbjcutils/config/EsProperties.java 0 → 100644
package com.zhiwei.qbjcutils.config;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
/**
* @Description:
* @Author: shentao
* @Date: 2020/4/27 13:47
*/
@Component
@Data
@ConfigurationProperties(prefix = "es")
public class EsProperties {
/**
* httpHost httpClient端口
*/
private int httpHost;
/**
* httpIp httpClientIp
*/
private String httpIp;
/**
* clusterName 集群名
*/
private String clusterName;
/**
* 自动嗅探
*/
private boolean transportSniff;
/**
* 集群节点s
*/
private String clusterNodes;
}
src/main/java/com/zhiwei/qbjcutils/config/GenericAttribute.java 0 → 100644
package com.zhiwei.qbjcutils.config;
/**
* Description:
*
* @author LiuMingHuan
* @classname GenericAttribute
* @date 2020/4/2
*/
public class GenericAttribute {
/**文件地址**/
public static final String FILE_PATH = "D:\\stWork\\download\\excel\\";
/**时间常量**/
public static final long AN_HOUR = 60 * 60 * 1000L;
/**es字段**/
public static final String ES_ORIGIN = "origin";
public static final String ES_FOREIGN = "foreign";
public static final String ES_C5 = "c5";
public static final String ES_IND_FULL_TEXT = "ind_full_text";
public static final String ES_IND_TITLE = "ind_title";
public static final String ES_SOURCE = "source";
public static final String ES_MGROUP = "mgroup";
public static final String ES_MTIME = "mtime";
public static final String ES_MTAG = "mtag";
public static final String ES_MUPDATE = "mupdate";
public static final String ES_MUPDATE_TWO = "mupdate_two";
public static final int SEARCH_TOTAL_LIMIT =10000 ;
/** redisKey **/
public static final String SYSTEM_KEY="YUQING:System:";
/** URL **/
public static final String WARN_URL = "https://login.zhiweidata.com/wechatwork/sendMessage";
}
src/main/java/com/zhiwei/qbjcutils/config/MiddlewareConfig.java 0 → 100644
package com.zhiwei.qbjcutils.config;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
/**
* @ClassName: MiddlewareConfig
* @Description: 中间件配置
* @author shentao
* @date 2019年7月16日 下午7:08:07
*/
@Component
public class MiddlewareConfig {
public static String REGISTRY;
public static String CHANNEL_INDEX_REGISTRY;
public static String CLEAN_FILTER_GROUP;
public static String AUTOMARK_GROUP;
public static String CHANNEL_GROUP;
@Value("${zookeeper.registry}")
public void setRegistry(String registry) {
REGISTRY = registry;
System.out.println("REGISTRY:" + REGISTRY);
}
@Value("${channel-index.zookeeper.registry}")
public void setChannelIndexRegistry(String registry) {
CHANNEL_INDEX_REGISTRY = registry;
System.out.println("CHANNEL_INDEX_REGISTRY:" + CHANNEL_INDEX_REGISTRY);
}
@Value("${dubbo.filter.group}")
public void setCleanFilterGroup(String group) {
CLEAN_FILTER_GROUP = group;
System.out.println("CLEAN_FILTER_GROUP:" + CLEAN_FILTER_GROUP);
}
@Value("${dubbo.automark.group}")
public void setAutoMarkGroup(String group) {
AUTOMARK_GROUP = group;
System.out.println("AUTOMARK_GROUP:" + AUTOMARK_GROUP);
}
@Value("${dubbo.channel.group}")
public void setChannelGroup(String group) {
CHANNEL_GROUP = group;
System.out.println("CHANNEL_GROUP:" + CHANNEL_GROUP);
}
}
src/main/java/com/zhiwei/qbjcutils/config/MongoConfig.java 0 → 100644
package com.zhiwei.qbjcutils.config;
import com.mongodb.MongoClientOptions;
import com.mongodb.MongoClientURI;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import org.springframework.data.mongodb.MongoDbFactory;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.SimpleMongoDbFactory;
import org.springframework.data.mongodb.core.convert.DbRefResolver;
import org.springframework.data.mongodb.core.convert.DefaultDbRefResolver;
import org.springframework.data.mongodb.core.convert.DefaultMongoTypeMapper;
import org.springframework.data.mongodb.core.convert.MappingMongoConverter;
import org.springframework.data.mongodb.core.mapping.MongoMappingContext;
/**
* MongoConfig
*
* @author shentao
* @ClassName: MongoConfig
* @Description: MongoConfig
* @date 2019年7月8日 上午10:32:02
*/
@Configuration
public class MongoConfig {
private final static Logger log = LogManager.getLogger(MongoConfig.class);
@Value("${mongo.connectionsPerHost}")
private int connectionsPerHost;
@Value("${mongo.threadsAllowedToBlockForConnectionMultiplier}")
private int threadsAllowedToBlockForConnectionMultiplier;
@Value("${mongo.connectTimeout}")
private int connectTimeout;
@Value("${mongo.maxWaitTime}")
private int maxWaitTime;
@Value("${mongo.autoConnectRetry}")
private boolean autoConnectRetry;
@Value("${mongo.socketKeepAlive}")
private boolean socketKeepAlive;
@Value("${mongo.socketTimeout}")
private int socketTimeout;
@Value("${mongo.slaveOk}")
private boolean slaveOk;
@Value("${primary.uri}")
private String uri;
private MongoDbFactory mongoDbFactory() {
MongoClientOptions.Builder builder = new MongoClientOptions.Builder();
builder.connectionsPerHost(connectionsPerHost);
builder.connectTimeout(connectTimeout);
builder.threadsAllowedToBlockForConnectionMultiplier(threadsAllowedToBlockForConnectionMultiplier);
builder.maxWaitTime(maxWaitTime);
return new SimpleMongoDbFactory(new MongoClientURI(uri, builder));
}
@Primary
@Bean(name = "primaryMongoTemplate")
public MongoTemplate getMongoTemplate() {
log.info("@Primary");
MongoDbFactory mongoDbFactory = mongoDbFactory();
DbRefResolver dbRefResolver = new DefaultDbRefResolver(mongoDbFactory);
MappingMongoConverter converter = new MappingMongoConverter(dbRefResolver, new MongoMappingContext());
// 不插入_class
converter.setTypeMapper(new DefaultMongoTypeMapper(null));
return new MongoTemplate(mongoDbFactory(), converter);
}
private MongoDbFactory mongoDbFactory2() {
MongoClientOptions.Builder builder = new MongoClientOptions.Builder();
builder.connectionsPerHost(connectionsPerHost);
builder.connectTimeout(connectTimeout);
builder.threadsAllowedToBlockForConnectionMultiplier(threadsAllowedToBlockForConnectionMultiplier);
builder.maxWaitTime(maxWaitTime);
return new SimpleMongoDbFactory(new MongoClientURI(secondUri, builder));
}
@Value("${secondary.uri}")
private String secondUri;
@Bean(name = "secondaryMongoTemplate")
public MongoTemplate getSecondaryMongoTemplate() {
log.info("@Secondary");
MongoDbFactory mongoDbFactory = mongoDbFactory2();
DbRefResolver dbRefResolver = new DefaultDbRefResolver(mongoDbFactory);
MappingMongoConverter converter = new MappingMongoConverter(dbRefResolver, new MongoMappingContext());
// 不插入_class
converter.setTypeMapper(new DefaultMongoTypeMapper(null));
return new MongoTemplate(mongoDbFactory2(), converter);
}
}
src/main/java/com/zhiwei/qbjcutils/config/TaskConfig.java 0 → 100644
package com.zhiwei.qbjcutils.config;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import java.util.concurrent.Executor;
import java.util.concurrent.ThreadPoolExecutor;
/**
* Description:
*
* @author LiuMingHuan
* @classname TaskConfig
* @date 2020/3/10
*/
@Configuration
@EnableAsync
public class TaskConfig {
@Bean
public Executor scheduleExecutor(){
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
executor.setCorePoolSize(2);
executor.setMaxPoolSize(2);
executor.setThreadNamePrefix("schedule-");
// rejection-policy：当pool已经达到max size的时候，如何处理新任务
// CALLER_RUNS：不在新线程中执行任务，而是有调用者所在的线程来执行
executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
executor.initialize();
return executor;
}
}
src/main/java/com/zhiwei/qbjcutils/dao/DirectGroupDao.java 0 → 100644
package com.zhiwei.qbjcutils.dao;
import com.zhiwei.qbjcutils.pojo.entity.DirectGroup;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.data.mongodb.core.query.Update;
/**
* Description:
*
* @author LiuMingHuan
* @classname DirectGroupDao
* @date 2020/4/16
*/
public interface DirectGroupDao {
/**
* updateFirst
* @param query
* @param update
*/
void updateFirst(Query query, Update update);
/**
* insert
* @param group
*/
void insert(DirectGroup group);
/**
*
* @param query
* @return
*/
DirectGroup findFirst(Query query);
}
src/main/java/com/zhiwei/qbjcutils/dao/EventDao.java 0 → 100644
package com.zhiwei.qbjcutils.dao;
import com.zhiwei.qbjcutils.pojo.entity.Event;
import org.springframework.data.mongodb.core.aggregation.Aggregation;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.data.mongodb.core.query.Update;
import java.util.List;
/**
* Description:
*
* @author LiuMingHuan
* @classname EventDao
* @date 2020/4/8
*/
public interface EventDao {
List<Event> aggregate(Aggregation aggregation);
List<Event> find(Query query);
void update(Query query, Update update);
}
src/main/java/com/zhiwei/qbjcutils/dao/PlatformDao.java 0 → 100644
package com.zhiwei.qbjcutils.dao;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.data.mongodb.core.query.Update;
/**
* Description:
*
* @author LiuMingHuan
* @classname PlatformDao
* @date 2020/4/3
*/
public interface PlatformDao {
/**
* 更新平台
* @param query
* @param update
*/
void update(Query query, Update update);
}
src/main/java/com/zhiwei/qbjcutils/dao/TestController.java 0 → 100644
package com.zhiwei.qbjcutils.dao;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.slf4j.log4j12.Log4jLoggerAdapter;
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RestController;
/**
* Description:
*
* @author LiuMingHuan
* @classname TestController
* @date 2020/6/4
*/
@RestController
public class TestController {
private Logger log = LogManager.getLogger(TestController.class);
@RequestMapping(method = RequestMethod.GET, value = "/get")
public void test(){
AnnotationConfigApplicationContext ann = new AnnotationConfigApplicationContext(Log4jLoggerAdapter.class);
ann.getBeanFactory().registerSingleton("log", log);
log.error("测试出错{}", "user");
log.error("测试出错{},{}", "user1", "user2");
}
@RequestMapping(method = RequestMethod.GET, value = "/get1")
public void test1(){
try {
int i = 10/0;
}catch (Exception e){
System.out.println(e.getCause());
}
}
}
src/main/java/com/zhiwei/qbjcutils/dao/impl/DirectGroupDaoImpl.java 0 → 100644
package com.zhiwei.qbjcutils.dao.impl;
import com.zhiwei.qbjcutils.dao.DirectGroupDao;
import com.zhiwei.qbjcutils.pojo.entity.DirectGroup;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.data.mongodb.core.query.Update;
import org.springframework.stereotype.Component;
/**
* Description:
*
* @author LiuMingHuan
* @classname DirectGroupDaoImpl
* @date 2020/4/16
*/
@Component
public class DirectGroupDaoImpl implements DirectGroupDao {
private final MongoTemplate mongoTemplate;
public DirectGroupDaoImpl(@Qualifier("primaryMongoTemplate") MongoTemplate mongoTemplate) {
this.mongoTemplate = mongoTemplate;
}
@Override
public void updateFirst(Query query, Update update) {
mongoTemplate.updateFirst(query, update, DirectGroup.class);
}
@Override
public void insert(DirectGroup group) {
mongoTemplate.insert(group);
}
@Override
public DirectGroup findFirst(Query query) {
return mongoTemplate.findOne(query, DirectGroup.class);
}
}
src/main/java/com/zhiwei/qbjcutils/dao/impl/EventDaoImpl.java 0 → 100644
package com.zhiwei.qbjcutils.dao.impl;
import com.zhiwei.qbjcutils.dao.EventDao;
import com.zhiwei.qbjcutils.pojo.entity.Event;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.aggregation.Aggregation;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.data.mongodb.core.query.Update;
import org.springframework.stereotype.Component;
import java.util.List;
/**
* Description:
*
* @author LiuMingHuan
* @classname EventDaoImpl
* @date 2020/4/8
*/
@Component
public class EventDaoImpl implements EventDao {
private final MongoTemplate mongoTemplate;
public EventDaoImpl(MongoTemplate mongoTemplate) {
this.mongoTemplate = mongoTemplate;
}
@Override
public List<Event> aggregate(Aggregation aggregation) {
return mongoTemplate.aggregate(aggregation, Event.class, Event.class).getMappedResults();
}
@Override
public List<Event> find(Query query) {
return mongoTemplate.find(query, Event.class);
}
@Override
public void update(Query query, Update update) {
mongoTemplate.updateFirst(query, update, Event.class);
}
}
src/main/java/com/zhiwei/qbjcutils/dao/impl/PlatformDaoImpl.java 0 → 100644
package com.zhiwei.qbjcutils.dao.impl;
import com.zhiwei.qbjcutils.dao.PlatformDao;
import com.zhiwei.qbjcutils.pojo.entity.MessagePlatform;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.data.mongodb.core.query.Update;
import org.springframework.stereotype.Component;
/**
* Description:
*
* @author LiuMingHuan
* @classname PlatformDaoImpl
* @date 2020/4/3
*/
@Component
public class PlatformDaoImpl implements PlatformDao {
private final MongoTemplate mongoTemplate;
public PlatformDaoImpl(@Qualifier("primaryMongoTemplate") MongoTemplate mongoTemplate) {
this.mongoTemplate = mongoTemplate;
}
@Override
public void update(Query query, Update update) {
mongoTemplate.updateFirst(query, update, MessagePlatform.class);
}
}
src/main/java/com/zhiwei/qbjcutils/dubbo/DubboClient.java 0 → 100644
package com.zhiwei.qbjcutils.dubbo;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.base.filter.FilterInfo;
import com.zhiwei.channel.index.middleware.core.client.ChannelDataClient;
import com.zhiwei.channel.index.middleware.core.client.ChannelIndexClientFactory;
import com.zhiwei.channel.index.middleware.core.client.ChannelStatisticsClient;
import com.zhiwei.middleware.automaticmark.dubbo.client.AutomaticClientFactory;
import com.zhiwei.middleware.automaticmark.dubbo.client.AutomaticMarkClient;
import com.zhiwei.middleware.cleaner.filter.UnifiedFilterClient;
import com.zhiwei.qbjcutils.config.MiddlewareConfig;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* @author shentao
* @ClassName: DubboClient
* @Description: 中间件接口类
* @date 2019年7月16日 下午6:44:56
*/
@Component
public class DubboClient {
private static final Logger log = LogManager.getLogger(DubboClient.class);
private static final String APP_NAME = "qbjc-utils";
public final MiddlewareConfig config;
private static AutomaticMarkClient AUTO_MATICMARK_CLIENT;
private static UnifiedFilterClient FILTER_CLIENT;
private static ChannelDataClient CHANNEL_DATA_CLIENT;
private static ChannelStatisticsClient CHANNEL_STATICS_CLIENT;
public DubboClient(MiddlewareConfig config) {
this.config = config;
}
@PostConstruct
public void init() {
try {
AUTO_MATICMARK_CLIENT = AutomaticClientFactory.automaticMarkClient(MiddlewareConfig.REGISTRY, MiddlewareConfig.AUTOMARK_GROUP, APP_NAME);
FILTER_CLIENT = UnifiedFilterClient.getClient(APP_NAME, MiddlewareConfig.REGISTRY, MiddlewareConfig.CLEAN_FILTER_GROUP);
CHANNEL_DATA_CLIENT = ChannelIndexClientFactory.channelDataClient(MiddlewareConfig.CHANNEL_INDEX_REGISTRY, MiddlewareConfig.CHANNEL_GROUP, APP_NAME);
CHANNEL_STATICS_CLIENT = ChannelIndexClientFactory.channelStatisticsClient(MiddlewareConfig.CHANNEL_INDEX_REGISTRY, MiddlewareConfig.CHANNEL_GROUP, APP_NAME);
} catch (Exception e) {
log.error("初始化中间件Client失败", e);
}
}
/**
* contains 去重 true重复（库里存在）/false不重复（不存在）
*
* @param filterInfo
* @return boolean
*/
public boolean contains(FilterInfo filterInfo) {
return FILTER_CLIENT.contains(filterInfo, false);
}
/**
* 返回去重条件
*
* @param filterInfo
* @return String []
*/
public String[] getUpdateInfo(FilterInfo filterInfo) {
return FILTER_CLIENT.getUpdateInfo(filterInfo);
}
/**
* 渠道索引获取对应账号
* @param mediaId
* @param keyword
* @param isCollected
* @param page
* @param pageSize
* @return
*/
public JSONObject searchCollectedAccountList(int mediaId, String keyword, Boolean isCollected, int page, int pageSize){
return CHANNEL_DATA_CLIENT.pageSearchAccount(mediaId, keyword, isCollected, page, pageSize);
}
/**
* searchCollectedAccountList
* @param queryTags
* @param mediaId
* @param keyword
* @param isCollected
* @param page
* @param pageSize
* @return
*/
public JSONObject searchCollectedAccountList(List<Long> queryTags, int mediaId, String keyword, Boolean isCollected, int page, int pageSize){
return CHANNEL_DATA_CLIENT.pageSearchAccount(queryTags, keyword, isCollected, mediaId, page, pageSize, null, null);
}
/**
* getPlatformsPermutation
* @return
*/
public Set<Map<String,Object>> getPlatformsPermutation(){
return CHANNEL_STATICS_CLIENT.channelArticleDataTypes();
}
}
src/main/java/com/zhiwei/qbjcutils/es/EsClient.java 0 → 100644
package com.zhiwei.qbjcutils.es;
import com.zhiwei.es.util.ESIndexesUtil;
import com.zhiwei.qbjcutils.config.EsProperties;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.transport.client.PreBuiltTransportClient;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.util.Assert;
import org.springframework.util.StringUtils;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.Arrays;
/**
* es连接客户端
*
* @ClassName: EsClient
* @Description: es连接客户端
* @author shentao
* @date 2019年7月8日 下午3:08:04
*/
@Configuration
public class EsClient {
public static final Logger log = LogManager.getLogger(EsClient.class);
private static final String COLON = ":";
private static final String COMMA = ",";
private final EsProperties esProperties;
public EsClient(EsProperties esProperties) {
this.esProperties = esProperties;
}
@Bean(name = "esIndexes")
public ESIndexesUtil.ESIndexes esIndexes(){
return ESIndexesUtil.create(esProperties.getHttpIp(), esProperties.getHttpHost());
}
@Bean(name = "esTransportClient")
public TransportClient esTransportClient() {
System.setProperty("es.set.netty.runtime.available.processors", "false");
Settings esSettings = Settings.builder()
// 设置ES实例的名称
.put("cluster.name", esProperties.getClusterName())
// 自动嗅探整个集群的状态，把集群中其他ES节点的ip添加到本地的客户端列表中
.put("client.transport.sniff", esProperties.isTransportSniff())
// .put("client.transport.ping_timeout", "15s")
.build();
TransportClient client = new PreBuiltTransportClient(esSettings);
// 此步骤添加IP，至少一个，其实一个就够了，因为添加了自动嗅探配置
try {
Assert.hasText(esProperties.getClusterNodes(), "Cluster nodes source must not be null or empty!");
String[] nodes = StringUtils.delimitedListToStringArray(esProperties.getClusterNodes(), COMMA);
Arrays.stream(nodes).forEach(node -> {
String[] segments = StringUtils.delimitedListToStringArray(node, COLON);
Assert.isTrue(segments.length == 2,
() -> String.format("Invalid cluster node %s in %s! Must be in the format host:port!", node,
esProperties.getClusterNodes()));
String host = segments[0].trim();
String port = segments[1].trim();
Assert.hasText(host, () -> String.format("No host name given cluster node %s!", node));
Assert.hasText(port, () -> String.format("No port given in cluster node %s!", node));
client.addTransportAddress(new InetSocketTransportAddress(toInetAddress(host), Integer.parseInt(port)));
});
} catch (Exception e) {
log.error("es client初始化异常",e);
}
return client;
}
private static InetAddress toInetAddress(String host) {
try {
return InetAddress.getByName(host);
} catch (UnknownHostException e) {
throw new IllegalArgumentException(e);
}
}
}
src/main/java/com/zhiwei/qbjcutils/es/EsIndexTools.java 0 → 100644
package com.zhiwei.qbjcutils.es;
import com.zhiwei.es.index.Index;
import com.zhiwei.es.util.ESIndexesUtil;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
/**
* es索引辅助类
*
* @author shentao
* @ClassName: EsIndexTools
* @Description: es索引辅助类
* @date 2019年7月8日 下午6:05:22
*/
@Service
public class EsIndexTools {
@Resource(name ="esIndexes")
private ESIndexesUtil.ESIndexes indexes;
/**
* 获取标注indexs-近三年
*
* @return String []
*/
public String[] getMarkIndexs() {
List<String> list = indexes.getMarkIndexes();
// List<String> list = new ArrayList<>();
// list.add("mark_test");
return list.toArray(new String[list.size()]);
}
/**
* 获取四大库indexs
*
* @param start
* @param end
* @return String []
*/
public String[] getAllIndex(Long start, Long end) {
List<String> list = new ArrayList<>();
list.addAll(indexes.getIndexes(Index.complete_text.name(), new Date(start), new Date(end)));
list.addAll(indexes.getIndexes(Index.incomplete_text.name(), new Date(start), new Date(end)));
list.addAll(indexes.getIndexes(Index.qa_text.name(), new Date(start), new Date(end)));
list.addAll(indexes.getIndexes(Index.video.name(), new Date(start), new Date(end)));
list = new ArrayList<>(new HashSet<>(list));
return list.toArray(new String[list.size()]);
}
}
src/main/java/com/zhiwei/qbjcutils/es/EsQueryTools.java 0 → 100644
package com.zhiwei.qbjcutils.es;
import com.zhiwei.qbjcutils.config.GenericAttribute;
import com.zhiwei.qbjcutils.pojo.entity.MessagePlatform;
import com.zhiwei.qbjcutils.utils.Tools;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
/**
* es语句辅助类
*
* @ClassName: EsQueryTools
* @Description: es语句辅助类
* @author shentao
* @date 2019年7月8日 上午11:27:19
*/
public class EsQueryTools {
private static final String FOREIGN_0 = "0";
public static final String ES_QUERY_AND = "must";
public static final String ES_QUERY_OR = "should";
public static final String ES_QUERY_NOT = "not";
/**
* rangeQuery(gte,lte)
*
* @Title: rangeQuery
* @Description: rangeQuery(gte,lte)
* @param @param name
* @param @param gte
* @param @param lte
* @param @return 设定文件
* @return QueryBuilder 返回类型
*/
public static QueryBuilder rangeQuery(String name, long gte, long lte) {
return QueryBuilders.rangeQuery(name).gte(gte).lte(lte);
}
/**
* rangeQuery(gte,lt)
*
* @param name
* @param gte
* @param lte
* @return QueryBuilder
*/
public static QueryBuilder rangeQueryGteLt(String name, long gte, long lte) {
return QueryBuilders.rangeQuery(name).gte(gte).lt(lte);
}
/**
* termQuery 单一字段，多匹配值
*
* @Title: termQueryField
* @Description: termQuery 单一字段，多匹配值
* @param @param boolQueryBuilder
* @param @param esQuery
* @param @param words
* @param @param field
* @param @return 设定文件
* @return BoolQueryBuilder 返回类型
*/
public static <T> BoolQueryBuilder termQueryField(BoolQueryBuilder boolQueryBuilder, String esQuery, List<T> words,
String field) {
if (ES_QUERY_AND.equals(esQuery)) {
for (T word : words) {
boolQueryBuilder.must(QueryBuilders.termQuery(field, word));
}
} else if (ES_QUERY_OR.equals(esQuery)) {
for (T word : words) {
boolQueryBuilder.should(QueryBuilders.termQuery(field, word));
}
} else if (ES_QUERY_NOT.equals(esQuery)) {
for (T word : words) {
boolQueryBuilder.mustNot(QueryBuilders.termQuery(field, word));
}
}
return boolQueryBuilder;
}
/**
* termQuery 多字段，单一匹配值
*
* @Title: termQueryField
* @Description: termQuery 多字段，单一匹配值
* @param @param boolQueryBuilder
* @param @param esQuery
* @param @param word
* @param @param fields
* @param @return 设定文件
* @return BoolQueryBuilder 返回类型
*/
public static <T> BoolQueryBuilder termQueryField(BoolQueryBuilder boolQueryBuilder, String esQuery, T word,
String[] fields) {
if (ES_QUERY_AND.equals(esQuery)) {
for (String field : fields) {
boolQueryBuilder.must(QueryBuilders.matchPhraseQuery(field, word));
}
} else if (ES_QUERY_OR.equals(esQuery)) {
for (String field : fields) {
boolQueryBuilder.should(QueryBuilders.matchPhraseQuery(field, word));
}
} else if (ES_QUERY_NOT.equals(esQuery)) {
for (String field : fields) {
boolQueryBuilder.mustNot(QueryBuilders.matchPhraseQuery(field, word));
}
}
return boolQueryBuilder;
}
/**
* wildcardQuery 多字段，单一匹配值
*
* @Title: wildcardQueryField
* @Description: wildcardQuery 多字段，单一匹配值
* @param @param boolQueryBuilder
* @param @param esQuery
* @param @param word
* @param @param fields
* @param @return 设定文件
* @return BoolQueryBuilder 返回类型
*/
public static BoolQueryBuilder wildcardQueryField(BoolQueryBuilder boolQueryBuilder, String esQuery, String word,
String[] fields) {
if (ES_QUERY_AND.equals(esQuery)) {
for (String field : fields) {
boolQueryBuilder.must(QueryBuilders.wildcardQuery(field, word));
}
} else if (ES_QUERY_OR.equals(esQuery)) {
for (String field : fields) {
boolQueryBuilder.should(QueryBuilders.wildcardQuery(field, word));
}
} else if (ES_QUERY_NOT.equals(esQuery)) {
for (String field : fields) {
boolQueryBuilder.mustNot(QueryBuilders.wildcardQuery(field, word));
}
}
return boolQueryBuilder;
}
/**
* 字段不拆封,多字段 同关键词
*
* @Title: matchPhraseQueryFields
* @Description: 字段不拆封,多字段 同关键词
* @param @param boolQueryBuilder
* @param @param esQuery
* @param @param word
* @param @param fields
* @param @return 设定文件
* @return BoolQueryBuilder 返回类型
*/
public static BoolQueryBuilder matchPhraseQueryFields(BoolQueryBuilder boolQueryBuilder, String esQuery,
String word, String... fields) {
if (ES_QUERY_AND.equals(esQuery)) {
for (String field : fields) {
boolQueryBuilder.must(QueryBuilders.matchPhraseQuery(field, word));
}
} else if (ES_QUERY_OR.equals(esQuery)) {
for (String field : fields) {
boolQueryBuilder.should(QueryBuilders.matchPhraseQuery(field, word));
}
} else if (ES_QUERY_NOT.equals(esQuery)) {
for (String field : fields) {
boolQueryBuilder.mustNot(QueryBuilders.matchPhraseQuery(field, word));
}
}
return boolQueryBuilder;
}
/**
* 多字段 keyword格式为【腾讯 马化腾|京东 刘强东】 这是（腾讯且马化腾）或（京东且刘强东）
*
* @Title: assembleNormalKeywordQuery
* @Description: 多字段 search格式为【腾讯 马化腾|京东 刘强东】 这是（腾讯且马化腾）或（京东且刘强东）
* @param @param keyword
* @param @param strings
* @param @return 设定文件
* @return QueryBuilder 返回类型
*/
public static BoolQueryBuilder assembleNormalKeywordQuery(String keyword, String[] fieldlist) {
String orStr = "|";
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
if (Tools.isEmpty(keyword)) {
return queryBuilder;
} else if (keyword.indexOf(StringUtils.SPACE) != -1 && keyword.indexOf(orStr) == -1) {
// 单组关键词
BoolQueryBuilder mixboolQueryBuilder = QueryBuilders.boolQuery();
String[] words = keyword.split(" +");
for (int i = 0; i < words.length; i++) {
String word = words[i];
mixboolQueryBuilder.must(
EsQueryTools.matchPhraseQueryFields(QueryBuilders.boolQuery(), ES_QUERY_OR, word, fieldlist));
}
queryBuilder.should(mixboolQueryBuilder);
} else if (keyword.indexOf(orStr) != -1) {
// 多组关键词
BoolQueryBuilder mixboolQueryBuilder = QueryBuilders.boolQuery();
String[] wordGroups = keyword.split("\\|");
for (int i = 0; i < wordGroups.length; i++) {
String wordGroup = wordGroups[i];
BoolQueryBuilder wordboolQueryBuilder = QueryBuilders.boolQuery();
String[] words = wordGroup.split(" +");
for (int j = 0; j < words.length; j++) {
String word = words[j];
wordboolQueryBuilder.must(EsQueryTools.matchPhraseQueryFields(QueryBuilders.boolQuery(),
ES_QUERY_OR, word, fieldlist));
}
mixboolQueryBuilder.should(wordboolQueryBuilder);
}
queryBuilder.should(mixboolQueryBuilder);
} else {
// 单个关键词
EsQueryTools.matchPhraseQueryFields(queryBuilder, ES_QUERY_OR, keyword, fieldlist);
}
return queryBuilder;
}
/**
* allWords anyWords notWords 高级搜索查询语句
*
* @Title: assembleSeniorKeywordQuery
* @Description: allWords anyWords notWords 高级搜索查询语句
* @param @param allWords
* @param @param anyWords
* @param @param notWords
* @param @param fieldlist
* @param @return 设定文件
* @return BoolQueryBuilder 返回类型
*/
public static BoolQueryBuilder assembleSeniorKeywordQuery(String allWords, String anyWords, String notWords,
String[] fieldlist) {
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
BoolQueryBuilder anyboolQueryBuilder = QueryBuilders.boolQuery();
if (!Tools.isEmpty(anyWords)) {
String[] anywords = anyWords.split(" +");
for (int i = 0; i < anywords.length; i++) {
String anyword = anywords[i];
anyboolQueryBuilder
.should(matchPhraseQueryFields(QueryBuilders.boolQuery(), ES_QUERY_OR, anyword, fieldlist));
}
boolQuery.must(anyboolQueryBuilder);
}
BoolQueryBuilder allboolQueryBuilder = QueryBuilders.boolQuery();
if (!Tools.isEmpty(allWords)) {
String[] allwords = allWords.split(" +");
for (int i = 0; i < allwords.length; i++) {
String allword = allwords[i];
allboolQueryBuilder
.must(matchPhraseQueryFields(QueryBuilders.boolQuery(), ES_QUERY_OR, allword, fieldlist));
}
boolQuery.must(allboolQueryBuilder);
}
BoolQueryBuilder notboolQueryBuilder = QueryBuilders.boolQuery();
if (!Tools.isEmpty(notWords)) {
String[] notwords = notWords.split(" +");
for (int i = 0; i < notwords.length; i++) {
String notword = notwords[i];
notboolQueryBuilder
.must(matchPhraseQueryFields(QueryBuilders.boolQuery(), ES_QUERY_NOT, notword, fieldlist));
}
boolQuery.must(notboolQueryBuilder);
}
return boolQuery;
}
/**
* keyword类型字段精准匹配
*
* @Title: assembleSeniorTermQuery
* @Description: keyword类型字段精准匹配
* @param @param source
* @param @param fieldlist
* @param @return 设定文件
* @return BoolQueryBuilder 返回类型
*/
public static BoolQueryBuilder assembleSeniorTermQuery(String source, String[] fieldlist) {
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
if (!Tools.isEmpty(source)) {
String[] sourcewords = source.split(" +");
for (int i = 0; i < sourcewords.length; i++) {
String sourceword = sourcewords[i];
boolQuery.should(termQueryField(QueryBuilders.boolQuery(), "should", sourceword, fieldlist));
}
}
return boolQuery;
}
public static BoolQueryBuilder assembleSeniorWildcardQuery(String source, String[] fieldlist) {
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
if (!Tools.isEmpty(source)) {
String[] sourcewords = source.split(" +");
for (int i = 0; i < sourcewords.length; i++) {
String sourceword = "*" + sourcewords[i] + "*";
boolQuery.should(wildcardQueryField(QueryBuilders.boolQuery(), "should", sourceword, fieldlist));
}
}
return boolQuery;
}
@Deprecated
public static BoolQueryBuilder assemblePlatformQuery(List<Integer> c4, List<String> origin) {
String originApp = "APP";
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
if (!Tools.isEmpty(c4)) {
if (!Tools.isEmpty(origin)) {
BoolQueryBuilder sonQuery = QueryBuilders.boolQuery();
if (origin.contains(originApp)) {
BoolQueryBuilder appQuery = QueryBuilders.boolQuery();
appQuery.should(QueryBuilders.termQuery("origin", originApp));
origin.remove(originApp);
sonQuery.must(EsQueryTools.termQueryField(QueryBuilders.boolQuery(), ES_QUERY_OR, c4, "c4"));
sonQuery.must(
EsQueryTools.termQueryField(QueryBuilders.boolQuery(), ES_QUERY_OR, origin, "origin"));
appQuery.should(sonQuery);
queryBuilder.must(appQuery);
} else {
sonQuery.must(EsQueryTools.termQueryField(QueryBuilders.boolQuery(), ES_QUERY_OR, c4, "c4"));
sonQuery.must(
EsQueryTools.termQueryField(QueryBuilders.boolQuery(), ES_QUERY_OR, origin, "origin"));
queryBuilder.must(sonQuery);
}
} else {
queryBuilder.must(EsQueryTools.termQueryField(QueryBuilders.boolQuery(), ES_QUERY_OR, c4, "c4"));
}
} else {
if (!Tools.isEmpty(origin)) {
queryBuilder
.must(EsQueryTools.termQueryField(QueryBuilders.boolQuery(), ES_QUERY_OR, origin, "origin"));
}
}
return queryBuilder;
}
/**
* platform 组合语句
*
* @Title: assemblePlatformQuery
* @Description: platform 组合语句
* @param @param platforms
* @param @return 设定文件
* @return BoolQueryBuilder 返回类型
*/
public static BoolQueryBuilder assemblePlatformQuery(List<MessagePlatform> platforms) {
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
Map<String, List<Integer>> queryMap = newQueryMap(platforms);
for (Entry<String, List<Integer>> queryEntry : queryMap.entrySet()) {
if (Tools.isEmpty(queryEntry.getValue())) {
continue;
}
BoolQueryBuilder ptBuilder = QueryBuilders.boolQuery();
switch (queryEntry.getKey()) {
case "PC_0":
ptBuilder.must(QueryBuilders.termQuery(GenericAttribute.ES_ORIGIN, "PC"))
.must(QueryBuilders.termQuery(GenericAttribute.ES_FOREIGN, 0))
.must(QueryBuilders.termsQuery(GenericAttribute.ES_C5, queryEntry.getValue()));
break;
case "APP_0":
ptBuilder.must(QueryBuilders.termQuery(GenericAttribute.ES_ORIGIN, "APP"))
.must(QueryBuilders.termQuery(GenericAttribute.ES_FOREIGN, 0))
.must(QueryBuilders.termsQuery(GenericAttribute.ES_C5, queryEntry.getValue()));
break;
case "MOBI_0":
ptBuilder.must(QueryBuilders.termQuery(GenericAttribute.ES_ORIGIN, "MOBI"))
.must(QueryBuilders.termQuery(GenericAttribute.ES_FOREIGN, 0))
.must(QueryBuilders.termsQuery(GenericAttribute.ES_C5, queryEntry.getValue()));
break;
case "PC_1":
ptBuilder.must(QueryBuilders.termQuery(GenericAttribute.ES_ORIGIN, "PC"))
.must(QueryBuilders.termQuery(GenericAttribute.ES_FOREIGN, 1))
.must(QueryBuilders.termsQuery(GenericAttribute.ES_C5, queryEntry.getValue()));
break;
case "APP_1":
ptBuilder.must(QueryBuilders.termQuery(GenericAttribute.ES_ORIGIN, "APP"))
.must(QueryBuilders.termQuery(GenericAttribute.ES_FOREIGN, 1))
.must(QueryBuilders.termsQuery(GenericAttribute.ES_C5, queryEntry.getValue()));
break;
case "MOBI_1":
ptBuilder.must(QueryBuilders.termQuery(GenericAttribute.ES_ORIGIN, "MOBI"))
.must(QueryBuilders.termQuery(GenericAttribute.ES_FOREIGN, 1))
.must(QueryBuilders.termsQuery(GenericAttribute.ES_C5, queryEntry.getValue()));
break;
default:
break;
}
queryBuilder.should(ptBuilder);
}
return queryBuilder;
}
/**
* new QueryMap
*
* @param platforms
*
* @return Map<String,List<Integer>>
*/
private static Map<String, List<Integer>> newQueryMap(List<MessagePlatform> platforms) {
Map<String, List<Integer>> queryMap = new HashMap<>(6);
queryMap.put("PC_0", new ArrayList<>());
queryMap.put("APP_0", new ArrayList<>());
queryMap.put("MOBI_0", new ArrayList<>());
queryMap.put("PC_1", new ArrayList<>());
queryMap.put("APP_1", new ArrayList<>());
queryMap.put("MOBI_1", new ArrayList<>());
for (MessagePlatform messagePlatform : platforms) {
// 按templateHeaderIds来组装查询语句，按origin+foreign分组
for (String templateHeaderId : messagePlatform.getTemplateHeaders()) {
String[] templateHeader = templateHeaderId.split("_");
String c5 = templateHeader[0];
String origin = templateHeader[1];
String foreign = templateHeader[2];
if (FOREIGN_0.equals(foreign)) {
// 非外媒
switch (origin) {
case "APP":
queryMap.get("APP_0").add(Integer.parseInt(c5));
break;
case "MOBI":
queryMap.get("MOBI_0").add(Integer.parseInt(c5));
break;
default:
queryMap.get("PC_0").add(Integer.parseInt(c5));
break;
}
} else {
switch (origin) {
case "APP":
queryMap.get("APP_1").add(Integer.parseInt(c5));
break;
case "MOBI":
queryMap.get("MOBI_1").add(Integer.parseInt(c5));
break;
default:
queryMap.get("PC_1").add(Integer.parseInt(c5));
break;
}
}
}
}
return queryMap;
}
/**
* 按更新条件获取标注信息，不能限制范围
*
* @param mupdate
* @param mupdateTwo
*
* @return BoolQueryBuilder
*/
public static QueryBuilder assembleTagInfoQuery(String mupdate, String mupdateTwo) {
if (Tools.isEmpty(mupdateTwo)) {
return QueryBuilders.boolQuery().must(QueryBuilders.termQuery(GenericAttribute.ES_MUPDATE, mupdate));
} else {
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery()
.should(QueryBuilders.termQuery(GenericAttribute.ES_MUPDATE, mupdate))
.should(QueryBuilders.termQuery(GenericAttribute.ES_MUPDATE_TWO, mupdateTwo));
return QueryBuilders.boolQuery().must(queryBuilder);
}
}
/**
* 按批量更新条件获取标注信息，不能限制范围
*
* @param forTagInfoQuery
*
* @return QueryBuilder
*/
// public static QueryBuilder assembleTagInfoQuery(List<MessageFlowEntityTagInfoDTO> forTagInfoQuery) {
// BoolQueryBuilder res = QueryBuilders.boolQuery();
// forTagInfoQuery.stream().forEach(a -> {
// String mupdate = a.getMupdate();
// String mupdateTwo = Tools.isNull(a.getMupdateTwo()) ? null : a.getMupdateTwo();
// if (Tools.isEmpty(mupdateTwo)) {
// res.should(QueryBuilders.termQuery(GenericAttribute.ES_MUPDATE, mupdate));
// } else {
// BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery()
// .should(QueryBuilders.termQuery(GenericAttribute.ES_MUPDATE, mupdate))
// .should(QueryBuilders.termQuery(GenericAttribute.ES_MUPDATE_TWO, mupdateTwo));
// res.should(QueryBuilders.boolQuery().must(queryBuilder));
// }
// });
// return res;
// }
}
src/main/java/com/zhiwei/qbjcutils/es/dao/EsDao.java 0 → 100644
package com.zhiwei.qbjcutils.es.dao;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.sort.FieldSortBuilder;
/**
* EsDao-interface
*
* @ClassName: EsDao
* @Description: EsDao-interface
* @author shentao
* @date 2019年7月8日 下午6:05:54
*/
public interface EsDao {
/**
* search-es检索
*
* @param indexs
* @param types
* @param postFilter
* @param query
* @param sort
* @param from
* @param size
* @param highlightBuilder
* @return SearchHits
*/
SearchHits search(String[] indexs, String[] types, QueryBuilder postFilter, QueryBuilder query,
FieldSortBuilder sort, int from, int size, HighlightBuilder highlightBuilder);
/**
* search-es检索限制返回值
*
* @param indexs
* @param types
* @param postFilter
* @param query
* @param sort
* @param from
* @param size
* @param includes
* @param excludes
* @param highlightBuilder
* @return SearchHits
*/
SearchHits searchFatchSource(String[] indexs, String[] types, QueryBuilder postFilter, QueryBuilder query,
FieldSortBuilder sort, int from, int size, String[] includes, String[] excludes,
HighlightBuilder highlightBuilder);
/**
* es统计量
*
* @param indexes
* @param types
* @param postFilter
* @param query
* @return
*/
Long count(String[] indexes, String[] types, QueryBuilder postFilter, QueryBuilder query);
/**
* scroll query
*
* @param indexs
* @param types
* @param postFilter
* @param query
* @param sort
* @param from
* @param size
* @param includes
* @param excludes
* @param highlightBuilder
* @param timeValue
* @return
*/
SearchResponse scroll(String[] indexs, String[] types, QueryBuilder postFilter, QueryBuilder query,
FieldSortBuilder sort, int from, int size, String[] includes, String[] excludes, HighlightBuilder highlightBuilder, TimeValue timeValue);
/**
* scroll query
*
* @param indexs
* @param types
* @param postFilter
* @param query
* @param sort
* @param from
* @param size
* @param highlightBuilder
* @param timeValue
* @return
*/
SearchResponse scroll(String[] indexs, String[] types, QueryBuilder postFilter, QueryBuilder query,
FieldSortBuilder sort, int from, int size, HighlightBuilder highlightBuilder, TimeValue timeValue);
/**
* es scroll query
*
* @param scrollId
* @param timeValue
*/
SearchResponse scroll(String scrollId, TimeValue timeValue);
}
src/main/java/com/zhiwei/qbjcutils/es/dao/impl/EsDaoImpl.java 0 → 100644
package com.zhiwei.qbjcutils.es.dao.impl;
import com.zhiwei.qbjcutils.es.dao.EsDao;
import com.zhiwei.qbjcutils.utils.Tools;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchScrollRequestBuilder;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.sort.FieldSortBuilder;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
/**
* EsDaoImpl
*
* @ClassName: EsDaoImpl
* @Description: EsDaoImpl
* @author shentao
* @date 2019年7月8日 上午11:31:41
*/
@Component
public final class EsDaoImpl implements EsDao {
@Resource(name ="esTransportClient")
private TransportClient client;
@Override
public SearchHits search(String[] indexs, String[] types, QueryBuilder postFilter, QueryBuilder query,
FieldSortBuilder sort, int from, int size, HighlightBuilder highlighter) {
SearchRequestBuilder searchRequestBuilder = client.prepareSearch(indexs);
if (!Tools.isNull(types)) {
searchRequestBuilder.setTypes(types);
}
if (!Tools.isNull(postFilter)) {
searchRequestBuilder.setPostFilter(postFilter);
}
if (!Tools.isNull(query)) {
searchRequestBuilder.setQuery(query);
}
if (!Tools.isNull(sort)) {
searchRequestBuilder.addSort(sort);
}
if (!Tools.isNull(from)) {
searchRequestBuilder.setFrom(from);
}
if (!Tools.isNull(size)) {
searchRequestBuilder.setSize(size);
}
if (!Tools.isNull(highlighter)) {
searchRequestBuilder.highlighter(highlighter);
}
SearchResponse response = searchRequestBuilder.execute().actionGet();
return response.getHits();
}
@Override
public SearchHits searchFatchSource(String[] indexs, String[] types, QueryBuilder postFilter, QueryBuilder query,
FieldSortBuilder sort, int from, int size, String[] includes, String[] excludes,
HighlightBuilder highlightBuilder) {
SearchRequestBuilder searchRequestBuilder = client.prepareSearch(indexs);
if (!Tools.isNull(types)) {
searchRequestBuilder.setTypes(types);
}
if (!Tools.isNull(postFilter)) {
searchRequestBuilder.setPostFilter(postFilter);
}
if (!Tools.isNull(query)) {
searchRequestBuilder.setQuery(query);
}
if (!Tools.isNull(sort)) {
searchRequestBuilder.addSort(sort);
}
if (!Tools.isNull(from)) {
searchRequestBuilder.setFrom(from);
}
if (!Tools.isNull(size)) {
searchRequestBuilder.setSize(size);
}
if (!Tools.isNull(highlightBuilder)) {
searchRequestBuilder.highlighter(highlightBuilder);
}
searchRequestBuilder.setFetchSource(includes, excludes);
SearchResponse response = searchRequestBuilder.execute().actionGet();
return response.getHits();
}
@Override
public Long count(String[] indexes, String[] types, QueryBuilder postFilter, QueryBuilder query) {
SearchRequestBuilder searchRequestBuilder = client.prepareSearch(indexes);
if (!Tools.isNull(types)) {
searchRequestBuilder.setTypes(types);
}
if (!Tools.isNull(postFilter)) {
searchRequestBuilder.setPostFilter(postFilter);
}
if (!Tools.isNull(query)) {
searchRequestBuilder.setQuery(query);
}
SearchResponse response = searchRequestBuilder.setFrom(0).setSize(0).execute().actionGet();
return response.getHits().getTotalHits();
}
@Override
public SearchResponse scroll(String[] indexs, String[] types, QueryBuilder postFilter, QueryBuilder query, FieldSortBuilder sort, int from, int size, String[] includes, String[] excludes, HighlightBuilder highlightBuilder, TimeValue timeValue) {
SearchRequestBuilder searchRequestBuilder = client.prepareSearch(indexs);
if (!Tools.isNull(types)) {
searchRequestBuilder.setTypes(types);
}
if (!Tools.isNull(postFilter)) {
searchRequestBuilder.setPostFilter(postFilter);
}
if (!Tools.isNull(query)) {
searchRequestBuilder.setQuery(query);
}
if (!Tools.isNull(sort)) {
searchRequestBuilder.addSort(sort);
}
if (!Tools.isNull(from)) {
searchRequestBuilder.setFrom(from);
}
if (!Tools.isNull(size)) {
searchRequestBuilder.setSize(size);
}
if(!Tools.isNull(includes) || !Tools.isNull(excludes)){
searchRequestBuilder.setFetchSource(includes, excludes);
}
if (!Tools.isNull(highlightBuilder)) {
searchRequestBuilder.highlighter(highlightBuilder);
}
if(!Tools.isNull(timeValue)){
searchRequestBuilder.setScroll(timeValue);
}
return searchRequestBuilder.execute().actionGet();
}
@Override
public SearchResponse scroll(String[] indexs, String[] types, QueryBuilder postFilter, QueryBuilder query, FieldSortBuilder sort, int from, int size, HighlightBuilder highlightBuilder, TimeValue timeValue) {
SearchRequestBuilder searchRequestBuilder = client.prepareSearch(indexs);
if (!Tools.isNull(types)) {
searchRequestBuilder.setTypes(types);
}
if (!Tools.isNull(postFilter)) {
searchRequestBuilder.setPostFilter(postFilter);
}
if (!Tools.isNull(query)) {
searchRequestBuilder.setQuery(query);
}
if (!Tools.isNull(sort)) {
searchRequestBuilder.addSort(sort);
}
if (!Tools.isNull(from)) {
searchRequestBuilder.setFrom(from);
}
if (!Tools.isNull(size)) {
searchRequestBuilder.setSize(size);
}
if (!Tools.isNull(highlightBuilder)) {
searchRequestBuilder.highlighter(highlightBuilder);
}
if(!Tools.isNull(timeValue)){
searchRequestBuilder.setScroll(timeValue);
}
return searchRequestBuilder.execute().actionGet();
}
@Override
public SearchResponse scroll(String scrollId, TimeValue timeValue) {
SearchScrollRequestBuilder searchScrollRequestBuilder = client.prepareSearchScroll(scrollId);
if(!Tools.isNull(timeValue)) {
searchScrollRequestBuilder.setScroll(timeValue);
}
return searchScrollRequestBuilder.execute().actionGet();
}
}
src/main/java/com/zhiwei/qbjcutils/pojo/dto/BaseObjectCacheDTO.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.dto;
import com.zhiwei.qbjcutils.pojo.entity.BaseObjectCache;
import lombok.*;
/**
* Description:
*
* @author LiuMingHuan
* @classname BaseObjectCacheDTO
* @date 2020/4/9
*/
@Getter
@Setter
@NoArgsConstructor
@AllArgsConstructor
public class BaseObjectCacheDTO extends BaseObjectCache {
private String mtag;
@Override
public String toString() {
return "BaseObjectCacheDTO{" +
"mtag='" + mtag + '\'' +
"} " + super.toString();
}
}
src/main/java/com/zhiwei/qbjcutils/pojo/emums/TimeUnit.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.emums;
import java.util.Calendar;
/**
* Description:
*
* @author LiuMingHuan
* @classname TimeUnit
* @date 2020/4/17
*/
public enum TimeUnit {
/**Second**/
SECOND(1000L),
/**MINUTE**/
MINUTE(60 * 1000L),
/**Hour**/
HOUR(60 * 60 * 1000L),
/**Day**/
DAY(24 * 60 * 60 * 1000L);
private Long timeMillSeconds;
private String name;
TimeUnit(long timeMillSeconds) {
this.name = this.toString();
this.timeMillSeconds = timeMillSeconds;
}
public long timeMillSeconds() {
return this.timeMillSeconds;
}
/**
* currentTimeMillSeconds
* @param timeType
* @param targetTime
* @return
*/
public static long currentTimeMillSeconds(String timeType, long targetTime) {
Calendar calendar = Calendar.getInstance();
calendar.setTimeInMillis(targetTime);
switch (timeType){
case "SECOND":
return SECOND.timeMillSeconds;
case "MINUTE":
return MINUTE.timeMillSeconds;
case "HOUR":
return HOUR.timeMillSeconds;
case "DAY":
return DAY.timeMillSeconds;
case "MONTH":
return calendar.getActualMaximum(Calendar.DAY_OF_MONTH) * DAY.timeMillSeconds;
case "YEAR":
return calendar.getActualMaximum(Calendar.DAY_OF_YEAR) * DAY.timeMillSeconds;
default:
return 0L;
}
}
}
src/main/java/com/zhiwei/qbjcutils/pojo/entity/BaseObjectCache.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.entity;
import lombok.AllArgsConstructor;
import lombok.NoArgsConstructor;
import lombok.ToString;
/**
* Description:
*
* @author LiuMingHuan
* @classname BaseObjectCache
* @date 2020/3/29
*/
@NoArgsConstructor
@AllArgsConstructor
@ToString
public class BaseObjectCache {
/**
* comment_num : -1
* like_num : -1
* real_source : 微信公众号
* gid : 1027174022
* origin : PC
* cname : 微信账号采集
* ind_full_text : ["长沙人注意!今年清明祭扫需先预约,这些你必须知道!","春暖花开芳草萋，又是一年清明时 按照传统 人们会在这天祭祀扫墓、踏青出游 受疫情影响 今年的清明注定与往年不同 文明祭扫 同舟共济，携手抗疫 在全国人民的共同努力下 疫情防控形势持续向好 但“革命”尚未结束，我们仍不能掉以轻心 清明时节，适逢全国上下共抗疫情关键时刻 为避免清明期间人员扎堆聚集 减少交叉感染风险 近期，全国多地发布倡议 提倡利用网络“云上祭扫”缅怀先人 为进一步弘扬优良传统、倡导文明新风 确保2020年清明节祭扫绿色、文明、低碳、安全 近日，湖南发布 《关于做好清明期间疫情防控和安全防范工作的通知》 湘疫防【2020】26号 倡导广大市民绿色低碳祭扫，推广网络祭扫 降低因人员聚集产生的疫情防控风险 对确有实地祭扫需求的 实行按日分时段限流预约祭扫 清明期间，如需现场祭扫 可通过以下方式预约： 01 点击下方二维码进入“云上清明”提前预约 02 拨打长沙市公墓电话提前预约 团团倡议 团子们： 　　今年，清明时节正值新冠肺炎疫情防控进入“下半场”的特殊时刻，为切实做好清明祭扫期间的疫情防控工作，营造安全、有序的祭祀环境，大力倡导绿色、文明、节俭的祭祀新风尚，弘扬移风易俗的文明新风，我们倡议： 安全祭祀，守护健康 提倡通过网络祭祀、撰写缅怀文章、选择通过公墓地提供的“代客祭扫”服务等方式，实现“云祭扫”和“云哀悼”；提倡外出祭扫人员做好个人防护，预约或错峰祭扫，不搞组团祭扫，简化祭扫流程，压缩祭扫时间。 文明祭祀，移风易俗 提倡以鲜花祭奠、植树纪念、踏青遥祭、家庭追思等文明、安全、低碳的方式，来替代烧纸钱、放爆竹等祭扫陋习，抵制迷信行为和低俗祭祀用品，将中华民族慎终追远的情感用现代文明的方式来表达。 绿色祭祀，爱护环境 倡导以献花、鞠躬等简约环保方式进行祭扫，不在路口、公园、小区周边绿化带内焚烧纸钱、燃放鞭炮，不影响他人正常生活，不影响公共环境卫生；不在林区焚烧纸钱、燃放鞭炮，防止造成火灾隐患。 以身作则，践行文明 人人争做文明新风的倡导者和传播者，自觉抵制和劝导不文明祭扫行为，以实际行动影响和带动身边的团员青年安全祭扫、践行文明、摒弃陋习。 团子们，参与疫情防控、践行文明祭祀、倡导移风易俗是你我共同的责任。保护好自身健康就是对家庭、对社会的最大负责，更是对逝者的最好告慰。让我们携起手来，从现在做起，从自身做起，从点滴做起，以文明祭祀的实际行动，弘扬时代新风，共度一个安全、文明、环保的清明节！ 本期小编 小巷 来源：长沙共青团新媒体中心 审稿：羅、格格 ☎ 0731-81859810 抖音号：1189978172 ✉ xcbtsw@163.com 团小青手机&微信：18674395454 爱 民 路 51 号 有态度 有深度 有温度 文明祭扫，从你我做起！"]
* stime : 1585296042222
* ind_title : ["长沙人注意!今年清明祭扫需先预约,这些你必须知道!"]
* source : 长沙共青团
* c1 : 134217728
* url : http://mp.weixin.qq.com/s?__biz=MzA5MjIxNDkxMQ==&mid=2651558191&idx=1&sn=50e5ad1a5fe372921b159569c1bf62ee&chksm=8b8fb23fbcf83b294d33b803e77af455456f747b3dff46f8d9f579423f5b3e07208df190d991&scene=0&xtrack=1#rd
* foreign : 0
* c2 : 8388608
* c3 : 524288
* c4 : 1000
* read_num : -1
* c5 : 143131624
* user_id : MzA5MjIxNDkxMQ==
* ctime : 1585296040212
* time : 1585295803000
* id : 20969db4040769f4b3d1f4182ba50672
* cid : 100030001
*/
private String real_source;
private int gid;
private String origin;
private String cname;
private String ind_full_text;
private long stime;
private String ind_title;
private String source;
private int c1;
private String url;
private int foreign;
private int c2;
private int c3;
private int c4;
private int c5;
private long ctime;
private long time;
private String id;
private int cid;
public String getReal_source() {
return real_source;
}
public void setReal_source(String real_source) {
this.real_source = real_source;
}
public int getGid() {
return gid;
}
public void setGid(int gid) {
this.gid = gid;
}
public String getOrigin() {
return origin;
}
public void setOrigin(String origin) {
this.origin = origin;
}
public String getCname() {
return cname;
}
public void setCname(String cname) {
this.cname = cname;
}
public String getInd_full_text() {
return ind_full_text;
}
public void setInd_full_text(String ind_full_text) {
this.ind_full_text = ind_full_text;
}
public long getStime() {
return stime;
}
public void setStime(long stime) {
this.stime = stime;
}
public String getInd_title() {
return ind_title;
}
public void setInd_title(String ind_title) {
this.ind_title = ind_title;
}
public String getSource() {
return source;
}
public void setSource(String source) {
this.source = source;
}
public int getC1() {
return c1;
}
public void setC1(int c1) {
this.c1 = c1;
}
public String getUrl() {
return url;
}
public void setUrl(String url) {
this.url = url;
}
public int getForeign() {
return foreign;
}
public void setForeign(int foreign) {
this.foreign = foreign;
}
public int getC2() {
return c2;
}
public void setC2(int c2) {
this.c2 = c2;
}
public int getC3() {
return c3;
}
public void setC3(int c3) {
this.c3 = c3;
}
public int getC4() {
return c4;
}
public void setC4(int c4) {
this.c4 = c4;
}
public int getC5() {
return c5;
}
public void setC5(int c5) {
this.c5 = c5;
}
public long getCtime() {
return ctime;
}
public void setCtime(long ctime) {
this.ctime = ctime;
}
public long getTime() {
return time;
}
public void setTime(long time) {
this.time = time;
}
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public int getCid() {
return cid;
}
public void setCid(int cid) {
this.cid = cid;
}
}
src/main/java/com/zhiwei/qbjcutils/pojo/entity/BaseObjectCacheKeyword.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.entity;
/**
* Description:
*
* @author LiuMingHuan
* @classname BaseObjectCacheKeyword
* @date 2020/5/7
*/
public class BaseObjectCacheKeyword extends BaseObjectCache {
private int wordFrequency;
public int getWordFrequency() {
return wordFrequency;
}
public void setWordFrequency(int wordFrequency) {
this.wordFrequency = wordFrequency;
}
}
src/main/java/com/zhiwei/qbjcutils/pojo/entity/CommonKeyword.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.entity;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.List;
/**
* Description:
*
* @author LiuMingHuan
* @classname KeywordCommon
* @date 2020/5/22
*/
@AllArgsConstructor
@NoArgsConstructor
@Data
public class CommonKeyword {
/**
* id
*/
private String id;
/**
* groupName
*/
private String groupName;
/**
* usedKeywords
*/
private List<String> usedKeywords;
/**
* keywordInfos all
*/
private List<KeywordInfo> keywordInfos;
/**
* platforms
*/
private List<String> platforms;
/**
* directGroups
*/
private List<String> directGroups;
/**
* groupUsed groupUsed
*/
private Boolean groupUsed;
/**
* show show
*/
private Boolean show;
/**
* project project
*/
private String project;
/**
* submitter submitter
*/
private String submitter;
/**
* createAt createAt
*/
private Long createAt;
/**
* updateAt updateAt
*/
private Long updateAt;
}
src/main/java/com/zhiwei/qbjcutils/pojo/entity/DirectGroup.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.entity;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import org.springframework.data.mongodb.core.mapping.Document;
import java.util.HashMap;
import java.util.List;
/**
* 定向监测组
*
* @ClassName: DirectGroup
* @Description: 定向监测组
* @author shentao
* @date 2019年7月8日 下午6:08:50
*/
@Getter
@Setter
@AllArgsConstructor
@NoArgsConstructor
@Document(collection = "qbjc_direct_group")
public class DirectGroup {
/**
* "id"
*/
private String id;
/**
* "name": 渠道组名
*/
private String name;
/**
* project 所属项目
*/
private String project;
/**
* submitter提交人员
*/
private String submitter;
/**
* 创建时间createAt
*/
private Long createAt;
/**
* 是否使用used
*/
private Boolean used;
/**
* 是否标注tag
*/
private Boolean tag;
/**
* memberList member使用中的list
*/
private List<String> memberList;
/**
* memberMaps member全信息list
*/
private List<HashMap<String, Object>> memberMaps;
}
src/main/java/com/zhiwei/qbjcutils/pojo/entity/Event.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.entity;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import org.springframework.data.mongodb.core.mapping.Document;
import java.util.List;
/**
* @ClassName: Event
* @Description: 事件
* @author shentao
* @date 2019年7月17日 上午10:27:48
*/
@Getter
@Setter
@AllArgsConstructor
@NoArgsConstructor
@Document(collection = "qbjc_event")
public class Event {
/**
* 事件id
*/
private String id;
/**
* 事件采集id
*/
private String collectionId;
/**
* 事件Name
*/
private String name;
/**
* 事件开始时间
*/
private Long startTime;
/**
* 事件结束时间
*/
private Long endTime;
/**
* 事件关键词
*/
private String keyword;
/**
* 事件提交人
*/
private String submitter;
/**
* 事件项目
*/
private String project;
/**
* 事件创建时间
*/
private Long createAt;
/**
* 事件更新时间
*/
private Long updateAt;
/**
* 事件传播状态spreadStatus
*/
private String spreadStatus;
/**
* 事件状态status（）
*/
private String status;
/**
* 事件标签markTag
*/
private String markTag;
/**
* 事件结果下载地址fileAddress
*/
private List<String> fileAddress;
/**
* 事件热度hotPoint
*/
private Double hotPoint;
/**
* 事件传播量
*/
private String spreadAmount;
/**
* 事件weibo互动量
*/
private Integer weiboInteraction;
/**
* 事件weibo评论量
*/
private Integer weiboReplyCount;
/**
* 事件weibo转发量
*/
private Integer weiboRetweetCount;
/**
* 网媒影响力
*/
private Integer mediaInfluence;
/**
* 网媒总条数
*/
private Integer mediaCount;
/**
* 平媒数据量
*/
private Integer printCount;
/**
* printH 平媒H因子量
*/
private Integer printInfluence;
/**
* 自媒体数据量
*/
private Integer selfMediaCount;
/**
* selfMediaH 自媒体H因子量
*/
private Integer selfMediaInfluence;
/**
* 微博影响力
*/
private Integer weiboInfluence;
/**
* 微博总条数
*/
private Integer weiboCount;
/**
* 微信影响力
*/
private Integer wechatInfluence;
/**
* 微信总条数
*/
private Integer wechatCount;
/**
* 头条影响力
*/
private Integer toutiaoInfluence;
/**
* 头条总条数
*/
private Integer toutiaoCount;
/**
* 是否为有上传文件
*/
private Integer fileSourceStatus;
/**
* 表格解析出错
*/
private Boolean errorStatus;
/**
* 事件数据源
*/
private String dataSource;
}
src/main/java/com/zhiwei/qbjcutils/pojo/entity/EventCollection.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.entity;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import org.springframework.data.mongodb.core.mapping.Document;
import java.util.List;
import java.util.Map;
/**
* @ClassName: EventCollection
* @Description: 事件采集
* @author shentao
* @date 2019年7月17日 上午10:28:40
*/
@Getter
@Setter
@AllArgsConstructor
@NoArgsConstructor
@Document(collection = "qbjc_event_collection")
public class EventCollection {
/**
* 事件采集id
*/
private String id;
/**
* 事件id
*/
private String eventId;
/**
* 事件type 0 普通|1 危机|2 带微博危机
*/
private Integer eventType;
/**
* 事件采集Name
*/
private String name;
/**
* 事件采集开始时间
*/
private Long startTime;
/**
* 事件采集结束时间
*/
private Long endTime;
/**
* 事件采集关键词
*/
private String keyword;
/**
* 已选关键词
*/
private String selectKeyword;
/**
* 事件采集提交人
*/
private String submitter;
/**
* 事件采集项目
*/
private String project;
/**
* 事件采集创建时间
*/
private Long createAt;
/**
* 事件采集更新时间
*/
private Long updateAt;
/**
* 事件采集状态status
*/
private String status;
/**
* 事件采集细节结果detials
*/
private String detials;
/**
* 事件采集结果下载地址fileAddress
*/
private List<String> fileAddress;
/**
* 事件历史采集记录
*/
private List<String> records;
/**
* 下载频率(下载人，/下载时间)
*/
private List<Map<String, Object>> downloadFrequency;
}
\ No newline at end of file
src/main/java/com/zhiwei/qbjcutils/pojo/entity/Keyword.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.entity;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.mapping.Document;
import java.util.HashMap;
import java.util.List;
/**
* Keyword
*
* @ClassName: Keyword
* @Description: Keyword
* @author shentao
* @date 2019年7月8日 下午4:06:52
*/
@Getter
@Setter
@AllArgsConstructor
@NoArgsConstructor
@Document(collection = "qbjc_keyword_back")
public class Keyword {
/**
* "id": keyTitle_project
*/
@Id
private String id;
/**
* "title": "小米产品",关键词主题名
*/
private String keyTitle;
/**
* "keyWords":关键词词组
*/
private List<String> keyWords;
/**
* "kwtimes":关键词词组
*/
private HashMap<String, Object> kwtimes;
/**
* project 所属项目
*/
private String project;
/**
* submitter提交人员
*/
private String submitter;
/**
* 创建时间createAt
*/
private Long createAt;
/**
* 创建时间是否使用used
*/
private Boolean used;
/**
* 定向监测是否使用dxIsUsed
*/
private Boolean dxIsUsed;
/**
* 平台列表
*/
private List<String> ptList;
/**
* 渠道列表
*/
private List<String> qdList;
}
src/main/java/com/zhiwei/qbjcutils/pojo/entity/KeywordInfo.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.entity;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* Description:
*
* @author LiuMingHuan
* @classname KeywordInfo
* @date 2020/5/21
*/
@AllArgsConstructor
@NoArgsConstructor
@Data
public class KeywordInfo {
/**
* name
*/
private String name;
/**
* used
*/
private Boolean used;
/**
* updateAt
*/
private Long updateAt;
/**
* submitter
*/
private String submitter;
}
src/main/java/com/zhiwei/qbjcutils/pojo/entity/MessagePlatform.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.entity;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import org.springframework.data.mongodb.core.mapping.Document;
import java.util.List;
/**
* @ClassName: MessagePlatform
* @Description: 最新消息流-平台实体
* @author shentao
* @date 2019年8月27日 下午7:14:28
*/
@Getter
@Setter
@AllArgsConstructor
@NoArgsConstructor
@Document(collection = "qbjc_message_platform")
public class MessagePlatform {
/**
* id 平台id
*/
private String id;
/**
* name 平台name
*/
private String name;
/**
* templateHeaders 模板头部id
*/
private List<String> templateHeaders;
/**
* createAt 创建时间
*/
private Long createAt;
/**
* submitter 提交人
*/
private String submitter;
}
src/main/java/com/zhiwei/qbjcutils/pojo/entity/NoiseRule.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.entity;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import org.springframework.data.mongodb.core.mapping.Document;
import java.util.List;
/**
* 去噪规则
*
* @ClassName: NoiseRule
* @Description: 去噪规则
* @author shentao
* @date 2019年7月8日 下午4:52:18
*/
@Getter
@Setter
@NoArgsConstructor
@Document(collection = "qbjc_noise_rule")
public class NoiseRule {
/**
* id
*/
private Long id;
/**
* ruleName 规则名
*/
private String ruleName;
/**
* ruleExplain 规则说明
*/
private String ruleExplain;
/**
* "ruleType": " (keyWords, channel)",
*/
private String ruleType;
/**
* keyWordPt 关键词规则适用平台
*/
private List<String> keyWordPt;
/**
* keyWordsInputOne 关键词规则第一个input输入框
*/
private String keyWordsInputOne;
/**
* andOr 且或者或
*/
private String andOr;
/**
* keyWordsInputTwo 关键词规则第二个input输入框
*/
private String keyWordsInputTwo;
/**
* 关键词汇总
*/
private String keyWords;
/**
* channelQd 渠道
*/
private String channelQd;
/**
* channelPt 渠道平台
*/
private List<String> channelPt;
/**
* project 所属项目
*/
private String project;
/**
* createAt 创建时间
*/
private Long createAt;
/**
* submitter 创建人
*/
private String submitter;
/**
* kwSearch 检索范围：全文、标题
*/
private String kwSearch;
/**
* used 是否被使用
*/
private Boolean used;
/**
* effectNum 生效次数
*/
private Long effectNum;
/**
* effectNumWeek 历史一周生效次数
*/
private List<Long> effectNumWeek;
}
src/main/java/com/zhiwei/qbjcutils/pojo/entity/NoiseRuleEntity.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.entity;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.springframework.data.mongodb.core.mapping.Document;
import java.util.List;
/**
* Description:
*
* @author LiuMingHuan
* @classname NoiseRuleEntity
* @date 2020/5/20
*/
@AllArgsConstructor
@NoArgsConstructor
@Data
@Document("qbjc_noise_rule_entity")
public class NoiseRuleEntity {
/**
* id
*/
private String id;
/**
* ruleName 规则名
*/
private String ruleName;
/**
* ruleContent 规则内容
*/
private String ruleContent;
/**
* ruleType 去噪类型
*/
private String ruleType;
/**
* kwSearch keyword 检索范围
*/
private String kwSearch;
/**
* currentNoiseNum 当日
*/
private Long currentNoiseNum;
/**
* totalNoiseNum 历史累计
*/
private Long totalNoiseNum;
/**
* noiseRuleExplain
*/
private String noiseRuleExplain;
/**
* platforms
*/
private List<String> platforms;
/**
* project
*/
private String project;
/**
* createAt
*/
private Long createAt;
/**
* updateAt
*/
private Long updateAt;
/**
* submitter
*/
private String submitter;
}
src/main/java/com/zhiwei/qbjcutils/pojo/entity/SentimentResult.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.entity;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.List;
import java.util.Map;
/**
* Description:
*
* @author LiuMingHuan
* @classname SentimentResult
* @date 2020/6/5
*/
@AllArgsConstructor
@NoArgsConstructor
@Data
public class SentimentResult {
private Long id;
private String text;
private List<Map<String, Object>> result;
}
src/main/java/com/zhiwei/qbjcutils/pojo/entity/Tag.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.entity;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import org.springframework.data.mongodb.core.mapping.Document;
/**
* Tag
*
* @ClassName: Tag
* @Description: Tag
* @author shentao
* @date 2019年7月8日 下午4:50:26
*/
@Getter
@Setter
@AllArgsConstructor
@NoArgsConstructor
@Document(collection = "qbjc_tag")
public class Tag {
/**
* id
*/
private Long id;
/**
* groupId 标签组id 删除时删除
*/
private Long groupId;
/**
* uniqueId
*/
private String uniqueId;
/**
* name 标签名称
*/
private String name;
/**
* groupName 标签组名称
*/
private String groupName;
/**
* createAt 标签创建时间
*/
private Long createAt;
/**
* submitter
*/
private String submitter;
/**
* project
*/
private String project;
/**
* show 是否呈现在事件列表
*/
private Boolean show;
}
src/main/java/com/zhiwei/qbjcutils/pojo/excelentity/BaseObjectExcelEntity.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.excelentity;
import com.alibaba.excel.annotation.ExcelProperty;
import com.alibaba.excel.annotation.write.style.ColumnWidth;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.springframework.data.mongodb.core.mapping.Document;
/**
* Description:
*
* @author LiuMingHuan
* @classname BaseObjectExcelEntity
* @date 2020/3/31
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
@Document(collection = "base_object_excel_entity")
public class BaseObjectExcelEntity {
@ExcelProperty(value = "id",index = 0)
@ColumnWidth(5)
private String id;
@ExcelProperty(value = "时间",index = 1)
@ColumnWidth(5)
private long time;
@ExcelProperty(value = "url",index = 2)
@ColumnWidth(8)
private String url;
@ExcelProperty(value = "标题",index = 3)
@ColumnWidth(12)
private String title;
@ExcelProperty(value = "内容",index = 4)
@ColumnWidth(12)
private String content;
@ExcelProperty(value = "平台",index = 5)
@ColumnWidth(7)
private String platform;
@ExcelProperty(value = "来源",index = 6)
@ColumnWidth(7)
private String source;
@ExcelProperty(value = "信源",index = 7)
@ColumnWidth(7)
private String origin;
@ExcelProperty(value = "foreign",index = 8)
@ColumnWidth(7)
private Integer foreign;
@ExcelProperty(value = "c1",index = 9)
@ColumnWidth(5)
private int c1;
@ExcelProperty(value = "c2",index = 10)
@ColumnWidth(5)
private int c2;
@ExcelProperty(value = "c3",index = 11)
@ColumnWidth(5)
private int c3;
@ExcelProperty(value = "c4",index = 12)
@ColumnWidth(5)
private int c4;
@ExcelProperty(value = "c5",index = 13)
@ColumnWidth(5)
private int c5;
@ExcelProperty(value = "采集时间",index = 14)
@ColumnWidth(7)
private long ctime;
@ExcelProperty(value = "cid",index = 15)
@ColumnWidth(7)
private int cid;
}
src/main/java/com/zhiwei/qbjcutils/pojo/excelentity/DirectChannel.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.excelentity;
import com.alibaba.excel.annotation.ExcelProperty;
import com.alibaba.excel.annotation.write.style.ColumnWidth;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
/**
* Description:
*
* @author LiuMingHuan
* @classname DirecChannel
* @date 2020/4/16
*/
@Data
@ToString
@AllArgsConstructor
@NoArgsConstructor
public class DirectChannel {
@ExcelProperty(value = "渠道组", index = 0)
@ColumnWidth(8)
private String directGroup;
@ExcelProperty(value = "渠道", index = 1)
@ColumnWidth(12)
private String channel;
@ColumnWidth(3)
@ExcelProperty(value = "平台", index = 2)
private String platform;
}
src/main/java/com/zhiwei/qbjcutils/pojo/excelentity/KeywordReader.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.excelentity;
import lombok.*;
/**
* Description:
*
* @author LiuMingHuan
* @classname KeywordReader
* @date 2020/4/22
*/
@Data
@AllArgsConstructor
public class KeywordReader {
}
src/main/java/com/zhiwei/qbjcutils/pojo/excelentity/MarkDownloadInfo.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.excelentity;
import com.alibaba.excel.annotation.ExcelProperty;
import com.alibaba.excel.annotation.write.style.ColumnWidth;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
/**
* @author shentao
* @ClassName: MarkDownloadInfo
* @Description: 标注下载表实体[序号 时间 标题 文本 地址 来源 平台 渠道 认证信息 采编权 粉丝数 认证类型 级别 领域 是否原发
* 原创作者 原创文本 命中词 标注时间 标注人 标签一 标签二 标签三 标签四 标签五]
* @date 2019年7月23日 上午9:38:21
*/
@Getter
@Setter
@AllArgsConstructor
@NoArgsConstructor
public class MarkDownloadInfo {
@ExcelProperty(value = "序号", index = 0)
@ColumnWidth(5)
private String id;
@ExcelProperty(value = "时间", index = 1)
@ColumnWidth(16)
private String time;
@ExcelProperty(value = "标题", index = 2)
@ColumnWidth(33)
private String title;
@ExcelProperty(value = "文本", index = 3)
@ColumnWidth(5)
private String content;
@ExcelProperty(value = "地址", index = 4)
@ColumnWidth(5)
private String url;
/**
* 固定平台
*/
@ExcelProperty(value = "平台", index = 5)
@ColumnWidth(10)
private String platform;
/**
* 信源 APP\PC\MOBI
*/
@ExcelProperty(value = "信源", index = 6)
@ColumnWidth(5)
private String origin;
/**
* c4 name realSource
*/
@ExcelProperty(value = "来源", index = 7)
@ColumnWidth(5)
private String clientFrom;
/**
* 来源 source
*/
@ExcelProperty(value = "渠道", index = 8)
@ColumnWidth(8)
private String source;
@ExcelProperty(value = "级别", index = 9)
@ColumnWidth(5)
private String level;
@ExcelProperty(value = "采编权", index = 10)
@ColumnWidth(5)
private String acquisitionRights;
@ExcelProperty(value = "领域", index = 11)
@ColumnWidth(5)
private String field;
@ExcelProperty(value = "认证信息", index = 12)
@ColumnWidth(10)
private String authenticationInformation;
@ExcelProperty(value = "粉丝数", index = 13)
@ColumnWidth(9)
private String fans;
@ExcelProperty(value = "认证类型", index = 14)
@ColumnWidth(5)
private String authenticationType;
@ExcelProperty(value = "是否原发", index = 15)
@ColumnWidth(8)
private String primary;
@ExcelProperty(value = "原创作者", index = 16)
@ColumnWidth(7)
private String rootSource;
@ExcelProperty(value = "原创文本", index = 17)
@ColumnWidth(7)
private String rootContent;
@ExcelProperty(value = "命中词", index = 18)
@ColumnWidth(7)
private String keywords;
@ExcelProperty(value = "UID", index = 19)
@ColumnWidth(5)
private String uid;
@ExcelProperty(value = "MID", index = 20)
@ColumnWidth(5)
private String mid;
@ExcelProperty(value = "标注时间", index = 21)
@ColumnWidth(8)
private String mtime;
@ExcelProperty(value = "标注人", index = 22)
@ColumnWidth(7)
private String mperson;
@ExcelProperty(value = "标签一", index = 23)
@ColumnWidth(7)
private String mtagOne;
@ExcelProperty(value = "标签二", index = 24)
@ColumnWidth(7)
private String mtagTwo;
@ExcelProperty(value = "标签三", index = 25)
@ColumnWidth(7)
private String mtagThree;
@ExcelProperty(value = "标签四", index = 26)
@ColumnWidth(7)
private String mtagFour;
@ExcelProperty(value = "标签五", index = 27)
@ColumnWidth(7)
private String mtagFive;
public void setCompleteTextMark(String time, String title, String content, String url, String platform, String origin,
String clientFrom, String source, String authenticationInformation, String acquisitionRights, String level,
String field, String primary, String rootSource, String mtime, String mperson) {
this.time = time;
this.title = title;
this.content = content;
this.url = url;
this.platform = platform;
this.origin = origin;
this.clientFrom = clientFrom;
this.source = source;
this.authenticationInformation = authenticationInformation;
this.acquisitionRights = acquisitionRights;
this.level = level;
this.field = field;
this.primary = primary;
this.rootSource = rootSource;
this.mtime = mtime;
this.mperson = mperson;
}
public void setIncompleteTextMark(String time, String content, String url, String platform, String origin, String clientFrom,
String source, String fans, String authenticationType, String level, String field, String primary,
String rootSource, String rootContent, String uid, String mid, String mtime, String mperson) {
this.time = time;
this.title = content;
this.content = content;
this.url = url;
this.platform = platform;
this.origin = origin;
this.clientFrom = clientFrom;
this.source = source;
this.fans = fans;
this.authenticationType = authenticationType;
this.level = level;
this.field = field;
this.primary = primary;
this.rootSource = rootSource;
this.rootContent = rootContent;
this.uid = uid;
this.mid = mid;
this.mtime = mtime;
this.mperson = mperson;
}
public void setQaTextMark(String time, String title, String content, String url, String platform,
String origin, String clientFrom,
String source, String mtime, String mperson) {
this.time = time;
this.title = title;
this.content = content;
this.url = url;
this.platform = platform;
this.origin = origin;
this.clientFrom = clientFrom;
this.source = source;
this.mtime = mtime;
this.mperson = mperson;
}
public void setVideoMark(String time, String title, String content, String url, String platform, String origin,
String clientFrom, String source, String mtime, String mperson) {
this.time = time;
this.title = title;
this.content = content;
this.url = url;
this.platform = platform;
this.origin = origin;
this.clientFrom = clientFrom;
this.source = source;
this.mtime = mtime;
this.mperson = mperson;
}
}
src/main/java/com/zhiwei/qbjcutils/pojo/excelentity/SentimentInfo.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.excelentity;
import com.alibaba.excel.annotation.ExcelProperty;
import lombok.Data;
/**
* Description:
*
* @author LiuMingHuan
* @classname SentimentInfo
* @date 2020/6/11
*/
@Data
public class SentimentInfo {
@ExcelProperty(value = "id", index = 0)
private String id;
@ExcelProperty(value = "url", index = 1)
private String url;
@ExcelProperty(value = "情感倾向", index = 10)
private String sentiment;
@ExcelProperty(value = "情感倾向2", index = 11)
private String sentimentBack;
}
src/main/java/com/zhiwei/qbjcutils/pojo/excelentity/UploadData.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.excelentity;
import com.alibaba.excel.annotation.ExcelProperty;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
/**
* Description:
*
* @author LiuMingHuan
* @classname UploadData
* @date 2020/4/2
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
@ToString
public class UploadData {
@ExcelProperty(value = "平台",index = 0)
private String platform;
@ExcelProperty(value = "来源",index = 1)
private String source;
@ExcelProperty(value = "时间",index = 2)
private String time;
@ExcelProperty(value = "标题",index = 3)
private String title;
@ExcelProperty(value = "地址",index = 4)
private String url;
@ExcelProperty(value = "文本",index = 5)
private String content;
@ExcelProperty(value = "UID",index = 6)
private String uid;
@ExcelProperty(value = "MID",index = 7)
private String mid;
}
src/main/java/com/zhiwei/qbjcutils/pojo/excelhadler/PlatformCountHandler.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.excelhadler;
import com.alibaba.excel.metadata.CellData;
import com.alibaba.excel.metadata.Head;
import com.alibaba.excel.write.handler.CellWriteHandler;
import com.alibaba.excel.write.metadata.holder.WriteSheetHolder;
import com.alibaba.excel.write.metadata.holder.WriteTableHolder;
import org.apache.poi.hssf.util.HSSFColor;
import org.apache.poi.ss.usermodel.*;
import java.util.List;
/**
* Description:
*
* @author LiuMingHuan
* @classname PlatformCountHandler
* @date 2020/4/20
*/
public class PlatformCountHandler implements CellWriteHandler {
@Override
public void beforeCellCreate(WriteSheetHolder writeSheetHolder, WriteTableHolder writeTableHolder, Row row, Head head, Integer integer, Integer integer1, Boolean aBoolean) {
}
@Override
public void afterCellCreate(WriteSheetHolder writeSheetHolder, WriteTableHolder writeTableHolder, Cell cell, Head head, Integer integer, Boolean aBoolean) {
}
@Override
public void afterCellDispose(WriteSheetHolder writeSheetHolder, WriteTableHolder writeTableHolder, List<CellData> list, Cell cell, Head head, Integer integer, Boolean aBoolean) {
Workbook workbook = writeSheetHolder.getSheet().getWorkbook();
CellStyle cellStyle = workbook.createCellStyle();
cellStyle.setAlignment(HorizontalAlignment.CENTER);
if (cell.getRowIndex() == 0) {
cellStyle.setFillBackgroundColor(HSSFColor.HSSFColorPredefined.PALE_BLUE.getIndex());
}
cell.setCellStyle(cellStyle);
}
}
src/main/java/com/zhiwei/qbjcutils/pojo/excellistener/DirectChannelListener.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.excellistener;
import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.event.AnalysisEventListener;
import com.zhiwei.qbjcutils.pojo.excelentity.DirectChannel;
import lombok.Getter;
import java.util.ArrayList;
import java.util.List;
/**
* Description:
*
* @author LiuMingHuan
* @classname DirectChannelListener
* @date 2020/4/16
*/
@Getter
public class DirectChannelListener extends AnalysisEventListener<DirectChannel> {
private List<DirectChannel> directChannels= new ArrayList<>();
@Override
public void invoke(DirectChannel directChannel, AnalysisContext analysisContext) {
directChannels.add(directChannel);
}
@Override
public void doAfterAllAnalysed(AnalysisContext analysisContext) {
}
}
src/main/java/com/zhiwei/qbjcutils/pojo/excellistener/Sentimentistener.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.excellistener;
import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.event.AnalysisEventListener;
import com.zhiwei.qbjcutils.pojo.excelentity.SentimentInfo;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
/**
* Description:
*
* @author LiuMingHuan
* @classname SentimenListener
* @date 2020/6/11
*/
@Data
public class Sentimentistener extends AnalysisEventListener<SentimentInfo> {
private List<SentimentInfo> infos = new ArrayList<>();
@Override
public void invoke(SentimentInfo sentimentInfo, AnalysisContext analysisContext) {
infos.add(sentimentInfo);
}
@Override
public void doAfterAllAnalysed(AnalysisContext analysisContext) {
}
}
src/main/java/com/zhiwei/qbjcutils/pojo/excellistener/UploadDataListener.java 0 → 100644
package com.zhiwei.qbjcutils.pojo.excellistener;
import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.event.AnalysisEventListener;
import com.zhiwei.qbjcutils.pojo.excelentity.UploadData;
import lombok.Data;
import lombok.EqualsAndHashCode;
import java.util.ArrayList;
import java.util.List;
/**
* Description:
*
* @author LiuMingHuan
* @classname UploadDataListener
* @date 2020/4/2
*/
@EqualsAndHashCode(callSuper = true)
@Data
public class UploadDataListener extends AnalysisEventListener<UploadData> {
private List<UploadData> infos =new ArrayList<>();
@Override
public void invoke(UploadData uploadData, AnalysisContext analysisContext) {
uploadData.setUid("无".equals(uploadData.getUid()) ? "" : uploadData.getUid());
uploadData.setMid("无".equals(uploadData.getMid()) ? "" : uploadData.getMid());
infos.add(uploadData);
}
@Override
public void doAfterAllAnalysed(AnalysisContext analysisContext) {
}
}
src/main/java/com/zhiwei/qbjcutils/redis/RedisTools.java 0 → 100644
package com.zhiwei.qbjcutils.redis;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.qbjcutils.config.GenericAttribute;
import com.zhiwei.qbjcutils.pojo.entity.MessagePlatform;
import com.zhiwei.qbjcutils.pojo.entity.Tag;
import org.redisson.api.RBucket;
import org.redisson.api.RMap;
import org.redisson.api.RScoredSortedSet;
import org.redisson.api.RedissonClient;
import org.redisson.client.codec.StringCodec;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
/**
* es 方法类
*
* @author shentao
* @ClassName: RedisTools
* @Description: es 方法类
* @date 2019年7月3日 上午9:00:10
*/
@Service
public class RedisTools {
private static final String KEY_INFO = "YUQING:Info:";
private final RedissonClient redissonClient;
public RedisTools(RedissonClient redissonClient) {
this.redissonClient = redissonClient;
}
/**
* key是否存在
*
* @param @param key
* @param @return 设定文件
* @return boolean 返回类型
* @Title: exists
* @Description: key是否存在
*/
public boolean exists(String key) {
RBucket<?> bucket = redissonClient.getBucket(key, StringCodec.INSTANCE);
return bucket.isExists();
}
/**
* 获取Score降序 info
*
* @param @param key
* @param @return 设定文件
* @return List<String> 返回类型
* @Title: zrevrange
* @Description: 获取Score降序 info
*/
public List<String> zrevrange(String key, int start, int end) {
RScoredSortedSet<String> set = redissonClient.getScoredSortedSet(key, StringCodec.INSTANCE);
return new ArrayList<>(set.valueRangeReversed(start, end));
}
/**
* 计算在有序集合中指定区间分数的成员数
*
* @param @param key
* @param @param startScore
* @param @param startScoreInclusive
* @param @param endScore
* @param @param endScoreInclusive
* @param @return 设定文件
* @return int 返回类型
* @Title: zcount
* @Description: 计算在有序集合中指定区间分数的成员数
*/
public int zcount(String key, double startScore, boolean startScoreInclusive, double endScore,
boolean endScoreInclusive) {
RScoredSortedSet<String> set = redissonClient.getScoredSortedSet(key, StringCodec.INSTANCE);
return set.count(startScore, startScoreInclusive, endScore, endScoreInclusive);
}
/**
* redis get String 方法
*
* @param @param key
* @param @return 设定文件
* @return String 返回类型
* @Title: get
* @Description: redis get String 方法
*/
public String get(String key) {
RBucket<String> bucket = redissonClient.getBucket(key, StringCodec.INSTANCE);
return bucket.get();
}
/**
* redis get RMap方法
*
* @param key
* @return
*/
public RMap<String, Object> getRMap(String key) {
return redissonClient.getMap(key, StringCodec.INSTANCE);
}
/**
* redis set String 方法
*
* @param @param key
* @param @param value 设定文件
* @return void 返回类型
* @Title: set
* @Description: redis set String 方法
*/
public void set(String key, String value) {
RBucket<String> bucket = redissonClient.getBucket(key, StringCodec.INSTANCE);
bucket.set(value);
}
/**
* redis setex String 方法
*
* @param @param key
* @param @param value 设定文件
* @return void 返回类型
* @Title: set
* @Description: redis setex String 方法
*/
public void setex(String key, String value, long timeToLive, TimeUnit timeUnit) {
RBucket<String> bucket = redissonClient.getBucket(key, StringCodec.INSTANCE);
bucket.set(value, timeToLive, timeUnit);
}
/**
* 统计SortSet总量
*
* @param @param key
* @param @return 设定文件
* @return long 返回类型
* @Title: countSortSet
* @Description: 统计SortSet总量
*/
public long countSortSet(String key) {
RScoredSortedSet<String> set = redissonClient.getScoredSortedSet(key, StringCodec.INSTANCE);
return set.size();
}
/**
* 获取info-Key
*
* @param @param pt
* @param @param keyWord
* @param @param project
* @param @return 设定文件
* @return String 返回类型
* @Title: getInfoKey
* @Description: 获取info-Key
*/
public String getInfoKey(String pt, String keyWord, String project) {
return new StringBuilder().append(KEY_INFO).append(project).append(":").append(pt).append(":")
.append(keyWord).toString();
}
/**
* 获取对应key的全部消息流集合
* @param infoKey
* @return
*/
public RScoredSortedSet<String> getAllMessageFlowEntity(String infoKey) {
return redissonClient.getScoredSortedSet(infoKey, StringCodec.INSTANCE);
}
/**
* 获取redis中保存的原有平台信息
* @param key
* @return
*/
public List<MessagePlatform> getAllMessagePlatform(String key) {
RMap<String, String> map = redissonClient.getMap(key, StringCodec.INSTANCE);
return new ArrayList<>(map.values()).stream().map(m -> JSONObject.parseObject(m, MessagePlatform.class)).collect(Collectors.toList());
}
/**
* 获取指定时段消息流缓存
* @param infoKey
* @param startIndex
* @param endIndex
* @return
*/
public List<JSONObject> getMessageFlowEntityWithTime(String infoKey, long startIndex, long endIndex) {
RScoredSortedSet<String> scoredSortedSet = redissonClient.getScoredSortedSet(infoKey, StringCodec.INSTANCE);
return new ArrayList<>(scoredSortedSet.valueRange(startIndex, false, endIndex, true))
.stream().map(JSONObject::parseObject).collect(Collectors.toList());
}
/**
* 获取全部平台信息
*/
public Map<String, String> getAllPlatForms() {
String key = GenericAttribute.SYSTEM_KEY+"platform";
return redissonClient.getMap(key, StringCodec.INSTANCE);
}
/**
*
* @param res
*/
public void cachePlatForms(Map<String, String> res) {
String key = GenericAttribute.SYSTEM_KEY+"platform";
RMap<String, String> map = redissonClient.getMap(key, StringCodec.INSTANCE);
map.putAll(res);
}
/**
* 更新缓存中的tags
* @param res
*/
public void setAllTags(List<Tag> res) {
String key = GenericAttribute.SYSTEM_KEY + "tag";
RBucket<String> bucket = redissonClient.getBucket(key, StringCodec.INSTANCE);
bucket.set(JSONObject.toJSONString(res));
}
}
src/main/java/com/zhiwei/qbjcutils/service/ClassifyUploadDataService.java 0 → 100644
package com.zhiwei.qbjcutils.service;
/**
* Description:
*
* @author LiuMingHuan
* @classname ClassfyUploadDataService
* @date 2020/4/2
*/
public interface ClassifyUploadDataService {
/**
* 分类上传数据
* @param path 文件路径
* @param project 项目
*/
void classifyUploadData(String path, String project);
}
src/main/java/com/zhiwei/qbjcutils/service/DirectChannelService.java 0 → 100644
package com.zhiwei.qbjcutils.service;
/**
* Description:
*
* @author LiuMingHuan
* @classname DirectChannelService
* @date 2020/4/16
*/
public interface DirectChannelService {
/**
* 定向渠道表格上传-新增逻辑（监测系统已实现该功能）
* @param project
* @param path
*/
@Deprecated
void readAndAddDirectChannel(String project, String path) throws Exception;
/**
* 定向渠道表格上传-补充逻辑（监测系统已实现该功能）
* @param project
* @param path
*/
@Deprecated
void read(String project, String path) throws Exception;
}
src/main/java/com/zhiwei/qbjcutils/service/MarkDataCountService.java 0 → 100644
package com.zhiwei.qbjcutils.service;
/**
* Description:
*
* @author LiuMingHuan
* @classname MarkDataCountService
* @date 2020/8/18
*/
public interface MarkDataCountService {
/**
* 按需统计标注库指定时段各平台数据量(excel导出)
* @param startTime
* @param endTime
* @param timeType
* @param timeUnit
* @param project
*/
void outputMarkDataCountByPlatforms(long startTime, long endTime, String timeType, String timeUnit, String project);
}
src/main/java/com/zhiwei/qbjcutils/service/MarkDataOutputService.java 0 → 100644
package com.zhiwei.qbjcutils.service;
/**
* Description:
*
* @author LiuMingHuan
* @classname MarkDataCountOutputService
* @date 2020/3/31
*/
public interface MarkDataOutputService {
}
src/main/java/com/zhiwei/qbjcutils/service/MessageFlowService.java 0 → 100644
package com.zhiwei.qbjcutils.service;
import com.zhiwei.qbjcutils.pojo.dto.BaseObjectCacheDTO;
import com.zhiwei.qbjcutils.pojo.entity.BaseObjectCache;
import com.zhiwei.qbjcutils.pojo.entity.BaseObjectCacheKeyword;
import com.zhiwei.qbjcutils.pojo.entity.MessagePlatform;
import com.zhiwei.qbjcutils.pojo.excelentity.BaseObjectExcelEntity;
import java.util.List;
/**
* Description:
*
* @author LiuMingHuan
* @classname CacheBaseObjectService
* @date 2020/3/30
*/
public interface MessageFlowService {
/**
* redis指定平台关键词项目数据导出 - mongo
* @param platform
* @param keyword
* @param project
* @return
*/
List<BaseObjectCache> getBaseObjectCacheFromMessageFlow(String platform,String keyword,String project);
/**
* redis指定平台关键词项目数据导出（附带词频）- mongo
* @param platform
* @param keyword
* @param project
* @return
*/
List<BaseObjectCacheKeyword> getBaseObjectCacheKeywordFromMessageFlow(String platform, String keyword, String project);
/**
* redis指定平台关键词项目数据导出 - excel
* @param platform
* @param keyword
* @param project
* @return
*/
List<BaseObjectExcelEntity> getBaseObjectCacheExcelFromMessageFlow(String platform,String keyword,String project);
/**
* redis指定平台关键词项目数据导出（附带标签） - mongo
* @param startTime
* @param endTime
* @param platform
* @param keyword
* @param project
* @return
*/
List<BaseObjectCacheDTO> getMessageFlowEntityWithTime(Long startTime, Long endTime, MessagePlatform platform, String keyword, String project);
/**
* 读取对应集合生成语料库（情感倾向匹配使用）
* @param collectionName
*/
void generateText(String collectionName);
}
src/main/java/com/zhiwei/qbjcutils/service/PlatformService.java 0 → 100644
package com.zhiwei.qbjcutils.service;
import com.zhiwei.qbjcutils.pojo.entity.MessagePlatform;
import java.util.List;
/**
* Description:
*
* @author LiuMingHuan
* @classname PlatformService
* @date 2020/4/3
*/
public interface PlatformService {
/**
* 获取全部平台（缓存日更新）
* @return
*/
List<MessagePlatform> getAllPlatforms();
/**
* 获取对应平台具体信息
* @param name
* @return
*/
MessagePlatform getPlatformByName(String name);
/**
* 按 c5_origin_foreign 组装header匹配 未匹配返回null
* @param c5
* @param origin
* @param foreign
* @return
*/
MessagePlatform getPlatform(int c5, String origin, int foreign);
/**
* 平台自动更新
*/
void autoUpdate();
/**
* 查询全量平台（mongo）
* @return
*/
List<MessagePlatform> getPlatforms();
}
src/main/java/com/zhiwei/qbjcutils/service/impl/ClassifyUploadDataServiceImpl.java 0 → 100644
package com.zhiwei.qbjcutils.service.impl;
import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.ExcelWriter;
import com.alibaba.excel.support.ExcelTypeEnum;
import com.alibaba.excel.write.metadata.WriteSheet;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.base.category.ClassB;
import com.zhiwei.qbjcutils.config.GenericAttribute;
import com.zhiwei.qbjcutils.es.EsIndexTools;
import com.zhiwei.qbjcutils.es.dao.EsDao;
import com.zhiwei.qbjcutils.pojo.excelentity.UploadData;
import com.zhiwei.qbjcutils.pojo.excellistener.UploadDataListener;
import com.zhiwei.qbjcutils.service.ClassifyUploadDataService;
import com.zhiwei.qbjcutils.utils.TimeUtils;
import com.zhiwei.qbjcutils.utils.Tools;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.springframework.stereotype.Service;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
/**
* Description:
*
* @author LiuMingHuan
* @classname ClassifyUploadDataServiceImpl
* @date 2020/4/2
*/
@Service
public class ClassifyUploadDataServiceImpl implements ClassifyUploadDataService {
private final EsDao esDao;
private final EsIndexTools esIndexTools;
public ClassifyUploadDataServiceImpl(EsDao esDao, EsIndexTools esIndexTools) {
this.esDao = esDao;
this.esIndexTools = esIndexTools;
}
@Override
public void classifyUploadData(String path, String project) {
try {
List<UploadData> markData = new ArrayList<>();
List<UploadData> dwData = new ArrayList<>();
List<UploadData> unchecked = new ArrayList<>();
UploadDataListener listener = new UploadDataListener();
try (InputStream in = new FileInputStream(path)) {
EasyExcel.read(in).sheet(0).registerReadListener(listener).head(UploadData.class).doRead();
} catch (Exception e) {
e.printStackTrace();
}
int i = 0;
for (UploadData info : listener.getInfos()) {
if (i >= 5000) {
break;
}
if (checkMarkExist(info, project)) {
markData.add(info);
} else if (checkDwExist(info)) {
dwData.add(info);
} else {
unchecked.add(info);
}
i++;
}
transportForm(Arrays.asList(markData, dwData, unchecked));
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* 标注库是否存在该条消息
* @param info
* @return
*/
private boolean checkMarkExist(UploadData info, String project) throws Exception {
if (Tools.isEmpty(info.getTitle()) || Tools.isEmpty(info.getSource()) || Tools.isEmpty(info.getUrl())) {
return false;
}
String[] markIndexes = esIndexTools.getMarkIndexs();
String[] types = new String[]{ClassB.TypeB.COMPLETE.name(), ClassB.TypeB.INCOMPLETE.name(), ClassB.TypeB.QA.name(), ClassB.TypeB.VIDEO.name()};
//优先指定url查询
BoolQueryBuilder urlQuery = QueryBuilders.boolQuery()
.must(QueryBuilders.boolQuery()
.should(QueryBuilders.termQuery("url", info.getUrl()))
.should(QueryBuilders.boolQuery()
.must(QueryBuilders.matchPhraseQuery("ind_full_text", info.getTitle()))
.must(QueryBuilders.termQuery("source", info.getSource()))))
.must(QueryBuilders.termQuery("mgroup", project));
SearchHits searchHits = esDao.searchFatchSource(markIndexes, types, null, urlQuery, null, 0, 10, new String[]{"id", "url"}, null, null);
if (searchHits.getTotalHits() >= 1) {
for (SearchHit hit : searchHits.getHits()) {
JSONObject source = JSONObject.parseObject(hit.getSourceAsString());
if (new URL(source.getString("url")).getHost().equals(new URL(info.getUrl()).getHost())) {
return true;
}
}
}
return false;
}
/**
* 舆情库是否存在该条消息
* @param info
* @return
*/
private boolean checkDwExist(UploadData info) throws Exception {
if(Tools.isEmpty(info.getTitle())|| Tools.isEmpty(info.getSource()) || Tools.isEmpty(info.getUrl())){
return false;
}
long lastMonth = TimeUtils.getFirstDayOfLastMonth(TimeUtils.TIME_FORMATE.parse(info.getTime()).getTime());
long nextMonth = TimeUtils.getFirstDayOfNextMonth(TimeUtils.TIME_FORMATE.parse(info.getTime()).getTime());
String[] indexes = esIndexTools.getAllIndex(lastMonth, nextMonth);
String[] types = new String[]{"status"};
//优先指定url查询
BoolQueryBuilder urlQuery = QueryBuilders.boolQuery()
.must(QueryBuilders.boolQuery()
.should(QueryBuilders.termQuery("url", info.getUrl()))
.should(QueryBuilders.boolQuery()
.must(QueryBuilders.matchPhraseQuery("ind_full_text", info.getTitle()))
.must(QueryBuilders.termQuery("source", info.getSource()))));
SearchHits searchHits = esDao.searchFatchSource(indexes, types, null, urlQuery, null, 0, 10, new String[]{"id", "url"}, null, null);
if (searchHits.getTotalHits() >= 1) {
for (SearchHit hit : searchHits.getHits()) {
JSONObject source = JSONObject.parseObject(hit.getSourceAsString());
if (new URL(source.getString("url")).getHost().equals(new URL(info.getUrl()).getHost())) {
return true;
}
}
}
return false;
}
/**
* 表格划分导出
* @param data
*/
private void transportForm(List<List<UploadData>> data) {
for (int i = 0; i < data.size(); i++) {
String filename = GenericAttribute.FILE_PATH + (i == 0 ? "mark" : (i == 1 ? "dw" : "unchecked"));
try (OutputStream out = new FileOutputStream(filename)) {
ExcelWriter writer = EasyExcel.write(out).needHead(true).head(UploadData.class).excelType(ExcelTypeEnum.XLSX).build();
WriteSheet writeSheet = EasyExcel.writerSheet(0, "总表").needHead(true).head(UploadData.class).build();
writer.write(data.get(i), writeSheet);
writer.finish();
out.flush();
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
src/main/java/com/zhiwei/qbjcutils/service/impl/DirectChannelServiceImpl.java 0 → 100644
package com.zhiwei.qbjcutils.service.impl;
import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.ExcelWriter;
import com.alibaba.excel.support.ExcelTypeEnum;
import com.alibaba.excel.write.metadata.WriteSheet;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.qbjcutils.config.GenericAttribute;
import com.zhiwei.qbjcutils.dao.DirectGroupDao;
import com.zhiwei.qbjcutils.dubbo.DubboClient;
import com.zhiwei.qbjcutils.pojo.entity.DirectGroup;
import com.zhiwei.qbjcutils.pojo.excelentity.DirectChannel;
import com.zhiwei.qbjcutils.pojo.excellistener.DirectChannelListener;
import com.zhiwei.qbjcutils.service.DirectChannelService;
import com.zhiwei.qbjcutils.utils.Tools;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/**
* Description:
*
* @author LiuMingHuan
* @classname DirectChannelServiceImpl
* @date 2020/4/16
*/
@Service
public class DirectChannelServiceImpl implements DirectChannelService {
private static final String SYSTEM_MANAGER = "系统管理员";
private final DubboClient client;
private final DirectGroupDao directGroupDao;
public DirectChannelServiceImpl(DubboClient client, DirectGroupDao directGroupDao) {
this.client = client;
this.directGroupDao = directGroupDao;
}
@Override
public void readAndAddDirectChannel(String project, String path) throws Exception {
operate(project, path, "insert");
}
@Override
public void read(String project, String path) throws Exception {
operate(project, path, "update");
}
private void operate(String project, String path,String type) throws Exception {
DirectChannelListener channelListener = new DirectChannelListener();
try (InputStream in = new FileInputStream(path)) {
EasyExcel.read(in).sheet(0).head(DirectChannel.class).registerReadListener(channelListener).doRead();
} catch (Exception e) {
e.printStackTrace();
}
List<DirectChannel> directChannels = channelListener.getDirectChannels();
Map<String, List<DirectChannel>> groups = directChannels.stream().collect(Collectors.groupingBy(DirectChannel::getDirectGroup));
List<DirectChannel> unMatched=new ArrayList<>();
for (Map.Entry<String, List<DirectChannel>> entry : groups.entrySet()) {
int groupLimit = 0;
DirectGroup group = "insert".equals(type) ? new DirectGroup(null, entry.getKey(), project, SYSTEM_MANAGER, System.currentTimeMillis(), true, true,
new ArrayList<>(), new ArrayList<>())
: directGroupDao.findFirst(new Query(Criteria.where("name").is(entry.getKey())));
for (DirectChannel directChannel : entry.getValue()) {
JSONObject res = client.searchCollectedAccountList(getMediaId(directChannel.getPlatform()), directChannel.getChannel(), true, 1, 1);
List<JSONObject> rows = res.getJSONArray("rows").toJavaList(JSONObject.class);
if (!Tools.isEmpty(rows)) {
JSONObject info = rows.get(0);
groupLimit = getDirectGroup(group, directChannel.getPlatform(), directChannel.getChannel(), info.getString("userId"), groupLimit);
} else {
unMatched.add(directChannel);
}
}
if("insert".equals(type)){
directGroupDao.insert(group);
}else {
directGroupDao.updateFirst(new Query(Criteria.where("_id").is(group.getId())), Tools.bean2Update(group));
}
}
getUnMatched(unMatched, project);
}
/**
* containsDirectMember
* @param mid
* @param directGroup
* @return
*/
public boolean containsDirectMember(String mid, DirectGroup directGroup) {
if (!Tools.isEmpty(directGroup.getMemberMaps())) {
List<String> mids = directGroup.getMemberMaps().stream()
.filter(a -> Boolean.parseBoolean(String.valueOf(a.get("used")))).map(a -> a.get("id") + "").collect(Collectors.toList());
return mids.contains(mid);
}
return false;
}
/**
* getDirectGroup
* @param directGroup
* @param platform
* @param name
* @param url
* @param groupLimit
* @return
* @throws Exception
*/
public int getDirectGroup(DirectGroup directGroup, String platform, String name, String url, int groupLimit) {
String mid = platform + "_" + name + "_" + url;
if (!containsDirectMember(mid, directGroup)) {
groupLimit++;
boolean isUsed = groupLimit <= 500;
HashMap<String, Object> memberHashMap = new HashMap<>(8);
memberHashMap.put("createAt", System.currentTimeMillis());
memberHashMap.put("id", mid);
memberHashMap.put("pt", platform);
memberHashMap.put("name", name);
memberHashMap.put("url", url);
memberHashMap.put("used", isUsed);
List<String> memberList = !Tools.isNull(directGroup.getMemberList()) ? directGroup.getMemberList()
: new ArrayList<>();
List<HashMap<String, Object>> existDirectGroupMapList = !Tools.isNull(directGroup.getMemberMaps())
? directGroup.getMemberMaps()
: new ArrayList<>();
if (isUsed) {
memberList.add(mid);
}
existDirectGroupMapList.add(memberHashMap);
directGroup.setMemberList(memberList);
directGroup.setMemberMaps(existDirectGroupMapList);
}
return groupLimit;
}
/**
*
* @param unMatched
*/
private void getUnMatched(List<DirectChannel> unMatched, String project) {
String fileName = project + "_" + System.currentTimeMillis() + "_UnMatched.xlsx";
try (OutputStream out = new FileOutputStream(GenericAttribute.FILE_PATH + fileName)) {
ExcelWriter writer = EasyExcel.write(out).needHead(true).head(DirectChannel.class).excelType(ExcelTypeEnum.XLSX).build();
WriteSheet writeSheet = EasyExcel.writerSheet(0, "总表").head(DirectChannel.class).build();
writer.write(unMatched, writeSheet);
writer.finish();
out.flush();
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* 获取mediaId
* @param pt
* @return
*/
private int getMediaId(String pt) {
int mediaId = 0;
switch (pt) {
case "微博":
mediaId = 4;
break;
case "微信":
mediaId = 3;
break;
case "网媒":
mediaId = 1;
break;
case "今日头条":
mediaId = 5;
break;
case "平媒":
mediaId = 2;
break;
case "抖音":
mediaId = 36;
break;
default:
break;
}
return mediaId;
}
}
src/main/java/com/zhiwei/qbjcutils/service/impl/MarkDataContServiceImpl.java 0 → 100644
package com.zhiwei.qbjcutils.service.impl;
import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.ExcelWriter;
import com.alibaba.excel.support.ExcelTypeEnum;
import com.alibaba.excel.write.metadata.WriteSheet;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.base.category.ClassB;
import com.zhiwei.qbjcutils.config.GenericAttribute;
import com.zhiwei.qbjcutils.es.EsIndexTools;
import com.zhiwei.qbjcutils.es.dao.EsDao;
import com.zhiwei.qbjcutils.pojo.emums.TimeUnit;
import com.zhiwei.qbjcutils.pojo.entity.MessagePlatform;
import com.zhiwei.qbjcutils.pojo.excelhadler.PlatformCountHandler;
import com.zhiwei.qbjcutils.service.MarkDataCountService;
import com.zhiwei.qbjcutils.service.PlatformService;
import com.zhiwei.qbjcutils.utils.TimeUtils;
import com.zhiwei.qbjcutils.utils.Tools;
import org.apache.commons.lang3.time.FastDateFormat;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.springframework.stereotype.Service;
import java.io.FileOutputStream;
import java.io.OutputStream;
import java.util.*;
import java.util.stream.Collectors;
/**
* Description:
*
* @author LiuMingHuan
* @classname MarkDataContServiceImpl
* @date 2020/8/18
*/
@Service
public class MarkDataContServiceImpl implements MarkDataCountService {
private final EsDao esDao;
private final EsIndexTools esIndexTools;
private final PlatformService platformService;
public MarkDataContServiceImpl(EsDao esDao, EsIndexTools esIndexTools, PlatformService platformService) {
this.esDao = esDao;
this.esIndexTools = esIndexTools;
this.platformService = platformService;
}
@Override
public void outputMarkDataCountByPlatforms(long startTime, long endTime, String timeType, String timeUnit, String project) {
List<List<Object>> data = divideTime(startTime, endTime, timeType, timeUnit, project);
String fileName = TimeUtils.DATE_FORMATE.format(startTime) + "-" + TimeUtils.DATE_FORMATE.format(endTime) + "-" + timeUnit + ".xlsx";
List<List<String>> header = platformService.getPlatforms().stream().map(MessagePlatform::getName).map(Arrays::asList).collect(Collectors.toList());
header.add(0, Collections.singletonList("时间"));
header.add(Collections.singletonList("未知"));
try (OutputStream out = new FileOutputStream(GenericAttribute.FILE_PATH + fileName)) {
ExcelWriter writer = EasyExcel.write(out).registerWriteHandler(new PlatformCountHandler()).excelType(ExcelTypeEnum.XLSX).build();
WriteSheet sheet = EasyExcel.writerSheet(0, "总表").needHead(Boolean.TRUE).head(header).build();
writer.write(data, sheet);
writer.finish();
out.flush();
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* 时间划分
* @param startTime
* @param endTime
* @param timeUnit
* @return
*/
private List<List<Object>> divideTime(long startTime, long endTime, String timeType, String timeUnit, String project) {
Map<String, Map<String, Long>> result = new LinkedHashMap<>(4);
List<List<Object>> data = new ArrayList<>();
List<String> header = platformService.getPlatforms().stream().map(MessagePlatform::getName).collect(Collectors.toList());
header.add("未知");
FastDateFormat format = getFormat(timeUnit);
for (long currentTime = startTime; currentTime < endTime; currentTime += TimeUnit.currentTimeMillSeconds(timeUnit, currentTime)) {
long currentEnd = Math.min(currentTime + TimeUnit.currentTimeMillSeconds(timeUnit, currentTime), endTime);
Map<String, Long> platformCount = dividePlatform(currentTime, currentEnd, timeType, project);
result.put(format.format(currentTime) , platformCount);
}
for (Map.Entry<String, Map<String, Long>> entry : result.entrySet()) {
List<Object> list = new ArrayList<>();
Map<String, Long> platformCount = entry.getValue();
list.add(entry.getKey());
for (String platform : header) {
list.add(platformCount.containsKey(platform) ? platformCount.get(platform) : "");
}
data.add(list);
}
return data;
}
/**
* getFormat
* @param timeUnit
*/
private FastDateFormat getFormat(String timeUnit) {
switch (timeUnit){
case "SECOND":
return TimeUtils.MINUTE_FORMATE;
case "HOUR":
return TimeUtils.HOUR_FORMATE;
case "DAY":
default:
return TimeUtils.DATE_FORMATE;
}
}
/**
* TODO （平台策略更新后需调整）
* dividePlatform
* @param currentTime
* @param currentEnd
* @param timeType
* @param project
* @return
*/
private Map<String, Long> dividePlatform(long currentTime, long currentEnd, String timeType, String project) {
Map<String, Long> result = new HashMap<>(4);
String[] indexes = esIndexTools.getMarkIndexs();
String[] types = new String[]{ClassB.TypeB.COMPLETE.name(), ClassB.TypeB.INCOMPLETE.name(), ClassB.TypeB.QA.name(), ClassB.TypeB.VIDEO.name()};
String[] includes = new String[]{GenericAttribute.ES_C5, GenericAttribute.ES_ORIGIN, GenericAttribute.ES_FOREIGN};
BoolQueryBuilder query = QueryBuilders.boolQuery()
.must(QueryBuilders.termQuery(GenericAttribute.ES_MGROUP, project))
.must(QueryBuilders.rangeQuery(timeType).gte(currentTime).lte(currentEnd));
SearchHits searchHits = esDao.searchFatchSource(indexes, types, null, query, null, 0, GenericAttribute.SEARCH_TOTAL_LIMIT, includes, null, null);
if (searchHits.getTotalHits() > GenericAttribute.SEARCH_TOTAL_LIMIT) {
int totalPage = Tools.totalPage(searchHits.getTotalHits(), 1000);
SearchResponse searchResponse = esDao.scroll(indexes, types, null, query, null, 0, 1000, null, TimeValue.timeValueMinutes(2));
for (int i = 0; i < totalPage; i++) {
SearchHit[] hits = searchResponse.getHits().getHits();
getHits(result, hits);
searchResponse = esDao.scroll(searchResponse.getScrollId(), TimeValue.timeValueMinutes(2));
}
return result;
} else if (searchHits.getTotalHits() > 0) {
getHits(result, searchHits.getHits());
}
return result;
}
/**
* getHits
* @param result
* @param hits
*/
private void getHits(Map<String, Long> result, SearchHit[] hits) {
Arrays.stream(hits).map(SearchHit::getSourceAsString).collect(Collectors.groupingBy(this::searchHit2Platform))
.forEach((m, n) -> {
long count = result.containsKey(m) ? result.get(m) : 0;
count += n.size();
result.put(m, count);
});
}
/**
* searchHit2Platform
* @param sourceAsString
* @return
*/
private String searchHit2Platform(String sourceAsString) {
JSONObject jsonObject = JSONObject.parseObject(sourceAsString);
MessagePlatform messagePlatform = platformService.getPlatform(jsonObject.getIntValue(GenericAttribute.ES_C5), jsonObject.getString(GenericAttribute.ES_ORIGIN), jsonObject.getIntValue(GenericAttribute.ES_FOREIGN));
return !Tools.isEmpty(messagePlatform) ? messagePlatform.getName() : "未知";
}
}
src/main/java/com/zhiwei/qbjcutils/service/impl/MarkDataOutputServiceImpl.java 0 → 100644
package com.zhiwei.qbjcutils.service.impl;
import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.ExcelWriter;
import com.alibaba.excel.support.ExcelTypeEnum;
import com.alibaba.excel.write.metadata.WriteSheet;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.base.category.ClassB;
import com.zhiwei.qbjcutils.config.GenericAttribute;
import com.zhiwei.qbjcutils.es.EsIndexTools;
import com.zhiwei.qbjcutils.es.dao.EsDao;
import com.zhiwei.qbjcutils.pojo.emums.TimeUnit;
import com.zhiwei.qbjcutils.pojo.entity.MessagePlatform;
import com.zhiwei.qbjcutils.pojo.excelhadler.PlatformCountHandler;
import com.zhiwei.qbjcutils.service.MarkDataOutputService;
import com.zhiwei.qbjcutils.service.PlatformService;
import com.zhiwei.qbjcutils.utils.TimeUtils;
import com.zhiwei.qbjcutils.utils.Tools;
import org.apache.commons.lang3.time.FastDateFormat;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.springframework.stereotype.Service;
import java.io.FileOutputStream;
import java.io.OutputStream;
import java.util.*;
import java.util.stream.Collectors;
/**
* Description:
*
* @author LiuMingHuan
* @classname MarkDataCountOutputServiceImpl
* @date 2020/3/31
*/
@Service
public class MarkDataOutputServiceImpl implements MarkDataOutputService {
}
src/main/java/com/zhiwei/qbjcutils/service/impl/MessageFlowServiceImpl.java 0 → 100644
package com.zhiwei.qbjcutils.service.impl;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.base.category.ClassB;
import com.zhiwei.base.entity.subclass.CompleteText;
import com.zhiwei.base.entity.subclass.IncompleteText;
import com.zhiwei.base.entity.subclass.QAText;
import com.zhiwei.base.entity.subclass.Video;
import com.zhiwei.base.entity.subclass.mark.CompleteTextMark;
import com.zhiwei.base.entity.subclass.mark.IncompleteTextMark;
import com.zhiwei.base.entity.subclass.mark.QATextMark;
import com.zhiwei.base.entity.subclass.mark.VideoMark;
import com.zhiwei.base.filter.FilterInfo;
import com.zhiwei.qbjcutils.config.GenericAttribute;
import com.zhiwei.qbjcutils.dubbo.DubboClient;
import com.zhiwei.qbjcutils.es.EsIndexTools;
import com.zhiwei.qbjcutils.es.EsQueryTools;
import com.zhiwei.qbjcutils.es.dao.EsDao;
import com.zhiwei.qbjcutils.pojo.dto.BaseObjectCacheDTO;
import com.zhiwei.qbjcutils.pojo.entity.BaseObjectCache;
import com.zhiwei.qbjcutils.pojo.entity.BaseObjectCacheKeyword;
import com.zhiwei.qbjcutils.pojo.entity.MessagePlatform;
import com.zhiwei.qbjcutils.pojo.entity.SentimentResult;
import com.zhiwei.qbjcutils.pojo.excelentity.BaseObjectExcelEntity;
import com.zhiwei.qbjcutils.redis.RedisTools;
import com.zhiwei.qbjcutils.service.MessageFlowService;
import com.zhiwei.qbjcutils.service.PlatformService;
import com.zhiwei.qbjcutils.utils.Tools;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.sort.FieldSortBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.redisson.api.RScoredSortedSet;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service;
import java.io.File;
import java.io.FileOutputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
/**
* Description:
*
* @author LiuMingHuan
* @classname CacheBaseObjectServiceImpl
* @date 2020/3/30
*/
@Service
public class MessageFlowServiceImpl implements MessageFlowService {
private final PlatformService platformService;
private final RedisTools redisTools;
private final EsDao esDao;
private final EsIndexTools esIndexTools;
private final DubboClient client;
private final MongoTemplate mongoTemplate;
public MessageFlowServiceImpl(PlatformService platformService, RedisTools redisTools, EsDao esDao, EsIndexTools esIndexTools, DubboClient client, @Qualifier("secondaryMongoTemplate") MongoTemplate mongoTemplate) {
this.platformService = platformService;
this.redisTools = redisTools;
this.esDao = esDao;
this.esIndexTools = esIndexTools;
this.client = client;
this.mongoTemplate = mongoTemplate;
}
@Override
public List<BaseObjectCache> getBaseObjectCacheFromMessageFlow(String platform, String keyword, String project) {
String infoKey = redisTools.getInfoKey(platform, keyword, project);
RScoredSortedSet<String> scoredSortedSet = redisTools.getAllMessageFlowEntity(infoKey);
return scoredSortedSet.stream().map(s -> Tools.gunzip(JSONObject.parseObject(s).getString("zip")))
.map(s -> JSONObject.parseObject(s, BaseObjectCache.class)).collect(Collectors.toList());
}
@Override
public List<BaseObjectCacheKeyword> getBaseObjectCacheKeywordFromMessageFlow(String platform, String keyword, String project) {
String infoKey = redisTools.getInfoKey(platform, keyword, project);
RScoredSortedSet<String> scoredSortedSet = redisTools.getAllMessageFlowEntity(infoKey);
return scoredSortedSet.stream().map(s ->
{
JSONObject jsonObject = JSONObject.parseObject(s);
int total = jsonObject.getJSONObject("hlMap").values().stream().mapToInt(m -> Integer.parseInt(m.toString())).sum();
BaseObjectCacheKeyword baseObjectCacheKeyword = JSONObject.parseObject(Tools.gunzip(jsonObject.getString("zip")), BaseObjectCacheKeyword.class);
baseObjectCacheKeyword.setWordFrequency(total);
return baseObjectCacheKeyword;
}).collect(Collectors.toList());
}
@Override
public List<BaseObjectExcelEntity> getBaseObjectCacheExcelFromMessageFlow(String platform, String keyword, String project) {
String infoKey = redisTools.getInfoKey(platform, keyword, project);
RScoredSortedSet<String> scoredSortedSet = redisTools.getAllMessageFlowEntity(infoKey);
List<JSONObject> infos = scoredSortedSet.stream().map(s -> JSONObject.parseObject(Tools.gunzip(JSONObject.parseObject(s).getString("zip"))))
.collect(Collectors.toList());
List<BaseObjectExcelEntity> res = new ArrayList<>();
for (JSONObject info : infos) {
BaseObjectExcelEntity entity = null;
switch (ClassB.TypeB.fromEncode(info.getIntValue("c2"))) {
case COMPLETE:
CompleteText completeText = CompleteText.restoreFromEs(info.getInnerMap());
entity = info.toJavaObject(BaseObjectExcelEntity.class);
entity.setPlatform(platform);
entity.setTitle(Tools.cellInterception(completeText.getTitle()));
entity.setContent(Tools.cellInterception(completeText.getContent()));
break;
case INCOMPLETE:
IncompleteText incompleteText = IncompleteText.restoreFromEs(info.getInnerMap());
entity = info.toJavaObject(BaseObjectExcelEntity.class);
entity.setPlatform(platform);
entity.setContent(Tools.cellInterception(incompleteText.getContent()));
break;
case QA:
QAText qaText = QAText.restoreFromEs(info.getInnerMap());
entity = info.toJavaObject(BaseObjectExcelEntity.class);
entity.setPlatform(platform);
entity.setTitle(qaText.getQuestionTitle());
entity.setContent(Tools.cellInterception(qaText.question()?qaText.getQuestionContent():qaText.getAnswerContent()));
break;
case VIDEO:
Video video = Video.restoreFromEs(info.getInnerMap());
entity = info.toJavaObject(BaseObjectExcelEntity.class);
entity.setPlatform(platform);
entity.setTitle(Tools.cellInterception(video.getTitle()));
entity.setContent(Tools.cellInterception(video.getContent()));
break;
default:
break;
}
res.add(entity);
}
return res;
}
@Override
public List<BaseObjectCacheDTO> getMessageFlowEntityWithTime(Long startTime, Long endTime, MessagePlatform platform, String keyword, String project) {
List<BaseObjectCacheDTO> result =new ArrayList<>();
long startIndex = getLastScore(startTime, platform);
long endIndex = getLastScore(endTime, platform);
if (startIndex == 0 || endIndex == 0) {
return null;
}
String infoKey = redisTools.getInfoKey(platform.getName(), keyword, project);
List<JSONObject> messages = redisTools.getMessageFlowEntityWithTime(infoKey, startIndex, endIndex);
List<JSONObject> infos = messages.stream().map(m -> JSONObject.parseObject(Tools.gunzip(m.getString("zip"))))
.collect(Collectors.toList());
for (JSONObject info : infos) {
String[] indexs = esIndexTools.getMarkIndexs();
String[] types = new String[]{ClassB.TypeB.COMPLETE.name(), ClassB.TypeB.INCOMPLETE.name(), ClassB.TypeB.QA.name(),
ClassB.TypeB.VIDEO.name()};
String[] mupdates = transformBaseObject(info, project);
String mtag = "";
if (!Tools.isNull(mupdates)) {
String mupdate = mupdates[0];
String mupdateTwo = mupdates.length > 1 ? mupdates[1] : null;
QueryBuilder query = EsQueryTools.assembleTagInfoQuery(mupdate, mupdateTwo);
SearchHits searchHits = esDao.searchFatchSource(indexs, types, null, query, null, 0, 1, new String[]{"mtag"}, null, null);
mtag = searchHits.getTotalHits() > 0 ? new JSONObject(searchHits.getHits()[0].getSource()).getString("mtag") : "";
}
BaseObjectCacheDTO baseObjectCacheDTO = info.toJavaObject(BaseObjectCacheDTO.class);
baseObjectCacheDTO.setMtag(mtag);
result.add(baseObjectCacheDTO);
}
return result;
}
/**
* mupdate 获取
* @param info
* @param project
* @return
*/
private String[] transformBaseObject(JSONObject info,String project) {
FilterInfo filterInfo = null;
switch (ClassB.TypeB.fromEncode(info.getIntValue("c2"))) {
case COMPLETE:
CompleteTextMark completeTextMark = CompleteTextMark.restoreFromEs(info);
completeTextMark.setMgroup(project);
filterInfo = completeTextMark.filterInfo();
break;
case INCOMPLETE:
IncompleteTextMark incompleteTextMark = IncompleteTextMark.restoreFromEs(info);
incompleteTextMark.setMgroup(project);
filterInfo = incompleteTextMark.filterInfo();
break;
case QA:
QATextMark qaTextMark = QATextMark.restoreFromEs(info);
qaTextMark.setMgroup(project);
filterInfo = qaTextMark.filterInfo();
break;
case VIDEO:
VideoMark videoMark = VideoMark.restoreFromEs(info);
videoMark.setMgroup(project);
filterInfo = videoMark.filterInfo();
break;
default:
break;
}
if (!Tools.isNull(filterInfo)) {
return client.getUpdateInfo(filterInfo);
}
return null;
}
/**
* 查询es对应库获取最大score值
*
* @param time
* @param platform
* @return
*/
public long getLastScore(long time, MessagePlatform platform) {
//查询截止时间
long deadLine = time - GenericAttribute.AN_HOUR;
//index及type
String[] indexes = esIndexTools.getAllIndex(deadLine, System.currentTimeMillis());
String[] types = new String[]{"status"};
BoolQueryBuilder query = QueryBuilders.boolQuery()
.must(QueryBuilders.rangeQuery("stime").gte(deadLine).lte(time))
.must(EsQueryTools.assemblePlatformQuery(Collections.singletonList(platform)));
FieldSortBuilder sort = SortBuilders.fieldSort("gid").order(SortOrder.DESC);
SearchHits searchHits = esDao.searchFatchSource(indexes, types, null, query, sort, 0,
1, new String[]{"gid"}, null, null);
return searchHits.getTotalHits() > 0 ? Long.parseLong(String.valueOf(searchHits.getHits()[0].getSource().get("gid"))) : 0;
}
@Override
public void generateText(String collectionName) {
long count = mongoTemplate.count(new Query(), SentimentResult.class, collectionName);
int totalPage = Tools.totalPage(count, 50000);
int lastPositive = 0;
int lastNegative = 0;
for (int k = 1; k <= totalPage; k++) {
long skip = 50000 * (k - 1);
List<SentimentResult> sentimentResults = mongoTemplate.find(new Query().skip(skip).limit(50000), SentimentResult.class, collectionName);
List<SentimentResult> positive = new ArrayList<>();
List<SentimentResult> negative = new ArrayList<>();
for (SentimentResult sentimentResult : sentimentResults) {
JSONObject jsonObject = new JSONObject(sentimentResult.getResult().get(0));
if (jsonObject.getDouble("positive_prob") >= 0.5) {
positive.add(sentimentResult);
} else {
negative.add(sentimentResult);
}
}
List<List<SentimentResult>> list = Arrays.asList(positive, negative);
for (int i = 0; i < list.size(); i++) {
String sentiment = i == 0 ? "D:\\stWork\\download\\excel\\sample\\" + collectionName + "positive\\"
: "D:\\stWork\\download\\excel\\sample\\" + collectionName + "negative\\";
List<SentimentResult> results = list.get(i);
for (int j = 0; j < results.size(); j++) {
File file = new File(sentiment);
file.mkdirs();
String fileName = i == 0 ? sentiment + (j + lastPositive) + ".txt" : sentiment + (j + lastNegative) + ".txt";
System.out.println(fileName);
try (OutputStream out = new FileOutputStream(fileName)) {
OutputStreamWriter os = new OutputStreamWriter(out);
os.write(results.get(j).getText());
os.close();
} catch (Exception e) {
e.printStackTrace();
}
}
}
lastPositive += positive.size();
lastNegative += negative.size();
}
}
}
src/main/java/com/zhiwei/qbjcutils/service/impl/PlatformServiceImpl.java 0 → 100644
package com.zhiwei.qbjcutils.service.impl;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.base.category.ClassB;
import com.zhiwei.base.category.ClassC;
import com.zhiwei.base.category.ClassCodec;
import com.zhiwei.base.category.ClassD;
import com.zhiwei.qbjcutils.config.GenericAttribute;
import com.zhiwei.qbjcutils.dubbo.DubboClient;
import com.zhiwei.qbjcutils.pojo.entity.MessagePlatform;
import com.zhiwei.qbjcutils.redis.RedisTools;
import com.zhiwei.qbjcutils.service.PlatformService;
import com.zhiwei.qbjcutils.utils.QYWeChatSendUtils;
import com.zhiwei.qbjcutils.utils.Tools;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.data.mongodb.core.query.Update;
import org.springframework.stereotype.Service;
import java.util.*;
import java.util.stream.Collectors;
/**
* Description:
*
* @author LiuMingHuan
* @classname PlatformServiceImpl
* @date 2020/4/3
*/
@Service
public class PlatformServiceImpl implements PlatformService {
private final RedisTools redisTools;
private final MongoTemplate mongoTemplate;
private final DubboClient dubboClient;
public PlatformServiceImpl(RedisTools redisTools, @Qualifier("primaryMongoTemplate") MongoTemplate mongoTemplate, DubboClient dubboClient) {
this.redisTools = redisTools;
this.mongoTemplate = mongoTemplate;
this.dubboClient = dubboClient;
}
@Override
public List<MessagePlatform> getAllPlatforms() {
String platformKey = GenericAttribute.SYSTEM_KEY + "platform";
return redisTools.getAllMessagePlatform(platformKey);
}
@Override
public MessagePlatform getPlatformByName(String name) {
return getPlatforms().stream().filter(p -> p.getName().equals(name)).findFirst().orElse(null);
}
@Override
public MessagePlatform getPlatform(int c5, String origin, int foreign) {
String header = c5 + "_" + origin + "_" + foreign;
return getPlatforms().stream().filter(m -> m.getTemplateHeaders().contains(header)).findFirst().orElse(null);
}
@Override
public void autoUpdate() {
//获取最新平台信息
List<MessagePlatform> allPlatform = mongoTemplate.findAll(MessagePlatform.class);
Map<String, List<JSONObject>> map = getNowPlatformMap(allPlatform);
List<MessagePlatform> newPlatforms = updateMessagePlatforms(allPlatform, map);
//更新库中平台
for (MessagePlatform me : newPlatforms) {
Query query = new Query(Criteria.where("_id").is(me.getId()));
Update update = new Update();
update.set("templateHeaders", me.getTemplateHeaders());
mongoTemplate.updateFirst(query, update, MessagePlatform.class);
}
Map<String, String> res = newPlatforms.stream().collect(Collectors.toMap(MessagePlatform::getName, JSON::toJSONString));
redisTools.cachePlatForms(res);
}
/**
* 查询全部平台信息
* @return
*/
private List<MessagePlatform> getAllPlatform() {
Collection<String> values = redisTools.getAllPlatForms().values();
return values.stream().map(m -> JSONObject.parseObject(m, MessagePlatform.class)).collect(Collectors.toList());
}
/**
* 获取新的平台信息（httpGet)
*
* @return
*/
private Map<String, List<JSONObject>> getNowPlatformMap(List<MessagePlatform> allPlatform) {
JSONArray data = JSON.parseArray(JSON.toJSONString(dubboClient.getPlatformsPermutation()));
List<String> allHeaders = allPlatform.stream().map(MessagePlatform::getTemplateHeaders).flatMap(Collection::stream).collect(Collectors.toList());
Map<String, List<JSONObject>> map = new HashMap<>(8);
for (int i = 0; i < data.size(); i++) {
JSONObject info = data.getJSONObject(i);
int c5 = info.getIntValue("c5");
String origin = info.getString("origin");
int foreign = info.getIntValue("foreign");
ClassD d = ClassCodec.decodeClassD(c5);
info.put("id", c5 + "_" + origin + "_" + foreign);
//普通
if (foreign == 1) {
if(d.typeD().equals(ClassD.TypeD.TWITTER)) {
addJson("Twitter", info, map);
}else{
addJson("外媒", info, map);
}
}else if (d.typeD().equals(ClassD.TypeD.抖音)) {
addJson("抖音", info, map);
} else if (d.typeB().equals(ClassB.TypeB.VIDEO)) {
addJson("视频", info, map);
}else if (d.typeD().equals(ClassD.TypeD.新浪微博)) {
addJson("微博", info, map);
} else if (d.typeD().equals(ClassD.TypeD.微信公众号)) {
addJson("微信", info, map);
} else if (d.typeD().equals(ClassD.TypeD.网媒) && d.typeC().equals(ClassC.TypeC.PGC)&&origin.equals("PC")) {
addJson("网媒", info, map);
} else if (d.typeD().equals(ClassD.TypeD.平媒) && d.typeC().equals(ClassC.TypeC.PGC)) {
addJson("平媒", info, map);
} else if (d.typeD().equals(ClassD.TypeD.今日头条) || d.typeD().equals(ClassD.TypeD.微头条) || d.typeD().equals(ClassD.TypeD.悟空问答) || d.typeD().equals(ClassD.TypeD.西瓜视频)) {
addJson("今日头条", info, map);
} else if (d.typeD().equals(ClassD.TypeD.知乎问答) || d.typeD().equals(ClassD.TypeD.知乎专栏)) {
addJson("知乎", info, map);
} else if (d.typeD().equals(ClassD.TypeD.脉脉职言) || d.typeD().equals(ClassD.TypeD.脉脉行业头条) || d.typeD().equals(ClassD.TypeD.脉脉问答)) {
addJson("脉脉", info, map);
} else if (d.typeB().equals(ClassB.TypeB.COMPLETE)) {
addJson("自媒体", info, map);
} else if (d.typeB().equals(ClassB.TypeB.QA)) {
if (d.typeD().name().equals("网媒") ) {
addJson("贴吧论坛", info, map);
}
if (d.typeD().name().contains("问答") || d.typeD().name().contains("知道") || d.typeD().name().contains("天天问")) {
addJson("问答", info, map);
} else if (d.typeD().name().contains("贴吧") || d.typeD().name().contains("论坛") || d.typeD().name().contains(
"投诉") || d.typeD().name().contains("社区") || d.typeD().name().contains("网")) {
addJson("贴吧论坛", info, map);
} else {
addJson("贴吧论坛", info, map);
}
} else {
if (!allHeaders.contains(info.getString("id"))) {
addJson("未知", info, map);
}
}
}
return map;
}
/**
* 组装finalMap
* @param pt
* @param info
* @param map
*/
private static void addJson(String pt, JSONObject info, Map<String, List<JSONObject>> map) {
List<JSONObject> list = map.get(pt);
if (Tools.isEmpty(list)) {
list = new ArrayList<>(Collections.singletonList(info));
map.put(pt, list);
} else {
list.add(info);
}
}
/**
* 更新消息平台
* @param allPlatform
*/
private List<MessagePlatform> updateMessagePlatforms(List<MessagePlatform> allPlatform, Map<String, List<JSONObject>> map) {
Map<String, List<String>> finalMap = new HashMap<>(8);
List<String> waitAddList = new ArrayList<>();
for (MessagePlatform me : allPlatform) {
List<String> list = JSONArray.parseArray(JSONObject.toJSONString(me.getTemplateHeaders()), String.class);
List<String> templateHeaders = finalMap.get(me.getName());
if (Tools.isEmpty(templateHeaders)) {
finalMap.put(me.getName(), list);
templateHeaders = finalMap.get(me.getName());
} else {
templateHeaders.addAll(list);
}
for (Map.Entry<String, List<JSONObject>> entry : map.entrySet()) {
String key = entry.getKey();
List<JSONObject> value = entry.getValue();
if (key.equals(me.getName())) {
for (JSONObject jsonObject : value) {
if (list.contains(jsonObject.getString("id"))) {
} else {
templateHeaders.add(jsonObject.getString("id"));
}
}
}
if ("未知".equals(key)) {
for (JSONObject jsonObject : value) {
waitAddList.add(jsonObject.getString("id"));
}
}
}
for (Map.Entry<String, List<JSONObject>> entry : map.entrySet()) {
String key = entry.getKey();
List<JSONObject> value = entry.getValue();
if (!key.equals(me.getName())) {
for (JSONObject jsonObject : value) {
if (list.contains(jsonObject.getString("id"))) {
waitAddList.add(jsonObject.getString("id"));
List<String> removeList = finalMap.get(key);
if (!Tools.isEmpty(removeList)) {
removeList.remove(jsonObject.getString("id"));
}
removeList = finalMap.get(me.getName());
if (!Tools.isEmpty(removeList)) {
removeList.remove(jsonObject.getString("id"));
}
}
}
}
}
}
//推送预警信息
Map<String, Map<String, List<String>>> allChanges = getChangeTemplateHeaders(allPlatform, finalMap);
//待处理header 去重
List<String> distinctWaitAddList = waitAddList.stream().distinct().collect(Collectors.toList());
sendWarningMessages(allPlatform, allChanges, distinctWaitAddList);
return allPlatform;
}
/**
* 获取改变的平台模板信息
* @param allPlatform
* @param finalMap
* @return
*/
private Map<String, Map<String, List<String>>> getChangeTemplateHeaders(List<MessagePlatform> allPlatform, Map<String, List<String>> finalMap) {
Map<String, Map<String, List<String>>> allChanges = new HashMap<>(4);
for (MessagePlatform me : allPlatform){
List<String> newHeaders = finalMap.get(me.getName());
//更新headers
List<String> oldHeaders = me.getTemplateHeaders();
List<String> retains = new ArrayList<>(oldHeaders);
List<String> savingHeaders = new ArrayList<>(newHeaders);
Map<String,List<String>> changeMap = new HashMap<>(2);
retains.retainAll(newHeaders);
oldHeaders.removeAll(retains);
newHeaders.removeAll(retains);
if(!Tools.isEmpty(oldHeaders)) {
changeMap.put("移除", oldHeaders);
}
if(!Tools.isEmpty(newHeaders)) {
changeMap.put("增加", newHeaders);
}
if(!Tools.isEmpty(changeMap)) {
allChanges.put(me.getName(), changeMap);
}
me.setTemplateHeaders(savingHeaders);
}
return allChanges;
}
/**
* 推送结果拼接，发送预警
* @param allPlatform
* @param allChanges
* @param waitAddList
*/
private void sendWarningMessages(List<MessagePlatform> allPlatform, Map<String, Map<String, List<String>>> allChanges, List<String> waitAddList) {
StringBuilder stringBuilder = new StringBuilder("待手动处理 ");
stringBuilder.append(Tools.isEmpty(waitAddList) ? "无" : waitAddList).append("\\r\\n\\r\\n").append("更新信息 ");
for (Map.Entry<String, Map<String, List<String>>> entry:allChanges.entrySet()){
stringBuilder.append(entry.getKey());
for (Map.Entry<String,List<String>> operateEntry : entry.getValue().entrySet()){
stringBuilder.append(operateEntry.getKey()).append("：").append(operateEntry.getValue());
}
stringBuilder.append(" ");
}
stringBuilder.append(" \\r\\n\\r\\n").append("各平台条数统计\\r\\n");
allPlatform.sort(Comparator.comparing(MessagePlatform::getCreateAt));
for (MessagePlatform me : allPlatform){
stringBuilder.append(me.getName()).append("：").append(me.getTemplateHeaders().size()).append("条 \\r\\n");
}
QYWeChatSendUtils.sendWarnMessage(new String(stringBuilder));
}
@Override
public List<MessagePlatform> getPlatforms() {
return mongoTemplate.findAll(MessagePlatform.class);
}
}
src/main/java/com/zhiwei/qbjcutils/task/AsyncTask.java 0 → 100644
package com.zhiwei.qbjcutils.task;
import com.zhiwei.qbjcutils.redis.RedisTools;
import org.springframework.stereotype.Component;
/**
* Description:
*
* @author LiuMingHuan
* @classname AsyncTask
* @date 2020/4/3
*/
@Component
public class AsyncTask {
private final RedisTools redisTools;
public AsyncTask(RedisTools redisTools) {
this.redisTools = redisTools;
}
public void initMessagePlatforms(){
// GenericAttribute.
// redisTools.getRMap()
}
}
src/main/java/com/zhiwei/qbjcutils/task/InitTask.java 0 → 100644
package com.zhiwei.qbjcutils.task;
import org.springframework.stereotype.Component;
/**
* Description:
*
* @author LiuMingHuan
* @classname initTask
* @date 2020/4/3
*/
@Component
public class InitTask {
public void init() {
}
}
src/main/java/com/zhiwei/qbjcutils/utils/QYWeChatSendUtils.java 0 → 100644
package com.zhiwei.qbjcutils.utils;
import com.zhiwei.qbjcutils.config.GenericAttribute;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/**
* Description:
*
* @author LiuMingHuan
* @classname QYWeChatSendUtils
* @date 2019/12/27
*/
public class QYWeChatSendUtils {
private static final Logger log = LogManager.getLogger(QYWeChatSendUtils.class);
private static long LAST_WARN;
private static final long WARN_LIMIT = 1L;
/**
* 发送预警消息
*
* @param s
*/
public static void sendWarnMessage(String s) {
try {
if ((LAST_WARN + WARN_LIMIT) < System.currentTimeMillis()) {
LAST_WARN = System.currentTimeMillis();
StringBuilder msgBuilder = new StringBuilder();
msgBuilder.append("{\"touser\": \"YangYongDeYu\",\"msgtype\": \"text\",\"text\": {\"content\": \"");
msgBuilder.append(s);
msgBuilder.append("\"},\"safe\": 0}");
log.info("预警发送结果：" + Tools.httpPostRequestBody(GenericAttribute.WARN_URL, msgBuilder.toString()));
}
} catch (Exception e) {
log.error("预警发送失败", e);
}
}
}
src/main/java/com/zhiwei/qbjcutils/utils/TimeUtils.java 0 → 100644
package com.zhiwei.qbjcutils.utils;
import org.apache.commons.lang3.time.FastDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.List;
/**
* @ClassName: TimeUtil
* @Description: 时间工具类
* @author shentao
* @date 2019年7月9日 上午9:31:37
*/
public class TimeUtils {
/** day **/
public static final String TIMETYPE_DAY = "day";
/** yyyy FastDateFormat **/
public static final FastDateFormat YEAR_FORMAT = FastDateFormat.getInstance("yyyy");
/** MM FastDateFormat **/
public static final FastDateFormat MONTH_FORMAT = FastDateFormat.getInstance("MM");
/** dd FastDateFormat **/
public static final FastDateFormat DAY_FORMAT = FastDateFormat.getInstance("dd");
/** yyyy-MM-dd HH:mm:ss FastDateFormat **/
public static final FastDateFormat TIME_FORMATE = FastDateFormat.getInstance("yyyy-MM-dd HH:mm:ss");
/** yyyy-MM-dd HH:mm FastDateFormat **/
public static final FastDateFormat MINUTE_FORMATE = FastDateFormat.getInstance("yyyy-MM-dd HH:mm");
/** yyyy-MM-dd HH FastDateFormat **/
public static final FastDateFormat HOUR_FORMATE = FastDateFormat.getInstance("yyyy-MM-dd HH");
/** yyyy-MM-dd FastDateFormat **/
public static final FastDateFormat DATE_FORMATE = FastDateFormat.getInstance("yyyy-MM-dd");
/** file yyyyMMddHHmmss FastDateFormat **/
public static final FastDateFormat FILE_FORMAT = FastDateFormat.getInstance("yyyyMMddHHmmss");
/** file yyyyMMdd FastDateFormat **/
public static final FastDateFormat FILE_DATE_FORMAT = FastDateFormat.getInstance("yyyyMMdd");
/**
* 按年月获取 当前月份第一天
*
* @param year
* @param month
* @return long
*/
public static long getStartTimeByYearAndMonth(int year, int month) {
Calendar calendar = Calendar.getInstance();
calendar.clear();
calendar.set(Calendar.YEAR, year);
calendar.set(Calendar.MONTH, month - 1);
calendar.set(Calendar.DAY_OF_MONTH, 1);
return calendar.getTimeInMillis();
}
/**
* 按年月获取 当前月份最后一天
*
* @param year
* @param month
* @return long
*/
public static long getEndTimeByYearAndMonth(int year, int month) {
Calendar calendar = Calendar.getInstance();
calendar.clear();
calendar.set(Calendar.YEAR, year);
calendar.set(Calendar.MONTH, month - 1);
calendar.set(Calendar.DAY_OF_MONTH, calendar.getActualMaximum(Calendar.DAY_OF_MONTH));
return calendar.getTimeInMillis();
}
/**
* 取时间点 day hour两钟
*
* @param start
* @param end
* @param timeType
* @return
* @return List<String>
*/
public static List<String> getTimePointToNowOrEnd(long start, long end, String timeType) {
Calendar car = Calendar.getInstance();
car.setTimeInMillis(start);
boolean flag = true;
// 为完善表现层，填充无数据时间点
// 创建完整时间点
List<String> times = new ArrayList<>();
if (TIMETYPE_DAY.equals(timeType)) {
while (flag) {
times.add(DATE_FORMATE.format(car.getTime()));
if (end <= car.getTimeInMillis()) {
flag = false;
}
car.add(Calendar.DAY_OF_MONTH, 1);
}
} else {
while (flag) {
times.add(HOUR_FORMATE.format(car.getTime()));
if (end <= car.getTimeInMillis()) {
flag = false;
}
car.add(Calendar.HOUR, 1);
}
}
return times;
}
/**
* 获取上个月第一天
* @param time
* @return
*/
public static long getFirstDayOfLastMonth(long time) {
Calendar calendar = Calendar.getInstance();
calendar.setTimeInMillis(time);
calendar.add(Calendar.MONTH, -1);
calendar.set(Calendar.DAY_OF_MONTH, 1);
return calendar.getTimeInMillis();
}
/**
* 获取下个月第一天
* @param time
* @return
*/
public static long getFirstDayOfNextMonth(long time) {
Calendar calendar = Calendar.getInstance();
calendar.setTimeInMillis(time);
calendar.add(Calendar.MONTH, 1);
calendar.set(Calendar.DAY_OF_MONTH, 1);
return calendar.getTimeInMillis();
}
}
src/main/java/com/zhiwei/qbjcutils/utils/Tools.java 0 → 100644
package com.zhiwei.qbjcutils.utils;
import com.mongodb.BasicDBObject;
import com.mongodb.DBObject;
import com.zhiwei.crawler.core.HttpBoot;
import com.zhiwei.crawler.utils.FormBodyUtils;
import com.zhiwei.crawler.utils.RequestUtils;
import okhttp3.MediaType;
import okhttp3.Request;
import okhttp3.RequestBody;
import okhttp3.Response;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.data.mongodb.core.query.Update;
import java.io.*;
import java.lang.reflect.Field;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import static java.util.Objects.nonNull;
/**
* 常用工具类
*
* @author shentao
* @ClassName: Tools
* @Description: 常用工具类
* @date 2019年5月23日 下午4:52:21
*/
public class Tools {
/** HttpBoot **/
private static HttpBoot httpBoot = null;
/**
* get请求
*
* @Title: httpGet
* @Description: get请求
* @param @param url
* @param @return 设定文件
* @return String 返回类型
*/
public static String httpGet(String url) {
Request request = RequestUtils.wrapGet(url);
try (Response response = httpBoot.syncCall(request)) {
return response.body().string();
} catch (Exception e) {
return null;
}
}
/**
* post form请求
*
* @param @param urlNewProject
* @param @param form
* @param @return 设定文件
* @return String 返回类型
* @Title: httpPostForm
* @Description: post form请求
*/
public static String httpPostForm(String url, Map<String, Object> form) {
Request request = RequestUtils.wrapPost(url, new HashMap<>(16), FormBodyUtils.wrap(form));
try (Response response = httpBoot.syncCall(request)) {
return response.body().string();
} catch (Exception e) {
return null;
}
}
/**
* post RequestBody请求
*
* @Title: httpPostRequestBody
* @Description: post RequestBody请求
* @param @param url
* @param @param info
* @param @return 设定文件
* @return String 返回类型
*/
public static String httpPostRequestBody(String url, String info) {
Request request = RequestUtils.wrapPost(url, RequestBody.create(MediaType.get("application/json"), info));
try (Response response = httpBoot.syncCall(request)) {
return response.body().string();
} catch (Exception e) {
return null;
}
}
/**
* delete 请求
*
* @param @param url
* @param @param info
* @param @return 设定文件
* @return String 返回类型
* @Title: httpDelete
* @Description: delete 请求
*/
public static String httpDelete(String url, String info) {
Request request = RequestUtils.wrap(RequestUtils.HttpMethod.DELETE, url,
RequestBody.create(MediaType.get("application/json"), info));
try (Response response = httpBoot.syncCall(request)) {
return response.body().string();
} catch (Exception e) {
return null;
}
}
/**
* put 请求
*
* @param @param url
* @param @param info
* @param @return 设定文件
* @return String 返回类型
* @Title: httpPut
* @Description: put 请求
*/
public static String httpPut(String url, String info) {
Request request = RequestUtils.wrap(RequestUtils.HttpMethod.PUT, url,
RequestBody.create(MediaType.get("application/json"), info));
try (Response response = httpBoot.syncCall(request)) {
return response.body().string();
} catch (Exception e) {
return null;
}
}
/**
* 实体转DBObject
*
* @param @param t
* @param @return
* @param @throws IllegalArgumentException
* @param @throws IllegalAccessException 设定文件
* @return DBObject 返回类型
* @Title: bean2Dbobject
* @Description: 实体转DBObject
*/
public static <T> DBObject bean2Dbobject(T t) throws IllegalArgumentException, IllegalAccessException {
if (isNull(t)) {
return null;
}
DBObject obj = new BasicDBObject();
// 获取对象对应类中的所有属性域
Field[] fields = t.getClass().getDeclaredFields();
for (Field field : fields) {
// 修改访问控制权限
boolean accessFlag = field.isAccessible();
if (!accessFlag) {
field.setAccessible(true);
}
Object param;
String key;
key = "id".equals(field.getName()) ? "_id" : field.getName();
param = field.get(t);
if (param == null) {
continue;
} else if (param instanceof String) {
obj.put(key, param);
} else if (param instanceof Integer) {
obj.put(key, param);
} else if (param instanceof Long) {
obj.put(key, param);
} else if (param instanceof List) {
obj.put(key, ((List<?>) param).toArray());
} else if (param instanceof Boolean) {
obj.put(key, param);
} else if (param instanceof Map) {
obj.put(key, param);
} else {
obj.put(key, param);
}
// 恢复访问控制权限
field.setAccessible(false);
}
return obj;
}
/**
* bean2Update 参数可为null
*
* @param @param t
* @param @return
* @param @throws Exception 设定文件
* @return Update 返回类型
* @Title: bean2UpdateCanNull
* @Description: bean2Update 参数可为null
*/
public static <T> Update bean2UpdateCanNull(T t) throws Exception {
Update res = new Update();
// 获取对象对应类中的所有属性域
Field[] fields = t.getClass().getDeclaredFields();
for (Field field : fields) {
// 修改访问控制权限
boolean accessFlag = field.isAccessible();
if (!accessFlag) {
field.setAccessible(true);
}
Object param;
String key;
if ("id".equals(field.getName()) || "_id".equals(field.getName())) {
continue;
}
key = field.getName();
param = field.get(t);
if (param instanceof String) {
res.set(key, param);
} else if (param instanceof Integer) {
res.set(key, param);
} else if (param instanceof Long) {
res.set(key, param);
} else if (param instanceof List) {
res.set(key, ((List<?>) param).toArray());
} else if (param instanceof Boolean) {
res.set(key, param);
} else if (param instanceof Map) {
res.set(key, param);
} else {
res.set(key, param);
}
// 恢复访问控制权限
field.setAccessible(false);
}
return res;
}
/**
* bean2Update 参数为null跳过
*
* @param @param t
* @param @return
* @param @throws Exception 设定文件
* @return Update 返回类型
* @Title: bean2Update
* @Description: bean2Update 参数为null跳过
*/
public static <T> Update bean2Update(T t) throws Exception {
Update res = new Update();
// 获取对象对应类中的所有属性域
Field[] fields = t.getClass().getDeclaredFields();
for (Field field : fields) {
// 修改访问控制权限
boolean accessFlag = field.isAccessible();
if (!accessFlag) {
field.setAccessible(true);
}
Object param;
String key;
if ("id".equals(field.getName()) || "_id".equals(field.getName())) {
continue;
}
key = field.getName();
param = field.get(t);
if (param == null) {
continue;
} else if (param instanceof String) {
res.set(key, param);
} else if (param instanceof Integer) {
res.set(key, param);
} else if (param instanceof Long) {
res.set(key, param);
} else if (param instanceof List) {
res.set(key, ((List<?>) param).toArray());
} else if (param instanceof Boolean) {
res.set(key, param);
} else if (param instanceof Map) {
res.set(key, param);
} else {
res.set(key, param);
}
// 恢复访问控制权限
field.setAccessible(false);
}
return res;
}
/**
* dbobject 转 Update
*
* @param @param t
* @param @return 设定文件
* @return Update 返回类型
* @Title: dbobjectToUpdate
* @Description: dbobject 转 Update
*/
public static Update dbobjectToUpdate(DBObject t) {
Update res = new Update();
// 获取对象对应类中的所有属性域
Set<String> fields = t.keySet();
for (String field : fields) {
Object param;
String key;
if ("id".equals(field) || "_id".equals(field)) {
continue;
}
key = field;
param = t.get(key);
if (param == null) {
} else if (param instanceof String) {
res.set(key, param);
} else if (param instanceof Integer) {
res.set(key, param);
} else if (param instanceof Long) {
res.set(key, param);
} else if (param instanceof List && ((List<?>) param).size() > 0) {
res.set(key, ((List<?>) param).toArray());
} else if (param instanceof Boolean) {
res.set(key, param);
} else if (param instanceof Map && ((Map<?, ?>) param).size() > 0) {
res.set(key, param);
}
}
return res;
}
/**
* 是否为空，数据为空
*
* @param @param obj
* @param @return 设定文件
* @return boolean 返回类型
* @Title: isEmpty
* @Description: 是否为空，数据为空
*/
public static boolean isEmpty(Object obj) {
if (!nonNull(obj)) {
return true;
}
if ((obj instanceof List)) {
return CollectionUtils.isEmpty((List<?>) obj);
}
if ((obj instanceof String)) {
return StringUtils.isEmpty((String) obj);
}
if ((obj instanceof Map)) {
return MapUtils.isEmpty((Map<?, ?>) obj);
}
return false;
}
/**
* 是否为null
*
* @param @param obj
* @param @return 设定文件
* @return boolean 返回类型
* @Title: isNull
* @Description: 是否为null
*/
public static boolean isNull(Object obj) {
return !nonNull(obj);
}
/**
* 模糊匹配
*
* @param @param kw
* @param @param word
* @param @return 设定文件
* @return boolean 返回类型
* @Title: approximateStringMatching
* @Description: 模糊匹配
*/
public static boolean approximateStringMatching(String beMatching, String mathingWord) {
boolean res = false;
beMatching = beMatching.toLowerCase();
mathingWord = mathingWord.toLowerCase();
if (beMatching.contains(mathingWord)) {
res = true;
}
return res;
}
/**
* 模糊匹配 多对多 多个matchs中有一个match匹配上多个key 为true
*
* @param @param matchs
* @param @param keys
* @param @return 设定文件
* @return boolean 返回类型
* @Title: approximateStringMatching
* @Description: 模糊匹配 多对多 多个matchs中有一个match匹配上多个key 为true
*/
public static boolean approximateStringMatching(String[] matchs, String[] keys) {
boolean res = false;
for (String match : matchs) {
res = Arrays.asList(keys).stream().allMatch(a -> approximateStringMatching(match, a));
if (res) {
break;
}
}
return res;
}
/**
* 深拷贝
*
* @param @param obj
* @param @return 设定文件
* @return T 返回类型
* @Title: clone
* @Description: 深拷贝
*/
@SuppressWarnings("unchecked")
public static <T extends Serializable> T clone(T obj) {
T cloneObj = null;
try {
// 写入字节流
ByteArrayOutputStream baos = new ByteArrayOutputStream();
ObjectOutputStream oos = new ObjectOutputStream(baos);
oos.writeObject(obj);
oos.close();
// 分配内存,写入原始对象,生成新对象
// 获取上面的输出字节流
ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
ObjectInputStream ois = new ObjectInputStream(bais);
// 返回生成的新对象
cloneObj = (T) ois.readObject();
ois.close();
} catch (Exception e) {
e.printStackTrace();
}
return cloneObj;
}
/**
* gzip压缩字符串
*
* @param primStr
* @return
*/
public static String gzip(String primStr) {
if (primStr == null || primStr.length() == 0) {
return primStr;
}
ByteArrayOutputStream out = new ByteArrayOutputStream();
GZIPOutputStream gzip = null;
try {
gzip = new GZIPOutputStream(out);
gzip.write(primStr.getBytes());
} catch (IOException e) {
e.printStackTrace();
} finally {
if (gzip != null) {
try {
gzip.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return new String(new Base64().encode(out.toByteArray()));
}
/**
* 使用gzip进行解压缩
*
* @param compressedStr
* @return
*/
public static String gunzip(String compressedStr) {
if (compressedStr == null) {
return null;
}
ByteArrayOutputStream out = new ByteArrayOutputStream();
ByteArrayInputStream in = null;
GZIPInputStream ginzip = null;
byte[] compressed = null;
String decompressed = null;
try {
compressed = new Base64().decode(compressedStr);
in = new ByteArrayInputStream(compressed);
ginzip = new GZIPInputStream(in);
byte[] buffer = new byte[102400];
int offset = -1;
while ((offset = ginzip.read(buffer)) != -1) {
out.write(buffer, 0, offset);
}
decompressed = out.toString();
} catch (IOException e) {
e.printStackTrace();
} finally {
if (ginzip != null) {
try {
ginzip.close();
} catch (IOException e) {
}
}
if (in != null) {
try {
in.close();
} catch (IOException e) {
}
}
if (out != null) {
try {
out.close();
} catch (IOException e) {
}
}
}
return decompressed;
}
public static byte[] gzip(byte[] data) throws Exception {
ByteArrayOutputStream bos = new ByteArrayOutputStream();
GZIPOutputStream gzip = new GZIPOutputStream(bos);
gzip.write(data);
gzip.finish();
gzip.close();
byte[] ret = bos.toByteArray();
bos.close();
return ret;
}
public static byte[] ungzip(byte[] data) throws Exception {
ByteArrayInputStream bis = new ByteArrayInputStream(data);
GZIPInputStream gzip = new GZIPInputStream(bis);
byte[] buf = new byte[102400];
int num = -1;
ByteArrayOutputStream bos = new ByteArrayOutputStream();
while ((num = gzip.read(buf, 0, buf.length)) != -1) {
bos.write(buf, 0, num);
}
gzip.close();
bis.close();
byte[] ret = bos.toByteArray();
bos.flush();
bos.close();
return ret;
}
/**
* 删除文件
*
* @param @param path
* @param @return 设定文件
* @return Boolean 返回类型
* @Title: deleteFile
* @Description:删除文件
*/
public static Boolean deleteFile(String path) {
int tryNum = 10;
Boolean succeedDelete = true;
File file = new File(path);
if (file.isFile() && file.exists()) {
for (int i = 0; i < tryNum; i++) {
succeedDelete = file.delete();
if (succeedDelete) {
// LOGGER.info("尝试删除" + file.getPath() + "的次数：" + (i + 1), ".*");
break;
} else {
System.gc();
try {
Thread.sleep(200);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
}
return succeedDelete;
}
/**
* 删除文件前非空校验，防止空指针
*
* @param path
*/
public static void checkEmptyAndDeleteFile(String path) {
if (!isEmpty(path)) {
deleteFile(path);
}
}
/**
* 组装存储文件名
*
* @param @param fileName
* @param @param specialid
* @param @return 设定文件
* @return String 返回类型
* @Title: assembleFileName
* @Description: 组装存储文件名
*/
public static String assembleFileName(String fileName, long specialid) {
String[] fileStrs = fileName.split("\\.");
fileName = fileStrs[0] + "_" + specialid + "." + fileStrs[1];
return fileName;
}
/**
* 组装可传递的urlid，无特殊符号,无空格，换行符
*
* @param @param project
* @param @param name
* @param @param createAt
* @param @return 设定文件
* @return String 返回类型
* @Title: assembleUrlId
* @Description: 组装可传递的urlid，无特殊符号,无空格，换行符
*/
public static String assembleUrlId(String... strings) {
StringBuilder res = new StringBuilder();
for (int i = 0; i < strings.length; i++) {
String str = strings[i];
str = StringUtils.deleteWhitespace(str);
// 去除全部标点
str = str.replaceAll("[\\p{P}+~$`^=|<>～｀＄＾＋＝｜＜＞￥×]", "");
if (isEmpty(res.toString())) {
res.append(str);
} else {
res.append("_").append(str);
}
}
return res.toString();
}
/**
* "xxx xx，xx xxx"内容判断是否命中关键词
*
* @param @param content
* @param @param keyword
* @param @return 设定文件
* @return boolean 返回类型
* @Title: eventIsHitkeyword
* @Description: "xxx xx，xx xxx"内容判断是否命中关键词
*/
public static boolean eventIsHitkeyword(String content, String keyword) {
boolean res = false;
// 处理自定义关键词,分割或
String[] words = keyword.split("\\|");
for (int i = 0; i < words.length; i++) {
String word = words[i];
// 分割且
String[] keys = word.split(" ");
boolean ishit = true;
for (int j = 0; j < keys.length; j++) {
String key = keys[j];
ishit = Tools.approximateStringMatching(content, key);
if (!ishit) {
break;
}
}
// 如果且关系都有命中则，满足条件
if (ishit) {
res = true;
break;
}
}
return res;
}
/**
* 按总量，页大小计算总页数
*
* @param total
* @param pageSize
* @return int
*/
public static int totalPage(long total, int pageSize) {
return (int) (total + pageSize - 1) / pageSize;
}
/**
* 计算图片字节大小
*
* @param @param image
* @param @return 设定文件
* @return Integer 返回类型
* @Title: imageSize
* @Description: 计算图片字节大小
*/
public static Integer imageSize(String image) {
String equal = "=";
// 1.需要计算文件流大小，首先把头部的data:image/png;base64,（注意有逗号）去掉。
String str = image.substring(22);
// 2.找到等号，把等号也去掉
Integer equalIndex = str.indexOf(equal);
if (str.indexOf(equal) > 0) {
str = str.substring(0, equalIndex);
}
// 3.原来的字符流大小，单位为字节
Integer strLength = str.length();
// 4.计算后得到的文件流大小，单位为字节
Integer size = strLength - (strLength / 8) * 2;
return size;
}
/**
* Base64存图片指定位置
*
* @param @param pic
* @param @param path 设定文件
* @return void 返回类型
* @throws IOException
* @Title: convertBase64DataToImage
* @Description: Base64存图片指定位置
*/
public static void convertBase64DataToImage(String base64ImgData, String filePath) throws IOException {
byte[] imageByte = Base64.decodeBase64(base64ImgData);
// 生成文件
File imageFile = new File(filePath);
imageFile.createNewFile();
if (!imageFile.exists()) {
imageFile.createNewFile();
}
OutputStream imageStream = new FileOutputStream(imageFile);
imageStream.write(imageByte);
imageStream.flush();
imageStream.close();
}
/**
* 正则表达式匹配结果
*
* @param line
* @param pattern
* @return List<String>
*/
public static List<String> patternMatchFind(String line, String pattern) {
List<String> res = new ArrayList<>();
// 创建 Pattern 对象
Pattern r = Pattern.compile(pattern);
// 现在创建 matcher 对象
Matcher m = r.matcher(line);
while (m.find()) {
res.add(m.group());
}
return res;
}
/**
* 排序多个标签，防止相同标签不同顺序，被认为是两种标签
*
* @param tag
* @return
*/
public static String sortTag(String tag) {
int lengthLimit = 2;
String[] tags = tag.split(",");
if (null != tags && tags.length > lengthLimit) {
List<String> tempTagList = Arrays.asList(tags).subList(1, tags.length);
tempTagList.sort(Comparator.naturalOrder());
return "," + StringUtils.join(tempTagList, ",");
}
return tag;
}
/**
* 获取主机名
*
* @param url
* @return
*/
public static String getHost(String url) {
try {
return new URL(url).getHost();
} catch (MalformedURLException e) {
e.printStackTrace();
}
return null;
}
/**
* 去掉头部的“//”、去掉尾部的“/”
*
* @param url
* @return String
*/
public static String cutUrl(String url) {
url = url.substring((url.indexOf("//") == -1 ? -2 : url.indexOf("//")) + 2);
url = url.substring(0, url.indexOf("/") == -1 ? url.length() : url.indexOf("/"));
return url;
}
/**
* 防止表格单元格内容过长，截取
*
* @param cell
* @return String
*/
public static String cellInterception(String cell) {
int start = 0;
int max = 5000;
if (isEmpty(cell)) {
return cell;
} else {
if (cell.length() >= max) {
cell = cell.substring(start, max);
}
}
return cell;
}
/**
* 获取进度
*
* @param part
* @param all
* @param max
* @return int
*/
public static int getSchedule(int part, int all, int max) {
int schedule = (int) (((double) part / (double) all) * max);
return schedule;
}
/**
* 处理微博的用户的认证类型
*
* @param vtype
* @return String
*/
public static String changeVtype(int vtype) {
String type;
switch (vtype) {
case -2:
type = "未知";
break;
case -1:
type = "普通用户";
break;
case 0:
type = "名人";
break;
case 1:
type = "政府";
break;
case 2:
type = "企业";
break;
case 3:
type = "媒体";
break;
case 4:
type = "校园";
break;
case 5:
type = "网站";
break;
case 6:
type = "应用";
break;
case 7:
type = "团体";
break;
case 10:
type = "微博女郎";
break;
case 220:
type = "达人";
break;
case 200:
type = "达人";
break;
default:
type = "未知";
break;
}
return type;
}
/**
* 获取字符串里的数字
*
* @param @param text
* @param @return 设定文件
* @return String 返回类型
* @Title: getIntStr
* @Description: 获取字符串里的数字
*/
public static String getIntStr(String text) {
String num = "(\\d+)";
Pattern p = Pattern.compile(num);
Matcher m = p.matcher(text);
String find = "";
while (m.find()) {
find = find + m.group(1).toString();
}
return find;
}
/**
* sleep
*
* @param millis
* @return void
*/
public static void sleep(long millis) {
try {
Thread.sleep(millis);
} catch (InterruptedException e) {
}
}
/**
* 递归获取集合排列组合
*
* @param list
* @param preStr
* @param saveList
*/
public static void permutation(List<List<String>> list, String preStr, List<String> saveList) {
int size = list.size();
if (1 == size) {
for (int i = 0; i < list.get(0).size(); i++) {
saveList.add(preStr + list.get(0).get(i));
}
} else {
List<String> permu = new ArrayList<>(list.get(0));
List<List<String>> now = new ArrayList<>(list);
now.remove(0);
for (int i = 0; i < permu.size(); i++) {
permutation(now, preStr + permu.get(i), saveList);
}
}
}
public static Boolean hitKeywords(String content, String keyword) {
List<String> res = new ArrayList<>();
// 处理自定义关键词,分割或
String[] words = keyword.split("\\|");
for (String word : words) {
// 分割且
String[] keys = word.split(" ");
boolean ishit = true;
for (String key : keys) {
ishit = Tools.approximateStringMatching(content, key);
if (!ishit) {
break;
}
}
// 如果且关系都有命中则，满足条件
if (ishit) {
res.add(word);
}
}
return !Tools.isEmpty(res);
}
}
src/main/resources/application-dev.properties 0 → 100644
#primary-mongo
primary.uri=mongodb://qbjc:asSADf5ffs@115.236.59.88:30001/qbjc?authSource=admin
#second-mongo
secondary.uri=mongodb://192.168.0.246:27017/channel-platform
#mongo 连接池配置
mongo.connectionsPerHost=50
mongo.threadsAllowedToBlockForConnectionMultiplier=10
mongo.connectTimeout=30000
mongo.maxWaitTime=50000
mongo.autoConnectRetry=true
mongo.socketKeepAlive=true
mongo.socketTimeout=120000
mongo.slaveOk=true
#es
es.httpHost=9400
es.httpIp=202.107.192.94
es.clusterNodes=202.107.192.94:9500
es.clusterName=zhiweidata-es
es.transportSniff=false
#path to redisson.yaml or redisson-local.json
spring.redis.redisson.config=classpath:redisson-dev.json
#zookeeper
zookeeper.registry=zookeeper://192.168.0.11:2181?backup=192.168.0.30:2181,192.168.0.35:2181
channel-index.zookeeper.registry=zookeeper://192.168.0.11:2182?backup=192.168.0.30:2182,192.168.0.35:2182
#dubbo
dubbo.mark.group=local-marker
dubbo.filter.group=local-filter
dubbo.automark.group=automatic-test_sjj
dubbo.channel.group=channel-index-local
#baiduMap
baiduMap.ak=2rkFM5ofHuhLvfQnEuuv5GkTXnSPEdiV
src/main/resources/application-prod.properties 0 → 100644
#primary-mongo
primary.uri=mongodb://qbjc:asSADf5ffs@192.168.0.101:30000/qbjc?authSource=admin
#second-mongo
secondary.uri=secondary.uri=mongodb://qbjcuser:qbjc1q2w3e4r@202.107.192.94:30000/eventMuseum?authSource=admin
#mongo 连接池配置
mongo.connectionsPerHost=200
mongo.threadsAllowedToBlockForConnectionMultiplier=10
mongo.connectTimeout=30000
mongo.maxWaitTime=50000
mongo.autoConnectRetry=true
mongo.socketKeepAlive=true
mongo.socketTimeout=120000
mongo.slaveOk=true
#es
es.httpHost=9200
es.httpIp=192.168.0.25
es.clusterNodes=192.168.0.25:9300,192.168.0.26:9300,192.168.0.27:9300
es.clusterName=zhiweidata-es
es.transportSniff=true
#path to redisson.yaml or redisson-local.json
spring.redis.redisson.config=classpath:redisson-prod.json
#zookeeper
zookeeper.registry=zookeeper://192.168.0.11:2181?backup=192.168.0.30:2181,192.168.0.35:2181
channel-index.zookeeper.registry=zookeeper://192.168.0.11:2182?backup=192.168.0.30:2182,192.168.0.35:2182
#dubbo
dubbo.mark.group=local-marker
dubbo.filter.group=local-filter
dubbo.automark.group=automatic-test_sjj
dubbo.channel.group=channel-index-local
\ No newline at end of file
src/main/resources/application.properties 0 → 100644
server.port=5788
spring.profiles.active=prod
\ No newline at end of file
src/main/resources/log4j2.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE xml>
<!-- log4j2 自身的日志级别 -->
<Configuration status="WARN">
<Properties> <!-- 配置日志文件输出目录 -->
<Property name="LOG_HOME">./log/</Property>
<property name="APP_NAME">qbjc-back</property>
</Properties>
<Appenders>
<!-- 定义日志输出地 -->
<Console name="Console" target="SYSTEM_OUT">
<PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level %logger{36} %L [%t] - %msg%n"/>
</Console>
<!--[%t]-->
<!-- INFO级别日志 -->
<RollingFile name="info_appender" fileName="${LOG_HOME}/${APP_NAME}.log"
filePattern="${LOG_HOME}/${APP_NAME}-%d{yyyy-MM-dd}-%i.log">
<Filters>     
<PushLogFilter onMatch="NEUTRAL" onMismatch="DENY"/>   
</Filters>
<PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level %logger{36} %L [%t] - %msg%n"/>
<Policies>
<TimeBasedTriggeringPolicy/>
<SizeBasedTriggeringPolicy size="20 MB"/>
</Policies>
</RollingFile>
</Appenders>
<Loggers>
<AsyncRoot level="info">
<AppenderRef ref="Console"/>
<AppenderRef ref="info_appender"/>
</AsyncRoot>
<!--<AsyncLogger level="info">-->
<!--<AppenderRef ref="Console"/>-->
<!--<AppenderRef ref="info_appender"/>-->
<!--</AsyncLogger>-->
<!-- 所有的 logger 均继承 Root
当 additivity 为 true 时, 父子 logger 均会打印
当 additivity 为 false 时, 仅子 logger 会打印 -->
<Logger name="mylog" level="error" additivity="false">
<AppenderRef ref="Console"/>
</Logger>
<Logger name="com.alibaba.dubbo.monitor.dubbo.DubboMonitor" level="OFF" >
</Logger>
<Logger name="org.apache.dubbo.monitor.dubbo.DubboMonitor" level="OFF" >
</Logger>
</Loggers>
</Configuration>
src/main/resources/pushlog.properties 0 → 100644
robot.push.address=https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=e2218c6e-af6a-4296-9d75-7178b941a3b5
robot.push.enable=true
robot.push.filterclass=org.apache.dubbo.common.Version,com.alibaba.dubbo.common.Version,org.apache.dubbo.monitor.dubbo.DubboMonitor,com.alibaba.dubbo.monitor.dubbo.DubboMonitor
robot.push.level=warn
\ No newline at end of file
src/main/resources/redisson-dev.json 0 → 100644
{
"singleServerConfig": {
"idleConnectionTimeout": 60000,
"pingTimeout": 1000,
"connectTimeout": 10000,
"timeout": 100000,
"retryAttempts": 3,
"retryInterval": 1500,
"address": "redis://115.236.59.91:7373",
"connectionMinimumIdleSize": 32,
"connectionPoolSize": 64,
"database": 4
},
"codec": {
"class": "org.redisson.client.codec.StringCodec"
},
"transportMode": "NIO"
}
\ No newline at end of file
src/main/resources/redisson-local.json 0 → 100644
{
"singleServerConfig": {
"idleConnectionTimeout": 60000,
"pingTimeout": 1000,
"connectTimeout": 10000,
"timeout": 3000,
"retryAttempts": 3,
"retryInterval": 1500,
"address": "redis://192.168.0.41:6379",
"connectionMinimumIdleSize": 64,
"connectionPoolSize": 128,
"database": 12
},
"codec": {
"class": "org.redisson.client.codec.ByteArrayCodec"
},
"transportMode": "NIO"
}
\ No newline at end of file
src/main/resources/redisson-prod.json 0 → 100644
{
"singleServerConfig": {
"idleConnectionTimeout": 60000,
"pingTimeout": 1000,
"connectTimeout": 10000,
"timeout": 3000,
"retryAttempts": 3,
"retryInterval": 1500,
"address": "redis://192.168.0.2:7373",
"connectionMinimumIdleSize": 128,
"connectionPoolSize": 256,
"database": 4
},
"codec": {
"class": "org.redisson.client.codec.StringCodec"
},
"transportMode": "NIO"
}
\ No newline at end of file
src/test/java/com/zhiwei/qbjcutils/QbjcUtilsApplicationTests.java 0 → 100644
package com.zhiwei.qbjcutils;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.base.category.ClassB;
import com.zhiwei.qbjcutils.config.GenericAttribute;
import com.zhiwei.qbjcutils.dubbo.DubboClient;
import com.zhiwei.qbjcutils.es.EsIndexTools;
import com.zhiwei.qbjcutils.es.dao.EsDao;
import com.zhiwei.qbjcutils.pojo.dto.BaseObjectCacheDTO;
import com.zhiwei.qbjcutils.pojo.emums.TimeUnit;
import com.zhiwei.qbjcutils.pojo.entity.*;
import com.zhiwei.qbjcutils.redis.RedisTools;
import com.zhiwei.qbjcutils.service.*;
import com.zhiwei.qbjcutils.utils.TimeUtils;
import com.zhiwei.qbjcutils.utils.Tools;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.redisson.api.RScoredSortedSet;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.test.context.junit4.SpringRunner;
import org.springframework.test.context.testng.AbstractTestNGSpringContextTests;
import java.util.*;
import java.util.stream.Collectors;
@SpringBootTest
@RunWith(SpringRunner.class)
public class QbjcUtilsApplicationTests extends AbstractTestNGSpringContextTests {
private static final Logger log = LogManager.getLogger(QbjcUtilsApplicationTests.class);
@Autowired
private DubboClient client;
@Autowired
private MessageFlowService messageFlowService;
@Autowired
private PlatformService platformService;
@Test
public void contextLoads() {
System.out.println(TimeUtils.getFirstDayOfLastMonth(1504195140000L));
}
@Autowired
@Qualifier("primaryMongoTemplate")
private MongoTemplate mongoTemplate;
@Test
public void run() {
String[] plaforms = {"平媒","自媒体"};
for (String plaform : plaforms) {
MessagePlatform messagePlatform = platformService.getPlatformByName(plaform);
List<BaseObjectCacheDTO> baseObjectDTOs = messageFlowService.
getMessageFlowEntityWithTime(1586318400000L, 1586404800000L, messagePlatform, "全部", "腾讯");
for (BaseObjectCacheDTO baseObjectDTO : baseObjectDTOs) {
secondMongoTemplate.insert(baseObjectDTO,"tencent_base_object_cache_tag");
}
}
}
@Autowired
@Qualifier("secondaryMongoTemplate")
private MongoTemplate secondMongoTemplate;
@Test
public void cacheData() {
// String[] keys = {"微信", "网媒", "今日头条", "自媒体"};
// String[] keys = {"微博", "贴吧论坛", "问答"};
List<String> keys = platformService.getPlatforms().stream().map(MessagePlatform::getName).collect(Collectors.toList());
for (String pt : keys) {
List<BaseObjectCache> excelEntities = messageFlowService.getBaseObjectCacheFromMessageFlow(pt, "全部", "Boss直聘");
for (BaseObjectCache baseObjectCache : excelEntities) {
try {
if(baseObjectCache.getId().equals("82d2a53c812c896ad91d1984a2402a1a")){
System.out.println(pt);
}
secondMongoTemplate.insert(baseObjectCache, "Boss_nlp_base_object_cache");
} catch (Exception e) {
System.err.println(baseObjectCache + " " + pt);
}
}
}
}
@Autowired
private ClassifyUploadDataService classifyUploadDataService;
@Test
public void classifyUploadData(){
String path="D:\\stWork\\download\\upload\\字节跳动-测试综合数据.xlsx";
String project="今日头条";
classifyUploadDataService.classifyUploadData(path, project);
}
@Autowired
private DirectChannelService directChannelService;
@Test
public void direct() throws Exception {
directChannelService.readAndAddDirectChannel("美团", "C:\\Users\\pread\\Documents\\WXWork\\1688854025205527\\Cache\\File\\2020-05\\定向渠道.xlsx");
}
@Autowired
private MarkDataCountService markDataCountService;
@Test
public void platformCount(){
markDataCountService.outputMarkDataCountByPlatforms(1587312000000L, 1587398400000L, "time", TimeUnit.HOUR.name(), "测试");
}
@Autowired
private RedisTools redisTools;
@Test
public void dataTest(){
String key = "YUQING:Supply:Cache:测试_4-21测试任务2_1587448264766";
RScoredSortedSet<String> scoredSortedSet = redisTools.getAllMessageFlowEntity(key);
List<JSONObject> list = scoredSortedSet.stream().map(JSONObject::parseObject).collect(Collectors.toList());
for (JSONObject info : list) {
String data = Tools.gunzip(info.getString("data"));
if(data.contains("#腾讯新闻# 病毒疑似来自美国，为何无人向其索赔？反而对中国狮子大开口病毒疑似来自美国，为何无人向其索赔？反而对中国狮子大开口 病毒疑似来自美国，为何无人向其索赔？反而对中国狮子大开口 来自@腾讯新闻")){
System.out.println(data);
}
}
}
@Autowired
private DubboClient dubboClient;
@Test
public void test1(){
// List<Tag> res = mongoTemplate.find(new Query(Criteria.where("project").ne(null)), Tag.class);
// redisTools.setAllTags(res);
platformService.autoUpdate();
// List<String> list = Arrays.asList("151521234_PC_0", "151528233_PC_0", "151520263_PC_0", "151519234_PC_0");
// for (String header : list) {
// String[] s = header.split("_");
// Integer integer = Integer.valueOf(s[0]);
// ClassD classD = ClassCodec.decodeClassD(integer);
// System.out.println("header: " + header + " c1: " + classD.typeA().name() + " c2: " + classD.typeB().name() + " c3: " + classD.typeC().name() + " c4: " + classD.typeD().title());
// }
}
@Test
public void test2(){
List<String> keys = platformService.getPlatforms().stream().map(MessagePlatform::getName).collect(Collectors.toList());
for (String pt : keys) {
List<BaseObjectCacheKeyword> excelEntities = messageFlowService.getBaseObjectCacheKeywordFromMessageFlow(pt, "全部", "腾讯");
for (BaseObjectCacheKeyword baseObjectCache : excelEntities) {
try {
if(baseObjectCache.getId().equals("82d2a53c812c896ad91d1984a2402a1a")){
System.out.println(JSONObject.toJSONString(baseObjectCache));
}
// secondMongoTemplate.insert(baseObjectCache, "tencent_nlp_base_object_cache_20200507");
} catch (Exception e) {
System.err.println(baseObjectCache + " " + pt);
}
}
}
}
@Autowired
private EsDao esDao;
@Test
public void test3(){
// cal2(1514736000000L, 1546272000000L,"腾讯", "微信|微信支付|张小龙" );
// cal2(1546272000000L, 1577808000000L,"腾讯", "微信|微信支付|张小龙" );
// cal2(1577808000000L, 1590059825000L,"腾讯", "微信|微信支付|张小龙" );
// cal1(1546272000000L, 1548950400000L, "京东");
// cal1(1548950400000L, 1551369600000L, "京东");
// cal1(1551369600000L, 1554048000000L, "京东");
// cal1(1554048000000L, 1556640000000L, "京东");
cal(1588262400000L, 1590940800000L, "美团");
cal(1590940800000L, 1593532800000L, "美团");
cal(1593532800000L, 1596211200000L, "美团");
cal(1596211200000L, 1597507200000L, "美团");
// cal(1567267200000L, 1569859200000L, "京东");
// cal(1569859200000L, 1572537600000L, "京东");
// cal(1572537600000L, 1575129600000L, "京东");
// cal(1575129600000L, 1577808000000L, "京东");
}
@Autowired
private EsIndexTools esIndexTools;
public void cal(long startTime, long endTime, String project){
String[] indexs = esIndexTools.getMarkIndexs();
String[] type = {ClassB.TypeB.COMPLETE.name(), ClassB.TypeB.INCOMPLETE.name(), ClassB.TypeB.QA.name(), ClassB.TypeB.VIDEO.name()};
BoolQueryBuilder query = QueryBuilders.boolQuery()
.must(QueryBuilders.termQuery(GenericAttribute.ES_MGROUP, project))
.mustNot(QueryBuilders.termQuery(GenericAttribute.ES_MTAG, ""))
.must(QueryBuilders.rangeQuery(GenericAttribute.ES_MTIME).gte(startTime).lt(endTime));
SearchHits searchHits = esDao.search(indexs, type, null, query, null, 0, 1, null);
System.out.println(project+" "+TimeUtils.DATE_FORMATE.format(startTime) + "-" + TimeUtils.DATE_FORMATE.format(endTime) + "：" + searchHits.getTotalHits());
}
public void cal1(long startTime, long endTime, String project){
Map<String, Integer> count =new HashMap<>();
String[] indexs = esIndexTools.getMarkIndexs();
String[] type = {ClassB.TypeB.COMPLETE.name(), ClassB.TypeB.INCOMPLETE.name(), ClassB.TypeB.QA.name(), ClassB.TypeB.VIDEO.name()};
BoolQueryBuilder query = QueryBuilders.boolQuery()
.must(QueryBuilders.termQuery(GenericAttribute.ES_MGROUP, project))
.mustNot(QueryBuilders.termQuery(GenericAttribute.ES_MTAG, ""))
.must(QueryBuilders.rangeQuery(GenericAttribute.ES_MTIME).gte(startTime).lt(endTime));
List<Tag> tags = mongoTemplate.find(new Query(Criteria.where("project").is(project).and("groupName").is("情感倾向")), Tag.class);
SearchResponse searchResponse = esDao.scroll(indexs, type, null, query, null, 0, 10000, new String[]{"mtag"}, null, null, TimeValue.timeValueMinutes(2));
int totalPage = (int) searchResponse.getHits().getTotalHits() / 10000;
for (SearchHit hit : searchResponse.getHits()) {
countHits(count, tags, hit);
}
for (int i = 0; i < totalPage; i++) {
searchResponse = esDao.scroll(searchResponse.getScrollId(), TimeValue.timeValueMinutes(2));
for (SearchHit searchHit : searchResponse.getHits()) {
countHits(count, tags, searchHit);
}
}
System.out.println(project+" "+TimeUtils.DATE_FORMATE.format(startTime) + "-" + TimeUtils.DATE_FORMATE.format(endTime) + "：" + count);
}
public void cal2(long startTime, long endTime, String project, String keyword) {
int num = 0;
String[] indexs = esIndexTools.getMarkIndexs();
String[] type = {ClassB.TypeB.COMPLETE.name(), ClassB.TypeB.INCOMPLETE.name(), ClassB.TypeB.QA.name(), ClassB.TypeB.VIDEO.name()};
BoolQueryBuilder query = QueryBuilders.boolQuery()
.must(QueryBuilders.termQuery(GenericAttribute.ES_MGROUP, project))
.mustNot(QueryBuilders.termQuery(GenericAttribute.ES_MTAG, ""))
.must(QueryBuilders.rangeQuery(GenericAttribute.ES_MTIME).gte(startTime).lt(endTime));
SearchResponse searchResponse = esDao.scroll(indexs, type, null, query, null, 0, 10000, new String[]{"ind_full_text"}, null, null, TimeValue.timeValueMinutes(2));
int totalPage = (int) searchResponse.getHits().getTotalHits() / 10000;
for (SearchHit hit : searchResponse.getHits()) {
String text = JSONObject.parseObject(hit.getSourceAsString()).getString(GenericAttribute.ES_IND_FULL_TEXT);
if (Tools.hitKeywords(text, keyword)) {
num++;
}
}
for (int i = 0; i < totalPage; i++) {
searchResponse = esDao.scroll(searchResponse.getScrollId(), TimeValue.timeValueMinutes(2));
for (SearchHit searchHit : searchResponse.getHits()) {
String text = JSONObject.parseObject(searchHit.getSourceAsString()).getString(GenericAttribute.ES_IND_FULL_TEXT);
if (Tools.hitKeywords(text, keyword)) {
num++;
}
}
}
System.out.println(project + " " + TimeUtils.DATE_FORMATE.format(startTime) + "-" + TimeUtils.DATE_FORMATE.format(endTime) + "：" + num);
}
private void countHits(Map<String, Integer> count, List<Tag> tags, SearchHit hit) {
String mtag = JSONObject.parseObject(hit.getSourceAsString()).getString("mtag");
for (Tag tag : tags) {
if(mtag.contains(tag.getUniqueId())){
Integer num = count.getOrDefault(tag.getName(), 0);
num++;
count.put(tag.getName(), num);
break;
}
}
}
@Test
public void calDistance(){
List<NoiseRule> all = mongoTemplate.findAll(NoiseRule.class);
for (NoiseRule noiseRule : all) {
String content = "channel".equals(noiseRule.getRuleType()) ? noiseRule.getChannelQd() : noiseRule.getKeyWords();
List<String> list = "channel".equals(noiseRule.getRuleType()) ? noiseRule.getChannelPt() : noiseRule.getKeyWordPt();
String ruleType = "channel".equals(noiseRule.getRuleType()) ? "渠道" : "关键词";
String noiseRuleExplain = "channel".equals(noiseRule.getRuleType()) ? "渠道" : "关键词-" + noiseRule.getKwSearch();
NoiseRuleEntity noiseRuleEntity = new NoiseRuleEntity(null, noiseRule.getRuleName(), content, ruleType, noiseRule.getKwSearch(), 0L, 0L,noiseRuleExplain ,list, noiseRule.getProject(), noiseRule.getCreateAt(), System.currentTimeMillis(), noiseRule.getSubmitter());
mongoTemplate.insert(noiseRuleEntity);
}
}
@Test
public void runtest(){
List<String> list = Arrays.asList("腾讯", "腾讯竞品", "腾讯视频", "快手", "京东金融", "趣头条", "跟谁学", "科慕化学", "微保");
for (String project : list) {
cal(1585670400000L, 1593532800000L, project);
}
}
}
src/test/java/com/zhiwei/qbjcutils/Tests.java 0 → 100644
package com.zhiwei.qbjcutils;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.qbjcutils.pojo.excelentity.SentimentInfo;
import com.zhiwei.qbjcutils.utils.Tools;
import org.apache.logging.log4j.Level;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.junit.Test;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.text.NumberFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/**
* Description:
*
* @author LiuMingHuan
* @classname Test
* @date 2020/5/18
*/
public class Tests {
private final static String GEO_CODING = "http://api.map.baidu.com/geocoding/v3/?&output=json&ak=";
private final static String DRIVING = "http://api.map.baidu.com/routematrix/v2/driving?output=json&tactics=13&ak=2rkFM5ofHuhLvfQnEuuv5GkTXnSPEdiV&origins=&destinations=";
@Test
public void run(){
String laiLngUrl = new StringBuilder(GEO_CODING).append("2rkFM5ofHuhLvfQnEuuv5GkTXnSPEdiV").append("&address=").append("广州").toString();
JSONObject latLngInfo = JSONObject.parseObject(Tools.httpGet(laiLngUrl));
System.out.println(latLngInfo.toJSONString());
}
@Test
public void aopTest() {
List<String> excelNames = Arrays.asList("地摊经济引关注", "广州市方圆实验小学教师涉嫌体罚学生", "十三届全国人大三次会议闭幕", "微软中国被列为被执行人", "中共中央、国务院印发《海南自由贸易港建设总体方案》");
for (String excelName : excelNames) {
System.out.println(excelName + "分析：");
String predictPath = "C:\\Users\\pread\\Desktop\\新建文件夹\\predict\\";
String zhiweiPath = "C:\\Users\\pread\\Desktop\\新建文件夹\\result\\";
String baiduPath = "C:\\Users\\pread\\Desktop\\新建文件夹\\baidu\\";
Map<String, SentimentInfo> local = readExcel(new File(zhiweiPath + excelName + ".xlsx")).stream().collect(Collectors.toMap(SentimentInfo::getId, m -> m));
Map<String, SentimentInfo> baidu = readExcel(new File(baiduPath + excelName + ".xlsx")).stream().collect(Collectors.toMap(SentimentInfo::getId, m -> m));
Map<String, SentimentInfo> module = readExcel(new File(predictPath + excelName + ".xlsx")).stream().collect(Collectors.toMap(SentimentInfo::getId, m -> m));
System.out.println(str(excelName, permunate(local, baidu), "本地", "百度"));
System.out.println(str(excelName, permunate(local, module), "本地", "模型"));
System.out.println(str(excelName, permunate(module, baidu), "模型", "百度"));
System.out.println("--------------------");
}
}
public List<SentimentInfo> readExcel(File file) {
try {
//create input stream ,get data
InputStream stream = new FileInputStream(file);
Workbook wb = new XSSFWorkbook(stream);
int sheetSize = wb.getNumberOfSheets();
List<SentimentInfo> sheetList = new ArrayList<>();
for (int i = 0; i < sheetSize; i++) {
List<SentimentInfo> outerList = new ArrayList<>();
Sheet sheet = wb.getSheetAt(i);
column:for (int j = 1; j <= sheet.getLastRowNum(); j++) {
SentimentInfo sentimentInfo = new SentimentInfo();
sentimentInfo.setId(sheet.getRow(j).getCell(0).getStringCellValue());
for (int c = 0; c < sheet.getRow(j).getLastCellNum(); c++) {
Cell cell = sheet.getRow(j).getCell(c);
String head = sheet.getRow(0).getCell(c, Row.MissingCellPolicy.CREATE_NULL_AS_BLANK).toString();
if("分类结果".equals(head)){
Double numericCellValue = cell.getNumericCellValue();
sentimentInfo.setSentiment(switchSentiment(numericCellValue.intValue()));
break;
}else if("".equals(head)){
String str = cell.getStringCellValue();
sentimentInfo.setSentiment(str);
break;
}
}
outerList.add(sentimentInfo);
}
sheetList.addAll(outerList);
}
return sheetList;
} catch (Exception e) {
e.printStackTrace();
}
return null;
}
public String switchSentiment(int sentiment){
switch (sentiment){
case 0:
return "负面";
case 1:
return "中性";
case 2:
return "正面";
default:
return "";
}
}
public JSONObject permunate(Map<String, SentimentInfo> matched, Map<String, SentimentInfo> bematched){
JSONObject jsonObject = new JSONObject();
int count =0;
int matchedCount = 0;
for (Map.Entry<String, SentimentInfo> entry : matched.entrySet()) {
SentimentInfo value = entry.getValue();
SentimentInfo sentimentInfo = bematched.get(entry.getKey());
if(!Tools.isNull(sentimentInfo)){
count++;
if(value.getSentiment().equals(sentimentInfo.getSentiment())){
matchedCount++;
}
}
}
jsonObject.put("matchCount", matchedCount);
jsonObject.put("unMatchCount", count - matchedCount);
jsonObject.put("hasId", count);
jsonObject.put("matchMapSize", matched.size());
jsonObject.put("beMatchMapSize", bematched.size());
return jsonObject;
}
public String str(String fileName, JSONObject jsonObject, String matched, String bematched){
StringBuilder stringBuilder = new StringBuilder(fileName);
float v = (float) jsonObject.getIntValue("matchCount") / jsonObject.getInteger("hasId");
float v1 = (float) jsonObject.getIntValue("unMatchCount") / jsonObject.getInteger("hasId");
NumberFormat instance = NumberFormat.getInstance();
instance.setMaximumFractionDigits(2);
stringBuilder.append(" ").append(matched).append("-").append(bematched).append(" 判断一致: ").append(instance.format(v)).append(" 判断不一致：").append(instance.format(v1));
return stringBuilder.toString();
}
@Test
public void normalTest(){
System.out.println(Arrays.toString(Level.values()));
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment