Skip to content

D
data-warning
Commit 4770a03c by chenyu
Options

init

parents
Showing with 2613 additions and 0 deletions
.gitignore 0 → 100644
<<<<<<< HEAD
# maven ignore
target/
*.jar
*.war
*.zip
*.tar
*.tar.gz
dependency-reduced-pom.xml
# eclipse ignore
.settings/
.project
.factorypath
.classpath
# idea ignore
.idea/
*.ipr
*.iml
*.iws
# git ignore
*/.gitignore
# temp ignore
*.log
*.cache
*.diff
*.patch
*.tmp
# test ignore
test-output/
# system ignore
.DS_Store
Thumbs.db
# directory ignore
log/
backup/
# jekins ignore
Jenkinsfile
=======
.idea
target
*.iml
testFile
>>>>>>> 91ff765fc773b7ac5371845c20ce42f604a5e83e
pom.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.zhiwei</groupId>
<artifactId>data-warning</artifactId>
<version>1.0-SNAPSHOT</version>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<spring-boot.version>2.2.6.RELEASE</spring-boot.version>
<spring-jdbc.version>5.2.5.RELEASE</spring-jdbc.version>
<redisson-spring.version>3.12.5</redisson-spring.version>
<mybatis-spring-boot.version>1.3.1</mybatis-spring-boot.version>
<druid-spring.version>1.1.22</druid-spring.version>
<mysql.version>8.0.19</mysql.version>
<commons-lang3.version>3.10</commons-lang3.version>
<fastjson.version>1.2.68</fastjson.version>
<easyexcel.version>2.1.7</easyexcel.version>
<snappy.version>1.1.7.3</snappy.version>
<java-jwt.version>2.2.0</java-jwt.version>
<elasticsearch.version>7.9.2</elasticsearch.version>
<log4j.version>2.13.1</log4j.version>
<slf4j.version>1.8.0-beta4</slf4j.version>
<disruptor.version>3.4.2</disruptor.version>
<crawler-core.vsersion>0.6.6.9-RELEASE</crawler-core.vsersion>
</properties>
<dependencies>
<!-- spring相关依赖 -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
<version>${spring-boot.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<version>${spring-boot.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-aop</artifactId>
<version>${spring-boot.version}</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-jdbc</artifactId>
<version>${spring-jdbc.version}</version>
</dependency>
<dependency>
<groupId>org.redisson</groupId>
<artifactId>redisson-spring-boot-starter</artifactId>
<version>${redisson-spring.version}</version>
<exclusions>
<exclusion>
<artifactId>spring-boot</artifactId>
<groupId>org.springframework.boot</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.mybatis.spring.boot</groupId>
<artifactId>mybatis-spring-boot-starter</artifactId>
<version>${mybatis-spring-boot.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>druid-spring-boot-starter</artifactId>
<version>${druid-spring.version}</version>
<exclusions>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-autoconfigure</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- 其他三方依赖 -->
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>${mysql.version}</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>${commons-lang3.version}</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>${fastjson.version}</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>easyexcel</artifactId>
<version>${easyexcel.version}</version>
<exclusions>
<exclusion>
<artifactId>jaxb-api</artifactId>
<groupId>javax.xml.bind</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
<version>${snappy.version}</version>
</dependency>
<dependency>
<groupId>com.auth0</groupId>
<artifactId>java-jwt</artifactId>
<version>${java-jwt.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>${elasticsearch.version}</version>
</dependency>
<!-- 本地依赖 -->
<dependency>
<groupId>com.zhiwei.crawler</groupId>
<artifactId>crawler-core</artifactId>
<version>${crawler-core.vsersion}</version>
</dependency>
<!-- 日志依赖 -->
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-1.2-api</artifactId>
<version>${log4j.version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>${slf4j.version}</version>
</dependency>
<dependency>
<groupId>com.lmax</groupId>
<artifactId>disruptor</artifactId>
<version>${disruptor.version}</version>
</dependency>
</dependencies>
<!-- 服务端打包 -->
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<executions>
<execution>
<goals>
<!-- 打包添加依赖 -->
<goal>repackage</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
\ No newline at end of file
src/main/java/com/zhiwei/data/DataWarningApplication.java 0 → 100644
package com.zhiwei.data;
import org.mybatis.spring.annotation.MapperScan;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.scheduling.annotation.EnableScheduling;
/**
* @author chenyu
* @ClassName: DataWarningApplication
* @Description: 启动类
* @date 2021/3/10 15:51
*/
@SpringBootApplication
@EnableScheduling
@MapperScan(basePackages = "com.zhiwei.data.repository.mapper")
public class DataWarningApplication {
public static void main(String[] args) {
SpringApplication.run(DataWarningApplication.class, args);
}
}
src/main/java/com/zhiwei/data/cache/CacheLoader.java 0 → 100644
package com.zhiwei.data.cache;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.time.Duration;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import static java.util.Objects.nonNull;
/**
* @author chenyu
* @ClassName: CacheLoader
* @Description: 缓存加载器
* @date 2021/2/1 17:46
*/
public abstract class CacheLoader<T> {
private static final Logger logger = LogManager.getLogger(CacheLoader.class);
private final Object lock = new Object();
private volatile Thread loadingThread;
private volatile boolean initialized;
private volatile T cache;
public CacheLoader(String name, Duration duration) {
this(name, duration, null);
}
public CacheLoader(ScheduledExecutorService executor, Duration duration) {
this(executor, duration, null);
}
public CacheLoader(String name, Duration duration, String exceptionMsg) {
this(Executors.newScheduledThreadPool(1,
new ThreadFactoryBuilder().setNameFormat(name + "-[%d]").build()),
duration,
exceptionMsg);
}
public CacheLoader(ScheduledExecutorService executor, Duration duration, String exceptionMsg) {
this(executor, duration, exceptionMsg, null);
}
public CacheLoader(ScheduledExecutorService executor, Duration duration, String exceptionMsg, InitializationPreProcessor<T> processor) {
executor.scheduleWithFixedDelay(() -> {
try {
if (!this.initialized && nonNull(processor)) {
//前置处理 可拦截正常初始化
this.initialized = processor.processBeforeInitialization(cache);
}
initOrRefresh();
} catch (Exception e) {
if (nonNull(exceptionMsg)) {
logger.error(exceptionMsg, e);
} else {
logger.error(e);
}
}
}, 0, duration.toMillis(), TimeUnit.MILLISECONDS);
}
/**
* 初始化或刷新
*
*/
private void initOrRefresh() {
synchronized (lock) {
if (!initialized) {
loadingThread = Thread.currentThread();
cache = load();
initialized = true;
lock.notifyAll();
} else {
//refresh
cache = load();
}
}
if (nonNull(loadingThread)) {
loadingThread = null;
}
}
/**
* 检查数据是否正常初始化和刷新
*
*/
private void check() throws InterruptedException {
if (!initialized) {
synchronized (lock) {
if (!initialized) {
if (Thread.currentThread() == loadingThread) {
throw new IllegalStateException("The current thread is being initialized");
}
lock.wait();
}
}
}
}
/**
* loading data
*
* @return T
*/
protected abstract T load();
/**
* 获取锁
*
* @return Object
*/
protected Object getLock() {
return lock;
}
/**
* get cache
*
* @return T
*/
public T get() {
try {
check();
} catch (InterruptedException e) {
logger.error("The thread is interrupted");
return null;
}
return cache;
}
/**
* 可对缓存初始化进行拦截
*
*/
@FunctionalInterface
public interface InitializationPreProcessor<T> {
/**
* 初始化拦截方法
*
* @param cache cache
* @return boolean initialized
*/
boolean processBeforeInitialization(T cache);
}
}
src/main/java/com/zhiwei/data/common/CommonConstants.java 0 → 100644
package com.zhiwei.data.common;
/**
* @author chenyu
* @ClassName: CommonConstants
* @Description: 公用字符常量
* @date 2021/3/10 17:28
*/
public class CommonConstants {
private CommonConstants() {}
public static final String DATA = "data";
public static final String MESSAGES = "messages";
public static final String STATUS = "status";
public static final String ID = "id";
public static final String NAME = "name";
public static final String ENABLE = "enable";
public static final String CLASS_B = "classB";
public static final String CLASS_C = "classC";
public static final String CLASS_D = "classD";
public static final String DIVISIBLE = "divisible";
public static final String TOP_DOMAIN = "topDomain";
public static final String CREATE_TIME = "createTime";
public static final String UPDATE_TIME = "updateTime";
public static final String TEMPLATE_ID = "templateId";
public static final String COUNT = "count";
}
src/main/java/com/zhiwei/data/common/SpecialConstants.java 0 → 100644
package com.zhiwei.data.common;
import java.util.Arrays;
import java.util.List;
/**
* @author chenyu
* @ClassName: SpecialConstants
* @Description: 公用特殊常量
* @date 2021/3/12 10:16
*/
public class SpecialConstants {
private SpecialConstants() {}
//es 查询索引名集合
public static final List<String> INDEX_NAME_FORMAT_LIST = Arrays.asList("complete_text_%s", "incomplete_text_%s", "qa_text_%s", "video_%s");
}
src/main/java/com/zhiwei/data/common/WarningLevel.java 0 → 100644
package com.zhiwei.data.common;
/**
* @author chenyu
* @ClassName: WarningLevel
* @Description: 预警等级
* @date 2021/3/10 11:07
*/
public enum WarningLevel {
NORMAL(0, "正常"),
PRIMARY(10, "初级预警"),
INTERMEDIATE(20, "中级预警"),
SERIOUS(30, "严重预警");
private int value;
private String desc;
WarningLevel(int value, String desc) {
this.value = value;
this.desc = desc;
}
public int value() {
return value;
}
public String desc() {
return desc;
}
}
src/main/java/com/zhiwei/data/config/ESClientConfig.java 0 → 100644
package com.zhiwei.data.config;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.env.Environment;
import javax.annotation.Resource;
import static java.util.Objects.nonNull;
import static java.util.Objects.requireNonNull;
/**
* @author chenyu
* @ClassName: ESClientConfig
* @Description: ES客户端配置
* @date 2020/5/6 10:56
*/
@Configuration
public class ESClientConfig {
private static final String ES_CONFIG_IP_KEY = "elasticsearch.ip";
private static final String ES_CONFIG_PORT_KEY = "elasticsearch.port";
private static final String ES_CONFIG_USERNAME_KEY = "elasticsearch.username";
private static final String ES_CONFIG_PASSWORD_KEY = "elasticsearch.password";
@Resource
private Environment environment;
private static RestHighLevelClient getOneESClient(String ipAddress, int port, String username, String password) {
requireNonNull(ipAddress);
RestClientBuilder restClientBuilder = RestClient.builder(new HttpHost(ipAddress, port, "http"));
BasicCredentialsProvider basicCredentialsProvider;
if (nonNull(username) && nonNull(password)) {
basicCredentialsProvider = new BasicCredentialsProvider();
UsernamePasswordCredentials usernamePasswordCredentials = new UsernamePasswordCredentials(username, password);
basicCredentialsProvider.setCredentials(AuthScope.ANY, usernamePasswordCredentials);
restClientBuilder.setHttpClientConfigCallback(httpAsyncClientBuilder ->
httpAsyncClientBuilder.setDefaultCredentialsProvider(basicCredentialsProvider));
}
return new RestHighLevelClient(restClientBuilder);
}
@Bean
public RestHighLevelClient restHighLevelClient() {
String ip = requireNonNull(environment.getProperty(ES_CONFIG_IP_KEY));
String port = requireNonNull(environment.getProperty(ES_CONFIG_PORT_KEY));
String username = requireNonNull(environment.getProperty(ES_CONFIG_USERNAME_KEY));
String password = requireNonNull(environment.getProperty(ES_CONFIG_PASSWORD_KEY));
return getOneESClient(ip, Integer.parseInt(port), username, password);
}
}
src/main/java/com/zhiwei/data/config/ScheduleConfig.java 0 → 100644
package com.zhiwei.data.config;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler;
/**
* @author chenyu
* @ClassName: AdvancedScheduleConfig
* @Description: 定时任务配置
* @date 2021/03/01 11:17
*/
@Configuration
public class ScheduleConfig {
@Bean
public ThreadPoolTaskScheduler threadPoolTaskScheduler() {
ThreadPoolTaskScheduler pool = new ThreadPoolTaskScheduler();
pool.initialize();
pool.setPoolSize(10);
return pool;
}
}
src/main/java/com/zhiwei/data/entity/Delivery.java 0 → 100644
package com.zhiwei.data.entity;
/**
* @author chenyu
* @ClassName: Delivery
* @Description: 预警传递参数的泛型规范接口
* @date 2021/3/17 15:36
*/
public interface Delivery {
}
src/main/java/com/zhiwei/data/entity/Template.java 0 → 100644
package com.zhiwei.data.entity;
import java.util.Date;
/**
* @author chenyu
* @ClassName: Template
* @Description: 模板信息
* @date 2021/3/10 17:10
*/
public class Template {
private Integer id;
private Integer templateId;
private String name;
private Boolean enable;
private String typeB;
private String typeC;
private String typeD;
private Integer divisible;
private String topDomain;
private Date templateCreateTime;
private Boolean isAutoMatchCombination;
private Integer combinationId;
private Date createTime;
private Date updateTime;
public Integer getId() {
return id;
}
public void setId(Integer id) {
this.id = id;
}
public Integer getTemplateId() {
return templateId;
}
public void setTemplateId(Integer templateId) {
this.templateId = templateId;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public Boolean getEnable() {
return enable;
}
public void setEnable(Boolean enable) {
this.enable = enable;
}
public String getTypeB() {
return typeB;
}
public void setTypeB(String typeB) {
this.typeB = typeB;
}
public String getTypeC() {
return typeC;
}
public void setTypeC(String typeC) {
this.typeC = typeC;
}
public String getTypeD() {
return typeD;
}
public void setTypeD(String typeD) {
this.typeD = typeD;
}
public Integer getDivisible() {
return divisible;
}
public void setDivisible(Integer divisible) {
this.divisible = divisible;
}
public String getTopDomain() {
return topDomain;
}
public void setTopDomain(String topDomain) {
this.topDomain = topDomain;
}
public Date getTemplateCreateTime() {
return templateCreateTime;
}
public void setTemplateCreateTime(Date templateCreateTime) {
this.templateCreateTime = templateCreateTime;
}
public Boolean getIsAutoMatchCombination() {
return isAutoMatchCombination;
}
public void setIsAutoMatchCombination(Boolean isAutoMatchCombination) {
this.isAutoMatchCombination = isAutoMatchCombination;
}
public Integer getCombinationId() {
return combinationId;
}
public void setCombinationId(Integer combinationId) {
this.combinationId = combinationId;
}
public Date getCreateTime() {
return createTime;
}
public void setCreateTime(Date createTime) {
this.createTime = createTime;
}
public Date getUpdateTime() {
return updateTime;
}
public void setUpdateTime(Date updateTime) {
this.updateTime = updateTime;
}
}
src/main/java/com/zhiwei/data/entity/TemplateWarningConfig.java 0 → 100644
package com.zhiwei.data.entity;
import java.util.Date;
/**
* @author chenyu
* @ClassName: TemplateWarningConfig
* @Description: 普通模板预警配置
* @date 2021/3/11 10:44
*/
public class TemplateWarningConfig {
private Integer id;
private Integer templateId;
private Boolean enableWarning;
private Integer period;
private Integer threshold;
private Boolean enableDelay;
private Integer delayPeriod;
private Date nextWarnTime;
private Date createTime;
private Date updateTime;
public Integer getId() {
return id;
}
public void setId(Integer id) {
this.id = id;
}
public Integer getTemplateId() {
return templateId;
}
public void setTemplateId(Integer templateId) {
this.templateId = templateId;
}
public Boolean getEnableWarning() {
return enableWarning;
}
public void setEnableWarning(Boolean enableWarning) {
this.enableWarning = enableWarning;
}
public Integer getPeriod() {
return period;
}
public void setPeriod(Integer period) {
this.period = period;
}
public Integer getThreshold() {
return threshold;
}
public void setThreshold(Integer threshold) {
this.threshold = threshold;
}
public Boolean getEnableDelay() {
return enableDelay;
}
public void setEnableDelay(Boolean enableDelay) {
this.enableDelay = enableDelay;
}
public Integer getDelayPeriod() {
return delayPeriod;
}
public void setDelayPeriod(Integer delayPeriod) {
this.delayPeriod = delayPeriod;
}
public Date getNextWarnTime() {
return nextWarnTime;
}
public void setNextWarnTime(Date nextWarnTime) {
this.nextWarnTime = nextWarnTime;
}
public Date getCreateTime() {
return createTime;
}
public void setCreateTime(Date createTime) {
this.createTime = createTime;
}
public Date getUpdateTime() {
return updateTime;
}
public void setUpdateTime(Date updateTime) {
this.updateTime = updateTime;
}
}
src/main/java/com/zhiwei/data/entity/TemplateWarningDelivery.java 0 → 100644
package com.zhiwei.data.entity;
import java.util.Date;
/**
* @author chenyu
* @ClassName: TemplateWarningDelivery
* @Description: 模板预警传递参数
* @date 2021/3/17 16:12
*/
public class TemplateWarningDelivery implements Delivery {
private Integer templateId;
private String name;
private Boolean enableWarning;
private Integer period;
private Integer count;
private Integer threshold;
private Boolean enableDelay;
private Integer delayPeriod;
private Date nextWarnTime;
private Integer warningLevel;
public Integer getTemplateId() {
return templateId;
}
public void setTemplateId(Integer templateId) {
this.templateId = templateId;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public Boolean getEnableWarning() {
return enableWarning;
}
public void setEnableWarning(Boolean enableWarning) {
this.enableWarning = enableWarning;
}
public Integer getPeriod() {
return period;
}
public void setPeriod(Integer period) {
this.period = period;
}
public Integer getCount() {
return count;
}
public void setCount(Integer count) {
this.count = count;
}
public Integer getThreshold() {
return threshold;
}
public void setThreshold(Integer threshold) {
this.threshold = threshold;
}
public Boolean getEnableDelay() {
return enableDelay;
}
public void setEnableDelay(Boolean enableDelay) {
this.enableDelay = enableDelay;
}
public Integer getDelayPeriod() {
return delayPeriod;
}
public void setDelayPeriod(Integer delayPeriod) {
this.delayPeriod = delayPeriod;
}
public Date getNextWarnTime() {
return nextWarnTime;
}
public void setNextWarnTime(Date nextWarnTime) {
this.nextWarnTime = nextWarnTime;
}
public Integer getWarningLevel() {
return warningLevel;
}
public void setWarningLevel(Integer warningLevel) {
this.warningLevel = warningLevel;
}
}
src/main/java/com/zhiwei/data/repository/elasticsearch/EsRepository.java 0 → 100644
package com.zhiwei.data.repository.elasticsearch;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.bucket.terms.ParsedLongTerms;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.metrics.ParsedValueCount;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.sort.SortBuilder;
import org.springframework.stereotype.Repository;
import org.springframework.util.Assert;
import javax.annotation.Resource;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static java.util.Objects.*;
/**
* @author chenyu
* @ClassName: EsRepository
* @Description: es 查询层
* @date 2021/3/12 11:37
*/
@Repository
public class EsRepository {
private static final Logger logger = LogManager.getLogger(EsRepository.class);
private static final int DEFAULT_SINGLE_QUERY_SIZE = 1000;
@Resource
private RestHighLevelClient client;
/**
* 单字段分组并获取结果
*
* @param indies 索引组
* @param searchSourceBuilder 搜索条件
* @param groupKey 分组 key
* @param groupResValueKey 分组结果value key
* @return Map<Object, Object>
*/
public Map<Object, Object> groupAndGetAsMap(String[] indies, SearchSourceBuilder searchSourceBuilder, String groupKey, String groupResValueKey) {
//check
Assert.isTrue(nonNull(indies) && nonNull(searchSourceBuilder) && nonNull(groupKey) && nonNull(groupResValueKey),
"params are must be not null");
Map<Object, Object> resultMap = new HashMap<>();
SearchRequest searchRequest = new SearchRequest(indies);
searchRequest.source(searchSourceBuilder);
try {
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
ParsedLongTerms idTerms = searchResponse.getAggregations().get(groupKey);
List<? extends Terms.Bucket> buckets = idTerms.getBuckets();
buckets.forEach(bucket -> {
ParsedValueCount count = bucket.getAggregations().get(groupResValueKey);
resultMap.put(bucket.getKey(), count.value());
});
} catch (Exception e) {
logger.error("查询ES时出现异常", e);
}
return resultMap;
}
/**
* 搜索es并获取所有的数据
*
* @param indies 索引组
* @param searchSourceBuilder 搜索条件
* @return List<Map<String, Object>>
*/
public List<Map<String, Object>> queryAllAsMap(String[] indies, SearchSourceBuilder searchSourceBuilder) {
//check
Assert.isTrue(nonNull(indies) && nonNull(searchSourceBuilder), "params are must be not null");
setDefaultIfNecessary(searchSourceBuilder);
int size = searchSourceBuilder.size();
List<Map<String, Object>> result = new ArrayList<>(size);
SearchAfterResponse searchAfterResponse = prepareSearchAfter(indies, searchSourceBuilder);
if (isNull(searchAfterResponse)) {
return result;
}
SearchHits hits = searchAfterResponse.searchResponse.getHits();
int length;
while (nonNull(hits) && ((length = hits.getHits().length) == size || length != 0)) {
hits.forEach(hit -> result.add(hit.getSourceAsMap()));
if (length < size) {
break;
}
searchAfterResponse = searchAfter(indies, searchSourceBuilder, searchAfterResponse.searchAfterId);
hits = nonNull(searchAfterResponse) ? searchAfterResponse.searchResponse.getHits() : null;
}
return result;
}
/**
* 默认设置
*
* @param searchSourceBuilder 搜索条件
*/
private void setDefaultIfNecessary(SearchSourceBuilder searchSourceBuilder) {
if (searchSourceBuilder.size() < 0) {
searchSourceBuilder.size(DEFAULT_SINGLE_QUERY_SIZE);
}
}
/**
* start searchAfter
*
* @param indies 索引组
* @param searchSourceBuilder 搜索条件
* @return SearchAfterResponse
*/
private SearchAfterResponse prepareSearchAfter(String[] indies, SearchSourceBuilder searchSourceBuilder) {
List<SortBuilder<?>> sorts = searchSourceBuilder.sorts();
Assert.isTrue(CollectionUtils.isNotEmpty(sorts), "searchAfter query must be sorted");
try {
SearchRequest searchRequest = new SearchRequest(indies);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
return new SearchAfterResponse(searchResponse);
} catch (IOException e) {
throw new IllegalStateException("查询ES时出现IO异常", e);
} catch (Exception e) {
logger.error("查询ES时出现异常", e);
return null;
}
}
/**
* searchAfter continue
*
* @param indies 索引组
* @param searchSourceBuilder 搜索条件
* @param searchAfterId SortValues Object[]
* @return SearchAfterResponse
*/
private SearchAfterResponse searchAfter(String[] indies, SearchSourceBuilder searchSourceBuilder, Object[] searchAfterId) {
List<SortBuilder<?>> sorts = searchSourceBuilder.sorts();
Assert.isTrue(CollectionUtils.isNotEmpty(sorts), "searchAfter query must be sorted");
try {
SearchRequest searchRequest = new SearchRequest(indies);
searchSourceBuilder.searchAfter(searchAfterId);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
return new SearchAfterResponse(searchResponse);
} catch (IOException e) {
throw new IllegalStateException("查询ES时出现IO异常", e);
} catch (Exception e) {
logger.error("查询ES时出现异常", e);
return null;
}
}
/**
* 附带searchAfterId的response
*
*/
static class SearchAfterResponse {
SearchResponse searchResponse;
Object[] searchAfterId;
SearchAfterResponse(SearchResponse response) {
this.searchResponse = response;
//get searchAfterId
SearchHits hits = response.getHits();
int length = hits.getHits().length;
if (length > 0) {
this.searchAfterId = hits.getAt( length - 1).getSortValues();
}
}
}
}
src/main/java/com/zhiwei/data/repository/mapper/StatisticsMapper.java 0 → 100644
package com.zhiwei.data.repository.mapper;
import org.apache.ibatis.annotations.Param;
import org.springframework.stereotype.Repository;
import java.util.Date;
import java.util.List;
import java.util.Map;
/**
* @author chenyu
* @ClassName: StatisticsMapper
* @Description: statistics mapper
* @date 2021/3/10 16:35
*/
@Repository
public interface StatisticsMapper {
/**
* 查询时间区间内的采集统计量
*
* @param templateId 模板id
* @param start 起始时间
* @param end 结束时间
* @param table 表名
* @return Integer
*/
Integer queryCountDuringPeriod(@Param("templateId") int templateId, @Param("start") Date start, @Param("end") Date end,
@Param("table") String table);
/**
* 插入模板采集统计量数据记录
*
* @param countMap 计数暂存容器
* @param time 时间节点
* @param table table name
* @return int
*/
int insertTemplateStatistic(@Param("countMap") Map<String, Integer> countMap, @Param("time") Date time, @Param("table") String table);
/**
* 创建模板统计表
*
* @param table table name
*/
void createTemplateStatisticsTable(@Param("table") String table);
}
src/main/java/com/zhiwei/data/repository/mapper/TemplateMapper.java 0 → 100644
package com.zhiwei.data.repository.mapper;
import com.zhiwei.data.entity.Template;
import com.zhiwei.data.entity.TemplateWarningConfig;
import com.zhiwei.data.entity.TemplateWarningDelivery;
import org.apache.ibatis.annotations.Param;
import org.springframework.stereotype.Repository;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* @author chenyu
* @ClassName: TemplateMapper
* @Description: template mapper
* @date 2021/3/10 16:35
*/
@Repository
public interface TemplateMapper {
/**
* 获取所有模板id
*
* @return Set<Integer>
*/
Set<Integer> queryAllTemplateIds();
/**
* 获取运行中的模板id
*
* @return List<Integer>
*/
List<Integer> queryRunningTemplateIds();
/**
* 获取开启预警的模板信息
*
* @return List<TemplateWarningDelivery>
*/
List<TemplateWarningDelivery> queryEnableWarningTemplate();
/**
* 批量插入模板基本属性数据
*
* @param templates 模板参数
* @return int
*/
int batchInsertTemplate(@Param("templates") List<Template> templates);
/**
* 批量插入模板配置
*
* @param configs 配置参数
* @return int
*/
int batchInsertTemplateConfig(@Param("configs") List<TemplateWarningConfig> configs);
/**
* 批量更新模板基本属性数据
*
* @param templates 模板参数
* @return int
*/
int batchUpdateTemplate(@Param("templates") List<Template> templates);
/**
* 新增或更新预警周期内采集量
*
* @param templateId 模板id
* @param count 预警周期内采集量
* @return int
*/
int upsertCount(@Param("templateId") int templateId, @Param("count") int count);
}
src/main/java/com/zhiwei/data/strategy/CombTemplateStrategy.java 0 → 100644
package com.zhiwei.data.strategy;
import com.zhiwei.data.entity.Delivery;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.stereotype.Component;
import java.util.Collection;
import java.util.Collections;
/**
* @author chenyu
* @ClassName: CombTemplateStrategy
* @Description: 组合模板预警策略
* @date 2021/3/1 14:02
*/
@Component
public class CombTemplateStrategy implements Strategy {
private static final Logger logger = LogManager.getLogger(CombTemplateStrategy.class);
@Override
public Collection<? extends Delivery> obtainWarningParam() {
return Collections.emptyList();
}
@Override
public boolean syncCount(Delivery param) {
logger.info("sync combination template data count");
return true;
}
@Override
public boolean shouldSkip(Delivery param) {
return true;
}
@Override
public boolean compute(Delivery param) {
return true;
}
@Override
public void refreshLevel(Delivery param) {
logger.info("refresh combination template level");
}
@Override
public void warningProcess(Delivery param) {
logger.info("process combination template warning");
}
@Override
public void postProcess(Delivery param) {
logger.info("post process combination template");
}
}
src/main/java/com/zhiwei/data/strategy/MediaStrategy.java 0 → 100644
package com.zhiwei.data.strategy;
import com.zhiwei.data.entity.Delivery;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.stereotype.Component;
import java.util.Collection;
import java.util.Collections;
/**
* @author chenyu
* @ClassName: MediaStrategy
* @Description: 媒体平台预警策略
* @date 2021/2/26 9:57
*/
@Component
public class MediaStrategy implements Strategy {
private static final Logger logger = LogManager.getLogger(MediaStrategy.class);
@Override
public Collection<? extends Delivery> obtainWarningParam() {
return Collections.emptyList();
}
@Override
public boolean syncCount(Delivery param) {
logger.info("sync type data count");
return true;
}
@Override
public boolean shouldSkip(Delivery param) {
return true;
}
@Override
public boolean compute(Delivery param) {
return false;
}
@Override
public void refreshLevel(Delivery param) {
logger.info("refresh type level");
}
@Override
public void warningProcess(Delivery param) {
logger.info("process type warning");
}
@Override
public void postProcess(Delivery param) {
logger.info("post process type");
}
}
src/main/java/com/zhiwei/data/strategy/Strategy.java 0 → 100644
package com.zhiwei.data.strategy;
import com.zhiwei.data.entity.Delivery;
import java.util.Collection;
/**
* @author chenyu
* @ClassName: Strategy
* @Description: 预警策略接口
* @date 2021/2/26 9:35
*/
public interface Strategy {
/**
* 获取预警处理必要参数
*
* @return Collection<Delivery>
*/
Collection<? extends Delivery> obtainWarningParam();
/**
* 同步预警周期内的数据量
*
* @param param 预警传递参数
* @return boolean
*/
boolean syncCount(Delivery param);
/**
* 是否跳过本次处理
*
* @param param 预警传递参数
* @return boolean
*/
boolean shouldSkip(Delivery param);
/**
* 预警等级刷新
*
* @param param 预警传递参数
*/
void refreshLevel(Delivery param);
/**
* 计算是否触发预警
*
* @param param 预警传递参数
* @return boolean
*/
boolean compute(Delivery param);
/**
* 触发预警处理
*
* @param param 预警传递参数
*/
void warningProcess(Delivery param);
/**
* 后置处理
*
* @param param 预警传递参数
*/
void postProcess(Delivery param);
}
src/main/java/com/zhiwei/data/strategy/TemplateStrategy.java 0 → 100644
package com.zhiwei.data.strategy;
import com.zhiwei.data.entity.Delivery;
import com.zhiwei.data.entity.TemplateWarningDelivery;
import com.zhiwei.data.repository.mapper.StatisticsMapper;
import com.zhiwei.data.repository.mapper.TemplateMapper;
import com.zhiwei.data.util.DaoUtils;
import com.zhiwei.data.util.TimeUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import java.text.ParseException;
import java.time.Duration;
import java.util.Collection;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import static java.util.Objects.isNull;
import static java.util.Objects.nonNull;
/**
* @author chenyu
* @ClassName: TemplateStrategy
* @Description: 单模板预警策略
* @date 2021/3/1 14:01
*/
@Component
public class TemplateStrategy implements Strategy {
private static final Logger logger = LogManager.getLogger(TemplateStrategy.class);
//缓存month info 与当月起始时间的关系
private final Map<String, Date> dateCacheMap = new ConcurrentHashMap<>();
@Resource
private TemplateMapper templateMapper;
@Resource
private StatisticsMapper statisticsMapper;
@Override
public Collection<TemplateWarningDelivery> obtainWarningParam() {
return templateMapper.queryEnableWarningTemplate();
}
@Override
public boolean syncCount(Delivery param) {
TemplateWarningDelivery twd = (TemplateWarningDelivery) param;
Date end = new Date(TimeUtils.getLastHourEndTimestamp());
Integer templateId = twd.getTemplateId();
Integer period = twd.getPeriod();
Date start = TimeUtils.getDateAfterChanged(end, Duration.ofHours(-period));
int count = queryCountDuringPeriod(templateId, start, end);
templateMapper.upsertCount(templateId, count);
twd.setCount(count);
return true;
}
@Override
public boolean shouldSkip(Delivery param) {
TemplateWarningDelivery twd = (TemplateWarningDelivery) param;
return twd.getNextWarnTime().getTime() > System.currentTimeMillis();
}
@Override
public void refreshLevel(Delivery param) {
TemplateWarningDelivery twd = (TemplateWarningDelivery) param;
Integer warningLevel = twd.getWarningLevel();
Integer count = twd.getCount();
Integer threshold = twd.getThreshold();
if (count < threshold) {
twd.setWarningLevel(warningLevel + 1);
} else {
twd.setWarningLevel(warningLevel - 1);
}
}
@Override
public boolean compute(Delivery param) {
TemplateWarningDelivery twd = (TemplateWarningDelivery) param;
return false;
}
@Override
public void warningProcess(Delivery param) {
TemplateWarningDelivery twd = (TemplateWarningDelivery) param;
}
@Override
public void postProcess(Delivery param) {
logger.info("post process template");
}
/**
* 查询单模板预警周期内采集量
*
* @param templateId 模板id
* @param start 起始时间
* @param end 结束时间
* @return int
*/
private int queryCountDuringPeriod(int templateId, Date start, Date end) {
List<String> tables = DaoUtils.getSortedTemplateTablesByTime(start, end);
int res = 0;
if (tables.size() == 1) {
Integer count = statisticsMapper.queryCountDuringPeriod(templateId, start, end, tables.get(0));
return nonNull(count) ? count : 0;
} else {
String firstTable = tables.get(0);
Integer fCount = statisticsMapper.queryCountDuringPeriod(templateId,
start, new Date(TimeUtils.getCurrentMonthEndTimestamp(start.getTime())), firstTable);
res += nonNull(fCount) ? fCount : 0;
tables.remove(0);
String lastTables = tables.get(tables.size() - 1);
Integer lCount = statisticsMapper.queryCountDuringPeriod(templateId,
new Date(TimeUtils.getCurrentMonthStartTimestamp(end.getTime())), end, lastTables);
res += nonNull(lCount) ? lCount : 0;
tables.remove(tables.size() - 1);
if (!tables.isEmpty()) {
for (String table : tables) {
String[] split = table.split("_");
String monthInfo = split[3];
Date targetMonthStart = dateCacheMap.computeIfAbsent(monthInfo, key -> {
try {
return TimeUtils.MONTH_FORMAT.parse(monthInfo);
} catch (ParseException e) {
logger.error("month parse 异常", e);
return null;
}
});
if (isNull(targetMonthStart)) {
continue;
}
Integer count = statisticsMapper.queryCountDuringPeriod(templateId, targetMonthStart,
new Date(TimeUtils.getCurrentMonthEndTimestamp(targetMonthStart.getTime())), table);
res += nonNull(count) ? count : 0;
}
}
}
return res;
}
}
src/main/java/com/zhiwei/data/sync/TemplateBaseSync.java 0 → 100644
package com.zhiwei.data.sync;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.crawler.core.utils.RequestUtils;
import com.zhiwei.data.cache.CacheLoader;
import com.zhiwei.data.entity.Template;
import com.zhiwei.data.entity.TemplateWarningConfig;
import com.zhiwei.data.repository.mapper.TemplateMapper;
import com.zhiwei.data.util.BaseUtils;
import com.zhiwei.data.util.TimeUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import static com.zhiwei.data.common.CommonConstants.*;
import static java.util.Objects.nonNull;
import static java.util.Objects.requireNonNull;
/**
* @author chenyu
* @ClassName: TemplateBaseSync
* @Description: 模板基础数据同步初始化程序
* @date 2021/3/10 16:43
*/
@Component
public class TemplateBaseSync {
private static final Logger logger = LogManager.getLogger(TemplateBaseSync.class);
//模板同步数据获取接口
private static final String DATA_SOURCE_URL = "https://crawler.zhiweidata.com/crawlerPannel/template/synchronizationData";
//允许拆分子list的最大size
private static final int PARTITION_LIST_MAX_SIZE = 1000;
private final CacheLoader<Set<Integer>> templateIdCache;
private final TemplateMapper templateMapper;
@Autowired
public TemplateBaseSync(TemplateMapper templateMapper) {
this.templateMapper = templateMapper;
this.templateIdCache = new CacheLoader<Set<Integer>>("templateId-cache", Duration.ofMinutes(10)) {
@Override
protected Set<Integer> load() {
return templateMapper.queryAllTemplateIds();
}
};
}
@Scheduled(fixedDelay = 10 * 60 * 1000)
public void startSync() {
try {
doSync();
} catch (Exception e) {
logger.error("模板基础数据同步初始化异常", e);
}
}
/**
* 同步逻辑
*
*/
private void doSync() {
String bodyStr = BaseUtils.httpRequest(RequestUtils.wrapGet(DATA_SOURCE_URL), logger);
if (StringUtils.isEmpty(bodyStr)) {
return;
}
JSONObject msgBody = JSON.parseObject(bodyStr);
boolean status = msgBody.getBooleanValue(STATUS);
if (status) {
JSONArray dataArr = msgBody.getJSONArray(DATA);
Set<Integer> idSet = templateIdCache.get();
List<Template> insertionList = new ArrayList<>();
List<Template> updateList = new ArrayList<>();
dataArr.forEach(data -> {
JSONObject json = (JSONObject) data;
Template template = buildTemplateFromJson(json);
if (idSet.contains(template.getTemplateId())) {
updateList.add(template);
} else {
insertionList.add(template);
}
});
if (!insertionList.isEmpty()) {
templateMapper.batchInsertTemplate(insertionList);
//初始化预警配置
initTemplateWarningConfig(insertionList);
}
if (!updateList.isEmpty()) {
BaseUtils.partitionProcess(updateList, PARTITION_LIST_MAX_SIZE, templateMapper::batchUpdateTemplate);
}
} else {
logger.info("模板基础数据获取接口状态异常 status : false messages : {}",
msgBody.getString(MESSAGES));
}
}
/**
* 从json中获取数据组装入库参数
*
* @param json json data
* @return Template
*/
private Template buildTemplateFromJson(JSONObject json) {
requireNonNull(json);
Template template = new Template();
template.setTemplateId(json.getInteger(ID));
template.setName(json.getString(NAME));
template.setEnable(json.getBoolean(ENABLE));
template.setTypeB(json.getString(CLASS_B));
template.setTypeC(json.getString(CLASS_C));
template.setTypeD(json.getString(CLASS_D));
template.setDivisible(json.getInteger(DIVISIBLE));
String topDomain = json.getString(TOP_DOMAIN);
template.setTopDomain(nonNull(topDomain) ? topDomain : StringUtils.EMPTY);
template.setTemplateCreateTime(json.getDate(CREATE_TIME));
return template;
}
/**
* 默认值赋予
*
* @param template Template object
*/
private void populateDefaultValues(Template template) {
requireNonNull(template);
Date now = new Date();
template.setIsAutoMatchCombination(false);
template.setCombinationId(0);
template.setCreateTime(now);
template.setUpdateTime(now);
}
/**
* 初始化模板预警配置
*
* @param templates 新入库的模板
*/
private void initTemplateWarningConfig(List<Template> templates) {
requireNonNull(templates);
//开启预警的模板默认在7天后进行第一次预警判断
Date nextWarnTime = TimeUtils.getDateAfterChanged(new Date(), Duration.ofDays(7));
List<TemplateWarningConfig> configs = templates.stream().map(template -> {
TemplateWarningConfig config = generateDefaultConfig(false);
config.setTemplateId(template.getTemplateId());
boolean enable = template.getEnable();
//开启状态的模板默认开启普通预警并设置下一次预警时间
if (enable) {
config.setEnableWarning(true);
config.setNextWarnTime(nextWarnTime);
}
return config;
}).collect(Collectors.toList());
if (!configs.isEmpty()) {
templateMapper.batchInsertTemplateConfig(configs);
}
}
/**
* 生成默认配置
*
* @param timeInitialized 是否初始化当前时间
* @return TemplateWarningConfig
*/
private TemplateWarningConfig generateDefaultConfig(boolean timeInitialized) {
int defaultPeriod = 24;
int defaultThreshold = 50;
int defaultDelayPeriod = 24;
TemplateWarningConfig config = new TemplateWarningConfig();
config.setEnableWarning(false);
config.setPeriod(defaultPeriod);
config.setThreshold(defaultThreshold);
config.setEnableDelay(false);
config.setDelayPeriod(defaultDelayPeriod);
config.setNextWarnTime(TimeUtils.ZERO_DATE);
if (timeInitialized) {
Date now = new Date();
config.setCreateTime(now);
config.setUpdateTime(now);
}
return config;
}
}
src/main/java/com/zhiwei/data/sync/TemplateDataCountSync.java 0 → 100644
package com.zhiwei.data.sync;
import com.zhiwei.data.repository.elasticsearch.EsRepository;
import com.zhiwei.data.repository.mapper.StatisticsMapper;
import com.zhiwei.data.repository.mapper.TemplateMapper;
import com.zhiwei.data.util.BaseUtils;
import com.zhiwei.data.util.DaoUtils;
import com.zhiwei.data.util.TimeUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.ValueCountAggregationBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.springframework.scheduling.annotation.Async;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static com.zhiwei.data.common.CommonConstants.COUNT;
import static com.zhiwei.data.common.CommonConstants.TEMPLATE_ID;
/**
* @author chenyu
* @ClassName: TemplateDataCountSync
* @Description: 模板采集数据量同步程序
* @date 2021/3/11 16:05
*/
@Component
public class TemplateDataCountSync {
private static final Logger logger = LogManager.getLogger(TemplateDataCountSync.class);
//允许拆分子list的最大size
private static final int PARTITION_LIST_MAX_SIZE = 1000;
//es 查询字段
private static final String CID = "cid";
private static final String S_TIME = "stime";
@Resource
private EsRepository esRepository;
@Resource
private TemplateMapper templateMapper;
@Resource
private StatisticsMapper statisticsMapper;
@Scheduled(cron = "0 0 12 25 * ?")
@Async
public void createTable() {
String tableName = null;
try {
tableName = generateNextMonthTableName(new Date());
statisticsMapper.createTemplateStatisticsTable(tableName);
} catch (Exception e) {
logger.error("创建模板统计表异常 name : {}", tableName, e);
}
}
/**
* 生成新表名
*
* @param date 目标日期
* @return String
*/
private static String generateNextMonthTableName(Date date) {
Date nextMonthDay = TimeUtils.getNextMonthDay(date);
return DaoUtils.getTemplateTableByTime(nextMonthDay);
}
@Scheduled(cron = "0 30 */1 * * ?")
public void startSync() {
try {
doSync();
} catch (Exception e) {
logger.error("模板采集数据量同步异常", e);
}
}
/**
* 同步逻辑
*
*/
private void doSync() {
long start = TimeUtils.getLastHourStartTimestamp();
long end = TimeUtils.getLastHourEndTimestamp();
String[] indies = DaoUtils.getIndexNamesByTime(start, end);
List<Integer> templateIds = templateMapper.queryRunningTemplateIds();
Map<String, Integer> countMap = new HashMap<>();
BaseUtils.partitionProcess(templateIds, PARTITION_LIST_MAX_SIZE, ids -> {
SearchSourceBuilder searchSourceBuilder = buildSearchSourceBuilder(ids, start, end);
Map<Object, Object> groupRes = esRepository.groupAndGetAsMap(indies, searchSourceBuilder, TEMPLATE_ID, COUNT);
groupRes.forEach((k, v) -> countMap.put(String.valueOf(k), ((Number) v).intValue()));
});
if (countMap.isEmpty()) {
return;
}
//以起始时间为时间点
Date datetime = new Date(start);
String table = DaoUtils.getTemplateTableByTime(datetime);
statisticsMapper.insertTemplateStatistic(countMap, datetime, table);
}
/**
* 创建查询构造器
*
* @param templateIds 模板id列表
* @param start 起始时间
* @param end 结束时间
* @return SearchSourceBuilder
*/
private SearchSourceBuilder buildSearchSourceBuilder(List<Integer> templateIds, long start, long end) {
BoolQueryBuilder parentQuery = new BoolQueryBuilder();
BoolQueryBuilder childQuery = new BoolQueryBuilder();
templateIds.forEach(templateId -> childQuery.should(QueryBuilders.termQuery(CID, templateId)));
parentQuery.must(childQuery);
parentQuery.must(QueryBuilders.rangeQuery(S_TIME).gte(start).lte(end));
TermsAggregationBuilder terms = AggregationBuilders.terms(TEMPLATE_ID).field(CID);
ValueCountAggregationBuilder valueCount = AggregationBuilders.count(COUNT).field(CID);
terms.subAggregation(valueCount);
terms.size(templateIds.size());
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(parentQuery);
searchSourceBuilder.aggregation(terms);
return searchSourceBuilder;
}
}
src/main/java/com/zhiwei/data/util/BaseUtils.java 0 → 100644
package com.zhiwei.data.util;
import com.zhiwei.crawler.core.HttpBoot;
import okhttp3.Request;
import okhttp3.Response;
import org.apache.commons.collections4.ListUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.IOException;
import java.util.List;
import java.util.Properties;
import java.util.function.Consumer;
import java.util.function.Supplier;
import static java.util.Objects.nonNull;
/**
* @author chenyu
* @ClassName: BaseUtils
* @Description: 基本工具类
* @date 2021/3/2 16:01
*/
public class BaseUtils {
private static final Logger logger = LogManager.getLogger(BaseUtils.class);
private static final HttpBoot HTTP_BOOT = new HttpBoot.Builder().retryTimes(3).build();
private static final String CURRENT_CONFIG_TAG;
public static final boolean IS_PRODUCT;
private BaseUtils() {}
static {
//default
String currentConfigTag = "dev";
boolean isProduct = false;
Properties properties = new Properties();
ClassLoader loader = Thread.currentThread().getContextClassLoader();
try {
properties.load(loader.getResourceAsStream("application.properties"));
String choose = properties.getProperty("spring.profiles.active", "dev");
currentConfigTag = choose;
isProduct = choose.equals("pro");
} catch (Exception e) {
logger.error("读取配置文件失败", e);
}
CURRENT_CONFIG_TAG = currentConfigTag;
IS_PRODUCT = isProduct;
}
/**
* 静态获取properties文件属性
*
* @param key 属性key
* @param defaultValue 设置默认值
* @return String
*/
public static String getProperties(String key, String defaultValue) {
Properties properties = new Properties();
try {
ClassLoader loader = Thread.currentThread().getContextClassLoader();
switch (CURRENT_CONFIG_TAG) {
case "dev":
properties.load(loader.getResourceAsStream("application-dev.properties"));
break;
case "pro":
properties.load(loader.getResourceAsStream("application-pro.properties"));
break;
}
return properties.getProperty(key, defaultValue);
} catch (Exception e) {
logger.error("读取配置文件失败：{}", key, e);
return null;
}
}
/**
* 耗时日志打印处理
*
* @param supplier supplier
* @param logger 日志
* @param description 简述
* @param <T> 返回泛型
* @return T
*/
public static <T> T processAroundTimeLog(Supplier<T> supplier, Logger logger, String description) {
long start = System.currentTimeMillis();
T result = supplier.get();
logger.info(StringUtils.join(description, " 总耗时：", System.currentTimeMillis() - start, " ms"));
return result;
}
/**
* 耗时日志打印处理
*
* @param consumer consumer
* @param <T> param
* @param logger 日志
* @param description 简述
*/
public static <T> void processAroundTimeLog(Consumer<T> consumer, T param, Logger logger, String description) {
long start = System.currentTimeMillis();
consumer.accept(param);
logger.info(StringUtils.join(description, " 总耗时：", System.currentTimeMillis() - start, " ms"));
}
/**
* 耗时日志打印处理
*
* @param insider insider
* @param logger 日志
* @param description 简述
*/
public static void processAroundTimeLog(Insider insider, Logger logger, String description) {
long start = System.currentTimeMillis();
insider.deal();
logger.info(StringUtils.join(description, " 总耗时：", System.currentTimeMillis() - start, " ms"));
}
/**
* 无返回值无参数的方法执行函数接口
*
*/
@FunctionalInterface
public interface Insider {
void deal();
}
/**
* HTTP请求发送简单封装
*
* @param request http request
* @param logger 日志对象
* @return String 响应体字符信息
*/
public static String httpRequest(Request request, Logger logger) {
try (Response response = HTTP_BOOT.syncCall(request)) {
if (response.isSuccessful() && nonNull(response.body())) {
return response.body().string();
}
} catch (IOException e) {
logger.error("http请求发送异常, url:{}", request.url(), e);
}
logger.info("http请求响应失败或响应结果为空");
return null;
}
/**
* 分区处理（list）
*
* @param list 待处理数据集合
* @param pSize 分区大小
* @param consumer 消费者
* @param <T> 数据类型
*/
public static <T> void partitionProcess(List<T> list, int pSize, Consumer<List<T>> consumer) {
if (list.size() > pSize) {
List<List<T>> partitions = ListUtils.partition(list, pSize);
partitions.forEach(consumer);
} else {
consumer.accept(list);
}
}
}
src/main/java/com/zhiwei/data/util/DaoUtils.java 0 → 100644
package com.zhiwei.data.util;
import com.zhiwei.data.common.SpecialConstants;
import org.springframework.util.Assert;
import java.util.*;
/**
* @author chenyu
* @ClassName: DaoUtils
* @Description: 数据层相关的工具方法
* @date 2021/3/16 17:18
*/
public class DaoUtils {
private static final String STATISTICS_TEMPLATE_TABLE_PREFIX = "manage_template_statistics_";
private DaoUtils() {}
/**
* 获取有序的模板日期表名
*
* @param start 目标起始日期
* @param end 目标结束日期
* @return List<String>
*/
public static List<String> getSortedTemplateTablesByTime(Date start, Date end) {
String startSuffix = TimeUtils.MONTH_FORMAT.format(start);
if (startSuffix.equals(TimeUtils.MONTH_FORMAT.format(end))) {
return Collections.singletonList(STATISTICS_TEMPLATE_TABLE_PREFIX + startSuffix);
}
long endMonthEndTime = TimeUtils.getCurrentMonthEndTimestamp(end.getTime());
Set<String> result = new LinkedHashSet<>();
Calendar calendar = Calendar.getInstance();
calendar.setTime(start);
long time = calendar.getTimeInMillis();
while ((time >= start.getTime()) && (time <= endMonthEndTime)) {
result.add(getMysqlTableByTime(STATISTICS_TEMPLATE_TABLE_PREFIX, time));
calendar.add(Calendar.MONTH, 1);
time = calendar.getTimeInMillis();
}
return new ArrayList<>(result);
}
/**
* 获取模板日期表名
*
* @param date 目标日期
* @return String
*/
public static String getTemplateTableByTime(Date date) {
return getMysqlTableByTime(STATISTICS_TEMPLATE_TABLE_PREFIX, date.getTime());
}
/**
* 获取日期表名
*
* @param prefix 表前缀
* @param time 目标日期
* @return String
*/
private static String getMysqlTableByTime(String prefix, long time) {
String suffix = TimeUtils.MONTH_FORMAT.format(time);
return prefix + suffix;
}
/**
* 根据起始时间获取查询的索引表名
*
* @param startTime 起始时间
* @param endTime 终止时间
* @return String
*/
public static String[] getIndexNamesByTime(long startTime, long endTime) {
String indexNameSuffix = TimeUtils.MONTH_FORMAT.format(startTime);
Assert.state(indexNameSuffix.equals(TimeUtils.MONTH_FORMAT.format(endTime)), "时间区段选择不合法");
return SpecialConstants.INDEX_NAME_FORMAT_LIST.stream()
.map(prefix -> String.format(prefix, indexNameSuffix))
.toArray(String[]::new);
}
}
src/main/java/com/zhiwei/data/util/TimeUtils.java 0 → 100644
package com.zhiwei.data.util;
import org.apache.commons.lang3.time.FastDateFormat;
import org.springframework.util.Assert;
import java.time.Duration;
import java.util.Calendar;
import java.util.Date;
import static java.util.Objects.requireNonNull;
/**
* @author chenyu
* @ClassName: TimeUtils
* @Description: 时间工具
* @date 2021/3/11 14:29
*/
public class TimeUtils {
//zero time
public static final Date ZERO_DATE = new Date(0);
//一小时的时间量
public static final long MILLISECONDS_IN_HOUR = 3600_000L;
//一天的时间量
public static final long MILLISECONDS_IN_DAY = 86400_000L;
//七天的时间量
public static final long MILLISECONDS_IN_SEVEN_DAY = 86400_000L * 7;
//月份格式器
public static final FastDateFormat MONTH_FORMAT = FastDateFormat.getInstance("yyyyMM");
private TimeUtils() {}
/**
* 计算目标日期偏移指定时间之后的日期
*
* @param targetDate 目标日期
* @param offset 偏移时间量
* @return Date
*/
public static Date getDateAfterChanged(Date targetDate, Duration offset) {
Calendar calendar = Calendar.getInstance();
calendar.setTime(targetDate);
calenderAddWithDuration(calendar, offset);
return calendar.getTime();
}
/**
* 计算目标时间戳偏移指定时间之后的时间戳
*
* @param targetTime 目标时间
* @param offset 偏移时间量
* @return Date
*/
public static long getDateAfterChanged(long targetTime, Duration offset) {
Calendar calendar = Calendar.getInstance();
calendar.setTimeInMillis(targetTime);
calenderAddWithDuration(calendar, offset);
return calendar.getTimeInMillis();
}
/**
* 时间的位移操作
*
* @param calendar 计算器对象
* @param duration 偏移时间量
*/
private static void calenderAddWithDuration(Calendar calendar, Duration duration) {
requireNonNull(calendar);
requireNonNull(duration);
long hours = duration.toHours();
Assert.isTrue(hours < Integer.MAX_VALUE, "时间位移量超标");
calendar.add(Calendar.HOUR, (int) hours);
}
/**
* 获取下个月的当天时间
*
* @param targetDate 目标日期
* @return Date
*/
public static Date getNextMonthDay(Date targetDate) {
requireNonNull(targetDate);
Calendar calendar = Calendar.getInstance();
calendar.setTime(targetDate);
calendar.add(Calendar.MONTH, 1);
return calendar.getTime();
}
/**
* 获取上小时开始的时间戳
*
* @return long
*/
public static long getLastHourStartTimestamp() {
Calendar calendar = Calendar.getInstance();
calendar.setTimeInMillis(System.currentTimeMillis());
calendar.add(Calendar.HOUR, -1);
calendar.set(Calendar.MINUTE, 0);
calendar.set(Calendar.SECOND, 0);
calendar.set(Calendar.MILLISECOND, 0);
return calendar.getTimeInMillis();
}
/**
* 获取上小时结束的时间戳
*
* @return long
*/
public static long getLastHourEndTimestamp() {
return getLastHourStartTimestamp() + MILLISECONDS_IN_HOUR - 1;
}
/**
* 获取当月初日00:00:00:000的时间戳
*
* @param timestamp 指定时间点
* @return long
*/
public static long getCurrentMonthStartTimestamp(long timestamp) {
Calendar calendar = Calendar.getInstance();
calendar.setTimeInMillis(timestamp);
calendar.set(Calendar.DAY_OF_MONTH, calendar.getActualMinimum(Calendar.DAY_OF_MONTH));
calendar.set(Calendar.HOUR_OF_DAY, 0);
calendar.set(Calendar.MINUTE, 0);
calendar.set(Calendar.SECOND, 0);
calendar.set(Calendar.MILLISECOND, 0);
return calendar.getTimeInMillis();
}
/**
* 获取当月末日23:59:59:99的时间戳
*
* @param timestamp 指定时间点
* @return long
*/
public static long getCurrentMonthEndTimestamp(long timestamp) {
Calendar calendar = Calendar.getInstance();
calendar.setTimeInMillis(timestamp);
calendar.set(Calendar.DAY_OF_MONTH, calendar.getActualMaximum(Calendar.DAY_OF_MONTH));
calendar.set(Calendar.HOUR_OF_DAY, 23);
calendar.set(Calendar.MINUTE, 59);
calendar.set(Calendar.SECOND, 59);
calendar.set(Calendar.MILLISECOND, 999);
return calendar.getTimeInMillis();
}
}
src/main/java/com/zhiwei/data/warning/WarningStarter.java 0 → 100644
package com.zhiwei.data.warning;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.zhiwei.data.entity.Delivery;
import com.zhiwei.data.strategy.CombTemplateStrategy;
import com.zhiwei.data.strategy.Strategy;
import com.zhiwei.data.strategy.TemplateStrategy;
import com.zhiwei.data.strategy.MediaStrategy;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Async;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
/**
* @author chenyu
* @ClassName: WarningStarter
* @Description: 预警启动类
* @date 2021/3/1 11:48
*/
@Component
public class WarningStarter {
private static final Logger logger = LogManager.getLogger(WarningStarter.class);
private final List<Strategy> strategies = new ArrayList<>(5);
private final ExecutorService warningExecutor = Executors.newFixedThreadPool(3,
new ThreadFactoryBuilder().setNameFormat("strategy-warn-[%d]").build());
@Autowired
public WarningStarter(TemplateStrategy teStrategy,
CombTemplateStrategy coStrategy,
MediaStrategy meStrategy) {
strategies.add(teStrategy);
strategies.add(coStrategy);
strategies.add(meStrategy);
}
/**
* 启动定时预警机制
*
*/
@Scheduled(cron = "0 10 */1 * * ?")
@Async
public void warningStart() {
strategies.forEach(strategy -> warningExecutor.execute(() -> {
try {
Collection<? extends Delivery> warningConfigs = strategy.obtainWarningParam();
warningConfigs.forEach(config -> {
if (!strategy.syncCount(config)) {
return;
}
if (strategy.shouldSkip(config)) {
return;
}
//刷新预警等级
strategy.refreshLevel(config);
//预警处理决策
if (strategy.compute(config)) {
strategy.warningProcess(config);
}
//后续处理
strategy.postProcess(config);
});
} catch (Exception e) {
logger.error("定时预警策略任务执行异常 strategy:{}", strategy.getClass(), e);
}
}));
}
}
src/main/resources/application-dev.properties 0 → 100644
#mysql
spring.datasource.druid.url=jdbc:mysql://202.107.192.94:3308/channeltag?useUnicode=true&characterEncoding=utf-8&serverTimezone=Asia/Shanghai&zeroDateTimeBehavior=CONVERT_TO_NULL&useSSL=false
spring.datasource.druid.username=channeltag
spring.datasource.druid.password=234A34FFR3
spring.datasource.druid.driver-class-name=com.mysql.cj.jdbc.Driver
spring.datasource.druid.initial-size=6
spring.datasource.druid.max-active=15
spring.datasource.druid.min-idle=6
spring.datasource.druid.max-wait=60000
spring.datasource.druid.time-between-eviction-runs-millis=60000
spring.datasource.druid.min-evictable-idle-time-millis=300000
spring.datasource.druid.validation-query=select 'x' from dual
spring.datasource.druid.test-while-idle=true
spring.datasource.druid.test-on-borrow=false
spring.datasource.druid.test-on-return=false
spring.datasource.druid.pool-prepared-statements=true
spring.datasource.druid.max-pool-prepared-statement-per-connection-size=20
spring.datasource.druid.filters=stat,wall,log4j
# mybatis
mybatis.mapper-locations=classpath:mapper/*.xml
mybatis.configuration.map-underscore-to-camel-case=true
#redis
spring.redis.host=localhost
spring.redis.port=6379
spring.redis.database=3
#elasticsearch
elasticsearch.ip = 202.107.192.94
elasticsearch.port = 1443
elasticsearch.username = crawler-platform-pannel
elasticsearch.password = www.zhiweidata.com
\ No newline at end of file
src/main/resources/application-pro.properties 0 → 100644
#mysql
spring.datasource.url=jdbc:mysql://192.168.0.37:3306/data_manage?useUnicode=true&characterEncoding=utf-8&serverTimezone=GMT%2B8&zeroDateTimeBehavior=CONVERT_TO_NULL&useSSL=false
spring.datasource.username=data-manage
spring.datasource.password=9kdsfsaf&87as-
spring.datasource.druid.driver-class-name=com.mysql.cj.jdbc.Driver
spring.datasource.druid.initial-size=6
spring.datasource.druid.max-active=15
spring.datasource.druid.min-idle=6
spring.datasource.druid.max-wait=60000
spring.datasource.druid.time-between-eviction-runs-millis=60000
spring.datasource.druid.min-evictable-idle-time-millis=300000
spring.datasource.druid.validation-query=select 'x' from dual
spring.datasource.druid.test-while-idle=true
spring.datasource.druid.test-on-borrow=false
spring.datasource.druid.test-on-return=false
spring.datasource.druid.pool-prepared-statements=true
spring.datasource.druid.max-pool-prepared-statement-per-connection-size=20
spring.datasource.druid.filters=stat,wall,log4j
# mybatis
mybatis.mapper-locations=classpath:mapper/*.xml
mybatis.configuration.map-underscore-to-camel-case=true
#redis
spring.redis.host=192.168.0.39
spring.redis.port=7375
spring.redis.database=9
#elasticsearch
elasticsearch.ip = 192.168.0.130
elasticsearch.port = 9200
elasticsearch.username = crawler-platform-pannel
elasticsearch.password = www.zhiweidata.com
\ No newline at end of file
src/main/resources/application.properties 0 → 100644
spring.profiles.active=dev
\ No newline at end of file
src/main/resources/log4j2.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<!-- log4j2 自身的日志级别 -->
<Configuration status="WARN">
<Appenders>
<!-- 定义日志输出地 -->
<Console name="Console" target="SYSTEM_OUT">
<!-- <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %-5level
%c:%L - %msg%n" /> -->
<PatternLayout charset="UTF-8"
pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level %logger{36} %L [%t] - %msg%n" />
</Console>
<RollingFile name="RollingFile" fileName="info.log"
filePattern="logs/warning.%d{yyyy-MM-dd}.log.gz">
<PatternLayout charset="UTF-8"
pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level %logger{36} %L [%t] - %msg%n" />
<Policies>
<TimeBasedTriggeringPolicy interval="1"
modulate="true" />
</Policies>
</RollingFile>
</Appenders>
<Loggers>
<AsyncRoot level="info">
<AppenderRef ref="Console" />
<AppenderRef ref="RollingFile" />
</AsyncRoot>
</Loggers>
</Configuration>
\ No newline at end of file
src/main/resources/mapper/StatisticsMapper.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
<mapper namespace="com.zhiwei.data.repository.mapper.StatisticsMapper" >
<select id="queryCountDuringPeriod" resultType="java.lang.Integer">
select sum(`count`) `count` from ${table} where template_id = #{templateId} and `time` &gt;= #{start} and `time` &lt;= #{end}
</select>
<insert id="insertTemplateStatistic">
insert ignore into ${table} (template_id, `time`, `count`, create_time, update_time) values
<foreach collection="countMap.entrySet()" item="value" index="key" separator=",">
(#{key}, #{time}, #{value}, CURTIME(), CURTIME())
</foreach>
</insert>
<update id="createTemplateStatisticsTable">
CREATE TABLE ${table} (
`id` int(10) NOT NULL AUTO_INCREMENT COMMENT '主键id',
`template_id` int(10) NOT NULL COMMENT '模板id',
`time` datetime(0) NOT NULL COMMENT '时间节点',
`count` int(10) NOT NULL COMMENT '采集数据量',
`create_time` datetime(0) NOT NULL DEFAULT CURRENT_TIMESTAMP(0) COMMENT '记录创建时间',
`update_time` datetime(0) NOT NULL DEFAULT CURRENT_TIMESTAMP(0) COMMENT '记录更新时间',
PRIMARY KEY (`id`) USING BTREE,
UNIQUE INDEX `template_id_time_unique`(`template_id`, `time`) USING BTREE COMMENT '唯一约束'
) ENGINE = InnoDB AUTO_INCREMENT = 1 CHARACTER SET = utf8
COLLATE = utf8_general_ci ROW_FORMAT = Dynamic;
</update>
</mapper>
\ No newline at end of file
src/main/resources/mapper/TemplateMapper.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
<mapper namespace="com.zhiwei.data.repository.mapper.TemplateMapper" >
<select id="queryAllTemplateIds" resultType="java.lang.Integer">
select distinct template_id from manage_template
</select>
<select id="queryRunningTemplateIds" resultType="java.lang.Integer">
select distinct template_id from manage_template where enable = 1
</select>
<select id="queryEnableWarningTemplate" resultType="com.zhiwei.data.entity.TemplateWarningDelivery">
select a.template_id, a.name, b.enable_warning, b.period, b.threshold, b.enable_delay, b.delay_period, b.next_warn_time, c.warning_level
from manage_template a inner join manage_template_warning_config b on a.template_id = b.template_id
left join manage_template_warning c on a.template_id = c.template_id
where a.enable = 1 and b.enable_warning = 1
</select>
<insert id="batchInsertTemplate">
insert ignore into manage_template (template_id, `name`, enable, typeB, typeC, typeD,
divisible, top_domain, template_create_time, is_auto_match_combination, combination_id, create_time, update_time) values
<foreach collection="templates" item="template" separator=','>
(#{template.templateId}, #{template.name}, #{template.enable}, #{template.typeB}, #{template.typeC}, #{template.typeD},
#{template.divisible}, #{template.topDomain}, #{template.templateCreateTime},
#{template.isAutoMatchCombination}, #{template.combinationId}, CURTIME(), CURTIME())
</foreach>
</insert>
<insert id="batchInsertTemplateConfig">
insert ignore into manage_template_warning_config (template_id, enable_warning, period, threshold,
enable_delay, delay_period, next_warn_time, create_time, update_time) values
<foreach collection="configs" item="config" separator=','>
(#{config.templateId}, #{config.enableWarning}, #{config.period}, #{config.threshold},
#{config.enableDelay}, #{config.delayPeriod}, #{config.nextWarnTime}, CURTIME(), CURTIME())
</foreach>
</insert>
<update id="batchUpdateTemplate">
update manage_template
<trim prefix="set" suffixOverrides=",">
<trim prefix="name = case" suffix="end,">
<foreach collection="templates" item="template" index="index">
<if test="template.name != null">
when template_id = #{template.templateId}
then #{template.templateId}
</if>
</foreach>
</trim>
<trim prefix="enable = case" suffix="end,">
<foreach collection="templates" item="template" index="index">
<if test="template.enable != null">
when template_id = #{template.templateId}
then #{template.enable}
</if>
</foreach>
</trim>
<trim prefix="typeB = case" suffix="end,">
<foreach collection="templates" item="template" index="index">
<if test="template.typeB != null">
when template_id = #{template.templateId}
then #{template.typeB}
</if>
</foreach>
</trim>
<trim prefix="typeC = case" suffix="end,">
<foreach collection="templates" item="template" index="index">
<if test="template.typeC != null">
when template_id = #{template.templateId}
then #{template.typeC}
</if>
</foreach>
</trim>
<trim prefix="typeD = case" suffix="end,">
<foreach collection="templates" item="template" index="index">
<if test="template.typeD != null">
when template_id = #{template.templateId}
then #{template.typeD}
</if>
</foreach>
</trim>
<trim prefix="divisible = case" suffix="end,">
<foreach collection="templates" item="template" index="index">
<if test="template.divisible != null">
when template_id = #{template.templateId}
then #{template.divisible}
</if>
</foreach>
</trim>
<trim prefix="top_domain = case" suffix="end,">
<foreach collection="templates" item="template" index="index">
<if test="template.topDomain != null">
when template_id = #{template.templateId}
then #{template.topDomain}
</if>
</foreach>
</trim>
<trim prefix="template_create_time = case" suffix="end,">
<foreach collection="templates" item="template" index="index">
<if test="template.templateCreateTime != null">
when template_id = #{template.templateId}
then #{template.templateCreateTime}
</if>
</foreach>
</trim>
<trim prefix="is_auto_match_combination = case" suffix="end,">
<foreach collection="templates" item="template" index="index">
<if test="template.isAutoMatchCombination != null">
when template_id = #{template.templateId}
then #{template.isAutoMatchCombination}
</if>
</foreach>
</trim>
<trim prefix="combination_id = case" suffix="end,">
<foreach collection="templates" item="template" index="index">
<if test="template.combinationId != null">
when template_id = #{template.templateId}
then #{template.combinationId}
</if>
</foreach>
</trim>
update_time = CURTIME()
</trim>
where template_id in
<foreach collection="templates" item="template" open="(" close=")" separator=",">
#{template.templateId}
</foreach>
</update>
<insert id="upsertCount">
insert manage_template_warning (template_id, warning_level, `count`, create_time, update_time)
values (#{templateId}, 0, #{count}, CURTIME(), CURTIME())
on duplicate key update `count` = values(`count`), update_time = CURTIME()
</insert>
</mapper>
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment