Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
A
automatic-test
Overview
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
钟涨钱
automatic-test
Commits
98fece20
Commit
98fece20
authored
Sep 09, 2021
by
朽木不可雕也
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
backup
parent
20dfe82d
Hide whitespace changes
Inline
Side-by-side
Showing
9 changed files
with
132 additions
and
213 deletions
+132
-213
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/createtasks/BiliBiliCreateTaskJob.java
+2
-2
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/createtasks/CreateTaskJob.java
+21
-4
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/createtasks/DouyuCreateTaskJob.java
+17
-21
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/createtasks/HuyaCreateTaskJob.java
+1
-10
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/BarrageExportTaskJob.java
+0
-107
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/DouyuExportTaskJob.java
+0
-30
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/ExportTaskJob.java
+89
-7
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/HuyaExportTaskJob.java
+0
-30
barrage-crawler-server/src/test/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/ExportTaskJobTest.java
+2
-2
No files found.
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/createtasks/BiliBiliCreateTaskJob.java
View file @
98fece20
...
@@ -2,7 +2,7 @@ package com.zhiweidata.automatictest.barragecrawlerserver.createtasks;
...
@@ -2,7 +2,7 @@ package com.zhiweidata.automatictest.barragecrawlerserver.createtasks;
import
com.fasterxml.jackson.databind.ObjectMapper
;
import
com.fasterxml.jackson.databind.ObjectMapper
;
import
com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTestTask
;
import
com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTestTask
;
import
com.zhiweidata.automatictest.barragecrawlerserver.exporttasks.
BiliBili
ExportTaskJob
;
import
com.zhiweidata.automatictest.barragecrawlerserver.exporttasks.ExportTaskJob
;
import
java.net.URL
;
import
java.net.URL
;
import
java.util.List
;
import
java.util.List
;
import
java.util.Map
;
import
java.util.Map
;
...
@@ -39,7 +39,7 @@ public class BiliBiliCreateTaskJob extends CreateTaskJob {
...
@@ -39,7 +39,7 @@ public class BiliBiliCreateTaskJob extends CreateTaskJob {
try
{
try
{
Map
<?,
?>
roomMap
=
(
Map
<?,
?>)
room
;
Map
<?,
?>
roomMap
=
(
Map
<?,
?>)
room
;
BarrageCollectionTestTask
collectionTask
=
BiliBiliCreateTaskJob
.
super
.
createCollectionTask
(
baseUrl
+
roomMap
.
get
(
"link"
),
currentTime
);
BarrageCollectionTestTask
collectionTask
=
BiliBiliCreateTaskJob
.
super
.
createCollectionTask
(
baseUrl
+
roomMap
.
get
(
"link"
),
currentTime
);
BiliBiliCreateTaskJob
.
super
.
createExportTask
(
collectionTask
,
BiliBiliExportTaskJob
.
class
);
BiliBiliCreateTaskJob
.
super
.
createExportTask
(
collectionTask
);
}
catch
(
Exception
e
)
{
}
catch
(
Exception
e
)
{
log
.
error
(
e
.
getMessage
(),
e
);
log
.
error
(
e
.
getMessage
(),
e
);
}
}
...
...
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/createtasks/CreateTaskJob.java
View file @
98fece20
...
@@ -3,11 +3,12 @@ package com.zhiweidata.automatictest.barragecrawlerserver.createtasks;
...
@@ -3,11 +3,12 @@ package com.zhiweidata.automatictest.barragecrawlerserver.createtasks;
import
com.fasterxml.jackson.databind.ObjectMapper
;
import
com.fasterxml.jackson.databind.ObjectMapper
;
import
com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTestTask
;
import
com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTestTask
;
import
com.zhiweidata.automatictest.barragecrawlerserver.entity.ServerResponseMessage
;
import
com.zhiweidata.automatictest.barragecrawlerserver.entity.ServerResponseMessage
;
import
com.zhiweidata.automatictest.barragecrawlerserver.exporttasks.
Barrage
ExportTaskJob
;
import
com.zhiweidata.automatictest.barragecrawlerserver.exporttasks.ExportTaskJob
;
import
com.zhiweidata.automatictest.barragecrawlerserver.mapper.BarrageCollectionTestTaskMapper
;
import
com.zhiweidata.automatictest.barragecrawlerserver.mapper.BarrageCollectionTestTaskMapper
;
import
com.zhiweidata.automatictest.barragecrawlerserver.mapper.ServerResponseMessageMapper
;
import
com.zhiweidata.automatictest.barragecrawlerserver.mapper.ServerResponseMessageMapper
;
import
com.zhiweidata.automatictest.barragecrawlerserver.util.TimeUtil
;
import
com.zhiweidata.automatictest.barragecrawlerserver.util.TimeUtil
;
import
java.io.IOException
;
import
java.io.IOException
;
import
java.util.Set
;
import
java.util.UUID
;
import
java.util.UUID
;
import
lombok.extern.slf4j.Slf4j
;
import
lombok.extern.slf4j.Slf4j
;
import
org.apache.http.HttpResponse
;
import
org.apache.http.HttpResponse
;
...
@@ -40,6 +41,21 @@ import static com.zhiweidata.automatictest.publics.BeanContainer.STANDARD_DATE_F
...
@@ -40,6 +41,21 @@ import static com.zhiweidata.automatictest.publics.BeanContainer.STANDARD_DATE_F
@SuppressWarnings
({
"JavaDoc"
})
@SuppressWarnings
({
"JavaDoc"
})
public
abstract
class
CreateTaskJob
implements
Job
{
public
abstract
class
CreateTaskJob
implements
Job
{
/**
/**
* 创建直播间的弹幕采集,和弹幕导出测试任务
*/
protected
final
void
createTasks
(
@NotNull
Set
<
String
>
liveUrls
)
{
final
long
currentTime
=
System
.
currentTimeMillis
();
liveUrls
.
forEach
(
liveUrl
->
{
try
{
BarrageCollectionTestTask
collectionTask
=
this
.
createCollectionTask
(
liveUrl
,
currentTime
);
this
.
createExportTask
(
collectionTask
);
}
catch
(
Exception
e
)
{
log
.
error
(
e
.
getMessage
(),
e
);
}
});
}
/**
* 请求弹幕采集服务器创建采集任务
* 请求弹幕采集服务器创建采集任务
*/
*/
@NotNull
@NotNull
...
@@ -70,9 +86,10 @@ public abstract class CreateTaskJob implements Job {
...
@@ -70,9 +86,10 @@ public abstract class CreateTaskJob implements Job {
/**
/**
* 创建定时导出任务
* 创建定时导出任务
*/
*/
protected
final
<
T
extends
BarrageExportTaskJob
>
void
createExportTask
(
@NotNull
BarrageCollectionTestTask
collectionTask
,
Class
<
T
>
tClass
)
throws
SchedulerException
{
protected
final
<
T
extends
ExportTaskJob
>
void
createExportTask
(
@NotNull
BarrageCollectionTestTask
collectionTask
)
throws
SchedulerException
{
JobBuilder
jobBuilder
=
JobBuilder
.
newJob
(
tClass
);
Class
<
ExportTaskJob
>
taskJobClass
=
ExportTaskJob
.
class
;
String
name
=
collectionTask
.
getName
(),
group
=
tClass
.
getSimpleName
();
JobBuilder
jobBuilder
=
JobBuilder
.
newJob
(
taskJobClass
);
String
name
=
collectionTask
.
getName
(),
group
=
taskJobClass
.
getSimpleName
();
jobBuilder
.
withIdentity
(
name
,
group
);
jobBuilder
.
withIdentity
(
name
,
group
);
jobBuilder
.
withDescription
(
"弹幕导出,以及导出的excel文件的分析"
);
jobBuilder
.
withDescription
(
"弹幕导出,以及导出的excel文件的分析"
);
JobDataMap
dataMap
=
new
JobDataMap
();
JobDataMap
dataMap
=
new
JobDataMap
();
...
...
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/createtasks/DouyuCreateTaskJob.java
View file @
98fece20
...
@@ -4,7 +4,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
...
@@ -4,7 +4,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import
com.fasterxml.jackson.databind.ObjectMapper
;
import
com.fasterxml.jackson.databind.ObjectMapper
;
import
com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTestTask
;
import
com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTestTask
;
import
com.zhiweidata.automatictest.barragecrawlerserver.exception.BarrageHomePageException
;
import
com.zhiweidata.automatictest.barragecrawlerserver.exception.BarrageHomePageException
;
import
com.zhiweidata.automatictest.barragecrawlerserver.exporttasks.
Douyu
ExportTaskJob
;
import
com.zhiweidata.automatictest.barragecrawlerserver.exporttasks.ExportTaskJob
;
import
java.util.HashSet
;
import
java.util.HashSet
;
import
java.util.List
;
import
java.util.List
;
import
java.util.Map
;
import
java.util.Map
;
...
@@ -31,7 +31,7 @@ import static com.zhiweidata.automatictest.publics.BeanContainer.HTTP_CLIENT;
...
@@ -31,7 +31,7 @@ import static com.zhiweidata.automatictest.publics.BeanContainer.HTTP_CLIENT;
*/
*/
@Slf4j
@Slf4j
@DisallowConcurrentExecution
@DisallowConcurrentExecution
@SuppressWarnings
({
"JavaDoc"
,
"unused"
,
"SpellCheckingInspection"
,
"GrazieInspection"
,
"DuplicatedCode"
})
@SuppressWarnings
({
"JavaDoc"
,
"unused"
,
"SpellCheckingInspection"
,
"GrazieInspection"
})
public
class
DouyuCreateTaskJob
extends
CreateTaskJob
{
public
class
DouyuCreateTaskJob
extends
CreateTaskJob
{
private
static
final
Pattern
PATTERN
=
Pattern
.
compile
(
"var \\$mainData = \\[[\\w\\W]+?(];)"
);
private
static
final
Pattern
PATTERN
=
Pattern
.
compile
(
"var \\$mainData = \\[[\\w\\W]+?(];)"
);
...
@@ -41,15 +41,7 @@ public class DouyuCreateTaskJob extends CreateTaskJob {
...
@@ -41,15 +41,7 @@ public class DouyuCreateTaskJob extends CreateTaskJob {
HttpResponse
response
=
HTTP_CLIENT
.
execute
(
new
HttpGet
(
"https://www.douyu.com/"
));
HttpResponse
response
=
HTTP_CLIENT
.
execute
(
new
HttpGet
(
"https://www.douyu.com/"
));
String
html
=
EntityUtils
.
toString
(
response
.
getEntity
());
String
html
=
EntityUtils
.
toString
(
response
.
getEntity
());
Set
<
String
>
liveUrls
=
this
.
getLiveUrl
(
html
);
Set
<
String
>
liveUrls
=
this
.
getLiveUrl
(
html
);
final
long
currentTime
=
System
.
currentTimeMillis
();
super
.
createTasks
(
liveUrls
);
liveUrls
.
forEach
(
liveUrl
->
{
try
{
BarrageCollectionTestTask
collectionTask
=
super
.
createCollectionTask
(
liveUrl
,
currentTime
);
super
.
createExportTask
(
collectionTask
,
DouyuExportTaskJob
.
class
);
}
catch
(
Exception
e
)
{
log
.
error
(
e
.
getMessage
(),
e
);
}
});
}
catch
(
Exception
e
)
{
}
catch
(
Exception
e
)
{
log
.
error
(
e
.
getMessage
(),
e
);
log
.
error
(
e
.
getMessage
(),
e
);
}
}
...
@@ -86,21 +78,13 @@ public class DouyuCreateTaskJob extends CreateTaskJob {
...
@@ -86,21 +78,13 @@ public class DouyuCreateTaskJob extends CreateTaskJob {
for
(
Object
moduleObj
:
modules
)
{
for
(
Object
moduleObj
:
modules
)
{
List
<?>
rl
=
(
List
<?>)
((
Map
<?,
?>)
moduleObj
).
get
(
"rl"
);
List
<?>
rl
=
(
List
<?>)
((
Map
<?,
?>)
moduleObj
).
get
(
"rl"
);
if
(
Objects
.
isNull
(
rl
))
continue
;
if
(
Objects
.
isNull
(
rl
))
continue
;
for
(
Object
rlObj
:
rl
)
{
this
.
assemblyPruneUrl
(
baseUrl
,
rl
,
liveUrls
);
String
urlString
=
(
String
)
((
Map
<?,
?>)
rlObj
).
get
(
"url"
);
if
(
urlString
.
startsWith
(
"/"
))
urlString
=
urlString
.
replaceFirst
(
"/"
,
""
);
liveUrls
.
add
(
baseUrl
+
urlString
);
}
}
}
}
else
if
(
"recommend"
.
equals
(
type
)
||
"roomList10"
.
equals
(
type
))
{
}
else
if
(
"recommend"
.
equals
(
type
)
||
"roomList10"
.
equals
(
type
))
{
// 精彩推荐 和 热门游戏(如 英雄联盟,穿越火线,王者荣耀,CFHD,和平精英,COD手游,云顶模式,天刀手游,竞技游戏,网络游戏,客厅游戏,手机游戏,暴雪专区,科技文化,官方活动 等)
// 精彩推荐 和 热门游戏(如 英雄联盟,穿越火线,王者荣耀,CFHD,和平精英,COD手游,云顶模式,天刀手游,竞技游戏,网络游戏,客厅游戏,手机游戏,暴雪专区,科技文化,官方活动 等)
List
<?>
rl
=
(
List
<?>)
jsonMap
.
get
(
"rl"
);
List
<?>
rl
=
(
List
<?>)
jsonMap
.
get
(
"rl"
);
if
(
Objects
.
isNull
(
rl
))
continue
;
if
(
Objects
.
isNull
(
rl
))
continue
;
for
(
Object
rlObj
:
rl
)
{
this
.
assemblyPruneUrl
(
baseUrl
,
rl
,
liveUrls
);
String
urlString
=
(
String
)
((
Map
<?,
?>)
rlObj
).
get
(
"url"
);
if
(
urlString
.
startsWith
(
"/"
))
urlString
=
urlString
.
replaceFirst
(
"/"
,
""
);
liveUrls
.
add
(
baseUrl
+
urlString
);
}
}
else
if
(
"footerHotroom"
.
equals
(
type
))
{
}
else
if
(
"footerHotroom"
.
equals
(
type
))
{
List
<?>
ds
=
(
List
<?>)
jsonMap
.
get
(
"ds"
);
List
<?>
ds
=
(
List
<?>)
jsonMap
.
get
(
"ds"
);
for
(
Object
dsObj
:
ds
)
{
for
(
Object
dsObj
:
ds
)
{
...
@@ -117,4 +101,16 @@ public class DouyuCreateTaskJob extends CreateTaskJob {
...
@@ -117,4 +101,16 @@ public class DouyuCreateTaskJob extends CreateTaskJob {
throw
new
BarrageHomePageException
(
"斗鱼直播间弹幕获取失败"
);
throw
new
BarrageHomePageException
(
"斗鱼直播间弹幕获取失败"
);
}
}
}
}
/**
* 拼接url,以及修建url尾部的“/”
*/
@SuppressWarnings
(
"SameParameterValue"
)
private
void
assemblyPruneUrl
(
String
baseUrl
,
@NotNull
List
<?>
rl
,
@NotNull
Set
<
String
>
liveUrls
)
{
for
(
Object
rlObj
:
rl
)
{
String
urlString
=
(
String
)
((
Map
<?,
?>)
rlObj
).
get
(
"url"
);
if
(
urlString
.
startsWith
(
"/"
))
urlString
=
urlString
.
replaceFirst
(
"/"
,
""
);
liveUrls
.
add
(
baseUrl
+
urlString
);
}
}
}
}
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/createtasks/HuyaCreateTaskJob.java
View file @
98fece20
...
@@ -4,7 +4,6 @@ import com.fasterxml.jackson.core.JsonProcessingException;
...
@@ -4,7 +4,6 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import
com.fasterxml.jackson.databind.ObjectMapper
;
import
com.fasterxml.jackson.databind.ObjectMapper
;
import
com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTestTask
;
import
com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTestTask
;
import
com.zhiweidata.automatictest.barragecrawlerserver.exception.BarrageHomePageException
;
import
com.zhiweidata.automatictest.barragecrawlerserver.exception.BarrageHomePageException
;
import
com.zhiweidata.automatictest.barragecrawlerserver.exporttasks.HuyaExportTaskJob
;
import
java.util.HashSet
;
import
java.util.HashSet
;
import
java.util.List
;
import
java.util.List
;
import
java.util.Map
;
import
java.util.Map
;
...
@@ -93,14 +92,6 @@ public class HuyaCreateTaskJob extends CreateTaskJob {
...
@@ -93,14 +92,6 @@ public class HuyaCreateTaskJob extends CreateTaskJob {
}
else
{
}
else
{
log
.
warn
(
"虎牙直播间地址数量:{}"
,
size
);
log
.
warn
(
"虎牙直播间地址数量:{}"
,
size
);
}
}
final
long
currentTime
=
System
.
currentTimeMillis
();
super
.
createTasks
(
liveUrls
);
liveUrls
.
forEach
(
liveUrl
->
{
try
{
BarrageCollectionTestTask
collectionTask
=
super
.
createCollectionTask
(
liveUrl
,
currentTime
);
super
.
createExportTask
(
collectionTask
,
HuyaExportTaskJob
.
class
);
}
catch
(
Exception
e
)
{
log
.
error
(
e
.
getMessage
(),
e
);
}
});
}
}
}
}
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/BarrageExportTaskJob.java
deleted
100644 → 0
View file @
20dfe82d
package
com
.
zhiweidata
.
automatictest
.
barragecrawlerserver
.
exporttasks
;
import
com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionInfo
;
import
com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageExportResult
;
import
com.zhiweidata.automatictest.barragecrawlerserver.entity.ServerResponseMessage
;
import
com.zhiweidata.automatictest.barragecrawlerserver.mapper.BarrageExportResultMapper
;
import
com.zhiweidata.automatictest.barragecrawlerserver.mapper.ServerResponseMessageMapper
;
import
java.io.File
;
import
java.io.FileOutputStream
;
import
java.io.IOException
;
import
java.io.InputStream
;
import
java.util.ArrayList
;
import
java.util.List
;
import
java.util.zip.ZipEntry
;
import
java.util.zip.ZipInputStream
;
import
org.apache.ibatis.session.SqlSession
;
import
org.jetbrains.annotations.NotNull
;
import
org.quartz.Job
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
static
com
.
zhiweidata
.
automatictest
.
publics
.
BeanContainer
.
SQL_SESSION_FACTORY
;
import
static
java
.
util
.
Objects
.
isNull
;
import
static
java
.
util
.
Objects
.
nonNull
;
import
static
java
.
util
.
Objects
.
requireNonNull
;
/**
* 弹幕导出任务
*
* @author aszswaz
* @createTime 2021-08-25 17:29:39
* @ide IntelliJ IDEA
*/
@SuppressWarnings
(
"JavaDoc"
)
public
interface
BarrageExportTaskJob
extends
Job
{
Logger
log
=
LoggerFactory
.
getLogger
(
BarrageExportTaskJob
.
class
);
/**
* 下载并解压zip文件
*
* @return 解压得到的文件
*/
default
List
<
File
>
write
(
File
cacheDir
,
InputStream
inputStream
)
throws
IOException
{
if
(!
cacheDir
.
exists
())
if
(!
cacheDir
.
mkdirs
())
log
.
error
(
"文件夹{}创建失败"
,
cacheDir
.
getName
());
final
List
<
File
>
outFiles
=
new
ArrayList
<>();
try
(
ZipInputStream
zipInputStream
=
new
ZipInputStream
(
inputStream
))
{
byte
[]
buff
=
new
byte
[
8192
];
int
len
;
while
(
true
)
{
ZipEntry
zipEntry
=
zipInputStream
.
getNextEntry
();
if
(
isNull
(
zipEntry
))
break
;
File
execFile
=
new
File
(
cacheDir
,
requireNonNull
(
zipEntry
.
getName
()));
try
(
FileOutputStream
fileOutputStream
=
new
FileOutputStream
(
execFile
))
{
while
((
len
=
zipInputStream
.
read
(
buff
))
!=
-
1
)
fileOutputStream
.
write
(
buff
,
0
,
len
);
}
zipInputStream
.
closeEntry
();
outFiles
.
add
(
execFile
);
}
}
return
outFiles
;
}
/**
* 删除文件
*/
default
void
removeFile
(
File
file
)
{
// 如果是文件夹,先删除文件夹中的文件
if
(
file
.
isDirectory
())
{
File
[]
files
=
file
.
listFiles
();
if
(
nonNull
(
files
))
{
for
(
File
subFile
:
files
)
this
.
removeFile
(
subFile
);
}
}
// 删除文件夹或文件
if
(!
file
.
delete
())
log
.
error
(
"文件夹:{}删除失败"
,
file
.
getName
());
}
/**
* 保存服务器的响应信息
*/
default
void
saveResponseMessage
(
ServerResponseMessage
message
)
{
try
(
SqlSession
session
=
SQL_SESSION_FACTORY
.
openSession
(
true
))
{
ServerResponseMessageMapper
messageMapper
=
session
.
getMapper
(
ServerResponseMessageMapper
.
class
);
messageMapper
.
insert
(
message
);
}
}
/**
* 保存弹幕的导出信息
*
* @param collectionInfo 弹幕采集任务信息
* @param exportCount 导出的弹幕数量
*/
default
void
saveExportInfo
(
@NotNull
BarrageCollectionInfo
collectionInfo
,
int
exportCount
)
{
try
(
SqlSession
session
=
SQL_SESSION_FACTORY
.
openSession
(
true
))
{
BarrageExportResult
exportResult
=
new
BarrageExportResult
();
exportResult
.
setName
(
collectionInfo
.
getName
());
exportResult
.
setExportVolume
(
exportCount
);
exportResult
.
setTotalCollection
(
collectionInfo
.
getCollectionCount
());
exportResult
.
setPercentage
((
double
)
exportCount
/
collectionInfo
.
getCollectionCount
());
BarrageExportResultMapper
resultMapper
=
session
.
getMapper
(
BarrageExportResultMapper
.
class
);
resultMapper
.
insert
(
exportResult
);
}
}
}
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/DouyuExportTaskJob.java
deleted
100644 → 0
View file @
20dfe82d
package
com
.
zhiweidata
.
automatictest
.
barragecrawlerserver
.
exporttasks
;
import
com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTestTask
;
import
lombok.extern.slf4j.Slf4j
;
import
org.quartz.JobDataMap
;
import
org.quartz.JobExecutionContext
;
import
static
com
.
zhiweidata
.
automatictest
.
barragecrawlerserver
.
config
.
BaseConfig
.
BARRAGE_COLLECTION_TASK_JOB_KEY
;
import
static
java
.
util
.
Objects
.
requireNonNull
;
/**
* 斗鱼弹幕导出任务
*
* @author aszswaz
* @createTime 2021-08-25 17:30:18
* @ide IntelliJ IDEA
*/
@Slf4j
@SuppressWarnings
(
"JavaDoc"
)
public
class
DouyuExportTaskJob
implements
BarrageExportTaskJob
{
@Override
public
void
execute
(
JobExecutionContext
context
)
{
try
{
JobDataMap
dataMap
=
requireNonNull
(
context
.
getMergedJobDataMap
());
BarrageCollectionTestTask
collectionTask
=
(
BarrageCollectionTestTask
)
requireNonNull
(
dataMap
.
get
(
BARRAGE_COLLECTION_TASK_JOB_KEY
));
}
catch
(
Exception
e
)
{
log
.
error
(
e
.
getMessage
(),
e
);
}
}
}
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/
BiliBili
ExportTaskJob.java
→
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/ExportTaskJob.java
View file @
98fece20
...
@@ -7,16 +7,25 @@ import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectio
...
@@ -7,16 +7,25 @@ import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectio
import
com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionInfo
;
import
com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionInfo
;
import
com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageExportResult
;
import
com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageExportResult
;
import
com.zhiweidata.automatictest.barragecrawlerserver.entity.ServerResponseMessage
;
import
com.zhiweidata.automatictest.barragecrawlerserver.entity.ServerResponseMessage
;
import
com.zhiweidata.automatictest.barragecrawlerserver.mapper.BarrageExportResultMapper
;
import
com.zhiweidata.automatictest.barragecrawlerserver.mapper.ServerResponseMessageMapper
;
import
java.io.File
;
import
java.io.File
;
import
java.io.FileOutputStream
;
import
java.io.IOException
;
import
java.io.IOException
;
import
java.io.InputStream
;
import
java.util.ArrayList
;
import
java.util.List
;
import
java.util.List
;
import
java.util.Map
;
import
java.util.Map
;
import
java.util.zip.ZipEntry
;
import
java.util.zip.ZipInputStream
;
import
lombok.extern.slf4j.Slf4j
;
import
lombok.extern.slf4j.Slf4j
;
import
org.apache.http.HttpResponse
;
import
org.apache.http.HttpResponse
;
import
org.apache.http.client.methods.HttpGet
;
import
org.apache.http.client.methods.HttpGet
;
import
org.apache.http.entity.ContentType
;
import
org.apache.http.entity.ContentType
;
import
org.apache.http.util.EntityUtils
;
import
org.apache.http.util.EntityUtils
;
import
org.apache.ibatis.session.SqlSession
;
import
org.jetbrains.annotations.NotNull
;
import
org.jetbrains.annotations.NotNull
;
import
org.quartz.Job
;
import
org.quartz.JobDataMap
;
import
org.quartz.JobDataMap
;
import
org.quartz.JobExecutionContext
;
import
org.quartz.JobExecutionContext
;
import
org.quartz.SchedulerException
;
import
org.quartz.SchedulerException
;
...
@@ -25,6 +34,9 @@ import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfi
...
@@ -25,6 +34,9 @@ import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfi
import
static
com
.
zhiweidata
.
automatictest
.
barragecrawlerserver
.
config
.
BaseConfig
.
CACHE_DIR
;
import
static
com
.
zhiweidata
.
automatictest
.
barragecrawlerserver
.
config
.
BaseConfig
.
CACHE_DIR
;
import
static
com
.
zhiweidata
.
automatictest
.
barragecrawlerserver
.
config
.
BaseConfig
.
TASK_URL
;
import
static
com
.
zhiweidata
.
automatictest
.
barragecrawlerserver
.
config
.
BaseConfig
.
TASK_URL
;
import
static
com
.
zhiweidata
.
automatictest
.
publics
.
BeanContainer
.
HTTP_CLIENT
;
import
static
com
.
zhiweidata
.
automatictest
.
publics
.
BeanContainer
.
HTTP_CLIENT
;
import
static
com
.
zhiweidata
.
automatictest
.
publics
.
BeanContainer
.
SQL_SESSION_FACTORY
;
import
static
java
.
util
.
Objects
.
isNull
;
import
static
java
.
util
.
Objects
.
nonNull
;
import
static
java
.
util
.
Objects
.
requireNonNull
;
import
static
java
.
util
.
Objects
.
requireNonNull
;
/**
/**
...
@@ -36,14 +48,13 @@ import static java.util.Objects.requireNonNull;
...
@@ -36,14 +48,13 @@ import static java.util.Objects.requireNonNull;
*/
*/
@Slf4j
@Slf4j
@SuppressWarnings
(
"JavaDoc"
)
@SuppressWarnings
(
"JavaDoc"
)
public
class
BiliBiliExportTaskJob
implements
BarrageExportTask
Job
{
public
class
ExportTaskJob
implements
Job
{
/**
/**
* TODO:
* 1. 使用任务的 name 作为关键词,请求接口,获得对应任务的任务ID
* 1. 使用任务的 name 作为关键词,请求接口,获得对应任务的任务ID
* 2. 根据获得的任务ID,尝试从弹幕导出接口,导出弹幕
* 2. 根据获得的任务ID,尝试从弹幕导出接口,导出弹幕
* 3. 判断服务的响应是 excel 文件,还是 application/json
* 3. 判断服务的响应是 excel 文件,还是 application/json
* 1. 如果是 application/json,就代表导出失败,内容是服务器响应的失败信息
* 1. 如果是 application/json,就代表导出失败,内容是服务器响应的失败信息
* 2. 如果是 zip 文件就表示,文件导出成功,分析一下文件中的弹幕
* 2. 如果是 zip 文件就表示,文件导出成功,分析一下文件中的弹幕
*/
*/
@Override
@Override
public
void
execute
(
JobExecutionContext
context
)
{
public
void
execute
(
JobExecutionContext
context
)
{
...
@@ -111,4 +122,75 @@ public class BiliBiliExportTaskJob implements BarrageExportTaskJob {
...
@@ -111,4 +122,75 @@ public class BiliBiliExportTaskJob implements BarrageExportTaskJob {
this
.
saveExportInfo
(
collectionInfo
,
exportCount
);
this
.
saveExportInfo
(
collectionInfo
,
exportCount
);
}
}
}
}
/**
* 下载并解压zip文件
*
* @return 解压得到的文件
*/
@NotNull
private
List
<
File
>
write
(
@NotNull
File
cacheDir
,
InputStream
inputStream
)
throws
IOException
{
if
(!
cacheDir
.
exists
())
if
(!
cacheDir
.
mkdirs
())
log
.
error
(
"文件夹{}创建失败"
,
cacheDir
.
getName
());
final
List
<
File
>
outFiles
=
new
ArrayList
<>();
try
(
ZipInputStream
zipInputStream
=
new
ZipInputStream
(
inputStream
))
{
byte
[]
buff
=
new
byte
[
8192
];
int
len
;
while
(
true
)
{
ZipEntry
zipEntry
=
zipInputStream
.
getNextEntry
();
if
(
isNull
(
zipEntry
))
break
;
File
execFile
=
new
File
(
cacheDir
,
requireNonNull
(
zipEntry
.
getName
()));
try
(
FileOutputStream
fileOutputStream
=
new
FileOutputStream
(
execFile
))
{
while
((
len
=
zipInputStream
.
read
(
buff
))
!=
-
1
)
fileOutputStream
.
write
(
buff
,
0
,
len
);
}
zipInputStream
.
closeEntry
();
outFiles
.
add
(
execFile
);
}
}
return
outFiles
;
}
/**
* 删除文件
*/
private
void
removeFile
(
@NotNull
File
file
)
{
// 如果是文件夹,先删除文件夹中的文件
if
(
file
.
isDirectory
())
{
File
[]
files
=
file
.
listFiles
();
if
(
nonNull
(
files
))
{
for
(
File
subFile
:
files
)
this
.
removeFile
(
subFile
);
}
}
// 删除文件夹或文件
if
(!
file
.
delete
())
log
.
error
(
"文件夹:{}删除失败"
,
file
.
getName
());
}
/**
* 保存服务器的响应信息
*/
private
void
saveResponseMessage
(
ServerResponseMessage
message
)
{
try
(
SqlSession
session
=
SQL_SESSION_FACTORY
.
openSession
(
true
))
{
ServerResponseMessageMapper
messageMapper
=
session
.
getMapper
(
ServerResponseMessageMapper
.
class
);
messageMapper
.
insert
(
message
);
}
}
/**
* 保存弹幕的导出信息
*
* @param collectionInfo 弹幕采集任务信息
* @param exportCount 导出的弹幕数量
*/
private
void
saveExportInfo
(
@NotNull
BarrageCollectionInfo
collectionInfo
,
int
exportCount
)
{
try
(
SqlSession
session
=
SQL_SESSION_FACTORY
.
openSession
(
true
))
{
BarrageExportResult
exportResult
=
new
BarrageExportResult
();
exportResult
.
setName
(
collectionInfo
.
getName
());
exportResult
.
setExportVolume
(
exportCount
);
exportResult
.
setTotalCollection
(
collectionInfo
.
getCollectionCount
());
exportResult
.
setPercentage
((
double
)
exportCount
/
collectionInfo
.
getCollectionCount
());
BarrageExportResultMapper
resultMapper
=
session
.
getMapper
(
BarrageExportResultMapper
.
class
);
resultMapper
.
insert
(
exportResult
);
}
}
}
}
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/HuyaExportTaskJob.java
deleted
100644 → 0
View file @
20dfe82d
package
com
.
zhiweidata
.
automatictest
.
barragecrawlerserver
.
exporttasks
;
import
com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTestTask
;
import
lombok.extern.slf4j.Slf4j
;
import
org.quartz.JobDataMap
;
import
org.quartz.JobExecutionContext
;
import
static
com
.
zhiweidata
.
automatictest
.
barragecrawlerserver
.
config
.
BaseConfig
.
BARRAGE_COLLECTION_TASK_JOB_KEY
;
import
static
java
.
util
.
Objects
.
requireNonNull
;
/**
* 虎牙弹幕导出任务
*
* @author aszswaz
* @createTime 2021-08-25 17:30:18
* @ide IntelliJ IDEA
*/
@Slf4j
@SuppressWarnings
(
"JavaDoc"
)
public
class
HuyaExportTaskJob
implements
BarrageExportTaskJob
{
@Override
public
void
execute
(
JobExecutionContext
context
)
{
try
{
JobDataMap
dataMap
=
requireNonNull
(
context
.
getMergedJobDataMap
());
BarrageCollectionTestTask
collectionTask
=
(
BarrageCollectionTestTask
)
requireNonNull
(
dataMap
.
get
(
BARRAGE_COLLECTION_TASK_JOB_KEY
));
}
catch
(
Exception
e
)
{
log
.
error
(
e
.
getMessage
(),
e
);
}
}
}
barrage-crawler-server/src/test/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/
BiliBili
ExportTaskJobTest.java
→
barrage-crawler-server/src/test/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/ExportTaskJobTest.java
View file @
98fece20
...
@@ -12,14 +12,14 @@ import static com.zhiweidata.automatictest.publics.BeanContainer.SCHEDULER;
...
@@ -12,14 +12,14 @@ import static com.zhiweidata.automatictest.publics.BeanContainer.SCHEDULER;
* @ide IntelliJ IDEA
* @ide IntelliJ IDEA
*/
*/
@SuppressWarnings
(
"JavaDoc"
)
@SuppressWarnings
(
"JavaDoc"
)
class
BiliBili
ExportTaskJobTest
{
class
ExportTaskJobTest
{
public
static
void
main
(
String
[]
args
)
throws
SchedulerException
,
IOException
{
public
static
void
main
(
String
[]
args
)
throws
SchedulerException
,
IOException
{
BarrageCollectionTestTask
collectionTask
=
new
BarrageCollectionTestTask
(
BarrageCollectionTestTask
collectionTask
=
new
BarrageCollectionTestTask
(
"https://live.bilibili.com/12265?hotRank=0"
,
"62aae6a3-e563-4f24-9e3a-135c6e74a8ac"
,
System
.
currentTimeMillis
()
"https://live.bilibili.com/12265?hotRank=0"
,
"62aae6a3-e563-4f24-9e3a-135c6e74a8ac"
,
System
.
currentTimeMillis
()
);
);
BiliBiliExportTaskJob
exportTaskJob
=
new
BiliBili
ExportTaskJob
();
ExportTaskJob
exportTaskJob
=
new
ExportTaskJob
();
exportTaskJob
.
export
(
collectionTask
);
exportTaskJob
.
export
(
collectionTask
);
SCHEDULER
.
shutdown
(
true
);
SCHEDULER
.
shutdown
(
true
);
}
}
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment