Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
S
source_forward
Overview
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
zhiwei
source_forward
Commits
a71c606b
Commit
a71c606b
authored
Jan 10, 2019
by
yangchen
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
链接是否删除 修改
parent
37ac4e23
Expand all
Show whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
32 additions
and
5 deletions
+32
-5
src/main/java/com/zhiwei/source_forward/bean/UrlLiveBean.java
+21
-2
src/main/java/com/zhiwei/source_forward/crawler/UrlLiveCrawler.java
+0
-0
src/main/java/com/zhiwei/source_forward/run/URLLive.java
+11
-3
No files found.
src/main/java/com/zhiwei/source_forward/bean/UrlLiveBean.java
View file @
a71c606b
...
@@ -6,6 +6,8 @@ public class UrlLiveBean {
...
@@ -6,6 +6,8 @@ public class UrlLiveBean {
private
Integer
isLive
;
private
Integer
isLive
;
private
String
title
;
public
UrlLiveBean
()
{
public
UrlLiveBean
()
{
super
();
super
();
}
}
...
@@ -16,9 +18,17 @@ public class UrlLiveBean {
...
@@ -16,9 +18,17 @@ public class UrlLiveBean {
this
.
isLive
=
isLive
;
this
.
isLive
=
isLive
;
}
}
public
UrlLiveBean
(
String
url
,
boolean
isLive
)
{
public
UrlLiveBean
(
String
url
,
Integer
isLive
,
String
title
)
{
super
();
this
.
url
=
url
;
this
.
isLive
=
isLive
;
this
.
title
=
title
;
}
public
UrlLiveBean
(
String
url
,
boolean
isLive
,
String
title
)
{
super
();
super
();
this
.
url
=
url
;
this
.
url
=
url
;
this
.
title
=
title
;
if
(
isLive
)
{
if
(
isLive
)
{
this
.
isLive
=
1
;
//已删除
this
.
isLive
=
1
;
//已删除
}
else
{
}
else
{
...
@@ -26,6 +36,14 @@ public class UrlLiveBean {
...
@@ -26,6 +36,14 @@ public class UrlLiveBean {
}
}
}
}
public
String
getTitle
()
{
return
title
;
}
public
void
setTitle
(
String
title
)
{
this
.
title
=
title
;
}
public
String
getUrl
()
{
public
String
getUrl
()
{
return
url
;
return
url
;
}
}
...
@@ -44,7 +62,8 @@ public class UrlLiveBean {
...
@@ -44,7 +62,8 @@ public class UrlLiveBean {
@Override
@Override
public
String
toString
()
{
public
String
toString
()
{
return
"UrlLiveBean [url="
+
url
+
", isLive="
+
isLive
+
"]"
;
return
"UrlLiveBean [url="
+
url
+
", isLive="
+
isLive
+
", title="
+
title
+
"]"
;
}
}
/**
/**
...
...
src/main/java/com/zhiwei/source_forward/crawler/UrlLiveCrawler.java
View file @
a71c606b
This diff is collapsed.
Click to expand it.
src/main/java/com/zhiwei/source_forward/run/URLLive.java
View file @
a71c606b
package
com
.
zhiwei
.
source_forward
.
run
;
package
com
.
zhiwei
.
source_forward
.
run
;
import
java.io.IOException
;
import
java.util.ArrayList
;
import
java.util.ArrayList
;
import
java.util.Collections
;
import
java.util.Collections
;
import
java.util.List
;
import
java.util.List
;
import
java.util.Map
;
import
java.util.Map
;
import
java.util.Map.Entry
;
import
java.util.Map.Entry
;
import
com.zhiwei.common.config.GroupType
;
import
com.zhiwei.crawler.core.HttpBoot
;
import
com.zhiwei.crawler.core.RequestUtils
;
import
com.zhiwei.crawler.proxy.ProxyFactory
;
import
com.zhiwei.source_forward.bean.UrlLiveBean
;
import
com.zhiwei.source_forward.bean.UrlLiveBean
;
import
com.zhiwei.source_forward.bean.UrlLiveBean.Attribution
;
import
com.zhiwei.source_forward.bean.UrlLiveBean.Attribution
;
import
com.zhiwei.source_forward.crawler.UrlLiveCrawler
;
import
com.zhiwei.source_forward.crawler.UrlLiveCrawler
;
...
@@ -46,6 +51,7 @@ public class URLLive {
...
@@ -46,6 +51,7 @@ public class URLLive {
}
else
if
(
i
==
0
)
{
}
else
if
(
i
==
0
)
{
map
.
put
(
"是否删除"
,
false
);
map
.
put
(
"是否删除"
,
false
);
}
}
map
.
put
(
"title"
,
ub
.
getTitle
());
dataMap
.
put
(
url
,
map
);
dataMap
.
put
(
url
,
map
);
}
}
}
}
...
@@ -60,13 +66,15 @@ public class URLLive {
...
@@ -60,13 +66,15 @@ public class URLLive {
*/
*/
public
static
List
<
UrlLiveBean
>
verificationURLLive
(
List
<
String
>
urlList
){
public
static
List
<
UrlLiveBean
>
verificationURLLive
(
List
<
String
>
urlList
){
//启动验证链接是否有效程序程序
//启动验证链接是否有效程序程序
List
<
UrlLiveBean
>
dataList
=
UrlLiveCrawlerThread
.
getUrlLiveCrawle
(
urlList
);
return
UrlLiveCrawlerThread
.
getUrlLiveCrawle
(
urlList
);
return
dataList
;
}
}
public
static
void
main
(
String
[]
args
)
{
public
static
void
main
(
String
[]
args
)
{
ProxyFactory
.
init
(
"zookeeper://192.168.0.36:2181"
,
"local"
,
GroupType
.
PROVIDER
);
List
<
String
>
urlList
=
new
ArrayList
<>();
List
<
String
>
urlList
=
new
ArrayList
<>();
urlList
.
add
(
"http://www.teso.cc/html/zixun/201606/233848.html"
);
urlList
.
add
(
"https://www.hao123.com/mid/16981890690654602094"
);
List
<
UrlLiveBean
>
u
=
URLLive
.
verificationURLLive
(
urlList
);
List
<
UrlLiveBean
>
u
=
URLLive
.
verificationURLLive
(
urlList
);
for
(
UrlLiveBean
b
:
u
)
{
for
(
UrlLiveBean
b
:
u
)
{
System
.
out
.
println
(
b
.
toString
());
System
.
out
.
println
(
b
.
toString
());
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment