Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
S
searchhotcrawler
Overview
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
zhiwei
searchhotcrawler
Commits
d431fc6a
Commit
d431fc6a
authored
Sep 17, 2021
by
chenweitao
Browse files
Options
Browse Files
Download
Plain Diff
Merge branch 'working' into 'master'
Working See merge request
!130
parents
1e591b17
8ae432b8
Show whitespace changes
Inline
Side-by-side
Showing
5 changed files
with
458 additions
and
20 deletions
+458
-20
README.md
+1
-1
src/main/java/com/zhiwei/searchhotcrawler/crawler/WeiboHotSearchCrawler.java
+103
-13
src/main/java/com/zhiwei/searchhotcrawler/dao/HotSearchCacheDAO.java
+1
-1
src/main/java/com/zhiwei/searchhotcrawler/util/AESUtils.java
+88
-5
src/test/java/weiboTest/WeiboTopInfoTest.java
+265
-0
No files found.
README.md
View file @
d431fc6a
...
...
@@ -50,7 +50,7 @@ redis.host = 192.168.0.39
redis.port = 6379
redis.database = 1
重启
2
重启
3
...
...
src/main/java/com/zhiwei/searchhotcrawler/crawler/WeiboHotSearchCrawler.java
View file @
d431fc6a
...
...
@@ -6,6 +6,8 @@ import java.net.URLEncoder;
import
java.text.ParseException
;
import
java.text.SimpleDateFormat
;
import
java.util.*
;
import
java.util.regex.Matcher
;
import
java.util.regex.Pattern
;
import
java.util.stream.Collectors
;
import
com.alibaba.fastjson.JSON
;
...
...
@@ -130,7 +132,8 @@ public class WeiboHotSearchCrawler {
* @Description: TODO(手机端Iphone 微博热搜采集)
*/
public
static
List
<
HotSearchList
>
weiboHotSearchByPhone
(
Date
date
)
{
String
url
=
"https://m.weibo.cn/api/container/getIndex?containerid=106003type%3D25%26t%3D3%26disable_hot%3D1%26filter_type%3Drealtimehot&title=%E5%BE%AE%E5%8D%9A%E7%83%AD%E6%90%9C&extparam=pos%3D0_0%26mi_cid%3D100103%26cate%3D10103%26filter_type%3Drealtimehot%26c_type%3D30&luicode=10000011&lfid=231583"
;
//String url = "https://m.weibo.cn/api/container/getIndex?containerid=106003type%3D25%26t%3D3%26disable_hot%3D1%26filter_type%3Drealtimehot&title=%E5%BE%AE%E5%8D%9A%E7%83%AD%E6%90%9C&extparam=pos%3D0_0%26mi_cid%3D100103%26cate%3D10103%26filter_type%3Drealtimehot%26c_type%3D30&luicode=10000011&lfid=231583";
String
url
=
"https://api.weibo.cn/2/guest/page?c=android&s=3d477777&from=10A8395010&gsid=_2AkMoFNQvf8NhqwJRm_gWy2rkbo1_yA7EieKeSCX0JRM3HRl-wT9kqkIltRV6A-gElEGNj31RgrfclQ31YPAf7UBZPBx2&containerid=106003type%3D25%26t%3D3%26disable_hot%3D1%26filter_type%3Drealtimehot"
;
Map
<
String
,
String
>
headerMap
=
new
HashMap
<>();
headerMap
.
put
(
"User-Agent"
,
"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36"
);
String
htmlBody
=
null
;
...
...
@@ -144,12 +147,11 @@ public class WeiboHotSearchCrawler {
List
<
HotSearchList
>
result
=
new
ArrayList
<
HotSearchList
>();
if
(
StringUtils
.
isNotBlank
(
htmlBody
)
&&
htmlBody
.
contains
(
"cards"
))
{
try
{
JSONObject
json
=
JSONObject
.
parseObject
(
htmlBody
).
getJSONObject
(
"data"
);
JSONArray
cards
=
json
.
getJSONArray
(
"cards"
);
JSONArray
json
=
JSONObject
.
parseObject
(
htmlBody
).
getJSONArray
(
"cards"
);
int
rank
=
0
;
// for (int i = 0; i < cards.size(); i++) {
try
{
JSONObject
card
=
cards
.
getJSONObject
(
0
);
JSONObject
card
=
json
.
getJSONObject
(
0
);
JSONArray
cardGroup
=
card
.
getJSONArray
(
"card_group"
);
JSONObject
topCard
=
cardGroup
.
getJSONObject
(
0
);
if
(!
topCard
.
containsKey
(
"pic"
))
{
...
...
@@ -168,15 +170,13 @@ public class WeiboHotSearchCrawler {
String
desc_extr
=
cardInfo
.
getString
(
"desc_extr"
);
String
heatLabel
=
null
;
Long
hotCount
=
null
;
if
(
Objects
.
nonNull
(
desc_extr
)){
String
[]
split
=
desc_extr
.
split
(
" "
);
if
(
split
.
length
>
1
){
heatLabel
=
split
[
0
].
trim
();
hotCount
=
Long
.
valueOf
(
split
[
1
].
trim
());
}
else
{
hotCount
=
cardInfo
.
getLongValue
(
"desc_extr"
);
}
if
(!
StringUtils
.
isEmpty
(
desc_extr
)&&
Objects
.
nonNull
(
desc_extr
)){
String
regEx
=
"[^0-9]"
;
Pattern
p
=
Pattern
.
compile
(
regEx
);
Matcher
m
=
p
.
matcher
(
desc_extr
);
String
num
=
m
.
replaceAll
(
""
).
trim
();
hotCount
=
Long
.
valueOf
(
num
);
heatLabel
=
desc_extr
.
split
(
" "
)[
0
];
}
String
iconUrl
=
cardInfo
.
getString
(
"icon"
);
String
icon
=
null
;
...
...
@@ -212,6 +212,96 @@ public class WeiboHotSearchCrawler {
return
Collections
.
emptyList
();
}
// /**
// * @return void 返回类型
// * @Title: weiboHotSearchByPhoneTest
// * @author hero
// * @Description: TODO(手机端Iphone 微博热搜采集)
// */
// public static List<HotSearchList> weiboHotSearchByPhone(Date date) {
// //String url = "https://m.weibo.cn/api/container/getIndex?containerid=106003type%3D25%26t%3D3%26disable_hot%3D1%26filter_type%3Drealtimehot&title=%E5%BE%AE%E5%8D%9A%E7%83%AD%E6%90%9C&extparam=pos%3D0_0%26mi_cid%3D100103%26cate%3D10103%26filter_type%3Drealtimehot%26c_type%3D30&luicode=10000011&lfid=231583";
// String url = "https://api.weibo.cn/2/guest/page?c=android&s=3d477777&from=10A8395010&gsid=_2AkMoFNQvf8NhqwJRm_gWy2rkbo1_yA7EieKeSCX0JRM3HRl-wT9kqkIltRV6A-gElEGNj31RgrfclQ31YPAf7UBZPBx2&containerid=106003type%3D25%26t%3D3%26disable_hot%3D1%26filter_type%3Drealtimehot";
// Map<String, String> headerMap = new HashMap<>();
// headerMap.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36");
// String htmlBody = null;
// Request request = RequestUtils.wrapGet(url, headerMap);
// for (int count = 0; count <= 5; count++) {
// try (Response response = httpBoot.syncCall(request, ProxyHolder.NAT_HEAVY_PROXY)) {
// htmlBody = response.body().string();
// } catch (IOException e) {
// log.error("解析微博时热搜时出现连接失败", e);
// }
// List<HotSearchList> result = new ArrayList<HotSearchList>();
// if (StringUtils.isNotBlank(htmlBody) && htmlBody.contains("cards")) {
// try {
// JSONObject json = JSONObject.parseObject(htmlBody).getJSONObject("data");
// JSONArray cards = json.getJSONArray("cards");
// int rank = 0;
//// for (int i = 0; i < cards.size(); i++) {
// try {
// JSONObject card = cards.getJSONObject(0);
// JSONArray cardGroup = card.getJSONArray("card_group");
// JSONObject topCard = cardGroup.getJSONObject(0);
// if (!topCard.containsKey("pic")) {
// rank = 1;
// }
// if (Objects.nonNull(cardGroup) && !cardGroup.isEmpty()) {
//// String title = card.getString("title");
// boolean hot = true;
//// if (Objects.nonNull(title) && title.contains("实时上升热点")) {
//// hot = false;
//// rank = 51;
//// }
// for (int j = 0; j < cardGroup.size(); j++) {
// JSONObject cardInfo = cardGroup.getJSONObject(j);
// String name = cardInfo.getString("desc");
// String desc_extr = cardInfo.getString("desc_extr");
// String heatLabel=null;
// Long hotCount =null;
// if (Objects.nonNull(desc_extr)){
// String[] split = desc_extr.split(" ");
// if (split.length>1){
// heatLabel= split[0].trim();
// hotCount= Long.valueOf(split[1].trim());
//
// }else {
// hotCount = cardInfo.getLongValue("desc_extr");
// }
// }
// String iconUrl = cardInfo.getString("icon");
// String icon=null;
// if (StringUtils.isNotBlank(iconUrl)) {
// icon = iconUrl.split("_")[1].split(".png")[0];
// }
//// String id = "http://s.weibo.com/weibo/" + URLCodeUtil.getURLEncode(name, "utf-8") + "&Refer=top";
// String id = cardInfo.getString("scheme");
// HotSearchList hotSearch = new HotSearchList(id, name, hotCount, hot, rank, HotSearchType.微博热搜.name(), icon, date);
// hotSearch.setHeatLabel(heatLabel);
// if (Objects.nonNull(iconUrl)){hotSearch.setIconUrl(iconUrl);}
// result.add(hotSearch);
// rank++;
// redisDao.addDataToSet(RedisConfig.WEIBO_HOTSEARCHIDS, name + "_微博热搜");
// }
// } else {
// log.info("card 数据结构为:{}", card);
// }
// } catch (Exception e) {
// log.error("解析微博时热搜时出现解析错误", e);
// continue;
// }
//// }
//
// return result;
// } catch (Exception e) {
// log.error("解析微博时热搜时出现解析错误,数据不是json结构", e);
// }
// } else {
// log.info("解析微博时热搜时出现解析错误,页面结构有问题");
// }
// }
// return Collections.emptyList();
// }
/**
* 微博预热榜(实时上升热点采集)
*
...
...
src/main/java/com/zhiwei/searchhotcrawler/dao/HotSearchCacheDAO.java
View file @
d431fc6a
...
...
@@ -218,7 +218,7 @@ public class HotSearchCacheDAO {
nowDoc
.
put
(
"pictureUrl"
,
pictureUrl
);
}
if
(
"微博热搜"
.
equals
(
type
)){
nowDoc
=
WeiboHotSearchCrawler
.
weiboUpdate
(
nowDoc
);
//
nowDoc = WeiboHotSearchCrawler.weiboUpdate(nowDoc);
//更新微博话题贡献者,关于功能
Document
documentPC
=
WeiboHotSearchCrawler
.
weiboUpdatePC
(
nowDoc
);
if
(
documentPC
.
containsKey
(
"分类"
))
{
...
...
src/main/java/com/zhiwei/searchhotcrawler/util/AESUtils.java
View file @
d431fc6a
package
com
.
zhiwei
.
searchhotcrawler
.
util
;
import
org.apache.commons.codec.binary.Base64
;
import
org.apache.commons.lang3.StringUtils
;
import
sun.misc.BASE64Decoder
;
import
java.io.UnsupportedEncodingException
;
import
java.math.BigInteger
;
import
java.security.SecureRandom
;
import
java.util.Arrays
;
import
java.util.Objects
;
import
javax.crypto.Cipher
;
import
javax.crypto.KeyGenerator
;
import
javax.crypto.spec.IvParameterSpec
;
import
javax.crypto.spec.SecretKeySpec
;
import
static
java
.
util
.
Objects
.
isNull
;
/**
* @author
* @version V1.0
...
...
@@ -14,6 +24,9 @@ import javax.crypto.spec.SecretKeySpec;
* @date 2017-12-28 14:26
**/
public
class
AESUtils
{
//默认偏移
public
static
final
String
VI_STR
=
"0102030405060708"
;
private
static
final
String
ALGORITHMSTR
=
"AES/ECB/PKCS5Padding"
;
private
AESUtils
()
{
}
...
...
@@ -25,11 +38,19 @@ public class AESUtils {
* @return 加密后的字符串
*/
public
static
String
encrypt
(
String
secret
,
String
value
)
{
return
encrypt
(
secret
,
value
,
VI_STR
);
}
public
static
String
encrypt
(
String
secret
,
String
value
,
String
ivStr
)
{
SecretKeySpec
keySpec
=
getKey
(
secret
);
IvParameterSpec
iv
=
new
IvParameterSpec
(
"0102030405060708"
.
getBytes
());
try
{
Cipher
cipher
=
Cipher
.
getInstance
(
"AES/CBC/PKCS5Padding"
);
Cipher
cipher
=
Cipher
.
getInstance
(
ALGORITHMSTR
);
if
(
isNull
(
ivStr
)){
cipher
.
init
(
Cipher
.
ENCRYPT_MODE
,
keySpec
);
}
else
{
IvParameterSpec
iv
=
new
IvParameterSpec
(
ivStr
.
getBytes
());
cipher
.
init
(
Cipher
.
ENCRYPT_MODE
,
keySpec
,
iv
);
}
byte
[]
encrypted
=
cipher
.
doFinal
(
value
.
getBytes
(
"UTF-8"
));
return
parseByte2HexStr
(
encrypted
);
}
catch
(
Exception
e
)
{
...
...
@@ -37,6 +58,24 @@ public class AESUtils {
}
}
public
static
byte
[]
encrypt
(
byte
[]
secretBytes
,
byte
[]
valueBytes
,
String
ivStr
)
{
SecretKeySpec
keySpec
=
new
SecretKeySpec
(
Arrays
.
copyOf
(
secretBytes
,
16
),
"AES"
);
try
{
Cipher
cipher
=
Cipher
.
getInstance
(
ALGORITHMSTR
);
if
(
isNull
(
ivStr
)){
cipher
.
init
(
Cipher
.
ENCRYPT_MODE
,
keySpec
);
}
else
{
IvParameterSpec
iv
=
new
IvParameterSpec
(
ivStr
.
getBytes
());
cipher
.
init
(
Cipher
.
ENCRYPT_MODE
,
keySpec
,
iv
);
}
byte
[]
encrypted
=
cipher
.
doFinal
(
valueBytes
);
return
encrypted
;
// return parseByte2HexStr(encrypted);
}
catch
(
Exception
e
)
{
throw
new
RuntimeException
(
e
);
}
}
public
static
void
main
(
String
[]
args
)
{
String
jsm
=
AESUtils
.
encrypt
(
"wechat"
,
"shenjinzhu"
);
System
.
out
.
println
(
jsm
);
...
...
@@ -52,11 +91,27 @@ public class AESUtils {
* @return 解密后的字符串
*/
public
static
String
decrypt
(
String
secret
,
String
value
)
{
return
decrypt
(
secret
,
value
,
VI_STR
);
}
/**
* 解密
*
* @param secret 密钥
* @param value 待解密字符串
* @param ivStr 偏移字符
* @return 解密后的字符串
*/
public
static
String
decrypt
(
String
secret
,
String
value
,
String
ivStr
)
{
SecretKeySpec
keySpec
=
getKey
(
secret
);
IvParameterSpec
iv
=
new
IvParameterSpec
(
"0102030405060708"
.
getBytes
());
try
{
Cipher
cipher
=
Cipher
.
getInstance
(
"AES/CBC/PKCS5Padding"
);
Cipher
cipher
=
Cipher
.
getInstance
(
ALGORITHMSTR
);
if
(
isNull
(
ivStr
)){
cipher
.
init
(
Cipher
.
DECRYPT_MODE
,
keySpec
);
}
else
{
IvParameterSpec
iv
=
new
IvParameterSpec
(
ivStr
.
getBytes
());
cipher
.
init
(
Cipher
.
DECRYPT_MODE
,
keySpec
,
iv
);
}
byte
[]
encrypted1
=
parseHexStr2Byte
(
value
);
byte
[]
original
=
cipher
.
doFinal
(
encrypted1
);
return
new
String
(
original
,
"UTF-8"
);
...
...
@@ -66,6 +121,32 @@ public class AESUtils {
}
/**
* 解密
*
* @param secretBytes 密钥组
* @param valueBytes 待解密字符组
* @param ivStr 偏移字符
* @return 解密后的字符串
*/
public
static
String
decrypt
(
byte
[]
secretBytes
,
byte
[]
valueBytes
,
String
ivStr
)
{
SecretKeySpec
keySpec
=
new
SecretKeySpec
(
Arrays
.
copyOf
(
secretBytes
,
16
),
"AES"
);
try
{
Cipher
cipher
=
Cipher
.
getInstance
(
ALGORITHMSTR
);
if
(
isNull
(
ivStr
)){
cipher
.
init
(
Cipher
.
DECRYPT_MODE
,
keySpec
);
}
else
{
IvParameterSpec
iv
=
new
IvParameterSpec
(
ivStr
.
getBytes
());
cipher
.
init
(
Cipher
.
DECRYPT_MODE
,
keySpec
,
iv
);
}
// byte[] encrypted1 = parseHexStr2Byte(value);
byte
[]
original
=
cipher
.
doFinal
(
valueBytes
);
return
new
String
(
original
,
"UTF-8"
);
}
catch
(
Exception
e
)
{
throw
new
RuntimeException
(
e
);
}
}
/**
* 生成加密的密钥,保证长度为16位
*
* @param secret 用户的密钥
...
...
@@ -75,6 +156,7 @@ public class AESUtils {
byte
[]
bytes
;
try
{
bytes
=
secret
.
getBytes
(
"UTF-8"
);
// return new SecretKeySpec(Arrays.copyOf(bytes, 32), "AES");
return
new
SecretKeySpec
(
Arrays
.
copyOf
(
bytes
,
16
),
"AES"
);
}
catch
(
UnsupportedEncodingException
e
)
{
e
.
printStackTrace
();
...
...
@@ -107,8 +189,9 @@ public class AESUtils {
* @return
*/
public
static
byte
[]
parseHexStr2Byte
(
String
hexStr
)
{
if
(
hexStr
.
length
()
<
1
)
if
(
hexStr
.
length
()
<
1
)
{
return
null
;
}
byte
[]
result
=
new
byte
[
hexStr
.
length
()
/
2
];
for
(
int
i
=
0
;
i
<
hexStr
.
length
()
/
2
;
i
++)
{
int
high
=
Integer
.
parseInt
(
hexStr
.
substring
(
i
*
2
,
i
*
2
+
1
),
16
);
...
...
src/test/java/weiboTest/WeiboTopInfoTest.java
0 → 100644
View file @
d431fc6a
package
weiboTest
;
import
com.alibaba.fastjson.JSONArray
;
import
com.zhiwei.crawler.core.HttpBoot
;
import
com.zhiwei.crawler.core.proxy.ProxyHolder
;
import
com.zhiwei.crawler.core.utils.RequestUtils
;
import
com.zhiwei.searchhotcrawler.util.AESUtils
;
import
com.zhiwei.searchhotcrawler.util.HttpClientUtils
;
import
okhttp3.Request
;
import
okhttp3.Response
;
import
org.apache.commons.codec.DecoderException
;
import
org.apache.commons.codec.binary.Hex
;
import
org.junit.Test
;
import
sun.misc.BASE64Decoder
;
import
javax.crypto.Cipher
;
import
javax.crypto.spec.IvParameterSpec
;
import
javax.crypto.spec.SecretKeySpec
;
import
java.beans.Encoder
;
import
java.io.IOException
;
import
java.io.UnsupportedEncodingException
;
import
java.net.URLDecoder
;
import
java.net.URLEncoder
;
import
java.nio.charset.Charset
;
import
java.nio.charset.StandardCharsets
;
import
java.util.Base64
;
import
java.util.HashMap
;
import
java.util.Map
;
/**
* @author cwt
* @date 2021/9/16 10:11
*/
public
class
WeiboTopInfoTest
{
private
static
HttpBoot
httpBoot
=
new
HttpBoot
.
Builder
().
retryTimes
(
3
).
build
();
/**
* 加密测试
* @author cwt
* @date 2021/9/16 16:51
* @return void
**/
@Test
public
void
encodetest
(){
String
str
=
"496761"
;
//秘钥
String
secret
=
"a1aa63b395e053e0c3c67cf1366284a4"
;
try
{
byte
[]
secretK
=
Hex
.
decodeHex
(
secret
.
toCharArray
());
byte
[]
resultBytes
=
AESUtils
.
encrypt
(
secretK
,
str
.
getBytes
(
StandardCharsets
.
UTF_8
),
null
);
byte
[]
result
=
Base64
.
getEncoder
().
encode
(
resultBytes
);
System
.
out
.
println
(
new
String
(
result
));
}
catch
(
Exception
e
)
{
e
.
printStackTrace
();
}
}
/**
* 解码测试
* @date 2021/9/16 16:50
* @return void
**/
@Test
public
void
decodeTest
(){
String
str
=
"GWlCSTFWf4l4yVlL/f45nA=="
;
byte
[]
strBytes
=
Base64
.
getDecoder
().
decode
(
str
);
//秘钥
String
secret
=
"a1aa63b395e053e0c3c67cf1366284a4"
;
try
{
byte
[]
secretK
=
Hex
.
decodeHex
(
secret
.
toCharArray
());
System
.
out
.
println
(
AESUtils
.
decrypt
(
secretK
,
strBytes
,
null
));
}
catch
(
Exception
e
)
{
e
.
printStackTrace
();
}
}
/**
* 数据id获取测试
* @author cwt
* @date 2021/9/16 17:11
* @return void
**/
@Test
public
void
timeIdGetTest
(){
String
key
=
"a1aa63b395e053e0c3c67cf1366284a4"
;
String
timestamp
=
"t5y+6whLpyGvG8Rd1p16jkWdy40+5AzxLy/lY1ik4ho="
;
//时间解码测试
System
.
out
.
println
(
decodeStr
(
key
,
timestamp
));
//时间格式为 yyyy-MM-ddTHH:mm:ss 2021-09-15T11:28:02
//获取指定时间
String
time
=
"2021-09-15T11:28:02"
;
String
encodeTime
=
encodeStr
(
key
,
time
);
//地址参数转码
try
{
encodeTime
=
URLEncoder
.
encode
(
encodeTime
,
"UTF-8"
);
}
catch
(
UnsupportedEncodingException
e
)
{
e
.
printStackTrace
();
}
String
url
=
"https://api.weibotop.cn/getclosesttime?timestamp="
+
encodeTime
;
System
.
out
.
println
(
url
);
Request
request
=
RequestUtils
.
wrapGet
(
url
,
getHeaderMap
());
//测试使用空代理
try
(
Response
response
=
httpBoot
.
syncCall
(
request
,
ProxyHolder
.
NONE_PROXY
))
{
System
.
out
.
println
(
response
.
body
().
string
());
}
catch
(
IOException
e
)
{
e
.
printStackTrace
();
}
}
/**
* 指定时间id获取数据
* @date 2021/9/16 17:10
* @return void
**/
@Test
public
void
currentitemsHttpTest
(){
String
key
=
"a1aa63b395e053e0c3c67cf1366284a4"
;
String
timeId
=
"496115"
;
String
encodeTimeId
=
encodeStr
(
key
,
timeId
);
//地址参数转码
try
{
encodeTimeId
=
URLEncoder
.
encode
(
encodeTimeId
,
"UTF-8"
);
}
catch
(
UnsupportedEncodingException
e
)
{
e
.
printStackTrace
();
}
String
url
=
"https://api.weibotop.cn/currentitems?timeid="
+
encodeTimeId
;
System
.
out
.
println
(
url
);
Request
request
=
RequestUtils
.
wrapGet
(
url
,
getHeaderMap
());
//测试使用空代理
try
(
Response
response
=
httpBoot
.
syncCall
(
request
,
ProxyHolder
.
NONE_PROXY
))
{
String
result
=
response
.
body
().
string
();
//结果解密
String
decodeResult
=
decodeStr
(
key
,
result
);
System
.
out
.
println
(
decodeResult
);
JSONArray
jsonArray
=
JSONArray
.
parseArray
(
decodeResult
);
for
(
Object
o
:
jsonArray
)
{
System
.
out
.
println
(
o
);
}
}
catch
(
IOException
e
)
{
e
.
printStackTrace
();
}
}
/**
* 封装个简单的请求头
* @author cwt
* @date 2021/9/16 17:36
* @return java.util.Map<java.lang.String,java.lang.String>
**/
private
Map
<
String
,
String
>
getHeaderMap
(){
Map
<
String
,
String
>
headerMap
=
new
HashMap
<>();
headerMap
.
put
(
"Accept"
,
"application/json, text/javascript, */*; q=0.01"
);
headerMap
.
put
(
"Accept-Encoding"
,
"gzip, deflate, br"
);
headerMap
.
put
(
"Accept-Language"
,
"zh-CN,zh;q=0.9"
);
headerMap
.
put
(
"Connection"
,
"keep-alive"
);
headerMap
.
put
(
"Host"
,
"api.weibotop.cn"
);
headerMap
.
put
(
"Origin"
,
"https://www.weibotop.cn"
);
headerMap
.
put
(
"Referer"
,
"https://www.weibotop.cn/"
);
headerMap
.
put
(
"sec-ch-ua"
,
"\"Google Chrome\";v=\"93\", \" Not;A Brand\";v=\"99\", \"Chromium\";v=\"93\""
);
headerMap
.
put
(
"User-Agent"
,
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.63 Safari/537.36"
);
return
headerMap
;
}
/**
* 数据解码测试
* @date 2021/9/16 17:06
* @return void
**/
@Test
public
void
currentitemsDataTest
(){
String
key
=
"a1aa63b395e053e0c3c67cf1366284a4"
;
String
str
=
"hV6ghSBb/p2wj7m+eUO49gOSs61I40xZbW7B3p/ck5qW+HrbmbkpXb1R2CLNvfaUWvmnuXwXyFn3kqn+hQp6yGNH6tYrT7noC6SbeQTTXZTURTqwZl/FEjFOJUNqXh9Mbb5T83NF1BEabPKjEvObnpl+f6kPCuuVG3Q517jT+h8eDpqACeHtkDrfoFQ6BTegLzaoUeyDEhZ34M88a0XC35mpo3BPrBNCIklU5dI3u5e1zoxBgyiPhHqL3OEBre9zJ7o42liporFkLQ+R3c77qs5CTMsGR4arEBRhjBTr8uqJZHnWf5i0VWoD25GaJApclYaiw/ZS2HVYCve5ctpsPY7f0aebVP+zmQuhsFpovl20DCym8sVSKd8eXgUxpBNxHnYSzDEPzbx5V+VLpSQPijAA94RlxoQzjCff43Ols6k7ZEKaG/7uZSxw7KnCkHfRqiG15PDXlr+1mxQPZW0g29+1PNquQdmr9Z0rhdu9SNmdiQl2bwSclKIt9hEe+UCUcP62M0zCdqmGnRYyqcu084IZ2aacZVQmxFKw/D2U9Q+SinM5J25WEgv/a7O+2S38rjHize2JxizEe+YUuTUfqaHnnx20a5IFD1PbFxp83vmsHNDNQXK/x24mEQhpxoBv9nJdlHQS/8kZO1jJ4mmRCbUx60N0TBoGHEk2I/uySGxjhd6LYXS5CP77AnWmEsXneH9taIKpB0w+SSkRv/DU6eEXXPh30Q5z1fDeQ2tXbEC9tNhZI3XBaRRfLyU60Cs3v+GDPJ9R8C6q3Ab3w8SD5JVy8swA8/UQNLw/4vUUEHTo0xtRa40buY3t86WMQHzZU8HHFz6UWpvY5r3Wsj3t4rrO+lnblCmCFbRJXK2xWdmdSKRwp8mfJ9Cn4w/RRJ9zErse00jB1HsLwJrKx4+BYVp0KhCUAek7ZN+pOVs62Fftio4/U4kJ/at/I+5A/qPXfb48ZQ6uN8R7lRxOqfaDdc3nHFeS/fPiOOvj5ottuaDSVf9BsLDGpzKh0cKGjVBO/qhcRTfPSq52vTWIF5bDWG9zVObbnaML9Px7203sVU3cVZoelFPXOBjyD1vJtrLAiWR51n+YtFVqA9uRmiQKXJWGosP2Uth1WAr3uXLabD20XysVwPY9MzrOC4sSllyltD+6rF4g6U+DRWOSS28no+ZXYpjmrGhQ7pkcQ6tvajdDyJ3GUeh+VxAUcgiqusYcI+tdvLDtMQs2Bpds3QYhb3dKgcbLspUYOA3LA1wMRtYu+PK44d/XTvOkJG6O4xi8QuPwhdHGU5PgB9/bZ12mk22iSk1I6BE6EByneulMAPq+c4z4E8PK0KmIhVMuEu3n+1HrDgpg70L0TBrWWsJ/eFlkoSv1prDrd9zmYLAcGJqFUXn4n1raP7XEvooha82eTDvlompsOt2TYsdFLFs2A5JVPerJFL3jw7+hQ82UjQV+wiAkfTSH7vFoZ45JkUTXg8Q1HLxFqAmOOUZh1As0I/dYRGjMQ5kxGCrvRI47UK0B6/kMLQo3katMX+AvG7VWVo/r/ZfmewG3BPDJqwlipXNSoY315St1ArW9xq7QkDV6reJi7WvCRFvJnawXiXoEvYMnzcVGd3LsNLIXCvISrFlkoSv1prDrd9zmYLAcGJraAhwS9Q4/vN5Y3Xeoe+JQsWkytx6p3etQ53rf0cEoy+wpUSRa0nshwAJpkV0/6UIOG2/IxbodBELndSpHehvm5GDL2UQEBI1IWY7/JxwJzAF7Jfz6Is1FdKjXPDsvVtdj7hAcqV2JnJKurow7USd26BLOCUYhZzD74kZi7fVZRv6aNXYv/IxCVBl3RYjAbj4Lkcej2YKtUNTx37iO/nB1cb5PKgxJw+l+Z6HFQgDrvFkGCvysZjbAFumWv95MCLmr3/NsKLV7hPgqu5lCGrIfCTA0+U+8SyD9K4FSPSHNJDhkVR4do0+Jo0hdplNC+VEOEFzTwWHzdzKgEdN8jRcV1r63nsJdWRwElrOGBROU6LKirBEb1wnuBmYeARkWgG2T7Qy3pX6smrBoDLyvy7aX+uc13VY5lV9vXQOCF7sPDyzJG83unepuvDiCQlmKv2qWp0Q2J+uePTRjEd164LypTqcJgYVENdh9y9dQRRo8DV9J4knuKR1Nj9nxtzTPtYyslOn8ZQFATeFcS5SL+mg1un58cEsA0i6Mx5oXO3zZ0KXlz5gT7BRv61373woebQnHOCpgUj08Tfb3Z9+GhRFkn/nKkKwbMGopp0oUd0z8ARL+B+uClbYxCyYSaLzK4A8PqwsqJ4dGTZBJV5/nvD8wq8EzcRccWqd3P65LfipkVWl0iMpZAY8xo2x/pPhPNE6fCf1sQf7OdvrJwD+ikGGICpB2ZNzS0IJM7MOSQnflJo7e/7f2A+qU7Q6QeYyzqyFcS3FHgi9GERdQhNzmgEvGvWeWJ5bBiuK0L76rTa1g5uwLGhh3ZXQ2x9qGp4BW/NLoarFWWZJ4hZUiyUlJKDAhs0E5x5tFspWBtAtbAhImh8/mVTZZdAoM0tsEyL/0Gl7ptfFvFJlBBbFN+k9Nul94FAgkPNHB1AfklHezxqqxZVx2R3Mhy4gzOjoPTMODRbpK9M8EZY1n9iJX2siGgmdpah3/bQrWcOOkWV0cPaFmjG/ZBEuhvtASQuHz+IgS0IOVsyWDpNO/IYBMsNYtgLludbbcJ/s+K+qwhWu3aWkt7/fxWJCOrqIM9Uydpmg3IF7ZU7spMman0lHDZu03hXjkqarDAxcqq5ddam/Yfw7inC4JPtybMiVp4r6yHDSksEEPrU+Hg3eoJoZLevGtoqojOt/pQmJRk5YtSazArr3N4vkYTExD+DmhbgeTE3j5/PKesT9uoenlxzkzDx264jH4fMO/Tc3G42kwyyrWYu0ElN+BXYAZ8Bq50i3QPRKCN9YY+NFLoDVXqeAtf1ojfYvvqg3LQjHjIB71r0C22Dl9QlJmlgL9RRmn/fiNNEOW9/BXtLBnO1MHLVuZhb+CgVhjjEiVO8LKct0KLeQnTOv1hyGWTK0NspieaDBx7JYwMARXpOs/4M0YodZWmDqNGL+TOtTFhlQkbZDvPTtOsXmmNt1DnPXhes56MGSJRnMejg5+aSNvz0ug9GsiAKJFqMpxw6xhXUK9jSX+8SgazCM/HoVYeCbS5znuPTfH+b8Xm+U8X0h6oTNgwoKGP41KVtoJSTTSskO9n26/HVcuS1Vf1YL4l3LPYzRVeYdyvcNgY8Z2M5F3TG3CT3sqJmLhA/2QqbuQAJsHh/DHxeH5e/7AKUaUXY9yYqUtvKQKUmHTx0Ns6k1QP7wILILv5FdILifl6pDhw1zEFmklB/1pO9MHk9WEAPCwQwyiaBO1ihYW2q/KDBzudDqgzA8E7hwAzeXXY7g4NNMJcwjH/tQueTpy4U58pHNhbN290Zq7irwUkcxYdf5qCE5fwgdoXj2vnF+apiU7YYIz4CJScRoDE3AuE2OtR/bswE2rp71TgPFUvmMtZaP7ZoitcpQtbRqq+6xNzkMndTCwk108THKbvuJXbPcsDLqKt55WWjPucdzMjQ9Afr2ivsq414fyo1C7Gd83gMPeDAt1m4FqT3gHN6YFosVbqN44uQ5LFzD0n1hvSoSMA7cexu5aKrUZEhAMO2u9Pl7bwwC+zsT+b8B2Q5Whe8jymBdne43kLBGhjNlZqJD1MYu466DJm6Z3D5UeI1mWNfHyRt3LyUKHRivA1t8cHFjBKZHglRN93cNzYipml001VbJS6YgYl0SkPqQOeqSkRjZSWmelhKnKEiZd2gF0qB+5hY4/Oiyy7+YhG703Hz2L5kZL/ppGWWY4+x6EfauIKIHpDPA+lHhDaB5NfxB53oUhMfdEGRikwjNZs8WBtqSlwyb9F7zR5BxTs2pQYC1gwtgZiHGPFNdsvY67ePPWGogNQx5/hLqLoCSuDwR7hny/YzE3VkhQGoNsX3vHvv9IS9NfNequCHBnKUCyvNuaaIndBA/qJrNas4lUPVMuDTBZZKEr9aaw63fc5mCwHBiatzyplQtLo2aixKDzNhMpWVuJupoOEmIEZ0MgcBsO7YFbSEj4/IShO9+tI4o5l0GXjuK5VR9xqKYm6U4sTzSZFiYMcKVAG5m1s2i2vIASmTAvNqhR7IMSFnfgzzxrRcLfzXFnX3cpvMwA2DqtKzazU8vLLUJaCEAL/MMVeiyV//dBJkT821R9TV1Wmm1996qdePt8k86+6HeMUNB0JdS/E5WzJYOk078hgEyw1i2AuW5GO2Dzs2D8OVrF+t8KxI+Mp7Q96LMUuPab+tfCO87dlkzdO1Jg2bGZVfZeR9M3QjrYYKqQwC1ftZ5K9xJ+PSnqGKMtgeegWXl3bfCvs6fiepnbrLe3lfM9XOb89/qs6mPMcN/cxyxneUAMkJ73pu13YPBUNW64M8j4gq2+g5ngdSgC/A7fXmXk8mXhMyqqHu9qLgY/lVdaR+q9yB+dXLOcQfHTDxEWEB00b2ct54Kq4XUq6ZfiyXvg+9XGJIvLybWmVucYxQxmfJLhLFI/ZCTYFPX48il33/u4+kI/hE8YDeEXXPh30Q5z1fDeQ2tXbED81IG5XndQqWgvuJjS2Zm7t7gaHWdtoXI2rFKSEBKqVk5SeUex8wEDcALYfH6w+pUekofhF/P9fXIA0OeYPDlZyTmzpxALj3XC4mxe0vxy5gqvF+jaokF63PDRch+oyZt6GAlFoDZ5rX6FA/RQzt4XlMXUAXFbQ6UXECKPSbSyz0nxQpduY4IRFTtRUsT+hY9uGpxY/cT9LommUJOedN2yoI4jRCf3ZXNXgBLdwWXPVkckbq7XjWtJEqT0mDr7R95NxEXmbOs6H24Te75HUQ2Me0Z9GWZ64rJu10Vepk+wY9tJSXq2Xziz1XqKWl/aFcY6JnDoXmhcqHMrKq5raklXf3Z0f+2/qacU+8rm/Fd2C4wL6VXaON8/H4js4SiysAbirqUcNEJDecSQWIJnNQuPoE/SSrCU4aEHOhXD8MVbZwNIysu5CpekdktmgqifPK4YUIIL15BLf1AT+JHgPKHtySrDlYMd0zOP5IhObraIyGVQlvE7t8bq521TIbWhS+jfRkW1v3KmoimUUGCeqR7Cpz/k0Rkee+EOY+WmLCGG8rpUmSVBT4oSoYsnP26OQOYgdrxG6IXtufdxq0cakn8TrwjNU2jBaYuLZQerC/FKv+EXXPh30Q5z1fDeQ2tXbEA6sVpSXd7cZovB2bC7VWUHfawnaxjdWZGbDWNts68iHkAkjsC94hDXNL8RpFjIpwi6GboCMItNkYvm7kwttraYyTmzpxALj3XC4mxe0vxy5jfvFrsA1rxYAKmTk+8ROlcmCpf2Fa0wrqlPRgOO1xINjUbKHlg45ghucGV0/Lfn6v2trJSmz3duY9Cz/iwXKgqbRU7QR2l+BhDZOh0PP5zdE9M1hNlf/3ShyTZtCmLuqboYrNVl1r0/JcAUf5Q8tV4/QievsGVZ91ZsQgGG5sHWqJ32RWdRRtOhKcVvF0NjWqhbZ+DYmWfAal/V91QrA/49ktBzpbn8Z6FCb2aGURC1xVuo3ji5DksXMPSfWG9KhFo+UG9XYAeKTOMTU22V1PnlWSWrwAhAUnB9GEiBERXcjeC5sGNFoH8CxxoyTpfRbtgn7DrM5pZr9/PpSHPBLVHDgr7QO6/6kj/NtduE/XSQ6KEsbSDtxSBHWnjM0ToQVL9OnwmscqFzNtX4S/wjX5Y3OzC2SBRfINUQzuJaeXTj8TjnWYe1f5FEmMKBiRkjdKYszf7Dz6xYWd1ybLjI4Xy2gKD5ItUczGUn23xEMwe1WmV0Tou/tuglUOMAUoMur9v1TCmEUcjQD3jKZdqykBZdYc4dX19/qBktPZ1VDKLtaKBAFfiyQ8o5HVgC6ejvMT/5yBft5wTQTYA7Xd57gCEtGVWlt8XLIxsi3FB5d+bMMtjah5OWaW+ZIikGmJ1thGicOam2/s2R8hi9o/XblEllP/80J51UB2xuALxLjD9cFrqk/JnKtbI29UiSfbf5nFq+Y3iMx9Hp/KRufGgixkD2jSCEqNd1caeRQdCEF7xr6KEsbSDtxSBHWnjM0ToQVExKZH6xTFtdxib8LYYJ3yblHSHmjPYmCjdp3tMOL0wFa6VDpHCuKFwLbctKYdKQtYAvecchTOP6IWS14zhG2c1ZQDs3DsjCi3ojQkHDTLSAwduevatAHUAse4sW1qKR/XM/V0jygbF6lmEe73CjT8uTzdkSRTbLZStRLvyjuD1UX8SoTj2nZKSpLTQMrtS76t3nVZH6XpBE3wg2pAKlEofzLq841tFojIeM0vkpVKsf997MD0qoYZz381iYfAX1PG04EQSfFq+AZyENiWetpBF0/t/TQ5M83Dj3X4rn08Fo/Q9o2nB4RQPuY9gMR5FHHdCs2gJpMHeuWdZQGXRSG6Gejay2YVRPRYWUPe5Be6Eqbbfjmpz4dOkIUCRDBwtOzcN7BUDsrK1dNUv3cNwdJIW8R6qkT9VytPVMWf21SCqR"
;
System
.
out
.
println
(
decodeStr
(
key
,
str
));
}
/**
* 加密测试版
* @author cwt
* @param key
* @param str
* @date 2021/9/16 16:25
* @return java.lang.String
**/
private
String
encodeStr
(
String
key
,
String
str
){
try
{
byte
[]
secretK
=
Hex
.
decodeHex
(
key
.
toCharArray
());
byte
[]
resultBytes
=
AESUtils
.
encrypt
(
secretK
,
str
.
getBytes
(
StandardCharsets
.
UTF_8
),
null
);
byte
[]
result
=
Base64
.
getEncoder
().
encode
(
resultBytes
);
return
new
String
(
result
);
}
catch
(
Exception
e
)
{
e
.
printStackTrace
();
}
return
null
;
}
/**
* 解密测试版
* @author cwt
* @param key
* @param str
* @date 2021/9/16 16:25
* @return java.lang.String
**/
private
String
decodeStr
(
String
key
,
String
str
){
byte
[]
strBytes
=
Base64
.
getDecoder
().
decode
(
str
);
try
{
byte
[]
secretK
=
Hex
.
decodeHex
(
key
.
toCharArray
());
return
AESUtils
.
decrypt
(
secretK
,
strBytes
,
null
);
}
catch
(
Exception
e
)
{
e
.
printStackTrace
();
}
return
null
;
}
@Test
public
void
test64EncodeHex
(){
String
str
=
"MZRvb1+5duvsvk4mYPyrvQ=="
;
byte
[]
strBytes
=
Base64
.
getDecoder
().
decode
(
str
);
System
.
out
.
println
(
strBytes
.
length
);
System
.
out
.
println
(
new
String
(
Hex
.
encodeHex
(
strBytes
)));
}
/**
* 32位秘钥转16位字符key
* @author cwt
* @date 2021/9/16 16:43
* @return void
**/
@Test
public
void
test64DecodeHex
(){
String
key
=
"a1aa63b395e053e0c3c67cf1366284a4"
;
try
{
byte
[]
keyByte
=
Hex
.
decodeHex
(
key
.
toCharArray
());
System
.
out
.
println
(
keyByte
.
length
);
byte
[]
result
=
Base64
.
getEncoder
().
encode
(
keyByte
);
System
.
out
.
println
(
new
String
(
result
));
}
catch
(
DecoderException
e
)
{
e
.
printStackTrace
();
}
}
}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment