Skip to content

T
toutiao
Commit 061bc022 by zhiwei
Options

提交新的js加密参数

parent 048700cc
Showing with 17 additions and 11 deletions
src/main/java/com/zhiwei/toutiao/parse/TouTiaoArticleParse.java
...@@ -117,8 +117,8 @@ public class TouTiaoArticleParse { ...@@ -117,8 +117,8 @@ public class TouTiaoArticleParse {
/** /**
* 获取今日头条历史文章接口新 * 获取今日头条历史文章接口新
* *
* @param user_id * @param userId
* @param max_behot_time * @param maxBehotTime
* @param endData * @param endData
* @param proxy * @param proxy
* @return * @return
...@@ -353,12 +353,15 @@ public class TouTiaoArticleParse { ...@@ -353,12 +353,15 @@ public class TouTiaoArticleParse {
*/ */
public static Map<String, Object> getMicroTouTiaoCrawler(String userId, Date endDate, Proxy proxy, public static Map<String, Object> getMicroTouTiaoCrawler(String userId, Date endDate, Proxy proxy,
String maxBehotTime) throws IOException { String maxBehotTime) throws IOException {
String url = "https://www.toutiao.com/api/pc/feed/?category=pc_profile_ugc&utm_source=toutiao&visit_user_id=" + userId; Signature signature = new Signature(userId, maxBehotTime);
if (maxBehotTime != null) { String signatureStr = signature.getSignature();
url = url + "&max_behot_time=" + maxBehotTime; if (StringUtils.isBlank(maxBehotTime)) {
maxBehotTime = "0";
} }
String url = "https://www.toutiao.com/api/pc/feed/?category=pc_profile_ugc&utm_source=toutiao&visit_user_id=" + userId + "&max_behot_time=" + maxBehotTime+ "&_signature=" + signatureStr;
System.out.println(url); System.out.println(url);
Map<String, String> headerMap = Tools.getTouTiaoHeader(); Map<String, String> headerMap = Tools.getTouTiaoHeader();
headerMap.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36");
headerMap.put("Referer", "https://www.toutiao.com/c/user/" + userId + "/"); headerMap.put("Referer", "https://www.toutiao.com/c/user/" + userId + "/");
try { try {
String htmlBody = downloadHtml(url, proxy, headerMap); String htmlBody = downloadHtml(url, proxy, headerMap);
...@@ -379,10 +382,13 @@ public class TouTiaoArticleParse { ...@@ -379,10 +382,13 @@ public class TouTiaoArticleParse {
public static Map<String, Object> getMicroTouTiaoCrawler(String userId, Date endDate, ProxyHolder proxy, public static Map<String, Object> getMicroTouTiaoCrawler(String userId, Date endDate, ProxyHolder proxy,
Long maxBehotTime) throws IOException { Long maxBehotTime) throws IOException {
String url = "https://www.toutiao.com/api/pc/feed/?category=pc_profile_ugc&utm_source=toutiao&visit_user_id=" + userId; Signature signature = new Signature(String.valueOf(userId), String.valueOf(maxBehotTime));
if (maxBehotTime != null) { String signatureStr = signature.getSignature();
url = url + "&max_behot_time=" + maxBehotTime; if (Objects.isNull(maxBehotTime)) {
maxBehotTime = 0L;
} }
String url = "https://www.toutiao.com/api/pc/feed/?category=pc_profile_ugc&utm_source=toutiao&visit_user_id=" + userId + "&max_behot_time=" + maxBehotTime+ "&_signature=" + signatureStr;
System.out.println(url);
logger.info("微头条采集链接:::{}", url); logger.info("微头条采集链接:::{}", url);
Map<String, String> headerMap = Tools.getTouTiaoHeader(); Map<String, String> headerMap = Tools.getTouTiaoHeader();
headerMap.put("Referer", "https://www.toutiao.com/c/user/" + userId + "/"); headerMap.put("Referer", "https://www.toutiao.com/c/user/" + userId + "/");
...@@ -407,9 +413,8 @@ public class TouTiaoArticleParse { ...@@ -407,9 +413,8 @@ public class TouTiaoArticleParse {
* *
* @Description 微头条客户端解析 * @Description 微头条客户端解析
* @param userId * @param userId
* @param endDate
* @param proxy * @param proxy
* @param max_behot_time * @param maxBehotTime
* @return * @return
*/ */
public static List<Map<String,Object>> getClientMicroToutiaoCrawler(String userId, ProxyHolder proxy, public static List<Map<String,Object>> getClientMicroToutiaoCrawler(String userId, ProxyHolder proxy,
...@@ -487,6 +492,7 @@ public class TouTiaoArticleParse { ...@@ -487,6 +492,7 @@ public class TouTiaoArticleParse {
Long maxBehotTime = null; Long maxBehotTime = null;
List<TouTiaoArticle> dataList = new ArrayList<>(); List<TouTiaoArticle> dataList = new ArrayList<>();
try { try {
System.out.println("htmlBody====="+htmlBody);
JSONObject json = JSONObject.parseObject(htmlBody); JSONObject json = JSONObject.parseObject(htmlBody);
boolean more = false; boolean more = false;
if(json.containsKey("has_more")) { if(json.containsKey("has_more")) {
...@@ -726,7 +732,7 @@ public class TouTiaoArticleParse { ...@@ -726,7 +732,7 @@ public class TouTiaoArticleParse {
if(proxy != null) { if(proxy != null) {
response = httpBoot.syncCall(RequestUtils.wrapGet(url,headMap),proxy); response = httpBoot.syncCall(RequestUtils.wrapGet(url,headMap),proxy);
}else { }else {
response = httpBoot.syncCall(RequestUtils.wrapGet(url,headMap), ProxyHolder.NAT_HEAVY_PROXY); response = httpBoot.syncCall(RequestUtils.wrapGet(url,headMap));
} }
return response.body().string(); return response.body().string();
} catch (Exception e) { } catch (Exception e) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment