Skip to content

L
live-crawler
Commit 87226592 by zhiwei
Options

B站弹幕获取

parent fe6e0d5b
Showing with 791 additions and 858 deletions
pom.xml
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<groupId>com.zhiwei.live</groupId> <groupId>com.zhiwei.live</groupId>
<artifactId>live-crawler</artifactId> <artifactId>live-crawler</artifactId>
<version>0.0.1-SNAPSHOT</version> <version>0.0.1-SNAPSHOT</version>
<dependencies> <dependencies>
<dependency> <dependency>
<groupId>com.zhiwei.tools</groupId> <groupId>com.zhiwei.tools</groupId>
<artifactId>zhiwei-tools</artifactId> <artifactId>zhiwei-tools</artifactId>
<version>0.1.1-SNAPSHOT</version> <version>0.1.1-SNAPSHOT</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.zhiwei.crawler</groupId> <groupId>com.zhiwei.crawler</groupId>
<artifactId>crawler-core</artifactId> <artifactId>crawler-core</artifactId>
<version>0.1.1-RELEASE</version> <version>0.6.2.1-RELEASE</version>
</dependency> </dependency>
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 --> <!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
<dependency> <dependency>
<groupId>org.apache.commons</groupId> <groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId> <artifactId>commons-lang3</artifactId>
<version>3.8.1</version> <version>3.8.1</version>
</dependency> </dependency>
<!-- https://mvnrepository.com/artifact/io.netty/netty-all --> <!-- https://mvnrepository.com/artifact/io.netty/netty-all -->
<dependency> <dependency>
<groupId>io.netty</groupId> <groupId>io.netty</groupId>
<artifactId>netty-all</artifactId> <artifactId>netty-all</artifactId>
<version>4.1.33.Final</version> <version>4.1.48.Final</version>
</dependency> </dependency>
</dependencies> <dependency>
<groupId>org.projectlombok</groupId>
<!-- 打包管理 --> <artifactId>lombok</artifactId>
<build> <version>1.18.12</version>
<plugins> </dependency>
<!-- 发布源码 --> </dependencies>
<plugin>
<artifactId>maven-source-plugin</artifactId> <!-- 打包管理 -->
<version>2.4</version> <build>
<configuration> <plugins>
<attach>true</attach> <!-- 发布源码 -->
</configuration> <plugin>
<executions> <artifactId>maven-source-plugin</artifactId>
<execution> <version>2.4</version>
<phase>compile</phase> <configuration>
<goals> <attach>true</attach>
<goal>jar</goal> </configuration>
</goals> <executions>
</execution> <execution>
</executions> <phase>compile</phase>
</plugin> <goals>
<plugin> <goal>jar</goal>
<groupId>org.apache.maven.plugins</groupId> </goals>
<artifactId>maven-javadoc-plugin</artifactId> </execution>
<version>2.10.4</version> </executions>
</plugin> </plugin>
<!-- 解决maven test命令时console出现中文乱码乱码 --> <plugin>
<plugin> <groupId>org.apache.maven.plugins</groupId>
<groupId>org.apache.maven.plugins</groupId> <artifactId>maven-javadoc-plugin</artifactId>
<artifactId>maven-surefire-plugin</artifactId> <version>2.10.4</version>
<version>2.7.2</version> </plugin>
<configuration> <!-- 解决maven test命令时console出现中文乱码乱码 -->
<forkMode>once</forkMode> <plugin>
<argLine>-Dfile.encoding=UTF-8</argLine> <groupId>org.apache.maven.plugins</groupId>
</configuration> <artifactId>maven-surefire-plugin</artifactId>
</plugin> <version>2.7.2</version>
</plugins> <configuration>
</build> <forkMode>once</forkMode>
<!-- 分发管理:管理distribution和supporting files --> <argLine>-Dfile.encoding=UTF-8</argLine>
<distributionManagement> </configuration>
<snapshotRepository> </plugin>
<id>nexus-releases</id> </plugins>
<name>User Porject Snapshot</name> </build>
<url>http://192.168.0.30:8081/nexus/content/repositories/snapshots/</url> <!-- 分发管理:管理distribution和supporting files -->
<uniqueVersion>true</uniqueVersion> <distributionManagement>
</snapshotRepository> <snapshotRepository>
<repository> <id>nexus-releases</id>
<id>nexus-releases</id> <name>User Porject Snapshot</name>
<name>User Porject Release</name> <url>http://192.168.0.30:8081/nexus/content/repositories/snapshots/</url>
<url>http://192.168.0.30:8081/nexus/content/repositories/releases/</url> <uniqueVersion>true</uniqueVersion>
</repository> </snapshotRepository>
</distributionManagement> <repository>
<id>nexus-releases</id>
<name>User Porject Release</name>
<url>http://192.168.0.30:8081/nexus/content/repositories/releases/</url>
</repository>
</distributionManagement>
</project> </project>
\ No newline at end of file
src/main/java/com/zhiwei/live/bean/RoomInfo.java
package com.zhiwei.live.bean; package com.zhiwei.live.bean;
public class RoomInfo { import lombok.Data;
import lombok.ToString;
String pt; // 平台类型
@Data
String roomId; // 房间号 @ToString
public class RoomInfo {
String nickName; // 主播昵称
/**
String roomName; // 房间名称 * 平台类型
*/
Integer hotNum; // 直播间热度 String pt;
Integer fans; // 订阅数 /**
* 房间号
public RoomInfo() { */
} String roomId;
public RoomInfo(String pt, String roomId, String nickName, String roomName, Integer hotNum, Integer fans) { /**
this.pt = pt; * 主播昵称
this.roomId = roomId; */
this.nickName = nickName; String nickName;
this.roomName = roomName;
this.hotNum = hotNum; /**
this.fans = fans; * 房间名称
} */
String roomName;
@Override
public String toString() { /**
return "new RoomInfo[" + "pt = " + pt + ", roomId = " + roomId + ", roomName = " + roomName + ", nickName = " * 直播间热度
+ nickName + ", hotNum = " + hotNum + ", fans = " + fans + "]"; */
} Integer hotNum;
public Integer getFans() { /**
return fans; * 订阅数
} */
Integer fans;
public void setHotNum(Integer hotNum) {
this.hotNum = hotNum; public RoomInfo() {
} }
public void setFans(Integer fans) { public RoomInfo(String pt, String roomId, String nickName, String roomName, Integer hotNum, Integer fans) {
this.fans = fans; this.pt = pt;
} this.roomId = roomId;
this.nickName = nickName;
public String getPt() { this.roomName = roomName;
return pt; this.hotNum = hotNum;
} this.fans = fans;
}
public String getRoomId() {
return roomId; }
}
public String getNickName() {
return nickName;
}
public String getRoomName() {
return roomName;
}
public int getHotNum() {
return hotNum;
}
public void setPt(String pt) {
this.pt = pt;
}
public void setRoomId(String roomId) {
this.roomId = roomId;
}
public void setNickName(String nickName) {
this.nickName = nickName;
}
public void setRoomName(String roomName) {
this.roomName = roomName;
}
public void setHotNum(int hotNum) {
this.hotNum = hotNum;
}
}
src/main/java/com/zhiwei/live/danmu/bilibili/BilibiliClient.java
package com.zhiwei.live.danmu.bilibili; package com.zhiwei.live.danmu.bilibili;
import java.util.Objects; import java.util.Objects;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import org.apache.logging.log4j.LogManager; import com.zhiwei.crawler.utils.RequestUtils;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.crawler.core.HttpBoot; import com.alibaba.fastjson.JSONObject;
import com.zhiwei.crawler.core.RequestUtils; import com.zhiwei.crawler.core.HttpBoot;
import com.zhiwei.live.bean.RoomInfo; import com.zhiwei.live.bean.RoomInfo;
import com.zhiwei.live.danmu.util.Connector; import com.zhiwei.live.danmu.util.Connector;
import com.zhiwei.live.danmu.util.DataCallBack; import com.zhiwei.live.danmu.util.DataCallBack;
import com.zhiwei.live.roominfo.BilibiliRoomInfoCrawler; import com.zhiwei.live.roominfo.BilibiliRoomInfoCrawler;
import io.netty.channel.nio.NioEventLoopGroup; import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.handler.timeout.IdleStateHandler; import io.netty.handler.timeout.IdleStateHandler;
import okhttp3.Request; import okhttp3.Request;
import okhttp3.Response; import okhttp3.Response;
public class BilibiliClient { public class BilibiliClient {
private static HttpBoot httpBoot = new HttpBoot(); private static HttpBoot httpBoot = new HttpBoot.Builder().build();
private static Logger logger = LogManager.getLogger(BilibiliClient.class); private static Logger logger = LogManager.getLogger(BilibiliClient.class);
private static final int PORT = 2243; private static final int PORT = 2243;
/** /**
* 根据房间号获取弹幕信息 * 根据房间号获取弹幕信息
* @param roomId * @param roomUrl
* @throws Exception * @throws Exception
*/ */
public static void getDanmu(DataCallBack dataCallBack,String roomUrl) throws Exception { public static void getDanmu(DataCallBack dataCallBack,String roomUrl) throws Exception {
//根据房间号获取真实房间号 //根据房间号获取真实房间号
RoomInfo roomInfo = BilibiliRoomInfoCrawler.getRoomInfoByRoomUrl(roomUrl); RoomInfo roomInfo = BilibiliRoomInfoCrawler.getRoomInfoByRoomUrl(roomUrl);
if(Objects.nonNull(roomInfo)) { if(Objects.nonNull(roomInfo)) {
//获取弹幕服务器地址 //获取弹幕服务器地址
String url = "https://api.live.bilibili.com/room/v1/Danmu/getConf?room_id=" + roomInfo.getRoomId(); String url = "https://api.live.bilibili.com/room/v1/Danmu/getConf?room_id=" + roomInfo.getRoomId();
Request request = RequestUtils.wrapGet(url); Request request = RequestUtils.wrapGet(url);
String host = null; String host = null;
try(Response response = httpBoot.syncCall(request)) { try(Response response = httpBoot.syncCall(request)) {
String result = response.body().string(); String result = response.body().string();
JSONObject json = JSONObject.parseObject(result); JSONObject json = JSONObject.parseObject(result);
host = json.getJSONObject("data").getJSONArray("host_server_list").getJSONObject(0).getString("host"); host = json.getJSONObject("data").getJSONArray("host_server_list").getJSONObject(0).getString("host");
} catch(Exception e) { } catch(Exception e) {
throw new IllegalArgumentException("获取聊天服务器地址失败", e); throw new IllegalArgumentException("获取聊天服务器地址失败", e);
} }
//建立弹幕连接 //建立弹幕连接
Connector.asynchronizedTcpConnect(new NioEventLoopGroup(), host, PORT, Connector.asynchronizedTcpConnect(new NioEventLoopGroup(), host, PORT,
new IdleStateHandler(0, 30, 0,TimeUnit.SECONDS), new BilibiliMessageHandler(dataCallBack,roomInfo.getRoomId())).sync(); new IdleStateHandler(0, 30, 0,TimeUnit.SECONDS), new BilibiliMessageHandler(dataCallBack,roomInfo.getRoomId())).sync();
}else { }else {
logger.info("获取真实房间号出现问题,请及时检查程序"); logger.info("获取真实房间号出现问题,请及时检查程序");
} }
} }
} }
src/main/java/com/zhiwei/live/danmu/bilibili/BilibiliMessage.java
package com.zhiwei.live.danmu.bilibili; package com.zhiwei.live.danmu.bilibili;
import java.util.Date; import java.util.Date;
import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSONObject;
import lombok.Data;
public class BilibiliMessage { import lombok.ToString;
String messageType; //弹幕消息类型 @Data
@ToString
String user_id; //发布者uid public class BilibiliMessage {
String nickName; //发布者昵称 /**
* 弹幕消息类型
Date time; //弹幕时间 */
String messageType;
String content; //弹幕内容
/**
String room_id; //房间id * 发布者uid
*/
public BilibiliMessage(JSONObject json) throws Exception { String userId;
constructJson(json);
} /**
* 发布者昵称
public BilibiliMessage(){ */
} String nickName;
private void constructJson(JSONObject json) throws Exception{ /**
try { * 弹幕时间
System.out.println(json); */
JSONArray jsonArray = json.getJSONArray("info"); Date time;
messageType = json.getString("cmd"); /**
user_id = jsonArray.getJSONArray(2).getString(0); * 弹幕内容
time = new Date(); */
nickName = jsonArray.getJSONArray(2).getString(1); String content;
content = jsonArray.getString(1);
/**
} catch (Exception e) { * 房间id
throw new Exception(); */
} String roomId;
}
public BilibiliMessage(JSONObject json) throws Exception {
constructJson(json);
}
@Override
public String toString() { public BilibiliMessage(){
return "new BilibiliMessage[" }
+ " user_id = " + user_id
+ ", nickName = " + nickName private void constructJson(JSONObject json) throws Exception{
+ ", messageType = " + messageType try {
+ ", time = " + time JSONArray jsonArray = json.getJSONArray("info");
+ ", content = " + content messageType = json.getString("cmd");
+ ", room_id = " + room_id userId = jsonArray.getJSONArray(2).getString(0);
+ "]"; time = new Date();
} nickName = jsonArray.getJSONArray(2).getString(1);
content = jsonArray.getString(1);
public Date getTime() { } catch (Exception e) {
return time; throw new Exception();
} }
}
public void setTime(Date time) {
this.time = time; }
}
public String getMessageType() {
return messageType;
}
public String getUser_id() {
return user_id;
}
public String getNickName() {
return nickName;
}
public String getContent() {
return content;
}
public void setMessageType(String messageType) {
this.messageType = messageType;
}
public void setUser_id(String user_id) {
this.user_id = user_id;
}
public void setNickName(String nickName) {
this.nickName = nickName;
}
public void setContent(String content) {
this.content = content;
}
public String getRoom_id() {
return room_id;
}
public void setRoom_id(String room_id) {
this.room_id = room_id;
}
}
src/main/java/com/zhiwei/live/danmu/douyu/DouyuClient.java
...@@ -22,8 +22,8 @@ import io.netty.handler.timeout.IdleStateHandler; ...@@ -22,8 +22,8 @@ import io.netty.handler.timeout.IdleStateHandler;
public class DouyuClient { public class DouyuClient {
private static Logger logger = LogManager.getLogger(DouyuClient.class); private static Logger logger = LogManager.getLogger(DouyuClient.class);
private static final int PORT = 8601; private static final int PORT = 8504;
private static final String HOST = "openbarrage.douyutv.com"; private static final String HOST = "danmuproxy.douyu.com";
/** /**
* 根据房间号获取弹幕信息 * 根据房间号获取弹幕信息
......
src/main/java/com/zhiwei/live/danmu/douyu/DouyuMessageHandler.java
...@@ -70,7 +70,7 @@ public class DouyuMessageHandler extends ChannelInboundHandlerAdapter { ...@@ -70,7 +70,7 @@ public class DouyuMessageHandler extends ChannelInboundHandlerAdapter {
if (msg instanceof ByteBuf) { if (msg instanceof ByteBuf) {
ByteBuf buf = (ByteBuf) msg; ByteBuf buf = (ByteBuf) msg;
String source = buf.toString(CharsetUtil.UTF_8); String source = buf.toString(CharsetUtil.UTF_8);
// System.out.println("source========="+source); System.out.println("source========="+source);
if(source.contains("chatmsg")) { if(source.contains("chatmsg")) {
Map<String,Object> messageMap = DouYuUtil.toMap(source); Map<String,Object> messageMap = DouYuUtil.toMap(source);
String data = JSONObject.toJSONString(messageMap); String data = JSONObject.toJSONString(messageMap);
...@@ -84,6 +84,10 @@ public class DouyuMessageHandler extends ChannelInboundHandlerAdapter { ...@@ -84,6 +84,10 @@ public class DouyuMessageHandler extends ChannelInboundHandlerAdapter {
ByteBuf groupMsg = buildMsg(StringUtils.join("type@=joingroup/rid@=",roomId,"/gid@=-9999/")); ByteBuf groupMsg = buildMsg(StringUtils.join("type@=joingroup/rid@=",roomId,"/gid@=-9999/"));
System.out.println("发送入组消息: \n" + ByteBufUtil.prettyHexDump(groupMsg)); System.out.println("发送入组消息: \n" + ByteBufUtil.prettyHexDump(groupMsg));
ctx.writeAndFlush(groupMsg); ctx.writeAndFlush(groupMsg);
// ByteBuf giftMsg = buildMsg(StringUtils.join("type@=dmfbdreq/dfl@=sn@AA=105@ASss@AA=0@AS@Ssn@AA=106@ASss@AA=0@AS@Ssn@AA=107@ASss@AA=0@AS@Ssn@AA=108@ASss@AA=0@AS@Ssn@AA=110@ASss@AA=0@AS@S/"));
// System.out.println("发送礼物消息: \n" + ByteBufUtil.prettyHexDump(giftMsg));
// ctx.writeAndFlush(giftMsg);
finish = true; finish = true;
} }
ctx.fireChannelRead(msg); ctx.fireChannelRead(msg);
...@@ -97,7 +101,7 @@ public class DouyuMessageHandler extends ChannelInboundHandlerAdapter { ...@@ -97,7 +101,7 @@ public class DouyuMessageHandler extends ChannelInboundHandlerAdapter {
if (evt instanceof IdleStateEvent) { if (evt instanceof IdleStateEvent) {
IdleStateEvent event = (IdleStateEvent) evt; IdleStateEvent event = (IdleStateEvent) evt;
if (event.state() == IdleState.WRITER_IDLE) { if (event.state() == IdleState.WRITER_IDLE) {
ByteBuf pingMsg = buildMsg(StringUtils.join("type@=keeplive/tick@=", System.currentTimeMillis() / 1000, "/")); ByteBuf pingMsg = buildMsg(StringUtils.join("type@=mrkl/"));
System.out.println("发送心跳消息: \n" + ByteBufUtil.prettyHexDump(pingMsg)); System.out.println("发送心跳消息: \n" + ByteBufUtil.prettyHexDump(pingMsg));
ctx.writeAndFlush(pingMsg); ctx.writeAndFlush(pingMsg);
} else { } else {
...@@ -109,7 +113,7 @@ public class DouyuMessageHandler extends ChannelInboundHandlerAdapter { ...@@ -109,7 +113,7 @@ public class DouyuMessageHandler extends ChannelInboundHandlerAdapter {
} }
private ByteBuf buildMsg(String content) { private ByteBuf buildMsg(String content) {
int fixedLen = 4 + 4 + 1; int fixedLen = 4 + 4 + 1 + content.length();
byte[] body = content.getBytes(CharsetUtil.UTF_8); byte[] body = content.getBytes(CharsetUtil.UTF_8);
int length = fixedLen + body.length; int length = fixedLen + body.length;
return Unpooled.buffer(length).writeIntLE(length).writeIntLE(length).writeShortLE(689) return Unpooled.buffer(length).writeIntLE(length).writeIntLE(length).writeShortLE(689)
......
src/main/java/com/zhiwei/live/danmu/pandam/PandamClient.java
package com.zhiwei.live.danmu.pandam; package com.zhiwei.live.danmu.pandam;
import java.util.Objects; import java.util.Objects;
import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSONObject;
import com.zhiwei.crawler.core.HttpBoot; import com.zhiwei.crawler.core.HttpBoot;
import com.zhiwei.crawler.core.RequestUtils; import com.zhiwei.crawler.utils.RequestUtils;
import com.zhiwei.live.bean.RoomInfo; import com.zhiwei.live.bean.RoomInfo;
import com.zhiwei.live.danmu.util.Connector; import com.zhiwei.live.danmu.util.Connector;
import com.zhiwei.live.danmu.util.DataCallBack; import com.zhiwei.live.danmu.util.DataCallBack;
import com.zhiwei.live.roominfo.PandamTVRoomInfoCrawler; import com.zhiwei.live.roominfo.PandamTVRoomInfoCrawler;
import io.netty.channel.nio.NioEventLoopGroup; import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.handler.timeout.IdleStateHandler; import io.netty.handler.timeout.IdleStateHandler;
import okhttp3.Request; import okhttp3.Request;
import okhttp3.Response; import okhttp3.Response;
/** /**
* @ClassName: PandaClient * @ClassName: PandaClient
* @Description: TODO * @Description: TODO
* @author 0xff * @author 0xff
* @date Jan 25, 2019 3:24:33 PM * @date Jan 25, 2019 3:24:33 PM
*/ */
public class PandamClient { public class PandamClient {
private static HttpBoot httpBoot = new HttpBoot(); private static HttpBoot httpBoot = new HttpBoot.Builder().build();
/** /**
* 根据房间号获取弹幕信息 * 根据房间号获取弹幕信息
* @param roomId * @param roomId
* @throws Exception * @throws Exception
*/ */
public static void getDanmu(DataCallBack dataCallBack,String roomUrl) throws Exception { public static void getDanmu(DataCallBack dataCallBack,String roomUrl) throws Exception {
//根据房间号获取弹幕服务器地址 //根据房间号获取弹幕服务器地址
RoomInfo roomInfo = PandamTVRoomInfoCrawler.getRoomInfoByRoomUrl(roomUrl); RoomInfo roomInfo = PandamTVRoomInfoCrawler.getRoomInfoByRoomUrl(roomUrl);
if(Objects.nonNull(roomInfo)) { if(Objects.nonNull(roomInfo)) {
String roomId = roomInfo.getRoomId(); String roomId = roomInfo.getRoomId();
Request request = RequestUtils.wrapGet("http://riven.panda.tv/chatroom/getinfo?roomid=" + roomId); Request request = RequestUtils.wrapGet("http://riven.panda.tv/chatroom/getinfo?roomid=" + roomId);
String host = null; String host = null;
int port = -1; int port = -1;
JSONObject json = null; JSONObject json = null;
try(Response response = httpBoot.syncCall(request)) { try(Response response = httpBoot.syncCall(request)) {
json = JSON.parseObject(response.body().string()); json = JSON.parseObject(response.body().string());
if(json.getInteger("errno") != 0) { if(json.getInteger("errno") != 0) {
throw new IllegalStateException("房间号: " + roomUrl + " 不存在"); throw new IllegalStateException("房间号: " + roomUrl + " 不存在");
} }
json = json.getJSONObject("data"); json = json.getJSONObject("data");
String[] address = json.getJSONArray("chat_addr_list").getString(0).split(":"); String[] address = json.getJSONArray("chat_addr_list").getString(0).split(":");
host = address[0]; host = address[0];
port = Integer.parseInt(address[1]); port = Integer.parseInt(address[1]);
} catch(Exception e) { } catch(Exception e) {
throw new IllegalArgumentException("获取聊天服务器地址失败", e); throw new IllegalArgumentException("获取聊天服务器地址失败", e);
} }
String appid = json.getString("appid"); String appid = json.getString("appid");
String rid = json.getString("rid"); String rid = json.getString("rid");
String sign = json.getString("sign"); String sign = json.getString("sign");
String authType = json.getString("authType"); String authType = json.getString("authType");
String ts = json.getString("ts"); String ts = json.getString("ts");
Connector.asynchronizedTcpConnect(new NioEventLoopGroup(), host, port, Connector.asynchronizedTcpConnect(new NioEventLoopGroup(), host, port,
new IdleStateHandler(0, 30, 45), new PandamMessageHandler(dataCallBack, appid, rid, ts, sign, authType)).sync(); new IdleStateHandler(0, 30, 45), new PandamMessageHandler(dataCallBack, appid, rid, ts, sign, authType)).sync();
}else { }else {
} }
} }
} }
src/main/java/com/zhiwei/live/danmu/pandam/PandamMessageHandler.java
package com.zhiwei.live.danmu.pandam; package com.zhiwei.live.danmu.pandam;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import org.apache.commons.lang3.StringUtils; import lombok.extern.log4j.Log4j2;
import org.apache.commons.lang3.StringUtils;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.live.danmu.util.DataCallBack; import com.alibaba.fastjson.JSONObject;
import com.zhiwei.live.danmu.util.DataCallBack;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufUtil; import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled; import io.netty.buffer.ByteBufUtil;
import io.netty.channel.ChannelHandlerContext; import io.netty.buffer.Unpooled;
import io.netty.channel.ChannelInboundHandlerAdapter; import io.netty.channel.ChannelHandlerContext;
import io.netty.handler.timeout.IdleState; import io.netty.channel.ChannelInboundHandlerAdapter;
import io.netty.handler.timeout.IdleStateEvent; import io.netty.handler.timeout.IdleState;
import io.netty.util.CharsetUtil; import io.netty.handler.timeout.IdleStateEvent;
import io.netty.util.ReferenceCountUtil; import io.netty.util.CharsetUtil;
import io.netty.util.ReferenceCountUtil;
/**
* @ClassName: PandaMessageHandler /**
* @Description: TODO * @ClassName: PandaMessageHandler
* @author 0xff * @Description: TODO
* @date Jan 25, 2019 11:03:20 AM * @author 0xff
*/ * @date Jan 25, 2019 11:03:20 AM
public class PandamMessageHandler extends ChannelInboundHandlerAdapter { */
private static final byte[] FIRST_REQ = new byte[] { 0x00, 0x06, 0x00, 0x02, 0x00 }; @Log4j2
private static final byte[] FIRST_RES = new byte[] { 0x00, 0x06, 0x00, 0x06 }; public class PandamMessageHandler extends ChannelInboundHandlerAdapter {
private static final byte[] PING = new byte[] { 0x00, 0x06, 0x00, 0x00 }; private static final byte[] FIRST_REQ = new byte[] { 0x00, 0x06, 0x00, 0x02, 0x00 };
private static final byte[] REQ = new byte[] { 0x00, 0x06, 0x00, 0x03 }; private static final byte[] FIRST_RES = new byte[] { 0x00, 0x06, 0x00, 0x06 };
private static Pattern pattern = Pattern.compile("\\{\"type\":\"1\".+?\\}\\}"); private static final byte[] PING = new byte[] { 0x00, 0x06, 0x00, 0x00 };
private String rid; private static final byte[] REQ = new byte[] { 0x00, 0x06, 0x00, 0x03 };
private String appid; private static Pattern pattern = Pattern.compile("\\{\"type\":\"1\".+?\\}\\}");
private String ts; private String rid;
private String sign; private String appid;
private String authType; private String ts;
private DataCallBack dataCallBack; private String sign;
private String authType;
private DataCallBack dataCallBack;
/**
* Constructor
* /**
* @param roomId * Constructor
* 房间号 *
*/ * @param roomId
public PandamMessageHandler(DataCallBack dataCallBack,String appid, String rid, String ts, String sign, String authType) { * 房间号
this.dataCallBack = dataCallBack; */
this.appid = appid; public PandamMessageHandler(DataCallBack dataCallBack,String appid, String rid, String ts, String sign, String authType) {
this.rid = rid; this.dataCallBack = dataCallBack;
this.sign = sign; this.appid = appid;
this.authType = authType; this.rid = rid;
this.ts = ts; this.sign = sign;
this.authType = authType;
} this.ts = ts;
/* }
* @see
* io.netty.channel.ChannelInboundHandlerAdapter#channelActive(io.netty.channel. /*
* ChannelHandlerContext) * @see
*/ * io.netty.channel.ChannelInboundHandlerAdapter#channelActive(io.netty.channel.
@Override * ChannelHandlerContext)
public void channelActive(ChannelHandlerContext ctx) throws Exception { */
System.out.println("TCP 连接建立成功: " + ctx.channel()); @Override
byte[] body = StringUtils public void channelActive(ChannelHandlerContext ctx) throws Exception {
.join("u:", rid, "@", appid, "\nk:1\nt:300\nts:", ts, "\nsign:", sign, "\nauthtype:", authType) System.out.println("TCP 连接建立成功: " + ctx.channel());
.getBytes(); byte[] body = StringUtils
ByteBuf loginMsg = Unpooled.wrappedBuffer(Unpooled.wrappedBuffer(FIRST_REQ), .join("u:", rid, "@", appid, "\nk:1\nt:300\nts:", ts, "\nsign:", sign, "\nauthtype:", authType)
Unpooled.buffer(1).writeByte(body.length), Unpooled.wrappedBuffer(body)); .getBytes();
System.out.println("发送登录消息: \n" + ByteBufUtil.prettyHexDump(loginMsg)); ByteBuf loginMsg = Unpooled.wrappedBuffer(Unpooled.wrappedBuffer(FIRST_REQ),
ctx.writeAndFlush(loginMsg); Unpooled.buffer(1).writeByte(body.length), Unpooled.wrappedBuffer(body));
} System.out.println("发送登录消息: \n" + ByteBufUtil.prettyHexDump(loginMsg));
ctx.writeAndFlush(loginMsg);
/* }
* @see
* io.netty.channel.ChannelInboundHandlerAdapter#channelRead(io.netty.channel. /*
* ChannelHandlerContext, java.lang.Object) * @see
*/ * io.netty.channel.ChannelInboundHandlerAdapter#channelRead(io.netty.channel.
@Override * ChannelHandlerContext, java.lang.Object)
public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { */
if (msg instanceof ByteBuf) { @Override
ByteBuf buf = (ByteBuf) msg; public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
// System.out.println("收到消息: \n" + ByteBufUtil.prettyHexDump(buf)); if (msg instanceof ByteBuf) {
String source = buf.toString(CharsetUtil.UTF_8); ByteBuf buf = (ByteBuf) msg;
Matcher matcher = pattern.matcher(source); // System.out.println("收到消息: \n" + ByteBufUtil.prettyHexDump(buf));
while(matcher.find()) { String source = buf.toString(CharsetUtil.UTF_8);
JSONObject dataJson = JSONObject.parseObject(matcher.group()); Matcher matcher = pattern.matcher(source);
PandamMessage pandamMessage = new PandamMessage(dataJson); while(matcher.find()) {
dataCallBack.onData(pandamMessage); JSONObject dataJson = JSONObject.parseObject(matcher.group());
} PandamMessage pandamMessage = new PandamMessage(dataJson);
} dataCallBack.onData(pandamMessage);
ReferenceCountUtil.release(msg); }
} }
ReferenceCountUtil.release(msg);
/* }
* @see
* io.netty.channel.ChannelInboundHandlerAdapter#userEventTriggered(io.netty. /*
* channel.ChannelHandlerContext, java.lang.Object) * @see
*/ * io.netty.channel.ChannelInboundHandlerAdapter#userEventTriggered(io.netty.
@Override * channel.ChannelHandlerContext, java.lang.Object)
public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception { */
if (evt instanceof IdleStateEvent) { @Override
IdleStateEvent event = (IdleStateEvent) evt; public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception {
if (event.state() == IdleState.WRITER_IDLE) { if (evt instanceof IdleStateEvent) {
ByteBuf pingMsg = Unpooled.wrappedBuffer(PING); IdleStateEvent event = (IdleStateEvent) evt;
System.out.println("发送心跳消息: \n" + ByteBufUtil.prettyHexDump(pingMsg)); if (event.state() == IdleState.WRITER_IDLE) {
ctx.writeAndFlush(pingMsg); ByteBuf pingMsg = Unpooled.wrappedBuffer(PING);
} else { System.out.println("发送心跳消息: \n" + ByteBufUtil.prettyHexDump(pingMsg));
ctx.fireUserEventTriggered(evt); ctx.writeAndFlush(pingMsg);
} } else {
} else { ctx.fireUserEventTriggered(evt);
ctx.fireUserEventTriggered(evt); }
} } else {
} ctx.fireUserEventTriggered(evt);
} }
}
}
src/main/java/com/zhiwei/live/roominfo/BilibiliRoomInfoCrawler.java
package com.zhiwei.live.roominfo; package com.zhiwei.live.roominfo;
import org.apache.commons.lang3.StringUtils; import com.zhiwei.crawler.utils.RequestUtils;
import org.apache.logging.log4j.LogManager; import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.crawler.core.HttpBoot; import com.alibaba.fastjson.JSONObject;
import com.zhiwei.crawler.core.RequestUtils; import com.zhiwei.crawler.core.HttpBoot;
import com.zhiwei.crawler.proxy.ProxyHolder; import com.zhiwei.crawler.proxy.ProxyHolder;
import com.zhiwei.live.bean.RoomInfo; import com.zhiwei.live.bean.RoomInfo;
import com.zhiwei.tools.tools.ZhiWeiTools; import com.zhiwei.tools.tools.ZhiWeiTools;
/** /**
* bilibili 直播间信息爬取 * bilibili 直播间信息爬取
* @author qq859 * @author qq859
* *
*/ */
public class BilibiliRoomInfoCrawler { public class BilibiliRoomInfoCrawler {
private static HttpBoot httpBoot = new HttpBoot(); private static HttpBoot httpBoot = new HttpBoot.Builder().build();
private static Logger logger = LogManager.getLogger(BilibiliRoomInfoCrawler.class); private static Logger logger = LogManager.getLogger(BilibiliRoomInfoCrawler.class);
private static final String PT = "B站"; private static final String PT = "B站";
/** /**
* 根据房间id获取房间信息 * 根据房间id获取房间信息
* @param roomId * @param roomId
* @return * @return
* @throws Exception * @throws Exception
*/ */
public static RoomInfo getRoomInfoByRoomUrl(String roomUrl) throws Exception{ public static RoomInfo getRoomInfoByRoomUrl(String roomUrl) throws Exception{
String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl)).body().string(); String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl)).body().string();
if(!StringUtils.isBlank(htmlBody)) { if(!StringUtils.isBlank(htmlBody)) {
//判断页面中是否包含房间信息 //判断页面中是否包含房间信息
if(htmlBody.contains("window.__NEPTUNE_IS_MY_WAIFU__=")) { if(htmlBody.contains("window.__NEPTUNE_IS_MY_WAIFU__=")) {
//通过截取获取直播间信息字段，将截取的字段处理为json格式方便解析 //通过截取获取直播间信息字段，将截取的字段处理为json格式方便解析
htmlBody = htmlBody.split("<script>window.__NEPTUNE_IS_MY_WAIFU__=")[1].split("</script>")[0]; htmlBody = htmlBody.split("<script>window.__NEPTUNE_IS_MY_WAIFU__=")[1].split("</script>")[0];
htmlBody = ZhiWeiTools.decodeUnicode(htmlBody); htmlBody = ZhiWeiTools.decodeUnicode(htmlBody);
htmlBody = ZhiWeiTools.delHTMLTag(htmlBody); htmlBody = ZhiWeiTools.delHTMLTag(htmlBody);
htmlBody = htmlBody.replaceAll("\\\\", "").replaceAll("'", "\""); htmlBody = htmlBody.replaceAll("\\\\", "").replaceAll("'", "\"");
//解析json数据 //解析json数据
JSONObject baseInfoRes = JSONObject.parseObject(htmlBody); JSONObject baseInfoRes = JSONObject.parseObject(htmlBody);
JSONObject data = baseInfoRes.getJSONObject("baseInfoRes").getJSONObject("data"); JSONObject data = baseInfoRes.getJSONObject("baseInfoRes").getJSONObject("data");
Integer person_num = data.getIntValue("online"); Integer person_num = data.getIntValue("online");
String roomname = data.getString("title"); String roomname = data.getString("title");
Integer room_id = data.getInteger("room_id"); Integer room_id = data.getInteger("room_id");
Integer fans = data.getInteger("attention"); Integer fans = data.getInteger("attention");
String roomId = room_id!=null?room_id.toString():null; String roomId = room_id!=null?room_id.toString():null;
String username = null; String username = null;
//通过房间id获取用户信息 //通过房间id获取用户信息
roomUrl = "https://api.live.bilibili.com/live_user/v1/UserInfo/get_anchor_in_room?roomid="+roomId; roomUrl = "https://api.live.bilibili.com/live_user/v1/UserInfo/get_anchor_in_room?roomid="+roomId;
String roomBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl)).body().string(); String roomBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl)).body().string();
if(!StringUtils.isBlank(roomBody)) { if(!StringUtils.isBlank(roomBody)) {
JSONObject roomData = JSONObject.parseObject(roomBody).getJSONObject("data"); JSONObject roomData = JSONObject.parseObject(roomBody).getJSONObject("data");
username = roomData.getJSONObject("info").getString("uname"); username = roomData.getJSONObject("info").getString("uname");
} }
return new RoomInfo(PT, roomId, roomname, username, person_num, fans); return new RoomInfo(PT, roomId, roomname, username, person_num, fans);
}else { }else {
logger.info("此次采集页面中不包含房间信息字段, 此次页面信息为:{}", htmlBody); logger.info("此次采集页面中不包含房间信息字段, 此次页面信息为:{}", htmlBody);
return null; return null;
} }
} }
return null; return null;
} }
public static RoomInfo getRoomInfoProxyByRoomUrl(String roomUrl) throws Exception{ public static RoomInfo getRoomInfoProxyByRoomUrl(String roomUrl) throws Exception{
String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl), ProxyHolder.NAT_PROXY).body().string(); String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl), ProxyHolder.NAT_HEAVY_PROXY).body().string();
if(!StringUtils.isBlank(htmlBody)) { if(!StringUtils.isBlank(htmlBody)) {
//判断页面中是否包含房间信息 //判断页面中是否包含房间信息
if(htmlBody.contains("window.__NEPTUNE_IS_MY_WAIFU__=")) { if(htmlBody.contains("window.__NEPTUNE_IS_MY_WAIFU__=")) {
//通过截取获取直播间信息字段，将截取的字段处理为json格式方便解析 //通过截取获取直播间信息字段，将截取的字段处理为json格式方便解析
htmlBody = htmlBody.split("<script>window.__NEPTUNE_IS_MY_WAIFU__=")[1].split("</script>")[0]; htmlBody = htmlBody.split("<script>window.__NEPTUNE_IS_MY_WAIFU__=")[1].split("</script>")[0];
htmlBody = ZhiWeiTools.decodeUnicode(htmlBody); htmlBody = ZhiWeiTools.decodeUnicode(htmlBody);
htmlBody = ZhiWeiTools.delHTMLTag(htmlBody); htmlBody = ZhiWeiTools.delHTMLTag(htmlBody);
htmlBody = htmlBody.replaceAll("\\\\", "").replaceAll("'", "\""); htmlBody = htmlBody.replaceAll("\\\\", "").replaceAll("'", "\"");
//解析json数据 //解析json数据
JSONObject baseInfoRes = JSONObject.parseObject(htmlBody); JSONObject baseInfoRes = JSONObject.parseObject(htmlBody);
JSONObject data = baseInfoRes.getJSONObject("baseInfoRes").getJSONObject("data"); JSONObject data = baseInfoRes.getJSONObject("baseInfoRes").getJSONObject("data");
Integer person_num = data.getIntValue("online"); Integer person_num = data.getIntValue("online");
String roomname = data.getString("title"); String roomname = data.getString("title");
Integer room_id = data.getInteger("room_id"); Integer room_id = data.getInteger("room_id");
String roomId = room_id!=null?room_id.toString():null; String roomId = room_id!=null?room_id.toString():null;
Integer fans = data.getInteger("attention"); Integer fans = data.getInteger("attention");
String username = null; String username = null;
//通过房间id获取用户信息 //通过房间id获取用户信息
roomUrl = "https://api.live.bilibili.com/live_user/v1/UserInfo/get_anchor_in_room?roomid="+room_id; roomUrl = "https://api.live.bilibili.com/live_user/v1/UserInfo/get_anchor_in_room?roomid="+room_id;
String roomBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl), ProxyHolder.NAT_PROXY).body().string(); String roomBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl), ProxyHolder.NAT_HEAVY_PROXY).body().string();
if(!StringUtils.isBlank(roomBody)) { if(!StringUtils.isBlank(roomBody)) {
JSONObject roomData = JSONObject.parseObject(roomBody).getJSONObject("data"); JSONObject roomData = JSONObject.parseObject(roomBody).getJSONObject("data");
username = roomData.getJSONObject("info").getString("uname"); username = roomData.getJSONObject("info").getString("uname");
} }
return new RoomInfo(PT, roomId, roomname, username, person_num, fans); return new RoomInfo(PT, roomId, roomname, username, person_num, fans);
}else { }else {
logger.info("此次采集页面中不包含房间信息字段, 此次页面信息为:{}", htmlBody); logger.info("此次采集页面中不包含房间信息字段, 此次页面信息为:{}", htmlBody);
return null; return null;
} }
} }
return null; return null;
} }
} }
src/main/java/com/zhiwei/live/roominfo/DouYuRoomInfoCrawler.java
package com.zhiwei.live.roominfo; package com.zhiwei.live.roominfo;
import org.apache.commons.lang3.StringUtils; import com.zhiwei.crawler.utils.RequestUtils;
import org.apache.logging.log4j.LogManager; import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.crawler.core.HttpBoot; import com.alibaba.fastjson.JSONObject;
import com.zhiwei.crawler.core.RequestUtils; import com.zhiwei.crawler.core.HttpBoot;
import com.zhiwei.crawler.proxy.ProxyHolder; import com.zhiwei.crawler.proxy.ProxyHolder;
import com.zhiwei.live.bean.RoomInfo; import com.zhiwei.live.bean.RoomInfo;
/** /**
* 斗鱼直播间信息获取 * 斗鱼直播间信息获取
* @author qq859 * @author qq859
* *
*/ */
public class DouYuRoomInfoCrawler { public class DouYuRoomInfoCrawler {
private static HttpBoot httpBoot = new HttpBoot(); private static HttpBoot httpBoot = new HttpBoot.Builder().build();
private static Logger logger = LogManager.getLogger(DouYuRoomInfoCrawler.class); private static Logger logger = LogManager.getLogger(DouYuRoomInfoCrawler.class);
private static final String PT = "斗鱼"; private static final String PT = "斗鱼";
/** /**
* 根据房间id获取房间信息 * 根据房间id获取房间信息
* @param roomId * @param roomUrl
* @return * @return
* @throws Exception * @throws Exception
*/ */
public static RoomInfo getRoomInfoByRoomUrl(String roomUrl) throws Exception{ public static RoomInfo getRoomInfoByRoomUrl(String roomUrl) throws Exception{
String roomBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl)).body().string(); String roomBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl)).body().string();
if(!StringUtils.isBlank(roomBody) && roomBody.contains("ROOM.room_id =")) { if(!StringUtils.isBlank(roomBody) && roomBody.contains("ROOM.room_id =")) {
String roomId = roomBody.split("ROOM\\.room_id = ")[1].split("; ")[0].trim(); String roomId = roomBody.split("ROOM\\.room_id = ")[1].split("; ")[0].trim();
//获取房间信息 //获取房间信息
String url = "http://open.douyucdn.cn/api/RoomApi/room/" + roomId; String url = "http://open.douyucdn.cn/api/RoomApi/room/" + roomId;
String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(url)).body().string(); String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(url)).body().string();
if(!StringUtils.isBlank(htmlBody)) { if(!StringUtils.isBlank(htmlBody)) {
JSONObject data = JSONObject.parseObject(htmlBody).getJSONObject("data"); JSONObject data = JSONObject.parseObject(htmlBody).getJSONObject("data");
String room_name = data.getString("room_name"); String room_name = data.getString("room_name");
String user_name = data.getString("owner_name"); String user_name = data.getString("owner_name");
Integer hn = data.getInteger("hn"); Integer hn = data.getInteger("hn");
int online = data.getInteger("online"); int online = data.getInteger("online");
Integer fans = 0; Integer fans = 0;
//获取用户信息 //获取用户信息
String userUrl = "https://www.douyu.com/swf_api/h5room/" + roomId; String userUrl = "https://www.douyu.com/swf_api/h5room/" + roomId;
String userBody = httpBoot.syncCall(RequestUtils.wrapGet(userUrl)).body().string(); String userBody = httpBoot.syncCall(RequestUtils.wrapGet(userUrl)).body().string();
if(!StringUtils.isBlank(userBody)) { if(!StringUtils.isBlank(userBody)) {
JSONObject userData = JSONObject.parseObject(userBody).getJSONObject("data"); JSONObject userData = JSONObject.parseObject(userBody).getJSONObject("data");
fans = Integer.valueOf(userData.getString("fans")); fans = Integer.valueOf(userData.getString("fans"));
} }
return new RoomInfo(PT, roomId, room_name, user_name , hn, fans); return new RoomInfo(PT, roomId, room_name, user_name , hn, fans);
}else { }else {
logger.info("此次采集页面中不包含房间信息字段, 此次页面信息为:{}", htmlBody); logger.info("此次采集页面中不包含房间信息字段, 此次页面信息为:{}", htmlBody);
return null; return null;
} }
}else { }else {
System.out.println("------------------"); System.out.println("------------------");
logger.info("此次采集页面中不包含房间信息字段, 此次页面信息为:{}", roomBody); logger.info("此次采集页面中不包含房间信息字段, 此次页面信息为:{}", roomBody);
return null; return null;
} }
} }
/** /**
* 根据房间id获取房间信息 * 根据房间id获取房间信息
* @param roomId * @param roomUrl
* @return * @return
* @throws Exception * @throws Exception
*/ */
public static RoomInfo getRoomInfoProxyByRoomUrl(String roomUrl) throws Exception{ public static RoomInfo getRoomInfoProxyByRoomUrl(String roomUrl) throws Exception{
String roomBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl), ProxyHolder.NAT_PROXY).body().string(); String roomBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl), ProxyHolder.NAT_HEAVY_PROXY).body().string();
if(!StringUtils.isBlank(roomBody) && roomBody.contains("ROOM.room_id =")) { if(!StringUtils.isBlank(roomBody) && roomBody.contains("ROOM.room_id =")) {
String roomId = roomBody.split("ROOM\\.room_id = ")[1].split("; ")[0].trim(); String roomId = roomBody.split("ROOM\\.room_id = ")[1].split("; ")[0].trim();
String url = "http://open.douyucdn.cn/api/RoomApi/room/" + roomId; String url = "http://open.douyucdn.cn/api/RoomApi/room/" + roomId;
String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(url), ProxyHolder.NAT_PROXY).body().string(); String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(url), ProxyHolder.NAT_HEAVY_PROXY).body().string();
if(!StringUtils.isBlank(htmlBody)) { if(!StringUtils.isBlank(htmlBody)) {
JSONObject data = JSONObject.parseObject(htmlBody).getJSONObject("data"); JSONObject data = JSONObject.parseObject(htmlBody).getJSONObject("data");
String room_name = data.getString("room_name"); String room_name = data.getString("room_name");
String user_name = data.getString("owner_name"); String user_name = data.getString("owner_name");
Integer hn = data.getInteger("hn"); Integer hn = data.getInteger("hn");
int online = data.getInteger("online"); int online = data.getInteger("online");
Integer fans = 0; Integer fans = 0;
//获取用户信息 //获取用户信息
String userUrl = "https://www.douyu.com/swf_api/h5room/" + roomId; String userUrl = "https://www.douyu.com/swf_api/h5room/" + roomId;
String userBody = httpBoot.syncCall(RequestUtils.wrapGet(userUrl)).body().string(); String userBody = httpBoot.syncCall(RequestUtils.wrapGet(userUrl)).body().string();
if(!StringUtils.isBlank(userBody)) { if(!StringUtils.isBlank(userBody)) {
JSONObject userData = JSONObject.parseObject(userBody).getJSONObject("data"); JSONObject userData = JSONObject.parseObject(userBody).getJSONObject("data");
fans = Integer.valueOf(userData.getString("fans")); fans = Integer.valueOf(userData.getString("fans"));
} }
return new RoomInfo(PT, roomId, room_name, user_name , hn, fans); return new RoomInfo(PT, roomId, room_name, user_name , hn, fans);
}else { }else {
logger.info("此次采集页面中不包含房间信息字段, 此次页面信息为:{}", htmlBody); logger.info("此次采集页面中不包含房间信息字段, 此次页面信息为:{}", htmlBody);
return null; return null;
} }
}else { }else {
logger.info("此次采集页面中不包含房间信息字段, 此次页面信息为:{}", roomBody); logger.info("此次采集页面中不包含房间信息字段, 此次页面信息为:{}", roomBody);
return null; return null;
} }
} }
} }
src/main/java/com/zhiwei/live/roominfo/HuYaRoomInfoCrawler.java
package com.zhiwei.live.roominfo; package com.zhiwei.live.roominfo;
import org.apache.commons.lang3.StringUtils; import com.zhiwei.crawler.utils.RequestUtils;
import org.apache.logging.log4j.LogManager; import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager;
import org.jsoup.Jsoup; import org.apache.logging.log4j.Logger;
import org.jsoup.nodes.Document; import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import com.zhiwei.crawler.core.HttpBoot;
import com.zhiwei.crawler.core.RequestUtils; import com.zhiwei.crawler.core.HttpBoot;
import com.zhiwei.crawler.proxy.ProxyHolder; import com.zhiwei.crawler.proxy.ProxyHolder;
import com.zhiwei.live.bean.RoomInfo; import com.zhiwei.live.bean.RoomInfo;
public class HuYaRoomInfoCrawler { public class HuYaRoomInfoCrawler {
private static HttpBoot httpBoot = new HttpBoot(); private static HttpBoot httpBoot = new HttpBoot.Builder().build();
private static Logger logger = LogManager.getLogger(HuYaRoomInfoCrawler.class); private static Logger logger = LogManager.getLogger(HuYaRoomInfoCrawler.class);
private static final String PT = "虎牙"; private static final String PT = "虎牙";
/** /**
* 根据房间id获取房间信息 * 根据房间id获取房间信息
* *
* @param roomId * @param roomId
* @return * @return
* @throws Exception * @throws Exception
*/ */
public static RoomInfo getRoomInfoByRoomUrl(String roomUrl) throws Exception { public static RoomInfo getRoomInfoByRoomUrl(String roomUrl) throws Exception {
String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl)).body().string(); String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl)).body().string();
if (!StringUtils.isBlank(htmlBody)) { if (!StringUtils.isBlank(htmlBody)) {
Document document = Jsoup.parse(htmlBody); Document document = Jsoup.parse(htmlBody);
String roomName = document.select("h1#J_roomTitle").text(); String roomName = document.select("h1#J_roomTitle").text();
Integer liveCount = Integer.valueOf(document.select("em#live-count").text().replaceAll(",", "")); Integer liveCount = Integer.valueOf(document.select("em#live-count").text().replaceAll(",", ""));
String username = document.select("h3.host-name").text(); String username = document.select("h3.host-name").text();
String activityCount = document.select("div#activityCount").text(); String activityCount = document.select("div#activityCount").text();
Integer fans = 0; Integer fans = 0;
try { try {
fans = Integer.valueOf(activityCount.replaceAll(",", "")); fans = Integer.valueOf(activityCount.replaceAll(",", ""));
} catch (Exception e) { } catch (Exception e) {
fans = 0; fans = 0;
} }
String room_id = document.select("span.host-rid").text(); String room_id = document.select("span.host-rid").text();
return new RoomInfo(PT, room_id, roomName, username, liveCount, fans); return new RoomInfo(PT, room_id, roomName, username, liveCount, fans);
} else { } else {
logger.info("此次采集页面中不包含房间信息字段, 此次页面信息为:{}", htmlBody); logger.info("此次采集页面中不包含房间信息字段, 此次页面信息为:{}", htmlBody);
return null; return null;
} }
} }
/** /**
* 根据房间id获取房间信息 * 根据房间id获取房间信息
* *
* @param roomId * @param roomId
* @return * @return
* @throws Exception * @throws Exception
*/ */
public static RoomInfo getRoomInfoProxyByRoomUrl(String roomUrl) throws Exception { public static RoomInfo getRoomInfoProxyByRoomUrl(String roomUrl) throws Exception {
String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl), ProxyHolder.NAT_PROXY).body().string(); String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl), ProxyHolder.NAT_HEAVY_PROXY).body().string();
if (!StringUtils.isBlank(htmlBody)) { if (!StringUtils.isBlank(htmlBody)) {
Document document = Jsoup.parse(htmlBody); Document document = Jsoup.parse(htmlBody);
String roomName = document.select("h1#J_roomTitle").text(); String roomName = document.select("h1#J_roomTitle").text();
Integer liveCount = Integer.valueOf(document.select("em#live-count").text().replaceAll(",", "")); Integer liveCount = Integer.valueOf(document.select("em#live-count").text().replaceAll(",", ""));
String username = document.select("h3.host-name").text(); String username = document.select("h3.host-name").text();
String activityCount = document.select("div#activityCount").text(); String activityCount = document.select("div#activityCount").text();
String room_id = document.select("span.host-rid").text(); String room_id = document.select("span.host-rid").text();
Integer fans = 0; Integer fans = 0;
try { try {
fans = Integer.valueOf(activityCount.replaceAll(",", "")); fans = Integer.valueOf(activityCount.replaceAll(",", ""));
} catch (Exception e) { } catch (Exception e) {
fans = 0; fans = 0;
} }
return new RoomInfo(PT, room_id, roomName, username, liveCount, fans); return new RoomInfo(PT, room_id, roomName, username, liveCount, fans);
} else { } else {
logger.info("此次采集页面中不包含房间信息字段, 此次页面信息为:{}", htmlBody); logger.info("此次采集页面中不包含房间信息字段, 此次页面信息为:{}", htmlBody);
return null; return null;
} }
} }
} }
src/test/java/com/zhiwei/live/test/danmu/BilibiliDanMuTest.java
package com.zhiwei.live.test.danmu; package com.zhiwei.live.test.danmu;
import com.zhiwei.live.danmu.bilibili.BilibiliClient; import com.zhiwei.live.danmu.bilibili.BilibiliClient;
import com.zhiwei.live.danmu.bilibili.BilibiliMessage; import com.zhiwei.live.danmu.bilibili.BilibiliMessage;
import com.zhiwei.live.danmu.util.DataCallBack; import com.zhiwei.live.danmu.util.DataCallBack;
public class BilibiliDanMuTest { public class BilibiliDanMuTest {
public static void main(String[] args) throws InterruptedException { public static void main(String[] args) throws InterruptedException {
String roomUrl = "https://live.bilibili.com/529"; String roomUrl = "https://live.bilibili.com/387";
try { try {
BilibiliClient.getDanmu(new DataCallBack() { BilibiliClient.getDanmu(new DataCallBack() {
@Override @Override
public void onData(Object message) { public void onData(Object message) {
if(message instanceof BilibiliMessage) { if(message instanceof BilibiliMessage) {
BilibiliMessage bilibiliMessage = (BilibiliMessage)message; BilibiliMessage bilibiliMessage = (BilibiliMessage)message;
System.out.println("-------------" + bilibiliMessage.toString()); System.out.println("-------------" + bilibiliMessage.toString());
} }
} }
}, roomUrl); }, roomUrl);
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} }
} }
} }
src/test/java/com/zhiwei/live/test/danmu/DouYuDanMuTest.java
package com.zhiwei.live.test.danmu; package com.zhiwei.live.test.danmu;
import com.zhiwei.live.danmu.douyu.DouYuMessage; import com.zhiwei.live.danmu.douyu.DouYuMessage;
import com.zhiwei.live.danmu.douyu.DouyuClient; import com.zhiwei.live.danmu.douyu.DouyuClient;
import com.zhiwei.live.danmu.util.DataCallBack; import com.zhiwei.live.danmu.util.DataCallBack;
public class DouYuDanMuTest { public class DouYuDanMuTest {
public static void main(String[] args) throws InterruptedException { public static void main(String[] args) throws InterruptedException {
String roomUrl = "https://www.douyu.com/71017"; String roomUrl = "https://www.douyu.com/topic/NarutoMatch?rid=535534";
try { try {
DouyuClient.getDanmu(new DataCallBack() { DouyuClient.getDanmu(new DataCallBack() {
@Override @Override
public void onData(Object message) { public void onData(Object message) {
if (message instanceof DouYuMessage) { if (message instanceof DouYuMessage) {
DouYuMessage douyuMessage = (DouYuMessage) message; DouYuMessage douyuMessage = (DouYuMessage) message;
System.out.println("-------------" + douyuMessage.toString()); System.out.println("-------------" + douyuMessage.toString());
} }
} }
}, roomUrl); }, roomUrl);
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} }
} }
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment