Skip to content

L
live-crawler
Commit 87226592 by zhiwei
Options

B站弹幕获取

parent fe6e0d5b
Showing with 791 additions and 858 deletions
pom.xml
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.zhiwei.live</groupId>
<artifactId>live-crawler</artifactId>
<version>0.0.1-SNAPSHOT</version>
<dependencies>
<dependency>
<groupId>com.zhiwei.tools</groupId>
<artifactId>zhiwei-tools</artifactId>
<version>0.1.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.zhiwei.crawler</groupId>
<artifactId>crawler-core</artifactId>
<version>0.1.1-RELEASE</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.8.1</version>
</dependency>
<!-- https://mvnrepository.com/artifact/io.netty/netty-all -->
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
<version>4.1.33.Final</version>
</dependency>
</dependencies>
<!-- 打包管理 -->
<build>
<plugins>
<!-- 发布源码 -->
<plugin>
<artifactId>maven-source-plugin</artifactId>
<version>2.4</version>
<configuration>
<attach>true</attach>
</configuration>
<executions>
<execution>
<phase>compile</phase>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>2.10.4</version>
</plugin>
<!-- 解决maven test命令时console出现中文乱码乱码 -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.7.2</version>
<configuration>
<forkMode>once</forkMode>
<argLine>-Dfile.encoding=UTF-8</argLine>
</configuration>
</plugin>
</plugins>
</build>
<!-- 分发管理:管理distribution和supporting files -->
<distributionManagement>
<snapshotRepository>
<id>nexus-releases</id>
<name>User Porject Snapshot</name>
<url>http://192.168.0.30:8081/nexus/content/repositories/snapshots/</url>
<uniqueVersion>true</uniqueVersion>
</snapshotRepository>
<repository>
<id>nexus-releases</id>
<name>User Porject Release</name>
<url>http://192.168.0.30:8081/nexus/content/repositories/releases/</url>
</repository>
</distributionManagement>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.zhiwei.live</groupId>
<artifactId>live-crawler</artifactId>
<version>0.0.1-SNAPSHOT</version>
<dependencies>
<dependency>
<groupId>com.zhiwei.tools</groupId>
<artifactId>zhiwei-tools</artifactId>
<version>0.1.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.zhiwei.crawler</groupId>
<artifactId>crawler-core</artifactId>
<version>0.6.2.1-RELEASE</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.8.1</version>
</dependency>
<!-- https://mvnrepository.com/artifact/io.netty/netty-all -->
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
<version>4.1.48.Final</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.12</version>
</dependency>
</dependencies>
<!-- 打包管理 -->
<build>
<plugins>
<!-- 发布源码 -->
<plugin>
<artifactId>maven-source-plugin</artifactId>
<version>2.4</version>
<configuration>
<attach>true</attach>
</configuration>
<executions>
<execution>
<phase>compile</phase>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>2.10.4</version>
</plugin>
<!-- 解决maven test命令时console出现中文乱码乱码 -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.7.2</version>
<configuration>
<forkMode>once</forkMode>
<argLine>-Dfile.encoding=UTF-8</argLine>
</configuration>
</plugin>
</plugins>
</build>
<!-- 分发管理:管理distribution和supporting files -->
<distributionManagement>
<snapshotRepository>
<id>nexus-releases</id>
<name>User Porject Snapshot</name>
<url>http://192.168.0.30:8081/nexus/content/repositories/snapshots/</url>
<uniqueVersion>true</uniqueVersion>
</snapshotRepository>
<repository>
<id>nexus-releases</id>
<name>User Porject Release</name>
<url>http://192.168.0.30:8081/nexus/content/repositories/releases/</url>
</repository>
</distributionManagement>
</project>
\ No newline at end of file
src/main/java/com/zhiwei/live/bean/RoomInfo.java
package com.zhiwei.live.bean;
public class RoomInfo {
String pt; // 平台类型
String roomId; // 房间号
String nickName; // 主播昵称
String roomName; // 房间名称
Integer hotNum; // 直播间热度
Integer fans; // 订阅数
public RoomInfo() {
}
public RoomInfo(String pt, String roomId, String nickName, String roomName, Integer hotNum, Integer fans) {
this.pt = pt;
this.roomId = roomId;
this.nickName = nickName;
this.roomName = roomName;
this.hotNum = hotNum;
this.fans = fans;
}
@Override
public String toString() {
return "new RoomInfo[" + "pt = " + pt + ", roomId = " + roomId + ", roomName = " + roomName + ", nickName = "
+ nickName + ", hotNum = " + hotNum + ", fans = " + fans + "]";
}
public Integer getFans() {
return fans;
}
public void setHotNum(Integer hotNum) {
this.hotNum = hotNum;
}
public void setFans(Integer fans) {
this.fans = fans;
}
public String getPt() {
return pt;
}
public String getRoomId() {
return roomId;
}
public String getNickName() {
return nickName;
}
public String getRoomName() {
return roomName;
}
public int getHotNum() {
return hotNum;
}
public void setPt(String pt) {
this.pt = pt;
}
public void setRoomId(String roomId) {
this.roomId = roomId;
}
public void setNickName(String nickName) {
this.nickName = nickName;
}
public void setRoomName(String roomName) {
this.roomName = roomName;
}
public void setHotNum(int hotNum) {
this.hotNum = hotNum;
}
}
package com.zhiwei.live.bean;
import lombok.Data;
import lombok.ToString;
@Data
@ToString
public class RoomInfo {
/**
* 平台类型
*/
String pt;
/**
* 房间号
*/
String roomId;
/**
* 主播昵称
*/
String nickName;
/**
* 房间名称
*/
String roomName;
/**
* 直播间热度
*/
Integer hotNum;
/**
* 订阅数
*/
Integer fans;
public RoomInfo() {
}
public RoomInfo(String pt, String roomId, String nickName, String roomName, Integer hotNum, Integer fans) {
this.pt = pt;
this.roomId = roomId;
this.nickName = nickName;
this.roomName = roomName;
this.hotNum = hotNum;
this.fans = fans;
}
}
src/main/java/com/zhiwei/live/danmu/bilibili/BilibiliClient.java
package com.zhiwei.live.danmu.bilibili;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.crawler.core.HttpBoot;
import com.zhiwei.crawler.core.RequestUtils;
import com.zhiwei.live.bean.RoomInfo;
import com.zhiwei.live.danmu.util.Connector;
import com.zhiwei.live.danmu.util.DataCallBack;
import com.zhiwei.live.roominfo.BilibiliRoomInfoCrawler;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.handler.timeout.IdleStateHandler;
import okhttp3.Request;
import okhttp3.Response;
public class BilibiliClient {
private static HttpBoot httpBoot = new HttpBoot();
private static Logger logger = LogManager.getLogger(BilibiliClient.class);
private static final int PORT = 2243;
/**
* 根据房间号获取弹幕信息
* @param roomId
* @throws Exception
*/
public static void getDanmu(DataCallBack dataCallBack,String roomUrl) throws Exception {
//根据房间号获取真实房间号
RoomInfo roomInfo = BilibiliRoomInfoCrawler.getRoomInfoByRoomUrl(roomUrl);
if(Objects.nonNull(roomInfo)) {
//获取弹幕服务器地址
String url = "https://api.live.bilibili.com/room/v1/Danmu/getConf?room_id=" + roomInfo.getRoomId();
Request request = RequestUtils.wrapGet(url);
String host = null;
try(Response response = httpBoot.syncCall(request)) {
String result = response.body().string();
JSONObject json = JSONObject.parseObject(result);
host = json.getJSONObject("data").getJSONArray("host_server_list").getJSONObject(0).getString("host");
} catch(Exception e) {
throw new IllegalArgumentException("获取聊天服务器地址失败", e);
}
//建立弹幕连接
Connector.asynchronizedTcpConnect(new NioEventLoopGroup(), host, PORT,
new IdleStateHandler(0, 30, 0,TimeUnit.SECONDS), new BilibiliMessageHandler(dataCallBack,roomInfo.getRoomId())).sync();
}else {
logger.info("获取真实房间号出现问题,请及时检查程序");
}
}
}
package com.zhiwei.live.danmu.bilibili;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
import com.zhiwei.crawler.utils.RequestUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.crawler.core.HttpBoot;
import com.zhiwei.live.bean.RoomInfo;
import com.zhiwei.live.danmu.util.Connector;
import com.zhiwei.live.danmu.util.DataCallBack;
import com.zhiwei.live.roominfo.BilibiliRoomInfoCrawler;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.handler.timeout.IdleStateHandler;
import okhttp3.Request;
import okhttp3.Response;
public class BilibiliClient {
private static HttpBoot httpBoot = new HttpBoot.Builder().build();
private static Logger logger = LogManager.getLogger(BilibiliClient.class);
private static final int PORT = 2243;
/**
* 根据房间号获取弹幕信息
* @param roomUrl
* @throws Exception
*/
public static void getDanmu(DataCallBack dataCallBack,String roomUrl) throws Exception {
//根据房间号获取真实房间号
RoomInfo roomInfo = BilibiliRoomInfoCrawler.getRoomInfoByRoomUrl(roomUrl);
if(Objects.nonNull(roomInfo)) {
//获取弹幕服务器地址
String url = "https://api.live.bilibili.com/room/v1/Danmu/getConf?room_id=" + roomInfo.getRoomId();
Request request = RequestUtils.wrapGet(url);
String host = null;
try(Response response = httpBoot.syncCall(request)) {
String result = response.body().string();
JSONObject json = JSONObject.parseObject(result);
host = json.getJSONObject("data").getJSONArray("host_server_list").getJSONObject(0).getString("host");
} catch(Exception e) {
throw new IllegalArgumentException("获取聊天服务器地址失败", e);
}
//建立弹幕连接
Connector.asynchronizedTcpConnect(new NioEventLoopGroup(), host, PORT,
new IdleStateHandler(0, 30, 0,TimeUnit.SECONDS), new BilibiliMessageHandler(dataCallBack,roomInfo.getRoomId())).sync();
}else {
logger.info("获取真实房间号出现问题,请及时检查程序");
}
}
}
src/main/java/com/zhiwei/live/danmu/bilibili/BilibiliMessage.java
package com.zhiwei.live.danmu.bilibili;
import java.util.Date;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
public class BilibiliMessage {
String messageType; //弹幕消息类型
String user_id; //发布者uid
String nickName; //发布者昵称
Date time; //弹幕时间
String content; //弹幕内容
String room_id; //房间id
public BilibiliMessage(JSONObject json) throws Exception {
constructJson(json);
}
public BilibiliMessage(){
}
private void constructJson(JSONObject json) throws Exception{
try {
System.out.println(json);
JSONArray jsonArray = json.getJSONArray("info");
messageType = json.getString("cmd");
user_id = jsonArray.getJSONArray(2).getString(0);
time = new Date();
nickName = jsonArray.getJSONArray(2).getString(1);
content = jsonArray.getString(1);
} catch (Exception e) {
throw new Exception();
}
}
@Override
public String toString() {
return "new BilibiliMessage["
+ " user_id = " + user_id
+ ", nickName = " + nickName
+ ", messageType = " + messageType
+ ", time = " + time
+ ", content = " + content
+ ", room_id = " + room_id
+ "]";
}
public Date getTime() {
return time;
}
public void setTime(Date time) {
this.time = time;
}
public String getMessageType() {
return messageType;
}
public String getUser_id() {
return user_id;
}
public String getNickName() {
return nickName;
}
public String getContent() {
return content;
}
public void setMessageType(String messageType) {
this.messageType = messageType;
}
public void setUser_id(String user_id) {
this.user_id = user_id;
}
public void setNickName(String nickName) {
this.nickName = nickName;
}
public void setContent(String content) {
this.content = content;
}
public String getRoom_id() {
return room_id;
}
public void setRoom_id(String room_id) {
this.room_id = room_id;
}
}
package com.zhiwei.live.danmu.bilibili;
import java.util.Date;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import lombok.Data;
import lombok.ToString;
@Data
@ToString
public class BilibiliMessage {
/**
* 弹幕消息类型
*/
String messageType;
/**
* 发布者uid
*/
String userId;
/**
* 发布者昵称
*/
String nickName;
/**
* 弹幕时间
*/
Date time;
/**
* 弹幕内容
*/
String content;
/**
* 房间id
*/
String roomId;
public BilibiliMessage(JSONObject json) throws Exception {
constructJson(json);
}
public BilibiliMessage(){
}
private void constructJson(JSONObject json) throws Exception{
try {
JSONArray jsonArray = json.getJSONArray("info");
messageType = json.getString("cmd");
userId = jsonArray.getJSONArray(2).getString(0);
time = new Date();
nickName = jsonArray.getJSONArray(2).getString(1);
content = jsonArray.getString(1);
} catch (Exception e) {
throw new Exception();
}
}
}
src/main/java/com/zhiwei/live/danmu/douyu/DouyuClient.java
......@@ -22,8 +22,8 @@ import io.netty.handler.timeout.IdleStateHandler;
public class DouyuClient {
private static Logger logger = LogManager.getLogger(DouyuClient.class);
private static final int PORT = 8601;
private static final String HOST = "openbarrage.douyutv.com";
private static final int PORT = 8504;
private static final String HOST = "danmuproxy.douyu.com";
/**
* 根据房间号获取弹幕信息
......
src/main/java/com/zhiwei/live/danmu/douyu/DouyuMessageHandler.java
......@@ -70,7 +70,7 @@ public class DouyuMessageHandler extends ChannelInboundHandlerAdapter {
if (msg instanceof ByteBuf) {
ByteBuf buf = (ByteBuf) msg;
String source = buf.toString(CharsetUtil.UTF_8);
// System.out.println("source========="+source);
System.out.println("source========="+source);
if(source.contains("chatmsg")) {
Map<String,Object> messageMap = DouYuUtil.toMap(source);
String data = JSONObject.toJSONString(messageMap);
......@@ -84,6 +84,10 @@ public class DouyuMessageHandler extends ChannelInboundHandlerAdapter {
ByteBuf groupMsg = buildMsg(StringUtils.join("type@=joingroup/rid@=",roomId,"/gid@=-9999/"));
System.out.println("发送入组消息: \n" + ByteBufUtil.prettyHexDump(groupMsg));
ctx.writeAndFlush(groupMsg);
// ByteBuf giftMsg = buildMsg(StringUtils.join("type@=dmfbdreq/dfl@=sn@AA=105@ASss@AA=0@AS@Ssn@AA=106@ASss@AA=0@AS@Ssn@AA=107@ASss@AA=0@AS@Ssn@AA=108@ASss@AA=0@AS@Ssn@AA=110@ASss@AA=0@AS@S/"));
// System.out.println("发送礼物消息: \n" + ByteBufUtil.prettyHexDump(giftMsg));
// ctx.writeAndFlush(giftMsg);
finish = true;
}
ctx.fireChannelRead(msg);
......@@ -97,7 +101,7 @@ public class DouyuMessageHandler extends ChannelInboundHandlerAdapter {
if (evt instanceof IdleStateEvent) {
IdleStateEvent event = (IdleStateEvent) evt;
if (event.state() == IdleState.WRITER_IDLE) {
ByteBuf pingMsg = buildMsg(StringUtils.join("type@=keeplive/tick@=", System.currentTimeMillis() / 1000, "/"));
ByteBuf pingMsg = buildMsg(StringUtils.join("type@=mrkl/"));
System.out.println("发送心跳消息: \n" + ByteBufUtil.prettyHexDump(pingMsg));
ctx.writeAndFlush(pingMsg);
} else {
......@@ -109,7 +113,7 @@ public class DouyuMessageHandler extends ChannelInboundHandlerAdapter {
}
private ByteBuf buildMsg(String content) {
int fixedLen = 4 + 4 + 1;
int fixedLen = 4 + 4 + 1 + content.length();
byte[] body = content.getBytes(CharsetUtil.UTF_8);
int length = fixedLen + body.length;
return Unpooled.buffer(length).writeIntLE(length).writeIntLE(length).writeShortLE(689)
......
src/main/java/com/zhiwei/live/danmu/pandam/PandamClient.java
package com.zhiwei.live.danmu.pandam;
import java.util.Objects;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.crawler.core.HttpBoot;
import com.zhiwei.crawler.core.RequestUtils;
import com.zhiwei.live.bean.RoomInfo;
import com.zhiwei.live.danmu.util.Connector;
import com.zhiwei.live.danmu.util.DataCallBack;
import com.zhiwei.live.roominfo.PandamTVRoomInfoCrawler;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.handler.timeout.IdleStateHandler;
import okhttp3.Request;
import okhttp3.Response;
/**
* @ClassName: PandaClient
* @Description: TODO
* @author 0xff
* @date Jan 25, 2019 3:24:33 PM
*/
public class PandamClient {
private static HttpBoot httpBoot = new HttpBoot();
/**
* 根据房间号获取弹幕信息
* @param roomId
* @throws Exception
*/
public static void getDanmu(DataCallBack dataCallBack,String roomUrl) throws Exception {
//根据房间号获取弹幕服务器地址
RoomInfo roomInfo = PandamTVRoomInfoCrawler.getRoomInfoByRoomUrl(roomUrl);
if(Objects.nonNull(roomInfo)) {
String roomId = roomInfo.getRoomId();
Request request = RequestUtils.wrapGet("http://riven.panda.tv/chatroom/getinfo?roomid=" + roomId);
String host = null;
int port = -1;
JSONObject json = null;
try(Response response = httpBoot.syncCall(request)) {
json = JSON.parseObject(response.body().string());
if(json.getInteger("errno") != 0) {
throw new IllegalStateException("房间号: " + roomUrl + " 不存在");
}
json = json.getJSONObject("data");
String[] address = json.getJSONArray("chat_addr_list").getString(0).split(":");
host = address[0];
port = Integer.parseInt(address[1]);
} catch(Exception e) {
throw new IllegalArgumentException("获取聊天服务器地址失败", e);
}
String appid = json.getString("appid");
String rid = json.getString("rid");
String sign = json.getString("sign");
String authType = json.getString("authType");
String ts = json.getString("ts");
Connector.asynchronizedTcpConnect(new NioEventLoopGroup(), host, port,
new IdleStateHandler(0, 30, 45), new PandamMessageHandler(dataCallBack, appid, rid, ts, sign, authType)).sync();
}else {
}
}
}
package com.zhiwei.live.danmu.pandam;
import java.util.Objects;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.crawler.core.HttpBoot;
import com.zhiwei.crawler.utils.RequestUtils;
import com.zhiwei.live.bean.RoomInfo;
import com.zhiwei.live.danmu.util.Connector;
import com.zhiwei.live.danmu.util.DataCallBack;
import com.zhiwei.live.roominfo.PandamTVRoomInfoCrawler;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.handler.timeout.IdleStateHandler;
import okhttp3.Request;
import okhttp3.Response;
/**
* @ClassName: PandaClient
* @Description: TODO
* @author 0xff
* @date Jan 25, 2019 3:24:33 PM
*/
public class PandamClient {
private static HttpBoot httpBoot = new HttpBoot.Builder().build();
/**
* 根据房间号获取弹幕信息
* @param roomId
* @throws Exception
*/
public static void getDanmu(DataCallBack dataCallBack,String roomUrl) throws Exception {
//根据房间号获取弹幕服务器地址
RoomInfo roomInfo = PandamTVRoomInfoCrawler.getRoomInfoByRoomUrl(roomUrl);
if(Objects.nonNull(roomInfo)) {
String roomId = roomInfo.getRoomId();
Request request = RequestUtils.wrapGet("http://riven.panda.tv/chatroom/getinfo?roomid=" + roomId);
String host = null;
int port = -1;
JSONObject json = null;
try(Response response = httpBoot.syncCall(request)) {
json = JSON.parseObject(response.body().string());
if(json.getInteger("errno") != 0) {
throw new IllegalStateException("房间号: " + roomUrl + " 不存在");
}
json = json.getJSONObject("data");
String[] address = json.getJSONArray("chat_addr_list").getString(0).split(":");
host = address[0];
port = Integer.parseInt(address[1]);
} catch(Exception e) {
throw new IllegalArgumentException("获取聊天服务器地址失败", e);
}
String appid = json.getString("appid");
String rid = json.getString("rid");
String sign = json.getString("sign");
String authType = json.getString("authType");
String ts = json.getString("ts");
Connector.asynchronizedTcpConnect(new NioEventLoopGroup(), host, port,
new IdleStateHandler(0, 30, 45), new PandamMessageHandler(dataCallBack, appid, rid, ts, sign, authType)).sync();
}else {
}
}
}
src/main/java/com/zhiwei/live/danmu/pandam/PandamMessageHandler.java
package com.zhiwei.live.danmu.pandam;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.lang3.StringUtils;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.live.danmu.util.DataCallBack;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufUtil;
import io.netty.buffer.Unpooled;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInboundHandlerAdapter;
import io.netty.handler.timeout.IdleState;
import io.netty.handler.timeout.IdleStateEvent;
import io.netty.util.CharsetUtil;
import io.netty.util.ReferenceCountUtil;
/**
* @ClassName: PandaMessageHandler
* @Description: TODO
* @author 0xff
* @date Jan 25, 2019 11:03:20 AM
*/
public class PandamMessageHandler extends ChannelInboundHandlerAdapter {
private static final byte[] FIRST_REQ = new byte[] { 0x00, 0x06, 0x00, 0x02, 0x00 };
private static final byte[] FIRST_RES = new byte[] { 0x00, 0x06, 0x00, 0x06 };
private static final byte[] PING = new byte[] { 0x00, 0x06, 0x00, 0x00 };
private static final byte[] REQ = new byte[] { 0x00, 0x06, 0x00, 0x03 };
private static Pattern pattern = Pattern.compile("\\{\"type\":\"1\".+?\\}\\}");
private String rid;
private String appid;
private String ts;
private String sign;
private String authType;
private DataCallBack dataCallBack;
/**
* Constructor
*
* @param roomId
* 房间号
*/
public PandamMessageHandler(DataCallBack dataCallBack,String appid, String rid, String ts, String sign, String authType) {
this.dataCallBack = dataCallBack;
this.appid = appid;
this.rid = rid;
this.sign = sign;
this.authType = authType;
this.ts = ts;
}
/*
* @see
* io.netty.channel.ChannelInboundHandlerAdapter#channelActive(io.netty.channel.
* ChannelHandlerContext)
*/
@Override
public void channelActive(ChannelHandlerContext ctx) throws Exception {
System.out.println("TCP 连接建立成功: " + ctx.channel());
byte[] body = StringUtils
.join("u:", rid, "@", appid, "\nk:1\nt:300\nts:", ts, "\nsign:", sign, "\nauthtype:", authType)
.getBytes();
ByteBuf loginMsg = Unpooled.wrappedBuffer(Unpooled.wrappedBuffer(FIRST_REQ),
Unpooled.buffer(1).writeByte(body.length), Unpooled.wrappedBuffer(body));
System.out.println("发送登录消息: \n" + ByteBufUtil.prettyHexDump(loginMsg));
ctx.writeAndFlush(loginMsg);
}
/*
* @see
* io.netty.channel.ChannelInboundHandlerAdapter#channelRead(io.netty.channel.
* ChannelHandlerContext, java.lang.Object)
*/
@Override
public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
if (msg instanceof ByteBuf) {
ByteBuf buf = (ByteBuf) msg;
// System.out.println("收到消息: \n" + ByteBufUtil.prettyHexDump(buf));
String source = buf.toString(CharsetUtil.UTF_8);
Matcher matcher = pattern.matcher(source);
while(matcher.find()) {
JSONObject dataJson = JSONObject.parseObject(matcher.group());
PandamMessage pandamMessage = new PandamMessage(dataJson);
dataCallBack.onData(pandamMessage);
}
}
ReferenceCountUtil.release(msg);
}
/*
* @see
* io.netty.channel.ChannelInboundHandlerAdapter#userEventTriggered(io.netty.
* channel.ChannelHandlerContext, java.lang.Object)
*/
@Override
public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception {
if (evt instanceof IdleStateEvent) {
IdleStateEvent event = (IdleStateEvent) evt;
if (event.state() == IdleState.WRITER_IDLE) {
ByteBuf pingMsg = Unpooled.wrappedBuffer(PING);
System.out.println("发送心跳消息: \n" + ByteBufUtil.prettyHexDump(pingMsg));
ctx.writeAndFlush(pingMsg);
} else {
ctx.fireUserEventTriggered(evt);
}
} else {
ctx.fireUserEventTriggered(evt);
}
}
}
package com.zhiwei.live.danmu.pandam;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import lombok.extern.log4j.Log4j2;
import org.apache.commons.lang3.StringUtils;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.live.danmu.util.DataCallBack;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufUtil;
import io.netty.buffer.Unpooled;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInboundHandlerAdapter;
import io.netty.handler.timeout.IdleState;
import io.netty.handler.timeout.IdleStateEvent;
import io.netty.util.CharsetUtil;
import io.netty.util.ReferenceCountUtil;
/**
* @ClassName: PandaMessageHandler
* @Description: TODO
* @author 0xff
* @date Jan 25, 2019 11:03:20 AM
*/
@Log4j2
public class PandamMessageHandler extends ChannelInboundHandlerAdapter {
private static final byte[] FIRST_REQ = new byte[] { 0x00, 0x06, 0x00, 0x02, 0x00 };
private static final byte[] FIRST_RES = new byte[] { 0x00, 0x06, 0x00, 0x06 };
private static final byte[] PING = new byte[] { 0x00, 0x06, 0x00, 0x00 };
private static final byte[] REQ = new byte[] { 0x00, 0x06, 0x00, 0x03 };
private static Pattern pattern = Pattern.compile("\\{\"type\":\"1\".+?\\}\\}");
private String rid;
private String appid;
private String ts;
private String sign;
private String authType;
private DataCallBack dataCallBack;
/**
* Constructor
*
* @param roomId
* 房间号
*/
public PandamMessageHandler(DataCallBack dataCallBack,String appid, String rid, String ts, String sign, String authType) {
this.dataCallBack = dataCallBack;
this.appid = appid;
this.rid = rid;
this.sign = sign;
this.authType = authType;
this.ts = ts;
}
/*
* @see
* io.netty.channel.ChannelInboundHandlerAdapter#channelActive(io.netty.channel.
* ChannelHandlerContext)
*/
@Override
public void channelActive(ChannelHandlerContext ctx) throws Exception {
System.out.println("TCP 连接建立成功: " + ctx.channel());
byte[] body = StringUtils
.join("u:", rid, "@", appid, "\nk:1\nt:300\nts:", ts, "\nsign:", sign, "\nauthtype:", authType)
.getBytes();
ByteBuf loginMsg = Unpooled.wrappedBuffer(Unpooled.wrappedBuffer(FIRST_REQ),
Unpooled.buffer(1).writeByte(body.length), Unpooled.wrappedBuffer(body));
System.out.println("发送登录消息: \n" + ByteBufUtil.prettyHexDump(loginMsg));
ctx.writeAndFlush(loginMsg);
}
/*
* @see
* io.netty.channel.ChannelInboundHandlerAdapter#channelRead(io.netty.channel.
* ChannelHandlerContext, java.lang.Object)
*/
@Override
public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
if (msg instanceof ByteBuf) {
ByteBuf buf = (ByteBuf) msg;
// System.out.println("收到消息: \n" + ByteBufUtil.prettyHexDump(buf));
String source = buf.toString(CharsetUtil.UTF_8);
Matcher matcher = pattern.matcher(source);
while(matcher.find()) {
JSONObject dataJson = JSONObject.parseObject(matcher.group());
PandamMessage pandamMessage = new PandamMessage(dataJson);
dataCallBack.onData(pandamMessage);
}
}
ReferenceCountUtil.release(msg);
}
/*
* @see
* io.netty.channel.ChannelInboundHandlerAdapter#userEventTriggered(io.netty.
* channel.ChannelHandlerContext, java.lang.Object)
*/
@Override
public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception {
if (evt instanceof IdleStateEvent) {
IdleStateEvent event = (IdleStateEvent) evt;
if (event.state() == IdleState.WRITER_IDLE) {
ByteBuf pingMsg = Unpooled.wrappedBuffer(PING);
System.out.println("发送心跳消息: \n" + ByteBufUtil.prettyHexDump(pingMsg));
ctx.writeAndFlush(pingMsg);
} else {
ctx.fireUserEventTriggered(evt);
}
} else {
ctx.fireUserEventTriggered(evt);
}
}
}
src/main/java/com/zhiwei/live/roominfo/BilibiliRoomInfoCrawler.java
package com.zhiwei.live.roominfo;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.crawler.core.HttpBoot;
import com.zhiwei.crawler.core.RequestUtils;
import com.zhiwei.crawler.proxy.ProxyHolder;
import com.zhiwei.live.bean.RoomInfo;
import com.zhiwei.tools.tools.ZhiWeiTools;
/**
* bilibili 直播间信息爬取
* @author qq859
*
*/
public class BilibiliRoomInfoCrawler {
private static HttpBoot httpBoot = new HttpBoot();
private static Logger logger = LogManager.getLogger(BilibiliRoomInfoCrawler.class);
private static final String PT = "B站";
/**
* 根据房间id获取房间信息
* @param roomId
* @return
* @throws Exception
*/
public static RoomInfo getRoomInfoByRoomUrl(String roomUrl) throws Exception{
String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl)).body().string();
if(!StringUtils.isBlank(htmlBody)) {
//判断页面中是否包含房间信息
if(htmlBody.contains("window.__NEPTUNE_IS_MY_WAIFU__=")) {
//通过截取获取直播间信息字段，将截取的字段处理为json格式方便解析
htmlBody = htmlBody.split("<script>window.__NEPTUNE_IS_MY_WAIFU__=")[1].split("</script>")[0];
htmlBody = ZhiWeiTools.decodeUnicode(htmlBody);
htmlBody = ZhiWeiTools.delHTMLTag(htmlBody);
htmlBody = htmlBody.replaceAll("\\\\", "").replaceAll("'", "\"");
//解析json数据
JSONObject baseInfoRes = JSONObject.parseObject(htmlBody);
JSONObject data = baseInfoRes.getJSONObject("baseInfoRes").getJSONObject("data");
Integer person_num = data.getIntValue("online");
String roomname = data.getString("title");
Integer room_id = data.getInteger("room_id");
Integer fans = data.getInteger("attention");
String roomId = room_id!=null?room_id.toString():null;
String username = null;
//通过房间id获取用户信息
roomUrl = "https://api.live.bilibili.com/live_user/v1/UserInfo/get_anchor_in_room?roomid="+roomId;
String roomBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl)).body().string();
if(!StringUtils.isBlank(roomBody)) {
JSONObject roomData = JSONObject.parseObject(roomBody).getJSONObject("data");
username = roomData.getJSONObject("info").getString("uname");
}
return new RoomInfo(PT, roomId, roomname, username, person_num, fans);
}else {
logger.info("此次采集页面中不包含房间信息字段, 此次页面信息为:{}", htmlBody);
return null;
}
}
return null;
}
public static RoomInfo getRoomInfoProxyByRoomUrl(String roomUrl) throws Exception{
String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl), ProxyHolder.NAT_PROXY).body().string();
if(!StringUtils.isBlank(htmlBody)) {
//判断页面中是否包含房间信息
if(htmlBody.contains("window.__NEPTUNE_IS_MY_WAIFU__=")) {
//通过截取获取直播间信息字段，将截取的字段处理为json格式方便解析
htmlBody = htmlBody.split("<script>window.__NEPTUNE_IS_MY_WAIFU__=")[1].split("</script>")[0];
htmlBody = ZhiWeiTools.decodeUnicode(htmlBody);
htmlBody = ZhiWeiTools.delHTMLTag(htmlBody);
htmlBody = htmlBody.replaceAll("\\\\", "").replaceAll("'", "\"");
//解析json数据
JSONObject baseInfoRes = JSONObject.parseObject(htmlBody);
JSONObject data = baseInfoRes.getJSONObject("baseInfoRes").getJSONObject("data");
Integer person_num = data.getIntValue("online");
String roomname = data.getString("title");
Integer room_id = data.getInteger("room_id");
String roomId = room_id!=null?room_id.toString():null;
Integer fans = data.getInteger("attention");
String username = null;
//通过房间id获取用户信息
roomUrl = "https://api.live.bilibili.com/live_user/v1/UserInfo/get_anchor_in_room?roomid="+room_id;
String roomBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl), ProxyHolder.NAT_PROXY).body().string();
if(!StringUtils.isBlank(roomBody)) {
JSONObject roomData = JSONObject.parseObject(roomBody).getJSONObject("data");
username = roomData.getJSONObject("info").getString("uname");
}
return new RoomInfo(PT, roomId, roomname, username, person_num, fans);
}else {
logger.info("此次采集页面中不包含房间信息字段, 此次页面信息为:{}", htmlBody);
return null;
}
}
return null;
}
}
package com.zhiwei.live.roominfo;
import com.zhiwei.crawler.utils.RequestUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.crawler.core.HttpBoot;
import com.zhiwei.crawler.proxy.ProxyHolder;
import com.zhiwei.live.bean.RoomInfo;
import com.zhiwei.tools.tools.ZhiWeiTools;
/**
* bilibili 直播间信息爬取
* @author qq859
*
*/
public class BilibiliRoomInfoCrawler {
private static HttpBoot httpBoot = new HttpBoot.Builder().build();
private static Logger logger = LogManager.getLogger(BilibiliRoomInfoCrawler.class);
private static final String PT = "B站";
/**
* 根据房间id获取房间信息
* @param roomId
* @return
* @throws Exception
*/
public static RoomInfo getRoomInfoByRoomUrl(String roomUrl) throws Exception{
String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl)).body().string();
if(!StringUtils.isBlank(htmlBody)) {
//判断页面中是否包含房间信息
if(htmlBody.contains("window.__NEPTUNE_IS_MY_WAIFU__=")) {
//通过截取获取直播间信息字段，将截取的字段处理为json格式方便解析
htmlBody = htmlBody.split("<script>window.__NEPTUNE_IS_MY_WAIFU__=")[1].split("</script>")[0];
htmlBody = ZhiWeiTools.decodeUnicode(htmlBody);
htmlBody = ZhiWeiTools.delHTMLTag(htmlBody);
htmlBody = htmlBody.replaceAll("\\\\", "").replaceAll("'", "\"");
//解析json数据
JSONObject baseInfoRes = JSONObject.parseObject(htmlBody);
JSONObject data = baseInfoRes.getJSONObject("baseInfoRes").getJSONObject("data");
Integer person_num = data.getIntValue("online");
String roomname = data.getString("title");
Integer room_id = data.getInteger("room_id");
Integer fans = data.getInteger("attention");
String roomId = room_id!=null?room_id.toString():null;
String username = null;
//通过房间id获取用户信息
roomUrl = "https://api.live.bilibili.com/live_user/v1/UserInfo/get_anchor_in_room?roomid="+roomId;
String roomBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl)).body().string();
if(!StringUtils.isBlank(roomBody)) {
JSONObject roomData = JSONObject.parseObject(roomBody).getJSONObject("data");
username = roomData.getJSONObject("info").getString("uname");
}
return new RoomInfo(PT, roomId, roomname, username, person_num, fans);
}else {
logger.info("此次采集页面中不包含房间信息字段, 此次页面信息为:{}", htmlBody);
return null;
}
}
return null;
}
public static RoomInfo getRoomInfoProxyByRoomUrl(String roomUrl) throws Exception{
String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl), ProxyHolder.NAT_HEAVY_PROXY).body().string();
if(!StringUtils.isBlank(htmlBody)) {
//判断页面中是否包含房间信息
if(htmlBody.contains("window.__NEPTUNE_IS_MY_WAIFU__=")) {
//通过截取获取直播间信息字段，将截取的字段处理为json格式方便解析
htmlBody = htmlBody.split("<script>window.__NEPTUNE_IS_MY_WAIFU__=")[1].split("</script>")[0];
htmlBody = ZhiWeiTools.decodeUnicode(htmlBody);
htmlBody = ZhiWeiTools.delHTMLTag(htmlBody);
htmlBody = htmlBody.replaceAll("\\\\", "").replaceAll("'", "\"");
//解析json数据
JSONObject baseInfoRes = JSONObject.parseObject(htmlBody);
JSONObject data = baseInfoRes.getJSONObject("baseInfoRes").getJSONObject("data");
Integer person_num = data.getIntValue("online");
String roomname = data.getString("title");
Integer room_id = data.getInteger("room_id");
String roomId = room_id!=null?room_id.toString():null;
Integer fans = data.getInteger("attention");
String username = null;
//通过房间id获取用户信息
roomUrl = "https://api.live.bilibili.com/live_user/v1/UserInfo/get_anchor_in_room?roomid="+room_id;
String roomBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl), ProxyHolder.NAT_HEAVY_PROXY).body().string();
if(!StringUtils.isBlank(roomBody)) {
JSONObject roomData = JSONObject.parseObject(roomBody).getJSONObject("data");
username = roomData.getJSONObject("info").getString("uname");
}
return new RoomInfo(PT, roomId, roomname, username, person_num, fans);
}else {
logger.info("此次采集页面中不包含房间信息字段, 此次页面信息为:{}", htmlBody);
return null;
}
}
return null;
}
}
src/main/java/com/zhiwei/live/roominfo/DouYuRoomInfoCrawler.java
package com.zhiwei.live.roominfo;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.crawler.core.HttpBoot;
import com.zhiwei.crawler.core.RequestUtils;
import com.zhiwei.crawler.proxy.ProxyHolder;
import com.zhiwei.live.bean.RoomInfo;
/**
* 斗鱼直播间信息获取
* @author qq859
*
*/
public class DouYuRoomInfoCrawler {
private static HttpBoot httpBoot = new HttpBoot();
private static Logger logger = LogManager.getLogger(DouYuRoomInfoCrawler.class);
private static final String PT = "斗鱼";
/**
* 根据房间id获取房间信息
* @param roomId
* @return
* @throws Exception
*/
public static RoomInfo getRoomInfoByRoomUrl(String roomUrl) throws Exception{
String roomBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl)).body().string();
if(!StringUtils.isBlank(roomBody) && roomBody.contains("ROOM.room_id =")) {
String roomId = roomBody.split("ROOM\\.room_id = ")[1].split("; ")[0].trim();
//获取房间信息
String url = "http://open.douyucdn.cn/api/RoomApi/room/" + roomId;
String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(url)).body().string();
if(!StringUtils.isBlank(htmlBody)) {
JSONObject data = JSONObject.parseObject(htmlBody).getJSONObject("data");
String room_name = data.getString("room_name");
String user_name = data.getString("owner_name");
Integer hn = data.getInteger("hn");
int online = data.getInteger("online");
Integer fans = 0;
//获取用户信息
String userUrl = "https://www.douyu.com/swf_api/h5room/" + roomId;
String userBody = httpBoot.syncCall(RequestUtils.wrapGet(userUrl)).body().string();
if(!StringUtils.isBlank(userBody)) {
JSONObject userData = JSONObject.parseObject(userBody).getJSONObject("data");
fans = Integer.valueOf(userData.getString("fans"));
}
return new RoomInfo(PT, roomId, room_name, user_name , hn, fans);
}else {
logger.info("此次采集页面中不包含房间信息字段, 此次页面信息为:{}", htmlBody);
return null;
}
}else {
System.out.println("------------------");
logger.info("此次采集页面中不包含房间信息字段, 此次页面信息为:{}", roomBody);
return null;
}
}
/**
* 根据房间id获取房间信息
* @param roomId
* @return
* @throws Exception
*/
public static RoomInfo getRoomInfoProxyByRoomUrl(String roomUrl) throws Exception{
String roomBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl), ProxyHolder.NAT_PROXY).body().string();
if(!StringUtils.isBlank(roomBody) && roomBody.contains("ROOM.room_id =")) {
String roomId = roomBody.split("ROOM\\.room_id = ")[1].split("; ")[0].trim();
String url = "http://open.douyucdn.cn/api/RoomApi/room/" + roomId;
String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(url), ProxyHolder.NAT_PROXY).body().string();
if(!StringUtils.isBlank(htmlBody)) {
JSONObject data = JSONObject.parseObject(htmlBody).getJSONObject("data");
String room_name = data.getString("room_name");
String user_name = data.getString("owner_name");
Integer hn = data.getInteger("hn");
int online = data.getInteger("online");
Integer fans = 0;
//获取用户信息
String userUrl = "https://www.douyu.com/swf_api/h5room/" + roomId;
String userBody = httpBoot.syncCall(RequestUtils.wrapGet(userUrl)).body().string();
if(!StringUtils.isBlank(userBody)) {
JSONObject userData = JSONObject.parseObject(userBody).getJSONObject("data");
fans = Integer.valueOf(userData.getString("fans"));
}
return new RoomInfo(PT, roomId, room_name, user_name , hn, fans);
}else {
logger.info("此次采集页面中不包含房间信息字段, 此次页面信息为:{}", htmlBody);
return null;
}
}else {
logger.info("此次采集页面中不包含房间信息字段, 此次页面信息为:{}", roomBody);
return null;
}
}
}
package com.zhiwei.live.roominfo;
import com.zhiwei.crawler.utils.RequestUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.crawler.core.HttpBoot;
import com.zhiwei.crawler.proxy.ProxyHolder;
import com.zhiwei.live.bean.RoomInfo;
/**
* 斗鱼直播间信息获取
* @author qq859
*
*/
public class DouYuRoomInfoCrawler {
private static HttpBoot httpBoot = new HttpBoot.Builder().build();
private static Logger logger = LogManager.getLogger(DouYuRoomInfoCrawler.class);
private static final String PT = "斗鱼";
/**
* 根据房间id获取房间信息
* @param roomUrl
* @return
* @throws Exception
*/
public static RoomInfo getRoomInfoByRoomUrl(String roomUrl) throws Exception{
String roomBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl)).body().string();
if(!StringUtils.isBlank(roomBody) && roomBody.contains("ROOM.room_id =")) {
String roomId = roomBody.split("ROOM\\.room_id = ")[1].split("; ")[0].trim();
//获取房间信息
String url = "http://open.douyucdn.cn/api/RoomApi/room/" + roomId;
String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(url)).body().string();
if(!StringUtils.isBlank(htmlBody)) {
JSONObject data = JSONObject.parseObject(htmlBody).getJSONObject("data");
String room_name = data.getString("room_name");
String user_name = data.getString("owner_name");
Integer hn = data.getInteger("hn");
int online = data.getInteger("online");
Integer fans = 0;
//获取用户信息
String userUrl = "https://www.douyu.com/swf_api/h5room/" + roomId;
String userBody = httpBoot.syncCall(RequestUtils.wrapGet(userUrl)).body().string();
if(!StringUtils.isBlank(userBody)) {
JSONObject userData = JSONObject.parseObject(userBody).getJSONObject("data");
fans = Integer.valueOf(userData.getString("fans"));
}
return new RoomInfo(PT, roomId, room_name, user_name , hn, fans);
}else {
logger.info("此次采集页面中不包含房间信息字段, 此次页面信息为:{}", htmlBody);
return null;
}
}else {
System.out.println("------------------");
logger.info("此次采集页面中不包含房间信息字段, 此次页面信息为:{}", roomBody);
return null;
}
}
/**
* 根据房间id获取房间信息
* @param roomUrl
* @return
* @throws Exception
*/
public static RoomInfo getRoomInfoProxyByRoomUrl(String roomUrl) throws Exception{
String roomBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl), ProxyHolder.NAT_HEAVY_PROXY).body().string();
if(!StringUtils.isBlank(roomBody) && roomBody.contains("ROOM.room_id =")) {
String roomId = roomBody.split("ROOM\\.room_id = ")[1].split("; ")[0].trim();
String url = "http://open.douyucdn.cn/api/RoomApi/room/" + roomId;
String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(url), ProxyHolder.NAT_HEAVY_PROXY).body().string();
if(!StringUtils.isBlank(htmlBody)) {
JSONObject data = JSONObject.parseObject(htmlBody).getJSONObject("data");
String room_name = data.getString("room_name");
String user_name = data.getString("owner_name");
Integer hn = data.getInteger("hn");
int online = data.getInteger("online");
Integer fans = 0;
//获取用户信息
String userUrl = "https://www.douyu.com/swf_api/h5room/" + roomId;
String userBody = httpBoot.syncCall(RequestUtils.wrapGet(userUrl)).body().string();
if(!StringUtils.isBlank(userBody)) {
JSONObject userData = JSONObject.parseObject(userBody).getJSONObject("data");
fans = Integer.valueOf(userData.getString("fans"));
}
return new RoomInfo(PT, roomId, room_name, user_name , hn, fans);
}else {
logger.info("此次采集页面中不包含房间信息字段, 此次页面信息为:{}", htmlBody);
return null;
}
}else {
logger.info("此次采集页面中不包含房间信息字段, 此次页面信息为:{}", roomBody);
return null;
}
}
}
src/main/java/com/zhiwei/live/roominfo/HuYaRoomInfoCrawler.java
package com.zhiwei.live.roominfo;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import com.zhiwei.crawler.core.HttpBoot;
import com.zhiwei.crawler.core.RequestUtils;
import com.zhiwei.crawler.proxy.ProxyHolder;
import com.zhiwei.live.bean.RoomInfo;
public class HuYaRoomInfoCrawler {
private static HttpBoot httpBoot = new HttpBoot();
private static Logger logger = LogManager.getLogger(HuYaRoomInfoCrawler.class);
private static final String PT = "虎牙";
/**
* 根据房间id获取房间信息
*
* @param roomId
* @return
* @throws Exception
*/
public static RoomInfo getRoomInfoByRoomUrl(String roomUrl) throws Exception {
String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl)).body().string();
if (!StringUtils.isBlank(htmlBody)) {
Document document = Jsoup.parse(htmlBody);
String roomName = document.select("h1#J_roomTitle").text();
Integer liveCount = Integer.valueOf(document.select("em#live-count").text().replaceAll(",", ""));
String username = document.select("h3.host-name").text();
String activityCount = document.select("div#activityCount").text();
Integer fans = 0;
try {
fans = Integer.valueOf(activityCount.replaceAll(",", ""));
} catch (Exception e) {
fans = 0;
}
String room_id = document.select("span.host-rid").text();
return new RoomInfo(PT, room_id, roomName, username, liveCount, fans);
} else {
logger.info("此次采集页面中不包含房间信息字段, 此次页面信息为:{}", htmlBody);
return null;
}
}
/**
* 根据房间id获取房间信息
*
* @param roomId
* @return
* @throws Exception
*/
public static RoomInfo getRoomInfoProxyByRoomUrl(String roomUrl) throws Exception {
String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl), ProxyHolder.NAT_PROXY).body().string();
if (!StringUtils.isBlank(htmlBody)) {
Document document = Jsoup.parse(htmlBody);
String roomName = document.select("h1#J_roomTitle").text();
Integer liveCount = Integer.valueOf(document.select("em#live-count").text().replaceAll(",", ""));
String username = document.select("h3.host-name").text();
String activityCount = document.select("div#activityCount").text();
String room_id = document.select("span.host-rid").text();
Integer fans = 0;
try {
fans = Integer.valueOf(activityCount.replaceAll(",", ""));
} catch (Exception e) {
fans = 0;
}
return new RoomInfo(PT, room_id, roomName, username, liveCount, fans);
} else {
logger.info("此次采集页面中不包含房间信息字段, 此次页面信息为:{}", htmlBody);
return null;
}
}
}
package com.zhiwei.live.roominfo;
import com.zhiwei.crawler.utils.RequestUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import com.zhiwei.crawler.core.HttpBoot;
import com.zhiwei.crawler.proxy.ProxyHolder;
import com.zhiwei.live.bean.RoomInfo;
public class HuYaRoomInfoCrawler {
private static HttpBoot httpBoot = new HttpBoot.Builder().build();
private static Logger logger = LogManager.getLogger(HuYaRoomInfoCrawler.class);
private static final String PT = "虎牙";
/**
* 根据房间id获取房间信息
*
* @param roomId
* @return
* @throws Exception
*/
public static RoomInfo getRoomInfoByRoomUrl(String roomUrl) throws Exception {
String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl)).body().string();
if (!StringUtils.isBlank(htmlBody)) {
Document document = Jsoup.parse(htmlBody);
String roomName = document.select("h1#J_roomTitle").text();
Integer liveCount = Integer.valueOf(document.select("em#live-count").text().replaceAll(",", ""));
String username = document.select("h3.host-name").text();
String activityCount = document.select("div#activityCount").text();
Integer fans = 0;
try {
fans = Integer.valueOf(activityCount.replaceAll(",", ""));
} catch (Exception e) {
fans = 0;
}
String room_id = document.select("span.host-rid").text();
return new RoomInfo(PT, room_id, roomName, username, liveCount, fans);
} else {
logger.info("此次采集页面中不包含房间信息字段, 此次页面信息为:{}", htmlBody);
return null;
}
}
/**
* 根据房间id获取房间信息
*
* @param roomId
* @return
* @throws Exception
*/
public static RoomInfo getRoomInfoProxyByRoomUrl(String roomUrl) throws Exception {
String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl), ProxyHolder.NAT_HEAVY_PROXY).body().string();
if (!StringUtils.isBlank(htmlBody)) {
Document document = Jsoup.parse(htmlBody);
String roomName = document.select("h1#J_roomTitle").text();
Integer liveCount = Integer.valueOf(document.select("em#live-count").text().replaceAll(",", ""));
String username = document.select("h3.host-name").text();
String activityCount = document.select("div#activityCount").text();
String room_id = document.select("span.host-rid").text();
Integer fans = 0;
try {
fans = Integer.valueOf(activityCount.replaceAll(",", ""));
} catch (Exception e) {
fans = 0;
}
return new RoomInfo(PT, room_id, roomName, username, liveCount, fans);
} else {
logger.info("此次采集页面中不包含房间信息字段, 此次页面信息为:{}", htmlBody);
return null;
}
}
}
src/test/java/com/zhiwei/live/test/danmu/BilibiliDanMuTest.java
package com.zhiwei.live.test.danmu;
import com.zhiwei.live.danmu.bilibili.BilibiliClient;
import com.zhiwei.live.danmu.bilibili.BilibiliMessage;
import com.zhiwei.live.danmu.util.DataCallBack;
public class BilibiliDanMuTest {
public static void main(String[] args) throws InterruptedException {
String roomUrl = "https://live.bilibili.com/529";
try {
BilibiliClient.getDanmu(new DataCallBack() {
@Override
public void onData(Object message) {
if(message instanceof BilibiliMessage) {
BilibiliMessage bilibiliMessage = (BilibiliMessage)message;
System.out.println("-------------" + bilibiliMessage.toString());
}
}
}, roomUrl);
} catch (Exception e) {
e.printStackTrace();
}
}
}
package com.zhiwei.live.test.danmu;
import com.zhiwei.live.danmu.bilibili.BilibiliClient;
import com.zhiwei.live.danmu.bilibili.BilibiliMessage;
import com.zhiwei.live.danmu.util.DataCallBack;
public class BilibiliDanMuTest {
public static void main(String[] args) throws InterruptedException {
String roomUrl = "https://live.bilibili.com/387";
try {
BilibiliClient.getDanmu(new DataCallBack() {
@Override
public void onData(Object message) {
if(message instanceof BilibiliMessage) {
BilibiliMessage bilibiliMessage = (BilibiliMessage)message;
System.out.println("-------------" + bilibiliMessage.toString());
}
}
}, roomUrl);
} catch (Exception e) {
e.printStackTrace();
}
}
}
src/test/java/com/zhiwei/live/test/danmu/DouYuDanMuTest.java
package com.zhiwei.live.test.danmu;
import com.zhiwei.live.danmu.douyu.DouYuMessage;
import com.zhiwei.live.danmu.douyu.DouyuClient;
import com.zhiwei.live.danmu.util.DataCallBack;
public class DouYuDanMuTest {
public static void main(String[] args) throws InterruptedException {
String roomUrl = "https://www.douyu.com/71017";
try {
DouyuClient.getDanmu(new DataCallBack() {
@Override
public void onData(Object message) {
if (message instanceof DouYuMessage) {
DouYuMessage douyuMessage = (DouYuMessage) message;
System.out.println("-------------" + douyuMessage.toString());
}
}
}, roomUrl);
} catch (Exception e) {
e.printStackTrace();
}
}
}
package com.zhiwei.live.test.danmu;
import com.zhiwei.live.danmu.douyu.DouYuMessage;
import com.zhiwei.live.danmu.douyu.DouyuClient;
import com.zhiwei.live.danmu.util.DataCallBack;
public class DouYuDanMuTest {
public static void main(String[] args) throws InterruptedException {
String roomUrl = "https://www.douyu.com/topic/NarutoMatch?rid=535534";
try {
DouyuClient.getDanmu(new DataCallBack() {
@Override
public void onData(Object message) {
if (message instanceof DouYuMessage) {
DouYuMessage douyuMessage = (DouYuMessage) message;
System.out.println("-------------" + douyuMessage.toString());
}
}
}, roomUrl);
} catch (Exception e) {
e.printStackTrace();
}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment