Skip to content

L
live-crawler
Commit 87226592 by zhiwei
Options

B站弹幕获取

parent fe6e0d5b
Showing with 103 additions and 171 deletions
pom.xml
......@@ -12,7 +12,7 @@
<dependency>
<groupId>com.zhiwei.crawler</groupId>
<artifactId>crawler-core</artifactId>
<version>0.1.1-RELEASE</version>
<version>0.6.2.1-RELEASE</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
......@@ -25,7 +25,12 @@
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
<version>4.1.33.Final</version>
<version>4.1.48.Final</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.12</version>
</dependency>
</dependencies>
......
src/main/java/com/zhiwei/live/bean/RoomInfo.java
package com.zhiwei.live.bean;
import lombok.Data;
import lombok.ToString;
@Data
@ToString
public class RoomInfo {
String pt; // 平台类型
/**
* 平台类型
*/
String pt;
String roomId; // 房间号
/**
* 房间号
*/
String roomId;
String nickName; // 主播昵称
/**
* 主播昵称
*/
String nickName;
String roomName; // 房间名称
/**
* 房间名称
*/
String roomName;
Integer hotNum; // 直播间热度
/**
* 直播间热度
*/
Integer hotNum;
Integer fans; // 订阅数
/**
* 订阅数
*/
Integer fans;
public RoomInfo() {
}
......@@ -26,62 +49,4 @@ public class RoomInfo {
this.fans = fans;
}
@Override
public String toString() {
return "new RoomInfo[" + "pt = " + pt + ", roomId = " + roomId + ", roomName = " + roomName + ", nickName = "
+ nickName + ", hotNum = " + hotNum + ", fans = " + fans + "]";
}
public Integer getFans() {
return fans;
}
public void setHotNum(Integer hotNum) {
this.hotNum = hotNum;
}
public void setFans(Integer fans) {
this.fans = fans;
}
public String getPt() {
return pt;
}
public String getRoomId() {
return roomId;
}
public String getNickName() {
return nickName;
}
public String getRoomName() {
return roomName;
}
public int getHotNum() {
return hotNum;
}
public void setPt(String pt) {
this.pt = pt;
}
public void setRoomId(String roomId) {
this.roomId = roomId;
}
public void setNickName(String nickName) {
this.nickName = nickName;
}
public void setRoomName(String roomName) {
this.roomName = roomName;
}
public void setHotNum(int hotNum) {
this.hotNum = hotNum;
}
}
src/main/java/com/zhiwei/live/danmu/bilibili/BilibiliClient.java
......@@ -3,12 +3,12 @@ package com.zhiwei.live.danmu.bilibili;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
import com.zhiwei.crawler.utils.RequestUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.crawler.core.HttpBoot;
import com.zhiwei.crawler.core.RequestUtils;
import com.zhiwei.live.bean.RoomInfo;
import com.zhiwei.live.danmu.util.Connector;
import com.zhiwei.live.danmu.util.DataCallBack;
......@@ -21,13 +21,13 @@ import okhttp3.Response;
public class BilibiliClient {
private static HttpBoot httpBoot = new HttpBoot();
private static HttpBoot httpBoot = new HttpBoot.Builder().build();
private static Logger logger = LogManager.getLogger(BilibiliClient.class);
private static final int PORT = 2243;
/**
* 根据房间号获取弹幕信息
* @param roomId
* @param roomUrl
* @throws Exception
*/
public static void getDanmu(DataCallBack dataCallBack,String roomUrl) throws Exception {
......
src/main/java/com/zhiwei/live/danmu/bilibili/BilibiliMessage.java
......@@ -4,20 +4,41 @@ import java.util.Date;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import lombok.Data;
import lombok.ToString;
@Data
@ToString
public class BilibiliMessage {
String messageType; //弹幕消息类型
String user_id; //发布者uid
String nickName; //发布者昵称
Date time; //弹幕时间
String content; //弹幕内容
String room_id; //房间id
/**
* 弹幕消息类型
*/
String messageType;
/**
* 发布者uid
*/
String userId;
/**
* 发布者昵称
*/
String nickName;
/**
* 弹幕时间
*/
Date time;
/**
* 弹幕内容
*/
String content;
/**
* 房间id
*/
String roomId;
public BilibiliMessage(JSONObject json) throws Exception {
constructJson(json);
......@@ -28,80 +49,15 @@ public class BilibiliMessage {
private void constructJson(JSONObject json) throws Exception{
try {
System.out.println(json);
JSONArray jsonArray = json.getJSONArray("info");
messageType = json.getString("cmd");
user_id = jsonArray.getJSONArray(2).getString(0);
userId = jsonArray.getJSONArray(2).getString(0);
time = new Date();
nickName = jsonArray.getJSONArray(2).getString(1);
content = jsonArray.getString(1);
} catch (Exception e) {
throw new Exception();
}
}
@Override
public String toString() {
return "new BilibiliMessage["
+ " user_id = " + user_id
+ ", nickName = " + nickName
+ ", messageType = " + messageType
+ ", time = " + time
+ ", content = " + content
+ ", room_id = " + room_id
+ "]";
}
public Date getTime() {
return time;
}
public void setTime(Date time) {
this.time = time;
}
public String getMessageType() {
return messageType;
}
public String getUser_id() {
return user_id;
}
public String getNickName() {
return nickName;
}
public String getContent() {
return content;
}
public void setMessageType(String messageType) {
this.messageType = messageType;
}
public void setUser_id(String user_id) {
this.user_id = user_id;
}
public void setNickName(String nickName) {
this.nickName = nickName;
}
public void setContent(String content) {
this.content = content;
}
public String getRoom_id() {
return room_id;
}
public void setRoom_id(String room_id) {
this.room_id = room_id;
}
}
src/main/java/com/zhiwei/live/danmu/douyu/DouyuClient.java
......@@ -22,8 +22,8 @@ import io.netty.handler.timeout.IdleStateHandler;
public class DouyuClient {
private static Logger logger = LogManager.getLogger(DouyuClient.class);
private static final int PORT = 8601;
private static final String HOST = "openbarrage.douyutv.com";
private static final int PORT = 8504;
private static final String HOST = "danmuproxy.douyu.com";
/**
* 根据房间号获取弹幕信息
......
src/main/java/com/zhiwei/live/danmu/douyu/DouyuMessageHandler.java
......@@ -70,7 +70,7 @@ public class DouyuMessageHandler extends ChannelInboundHandlerAdapter {
if (msg instanceof ByteBuf) {
ByteBuf buf = (ByteBuf) msg;
String source = buf.toString(CharsetUtil.UTF_8);
// System.out.println("source========="+source);
System.out.println("source========="+source);
if(source.contains("chatmsg")) {
Map<String,Object> messageMap = DouYuUtil.toMap(source);
String data = JSONObject.toJSONString(messageMap);
......@@ -84,6 +84,10 @@ public class DouyuMessageHandler extends ChannelInboundHandlerAdapter {
ByteBuf groupMsg = buildMsg(StringUtils.join("type@=joingroup/rid@=",roomId,"/gid@=-9999/"));
System.out.println("发送入组消息: \n" + ByteBufUtil.prettyHexDump(groupMsg));
ctx.writeAndFlush(groupMsg);
// ByteBuf giftMsg = buildMsg(StringUtils.join("type@=dmfbdreq/dfl@=sn@AA=105@ASss@AA=0@AS@Ssn@AA=106@ASss@AA=0@AS@Ssn@AA=107@ASss@AA=0@AS@Ssn@AA=108@ASss@AA=0@AS@Ssn@AA=110@ASss@AA=0@AS@S/"));
// System.out.println("发送礼物消息: \n" + ByteBufUtil.prettyHexDump(giftMsg));
// ctx.writeAndFlush(giftMsg);
finish = true;
}
ctx.fireChannelRead(msg);
......@@ -97,7 +101,7 @@ public class DouyuMessageHandler extends ChannelInboundHandlerAdapter {
if (evt instanceof IdleStateEvent) {
IdleStateEvent event = (IdleStateEvent) evt;
if (event.state() == IdleState.WRITER_IDLE) {
ByteBuf pingMsg = buildMsg(StringUtils.join("type@=keeplive/tick@=", System.currentTimeMillis() / 1000, "/"));
ByteBuf pingMsg = buildMsg(StringUtils.join("type@=mrkl/"));
System.out.println("发送心跳消息: \n" + ByteBufUtil.prettyHexDump(pingMsg));
ctx.writeAndFlush(pingMsg);
} else {
......@@ -109,7 +113,7 @@ public class DouyuMessageHandler extends ChannelInboundHandlerAdapter {
}
private ByteBuf buildMsg(String content) {
int fixedLen = 4 + 4 + 1;
int fixedLen = 4 + 4 + 1 + content.length();
byte[] body = content.getBytes(CharsetUtil.UTF_8);
int length = fixedLen + body.length;
return Unpooled.buffer(length).writeIntLE(length).writeIntLE(length).writeShortLE(689)
......
src/main/java/com/zhiwei/live/danmu/pandam/PandamClient.java
......@@ -5,7 +5,7 @@ import java.util.Objects;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.crawler.core.HttpBoot;
import com.zhiwei.crawler.core.RequestUtils;
import com.zhiwei.crawler.utils.RequestUtils;
import com.zhiwei.live.bean.RoomInfo;
import com.zhiwei.live.danmu.util.Connector;
import com.zhiwei.live.danmu.util.DataCallBack;
......@@ -23,7 +23,7 @@ import okhttp3.Response;
* @date Jan 25, 2019 3:24:33 PM
*/
public class PandamClient {
private static HttpBoot httpBoot = new HttpBoot();
private static HttpBoot httpBoot = new HttpBoot.Builder().build();
/**
......
src/main/java/com/zhiwei/live/danmu/pandam/PandamMessageHandler.java
......@@ -3,6 +3,7 @@ package com.zhiwei.live.danmu.pandam;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import lombok.extern.log4j.Log4j2;
import org.apache.commons.lang3.StringUtils;
import com.alibaba.fastjson.JSONObject;
......@@ -24,6 +25,7 @@ import io.netty.util.ReferenceCountUtil;
* @author 0xff
* @date Jan 25, 2019 11:03:20 AM
*/
@Log4j2
public class PandamMessageHandler extends ChannelInboundHandlerAdapter {
private static final byte[] FIRST_REQ = new byte[] { 0x00, 0x06, 0x00, 0x02, 0x00 };
private static final byte[] FIRST_RES = new byte[] { 0x00, 0x06, 0x00, 0x06 };
......
src/main/java/com/zhiwei/live/roominfo/BilibiliRoomInfoCrawler.java
package com.zhiwei.live.roominfo;
import com.zhiwei.crawler.utils.RequestUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.crawler.core.HttpBoot;
import com.zhiwei.crawler.core.RequestUtils;
import com.zhiwei.crawler.proxy.ProxyHolder;
import com.zhiwei.live.bean.RoomInfo;
import com.zhiwei.tools.tools.ZhiWeiTools;
......@@ -18,7 +18,7 @@ import com.zhiwei.tools.tools.ZhiWeiTools;
*/
public class BilibiliRoomInfoCrawler {
private static HttpBoot httpBoot = new HttpBoot();
private static HttpBoot httpBoot = new HttpBoot.Builder().build();
private static Logger logger = LogManager.getLogger(BilibiliRoomInfoCrawler.class);
private static final String PT = "B站";
......@@ -69,7 +69,7 @@ public class BilibiliRoomInfoCrawler {
public static RoomInfo getRoomInfoProxyByRoomUrl(String roomUrl) throws Exception{
String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl), ProxyHolder.NAT_PROXY).body().string();
String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl), ProxyHolder.NAT_HEAVY_PROXY).body().string();
if(!StringUtils.isBlank(htmlBody)) {
//判断页面中是否包含房间信息
if(htmlBody.contains("window.__NEPTUNE_IS_MY_WAIFU__=")) {
......@@ -90,7 +90,7 @@ public class BilibiliRoomInfoCrawler {
String username = null;
//通过房间id获取用户信息
roomUrl = "https://api.live.bilibili.com/live_user/v1/UserInfo/get_anchor_in_room?roomid="+room_id;
String roomBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl), ProxyHolder.NAT_PROXY).body().string();
String roomBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl), ProxyHolder.NAT_HEAVY_PROXY).body().string();
if(!StringUtils.isBlank(roomBody)) {
JSONObject roomData = JSONObject.parseObject(roomBody).getJSONObject("data");
username = roomData.getJSONObject("info").getString("uname");
......
src/main/java/com/zhiwei/live/roominfo/DouYuRoomInfoCrawler.java
package com.zhiwei.live.roominfo;
import com.zhiwei.crawler.utils.RequestUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.crawler.core.HttpBoot;
import com.zhiwei.crawler.core.RequestUtils;
import com.zhiwei.crawler.proxy.ProxyHolder;
import com.zhiwei.live.bean.RoomInfo;
......@@ -17,13 +17,13 @@ import com.zhiwei.live.bean.RoomInfo;
*/
public class DouYuRoomInfoCrawler {
private static HttpBoot httpBoot = new HttpBoot();
private static HttpBoot httpBoot = new HttpBoot.Builder().build();
private static Logger logger = LogManager.getLogger(DouYuRoomInfoCrawler.class);
private static final String PT = "斗鱼";
/**
* 根据房间id获取房间信息
* @param roomId
* @param roomUrl
* @return
* @throws Exception
*/
......@@ -64,16 +64,16 @@ public class DouYuRoomInfoCrawler {
/**
* 根据房间id获取房间信息
* @param roomId
* @param roomUrl
* @return
* @throws Exception
*/
public static RoomInfo getRoomInfoProxyByRoomUrl(String roomUrl) throws Exception{
String roomBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl), ProxyHolder.NAT_PROXY).body().string();
String roomBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl), ProxyHolder.NAT_HEAVY_PROXY).body().string();
if(!StringUtils.isBlank(roomBody) && roomBody.contains("ROOM.room_id =")) {
String roomId = roomBody.split("ROOM\\.room_id = ")[1].split("; ")[0].trim();
String url = "http://open.douyucdn.cn/api/RoomApi/room/" + roomId;
String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(url), ProxyHolder.NAT_PROXY).body().string();
String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(url), ProxyHolder.NAT_HEAVY_PROXY).body().string();
if(!StringUtils.isBlank(htmlBody)) {
JSONObject data = JSONObject.parseObject(htmlBody).getJSONObject("data");
String room_name = data.getString("room_name");
......
src/main/java/com/zhiwei/live/roominfo/HuYaRoomInfoCrawler.java
package com.zhiwei.live.roominfo;
import com.zhiwei.crawler.utils.RequestUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
......@@ -7,13 +8,12 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import com.zhiwei.crawler.core.HttpBoot;
import com.zhiwei.crawler.core.RequestUtils;
import com.zhiwei.crawler.proxy.ProxyHolder;
import com.zhiwei.live.bean.RoomInfo;
public class HuYaRoomInfoCrawler {
private static HttpBoot httpBoot = new HttpBoot();
private static HttpBoot httpBoot = new HttpBoot.Builder().build();
private static Logger logger = LogManager.getLogger(HuYaRoomInfoCrawler.class);
private static final String PT = "虎牙";
......@@ -56,7 +56,7 @@ public class HuYaRoomInfoCrawler {
* @throws Exception
*/
public static RoomInfo getRoomInfoProxyByRoomUrl(String roomUrl) throws Exception {
String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl), ProxyHolder.NAT_PROXY).body().string();
String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl), ProxyHolder.NAT_HEAVY_PROXY).body().string();
if (!StringUtils.isBlank(htmlBody)) {
Document document = Jsoup.parse(htmlBody);
String roomName = document.select("h1#J_roomTitle").text();
......
src/main/java/com/zhiwei/live/roominfo/PandamTVRoomInfoCrawler.java
package com.zhiwei.live.roominfo;
import com.zhiwei.crawler.utils.RequestUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.crawler.core.HttpBoot;
import com.zhiwei.crawler.core.RequestUtils;
import com.zhiwei.crawler.proxy.ProxyHolder;
import com.zhiwei.live.bean.RoomInfo;
import com.zhiwei.tools.tools.ZhiWeiTools;
......@@ -17,7 +17,7 @@ import com.zhiwei.tools.tools.ZhiWeiTools;
*
*/
public class PandamTVRoomInfoCrawler {
private static HttpBoot httpBoot = new HttpBoot();
private static HttpBoot httpBoot = new HttpBoot.Builder().build();
private static Logger logger = LogManager.getLogger(PandamTVRoomInfoCrawler.class);
private static final String PT = "熊猫TV";
......@@ -85,7 +85,7 @@ public class PandamTVRoomInfoCrawler {
* @throws Exception
*/
public static RoomInfo getRoomInfoProxyByRoomUrl(String roomUrl) throws Exception{
String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl),ProxyHolder.NAT_PROXY).body().string();
String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl),ProxyHolder.NAT_HEAVY_PROXY).body().string();
if(!StringUtils.isBlank(htmlBody)) {
//判断页面中是否包含房间信息,此为pc端直播
if(htmlBody.contains("window._config_roominfo = ")) {
......
src/test/java/com/zhiwei/live/test/danmu/BilibiliDanMuTest.java
......@@ -7,7 +7,7 @@ import com.zhiwei.live.danmu.util.DataCallBack;
public class BilibiliDanMuTest {
public static void main(String[] args) throws InterruptedException {
String roomUrl = "https://live.bilibili.com/529";
String roomUrl = "https://live.bilibili.com/387";
try {
BilibiliClient.getDanmu(new DataCallBack() {
@Override
......
src/test/java/com/zhiwei/live/test/danmu/DouYuDanMuTest.java
......@@ -7,7 +7,7 @@ import com.zhiwei.live.danmu.util.DataCallBack;
public class DouYuDanMuTest {
public static void main(String[] args) throws InterruptedException {
String roomUrl = "https://www.douyu.com/71017";
String roomUrl = "https://www.douyu.com/topic/NarutoMatch?rid=535534";
try {
DouyuClient.getDanmu(new DataCallBack() {
@Override
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment