B站弹幕获取

pom.xml

@@ -12,7 +12,7 @@
   	<dependency>
   		<groupId>com.zhiwei.crawler</groupId>
   		<artifactId>crawler-core</artifactId>
-  		<version>0.1.1-RELEASE</version>
+  		<version>0.6.2.1-RELEASE</version>
   	</dependency>
   	
   	<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
@@ -25,7 +25,12 @@
 		<dependency>
 			<groupId>io.netty</groupId>
 			<artifactId>netty-all</artifactId>
-			<version>4.1.33.Final</version>
+			<version>4.1.48.Final</version>
+		</dependency>
+	  <dependency>
+		  <groupId>org.projectlombok</groupId>
+		  <artifactId>lombok</artifactId>
+		  <version>1.18.12</version>
 	  </dependency>
   </dependencies>
   

src/main/java/com/zhiwei/live/bean/RoomInfo.java

 package com.zhiwei.live.bean;
 
+import lombok.Data;
+import lombok.ToString;
+
+@Data
+@ToString
 public class RoomInfo {
 
-	String pt; // 平台类型
+	/**
+	 * 平台类型
+	 */
+	String pt;
 
-	String roomId; // 房间号
+	/**
+	 * 房间号
+	 */
+	String roomId;
 
-	String nickName; // 主播昵称
+	/**
+	 * 主播昵称
+	 */
+	String nickName;
 
-	String roomName; // 房间名称
+	/**
+	 * 房间名称
+	 */
+	String roomName;
 
-	Integer hotNum; // 直播间热度
+	/**
+	 * 直播间热度
+	 */
+	Integer hotNum;
 
-	Integer fans; // 订阅数
+	/**
+	 * 订阅数
+	 */
+	Integer fans;
 
 	public RoomInfo() {
 	}
@@ -26,62 +49,4 @@ public class RoomInfo {
 		this.fans = fans;
 	}
 
-	@Override
-	public String toString() {
-		return "new RoomInfo[" + "pt = " + pt + ", roomId = " + roomId + ", roomName = " + roomName + ", nickName = "
-				+ nickName + ", hotNum = " + hotNum + ", fans = " + fans + "]";
-	}
-
-	public Integer getFans() {
-		return fans;
-	}
-
-	public void setHotNum(Integer hotNum) {
-		this.hotNum = hotNum;
-	}
-
-	public void setFans(Integer fans) {
-		this.fans = fans;
-	}
-
-	public String getPt() {
-		return pt;
-	}
-
-	public String getRoomId() {
-		return roomId;
-	}
-
-	public String getNickName() {
-		return nickName;
-	}
-
-	public String getRoomName() {
-		return roomName;
-	}
-
-	public int getHotNum() {
-		return hotNum;
-	}
-
-	public void setPt(String pt) {
-		this.pt = pt;
-	}
-
-	public void setRoomId(String roomId) {
-		this.roomId = roomId;
-	}
-
-	public void setNickName(String nickName) {
-		this.nickName = nickName;
-	}
-
-	public void setRoomName(String roomName) {
-		this.roomName = roomName;
-	}
-
-	public void setHotNum(int hotNum) {
-		this.hotNum = hotNum;
-	}
-
 }

src/main/java/com/zhiwei/live/danmu/bilibili/BilibiliClient.java

@@ -3,12 +3,12 @@ package com.zhiwei.live.danmu.bilibili;
 import java.util.Objects;
 import java.util.concurrent.TimeUnit;
 
+import com.zhiwei.crawler.utils.RequestUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.core.RequestUtils;
 import com.zhiwei.live.bean.RoomInfo;
 import com.zhiwei.live.danmu.util.Connector;
 import com.zhiwei.live.danmu.util.DataCallBack;
@@ -21,13 +21,13 @@ import okhttp3.Response;
 
 public class BilibiliClient {
     
-    private static HttpBoot httpBoot = new HttpBoot();
+    private static HttpBoot httpBoot = new HttpBoot.Builder().build();
     private static Logger logger = LogManager.getLogger(BilibiliClient.class);
     private static final  int PORT = 2243;
     
     /**
      * 根据房间号获取弹幕信息
-     * @param roomId
+     * @param roomUrl
      * @throws Exception
      */
     public static void getDanmu(DataCallBack dataCallBack,String roomUrl) throws Exception {

src/main/java/com/zhiwei/live/danmu/bilibili/BilibiliMessage.java

@@ -4,20 +4,41 @@ import java.util.Date;
 
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
+import lombok.Data;
+import lombok.ToString;
 
+@Data
+@ToString
 public class BilibiliMessage {
 
-    String messageType;    //弹幕消息类型
-    
-    String user_id;        //发布者uid
-    
-    String nickName;       //发布者昵称
-    
-    Date time;           //弹幕时间
-    
-    String content;        //弹幕内容
-    
-    String room_id;     //房间id
+    /**
+     * 弹幕消息类型
+     */
+    String messageType;
+
+    /**
+     * 发布者uid
+     */
+    String userId;
+
+    /**
+     * 发布者昵称
+     */
+    String nickName;
+
+    /**
+     * 弹幕时间
+     */
+    Date time;
+    /**
+     * 弹幕内容
+     */
+    String content;
+
+    /**
+     *  房间id
+     */
+    String roomId;
     
     public BilibiliMessage(JSONObject json) throws Exception {
         constructJson(json);
@@ -28,80 +49,15 @@ public class BilibiliMessage {
     
     private void constructJson(JSONObject json) throws Exception{
         try {
-        	System.out.println(json);
             JSONArray jsonArray = json.getJSONArray("info");
             messageType = json.getString("cmd");
-            user_id = jsonArray.getJSONArray(2).getString(0);
+            userId = jsonArray.getJSONArray(2).getString(0);
             time = new Date();
             nickName = jsonArray.getJSONArray(2).getString(1);
             content = jsonArray.getString(1);
-            
         } catch (Exception e) {
             throw new Exception();
         }
     }
 
-
-
-    @Override
-    public String toString() {
-        return "new BilibiliMessage["
-                + "  user_id = " + user_id
-                + ", nickName = " + nickName
-                + ", messageType = " + messageType
-                + ", time = " + time
-                + ", content = " + content
-                + ", room_id = " + room_id
-                + "]";
-    }
-    
-    public Date getTime() {
-        return time;
-    }
-
-    public void setTime(Date time) {
-        this.time = time;
-    }
-
-    public String getMessageType() {
-        return messageType;
-    }
-
-    public String getUser_id() {
-        return user_id;
-    }
-
-    public String getNickName() {
-        return nickName;
-    }
-
-    public String getContent() {
-        return content;
-    }
-
-    public void setMessageType(String messageType) {
-        this.messageType = messageType;
-    }
-
-    public void setUser_id(String user_id) {
-        this.user_id = user_id;
-    }
-
-    public void setNickName(String nickName) {
-        this.nickName = nickName;
-    }
-
-    public void setContent(String content) {
-        this.content = content;
-    }
-
-	public String getRoom_id() {
-		return room_id;
-	}
-
-	public void setRoom_id(String room_id) {
-		this.room_id = room_id;
-	}
-
-    
 }

src/main/java/com/zhiwei/live/danmu/douyu/DouyuClient.java

@@ -22,8 +22,8 @@ import io.netty.handler.timeout.IdleStateHandler;
 public class DouyuClient {
 
 	private static Logger logger = LogManager.getLogger(DouyuClient.class);
-	private static final int PORT = 8601;
-	private static final String HOST = "openbarrage.douyutv.com";
+	private static final int PORT = 8504;
+	private static final String HOST = "danmuproxy.douyu.com";
 
 	/**
 	 * 根据房间号获取弹幕信息

src/main/java/com/zhiwei/live/danmu/douyu/DouyuMessageHandler.java

@@ -70,7 +70,7 @@ public class DouyuMessageHandler extends ChannelInboundHandlerAdapter {
 		 if (msg instanceof ByteBuf) {
 	            ByteBuf buf = (ByteBuf) msg;
 	            String source = buf.toString(CharsetUtil.UTF_8);
-//	            System.out.println("source========="+source);
+	            System.out.println("source========="+source);
 	            if(source.contains("chatmsg")) {
 	            	Map<String,Object> messageMap = DouYuUtil.toMap(source);
 	            	String data = JSONObject.toJSONString(messageMap);
@@ -84,6 +84,10 @@ public class DouyuMessageHandler extends ChannelInboundHandlerAdapter {
 			ByteBuf groupMsg = buildMsg(StringUtils.join("type@=joingroup/rid@=",roomId,"/gid@=-9999/"));
 			System.out.println("发送入组消息: \n" + ByteBufUtil.prettyHexDump(groupMsg));
 			ctx.writeAndFlush(groupMsg);
+
+//			ByteBuf giftMsg = buildMsg(StringUtils.join("type@=dmfbdreq/dfl@=sn@AA=105@ASss@AA=0@AS@Ssn@AA=106@ASss@AA=0@AS@Ssn@AA=107@ASss@AA=0@AS@Ssn@AA=108@ASss@AA=0@AS@Ssn@AA=110@ASss@AA=0@AS@S/"));
+//			System.out.println("发送礼物消息: \n" + ByteBufUtil.prettyHexDump(giftMsg));
+//			ctx.writeAndFlush(giftMsg);
 			finish = true;
 		}
 		ctx.fireChannelRead(msg);
@@ -97,7 +101,7 @@ public class DouyuMessageHandler extends ChannelInboundHandlerAdapter {
 		if (evt instanceof IdleStateEvent) {
 			IdleStateEvent event = (IdleStateEvent) evt;
 			if (event.state() == IdleState.WRITER_IDLE) {
-				ByteBuf pingMsg = buildMsg(StringUtils.join("type@=keeplive/tick@=", System.currentTimeMillis() / 1000, "/"));
+				ByteBuf pingMsg = buildMsg(StringUtils.join("type@=mrkl/"));
 				System.out.println("发送心跳消息: \n" + ByteBufUtil.prettyHexDump(pingMsg));
 				ctx.writeAndFlush(pingMsg);
 			} else {
@@ -109,7 +113,7 @@ public class DouyuMessageHandler extends ChannelInboundHandlerAdapter {
 	}
 
 	private ByteBuf buildMsg(String content) {
-		int fixedLen = 4 + 4 + 1;
+		int fixedLen = 4 + 4 + 1 + content.length();
 		byte[] body = content.getBytes(CharsetUtil.UTF_8);
 		int length = fixedLen + body.length;
 		return Unpooled.buffer(length).writeIntLE(length).writeIntLE(length).writeShortLE(689)

src/main/java/com/zhiwei/live/danmu/pandam/PandamClient.java

@@ -5,7 +5,7 @@ import java.util.Objects;
 import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.core.RequestUtils;
+import com.zhiwei.crawler.utils.RequestUtils;
 import com.zhiwei.live.bean.RoomInfo;
 import com.zhiwei.live.danmu.util.Connector;
 import com.zhiwei.live.danmu.util.DataCallBack;
@@ -23,7 +23,7 @@ import okhttp3.Response;
  * @date Jan 25, 2019 3:24:33 PM
  */
 public class PandamClient {
-	private static HttpBoot httpBoot = new HttpBoot();
+	private static HttpBoot httpBoot = new HttpBoot.Builder().build();
 	
 	
 	 /**

src/main/java/com/zhiwei/live/danmu/pandam/PandamMessageHandler.java

@@ -3,6 +3,7 @@ package com.zhiwei.live.danmu.pandam;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import lombok.extern.log4j.Log4j2;
 import org.apache.commons.lang3.StringUtils;
 
 import com.alibaba.fastjson.JSONObject;
@@ -24,6 +25,7 @@ import io.netty.util.ReferenceCountUtil;
  * @author 0xff
  * @date Jan 25, 2019 11:03:20 AM
  */
+@Log4j2
 public class PandamMessageHandler extends ChannelInboundHandlerAdapter {
 	private static final byte[] FIRST_REQ = new byte[] { 0x00, 0x06, 0x00, 0x02, 0x00 };
 	private static final byte[] FIRST_RES = new byte[] { 0x00, 0x06, 0x00, 0x06 };

src/main/java/com/zhiwei/live/roominfo/BilibiliRoomInfoCrawler.java

 package com.zhiwei.live.roominfo;
 
+import com.zhiwei.crawler.utils.RequestUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.core.RequestUtils;
 import com.zhiwei.crawler.proxy.ProxyHolder;
 import com.zhiwei.live.bean.RoomInfo;
 import com.zhiwei.tools.tools.ZhiWeiTools;
@@ -18,7 +18,7 @@ import com.zhiwei.tools.tools.ZhiWeiTools;
  */
 public class BilibiliRoomInfoCrawler {
 	
-	private static HttpBoot httpBoot = new HttpBoot();
+	private static HttpBoot httpBoot = new HttpBoot.Builder().build();
 	private static Logger logger = LogManager.getLogger(BilibiliRoomInfoCrawler.class);
 	private static final String PT = "B站";
 	
@@ -69,7 +69,7 @@ public class BilibiliRoomInfoCrawler {
 	
 	
 	public static RoomInfo getRoomInfoProxyByRoomUrl(String roomUrl) throws Exception{
-		String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl), ProxyHolder.NAT_PROXY).body().string();
+		String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl), ProxyHolder.NAT_HEAVY_PROXY).body().string();
 		if(!StringUtils.isBlank(htmlBody)) {
 			//判断页面中是否包含房间信息
 			if(htmlBody.contains("window.__NEPTUNE_IS_MY_WAIFU__=")) {
@@ -90,7 +90,7 @@ public class BilibiliRoomInfoCrawler {
 				String username = null;
 				//通过房间id获取用户信息
 				roomUrl = "https://api.live.bilibili.com/live_user/v1/UserInfo/get_anchor_in_room?roomid="+room_id;
-				String roomBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl), ProxyHolder.NAT_PROXY).body().string();
+				String roomBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl), ProxyHolder.NAT_HEAVY_PROXY).body().string();
 				if(!StringUtils.isBlank(roomBody)) {
 					JSONObject roomData = JSONObject.parseObject(roomBody).getJSONObject("data");
 					username = roomData.getJSONObject("info").getString("uname");

src/main/java/com/zhiwei/live/roominfo/DouYuRoomInfoCrawler.java

 package com.zhiwei.live.roominfo;
 
+import com.zhiwei.crawler.utils.RequestUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.core.RequestUtils;
 import com.zhiwei.crawler.proxy.ProxyHolder;
 import com.zhiwei.live.bean.RoomInfo;
 
@@ -17,13 +17,13 @@ import com.zhiwei.live.bean.RoomInfo;
  */
 public class DouYuRoomInfoCrawler {
 	
-	private static HttpBoot httpBoot = new HttpBoot();
+	private static HttpBoot httpBoot = new HttpBoot.Builder().build();
 	private static Logger logger = LogManager.getLogger(DouYuRoomInfoCrawler.class);
 	private static final String PT = "斗鱼";
 	
 	/**
 	 * 根据房间id获取房间信息
-	 * @param roomId
+	 * @param roomUrl
 	 * @return
 	 * @throws Exception
 	 */
@@ -64,16 +64,16 @@ public class DouYuRoomInfoCrawler {
 	
 	/**
 	 * 根据房间id获取房间信息
-	 * @param roomId
+	 * @param roomUrl
 	 * @return
 	 * @throws Exception
 	 */
 	public static RoomInfo getRoomInfoProxyByRoomUrl(String roomUrl) throws Exception{
-		String roomBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl), ProxyHolder.NAT_PROXY).body().string();
+		String roomBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl), ProxyHolder.NAT_HEAVY_PROXY).body().string();
 		if(!StringUtils.isBlank(roomBody) && roomBody.contains("ROOM.room_id =")) {
 			String roomId = roomBody.split("ROOM\\.room_id = ")[1].split("; ")[0].trim();
 			String url = "http://open.douyucdn.cn/api/RoomApi/room/" + roomId;
-			String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(url), ProxyHolder.NAT_PROXY).body().string();
+			String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(url), ProxyHolder.NAT_HEAVY_PROXY).body().string();
 			if(!StringUtils.isBlank(htmlBody)) {
 				JSONObject data = JSONObject.parseObject(htmlBody).getJSONObject("data");
 				String room_name = data.getString("room_name");

src/main/java/com/zhiwei/live/roominfo/HuYaRoomInfoCrawler.java

 package com.zhiwei.live.roominfo;
 
+import com.zhiwei.crawler.utils.RequestUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -7,13 +8,12 @@ import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.core.RequestUtils;
 import com.zhiwei.crawler.proxy.ProxyHolder;
 import com.zhiwei.live.bean.RoomInfo;
 
 public class HuYaRoomInfoCrawler {
 
-	private static HttpBoot httpBoot = new HttpBoot();
+	private static HttpBoot httpBoot = new HttpBoot.Builder().build();
 	private static Logger logger = LogManager.getLogger(HuYaRoomInfoCrawler.class);
 	private static final String PT = "虎牙";
 
@@ -56,7 +56,7 @@ public class HuYaRoomInfoCrawler {
 	 * @throws Exception
 	 */
 	public static RoomInfo getRoomInfoProxyByRoomUrl(String roomUrl) throws Exception {
-		String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl), ProxyHolder.NAT_PROXY).body().string();
+		String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl), ProxyHolder.NAT_HEAVY_PROXY).body().string();
 		if (!StringUtils.isBlank(htmlBody)) {
 			Document document = Jsoup.parse(htmlBody);
 			String roomName = document.select("h1#J_roomTitle").text();

src/main/java/com/zhiwei/live/roominfo/PandamTVRoomInfoCrawler.java

 package com.zhiwei.live.roominfo;
 
+import com.zhiwei.crawler.utils.RequestUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.core.RequestUtils;
 import com.zhiwei.crawler.proxy.ProxyHolder;
 import com.zhiwei.live.bean.RoomInfo;
 import com.zhiwei.tools.tools.ZhiWeiTools;
@@ -17,7 +17,7 @@ import com.zhiwei.tools.tools.ZhiWeiTools;
  *
  */
 public class PandamTVRoomInfoCrawler {
-	private static HttpBoot httpBoot = new HttpBoot();
+	private static HttpBoot httpBoot = new HttpBoot.Builder().build();
 	private static Logger logger = LogManager.getLogger(PandamTVRoomInfoCrawler.class);
 	private static final String PT = "熊猫TV";
 	
@@ -85,7 +85,7 @@ public class PandamTVRoomInfoCrawler {
 	 * @throws Exception
 	 */
 	public static RoomInfo getRoomInfoProxyByRoomUrl(String roomUrl) throws Exception{
-		String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl),ProxyHolder.NAT_PROXY).body().string();
+		String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(roomUrl),ProxyHolder.NAT_HEAVY_PROXY).body().string();
 		if(!StringUtils.isBlank(htmlBody)) {
 			//判断页面中是否包含房间信息,此为pc端直播
 			if(htmlBody.contains("window._config_roominfo = ")) {

src/test/java/com/zhiwei/live/test/danmu/BilibiliDanMuTest.java

@@ -7,7 +7,7 @@ import com.zhiwei.live.danmu.util.DataCallBack;
 public class BilibiliDanMuTest {
 	
 	public static void main(String[] args) throws InterruptedException {
-    	String roomUrl = "https://live.bilibili.com/529";
+    	String roomUrl = "https://live.bilibili.com/387";
     	try {
 			BilibiliClient.getDanmu(new DataCallBack() {
                 @Override

src/test/java/com/zhiwei/live/test/danmu/DouYuDanMuTest.java

@@ -7,7 +7,7 @@ import com.zhiwei.live.danmu.util.DataCallBack;
 public class DouYuDanMuTest {
 
 	public static void main(String[] args) throws InterruptedException {
-		String roomUrl = "https://www.douyu.com/71017";
+		String roomUrl = "https://www.douyu.com/topic/NarutoMatch?rid=535534";
 		try {
 			DouyuClient.getDanmu(new DataCallBack() {
 				@Override