Skip to content

A
articlenewscrawler
Commit f9343985 by win 10
Options

解决冲突

parent 6018f0b3
Showing with 42 additions and 42 deletions
src/test/java/com/zhiwei/shipin/BilibiliTest.java
package com.zhiwei.shipin; //package com.zhiwei.shipin;
//
import java.util.ArrayList; //import java.util.ArrayList;
import java.util.List; //import java.util.List;
import java.util.Map; //import java.util.Map;
//
import org.junit.Test; //import org.junit.Test;
//
import com.zhiwei.common.config.GroupType; //import com.zhiwei.common.config.GroupType;
import com.zhiwei.crawler.proxy.ProxyFactory; //import com.zhiwei.crawler.proxy.ProxyFactory;
import com.zhiwei.excelpoi.excel.PoiExcelUtil; //import com.zhiwei.excelpoi.excel.PoiExcelUtil;
import com.zhiwei.parse.BiliBili; //import com.zhiwei.parse.BiliBili;
import com.zhiwei.util.WordReadFile; //import com.zhiwei.util.WordReadFile;
//
public class BilibiliTest { //public class BilibiliTest {
@Test // @Test
public void f() { // public void f() {
ProxyFactory.init("zookeeper://192.168.0.11:2181?backup=192.168.0.30:2181,192.168.0.35:2181", "local", GroupType.PROVIDER , 10000002L); // ProxyFactory.init("zookeeper://192.168.0.11:2181?backup=192.168.0.30:2181,192.168.0.35:2181", "local", GroupType.PROVIDER , 10000002L);
List<String> wordList = WordReadFile.getWords("D://crawlerdata//关键词.txt"); // List<String> wordList = WordReadFile.getWords("E://crawlerdata//关键词.txt");
List<Map<String, Object>> bodyList = new ArrayList<>(); // List<Map<String, Object>> bodyList = new ArrayList<>();
String cookie = "LIVE_BUVID=AUTO8715300758995538; sid=kp5rluge; fts=1530161621; im_notify_type_35324319=0; buvid3=08ABE6AE-5061-4CE5-B34F-1A8AAB64DB3320712infoc; rpdid=olppsmkxmpdoskwoxiwww; finger=edc6ecda; stardustvideo=1; UM_distinctid=164fe68fb31996-01f161c3523abe-6114167a-1fa400-164fe68fb32274"; // String cookie = "LIVE_BUVID=AUTO8715300758995538; sid=kp5rluge; fts=1530161621; im_notify_type_35324319=0; buvid3=08ABE6AE-5061-4CE5-B34F-1A8AAB64DB3320712infoc; rpdid=olppsmkxmpdoskwoxiwww; finger=edc6ecda; stardustvideo=1; UM_distinctid=164fe68fb31996-01f161c3523abe-6114167a-1fa400-164fe68fb32274";
for (String word : wordList) { // for (String word : wordList) {
List<Map<String, Object>> dataList = BiliBili.getData(word, null, "2001-01-14 00:00:00", // List<Map<String, Object>> dataList = BiliBili.getData(word, null, "2001-01-14 00:00:00",
cookie); // cookie);
if (dataList != null) { // if (dataList != null) {
System.out.println(word + " ----- " + dataList.size()); // System.out.println(word + " ----- " + dataList.size());
bodyList.addAll(dataList); // bodyList.addAll(dataList);
} // }
} // }
List<String> headlist = new ArrayList<>(); // List<String> headlist = new ArrayList<>();
headlist.add("submitcount"); // headlist.add("submitcount");
headlist.add("playcount"); // headlist.add("playcount");
headlist.add("time"); // headlist.add("time");
headlist.add("source"); // headlist.add("source");
headlist.add("title"); // headlist.add("title");
headlist.add("url"); // headlist.add("url");
headlist.add("word"); // headlist.add("word");
PoiExcelUtil poi = PoiExcelUtil.getInstance(); // PoiExcelUtil poi = PoiExcelUtil.getInstance();
poi.exportExcel("D://crawlerdata//视频//bilibili关键词采集数据-封神神话-收藏.xlsx", "B站数据", headlist, bodyList); // poi.exportExcel("E://crawlerdata//视频//bilibili关键词采集数据-发布.xlsx", "B站数据", headlist, bodyList);
//
} // }
} //}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment