Skip to content

T
toutiao
Commit c692ae5b by zhiwei
Options

添加更新今日头条阅读数功能

parent 9d384b56
Showing with 143 additions and 145 deletions
pom.xml
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<groupId>com.zhiwei</groupId> <groupId>com.zhiwei</groupId>
<artifactId>toutiao</artifactId> <artifactId>toutiao</artifactId>
<version>0.4.0-SNAPSHOT</version> <version>0.4.1-SNAPSHOT</version>
<dependencies> <dependencies>
<dependency> <dependency>
<groupId>com.zhiwei.tools</groupId> <groupId>com.zhiwei.tools</groupId>
<artifactId>zhiwei-tools</artifactId> <artifactId>zhiwei-tools</artifactId>
<version>0.1.4-SNAPSHOT</version> <version>0.1.5-SNAPSHOT</version>
<scope>provided</scope> <scope>provided</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.zhiwei.crawler</groupId> <groupId>com.zhiwei.crawler</groupId>
<artifactId>crawler-core</artifactId> <artifactId>crawler-core</artifactId>
<version>0.5.2-SNAPSHOT</version> <version>0.5.2-SNAPSHOT</version>
<scope>provided</scope> <scope>provided</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.zhiwei</groupId> <groupId>com.zhiwei</groupId>
<artifactId>excelpoi</artifactId> <artifactId>excelpoi</artifactId>
<version>0.0.5-SNAPSHOT</version> <version>0.0.5-SNAPSHOT</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.mongodb</groupId> <groupId>org.mongodb</groupId>
<artifactId>mongo-java-driver</artifactId> <artifactId>mongo-java-driver</artifactId>
<version>3.8.1</version> <version>3.8.1</version>
</dependency> </dependency>
</dependencies> </dependencies>
<!-- 打包管理 --> <!-- 打包管理 -->
<build> <build>
<plugins> <plugins>
<!-- 发布源码 --> <!-- 发布源码 -->
<plugin> <plugin>
<artifactId>maven-source-plugin</artifactId> <artifactId>maven-source-plugin</artifactId>
<version>2.4</version> <version>2.4</version>
<configuration> <configuration>
<attach>true</attach> <attach>true</attach>
</configuration> </configuration>
<executions> <executions>
<execution> <execution>
<phase>compile</phase> <phase>compile</phase>
<goals> <goals>
<goal>jar</goal> <goal>jar</goal>
</goals> </goals>
</execution> </execution>
</executions> </executions>
</plugin> </plugin>
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId> <artifactId>maven-javadoc-plugin</artifactId>
<version>2.10.4</version> <version>2.10.4</version>
</plugin> </plugin>
<!-- 解决maven test命令时console出现中文乱码乱码 --> <!-- 解决maven test命令时console出现中文乱码乱码 -->
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId> <artifactId>maven-surefire-plugin</artifactId>
<version>2.7.2</version> <version>2.7.2</version>
<configuration> <configuration>
<forkMode>once</forkMode> <forkMode>once</forkMode>
<argLine>-Dfile.encoding=UTF-8</argLine> <argLine>-Dfile.encoding=UTF-8</argLine>
</configuration> </configuration>
</plugin> </plugin>
</plugins> </plugins>
</build> </build>
<!-- 分发管理:管理distribution和supporting files --> <!-- 分发管理:管理distribution和supporting files -->
<distributionManagement> <distributionManagement>
<snapshotRepository> <snapshotRepository>
<id>nexus-releases</id> <id>nexus-releases</id>
<name>User Porject Snapshot</name> <name>User Porject Snapshot</name>
<url>http://192.168.0.30:8081/nexus/content/repositories/snapshots/</url > <url>http://192.168.0.30:8081/nexus/content/repositories/snapshots/</url >
<uniqueVersion>true</uniqueVersion> <uniqueVersion>true</uniqueVersion>
</snapshotRepository> </snapshotRepository>
<repository> <repository>
<id>nexus-releases</id> <id>nexus-releases</id>
<name>User Porject Release</name> <name>User Porject Release</name>
<url>http://192.168.0.30:8081/nexus/content/repositories/releases/</url <url>http://192.168.0.30:8081/nexus/content/repositories/releases/</url
> >
</repository> </repository>
</distributionManagement> </distributionManagement>
</project> </project>
\ No newline at end of file
src/test/java/com/zhiwei/toutiao/test/TouTiaoQuestionExample.java
package com.zhiwei.toutiao.test; package com.zhiwei.toutiao.test;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import com.alibaba.fastjson.JSONObject;
import org.junit.jupiter.api.Test; import com.mongodb.BasicDBObject;
import com.mongodb.DB;
import com.alibaba.fastjson.JSONObject; import com.mongodb.DBCollection;
import com.mongodb.BasicDBObject; import com.mongodb.DBObject;
import com.mongodb.DB; import com.mongodb.Mongo;
import com.mongodb.DBCollection; import com.zhiwei.common.config.GroupType;
import com.mongodb.DBObject; import com.zhiwei.crawler.proxy.ProxyFactory;
import com.mongodb.Mongo; import com.zhiwei.excelpoi.bean.ExcelResult;
import com.zhiwei.common.config.GroupType; import com.zhiwei.excelpoi.excel.PoiExcelUtil;
import com.zhiwei.crawler.proxy.ProxyFactory; import com.zhiwei.toutiao.bean.TouTiaoQuestion;
import com.zhiwei.excelpoi.bean.ExcelResult; import com.zhiwei.toutiao.parse.TouTiaoQuestionParse;
import com.zhiwei.excelpoi.excel.PoiExcelUtil;
import com.zhiwei.toutiao.bean.TouTiaoQuestion; /**
import com.zhiwei.toutiao.parse.TouTiaoQuestionParse; * @ClassName: TouTiaoQuestionExample
* @Description: TODO(头条问答采集测试类)
/** * @author hero
* @ClassName: TouTiaoQuestionExample * @date 2017年7月20日 下午3:06:51
* @Description: TODO(头条问答采集测试类) */
* @author hero public class TouTiaoQuestionExample {
* @date 2017年7月20日 下午3:06:51
*/ private static final String registry = "zookeeper://192.168.0.36:2181";
public class TouTiaoQuestionExample { private static final String group = "local";
private static final String registry = "zookeeper://192.168.0.36:2181"; public static void main(String[] args) {
private static final String group = "local"; ProxyFactory.init(registry, group, GroupType.PROVIDER);
public static void main(String[] args) { Mongo mongo = new Mongo("192.168.0.81", 27017);
ProxyFactory.init(registry, group, GroupType.PROVIDER); DB db = mongo.getDB("wukong");
DBCollection coll = db.getCollection("wukong");
Mongo mongo = new Mongo("192.168.0.81", 27017);
DB db = mongo.getDB("wukong"); touTiaoQuestionTest(coll);
DBCollection coll = db.getCollection("wukong");
}
touTiaoQuestionTest(coll);
public static void touTiaoQuestionTest(DBCollection coll) {
} PoiExcelUtil poi = PoiExcelUtil.getInstance();
ExcelResult excelResult = poi.importExcelResult("C:\\Users\\qq859\\Desktop\\悟空问答关键词.xlsx", 0);
public static void touTiaoQuestionTest(DBCollection coll) { List<Map<String,Object>> dataList = excelResult.getBodyList();
PoiExcelUtil poi = PoiExcelUtil.getInstance(); for(Map<String,Object> data : dataList) {
ExcelResult excelResult = poi.importExcelResult("C:\\Users\\qq859\\Desktop\\悟空问答关键词.xlsx", 0); String word = data.get("关键词").toString();
List<Map<String,Object>> dataList = excelResult.getBodyList(); System.out.println("word================"+word);
for(Map<String,Object> data : dataList) { List<TouTiaoQuestion> list = TouTiaoQuestionParse.getSearchTouTiaoQuestion(word);
String word = data.get("关键词").toString(); System.out.println(list.size());
System.out.println("word================"+word); for(TouTiaoQuestion question : list){
List<TouTiaoQuestion> list = TouTiaoQuestionParse.getSearchTouTiaoQuestion(word); String jsonStr = JSONObject.toJSONString(question);
System.out.println(list.size()); Map dataMap = JSONObject.toJavaObject(JSONObject.parseObject(jsonStr), Map.class);
for(TouTiaoQuestion question : list){ dataMap.put("word", word);
String jsonStr = JSONObject.toJSONString(question); coll.save(new BasicDBObject(dataMap));
Map dataMap = JSONObject.toJavaObject(JSONObject.parseObject(jsonStr), Map.class); }
dataMap.put("word", word); }
coll.save(new BasicDBObject(dataMap)); }
}
} }
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment