package com.zhiweidata.bjfzxWebapp.listener;

import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.zhiweidata.bjfzxWebapp.conf.GenericAttribute;
import com.zhiweidata.bjfzxWebapp.entity.Keyword;
import com.zhiweidata.bjfzxWebapp.entity.MediaAndWechatEvent;
import com.zhiweidata.bjfzxWebapp.htttpClient.HttpClientTemplate;
import com.zhiweidata.bjfzxWebapp.util.TimeUtil;
import com.zhiweidata.bjfzxWebapp.util.Tools;

/**
 * @ClassName: MediaBaiDuSerach 
 * @Description: TODO(这里用一句话描述这个类的作用) 
 * @author xuyimeng 
 * @date 2018年2月6日 上午10:35:24
 */
public class MediaBaiDuSerach
{
	
	private static Logger log = LoggerFactory.getLogger(MediaBaiDuSerach.class);
//	@Resource(name="keywordDao")
//	private KeywordDao keywordDao;
	
	private static HttpClientTemplate httpClient = new HttpClientTemplate();

	private static boolean f = true; // 判断是否有下一页

	private static Map<String, String> headerMap; // 采集头信息

	public MediaBaiDuSerach() {
		/** 设置采集头信息 **/
		headerMap = new HashMap<String, String>();
		headerMap.put("User-Agent",
				"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.116 Safari/537.36");
		headerMap.put("Host", "news.baidu.com");
		headerMap.put("Upgrade-Insecure-Requests", "1");
	}

	/**
	 * @Title: mediaKeywordSearchByBaiDu
	 * @Description: TODO(根据时间和关键词获取相关数据)
	 * @param @param
	 *            start
	 * @param @param
	 *            end
	 * @param @param
	 *            keyword
	 * @param @return
	 * @param @throws
	 *            UnsupportedEncodingException
	 * @param @throws
	 *            Exception 设定文件
	 * @return List<Media> 返回类型
	 */
	public List<MediaAndWechatEvent> mediaKeywordSearchByBaiDu(long start, long end, Keyword keyword,List<Keyword> keywords)
			throws UnsupportedEncodingException, Exception
	{
		List<MediaAndWechatEvent> result = new ArrayList<MediaAndWechatEvent>();
		int page = 1;
		int a = 0;
		String word = URLEncoder.encode(keyword.getKeyword(), "UTF-8");
		/** 循环翻页 **/
		while (f)
		{
			try {
				Thread.sleep(100);
			} catch (Exception e1) {
				e1.printStackTrace();
			}
			
			String url = "http://news.baidu.com/ns?from=news&cl=2&bt=" + start + "&et=" + end + "&q1=" + word
					+ "&q3=&q4=&tn=newsdy" + "&ct=0&rn=50&clk=sortbytime&q6=&pn=" + (page - 1) * 50;
			headerMap.put("Referer", url);
			log.debug("{}【page:{}】，url:【{}】",keyword, page, url);
			String htmlBody = httpClient.executeHttpRequestGet(url, headerMap);
			if (htmlBody != null)
			{
				for (int i = 0; i < 3; i++)
				{
					if (htmlBody.contains("抱歉，没有找到与"))
					{
						log.debug("抱歉，没有找到与“" + keyword + "” 相关的新闻内容。");
					}
					else if (htmlBody.contains("此页面不存在"))
					{
						log.debug("此页面不存在");
					}
					else
					{
						List<MediaAndWechatEvent> maweList = null;
						try
						{
							maweList = analysisNews(htmlBody, keyword, keywords);
							result.addAll(maweList);
							log.debug("第{}页数据量为:{}", page++, maweList.size());
						}
						catch (Exception e)
						{
							log.debug("第{}页未采集到数据,原因:【{}】", page++, e.getMessage());
						}
						break;
					}
					if (i == 2)
					{
						f = false;
					}
				}
			}
			else
			{

				log.debug("第一层采集数据出现连接超时等问题.....尝试【{}】,到达3是跳出", ++a);
				if (a >= 3)
				{
					f = false;
				}
			}
		}
		f = true;
		if (log.isInfoEnabled())
		{
			log.debug("去重后数据量为:{}", goHeavy(result).size());
		}		
		return goHeavy(result);		
	}

	/**
	 * @Title: analysisXML
	 * @Description: TODO(解析百度新闻数据结构结构)
	 * @param @param
	 *            htmlBody
	 * @param @return
	 * @param @throws
	 *            Exception 设定文件
	 * @return List<MediaAndWechatEvent> 返回类型
	 */
	private List<MediaAndWechatEvent> analysisNews(String htmlBody, Keyword keyword,List<Keyword> keywords) throws Exception
	{
		/** 解析页面 */
		Document document = Jsoup.parse(htmlBody);
		/** 判断是否有下一页 **/
		if (document.select("p#page") == null)
		{
			f = false;
		}
		else
		{
			if (!document.select("p#page").text().contains("下一页"))
			{
				f = false;
			}
		}

		/** 解析数据 **/
		return this.getNewsData(document, keyword, keywords);
	}

	/**
	 * @deprecated:解析百度新闻结构
	 * @param Document
	 *            document 要解析的数据结构
	 * @param HashMap
	 *            otherLinkMap 相同新闻地址map
	 * @return List<MediaAndWechatEvent>
	 **/
	private List<MediaAndWechatEvent> getNewsData(Document document, Keyword keyword, List<Keyword> keywords)
	{
		List<MediaAndWechatEvent> result = new ArrayList<MediaAndWechatEvent>();
		/** 相似新闻链接地址 **/
		Map<String, Integer> similarLink = new HashMap<String, Integer>();
		/** 解析数据 */
		Elements elm = document.select("div.result");
		String time = "";
		String source = "";
		for (Element el : elm)
		{
			try
			{
				String link = el.select("h3.c-title").select("a").attr("href");
				String title = el.select("h3.c-title").select("a").text();
				String soure_time = el.select("div.c-row").select("p.c-author").html();
				/** 截取时间 **/
				if (soure_time.contains("&nbsp;&nbsp;"))
				{
					String tim[] = soure_time.split("&nbsp;&nbsp;");
					time = tim[1];
					source = tim[0];
				}
				else
				{
					time = el.select("div.c-row").select("p.c-author").text();
				}
				// 处理文章简介
				String descript = el.select("div.c-row").text();
				String descriptA = el.select("div.c-row").select("p.c-author").text();
				String descriptC = descript.substring(descriptA.length(), descript.length());
				Pattern pattern = Pattern.compile("\\d*条相同新闻");
				Matcher matcher = pattern.matcher(descriptC);
				String content = matcher.replaceAll("").replaceAll("-", "").replaceAll("百度快照", "");
				/** 文章发布时间处理 **/
				Date date = TimeUtil.conversionDate(time);
				/** 采集相同新闻链接 **/
				if (el.select("div.c-row").select("a.c-more_link") != null)
				{
					String otherLink = "http://news.baidu.com"
							+ el.select("div.c-row").select("a.c-more_link").attr("href");
					// 判断之前的链接中是否存在这个key
					similarLink.put(otherLink, 1);
				}
				//去除杂数据
				String key = keyword.getKeyword();
				if (key.contains("+"))
				{
					key = key.replaceAll("\\+", " ");
				}
				if (key.contains("-"))
				{
					key = key.replaceAll("\\-", " ");
				}
				
				String[] keys = key.split("\\s+");
				
				String str = ".*";
				for (String keyStr : keys)
				{
					str += keyStr + ".*";
				}
				
				Pattern patt = Pattern.compile(str);
				
				if (!patt.matcher(content+title).find())
				{
					continue;
				}
				
				/** 存储数据 **/
				log.debug("【title:{}】", title);
				
				MediaAndWechatEvent mediaAndWechatEvent = new MediaAndWechatEvent(link, title, date, source, content,
						GenericAttribute.PT_MEDIADATA, GenericAttribute.PT_MEDIADATA, keyword.getKeyword(),keyword.getGroupName());
				result.add(mediaAndWechatEvent);
//				
			}
			catch (Exception e)
			{
				log.error("解析百度新闻数据时出现bug" + e.getMessage());
				continue;
			}
		}
		/** 添加相同新闻 **/
		for (Entry<String, Integer> entry : similarLink.entrySet())
		{
			String otherLink = entry.getKey();
			// 通过httpClient下载相关数据
			String htmlBodyOther = null;
			try
			{
				htmlBodyOther = httpClient.executeHttpRequestGet(otherLink, headerMap);
			}
			catch (Exception e)
			{
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
			Document documentOther = null;
			if (htmlBodyOther != null)
			{
				documentOther = Jsoup.parse(htmlBodyOther);
				List<MediaAndWechatEvent> otherList = getNewsData(documentOther, keyword, keywords);
				result.addAll(otherList);
				documentOther = null;
				Tools.sleep(50);
			}
			else
			{
				log.debug("百度采集【OtherLink为null】");
				break;
			}
		}
		return result;
	}

	/***
	 * @Title: goHeavy
	 * @Description: TODO(数据去重,去重重复的数据结果集)
	 * @param @param
	 *            dataList
	 * @param @return
	 *            设定文件
	 * @return List<MediaAndWechatEvent> 返回类型
	 */
	private List<MediaAndWechatEvent> goHeavy(List<MediaAndWechatEvent> dataList)
	{
		List<MediaAndWechatEvent> result = new ArrayList<MediaAndWechatEvent>();
		Map<String, MediaAndWechatEvent> dataMap = new HashMap<String, MediaAndWechatEvent>();
		/** 根据标题+来源去重 **/
		for (MediaAndWechatEvent mawe : dataList)
		{
			String key = mawe.getTitle().hashCode() + "_" + mawe.getSource().hashCode()
					+"_"+mawe.getKeyword();
			if (!dataMap.containsKey(key))
			{
				dataMap.put(key, mawe);
			}
		}
		/** 整合最后的数据结果集 **/
		for (Entry<String, MediaAndWechatEvent> entry : dataMap.entrySet())
		{
			result.add(entry.getValue());
		}
		return result;
	}

}
