using HtmlAgilityPack;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading.Tasks;
namespace HttpCrawler
{
class Program
{
static void Main(string[] args)
{
Stopwatch sw = new Stopwatch();
sw.Start();
var titles = from row in GetHtml("http://bbs.csdn.net/forums/DotNET/").DocumentNode.SelectSingleNode("//table[@class='table_list parent_forum ']").Elements("tr").Skip(1)
let td = row.Element("td")
where td != null
let a = td.Descendants("a").FirstOrDefault()
where a != null
select new
{
href = a.Attributes["href"].Value,
text = a.InnerText
};
var pages = from t in titles
.AsParallel().WithDegreeOfParallelism(20)
where t.href != null
let path = "http://bbs.csdn.net" + t.href
let subQuery = from nick in GetHtml(path).DocumentNode.SelectNodes("//span[@class='name2nick']")
where nick.InnerText == "sp1234"
select nick
where subQuery.Any()
select new
{
title = t.text,
href = path
};
var results = pages.ToList();
sw.Stop();
Console.WriteLine("不加并发的时间:"+sw.ElapsedMilliseconds);
Console.ReadKey();
}
static HtmlDocument GetHtml(string url)
{
var content = Encoding.UTF8.GetString(new WebClient().DownloadData(url));
var doc = new HtmlDocument();
doc.Load(new StringReader(content));
return doc;
}
}
}
转载于:https://www.cnblogs.com/c-x-a/p/7792750.html
相关资源：JAVA上百实例源码以及开源项目
