我试图让使用C＃的网络爬虫先贤篇
我也跟着快速入门教程 ，但我似乎无法使它工作。
它在方法未处理的异常crawler_ProcessPageCrawlCompleted ，恰好这一行：
if (crawledPage.WebException != null || crawledPage.HttpWebResponse.StatusCode != HttpStatusCode.OK)
{
Console.WriteLine("Crawl of page failed {0}", crawledPage.Uri.AbsoluteUri);
}
由于crawledPage.HttpWebResponse为空。
我可能失去了一些东西，但什么？
注释和全码：
我编辑app.config文件作为教程提出，这里是我的类（即引用Abot.dll）：
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using Abot.Crawler;
using Abot.Poco;
using System.Net;
using System.Windows.Forms; // for HttpStatusCode
namespace WebCrawler
{
public class MyCrawler
{
public MyCrawler()
{
}
public PoliteWebCrawler crawler;
public void initialize()
{
// 3. Create an instance of Abot.Crawler.PoliteWebCrawler
// 3.2 Will use app.config for confguration
// because I choose 2.1 === edited app.config
crawler = new PoliteWebCrawler();
// 4. Register for events and create processing methods (both synchronous and asynchronous versions available)
crawler.PageCrawlStartingAsync += crawler_ProcessPageCrawlStarting;
crawler.PageCrawlCompletedAsync += crawler_ProcessPageCrawlCompleted;
crawler.PageCrawlDisallowedAsync += crawler_PageCrawlDisallowed;
crawler.PageLinksCrawlDisallowedAsync += crawler_PageLinksCrawlDisallowed;
#region(Step 5. Add custom objects to crawl bag ?)
//5. Add any number of custom objects to the dynamic crawl bag. These objects will be available in the CrawlContext.CrawlBag object.
// ???
/*
PoliteWebCrawler crawler = new PoliteWebCrawler();
crawler.CrawlBag.MyFoo1 = new Foo();
crawler.CrawlBag.MyFoo2 = new Foo();
crawler.PageCrawlStartingAsync += crawler_ProcessPageCrawlStarting;
void crawler_ProcessPageCrawlStarting(object sender, PageCrawlStartingArgs e)
{
//Get your Foo instances from the CrawlContext object
CrawlContext context = e.CrawlContext;
context.CrawlBag.MyFoo1.Bar();
context.CrawlBag.MyFoo2.Bar();
}
*/
#endregion
}// initialize()
public void doCrawl()
{
CrawlResult result = crawler.Crawl(new Uri("http://yahoo.com"));
if (result.ErrorOccurred)
{
/* line 60 : */ // Console.WriteLine("Crawl of {0} completed with error: {1}", result.RootUri.AbsoluteUri, result.ErrorMessage);
// I commented out because it outputs the error : 'Abot.Poco.CrawlResult' does not contain a definition for 'ErrorMessage'
}
else
{
Console.WriteLine("Crawl of {0} completed without error.", result.RootUri.AbsoluteUri);
}
}
void crawler_ProcessPageCrawlStarting(object sender, PageCrawlStartingArgs e)
{
PageToCrawl pageToCrawl = e.PageToCrawl;
Console.WriteLine("About to crawl link {0} which was found on page {1}", pageToCrawl.Uri.AbsoluteUri, pageToCrawl.ParentUri.AbsoluteUri);
}
void crawler_ProcessPageCrawlCompleted(object sender, PageCrawlCompletedArgs e)
{
CrawledPage crawledPage = e.CrawledPage;
if (crawledPage.HttpWebResponse == null)
{
MessageBox.Show("HttpWebResponse null");
}
/* line 84 : */ if (crawledPage.WebException != null || crawledPage.HttpWebResponse.StatusCode != HttpStatusCode.OK)
Console.WriteLine("Crawl of page failed {0}", crawledPage.Uri.AbsoluteUri);
else
Console.WriteLine("Crawl of page succeeded {0}", crawledPage.Uri.AbsoluteUri);
if (string.IsNullOrEmpty(crawledPage.RawContent))
Console.WriteLine("Page had no content {0}", crawledPage.Uri.AbsoluteUri);
}
void crawler_PageLinksCrawlDisallowed(object sender, PageLinksCrawlDisallowedArgs e)
{
CrawledPage crawledPage = e.CrawledPage;
Console.WriteLine("Did not crawl the links on page {0} due to {1}", crawledPage.Uri.AbsoluteUri, e.DisallowedReason);
}
void crawler_PageCrawlDisallowed(object sender, PageCrawlDisallowedArgs e)
{
PageToCrawl pageToCrawl = e.PageToCrawl;
Console.WriteLine("Did not crawl page {0} due to {1}", pageToCrawl.Uri.AbsoluteUri, e.DisallowedReason);
}
}// end of public class MyCrawler
}
该错误是在管线84。
此外，附加的细节（也许这表示什么，我缺少的）是第60行中，这就是：
'Abot.Poco.CrawlResult' does not contain a definition for 'ErrorMessage' and no extension method 'ErrorMessage' accepting a first argument of type 'Abot.Poco.CrawlResult' could be found (are you missing a using directive or an assembly reference?)
谢谢你的帮助 ！
