using System; using System.IO; using System.Linq; using System.Threading.Tasks; using HtmlAgilityPack; using ScrapySharp.Extensions; using ScrapySharp.Network;
namespaceScrapySharpDemo { classProgram { staticvoidMain(string[] args) { //示例网站地址 var url = "http://bbs.tianya.cn/post-12-563201-1.shtml"; var web = new ScrapingBrowser(); var html = web.DownloadString(new Uri(url)); var doc = new HtmlDocument(); doc.LoadHtml(html); //获取网站中的图片地址 var urls= doc.DocumentNode.CssSelect("div.bbs-content > img").Select(node => node.GetAttributeValue("original")).ToList(); //并行下载图片 Parallel.ForEach(urls, SavePic); }
publicstaticvoidSavePic(string url) { var web = new ScrapingBrowser(); //因天涯网站限制,所有站外来源都无法访问图片,故先设置请求头Refer属性为当前页地址 web.Headers.Add("Referer", "http://bbs.tianya.cn/post-12-563201-1.shtml"); var pic = web.NavigateToPage(new Uri(url)).RawResponse.Body; var file = url.Substring(url.LastIndexOf("/", StringComparison.Ordinal)); if (!Directory.Exists("imgs")) Directory.CreateDirectory("imgs"); File.WriteAllBytes("imgs" + file, pic); } } }