NET抓取和分析网页的类

主要功能有：1、提取网页的纯文本，去所有html标签和javascript代码2、提取网页的链接，包括href和frame及iframe3、提取网页的title等(其它的标签可依此类推，正则是一样的)4、可以实现简单的表单提交及cookie保存第一部分using System;using System.Data;using System.Configuration;using ;using System.IO;using System.Text;using System.Collections.Generic;using System.Text.RegularExpressions;using System.Threading;using System.Web;/// <summary>/// 网页类/// </summary>public class WebPage{#region 私有成员private Uri m_uri; //网址private List<Link> m_links; //此网页上的链接private string m_title; //此网页的标题private string m_html; //此网页的HTML代码private string m_outstr; //此网页可输出的纯文本private bool m_good; //此网页是否可用private int m_pagesize; //此网页的大小private static Dictionary<string, CookieContainer> webcookies = new Diction ary<string, CookieContainer>();//存放所有网页的Cookieprivate string m_post; //此网页的登陆页需要的POST数据private string m_loginurl; //此网页的登陆页#endregion#region 私有方法/// <summary>/// 这私有方法从网页的HTML代码中分析出链接信息/// </summary>/// <returns>List<Link></returns>private List<Link> getLinks(){if (m_links.Count == 0){Regex[] regex = new Regex[2];regex[0] = new Regex("(?m)<a[^><]+href=(\"|')?(?<url>([^>\"'\\s)])+ )(\"|')?[^>]*>(?<text>(\\w|\\W)*?)</", RegexOptions.Multiline | RegexOptions.Ig noreCase);regex[1] = new Regex("<[i]*frame[^><]+src=(\"|')?(?<url>([^>\"'\\s) ])+)(\"|')?[^>]*>", RegexOptions.Multiline | RegexOptions.IgnoreCase);for (int i = 0; i < 2; i++){Match match = regex[i].Match(m_html);while (match.Success){try{string url = new Uri(m_uri, match.Groups["url"].Value). AbsoluteUri;string text = "";if (i == 0) text = new Regex("(<[^>]+>)|(\\s)|(&nbsp;)| &|\"", RegexOptions.Multiline | RegexOptions.IgnoreCase).Replace(match.Groups[" text"].Value, "");Link link = new Link(url, text);m_links.Add(link);}catch(Exception ex){Console.WriteLine(ex.Message); };match = match.NextMatch();}}}return m_links;}/// <summary>/// 此私有方法从一段HTML文本中提取出一定字数的纯文本/// </summary>/// <param name="instr">HTML代码</param>/// <param name="firstN">提取从头数多少个字</param>/// <param name="withLink">是否要链接里面的字</param>/// <returns>纯文本</returns>private string getFirstNchar(string instr, int firstN, bool withLink){if (m_outstr == ""){m_outstr = instr.Clone() as string;m_outstr = new Regex(@"(?m)<script[^>]*>(\w|\W)*?</script[^>]*>", R egexOptions.Multiline | RegexOptions.IgnoreCase ).Replace(m_outstr, "");m_outstr = new Regex(@"(?m)<style[^>]*>(\w|\W)*?</style[^>]*>", Reg exOptions.Multiline | RegexOptions.IgnoreCase ).Replace(m_outstr, "");m_outstr = new Regex(@"(?m)<select[^>]*>(\w|\W)*?</select[^>]*>", R egexOptions.Multiline | RegexOptions.IgnoreCase ).Replace(m_outstr, "");if (!withLink) m_outstr = new Regex(@"(?m)<a[^>]*>(\w|\W)*?</a[^>]* >", RegexOptions.Multiline | RegexOptions.IgnoreCase).Replace(m_outstr, ""); Regex objReg = new System.Text.RegularExpressions.Regex("(<[^>]+?>) |&nbsp;", RegexOptions.Multiline | RegexOptions.IgnoreCase);m_outstr = objReg.Replace(m_outstr, "");Regex objReg2 = new System.Text.RegularExpressions.Regex("(\\s)+", RegexOptions.Multiline | RegexOptions.IgnoreCase);m_outstr = objReg2.Replace(m_outstr, " ");}return m_outstr.Length > firstN ? m_outstr.Substring(0, firstN) : m_out str;}/// <summary>/// 此私有方法返回一个IP地址对应的无符号整数/// </summary>/// <param name="x">IP地址</param>/// <returns></returns>private uint getuintFromIP(IPAddress x){Byte[] bt = x.GetAddressBytes();uint i = (uint)(bt[0] * 256 * 256 * 256);i += (uint)(bt[1] * 256 * 256);i += (uint)(bt[2] * 256);i += (uint)(bt[3]);return i;}#endregion#region 公有文法/// <summary>/// 此公有方法提取网页中一定字数的纯文本，包括链接文字/// </summary>/// <param name="firstN">字数</param>/// <returns></returns>public string getContext(int firstN){return getFirstNchar(m_html, firstN, true);}/// <summary>/// 此公有方法提取网页中一定字数的纯文本，不包括链接文字/// </summary>/// <param name="firstN"></param>/// <returns></returns>public string getContextWithOutLink(int firstN){return getFirstNchar(m_html, firstN, false);}/// <summary>/// 此公有方法从本网页的链接中提取一定数量的链接，该链接的URL满足某正则式 /// </summary>/// <param name="pattern">正则式</param>/// <param name="count">返回的链接的个数</param>/// <returns>List<Link></returns>public List<Link> getSpecialLinksByUrl(string pattern,int count){if(m_links.Count==0)getLinks();List<Link> SpecialLinks = new List<Link>();List<Link>.Enumerator i;i = m_links.GetEnumerator();int cnt = 0;while (i.MoveNext() && cnt<count){if (new Regex(pattern, RegexOptions.Multiline | RegexOptions.Ignore Case ).Match(i.Current.url).Success){SpecialLinks.Add(i.Current);cnt++;}}return SpecialLinks;}/// <summary>/// 此公有方法从本网页的链接中提取一定数量的链接，该链接的文字满足某正则式/// </summary>/// <param name="pattern">正则式</param>/// <param name="count">返回的链接的个数</param>/// <returns>List<Link></returns>public List<Link> getSpecialLinksByText(string pattern,int count){if (m_links.Count == 0) getLinks();List<Link> SpecialLinks = new List<Link>();List<Link>.Enumerator i;i = m_links.GetEnumerator();int cnt = 0;while (i.MoveNext() && cnt < count){if (new Regex(pattern, RegexOptions.Multiline | RegexOptions.Ignore Case ).Match(i.Current.text).Success){SpecialLinks.Add(i.Current);cnt++;}}return SpecialLinks;}/// <summary>/// 此公有方法获得所有链接中在一定IP范围的链接/// </summary>/// <param name="_ip_start">起始IP</param>/// <param name="_ip_end">终止IP</param>/// <returns></returns>public List<Link> getSpecialLinksByIP(string _ip_start, string _ip_end){IPAddress ip_start = IPAddress.Parse(_ip_start);IPAddress ip_end = IPAddress.Parse(_ip_end);if (m_links.Count == 0) getLinks();List<Link> SpecialLinks = new List<Link>();List<Link>.Enumerator i;i = m_links.GetEnumerator();while (i.MoveNext()){IPAddress ip;try{ip = Dns.GetHostEntry(new Uri(i.Current.url).Host).AddressList[ 0];}catch { continue; }if(getuintFromIP(ip)>=getuintFromIP(ip_start) && getuintFromIP(ip)< =getuintFromIP(ip_end)){SpecialLinks.Add(i.Current);}}return SpecialLinks;}/// <summary>/// 这公有方法提取本网页的纯文本中满足某正则式的文字/// </summary>/// <param name="pattern">正则式</param>/// <returns>返回文字</returns>public string getSpecialWords(string pattern){if (m_outstr == "") getContext(Int16.MaxValue);Regex regex = new Regex(pattern, RegexOptions.Multiline | RegexOptions. IgnoreCase );Match mc=regex.Match(m_outstr);if (mc.Success)return mc.Groups[1].Value;return string.Empty;}#endregion#region 构造函数private void Init(string _url){try{m_uri = new Uri(_url);m_links = new List<Link>();m_html = "";m_outstr = "";m_title = "";m_good = true;if (_url.EndsWith(".rar") || _url.EndsWith(".dat") || _url.EndsWith (".msi")){m_good = false;return;}HttpWebRequest rqst = (HttpWebRequest)WebRequest.Create(m_uri);rqst.AllowAutoRedirect = true;rqst.MaximumAutomaticRedirections = 3;erAgent = "Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0)";rqst.KeepAlive = true;rqst.Timeout = 30000;lock (WebPage.webcookies){if (WebPage.webcookies.ContainsKey(m_uri.Host))rqst.CookieContainer = WebPage.webcookies[m_uri.Host];else{CookieContainer cc = new CookieContainer();WebPage.webcookies[m_uri.Host] = cc;rqst.CookieContainer = cc;}}HttpWebResponse rsps = (HttpWebResponse)rqst.GetResponse();Stream sm = rsps.GetResponseStream();if (!rsps.ContentType.ToLower().StartsWith("text/") || rsps.Content Length > 1 << 22){rsps.Close();m_good = false;return;}Encoding cding = System.Text.Encoding.Default;string contenttype=rsps.ContentType.ToLower();int ix = contenttype.IndexOf("charset=");if (ix != -1){try{cding = System.Text.Encoding.GetEncoding(rsps.ContentType.S ubstring(ix + "charset".Length + 1));}catch{cding = Encoding.Default;}m_html = new StreamReader(sm, cding).ReadToEnd();}else{m_html = new StreamReader(sm, cding).ReadToEnd();Regex regex = new Regex("charset=(?<cding>[^=]+)?\"",RegexOptio ns.IgnoreCase);string strcding = regex.Match(m_html).Groups["cding"].Value; try{cding = Encoding.GetEncoding(strcding);}catch{cding = Encoding.Default;}byte[] bytes=Encoding.Default.GetBytes(m_html.ToCharArray()); m_html = cding.GetString(bytes);if (m_html.Split('?').Length > 100){m_html=Encoding.Default.GetString(bytes);}}m_pagesize = m_html.Length;m_uri = rsps.ResponseUri;rsps.Close();}catch (Exception ex){Console.WriteLine(ex.Message+m_uri.ToString());m_good = false;}}第二部分public WebPage(string _url){string uurl = "";try{uurl = Uri.UnescapeDataString(_url);_url = uurl;}catch { };Regex re = new Regex("(?<h>[^\x00-\xff]+)");Match mc = re.Match(_url);if (mc.Success){string han = mc.Groups["h"].Value;_url = _url.Replace(han, System.Web.HttpUtility.UrlEncode(han, Enco ding.GetEncoding("GB2312")));}Init(_url);}public WebPage(string _url, string _loginurl, string _post){string uurl = "";try{uurl = Uri.UnescapeDataString(_url);_url = uurl;}catch { };Regex re = new Regex("(?<h>[^\x00-\xff]+)");Match mc = re.Match(_url);if (mc.Success){string han = mc.Groups["h"].Value;_url = _url.Replace(han, System.Web.HttpUtility.UrlEncode(han, Enco ding.GetEncoding("GB2312")));}if (_loginurl.Trim() == "" || _post.Trim() == "" || WebPage.webcookies. ContainsKey(new Uri(_url).Host)){Init(_url);}else{#region 登陆string indata = _post;m_post = _post;m_loginurl = _loginurl;byte[] bytes = Encoding.Default.GetBytes(_post);CookieContainer myCookieContainer = new CookieContainer();try{//新建一个CookieContainer来存放Cookie集合HttpWebRequest myHttpWebRequest = (HttpWebRequest)WebRequest.Cr eate(_loginurl);//新建一个HttpWebRequestmyHttpWebRequest.ContentType = "application/x-www-form-urlencod ed";myHttpWebRequest.AllowAutoRedirect = false;erAgent = "Mozilla/4.0 (compatible; MSIE 5.0 1; Windows NT 5.0)";myHttpWebRequest.Timeout = 60000;myHttpWebRequest.KeepAlive = true;myHttpWebRequest.ContentLength = bytes.Length;myHttpWebRequest.Method = "POST";myHttpWebRequest.CookieContainer = myCookieContainer;//设置HttpWebRequest的CookieContainer为刚才建立的那个myCookieContainerStream myRequestStream = myHttpWebRequest.GetRequestStream(); myRequestStream.Write(bytes, 0, bytes.Length);myRequestStream.Close();HttpWebResponse myHttpWebResponse = (HttpWebResponse)myHttpWebR equest.GetResponse();foreach (Cookie ck in myHttpWebResponse.Cookies){myCookieContainer.Add(ck);}myHttpWebResponse.Close();}catch{Init(_url);return;}#endregion#region 登陆后再访问页面try{m_uri = new Uri(_url);m_links = new List<Link>();m_html = "";m_outstr = "";m_title = "";m_good = true;if (_url.EndsWith(".rar") || _url.EndsWith(".dat") || _url.Ends With(".msi")){m_good = false;return;}HttpWebRequest rqst = (HttpWebRequest)WebRequest.Create(m_uri); rqst.AllowAutoRedirect = true;rqst.MaximumAutomaticRedirections = 3;erAgent = "Mozilla/4.0 (compatible; MSIE 5.01; Windows N T 5.0)";rqst.KeepAlive = true;rqst.Timeout = 30000;rqst.CookieContainer = myCookieContainer;lock (WebPage.webcookies){WebPage.webcookies[m_uri.Host] = myCookieContainer;}HttpWebResponse rsps = (HttpWebResponse)rqst.GetResponse();Stream sm = rsps.GetResponseStream();if (!rsps.ContentType.ToLower().StartsWith("text/") || rsps.Con tentLength > 1 << 22){rsps.Close();m_good = false;return;}Encoding cding = System.Text.Encoding.Default;int ix = rsps.ContentType.ToLower().IndexOf("charset=");if (ix != -1){try{cding = System.Text.Encoding.GetEncoding(rsps.ContentType.Substring(ix + "charset".Length + 1));}catch{cding = Encoding.Default;}}m_html = new StreamReader(sm, cding).ReadToEnd();m_pagesize = m_html.Length;m_uri = rsps.ResponseUri;rsps.Close();}catch (Exception ex){Console.WriteLine(ex.Message+m_uri.ToString()); m_good = false;}#endregion}}#endregion#region 属性/// <summary>/// 通过此属性可获得本网页的网址，只读/// </summary>public string URL{get{return m_uri.AbsoluteUri;}}/// <summary>/// 通过此属性可获得本网页的标题，只读public string Title{get{if (m_title == ""){Regex reg = new Regex(@"(?m)<title[^>]*>(?<title>(?:\w|\W)*?)</ title[^>]*>", RegexOptions.Multiline | RegexOptions.IgnoreCase );Match mc = reg.Match(m_html);if (mc.Success)m_title= mc.Groups["title"].Value.Trim();}return m_title;}}/// <summary>/// 此属性获得本网页的所有链接信息，只读/// </summary>public List<Link> Links{get{if (m_links.Count == 0) getLinks();return m_links;}}/// <summary>/// 此属性返回本网页的全部纯文本信息，只读/// </summary>public string Context{get{if (m_outstr == "") getContext(Int16.MaxValue);return m_outstr;}}/// <summary>/// 此属性获得本网页的大小public int PageSize{get{return m_pagesize;}}/// <summary>/// 此属性获得本网页的所有站内链接/// </summary>public List<Link> InsiteLinks{get{return getSpecialLinksByUrl("^http://"+m_uri.Host,Int16.MaxValue); }}/// <summary>/// 此属性表示本网页是否可用/// </summary>public bool IsGood{get{return m_good;}}/// <summary>/// 此属性表示网页的所在的网站/// </summary>public string Host{get{return m_uri.Host;}}/// <summary>/// 此网页的登陆页所需的POST数据/// </summary>public string PostStr{get{return m_post;}}/// <summary>/// 此网页的登陆页/// </summary>public string LoginURL{get{return m_loginurl;}}#endregion}/// <summary>/// 链接类/// </summary>public class Link{public string url; //链接网址public string text; //链接文字public Link(string _url, string _text) {url = _url;text = _text;}}。