zoukankan
html css js c++ java
CollectionHelper网页采集辅助类
using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Text.RegularExpressions; using System.IO; using System.Net; namespace Framework { /// <summary> /// 网页采集辅助类 /// </summary> public static class CollectionHelper { /// <summary> /// 取得字符里的Dom元素 不包含元素属性 /// </summary> /// <param name="orgStr"></param> /// <param name="domElem"></param> /// <returns></returns> public static List<string> GetDomElem(string orgStr, string domElem) { List<string> matchList = new List<string>(); string regStr = string.Format("<{0}[^>]*?>[\\s\\S]+?<\\/{0}>", domElem); try { Regex regex = new Regex(regStr, RegexOptions.Compiled | RegexOptions.IgnoreCase); MatchCollection matches = regex.Matches(orgStr); StringBuilder sb = new StringBuilder(); foreach (Match match in matches) { matchList.Add(match.Value); } } catch (Exception ex) { matchList.Add(ex.Message); } return matchList; } /// <summary> /// 取得字符里的Dom元素 包含元素属性 如:class="aa" /// </summary> /// <param name="orgStr"></param> /// <param name="tagName"></param> /// <param name="tagValue"></param> /// <returns></returns> public static List<string> GetDomElemByAttr(string orgStr, string tagName, string tagValue) { List<string> matchList = new List<string>(); string regStr = string.Format(@"<(?<HtmlTag>[\w]+)[^>]*\s{0}=(?<Quote>[""']?){1}(?(Quote)\k<Quote>)[""']?[^>]*>((?<Nested><\k<HtmlTag>[^>]*>)|</\k<HtmlTag>>(?<-Nested>)|[\s\S]*?)*</\k<HtmlTag>>", tagName.ToLower(), tagValue); try { Regex regex = new Regex(regStr, RegexOptions.Compiled | RegexOptions.IgnoreCase); MatchCollection matches = regex.Matches(orgStr); StringBuilder sb = new StringBuilder(); foreach (Match match in matches) { matchList.Add(match.Value); } } catch (Exception ex) { matchList.Add(ex.Message); } return matchList; } /// <summary> /// 取得字符里的A元素键值对 [name,url] /// </summary> /// <param name="orgStr"></param> /// <param name="domElem"></param> /// <returns></returns> public static Dictionary<string, string> GetDomElem_A(string orgStr) { Dictionary<string, string> matchList = new Dictionary<string, string>(); string regStr1 = "<a[^>]*? href=[\"'](?<url>[^\"']*?)[\"'][^>]*?>(?<text>[\\w\\W]*?)</a>"; try { Regex regex = new Regex(regStr1, RegexOptions.Compiled | RegexOptions.IgnoreCase); MatchCollection matches = regex.Matches(orgStr); StringBuilder sb = new StringBuilder(); foreach (Match match in matches) { string key = match.Value.RemoveHtml(); if (!matchList.ContainsKey(key)) { matchList.Add(key, GetUrlArray(match.Value)[0]); } } } catch (Exception ex) { matchList.Add(ex.Message, ""); } return matchList; } /// <summary> /// 获取网页源码 /// </summary> /// <param name="url">要获取源码的网页地址</param> /// <param name="coding">编码</param> /// <returns>返回获取的网页源代码</returns> public static string GetPageSourceByUrl(string url, string coding = "gb2312") { return GetPageSourceByUrl(new Uri(url), coding); } /// <summary> /// 获取网页源码 /// </summary> /// <param name="url">要获取源码的网页地址</param> /// <param name="coding">编码</param> /// <returns>返回获取的网页源代码</returns> public static string GetPageSourceByUrl(Uri url, string coding = "gb2312") { string getSource = string.Empty; try { HttpWebRequest httpwebrequest = (HttpWebRequest)WebRequest.Create(url); HttpWebResponse httpwebresponse = (HttpWebResponse)httpwebrequest.GetResponse(); Stream stream = httpwebresponse.GetResponseStream(); StreamReader streamreader = new StreamReader(stream, Encoding.GetEncoding(coding)); getSource = streamreader.ReadToEnd(); stream.Close(); httpwebresponse.Close(); } catch (NotSupportedException exception) { getSource = exception.Message; } catch (InvalidOperationException exception) { getSource = exception.Message; } catch (IOException exception) { getSource = exception.Message; } return getSource; } /// <summary> /// 获取页面内容后,用匹配url正则表达式抓取内容中的url /// </summary> /// <param name="code">列表代码</param> /// <returns>返回截取后的URL地址</returns> public static List<string> GetUrlArray(string code) { List<string> urlList = new List<string>(); Regex regex = new Regex(@"(http://)?[\w-\.]*([\/]?[\w-])+[\w-]*\.(htm|html|shtm|shtml|aspx|asp|php|jsp)+[\w-\=\?]*", RegexOptions.Compiled | RegexOptions.IgnoreCase); MatchCollection matches = regex.Matches(code); foreach (Match match in matches) { urlList.Add(match.Value); } return urlList; } /// <summary> /// 获取内容code中所有都图片地址 /// </summary> /// <param name="code">内容</param> /// <returns>返回截取后都图片地址</returns> public static Dictionary<string, string> GetImgUrlArray(string content) { Dictionary<string, string> imgList = new Dictionary<string, string>(); Regex reg = new Regex(@"<img[\s\S]*?src=(""(?<src>[^']*?)""|'(?<src>[^']*?)'|(?<src>[^>\s]*))[^>]*?>(.*?)"); MatchCollection m = reg.Matches(content.ToLower()); foreach (Match match in m) { string matchValue = match.Groups["src"].Value; if (!imgList.ContainsKey(matchValue)) { imgList.Add(matchValue, matchValue); } } return imgList; } /// <summary> /// 将相对地址转换为绝对地址 /// </summary> /// <param name="relativeAddress">要转换的相对地址</param> /// <param name="absoluteAddress">当前网页地址</param> /// <returns>返回转换后的地址</returns> public static string ConvertToAbsluteUrl(string relativeAddress, string absoluteAddress) { if (string.IsNullOrEmpty(relativeAddress)) { return string.Empty; } if (relativeAddress.Contains("://")) { return relativeAddress; } if (string.IsNullOrEmpty(absoluteAddress)) { return string.Empty; } if (!absoluteAddress.Contains("://")) { return string.Empty; } Uri baseUrl = new Uri(absoluteAddress); Uri webrul = new Uri(baseUrl, relativeAddress); return webrul.ToString(); } /// <summary> /// 替换所有HTML标签为空 /// </summary> /// <param name="input">The string whose values should be replaced.</param> /// <returns>A string.</returns> public static string RemoveHtml(this string input) { var stripTags = new Regex("</?[a-z][^<>]*>", RegexOptions.IgnoreCase); return stripTags.Replace(input, string.Empty); } } }
查看全文
相关阅读:
使用Word发布文章到 WordPress 博客
Wordpress上传到阿里云服务器
IntelliJ设置鼠标悬浮提示和修改快捷键
梅塔幻灯片如何设置图片高度不被裁减
更改XAMPP中MySQL数据库的端口号
PHP开启cURL功能
Android Studio使用百度地图示例BaiduMapsApiASDemo
CocosCreator反射在Android中的使用
Android Studio新建一个HelloWorld 程序(App)
无法中止进程无法访问操作拒绝访问
原文地址:https://www.cnblogs.com/zhangqs008/p/2341094.html
最新文章
Oracle中为表设置自动增长的标识列
Oracle 触发器和序列的创建和使用 (自动增长列)
安装包制作
安装包制作2
安装包制作
窗体间传多个值
FTP上传及下载
无底图雷达回波生成
手动搭建简单的三层框架
一般处理程序对数据库的怎删改查操作
热门文章
SqlDataAdapter与SqlCommand之间的区别
asp.net一般处理程序对数据库的操作
ava中jsp的学习中避免乱码的问题
Hashtable的子类properties,外部读取属性值
Collection接口下的Set接口TreeSet类中的自定义比较策略
空指针异常的处理
数组元素的查找,添加,修改,删除
二维数组的练习
eclipse使用步骤,javac和java指令的区别
java中应用循环及结构打印图案
Copyright © 2011-2022 走看看