zoukankan
html css js c++ java
CollectionHelper网页采集辅助类
using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Text.RegularExpressions; using System.IO; using System.Net; namespace Framework { /// <summary> /// 网页采集辅助类 /// </summary> public static class CollectionHelper { /// <summary> /// 取得字符里的Dom元素 不包含元素属性 /// </summary> /// <param name="orgStr"></param> /// <param name="domElem"></param> /// <returns></returns> public static List<string> GetDomElem(string orgStr, string domElem) { List<string> matchList = new List<string>(); string regStr = string.Format("<{0}[^>]*?>[\\s\\S]+?<\\/{0}>", domElem); try { Regex regex = new Regex(regStr, RegexOptions.Compiled | RegexOptions.IgnoreCase); MatchCollection matches = regex.Matches(orgStr); StringBuilder sb = new StringBuilder(); foreach (Match match in matches) { matchList.Add(match.Value); } } catch (Exception ex) { matchList.Add(ex.Message); } return matchList; } /// <summary> /// 取得字符里的Dom元素 包含元素属性 如:class="aa" /// </summary> /// <param name="orgStr"></param> /// <param name="tagName"></param> /// <param name="tagValue"></param> /// <returns></returns> public static List<string> GetDomElemByAttr(string orgStr, string tagName, string tagValue) { List<string> matchList = new List<string>(); string regStr = string.Format(@"<(?<HtmlTag>[\w]+)[^>]*\s{0}=(?<Quote>[""']?){1}(?(Quote)\k<Quote>)[""']?[^>]*>((?<Nested><\k<HtmlTag>[^>]*>)|</\k<HtmlTag>>(?<-Nested>)|[\s\S]*?)*</\k<HtmlTag>>", tagName.ToLower(), tagValue); try { Regex regex = new Regex(regStr, RegexOptions.Compiled | RegexOptions.IgnoreCase); MatchCollection matches = regex.Matches(orgStr); StringBuilder sb = new StringBuilder(); foreach (Match match in matches) { matchList.Add(match.Value); } } catch (Exception ex) { matchList.Add(ex.Message); } return matchList; } /// <summary> /// 取得字符里的A元素键值对 [name,url] /// </summary> /// <param name="orgStr"></param> /// <param name="domElem"></param> /// <returns></returns> public static Dictionary<string, string> GetDomElem_A(string orgStr) { Dictionary<string, string> matchList = new Dictionary<string, string>(); string regStr1 = "<a[^>]*? href=[\"'](?<url>[^\"']*?)[\"'][^>]*?>(?<text>[\\w\\W]*?)</a>"; try { Regex regex = new Regex(regStr1, RegexOptions.Compiled | RegexOptions.IgnoreCase); MatchCollection matches = regex.Matches(orgStr); StringBuilder sb = new StringBuilder(); foreach (Match match in matches) { string key = match.Value.RemoveHtml(); if (!matchList.ContainsKey(key)) { matchList.Add(key, GetUrlArray(match.Value)[0]); } } } catch (Exception ex) { matchList.Add(ex.Message, ""); } return matchList; } /// <summary> /// 获取网页源码 /// </summary> /// <param name="url">要获取源码的网页地址</param> /// <param name="coding">编码</param> /// <returns>返回获取的网页源代码</returns> public static string GetPageSourceByUrl(string url, string coding = "gb2312") { return GetPageSourceByUrl(new Uri(url), coding); } /// <summary> /// 获取网页源码 /// </summary> /// <param name="url">要获取源码的网页地址</param> /// <param name="coding">编码</param> /// <returns>返回获取的网页源代码</returns> public static string GetPageSourceByUrl(Uri url, string coding = "gb2312") { string getSource = string.Empty; try { HttpWebRequest httpwebrequest = (HttpWebRequest)WebRequest.Create(url); HttpWebResponse httpwebresponse = (HttpWebResponse)httpwebrequest.GetResponse(); Stream stream = httpwebresponse.GetResponseStream(); StreamReader streamreader = new StreamReader(stream, Encoding.GetEncoding(coding)); getSource = streamreader.ReadToEnd(); stream.Close(); httpwebresponse.Close(); } catch (NotSupportedException exception) { getSource = exception.Message; } catch (InvalidOperationException exception) { getSource = exception.Message; } catch (IOException exception) { getSource = exception.Message; } return getSource; } /// <summary> /// 获取页面内容后,用匹配url正则表达式抓取内容中的url /// </summary> /// <param name="code">列表代码</param> /// <returns>返回截取后的URL地址</returns> public static List<string> GetUrlArray(string code) { List<string> urlList = new List<string>(); Regex regex = new Regex(@"(http://)?[\w-\.]*([\/]?[\w-])+[\w-]*\.(htm|html|shtm|shtml|aspx|asp|php|jsp)+[\w-\=\?]*", RegexOptions.Compiled | RegexOptions.IgnoreCase); MatchCollection matches = regex.Matches(code); foreach (Match match in matches) { urlList.Add(match.Value); } return urlList; } /// <summary> /// 获取内容code中所有都图片地址 /// </summary> /// <param name="code">内容</param> /// <returns>返回截取后都图片地址</returns> public static Dictionary<string, string> GetImgUrlArray(string content) { Dictionary<string, string> imgList = new Dictionary<string, string>(); Regex reg = new Regex(@"<img[\s\S]*?src=(""(?<src>[^']*?)""|'(?<src>[^']*?)'|(?<src>[^>\s]*))[^>]*?>(.*?)"); MatchCollection m = reg.Matches(content.ToLower()); foreach (Match match in m) { string matchValue = match.Groups["src"].Value; if (!imgList.ContainsKey(matchValue)) { imgList.Add(matchValue, matchValue); } } return imgList; } /// <summary> /// 将相对地址转换为绝对地址 /// </summary> /// <param name="relativeAddress">要转换的相对地址</param> /// <param name="absoluteAddress">当前网页地址</param> /// <returns>返回转换后的地址</returns> public static string ConvertToAbsluteUrl(string relativeAddress, string absoluteAddress) { if (string.IsNullOrEmpty(relativeAddress)) { return string.Empty; } if (relativeAddress.Contains("://")) { return relativeAddress; } if (string.IsNullOrEmpty(absoluteAddress)) { return string.Empty; } if (!absoluteAddress.Contains("://")) { return string.Empty; } Uri baseUrl = new Uri(absoluteAddress); Uri webrul = new Uri(baseUrl, relativeAddress); return webrul.ToString(); } /// <summary> /// 替换所有HTML标签为空 /// </summary> /// <param name="input">The string whose values should be replaced.</param> /// <returns>A string.</returns> public static string RemoveHtml(this string input) { var stripTags = new Regex("</?[a-z][^<>]*>", RegexOptions.IgnoreCase); return stripTags.Replace(input, string.Empty); } } }
查看全文
相关阅读:
ThreadPoolExecutor使用介绍
apache和tomcat区别(转)
ThreadFactory的理解
Tomcat报45秒无法启动错误修改方法
快速排序
冒泡排序
矩阵快速幂
CF#524-C
CF#524-B
hdu3308—LCIS
原文地址:https://www.cnblogs.com/zhangqs008/p/2341094.html
最新文章
毕业设计-基于图像处理的垃圾分类系统2020.03.11
毕业设计-基于图像处理的垃圾分类系统2020.03.10
html中传值 使用js
JavaScript 中乱码解决
毕业设计-基于图像处理的垃圾分类系统2020.03.04
大数据开发平台架构
架构设计思维
云平台的软件架构
分布式架构知识
架构设计之「 CAP 定理 」
热门文章
互联网架构的三马车
支付平台架构
架构层级的“开闭原则”
微博平台架构
Netty笔记:使用WebSocket协议开发聊天系统
经验分享:如何做好棋牌游戏的产品运营工作
Spring+MyBatis多数据源配置实现
Date,Calendar,SimpleDateFormat时间操作整理
spring和jsp的数据传递。。。
mysql合并相同字段,不同的拼接在起后
Copyright © 2011-2022 走看看