爬虫

************** 反射和泛型写的增删改查******************************

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using ConsoleApplication1.Model;
using System.Data;
using System.Data.SqlClient;
using System.Reflection;
using System.Configuration;
namespace ConsoleApplication1.DAL
{
    public class DbHelper
    {
        /// <summary>
        /// 根据ID查看
        /// </summary>
        /// <typeparam name="T"></typeparam>
        /// <param name="id"></param>
        /// <returns></returns>
        public T GetModel<T>(int id)
        {
            Type type = typeof(T); //获取传进来对像
            string properties = "";
            //获取对象的属性,
            foreach (PropertyInfo item in type.GetProperties())
            {
                properties += "[" + item.Name + "],";
            }
            properties = properties.Remove(properties.Length - 1, 1);  //删除多余的逗号
            string propertyFirst = type.GetProperties()[0].Name;//获取第一个数据
            //链接数据库，得到数据
            string sqlStr = "select top 1 " + properties + "  from  [" + type.Name + "] where [" + propertyFirst + "]=" + id + "";
            SqlDataAdapter db = new SqlDataAdapter(sqlStr, new SqlConnection(ConfigurationManager.AppSettings["conn"]));
            DataSet ds = new DataSet();
            db.Fill(ds);
            T t = (T)Activator.CreateInstance(type);//实例化对象
            foreach (PropertyInfo item in type.GetProperties())
            {
                //得到属性的名字
                item.SetValue(t, ds.Tables[0].Rows[0][item.Name]);
            }
            return t;
        }
        //有条件的查全表
        public DataSet GetList<T>(string strWhere)
        {
            Type type = typeof(T);
            StringBuilder str = new StringBuilder();
            string properties = "";
            foreach (PropertyInfo item in type.GetProperties())
            {
                properties += "[" + item.Name + "],";
            }
            properties = properties.Remove(properties.Length - 1, 1);//删除最后多余的逗号
            str.Append("select " + properties + " ");
            str.Append("  FROM  [" + type.Name + "]  ");
            if (strWhere.Trim() != "")
            {
                str.Append(" where  " + strWhere);
            }
            SqlDataAdapter sda = new SqlDataAdapter(str.ToString(), new SqlConnection(ConfigurationManager.AppSettings["conn"]));
            DataSet ds = new DataSet();
            sda.Fill(ds);
            return ds;
        }
        /// <summary>
        /// 添加数据
        /// </summary>
        /// <typeparam name="T"></typeparam>
        /// <param name="t"></param>
        /// <returns></returns>
        public int Add<T>(T t)
        {
            StringBuilder str = new StringBuilder();
            str.Append("insert into");
            //table() values( ?);select @@IDENTITY");
            Type type = typeof(T);
            str.Append(" [" + type.Name + "] ");
            str.Append("( ");
            for (int i = 1; i < type.GetProperties().Length; i++)
            {
                PropertyInfo item = type.GetProperties()[i];
                str.Append(" [" + item.Name + "],");
            }
            str.Remove(str.Length - 1, 1);//去调逗号
            str.Append(") values(");
            //----------------------
            for (int i = 1; i < type.GetProperties().Length; i++)
            {
                PropertyInfo item = type.GetProperties()[i];
                str.Append(" '" + item.GetValue(t) + "',");
            }
            str.Remove(str.Length - 1, 1);
            str.Append(");select @@IDENTITY");
            SqlConnection conn = new SqlConnection(ConfigurationManager.AppSettings["conn"]);
            SqlCommand cmd = new SqlCommand(str.ToString(), conn);
            conn.Open();
            object obj = cmd.ExecuteScalar();
            if (obj != null)
            {
                return Convert.ToInt32(obj);
            }
            else
            {
                return 0;
            }
        }

        /// <summary>
        /// 增加一条数据
        /// </summary>
        ///
        public int ADD<T>(T t)
        {
            Type type = typeof(T);  //对象
            Type tp = t.GetType();
            string val = "";
            foreach (PropertyInfo item in tp.GetProperties())
            {
                val += "'" + tp.GetProperty(item.Name).GetValue(t, null) + "',";
            }
            val = val.Remove(val.Length - 1, 1);
            val = val.Remove(1, 4);
            string sqlStr = "insert into " + type.Name + " values(" + val + ");select @@IDENTITY";
            SqlConnection conn = new SqlConnection(ConfigurationManager.AppSettings["conn"]);
            SqlCommand cmd = new SqlCommand(sqlStr, conn);
            conn.Open();
            int flag = Convert.ToInt32(cmd.ExecuteScalar());
            conn.Close();
            return flag;
        }
        /// <summary>
        /// 修改数据
        /// </summary>
        /// <typeparam name="T"></typeparam>
        /// <param name="t"></param>
        /// <returns></returns>
        public bool Update<T>(T t, int id)
        {
            StringBuilder strSql = new StringBuilder();
            Type type = typeof(T);
            strSql.Append("update [" + type.Name + "]  set ");
            for (int i = 1; i < type.GetProperties().Length; i++)
            {
                strSql.Append("" + type.GetProperties()[i].Name + "='" + type.GetProperties()[i].GetValue(t) + "',");
            }
            strSql = strSql.Remove(strSql.Length - 1, 1);
            string PropertiesFirst = type.GetProperties()[0].Name;
            int PropertiesFirstValue = (int)type.GetProperties()[0].GetValue(t);
            strSql.Append(" where " + PropertiesFirst + "=" + id + "");
            SqlConnection conn = new SqlConnection(ConfigurationManager.AppSettings["conn"]);
            conn.Open();
            SqlCommand cmd = new SqlCommand(strSql.ToString(), conn);
            int rows = cmd.ExecuteNonQuery();
            conn.Close();
            if (rows > 0)
            {
                return true;
            }
            else
            {
                return false;
            }
        }
        /// <summary>
        ///修改数据
        /// </summary>
        /// <typeparam name="T"></typeparam>
        /// <param name="t"></param>
        /// <returns></returns>
        public bool UpdateData<T>(T t)
        {
            Type type = typeof(T);
            Type t1 = t.GetType();
            //获取表中的字段（属性）
            string shuxing = "";
            foreach (PropertyInfo item in type.GetProperties())
            { shuxing += "" + item.Name + ","; }
            shuxing = shuxing.Remove(shuxing.Length - 1, 1);
            string [] s = shuxing.Split(',');
            //获取t中的值
            string val = "";
            foreach (PropertyInfo i in t1.GetProperties())
            {
                val += "'" + t1.GetProperty(i.Name).GetValue(t, null) + "',";
            }
            val = val.Remove(val.Length - 1, 1);
            string [] v = val.Split(',');
            //拼接字段
            string s1 = "";
            for (int i = 1; i < s.Length; i++) { s1 += "" + s[i] + " =" + v[i] + ","; }
            s1 = s1.Remove(s1.Length - 1, 1);
            StringBuilder str = new StringBuilder();
            str.Append("update  ["+type.Name+"] set ");
            str.Append(""+s1+"");
            str.Append(" where "+s[0]+" = "+v[0]+"");
            SqlConnection conn = new SqlConnection(ConfigurationManager.AppSettings["conn"]);
            SqlCommand cmd = new SqlCommand(str.ToString(), conn);
            conn.Open();
            int flag = Convert.ToInt32(cmd.ExecuteNonQuery());
            conn.Close();
            if (flag > 0) { return true; } else { return false; }
        }
        /// <summary>
        /// 删除
        /// </summary>
        /// <typeparam name="T"></typeparam>
        /// <param name="ID"></param>
        /// <returns></returns
        public int Delete<T>(int id)
        {
            Type type = typeof(T);
            string ID = type.GetProperties()[0].Name;
            string sql = "delete from " + type.Name + " where " + ID + " = " + id + "";
            SqlConnection conn = new SqlConnection(ConfigurationManager.AppSettings["conn"]);
            SqlCommand cmd = new SqlCommand(sql, conn);
            conn.Open();
            int flag = cmd.ExecuteNonQuery();
            conn.Close();
            return flag;
        }
    }
}

******************** 简单爬虫*****

//前台

  <asp:Button ID="Button1" runat="server" Text="抓取" OnClick="Button1_Click" />

//后台

using System;
using System.Collections.Generic;
using System.Linq;
using System.Web;
using System.Web.UI;
using System.Web.UI.WebControls;
using System.Configuration;
using System.Data.SqlClient;
using System.Data;
using System.Reflection;
using ConsoleApplication1.Model;
using ConsoleApplication1.DAL;
using HtmlAgilityPack;                            //////////////引用第三类库，Install-Package HtmlAgilityPack -Version 1.11.2

 

public partial class ReptileText : System.Web.UI.Page
{
    protected void Page_Load(object sender, EventArgs e)
    {

    }
    protected void Button1_Click(object sender, EventArgs e)
    {
        DbHelper db = new DbHelper();    /// 通用增删改查的类名
        HtmlWeb webClient = new HtmlWeb();  /////////model类，表名
        HtmlAgilityPack.HtmlDocument doc = webClient.Load("https://ly.esf.fang.com/house-a010204-b012374/");
        HtmlNodeCollection html = doc.DocumentNode.SelectNodes(".//dl[@class='clearfix']");
        HtmlNodeCollection nn = doc.DocumentNode.SelectNodes(".//span[@class='tit_shop']");
        HtmlNodeCollection price = doc.DocumentNode.SelectNodes(".//span[@class='red']");
        Table_BC home = new Table_BC();
        if (html != null)
        {
            for (int i = 1; i < html.Count;i++ )
            {

                HtmlNode name = doc.DocumentNode.SelectSingleNode(".//dl[@class='clearfix'][" + i + "]/dd[1]/h4[1]/a[1]/span[1]");
                HtmlNode Prace = doc.DocumentNode.SelectSingleNode(".//dl[@class='clearfix'][" + i + "]/dd[2]/span[2]");
                HtmlNode mm = doc.DocumentNode.SelectSingleNode(".//dl[@class='clearfix'][" + i + "]/dd[3]/span[3]");
                if (name != null && Prace != null&&mm!=null)
                {

             ///////////把抓到的数据。存到数据库，
                    home.Name = name.InnerText;
                    home.Title = Prace.InnerText;
                    home.MMNN = mm.InnerText;
                    db.ADD<Table_BC>(home);
                }
                
            }

        }

    }
}

*************************************----需要解压页面，爬虫************

   private static string GetHtmlCode(string url)
        {
            string htmlCode;
            HttpWebRequest webRequest = (System.Net.HttpWebRequest)System.Net.WebRequest.Create(url);
            webRequest.Timeout = 30000;
            webRequest.Method = "GET";
            webRequest.UserAgent = "Mozilla/4.0";
            webRequest.Headers.Add("Accept-Encoding", "gzip, deflate");
            HttpWebResponse webResponse = (System.Net.HttpWebResponse)webRequest.GetResponse();
            if (webResponse.ContentEncoding.ToLower() == "gzip")//如果使用了GZip则先解压  
            {
                using (System.IO.Stream streamReceive = webResponse.GetResponseStream())
                {
                    using (var zipStream =
                        new System.IO.Compression.GZipStream(streamReceive, System.IO.Compression.CompressionMode.Decompress))
                    {
                        Encoding enc = System.Text.Encoding.GetEncoding("gb2312");
                        using (StreamReader sr = new System.IO.StreamReader(zipStream, enc))
                        {
                            htmlCode = sr.ReadToEnd();
                        }
                    }
                }
            }
            else
            {
                using (System.IO.Stream streamReceive = webResponse.GetResponseStream())
                {
                    Encoding enc = System.Text.Encoding.GetEncoding("gb2312");
                    using (System.IO.StreamReader sr = new System.IO.StreamReader(streamReceive, enc))
                    {
                        htmlCode = sr.ReadToEnd();
                    }
                }
            }

            return htmlCode;
        }
  //比特流 数据单位=》字符串  内存流Stream  =>图片  MP3  MP4 BASE64
            //byte[] btWeb = wc.DownloadData("https://ly.newhouse.fang.com/house/s/jianxiqu/?ctm=1.ly.xf_search.lpsearch_area.3");
            //string strWeb = System.Text.Encoding.GetEncoding("gb2312").GetString(btWeb);//gbk gb2312 utf-8 ASCII
            string html = GetHtmlCode("https://ly.esf.fang.com/house-a010204-b012374/");
            HtmlDocument doc = new HtmlDocument();
            doc.LoadHtml(html);
         
            HtmlNodeCollection dllist = doc.DocumentNode.SelectNodes("//dl[@class='clearfix']");
            foreach (HtmlNode item in dllist)
            {
                
            }
            int aa = dllist.Count;
            //HtmlNodeCollection 节点列表    XMLNodeList
            //HtmlAttributeCollection 属性列表 
            //HtmlNode节点 XMLNode