爬虫之初试

　　MD，这个星期简直了~~有时候真是一步错步步错啊，各种套路，好了，废话不多说，因为急求数据，于是按照同事的方法，自己写了一份爬虫，算是初试吧。

<?php
header("Content-type: text/html; charset=utf-8");
include_once "Connection.php";
include_once "autoload.php";

$db = NewADOConnection('pgsql');
$db->Connect(~~~~~);
$db->Execute("set names utf8;");
$city= [
    ['beijing','北京'],
    ['tianjin','天津'],
    ['shanghai','上海'],
    ['chongqing','重庆'],
    ['guangzhou','广州'],
    ['shenzhen','深圳'],
    ['shijiazhuang','石家庄'],
    ['taiyuan','太原'],
    ['huhehaote','呼和浩特'],
    ['shenyang','沈阳'],
    ['changchun','长春'],
    ['haerbin','哈尔滨'],
    ['nanjing','南京'],
    ['hangzhou','杭州'],
    ['hefei','合肥'],
    ['fujianfuzhou','福州'],
    ['nanchang','南昌'],
    ['jinan','济南'],
    ['zhengzhou','郑州'],
    ['wuhan','武汉'],
    ['changsha','长沙'],
    ['guangzhou','广州'],
    ['nanning','南宁'],
    ['haikou','海口'],
    ['chengdu','成都'],
    ['guiyang','贵阳'],
    ['kunming','昆明'],
    ['lasa','拉萨'],
    ['xian','西安'],
    ['lanzhou','兰州'],
    ['xining','西宁'],
    ['yinchuan','银川'],
    ['wulumuqi','乌鲁木齐']
];


for($i = 1; $i<count($city);$i++){
    $cityData = (getCityData($city[$i][0]));
    var_dump($cityData);
    $res = toDataBase($cityData,$city[$i][1]);
    if($res === false){
        var_dump($city[$i]);
        break;
    }
    var_dump($res);
}



//获取城市12月数据
function getCityData($city="beijing"){
    $beginYear = "2014";
    $month = ["01","02","03","04","05","06","07","08","09","10","11","12"];

    $result = [];
    for($i = 0; $i<12;$i++){
        $time = $beginYear.$month[$i];
        $url = "http://www.tianqihoubao.com/lishi/".$city."/month/".$time.".html";   //地址拼接
        $output = httpcurl($url);
        $output = iconv("GBK", "UTF-8", $output);   //转码
        $output = pregData($output);
        $result[]  = $output;
    }
    return $result;
}

function toDataBase($data, $city){
    if(!$data){
        return false;
    }
    $sql = "insert into air_report(date,state,temperature,wind,city) values ";
    $monthLength = count($data);
    $outerValues = "";
    for($i=0; $i<$monthLength; $i++ ){
        $dayLength = count($data[$i]);
        $values = "";
        for($j=0; $j<$dayLength; $j++){
            $values .= "(";
            $values .= implode($data[$i][$j],',').",'".$city."'";
            $values.="),";
//            var_dump($values);
        }
        $outerValues = $outerValues.$values;
    }
    $outerValues = substr($outerValues,0,strlen($outerValues)-1);
    $sql = $sql.$outerValues;
    global $db;
    $res = $db->Execute($sql);
    if($res == false){
        return false;
    }
//    var_dump($db->errorMsg());
    return $res;
}


//解析数据
function pregData($str){
    $rule = "/<table width="100%".*?>.*?</table>/ism";
    $ruleTable = "/<td.*?>.*?</td>/ism";
    $ruleTr = "/<tr.*?>.*?</tr>/ism";
    $output = "";
    preg_match_all($rule, $str, $output);
    preg_match_all($ruleTable, $output[0][0], $output);
    $output = $output[0];
    $output = array_slice($output, 4);
    $trLength = count($output);
    $info = [];
    for($i = 0; $i<$trLength; $i++){
        $output[$i] = "'".trim(strip_tags($output[$i]))."'";　　　　//直接strip_tags()取数据
    }
    $info = array_chunk($output, 4);
    return $info;
}


function httpcurl($url, $post_data = null)
{
    $ch = curl_init();
    curl_setopt($ch, CURLOPT_URL, $url);//x
    curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
//    curl_setopt($ch, CURLOPT_POST, 1);
//    curl_setopt($ch, CURLOPT_BINARYTRANSFER, true);
//    curl_setopt($ch, CURLOPT_POSTFIELDS, $post_data);
    $output = curl_exec($ch);
    curl_close($ch);
    return $output;
}

　　数据有时取出来后格式还是有问题的，刚好碰上的一个取出来的值里面包含不可见的字符，想要去除暂时还没有想到合适的方法。