Crawling.php

Crawling.php

  array('jQuery选择器','要采集的属性'[,"标签过滤列表"][,"回调函数"]),
*   '规则名2' => array('jQuery选择器','要采集的属性'[,"标签过滤列表"][,"回调函数"]),
*    ..........
*    [,"callback"=>"全局回调函数"]
* );
 */
require 'vendor/autoload.php';


use QL\QueryList;
//长江航运指数分析采类


/*
  基类
  获取页面的地址库
  $url 地址链接
  $str 剔除某些不好的标题  
*/
function getListUrl($url,$str){
    $data=array();
    $pages = QueryList::Query($url,array(
        'page' => array('.pages','html','',function($content){preg_match('{.*\((\d+).*}',$content,$match);return $match[1];}) 
        ),'.dfxw_main')->data;
    for($i=1;$i<$pages[0]['page'];$i++){
        $cururl=$url.'index_'.$i.".html";
        $da[] = QueryList::Query($cururl,array(
        'page' => array('a','html'),
        'list-src' => array('a','href')   
        ),'.dfxw_main_bottom ul li')->data;
        $cururl="";
    }
    for($i=0;$i
 
  ';
print_r($data);
echo '
 

'; /* 基类 获取页面的详情信息 $url 地址链接 string $str 选取关键元素字段 string */ function getDetailData($url,$str){ $num=''; $da = QueryList::Query($url,array( 'key' => array('','text') ),'.MsoNormalTable tr:eq(0) td')->data; foreach ($da as $k => $v) { foreach ($v as $a => $b) { if($b==$str){ $num = $k; break; } } } unset($da); $da = QueryList::Query($url,array( 'key' => array('td:eq(0)','text'), 'val' => array('td:eq('.$num.')','text') ),'.MsoNormalTable tr:gt(0)')->data; return $da; } /*$url="http://zizhan.mot.gov.cn/zizhan/zhishuJG/changhangju/tongjishuju/changjiangyunjiaZS/201311/t20131108_1509615.html"; $str="本月"; $da=getDetailData($url,$str); echo '

';
print_r($da);
echo '

';*/ /* 基类 获取年月 $url 标题组的数据 array */ function getYearMonth($data){ for($i=0;$i '; print_r($da); echo '';*/ /*************************************************/ //使用插件 /*$urls = QueryList::run('Request',array( 'target' => 'http://cms.querylist.cc/news/list_2.html', 'referrer'=>'http://cms.querylist.cc', 'method' => 'GET', 'params' => array('var1' => 'testvalue', 'var2' => 'somevalue'), 'user_agent'=>'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:21.0) Gecko/20100101 Firefox/21.0', 'cookiePath' => './cookie.txt', 'timeout' =>'30' ))->setQuery(array('link' => array('h2>a','html','',function($content){ //利用回调函数补全相对链接 $baseUrl = 'http://cms.querylist.cc'; return $content; })),'.cate_list li')->getData(function($item){ return $item['link']; }); print_r($urls); */