通过curl模拟多线程抓取

<?php
//此模型虽然是一次多个url请求，但缺陷是 要等所有数据请求结束一起返回，才能逐个处理数据。
$start = microtime(true);

header('Content-type:text/html;charset=utf-8');

$arrs = [
    'https://www.yahoo.com/',
    'https://www.sohu.com/',
    'http://www.qq.com/',
    'http://www.sina.com.cn/',
    'http://www.163.com/'
];

$headers = array(
    'User-Agent:Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.107 Safari/537.36',
);

$mh = curl_multi_init();

foreach ($arrs as $i=>$url){
    $conn[$i] = curl_init($url);
    curl_setopt($conn[$i],CURLOPT_RETURNTRANSFER,1);
    curl_setopt($conn[$i], CURLOPT_HTTPHEADER,$headers);
    curl_setopt($conn[$i], CURLOPT_HEADER, 0);
    curl_setopt($conn[$i], CURLOPT_TIMEOUT, 20);


    if (strpos($url,'https')){
        curl_setopt ( $conn[$i], CURLOPT_SSL_VERIFYPEER, false );
        curl_setopt ( $conn[$i], CURLOPT_SSL_VERIFYHOST, 2 );
    }
    curl_multi_add_handle($mh,$conn[$i]);
}

$active = null;
/*
 * 这样写会轻易导致CPU占用100%

do {
    $n=curl_multi_exec($mh,$active);
} while ($active);
 *
 */

//改写
/*
do {
    $mrc = curl_multi_exec($mh,$active);
}while($mrc == CURLM_CALL_MULTI_PERFORM);

while ($active and $mrc == CURLM_OK){
    if (curl_multi_select($mh) != -1) {
        do {
            $mrc = curl_multi_exec($mh, $active);
        } while ($mrc == CURLM_CALL_MULTI_PERFORM);
    }
}
*/

//最简单方案
do {
    curl_multi_exec($mh, $running);
    curl_multi_select($mh);
} while ($running > 0);


//获取内容
foreach ($arrs as $i => $url) {
    $res[$i]=curl_multi_getcontent($conn[$i]);
    var_dump($res[$i]);
    curl_close($conn[$i]);
    //等待所有http请求结束返回数据依次生成文件。
    file_put_contents('curl_multi.log', $res[$i]."\r\n\r\n\r\n\r\n", FILE_APPEND);
}

$end = microtime(true) - $start;

echo '<br/>';
echo $end; // 平均 10.091157913208s