swoole_process 实现了多进程的管理,多个进程同时进行采集任务,
公司的框架比较low,用的tp框架,结合tp框架实现多进程的采集
这是swoole好的学习资源
https://segmentfault.com/a/1190000002946586
1 /** 2 * 用swoole多进程方法进行采集 3 * 采集网贷天眼下各个平台的数据,由于网贷天眼的平台的每个参数指标对应一个ajax请求, 4 * 所有设计每个参数指标对应一张表,这样看起来虽然比较繁琐,但有助于数据的更新和维护. 5 * 先循环遍历出url表中所有的属于网贷天眼的url,和cid,调用各个抓取参数方法,进行 6 * 入库操作 7 */ 8 public function addDataSwoole() 9 { 10 11 for ($i = 0; $i < 14; $i++) { 12 13 $process = new swoole_process([$this, 'swooleGetContents'], true); 14 $start = $process->start(); 15 $process->write($i); 16 $workers[$start] = $process; 17 } 18 19 //防止产生僵尸进程. 20 while (true) { 21 $res = swoole_process::wait(); 22 23 if ($res) { 24 $pid = $res['pid']; 25 write_logs('exit '.$pid); 26 }else{ 27 break; 28 } 29 } 30 31 } 32 33 /** 34 * 这是swoole回调函数用的 35 * @param swoole_process $worker 36 */ 37 public function swooleGetContents(swoole_process $worker) 38 { 39 $i = $worker->read(); 40 41 $urls = $this->allUrls(); 42 43 switch ($i) { 44 case 0: 45 $method = 'add_new_collect_money'; 46 break; 47 case 1: 48 $method = 'add_pure_money'; 49 break; 50 case 2: 51 $method = 'add_rate'; 52 break; 53 case 3: 54 $method = 'add_remainder'; 55 break; 56 case 4: 57 $method = 'add_paid_invest_people'; 58 break; 59 case 5: 60 $method = 'add_plat_paid'; 61 break; 62 case 6: 63 $method = 'add_top_ten_borrower'; 64 break; 65 case 7: 66 $method = 'add_top_ten_invest'; 67 break; 68 case 8: 69 $method = 'add_full_scale_time'; 70 break; 71 case 9: 72 $method = 'add_average_borrowing_time'; 73 break; 74 case 10: 75 $method = 'add_borrowing_to_invest'; 76 break; 77 case 11: 78 $method = 'add_invest_vs'; 79 break; 80 case 12: 81 $method = 'add_invest_total_vs'; 82 break; 83 case 13: 84 $method = 'add_new_borrowing_distribution'; 85 break; 86 } 87 88 foreach ($urls as $v) { 89 $res = $this->$method($v['url'], $v['cid']); 90 } 91 92 var_dump('this process' . $method); 93 94 }