首页 / PHP / URL抓取工具

URL抓取工具

内容导读

互联网集市收集整理的这篇技术教程文章主要介绍了URL抓取工具，小编现在分享给大家，供广大互联网技能从业者学习和参考。文章包含8027字，纯文字阅读大概需要12分钟。

内容图文

有需要csdn免积分下载、pudn免积分下载、51cto免积分，请到http://www.itziy.com/
命令行下执行，直接php调用将显示使用方式
功能说明
1.支持代理
2.支持设置递归检查次数
3.支持输出类型控制、检查内容控制

作用：
主要代替肉眼尽量多的抓取可能的请求包及url地址等，方便渗透测试
  1. error_reporting(E_ERROR | E_WARNING | E_PARSE);
  2. ini_set('memory_limit','1024M');
  3. set_time_limit(0);
  4. define('CHECK_A_TAG', false);
  5. define('CHECK_JS_TAG', true);
  6. define('CHECK_URL', true);
  7. define('SAVE_ERROR', true);
  9. $checkArr = array(
  10. '$.load',
  11. '.ajax',
  12. '$.post',
  13. '$.get',
  14. '.getJSON'
  15. );
  17. if ($argc < 2)
  18. die(showerror('sorry, parameter error', array('example: php debug.php url num filename header proxy', 'detail information:', 'url: target url address which you want to check it', 'num: The number of pages of recursive,default 3', 'filename: output filename default name ret.txt', 'header: The request header file default null', 'proxy: if you want to use proxy set it here default no use proxy')));
  20. if (!check_extension())
  21. die(showerror('extension curl not support', 'please open php curl extension support'));
  23. //global variable
  24. $url = trim($argv[1]);
  25. if (stripos($url, 'http') === false)
  26. $url = 'http://'.$url;
  27. $num = isset($argv[2]) ? intval($argv[2]) : 3;
  28. $output = isset($argv[3]) ? trim(str_replace("\\", '/', $argv[3])) : str_replace("\\", '/', dirname(__FILE__)).'/ret.txt';
  29. $header = null;
  30. $proxy = null;
  31. $host = null;
  33. if (isset($argv[4]))
  34. {
  35. $header = trim(str_replace("\\", '/', $argv[4]));
  36. if (file_exists($header))
  37. $header = array_filter(explode("\n", str_replace("\r", '', file_get_contents($header))));
  38. else
  39. {
  40. $file = str_replace("\\", '/', dirname(__FILE__)).'/'.$header;
  41. if (file_exists($file))
  42. $header = array_filter(explode("\n", str_replace("\r", '', file_get_contents($file))));
  43. else
  44. $header = null;
  45. }
  46. }
  48. if (isset($argv[5]))
  49. $proxy = trim($argv[5]);
  51. if (!is_array($header) || empty($header))
  52. $header = null;
  53. $result = check_valid_url($url);
  54. $outputArr = array();
  56. if (!empty($result))
  57. {
  58. $result = str_replace("\r", '', $result);
  59. $result = str_replace("\n", '', $result);
  60. $tmpArr = parse_url($url);
  61. if (!isset($tmpArr['host']))
  62. die(showerror('parse url error', 'can not get host form url: '.$url));
  63. $host = $tmpArr['host'];
  64. if (stripos($host, 'http') === false)
  65. $host = 'http://'.$host;
  66. unset($tmpArr);
  68. //check for current page
  69. if (!isset($outputArr[md5($url)]))
  70. {
  71. $outputArr[md5($url)] = $url;
  72. file_put_contents($output, $url."\n", FILE_APPEND);
  73. echo 'url: ',$url,' find ajax require so save it',PHP_EOL;
  74. }
  75. work($result);
  76. }
  77. echo 'run finish',PHP_EOL;
  79. function work($result, $reverse = false)
  80. {
  81. global $num, $host, $outputArr, $checkArr, $output;
  83. if (!$result)
  84. return;
  85. $result = str_replace("\r", '', $result);
  86. $result = str_replace("\n", '', $result);
  88. while ($num > 0)
  89. {
  90. echo 'remain: ',$num,' now start to check for url address',PHP_EOL,PHP_EOL;
  91. preg_match_all('//i', $result, $match);
  92. if (CHECK_A_TAG && isset($match[2]) && !empty($match[2]))
  93. {
  94. foreach ($match[2] as $mc)
  95. {
  96. $mc = trim($mc);
  97. if ($mc == '#')
  98. continue;
  99. if (stripos($mc, 'http') === false)
  100. $mc = $host.$mc;
  102. if (($ret = check_valid_url($mc)))
  103. {
  104. if (!isset($outputArr[md5($mc)]))
  105. {
  106. $outputArr[md5($mc)] = $mc;
  107. file_put_contents($output, $mc."\n", FILE_APPEND);
  108. echo 'url: ',$mc,' find ajax require so save it',PHP_EOL;
  109. }
  110. }
  111. }
  112. }
  114. //check for page url
  115. echo 'remain: ',$num,' now start to check for page url',PHP_EOL,PHP_EOL;
  116. preg_match_all('/(https?|ftp|mms):\/\/([A-z0-9]+[_\-]?[A-z0-9]+\.)*[A-z0-9]+\-?[A-z0-9]+\.[A-z]{2,}(\/.*)*\/?/i', $result, $match);
  117. if (CHECK_URL && isset($match[2]) && !empty($match[2]))
  118. {
  119. foreach ($match[2] as $mc)
  120. {
  121. $mc = trim($mc);
  122. if ($mc == '#')
  123. continue;
  124. if (stripos($mc, 'http') === false)
  125. $mc = $host.$mc;
  127. if (($ret = check_valid_url($mc)))
  128. {
  129. if (!isset($outputArr[md5($mc)]))
  130. {
  131. $outputArr[md5($mc)] = $mc;
  132. file_put_contents($output, $mc."\n", FILE_APPEND);
  133. echo 'url: ',$mc,' find ajax require so save it',PHP_EOL;
  134. }
  135. }
  136. }
  137. }
  139. //check for javascript ajax require
  140. echo 'remain: ',$num,' now start to check for javascript ajax require',PHP_EOL,PHP_EOL;
  141. preg_match_all('//i', $result, $match);
  142. if (CHECK_JS_TAG && isset($match[2]) && !empty($match[2]))
  143. {
  144. foreach ($match[2] as $mc)
  145. {
  146. $mc = trim($mc);
  147. if ($mc == '#')
  148. continue;
  149. if (stripos($mc, 'http') === false)
  150. $mc = $host.$mc;
  152. if (($ret = check_valid_url($mc)))
  153. {
  154. //check for current page
  155. foreach ($checkArr as $ck)
  156. {
  157. if (!isset($outputArr[md5($mc)]) && strpos($ret, $ck) !== false)
  158. {
  159. $outputArr[md5($mc)] = $mc;
  160. file_put_contents($output, $mc."\n", FILE_APPEND);
  161. echo 'url: ',$mc,' find ajax require so save it',PHP_EOL;
  162. break;
  163. }
  164. }
  165. }
  166. }
  167. }
  169. if ($reverse)
  170. return;
  172. //check for next page
  173. preg_match_all('//i', $result, $match);
  174. if (isset($match[2]) && !empty($match[2]))
  175. {
  176. echo 'check for next page, remain page counts: ',$num,PHP_EOL;
  177. foreach ($match[2] as $mc)
  178. {
  179. $mc = trim($mc);
  180. if ($mc == '#')
  181. continue;
  182. if (stripos($mc, 'http') === false)
  183. $mc = $host.$mc;
  184. echo 'check for next page: ',$mc,PHP_EOL;
  185. work(check_valid_url($mc), true);
  186. }
  187. }
  188. $num--;
  189. sleep(3);
  190. }
  191. }
  193. function check_valid_url($url)
  194. {
  195. if (stripos($url, 'http') === false)
  196. $url = 'http://'.$url;
  197. $ch = curl_init();
  198. curl_setopt($ch, CURLOPT_URL, $url);
  199. curl_setopt($ch, CURLOPT_HEADER, true);
  200. curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
  201. curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
  202. curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)');
  203. if (!is_null($header))
  204. curl_setopt($ch, CURLOPT_HTTPHEADER, $header);
  205. if (!is_null($proxy))
  206. curl_setopt($ch, CURLOPT_PROXY, $proxy);
  207. $ret = curl_exec($ch);
  208. $errinfo = curl_error($ch);
  209. curl_close($ch);
  210. unset($ch);
  211. if (!empty($errinfo) || ((strpos($ret, '200 OK') === false) && (strpos($ret, '302 Moved') === false)) || strpos($ret, '114so.cn') !== false)
  212. {
  213. showerror('check url: '.$url. ' find some errors', array($errinfo, $ret));
  214. if (SAVE_ERROR)
  215. file_put_contents(dirname(__FILE__).'/error.txt', $url."\n", FILE_APPEND);
  216. return false;
  217. }
  218. return $ret;
  219. }
  221. function check_extension()
  222. {
  223. if (!function_exists('curl_init') || !extension_loaded('curl'))
  224. return false;
  225. return true;
  226. }
  228. function showerror($t, $c)
  229. {
  230. $str = "#########################################################################\n";
  231. $str .= "# ".$t."\n";
  232. if (is_string($c))
  233. $str .= "# ".$c;
  234. elseif (is_array($c) && !empty($c))
  235. {
  236. foreach ($c as $c1)
  237. $str .= "# ".$c1."\n";
  238. }
  239. $str .= "\n#########################################################################\n";
  240. echo $str;
  241. unset($str);
  242. }

内容总结

以上是互联网集市为您收集整理的URL抓取工具全部内容，希望文章能够帮你解决URL抓取工具所遇到的程序开发问题。 如果觉得互联网集市技术教程内容还不错，欢迎将互联网集市网站推荐给程序员好友。

内容备注

版权声明：本文内容由互联网用户自发贡献，该文观点与技术仅代表作者本人。本站仅提供信息存储空间服务，不拥有所有权，不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容， 请发送邮件至 gblab@vip.qq.com 举报，一经查实，本站将立刻删除。

内容手机端

扫描二维码推送至手机访问。

本文链接：https://qyyshop.com/info/164444.html

来源：【匿名】