Здесь мой искатель основан на приведенных выше ответах.
Curl
, поэтому мы можем использовать HTTP-аутентификацию КЛАСС КЛАССА CRAWL
class crawler
{
protected $_url;
protected $_depth;
protected $_host;
public function __construct($url, $depth = 5)
{
$this->_url = $url;
$this->_depth = $depth;
$parse = parse_url($url);
$this->_host = $parse['host'];
}
public function run()
{
$this->crawl_page($this->_url, $this->_depth = 5);
}
public function crawl_page($url, $depth = 5)
{
static $seen = array();
if (isset($seen[$url]) || $depth === 0) {
return;
}
$seen[$url] = true;
list($content, $httpcode) = $this->getContent($url);
$dom = new DOMDocument('1.0');
@$dom->loadHTML($content);
$this->processAnchors($dom, $url, $depth);
ob_end_flush();
echo "CODE::$httpcode, URL::$url <br>";
ob_start();
flush();
// echo "URL:", $url, PHP_EOL, "CONTENT:", PHP_EOL, $dom->saveHTML(), PHP_EOL, PHP_EOL;
}
public function processAnchors($dom, $url, $depth)
{
$anchors = $dom->getElementsByTagName('a');
foreach ($anchors as $element) {
$href = $element->getAttribute('href');
if (0 !== strpos($href, 'http')) {
$path = '/' . ltrim($href, '/');
if (extension_loaded('http')) {
$href = http_build_url($url, array('path' => $path));
} else {
$parts = parse_url($url);
$href = $parts['scheme'] . '://';
if (isset($parts['user']) && isset($parts['pass'])) {
$href .= $parts['user'] . ':' . $parts['pass'] . '@';
}
$href .= $parts['host'];
if (isset($parts['port'])) {
$href .= ':' . $parts['port'];
}
$href .= $path;
}
}
// Crawl only link that belongs to the start domain
if (strpos($href, $this->_host) !== false)
$this->crawl_page($href, $depth - 1);
}
}
public function getContent($url)
{
$handle = curl_init($url);
curl_setopt($handle, CURLOPT_RETURNTRANSFER, TRUE);
/* Get the HTML or whatever is linked in $url. */
$response = curl_exec($handle);
/* Check for 404 (file not found). */
$httpCode = curl_getinfo($handle, CURLINFO_HTTP_CODE);
if ($httpCode == 404) {
/* Handle 404 here. */
}
curl_close($handle);
return array($response, $httpCode);
}
}
// USAGE
$startURL = 'http://YOUR_START_ULR';
$depth = 2;
$crawler = new crawler($startURL, $depth);
$crawler->run();