代码之家 › 专栏 › 技术社区 › ZombieChowder

undefined不是有效的uri或options对象。节点

web-crawler node.js javascript

0

ZombieChowder · 技术社区 · 8 年前

我正在建立一个 web scraper 在NodeJS和我正面临着一个问题。运行代码后,我收到以下错误:

我不知道如何绕过这个错误,我看了以下示例: Example One Example Two

这是我所有的代码:

var request = require('request');
var cheerio = require('cheerio');
var URL = require('url-parse');

var START_URL = "http://example.com";

var pagesVisited = {};
var numPagesVisited = 0;
var pagesToVisit = [];
var url = new URL(START_URL);
var baseUrl = url.protocol + "//" + url.hostname;

pagesToVisit.push(START_URL);
setInterval(crawl,5000);

function crawl() {
  var nextPage = pagesToVisit.pop();
  if (nextPage in pagesVisited) {
    // We've already visited this page, so repeat the crawl
    setInterval(crawl,5000);
  } else {
    // New page we haven't visited
    visitPage(nextPage, crawl);
  }
}

function visitPage(url, callback) {
  // Add page to our set
  pagesVisited[url] = true;
  numPagesVisited++;

  // Make the request
  console.log("Visiting page " + url);
  request(url, function(error, response, body) {
     // Check status code (200 is HTTP OK)
     console.log("Status code: " + response.statusCode);
     if(response.statusCode !== 200) {
       console.log(response.statusCode);
       callback();
       return;
     }else{
       console.log(error);
     }
     // Parse the document body
     var $ = cheerio.load(body);
       collectInternalLinks($);
       // In this short program, our callback is just calling crawl()
       callback();

  });
}

function collectInternalLinks($) {
    var relativeLinks = $("a[href^='/']");
    console.log("Found " + relativeLinks.length + " relative links on page");
    relativeLinks.each(function() {
        pagesToVisit.push(baseUrl + $(this).attr('href'));
    });
}

2 回复 | 直到 8 年前

1

0

Terry Lennox 8 年前

一旦pagesToVisit清空,url将是未定义的,因为在空数组上调用pop将返回此值。

function visitPage(url, callback) {
    if (!url) {
        // We're done
        return;
    }

或者在爬网中,检查pagesToVisit是否包含元素,例如。

function crawl() {
  var nextPage = pagesToVisit.pop();
  if (!nextPage) {
      // We're done!
      console.log('Crawl complete!');
  } else if (nextPage in pagesVisited) {
    // We've already visited this page, so repeat the crawl
    setInterval(crawl,5000);
  } else {
    // New page we haven't visited
    visitPage(nextPage, crawl);
  }
}

2

0

benSooraj Matt Jordan 8 年前

从中得到暗示 Terry Lennox's answer ,我修改了 crawl()

function crawl() {
    var nextPage = pagesToVisit.pop();
    if (nextPage in pagesVisited) {
        // We've already visited this page, so repeat the crawl
        setInterval(crawl, 5000);
    } else if(nextPage) {
        // New page we haven't visited
        visitPage(nextPage, crawl);
    } 
}

我所做的就是在调用之前检查弹出的元素是否存在 visitPage() .

我得到以下输出:

Visiting page http://example.com
Status code: 200
response.statusCode:  200
null
Found 0 relative links on page
^C