Make concurrent requests in NodeJS

Our API is designed to allow you to have multiple concurrent scraping operations. That means you can speed up scraping for hundreds, thousands or even millions of pages per day, depending on your plan.

The more concurrent requests limit you have the more calls you can have active in parallel, and the faster you can scrape.

Making concurrent requests in NodeJS is very straightforward using Cluster module. The code below will make two concurrent requests to ScrapingBee’s pages, and save the content in an HTML file.

const cluster = require('cluster')
const scrapingbee = require('scrapingbee')
const fs = require("fs")
const crypto = require("crypto")

async function get(url) {
  var client = new scrapingbee.ScrapingBeeClient('YOUR-API-KEY');
  console.log("Running...")
  var response = await client.get({url: url}).then(function (response){
    var path = 'spb'+crypto.randomBytes(4).readUInt32LE(0)+'.html';
    fs.writeFileSync(path, response.data)
    console.log("Done!")
    })
    .catch((e) => console.log('A problem occurs : ' + e.response.data))
  return response
}

// For Master process
if (cluster.isMaster) {
  console.log(`Master ${process.pid} is running`);
  var urls = ["https://scrapingbee.com/blog", "https://scrapingbee.com/documentation"]
  // Fork workers.
  for (let i = 0; i < 2; i++) {
    cluster.fork({WorkerName: urls[i]});
  }

}

// For Worker
else{
    get(process.env.WorkerName)
}