How to log in to a website using ScrapingBee with NodeJS

In this tutorial, we will see how you can log in to any website, using three different methods:

• A JavaScript scenario.
• A POST request.
• Cookies

As an example, we’re going to log into this  demo website  and take a screenshot of the account page. So make sure to create an account there before you start!

1. Login using a js_scenario:

This is the easiest solution among the three, as it mimics the behavior of a normal user. We first visit the login page, input our login credentials, and click on the login button.

The code below will do that for our example website, and will take a screenshot of the account page:

const scrapingbee = require('scrapingbee'); // Import ScrapingBee's SDK
const fs = require('fs');

async function login_and_take_screenshot(url, path) {
  var client = new scrapingbee.ScrapingBeeClient('YOUR-API-KEY'); // New ScrapingBee client
  var response = await client.get({
    url: url,
    params: {
        'js_scenario': {"instructions":[
            {"fill": ["#email", "your-email@address.com"]}, // Enter registration email
            {"fill": ["#passwd", "your-password"]}, // Enter password
            {"click": "#SubmitLogin"}, // Click on login
            {"wait": 1000} // Wait for a second
        ]},
        'screenshot': 'true'
    }
}).then((response)=>fs.writeFileSync(path, response.data)) // Save the contents of the request (screenshot) to the 'path' file destination
.catch((e)=>console.log("An error has occured: " + e.message))
}

login_and_take_screenshot("http://automationpractice.com/index.php?controller=authentication", "./my-account.png");

And the result is a successful login as you can see:

2. Login using a POST requests:

With this method, we will send a direct POST request to the form action URL.

First we need to see what fields are being sent when we login. We can do that easily via Chrome developers tools: 

As you can see, the fields we need to fill are: email and passwd.

The code below will make a POST request to our website, sending our data with it.

const fs = require('fs');

(function(callback) {
    'use strict';
    var url = encodeURIComponent("http://automationpractice.com/index.php?controller=authentication");
    var api_key = "YOUR-API-KEY";
    const httpTransport = require('https');
    const responseEncoding = 'utf8';
    const httpOptions = {
        hostname: 'app.scrapingbee.com',
        port: '443',
        path: '/api/v1?url='+ url +'&api_key='+api_key,
        method: 'POST',
        headers: {"Content-Type":"application/x-www-form-urlencoded; charset=utf-8", "Referer": "http://automationpractice.com/index.php?controller=authentication"}
    };

    const request = httpTransport.request(httpOptions, (res) => {
        let responseBufs = [];
        let responseStr = '';

        res.on('data', (chunk) => {
            if (Buffer.isBuffer(chunk)) {
                responseBufs.push(chunk);
            }
            else {
                responseStr = responseStr + chunk;
            }
        }).on('end', () => {
            responseStr = responseBufs.length > 0 ?
                Buffer.concat(responseBufs).toString(responseEncoding) : responseStr;

            callback(null, res.statusCode, res.headers, responseStr);
        });

    })
    .setTimeout(0)
    .on('error', (error) => {
        callback(error);
    });
    var data = JSON.stringify({
        'email' : 'your-email@address.com',
        'passwd': 'your-password'
    });
    request.write(data)
    request.end();

})((error, statusCode, headers, body) => {
    console.log('ERROR:', error);
    console.log('STATUS:', statusCode);
    console.log('HEADERS:', JSON.stringify(headers));
    fs.writeFileSync("./my-account.html", body);
});

3. Using cookies to login:

Log into our demo website, and open Chrome developers tools, go to Application → Cookies, and you will notice that a cookie with the name PrestaShop-a30a9934ef476d11b6cc3c983616e364 with a large hashed value is saved and being used on that domain. That cookie stores all of the information about the user, logging information, user status, etc…

Now that we have that cookie, we can copy its content and send it directly with a request to  My Account  area without having to send any credentials.

The final code will look like this:

const scrapingbee = require('scrapingbee');
const fs = require("fs");

async function get(url) {
  var client = new scrapingbee.ScrapingBeeClient('YOUR-API-KEY');
  var response = await client.get({
    url: url,
    params: {
        "screenshot_full_page": "true"
    },
    cookies: {"PrestaShop-a30a9934ef476d11b6cc3c983616e364":"Cookie-Text-Here"},
  })
  return response
}

get('http://automationpractice.com/index.php?controller=authentication')
.then((response)=>fs.writeFileSync("./screenshot.png", response.data))
.catch((e) => console.log('A problem occurs : ' + e.response.data));

And as you can see below, we managed to access that page without having to manually log in!