In this tutorial, we will see how you can use ScrapingBee’s API with GoLang, and use it to scrape web pages. As such, we will cover these topics:
- General structure of an API request
- Create your first API request.
Let’s get started!
1. General structure of an API request
The general structure of an API request made in Go will always look like this:
package main
import (
"fmt"
"io/ioutil"
"net/http"
"net/url"
)
func get_request() *http.Response {
// Create client
client := &http.Client{}
my_url := url.QueryEscape("YOUR-URL") // Encoding the URL
// Create request
req, err := http.NewRequest("GET", "https://app.scrapingbee.com/api/v1/?api_key=YOUR-API-KEY&url="+my_url, nil) // Create the request the request
parseFormErr := req.ParseForm()
if parseFormErr != nil {
fmt.Println(parseFormErr)
}
// Fetch Request
resp, err := client.Do(req)
if err != nil {
fmt.Println("Failure : ", err)
}
return resp // Return the response
}
And you can do whatever you want with the response variable! For example:
func main() {
request := get_request()
// Read Response Body
respBody, _ := ioutil.ReadAll(request.Body)
// Display Results
fmt.Println("response Status : ", request.Status)
fmt.Println("response Headers : ", request.Header)
fmt.Println("response Body : ", string(respBody))
}
2. Create your first API request:
Let’s create a tool that saves the HTML code of ScrapingBee’s blog:
func get_request(api_key string, user_url string) *http.Response {
// Create client
client := &http.Client{}
my_url := url.QueryEscape(user_url) // Encoding the URL
// Create request
req, err := http.NewRequest("GET", "https://app.scrapingbee.com/api/v1/?api_key="+api_key+"&url="+my_url, nil) // Create the request the request
parseFormErr := req.ParseForm()
if parseFormErr != nil {
fmt.Println(parseFormErr)
}
// Fetch Request
resp, err := client.Do(req)
if err != nil {
fmt.Println("Failure : ", err)
}
return resp // Return the response
}
func main() {
api_key := "YOUR-API-KEY"
request := get_request(api_key, "https://scrapingbee.com/blog")
// Read Response Body
respBody, _ := ioutil.ReadAll(request.Body)
file, err := os.Create("page.html")
if err != nil {
fmt.Println(err)
return
}
l, err := file.WriteString(string(respBody)) // Write content to the file.
if err != nil {
fmt.Println(err)
file.Close()
return
}
fmt.Println("File has been saved successfully")
err = file.Close()
if err != nil {
fmt.Println(err)
return
}
}
Go back to tutorials