Web Scraping with R Tutorial: Scraping BrickEconomy.com

# On Linux/Mac terminal - install dependencies first
sudo apt update
sudo apt install libcurl4-openssl-dev libxml2-dev libfontconfig1-dev libfreetype6-dev
sudo apt install build-essential libssl-dev libharfbuzz-dev libfribidi-dev
sudo apt install libfreetype6-dev libpng-dev libtiff5-dev libjpeg-dev 

# Then install the R packages
Rscript -e 'install.packages(c("tidyverse", "httr2", "jsonlite", "xml2", "rvest", "ragg"), repos="https://cloud.r-project.org", dependencies=TRUE)'

# Create a new R script
touch scraper.R

# Load the essential scraping packages
library(rvest)    # The star of our show - for HTML parsing
library(dplyr)    # For data wrangling
library(httr2)    # For advanced HTTP requests
library(stringr)  # For string manipulation

# Test that everything is working
cat("R scraping environment ready to go!\n")

# Load rvest package
library(rvest)

# Read HTML from a URL
url <- "https://example.com"
page <- read_html(url)
print(page)

# You can also read from a local HTML file
# local_html <- read_html("path/to/local/file.html")

# Extract all paragraph elements
paragraphs <- page %>% html_elements("p")

# Extract elements with a specific class
prices <- page %>% html_elements(".price")

# Extract an element with a specific ID
header <- page %>% html_elements("#main-header")

# Get text from elements
paragraph_text <- paragraphs %>% html_text2()

# Get attributes from elements (like href from links)
links <- page %>% 
  html_elements("a") %>% 
  html_attr("href")

# Get the src attribute from images
images <- page %>% 
  html_elements("img") %>% 
  html_attr("src")

# Extract a table into a data frame
tables <- page %>% html_table()
first_table <- tables[[1]]  # Get the first table

# If you have multiple tables, you can access them by index
second_table <- tables[[2]]

# You can also be more specific with your selector
specific_table <- page %>% 
  html_element("table.data-table") %>% 
  html_table()

# Find links and navigate to them
next_page_url <- page %>% 
  html_element(".pagination .next") %>% 
  html_attr("href")

# If the URL is relative, make it absolute
if(!grepl("^http", next_page_url)) {
  next_page_url <- paste0("https://example.com", next_page_url)
}

# Now fetch the next page
next_page <- read_html(next_page_url)
}

library(rvest)

# Let's scrape data about a cinematic masterpiece
movie_url <- "https://www.imdb.com/title/tt8031422/"
movie_page <- read_html(movie_url) # Read the main movie page

# First, let's grab the movie title
movie_title <- movie_page %>%
  html_element("h1") %>%
  html_text2()

cat("Movie:", movie_title, "\n")

# --- Extract Top Cast (from Main Page using Divs) ---
# Target the individual cast item containers using the data-testid
cast_items <- movie_page %>%
  html_nodes("[data-testid='title-cast-item']")

# Extract actor names from the specific link within each item
actor_names <- cast_items %>%
  html_node("[data-testid='title-cast-item__actor']") %>%
  html_text(trim = TRUE) # trim whitespace directly

# Extract character names similarly
character_names <- cast_items %>%
  html_node("[data-testid='cast-item-characters-link'] span") %>%
  html_text(trim = TRUE) # trim whitespace directly

# Combine actors and characters into a data frame
cast_df <- data.frame(
  Actor = actor_names,
  Character = character_names,
  stringsAsFactors = FALSE
)

# Display the full top cast list found
cat("\nCast (Top Billed):\n")
print(cast_df)

rating <- movie_page %>%
  html_element("[data-testid='hero-rating-bar__aggregate-rating__score'] span:first-child") %>%
  html_text() %>%
  as.numeric()

cat("\nIMDB Rating:", rating, "/ 10\n")

# Create a request with custom headers
req <- request("https://example.com") %>%
  req_headers(
    `User-Agent` = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
    `Accept-Language` = "en-US,en;q=0.9",
    `Referer` = "https://example.com"
  )

# Add cookies if needed
req <- req %>%
  req_cookies(session_id = "your_session_id")

# Perform the request
resp <- req %>% req_perform()

# Extract the HTML content
html <- resp %>% resp_body_html()

# Basic authentication (for APIs)
req <- request("https://api.example.com") %>%
  req_auth_basic("username", "password")

# Or for form-based login
login_data <- list(
  username = "your_username",
  password = "your_password"
)

resp <- request("https://example.com/login") %>%
  req_body_form(!!!login_data) %>%
  req_perform()

# Save cookies for subsequent requests
cookies <- resp %>% resp_cookies()

# Use those cookies in your next request
next_req <- request("https://example.com/protected-page") %>%
  req_cookies(!!!cookies) %>%
  req_perform()

# Set up rate limiting and retries
req <- request("https://example.com") %>%
  req_retry(max_tries = 3, backoff = ~ 5) %>%
  req_throttle(rate = 1/3)  # Max 1 request per 3 seconds

# Perform the request
resp <- req %>% req_perform()

# Make a request to a JSON API
req <- request("https://api.example.com/products") %>%
  req_headers(`Accept` = "application/json")

resp <- req %>% req_perform()

# Parse the JSON response
data <- resp %>% resp_body_json()

# Work with the data as a regular R list
product_names <- sapply(data$products, function(product) product$name)

library(httr2)
library(rvest)

# Configure a base request template with all our common settings
base_req <- request("https://example.com") %>%
  req_headers(
    `User-Agent` = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
    `Accept-Language` = "en-US,en;q=0.9"
  ) %>%
  req_throttle(rate = 1/5) %>%  # Be polite!
  req_retry(max_tries = 3)      # Be persistent!

# Make a specific request
resp <- base_req %>%
  req_url_path_append("/products") %>%
  req_perform()

# Extract data using rvest on the response
html <- resp %>% resp_body_html()
product_names <- html %>%
  html_elements(".product-name") %>%
  html_text2()

library(Rcrawler)

# Basic website crawling
Rcrawler(Website = "https://example.com", 
         no_cores = 4,  # Use 4 CPU cores for parallel crawling
         MaxDepth = 2)  # Only follow links 2 levels deep

library(Rcrawler)

# List of scientists we're interested in
list_of_scientists <- c("Niels Bohr", "Max Born", "Albert Einstein", "Enrico Fermi")

# Create Wikipedia search URLs for each scientist
target_pages <- paste0('https://en.wikipedia.org/wiki/', gsub(" ", "_", list_of_scientists))

# Scrape specific data points using XPath patterns
scientist_data <- ContentScraper(
  Url = target_pages, 
  XpathPatterns = c(
    "//th[contains(text(), 'Born')]/following-sibling::td",
    "//th[contains(text(), 'Died')]/following-sibling::td"
  ),
  PatternsName = c("born", "died"), 
  asDataFrame = TRUE
)

# Display the results
print(scientist_data)

# Crawl a site and extract its network structure
Rcrawler(Website = "https://small-website-example.com", 
         no_cores = 2,
         NetworkData = TRUE,  # Extract network data
         NetwExtract = TRUE)  # Build the network graph

# This creates a file called "Net-Graph.graphml" that you can 
# visualize with tools like Gephi or Cytoscape

install.packages("chromote")

# Download the latest stable Chrome .deb package
wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb

# Install the package (might require sudo)
sudo apt install ./google-chrome-stable_current_amd64.deb

# Clean up the downloaded file (optional)
# rm google-chrome-stable_current_amd64.deb

# If the install command complains about missing dependencies, try fixing them:
sudo apt --fix-broken install

# --- Step 1: Libraries and Configuration ---

print("Loading libraries...")
# Install packages if needed
# install.packages(c("chromote", "rvest", "dplyr", "stringr", "purrr", "jsonlite"))

# Load necessary libraries
suppressPackageStartupMessages({
  library(chromote) # For controlling Chrome
  library(rvest)    # For parsing HTML
  library(dplyr)    # For data manipulation (used in bind_rows)
  library(stringr)  # For string cleaning
  library(purrr)    # For list manipulation (used by helper function check)
  library(jsonlite) # For saving JSON output
})
print("Libraries loaded.")

# --- Configuration ---
start_url <- "https://www.brickeconomy.com/sets/theme/racers" # Target page
base_url <- "https://www.brickeconomy.com"                   # Base for joining relative URLs
ua_string <- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36" # User agent
timeout_ms <- 60000 # 60 seconds for navigation/load events
wait_timeout_seconds <- 30 # Max wait for page update after pagination click
polite_sleep_range <- c(2, 5) # Min/max sleep seconds between scraping detail pages
max_pages_scrape <- 10 # Safety break for pagination loop (adjust if theme has more pages)
max_sets_scrape <- 2 # <<< SET LOW FOR TESTING, set to Inf for full run >>>
rds_url_file <- "racers_set_urls.rds"     # File to save/load intermediate URLs
rds_data_file <- "racers_set_data.rds"    # Final data output (RDS)
json_data_file <- "racers_set_data.json" # Final data output (JSON)

# --- End Step 1 ---

# --- Function Definition Start ---
get_all_set_urls <- function() {
  print("--- Function: get_all_set_urls ---")
  # Initialize variables for this function's scope
  all_relative_urls <- list()
  page_num <- 1
  navigation_success <- FALSE
  b <- NULL # Will hold our chromote session object

  # --- Step 2: Launch Browser ---
  tryCatch({
    b <- ChromoteSession$new() # Create the session (launches Chrome)
    print("Chromote session started for URL collection.")
  }, error = function(e) { 
      print(paste("Error creating Chromote session:", e$message))
      # If session fails to start, return NULL (or empty list) from function
      return(NULL) 
  })

  # Proceed only if session started successfully
  if (!is.null(b)) {
      # (Code for Step 3 and onwards goes here)
      # ...
  # --- End Step 2 ---  (Function continues)

# (Continuing inside the 'if (!is.null(b))' block from Step 2)

    # --- Step 3: Initial Session Setup & Navigation ---
    # Ensure browser closes when function exits (even on error)
    on.exit({ 
        if (!is.null(b) && b$is_active()) { 
            print("Closing URL collection session...")
            b$close() 
        } 
    })

    # Set the User-Agent for this session
    tryCatch({
      print("Setting User-Agent..."); 
      b$Network$setUserAgentOverride(userAgent = ua_string)
      print("User-Agent set.")
    }, error = function(e) { 
        # Print warning but continue if UA fails
        print(paste("Error setting User-Agent:", e$message)) 
    })

    # Navigate to the first page of Racers sets
    print(paste("Navigating to:", start_url, "..."))
    tryCatch({
      b$Page$navigate(start_url, timeout = timeout_ms)
      print("Initial navigation successful.")
      navigation_success <- TRUE # Set flag for later steps
    }, error = function(e) { 
        print(paste("Error during initial navigation:", e$message))
        navigation_success <- FALSE # Ensure flag is FALSE on error
    })
    # --- End Step 3 --- (Function continues)

    # if (navigation_success) { ... (Code for Step 4 goes here) }

# (Continuing inside the 'if (!is.null(b))' block from Step 3)

    # Check if initial navigation was successful before starting loop
    if (navigation_success) {
      
      print("--- Starting Pagination Loop ---")
      # --- Step 4: Loop Start and Get Page HTML ---
      while(page_num <= max_pages_scrape) { # Loop until max pages or last page detected
        
        print(paste("Processing Page:", page_num))
        Sys.sleep(2) # Small pause for stability before interacting
        
        print("Getting HTML content...")
        html_content <- NULL # Reset variables for this iteration
        page_html <- NULL
        urls_from_current_page <- character(0)
        first_url_on_page <- NULL 

        # Try to get the current page's HTML from chromote
        tryCatch({
          # Check session activity before commands
          if(is.null(b) || !b$is_active()) { 
            print("Chromote session inactive during loop. Breaking.")
            break 
          }
          # Get the root node of the document's DOM
          doc <- b$DOM$getDocument() 
          root_node_id <- doc$root$nodeId
          # Get the full outer HTML based on the root node
          html_content_js <- b$DOM$getOuterHTML(nodeId = root_node_id)
          html_content <- html_content_js$outerHTML
          # Parse the retrieved HTML using rvest
          page_html <- read_html(html_content) 
          print("HTML content retrieved and parsed.")

        }, error = function(e) { 
            # If getting HTML fails, print error and stop the loop
            print(paste("Error processing page", page_num, ":", e$message))
            break # Exit while loop on error
        })
        
        # Check if HTML parsing failed silently (shouldn't happen if tryCatch works)
        if (is.null(page_html)) { 
            print("Error: page_html is null after tryCatch. Stopping loop.")
            break 
        }
        # --- End Step 4 --- (Loop continues)
        
        # (Code for Step 5 goes here)
        # ...

# (Continuing inside the 'while' loop from Step 4)
        
        # --- Step 5: Extract URLs and Check for Last Page ---
        
        # Use the specific CSS selector to find set link elements
        set_link_selector <- "table#ContentPlaceHolder1_ctlSets_GridViewSets > tbody > tr > td:nth-child(2) > h4 > a"
        set_link_nodes <- html_elements(page_html, set_link_selector)
        
        # Extract the 'href' attribute (the relative URL) from each link node
        urls_from_current_page <- html_attr(set_link_nodes, "href")
        num_found <- length(urls_from_current_page)

        # Store the found URLs if any exist
        if (num_found > 0) {
            # Store the first URL to detect page changes later
            first_url_on_page <- urls_from_current_page[[1]] 
            print(paste("Found", num_found, "URLs on page", page_num, "(First:", first_url_on_page, ")"))
            # Add this page's URLs to our main list
            all_relative_urls[[page_num]] <- as.character(urls_from_current_page) 
        } else {
            # If no URLs found on a page (unexpected), stop the loop
            print(paste("Warning: Found 0 URLs on page", page_num, ". Stopping pagination."))
            break 
        }

        # Check if the 'Next' button's parent li has the 'disabled' class
        print("Checking for 'Next' button state...")
        next_button_parent_disabled_selector <- "ul.pagination > li.disabled:last-child"
        disabled_check_nodes <- html_elements(page_html, next_button_parent_disabled_selector)

        # If the disabled element exists (length > 0), we are on the last page
        if (length(disabled_check_nodes) > 0) { 
            print("Next button parent is disabled. Reached last page.")
            break # Exit the while loop
        } 
        # --- End Step 5 --- (Loop continues if 'Next' is not disabled)

        # (Code for Step 6 goes here)
        # else { ... }

# (Continuing inside the 'while' loop from Step 5)

        # --- Step 6: Click Next and Wait for Update ---
        else { 
          # If 'Next' button is not disabled, proceed to click it
          print("'Next' button appears active.")
          
          # Define the selector for the clickable 'Next' link
          next_button_selector <- "ul.pagination > li:last-child > a.page-link"
          print(paste("Attempting JavaScript click for selector:", next_button_selector))
          
          # Prepare the JavaScript code to execute the click
          # We escape any quotes in the selector just in case
          js_click_code <- paste0("document.querySelector(\"", gsub("\"", "\\\\\"", next_button_selector), "\").click();")
          
          click_success <- FALSE # Flag to track if click worked
          # Try executing the JavaScript click
          tryCatch({ 
              b$Runtime$evaluate(js_click_code) 
              print("JavaScript click executed.")
              click_success <- TRUE 
            }, error = function(e) { 
                print(paste("Error with JS click:", e$message))
                click_success <- FALSE 
            })
            
          # If the click command failed, stop the loop
          if (!click_success) { 
              print("JS click failed. Stopping.")
              break 
          }

          # === Intelligent Wait ===
          print("Waiting for page content update...")
          wait_start_time <- Sys.time() # Record start time
          content_updated <- FALSE      # Flag to track if content changed
          
          # Loop until content changes or timeout occurs
          while (difftime(Sys.time(), wait_start_time, units = "secs") < wait_timeout_seconds) {
              Sys.sleep(1) # Check every second
              print("Checking update...")
              current_first_url <- NULL # Reset check variable
              
              # Try to get the first URL from the *current* state of the page
              tryCatch({
                  if(!b$is_active()) { stop("Session inactive during wait.") } # Check session
                  doc <- b$DOM$getDocument(); root_node_id <- doc$root$nodeId
                  html_content_js <- b$DOM$getOuterHTML(nodeId = root_node_id)
                  current_page_html <- read_html(html_content_js$outerHTML)
                  current_set_nodes <- html_elements(current_page_html, set_link_selector)
                  if (length(current_set_nodes) > 0) { 
                      current_first_url <- html_attr(current_set_nodes[[1]], "href") 
                  }
              }, error = function(e) { 
                  # Ignore errors during check, just means content might not be ready
                  print("Error during wait check (ignored).") 
              })
              
              # Compare the newly fetched first URL with the one from *before* the click
              if (!is.null(current_first_url) && !is.null(first_url_on_page) && current_first_url != first_url_on_page) { 
                  print("Content updated!")
                  content_updated <- TRUE # Set flag
                  break # Exit the wait loop
              }
          } # End of inner wait loop
          
          # If the content didn't update within the timeout, stop the main loop
          if (!content_updated) { 
              print("Wait timeout reached. Stopping pagination.")
              break 
          }
          
          # Increment page number ONLY if click succeeded and content updated
          page_num <- page_num + 1 
          
        } # End else block (if 'Next' button was active)
      } # End while loop (pagination)
      print("--- Finished Pagination Loop ---")
      
    } # End if (navigation_success)
# --- End Step 6 --- (Function continues to aggregate URLs)

# (Code for aggregating/saving URLs from get_all_set_urls goes here)
# ... 
# } # End function get_all_set_urls

# --- Step 7: Define Data Extraction Helper Function ---

# This function takes parsed HTML ('page_html') from a single set page 
# and extracts data from the 'Set Details' and 'Set Pricing' sections.
extract_set_data <- function(page_html) {
  set_data <- list() # Initialize empty list for this set's data
  
  # Try to extract the set title/heading (often in H1)
  title_node <- html_element(page_html, "h1")
  if (!is.null(title_node) && !purrr::is_empty(title_node)) {
    set_data$title <- str_trim(html_text(title_node))
    # print(paste("Extracted title:", set_data$title)) # Optional debug print
  } else {
    print("Warning: Could not extract title (h1 element)")
  }

  # --- Extract Set Details ---
  # Strategy: Find all potential blocks, then identify the correct one.
  print("Attempting to identify and extract Set Details...")
  details_nodes <- html_elements(page_html, ".side-box-body") # Find all side-box bodies
  
  if (length(details_nodes) > 0) {
    found_details_section <- FALSE
    # Loop through potential sections to find the one containing "Set number"
    for (i in 1:length(details_nodes)) {
      row_nodes <- html_elements(details_nodes[[i]], "div.rowlist")
      contains_set_number <- FALSE
      # Check rows within this section for the key identifier
      for (row_node in row_nodes) {
        key_node <- html_element(row_node, "div.col-xs-5.text-muted")
        if (!is.null(key_node) && !purrr::is_empty(key_node)) {
          if (grepl("Set number", html_text(key_node), ignore.case = TRUE)) {
            contains_set_number <- TRUE
            break # Found the identifier in this section
          }
        }
      }
      
      # If this section contained "Set number", extract all data from it
      if (contains_set_number) {
        print("Found Set Details section. Extracting items...")
        found_details_section <- TRUE
        for (row_node in row_nodes) {
          key_node <- html_element(row_node, "div.col-xs-5.text-muted")
          if (!is.null(key_node) && !purrr::is_empty(key_node)) {
            key <- str_trim(html_text(key_node))
            if (key == "") next # Skip empty keys (like minifig image rows)
            
            value_node <- html_element(row_node, "div.col-xs-7")
            value <- if (!is.null(value_node) && !purrr::is_empty(value_node)) str_trim(html_text(value_node)) else NA_character_
            
            # Clean key, ensure uniqueness, add to list
            clean_key <- tolower(gsub(":", "", key)); clean_key <- gsub("\\s+", "_", clean_key); clean_key <- gsub("[^a-z0-9_]", "", clean_key)
            original_clean_key <- clean_key; key_suffix <- 1
            while(clean_key %in% names(set_data)) { key_suffix <- key_suffix + 1; clean_key <- paste0(original_clean_key, "_", key_suffix) }
            set_data[[clean_key]] <- value
            # print(paste("  Extracted Detail:", clean_key, "=", value)) # Optional debug
          }
        }
        break # Stop checking other side-box-body divs once found
      }
    } # End loop through potential detail sections
    if (!found_details_section) print("Warning: Could not definitively identify Set Details section.")
  } else {
    print("Warning: Could not find any .side-box-body elements for Set Details.")
  }

  # --- Extract Set Pricing ---
  # Strategy: Find the specific block using its unique combination of classes.
  print("Attempting to identify and extract Set Pricing...")
  # Pricing section specifically has 'position-relative' class as well
  pricing_nodes <- html_elements(page_html, ".side-box-body.position-relative") 
  
  if (length(pricing_nodes) > 0) {
    # Assume the first one found is correct (usually specific enough)
    pricing_node <- pricing_nodes[[1]] 
    print("Found Set Pricing section. Extracting items...")
    row_nodes <- html_elements(pricing_node, "div.rowlist")
    
    for (row_node in row_nodes) {
      key_node <- html_element(row_node, "div.col-xs-5.text-muted")
      if (!is.null(key_node) && !purrr::is_empty(key_node)) {
        key <- str_trim(html_text(key_node))
        if (key == "") next # Skip empty keys
        
        value_node <- html_element(row_node, "div.col-xs-7")
        value <- if (!is.null(value_node) && !purrr::is_empty(value_node)) str_trim(html_text(value_node)) else NA_character_
          
        # Clean key, add 'pricing_' prefix, ensure uniqueness
        clean_key <- tolower(gsub(":", "", key)); clean_key <- gsub("\\s+", "_", clean_key); clean_key <- gsub("[^a-z0-9_]", "", clean_key)
        clean_key <- paste0("pricing_", clean_key) # Add prefix
        original_clean_key <- clean_key; key_suffix <- 1
        while(clean_key %in% names(set_data)) { key_suffix <- key_suffix + 1; clean_key <- paste0(original_clean_key, "_", key_suffix) }
        set_data[[clean_key]] <- value
        # print(paste("  Extracted Pricing:", clean_key, "=", value)) # Optional debug
      }
    }
  } else {
    print("Warning: Could not find Set Pricing section (.side-box-body.position-relative).")
  }
  
  return(set_data) # Return the list of extracted data for this set
}

# --- End Step 7 ---

# --- Function Definition Start ---
scrape_set_details <- function(url_list) {
  print("--- Function: scrape_set_details ---")
  if (length(url_list) == 0) { 
    print("No URLs provided to scrape.")
    return(NULL) 
  }

  # Initialize list to store data for ALL sets
  all_set_data <- list() 
  b_details <- NULL # Will hold the chromote session for this function

  # --- Step 8: Initialize Session and Start Loop ---
  # Start a new chromote session specifically for scraping details
  tryCatch({
    b_details <- ChromoteSession$new()
    print("New Chromote session started for scraping details.")
  }, error = function(e) { 
      print(paste("Error creating details session:", e$message))
      return(NULL) # Cannot proceed if session doesn't start
  })

  # Proceed only if session started
  if (!is.null(b_details)) {
    # Ensure session closes when function finishes or errors
    on.exit({ 
        if (!is.null(b_details) && b_details$is_active()) { 
            print("Closing details session...")
            b_details$close() 
        } 
    })

    # Set User-Agent for this session
    tryCatch({ 
        print("Setting User-Agent...")
        b_details$Network$setUserAgentOverride(userAgent = ua_string)
        print("User-Agent set.") 
      }, error = function(e) { 
          print(paste("Error setting User-Agent:", e$message)) 
      })

    # --- Loop Through Each Set URL ---
    for (i in 1:length(url_list)) {
      
      # Check if we've reached the test limit
      if (i > max_sets_scrape) { 
          print(paste("Reached limit of", max_sets_scrape, "sets."))
          break 
      }

      # Get the relative URL and construct absolute URL
      relative_url <- url_list[[i]]
      if(is.null(relative_url) || !is.character(relative_url) || !startsWith(relative_url, "/set/")) { 
          print(paste("Skipping invalid URL entry at index", i))
          next # Skip to next iteration
      }
      absolute_url <- paste0(base_url, relative_url)
      print(paste("Processing set", i, "/", length(url_list), ":", absolute_url))
      
      # Initialize list for THIS set's data, starting with the URL
      set_data <- list(url = absolute_url) 

      # Flags to track progress for this URL
      navigation_success <- FALSE
      html_retrieval_success <- FALSE
      page_html <- NULL # Will store parsed HTML

      # Try navigating to the page and waiting
      tryCatch({ 
        print("Navigating...")
        b_details$Page$navigate(absolute_url, timeout = timeout_ms)
        print("Waiting for load event...")
        b_details$Page$loadEventFired(timeout = timeout_ms)
        print("Load event fired.")
        Sys.sleep(3) # Add pause after load event fires
        print("Navigation and wait complete.")
        navigation_success <- TRUE # Mark success if no error
      }, error = function(e) { 
          print(paste("Error navigating/waiting:", e$message))
          set_data$error <- "Navigation failed" # Record error
      })
      # --- End Step 8 --- (Loop continues to Step 9: Get HTML & Extract)

      # (Code for Step 9 goes here)
      # if (navigation_success) { ... } else { ... }
      # ...

# (Continuing inside the 'for' loop of the scrape_set_details function, after Step 8)

      # --- Step 9: Get HTML, Extract Data, and Pause ---
      
      # Proceed only if navigation was successful
      if (navigation_success) {
          
          # Try to get the HTML content for the successfully navigated page
          tryCatch({ 
            # Check session activity before commands
            if(is.null(b_details) || !b_details$is_active()) { 
                print("Details session inactive before getting HTML.")
                stop("Session closed unexpectedly") # Stop this iteration
            }
            print("Getting HTML...")
            doc <- b_details$DOM$getDocument()
            root_node_id <- doc$root$nodeId
            html_content_js <- b_details$DOM$getOuterHTML(nodeId = root_node_id)
            page_html <- read_html(html_content_js$outerHTML) # Parse it
            print("HTML retrieved/parsed.")
            html_retrieval_success <- TRUE # Mark success
            
          }, error = function(e) { 
              # If getting HTML fails, record the error
              print(paste("Error getting HTML:", e$message))
              set_data$error <- "HTML retrieval failed" # Add error info
              # html_retrieval_success remains FALSE
          })
      } # End if(navigation_success)

      # Proceed with data extraction only if HTML was retrieved successfully
      if (html_retrieval_success && !is.null(page_html)) {
          
          print("Extracting Set Details and Pricing...")
          # Call our helper function to do the heavy lifting
          extracted_data <- extract_set_data(page_html) 
          
          # Check if the helper function returned anything meaningful
          if (length(extracted_data) > 0) {
              # Add the extracted data to the 'set_data' list (which already has the URL)
              set_data <- c(set_data, extracted_data) 
              print("Data extraction complete.")
          } else {
              # Helper function returned empty list (e.g., selectors failed)
              print("Warning: No data extracted from page by helper function.")
              set_data$error <- "No data extracted by helper" # Add error info
          }
      } else if (navigation_success) { 
          # Handle cases where navigation worked but HTML retrieval failed
          print("Skipping data extraction due to HTML retrieval failure.")
          if(is.null(set_data$error)) { # Assign error if not already set
             set_data$error <- "HTML retrieval failed post-nav" 
          }
      }
      
      # --- Store results for this set ---
      # 'set_data' now contains either the extracted data+URL or URL+error
      all_set_data[[i]] <- set_data 
      
      # --- Polite Pause ---
      # Wait a random amount of time before processing the next URL
      sleep_time <- runif(1, polite_sleep_range[1], polite_sleep_range[2]) 
      print(paste("Sleeping for", round(sleep_time, 1), "seconds..."))
      Sys.sleep(sleep_time)
      # --- End Step 9 --- (Loop continues to next iteration)

    } # End FOR loop through URLs

  # (Code for aggregating results goes here)
  # ...
# } # End scrape_set_details function

# (Continuing inside the scrape_set_details function, after the 'for' loop)

    # --- Step 10: Aggregate Results & Clean Data ---
    print("--- Aggregating Final Results ---")

    # Check if any data was collected before proceeding
    if (length(all_set_data) > 0) {
        
        # Remove any potential NULL entries if loop skipped iterations
        all_set_data <- all_set_data[!sapply(all_set_data, is.null)] 
        
        # Define a function to check if an entry is just an error placeholder
        is_error_entry <- function(entry) { 
            return(is.list(entry) && !is.null(entry$error)) 
        }
        # Separate successful data from errors
        successful_data <- all_set_data[!sapply(all_set_data, is_error_entry)]
        error_data <- all_set_data[sapply(all_set_data, is_error_entry)]
        
        print(paste("Sets processed successfully:", length(successful_data)))
        print(paste("Sets with errors:", length(error_data)))

        # Proceed only if there's successful data to combine
        if (length(successful_data) > 0) {
            
            # Combine the list of successful data lists into a data frame
            # bind_rows handles differing columns by filling with NA
            final_df <- bind_rows(successful_data) %>%
                        # Start by ensuring all columns are character type for simpler cleaning
                        mutate(across(everything(), as.character)) 

            # --- Basic Data Cleaning Examples ---
            # (Add more cleaning as needed for analysis)
            
            # Clean 'pieces': extract only the number at the beginning
            if ("pieces" %in% names(final_df)) { 
                final_df <- final_df %>% 
                    mutate(pieces = str_extract(pieces, "^\\d+")) 
            }
            # Clean 'pricing_value': remove '$' and ','
            if ("pricing_value" %in% names(final_df)) { 
                 final_df <- final_df %>% 
                    mutate(pricing_value = str_remove_all(pricing_value, "[$,]")) 
            }
            # Clean 'pricing_growth': remove '+' and '%'
            if ("pricing_growth" %in% names(final_df)) { 
                 final_df <- final_df %>% 
                    mutate(pricing_growth = str_remove_all(pricing_growth, "[+%]")) 
            }
            # Clean 'pricing_annual_growth': remove '+' and '%' (keep text like 'Decreasing')
             if ("pricing_annual_growth" %in% names(final_df)) { 
                 final_df <- final_df %>% 
                    mutate(pricing_annual_growth = str_remove_all(pricing_annual_growth, "[+%]"))
            }
            # --- End Basic Cleaning ---

            print(paste("Combined data for", nrow(final_df), "sets into data frame."))
            
            # (Code for Step 11: Saving Data, goes here)
            # ...

        } else { 
            print("No data was successfully scraped to create a final data frame.") 
            final_df <- NULL # Ensure final_df is NULL if no success
        }
    } else {
        print("No data was collected in the all_set_data list.")
        final_df <- NULL # Ensure final_df is NULL if no data
    }
    # --- End Step 10 --- (Function continues)

    # return(final_df) # Return value determined in Step 11
# } # End scrape_set_details function

# (Continuing inside the scrape_set_details function, after Step 10)

            # --- Step 11: Display Summary, Save Output ---
            print("Dimensions of final data frame:")
            print(dim(final_df))
            # Show sample data robustly in case of few columns
            cols_to_show <- min(ncol(final_df), 10) 
            print(paste("Sample data (first 6 rows, first", cols_to_show, "columns):"))
            print(head(final_df[, 1:cols_to_show])) 

            # --- Save RDS ---
            print(paste("Saving final data frame to", rds_data_file))
            saveRDS(final_df, rds_data_file)
            
            # --- Save Structured JSON ---
            print(paste("Creating structured JSON output for", json_data_file))
            scrape_time_wat <- format(Sys.time(), "%Y-%m-%d %H:%M:%S %Z", tz="Africa/Lagos") 
            json_output_object <- list(
                metadata = list(
                    description = "Scraped LEGO Racers set data from BrickEconomy",
                    source_theme_url = start_url, 
                    sets_scraped = nrow(final_df),
                    sets_with_errors = length(error_data), # Include error count
                    scrape_timestamp_wat = scrape_time_wat,
                    data_structure = "List of objects under 'data', each object represents a set."
                ),
                data = final_df 
            )
            print(paste("Saving structured data to", json_data_file))
            write_json(json_output_object, json_data_file, pretty = TRUE, auto_unbox = TRUE) 
            # --- End Saving ---

            return(final_df) # Return the data frame on success

        # (Else blocks from Step 10 handling no successful data)
        # ...
    # (Return NULL block from Step 10 if no data collected)
    # ...
# } # End scrape_set_details function

# === Main Execution Block ===
# This part runs when you execute the script.

# 1. Get URLs (or load if file exists)
# It first checks if we already collected URLs and saved them.
if (!file.exists(rds_url_file)) {
    print(paste("URL file", rds_url_file, "not found, running URL collection..."))
    # If not found, call the function to scrape them
    racers_urls <- get_all_set_urls() 
     # Stop if URL collection failed
     if (is.null(racers_urls) || length(racers_urls) == 0) {
         stop("URL collection failed or yielded no URLs. Cannot proceed.") 
     }
} else {
    # If file exists, load the URLs from it
    print(paste("Loading existing URLs from", rds_url_file))
    racers_urls <- readRDS(rds_url_file) 
    # Stop if loaded list is bad
    if (is.null(racers_urls) || length(racers_urls) == 0) {
         stop("Loaded URL list is empty or invalid. Cannot proceed.")
    }
}

# 2. Scrape Details for loaded/collected URLs
# Only proceed if we have a valid list of URLs
if (!is.null(racers_urls) && length(racers_urls) > 0) {
    print("Proceeding to scrape details...")
    # Call the main function to scrape details using the URLs
    final_dataframe <- scrape_set_details(racers_urls) 
    
    # Check if scraping returned a data frame
    if (!is.null(final_dataframe)) {
        print("Scraping and processing appears complete.")
    } else {
        print("Scraping details function returned NULL, indicating failure or no data.")
    }
} else {
    print("Cannot proceed to scrape details, no URLs available.")
}

print("--- End of Full Script ---")
# --- End Step 11 ---

Rscript scrape_brickeconomy_sets.R

# Load the scraped data
lego_data <- readRDS("racers_set_data.rds")

# Load necessary libraries for analysis
library(dplyr)
library(ggplot2)
library(stringr)
library(tidyr)
library(scales)  # For nice formatting in plots

# Clean the pricing and numeric columns
lego_clean <- lego_data %>%
  # Convert text columns with numbers to actual numeric values
  mutate(
    pieces = as.numeric(pieces),
    pricing_value = as.numeric(str_remove_all(pricing_value, "[$,]")),
    pricing_growth = as.numeric(str_remove_all(pricing_growth, "[+%]")),
    pricing_annual_growth = as.numeric(str_remove_all(pricing_annual_growth, "[+%]")),
    year = as.numeric(str_extract(year, "\\d{4}"))
  ) %>%
  # Some rows might have NA values after conversion - remove them for analysis
  filter(!is.na(pricing_value) & !is.na(pricing_growth))

# Check our cleaned data
summary(lego_clean %>% select(year, pieces, pricing_value, pricing_growth, pricing_annual_growth))

# First, extract car brands from the set titles
# This requires some domain knowledge about car brands
lego_brands <- lego_clean %>%
  mutate(
    brand = case_when(
      str_detect(tolower(title), "ferrari") ~ "Ferrari",
      str_detect(tolower(title), "porsche") ~ "Porsche",
      str_detect(tolower(title), "lamborghini") ~ "Lamborghini",
      str_detect(tolower(title), "mercedes") ~ "Mercedes",
      str_detect(tolower(title), "bmw") ~ "BMW",
      str_detect(tolower(title), "ford") ~ "Ford",
      TRUE ~ "Other"
    )
  )

# Calculate average price gain by brand
brand_growth <- lego_brands %>%
  group_by(brand) %>%
  summarise(
    avg_growth = mean(pricing_growth, na.rm = TRUE),
    avg_annual_growth = mean(pricing_annual_growth, na.rm = TRUE),
    avg_value = mean(pricing_value, na.rm = TRUE),
    count = n()
  ) %>%
  # Only include brands with at least 2 sets for more reliable results
  filter(count >= 2) %>%
  arrange(desc(avg_growth))

# Create a nice table
knitr::kable(brand_growth, digits = 1)

# Helper function to remove outliers for better visualization
remove_outliers <- function(df, cols) {
  for (col in cols) {
    q1 <- quantile(df[[col]], 0.025, na.rm = TRUE)
    q3 <- quantile(df[[col]], 0.975, na.rm = TRUE)
    df <- df %>% filter(between(!!sym(col), q1, q3))
  }
  return(df)
}

# Clean data for plotting - remove extreme outliers
plot_data <- lego_clean %>%
  filter(!is.na(pieces), !is.na(pricing_growth)) %>%
  remove_outliers(c("pricing_growth", "pieces"))

# Create scatter plot - piece count vs growth
piece_plot <- ggplot(plot_data, aes(x = pieces, y = pricing_growth)) +
  geom_point(aes(size = pricing_value, color = year), alpha = 0.7) +
  geom_smooth(method = "lm", se = TRUE, color = "red", linetype = "dashed") +
  scale_y_continuous(labels = function(x) paste0(x, "%")) +
  scale_color_viridis_c() +
  labs(
    title = "Does Piece Count Affect LEGO Racers Value Growth?",
    subtitle = "Exploring the relationship between set size and price appreciation",
    x = "Number of Pieces",
    y = "Price Growth (%)",
    size = "Current Value ($)",
    color = "Release Year"
  ) +
  theme_minimal()

# Display the plot
print(piece_plot)

# Save the plot
ggsave("lego_piece_count_analysis.png", piece_plot, width = 10, height = 6, dpi = 300)

# Calculate correlation between pieces and growth
correlation <- cor(plot_data$pieces, plot_data$pricing_growth, use = "complete.obs")
cat("Correlation between piece count and price growth:", round(correlation, 3))

# Run a simple linear model
model <- lm(pricing_growth ~ pieces, data = plot_data)
summary(model)

# Group sets by piece count categories
size_analysis <- lego_clean %>%
  mutate(
    size_category = case_when(
      pieces < 200 ~ "Small (< 200 pieces)",
      pieces < 500 ~ "Medium (200-499 pieces)",
      pieces < 1000 ~ "Large (500-999 pieces)",
      TRUE ~ "Very Large (1000+ pieces)"
    )
  ) %>%
  group_by(size_category) %>%
  summarise(
    avg_growth = mean(pricing_growth, na.rm = TRUE),
    median_growth = median(pricing_growth, na.rm = TRUE),
    avg_value = mean(pricing_value, na.rm = TRUE),
    count = n()
  ) %>%
  # Ensure proper ordering for visualization
  mutate(size_category = factor(size_category, 
                               levels = c("Small (< 200 pieces)", 
                                         "Medium (200-499 pieces)", 
                                         "Large (500-999 pieces)", 
                                         "Very Large (1000+ pieces)"))) %>%
  arrange(size_category)

# Create a bar chart for size categories
size_plot <- ggplot(size_analysis, aes(x = size_category, y = avg_growth, fill = size_category)) +
  geom_bar(stat = "identity") +
  geom_text(aes(label = paste0(round(avg_growth, 1), "%")), 
            position = position_stack(vjust = 0.5), color = "white") +
  scale_y_continuous(labels = function(x) paste0(x, "%")) +
  labs(
    title = "Average Growth by LEGO Set Size",
    subtitle = "How piece count correlates with value appreciation",
    x = "Size Category",
    y = "Average Growth (%)"
  ) +
  theme_minimal() +
  theme(legend.position = "none",
        axis.text.x = element_text(angle = 45, hjust = 1))

# Display the second plot
print(size_plot)

# Save the second plot
ggsave("lego_size_category_analysis.png", size_plot, width = 10, height = 6, dpi = 300)

# Create a nice table
knitr::kable(size_analysis, digits = 1)