How to Download Files via cURL With Battle Ready Examples

curl --version

mkdir ~/curl-downloads
cd ~/curl-downloads

sudo apt install ca-certificates

curl -O https://example.com/file.zip

curl [options] [URL]

curl -O https://example.com/awesome-file.pdf

curl -o my-awesome-name.pdf https://example.com/boring-original-name.pdf

# Real-world example from our financial data scraping
curl -o "company_report_$(date +%Y%m%d).pdf" https://example.com/report.pdf

# Create directory structure automatically
curl -o "./downloads/reports/2024/Q1/report.pdf" https://example.com/report.pdf --create-dirs

# Use content-disposition header (when available)
curl -OJ https://example.com/mystery-file

# Follow redirects like a pro
curl -L -O https://example.com/file-that-moves-around

# See where your file leads you
curl -IL https://example.com/mysterious-file

# Complete silence (no progress or error messages)
curl -s -O https://example.com/quiet-file.zip

# Silent but shows errors (my personal favorite)
curl -sS -O https://example.com/important-file.zip

# Silent with custom error redirection
curl -s -O https://example.com/file.zip 2>errors.log

# Classic progress bar
curl -# -O https://example.com/big-file.zip

# Fancy progress meter with stats
curl --progress-bar -O https://example.com/big-file.zip

# Custom progress format (my favorite for scripting)
curl -O https://example.com/big-file.zip \
  --progress-bar \
  --write-out "\nDownload completed!\nAverage Speed: %{speed_download}bytes/sec\nTime: %{time_total}s\n"

# Resume a partial download
curl -C - -O https://example.com/massive-dataset.zip

# Check file size before resuming
curl -I https://example.com/massive-dataset.zip

# Resume with retry logic (battle-tested version)
curl -C - --retry 3 --retry-delay 5 -O https://example.com/massive-dataset.zip

#!/bin/bash
download_with_resume() {
    local url=$1
    local max_retries=3
    local retry_count=0

    while [ $retry_count -lt $max_retries ]; do
        curl -C - --retry 3 --retry-delay 5 -O "$url"
        if [ $? -eq 0 ]; then
            echo "Download completed successfully! 🎉"
            return 0
        fi
        let retry_count++
        echo "Retry $retry_count of $max_retries... 🔄"
        sleep 5
    done
    return 1
}

# Usage
download_with_resume "https://example.com/huge-file.zip"

curl -C - \
     --retry 3 \
     --retry-delay 5 \
     -# \
     -o "./downloads/$(date +%Y%m%d)_${filename}" \
     --create-dirs \
     -L \
     "https://example.com/file.zip"

alias superdownload='function _dl() { curl -C - --retry 3 --retry-delay 5 -# -o "./downloads/$(date +%Y%m%d)_$2" --create-dirs -L "$1"; };_dl'

superdownload https://example.com/file.zip custom_name

# Basic authentication (the classic way)
curl -u username:password -O https://secure-site.com/file.zip

# Using netrc file (my preferred method for automation)
echo "machine secure-site.com login myuser password mypass" >> ~/.netrc
chmod 600 ~/.netrc  # Important security step!
curl -n -O https://secure-site.com/file.zip

# Create a secure credential handler
curl -u "$(security find-generic-password -a $USER -s "api-access" -w)"

# Save cookies
curl -c cookies.txt -O https://example.com/login-required-file.zip

# Use saved cookies
curl -b cookies.txt -O https://example.com/another-file.zip

# The power combo (save and use in one go)
curl -b cookies.txt -c cookies.txt -O https://example.com/file.zip

#!/bin/bash
SESSION_HANDLER() {
    local username="$1"
    local password="$2"
    local cookie_file=$(mktemp)
    local max_retries=3
    local retry_delay=2
    
    # Input validation
    if [[ -z "$username" || -z "$password" ]]; then
        echo "❌ Error: Username and password are required"
        rm -f "$cookie_file"
        return 1
    }
    
    echo "🔐 Initiating secure session..."
    
    # First, login and save cookies with error handling
    if ! curl -s -c "$cookie_file" \
              --connect-timeout 10 \
              --max-time 30 \
              --retry $max_retries \
              --retry-delay $retry_delay \
              --fail-with-body \
              -d "username=${username}&password=${password}" \
              "https://example.com/login" > /dev/null 2>&1; then
        echo "❌ Login failed!"
        rm -f "$cookie_file"
        return 1
    fi
    
    echo "✅ Login successful"
    
    # Verify cookie file existence and content
    if [[ ! -s "$cookie_file" ]]; then
        echo "❌ No cookies were saved"
        rm -f "$cookie_file"
        return 1
    fi
    
    # Now download with session
    echo "📥 Downloading protected file..."
    if ! curl -sS -b "$cookie_file" \
              -O \
              --retry $max_retries \
              --retry-delay $retry_delay \
              --connect-timeout 10 \
              --max-time 300 \
              "https://example.com/protected-file.zip"; then
        echo "❌ Download failed"
        rm -f "$cookie_file"
        return 1
    fi
    
    echo "✅ Download completed successfully"
    
    # Cleanup
    rm -f "$cookie_file"
    return 0
}

# Usage Example
SESSION_HANDLER "myusername" "mypassword"

# Single header
curl -H "User-Agent: Mozilla/5.0" -O https://example.com/file.zip

# Multiple headers (production-ready version)
curl -H "User-Agent: Mozilla/5.0" \
     -H "Accept: application/pdf" \
     -H "Referer: https://example.com" \
     -O https://example.com/file.pdf

#!/bin/bash
# Define browser headers
USER_AGENTS=(
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0.0.0"
    "Mozilla/5.0 (Windows NT 10.0; Firefox/121.0)"
    "Mozilla/5.0 (Macintosh; Safari/605.1.15)"
)

SMART_DOWNLOAD() {
    local url="$1"
    local output="$2"
    
    # Select random User-Agent
    local agent=${USER_AGENTS[$RANDOM % ${#USER_AGENTS[@]}]}
    
    # Execute download with common headers
    curl --fail \
         -H "User-Agent: $agent" \
         -H "Accept: text/html,application/xhtml+xml,*/*" \
         -H "Accept-Language: en-US,en;q=0.9" \
         -H "Connection: keep-alive" \
         -o "$output" \
         "$url"
    
    # Be nice to servers
    sleep 1
}

# Usage Example
SMART_DOWNLOAD "https://example.com/file.pdf" "downloaded.pdf"

# Basic proxy usage
curl -x proxy.example.com:8080 -O https://example.com/file.zip

# Authenticated proxy (my preferred setup)
curl -x "username:password@proxy.example.com:8080" \
    -O https://example.com/file.zip

# SOCKS5 proxy (for extra sneakiness)
curl --socks5 proxy.example.com:1080 -O https://example.com/file.zip

#!/bin/bash
# Proxy configurations with authentication
declare -A PROXY_CONFIGS=(
    ["proxy1"]="username1:password1@proxy1.example.com:8080"
    ["proxy2"]="username2:password2@proxy2.example.com:8080"
    ["proxy3"]="username3:password3@proxy3.example.com:8080"
)

PROXY_DOWNLOAD() {
    local url="$1"
    local output="$2"
    local max_retries=3
    local retry_delay=2
    local timeout=30
    local temp_log=$(mktemp)
    
    # Input validation
    if [[ -z "$url" ]]; then
        echo "❌ Error: URL is required"
        rm -f "$temp_log"
        return 1
    }
    
    # Get proxy list keys
    local proxy_keys=("${!PROXY_CONFIGS[@]}")
    
    for ((retry=0; retry<max_retries; retry++)); do
        # Select random proxy
        local selected_proxy=${proxy_keys[$RANDOM % ${#proxy_keys[@]}]}
        local proxy_auth="${PROXY_CONFIGS[$selected_proxy]}"
        
        echo "🔄 Attempt $((retry + 1))/$max_retries using proxy: ${selected_proxy}"
        
        # Test proxy before download
        if curl --connect-timeout 5 \
                -x "$proxy_auth" \
                -s "https://api.ipify.org" > /dev/null 2>&1; then
            echo "✅ Proxy connection successful"
            
            # Attempt download with the working proxy
            if curl -x "$proxy_auth" \
                    --connect-timeout "$timeout" \
                    --max-time $((timeout * 2)) \
                    --retry 2 \
                    --retry-delay "$retry_delay" \
                    --fail \
                    --silent \
                    --show-error \
                    -o "$output" \
                    "$url" 2> "$temp_log"; then
                echo "✅ Download completed successfully using ${selected_proxy}"
                rm -f "$temp_log"
                return 0
            else
                echo "⚠️ Download failed with ${selected_proxy}"
                cat "$temp_log"
            fi
        else
            echo "⚠️ Proxy ${selected_proxy} is not responding"
        fi
        
        # Wait before trying next proxy
        if ((retry < max_retries - 1)); then
            local wait_time=$((retry_delay * (retry + 1)))
            echo "⏳ Waiting ${wait_time}s before next attempt..."
            sleep "$wait_time"
        fi
    done
    
    echo "❌ All proxy attempts failed"
    rm -f "$temp_log"
    return 1
}

# Usage Example
PROXY_DOWNLOAD "https://example.com/file.zip" "downloaded_file.zip"

#!/bin/bash
# SOCKS5 specific download function
SOCKS5_DOWNLOAD() {
    local url="$1"
    local output="$2"
    local socks5_proxy="$3"
    
    echo "🧦 Using SOCKS5 proxy: $socks5_proxy"
    
    if curl --socks5 "$socks5_proxy" \
            --connect-timeout 10 \
            --max-time 60 \
            --retry 3 \
            --retry-delay 2 \
            --fail \
            --silent \
            --show-error \
            -o "$output" \
            "$url"; then
        echo "✅ SOCKS5 download successful"
        return 0
    else
        echo "❌ SOCKS5 download failed"
        return 1
    fi
}

# Usage Example
SOCKS5_DOWNLOAD "https://example.com/file.zip" "output.zip" "proxy.example.com:1080"

#!/bin/bash
# Batch download with proxy rotation
BATCH_PROXY_DOWNLOAD() {
    local -a urls=("$@")
    local success_count=0
    local fail_count=0
    
    echo "📦 Starting batch download with proxy rotation..."
    
    for url in "${urls[@]}"; do
        local filename="${url##*/}"
        if PROXY_DOWNLOAD "$url" "$filename"; then
            ((success_count++))
        else
            ((fail_count++))
            echo "⚠️ Failed to download: $url"
        fi
    done
    
    echo "📊 Download Summary:"
    echo "✅ Successful: $success_count"
    echo "❌ Failed: $fail_count"
}

# Usage Example
BATCH_PROXY_DOWNLOAD "https://example.com/file1.zip" "https://example.com/file2.zip"

# Limit download speed (1M = 1MB/s)
curl --limit-rate 1M -O https://example.com/huge-file.zip

# Bandwidth control with retry logic (500KB/s limit)
curl --limit-rate 500k \
     --retry 3 \
     --retry-delay 5 \
     -C - \
     -O https://example.com/huge-file.zip

#!/bin/bash
adaptive_download() {
    local url="$1"
    local output="$2"
    local base_rate="500k"
    local retry_count=0
    local max_retries=3
    local backoff_delay=5
    
    while [ $retry_count -lt $max_retries ]; do
        if curl --limit-rate $base_rate \
                --connect-timeout 10 \
                --max-time 3600 \
                -o "$output" \
                -C - \
                "$url"; then
            echo "✅ Download successful!"
            return 0
        else
            base_rate="250k"  # Reduce speed on failure
            let retry_count++
            echo "⚠️ Retry $retry_count with reduced speed: $base_rate"
            sleep $((backoff_delay * retry_count))
        fi
    done
    return 1
}

# Usage Example
adaptive_download "https://example.com/huge-file.zip" "my-download.zip"

exec 1> >(tee -a "${LOG_DIR}/download_$(date +%Y%m%d).log")
exec 2>&1

curl -v --tlsv1.2 -O https://example.com/file.zip

curl -I --proxy-insecure https://example.com/file.zip

curl -v --proxy proxy.example.com:8080 https://example.com/file.zip >/dev/null

#!/bin/bash
SMART_DOWNLOAD() {
    local url="$1"
    local output_name="$2"
    local proxy="${PROXY_LIST[RANDOM % ${#PROXY_LIST[@]}]}"
    
    curl -x "$proxy" \
         --limit-rate 1M \
         --retry 3 \
         --retry-delay 5 \
         -C - \
         -# \
         -H "User-Agent: ${USER_AGENTS[RANDOM % ${#USER_AGENTS[@]}]}" \
         -b cookies.txt \
         -c cookies.txt \
         --create-dirs \
         -o "./downloads/$(date +%Y%m%d)_${output_name}" \
         "$url"
}

# Usage Examples

# Single file download with all the bells and whistles
SMART_DOWNLOAD "https://example.com/dataset.zip" "important_dataset.zip"

# Multiple files
for file in file1.zip file2.zip file3.zip; do
    SMART_DOWNLOAD "https://example.com/$file" "$file"
    sleep 2  # Be nice to servers
done

# From a file containing URLs (my most-used method)
while read url; do
   curl -O "$url"
done < urls.txt

# Multiple files from same domain (clean and simple)
# Downloads: file1.pdf, file2.pdf, file3.pdf
curl -O http://example.com/file[1-100].pdf

#!/bin/bash
BATCH_DOWNLOAD() {
    local url="$1"
    local retries=3
    local wait_time=2
    
    echo "🎯 Downloading: $url"
    
    for ((i=1; i<=retries; i++)); do
        if curl -sS --fail \
                  --retry-connrefused \
                  --connect-timeout 10 \
                  --max-time 300 \
                  -O "$url"; then
            echo "✅ Success: $url"
            return 0
        else
            echo "⚠️ Attempt $i failed, waiting ${wait_time}s..."
            sleep $wait_time
            wait_time=$((wait_time * 2))  # Exponential backoff
        fi
    done
    
    echo "❌ Failed after $retries attempts: $url"
    return 1
}

# Usage Example
BATCH_DOWNLOAD "https://example.com/large-file.zip"
# Or in a loop:
while read url; do
    BATCH_DOWNLOAD "$url"
done < urls.txt

#!/bin/bash
PROCESS_URL_LIST() {
    trap 'echo "⚠️ Process interrupted"; exit 1' SIGINT SIGTERM
    local input_file="$1"
    local success_count=0
    local total_urls=$(wc -l < "$input_file")
    
    echo "🚀 Starting batch download of $total_urls files..."
    
    while IFS='' read -r url || [[ -n "$url" ]]; do
        if [[ "$url" =~ ^#.*$ ]] || [[ -z "$url" ]]; then
            continue  # Skip comments and empty lines
        fi
        
        if BATCH_DOWNLOAD "$url"; then
            ((success_count++))
            printf "Progress: [%d/%d] (%.2f%%)\n" \
                   $success_count $total_urls \
                   $((success_count * 100 / total_urls))
        fi
    done < "$input_file"
    
    echo "✨ Download complete! Success rate: $((success_count * 100 / total_urls))%"
}

# Add to the script above
failed_urls=()
if ! BATCH_DOWNLOAD "$url"; then
    failed_urls+=("$url")
    echo "$url" >> failed_downloads.txt
fi

# Basic recursive download
curl -r -O 'http://example.com/files/{file1,file2,file3}.pdf'

# Advanced recursive download with pattern matching

#!/bin/bash
PARALLEL_DOWNLOAD() {
   trap 'kill $(jobs -p) 2>/dev/null; exit 1' SIGINT SIGTERM 
   local max_parallel=5  # Adjust based on your needs
   local active_downloads=0
   
   while read -r url; do
       # Check if we've hit our parallel limit
       while [ $active_downloads -ge $max_parallel ]; do
           wait -n  # Wait for any child process to finish
           ((active_downloads--))
       done
       
       # Start new download in background
       (
           if curl -sS --fail -O "$url"; then
               echo "✅ Success: $url"
           else
               echo "❌ Failed: $url"
               echo "$url" >> failed_urls.txt
           fi
       ) &
       
       ((active_downloads++))
       echo "🚀 Started download: $url (Active: $active_downloads)"
   done < urls.txt
   
   # Wait for remaining downloads
   wait
}

# Usage Example
echo "https://example.com/file1.zip
https://example.com/file2.zip" > urls.txt
PARALLEL_DOWNLOAD

# Add dynamic throttling based on failure rate
SMART_PARALLEL_DOWNLOAD() {
    local fail_count=0
    local total_count=0
    local max_parallel=5
    
    monitor_failures() {
        if [ $((total_count % 10)) -eq 0 ]; then
            local failure_rate=$((fail_count * 100 / total_count))
            if [ $failure_rate -gt 20 ]; then
                ((max_parallel--))
                echo "⚠️ High failure rate detected! Reducing parallel downloads to $max_parallel"
            fi
        fi
    }
    # ... rest of parallel download logic
}

#!/bin/bash
DOWNLOAD_WITH_ERROR_HANDLING() {
    trap 'rm -f "$temp_file"' EXIT
    local url="$1"
    local retry_count=0
    local max_retries=3
    local backoff_time=5
    local temp_file=$(mktemp)
    
    while [ $retry_count -lt $max_retries ]; do
        if curl -sS \
                --fail \
                --connect-timeout 15 \
                --max-time 300 \
                --retry 3 \
                --retry-delay 5 \
                -o "$temp_file" \
                "$url"; then
            
            # Verify file integrity
            if [ -s "$temp_file" ]; then
                mv "$temp_file" "$(basename "$url")"
                echo "✅ Download successful: $url"
                return 0
            else
                echo "⚠️ Downloaded file is empty"
            fi
        fi
        
        ((retry_count++))
        echo "🔄 Retry $retry_count/$max_retries for $url"
        sleep $((backoff_time * retry_count))  # Exponential backoff
    done
    
    rm -f "$temp_file"
    return 1
}

# Usage Example
DOWNLOAD_WITH_ERROR_HANDLING "https://example.com/large-file.dat"

#!/bin/bash
MASTER_BATCH_DOWNLOAD() {
    set -eo pipefail  # Exit on error
    trap 'kill $(jobs -p) 2>/dev/null; echo "⚠️ Process interrupted"; exit 1' SIGINT SIGTERM
    local url_file="$1"
    local max_parallel=5
    local success_count=0
    local fail_count=0
    
    # Setup logging
    local log_dir="logs/$(date +%Y%m%d_%H%M%S)"
    mkdir -p "$log_dir"
    exec 1> >(tee -a "${log_dir}/download.log")
    exec 2>&1
    
    echo "🚀 Starting batch download at $(date)"
    
    # Process URLs in parallel with smart throttling
    cat "$url_file" | while read -r url; do
        while [ $(jobs -p | wc -l) -ge $max_parallel ]; do
            wait -n
        done
        
        (
            if DOWNLOAD_WITH_ERROR_HANDLING "$url"; then
                echo "$url" >> "${log_dir}/success.txt"
                ((success_count++))
            else
                echo "$url" >> "${log_dir}/failed.txt"
                ((fail_count++))
            fi
            
            # Progress update
            total=$((success_count + fail_count))
            echo "Progress: $total files processed (Success: $success_count, Failed: $fail_count)"
        ) &
    done
    
    wait  # Wait for all downloads to complete
    
    # Generate report
    echo "
    📊 Download Summary
    ==================
    Total Files: $((success_count + fail_count))
    Successful: $success_count
    Failed: $fail_count
    Success Rate: $((success_count * 100 / (success_count + fail_count)))%
    
    Log Location: $log_dir
    ==================
    "
}

# Usage Example
echo "https://example.com/file1.zip
https://example.com/file2.zip
https://example.com/file3.zip" > batch_urls.txt

# Execute master download
MASTER_BATCH_DOWNLOAD "batch_urls.txt"

#!/bin/bash
IMAGE_BATCH_DOWNLOAD() {
   local url_list="$1"
   local output_dir="images/$(date +%Y%m%d)"
   local log_dir="logs/$(date +%Y%m%d)"
   local max_size=$((10*1024*1024))  # 10MB limit by default
   
   # Setup directories
   mkdir -p "$output_dir" "$log_dir"
   
   # Initialize counters
   declare -A stats=([success]=0 [failed]=0 [invalid]=0 [oversized]=0)
   
   # Cleanup function
   cleanup() {
       local exit_code=$?
       rm -f "$temp_file" 2>/dev/null
       echo "🧹 Cleaning up temporary files..."
       exit $exit_code
   }
   
   # Set trap for cleanup
   trap cleanup EXIT INT TERM
   
   validate_image() {
       local file="$1"
       local mime_type=$(file -b --mime-type "$file")
       local file_size=$(stat -f%z "$file" 2>/dev/null || stat -c%s "$file")
       
       # Check file size
       if [ "$file_size" -gt "$max_size" ]; then
           ((stats[oversized]++))
           echo "⚠️ File too large: $file_size bytes (max: $max_size bytes)"
           return 1
       }
       
       case "$mime_type" in
           image/jpeg|image/png|image/gif|image/webp)
               return 0
               ;;
           *)
               return 1
               ;;
       esac
   }
   
   download_image() {
       local url="$1"
       local filename=$(basename "$url")
       local temp_file=$(mktemp)
       
       echo "🎯 Downloading: $url"
       
       if curl -sS --fail \
                 --retry 3 \
                 --retry-delay 2 \
                 --max-time 30 \
                 -o "$temp_file" \
                 "$url"; then
           
           if validate_image "$temp_file"; then
               mv "$temp_file" "$output_dir/$filename"
               ((stats[success]++))
               echo "✅ Success: $filename"
               return 0
           else
               rm "$temp_file"
               ((stats[invalid]++))
               echo "⚠️ Invalid image type or size: $url"
               return 1
           fi
       else
           rm "$temp_file"
           ((stats[failed]++))
           echo "❌ Download failed: $url"
           return 1
       fi
   }
}

# Usage Examples

# Create a file with image URLs
echo "https://example.com/image1.jpg
https://example.com/image2.png" > url_list.txt

# Execute with default 10MB limit
IMAGE_BATCH_DOWNLOAD "url_list.txt"

# Or modify max_size variable for different limit
max_size=$((20*1024*1024)) IMAGE_BATCH_DOWNLOAD "url_list.txt"  # 20MB limit

#!/bin/bash
API_DATA_DOWNLOAD() {
    local api_base="https://api.example.com"
    local token=""
    local rate_limit=60  # requests per minute
    local last_token_refresh=0
    local log_file="api_downloads.log"
    local max_retries=3
    
    log_message() {
        local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
        echo "[$timestamp] $1" | tee -a "$log_file"
    }
    
    refresh_token() {
        local refresh_response
        refresh_response=$(curl -sS \
                    -X POST \
                    -H "Content-Type: application/json" \
                    --max-time 30 \
                    -d '{"key": "YOUR_API_KEY"}' \
                    "${api_base}/auth")
        
        if [ $? -ne 0 ]; then
            log_message "❌ Token refresh failed: Network error"
            return 1
        fi
        
        token=$(echo "$refresh_response" | jq -r '.token')
        if [ -z "$token" ] || [ "$token" = "null" ]; then
            log_message "❌ Token refresh failed: Invalid response"
            return 1
        }
        
        last_token_refresh=$(date +%s)
        log_message "✅ Token refreshed successfully"
        return 0
    }
    
    calculate_backoff() {
        local retry_count=$1
        echo $((2 ** (retry_count - 1) * 5))  # Exponential backoff: 5s, 10s, 20s...
    }
    
    download_data() {
        local endpoint="$1"
        local output_file="$2"
        local retry_count=0
        local success=false
        
        # Check token age
        local current_time=$(date +%s)
        if [ $((current_time - last_token_refresh)) -gt 3600 ]; then
            log_message "🔄 Token expired, refreshing..."
            refresh_token || return 1
        fi
        
        # Rate limiting
        sleep $(( 60 / rate_limit ))
        
        while [ $retry_count -lt $max_retries ] && [ "$success" = false ]; do
            local response
            response=$(curl -sS \
                 -H "Authorization: Bearer $token" \
                 -H "Accept: application/json" \
                 --max-time 30 \
                 "${api_base}${endpoint}")
            
            if [ $? -eq 0 ] && [ "$(echo "$response" | jq -e 'type')" = "object" ]; then
                echo "$response" > "$output_file"
                log_message "✅ Successfully downloaded data to $output_file"
                success=true
            else
                retry_count=$((retry_count + 1))
                if [ $retry_count -lt $max_retries ]; then
                    local backoff_time=$(calculate_backoff $retry_count)
                    log_message "⚠️ Attempt $retry_count failed. Retrying in ${backoff_time}s..."
                    sleep $backoff_time
                else
                    log_message "❌ Download failed after $max_retries attempts"
                    return 1
                fi
            fi
        done
        
        return 0
    }
}

# Usage Examples

# Initialize the function
API_DATA_DOWNLOAD

# Download data from specific endpoint
download_data "/v1/users" "users_data.json"

# Download with custom rate limit
rate_limit=30 download_data "/v1/transactions" "transactions.json"

#!/bin/bash
FTP_BATCH_DOWNLOAD() {
    local host="$1"
    local user="$2"
    local pass="$3"
    local remote_dir="$4"
    local local_dir="downloads/ftp/$(date +%Y%m%d)"
    local log_file="$local_dir/ftp_transfer.log"
    local netrc_file
    local status=0
    
    # Create secure temporary .netrc file
    netrc_file=$(mktemp)
    
    log_message() {
        local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
        echo "[$timestamp] $1" | tee -a "$log_file"
    }
    
    cleanup() {
        local exit_code=$?
        if [ -f "$netrc_file" ]; then
            shred -u "$netrc_file" 2>/dev/null || rm -P "$netrc_file" 2>/dev/null || rm "$netrc_file"
        fi
        log_message "🧹 Cleanup completed"
        exit $exit_code
    }
    
    validate_downloads() {
        local failed_files=0
        log_message "🔍 Validating downloaded files..."
        
        while IFS= read -r file; do
            if [ ! -s "$file" ]; then
                log_message "⚠️ Empty or invalid file: $file"
                ((failed_files++))
            fi
        done < <(find "$local_dir" -type f -not -name "*.log")
        
        return $failed_files
    }
    
    # Set trap for cleanup
    trap cleanup EXIT INT TERM
    
    # Create directories
    mkdir -p "$local_dir"
    
    # Create .netrc with secure permissions
    umask 077
    echo "machine $host login $user password $pass" > "$netrc_file"
    
    log_message "🚀 Starting FTP download from $host..."
    log_message "📁 Remote directory: $remote_dir"
    
    # Download with enhanced options
    curl --retry 3 \
         --retry-delay 10 \
         --retry-all-errors \
         --ftp-create-dirs \
         --create-dirs \
         --connect-timeout 30 \
         --max-time 3600 \
         -C - \
         --netrc-file "$netrc_file" \
         --stderr - \
         --progress-bar \
         "ftp://$host/$remote_dir/*" \
         --output "$local_dir/#1" 2>&1 | tee -a "$log_file"
    
    status=$?
    
    if [ $status -eq 0 ]; then
        # Validate downloads
        validate_downloads
        local validation_status=$?
        
        if [ $validation_status -eq 0 ]; then
            log_message "✅ FTP download completed successfully!"
        else
            log_message "⚠️ Download completed but $validation_status files failed validation"
            status=1
        fi
    else
        log_message "❌ FTP download failed with status: $status"
    fi
    
    return $status
}

# Usage Example
FTP_BATCH_DOWNLOAD "ftp.example.com" "username" "password" "remote/directory"

# With error handling
if FTP_BATCH_DOWNLOAD "ftp.example.com" "username" "password" "remote/directory"; then
    echo "Transfer successful"
else
    echo "Transfer failed"
fi

#!/bin/bash
LARGE_FILE_DOWNLOAD() {
   local url="$1"
   local filename="$2"
   local min_speed=1000  # 1KB/s minimum
   local timeout=300     # 5 minutes
   local chunk_size="10M"
   
   echo "🎯 Starting large file download: $filename"
   
   # Create temporary directory for chunks
   local temp_dir=$(mktemp -d)
   local final_file="downloads/large/$filename"
   mkdir -p "$(dirname "$final_file")"
   
   download_chunk() {
       local start=$1
       local end=$2
       local chunk_file="$temp_dir/chunk_${start}-${end}"
       
       curl -sS \
            --range "$start-$end" \
            --retry 3 \
            --retry-delay 5 \
            --speed-limit $min_speed \
            --speed-time $timeout \
            -o "$chunk_file" \
            "$url"
       
       return $?
   }
   
   # Get file size
   local size=$(curl -sI "$url" | grep -i content-length | awk '{print $2}' | tr -d '\r')
   local chunks=$(( (size + (1024*1024*10) - 1) / (1024*1024*10) ))
   
   echo "📦 File size: $(( size / 1024 / 1024 ))MB, Split into $chunks chunks"
   
   # Download chunks in parallel
   for ((i=0; i<chunks; i++)); do
       local start=$((i * 1024 * 1024 * 10))
       local end=$(( (i + 1) * 1024 * 1024 * 10 - 1 ))
       if [ $end -ge $size ]; then
           end=$((size - 1))
       fi
       
       (download_chunk $start $end) &
       
       # Limit parallel downloads
       if [ $((i % 5)) -eq 0 ]; then
           wait
       fi
   done
   
   wait  # Wait for all chunks
   
   # Combine chunks
   echo "🔄 Combining chunks..."
   cat "$temp_dir"/chunk_* > "$final_file"
   
   # Verify file size
   local downloaded_size=$(stat --format=%s "$final_file" 2>/dev/null || stat -f %z "$final_file")
   if [ "$downloaded_size" -eq "$size" ]; then
       echo "✅ Download complete and verified: $filename"
       rm -rf "$temp_dir"
       return 0
   else
       echo "❌ Size mismatch! Expected: $size, Got: $downloaded_size"
       rm -rf "$temp_dir"
       return 1
   fi
}

# Usage Example
LARGE_FILE_DOWNLOAD "https://example.com/huge.zip" "backup.zip"

#!/bin/bash
monitor_progress() {
    local file="$1"
    local total_size="$2"
    local timeout="${3:-3600}"  # Default 1 hour timeout
    
    local start_time=$(date +%s)
    local last_size=0
    local last_check_time=$start_time
    
    # Function to format sizes
    format_size() {
        local size=$1
        if [ $size -ge $((1024*1024*1024)) ]; then
            printf "%.1fGB" $(echo "scale=1; $size/1024/1024/1024" | bc)
        elif [ $size -ge $((1024*1024)) ]; then
            printf "%.1fMB" $(echo "scale=1; $size/1024/1024" | bc)
        else
            printf "%.1fKB" $(echo "scale=1; $size/1024" | bc)
        fi
    }
    
    # Function to draw progress bar
    draw_progress_bar() {
        local percentage=$1
        local width=50
        local completed=$((percentage * width / 100))
        local remaining=$((width - completed))
        
        printf "["
        printf "%${completed}s" | tr " " "="
        printf ">"
        printf "%${remaining}s" | tr " " " "
        printf "] "
    }
    
    # Check if file exists
    if [ ! -f "$file" ]; then
        echo "❌ Error: File '$file' not found!"
        return 1
    }
    
    # Check if total size is valid
    if [ $total_size -le 0 ]; then
        echo "❌ Error: Invalid total size!"
        return 1
    }
    
    echo "🚀 Starting progress monitoring..."
    
    # Loop to continuously check the file size
    while true; do
        # Get the current file size
        local current_size=$(stat --format=%s "$file" 2>/dev/null || stat -f %z "$file" 2>/dev/null || echo 0)
        local current_time=$(date +%s)
        
        # Calculate time elapsed
        local elapsed=$((current_time - start_time))
        
        # Check timeout
        if [ $elapsed -gt $timeout ]; then
            echo -e "\n⚠️ Monitoring timed out after $(($timeout/60)) minutes"
            return 1
        }
        
        # Calculate speed and ETA
        local time_diff=$((current_time - last_check_time))
        local size_diff=$((current_size - last_size))
        if [ $time_diff -gt 0 ]; then
            local speed=$((size_diff / time_diff))
            local remaining_size=$((total_size - current_size))
            local eta=$((remaining_size / speed))
        fi
        
        # Calculate percentage
        local percentage=0
        if [ $total_size -gt 0 ]; then
            percentage=$(( (current_size * 100) / total_size ))
        fi
        
        # Clear line and show progress
        echo -ne "\r\033[K"
        
        # Draw progress bar
        draw_progress_bar $percentage
        
        # Show detailed stats
        printf "%3d%% " $percentage
        printf "$(format_size $current_size)/$(format_size $total_size) "
        
        if [ $speed ]; then
            printf "@ $(format_size $speed)/s "
            if [ $eta -gt 0 ]; then
                printf "ETA: %02d:%02d " $((eta/60)) $((eta%60))
            fi
        fi
        
        # Check if download is complete
        if [ "$current_size" -ge "$total_size" ]; then
            echo -e "\n✅ Download complete! Total time: $((elapsed/60))m $((elapsed%60))s"
            return 0
        fi
        
        # Update last check values
        last_size=$current_size
        last_check_time=$current_time
        
        # Wait before next check
        sleep 1
    done
}

# Usage Examples

# Basic usage (monitoring a 100MB download)
monitor_progress "downloading_file.zip" 104857600

# With custom timeout (2 hours)
monitor_progress "large_file.iso" 1073741824 7200

# Integration with curl
curl -o "download.zip" "https://example.com/file.zip" &
monitor_progress "download.zip" $(curl -sI "https://example.com/file.zip" | grep -i content-length | awk '{print $2}' | tr -d '\r')

# 1. Enable compression (huge performance boost!)
curl --compressed -O https://example.com/file.zip

# 2. Use keepalive connections
curl --keepalive-time 60 -O https://example.com/file.zip

# 3. Optimize DNS resolution
curl --resolve example.com:443:1.2.3.4 -O https://example.com/file.zip

#!/bin/bash
OPTIMIZED_DOWNLOAD() {
    local url="$1"
    local output="$2"
    
    # Pre-resolve DNS
    local domain=$(echo "$url" | awk -F[/:] '{print $4}')
    local ip=$(dig +short "$domain" | head -n1)
    
    # Performance optimization flags
    local opts=(
        --compressed
        --keepalive-time 60
        --resolve "${domain}:443:${ip}"
        --connect-timeout 10
        --max-time 300
        --retry 3
        --retry-delay 5
    )
    
    echo "🚀 Downloading with optimizations..."
    curl "${opts[@]}" -o "$output" "$url"
}

# Usage Examples

# Basic optimized download
OPTIMIZED_DOWNLOAD "https://example.com/large-file.zip" "downloaded-file.zip"

# Multiple files (using xargs for parallel downloads)
cat url-list.txt | xargs -P 4 -I {} sh -c 'OPTIMIZED_DOWNLOAD "{}" "$(basename {})"'

# With custom naming
OPTIMIZED_DOWNLOAD "https://example.com/data.zip" "backup-$(date +%Y%m%d).zip"

#!/bin/bash
ROBUST_DOWNLOAD() {
    local url="$1"
    local retries=3
    local timeout=30
    local success=false
    
    # Trap cleanup on script exit
    trap 'cleanup' EXIT
    
    cleanup() {
        rm -f "$temp_file"
        [ "$success" = true ] || echo "❌ Download failed for $url"
    }
    
    verify_download() {
        local file="$1"
        local expected_size="$2"
        
        # Basic existence check
        [ -f "$file" ] || return 1
        
        # Size verification (must be larger than 0 bytes)
        [ -s "$file" ] || return 1
        
        # If expected size is provided, verify it matches
        if [ -n "$expected_size" ]; then
            local actual_size=$(stat --format=%s "$file" 2>/dev/null || stat -f %z "$file" 2>/dev/null)
            [ "$actual_size" = "$expected_size" ] || return 1
        fi
        
        # Basic content verification
        head -c 512 "$file" | grep -q '[^\x00]' || return 1
        
        return 0
    }
    
    # Main download logic with comprehensive error handling
    local temp_file=$(mktemp)
    local output_file=$(basename "$url")
    
    for ((i=1; i<=retries; i++)); do
        echo "🔄 Attempt $i of $retries"
        
        if curl -sS \
                --fail \
                --show-error \
                --location \
                --connect-timeout "$timeout" \
                -o "$temp_file" \
                "$url" 2>error.log; then
            
            if verify_download "$temp_file"; then
                mv "$temp_file" "$output_file"
                success=true
                echo "✅ Download successful! Saved as $output_file"
                break
            else
                echo "⚠️ File verification failed, retrying..."
                sleep $((2 ** i))  # Exponential backoff
            fi
        else
            echo "⚠️ Error on attempt $i:"
            cat error.log
            sleep $((2 ** i))
        fi
    done
}

# Usage Examples

# Basic download with error handling
ROBUST_DOWNLOAD "https://example.com/file.zip"

# Integration with size verification
expected_size=$(curl -sI "https://example.com/file.zip" | grep -i content-length | awk '{print $2}' | tr -d '\r')
ROBUST_DOWNLOAD "https://example.com/file.zip" "$expected_size"

# Multiple files with error handling
cat urls.txt | while read url; do
    ROBUST_DOWNLOAD "$url"
    sleep 2  # Polite delay between downloads
done

#!/bin/bash
SECURE_DOWNLOAD() {
   local url="$1"
   local output="$2"
   local expected_hash="$3"
   
   # Security configuration
   local security_opts=( 
       --ssl-reqd
       --tlsv1.2
       --ssl-no-revoke
       --cert-status
       --remote-header-name
       --proto '=https'
   )
   
   # Hash verification function
   verify_checksum() {
       local file="$1"
       local expected_hash="$2"
       
       if [ -z "$expected_hash" ]; then
           echo "⚠️  No checksum provided. Skipping verification."
           return 0
       fi
       
       local actual_hash
       actual_hash=$(sha256sum "$file" | cut -d' ' -f1)
       
       if [[ "$actual_hash" == "$expected_hash" ]]; then
           return 0
       else
           echo "❌ Checksum verification failed!"
           return 1
       fi
   }
   
   echo "🔒 Initiating secure download..."
   
   # Create secure temporary directory
   local temp_dir=$(mktemp -d)
   chmod 700 "$temp_dir"
   
   # Set temporary filename
   local temp_file="${temp_dir}/$(basename "$output")"
   
   # Execute secure download
   if curl "${security_opts[@]}" \
           -o "$temp_file" \
           "$url"; then
       
       # Verify file integrity
       if verify_checksum "$temp_file" "$expected_hash"; then
           mv "$temp_file" "$output"
           echo "✅ Secure download complete! Saved as $output"
       else
           echo "❌ Integrity check failed. File removed."
           rm -rf "$temp_dir"
           return 1
       fi
   else
       echo "❌ Download failed for $url"
       rm -rf "$temp_dir"
       return 1
   fi
   
   # Cleanup
   rm -rf "$temp_dir"
}

# Usage Example
SECURE_DOWNLOAD "https://example.com/file.zip" "downloaded_file.zip" "expected_sha256_hash"

# DON'T do this
curl https://example.com/huge-file.zip > file.zip

# DO this instead
SMART_MEMORY_DOWNLOAD() {
    local url="$1"
    local buffer_size="16k"
    
    if [[ -z "$url" ]]; then
        echo "Error: URL required" >&2
        return 1
    }
    
    curl --limit-rate 50M \
         --max-filesize 10G \
         --buffer-size "$buffer_size" \
         --fail \
         --silent \
         --show-error \
         -O "$url"
         
    return $?
}
# Usage
SMART_MEMORY_DOWNLOAD "https://example.com/large-dataset.zip"

# DON'T disable SSL verification
curl -k https://example.com/file

# DO handle certificates properly
CERT_AWARE_DOWNLOAD() {
    local url="$1"
    local ca_path="/etc/ssl/certs"
    
    if [[ ! -d "$ca_path" ]]; then
        echo "Error: Certificate path not found" >&2
        return 1
    }
    
    curl --cacert "$ca_path/ca-certificates.crt" \
         --capath "$ca_path" \
         --ssl-reqd \
         --fail \
         -O "$url"
}

# Usage
CERT_AWARE_DOWNLOAD "https://api.example.com/secure-file.zip"

# Proper resource management
CLEAN_DOWNLOAD() {
    local url="$1"
    local temp_dir=$(mktemp -d)
    local temp_files=()
    
    cleanup() {
        local exit_code=$?
        echo "Cleaning up temporary files..."
        for file in "${temp_files[@]}"; do
            rm -f "$file"
        done
        rmdir "$temp_dir" 2>/dev/null
        exit $exit_code
    }
    
    trap cleanup EXIT INT TERM
    
    # Your download logic here
    curl --fail "$url" -o "$temp_dir/download"
    temp_files+=("$temp_dir/download")
}

# Usage
CLEAN_DOWNLOAD "https://example.com/file.tar.gz"    

...
PRODUCTION_DOWNLOAD() {
    # Pre-download checks
    [[ -z "$1" ]] && { echo "❌ URL required"; return 1; }
    [[ -d "$(dirname "$2")" ]] || mkdir -p "$(dirname "$2")"
    
    # Resource monitoring
    local start_time=$(date +%s)
    local disk_space=$(df -h . | awk 'NR==2 {print $4}')
    
    echo "📊 Pre-download stats:"
    echo "Available disk space: $disk_space"
    
    # The actual download with all best practices
    local result=$(SECURE_DOWNLOAD "$1" "$2" 2>&1)
    local status=$?
    
    # Post-download verification
    if [[ $status -eq 0 ]]; then
        local end_time=$(date +%s)
        local duration=$((end_time - start_time))
        
        echo "✨ Download Statistics:"
        echo "Duration: ${duration}s"
        echo "Final size: $(du -h "$2" | cut -f1)"
        echo "Success! 🎉"
    else
        echo "❌ Download failed: $result"
    fi
    
    return $status
}

# Usage
PRODUCTION_DOWNLOAD "https://example.com/file.zip" "/path/to/destination/file.zip"

CHECK_ENVIRONMENT() {
    local required_space=$((5 * 1024 * 1024))  # 5GB in KB
    local available_space=$(df . | awk 'NR==2 {print $4}')
    local curl_version=$(curl --version | head -n1)
    
    # Check disk space
    [[ $available_space -lt $required_space ]] && {
        echo "⚠️ Insufficient disk space!" >&2
        return 1
    }
    
    # Check curl installation
    command -v curl >/dev/null 2>&1 || {
        echo "⚠️ curl is not installed!" >&2
        return 1
    }
    
    # Check SSL support
    curl --version | grep -q "SSL" || {
        echo "⚠️ curl lacks SSL support!" >&2
        return 1
    }
    
    echo "✅ Environment check passed!"
    echo "🔍 curl version: $curl_version"
    echo "💾 Available space: $(numfmt --to=iec-i --suffix=B $available_space)"
}

# Usage
CHECK_ENVIRONMENT || exit 1

#!/bin/bash
DIAGNOSE_DOWNLOAD() {
   local url="$1"
   local error_log=$(mktemp)
   
   echo "🔍 Starting download diagnosis..."
   
   # Step 1: Basic connection test with status code check
   echo "Testing basic connectivity..."
   if ! curl -sS -w "\nHTTP Status: %{http_code}\n" --head "$url" > "$error_log" 2>&1; then
       echo "❌ Connection failed! Details:"
       cat "$error_log"
       
       # Check for common issues
       if grep -q "Could not resolve host" "$error_log"; then
           echo "💡 DNS resolution failed. Checking DNS..."
           dig +short "$(echo "$url" | awk -F[/:] '{print $4}')"
       elif grep -q "Connection timed out" "$error_log"; then
           echo "💡 Connection timeout. Try:"
           echo "curl --connect-timeout 30 --max-time 300 \"$url\""
       fi
   else
       echo "✅ Basic connectivity OK"
       # Check HTTP status code
       status_code=$(grep "HTTP Status:" "$error_log" | awk '{print $3}')
       if [[ $status_code -ge 400 ]]; then
           echo "⚠️ Server returned error status: $status_code"
       fi
   fi
   
   # Step 2: SSL verification
   echo "Checking SSL/TLS..."
   if ! curl -vI --ssl-reqd "$url" > "$error_log" 2>&1; then
       echo "❌ SSL verification failed! Details:"
       grep "SSL" "$error_log"
   else
       echo "✅ SSL verification OK"
   fi

   # Cleanup
   rm -f "$error_log"
}

# Usage Example
DIAGNOSE_DOWNLOAD "https://api.github.com/repos/curl/curl/releases/latest"

#!/bin/bash
NETWORK_DIAGNOSIS() {
    local url="$1"
    local domain=$(echo "$url" | awk -F[/:] '{print $4}')
    local trace_log=$(mktemp)
   
    echo "🌐 Starting network diagnosis..."
   
    # DNS Resolution
    echo "Testing DNS resolution..."
    local dns_result=$(dig +short "$domain")
    if [[ -z "$dns_result" ]]; then
        echo "❌ DNS resolution failed!"
        echo "Try: nslookup $domain 8.8.8.8"
        rm -f "$trace_log"
        return 1
    fi
   
    # Trace Route with timeout
    echo "Checking network path..."
    if command -v timeout >/dev/null 2>&1; then
        timeout 30 traceroute -n "$domain" | tee "$trace_log"
    else
        traceroute -w 2 -n "$domain" | tee "$trace_log"  # -w 2 sets 2-second timeout per hop
    fi
   
    # Bandwidth Test
    echo "Testing download speed..."
    curl -s -w "\nSpeed: %{speed_download} bytes/sec\nTime to first byte: %{time_starttransfer} seconds\n" \
         "$url" -o /dev/null

    # Cleanup
    rm -f "$trace_log"
}

# Usage Example
NETWORK_DIAGNOSIS "https://downloads.example.com/large-file.zip"

#!/bin/bash
PERMISSION_CHECK() {
   local target_path="$1"
   
   # Check if target path was provided
   if [[ -z "$target_path" ]]; then
       echo "❌ Error: No target path provided"
       return 1
   }

   local temp_dir=$(dirname "$target_path")
   
   echo "🔍 Checking permissions..."
   
   # Check if path exists
   if [[ ! -e "$temp_dir" ]]; then
       echo "❌ Directory does not exist: $temp_dir"
       return 1
   }

   # File system type check
   local fs_type=$(df -PT "$temp_dir" | awk 'NR==2 {print $2}')
   echo "📂 File system type: $fs_type"
   case "$fs_type" in
       "nfs"|"cifs")
           echo "⚠️ Network file system detected - special permissions may apply"
           ;;
       "tmpfs")
           echo "⚠️ Temporary file system - data will not persist after reboot"
           ;;
   esac

   # Directory permissions check
   check_directory_access() {
       local dir="$1"
       if [[ ! -w "$dir" ]]; then
           echo "❌ No write permission in: $dir"
           echo "Current permissions: $(ls -ld "$dir")"
           echo "💡 Fix with: sudo chown $(whoami) \"$dir\""
           return 1
       fi
       return 0
   }
   
   # Disk space verification with reserved space check
   check_disk_space() {
       local required_mb="$1"
       local available_kb=$(df . | awk 'NR==2 {print $4}')
       local available_mb=$((available_kb / 1024))
       local reserved_space_mb=100  # Reserve 100MB for safety
       
       if [[ $((available_mb - reserved_space_mb)) -lt $required_mb ]]; then
           echo "❌ Insufficient disk space!"
           echo "Required: ${required_mb}MB"
           echo "Available: ${available_mb}MB (with ${reserved_space_mb}MB reserved)"
           return 1
       fi
       return 0
   }
   
   # Full permission diagnostic
   DIAGNOSE_PERMISSIONS() {
       echo "📁 Directory structure:"
       namei -l "$target_path"
       
       echo -e "\n💾 Disk usage:"
       df -h "$(dirname "$target_path")"
       
       echo -e "\n👤 Current user context:"
       id
       
       echo -e "\n🔒 SELinux status (if applicable):"
       if command -v getenforce >/dev/null 2>&1; then
           getenforce
       fi
   }

   # Run the diagnostics
   check_directory_access "$temp_dir"
   check_disk_space 500  # Assuming 500MB minimum required
   DIAGNOSE_PERMISSIONS
}

# Usage Example
PERMISSION_CHECK "/var/www/downloads/target-file.zip"

#!/bin/bash
SSL_DIAGNOSTIC() {
    local url="$1"
    local domain=$(echo "$url" | awk -F[/:] '{print $4}')
    local temp_cert=$(mktemp)
    
    echo "🔐 Starting SSL diagnosis..."
    
    # Input validation
    if [[ -z "$domain" ]]; then
        echo "❌ Invalid URL provided"
        rm -f "$temp_cert"
        return 1
    }
    
    # Certificate validation with timeout
    check_cert() {
        echo "Checking certificate for $domain..."
        if timeout 10 openssl s_client -connect "${domain}:443" 2> "$temp_cert" | \
            openssl x509 -noout -dates -subject -issuer; then
            echo "✅ Certificate details retrieved successfully"
        else
            echo "❌ Certificate retrieval failed"
            cat "$temp_cert"
        fi
    }
    
    # Certificate chain verification with SNI support
    verify_chain() {
        echo "Verifying certificate chain..."
        if timeout 10 openssl s_client -connect "${domain}:443" \
                        -servername "${domain}" \
                        -showcerts \
                        -verify 5 \
                        -verifyCAfile /etc/ssl/certs/ca-certificates.crt \
                        < /dev/null 2> "$temp_cert"; then
            echo "✅ Certificate chain verification successful"
        else
            echo "❌ Certificate chain verification failed"
            cat "$temp_cert"
        fi
    }
    
    # Smart SSL handler with cipher suite check
    SMART_SSL_DOWNLOAD() {
        local url="$1"
        local output="$2"
        local retry_count=0
        local max_retries=3
        
        # Check supported ciphers first
        echo "Checking supported cipher suites..."
        openssl ciphers -v | grep TLSv1.2
        
        # Try different SSL versions with retry mechanism
        for version in "tlsv1.3" "tlsv1.2" "tlsv1.1"; do
            while [[ $retry_count -lt $max_retries ]]; do
                echo "Attempting with $version (attempt $((retry_count + 1)))..."
                if curl --tlsv1.2 \
                        --tls-max "$version" \
                        --retry 3 \
                        --retry-delay 2 \
                        --connect-timeout 10 \
                        -o "$output" \
                        "$url"; then
                    echo "✅ Success with $version!"
                    rm -f "$temp_cert"
                    return 0
                fi
                retry_count=$((retry_count + 1))
                sleep 2
            done
            retry_count=0
        done
        
        echo "❌ All SSL versions failed"
        rm -f "$temp_cert"
        return 1
    }

    # Run the checks
    check_cert
    verify_chain
}

# Usage Examples
SSL_DIAGNOSTIC "https://api.github.com"
SSL_DIAGNOSTIC "https://example.com" && SMART_SSL_DOWNLOAD "https://example.com/file.zip" "download.zip"

#!/bin/bash
MASTER_TROUBLESHOOT() {
    local url="$1"
    local output="$2"
    local start_time=$(date +%s)
    local temp_log=$(mktemp)
    
    echo "🔄 Starting comprehensive diagnosis..."
    
    # Parameter validation
    if [[ -z "$url" || -z "$output" ]]; then
        echo "❌ Usage: MASTER_TROUBLESHOOT <url> <output_path>"
        rm -f "$temp_log"
        return 1
    }
    
    # Step 1: Quick connectivity check with timeout
    echo "Step 1: Basic connectivity"
    if ! timeout 10 curl -Is "$url" &> "$temp_log"; then
        echo "❌ Connectivity check failed"
        cat "$temp_log"
        NETWORK_DIAGNOSIS "$url"
        rm -f "$temp_log"
        return 1
    fi
    
    # Step 2: SSL verification with protocol detection
    echo "Step 2: SSL verification"
    if [[ "$url" =~ ^https:// ]]; then
        if ! curl -Iv "$url" 2>&1 | grep -q "SSL connection"; then
            echo "❌ SSL verification failed"
            SSL_DIAGNOSTIC "$url"
            rm -f "$temp_log"
            return 1
        fi
    fi
    
    # Step 3: Permission check with space verification
    echo "Step 3: Permission verification"
    local target_dir=$(dirname "$output")
    if ! PERMISSION_CHECK "$target_dir"; then
        # Check if we have at least 1GB free space
        if ! df -P "$target_dir" | awk 'NR==2 {exit($4<1048576)}'; then
            echo "⚠️ Warning: Less than 1GB free space available"
        fi
        rm -f "$temp_log"
        return 1
    fi
    
    # Step 4: Trial download with resume capability
    echo "Step 4: Test download"
    if ! curl -sS --fail \
              --max-time 10 \
              --retry 3 \
              --retry-delay 2 \
              -C - \
              -r 0-1024 \
              "$url" -o "$temp_log"; then
        echo "❌ Download failed! Full diagnosis needed..."
        DIAGNOSE_DOWNLOAD "$url"
        rm -f "$temp_log"
        return 1
    fi
    
    # Calculate total diagnostic time
    local end_time=$(date +%s)
    local duration=$((end_time - start_time))
    
    echo "✅ All systems go! Ready for download."
    echo "📊 Diagnostic completed in $duration seconds"
    
    # Cleanup
    rm -f "$temp_log"
    return 0
}

# Usage Example
MASTER_TROUBLESHOOT "https://downloads.example.com/large-file.zip" "/path/to/output/file.zip"