Chapter 13 Exercises: Getting Data from the Web

import requests

response = requests.get('https://httpbin.org/get')

if response.status_code == 200:
    data = response.json()
    print(data)
    print(f"My IP: {data['origin']}")
else:
    print(f"Error: {response.status_code}")

import requests

params = {
    'name': 'Jordan',
    'course': 'DataScience',
    'chapter': 13
}

response = requests.get('https://httpbin.org/get',
                        params=params)
print(f"URL: {response.url}")

data = response.json()
print(f"Args: {data['args']}")

import requests

def safe_request(url, params=None):
    """Make a GET request with error handling."""
    try:
        resp = requests.get(url, params=params,
                            timeout=10)
    except requests.RequestException as e:
        print(f"Connection error: {e}")
        return None

    if resp.status_code == 200:
        return resp.json()
    elif resp.status_code == 401:
        print("Unauthorized: check your API key")
    elif resp.status_code == 404:
        print("Not found: check the URL")
    elif resp.status_code == 429:
        print("Rate limited: slow down requests")
    elif resp.status_code == 500:
        print("Server error: try again later")
    else:
        print(f"Unexpected status: {resp.status_code}")
    return None

import pandas as pd

response_data = {
    "status": "ok",
    "count": 3,
    "data": [
        {"id": 1, "name": "Station A",
         "location": {"city": "Portland", "country": "US"}},
        {"id": 2, "name": "Station B",
         "location": {"city": "Vancouver", "country": "CA"}},
        {"id": 3, "name": "Station C",
         "location": {"city": "London", "country": "UK"}}
    ]
}

# Extract the data array, then normalize
records = response_data['data']
df = pd.json_normalize(records, sep='_')
print(df)

from bs4 import BeautifulSoup

html = """..."""  # The HTML above

soup = BeautifulSoup(html, 'html.parser')
products = soup.find_all('div', class_='product')

data = []
for product in products:
    name = product.find('span', class_='name').text
    price = product.find('span', class_='price').text
    data.append({'name': name, 'price': price})

print(data)
# [{'name': 'Widget A', 'price': '$12.99'}, ...]

from bs4 import BeautifulSoup
import pandas as pd

html = """..."""  # The HTML above
soup = BeautifulSoup(html, 'html.parser')

rows = soup.find_all('tr')
headers = [th.text for th in rows[0].find_all('th')]

data = []
for row in rows[1:]:
    cells = [td.text for td in row.find_all('td')]
    data.append(cells)

df = pd.DataFrame(data, columns=headers)

tables = pd.read_html(html)
df = tables[0]

Exercise 13.12 — Pagination loop

Write a function that collects data from a paginated API. The API returns JSON like:

{"results": [...], "page": 1, "total_pages": 5}

Your function should loop through all pages, collect all results, respect a 1-second delay between requests, and return a combined DataFrame.

import requests
import pandas as pd
import time

def collect_all_pages(base_url, params=None, delay=1.0):
    all_records = []
    page = 1

    while True:
        if params is None:
            params = {}
        params['page'] = page

        resp = requests.get(base_url, params=params,
                            timeout=10)
        if resp.status_code != 200:
            print(f"Error on page {page}: "
                  f"{resp.status_code}")
            break

        data = resp.json()
        all_records.extend(data['results'])

        if page >= data['total_pages']:
            break

        page += 1
        time.sleep(delay)

    return pd.DataFrame(all_records)

Part C: Real-World Scenarios (2-star to 3-star)

Exercise 13.13 — API exploration ⭐⭐

Choose one of these real, free, no-authentication-required APIs and retrieve data from it: - https://api.open-meteo.com/v1/forecast?latitude=45.52&longitude=-122.68&current_weather=true (weather data) - https://api.agify.io?name=jordan (name-based age estimation) - https://catfact.ninja/fact (random cat facts)

Make the request, inspect the response structure, and create a pandas DataFrame from the relevant data.

import requests
import pandas as pd

resp = requests.get(
    'https://api.open-meteo.com/v1/forecast',
    params={
        'latitude': 45.52,
        'longitude': -122.68,
        'current_weather': True
    }
)

data = resp.json()
weather = pd.json_normalize(data['current_weather'])
print(weather)

Exercise 13.14 — Building a multi-city weather collector ⭐⭐⭐

Using the Open-Meteo API (no key required), write a script that: 1. Defines a list of 5 cities with their latitude and longitude 2. Loops through the cities, requesting current weather for each 3. Respects a 1-second delay between requests 4. Combines all results into a single DataFrame with a city column 5. Saves the result to a CSV file

import requests
import pandas as pd
import time

cities = {
    'Portland': (45.52, -122.68),
    'Denver': (39.74, -104.99),
    'Chicago': (41.88, -87.63),
    'Miami': (25.76, -80.19),
    'Seattle': (47.61, -122.33)
}

records = []
for city, (lat, lon) in cities.items():
    resp = requests.get(
        'https://api.open-meteo.com/v1/forecast',
        params={'latitude': lat, 'longitude': lon,
                'current_weather': True},
        timeout=10
    )
    if resp.status_code == 200:
        weather = resp.json()['current_weather']
        weather['city'] = city
        records.append(weather)
    time.sleep(1)

df = pd.DataFrame(records)
df.to_csv('weather_data.csv', index=False)
print(df)

Exercise 13.15 — Debugging a scraper ⭐⭐

A colleague wrote this scraping code, but it isn't working. Identify and fix all the bugs:

import request
from bs4 import BeautifulSoup

response = request.get('https://example.com/data')
soup = BeautifulSoup(response, 'html.parser')
rows = soup.find('tr')
for row in rows:
    cells = row.find('td')
    print(cells.text)

import requests
from bs4 import BeautifulSoup

response = requests.get('https://example.com/data')
soup = BeautifulSoup(response.text, 'html.parser')
rows = soup.find_all('tr')
for row in rows:
    cells = row.find_all('td')
    for cell in cells:
        print(cell.text)

Exercise 13.16 — API key safety ⭐⭐

Your teammate writes this code and asks you to review it before they push it to GitHub:

import requests

API_KEY = 'sk_live_a8f3k2j5h7g9d1m4n6p8'
response = requests.get(
    'https://api.dataservice.com/records',
    headers={'Authorization': f'Bearer {API_KEY}'}
)

What is the security problem? Write a corrected version that keeps the API key safe.

import requests
import os

API_KEY = os.environ.get('DATA_SERVICE_API_KEY')
if not API_KEY:
    raise ValueError(
        "Set DATA_SERVICE_API_KEY environment variable"
    )

response = requests.get(
    'https://api.dataservice.com/records',
    headers={'Authorization': f'Bearer {API_KEY}'}
)

Exercise 13.17 — robots.txt analysis ⭐⭐

Fetch and analyze the robots.txt file for three real websites (e.g., https://en.wikipedia.org/robots.txt, https://twitter.com/robots.txt, https://github.com/robots.txt). For each: 1. Which paths are disallowed? 2. Is there a crawl delay specified? 3. Based on the robots.txt, would your data collection script be allowed to access the main content pages?

import requests

sites = [
    'https://en.wikipedia.org/robots.txt',
    'https://github.com/robots.txt'
]

for url in sites:
    resp = requests.get(url, timeout=10)
    print(f"\n--- {url} ---")
    print(resp.text[:500])

Part D: Synthesis and Extension (3-star to 4-star)

Exercise 13.18 — Complete data pipeline ⭐⭐⭐

Build a complete data collection pipeline that: 1. Fetches data from a free public API 2. Handles errors and timeouts 3. Converts the response to a pandas DataFrame 4. Cleans the data (rename columns, fix types) 5. Saves to CSV 6. Includes an ethics documentation cell

Document each step in a Jupyter notebook with Markdown explanations.

Exercise 13.19 — Scraping vs. API comparison ⭐⭐⭐

Find a website that provides data both through a web page and through an API (many government data portals do this). Write two scripts: 1. One that scrapes the data from the web page using BeautifulSoup 2. One that retrieves the same data from the API using requests

Compare the two approaches: Which was easier? Which produced cleaner data? Which is more likely to break if the website changes?

Exercise 13.20 — Retry logic with exponential backoff ⭐⭐⭐⭐

Write a function request_with_backoff(url, max_retries=5) that implements exponential backoff: if a request fails, wait 1 second before retrying, then 2 seconds, then 4, then 8, then 16. This is a common pattern in production data pipelines. Explain why exponential backoff is better than a fixed retry delay.

import requests
import time

def request_with_backoff(url, params=None,
                         max_retries=5):
    for attempt in range(max_retries):
        try:
            resp = requests.get(url, params=params,
                                timeout=10)
            if resp.status_code == 200:
                return resp
            elif resp.status_code == 429:
                wait = 2 ** attempt
                print(f"Retry {attempt + 1}: "
                      f"waiting {wait}s")
                time.sleep(wait)
            else:
                return resp
        except requests.RequestException:
            wait = 2 ** attempt
            print(f"Connection error. "
                  f"Retry in {wait}s")
            time.sleep(wait)
    return None

Exercise 13.21 — Caching for repeated API calls ⭐⭐⭐⭐

Write a function that wraps API calls with local file caching. If the data was fetched within the last 24 hours, return the cached version instead of making a new request. Use file modification timestamps to determine freshness.

import requests
import json
import os
import time

def cached_request(url, cache_path, max_age=86400):
    """Fetch from API or return cached data."""
    if os.path.exists(cache_path):
        age = time.time() - os.path.getmtime(cache_path)
        if age < max_age:
            with open(cache_path, 'r') as f:
                print("Using cached data")
                return json.load(f)

    resp = requests.get(url, timeout=10)
    if resp.status_code == 200:
        data = resp.json()
        with open(cache_path, 'w') as f:
            json.dump(data, f)
        return data
    return None

Part E: Ethics and Judgment (All levels)

These problems don't have single right answers. They require ethical reasoning and judgment.

Exercise 13.22 — Ethical scenario analysis ⭐⭐

For each scenario, decide whether the data collection is ethical, unethical, or "it depends." Justify your reasoning.

1. A student scrapes public restaurant health inspection scores from a city government website to build a searchable database.
2. A company scrapes competitor pricing from e-commerce sites every hour to dynamically adjust their own prices.
3. A researcher scrapes social media profiles to study mental health patterns, using publicly posted data.
4. A journalist scrapes public court records to investigate patterns in sentencing.
5. A startup scrapes millions of images from personal blogs to train an AI model.

Part M: Mixed Review (Chapters 1-12)

Exercise 13.23 — Review: Data types (Chapter 7) ⭐⭐

After fetching JSON data from an API and loading it into a DataFrame, you notice that a population column has dtype object instead of int64. What are two possible causes and how would you fix each?

Exercise 13.24 — Review: Merge patterns (Chapter 9) ⭐⭐

You've collected data from two different APIs: one returns country-level COVID data and the other returns country-level economic data. The first uses a column called iso_code and the second uses country_code. Write the merge code and explain which how parameter you'd choose and why.

merged = covid_df.merge(
    econ_df,
    left_on='iso_code',
    right_on='country_code',
    how='inner'
)

Exercise 13.25 — Review: Dates (Chapter 11) + APIs ⭐⭐⭐

You fetch time series data from an API. The dates come as Unix timestamps (integers representing seconds since January 1, 1970). Write code to: 1. Convert the timestamp column to pandas datetime 2. Set it as the index 3. Resample to weekly frequency

# Convert Unix timestamps to datetime
df['date'] = pd.to_datetime(df['timestamp'], unit='s')

# Set as index
df = df.set_index('date')

# Resample to weekly
weekly = df.resample('W').mean()
print(weekly.head())