HTTP Requests & Web Scraping

Use the requests library and BeautifulSoup to make HTTP calls, scrape web content, probe APIs, and automate form submissions.

Medium 65m 3 tasks

Prerequisites: Networking with Sockets

Learning Objectives

→ Install and use the requests library for HTTP GET/POST
→ Inspect response status codes, headers, and body
→ Parse HTML with BeautifulSoup to extract links and data
→ Submit forms and handle sessions/cookies
→ Understand rate limiting and ethical scraping

The requests Library

Install once per venv:

pip install requests beautifulsoup4

GET Requests

import requests

r = requests.get("https://httpbin.org/get", timeout=5)
print(r.status_code)        # 200
print(r.headers["Content-Type"])
print(r.text[:300])
print(r.json())             # if response is JSON

Query Parameters

params = {"q": "site:example.com", "limit": 10}
r = requests.get("https://httpbin.org/get", params=params)
print(r.url)   # URL with ?q=site%3Aexample.com&limit=10

Custom Headers

headers = {
    "User-Agent": "Mozilla/5.0 (security-scanner/1.0)",
    "Accept":     "application/json",
}
r = requests.get("https://httpbin.org/headers", headers=headers)

POST Requests

# Form data
data = {"username": "admin", "password": "test"}
r = requests.post("https://httpbin.org/post", data=data)

# JSON body
payload = {"action": "scan", "target": "192.168.1.1"}
r = requests.post("https://api.example.com/scan", json=payload)

Status Codes

Code	Meaning
200	OK
301	Redirect (permanent)
401	Unauthorized
403	Forbidden
404	Not Found
500	Server Error

if r.status_code == 200:
    print("Success")
elif r.status_code == 401:
    print("Authentication required")
r.raise_for_status()   # raises HTTPError for 4xx/5xx

Sessions (Cookies)

s = requests.Session()
s.headers.update({"User-Agent": "Tester/1.0"})

# Login
s.post("https://example.com/login", data={"user": "test", "pass": "test"})
# Now subsequent requests carry the session cookie
r = s.get("https://example.com/dashboard")

BeautifulSoup

from bs4 import BeautifulSoup

r = requests.get("https://example.com")
soup = BeautifulSoup(r.text, "html.parser")

# Find elements
title = soup.find("title").text
links = [a["href"] for a in soup.find_all("a", href=True)]
forms = soup.find_all("form")
inputs = soup.find_all("input", {"type": "hidden"})

print(title)
print("Links found:", len(links))

Link Crawler (Simple)

import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin

def crawl(base_url, max_pages=10):
    visited = set()
    to_visit = [base_url]
    while to_visit and len(visited) < max_pages:
        url = to_visit.pop(0)
        if url in visited:
            continue
        try:
            r = requests.get(url, timeout=3)
            visited.add(url)
            soup = BeautifulSoup(r.text, "html.parser")
            for a in soup.find_all("a", href=True):
                link = urljoin(base_url, a["href"])
                if link.startswith(base_url) and link not in visited:
                    to_visit.append(link)
        except Exception:
            pass
    return visited

# pages = crawl("https://example.com")

Mini Project – Directory Bruteforcer

import requests

def dir_brute(base_url, wordlist, threads=10):
    found = []
    for word in wordlist:
        url = f"{base_url.rstrip('/')}/{word}"
        try:
            r = requests.get(url, timeout=2, allow_redirects=False)
            if r.status_code in (200, 301, 302, 403):
                print(f"  [{r.status_code}] {url}")
                found.append(url)
        except requests.RequestException:
            pass
    return found

words = ["admin", "login", "backup", "config", "api", "uploads", ".env"]
found = dir_brute("https://httpbin.org", words)
print(f"Found: {len(found)} paths")