Nicer APIs for urllib.request

Ideas
1

Hi Folks,

I have been looking at stdlib urllib.reqeust apis and they seem a little not nicer, since I am used to requests. So I was wondering if having similar APIs in the stdlib would be any improvement for user experience. I’m sure people reach for requests for anything production most of the time, regardless I think it would be nice to have nicer APIs in the stdlib, since the basic machinery is already there. I have a basic snippet for demo if it helps.

Thank you for taking a look.

import urllib.request
import http
import json as jsonlib


class Response:
    def __init__(self, url, content, status_code, headers, reason=""):
        self.url = url
        self.content = content
        self.status_code = status_code
        self.headers = headers if headers else {}
        self.reason = reason


def get(url, **kwargs):
    headers = kwargs.get("headers", {})
    return _request(
            url,
            method="GET",
            headers=headers,
            **kwargs
    )


def post(url, data="", json=None, **kwargs):
    headers = kwargs.get("headers", {})

    if json:
        data = jsonlib.dumps(json)
        headers["Content-Type"] = "application/json"

    return _request(
            url,
            method="POST",
            data=data,
            json=json,
            headers=headers,
            **kwargs
    )


def _request(url, method="GET", data="", json=None, headers={}, **kwargs):
    headers.setdefault("User-Agent", "Python-urllib/3")

    timeout = kwargs.pop("timeout", None)
    if method == "GET":
        req = urllib.request.Request(url, headers=headers, **kwargs)
    else:
        req = urllib.request.Request(
                url,
                data=data.encode("utf-8"),
                headers=headers,
                method=method,
                **kwargs
        )
    
    try:
        with urllib.request.urlopen(req, timeout=timeout) as f:
            content = f.read()
            headers = dict(f.headers)
        res = Response(
                url=url,
                content=content,
                status_code=http.HTTPStatus(f.code),
                headers=headers,
                **kwargs
        )
        return res
    except urllib.error.HTTPError as err:
        res = Response(
                url=url,
                content=b"",
                status_code=http.HTTPStatus(err.code),
                headers=dict(err.headers),
                reason=err.reason
        )
        return res
    # happens when the handshake fails due to timeout and maybe other scenarios
    # https://github.com/python/cpython/blob/3.14/Lib/urllib/request.py#L1323
    # which is unfortunate, because this exception is raised for `socket.gaierror`
    # or unknow url protocol etc
    # TODO: maybe we can filter the exception and narrow it down to raise better exceptions
    except urllib.error.URLError as err:
        raise RequestException(err) from None
    except TimeoutError as err:
        raise RequestTimeoutError(f"Request timed out after {timeout} seconds: {err}") from None


class RequestTimeoutError(Exception):
    pass

class RequestException(Exception):
    pass

Usage:

from requests import get, post


def print_response(res):
    print(f"[{res.url}]")
    print(f"\t{res.status_code=}")
    print(f"\t{res.headers=}")
    print(f"\t{res.content=}")
    print(f"\t{res.reason=}")


res = get("https://httpbin.org/json")
print_response(res)
res = get("https://httpbin.org/status/400")
print_response(res)
res = post("https://httpbin.org/post", data='Hello=World')
print_response(res)
res = post("https://httpbin.org/post", json={"x": 42}, timeout=5)
print_response(res)

I’m sure I have overlooked most aspects and am mostly considering for internal usage, so not sure how much it warrants the burden as well. But thought I’d just put it out there anyway in case someone had any similar thoughts.

1 Like