Add `robots.txt` tests about xil HOT 1 CLOSED

"""
Visited URLs should be allowed to visit according to the robots.txt file.

See also: http://www.robotstxt.org/orig.html
"""
from urllib.robotparser import RobotFileParser
from urllib.parse import urlsplit, urljoin

import pytest

from xil.boi import _BOI_URL, get_boi_url_response


ROBOTS_TXT = "robots.txt"


robots_url = urljoin("{0.scheme}://{0.netloc}".format(urlsplit(_BOI_URL)), ROBOTS_TXT)


@pytest.fixture(name="user_agent")
def fixture_user_agent() -> str:
    return "*"


def test_boi(user_agent: str) -> None:
    rp = RobotFileParser()
    with get_boi_url_response(robots_url) as response:  #
        rp.parse(response.read().decode().splitlines())
    assert rp.entries
    assert rp.can_fetch(user_agent, _BOI_URL)