requests_middleware.contrib.robotware.RobotsMiddleware

Here are the examples of the python api requests_middleware.contrib.robotware.RobotsMiddleware taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

6 Examples 7

Example 1

Project: requests-middleware
Source File: test_multi.py
View license
@pytest.fixture
def session():
    session = requests.Session()
    robots_middleware = robotware.RobotsMiddleware()
    ssl_middleware = sslware.SSLMiddleware(ssl.PROTOCOL_TLSv1)
    middlewares = [robots_middleware, ssl_middleware]
    adapter = MiddlewareHTTPAdapter(middlewares=middlewares)
    session.mount('http://', adapter)
    session.mount('https://', adapter)
    return session

Example 2

Project: requests-middleware
Source File: test_multi.py
View license
@pytest.fixture
def session():
    session = requests.Session()
    robots_middleware = robotware.RobotsMiddleware()
    ssl_middleware = sslware.SSLMiddleware(ssl.PROTOCOL_TLSv1)
    middlewares = [robots_middleware, ssl_middleware]
    adapter = MiddlewareHTTPAdapter(middlewares=middlewares)
    session.mount('http://', adapter)
    session.mount('https://', adapter)
    return session

Example 3

Project: requests-middleware
Source File: test_robots.py
View license
@pytest.fixture
def session():
    session = requests.Session()
    session.headers['User-Agent'] = 'robot'
    adapter = MiddlewareHTTPAdapter()
    robots_middleware = robotware.RobotsMiddleware()
    adapter.register(robots_middleware)
    session.mount('http://', adapter)
    session.mount('https://', adapter)
    return session

Example 4

Project: requests-middleware
Source File: test_robots.py
View license
def make_middleware_fixture(rules):
    @pytest.fixture
    def fixture():
        middleware = robotware.RobotsMiddleware()
        middleware.cache.add(
            Rules(
                'http://test.com/robots.txt',
                httplib.OK,
                rules,
                time.time() * 2,
            )
        )
        return middleware
    return fixture

Example 5

Project: requests-middleware
Source File: test_robots.py
View license
@pytest.fixture
def session():
    session = requests.Session()
    session.headers['User-Agent'] = 'robot'
    adapter = MiddlewareHTTPAdapter()
    robots_middleware = robotware.RobotsMiddleware()
    adapter.register(robots_middleware)
    session.mount('http://', adapter)
    session.mount('https://', adapter)
    return session

Example 6

Project: requests-middleware
Source File: test_robots.py
View license
def make_middleware_fixture(rules):
    @pytest.fixture
    def fixture():
        middleware = robotware.RobotsMiddleware()
        middleware.cache.add(
            Rules(
                'http://test.com/robots.txt',
                httplib.OK,
                rules,
                time.time() * 2,
            )
        )
        return middleware
    return fixture