Here are the examples of the python api ruia.Request taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
11 Examples
3
Source : hacker_news.py
with Apache License 2.0
from howie6879
with Apache License 2.0
from howie6879
async def parse(self, response):
self.mongo_db = MotorBase().get_db("ruia_test")
urls = [
"https://news.ycombinator.com/news?p=1",
"https://news.ycombinator.com/news?p=2",
]
for index, url in enumerate(urls):
yield Request(url, callback=self.parse_item, metadata={"index": index})
async def parse_item(self, response):
3
Source : request.py
with Apache License 2.0
from howie6879
with Apache License 2.0
from howie6879
async def request_example():
url = "https://httpbin.org/get"
params = {"name": "ruia"}
headers = {"User-Agent": "Python3.6"}
request = Request(url=url, method="GET", params=params, headers=headers)
response = await request.fetch()
res = await response.json()
assert res["args"]["name"] == "ruia"
assert res["headers"]["User-Agent"] == "Python3.6"
def test_request():
3
Source : test_request.py
with Apache License 2.0
from howie6879
with Apache License 2.0
from howie6879
async def make_post_request(sem, callback):
headers = {"Content-Type": "application/json"}
request = Request(
"https://httpbin.org/post",
method="POST",
headers=headers,
data=params,
callback=callback,
)
return await request.fetch_callback(sem)
def test_request_config():
3
Source : test_request.py
with Apache License 2.0
from howie6879
with Apache License 2.0
from howie6879
def test_request_config():
assert str(Request("https://httpbin.org/")) == " < GET https://httpbin.org/>"
_, response = asyncio.get_event_loop().run_until_complete(
make_get_request(sem=sem, callback=hello)
)
# assert response.callback_result == "hello ruia"
assert response.metadata == {"hello": "ruia"}
json_result = asyncio.get_event_loop().run_until_complete(response.json())
assert json_result["args"]["name"] == "ruia"
_, response = asyncio.get_event_loop().run_until_complete(
make_post_request(sem=sem, callback=None)
)
json_result = asyncio.get_event_loop().run_until_complete(response.json())
assert json_result["data"] == "name=ruia"
def test_method_error_request():
3
Source : test_request.py
with Apache License 2.0
from howie6879
with Apache License 2.0
from howie6879
def test_method_error_request():
try:
request = Request("https://httpbin.org/", method="PUT")
response = asyncio.get_event_loop().run_until_complete(request.fetch())
res = asyncio.get_event_loop().run_until_complete(response.text())
assert res == ""
except Exception as e:
assert isinstance(e, InvalidRequestMethod)
def test_sem_error_request():
3
Source : test_request.py
with Apache License 2.0
from howie6879
with Apache License 2.0
from howie6879
def test_retry_request():
request = Request("http://httpbin.org/404")
_, response = asyncio.get_event_loop().run_until_complete(
request.fetch_callback(sem=sem)
)
assert response.url == "http://httpbin.org/404"
def test_retry_delay():
3
Source : test_request.py
with Apache License 2.0
from howie6879
with Apache License 2.0
from howie6879
def test_retry_delay():
# Test invalid URL (to trigger retries) with 1s delay between retries
request_config = {"RETRIES": 2, "RETRY_DELAY": 1}
request = Request("http://127.0.0.1:5999/", request_config=request_config)
# Start a timer to time retries
timer = time.time()
_, response = asyncio.get_event_loop().run_until_complete(
request.fetch_callback(sem=sem)
)
# Ensure that for 2 retries the time taken is > 2s (1s between each retry)
assert time.time() - timer > 2
def test_delay_false():
3
Source : test_request.py
with Apache License 2.0
from howie6879
with Apache License 2.0
from howie6879
def test_delay_false():
request_config = {"DELAY": 10}
request = Request("https://httpbin.org/", request_config=request_config)
# Start a timer to time request
timer = time.time()
response = asyncio.get_event_loop().run_until_complete(request.fetch(delay=False))
# Ensure delay option was ignored (time taken is less than 10s)
assert time.time() - timer < 10
def test_timeout_request():
3
Source : test_request.py
with Apache License 2.0
from howie6879
with Apache License 2.0
from howie6879
def test_timeout_request():
async def timeout_request(sem):
request_config = {"RETRIES": 1, "DELAY": 1, "TIMEOUT": 0.1}
request = Request(
"https://httpbin.org/get",
method="GET",
metadata={"hello": "ruia"},
encoding="utf-8",
request_config=request_config,
params=params,
callback=hi,
)
return await request.fetch_callback(sem)
_, response = asyncio.get_event_loop().run_until_complete(timeout_request(sem=sem))
assert "https://httpbin.org/get" in response.url
3
Source : test_spider.py
with Apache License 2.0
from howie6879
with Apache License 2.0
from howie6879
async def parse(self, response):
yield Request(
url=response.url,
callback=self.parse_item,
headers=self.headers,
request_config=self.request_config,
**self.aiohttp_kwargs,
)
async def parse_item(self, response):
0
Source : test_request.py
with Apache License 2.0
from howie6879
with Apache License 2.0
from howie6879
async def make_get_request(sem, callback=None):
request_config = {
"RETRIES": 3,
"DELAY": 1,
"TIMEOUT": 0.1,
"VALID": valid_response,
"RETRY_FUNC": retry_func,
}
request = Request(
"https://httpbin.org/get",
method="GET",
metadata={"hello": "ruia"},
request_config=request_config,
params=params,
callback=callback,
)
return await request.fetch_callback(sem)
async def make_post_request(sem, callback):