Here are the examples of the python api scrapy.utils.python.to_native_str.splitlines taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
2 Examples
0
Source : robotstxt.py
with MIT License
from autofelix
with MIT License
from autofelix
def _parse_robots(self, response, netloc):
self.crawler.stats.inc_value('robotstxt/response_count')
self.crawler.stats.inc_value(
'robotstxt/response_status_count/{}'.format(response.status))
rp = robotparser.RobotFileParser(response.url)
body = ''
if hasattr(response, 'text'):
body = response.text
else: # last effort try
try:
body = response.body.decode('utf-8')
except UnicodeDecodeError:
# If we found garbage, disregard it:,
# but keep the lookup cached (in self._parsers)
# Running rp.parse() will set rp state from
# 'disallow all' to 'allow any'.
self.crawler.stats.inc_value('robotstxt/unicode_error_count')
# stdlib's robotparser expects native 'str' ;
# with unicode input, non-ASCII encoded bytes decoding fails in Python2
rp.parse(to_native_str(body).splitlines())
rp_dfd = self._parsers[netloc]
self._parsers[netloc] = rp
rp_dfd.callback(rp)
def _robots_error(self, failure, netloc):
0
Source : robotstxt.py
with Apache License 2.0
from lynings
with Apache License 2.0
from lynings
def _parse_robots(self, response, netloc):
rp = robotparser.RobotFileParser(response.url)
body = ''
if hasattr(response, 'text'):
body = response.text
else: # last effort try
try:
body = response.body.decode('utf-8')
except UnicodeDecodeError:
# If we found garbage, disregard it:,
# but keep the lookup cached (in self._parsers)
# Running rp.parse() will set rp state from
# 'disallow all' to 'allow any'.
pass
# stdlib's robotparser expects native 'str' ;
# with unicode input, non-ASCII encoded bytes decoding fails in Python2
rp.parse(to_native_str(body).splitlines())
rp_dfd = self._parsers[netloc]
self._parsers[netloc] = rp
rp_dfd.callback(rp)
def _robots_error(self, failure, netloc):