requests.packages.urllib3.connectionpool.HTTPConnectionPool

Here are the examples of the python api requests.packages.urllib3.connectionpool.HTTPConnectionPool taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

14 Examples 7

Example 1

Project: grr
Source File: data_server.py
View license
  def __init__(self, my_port, handler_cls):
    servers = config_lib.CONFIG["Dataserver.server_list"]
    if not servers:
      raise errors.DataServerError("List of data servers not available.")
    master_location = servers[0]
    loc = urlparse.urlparse(master_location, scheme="http")
    self.index = None
    self.master_addr = loc.hostname
    self.master_port = loc.port
    self.my_port = my_port
    self.pool = connectionpool.HTTPConnectionPool(
        self.master_addr, port=int(self.master_port), maxsize=1)
    self.registered = False
    self.periodic_fail = 0

    self.handler_cls = handler_cls

Example 2

Project: grr
Source File: manager.py
View license
  def __init__(self):
    servers = config_lib.CONFIG["Dataserver.server_list"]
    if not servers:
      raise errors.DataServerError("List of data servers not available.")
    master_location = servers[0]
    loc = urlparse.urlparse(master_location, scheme="http")
    self.addr = loc.hostname
    self.port = int(loc.port)
    self.pool = connectionpool.HTTPConnectionPool(self.addr, port=self.port)
    self.history_path = os.path.expanduser("~/.grr-data-store-manager")
    if os.path.exists(self.history_path):
      readline.read_history_file(self.history_path)
    self.periodic_thread = None
    self.mapping = None
    self.mapping_time = 0

Example 3

Project: grr
Source File: master.py
View license
  def SetRebalancing(self, reb):
    """Sets a new rebalance operation and starts communication with servers."""
    self.rebalance = reb
    self.rebalance_pool = []
    try:
      for serv in self.servers:
        pool = connectionpool.HTTPConnectionPool(
            serv.Address(), port=serv.Port())
        self.rebalance_pool.append(pool)
    except urllib3.exceptions.MaxRetryError:
      self.CancelRebalancing()
      return False
    return True

Example 4

Project: pex
Source File: test_http.py
View license
@pytest.mark.skipif(NO_REQUESTS)
def test_requests_context_retries_connect_timeout():
  with mock.patch.object(
      requests.packages.urllib3.connectionpool.HTTPConnectionPool,
      '_make_request') as mock_make_request:

    url, mock_make_request.side_effect = timeout_side_effect()

    context = RequestsContext(verify=False)

    data = context.read(Link.wrap(url))
    assert data == BLOB

Example 5

Project: pex
Source File: test_http.py
View license
@pytest.mark.skipif(NO_REQUESTS)
def test_requests_context_retries_connect_timeout_retries_exhausted():
  with mock.patch.object(
      requests.packages.urllib3.connectionpool.HTTPConnectionPool,
      '_make_request') as mock_make_request:

    url, mock_make_request.side_effect = timeout_side_effect(num_timeouts=3)
    env = Variables(environ={'PEX_HTTP_RETRIES': '2'})

    context = RequestsContext(verify=False, env=env)

    with pytest.raises(Context.Error):
      context.read(Link.wrap(url))

Example 6

Project: pex
Source File: test_http.py
View license
@pytest.mark.skipif(NO_REQUESTS)
def test_requests_context_retries_read_timeout():
  with mock.patch.object(
      requests.packages.urllib3.connectionpool.HTTPConnectionPool,
      '_make_request') as mock_make_request:

    url, mock_make_request.side_effect = timeout_side_effect(
        timeout_error=requests.packages.urllib3.exceptions.ReadTimeoutError)

    context = RequestsContext(verify=False)

    data = context.read(Link.wrap(url))
    assert data == BLOB

Example 7

Project: pex
Source File: test_http.py
View license
@pytest.mark.skipif(NO_REQUESTS)
def test_requests_context_retries_read_timeout_retries_exhausted():
  with mock.patch.object(
      requests.packages.urllib3.connectionpool.HTTPConnectionPool,
      '_make_request') as mock_make_request:

    url, mock_make_request.side_effect = timeout_side_effect(
        timeout_error=requests.packages.urllib3.exceptions.ReadTimeoutError,
        num_timeouts=3)
    env = Variables(environ={'PEX_HTTP_RETRIES': '2'})

    context = RequestsContext(verify=False, env=env)

    with pytest.raises(Context.Error):
      context.read(Link.wrap(url))

Example 8

Project: grr
Source File: manager.py
View license
  def _DoRebalance(self, new_mapping):
    """Performs a new rebalancing operation with the master server."""
    print "Contacting master server to start re-sharding...",
    # Send mapping information to master.
    pool = None
    try:
      pool = connectionpool.HTTPConnectionPool(self.addr, port=self.port)
    except urllib3.exceptions.MaxRetryError:
      print "Unable to contact master..."
      return
    body = new_mapping.SerializeToString()
    headers = {"Content-Length": len(body)}
    res = None
    try:
      res = pool.urlopen(
          "POST", "/rebalance/phase1", headers=headers, body=body)
    except urllib3.exceptions.MaxRetryError:
      print "Unable to talk with master..."
      pool.close()
      return
    if res.status != constants.RESPONSE_OK:
      print "Re-sharding cannot be done!"
      return
    rebalance = rdf_data_server.DataServerRebalance(res.data)
    print "OK"
    print
    print "The following servers will need to move data:"
    for i, move in enumerate(list(rebalance.moving)):
      print "Server %d moves %dKB" % (i, move / 1024)
    answer = raw_input("Proceed with re-sharding? (y/n) ")
    if answer != "y":
      return
    body = rebalance.SerializeToString()
    headers = {"Content-Length": len(body)}
    try:
      res = pool.urlopen(
          "POST", "/rebalance/phase2", headers=headers, body=body)
    except urllib3.exceptions.MaxRetryError:
      print "Unable to contact server for re-sharding."
      print "Make sure the data servers are up and try again."
      return
    if res.status != constants.RESPONSE_OK:
      print "Could not start copying files for re-sharding"
      print "Make sure the data servers are up and try again."
      return

    try:
      res = pool.urlopen(
          "POST", "/rebalance/commit", headers=headers, body=body)
    except urllib3.exceptions.MaxRetryError:
      print("Could not commit the re-sharding transaction with id "
            "%s") % rebalance.id
      print "Make sure the data servers are up and then run:"
      print "'recover %s' in order to re-run transaction" % rebalance.id
      return

    if res.status != constants.RESPONSE_OK:
      print "Could not commit the transaction %s" % rebalance.id
      print "Make sure the data servers are up and then run:"
      print "'recover %s' in order to re-run transaction" % rebalance.id
      return

    self.mapping = rdf_data_server.DataServerMapping(res.data)

    print "Rebalance with id %s fully performed." % rebalance.id

Example 9

Project: grr
Source File: manager.py
View license
  def _Recover(self, transid):
    """Completes a rebalancing transaction that was unsuccessful."""
    print "Contacting master about transaction %s..." % transid,
    pool = None
    try:
      pool = connectionpool.HTTPConnectionPool(self.addr, port=self.port)
    except urllib3.exceptions.MaxRetryError:
      print "Unable to contact master..."
      return

    print "OK."

    try:
      body = transid
      headers = {"Content-Length": len(body)}
      res = pool.urlopen(
          "POST", "/rebalance/recover", headers=headers, body=body)
    except urllib3.exceptions.MaxRetryError:
      print "Unable to contact master..."
      return

    if res.status == constants.RESPONSE_TRANSACTION_NOT_FOUND:
      print "Transaction %s was not found" % transid
      return
    if res.status != constants.RESPONSE_OK:
      print "Potential data master error. Giving up..."
      return
    rebalance = rdf_data_server.DataServerRebalance(res.data)
    print "Got transaction object %s" % rebalance.id
    answer = raw_input("Proceed with the recover process? (y/n) ")
    if answer != "y":
      return

    body = rebalance.SerializeToString()
    headers = {"Content-Length": len(body)}

    try:
      res = pool.urlopen(
          "POST", "/rebalance/commit", headers=headers, body=body)
    except urllib3.exceptions.MaxRetryError:
      print "Could not commit re-sharding transaction with id %s" % rebalance.id
      print "Make sure the data servers are up and then run:"
      print "'recover %s' in order to re-run transaction" % rebalance.id
      return

    if res.status != constants.RESPONSE_OK:
      print "Could not commit transaction %s" % rebalance.id
      print "Make sure the data servers are up and then run:"
      print "'recover %s' in order to re-run transaction" % rebalance.id
      return

    self.mapping = rdf_data_server.DataServerMapping(res.data)
    print "Rebalance with id %s fully performed." % rebalance.id

Example 10

Project: grr
Source File: manager.py
View license
  def _AddServer(self, addr, port):
    """Starts the process of adding a new server."""
    if port <= 0:
      print "Wrong port: %d" % port
      return
    pool = None
    try:
      pool = connectionpool.HTTPConnectionPool(self.addr, port=self.port)
    except urllib3.exceptions.MaxRetryError:
      print "Unable to contact master..."
      return

    body = self._PackNewServer(addr, port)
    headers = {"Content-Length": len(body)}
    try:
      res = pool.urlopen(
          "POST", "/servers/add/check", headers=headers, body=body)
    except urllib3.exceptions.MaxRetryError:
      print "Unable to contact master..."
      return

    if res.status == constants.RESPONSE_EQUAL_DATA_SERVER:
      print "Master server says there is already a similar server."
      print "Giving up..."
      return

    if res.status == constants.RESPONSE_DATA_SERVERS_UNREACHABLE:
      print "Master server says that some data servers are not running."
      print "Giving up..."
      return

    if res.status != constants.RESPONSE_OK:
      print "Master server error. Is the server running?"
      return

    print "Master server allows us to add server %s:%d" % (addr, port)

    answer = raw_input("Do you really want to add server //%s:%d? (y/n) " %
                       (addr, port))
    if answer != "y":
      return

    try:
      res = pool.urlopen("POST", "/servers/add", headers=headers, body=body)
    except urllib3.exceptions.MaxRetryError:
      print "Unable to contact master..."
      return

    if res.status == constants.RESPONSE_DATA_SERVERS_UNREACHABLE:
      print "Master server says that some data servers are not running."
      print "Giving up..."
      return

    if res.status == constants.RESPONSE_INCOMPLETE_SYNC:
      print("The master server has set up the new server, but the other "
            "servers may not know about it.")
      print "Please run 'sync' to fix the problem."
      print "Afterwards, you have to rebalance server data with the following:"
      self._CompleteAddServerHelp(addr, port)
      return

    if res.status != constants.RESPONSE_OK:
      print "Failed to contact master server."
      return

    print "============================================="
    print "Operation completed."
    print "To rebalance server data you have to do the following:"
    self._CompleteAddServerHelp(addr, port)

    # Update mapping.
    self.mapping = rdf_data_server.DataServerMapping(res.data)

Example 11

Project: grr
Source File: manager.py
View license
  def _Sync(self):
    """Forces the master to sync with the other data servers."""
    pool = None
    try:
      pool = connectionpool.HTTPConnectionPool(self.addr, port=self.port)
      body = ""
      headers = {"Content-Length": len(body)}
      res = pool.urlopen(
          "POST", "/servers/sync-all", headers=headers, body=body)

      if res.status == constants.RESPONSE_INCOMPLETE_SYNC:
        print "Master has tried to contact all the data servers, but failed."
        return False

      if res.status == constants.RESPONSE_DATA_SERVERS_UNREACHABLE:
        print "Master server says that some data servers are not running."
        print "Giving up..."
        return False

      if res.status != constants.RESPONSE_OK:
        print "Unable to sync servers."
        return False
    except urllib3.exceptions.MaxRetryError:
      print "Unable to contact master..."
      return False
    print "Sync done."
    # Update mapping.
    self.mapping = rdf_data_server.DataServerMapping(res.data)
    return True

Example 12

Project: grr
Source File: manager.py
View license
  def _RemServer(self, addr, port):
    """Remove server from group."""
    # Find server.
    server, _ = self._FindServer(addr, port)
    if not server:
      print "Server not found."
      return
    if server.interval.start != server.interval.end:
      print "Server has some data in it!"
      print "Giving up..."
      return

    pool = None
    try:
      pool = connectionpool.HTTPConnectionPool(self.addr, port=self.port)
    except urllib3.exceptions.MaxRetryError:
      print "Unable to contact master..."
      return

    body = self._PackNewServer(addr, port)
    headers = {"Content-Length": len(body)}
    try:
      res = pool.urlopen(
          "POST", "/servers/rem/check", headers=headers, body=body)
    except urllib3.exceptions.MaxRetryError:
      print "Unable to contact master..."
      return

    if res.status == constants.RESPONSE_DATA_SERVER_NOT_FOUND:
      print "Master server says the data server does not exist."
      return

    if res.status == constants.RESPONSE_RANGE_NOT_EMPTY:
      print "Master server says the data server has still some data."
      print "Giving up..."
      return

    if res.status == constants.RESPONSE_DATA_SERVERS_UNREACHABLE:
      print "Master server says some data servers are not running."
      print "Giving up..."
      return

    if res.status != constants.RESPONSE_OK:
      print "Master server error. Is the server running?"
      return

    print "Master server allows us to remove server %s:%d" % (addr, port)

    answer = raw_input("Do you really want to remove server //%s:%d? (y/n) " %
                       (addr, port))
    if answer != "y":
      return

    try:
      res = pool.urlopen("POST", "/servers/rem", headers=headers, body=body)
    except urllib3.exceptions.MaxRetryError:
      print "Unable to contact master..."
      return

    if res.status == constants.RESPONSE_DATA_SERVERS_UNREACHABLE:
      print "Master server says that some data servers are not running."
      print "Giving up..."
      return

    if res.status == constants.RESPONSE_OK:
      # Update mapping.
      self.mapping = rdf_data_server.DataServerMapping(res.data)
      self._CompleteRemServerHelpComplete(addr, port)
      return

    if res.status == constants.RESPONSE_INCOMPLETE_SYNC:
      # We were unable to sync, so we try again:
      if self._Sync():
        self._CompleteRemServerHelpComplete(addr, port)
        return
      else:
        # If we cannot sync in the second attempt, we give up.
        print("The master server has removed the new server, but the other "
              "servers may not know about it.")

        print "Please run 'sync' to fix the problem, followed by:"
        self._CompleteRemServerHelp(addr, port)
        return

    if res.status != constants.RESPONSE_OK:
      print "Master has returned an unknown error..."
      return

Example 13

Project: grr
Source File: master.py
View license
  def SyncMapping(self, skip=None):
    """Syncs mapping with other servers."""
    pools = []
    try:
      # Update my state.
      self._PeriodicThread()
      for serv in self.servers[1:]:
        if skip and serv in skip:
          continue
        pool = connectionpool.HTTPConnectionPool(
            serv.Address(), port=serv.Port())
        pools.append((serv, pool))
      body = self.mapping.SerializeToString()
      headers = {"Content-Length": len(body)}
      for serv, pool in pools:
        res = pool.urlopen("POST", "/servers/sync", headers=headers, body=body)
        if res.status != constants.RESPONSE_OK:
          logging.warning("Could not sync with server %s:%d",
                          serv.Address(), serv.Port())
          return False
        state = rdf_data_server.DataServerState()
        state.ParseFromString(res.data)
        serv.UpdateState(state)
    except urllib3.exceptions.MaxRetryError:
      return False
    finally:
      for _, pool in pools:
        pool.close()
    return True

Example 14

Project: grr
Source File: rebalance.py
View license
def _RecCopyFiles(rebalance, server_id, dspath, subpath, pool_cache,
                  removed_list):
  """Recursively send files for moving to the required data server."""
  fulldir = utils.JoinPath(dspath, subpath)
  mapping = rebalance.mapping
  for comp in os.listdir(fulldir):
    if comp == constants.REBALANCE_DIRECTORY:
      continue
    path = utils.JoinPath(fulldir, comp)
    name, unused_extension = os.path.splitext(comp)
    if name in COPY_EXCEPTIONS:
      continue
    if os.path.isdir(path):
      result = _RecCopyFiles(rebalance, server_id, dspath,
                             utils.JoinPath(subpath, comp), pool_cache,
                             removed_list)
      if not result:
        return False
      continue
    if not os.path.isfile(path):
      continue
    key = common.MakeDestinationKey(subpath, name)
    where = sutils.MapKeyToServer(mapping, key)
    if where != server_id:
      server = mapping.servers[where]
      addr = server.address
      port = server.port
      key = (addr, port)
      try:
        pool = pool_cache[key]
      except KeyError:
        pool = connectionpool.HTTPConnectionPool(addr, port=port)
        pool_cache[key] = pool
      logging.info("Need to move %s from %d to %d", key, server_id, where)
      if not _SendFileToServer(pool, path, subpath, comp, rebalance):
        return False
      removed_list.append(path)
    else:
      logging.info("File %s stays here", path)
  return True