org.apache.hadoop.hdds.protocol.DatanodeDetails

Here are the examples of the java api org.apache.hadoop.hdds.protocol.DatanodeDetails taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

445 Examples 7

19 Source : ReconNodeManager.java
with Apache License 2.0
from apache

@Override
protected void updateDatanodeOpState(DatanodeDetails reportedDn) throws NodeNotFoundException {
    super.updateDatanodeOpState(reportedDn);
    // Update NodeOperationalState in NodeStatus to keep it consistent for Recon
    super.getNodeStateManager().setNodeOperationalState(reportedDn, reportedDn.getPersistedOpState(), reportedDn.getPersistedOpStateExpiryEpochSec());
}

19 Source : ReconNodeManager.java
with Apache License 2.0
from apache

/**
 * Add a new new node to the NodeDB. Must be called after register.
 * @param datanodeDetails Datanode details.
 */
public void addNodeToDB(DatanodeDetails datanodeDetails) throws IOException {
    nodeDB.put(datanodeDetails.getUuid(), datanodeDetails);
    LOG.info("Adding new node {} to Node DB.", datanodeDetails.getUuid());
}

19 Source : ReconNodeManager.java
with Apache License 2.0
from apache

/**
 * Returns the last heartbeat time of the given node.
 *
 * @param datanodeDetails DatanodeDetails
 * @return last heartbeat time
 */
public long getLastHeartbeat(DatanodeDetails datanodeDetails) {
    return datanodeHeartbeatMap.getOrDefault(datanodeDetails.getUuid(), 0L);
}

19 Source : ReconNodeManager.java
with Apache License 2.0
from apache

private void loadExistingNodes() {
    try {
        int nodeCount = 0;
        TableIterator<UUID, ? extends Table.KeyValue<UUID, DatanodeDetails>> iterator = nodeDB.iterator();
        while (iterator.hasNext()) {
            DatanodeDetails datanodeDetails = iterator.next().getValue();
            register(datanodeDetails, null, null);
            nodeCount++;
        }
        LOG.info("Loaded {} nodes from node DB.", nodeCount);
    } catch (IOException ioEx) {
        LOG.error("Exception while loading existing nodes.", ioEx);
    }
}

19 Source : ScmBlockLocationTestingClient.java
with Apache License 2.0
from apache

private Pipeline createPipeline(DatanodeDetails datanode) {
    List<DatanodeDetails> dns = new ArrayList<>();
    dns.add(datanode);
    Pipeline pipeline = Pipeline.newBuilder().setState(Pipeline.PipelineState.OPEN).setId(PipelineID.randomId()).setType(HddsProtos.ReplicationType.STAND_ALONE).setFactor(HddsProtos.ReplicationFactor.ONE).setNodes(dns).build();
    return pipeline;
}

19 Source : TestDecommissionAndMaintenance.java
with Apache License 2.0
from apache

/**
 * Given a Datanode, return a string consisting of the hostname and one of its
 * ports in the for host:post.
 * @param dn Datanode for which to retrieve the host:post string
 * @return host:port for the given DN.
 */
private String getDNHostAndPort(DatanodeDetails dn) {
    return dn.getHostName() + ":" + dn.getPorts().get(0).getValue();
}

19 Source : MiniOzoneChaosCluster.java
with Apache License 2.0
from apache

// Should the selected node be stopped or started.
public boolean shouldStop(DatanodeDetails dn) {
    return !failedDnSet.contains(dn);
}

19 Source : InfoSubcommand.java
with Apache License 2.0
from apache

private static String buildDatanodeDetails(DatanodeDetails details) {
    return details.getUuidString() + "/" + details.getHostName();
}

19 Source : ReplicationNodeManagerMock.java
with Apache License 2.0
from apache

@Override
public Boolean isNodeRegistered(DatanodeDetails datanodeDetails) {
    return false;
}

19 Source : ReplicationNodeManagerMock.java
with Apache License 2.0
from apache

/**
 * Update set of containers available on a datanode.
 * @param uuid - DatanodeID
 * @param containerIds - Set of containerIDs
 * @throws NodeNotFoundException - if datanode is not known. For new datanode
 *                                 use addDatanodeInContainerMap call.
 */
@Override
public void setContainers(DatanodeDetails uuid, Set<ContainerID> containerIds) throws NodeNotFoundException {
    throw new UnsupportedOperationException("Not yet implemented");
}

19 Source : ReplicationNodeManagerMock.java
with Apache License 2.0
from apache

/**
 * Return the node stat of the specified datanode.
 *
 * @param dd - datanode details.
 * @return node stat if it is live/stale, null if it is decommissioned or
 * doesn't exist.
 */
@Override
public SCMNodeMetric getNodeStat(DatanodeDetails dd) {
    return null;
}

19 Source : ReplicationNodeManagerMock.java
with Apache License 2.0
from apache

@Override
public int pipelineLimit(DatanodeDetails dn) {
    return 0;
}

19 Source : ReplicationNodeManagerMock.java
with Apache License 2.0
from apache

/**
 * Get the count of pipelines a datanodes is replacedociated with.
 * @param dn DatanodeDetails
 * @return The number of pipelines
 */
@Override
public int getPipelinesCount(DatanodeDetails dn) {
    throw new UnsupportedOperationException("Not yet implemented");
}

19 Source : ReplicationNodeManagerMock.java
with Apache License 2.0
from apache

/**
 * Send heartbeat to indicate the datanode is alive and doing well.
 *
 * @param dd - Datanode Details.
 * @return SCMheartbeat response list
 */
@Override
public List<SCMCommand> processHeartbeat(DatanodeDetails dd) {
    return null;
}

19 Source : ReplicationNodeManagerMock.java
with Apache License 2.0
from apache

/**
 * Returns the node state of a specific node.
 *
 * @param dd - DatanodeDetails
 * @return Healthy/Stale/Dead.
 */
@Override
public NodeStatus getNodeStatus(DatanodeDetails dd) {
    return nodeStateMap.get(dd);
}

19 Source : ReplicationNodeManagerMock.java
with Apache License 2.0
from apache

@Override
public void addContainer(DatanodeDetails datanodeDetails, ContainerID containerId) throws NodeNotFoundException {
    throw new UnsupportedOperationException("Not yet implemented");
}

19 Source : ReplicationNodeManagerMock.java
with Apache License 2.0
from apache

/**
 * Return set of containerIDs available on a datanode.
 * @param uuid - DatanodeID
 * @return - set of containerIDs
 */
@Override
public Set<ContainerID> getContainers(DatanodeDetails uuid) {
    throw new UnsupportedOperationException("Not yet implemented");
}

19 Source : ReplicationNodeManagerMock.java
with Apache License 2.0
from apache

/**
 * Set the operation state of a node.
 * @param dd The datanode to set the new state for
 * @param newState The new operational state for the node
 */
@Override
public void setNodeOperationalState(DatanodeDetails dd, HddsProtos.NodeOperationalState newState) throws NodeNotFoundException {
    setNodeOperationalState(dd, newState, 0);
}

19 Source : ReplicationNodeManagerMock.java
with Apache License 2.0
from apache

/**
 * Get set of pipelines a datanode is part of.
 * @param dnId - datanodeID
 * @return Set of PipelineID
 */
@Override
public Set<PipelineID> getPipelines(DatanodeDetails dnId) {
    throw new UnsupportedOperationException("Not yet implemented");
}

19 Source : TestContainerPlacement.java
with Apache License 2.0
from apache

private void deleteContainer(MockNodeManager nodeManager, List<DatanodeDetails> nodes, long containerSize) {
    for (DatanodeDetails dd : nodes) {
        nodeManager.delContainer(dd, containerSize);
    }
}

19 Source : TestUtils.java
with Apache License 2.0
from apache

public static Set<ContainerReplica> getReplicas(final ContainerID containerId, final ContainerReplicaProto.State state, final DatanodeDetails... datanodeDetails) {
    return getReplicas(containerId, state, 10000L, datanodeDetails);
}

19 Source : TestDeadNodeHandler.java
with Apache License 2.0
from apache

/**
 * Update containers available on the datanode.
 * @param datanode
 * @param containers
 * @throws NodeNotFoundException
 */
private void registerContainers(DatanodeDetails datanode, ContainerInfo... containers) throws NodeNotFoundException {
    nodeManager.setContainers(datanode, Arrays.stream(containers).map(container -> new ContainerID(container.getContainerID())).collect(Collectors.toSet()));
}

19 Source : TestDatanodeAdminMonitor.java
with Apache License 2.0
from apache

/**
 * Generate a new ContainerReplica with the given containerID and State.
 * @param containerID The ID the replica is replacedociated with
 * @param nodeState The persistedOpState stored in datanodeDetails.
 * @param replicaState The state of the generated replica.
 * @return A containerReplica with the given ID and state
 */
private ContainerReplica generateReplica(ContainerID containerID, HddsProtos.NodeOperationalState nodeState, ContainerReplicaProto.State replicaState) {
    DatanodeDetails dn = MockDatanodeDetails.randomDatanodeDetails();
    dn.setPersistedOpState(nodeState);
    return ContainerReplica.newBuilder().setContainerState(replicaState).setContainerID(containerID).setSequenceId(1).setDatanodeDetails(dn).build();
}

19 Source : TestContainerStateManager.java
with Apache License 2.0
from apache

private void addReplica(ContainerInfo cont, DatanodeDetails node) throws ContainerNotFoundException {
    ContainerReplica replica = ContainerReplica.newBuilder().setContainerID(cont.containerID()).setContainerState(ContainerReplicaProto.State.CLOSED).setDatanodeDetails(node).build();
    containerStateManager.updateContainerReplica(cont.containerID(), replica);
}

19 Source : TestContainerReplicaCount.java
with Apache License 2.0
from apache

private Set<ContainerReplica> registerNodes(HddsProtos.NodeOperationalState... states) {
    Set<ContainerReplica> replica = new HashSet<>();
    for (HddsProtos.NodeOperationalState s : states) {
        DatanodeDetails dn = MockDatanodeDetails.randomDatanodeDetails();
        dn.setPersistedOpState(s);
        replica.add(new ContainerReplica.ContainerReplicaBuilder().setContainerID(new ContainerID(1)).setContainerState(CLOSED).setDatanodeDetails(dn).setOriginNodeId(dn.getUuid()).setSequenceId(1).build());
    }
    return replica;
}

19 Source : SimpleMockNodeManager.java
with Apache License 2.0
from apache

@Override
public RegisteredCommand register(DatanodeDetails datanodeDetails, StorageContainerDatanodeProtocolProtos.NodeReportProto nodeReport, StorageContainerDatanodeProtocolProtos.PipelineReportsProto pipelineReport) {
    return null;
}

19 Source : SimpleMockNodeManager.java
with Apache License 2.0
from apache

@Override
public List<SCMCommand> processHeartbeat(DatanodeDetails datanodeDetails) {
    return null;
}

19 Source : SimpleMockNodeManager.java
with Apache License 2.0
from apache

/**
 * Set the number of pipelines for the given node. This simply generates
 * new PipelineID objects and places them in a set. No actual pipelines are
 * created.
 *
 * Setting the count to zero effectively deletes the pipelines for the node
 *
 * @param dd The DatanodeDetails for which to create the pipelines
 * @param count The number of pipelines to create or zero to delete all
 *              pipelines
 */
public void setPipelines(DatanodeDetails dd, int count) {
    Set<PipelineID> pipelines = new HashSet<>();
    for (int i = 0; i < count; i++) {
        pipelines.add(PipelineID.randomId());
    }
    pipelineMap.put(dd.getUuid(), pipelines);
}

19 Source : SimpleMockNodeManager.java
with Apache License 2.0
from apache

@Override
public int getPipelinesCount(DatanodeDetails datanodeDetails) {
    return 0;
}

19 Source : SimpleMockNodeManager.java
with Apache License 2.0
from apache

@Override
public void processNodeReport(DatanodeDetails datanodeDetails, StorageContainerDatanodeProtocolProtos.NodeReportProto nodeReport) {
}

19 Source : SimpleMockNodeManager.java
with Apache License 2.0
from apache

@Override
public void addContainer(DatanodeDetails datanodeDetails, ContainerID containerId) throws NodeNotFoundException {
}

19 Source : SimpleMockNodeManager.java
with Apache License 2.0
from apache

/**
 * Return the set of ContainerID replacedociated with the datanode. If there are no
 * container present, an empty set is return to mirror the behaviour of
 * SCMNodeManaer
 *
 * @param dn The datanodeDetails for which to return the containers
 * @return A Set of ContainerID or an empty Set if none are present
 * @throws NodeNotFoundException
 */
@Override
public Set<ContainerID> getContainers(DatanodeDetails dn) throws NodeNotFoundException {
    // The concrete implementation of this method in SCMNodeManager will return
    // an empty set if there are no containers, and will never return null.
    return containerMap.computeIfAbsent(dn.getUuid(), key -> new HashSet<>());
}

19 Source : SimpleMockNodeManager.java
with Apache License 2.0
from apache

@Override
public void setNodeOperationalState(DatanodeDetails dn, HddsProtos.NodeOperationalState newState, long opStateExpiryEpocSec) throws NodeNotFoundException {
    DatanodeInfo dni = nodeMap.get(dn.getUuid());
    if (dni == null) {
        throw new NodeNotFoundException();
    }
    dni.setNodeStatus(new NodeStatus(newState, dni.getNodeStatus().getHealth(), opStateExpiryEpocSec));
}

19 Source : SimpleMockNodeManager.java
with Apache License 2.0
from apache

@Override
public void setNodeOperationalState(DatanodeDetails dn, HddsProtos.NodeOperationalState newState) throws NodeNotFoundException {
    setNodeOperationalState(dn, newState, 0);
}

19 Source : SimpleMockNodeManager.java
with Apache License 2.0
from apache

/**
 * Return the set of PipelineID replacedociated with the given DatanodeDetails.
 *
 * If there are no pipelines, null is return, to mirror the behaviour of
 * SCMNodeManager.
 *
 * @param datanodeDetails The datanode for which to return the pipelines
 * @return A set of PipelineID or null if there are none
 */
@Override
public Set<PipelineID> getPipelines(DatanodeDetails datanodeDetails) {
    Set<PipelineID> p = pipelineMap.get(datanodeDetails.getUuid());
    if (p == null || p.size() == 0) {
        return null;
    } else {
        return p;
    }
}

19 Source : SimpleMockNodeManager.java
with Apache License 2.0
from apache

@Override
public void setContainers(DatanodeDetails dn, Set<ContainerID> containerIds) throws NodeNotFoundException {
    containerMap.put(dn.getUuid(), containerIds);
}

19 Source : SimpleMockNodeManager.java
with Apache License 2.0
from apache

@Override
public int pipelineLimit(DatanodeDetails dn) {
    return 1;
}

19 Source : SimpleMockNodeManager.java
with Apache License 2.0
from apache

@Override
public SCMNodeMetric getNodeStat(DatanodeDetails datanodeDetails) {
    return null;
}

19 Source : SCMCommonPlacementPolicy.java
with Apache License 2.0
from apache

/**
 * Returns true if this node has enough space to meet our requirement.
 *
 * @param datanodeDetails DatanodeDetails
 * @return true if we have enough space.
 */
public boolean hasEnoughSpace(DatanodeDetails datanodeDetails, long sizeRequired) {
    SCMNodeMetric nodeMetric = nodeManager.getNodeStat(datanodeDetails);
    return (nodeMetric != null) && (nodeMetric.get() != null) && nodeMetric.get().getRemaining().hasResources(sizeRequired);
}

19 Source : SCMCommonPlacementPolicy.java
with Apache License 2.0
from apache

/**
 * This function invokes the derived clreplacedes chooseNode Function to build a
 * list of nodes. Then it verifies that invoked policy was able to return
 * expected number of nodes.
 *
 * @param nodesRequired - Nodes Required
 * @param healthyNodes  - List of Nodes in the result set.
 * @return List of Datanodes that can be used for placement.
 * @throws SCMException SCMException
 */
public List<DatanodeDetails> getResultSet(int nodesRequired, List<DatanodeDetails> healthyNodes) throws SCMException {
    List<DatanodeDetails> results = new ArrayList<>();
    for (int x = 0; x < nodesRequired; x++) {
        // invoke the choose function defined in the derived clreplacedes.
        DatanodeDetails nodeId = chooseNode(healthyNodes);
        if (nodeId != null) {
            results.add(nodeId);
        }
    }
    if (results.size() < nodesRequired) {
        LOG.error("Unable to find the required number of healthy nodes that " + "meet the criteria. Required nodes: {}, Found nodes: {}", nodesRequired, results.size());
        throw new SCMException("Unable to find required number of nodes.", SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
    }
    return results;
}

19 Source : PipelineReportHandler.java
with Apache License 2.0
from apache

protected void setReportedDatanode(Pipeline pipeline, DatanodeDetails dn) throws IOException {
    pipeline.reportDatanode(dn);
}

19 Source : PipelineReportHandler.java
with Apache License 2.0
from apache

protected void setPipelineLeaderId(PipelineReport report, Pipeline pipeline, DatanodeDetails dn) {
    // ONE replica pipeline doesn't have leader flag
    if (report.getIsLeader() || pipeline.getFactor() == HddsProtos.ReplicationFactor.ONE) {
        pipeline.setLeaderId(dn.getUuid());
        metrics.incNumPipelineBytesWritten(pipeline, report.getBytesWritten());
    }
}

19 Source : PipelinePlacementPolicy.java
with Apache License 2.0
from apache

/**
 * Find a node from the healthy list and return it after removing it from the
 * list that we are operating on.
 *
 * @param healthyNodes - Set of healthy nodes we can choose from.
 * @return chosen datanodeDetails
 */
@Override
public DatanodeDetails chooseNode(final List<DatanodeDetails> healthyNodes) {
    if (healthyNodes == null || healthyNodes.isEmpty()) {
        return null;
    }
    DatanodeDetails selectedNode = healthyNodes.get(getRand().nextInt(healthyNodes.size()));
    healthyNodes.remove(selectedNode);
    return selectedNode;
}

19 Source : PipelinePlacementPolicy.java
with Apache License 2.0
from apache

/**
 * Given a list of Datanodes, return false if the entire list is only on a
 * single rack, or the list is empty. If there is more than 1 rack, return
 * true.
 * @param dns List of datanodes to check
 * @return True if there are multiple racks, false otherwise
 */
private boolean multipleRacksAvailable(List<DatanodeDetails> dns) {
    if (dns.size() <= 1) {
        return false;
    }
    String initialRack = dns.get(0).getNetworkLocation();
    for (DatanodeDetails dn : dns) {
        if (!dn.getNetworkLocation().equals(initialRack)) {
            return true;
        }
    }
    return false;
}

19 Source : MinLeaderCountChoosePolicy.java
with Apache License 2.0
from apache

@Override
public DatanodeDetails chooseLeader(List<DatanodeDetails> dns) {
    Map<DatanodeDetails, Integer> suggestedLeaderCount = getSuggestedLeaderCount(dns, getNodeManager(), getPipelineStateManager());
    int minLeaderCount = Integer.MAX_VALUE;
    DatanodeDetails suggestedLeader = null;
    for (Map.Entry<DatanodeDetails, Integer> entry : suggestedLeaderCount.entrySet()) {
        if (entry.getValue() < minLeaderCount) {
            minLeaderCount = entry.getValue();
            suggestedLeader = entry.getKey();
        }
    }
    return suggestedLeader;
}

19 Source : SCMNodeManager.java
with Apache License 2.0
from apache

boolean opStateDiffers(DatanodeDetails dnDetails, NodeStatus nodeStatus) {
    return nodeStatus.getOperationalState() != dnDetails.getPersistedOpState() || nodeStatus.getOpStateExpiryEpochSeconds() != dnDetails.getPersistedOpStateExpiryEpochSec();
}

19 Source : NodeDecommissionManager.java
with Apache License 2.0
from apache

private NodeStatus getNodeStatus(DatanodeDetails dn) throws NodeNotFoundException {
    return nodeManager.getNodeStatus(dn);
}

19 Source : NodeDecommissionManager.java
with Apache License 2.0
from apache

public synchronized void decommissionNodes(List nodes) throws InvalidHostStringException {
    List<DatanodeDetails> dns = mapHostnamesToDatanodes(nodes);
    for (DatanodeDetails dn : dns) {
        try {
            startDecommission(dn);
        } catch (NodeNotFoundException e) {
            // We already validated the host strings and retrieved the DnDetails
            // object from the node manager. Therefore we should never get a
            // NodeNotFoundException here expect if the node is remove in the
            // very short window between validation and starting decom. Therefore
            // log a warning and ignore the exception
            LOG.warn("The host {} was not found in SCM. Ignoring the request to " + "decommission it", dn.getHostName());
        } catch (InvalidNodeStateException e) {
        // TODO - decide how to handle this. We may not want to fail all nodes
        // only one is in a bad state, as some nodes may have been OK
        // and already processed. Perhaps we should return a list of
        // error and feed that all the way back to the client?
        }
    }
}

19 Source : NodeDecommissionManager.java
with Apache License 2.0
from apache

/**
 * If a SCM is restarted, then upon re-registration the datanode will already
 * be in DECOMMISSIONING or ENTERING_MAINTENANCE state. In that case, it
 * needs to be added back into the monitor to track its progress.
 * @param dn Datanode to add back to tracking.
 * @throws NodeNotFoundException
 */
public synchronized void continueAdminForNode(DatanodeDetails dn) throws NodeNotFoundException {
    NodeOperationalState opState = getNodeStatus(dn).getOperationalState();
    if (opState == NodeOperationalState.DECOMMISSIONING || opState == NodeOperationalState.ENTERING_MAINTENANCE || opState == NodeOperationalState.IN_MAINTENANCE) {
        monitor.startMonitoring(dn);
    }
}

19 Source : NodeDecommissionManager.java
with Apache License 2.0
from apache

public synchronized void recommissionNodes(List nodes) throws InvalidHostStringException {
    List<DatanodeDetails> dns = mapHostnamesToDatanodes(nodes);
    for (DatanodeDetails dn : dns) {
        try {
            recommission(dn);
        } catch (NodeNotFoundException e) {
            // We already validated the host strings and retrieved the DnDetails
            // object from the node manager. Therefore we should never get a
            // NodeNotFoundException here expect if the node is remove in the
            // very short window between validation and starting decom. Therefore
            // log a warning and ignore the exception
            LOG.warn("Host {} was not found in SCM. Ignoring the request to " + "recommission it.", dn.getHostName());
        }
    }
}

See More Examples