Here are the examples of the java api org.apache.hadoop.hdds.protocol.DatanodeDetails taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
445 Examples
19
Source : ReconNodeManager.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
@Override
protected void updateDatanodeOpState(DatanodeDetails reportedDn) throws NodeNotFoundException {
super.updateDatanodeOpState(reportedDn);
// Update NodeOperationalState in NodeStatus to keep it consistent for Recon
super.getNodeStateManager().setNodeOperationalState(reportedDn, reportedDn.getPersistedOpState(), reportedDn.getPersistedOpStateExpiryEpochSec());
}
19
Source : ReconNodeManager.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
/**
* Add a new new node to the NodeDB. Must be called after register.
* @param datanodeDetails Datanode details.
*/
public void addNodeToDB(DatanodeDetails datanodeDetails) throws IOException {
nodeDB.put(datanodeDetails.getUuid(), datanodeDetails);
LOG.info("Adding new node {} to Node DB.", datanodeDetails.getUuid());
}
19
Source : ReconNodeManager.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
/**
* Returns the last heartbeat time of the given node.
*
* @param datanodeDetails DatanodeDetails
* @return last heartbeat time
*/
public long getLastHeartbeat(DatanodeDetails datanodeDetails) {
return datanodeHeartbeatMap.getOrDefault(datanodeDetails.getUuid(), 0L);
}
19
Source : ReconNodeManager.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
private void loadExistingNodes() {
try {
int nodeCount = 0;
TableIterator<UUID, ? extends Table.KeyValue<UUID, DatanodeDetails>> iterator = nodeDB.iterator();
while (iterator.hasNext()) {
DatanodeDetails datanodeDetails = iterator.next().getValue();
register(datanodeDetails, null, null);
nodeCount++;
}
LOG.info("Loaded {} nodes from node DB.", nodeCount);
} catch (IOException ioEx) {
LOG.error("Exception while loading existing nodes.", ioEx);
}
}
19
Source : ScmBlockLocationTestingClient.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
private Pipeline createPipeline(DatanodeDetails datanode) {
List<DatanodeDetails> dns = new ArrayList<>();
dns.add(datanode);
Pipeline pipeline = Pipeline.newBuilder().setState(Pipeline.PipelineState.OPEN).setId(PipelineID.randomId()).setType(HddsProtos.ReplicationType.STAND_ALONE).setFactor(HddsProtos.ReplicationFactor.ONE).setNodes(dns).build();
return pipeline;
}
19
Source : TestDecommissionAndMaintenance.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
/**
* Given a Datanode, return a string consisting of the hostname and one of its
* ports in the for host:post.
* @param dn Datanode for which to retrieve the host:post string
* @return host:port for the given DN.
*/
private String getDNHostAndPort(DatanodeDetails dn) {
return dn.getHostName() + ":" + dn.getPorts().get(0).getValue();
}
19
Source : MiniOzoneChaosCluster.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
// Should the selected node be stopped or started.
public boolean shouldStop(DatanodeDetails dn) {
return !failedDnSet.contains(dn);
}
19
Source : InfoSubcommand.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
private static String buildDatanodeDetails(DatanodeDetails details) {
return details.getUuidString() + "/" + details.getHostName();
}
19
Source : ReplicationNodeManagerMock.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
@Override
public Boolean isNodeRegistered(DatanodeDetails datanodeDetails) {
return false;
}
19
Source : ReplicationNodeManagerMock.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
/**
* Update set of containers available on a datanode.
* @param uuid - DatanodeID
* @param containerIds - Set of containerIDs
* @throws NodeNotFoundException - if datanode is not known. For new datanode
* use addDatanodeInContainerMap call.
*/
@Override
public void setContainers(DatanodeDetails uuid, Set<ContainerID> containerIds) throws NodeNotFoundException {
throw new UnsupportedOperationException("Not yet implemented");
}
19
Source : ReplicationNodeManagerMock.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
/**
* Return the node stat of the specified datanode.
*
* @param dd - datanode details.
* @return node stat if it is live/stale, null if it is decommissioned or
* doesn't exist.
*/
@Override
public SCMNodeMetric getNodeStat(DatanodeDetails dd) {
return null;
}
19
Source : ReplicationNodeManagerMock.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
@Override
public int pipelineLimit(DatanodeDetails dn) {
return 0;
}
19
Source : ReplicationNodeManagerMock.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
/**
* Get the count of pipelines a datanodes is replacedociated with.
* @param dn DatanodeDetails
* @return The number of pipelines
*/
@Override
public int getPipelinesCount(DatanodeDetails dn) {
throw new UnsupportedOperationException("Not yet implemented");
}
19
Source : ReplicationNodeManagerMock.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
/**
* Send heartbeat to indicate the datanode is alive and doing well.
*
* @param dd - Datanode Details.
* @return SCMheartbeat response list
*/
@Override
public List<SCMCommand> processHeartbeat(DatanodeDetails dd) {
return null;
}
19
Source : ReplicationNodeManagerMock.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
/**
* Returns the node state of a specific node.
*
* @param dd - DatanodeDetails
* @return Healthy/Stale/Dead.
*/
@Override
public NodeStatus getNodeStatus(DatanodeDetails dd) {
return nodeStateMap.get(dd);
}
19
Source : ReplicationNodeManagerMock.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
@Override
public void addContainer(DatanodeDetails datanodeDetails, ContainerID containerId) throws NodeNotFoundException {
throw new UnsupportedOperationException("Not yet implemented");
}
19
Source : ReplicationNodeManagerMock.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
/**
* Return set of containerIDs available on a datanode.
* @param uuid - DatanodeID
* @return - set of containerIDs
*/
@Override
public Set<ContainerID> getContainers(DatanodeDetails uuid) {
throw new UnsupportedOperationException("Not yet implemented");
}
19
Source : ReplicationNodeManagerMock.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
/**
* Set the operation state of a node.
* @param dd The datanode to set the new state for
* @param newState The new operational state for the node
*/
@Override
public void setNodeOperationalState(DatanodeDetails dd, HddsProtos.NodeOperationalState newState) throws NodeNotFoundException {
setNodeOperationalState(dd, newState, 0);
}
19
Source : ReplicationNodeManagerMock.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
/**
* Get set of pipelines a datanode is part of.
* @param dnId - datanodeID
* @return Set of PipelineID
*/
@Override
public Set<PipelineID> getPipelines(DatanodeDetails dnId) {
throw new UnsupportedOperationException("Not yet implemented");
}
19
Source : TestContainerPlacement.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
private void deleteContainer(MockNodeManager nodeManager, List<DatanodeDetails> nodes, long containerSize) {
for (DatanodeDetails dd : nodes) {
nodeManager.delContainer(dd, containerSize);
}
}
19
Source : TestUtils.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public static Set<ContainerReplica> getReplicas(final ContainerID containerId, final ContainerReplicaProto.State state, final DatanodeDetails... datanodeDetails) {
return getReplicas(containerId, state, 10000L, datanodeDetails);
}
19
Source : TestDeadNodeHandler.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
/**
* Update containers available on the datanode.
* @param datanode
* @param containers
* @throws NodeNotFoundException
*/
private void registerContainers(DatanodeDetails datanode, ContainerInfo... containers) throws NodeNotFoundException {
nodeManager.setContainers(datanode, Arrays.stream(containers).map(container -> new ContainerID(container.getContainerID())).collect(Collectors.toSet()));
}
19
Source : TestDatanodeAdminMonitor.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
/**
* Generate a new ContainerReplica with the given containerID and State.
* @param containerID The ID the replica is replacedociated with
* @param nodeState The persistedOpState stored in datanodeDetails.
* @param replicaState The state of the generated replica.
* @return A containerReplica with the given ID and state
*/
private ContainerReplica generateReplica(ContainerID containerID, HddsProtos.NodeOperationalState nodeState, ContainerReplicaProto.State replicaState) {
DatanodeDetails dn = MockDatanodeDetails.randomDatanodeDetails();
dn.setPersistedOpState(nodeState);
return ContainerReplica.newBuilder().setContainerState(replicaState).setContainerID(containerID).setSequenceId(1).setDatanodeDetails(dn).build();
}
19
Source : TestContainerStateManager.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
private void addReplica(ContainerInfo cont, DatanodeDetails node) throws ContainerNotFoundException {
ContainerReplica replica = ContainerReplica.newBuilder().setContainerID(cont.containerID()).setContainerState(ContainerReplicaProto.State.CLOSED).setDatanodeDetails(node).build();
containerStateManager.updateContainerReplica(cont.containerID(), replica);
}
19
Source : TestContainerReplicaCount.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
private Set<ContainerReplica> registerNodes(HddsProtos.NodeOperationalState... states) {
Set<ContainerReplica> replica = new HashSet<>();
for (HddsProtos.NodeOperationalState s : states) {
DatanodeDetails dn = MockDatanodeDetails.randomDatanodeDetails();
dn.setPersistedOpState(s);
replica.add(new ContainerReplica.ContainerReplicaBuilder().setContainerID(new ContainerID(1)).setContainerState(CLOSED).setDatanodeDetails(dn).setOriginNodeId(dn.getUuid()).setSequenceId(1).build());
}
return replica;
}
19
Source : SimpleMockNodeManager.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
@Override
public RegisteredCommand register(DatanodeDetails datanodeDetails, StorageContainerDatanodeProtocolProtos.NodeReportProto nodeReport, StorageContainerDatanodeProtocolProtos.PipelineReportsProto pipelineReport) {
return null;
}
19
Source : SimpleMockNodeManager.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
@Override
public List<SCMCommand> processHeartbeat(DatanodeDetails datanodeDetails) {
return null;
}
19
Source : SimpleMockNodeManager.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
/**
* Set the number of pipelines for the given node. This simply generates
* new PipelineID objects and places them in a set. No actual pipelines are
* created.
*
* Setting the count to zero effectively deletes the pipelines for the node
*
* @param dd The DatanodeDetails for which to create the pipelines
* @param count The number of pipelines to create or zero to delete all
* pipelines
*/
public void setPipelines(DatanodeDetails dd, int count) {
Set<PipelineID> pipelines = new HashSet<>();
for (int i = 0; i < count; i++) {
pipelines.add(PipelineID.randomId());
}
pipelineMap.put(dd.getUuid(), pipelines);
}
19
Source : SimpleMockNodeManager.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
@Override
public int getPipelinesCount(DatanodeDetails datanodeDetails) {
return 0;
}
19
Source : SimpleMockNodeManager.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
@Override
public void processNodeReport(DatanodeDetails datanodeDetails, StorageContainerDatanodeProtocolProtos.NodeReportProto nodeReport) {
}
19
Source : SimpleMockNodeManager.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
@Override
public void addContainer(DatanodeDetails datanodeDetails, ContainerID containerId) throws NodeNotFoundException {
}
19
Source : SimpleMockNodeManager.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
/**
* Return the set of ContainerID replacedociated with the datanode. If there are no
* container present, an empty set is return to mirror the behaviour of
* SCMNodeManaer
*
* @param dn The datanodeDetails for which to return the containers
* @return A Set of ContainerID or an empty Set if none are present
* @throws NodeNotFoundException
*/
@Override
public Set<ContainerID> getContainers(DatanodeDetails dn) throws NodeNotFoundException {
// The concrete implementation of this method in SCMNodeManager will return
// an empty set if there are no containers, and will never return null.
return containerMap.computeIfAbsent(dn.getUuid(), key -> new HashSet<>());
}
19
Source : SimpleMockNodeManager.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
@Override
public void setNodeOperationalState(DatanodeDetails dn, HddsProtos.NodeOperationalState newState, long opStateExpiryEpocSec) throws NodeNotFoundException {
DatanodeInfo dni = nodeMap.get(dn.getUuid());
if (dni == null) {
throw new NodeNotFoundException();
}
dni.setNodeStatus(new NodeStatus(newState, dni.getNodeStatus().getHealth(), opStateExpiryEpocSec));
}
19
Source : SimpleMockNodeManager.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
@Override
public void setNodeOperationalState(DatanodeDetails dn, HddsProtos.NodeOperationalState newState) throws NodeNotFoundException {
setNodeOperationalState(dn, newState, 0);
}
19
Source : SimpleMockNodeManager.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
/**
* Return the set of PipelineID replacedociated with the given DatanodeDetails.
*
* If there are no pipelines, null is return, to mirror the behaviour of
* SCMNodeManager.
*
* @param datanodeDetails The datanode for which to return the pipelines
* @return A set of PipelineID or null if there are none
*/
@Override
public Set<PipelineID> getPipelines(DatanodeDetails datanodeDetails) {
Set<PipelineID> p = pipelineMap.get(datanodeDetails.getUuid());
if (p == null || p.size() == 0) {
return null;
} else {
return p;
}
}
19
Source : SimpleMockNodeManager.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
@Override
public void setContainers(DatanodeDetails dn, Set<ContainerID> containerIds) throws NodeNotFoundException {
containerMap.put(dn.getUuid(), containerIds);
}
19
Source : SimpleMockNodeManager.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
@Override
public int pipelineLimit(DatanodeDetails dn) {
return 1;
}
19
Source : SimpleMockNodeManager.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
@Override
public SCMNodeMetric getNodeStat(DatanodeDetails datanodeDetails) {
return null;
}
19
Source : SCMCommonPlacementPolicy.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
/**
* Returns true if this node has enough space to meet our requirement.
*
* @param datanodeDetails DatanodeDetails
* @return true if we have enough space.
*/
public boolean hasEnoughSpace(DatanodeDetails datanodeDetails, long sizeRequired) {
SCMNodeMetric nodeMetric = nodeManager.getNodeStat(datanodeDetails);
return (nodeMetric != null) && (nodeMetric.get() != null) && nodeMetric.get().getRemaining().hasResources(sizeRequired);
}
19
Source : SCMCommonPlacementPolicy.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
/**
* This function invokes the derived clreplacedes chooseNode Function to build a
* list of nodes. Then it verifies that invoked policy was able to return
* expected number of nodes.
*
* @param nodesRequired - Nodes Required
* @param healthyNodes - List of Nodes in the result set.
* @return List of Datanodes that can be used for placement.
* @throws SCMException SCMException
*/
public List<DatanodeDetails> getResultSet(int nodesRequired, List<DatanodeDetails> healthyNodes) throws SCMException {
List<DatanodeDetails> results = new ArrayList<>();
for (int x = 0; x < nodesRequired; x++) {
// invoke the choose function defined in the derived clreplacedes.
DatanodeDetails nodeId = chooseNode(healthyNodes);
if (nodeId != null) {
results.add(nodeId);
}
}
if (results.size() < nodesRequired) {
LOG.error("Unable to find the required number of healthy nodes that " + "meet the criteria. Required nodes: {}, Found nodes: {}", nodesRequired, results.size());
throw new SCMException("Unable to find required number of nodes.", SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
}
return results;
}
19
Source : PipelineReportHandler.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
protected void setReportedDatanode(Pipeline pipeline, DatanodeDetails dn) throws IOException {
pipeline.reportDatanode(dn);
}
19
Source : PipelineReportHandler.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
protected void setPipelineLeaderId(PipelineReport report, Pipeline pipeline, DatanodeDetails dn) {
// ONE replica pipeline doesn't have leader flag
if (report.getIsLeader() || pipeline.getFactor() == HddsProtos.ReplicationFactor.ONE) {
pipeline.setLeaderId(dn.getUuid());
metrics.incNumPipelineBytesWritten(pipeline, report.getBytesWritten());
}
}
19
Source : PipelinePlacementPolicy.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
/**
* Find a node from the healthy list and return it after removing it from the
* list that we are operating on.
*
* @param healthyNodes - Set of healthy nodes we can choose from.
* @return chosen datanodeDetails
*/
@Override
public DatanodeDetails chooseNode(final List<DatanodeDetails> healthyNodes) {
if (healthyNodes == null || healthyNodes.isEmpty()) {
return null;
}
DatanodeDetails selectedNode = healthyNodes.get(getRand().nextInt(healthyNodes.size()));
healthyNodes.remove(selectedNode);
return selectedNode;
}
19
Source : PipelinePlacementPolicy.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
/**
* Given a list of Datanodes, return false if the entire list is only on a
* single rack, or the list is empty. If there is more than 1 rack, return
* true.
* @param dns List of datanodes to check
* @return True if there are multiple racks, false otherwise
*/
private boolean multipleRacksAvailable(List<DatanodeDetails> dns) {
if (dns.size() <= 1) {
return false;
}
String initialRack = dns.get(0).getNetworkLocation();
for (DatanodeDetails dn : dns) {
if (!dn.getNetworkLocation().equals(initialRack)) {
return true;
}
}
return false;
}
19
Source : MinLeaderCountChoosePolicy.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
@Override
public DatanodeDetails chooseLeader(List<DatanodeDetails> dns) {
Map<DatanodeDetails, Integer> suggestedLeaderCount = getSuggestedLeaderCount(dns, getNodeManager(), getPipelineStateManager());
int minLeaderCount = Integer.MAX_VALUE;
DatanodeDetails suggestedLeader = null;
for (Map.Entry<DatanodeDetails, Integer> entry : suggestedLeaderCount.entrySet()) {
if (entry.getValue() < minLeaderCount) {
minLeaderCount = entry.getValue();
suggestedLeader = entry.getKey();
}
}
return suggestedLeader;
}
19
Source : SCMNodeManager.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
boolean opStateDiffers(DatanodeDetails dnDetails, NodeStatus nodeStatus) {
return nodeStatus.getOperationalState() != dnDetails.getPersistedOpState() || nodeStatus.getOpStateExpiryEpochSeconds() != dnDetails.getPersistedOpStateExpiryEpochSec();
}
19
Source : NodeDecommissionManager.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
private NodeStatus getNodeStatus(DatanodeDetails dn) throws NodeNotFoundException {
return nodeManager.getNodeStatus(dn);
}
19
Source : NodeDecommissionManager.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public synchronized void decommissionNodes(List nodes) throws InvalidHostStringException {
List<DatanodeDetails> dns = mapHostnamesToDatanodes(nodes);
for (DatanodeDetails dn : dns) {
try {
startDecommission(dn);
} catch (NodeNotFoundException e) {
// We already validated the host strings and retrieved the DnDetails
// object from the node manager. Therefore we should never get a
// NodeNotFoundException here expect if the node is remove in the
// very short window between validation and starting decom. Therefore
// log a warning and ignore the exception
LOG.warn("The host {} was not found in SCM. Ignoring the request to " + "decommission it", dn.getHostName());
} catch (InvalidNodeStateException e) {
// TODO - decide how to handle this. We may not want to fail all nodes
// only one is in a bad state, as some nodes may have been OK
// and already processed. Perhaps we should return a list of
// error and feed that all the way back to the client?
}
}
}
19
Source : NodeDecommissionManager.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
/**
* If a SCM is restarted, then upon re-registration the datanode will already
* be in DECOMMISSIONING or ENTERING_MAINTENANCE state. In that case, it
* needs to be added back into the monitor to track its progress.
* @param dn Datanode to add back to tracking.
* @throws NodeNotFoundException
*/
public synchronized void continueAdminForNode(DatanodeDetails dn) throws NodeNotFoundException {
NodeOperationalState opState = getNodeStatus(dn).getOperationalState();
if (opState == NodeOperationalState.DECOMMISSIONING || opState == NodeOperationalState.ENTERING_MAINTENANCE || opState == NodeOperationalState.IN_MAINTENANCE) {
monitor.startMonitoring(dn);
}
}
19
Source : NodeDecommissionManager.java
with Apache License 2.0
from apache
with Apache License 2.0
from apache
public synchronized void recommissionNodes(List nodes) throws InvalidHostStringException {
List<DatanodeDetails> dns = mapHostnamesToDatanodes(nodes);
for (DatanodeDetails dn : dns) {
try {
recommission(dn);
} catch (NodeNotFoundException e) {
// We already validated the host strings and retrieved the DnDetails
// object from the node manager. Therefore we should never get a
// NodeNotFoundException here expect if the node is remove in the
// very short window between validation and starting decom. Therefore
// log a warning and ignore the exception
LOG.warn("Host {} was not found in SCM. Ignoring the request to " + "recommission it.", dn.getHostName());
}
}
}
See More Examples