Here are the examples of the java api org.apache.hadoop.yarn.api.records.ResourceRequest taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
123 Examples
19
Source : SchedulerUtils.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
/**
* Utility method to normalize a list of resource requests, by insuring that
* the memory for each request is a multiple of minMemory and is not zero.
*/
public static void normalizeRequests(List<ResourceRequest> asks, ResourceCalculator resourceCalculator, Resource clusterResource, Resource minimumResource, Resource maximumResource, Resource incrementResource) {
for (ResourceRequest ask : asks) {
normalizeRequest(ask, resourceCalculator, clusterResource, minimumResource, maximumResource, incrementResource);
}
}
19
Source : SchedulerUtils.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private static void normalizeNodeLabelExpressionInRequest(ResourceRequest resReq, QueueInfo queueInfo) {
String labelExp = resReq.getNodeLabelExpression();
// if queue has default label expression, and RR doesn't have, use the
// default label expression of queue
if (labelExp == null && queueInfo != null && ResourceRequest.ANY.equals(resReq.getResourceName())) {
labelExp = queueInfo.getDefaultNodeLabelExpression();
}
// If labelExp still equals to null, set it to be NO_LABEL
if (labelExp == null) {
labelExp = RMNodeLabelsManager.NO_LABEL;
}
resReq.setNodeLabelExpression(labelExp);
}
19
Source : SchedulerUtils.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
/**
* Utility method to normalize a list of resource requests, by insuring that
* the memory for each request is a multiple of minMemory and is not zero.
*/
public static void normalizeRequests(List<ResourceRequest> asks, ResourceCalculator resourceCalculator, Resource clusterResource, Resource minimumResource, Resource maximumResource) {
for (ResourceRequest ask : asks) {
normalizeRequest(ask, resourceCalculator, clusterResource, minimumResource, maximumResource, minimumResource);
}
}
19
Source : AppSchedulingInfo.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
synchronized private void decrementOutstanding(ResourceRequest offSwitchRequest) {
int numOffSwitchContainers = offSwitchRequest.getNumContainers() - 1;
// Do not remove ANY
offSwitchRequest.setNumContainers(numOffSwitchContainers);
// Do we have any outstanding requests?
// If there is nothing, we need to deactivate this application
if (numOffSwitchContainers == 0) {
checkForDeactivation();
}
}
19
Source : PreemptionResourceRequestPBImpl.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private ResourceRequestProto convertToProtoFormat(ResourceRequest t) {
return ((ResourceRequestPBImpl) t).getProto();
}
19
Source : ApplicationSubmissionContextPBImpl.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Override
public void setAMContainerResourceRequest(ResourceRequest request) {
maybeInitBuilder();
if (request == null) {
builder.clearAmContainerResourceRequest();
}
this.amResourceRequest = request;
}
19
Source : AppSchedulingInfo.java
with Apache License 2.0
from naver
with Apache License 2.0
from naver
private boolean isRequestLabelChanged(ResourceRequest requestOne, ResourceRequest requestTwo) {
String requestOneLabelExp = requestOne.getNodeLabelExpression();
String requestTwoLabelExp = requestTwo.getNodeLabelExpression();
return (!(requestOneLabelExp.equals(requestTwoLabelExp)));
}
18
Source : SchedulerUtils.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
public static void normalizeAndvalidateRequest(ResourceRequest resReq, Resource maximumResource, String queueName, YarnScheduler scheduler, RMContext rmContext) throws InvalidResourceRequestException {
normalizeAndValidateRequest(resReq, maximumResource, queueName, scheduler, false, rmContext);
}
18
Source : SchedulerUtils.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
/**
* Utility method to validate a resource request, by insuring that the
* requested memory/vcore is non-negative and not greater than max
*
* @throws <code>InvalidResourceRequestException</code> when there is invalid
* request
*/
private static void validateResourceRequest(ResourceRequest resReq, Resource maximumResource, QueueInfo queueInfo, RMContext rmContext) throws InvalidResourceRequestException {
if (resReq.getCapability().getMemory() < 0 || resReq.getCapability().getMemory() > maximumResource.getMemory()) {
throw new InvalidResourceRequestException("Invalid resource request" + ", requested memory < 0" + ", or requested memory > max configured" + ", requestedMemory=" + resReq.getCapability().getMemory() + ", maxMemory=" + maximumResource.getMemory());
}
if (resReq.getCapability().getVirtualCores() < 0 || resReq.getCapability().getVirtualCores() > maximumResource.getVirtualCores()) {
throw new InvalidResourceRequestException("Invalid resource request" + ", requested virtual cores < 0" + ", or requested virtual cores > max configured" + ", requestedVirtualCores=" + resReq.getCapability().getVirtualCores() + ", maxVirtualCores=" + maximumResource.getVirtualCores());
}
String labelExp = resReq.getNodeLabelExpression();
// we don't allow specify label expression other than resourceName=ANY now
if (!ResourceRequest.ANY.equals(resReq.getResourceName()) && labelExp != null && !labelExp.trim().isEmpty()) {
throw new InvalidResourceRequestException("Invailid resource request, queue=" + queueInfo.getQueueName() + " specified node label expression in a " + "resource request has resource name = " + resReq.getResourceName());
}
// we don't allow specify label expression with more than one node labels now
if (labelExp != null && labelExp.contains("&&")) {
throw new InvalidResourceRequestException("Invailid resource request, queue=" + queueInfo.getQueueName() + " specified more than one node label " + "in a node label expression, node label expression = " + labelExp);
}
if (labelExp != null && !labelExp.trim().isEmpty() && queueInfo != null) {
if (!checkQueueLabelExpression(queueInfo.getAccessibleNodeLabels(), labelExp, rmContext)) {
throw new InvalidResourceRequestException("Invalid resource request" + ", queue=" + queueInfo.getQueueName() + " doesn't have permission to access all labels " + "in resource request. labelExpression of resource request=" + labelExp + ". Queue labels=" + (queueInfo.getAccessibleNodeLabels() == null ? "" : StringUtils.join(queueInfo.getAccessibleNodeLabels().iterator(), ',')));
}
}
}
18
Source : SchedulerUtils.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
/**
* Utility method to normalize a resource request, by insuring that the
* requested memory is a multiple of minMemory and is not zero.
*/
public static void normalizeRequest(ResourceRequest ask, ResourceCalculator resourceCalculator, Resource clusterResource, Resource minimumResource, Resource maximumResource) {
Resource normalized = Resources.normalize(resourceCalculator, ask.getCapability(), minimumResource, maximumResource, minimumResource);
ask.setCapability(normalized);
}
18
Source : SchedulerUtils.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
/**
* Utility method to normalize a resource request, by insuring that the
* requested memory is a multiple of minMemory and is not zero.
*/
public static void normalizeRequest(ResourceRequest ask, ResourceCalculator resourceCalculator, Resource clusterResource, Resource minimumResource, Resource maximumResource, Resource incrementResource) {
Resource normalized = Resources.normalize(resourceCalculator, ask.getCapability(), minimumResource, maximumResource, incrementResource);
ask.setCapability(normalized);
}
18
Source : SchedulerUtils.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
public static void normalizeAndValidateRequest(ResourceRequest resReq, Resource maximumResource, String queueName, YarnScheduler scheduler, boolean isRecovery, RMContext rmContext) throws InvalidResourceRequestException {
QueueInfo queueInfo = null;
try {
queueInfo = scheduler.getQueueInfo(queueName, false, false);
} catch (IOException e) {
// it is possible queue cannot get when queue mapping is set, just ignore
// the queueInfo here, and move forward
}
SchedulerUtils.normalizeNodeLabelExpressionInRequest(resReq, queueInfo);
if (!isRecovery) {
validateResourceRequest(resReq, maximumResource, queueInfo, rmContext);
}
}
18
Source : FifoScheduler.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private int replacedignNodeLocalContainers(FiCaSchedulerNode node, FiCaSchedulerApp application, Priority priority) {
int replacedignedContainers = 0;
ResourceRequest request = application.getResourceRequest(priority, node.getNodeName());
if (request != null) {
// Don't allocate on this node if we don't need containers on this rack
ResourceRequest rackRequest = application.getResourceRequest(priority, node.getRMNode().getRackName());
if (rackRequest == null || rackRequest.getNumContainers() <= 0) {
return 0;
}
int replacedignableContainers = Math.min(getMaxAllocatableContainers(application, priority, node, NodeType.NODE_LOCAL), request.getNumContainers());
replacedignedContainers = replacedignContainer(node, application, priority, replacedignableContainers, request, NodeType.NODE_LOCAL);
}
return replacedignedContainers;
}
18
Source : PreemptionResourceRequestPBImpl.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Override
public synchronized void setResourceRequest(final ResourceRequest rr) {
maybeInitBuilder();
if (null == rr) {
builder.clearResource();
}
this.rr = rr;
}
18
Source : TestAMRMClient.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Test(timeout = 30000)
public void testAskWithNodeLabels() {
AMRMClientImpl<ContainerRequest> client = new AMRMClientImpl<ContainerRequest>();
// add exp=x to ANY
client.addContainerRequest(new ContainerRequest(Resource.newInstance(1024, 1), null, null, Priority.UNDEFINED, true, "x"));
replacedert.replacedertEquals(1, client.ask.size());
replacedert.replacedertEquals("x", client.ask.iterator().next().getNodeLabelExpression());
// add exp=x then add exp=a to ANY in same priority, only exp=a should kept
client.addContainerRequest(new ContainerRequest(Resource.newInstance(1024, 1), null, null, Priority.UNDEFINED, true, "x"));
client.addContainerRequest(new ContainerRequest(Resource.newInstance(1024, 1), null, null, Priority.UNDEFINED, true, "a"));
replacedert.replacedertEquals(1, client.ask.size());
replacedert.replacedertEquals("a", client.ask.iterator().next().getNodeLabelExpression());
// add exp=x to ANY, rack and node, only resource request has ANY resource
// name will be replacedigned the label expression
// add exp=x then add exp=a to ANY in same priority, only exp=a should kept
client.addContainerRequest(new ContainerRequest(Resource.newInstance(1024, 1), null, null, Priority.UNDEFINED, true, "y"));
replacedert.replacedertEquals(1, client.ask.size());
for (ResourceRequest req : client.ask) {
if (ResourceRequest.ANY.equals(req.getResourceName())) {
replacedert.replacedertEquals("y", req.getNodeLabelExpression());
} else {
replacedert.replacedertNull(req.getNodeLabelExpression());
}
}
}
18
Source : AppSchedulingInfo.java
with Apache License 2.0
from naver
with Apache License 2.0
from naver
synchronized private void decrementOutstanding(ResourceRequest offSwitchRequest) {
int numOffSwitchContainers = offSwitchRequest.getNumContainers() - 1;
// Do not remove ANY
offSwitchRequest.setNumContainers(numOffSwitchContainers);
// Do we have any outstanding requests?
// If there is nothing, we need to deactivate this application
if (numOffSwitchContainers == 0) {
checkForDeactivation();
}
queue.decPendingResource(offSwitchRequest.getNodeLabelExpression(), offSwitchRequest.getCapability());
}
18
Source : TestAMRMClient.java
with Apache License 2.0
from naver
with Apache License 2.0
from naver
@Test(timeout = 30000)
public void testAskWithNodeLabels() {
AMRMClientImpl<ContainerRequest> client = new AMRMClientImpl<ContainerRequest>();
// add exp=x to ANY
client.addContainerRequest(new ContainerRequest(Resource.newInstance(1024, 1), null, null, Priority.UNDEFINED, true, "x"));
replacedert.replacedertEquals(1, client.ask.size());
replacedert.replacedertEquals("x", client.ask.iterator().next().getNodeLabelExpression());
// add exp=x then add exp=a to ANY in same priority, only exp=a should kept
client.addContainerRequest(new ContainerRequest(Resource.newInstance(1024, 1), null, null, Priority.UNDEFINED, true, "x"));
client.addContainerRequest(new ContainerRequest(Resource.newInstance(1024, 1), null, null, Priority.UNDEFINED, true, "a"));
replacedert.replacedertEquals(1, client.ask.size());
replacedert.replacedertEquals("a", client.ask.iterator().next().getNodeLabelExpression());
// add exp=x to ANY, rack and node, only resource request has ANY resource
// name will be replacedigned the label expression
// add exp=x then add exp=a to ANY in same priority, only exp=a should kept
client.addContainerRequest(new ContainerRequest(Resource.newInstance(1024, 1), null, null, Priority.UNDEFINED, true, "y"));
replacedert.replacedertEquals(1, client.ask.size());
for (ResourceRequest req : client.ask) {
if (ResourceRequest.ANY.equals(req.getResourceName())) {
replacedert.replacedertEquals("y", req.getNodeLabelExpression());
} else {
replacedert.replacedertNull(req.getNodeLabelExpression());
}
}
// set container with nodes and racks with labels
client.addContainerRequest(new ContainerRequest(Resource.newInstance(1024, 1), new String[] { "rack1" }, new String[] { "node1", "node2" }, Priority.UNDEFINED, true, "y"));
for (ResourceRequest req : client.ask) {
if (ResourceRequest.ANY.equals(req.getResourceName())) {
replacedert.replacedertEquals("y", req.getNodeLabelExpression());
} else {
replacedert.replacedertNull(req.getNodeLabelExpression());
}
}
}
17
Source : FairSchedulerTestBase.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
protected void createSchedulingRequestExistingApplication(int memory, int vcores, int priority, ApplicationAttemptId attId) {
ResourceRequest request = createResourceRequest(memory, vcores, ResourceRequest.ANY, priority, 1, true);
createSchedulingRequestExistingApplication(request, attId);
}
17
Source : FairSchedulerTestBase.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
protected void createSchedulingRequestExistingApplication(int memory, int priority, ApplicationAttemptId attId) {
ResourceRequest request = createResourceRequest(memory, ResourceRequest.ANY, priority, 1, true);
createSchedulingRequestExistingApplication(request, attId);
}
17
Source : FairSchedulerTestBase.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
protected void createSchedulingRequestExistingApplication(ResourceRequest request, ApplicationAttemptId attId) {
List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
ask.add(request);
scheduler.allocate(attId, ask, new ArrayList<ContainerId>(), null, null);
}
17
Source : MockRMApp.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
public clreplaced MockRMApp implements RMApp {
// ms
static final int DT = 1000000;
String user = MockApps.newUserName();
String name = MockApps.newAppName();
String queue = MockApps.newQueue();
long start = System.currentTimeMillis() - (int) (Math.random() * DT);
long submit = start - (int) (Math.random() * DT);
long finish = 0;
RMAppState state = RMAppState.NEW;
int failCount = 0;
ApplicationId id;
String url = null;
String oUrl = null;
StringBuilder diagnostics = new StringBuilder();
RMAppAttempt attempt;
int maxAppAttempts = 1;
ResourceRequest amReq;
public MockRMApp(int newid, long time, RMAppState newState) {
finish = time;
id = MockApps.newAppID(newid);
state = newState;
}
public MockRMApp(int newid, long time, RMAppState newState, String userName) {
this(newid, time, newState);
user = userName;
}
public MockRMApp(int newid, long time, RMAppState newState, String userName, String diag) {
this(newid, time, newState, userName);
this.diagnostics = new StringBuilder(diag);
}
@Override
public ApplicationId getApplicationId() {
return id;
}
@Override
public ApplicationSubmissionContext getApplicationSubmissionContext() {
return new ApplicationSubmissionContextPBImpl();
}
@Override
public RMAppState getState() {
return state;
}
public void setState(RMAppState state) {
this.state = state;
}
@Override
public String getUser() {
return user;
}
public void setUser(String user) {
this.user = user;
}
@Override
public float getProgress() {
return (float) 0.0;
}
@Override
public RMAppAttempt getRMAppAttempt(ApplicationAttemptId appAttemptId) {
throw new UnsupportedOperationException("Not supported yet.");
}
@Override
public String getQueue() {
return queue;
}
public void setQueue(String queue) {
this.queue = queue;
}
@Override
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
@Override
public Map<ApplicationAttemptId, RMAppAttempt> getAppAttempts() {
Map<ApplicationAttemptId, RMAppAttempt> attempts = new LinkedHashMap<ApplicationAttemptId, RMAppAttempt>();
if (attempt != null) {
attempts.put(attempt.getAppAttemptId(), attempt);
}
return attempts;
}
@Override
public RMAppAttempt getCurrentAppAttempt() {
return attempt;
}
public void setCurrentAppAttempt(RMAppAttempt attempt) {
this.attempt = attempt;
}
@Override
public ApplicationReport createAndGetApplicationReport(String clientUserName, boolean allowAccess) {
throw new UnsupportedOperationException("Not supported yet.");
}
@Override
public long getFinishTime() {
return finish;
}
public void setFinishTime(long time) {
this.finish = time;
}
@Override
public long getStartTime() {
return start;
}
@Override
public long getSubmitTime() {
return submit;
}
public void setStartTime(long time) {
this.start = time;
}
@Override
public String getTrackingUrl() {
return url;
}
public void setTrackingUrl(String url) {
this.url = url;
}
@Override
public String getOriginalTrackingUrl() {
return oUrl;
}
public void setOriginalTrackingUrl(String oUrl) {
this.oUrl = oUrl;
}
@Override
public StringBuilder getDiagnostics() {
return diagnostics;
}
public void setDiagnostics(String diag) {
this.diagnostics = new StringBuilder(diag);
}
@Override
public int getMaxAppAttempts() {
return maxAppAttempts;
}
public void setNumMaxRetries(int maxAppAttempts) {
this.maxAppAttempts = maxAppAttempts;
}
@Override
public void handle(RMAppEvent event) {
}
@Override
public FinalApplicationStatus getFinalApplicationStatus() {
return FinalApplicationStatus.UNDEFINED;
}
@Override
public int pullRMNodeUpdates(Collection<RMNode> updatedNodes) {
throw new UnsupportedOperationException("Not supported yet.");
}
@Override
public String getApplicationType() {
return YarnConfiguration.DEFAULT_APPLICATION_TYPE;
}
@Override
public Set<String> getApplicationTags() {
return null;
}
@Override
public boolean isAppFinalStateStored() {
return true;
}
@Override
public YarnApplicationState createApplicationState() {
return null;
}
@Override
public Set<NodeId> getRanNodes() {
return null;
}
public Resource getResourcePreempted() {
throw new UnsupportedOperationException("Not supported yet.");
}
@Override
public RMAppMetrics getRMAppMetrics() {
throw new UnsupportedOperationException("Not supported yet.");
}
@Override
public ReservationId getReservationId() {
throw new UnsupportedOperationException("Not supported yet.");
}
@Override
public ResourceRequest getAMResourceRequest() {
return this.amReq;
}
}
17
Source : MockAM.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
public List<ResourceRequest> createReq(String[] hosts, int memory, int priority, int containers, String labelExpression) throws Exception {
List<ResourceRequest> reqs = new ArrayList<ResourceRequest>();
for (String host : hosts) {
// only add host/rack request when asked host isn't ANY
if (!host.equals(ResourceRequest.ANY)) {
ResourceRequest hostReq = createResourceReq(host, memory, priority, containers, labelExpression);
reqs.add(hostReq);
ResourceRequest rackReq = createResourceReq("/default-rack", memory, priority, containers, labelExpression);
reqs.add(rackReq);
}
}
ResourceRequest offRackReq = createResourceReq(ResourceRequest.ANY, memory, priority, containers, labelExpression);
reqs.add(offRackReq);
return reqs;
}
17
Source : FifoScheduler.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private int replacedignRackLocalContainers(FiCaSchedulerNode node, FiCaSchedulerApp application, Priority priority) {
int replacedignedContainers = 0;
ResourceRequest request = application.getResourceRequest(priority, node.getRMNode().getRackName());
if (request != null) {
// Don't allocate on this rack if the application doens't need containers
ResourceRequest offSwitchRequest = application.getResourceRequest(priority, ResourceRequest.ANY);
if (offSwitchRequest.getNumContainers() <= 0) {
return 0;
}
int replacedignableContainers = Math.min(getMaxAllocatableContainers(application, priority, node, NodeType.RACK_LOCAL), request.getNumContainers());
replacedignedContainers = replacedignContainer(node, application, priority, replacedignableContainers, request, NodeType.RACK_LOCAL);
}
return replacedignedContainers;
}
17
Source : FifoScheduler.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private int replacedignOffSwitchContainers(FiCaSchedulerNode node, FiCaSchedulerApp application, Priority priority) {
int replacedignedContainers = 0;
ResourceRequest request = application.getResourceRequest(priority, ResourceRequest.ANY);
if (request != null) {
replacedignedContainers = replacedignContainer(node, application, priority, request.getNumContainers(), request, NodeType.OFF_SWITCH);
}
return replacedignedContainers;
}
17
Source : FSAppAttempt.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
/**
* Whether this app has containers requests that could be satisfied on the
* given node, if the node had full space.
*/
public boolean hasContainerForNode(Priority prio, FSSchedulerNode node) {
ResourceRequest anyRequest = getResourceRequest(prio, ResourceRequest.ANY);
ResourceRequest rackRequest = getResourceRequest(prio, node.getRackName());
ResourceRequest nodeRequest = getResourceRequest(prio, node.getNodeName());
return // There must be outstanding requests at the given priority:
anyRequest != null && anyRequest.getNumContainers() > 0 && // If locality relaxation is turned off at *-level, there must be a
// non-zero request for the node's rack:
(anyRequest.getRelaxLocality() || (rackRequest != null && rackRequest.getNumContainers() > 0)) && // If locality relaxation is turned off at rack-level, there must be a
// non-zero request at the node:
(rackRequest == null || rackRequest.getRelaxLocality() || (nodeRequest != null && nodeRequest.getNumContainers() > 0)) && // The requested container must be able to fit on the node:
Resources.lessThanOrEqual(RESOURCE_CALCULATOR, null, anyRequest.getCapability(), node.getRMNode().getTotalCapability());
}
17
Source : AppSchedulingInfo.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
synchronized private void checkForDeactivation() {
boolean deactivate = true;
for (Priority priority : getPriorities()) {
ResourceRequest request = getResourceRequest(priority, ResourceRequest.ANY);
if (request != null) {
if (request.getNumContainers() > 0) {
deactivate = false;
break;
}
}
}
if (deactivate) {
activeUsersManager.deactivateApplication(user, applicationId);
}
}
17
Source : AppSchedulingInfo.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
public synchronized Resource getResource(Priority priority) {
ResourceRequest request = getResourceRequest(priority, ResourceRequest.ANY);
return (request == null) ? null : request.getCapability();
}
17
Source : AppSchedulingInfo.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
/**
* The {@link ResourceScheduler} is allocating data-local resources to the
* application.
*
* @param allocatedContainers
* resources allocated to the application
*/
synchronized private void allocateOffSwitch(SchedulerNode node, Priority priority, ResourceRequest offSwitchRequest, Container container, List<ResourceRequest> resourceRequests) {
// Update future requirements
decrementOutstanding(offSwitchRequest);
// Update cloned OffRack requests for recovery
resourceRequests.add(cloneResourceRequest(offSwitchRequest));
}
17
Source : RMServerUtils.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
/**
* Utility method to validate a list resource requests, by insuring that the
* requested memory/vcore is non-negative and not greater than max
*/
public static void normalizeAndValidateRequests(List<ResourceRequest> ask, Resource maximumResource, String queueName, YarnScheduler scheduler, RMContext rmContext) throws InvalidResourceRequestException {
for (ResourceRequest resReq : ask) {
SchedulerUtils.normalizeAndvalidateRequest(resReq, maximumResource, queueName, scheduler, rmContext);
}
}
17
Source : BuilderUtils.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
public static ResourceRequest newResourceRequest(Priority priority, String hostName, Resource capability, int numContainers) {
ResourceRequest request = recordFactory.newRecordInstance(ResourceRequest.clreplaced);
request.setPriority(priority);
request.setResourceName(hostName);
request.setCapability(capability);
request.setNumContainers(numContainers);
return request;
}
17
Source : BuilderUtils.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
public static ResourceRequest newResourceRequest(ResourceRequest r) {
ResourceRequest request = recordFactory.newRecordInstance(ResourceRequest.clreplaced);
request.setPriority(r.getPriority());
request.setResourceName(r.getResourceName());
request.setCapability(r.getCapability());
request.setNumContainers(r.getNumContainers());
return request;
}
17
Source : FairSchedulerTestBase.java
with Apache License 2.0
from naver
with Apache License 2.0
from naver
protected void createSchedulingRequestExistingApplication(int memory, int vcores, int gcores, int priority, ApplicationAttemptId attId) {
ResourceRequest request = createResourceRequest(memory, vcores, gcores, ResourceRequest.ANY, priority, 1, true);
createSchedulingRequestExistingApplication(request, attId);
}
17
Source : MockAM.java
with Apache License 2.0
from naver
with Apache License 2.0
from naver
public List<ResourceRequest> createReq(String[] hosts, int memory, int priority, int containers, String labelExpression) throws Exception {
List<ResourceRequest> reqs = new ArrayList<ResourceRequest>();
if (hosts != null) {
for (String host : hosts) {
// only add host/rack request when asked host isn't ANY
if (!host.equals(ResourceRequest.ANY)) {
ResourceRequest hostReq = createResourceReq(host, memory, priority, containers, labelExpression);
reqs.add(hostReq);
ResourceRequest rackReq = createResourceReq("/default-rack", memory, priority, containers, labelExpression);
reqs.add(rackReq);
}
}
}
ResourceRequest offRackReq = createResourceReq(ResourceRequest.ANY, memory, priority, containers, labelExpression);
reqs.add(offRackReq);
return reqs;
}
17
Source : SchedulerUtils.java
with Apache License 2.0
from naver
with Apache License 2.0
from naver
public static void normalizeAndvalidateRequest(ResourceRequest resReq, Resource maximumResource, String queueName, YarnScheduler scheduler, RMContext rmContext) throws InvalidResourceRequestException {
normalizeAndValidateRequest(resReq, maximumResource, queueName, scheduler, false, rmContext, null);
}
17
Source : SchedulerUtils.java
with Apache License 2.0
from naver
with Apache License 2.0
from naver
public static void normalizeAndValidateRequest(ResourceRequest resReq, Resource maximumResource, String queueName, YarnScheduler scheduler, boolean isRecovery, RMContext rmContext) throws InvalidResourceRequestException {
normalizeAndValidateRequest(resReq, maximumResource, queueName, scheduler, isRecovery, rmContext, null);
}
17
Source : AppSchedulingInfo.java
with Apache License 2.0
from naver
with Apache License 2.0
from naver
private void decResourceRequest(String resourceName, Priority priority, ResourceRequest request) {
request.setNumContainers(request.getNumContainers() - 1);
if (request.getNumContainers() == 0) {
requests.get(priority).remove(resourceName);
}
}
17
Source : BuilderUtils.java
with Apache License 2.0
from naver
with Apache License 2.0
from naver
public static ResourceRequest newResourceRequest(Priority priority, String hostName, Resource capability, int numContainers, String label) {
ResourceRequest request = recordFactory.newRecordInstance(ResourceRequest.clreplaced);
request.setPriority(priority);
request.setResourceName(hostName);
request.setCapability(capability);
request.setNumContainers(numContainers);
request.setNodeLabelExpression(label);
return request;
}
17
Source : RMContainerRequestor.java
with Apache License 2.0
from naver
with Apache License 2.0
from naver
private void addResourceRequestToAsk(ResourceRequest remoteRequest) {
// because objects inside the resource map can be deleted ask can end up
// containing an object that matches new resource object but with different
// numContainers. So existing values must be replaced explicitly
ask.remove(remoteRequest);
ask.add(remoteRequest);
}
17
Source : RMContainerRequestor.java
with Apache License 2.0
from lsds
with Apache License 2.0
from lsds
private void addResourceRequestToAsk(ResourceRequest remoteRequest) {
// because objects inside the resource map can be deleted ask can end up
// containing an object that matches new resource object but with different
// numContainers. So exisintg values must be replaced explicitly
if (ask.contains(remoteRequest)) {
ask.remove(remoteRequest);
}
ask.add(remoteRequest);
}
16
Source : TestSchedulerUtils.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Test(timeout = 30000)
public void testNormalizeRequestWithDominantResourceCalculator() {
ResourceCalculator resourceCalculator = new DominantResourceCalculator();
Resource minResource = Resources.createResource(1024, 1);
Resource maxResource = Resources.createResource(10240, 10);
Resource clusterResource = Resources.createResource(10 * 1024, 10);
ResourceRequest ask = new ResourceRequestPBImpl();
// case negative memory/vcores
ask.setCapability(Resources.createResource(-1024, -1));
SchedulerUtils.normalizeRequest(ask, resourceCalculator, clusterResource, minResource, maxResource);
replacedertEquals(minResource, ask.getCapability());
// case zero memory/vcores
ask.setCapability(Resources.createResource(0, 0));
SchedulerUtils.normalizeRequest(ask, resourceCalculator, clusterResource, minResource, maxResource);
replacedertEquals(minResource, ask.getCapability());
replacedertEquals(1, ask.getCapability().getVirtualCores());
replacedertEquals(1024, ask.getCapability().getMemory());
// case non-zero memory & zero cores
ask.setCapability(Resources.createResource(1536, 0));
SchedulerUtils.normalizeRequest(ask, resourceCalculator, clusterResource, minResource, maxResource);
replacedertEquals(Resources.createResource(2048, 1), ask.getCapability());
replacedertEquals(1, ask.getCapability().getVirtualCores());
replacedertEquals(2048, ask.getCapability().getMemory());
}
16
Source : TestSchedulerUtils.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Test(timeout = 30000)
public void testNormalizeRequest() {
ResourceCalculator resourceCalculator = new DefaultResourceCalculator();
final int minMemory = 1024;
final int maxMemory = 8192;
Resource minResource = Resources.createResource(minMemory, 0);
Resource maxResource = Resources.createResource(maxMemory, 0);
ResourceRequest ask = new ResourceRequestPBImpl();
// case negative memory
ask.setCapability(Resources.createResource(-1024));
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, maxResource);
replacedertEquals(minMemory, ask.getCapability().getMemory());
// case zero memory
ask.setCapability(Resources.createResource(0));
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, maxResource);
replacedertEquals(minMemory, ask.getCapability().getMemory());
// case memory is a multiple of minMemory
ask.setCapability(Resources.createResource(2 * minMemory));
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, maxResource);
replacedertEquals(2 * minMemory, ask.getCapability().getMemory());
// case memory is not a multiple of minMemory
ask.setCapability(Resources.createResource(minMemory + 10));
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, maxResource);
replacedertEquals(2 * minMemory, ask.getCapability().getMemory());
// case memory is equal to max allowed
ask.setCapability(Resources.createResource(maxMemory));
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, maxResource);
replacedertEquals(maxMemory, ask.getCapability().getMemory());
// case memory is just less than max
ask.setCapability(Resources.createResource(maxMemory - 10));
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, maxResource);
replacedertEquals(maxMemory, ask.getCapability().getMemory());
// max is not a multiple of min
maxResource = Resources.createResource(maxMemory - 10, 0);
ask.setCapability(Resources.createResource(maxMemory - 100));
// multiple of minMemory > maxMemory, then reduce to maxMemory
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, maxResource);
replacedertEquals(maxResource.getMemory(), ask.getCapability().getMemory());
// ask is more than max
maxResource = Resources.createResource(maxMemory, 0);
ask.setCapability(Resources.createResource(maxMemory + 100));
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, maxResource);
replacedertEquals(maxResource.getMemory(), ask.getCapability().getMemory());
}
16
Source : Application.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private void updateResourceRequest(ResourceRequest request) {
request.setNumContainers(request.getNumContainers() - 1);
// Note this for next interaction with ResourceManager
ask.remove(request);
ask.add(org.apache.hadoop.yarn.server.utils.BuilderUtils.newResourceRequest(// clone to ensure the RM doesn't manipulate the same obj
request));
if (LOG.isDebugEnabled()) {
LOG.debug("updateResourceRequest:" + " application=" + applicationId + " request=" + request);
}
}
16
Source : FifoScheduler.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private int getMaxAllocatableContainers(FiCaSchedulerApp application, Priority priority, FiCaSchedulerNode node, NodeType type) {
int maxContainers = 0;
ResourceRequest offSwitchRequest = application.getResourceRequest(priority, ResourceRequest.ANY);
if (offSwitchRequest != null) {
maxContainers = offSwitchRequest.getNumContainers();
}
if (type == NodeType.OFF_SWITCH) {
return maxContainers;
}
if (type == NodeType.RACK_LOCAL) {
ResourceRequest rackLocalRequest = application.getResourceRequest(priority, node.getRMNode().getRackName());
if (rackLocalRequest == null) {
return maxContainers;
}
maxContainers = Math.min(maxContainers, rackLocalRequest.getNumContainers());
}
if (type == NodeType.NODE_LOCAL) {
ResourceRequest nodeLocalRequest = application.getResourceRequest(priority, node.getRMNode().getNodeAddress());
if (nodeLocalRequest != null) {
maxContainers = Math.min(maxContainers, nodeLocalRequest.getNumContainers());
}
}
return maxContainers;
}
16
Source : AppSchedulingInfo.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
/**
* Resources have been allocated to this application by the resource
* scheduler. Track them.
*
* @param type
* the type of the node
* @param node
* the nodeinfo of the node
* @param priority
* the priority of the request.
* @param request
* the request
* @param container
* the containers allocated.
*/
synchronized public List<ResourceRequest> allocate(NodeType type, SchedulerNode node, Priority priority, ResourceRequest request, Container container) {
List<ResourceRequest> resourceRequests = new ArrayList<ResourceRequest>();
if (type == NodeType.NODE_LOCAL) {
allocateNodeLocal(node, priority, request, container, resourceRequests);
} else if (type == NodeType.RACK_LOCAL) {
allocateRackLocal(node, priority, request, container, resourceRequests);
} else {
allocateOffSwitch(node, priority, request, container, resourceRequests);
}
QueueMetrics metrics = queue.getMetrics();
if (pending) {
// once an allocation is done we replacedume the application is
// running from scheduler's POV.
pending = false;
metrics.runAppAttempt(applicationId, user);
}
if (LOG.isDebugEnabled()) {
LOG.debug("allocate: applicationId=" + applicationId + " container=" + container.getId() + " host=" + container.getNodeId().toString() + " user=" + user + " resource=" + request.getCapability());
}
metrics.allocateResources(user, 1, request.getCapability(), true);
return resourceRequests;
}
16
Source : PreemptionResourceRequestPBImpl.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Private
@Unstable
public clreplaced PreemptionResourceRequestPBImpl extends PreemptionResourceRequest {
PreemptionResourceRequestProto proto = PreemptionResourceRequestProto.getDefaultInstance();
PreemptionResourceRequestProto.Builder builder = null;
boolean viaProto = false;
private ResourceRequest rr;
public PreemptionResourceRequestPBImpl() {
builder = PreemptionResourceRequestProto.newBuilder();
}
public PreemptionResourceRequestPBImpl(PreemptionResourceRequestProto proto) {
this.proto = proto;
viaProto = true;
}
public synchronized PreemptionResourceRequestProto getProto() {
mergeLocalToProto();
proto = viaProto ? proto : builder.build();
viaProto = true;
return proto;
}
@Override
public int hashCode() {
return getProto().hashCode();
}
@Override
public boolean equals(Object other) {
if (other == null)
return false;
if (other.getClreplaced().isreplacedignableFrom(this.getClreplaced())) {
return this.getProto().equals(this.getClreplaced().cast(other).getProto());
}
return false;
}
@Override
public String toString() {
return TextFormat.shortDebugString(getProto());
}
private void mergeLocalToProto() {
if (viaProto)
maybeInitBuilder();
mergeLocalToBuilder();
proto = builder.build();
viaProto = true;
}
private void mergeLocalToBuilder() {
if (rr != null) {
builder.setResource(convertToProtoFormat(rr));
}
}
private void maybeInitBuilder() {
if (viaProto || builder == null) {
builder = PreemptionResourceRequestProto.newBuilder(proto);
}
viaProto = false;
}
@Override
public synchronized ResourceRequest getResourceRequest() {
PreemptionResourceRequestProtoOrBuilder p = viaProto ? proto : builder;
if (rr != null) {
return rr;
}
if (!p.hasResource()) {
return null;
}
rr = convertFromProtoFormat(p.getResource());
return rr;
}
@Override
public synchronized void setResourceRequest(final ResourceRequest rr) {
maybeInitBuilder();
if (null == rr) {
builder.clearResource();
}
this.rr = rr;
}
private ResourceRequestPBImpl convertFromProtoFormat(ResourceRequestProto p) {
return new ResourceRequestPBImpl(p);
}
private ResourceRequestProto convertToProtoFormat(ResourceRequest t) {
return ((ResourceRequestPBImpl) t).getProto();
}
}
16
Source : TestAMRMClientContainerRequest.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private void verifyResourceRequest(AMRMClientImpl<ContainerRequest> client, ContainerRequest request, String location, boolean expectedRelaxLocality) {
ResourceRequest ask = client.remoteRequestsTable.get(request.getPriority()).get(location).get(request.getCapability()).remoteRequest;
replacedertEquals(location, ask.getResourceName());
replacedertEquals(1, ask.getNumContainers());
replacedertEquals(expectedRelaxLocality, ask.getRelaxLocality());
}
16
Source : AMRMClientImpl.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private void addResourceRequestToAsk(ResourceRequest remoteRequest) {
// This code looks weird but is needed because of the following scenario.
// A ResourceRequest is removed from the remoteRequestTable. A 0 container
// request is added to 'ask' to notify the RM about not needing it any more.
// Before the call to allocate, the user now requests more containers. If
// the locations of the 0 size request and the new request are the same
// (with the difference being only container count), then the set comparator
// will consider both to be the same and not add the new request to ask. So
// we need to check for the "same" request being present and remove it and
// then add it back. The comparator is container count agnostic.
// This should happen only rarely but we do need to guard against it.
if (ask.contains(remoteRequest)) {
ask.remove(remoteRequest);
}
ask.add(remoteRequest);
}
16
Source : TestSchedulerUtils.java
with Apache License 2.0
from naver
with Apache License 2.0
from naver
@Test(timeout = 30000)
public void testNormalizeRequest() {
ResourceCalculator resourceCalculator = new DefaultResourceCalculator();
final int minMemory = 1024;
final int maxMemory = 8192;
Resource minResource = Resources.createResource(minMemory, 0, 0);
Resource maxResource = Resources.createResource(maxMemory, 0, 0);
ResourceRequest ask = new ResourceRequestPBImpl();
// case negative memory
ask.setCapability(Resources.createResource(-1024));
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, maxResource);
replacedertEquals(minMemory, ask.getCapability().getMemory());
// case zero memory
ask.setCapability(Resources.createResource(0));
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, maxResource);
replacedertEquals(minMemory, ask.getCapability().getMemory());
// case memory is a multiple of minMemory
ask.setCapability(Resources.createResource(2 * minMemory));
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, maxResource);
replacedertEquals(2 * minMemory, ask.getCapability().getMemory());
// case memory is not a multiple of minMemory
ask.setCapability(Resources.createResource(minMemory + 10));
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, maxResource);
replacedertEquals(2 * minMemory, ask.getCapability().getMemory());
// case memory is equal to max allowed
ask.setCapability(Resources.createResource(maxMemory));
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, maxResource);
replacedertEquals(maxMemory, ask.getCapability().getMemory());
// case memory is just less than max
ask.setCapability(Resources.createResource(maxMemory - 10));
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, maxResource);
replacedertEquals(maxMemory, ask.getCapability().getMemory());
// max is not a multiple of min
maxResource = Resources.createResource(maxMemory - 10, 0, 0);
ask.setCapability(Resources.createResource(maxMemory - 100));
// multiple of minMemory > maxMemory, then reduce to maxMemory
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, maxResource);
replacedertEquals(maxResource.getMemory(), ask.getCapability().getMemory());
// ask is more than max
maxResource = Resources.createResource(maxMemory, 0, 0);
ask.setCapability(Resources.createResource(maxMemory + 100));
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, maxResource);
replacedertEquals(maxResource.getMemory(), ask.getCapability().getMemory());
}
16
Source : TestSchedulerUtils.java
with Apache License 2.0
from naver
with Apache License 2.0
from naver
@Test(timeout = 30000)
public void testNormalizeRequestWithDominantResourceCalculator() {
ResourceCalculator resourceCalculator = new DominantResourceCalculator();
Resource minResource = Resources.createResource(1024, 1, 0);
Resource maxResource = Resources.createResource(10240, 10, 10);
Resource clusterResource = Resources.createResource(10 * 1024, 10, 10);
ResourceRequest ask = new ResourceRequestPBImpl();
// case negative memory/vcores/gcores
ask.setCapability(Resources.createResource(-1024, -1, -1));
SchedulerUtils.normalizeRequest(ask, resourceCalculator, clusterResource, minResource, maxResource);
replacedertEquals(minResource, ask.getCapability());
// case zero memory/vcores/gcores
ask.setCapability(Resources.createResource(0, 0, 0));
SchedulerUtils.normalizeRequest(ask, resourceCalculator, clusterResource, minResource, maxResource);
replacedertEquals(minResource, ask.getCapability());
replacedertEquals(1, ask.getCapability().getVirtualCores());
replacedertEquals(1024, ask.getCapability().getMemory());
replacedertEquals(0, ask.getCapability().getGpuCores());
// case non-zero memory & zero cores
ask.setCapability(Resources.createResource(1536, 0, 0));
SchedulerUtils.normalizeRequest(ask, resourceCalculator, clusterResource, minResource, maxResource);
replacedertEquals(Resources.createResource(2048, 1, 0), ask.getCapability());
replacedertEquals(1, ask.getCapability().getVirtualCores());
replacedertEquals(2048, ask.getCapability().getMemory());
replacedertEquals(0, ask.getCapability().getGpuCores());
}
16
Source : SchedulerUtils.java
with Apache License 2.0
from naver
with Apache License 2.0
from naver
public static void normalizeAndvalidateRequest(ResourceRequest resReq, Resource maximumResource, String queueName, YarnScheduler scheduler, RMContext rmContext, QueueInfo queueInfo) throws InvalidResourceRequestException {
normalizeAndValidateRequest(resReq, maximumResource, queueName, scheduler, false, rmContext, queueInfo);
}
See More Examples