org.apache.hadoop.yarn.api.records.ResourceRequest

Here are the examples of the java api org.apache.hadoop.yarn.api.records.ResourceRequest taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

123 Examples 7

19 Source : SchedulerUtils.java
with Apache License 2.0
from NJUJYB

/**
 * Utility method to normalize a list of resource requests, by insuring that
 * the memory for each request is a multiple of minMemory and is not zero.
 */
public static void normalizeRequests(List<ResourceRequest> asks, ResourceCalculator resourceCalculator, Resource clusterResource, Resource minimumResource, Resource maximumResource, Resource incrementResource) {
    for (ResourceRequest ask : asks) {
        normalizeRequest(ask, resourceCalculator, clusterResource, minimumResource, maximumResource, incrementResource);
    }
}

19 Source : SchedulerUtils.java
with Apache License 2.0
from NJUJYB

private static void normalizeNodeLabelExpressionInRequest(ResourceRequest resReq, QueueInfo queueInfo) {
    String labelExp = resReq.getNodeLabelExpression();
    // if queue has default label expression, and RR doesn't have, use the
    // default label expression of queue
    if (labelExp == null && queueInfo != null && ResourceRequest.ANY.equals(resReq.getResourceName())) {
        labelExp = queueInfo.getDefaultNodeLabelExpression();
    }
    // If labelExp still equals to null, set it to be NO_LABEL
    if (labelExp == null) {
        labelExp = RMNodeLabelsManager.NO_LABEL;
    }
    resReq.setNodeLabelExpression(labelExp);
}

19 Source : SchedulerUtils.java
with Apache License 2.0
from NJUJYB

/**
 * Utility method to normalize a list of resource requests, by insuring that
 * the memory for each request is a multiple of minMemory and is not zero.
 */
public static void normalizeRequests(List<ResourceRequest> asks, ResourceCalculator resourceCalculator, Resource clusterResource, Resource minimumResource, Resource maximumResource) {
    for (ResourceRequest ask : asks) {
        normalizeRequest(ask, resourceCalculator, clusterResource, minimumResource, maximumResource, minimumResource);
    }
}

19 Source : AppSchedulingInfo.java
with Apache License 2.0
from NJUJYB

synchronized private void decrementOutstanding(ResourceRequest offSwitchRequest) {
    int numOffSwitchContainers = offSwitchRequest.getNumContainers() - 1;
    // Do not remove ANY
    offSwitchRequest.setNumContainers(numOffSwitchContainers);
    // Do we have any outstanding requests?
    // If there is nothing, we need to deactivate this application
    if (numOffSwitchContainers == 0) {
        checkForDeactivation();
    }
}

19 Source : PreemptionResourceRequestPBImpl.java
with Apache License 2.0
from NJUJYB

private ResourceRequestProto convertToProtoFormat(ResourceRequest t) {
    return ((ResourceRequestPBImpl) t).getProto();
}

19 Source : ApplicationSubmissionContextPBImpl.java
with Apache License 2.0
from NJUJYB

@Override
public void setAMContainerResourceRequest(ResourceRequest request) {
    maybeInitBuilder();
    if (request == null) {
        builder.clearAmContainerResourceRequest();
    }
    this.amResourceRequest = request;
}

19 Source : AppSchedulingInfo.java
with Apache License 2.0
from naver

private boolean isRequestLabelChanged(ResourceRequest requestOne, ResourceRequest requestTwo) {
    String requestOneLabelExp = requestOne.getNodeLabelExpression();
    String requestTwoLabelExp = requestTwo.getNodeLabelExpression();
    return (!(requestOneLabelExp.equals(requestTwoLabelExp)));
}

18 Source : SchedulerUtils.java
with Apache License 2.0
from NJUJYB

public static void normalizeAndvalidateRequest(ResourceRequest resReq, Resource maximumResource, String queueName, YarnScheduler scheduler, RMContext rmContext) throws InvalidResourceRequestException {
    normalizeAndValidateRequest(resReq, maximumResource, queueName, scheduler, false, rmContext);
}

18 Source : SchedulerUtils.java
with Apache License 2.0
from NJUJYB

/**
 * Utility method to validate a resource request, by insuring that the
 * requested memory/vcore is non-negative and not greater than max
 *
 * @throws <code>InvalidResourceRequestException</code> when there is invalid
 *         request
 */
private static void validateResourceRequest(ResourceRequest resReq, Resource maximumResource, QueueInfo queueInfo, RMContext rmContext) throws InvalidResourceRequestException {
    if (resReq.getCapability().getMemory() < 0 || resReq.getCapability().getMemory() > maximumResource.getMemory()) {
        throw new InvalidResourceRequestException("Invalid resource request" + ", requested memory < 0" + ", or requested memory > max configured" + ", requestedMemory=" + resReq.getCapability().getMemory() + ", maxMemory=" + maximumResource.getMemory());
    }
    if (resReq.getCapability().getVirtualCores() < 0 || resReq.getCapability().getVirtualCores() > maximumResource.getVirtualCores()) {
        throw new InvalidResourceRequestException("Invalid resource request" + ", requested virtual cores < 0" + ", or requested virtual cores > max configured" + ", requestedVirtualCores=" + resReq.getCapability().getVirtualCores() + ", maxVirtualCores=" + maximumResource.getVirtualCores());
    }
    String labelExp = resReq.getNodeLabelExpression();
    // we don't allow specify label expression other than resourceName=ANY now
    if (!ResourceRequest.ANY.equals(resReq.getResourceName()) && labelExp != null && !labelExp.trim().isEmpty()) {
        throw new InvalidResourceRequestException("Invailid resource request, queue=" + queueInfo.getQueueName() + " specified node label expression in a " + "resource request has resource name = " + resReq.getResourceName());
    }
    // we don't allow specify label expression with more than one node labels now
    if (labelExp != null && labelExp.contains("&&")) {
        throw new InvalidResourceRequestException("Invailid resource request, queue=" + queueInfo.getQueueName() + " specified more than one node label " + "in a node label expression, node label expression = " + labelExp);
    }
    if (labelExp != null && !labelExp.trim().isEmpty() && queueInfo != null) {
        if (!checkQueueLabelExpression(queueInfo.getAccessibleNodeLabels(), labelExp, rmContext)) {
            throw new InvalidResourceRequestException("Invalid resource request" + ", queue=" + queueInfo.getQueueName() + " doesn't have permission to access all labels " + "in resource request. labelExpression of resource request=" + labelExp + ". Queue labels=" + (queueInfo.getAccessibleNodeLabels() == null ? "" : StringUtils.join(queueInfo.getAccessibleNodeLabels().iterator(), ',')));
        }
    }
}

18 Source : SchedulerUtils.java
with Apache License 2.0
from NJUJYB

/**
 * Utility method to normalize a resource request, by insuring that the
 * requested memory is a multiple of minMemory and is not zero.
 */
public static void normalizeRequest(ResourceRequest ask, ResourceCalculator resourceCalculator, Resource clusterResource, Resource minimumResource, Resource maximumResource) {
    Resource normalized = Resources.normalize(resourceCalculator, ask.getCapability(), minimumResource, maximumResource, minimumResource);
    ask.setCapability(normalized);
}

18 Source : SchedulerUtils.java
with Apache License 2.0
from NJUJYB

/**
 * Utility method to normalize a resource request, by insuring that the
 * requested memory is a multiple of minMemory and is not zero.
 */
public static void normalizeRequest(ResourceRequest ask, ResourceCalculator resourceCalculator, Resource clusterResource, Resource minimumResource, Resource maximumResource, Resource incrementResource) {
    Resource normalized = Resources.normalize(resourceCalculator, ask.getCapability(), minimumResource, maximumResource, incrementResource);
    ask.setCapability(normalized);
}

18 Source : SchedulerUtils.java
with Apache License 2.0
from NJUJYB

public static void normalizeAndValidateRequest(ResourceRequest resReq, Resource maximumResource, String queueName, YarnScheduler scheduler, boolean isRecovery, RMContext rmContext) throws InvalidResourceRequestException {
    QueueInfo queueInfo = null;
    try {
        queueInfo = scheduler.getQueueInfo(queueName, false, false);
    } catch (IOException e) {
    // it is possible queue cannot get when queue mapping is set, just ignore
    // the queueInfo here, and move forward
    }
    SchedulerUtils.normalizeNodeLabelExpressionInRequest(resReq, queueInfo);
    if (!isRecovery) {
        validateResourceRequest(resReq, maximumResource, queueInfo, rmContext);
    }
}

18 Source : FifoScheduler.java
with Apache License 2.0
from NJUJYB

private int replacedignNodeLocalContainers(FiCaSchedulerNode node, FiCaSchedulerApp application, Priority priority) {
    int replacedignedContainers = 0;
    ResourceRequest request = application.getResourceRequest(priority, node.getNodeName());
    if (request != null) {
        // Don't allocate on this node if we don't need containers on this rack
        ResourceRequest rackRequest = application.getResourceRequest(priority, node.getRMNode().getRackName());
        if (rackRequest == null || rackRequest.getNumContainers() <= 0) {
            return 0;
        }
        int replacedignableContainers = Math.min(getMaxAllocatableContainers(application, priority, node, NodeType.NODE_LOCAL), request.getNumContainers());
        replacedignedContainers = replacedignContainer(node, application, priority, replacedignableContainers, request, NodeType.NODE_LOCAL);
    }
    return replacedignedContainers;
}

18 Source : PreemptionResourceRequestPBImpl.java
with Apache License 2.0
from NJUJYB

@Override
public synchronized void setResourceRequest(final ResourceRequest rr) {
    maybeInitBuilder();
    if (null == rr) {
        builder.clearResource();
    }
    this.rr = rr;
}

18 Source : TestAMRMClient.java
with Apache License 2.0
from NJUJYB

@Test(timeout = 30000)
public void testAskWithNodeLabels() {
    AMRMClientImpl<ContainerRequest> client = new AMRMClientImpl<ContainerRequest>();
    // add exp=x to ANY
    client.addContainerRequest(new ContainerRequest(Resource.newInstance(1024, 1), null, null, Priority.UNDEFINED, true, "x"));
    replacedert.replacedertEquals(1, client.ask.size());
    replacedert.replacedertEquals("x", client.ask.iterator().next().getNodeLabelExpression());
    // add exp=x then add exp=a to ANY in same priority, only exp=a should kept
    client.addContainerRequest(new ContainerRequest(Resource.newInstance(1024, 1), null, null, Priority.UNDEFINED, true, "x"));
    client.addContainerRequest(new ContainerRequest(Resource.newInstance(1024, 1), null, null, Priority.UNDEFINED, true, "a"));
    replacedert.replacedertEquals(1, client.ask.size());
    replacedert.replacedertEquals("a", client.ask.iterator().next().getNodeLabelExpression());
    // add exp=x to ANY, rack and node, only resource request has ANY resource
    // name will be replacedigned the label expression
    // add exp=x then add exp=a to ANY in same priority, only exp=a should kept
    client.addContainerRequest(new ContainerRequest(Resource.newInstance(1024, 1), null, null, Priority.UNDEFINED, true, "y"));
    replacedert.replacedertEquals(1, client.ask.size());
    for (ResourceRequest req : client.ask) {
        if (ResourceRequest.ANY.equals(req.getResourceName())) {
            replacedert.replacedertEquals("y", req.getNodeLabelExpression());
        } else {
            replacedert.replacedertNull(req.getNodeLabelExpression());
        }
    }
}

18 Source : AppSchedulingInfo.java
with Apache License 2.0
from naver

synchronized private void decrementOutstanding(ResourceRequest offSwitchRequest) {
    int numOffSwitchContainers = offSwitchRequest.getNumContainers() - 1;
    // Do not remove ANY
    offSwitchRequest.setNumContainers(numOffSwitchContainers);
    // Do we have any outstanding requests?
    // If there is nothing, we need to deactivate this application
    if (numOffSwitchContainers == 0) {
        checkForDeactivation();
    }
    queue.decPendingResource(offSwitchRequest.getNodeLabelExpression(), offSwitchRequest.getCapability());
}

18 Source : TestAMRMClient.java
with Apache License 2.0
from naver

@Test(timeout = 30000)
public void testAskWithNodeLabels() {
    AMRMClientImpl<ContainerRequest> client = new AMRMClientImpl<ContainerRequest>();
    // add exp=x to ANY
    client.addContainerRequest(new ContainerRequest(Resource.newInstance(1024, 1), null, null, Priority.UNDEFINED, true, "x"));
    replacedert.replacedertEquals(1, client.ask.size());
    replacedert.replacedertEquals("x", client.ask.iterator().next().getNodeLabelExpression());
    // add exp=x then add exp=a to ANY in same priority, only exp=a should kept
    client.addContainerRequest(new ContainerRequest(Resource.newInstance(1024, 1), null, null, Priority.UNDEFINED, true, "x"));
    client.addContainerRequest(new ContainerRequest(Resource.newInstance(1024, 1), null, null, Priority.UNDEFINED, true, "a"));
    replacedert.replacedertEquals(1, client.ask.size());
    replacedert.replacedertEquals("a", client.ask.iterator().next().getNodeLabelExpression());
    // add exp=x to ANY, rack and node, only resource request has ANY resource
    // name will be replacedigned the label expression
    // add exp=x then add exp=a to ANY in same priority, only exp=a should kept
    client.addContainerRequest(new ContainerRequest(Resource.newInstance(1024, 1), null, null, Priority.UNDEFINED, true, "y"));
    replacedert.replacedertEquals(1, client.ask.size());
    for (ResourceRequest req : client.ask) {
        if (ResourceRequest.ANY.equals(req.getResourceName())) {
            replacedert.replacedertEquals("y", req.getNodeLabelExpression());
        } else {
            replacedert.replacedertNull(req.getNodeLabelExpression());
        }
    }
    // set container with nodes and racks with labels
    client.addContainerRequest(new ContainerRequest(Resource.newInstance(1024, 1), new String[] { "rack1" }, new String[] { "node1", "node2" }, Priority.UNDEFINED, true, "y"));
    for (ResourceRequest req : client.ask) {
        if (ResourceRequest.ANY.equals(req.getResourceName())) {
            replacedert.replacedertEquals("y", req.getNodeLabelExpression());
        } else {
            replacedert.replacedertNull(req.getNodeLabelExpression());
        }
    }
}

17 Source : FairSchedulerTestBase.java
with Apache License 2.0
from NJUJYB

protected void createSchedulingRequestExistingApplication(int memory, int vcores, int priority, ApplicationAttemptId attId) {
    ResourceRequest request = createResourceRequest(memory, vcores, ResourceRequest.ANY, priority, 1, true);
    createSchedulingRequestExistingApplication(request, attId);
}

17 Source : FairSchedulerTestBase.java
with Apache License 2.0
from NJUJYB

protected void createSchedulingRequestExistingApplication(int memory, int priority, ApplicationAttemptId attId) {
    ResourceRequest request = createResourceRequest(memory, ResourceRequest.ANY, priority, 1, true);
    createSchedulingRequestExistingApplication(request, attId);
}

17 Source : FairSchedulerTestBase.java
with Apache License 2.0
from NJUJYB

protected void createSchedulingRequestExistingApplication(ResourceRequest request, ApplicationAttemptId attId) {
    List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
    ask.add(request);
    scheduler.allocate(attId, ask, new ArrayList<ContainerId>(), null, null);
}

17 Source : MockRMApp.java
with Apache License 2.0
from NJUJYB

public clreplaced MockRMApp implements RMApp {

    // ms
    static final int DT = 1000000;

    String user = MockApps.newUserName();

    String name = MockApps.newAppName();

    String queue = MockApps.newQueue();

    long start = System.currentTimeMillis() - (int) (Math.random() * DT);

    long submit = start - (int) (Math.random() * DT);

    long finish = 0;

    RMAppState state = RMAppState.NEW;

    int failCount = 0;

    ApplicationId id;

    String url = null;

    String oUrl = null;

    StringBuilder diagnostics = new StringBuilder();

    RMAppAttempt attempt;

    int maxAppAttempts = 1;

    ResourceRequest amReq;

    public MockRMApp(int newid, long time, RMAppState newState) {
        finish = time;
        id = MockApps.newAppID(newid);
        state = newState;
    }

    public MockRMApp(int newid, long time, RMAppState newState, String userName) {
        this(newid, time, newState);
        user = userName;
    }

    public MockRMApp(int newid, long time, RMAppState newState, String userName, String diag) {
        this(newid, time, newState, userName);
        this.diagnostics = new StringBuilder(diag);
    }

    @Override
    public ApplicationId getApplicationId() {
        return id;
    }

    @Override
    public ApplicationSubmissionContext getApplicationSubmissionContext() {
        return new ApplicationSubmissionContextPBImpl();
    }

    @Override
    public RMAppState getState() {
        return state;
    }

    public void setState(RMAppState state) {
        this.state = state;
    }

    @Override
    public String getUser() {
        return user;
    }

    public void setUser(String user) {
        this.user = user;
    }

    @Override
    public float getProgress() {
        return (float) 0.0;
    }

    @Override
    public RMAppAttempt getRMAppAttempt(ApplicationAttemptId appAttemptId) {
        throw new UnsupportedOperationException("Not supported yet.");
    }

    @Override
    public String getQueue() {
        return queue;
    }

    public void setQueue(String queue) {
        this.queue = queue;
    }

    @Override
    public String getName() {
        return name;
    }

    public void setName(String name) {
        this.name = name;
    }

    @Override
    public Map<ApplicationAttemptId, RMAppAttempt> getAppAttempts() {
        Map<ApplicationAttemptId, RMAppAttempt> attempts = new LinkedHashMap<ApplicationAttemptId, RMAppAttempt>();
        if (attempt != null) {
            attempts.put(attempt.getAppAttemptId(), attempt);
        }
        return attempts;
    }

    @Override
    public RMAppAttempt getCurrentAppAttempt() {
        return attempt;
    }

    public void setCurrentAppAttempt(RMAppAttempt attempt) {
        this.attempt = attempt;
    }

    @Override
    public ApplicationReport createAndGetApplicationReport(String clientUserName, boolean allowAccess) {
        throw new UnsupportedOperationException("Not supported yet.");
    }

    @Override
    public long getFinishTime() {
        return finish;
    }

    public void setFinishTime(long time) {
        this.finish = time;
    }

    @Override
    public long getStartTime() {
        return start;
    }

    @Override
    public long getSubmitTime() {
        return submit;
    }

    public void setStartTime(long time) {
        this.start = time;
    }

    @Override
    public String getTrackingUrl() {
        return url;
    }

    public void setTrackingUrl(String url) {
        this.url = url;
    }

    @Override
    public String getOriginalTrackingUrl() {
        return oUrl;
    }

    public void setOriginalTrackingUrl(String oUrl) {
        this.oUrl = oUrl;
    }

    @Override
    public StringBuilder getDiagnostics() {
        return diagnostics;
    }

    public void setDiagnostics(String diag) {
        this.diagnostics = new StringBuilder(diag);
    }

    @Override
    public int getMaxAppAttempts() {
        return maxAppAttempts;
    }

    public void setNumMaxRetries(int maxAppAttempts) {
        this.maxAppAttempts = maxAppAttempts;
    }

    @Override
    public void handle(RMAppEvent event) {
    }

    @Override
    public FinalApplicationStatus getFinalApplicationStatus() {
        return FinalApplicationStatus.UNDEFINED;
    }

    @Override
    public int pullRMNodeUpdates(Collection<RMNode> updatedNodes) {
        throw new UnsupportedOperationException("Not supported yet.");
    }

    @Override
    public String getApplicationType() {
        return YarnConfiguration.DEFAULT_APPLICATION_TYPE;
    }

    @Override
    public Set<String> getApplicationTags() {
        return null;
    }

    @Override
    public boolean isAppFinalStateStored() {
        return true;
    }

    @Override
    public YarnApplicationState createApplicationState() {
        return null;
    }

    @Override
    public Set<NodeId> getRanNodes() {
        return null;
    }

    public Resource getResourcePreempted() {
        throw new UnsupportedOperationException("Not supported yet.");
    }

    @Override
    public RMAppMetrics getRMAppMetrics() {
        throw new UnsupportedOperationException("Not supported yet.");
    }

    @Override
    public ReservationId getReservationId() {
        throw new UnsupportedOperationException("Not supported yet.");
    }

    @Override
    public ResourceRequest getAMResourceRequest() {
        return this.amReq;
    }
}

17 Source : MockAM.java
with Apache License 2.0
from NJUJYB

public List<ResourceRequest> createReq(String[] hosts, int memory, int priority, int containers, String labelExpression) throws Exception {
    List<ResourceRequest> reqs = new ArrayList<ResourceRequest>();
    for (String host : hosts) {
        // only add host/rack request when asked host isn't ANY
        if (!host.equals(ResourceRequest.ANY)) {
            ResourceRequest hostReq = createResourceReq(host, memory, priority, containers, labelExpression);
            reqs.add(hostReq);
            ResourceRequest rackReq = createResourceReq("/default-rack", memory, priority, containers, labelExpression);
            reqs.add(rackReq);
        }
    }
    ResourceRequest offRackReq = createResourceReq(ResourceRequest.ANY, memory, priority, containers, labelExpression);
    reqs.add(offRackReq);
    return reqs;
}

17 Source : FifoScheduler.java
with Apache License 2.0
from NJUJYB

private int replacedignRackLocalContainers(FiCaSchedulerNode node, FiCaSchedulerApp application, Priority priority) {
    int replacedignedContainers = 0;
    ResourceRequest request = application.getResourceRequest(priority, node.getRMNode().getRackName());
    if (request != null) {
        // Don't allocate on this rack if the application doens't need containers
        ResourceRequest offSwitchRequest = application.getResourceRequest(priority, ResourceRequest.ANY);
        if (offSwitchRequest.getNumContainers() <= 0) {
            return 0;
        }
        int replacedignableContainers = Math.min(getMaxAllocatableContainers(application, priority, node, NodeType.RACK_LOCAL), request.getNumContainers());
        replacedignedContainers = replacedignContainer(node, application, priority, replacedignableContainers, request, NodeType.RACK_LOCAL);
    }
    return replacedignedContainers;
}

17 Source : FifoScheduler.java
with Apache License 2.0
from NJUJYB

private int replacedignOffSwitchContainers(FiCaSchedulerNode node, FiCaSchedulerApp application, Priority priority) {
    int replacedignedContainers = 0;
    ResourceRequest request = application.getResourceRequest(priority, ResourceRequest.ANY);
    if (request != null) {
        replacedignedContainers = replacedignContainer(node, application, priority, request.getNumContainers(), request, NodeType.OFF_SWITCH);
    }
    return replacedignedContainers;
}

17 Source : FSAppAttempt.java
with Apache License 2.0
from NJUJYB

/**
 * Whether this app has containers requests that could be satisfied on the
 * given node, if the node had full space.
 */
public boolean hasContainerForNode(Priority prio, FSSchedulerNode node) {
    ResourceRequest anyRequest = getResourceRequest(prio, ResourceRequest.ANY);
    ResourceRequest rackRequest = getResourceRequest(prio, node.getRackName());
    ResourceRequest nodeRequest = getResourceRequest(prio, node.getNodeName());
    return // There must be outstanding requests at the given priority:
    anyRequest != null && anyRequest.getNumContainers() > 0 && // If locality relaxation is turned off at *-level, there must be a
    // non-zero request for the node's rack:
    (anyRequest.getRelaxLocality() || (rackRequest != null && rackRequest.getNumContainers() > 0)) && // If locality relaxation is turned off at rack-level, there must be a
    // non-zero request at the node:
    (rackRequest == null || rackRequest.getRelaxLocality() || (nodeRequest != null && nodeRequest.getNumContainers() > 0)) && // The requested container must be able to fit on the node:
    Resources.lessThanOrEqual(RESOURCE_CALCULATOR, null, anyRequest.getCapability(), node.getRMNode().getTotalCapability());
}

17 Source : AppSchedulingInfo.java
with Apache License 2.0
from NJUJYB

synchronized private void checkForDeactivation() {
    boolean deactivate = true;
    for (Priority priority : getPriorities()) {
        ResourceRequest request = getResourceRequest(priority, ResourceRequest.ANY);
        if (request != null) {
            if (request.getNumContainers() > 0) {
                deactivate = false;
                break;
            }
        }
    }
    if (deactivate) {
        activeUsersManager.deactivateApplication(user, applicationId);
    }
}

17 Source : AppSchedulingInfo.java
with Apache License 2.0
from NJUJYB

public synchronized Resource getResource(Priority priority) {
    ResourceRequest request = getResourceRequest(priority, ResourceRequest.ANY);
    return (request == null) ? null : request.getCapability();
}

17 Source : AppSchedulingInfo.java
with Apache License 2.0
from NJUJYB

/**
 * The {@link ResourceScheduler} is allocating data-local resources to the
 * application.
 *
 * @param allocatedContainers
 *          resources allocated to the application
 */
synchronized private void allocateOffSwitch(SchedulerNode node, Priority priority, ResourceRequest offSwitchRequest, Container container, List<ResourceRequest> resourceRequests) {
    // Update future requirements
    decrementOutstanding(offSwitchRequest);
    // Update cloned OffRack requests for recovery
    resourceRequests.add(cloneResourceRequest(offSwitchRequest));
}

17 Source : RMServerUtils.java
with Apache License 2.0
from NJUJYB

/**
 * Utility method to validate a list resource requests, by insuring that the
 * requested memory/vcore is non-negative and not greater than max
 */
public static void normalizeAndValidateRequests(List<ResourceRequest> ask, Resource maximumResource, String queueName, YarnScheduler scheduler, RMContext rmContext) throws InvalidResourceRequestException {
    for (ResourceRequest resReq : ask) {
        SchedulerUtils.normalizeAndvalidateRequest(resReq, maximumResource, queueName, scheduler, rmContext);
    }
}

17 Source : BuilderUtils.java
with Apache License 2.0
from NJUJYB

public static ResourceRequest newResourceRequest(Priority priority, String hostName, Resource capability, int numContainers) {
    ResourceRequest request = recordFactory.newRecordInstance(ResourceRequest.clreplaced);
    request.setPriority(priority);
    request.setResourceName(hostName);
    request.setCapability(capability);
    request.setNumContainers(numContainers);
    return request;
}

17 Source : BuilderUtils.java
with Apache License 2.0
from NJUJYB

public static ResourceRequest newResourceRequest(ResourceRequest r) {
    ResourceRequest request = recordFactory.newRecordInstance(ResourceRequest.clreplaced);
    request.setPriority(r.getPriority());
    request.setResourceName(r.getResourceName());
    request.setCapability(r.getCapability());
    request.setNumContainers(r.getNumContainers());
    return request;
}

17 Source : FairSchedulerTestBase.java
with Apache License 2.0
from naver

protected void createSchedulingRequestExistingApplication(int memory, int vcores, int gcores, int priority, ApplicationAttemptId attId) {
    ResourceRequest request = createResourceRequest(memory, vcores, gcores, ResourceRequest.ANY, priority, 1, true);
    createSchedulingRequestExistingApplication(request, attId);
}

17 Source : MockAM.java
with Apache License 2.0
from naver

public List<ResourceRequest> createReq(String[] hosts, int memory, int priority, int containers, String labelExpression) throws Exception {
    List<ResourceRequest> reqs = new ArrayList<ResourceRequest>();
    if (hosts != null) {
        for (String host : hosts) {
            // only add host/rack request when asked host isn't ANY
            if (!host.equals(ResourceRequest.ANY)) {
                ResourceRequest hostReq = createResourceReq(host, memory, priority, containers, labelExpression);
                reqs.add(hostReq);
                ResourceRequest rackReq = createResourceReq("/default-rack", memory, priority, containers, labelExpression);
                reqs.add(rackReq);
            }
        }
    }
    ResourceRequest offRackReq = createResourceReq(ResourceRequest.ANY, memory, priority, containers, labelExpression);
    reqs.add(offRackReq);
    return reqs;
}

17 Source : SchedulerUtils.java
with Apache License 2.0
from naver

public static void normalizeAndvalidateRequest(ResourceRequest resReq, Resource maximumResource, String queueName, YarnScheduler scheduler, RMContext rmContext) throws InvalidResourceRequestException {
    normalizeAndValidateRequest(resReq, maximumResource, queueName, scheduler, false, rmContext, null);
}

17 Source : SchedulerUtils.java
with Apache License 2.0
from naver

public static void normalizeAndValidateRequest(ResourceRequest resReq, Resource maximumResource, String queueName, YarnScheduler scheduler, boolean isRecovery, RMContext rmContext) throws InvalidResourceRequestException {
    normalizeAndValidateRequest(resReq, maximumResource, queueName, scheduler, isRecovery, rmContext, null);
}

17 Source : AppSchedulingInfo.java
with Apache License 2.0
from naver

private void decResourceRequest(String resourceName, Priority priority, ResourceRequest request) {
    request.setNumContainers(request.getNumContainers() - 1);
    if (request.getNumContainers() == 0) {
        requests.get(priority).remove(resourceName);
    }
}

17 Source : BuilderUtils.java
with Apache License 2.0
from naver

public static ResourceRequest newResourceRequest(Priority priority, String hostName, Resource capability, int numContainers, String label) {
    ResourceRequest request = recordFactory.newRecordInstance(ResourceRequest.clreplaced);
    request.setPriority(priority);
    request.setResourceName(hostName);
    request.setCapability(capability);
    request.setNumContainers(numContainers);
    request.setNodeLabelExpression(label);
    return request;
}

17 Source : RMContainerRequestor.java
with Apache License 2.0
from naver

private void addResourceRequestToAsk(ResourceRequest remoteRequest) {
    // because objects inside the resource map can be deleted ask can end up
    // containing an object that matches new resource object but with different
    // numContainers. So existing values must be replaced explicitly
    ask.remove(remoteRequest);
    ask.add(remoteRequest);
}

17 Source : RMContainerRequestor.java
with Apache License 2.0
from lsds

private void addResourceRequestToAsk(ResourceRequest remoteRequest) {
    // because objects inside the resource map can be deleted ask can end up
    // containing an object that matches new resource object but with different
    // numContainers. So exisintg values must be replaced explicitly
    if (ask.contains(remoteRequest)) {
        ask.remove(remoteRequest);
    }
    ask.add(remoteRequest);
}

16 Source : TestSchedulerUtils.java
with Apache License 2.0
from NJUJYB

@Test(timeout = 30000)
public void testNormalizeRequestWithDominantResourceCalculator() {
    ResourceCalculator resourceCalculator = new DominantResourceCalculator();
    Resource minResource = Resources.createResource(1024, 1);
    Resource maxResource = Resources.createResource(10240, 10);
    Resource clusterResource = Resources.createResource(10 * 1024, 10);
    ResourceRequest ask = new ResourceRequestPBImpl();
    // case negative memory/vcores
    ask.setCapability(Resources.createResource(-1024, -1));
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, clusterResource, minResource, maxResource);
    replacedertEquals(minResource, ask.getCapability());
    // case zero memory/vcores
    ask.setCapability(Resources.createResource(0, 0));
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, clusterResource, minResource, maxResource);
    replacedertEquals(minResource, ask.getCapability());
    replacedertEquals(1, ask.getCapability().getVirtualCores());
    replacedertEquals(1024, ask.getCapability().getMemory());
    // case non-zero memory & zero cores
    ask.setCapability(Resources.createResource(1536, 0));
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, clusterResource, minResource, maxResource);
    replacedertEquals(Resources.createResource(2048, 1), ask.getCapability());
    replacedertEquals(1, ask.getCapability().getVirtualCores());
    replacedertEquals(2048, ask.getCapability().getMemory());
}

16 Source : TestSchedulerUtils.java
with Apache License 2.0
from NJUJYB

@Test(timeout = 30000)
public void testNormalizeRequest() {
    ResourceCalculator resourceCalculator = new DefaultResourceCalculator();
    final int minMemory = 1024;
    final int maxMemory = 8192;
    Resource minResource = Resources.createResource(minMemory, 0);
    Resource maxResource = Resources.createResource(maxMemory, 0);
    ResourceRequest ask = new ResourceRequestPBImpl();
    // case negative memory
    ask.setCapability(Resources.createResource(-1024));
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, maxResource);
    replacedertEquals(minMemory, ask.getCapability().getMemory());
    // case zero memory
    ask.setCapability(Resources.createResource(0));
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, maxResource);
    replacedertEquals(minMemory, ask.getCapability().getMemory());
    // case memory is a multiple of minMemory
    ask.setCapability(Resources.createResource(2 * minMemory));
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, maxResource);
    replacedertEquals(2 * minMemory, ask.getCapability().getMemory());
    // case memory is not a multiple of minMemory
    ask.setCapability(Resources.createResource(minMemory + 10));
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, maxResource);
    replacedertEquals(2 * minMemory, ask.getCapability().getMemory());
    // case memory is equal to max allowed
    ask.setCapability(Resources.createResource(maxMemory));
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, maxResource);
    replacedertEquals(maxMemory, ask.getCapability().getMemory());
    // case memory is just less than max
    ask.setCapability(Resources.createResource(maxMemory - 10));
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, maxResource);
    replacedertEquals(maxMemory, ask.getCapability().getMemory());
    // max is not a multiple of min
    maxResource = Resources.createResource(maxMemory - 10, 0);
    ask.setCapability(Resources.createResource(maxMemory - 100));
    // multiple of minMemory > maxMemory, then reduce to maxMemory
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, maxResource);
    replacedertEquals(maxResource.getMemory(), ask.getCapability().getMemory());
    // ask is more than max
    maxResource = Resources.createResource(maxMemory, 0);
    ask.setCapability(Resources.createResource(maxMemory + 100));
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, maxResource);
    replacedertEquals(maxResource.getMemory(), ask.getCapability().getMemory());
}

16 Source : Application.java
with Apache License 2.0
from NJUJYB

private void updateResourceRequest(ResourceRequest request) {
    request.setNumContainers(request.getNumContainers() - 1);
    // Note this for next interaction with ResourceManager
    ask.remove(request);
    ask.add(org.apache.hadoop.yarn.server.utils.BuilderUtils.newResourceRequest(// clone to ensure the RM doesn't manipulate the same obj
    request));
    if (LOG.isDebugEnabled()) {
        LOG.debug("updateResourceRequest:" + " application=" + applicationId + " request=" + request);
    }
}

16 Source : FifoScheduler.java
with Apache License 2.0
from NJUJYB

private int getMaxAllocatableContainers(FiCaSchedulerApp application, Priority priority, FiCaSchedulerNode node, NodeType type) {
    int maxContainers = 0;
    ResourceRequest offSwitchRequest = application.getResourceRequest(priority, ResourceRequest.ANY);
    if (offSwitchRequest != null) {
        maxContainers = offSwitchRequest.getNumContainers();
    }
    if (type == NodeType.OFF_SWITCH) {
        return maxContainers;
    }
    if (type == NodeType.RACK_LOCAL) {
        ResourceRequest rackLocalRequest = application.getResourceRequest(priority, node.getRMNode().getRackName());
        if (rackLocalRequest == null) {
            return maxContainers;
        }
        maxContainers = Math.min(maxContainers, rackLocalRequest.getNumContainers());
    }
    if (type == NodeType.NODE_LOCAL) {
        ResourceRequest nodeLocalRequest = application.getResourceRequest(priority, node.getRMNode().getNodeAddress());
        if (nodeLocalRequest != null) {
            maxContainers = Math.min(maxContainers, nodeLocalRequest.getNumContainers());
        }
    }
    return maxContainers;
}

16 Source : AppSchedulingInfo.java
with Apache License 2.0
from NJUJYB

/**
 * Resources have been allocated to this application by the resource
 * scheduler. Track them.
 *
 * @param type
 *          the type of the node
 * @param node
 *          the nodeinfo of the node
 * @param priority
 *          the priority of the request.
 * @param request
 *          the request
 * @param container
 *          the containers allocated.
 */
synchronized public List<ResourceRequest> allocate(NodeType type, SchedulerNode node, Priority priority, ResourceRequest request, Container container) {
    List<ResourceRequest> resourceRequests = new ArrayList<ResourceRequest>();
    if (type == NodeType.NODE_LOCAL) {
        allocateNodeLocal(node, priority, request, container, resourceRequests);
    } else if (type == NodeType.RACK_LOCAL) {
        allocateRackLocal(node, priority, request, container, resourceRequests);
    } else {
        allocateOffSwitch(node, priority, request, container, resourceRequests);
    }
    QueueMetrics metrics = queue.getMetrics();
    if (pending) {
        // once an allocation is done we replacedume the application is
        // running from scheduler's POV.
        pending = false;
        metrics.runAppAttempt(applicationId, user);
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("allocate: applicationId=" + applicationId + " container=" + container.getId() + " host=" + container.getNodeId().toString() + " user=" + user + " resource=" + request.getCapability());
    }
    metrics.allocateResources(user, 1, request.getCapability(), true);
    return resourceRequests;
}

16 Source : PreemptionResourceRequestPBImpl.java
with Apache License 2.0
from NJUJYB

@Private
@Unstable
public clreplaced PreemptionResourceRequestPBImpl extends PreemptionResourceRequest {

    PreemptionResourceRequestProto proto = PreemptionResourceRequestProto.getDefaultInstance();

    PreemptionResourceRequestProto.Builder builder = null;

    boolean viaProto = false;

    private ResourceRequest rr;

    public PreemptionResourceRequestPBImpl() {
        builder = PreemptionResourceRequestProto.newBuilder();
    }

    public PreemptionResourceRequestPBImpl(PreemptionResourceRequestProto proto) {
        this.proto = proto;
        viaProto = true;
    }

    public synchronized PreemptionResourceRequestProto getProto() {
        mergeLocalToProto();
        proto = viaProto ? proto : builder.build();
        viaProto = true;
        return proto;
    }

    @Override
    public int hashCode() {
        return getProto().hashCode();
    }

    @Override
    public boolean equals(Object other) {
        if (other == null)
            return false;
        if (other.getClreplaced().isreplacedignableFrom(this.getClreplaced())) {
            return this.getProto().equals(this.getClreplaced().cast(other).getProto());
        }
        return false;
    }

    @Override
    public String toString() {
        return TextFormat.shortDebugString(getProto());
    }

    private void mergeLocalToProto() {
        if (viaProto)
            maybeInitBuilder();
        mergeLocalToBuilder();
        proto = builder.build();
        viaProto = true;
    }

    private void mergeLocalToBuilder() {
        if (rr != null) {
            builder.setResource(convertToProtoFormat(rr));
        }
    }

    private void maybeInitBuilder() {
        if (viaProto || builder == null) {
            builder = PreemptionResourceRequestProto.newBuilder(proto);
        }
        viaProto = false;
    }

    @Override
    public synchronized ResourceRequest getResourceRequest() {
        PreemptionResourceRequestProtoOrBuilder p = viaProto ? proto : builder;
        if (rr != null) {
            return rr;
        }
        if (!p.hasResource()) {
            return null;
        }
        rr = convertFromProtoFormat(p.getResource());
        return rr;
    }

    @Override
    public synchronized void setResourceRequest(final ResourceRequest rr) {
        maybeInitBuilder();
        if (null == rr) {
            builder.clearResource();
        }
        this.rr = rr;
    }

    private ResourceRequestPBImpl convertFromProtoFormat(ResourceRequestProto p) {
        return new ResourceRequestPBImpl(p);
    }

    private ResourceRequestProto convertToProtoFormat(ResourceRequest t) {
        return ((ResourceRequestPBImpl) t).getProto();
    }
}

16 Source : TestAMRMClientContainerRequest.java
with Apache License 2.0
from NJUJYB

private void verifyResourceRequest(AMRMClientImpl<ContainerRequest> client, ContainerRequest request, String location, boolean expectedRelaxLocality) {
    ResourceRequest ask = client.remoteRequestsTable.get(request.getPriority()).get(location).get(request.getCapability()).remoteRequest;
    replacedertEquals(location, ask.getResourceName());
    replacedertEquals(1, ask.getNumContainers());
    replacedertEquals(expectedRelaxLocality, ask.getRelaxLocality());
}

16 Source : AMRMClientImpl.java
with Apache License 2.0
from NJUJYB

private void addResourceRequestToAsk(ResourceRequest remoteRequest) {
    // This code looks weird but is needed because of the following scenario.
    // A ResourceRequest is removed from the remoteRequestTable. A 0 container
    // request is added to 'ask' to notify the RM about not needing it any more.
    // Before the call to allocate, the user now requests more containers. If
    // the locations of the 0 size request and the new request are the same
    // (with the difference being only container count), then the set comparator
    // will consider both to be the same and not add the new request to ask. So
    // we need to check for the "same" request being present and remove it and
    // then add it back. The comparator is container count agnostic.
    // This should happen only rarely but we do need to guard against it.
    if (ask.contains(remoteRequest)) {
        ask.remove(remoteRequest);
    }
    ask.add(remoteRequest);
}

16 Source : TestSchedulerUtils.java
with Apache License 2.0
from naver

@Test(timeout = 30000)
public void testNormalizeRequest() {
    ResourceCalculator resourceCalculator = new DefaultResourceCalculator();
    final int minMemory = 1024;
    final int maxMemory = 8192;
    Resource minResource = Resources.createResource(minMemory, 0, 0);
    Resource maxResource = Resources.createResource(maxMemory, 0, 0);
    ResourceRequest ask = new ResourceRequestPBImpl();
    // case negative memory
    ask.setCapability(Resources.createResource(-1024));
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, maxResource);
    replacedertEquals(minMemory, ask.getCapability().getMemory());
    // case zero memory
    ask.setCapability(Resources.createResource(0));
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, maxResource);
    replacedertEquals(minMemory, ask.getCapability().getMemory());
    // case memory is a multiple of minMemory
    ask.setCapability(Resources.createResource(2 * minMemory));
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, maxResource);
    replacedertEquals(2 * minMemory, ask.getCapability().getMemory());
    // case memory is not a multiple of minMemory
    ask.setCapability(Resources.createResource(minMemory + 10));
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, maxResource);
    replacedertEquals(2 * minMemory, ask.getCapability().getMemory());
    // case memory is equal to max allowed
    ask.setCapability(Resources.createResource(maxMemory));
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, maxResource);
    replacedertEquals(maxMemory, ask.getCapability().getMemory());
    // case memory is just less than max
    ask.setCapability(Resources.createResource(maxMemory - 10));
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, maxResource);
    replacedertEquals(maxMemory, ask.getCapability().getMemory());
    // max is not a multiple of min
    maxResource = Resources.createResource(maxMemory - 10, 0, 0);
    ask.setCapability(Resources.createResource(maxMemory - 100));
    // multiple of minMemory > maxMemory, then reduce to maxMemory
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, maxResource);
    replacedertEquals(maxResource.getMemory(), ask.getCapability().getMemory());
    // ask is more than max
    maxResource = Resources.createResource(maxMemory, 0, 0);
    ask.setCapability(Resources.createResource(maxMemory + 100));
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource, maxResource);
    replacedertEquals(maxResource.getMemory(), ask.getCapability().getMemory());
}

16 Source : TestSchedulerUtils.java
with Apache License 2.0
from naver

@Test(timeout = 30000)
public void testNormalizeRequestWithDominantResourceCalculator() {
    ResourceCalculator resourceCalculator = new DominantResourceCalculator();
    Resource minResource = Resources.createResource(1024, 1, 0);
    Resource maxResource = Resources.createResource(10240, 10, 10);
    Resource clusterResource = Resources.createResource(10 * 1024, 10, 10);
    ResourceRequest ask = new ResourceRequestPBImpl();
    // case negative memory/vcores/gcores
    ask.setCapability(Resources.createResource(-1024, -1, -1));
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, clusterResource, minResource, maxResource);
    replacedertEquals(minResource, ask.getCapability());
    // case zero memory/vcores/gcores
    ask.setCapability(Resources.createResource(0, 0, 0));
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, clusterResource, minResource, maxResource);
    replacedertEquals(minResource, ask.getCapability());
    replacedertEquals(1, ask.getCapability().getVirtualCores());
    replacedertEquals(1024, ask.getCapability().getMemory());
    replacedertEquals(0, ask.getCapability().getGpuCores());
    // case non-zero memory & zero cores
    ask.setCapability(Resources.createResource(1536, 0, 0));
    SchedulerUtils.normalizeRequest(ask, resourceCalculator, clusterResource, minResource, maxResource);
    replacedertEquals(Resources.createResource(2048, 1, 0), ask.getCapability());
    replacedertEquals(1, ask.getCapability().getVirtualCores());
    replacedertEquals(2048, ask.getCapability().getMemory());
    replacedertEquals(0, ask.getCapability().getGpuCores());
}

16 Source : SchedulerUtils.java
with Apache License 2.0
from naver

public static void normalizeAndvalidateRequest(ResourceRequest resReq, Resource maximumResource, String queueName, YarnScheduler scheduler, RMContext rmContext, QueueInfo queueInfo) throws InvalidResourceRequestException {
    normalizeAndValidateRequest(resReq, maximumResource, queueName, scheduler, false, rmContext, queueInfo);
}

See More Examples