org.apache.hadoop.yarn.api.records.ReservationId

Here are the examples of the java api org.apache.hadoop.yarn.api.records.ReservationId taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

94 Examples 7

19 Source : ReservationInputValidator.java
with Apache License 2.0
from NJUJYB

/**
 * Quick validation on the input to check some obvious fail conditions (fail
 * fast) the input and returns the appropriate {@link Plan} replacedociated with
 * the specified {@link Queue} or throws an exception message illustrating the
 * details of any validation check failures
 *
 * @param reservationSystem the {@link ReservationSystem} to validate against
 * @param request the {@link ReservationUpdateRequest} defining the resources
 *          required over time for the request
 * @return the {@link Plan} to submit the request to
 * @throws YarnException
 */
public Plan validateReservationUpdateRequest(ReservationSystem reservationSystem, ReservationUpdateRequest request) throws YarnException {
    ReservationId reservationId = request.getReservationId();
    Plan plan = validateReservation(reservationSystem, reservationId, AuditConstants.UPDATE_RESERVATION_REQUEST);
    validateReservationDefinition(reservationId, request.getReservationDefinition(), plan, AuditConstants.UPDATE_RESERVATION_REQUEST);
    return plan;
}

19 Source : ReservationInputValidator.java
with Apache License 2.0
from NJUJYB

private Plan validateReservation(ReservationSystem reservationSystem, ReservationId reservationId, String auditConstant) throws YarnException {
    String message = "";
    // check if the reservation id is valid
    if (reservationId == null) {
        message = "Missing reservation id." + " Please try again by specifying a reservation id.";
        RMAuditLogger.logFailure("UNKNOWN", auditConstant, "validate reservation input", "ClientRMService", message);
        throw RPCUtil.getRemoteException(message);
    }
    String queueName = reservationSystem.getQueueForReservation(reservationId);
    if (queueName == null) {
        message = "The specified reservation with ID: " + reservationId + " is unknown. Please try again with a valid reservation.";
        RMAuditLogger.logFailure("UNKNOWN", auditConstant, "validate reservation input", "ClientRMService", message);
        throw RPCUtil.getRemoteException(message);
    }
    // check if the replacedociated plan is valid
    Plan plan = reservationSystem.getPlan(queueName);
    if (plan == null) {
        message = "The specified reservation: " + reservationId + " is not replacedociated with any valid plan." + " Please try again with a valid reservation.";
        RMAuditLogger.logFailure("UNKNOWN", auditConstant, "validate reservation input", "ClientRMService", message);
        throw RPCUtil.getRemoteException(message);
    }
    return plan;
}

19 Source : AbstractReservationSystem.java
with Apache License 2.0
from NJUJYB

@Override
public String getQueueForReservation(ReservationId reservationId) {
    readLock.lock();
    try {
        return resQMap.get(reservationId);
    } finally {
        readLock.unlock();
    }
}

19 Source : ApplicationSubmissionContextPBImpl.java
with Apache License 2.0
from NJUJYB

@Override
public void setReservationID(ReservationId reservationID) {
    maybeInitBuilder();
    if (reservationID == null) {
        builder.clearReservationId();
        return;
    }
    this.reservationId = reservationID;
}

19 Source : ApplicationSubmissionContextPBImpl.java
with Apache License 2.0
from NJUJYB

private ReservationIdProto convertToProtoFormat(ReservationId t) {
    return ((ReservationIdPBImpl) t).getProto();
}

19 Source : Job.java
with Apache License 2.0
from naver

/**
 * Set the reservation to which the job is submitted to
 *
 * @param reservationId the reservationId to set
 */
public void setReservationId(ReservationId reservationId) {
    this.reservationId = reservationId;
}

18 Source : TestNoOverCommitPolicy.java
with Apache License 2.0
from NJUJYB

@Test(expected = MismatchedUserException.clreplaced)
public void testUserMismatch() throws IOException, PlanningException {
    // generate allocation from single tenant that exceed capacity
    int[] f = generateData(3600, (int) (0.5 * totCont));
    ReservationId rid = ReservationSystemTestUtil.getNewReservationId();
    plan.addReservation(new InMemoryReservationAllocation(rid, null, "u1", "dedicated", initTime, initTime + f.length, ReservationSystemTestUtil.generateAllocation(initTime, step, f), res, minAlloc));
    // trying to update a reservation with a mismatching user
    plan.updateReservation(new InMemoryReservationAllocation(rid, null, "u2", "dedicated", initTime, initTime + f.length, ReservationSystemTestUtil.generateAllocation(initTime, step, f), res, minAlloc));
}

18 Source : AppAddedSchedulerEvent.java
with Apache License 2.0
from NJUJYB

public clreplaced AppAddedSchedulerEvent extends SchedulerEvent {

    private final ApplicationId applicationId;

    private final String queue;

    private final String user;

    private final ReservationId reservationID;

    private final boolean isAppRecovering;

    public AppAddedSchedulerEvent(ApplicationId applicationId, String queue, String user) {
        this(applicationId, queue, user, false, null);
    }

    public AppAddedSchedulerEvent(ApplicationId applicationId, String queue, String user, ReservationId reservationID) {
        this(applicationId, queue, user, false, reservationID);
    }

    public AppAddedSchedulerEvent(ApplicationId applicationId, String queue, String user, boolean isAppRecovering, ReservationId reservationID) {
        super(SchedulerEventType.APP_ADDED);
        this.applicationId = applicationId;
        this.queue = queue;
        this.user = user;
        this.reservationID = reservationID;
        this.isAppRecovering = isAppRecovering;
    }

    public ApplicationId getApplicationId() {
        return applicationId;
    }

    public String getQueue() {
        return queue;
    }

    public String getUser() {
        return user;
    }

    public boolean getIsAppRecovering() {
        return isAppRecovering;
    }

    public ReservationId getReservationID() {
        return reservationID;
    }
}

18 Source : InMemoryReservationAllocation.java
with Apache License 2.0
from NJUJYB

/**
 * An in memory implementation of a reservation allocation using the
 * {@link RLESparseResourceAllocation}
 */
clreplaced InMemoryReservationAllocation implements ReservationAllocation {

    private final String planName;

    private final ReservationId reservationID;

    private final String user;

    private final ReservationDefinition contract;

    private final long startTime;

    private final long endTime;

    private final Map<ReservationInterval, ReservationRequest> allocationRequests;

    private boolean hasGang = false;

    private long acceptedAt = -1;

    private RLESparseResourceAllocation resourcesOverTime;

    InMemoryReservationAllocation(ReservationId reservationID, ReservationDefinition contract, String user, String planName, long startTime, long endTime, Map<ReservationInterval, ReservationRequest> allocationRequests, ResourceCalculator calculator, Resource minAlloc) {
        this.contract = contract;
        this.startTime = startTime;
        this.endTime = endTime;
        this.reservationID = reservationID;
        this.user = user;
        this.allocationRequests = allocationRequests;
        this.planName = planName;
        resourcesOverTime = new RLESparseResourceAllocation(calculator, minAlloc);
        for (Map.Entry<ReservationInterval, ReservationRequest> r : allocationRequests.entrySet()) {
            resourcesOverTime.addInterval(r.getKey(), r.getValue());
            if (r.getValue().getConcurrency() > 1) {
                hasGang = true;
            }
        }
    }

    @Override
    public ReservationId getReservationId() {
        return reservationID;
    }

    @Override
    public ReservationDefinition getReservationDefinition() {
        return contract;
    }

    @Override
    public long getStartTime() {
        return startTime;
    }

    @Override
    public long getEndTime() {
        return endTime;
    }

    @Override
    public Map<ReservationInterval, ReservationRequest> getAllocationRequests() {
        return Collections.unmodifiableMap(allocationRequests);
    }

    @Override
    public String getPlanName() {
        return planName;
    }

    @Override
    public String getUser() {
        return user;
    }

    @Override
    public boolean containsGangs() {
        return hasGang;
    }

    @Override
    public void setAcceptanceTimestamp(long acceptedAt) {
        this.acceptedAt = acceptedAt;
    }

    @Override
    public long getAcceptanceTime() {
        return acceptedAt;
    }

    @Override
    public Resource getResourcesAtTime(long tick) {
        if (tick < startTime || tick >= endTime) {
            return Resource.newInstance(0, 0);
        }
        return Resources.clone(resourcesOverTime.getCapacityAtTime(tick));
    }

    @Override
    public String toString() {
        StringBuilder sBuf = new StringBuilder();
        sBuf.append(getReservationId()).append(" user:").append(getUser()).append(" startTime: ").append(getStartTime()).append(" endTime: ").append(getEndTime()).append(" alloc:[").append(resourcesOverTime.toString()).append("] ");
        return sBuf.toString();
    }

    @Override
    public int compareTo(ReservationAllocation other) {
        // reverse order of acceptance
        if (this.getAcceptanceTime() > other.getAcceptanceTime()) {
            return -1;
        }
        if (this.getAcceptanceTime() < other.getAcceptanceTime()) {
            return 1;
        }
        return 0;
    }

    @Override
    public int hashCode() {
        return reservationID.hashCode();
    }

    @Override
    public boolean equals(Object obj) {
        if (this == obj)
            return true;
        if (obj == null)
            return false;
        if (getClreplaced() != obj.getClreplaced())
            return false;
        InMemoryReservationAllocation other = (InMemoryReservationAllocation) obj;
        return this.reservationID.equals(other.getReservationId());
    }
}

18 Source : GreedyReservationAgent.java
with Apache License 2.0
from NJUJYB

@Override
public boolean deleteReservation(ReservationId reservationId, String user, Plan plan) throws PlanningException {
    return plan.deleteReservation(reservationId);
}

18 Source : AbstractReservationSystem.java
with Apache License 2.0
from NJUJYB

@Override
public ReservationId getNewReservationId() {
    writeLock.lock();
    try {
        ReservationId resId = ReservationId.newInstance(ResourceManager.getClusterTimeStamp(), resCounter.incrementAndGet());
        LOG.info("Allocated new reservationId: " + resId);
        return resId;
    } finally {
        writeLock.unlock();
    }
}

18 Source : AbstractReservationSystem.java
with Apache License 2.0
from NJUJYB

@Override
public void setQueueForReservation(ReservationId reservationId, String queueName) {
    writeLock.lock();
    try {
        resQMap.put(reservationId, queueName);
    } finally {
        writeLock.unlock();
    }
}

18 Source : ReservationUpdateRequestPBImpl.java
with Apache License 2.0
from NJUJYB

@Override
public void setReservationId(ReservationId reservationId) {
    maybeInitBuilder();
    if (reservationId == null) {
        builder.clearReservationId();
        return;
    }
    this.reservationId = reservationId;
}

18 Source : ReservationSubmissionResponsePBImpl.java
with Apache License 2.0
from NJUJYB

public clreplaced ReservationSubmissionResponsePBImpl extends ReservationSubmissionResponse {

    ReservationSubmissionResponseProto proto = ReservationSubmissionResponseProto.getDefaultInstance();

    ReservationSubmissionResponseProto.Builder builder = null;

    boolean viaProto = false;

    private ReservationId reservationId;

    public ReservationSubmissionResponsePBImpl() {
        builder = ReservationSubmissionResponseProto.newBuilder();
    }

    public ReservationSubmissionResponsePBImpl(ReservationSubmissionResponseProto proto) {
        this.proto = proto;
        viaProto = true;
    }

    public ReservationSubmissionResponseProto getProto() {
        mergeLocalToProto();
        proto = viaProto ? proto : builder.build();
        viaProto = true;
        return proto;
    }

    private void mergeLocalToBuilder() {
        if (this.reservationId != null) {
            builder.setReservationId(convertToProtoFormat(this.reservationId));
        }
    }

    private void mergeLocalToProto() {
        if (viaProto)
            maybeInitBuilder();
        mergeLocalToBuilder();
        proto = builder.build();
        viaProto = true;
    }

    private void maybeInitBuilder() {
        if (viaProto || builder == null) {
            builder = ReservationSubmissionResponseProto.newBuilder(proto);
        }
        viaProto = false;
    }

    @Override
    public ReservationId getReservationId() {
        ReservationSubmissionResponseProtoOrBuilder p = viaProto ? proto : builder;
        if (reservationId != null) {
            return reservationId;
        }
        if (!p.hasReservationId()) {
            return null;
        }
        reservationId = convertFromProtoFormat(p.getReservationId());
        return reservationId;
    }

    @Override
    public void setReservationId(ReservationId reservationId) {
        maybeInitBuilder();
        if (reservationId == null) {
            builder.clearReservationId();
            return;
        }
        this.reservationId = reservationId;
    }

    private ReservationIdPBImpl convertFromProtoFormat(ReservationIdProto p) {
        return new ReservationIdPBImpl(p);
    }

    private ReservationIdProto convertToProtoFormat(ReservationId t) {
        return ((ReservationIdPBImpl) t).getProto();
    }

    @Override
    public int hashCode() {
        return getProto().hashCode();
    }

    @Override
    public boolean equals(Object other) {
        if (other == null)
            return false;
        if (other.getClreplaced().isreplacedignableFrom(this.getClreplaced())) {
            return this.getProto().equals(this.getClreplaced().cast(other).getProto());
        }
        return false;
    }

    @Override
    public String toString() {
        return TextFormat.shortDebugString(getProto());
    }
}

18 Source : ReservationDeleteRequestPBImpl.java
with Apache License 2.0
from NJUJYB

public clreplaced ReservationDeleteRequestPBImpl extends ReservationDeleteRequest {

    ReservationDeleteRequestProto proto = ReservationDeleteRequestProto.getDefaultInstance();

    ReservationDeleteRequestProto.Builder builder = null;

    boolean viaProto = false;

    private ReservationId reservationId;

    public ReservationDeleteRequestPBImpl() {
        builder = ReservationDeleteRequestProto.newBuilder();
    }

    public ReservationDeleteRequestPBImpl(ReservationDeleteRequestProto proto) {
        this.proto = proto;
        viaProto = true;
    }

    public ReservationDeleteRequestProto getProto() {
        mergeLocalToProto();
        proto = viaProto ? proto : builder.build();
        viaProto = true;
        return proto;
    }

    private void mergeLocalToBuilder() {
        if (this.reservationId != null) {
            builder.setReservationId(convertToProtoFormat(this.reservationId));
        }
    }

    private void mergeLocalToProto() {
        if (viaProto)
            maybeInitBuilder();
        mergeLocalToBuilder();
        proto = builder.build();
        viaProto = true;
    }

    private void maybeInitBuilder() {
        if (viaProto || builder == null) {
            builder = ReservationDeleteRequestProto.newBuilder(proto);
        }
        viaProto = false;
    }

    @Override
    public ReservationId getReservationId() {
        ReservationDeleteRequestProtoOrBuilder p = viaProto ? proto : builder;
        if (reservationId != null) {
            return reservationId;
        }
        if (!p.hasReservationId()) {
            return null;
        }
        reservationId = convertFromProtoFormat(p.getReservationId());
        return reservationId;
    }

    @Override
    public void setReservationId(ReservationId reservationId) {
        maybeInitBuilder();
        if (reservationId == null) {
            builder.clearReservationId();
            return;
        }
        this.reservationId = reservationId;
    }

    private ReservationIdPBImpl convertFromProtoFormat(ReservationIdProto p) {
        return new ReservationIdPBImpl(p);
    }

    private ReservationIdProto convertToProtoFormat(ReservationId t) {
        return ((ReservationIdPBImpl) t).getProto();
    }

    @Override
    public int hashCode() {
        return getProto().hashCode();
    }

    @Override
    public boolean equals(Object other) {
        if (other == null)
            return false;
        if (other.getClreplaced().isreplacedignableFrom(this.getClreplaced())) {
            return this.getProto().equals(this.getClreplaced().cast(other).getProto());
        }
        return false;
    }

    @Override
    public String toString() {
        return TextFormat.shortDebugString(getProto());
    }
}

18 Source : InMemoryReservationAllocation.java
with Apache License 2.0
from naver

/**
 * An in memory implementation of a reservation allocation using the
 * {@link RLESparseResourceAllocation}
 */
clreplaced InMemoryReservationAllocation implements ReservationAllocation {

    private final String planName;

    private final ReservationId reservationID;

    private final String user;

    private final ReservationDefinition contract;

    private final long startTime;

    private final long endTime;

    private final Map<ReservationInterval, ReservationRequest> allocationRequests;

    private boolean hasGang = false;

    private long acceptedAt = -1;

    private RLESparseResourceAllocation resourcesOverTime;

    InMemoryReservationAllocation(ReservationId reservationID, ReservationDefinition contract, String user, String planName, long startTime, long endTime, Map<ReservationInterval, ReservationRequest> allocationRequests, ResourceCalculator calculator, Resource minAlloc) {
        this.contract = contract;
        this.startTime = startTime;
        this.endTime = endTime;
        this.reservationID = reservationID;
        this.user = user;
        this.allocationRequests = allocationRequests;
        this.planName = planName;
        resourcesOverTime = new RLESparseResourceAllocation(calculator, minAlloc);
        for (Map.Entry<ReservationInterval, ReservationRequest> r : allocationRequests.entrySet()) {
            resourcesOverTime.addInterval(r.getKey(), r.getValue());
            if (r.getValue().getConcurrency() > 1) {
                hasGang = true;
            }
        }
    }

    @Override
    public ReservationId getReservationId() {
        return reservationID;
    }

    @Override
    public ReservationDefinition getReservationDefinition() {
        return contract;
    }

    @Override
    public long getStartTime() {
        return startTime;
    }

    @Override
    public long getEndTime() {
        return endTime;
    }

    @Override
    public Map<ReservationInterval, ReservationRequest> getAllocationRequests() {
        return Collections.unmodifiableMap(allocationRequests);
    }

    @Override
    public String getPlanName() {
        return planName;
    }

    @Override
    public String getUser() {
        return user;
    }

    @Override
    public boolean containsGangs() {
        return hasGang;
    }

    @Override
    public void setAcceptanceTimestamp(long acceptedAt) {
        this.acceptedAt = acceptedAt;
    }

    @Override
    public long getAcceptanceTime() {
        return acceptedAt;
    }

    @Override
    public Resource getResourcesAtTime(long tick) {
        if (tick < startTime || tick >= endTime) {
            return Resource.newInstance(0, 0, 0);
        }
        return Resources.clone(resourcesOverTime.getCapacityAtTime(tick));
    }

    @Override
    public String toString() {
        StringBuilder sBuf = new StringBuilder();
        sBuf.append(getReservationId()).append(" user:").append(getUser()).append(" startTime: ").append(getStartTime()).append(" endTime: ").append(getEndTime()).append(" alloc:[").append(resourcesOverTime.toString()).append("] ");
        return sBuf.toString();
    }

    @Override
    public int compareTo(ReservationAllocation other) {
        // reverse order of acceptance
        if (this.getAcceptanceTime() > other.getAcceptanceTime()) {
            return -1;
        }
        if (this.getAcceptanceTime() < other.getAcceptanceTime()) {
            return 1;
        }
        return 0;
    }

    @Override
    public int hashCode() {
        return reservationID.hashCode();
    }

    @Override
    public boolean equals(Object obj) {
        if (this == obj)
            return true;
        if (obj == null)
            return false;
        if (getClreplaced() != obj.getClreplaced())
            return false;
        InMemoryReservationAllocation other = (InMemoryReservationAllocation) obj;
        return this.reservationID.equals(other.getReservationId());
    }
}

18 Source : FairSchedulerPlanFollower.java
with Apache License 2.0
from naver

@Override
protected Resource getReservationQueueResourceIfExists(Plan plan, ReservationId reservationId) {
    String reservationQueueName = getReservationQueueName(plan.getQueueName(), reservationId.toString());
    FSLeafQueue reservationQueue = fs.getQueueManager().getLeafQueue(reservationQueueName, false);
    Resource reservationResource = null;
    if (reservationQueue != null) {
        reservationResource = reservationQueue.getSteadyFairShare();
    }
    return reservationResource;
}

17 Source : TestSimpleCapacityReplanner.java
with Apache License 2.0
from NJUJYB

@Test
public void testReplanningPlanCapacityLoss() throws PlanningException {
    Resource clusterCapacity = Resource.newInstance(100 * 1024, 10);
    Resource minAlloc = Resource.newInstance(1024, 1);
    Resource maxAlloc = Resource.newInstance(1024 * 8, 8);
    ResourceCalculator res = new DefaultResourceCalculator();
    long step = 1L;
    Clock clock = mock(Clock.clreplaced);
    ReservationAgent agent = mock(ReservationAgent.clreplaced);
    SharingPolicy policy = new NoOverCommitPolicy();
    policy.init("root.dedicated", null);
    QueueMetrics queueMetrics = mock(QueueMetrics.clreplaced);
    when(clock.getTime()).thenReturn(0L);
    SimpleCapacityReplanner enf = new SimpleCapacityReplanner(clock);
    CapacitySchedulerConfiguration conf = mock(CapacitySchedulerConfiguration.clreplaced);
    when(conf.getEnforcementWindow(any(String.clreplaced))).thenReturn(6L);
    conf.setLong(CapacitySchedulerConfiguration.PREFIX + "blah" + CapacitySchedulerConfiguration.DOT + CapacitySchedulerConfiguration.RESERVATION_ENFORCEMENT_WINDOW, 6);
    enf.init("blah", conf);
    // Initialize the plan with more resources
    InMemoryPlan plan = new InMemoryPlan(queueMetrics, policy, agent, clusterCapacity, step, res, minAlloc, maxAlloc, "dedicated", enf, true, clock);
    // add reservation filling the plan (separating them 1ms, so we are sure
    // s2 follows s1 on acceptance
    long ts = System.currentTimeMillis();
    ReservationId r1 = ReservationId.newInstance(ts, 1);
    int[] f5 = { 20, 20, 20, 20, 20 };
    replacedertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r1, null, "u3", "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res, minAlloc)));
    when(clock.getTime()).thenReturn(1L);
    ReservationId r2 = ReservationId.newInstance(ts, 2);
    replacedertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r2, null, "u4", "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res, minAlloc)));
    when(clock.getTime()).thenReturn(2L);
    ReservationId r3 = ReservationId.newInstance(ts, 3);
    replacedertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r3, null, "u5", "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res, minAlloc)));
    when(clock.getTime()).thenReturn(3L);
    ReservationId r4 = ReservationId.newInstance(ts, 4);
    replacedertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r4, null, "u6", "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res, minAlloc)));
    when(clock.getTime()).thenReturn(4L);
    ReservationId r5 = ReservationId.newInstance(ts, 5);
    replacedertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r5, null, "u7", "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res, minAlloc)));
    int[] f6 = { 50, 50, 50, 50, 50 };
    ReservationId r6 = ReservationId.newInstance(ts, 6);
    replacedertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r6, null, "u3", "dedicated", 10, 10 + f6.length, generateAllocation(10, f6), res, minAlloc)));
    when(clock.getTime()).thenReturn(6L);
    ReservationId r7 = ReservationId.newInstance(ts, 7);
    replacedertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r7, null, "u4", "dedicated", 10, 10 + f6.length, generateAllocation(10, f6), res, minAlloc)));
    // remove some of the resources (requires replanning)
    plan.setTotalCapacity(Resource.newInstance(70 * 1024, 70));
    when(clock.getTime()).thenReturn(0L);
    // run the replanner
    enf.plan(plan, null);
    // check which reservation are still present
    replacedertNotNull(plan.getReservationById(r1));
    replacedertNotNull(plan.getReservationById(r2));
    replacedertNotNull(plan.getReservationById(r3));
    replacedertNotNull(plan.getReservationById(r6));
    replacedertNotNull(plan.getReservationById(r7));
    // and which ones are removed
    replacedertNull(plan.getReservationById(r4));
    replacedertNull(plan.getReservationById(r5));
    // check resources at each moment in time no more exceed capacity
    for (int i = 0; i < 20; i++) {
        int tot = 0;
        for (ReservationAllocation r : plan.getReservationsAtTime(i)) {
            tot = r.getResourcesAtTime(i).getMemory();
        }
        replacedertTrue(tot <= 70 * 1024);
    }
}

17 Source : TestInMemoryReservationAllocation.java
with Apache License 2.0
from NJUJYB

private void doreplacedertions(ReservationAllocation rAllocation, ReservationId reservationID, ReservationDefinition rDef, Map<ReservationInterval, ReservationRequest> allocations, int start, int[] alloc) {
    replacedert.replacedertEquals(reservationID, rAllocation.getReservationId());
    replacedert.replacedertEquals(rDef, rAllocation.getReservationDefinition());
    replacedert.replacedertEquals(allocations, rAllocation.getAllocationRequests());
    replacedert.replacedertEquals(user, rAllocation.getUser());
    replacedert.replacedertEquals(planName, rAllocation.getPlanName());
    replacedert.replacedertEquals(start, rAllocation.getStartTime());
    replacedert.replacedertEquals(start + alloc.length + 1, rAllocation.getEndTime());
}

17 Source : TestInMemoryPlan.java
with Apache License 2.0
from NJUJYB

private void doreplacedertions(Plan plan, ReservationAllocation rAllocation) {
    ReservationId reservationID = rAllocation.getReservationId();
    replacedert.replacedertNotNull(plan.getReservationById(reservationID));
    replacedert.replacedertEquals(rAllocation, plan.getReservationById(reservationID));
    replacedert.replacedertTrue(((InMemoryPlan) plan).getAllReservations().size() == 1);
    replacedert.replacedertEquals(rAllocation.getEndTime(), plan.getLastEndTime());
    replacedert.replacedertEquals(totalCapacity, plan.getTotalCapacity());
    replacedert.replacedertEquals(minAlloc, plan.getMinimumAllocation());
    replacedert.replacedertEquals(maxAlloc, plan.getMaximumAllocation());
    replacedert.replacedertEquals(resCalc, plan.getResourceCalculator());
    replacedert.replacedertEquals(planName, plan.getQueueName());
    replacedert.replacedertTrue(plan.getMoveOnExpiry());
}

17 Source : ReservationInputValidator.java
with Apache License 2.0
from NJUJYB

/**
 * Quick validation on the input to check some obvious fail conditions (fail
 * fast) the input and returns the appropriate {@link Plan} replacedociated with
 * the specified {@link Queue} or throws an exception message illustrating the
 * details of any validation check failures
 *
 * @param reservationSystem the {@link ReservationSystem} to validate against
 * @param request the {@link ReservationSubmissionRequest} defining the
 *          resources required over time for the request
 * @param reservationId the {@link ReservationId} replacedociated with the current
 *          request
 * @return the {@link Plan} to submit the request to
 * @throws YarnException
 */
public Plan validateReservationSubmissionRequest(ReservationSystem reservationSystem, ReservationSubmissionRequest request, ReservationId reservationId) throws YarnException {
    // Check if it is a managed queue
    String queueName = request.getQueue();
    if (queueName == null || queueName.isEmpty()) {
        String errMsg = "The queue to submit is not specified." + " Please try again with a valid reservable queue.";
        RMAuditLogger.logFailure("UNKNOWN", AuditConstants.SUBMIT_RESERVATION_REQUEST, "validate reservation input", "ClientRMService", errMsg);
        throw RPCUtil.getRemoteException(errMsg);
    }
    Plan plan = reservationSystem.getPlan(queueName);
    if (plan == null) {
        String errMsg = "The specified queue: " + queueName + " is not managed by reservation system." + " Please try again with a valid reservable queue.";
        RMAuditLogger.logFailure("UNKNOWN", AuditConstants.SUBMIT_RESERVATION_REQUEST, "validate reservation input", "ClientRMService", errMsg);
        throw RPCUtil.getRemoteException(errMsg);
    }
    validateReservationDefinition(reservationId, request.getReservationDefinition(), plan, AuditConstants.SUBMIT_RESERVATION_REQUEST);
    return plan;
}

17 Source : ReservationInputValidator.java
with Apache License 2.0
from NJUJYB

private void validateReservationDefinition(ReservationId reservationId, ReservationDefinition contract, Plan plan, String auditConstant) throws YarnException {
    String message = "";
    // check if deadline is in the past
    if (contract == null) {
        message = "Missing reservation definition." + " Please try again by specifying a reservation definition.";
        RMAuditLogger.logFailure("UNKNOWN", auditConstant, "validate reservation input definition", "ClientRMService", message);
        throw RPCUtil.getRemoteException(message);
    }
    if (contract.getDeadline() <= clock.getTime()) {
        message = "The specified deadline: " + contract.getDeadline() + " is the past. Please try again with deadline in the future.";
        RMAuditLogger.logFailure("UNKNOWN", auditConstant, "validate reservation input definition", "ClientRMService", message);
        throw RPCUtil.getRemoteException(message);
    }
    // Check if at least one RR has been specified
    ReservationRequests resReqs = contract.getReservationRequests();
    if (resReqs == null) {
        message = "No resources have been specified to reserve." + "Please try again by specifying the resources to reserve.";
        RMAuditLogger.logFailure("UNKNOWN", auditConstant, "validate reservation input definition", "ClientRMService", message);
        throw RPCUtil.getRemoteException(message);
    }
    List<ReservationRequest> resReq = resReqs.getReservationResources();
    if (resReq == null || resReq.isEmpty()) {
        message = "No resources have been specified to reserve." + " Please try again by specifying the resources to reserve.";
        RMAuditLogger.logFailure("UNKNOWN", auditConstant, "validate reservation input definition", "ClientRMService", message);
        throw RPCUtil.getRemoteException(message);
    }
    // compute minimum duration and max gang size
    long minDuration = 0;
    Resource maxGangSize = Resource.newInstance(0, 0);
    ReservationRequestInterpreter type = contract.getReservationRequests().getInterpreter();
    for (ReservationRequest rr : resReq) {
        if (type == ReservationRequestInterpreter.R_ALL || type == ReservationRequestInterpreter.R_ANY) {
            minDuration = Math.max(minDuration, rr.getDuration());
        } else {
            minDuration += rr.getDuration();
        }
        maxGangSize = Resources.max(plan.getResourceCalculator(), plan.getTotalCapacity(), maxGangSize, Resources.multiply(rr.getCapability(), rr.getConcurrency()));
    }
    // verify the allocation is possible (skip for ANY)
    if (contract.getDeadline() - contract.getArrival() < minDuration && type != ReservationRequestInterpreter.R_ANY) {
        message = "The time difference (" + (contract.getDeadline() - contract.getArrival()) + ") between arrival (" + contract.getArrival() + ") " + "and deadline (" + contract.getDeadline() + ") must " + " be greater or equal to the minimum resource duration (" + minDuration + ")";
        RMAuditLogger.logFailure("UNKNOWN", auditConstant, "validate reservation input definition", "ClientRMService", message);
        throw RPCUtil.getRemoteException(message);
    }
    // check that the largest gang does not exceed the inventory available
    // capacity (skip for ANY)
    if (Resources.greaterThan(plan.getResourceCalculator(), plan.getTotalCapacity(), maxGangSize, plan.getTotalCapacity()) && type != ReservationRequestInterpreter.R_ANY) {
        message = "The size of the largest gang in the reservation refinition (" + maxGangSize + ") exceed the capacity available (" + plan.getTotalCapacity() + " )";
        RMAuditLogger.logFailure("UNKNOWN", auditConstant, "validate reservation input definition", "ClientRMService", message);
        throw RPCUtil.getRemoteException(message);
    }
}

17 Source : InMemoryPlan.java
with Apache License 2.0
from NJUJYB

@Override
public boolean deleteReservation(ReservationId reservationID) {
    writeLock.lock();
    try {
        ReservationAllocation reservation = getReservationById(reservationID);
        if (reservation == null) {
            String errMsg = "The specified Reservation with ID " + reservationID + " does not exist in the plan";
            LOG.error(errMsg);
            throw new IllegalArgumentException(errMsg);
        }
        return removeReservation(reservation);
    } finally {
        writeLock.unlock();
    }
}

17 Source : GreedyReservationAgent.java
with Apache License 2.0
from NJUJYB

@Override
public boolean createReservation(ReservationId reservationId, String user, Plan plan, ReservationDefinition contract) throws PlanningException {
    return computeAllocation(reservationId, user, plan, contract, null);
}

17 Source : GreedyReservationAgent.java
with Apache License 2.0
from NJUJYB

@Override
public boolean updateReservation(ReservationId reservationId, String user, Plan plan, ReservationDefinition contract) throws PlanningException {
    return computeAllocation(reservationId, user, plan, contract, plan.getReservationById(reservationId));
}

17 Source : ReservationUpdateRequestPBImpl.java
with Apache License 2.0
from NJUJYB

public clreplaced ReservationUpdateRequestPBImpl extends ReservationUpdateRequest {

    ReservationUpdateRequestProto proto = ReservationUpdateRequestProto.getDefaultInstance();

    ReservationUpdateRequestProto.Builder builder = null;

    boolean viaProto = false;

    private ReservationDefinition reservationDefinition;

    private ReservationId reservationId;

    public ReservationUpdateRequestPBImpl() {
        builder = ReservationUpdateRequestProto.newBuilder();
    }

    public ReservationUpdateRequestPBImpl(ReservationUpdateRequestProto proto) {
        this.proto = proto;
        viaProto = true;
    }

    public ReservationUpdateRequestProto getProto() {
        mergeLocalToProto();
        proto = viaProto ? proto : builder.build();
        viaProto = true;
        return proto;
    }

    private void mergeLocalToBuilder() {
        if (this.reservationId != null) {
            builder.setReservationId(convertToProtoFormat(this.reservationId));
        }
        if (this.reservationDefinition != null) {
            builder.setReservationDefinition(convertToProtoFormat(reservationDefinition));
        }
    }

    private void mergeLocalToProto() {
        if (viaProto)
            maybeInitBuilder();
        mergeLocalToBuilder();
        proto = builder.build();
        viaProto = true;
    }

    private void maybeInitBuilder() {
        if (viaProto || builder == null) {
            builder = ReservationUpdateRequestProto.newBuilder(proto);
        }
        viaProto = false;
    }

    @Override
    public ReservationDefinition getReservationDefinition() {
        ReservationUpdateRequestProtoOrBuilder p = viaProto ? proto : builder;
        if (reservationDefinition != null) {
            return reservationDefinition;
        }
        if (!p.hasReservationDefinition()) {
            return null;
        }
        reservationDefinition = convertFromProtoFormat(p.getReservationDefinition());
        return reservationDefinition;
    }

    @Override
    public void setReservationDefinition(ReservationDefinition reservationDefinition) {
        maybeInitBuilder();
        if (reservationDefinition == null) {
            builder.clearReservationDefinition();
        }
        this.reservationDefinition = reservationDefinition;
    }

    @Override
    public ReservationId getReservationId() {
        ReservationUpdateRequestProtoOrBuilder p = viaProto ? proto : builder;
        if (reservationId != null) {
            return reservationId;
        }
        if (!p.hasReservationId()) {
            return null;
        }
        reservationId = convertFromProtoFormat(p.getReservationId());
        return reservationId;
    }

    @Override
    public void setReservationId(ReservationId reservationId) {
        maybeInitBuilder();
        if (reservationId == null) {
            builder.clearReservationId();
            return;
        }
        this.reservationId = reservationId;
    }

    private ReservationIdPBImpl convertFromProtoFormat(ReservationIdProto p) {
        return new ReservationIdPBImpl(p);
    }

    private ReservationIdProto convertToProtoFormat(ReservationId t) {
        return ((ReservationIdPBImpl) t).getProto();
    }

    private ReservationDefinitionProto convertToProtoFormat(ReservationDefinition r) {
        return ((ReservationDefinitionPBImpl) r).getProto();
    }

    private ReservationDefinitionPBImpl convertFromProtoFormat(ReservationDefinitionProto r) {
        return new ReservationDefinitionPBImpl(r);
    }

    @Override
    public int hashCode() {
        return getProto().hashCode();
    }

    @Override
    public boolean equals(Object other) {
        if (other == null)
            return false;
        if (other.getClreplaced().isreplacedignableFrom(this.getClreplaced())) {
            return this.getProto().equals(this.getClreplaced().cast(other).getProto());
        }
        return false;
    }

    @Override
    public String toString() {
        return TextFormat.shortDebugString(getProto());
    }
}

17 Source : TestFairSchedulerPlanFollower.java
with Apache License 2.0
from naver

@Override
protected void replacedertReservationQueueDoesNotExist(ReservationId r) {
    Queue q = getReservationQueue(r.toString());
    replacedertNull(q);
}

17 Source : TestFairSchedulerPlanFollower.java
with Apache License 2.0
from naver

@Override
protected void replacedertReservationQueueExists(ReservationId r) {
    Queue q = getReservationQueue(r.toString());
    replacedertNotNull(q);
}

17 Source : ReservationInputValidator.java
with Apache License 2.0
from naver

private void validateReservationDefinition(ReservationId reservationId, ReservationDefinition contract, Plan plan, String auditConstant) throws YarnException {
    String message = "";
    // check if deadline is in the past
    if (contract == null) {
        message = "Missing reservation definition." + " Please try again by specifying a reservation definition.";
        RMAuditLogger.logFailure("UNKNOWN", auditConstant, "validate reservation input definition", "ClientRMService", message);
        throw RPCUtil.getRemoteException(message);
    }
    if (contract.getDeadline() <= clock.getTime()) {
        message = "The specified deadline: " + contract.getDeadline() + " is the past. Please try again with deadline in the future.";
        RMAuditLogger.logFailure("UNKNOWN", auditConstant, "validate reservation input definition", "ClientRMService", message);
        throw RPCUtil.getRemoteException(message);
    }
    // Check if at least one RR has been specified
    ReservationRequests resReqs = contract.getReservationRequests();
    if (resReqs == null) {
        message = "No resources have been specified to reserve." + "Please try again by specifying the resources to reserve.";
        RMAuditLogger.logFailure("UNKNOWN", auditConstant, "validate reservation input definition", "ClientRMService", message);
        throw RPCUtil.getRemoteException(message);
    }
    List<ReservationRequest> resReq = resReqs.getReservationResources();
    if (resReq == null || resReq.isEmpty()) {
        message = "No resources have been specified to reserve." + " Please try again by specifying the resources to reserve.";
        RMAuditLogger.logFailure("UNKNOWN", auditConstant, "validate reservation input definition", "ClientRMService", message);
        throw RPCUtil.getRemoteException(message);
    }
    // compute minimum duration and max gang size
    long minDuration = 0;
    Resource maxGangSize = Resource.newInstance(0, 0, 0);
    ReservationRequestInterpreter type = contract.getReservationRequests().getInterpreter();
    for (ReservationRequest rr : resReq) {
        if (type == ReservationRequestInterpreter.R_ALL || type == ReservationRequestInterpreter.R_ANY) {
            minDuration = Math.max(minDuration, rr.getDuration());
        } else {
            minDuration += rr.getDuration();
        }
        maxGangSize = Resources.max(plan.getResourceCalculator(), plan.getTotalCapacity(), maxGangSize, Resources.multiply(rr.getCapability(), rr.getConcurrency()));
    }
    // verify the allocation is possible (skip for ANY)
    if (contract.getDeadline() - contract.getArrival() < minDuration && type != ReservationRequestInterpreter.R_ANY) {
        message = "The time difference (" + (contract.getDeadline() - contract.getArrival()) + ") between arrival (" + contract.getArrival() + ") " + "and deadline (" + contract.getDeadline() + ") must " + " be greater or equal to the minimum resource duration (" + minDuration + ")";
        RMAuditLogger.logFailure("UNKNOWN", auditConstant, "validate reservation input definition", "ClientRMService", message);
        throw RPCUtil.getRemoteException(message);
    }
    // check that the largest gang does not exceed the inventory available
    // capacity (skip for ANY)
    if (Resources.greaterThan(plan.getResourceCalculator(), plan.getTotalCapacity(), maxGangSize, plan.getTotalCapacity()) && type != ReservationRequestInterpreter.R_ANY) {
        message = "The size of the largest gang in the reservation refinition (" + maxGangSize + ") exceed the capacity available (" + plan.getTotalCapacity() + " )";
        RMAuditLogger.logFailure("UNKNOWN", auditConstant, "validate reservation input definition", "ClientRMService", message);
        throw RPCUtil.getRemoteException(message);
    }
}

17 Source : CapacitySchedulerPlanFollower.java
with Apache License 2.0
from naver

@Override
protected Resource getReservationQueueResourceIfExists(Plan plan, ReservationId reservationId) {
    CSQueue resQueue = cs.getQueue(reservationId.toString());
    Resource reservationResource = null;
    if (resQueue != null) {
        reservationResource = Resources.multiply(cs.getClusterResource(), resQueue.getAbsoluteCapacity());
    }
    return reservationResource;
}

17 Source : Job.java
with Apache License 2.0
from naver

/**
 * The job submitter's view of the Job.
 *
 * <p>It allows the user to configure the
 * job, submit it, control its execution, and query the state. The set methods
 * only work until the job is submitted, afterwards they will throw an
 * IllegalStateException. </p>
 *
 * <p>
 * Normally the user creates the application, describes various facets of the
 * job via {@link Job} and then submits the job and monitor its progress.</p>
 *
 * <p>Here is an example on how to submit a job:</p>
 * <p><blockquote><pre>
 *     // Create a new Job
 *     Job job = Job.getInstance();
 *     job.setJarByClreplaced(MyJob.clreplaced);
 *
 *     // Specify various job-specific parameters
 *     job.setJobName("myjob");
 *
 *     job.setInputPath(new Path("in"));
 *     job.setOutputPath(new Path("out"));
 *
 *     job.setMapperClreplaced(MyJob.MyMapper.clreplaced);
 *     job.setReducerClreplaced(MyJob.MyReducer.clreplaced);
 *
 *     // Submit the job, then poll for progress until the job is complete
 *     job.waitForCompletion(true);
 * </pre></blockquote>
 */
@InterfaceAudience.Public
@InterfaceStability.Evolving
public clreplaced Job extends JobContextImpl implements JobContext {

    private static final Log LOG = LogFactory.getLog(Job.clreplaced);

    @InterfaceStability.Evolving
    public static enum JobState {

        DEFINE, RUNNING
    }

    private static final long MAX_JOBSTATUS_AGE = 1000 * 2;

    public static final String OUTPUT_FILTER = "mapreduce.client.output.filter";

    /**
     * Key in mapred-*.xml that sets completionPollInvervalMillis
     */
    public static final String COMPLETION_POLL_INTERVAL_KEY = "mapreduce.client.completion.pollinterval";

    /**
     * Default completionPollIntervalMillis is 5000 ms.
     */
    static final int DEFAULT_COMPLETION_POLL_INTERVAL = 5000;

    /**
     * Key in mapred-*.xml that sets progMonitorPollIntervalMillis
     */
    public static final String PROGRESS_MONITOR_POLL_INTERVAL_KEY = "mapreduce.client.progressmonitor.pollinterval";

    /**
     * Default progMonitorPollIntervalMillis is 1000 ms.
     */
    static final int DEFAULT_MONITOR_POLL_INTERVAL = 1000;

    public static final String USED_GENERIC_PARSER = "mapreduce.client.genericoptionsparser.used";

    public static final String SUBMIT_REPLICATION = "mapreduce.client.submit.file.replication";

    public static final int DEFAULT_SUBMIT_REPLICATION = 10;

    @InterfaceStability.Evolving
    public static enum TaskStatusFilter {

        NONE, KILLED, FAILED, SUCCEEDED, ALL
    }

    static {
        ConfigUtil.loadResources();
    }

    private JobState state = JobState.DEFINE;

    private JobStatus status;

    private long statustime;

    private Cluster cluster;

    private ReservationId reservationId;

    /**
     * @deprecated Use {@link #getInstance()}
     */
    @Deprecated
    public Job() throws IOException {
        this(new JobConf(new Configuration()));
    }

    /**
     * @deprecated Use {@link #getInstance(Configuration)}
     */
    @Deprecated
    public Job(Configuration conf) throws IOException {
        this(new JobConf(conf));
    }

    /**
     * @deprecated Use {@link #getInstance(Configuration, String)}
     */
    @Deprecated
    public Job(Configuration conf, String jobName) throws IOException {
        this(new JobConf(conf));
        setJobName(jobName);
    }

    Job(JobConf conf) throws IOException {
        super(conf, null);
        // propagate existing user credentials to job
        this.credentials.mergeAll(this.ugi.getCredentials());
        this.cluster = null;
    }

    Job(JobStatus status, JobConf conf) throws IOException {
        this(conf);
        setJobID(status.getJobID());
        this.status = status;
        state = JobState.RUNNING;
    }

    /**
     * Creates a new {@link Job} with no particular {@link Cluster} .
     * A Cluster will be created with a generic {@link Configuration}.
     *
     * @return the {@link Job} , with no connection to a cluster yet.
     * @throws IOException
     */
    public static Job getInstance() throws IOException {
        // create with a null Cluster
        return getInstance(new Configuration());
    }

    /**
     * Creates a new {@link Job} with no particular {@link Cluster} and a
     * given {@link Configuration}.
     *
     * The <code>Job</code> makes a copy of the <code>Configuration</code> so
     * that any necessary internal modifications do not reflect on the incoming
     * parameter.
     *
     * A Cluster will be created from the conf parameter only when it's needed.
     *
     * @param conf the configuration
     * @return the {@link Job} , with no connection to a cluster yet.
     * @throws IOException
     */
    public static Job getInstance(Configuration conf) throws IOException {
        // create with a null Cluster
        JobConf jobConf = new JobConf(conf);
        return new Job(jobConf);
    }

    /**
     * Creates a new {@link Job} with no particular {@link Cluster} and a given jobName.
     * A Cluster will be created from the conf parameter only when it's needed.
     *
     * The <code>Job</code> makes a copy of the <code>Configuration</code> so
     * that any necessary internal modifications do not reflect on the incoming
     * parameter.
     *
     * @param conf the configuration
     * @return the {@link Job} , with no connection to a cluster yet.
     * @throws IOException
     */
    public static Job getInstance(Configuration conf, String jobName) throws IOException {
        // create with a null Cluster
        Job result = getInstance(conf);
        result.setJobName(jobName);
        return result;
    }

    /**
     * Creates a new {@link Job} with no particular {@link Cluster} and given
     * {@link Configuration} and {@link JobStatus}.
     * A Cluster will be created from the conf parameter only when it's needed.
     *
     * The <code>Job</code> makes a copy of the <code>Configuration</code> so
     * that any necessary internal modifications do not reflect on the incoming
     * parameter.
     *
     * @param status job status
     * @param conf job configuration
     * @return the {@link Job} , with no connection to a cluster yet.
     * @throws IOException
     */
    public static Job getInstance(JobStatus status, Configuration conf) throws IOException {
        return new Job(status, new JobConf(conf));
    }

    /**
     * Creates a new {@link Job} with no particular {@link Cluster}.
     * A Cluster will be created from the conf parameter only when it's needed.
     *
     * The <code>Job</code> makes a copy of the <code>Configuration</code> so
     * that any necessary internal modifications do not reflect on the incoming
     * parameter.
     *
     * @param ignored
     * @return the {@link Job} , with no connection to a cluster yet.
     * @throws IOException
     * @deprecated Use {@link #getInstance()}
     */
    @Deprecated
    public static Job getInstance(Cluster ignored) throws IOException {
        return getInstance();
    }

    /**
     * Creates a new {@link Job} with no particular {@link Cluster} and given
     * {@link Configuration}.
     * A Cluster will be created from the conf parameter only when it's needed.
     *
     * The <code>Job</code> makes a copy of the <code>Configuration</code> so
     * that any necessary internal modifications do not reflect on the incoming
     * parameter.
     *
     * @param ignored
     * @param conf job configuration
     * @return the {@link Job} , with no connection to a cluster yet.
     * @throws IOException
     * @deprecated Use {@link #getInstance(Configuration)}
     */
    @Deprecated
    public static Job getInstance(Cluster ignored, Configuration conf) throws IOException {
        return getInstance(conf);
    }

    /**
     * Creates a new {@link Job} with no particular {@link Cluster} and given
     * {@link Configuration} and {@link JobStatus}.
     * A Cluster will be created from the conf parameter only when it's needed.
     *
     * The <code>Job</code> makes a copy of the <code>Configuration</code> so
     * that any necessary internal modifications do not reflect on the incoming
     * parameter.
     *
     * @param cluster cluster
     * @param status job status
     * @param conf job configuration
     * @return the {@link Job} , with no connection to a cluster yet.
     * @throws IOException
     */
    @Private
    public static Job getInstance(Cluster cluster, JobStatus status, Configuration conf) throws IOException {
        Job job = getInstance(status, conf);
        job.setCluster(cluster);
        return job;
    }

    private void ensureState(JobState state) throws IllegalStateException {
        if (state != this.state) {
            throw new IllegalStateException("Job in state " + this.state + " instead of " + state);
        }
        if (state == JobState.RUNNING && cluster == null) {
            throw new IllegalStateException("Job in state " + this.state + ", but it isn't attached to any job tracker!");
        }
    }

    /**
     * Some methods rely on having a recent job status object.  Refresh
     * it, if necessary
     */
    synchronized void ensureFreshStatus() throws IOException {
        if (System.currentTimeMillis() - statustime > MAX_JOBSTATUS_AGE) {
            updateStatus();
        }
    }

    /**
     * Some methods need to update status immediately. So, refresh
     * immediately
     * @throws IOException
     */
    synchronized void updateStatus() throws IOException {
        try {
            this.status = ugi.doAs(new PrivilegedExceptionAction<JobStatus>() {

                @Override
                public JobStatus run() throws IOException, InterruptedException {
                    return cluster.getClient().getJobStatus(status.getJobID());
                }
            });
        } catch (InterruptedException ie) {
            throw new IOException(ie);
        }
        if (this.status == null) {
            throw new IOException("Job status not available ");
        }
        this.statustime = System.currentTimeMillis();
    }

    public JobStatus getStatus() throws IOException, InterruptedException {
        ensureState(JobState.RUNNING);
        updateStatus();
        return status;
    }

    /**
     * Returns the current state of the Job.
     *
     * @return JobStatus#State
     * @throws IOException
     * @throws InterruptedException
     */
    public JobStatus.State getJobState() throws IOException, InterruptedException {
        ensureState(JobState.RUNNING);
        updateStatus();
        return status.getState();
    }

    /**
     * Get the URL where some job progress information will be displayed.
     *
     * @return the URL where some job progress information will be displayed.
     */
    public String getTrackingURL() {
        ensureState(JobState.RUNNING);
        return status.getTrackingUrl().toString();
    }

    /**
     * Get the path of the submitted job configuration.
     *
     * @return the path of the submitted job configuration.
     */
    public String getJobFile() {
        ensureState(JobState.RUNNING);
        return status.getJobFile();
    }

    /**
     * Get start time of the job.
     *
     * @return the start time of the job
     */
    public long getStartTime() {
        ensureState(JobState.RUNNING);
        return status.getStartTime();
    }

    /**
     * Get finish time of the job.
     *
     * @return the finish time of the job
     */
    public long getFinishTime() throws IOException, InterruptedException {
        ensureState(JobState.RUNNING);
        updateStatus();
        return status.getFinishTime();
    }

    /**
     * Get scheduling info of the job.
     *
     * @return the scheduling info of the job
     */
    public String getSchedulingInfo() {
        ensureState(JobState.RUNNING);
        return status.getSchedulingInfo();
    }

    /**
     * Get scheduling info of the job.
     *
     * @return the scheduling info of the job
     */
    public JobPriority getPriority() throws IOException, InterruptedException {
        ensureState(JobState.RUNNING);
        updateStatus();
        return status.getPriority();
    }

    /**
     * The user-specified job name.
     */
    public String getJobName() {
        if (state == JobState.DEFINE) {
            return super.getJobName();
        }
        ensureState(JobState.RUNNING);
        return status.getJobName();
    }

    public String getHistoryUrl() throws IOException, InterruptedException {
        ensureState(JobState.RUNNING);
        updateStatus();
        return status.getHistoryFile();
    }

    public boolean isRetired() throws IOException, InterruptedException {
        ensureState(JobState.RUNNING);
        updateStatus();
        return status.isRetired();
    }

    @Private
    public Cluster getCluster() {
        return cluster;
    }

    /**
     * Only for mocks in unit tests.
     */
    @Private
    private void setCluster(Cluster cluster) {
        this.cluster = cluster;
    }

    /**
     * Dump stats to screen.
     */
    @Override
    public String toString() {
        ensureState(JobState.RUNNING);
        String reasonforFailure = " ";
        int numMaps = 0;
        int numReduces = 0;
        try {
            updateStatus();
            if (status.getState().equals(JobStatus.State.FAILED))
                reasonforFailure = getTaskFailureEventString();
            numMaps = getTaskReports(TaskType.MAP).length;
            numReduces = getTaskReports(TaskType.REDUCE).length;
        } catch (IOException e) {
        } catch (InterruptedException ie) {
        }
        StringBuffer sb = new StringBuffer();
        sb.append("Job: ").append(status.getJobID()).append("\n");
        sb.append("Job File: ").append(status.getJobFile()).append("\n");
        sb.append("Job Tracking URL : ").append(status.getTrackingUrl());
        sb.append("\n");
        sb.append("Uber job : ").append(status.isUber()).append("\n");
        sb.append("Number of maps: ").append(numMaps).append("\n");
        sb.append("Number of reduces: ").append(numReduces).append("\n");
        sb.append("map() completion: ");
        sb.append(status.getMapProgress()).append("\n");
        sb.append("reduce() completion: ");
        sb.append(status.getReduceProgress()).append("\n");
        sb.append("Job state: ");
        sb.append(status.getState()).append("\n");
        sb.append("retired: ").append(status.isRetired()).append("\n");
        sb.append("reason for failure: ").append(reasonforFailure);
        return sb.toString();
    }

    /**
     * @return taskid which caused job failure
     * @throws IOException
     * @throws InterruptedException
     */
    String getTaskFailureEventString() throws IOException, InterruptedException {
        int failCount = 1;
        TaskCompletionEvent lastEvent = null;
        TaskCompletionEvent[] events = ugi.doAs(new PrivilegedExceptionAction<TaskCompletionEvent[]>() {

            @Override
            public TaskCompletionEvent[] run() throws IOException, InterruptedException {
                return cluster.getClient().getTaskCompletionEvents(status.getJobID(), 0, 10);
            }
        });
        for (TaskCompletionEvent event : events) {
            if (event.getStatus().equals(TaskCompletionEvent.Status.FAILED)) {
                failCount++;
                lastEvent = event;
            }
        }
        if (lastEvent == null) {
            return "There are no failed tasks for the job. " + "Job is failed due to some other reason and reason " + "can be found in the logs.";
        }
        String[] taskAttemptID = lastEvent.getTaskAttemptId().toString().split("_", 2);
        String taskID = taskAttemptID[1].substring(0, taskAttemptID[1].length() - 2);
        return (" task " + taskID + " failed " + failCount + " times " + "For details check tasktracker at: " + lastEvent.getTaskTrackerHttp());
    }

    /**
     * Get the information of the current state of the tasks of a job.
     *
     * @param type Type of the task
     * @return the list of all of the map tips.
     * @throws IOException
     */
    public TaskReport[] getTaskReports(TaskType type) throws IOException, InterruptedException {
        ensureState(JobState.RUNNING);
        final TaskType tmpType = type;
        return ugi.doAs(new PrivilegedExceptionAction<TaskReport[]>() {

            public TaskReport[] run() throws IOException, InterruptedException {
                return cluster.getClient().getTaskReports(getJobID(), tmpType);
            }
        });
    }

    /**
     * Get the <i>progress</i> of the job's map-tasks, as a float between 0.0
     * and 1.0.  When all map tasks have completed, the function returns 1.0.
     *
     * @return the progress of the job's map-tasks.
     * @throws IOException
     */
    public float mapProgress() throws IOException {
        ensureState(JobState.RUNNING);
        ensureFreshStatus();
        return status.getMapProgress();
    }

    /**
     * Get the <i>progress</i> of the job's reduce-tasks, as a float between 0.0
     * and 1.0.  When all reduce tasks have completed, the function returns 1.0.
     *
     * @return the progress of the job's reduce-tasks.
     * @throws IOException
     */
    public float reduceProgress() throws IOException {
        ensureState(JobState.RUNNING);
        ensureFreshStatus();
        return status.getReduceProgress();
    }

    /**
     * Get the <i>progress</i> of the job's cleanup-tasks, as a float between 0.0
     * and 1.0.  When all cleanup tasks have completed, the function returns 1.0.
     *
     * @return the progress of the job's cleanup-tasks.
     * @throws IOException
     */
    public float cleanupProgress() throws IOException, InterruptedException {
        ensureState(JobState.RUNNING);
        ensureFreshStatus();
        return status.getCleanupProgress();
    }

    /**
     * Get the <i>progress</i> of the job's setup-tasks, as a float between 0.0
     * and 1.0.  When all setup tasks have completed, the function returns 1.0.
     *
     * @return the progress of the job's setup-tasks.
     * @throws IOException
     */
    public float setupProgress() throws IOException {
        ensureState(JobState.RUNNING);
        ensureFreshStatus();
        return status.getSetupProgress();
    }

    /**
     * Check if the job is finished or not.
     * This is a non-blocking call.
     *
     * @return <code>true</code> if the job is complete, else <code>false</code>.
     * @throws IOException
     */
    public boolean isComplete() throws IOException {
        ensureState(JobState.RUNNING);
        updateStatus();
        return status.isJobComplete();
    }

    /**
     * Check if the job completed successfully.
     *
     * @return <code>true</code> if the job succeeded, else <code>false</code>.
     * @throws IOException
     */
    public boolean isSuccessful() throws IOException {
        ensureState(JobState.RUNNING);
        updateStatus();
        return status.getState() == JobStatus.State.SUCCEEDED;
    }

    /**
     * Kill the running job.  Blocks until all job tasks have been
     * killed as well.  If the job is no longer running, it simply returns.
     *
     * @throws IOException
     */
    public void killJob() throws IOException {
        ensureState(JobState.RUNNING);
        try {
            cluster.getClient().killJob(getJobID());
        } catch (InterruptedException ie) {
            throw new IOException(ie);
        }
    }

    /**
     * Set the priority of a running job.
     * @param priority the new priority for the job.
     * @throws IOException
     */
    public void setPriority(JobPriority priority) throws IOException, InterruptedException {
        if (state == JobState.DEFINE) {
            conf.setJobPriority(org.apache.hadoop.mapred.JobPriority.valueOf(priority.name()));
        } else {
            ensureState(JobState.RUNNING);
            final JobPriority tmpPriority = priority;
            ugi.doAs(new PrivilegedExceptionAction<Object>() {

                @Override
                public Object run() throws IOException, InterruptedException {
                    cluster.getClient().setJobPriority(getJobID(), tmpPriority.toString());
                    return null;
                }
            });
        }
    }

    /**
     * Get events indicating completion (success/failure) of component tasks.
     *
     * @param startFrom index to start fetching events from
     * @param numEvents number of events to fetch
     * @return an array of {@link TaskCompletionEvent}s
     * @throws IOException
     */
    public TaskCompletionEvent[] getTaskCompletionEvents(final int startFrom, final int numEvents) throws IOException, InterruptedException {
        ensureState(JobState.RUNNING);
        return ugi.doAs(new PrivilegedExceptionAction<TaskCompletionEvent[]>() {

            @Override
            public TaskCompletionEvent[] run() throws IOException, InterruptedException {
                return cluster.getClient().getTaskCompletionEvents(getJobID(), startFrom, numEvents);
            }
        });
    }

    /**
     * Get events indicating completion (success/failure) of component tasks.
     *
     * @param startFrom index to start fetching events from
     * @return an array of {@link org.apache.hadoop.mapred.TaskCompletionEvent}s
     * @throws IOException
     */
    public org.apache.hadoop.mapred.TaskCompletionEvent[] getTaskCompletionEvents(final int startFrom) throws IOException {
        try {
            TaskCompletionEvent[] events = getTaskCompletionEvents(startFrom, 10);
            org.apache.hadoop.mapred.TaskCompletionEvent[] retEvents = new org.apache.hadoop.mapred.TaskCompletionEvent[events.length];
            for (int i = 0; i < events.length; i++) {
                retEvents[i] = org.apache.hadoop.mapred.TaskCompletionEvent.downgrade(events[i]);
            }
            return retEvents;
        } catch (InterruptedException ie) {
            throw new IOException(ie);
        }
    }

    /**
     * Kill indicated task attempt.
     * @param taskId the id of the task to kill.
     * @param shouldFail if <code>true</code> the task is failed and added
     *                   to failed tasks list, otherwise it is just killed,
     *                   w/o affecting job failure status.
     */
    @Private
    public boolean killTask(final TaskAttemptID taskId, final boolean shouldFail) throws IOException {
        ensureState(JobState.RUNNING);
        try {
            return ugi.doAs(new PrivilegedExceptionAction<Boolean>() {

                public Boolean run() throws IOException, InterruptedException {
                    return cluster.getClient().killTask(taskId, shouldFail);
                }
            });
        } catch (InterruptedException ie) {
            throw new IOException(ie);
        }
    }

    /**
     * Kill indicated task attempt.
     *
     * @param taskId the id of the task to be terminated.
     * @throws IOException
     */
    public void killTask(final TaskAttemptID taskId) throws IOException {
        killTask(taskId, false);
    }

    /**
     * Fail indicated task attempt.
     *
     * @param taskId the id of the task to be terminated.
     * @throws IOException
     */
    public void failTask(final TaskAttemptID taskId) throws IOException {
        killTask(taskId, true);
    }

    /**
     * Gets the counters for this job. May return null if the job has been
     * retired and the job is no longer in the completed job store.
     *
     * @return the counters for this job.
     * @throws IOException
     */
    public Counters getCounters() throws IOException {
        ensureState(JobState.RUNNING);
        try {
            return ugi.doAs(new PrivilegedExceptionAction<Counters>() {

                @Override
                public Counters run() throws IOException, InterruptedException {
                    return cluster.getClient().getJobCounters(getJobID());
                }
            });
        } catch (InterruptedException ie) {
            throw new IOException(ie);
        }
    }

    /**
     * Gets the diagnostic messages for a given task attempt.
     * @param taskid
     * @return the list of diagnostic messages for the task
     * @throws IOException
     */
    public String[] getTaskDiagnostics(final TaskAttemptID taskid) throws IOException, InterruptedException {
        ensureState(JobState.RUNNING);
        return ugi.doAs(new PrivilegedExceptionAction<String[]>() {

            @Override
            public String[] run() throws IOException, InterruptedException {
                return cluster.getClient().getTaskDiagnostics(taskid);
            }
        });
    }

    /**
     * Set the number of reduce tasks for the job.
     * @param tasks the number of reduce tasks
     * @throws IllegalStateException if the job is submitted
     */
    public void setNumReduceTasks(int tasks) throws IllegalStateException {
        ensureState(JobState.DEFINE);
        conf.setNumReduceTasks(tasks);
    }

    /**
     * Set the current working directory for the default file system.
     *
     * @param dir the new current working directory.
     * @throws IllegalStateException if the job is submitted
     */
    public void setWorkingDirectory(Path dir) throws IOException {
        ensureState(JobState.DEFINE);
        conf.setWorkingDirectory(dir);
    }

    /**
     * Set the {@link InputFormat} for the job.
     * @param cls the <code>InputFormat</code> to use
     * @throws IllegalStateException if the job is submitted
     */
    public void setInputFormatClreplaced(Clreplaced<? extends InputFormat> cls) throws IllegalStateException {
        ensureState(JobState.DEFINE);
        conf.setClreplaced(INPUT_FORMAT_CLreplaced_ATTR, cls, InputFormat.clreplaced);
    }

    /**
     * Set the {@link OutputFormat} for the job.
     * @param cls the <code>OutputFormat</code> to use
     * @throws IllegalStateException if the job is submitted
     */
    public void setOutputFormatClreplaced(Clreplaced<? extends OutputFormat> cls) throws IllegalStateException {
        ensureState(JobState.DEFINE);
        conf.setClreplaced(OUTPUT_FORMAT_CLreplaced_ATTR, cls, OutputFormat.clreplaced);
    }

    /**
     * Set the {@link Mapper} for the job.
     * @param cls the <code>Mapper</code> to use
     * @throws IllegalStateException if the job is submitted
     */
    public void setMapperClreplaced(Clreplaced<? extends Mapper> cls) throws IllegalStateException {
        ensureState(JobState.DEFINE);
        conf.setClreplaced(MAP_CLreplaced_ATTR, cls, Mapper.clreplaced);
    }

    /**
     * Set the Jar by finding where a given clreplaced came from.
     * @param cls the example clreplaced
     */
    public void setJarByClreplaced(Clreplaced<?> cls) {
        ensureState(JobState.DEFINE);
        conf.setJarByClreplaced(cls);
    }

    /**
     * Set the job jar
     */
    public void setJar(String jar) {
        ensureState(JobState.DEFINE);
        conf.setJar(jar);
    }

    /**
     * Set the reported username for this job.
     *
     * @param user the username for this job.
     */
    public void setUser(String user) {
        ensureState(JobState.DEFINE);
        conf.setUser(user);
    }

    /**
     * Set the combiner clreplaced for the job.
     * @param cls the combiner to use
     * @throws IllegalStateException if the job is submitted
     */
    public void setCombinerClreplaced(Clreplaced<? extends Reducer> cls) throws IllegalStateException {
        ensureState(JobState.DEFINE);
        conf.setClreplaced(COMBINE_CLreplaced_ATTR, cls, Reducer.clreplaced);
    }

    /**
     * Set the {@link Reducer} for the job.
     * @param cls the <code>Reducer</code> to use
     * @throws IllegalStateException if the job is submitted
     */
    public void setReducerClreplaced(Clreplaced<? extends Reducer> cls) throws IllegalStateException {
        ensureState(JobState.DEFINE);
        conf.setClreplaced(REDUCE_CLreplaced_ATTR, cls, Reducer.clreplaced);
    }

    /**
     * Set the {@link Parreplacedioner} for the job.
     * @param cls the <code>Parreplacedioner</code> to use
     * @throws IllegalStateException if the job is submitted
     */
    public void setParreplacedionerClreplaced(Clreplaced<? extends Parreplacedioner> cls) throws IllegalStateException {
        ensureState(JobState.DEFINE);
        conf.setClreplaced(PARreplacedIONER_CLreplaced_ATTR, cls, Parreplacedioner.clreplaced);
    }

    /**
     * Set the key clreplaced for the map output data. This allows the user to
     * specify the map output key clreplaced to be different than the final output
     * value clreplaced.
     *
     * @param theClreplaced the map output key clreplaced.
     * @throws IllegalStateException if the job is submitted
     */
    public void setMapOutputKeyClreplaced(Clreplaced<?> theClreplaced) throws IllegalStateException {
        ensureState(JobState.DEFINE);
        conf.setMapOutputKeyClreplaced(theClreplaced);
    }

    /**
     * Set the value clreplaced for the map output data. This allows the user to
     * specify the map output value clreplaced to be different than the final output
     * value clreplaced.
     *
     * @param theClreplaced the map output value clreplaced.
     * @throws IllegalStateException if the job is submitted
     */
    public void setMapOutputValueClreplaced(Clreplaced<?> theClreplaced) throws IllegalStateException {
        ensureState(JobState.DEFINE);
        conf.setMapOutputValueClreplaced(theClreplaced);
    }

    /**
     * Set the key clreplaced for the job output data.
     *
     * @param theClreplaced the key clreplaced for the job output data.
     * @throws IllegalStateException if the job is submitted
     */
    public void setOutputKeyClreplaced(Clreplaced<?> theClreplaced) throws IllegalStateException {
        ensureState(JobState.DEFINE);
        conf.setOutputKeyClreplaced(theClreplaced);
    }

    /**
     * Set the value clreplaced for job outputs.
     *
     * @param theClreplaced the value clreplaced for job outputs.
     * @throws IllegalStateException if the job is submitted
     */
    public void setOutputValueClreplaced(Clreplaced<?> theClreplaced) throws IllegalStateException {
        ensureState(JobState.DEFINE);
        conf.setOutputValueClreplaced(theClreplaced);
    }

    /**
     * Define the comparator that controls which keys are grouped together
     * for a single call to combiner,
     * {@link Reducer#reduce(Object, Iterable,
     * org.apache.hadoop.mapreduce.Reducer.Context)}
     *
     * @param cls the raw comparator to use
     * @throws IllegalStateException if the job is submitted
     */
    public void setCombinerKeyGroupingComparatorClreplaced(Clreplaced<? extends RawComparator> cls) throws IllegalStateException {
        ensureState(JobState.DEFINE);
        conf.setCombinerKeyGroupingComparator(cls);
    }

    /**
     * Define the comparator that controls how the keys are sorted before they
     * are preplaceded to the {@link Reducer}.
     * @param cls the raw comparator
     * @throws IllegalStateException if the job is submitted
     * @see #setCombinerKeyGroupingComparatorClreplaced(Clreplaced)
     */
    public void setSortComparatorClreplaced(Clreplaced<? extends RawComparator> cls) throws IllegalStateException {
        ensureState(JobState.DEFINE);
        conf.setOutputKeyComparatorClreplaced(cls);
    }

    /**
     * Define the comparator that controls which keys are grouped together
     * for a single call to
     * {@link Reducer#reduce(Object, Iterable,
     *                       org.apache.hadoop.mapreduce.Reducer.Context)}
     * @param cls the raw comparator to use
     * @throws IllegalStateException if the job is submitted
     * @see #setCombinerKeyGroupingComparatorClreplaced(Clreplaced)
     */
    public void setGroupingComparatorClreplaced(Clreplaced<? extends RawComparator> cls) throws IllegalStateException {
        ensureState(JobState.DEFINE);
        conf.setOutputValueGroupingComparator(cls);
    }

    /**
     * Set the user-specified job name.
     *
     * @param name the job's new name.
     * @throws IllegalStateException if the job is submitted
     */
    public void setJobName(String name) throws IllegalStateException {
        ensureState(JobState.DEFINE);
        conf.setJobName(name);
    }

    /**
     * Turn speculative execution on or off for this job.
     *
     * @param speculativeExecution <code>true</code> if speculative execution
     *                             should be turned on, else <code>false</code>.
     */
    public void setSpeculativeExecution(boolean speculativeExecution) {
        ensureState(JobState.DEFINE);
        conf.setSpeculativeExecution(speculativeExecution);
    }

    /**
     * Turn speculative execution on or off for this job for map tasks.
     *
     * @param speculativeExecution <code>true</code> if speculative execution
     *                             should be turned on for map tasks,
     *                             else <code>false</code>.
     */
    public void setMapSpeculativeExecution(boolean speculativeExecution) {
        ensureState(JobState.DEFINE);
        conf.setMapSpeculativeExecution(speculativeExecution);
    }

    /**
     * Turn speculative execution on or off for this job for reduce tasks.
     *
     * @param speculativeExecution <code>true</code> if speculative execution
     *                             should be turned on for reduce tasks,
     *                             else <code>false</code>.
     */
    public void setReduceSpeculativeExecution(boolean speculativeExecution) {
        ensureState(JobState.DEFINE);
        conf.setReduceSpeculativeExecution(speculativeExecution);
    }

    /**
     * Specify whether job-setup and job-cleanup is needed for the job
     *
     * @param needed If <code>true</code>, job-setup and job-cleanup will be
     *               considered from {@link OutputCommitter}
     *               else ignored.
     */
    public void setJobSetupCleanupNeeded(boolean needed) {
        ensureState(JobState.DEFINE);
        conf.setBoolean(SETUP_CLEANUP_NEEDED, needed);
    }

    /**
     * Set the given set of archives
     * @param archives The list of archives that need to be localized
     */
    public void setCacheArchives(URI[] archives) {
        ensureState(JobState.DEFINE);
        DistributedCache.setCacheArchives(archives, conf);
    }

    /**
     * Set the given set of files
     * @param files The list of files that need to be localized
     */
    public void setCacheFiles(URI[] files) {
        ensureState(JobState.DEFINE);
        DistributedCache.setCacheFiles(files, conf);
    }

    /**
     * Add a archives to be localized
     * @param uri The uri of the cache to be localized
     */
    public void addCacheArchive(URI uri) {
        ensureState(JobState.DEFINE);
        DistributedCache.addCacheArchive(uri, conf);
    }

    /**
     * Add a file to be localized
     * @param uri The uri of the cache to be localized
     */
    public void addCacheFile(URI uri) {
        ensureState(JobState.DEFINE);
        DistributedCache.addCacheFile(uri, conf);
    }

    /**
     * Add an file path to the current set of clreplacedpath entries It adds the file
     * to cache as well.
     *
     * Files added with this method will not be unpacked while being added to the
     * clreplacedpath.
     * To add archives to clreplacedpath, use the {@link #addArchiveToClreplacedPath(Path)}
     * method instead.
     *
     * @param file Path of the file to be added
     */
    public void addFileToClreplacedPath(Path file) throws IOException {
        ensureState(JobState.DEFINE);
        DistributedCache.addFileToClreplacedPath(file, conf, file.getFileSystem(conf));
    }

    /**
     * Add an archive path to the current set of clreplacedpath entries. It adds the
     * archive to cache as well.
     *
     * Archive files will be unpacked and added to the clreplacedpath
     * when being distributed.
     *
     * @param archive Path of the archive to be added
     */
    public void addArchiveToClreplacedPath(Path archive) throws IOException {
        ensureState(JobState.DEFINE);
        DistributedCache.addArchiveToClreplacedPath(archive, conf, archive.getFileSystem(conf));
    }

    /**
     * Originally intended to enable symlinks, but currently symlinks cannot be
     * disabled.
     */
    @Deprecated
    public void createSymlink() {
        ensureState(JobState.DEFINE);
        DistributedCache.createSymlink(conf);
    }

    /**
     * Expert: Set the number of maximum attempts that will be made to run a
     * map task.
     *
     * @param n the number of attempts per map task.
     */
    public void setMaxMapAttempts(int n) {
        ensureState(JobState.DEFINE);
        conf.setMaxMapAttempts(n);
    }

    /**
     * Expert: Set the number of maximum attempts that will be made to run a
     * reduce task.
     *
     * @param n the number of attempts per reduce task.
     */
    public void setMaxReduceAttempts(int n) {
        ensureState(JobState.DEFINE);
        conf.setMaxReduceAttempts(n);
    }

    /**
     * Set whether the system should collect profiler information for some of
     * the tasks in this job? The information is stored in the user log
     * directory.
     * @param newValue true means it should be gathered
     */
    public void setProfileEnabled(boolean newValue) {
        ensureState(JobState.DEFINE);
        conf.setProfileEnabled(newValue);
    }

    /**
     * Set the profiler configuration arguments. If the string contains a '%s' it
     * will be replaced with the name of the profiling output file when the task
     * runs.
     *
     * This value is preplaceded to the task child JVM on the command line.
     *
     * @param value the configuration string
     */
    public void setProfileParams(String value) {
        ensureState(JobState.DEFINE);
        conf.setProfileParams(value);
    }

    /**
     * Set the ranges of maps or reduces to profile. setProfileEnabled(true)
     * must also be called.
     * @param newValue a set of integer ranges of the map ids
     */
    public void setProfileTaskRange(boolean isMap, String newValue) {
        ensureState(JobState.DEFINE);
        conf.setProfileTaskRange(isMap, newValue);
    }

    private void ensureNotSet(String attr, String msg) throws IOException {
        if (conf.get(attr) != null) {
            throw new IOException(attr + " is incompatible with " + msg + " mode.");
        }
    }

    /**
     * Sets the flag that will allow the JobTracker to cancel the HDFS delegation
     * tokens upon job completion. Defaults to true.
     */
    public void setCancelDelegationTokenUponJobCompletion(boolean value) {
        ensureState(JobState.DEFINE);
        conf.setBoolean(JOB_CANCEL_DELEGATION_TOKEN, value);
    }

    /**
     * Default to the new APIs unless they are explicitly set or the old mapper or
     * reduce attributes are used.
     * @throws IOException if the configuration is inconsistant
     */
    private void setUseNewAPI() throws IOException {
        int numReduces = conf.getNumReduceTasks();
        String oldMapperClreplaced = "mapred.mapper.clreplaced";
        String oldReduceClreplaced = "mapred.reducer.clreplaced";
        conf.setBooleanIfUnset("mapred.mapper.new-api", conf.get(oldMapperClreplaced) == null);
        if (conf.getUseNewMapper()) {
            String mode = "new map API";
            ensureNotSet("mapred.input.format.clreplaced", mode);
            ensureNotSet(oldMapperClreplaced, mode);
            if (numReduces != 0) {
                ensureNotSet("mapred.parreplacedioner.clreplaced", mode);
            } else {
                ensureNotSet("mapred.output.format.clreplaced", mode);
            }
        } else {
            String mode = "map compatability";
            ensureNotSet(INPUT_FORMAT_CLreplaced_ATTR, mode);
            ensureNotSet(MAP_CLreplaced_ATTR, mode);
            if (numReduces != 0) {
                ensureNotSet(PARreplacedIONER_CLreplaced_ATTR, mode);
            } else {
                ensureNotSet(OUTPUT_FORMAT_CLreplaced_ATTR, mode);
            }
        }
        if (numReduces != 0) {
            conf.setBooleanIfUnset("mapred.reducer.new-api", conf.get(oldReduceClreplaced) == null);
            if (conf.getUseNewReducer()) {
                String mode = "new reduce API";
                ensureNotSet("mapred.output.format.clreplaced", mode);
                ensureNotSet(oldReduceClreplaced, mode);
            } else {
                String mode = "reduce compatability";
                ensureNotSet(OUTPUT_FORMAT_CLreplaced_ATTR, mode);
                ensureNotSet(REDUCE_CLreplaced_ATTR, mode);
            }
        }
    }

    private synchronized void connect() throws IOException, InterruptedException, ClreplacedNotFoundException {
        if (cluster == null) {
            cluster = ugi.doAs(new PrivilegedExceptionAction<Cluster>() {

                public Cluster run() throws IOException, InterruptedException, ClreplacedNotFoundException {
                    return new Cluster(getConfiguration());
                }
            });
        }
    }

    boolean isConnected() {
        return cluster != null;
    }

    /**
     * Only for mocking via unit tests.
     */
    @Private
    public JobSubmitter getJobSubmitter(FileSystem fs, ClientProtocol submitClient) throws IOException {
        return new JobSubmitter(fs, submitClient);
    }

    /**
     * Submit the job to the cluster and return immediately.
     * @throws IOException
     */
    public void submit() throws IOException, InterruptedException, ClreplacedNotFoundException {
        ensureState(JobState.DEFINE);
        setUseNewAPI();
        connect();
        final JobSubmitter submitter = getJobSubmitter(cluster.getFileSystem(), cluster.getClient());
        status = ugi.doAs(new PrivilegedExceptionAction<JobStatus>() {

            public JobStatus run() throws IOException, InterruptedException, ClreplacedNotFoundException {
                return submitter.submitJobInternal(Job.this, cluster);
            }
        });
        state = JobState.RUNNING;
        LOG.info("The url to track the job: " + getTrackingURL());
    }

    /**
     * Submit the job to the cluster and wait for it to finish.
     * @param verbose print the progress to the user
     * @return true if the job succeeded
     * @throws IOException thrown if the communication with the
     *         <code>JobTracker</code> is lost
     */
    public boolean waitForCompletion(boolean verbose) throws IOException, InterruptedException, ClreplacedNotFoundException {
        if (state == JobState.DEFINE) {
            submit();
        }
        if (verbose) {
            monitorAndPrintJob();
        } else {
            // get the completion poll interval from the client.
            int completionPollIntervalMillis = Job.getCompletionPollInterval(cluster.getConf());
            while (!isComplete()) {
                try {
                    Thread.sleep(completionPollIntervalMillis);
                } catch (InterruptedException ie) {
                }
            }
        }
        return isSuccessful();
    }

    /**
     * Monitor a job and print status in real-time as progress is made and tasks
     * fail.
     * @return true if the job succeeded
     * @throws IOException if communication to the JobTracker fails
     */
    public boolean monitorAndPrintJob() throws IOException, InterruptedException {
        String lastReport = null;
        Job.TaskStatusFilter filter;
        Configuration clientConf = getConfiguration();
        filter = Job.getTaskOutputFilter(clientConf);
        JobID jobId = getJobID();
        LOG.info("Running job: " + jobId);
        int eventCounter = 0;
        boolean profiling = getProfileEnabled();
        IntegerRanges mapRanges = getProfileTaskRange(true);
        IntegerRanges reduceRanges = getProfileTaskRange(false);
        int progMonitorPollIntervalMillis = Job.getProgressPollInterval(clientConf);
        /* make sure to report full progress after the job is done */
        boolean reportedAfterCompletion = false;
        boolean reportedUberMode = false;
        while (!isComplete() || !reportedAfterCompletion) {
            if (isComplete()) {
                reportedAfterCompletion = true;
            } else {
                Thread.sleep(progMonitorPollIntervalMillis);
            }
            if (status.getState() == JobStatus.State.PREP) {
                continue;
            }
            if (!reportedUberMode) {
                reportedUberMode = true;
                LOG.info("Job " + jobId + " running in uber mode : " + isUber());
            }
            String report = (" map " + StringUtils.formatPercent(mapProgress(), 0) + " reduce " + StringUtils.formatPercent(reduceProgress(), 0));
            if (!report.equals(lastReport)) {
                LOG.info(report);
                lastReport = report;
            }
            TaskCompletionEvent[] events = getTaskCompletionEvents(eventCounter, 10);
            eventCounter += events.length;
            printTaskEvents(events, filter, profiling, mapRanges, reduceRanges);
        }
        boolean success = isSuccessful();
        if (success) {
            LOG.info("Job " + jobId + " completed successfully");
        } else {
            LOG.info("Job " + jobId + " failed with state " + status.getState() + " due to: " + status.getFailureInfo());
        }
        Counters counters = getCounters();
        if (counters != null) {
            LOG.info(counters.toString());
        }
        return success;
    }

    private void printTaskEvents(TaskCompletionEvent[] events, Job.TaskStatusFilter filter, boolean profiling, IntegerRanges mapRanges, IntegerRanges reduceRanges) throws IOException, InterruptedException {
        for (TaskCompletionEvent event : events) {
            switch(filter) {
                case NONE:
                    break;
                case SUCCEEDED:
                    if (event.getStatus() == TaskCompletionEvent.Status.SUCCEEDED) {
                        LOG.info(event.toString());
                    }
                    break;
                case FAILED:
                    if (event.getStatus() == TaskCompletionEvent.Status.FAILED) {
                        LOG.info(event.toString());
                        // Displaying the task diagnostic information
                        TaskAttemptID taskId = event.getTaskAttemptId();
                        String[] taskDiagnostics = getTaskDiagnostics(taskId);
                        if (taskDiagnostics != null) {
                            for (String diagnostics : taskDiagnostics) {
                                System.err.println(diagnostics);
                            }
                        }
                    }
                    break;
                case KILLED:
                    if (event.getStatus() == TaskCompletionEvent.Status.KILLED) {
                        LOG.info(event.toString());
                    }
                    break;
                case ALL:
                    LOG.info(event.toString());
                    break;
            }
        }
    }

    /**
     * The interval at which monitorAndPrintJob() prints status
     */
    public static int getProgressPollInterval(Configuration conf) {
        // Read progress monitor poll interval from config. Default is 1 second.
        int progMonitorPollIntervalMillis = conf.getInt(PROGRESS_MONITOR_POLL_INTERVAL_KEY, DEFAULT_MONITOR_POLL_INTERVAL);
        if (progMonitorPollIntervalMillis < 1) {
            LOG.warn(PROGRESS_MONITOR_POLL_INTERVAL_KEY + " has been set to an invalid value; " + " replacing with " + DEFAULT_MONITOR_POLL_INTERVAL);
            progMonitorPollIntervalMillis = DEFAULT_MONITOR_POLL_INTERVAL;
        }
        return progMonitorPollIntervalMillis;
    }

    /**
     * The interval at which waitForCompletion() should check.
     */
    public static int getCompletionPollInterval(Configuration conf) {
        int completionPollIntervalMillis = conf.getInt(COMPLETION_POLL_INTERVAL_KEY, DEFAULT_COMPLETION_POLL_INTERVAL);
        if (completionPollIntervalMillis < 1) {
            LOG.warn(COMPLETION_POLL_INTERVAL_KEY + " has been set to an invalid value; " + "replacing with " + DEFAULT_COMPLETION_POLL_INTERVAL);
            completionPollIntervalMillis = DEFAULT_COMPLETION_POLL_INTERVAL;
        }
        return completionPollIntervalMillis;
    }

    /**
     * Get the task output filter.
     *
     * @param conf the configuration.
     * @return the filter level.
     */
    public static TaskStatusFilter getTaskOutputFilter(Configuration conf) {
        return TaskStatusFilter.valueOf(conf.get(Job.OUTPUT_FILTER, "FAILED"));
    }

    /**
     * Modify the Configuration to set the task output filter.
     *
     * @param conf the Configuration to modify.
     * @param newValue the value to set.
     */
    public static void setTaskOutputFilter(Configuration conf, TaskStatusFilter newValue) {
        conf.set(Job.OUTPUT_FILTER, newValue.toString());
    }

    public boolean isUber() throws IOException, InterruptedException {
        ensureState(JobState.RUNNING);
        updateStatus();
        return status.isUber();
    }

    /**
     * Get the reservation to which the job is submitted to, if any
     *
     * @return the reservationId the identifier of the job's reservation, null if
     *         the job does not have any reservation replacedociated with it
     */
    public ReservationId getReservationId() {
        return reservationId;
    }

    /**
     * Set the reservation to which the job is submitted to
     *
     * @param reservationId the reservationId to set
     */
    public void setReservationId(ReservationId reservationId) {
        this.reservationId = reservationId;
    }
}

16 Source : TestInMemoryPlan.java
with Apache License 2.0
from NJUJYB

@Test
public void testDeleteNonExistingReservation() {
    Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, resCalc, minAlloc, maxAlloc, planName, replanner, true);
    ReservationId reservationID = ReservationSystemTestUtil.getNewReservationId();
    // Try to delete a reservation without adding
    replacedert.replacedertNull(plan.getReservationById(reservationID));
    try {
        plan.deleteReservation(reservationID);
        replacedert.fail("Delete should fail as it does not exist in the plan");
    } catch (IllegalArgumentException e) {
        replacedert.replacedertTrue(e.getMessage().endsWith("does not exist in the plan"));
    } catch (PlanningException e) {
        replacedert.fail(e.getMessage());
    }
    replacedert.replacedertNull(plan.getReservationById(reservationID));
}

16 Source : InMemoryPlan.java
with Apache License 2.0
from NJUJYB

@Override
public ReservationAllocation getReservationById(ReservationId reservationID) {
    if (reservationID == null) {
        return null;
    }
    readLock.lock();
    try {
        return reservationTable.get(reservationID);
    } finally {
        readLock.unlock();
    }
}

16 Source : InMemoryPlan.java
with Apache License 2.0
from NJUJYB

@Override
public boolean updateReservation(ReservationAllocation reservation) throws PlanningException {
    writeLock.lock();
    boolean result = false;
    try {
        ReservationId resId = reservation.getReservationId();
        ReservationAllocation currReservation = getReservationById(resId);
        if (currReservation == null) {
            String errMsg = "The specified Reservation with ID " + resId + " does not exist in the plan";
            LOG.error(errMsg);
            throw new IllegalArgumentException(errMsg);
        }
        // validate if we can accept this reservation, throws exception if
        // validation fails
        policy.validate(this, reservation);
        if (!removeReservation(currReservation)) {
            LOG.error("Unable to replace reservation: {} from plan.", reservation.getReservationId());
            return result;
        }
        try {
            result = addReservation(reservation);
        } catch (PlanningException e) {
            LOG.error("Unable to update reservation: {} from plan due to {}.", reservation.getReservationId(), e.getMessage());
        }
        if (result) {
            LOG.info("Sucessfully updated reservation: {} in plan.", reservation.getReservationId());
            return result;
        } else {
            // rollback delete
            addReservation(currReservation);
            LOG.info("Rollbacked update reservation: {} from plan.", reservation.getReservationId());
            return result;
        }
    } finally {
        writeLock.unlock();
    }
}

16 Source : ReservationSubmissionResponse.java
with Apache License 2.0
from NJUJYB

@Private
@Unstable
public static ReservationSubmissionResponse newInstance(ReservationId reservationId) {
    ReservationSubmissionResponse response = Records.newRecord(ReservationSubmissionResponse.clreplaced);
    response.setReservationId(reservationId);
    return response;
}

16 Source : ReservationDeleteRequest.java
with Apache License 2.0
from NJUJYB

@Public
@Unstable
public static ReservationDeleteRequest newInstance(ReservationId reservationId) {
    ReservationDeleteRequest request = Records.newRecord(ReservationDeleteRequest.clreplaced);
    request.setReservationId(reservationId);
    return request;
}

16 Source : TestCapacitySchedulerPlanFollower.java
with Apache License 2.0
from naver

@Override
protected void replacedertReservationQueueExists(ReservationId r) {
    CSQueue q = cs.getQueue(r.toString());
    replacedertNotNull(q);
}

16 Source : TestCapacitySchedulerPlanFollower.java
with Apache License 2.0
from naver

@Override
protected void replacedertReservationQueueDoesNotExist(ReservationId r2) {
    CSQueue q2 = cs.getQueue(r2.toString());
    replacedertNull(q2);
}

15 Source : TestReservationInputValidator.java
with Apache License 2.0
from NJUJYB

@Test
public void testDeleteReservationNormal() {
    ReservationDeleteRequest request = new ReservationDeleteRequestPBImpl();
    ReservationId reservationID = ReservationSystemTestUtil.getNewReservationId();
    request.setReservationId(reservationID);
    ReservationAllocation reservation = mock(ReservationAllocation.clreplaced);
    when(plan.getReservationById(reservationID)).thenReturn(reservation);
    Plan plan = null;
    try {
        plan = rrValidator.validateReservationDeleteRequest(rSystem, request);
    } catch (YarnException e) {
        replacedert.fail(e.getMessage());
    }
    replacedert.replacedertNotNull(plan);
}

15 Source : ReservationUpdateRequest.java
with Apache License 2.0
from NJUJYB

@Public
@Unstable
public static ReservationUpdateRequest newInstance(ReservationDefinition reservationDefinition, ReservationId reservationId) {
    ReservationUpdateRequest request = Records.newRecord(ReservationUpdateRequest.clreplaced);
    request.setReservationDefinition(reservationDefinition);
    request.setReservationId(reservationId);
    return request;
}

15 Source : TestFairSchedulerPlanFollower.java
with Apache License 2.0
from naver

@Override
protected void replacedertReservationQueueExists(ReservationId r, double expectedCapacity, double expectedMaxCapacity) {
    FSLeafQueue q = fs.getQueueManager().getLeafQueue(plan.getQueueName() + "" + "." + r, false);
    replacedertNotNull(q);
    // For now we are setting both to same weight
    replacedert.replacedertEquals(expectedCapacity, q.getWeights().getWeight(ResourceType.MEMORY), 0.01);
}

14 Source : TestReservationInputValidator.java
with Apache License 2.0
from NJUJYB

@Test
public void testDeleteReservationInvalidPlan() {
    ReservationDeleteRequest request = new ReservationDeleteRequestPBImpl();
    ReservationId reservationID = ReservationSystemTestUtil.getNewReservationId();
    request.setReservationId(reservationID);
    when(rSystem.getPlan(PLAN_NAME)).thenReturn(null);
    Plan plan = null;
    try {
        plan = rrValidator.validateReservationDeleteRequest(rSystem, request);
        replacedert.fail();
    } catch (YarnException e) {
        replacedert.replacedertNull(plan);
        String message = e.getMessage();
        replacedert.replacedertTrue(message.endsWith(" is not replacedociated with any valid plan. Please try again with a valid reservation."));
        LOG.info(message);
    }
}

14 Source : TestInMemoryReservationAllocation.java
with Apache License 2.0
from NJUJYB

@Test
public void testZeroAlloaction() {
    ReservationId reservationID = ReservationId.newInstance(rand.nextLong(), rand.nextLong());
    int[] alloc = {};
    long start = 0;
    ReservationDefinition rDef = createSimpleReservationDefinition(start, start + alloc.length + 1, alloc.length);
    Map<ReservationInterval, ReservationRequest> allocations = new HashMap<ReservationInterval, ReservationRequest>();
    ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, start, start + alloc.length + 1, allocations, resCalc, minAlloc);
    doreplacedertions(rAllocation, reservationID, rDef, allocations, (int) start, alloc);
    replacedert.replacedertFalse(rAllocation.containsGangs());
}

13 Source : TestReservationInputValidator.java
with Apache License 2.0
from NJUJYB

@Test
public void testDeleteReservationDoesnotExist() {
    ReservationDeleteRequest request = new ReservationDeleteRequestPBImpl();
    ReservationId rId = ReservationSystemTestUtil.getNewReservationId();
    request.setReservationId(rId);
    when(rSystem.getQueueForReservation(rId)).thenReturn(null);
    Plan plan = null;
    try {
        plan = rrValidator.validateReservationDeleteRequest(rSystem, request);
        replacedert.fail();
    } catch (YarnException e) {
        replacedert.replacedertNull(plan);
        String message = e.getMessage();
        replacedert.replacedertTrue(message.equals(MessageFormat.format("The specified reservation with ID: {0} is unknown. Please try again with a valid reservation.", rId)));
        LOG.info(message);
    }
}

13 Source : TestInMemoryReservationAllocation.java
with Apache License 2.0
from NJUJYB

@Test
public void testGangAlloaction() {
    ReservationId reservationID = ReservationId.newInstance(rand.nextLong(), rand.nextLong());
    int[] alloc = { 10, 10, 10, 10, 10, 10 };
    int start = 100;
    ReservationDefinition rDef = createSimpleReservationDefinition(start, start + alloc.length + 1, alloc.length);
    Map<ReservationInterval, ReservationRequest> allocations = generateAllocation(start, alloc, false, true);
    ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, start, start + alloc.length + 1, allocations, resCalc, minAlloc);
    doreplacedertions(rAllocation, reservationID, rDef, allocations, start, alloc);
    replacedert.replacedertTrue(rAllocation.containsGangs());
    for (int i = 0; i < alloc.length; i++) {
        replacedert.replacedertEquals(Resource.newInstance(1024 * (alloc[i]), (alloc[i])), rAllocation.getResourcesAtTime(start + i));
    }
}

13 Source : TestInMemoryReservationAllocation.java
with Apache License 2.0
from NJUJYB

@Test
public void testSteps() {
    ReservationId reservationID = ReservationId.newInstance(rand.nextLong(), rand.nextLong());
    int[] alloc = { 10, 10, 10, 10, 10, 10 };
    int start = 100;
    ReservationDefinition rDef = createSimpleReservationDefinition(start, start + alloc.length + 1, alloc.length);
    Map<ReservationInterval, ReservationRequest> allocations = generateAllocation(start, alloc, true, false);
    ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, start, start + alloc.length + 1, allocations, resCalc, minAlloc);
    doreplacedertions(rAllocation, reservationID, rDef, allocations, start, alloc);
    replacedert.replacedertFalse(rAllocation.containsGangs());
    for (int i = 0; i < alloc.length; i++) {
        replacedert.replacedertEquals(Resource.newInstance(1024 * (alloc[i] + i), (alloc[i] + i)), rAllocation.getResourcesAtTime(start + i));
    }
}

13 Source : TestInMemoryReservationAllocation.java
with Apache License 2.0
from NJUJYB

@Test
public void testBlocks() {
    ReservationId reservationID = ReservationId.newInstance(rand.nextLong(), rand.nextLong());
    int[] alloc = { 10, 10, 10, 10, 10, 10 };
    int start = 100;
    ReservationDefinition rDef = createSimpleReservationDefinition(start, start + alloc.length + 1, alloc.length);
    Map<ReservationInterval, ReservationRequest> allocations = generateAllocation(start, alloc, false, false);
    ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, start, start + alloc.length + 1, allocations, resCalc, minAlloc);
    doreplacedertions(rAllocation, reservationID, rDef, allocations, start, alloc);
    replacedert.replacedertFalse(rAllocation.containsGangs());
    for (int i = 0; i < alloc.length; i++) {
        replacedert.replacedertEquals(Resource.newInstance(1024 * (alloc[i]), (alloc[i])), rAllocation.getResourcesAtTime(start + i));
    }
}

13 Source : TestInMemoryReservationAllocation.java
with Apache License 2.0
from NJUJYB

@Test
public void testSkyline() {
    ReservationId reservationID = ReservationId.newInstance(rand.nextLong(), rand.nextLong());
    int[] alloc = { 0, 5, 10, 10, 5, 0 };
    int start = 100;
    ReservationDefinition rDef = createSimpleReservationDefinition(start, start + alloc.length + 1, alloc.length);
    Map<ReservationInterval, ReservationRequest> allocations = generateAllocation(start, alloc, true, false);
    ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, start, start + alloc.length + 1, allocations, resCalc, minAlloc);
    doreplacedertions(rAllocation, reservationID, rDef, allocations, start, alloc);
    replacedert.replacedertFalse(rAllocation.containsGangs());
    for (int i = 0; i < alloc.length; i++) {
        replacedert.replacedertEquals(Resource.newInstance(1024 * (alloc[i] + i), (alloc[i] + i)), rAllocation.getResourcesAtTime(start + i));
    }
}

13 Source : TestSimpleCapacityReplanner.java
with Apache License 2.0
from naver

@Test
public void testReplanningPlanCapacityLoss() throws PlanningException {
    Resource clusterCapacity = Resource.newInstance(100 * 1024, 10, 10);
    Resource minAlloc = Resource.newInstance(1024, 1, 1);
    Resource maxAlloc = Resource.newInstance(1024 * 8, 8, 8);
    ResourceCalculator res = new DefaultResourceCalculator();
    long step = 1L;
    Clock clock = mock(Clock.clreplaced);
    ReservationAgent agent = mock(ReservationAgent.clreplaced);
    SharingPolicy policy = new NoOverCommitPolicy();
    policy.init("root.dedicated", null);
    QueueMetrics queueMetrics = mock(QueueMetrics.clreplaced);
    when(clock.getTime()).thenReturn(0L);
    SimpleCapacityReplanner enf = new SimpleCapacityReplanner(clock);
    ReservationSchedulerConfiguration conf = mock(ReservationSchedulerConfiguration.clreplaced);
    when(conf.getEnforcementWindow(any(String.clreplaced))).thenReturn(6L);
    enf.init("blah", conf);
    // Initialize the plan with more resources
    InMemoryPlan plan = new InMemoryPlan(queueMetrics, policy, agent, clusterCapacity, step, res, minAlloc, maxAlloc, "dedicated", enf, true, clock);
    // add reservation filling the plan (separating them 1ms, so we are sure
    // s2 follows s1 on acceptance
    long ts = System.currentTimeMillis();
    ReservationId r1 = ReservationId.newInstance(ts, 1);
    int[] f5 = { 20, 20, 20, 20, 20 };
    replacedertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r1, null, "u3", "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res, minAlloc)));
    when(clock.getTime()).thenReturn(1L);
    ReservationId r2 = ReservationId.newInstance(ts, 2);
    replacedertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r2, null, "u4", "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res, minAlloc)));
    when(clock.getTime()).thenReturn(2L);
    ReservationId r3 = ReservationId.newInstance(ts, 3);
    replacedertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r3, null, "u5", "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res, minAlloc)));
    when(clock.getTime()).thenReturn(3L);
    ReservationId r4 = ReservationId.newInstance(ts, 4);
    replacedertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r4, null, "u6", "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res, minAlloc)));
    when(clock.getTime()).thenReturn(4L);
    ReservationId r5 = ReservationId.newInstance(ts, 5);
    replacedertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r5, null, "u7", "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res, minAlloc)));
    int[] f6 = { 50, 50, 50, 50, 50 };
    ReservationId r6 = ReservationId.newInstance(ts, 6);
    replacedertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r6, null, "u3", "dedicated", 10, 10 + f6.length, generateAllocation(10, f6), res, minAlloc)));
    when(clock.getTime()).thenReturn(6L);
    ReservationId r7 = ReservationId.newInstance(ts, 7);
    replacedertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r7, null, "u4", "dedicated", 10, 10 + f6.length, generateAllocation(10, f6), res, minAlloc)));
    // remove some of the resources (requires replanning)
    plan.setTotalCapacity(Resource.newInstance(70 * 1024, 70, 70));
    when(clock.getTime()).thenReturn(0L);
    // run the replanner
    enf.plan(plan, null);
    // check which reservation are still present
    replacedertNotNull(plan.getReservationById(r1));
    replacedertNotNull(plan.getReservationById(r2));
    replacedertNotNull(plan.getReservationById(r3));
    replacedertNotNull(plan.getReservationById(r6));
    replacedertNotNull(plan.getReservationById(r7));
    // and which ones are removed
    replacedertNull(plan.getReservationById(r4));
    replacedertNull(plan.getReservationById(r5));
    // check resources at each moment in time no more exceed capacity
    for (int i = 0; i < 20; i++) {
        int tot = 0;
        for (ReservationAllocation r : plan.getReservationsAtTime(i)) {
            tot = r.getResourcesAtTime(i).getMemory();
        }
        replacedertTrue(tot <= 70 * 1024);
    }
}

13 Source : TestInMemoryReservationAllocation.java
with Apache License 2.0
from naver

@Test
public void testBlocks() {
    ReservationId reservationID = ReservationId.newInstance(rand.nextLong(), rand.nextLong());
    int[] alloc = { 10, 10, 10, 10, 10, 10 };
    int start = 100;
    ReservationDefinition rDef = createSimpleReservationDefinition(start, start + alloc.length + 1, alloc.length);
    Map<ReservationInterval, ReservationRequest> allocations = generateAllocation(start, alloc, false, false);
    ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, start, start + alloc.length + 1, allocations, resCalc, minAlloc);
    doreplacedertions(rAllocation, reservationID, rDef, allocations, start, alloc);
    replacedert.replacedertFalse(rAllocation.containsGangs());
    for (int i = 0; i < alloc.length; i++) {
        replacedert.replacedertEquals(Resource.newInstance(1024 * (alloc[i]), (alloc[i]), (alloc[i])), rAllocation.getResourcesAtTime(start + i));
    }
}

See More Examples