Here are the examples of the java api org.apache.hadoop.yarn.api.records.ReservationId taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
94 Examples
19
Source : ReservationInputValidator.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
/**
* Quick validation on the input to check some obvious fail conditions (fail
* fast) the input and returns the appropriate {@link Plan} replacedociated with
* the specified {@link Queue} or throws an exception message illustrating the
* details of any validation check failures
*
* @param reservationSystem the {@link ReservationSystem} to validate against
* @param request the {@link ReservationUpdateRequest} defining the resources
* required over time for the request
* @return the {@link Plan} to submit the request to
* @throws YarnException
*/
public Plan validateReservationUpdateRequest(ReservationSystem reservationSystem, ReservationUpdateRequest request) throws YarnException {
ReservationId reservationId = request.getReservationId();
Plan plan = validateReservation(reservationSystem, reservationId, AuditConstants.UPDATE_RESERVATION_REQUEST);
validateReservationDefinition(reservationId, request.getReservationDefinition(), plan, AuditConstants.UPDATE_RESERVATION_REQUEST);
return plan;
}
19
Source : ReservationInputValidator.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private Plan validateReservation(ReservationSystem reservationSystem, ReservationId reservationId, String auditConstant) throws YarnException {
String message = "";
// check if the reservation id is valid
if (reservationId == null) {
message = "Missing reservation id." + " Please try again by specifying a reservation id.";
RMAuditLogger.logFailure("UNKNOWN", auditConstant, "validate reservation input", "ClientRMService", message);
throw RPCUtil.getRemoteException(message);
}
String queueName = reservationSystem.getQueueForReservation(reservationId);
if (queueName == null) {
message = "The specified reservation with ID: " + reservationId + " is unknown. Please try again with a valid reservation.";
RMAuditLogger.logFailure("UNKNOWN", auditConstant, "validate reservation input", "ClientRMService", message);
throw RPCUtil.getRemoteException(message);
}
// check if the replacedociated plan is valid
Plan plan = reservationSystem.getPlan(queueName);
if (plan == null) {
message = "The specified reservation: " + reservationId + " is not replacedociated with any valid plan." + " Please try again with a valid reservation.";
RMAuditLogger.logFailure("UNKNOWN", auditConstant, "validate reservation input", "ClientRMService", message);
throw RPCUtil.getRemoteException(message);
}
return plan;
}
19
Source : AbstractReservationSystem.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Override
public String getQueueForReservation(ReservationId reservationId) {
readLock.lock();
try {
return resQMap.get(reservationId);
} finally {
readLock.unlock();
}
}
19
Source : ApplicationSubmissionContextPBImpl.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Override
public void setReservationID(ReservationId reservationID) {
maybeInitBuilder();
if (reservationID == null) {
builder.clearReservationId();
return;
}
this.reservationId = reservationID;
}
19
Source : ApplicationSubmissionContextPBImpl.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private ReservationIdProto convertToProtoFormat(ReservationId t) {
return ((ReservationIdPBImpl) t).getProto();
}
19
Source : Job.java
with Apache License 2.0
from naver
with Apache License 2.0
from naver
/**
* Set the reservation to which the job is submitted to
*
* @param reservationId the reservationId to set
*/
public void setReservationId(ReservationId reservationId) {
this.reservationId = reservationId;
}
18
Source : TestNoOverCommitPolicy.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Test(expected = MismatchedUserException.clreplaced)
public void testUserMismatch() throws IOException, PlanningException {
// generate allocation from single tenant that exceed capacity
int[] f = generateData(3600, (int) (0.5 * totCont));
ReservationId rid = ReservationSystemTestUtil.getNewReservationId();
plan.addReservation(new InMemoryReservationAllocation(rid, null, "u1", "dedicated", initTime, initTime + f.length, ReservationSystemTestUtil.generateAllocation(initTime, step, f), res, minAlloc));
// trying to update a reservation with a mismatching user
plan.updateReservation(new InMemoryReservationAllocation(rid, null, "u2", "dedicated", initTime, initTime + f.length, ReservationSystemTestUtil.generateAllocation(initTime, step, f), res, minAlloc));
}
18
Source : AppAddedSchedulerEvent.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
public clreplaced AppAddedSchedulerEvent extends SchedulerEvent {
private final ApplicationId applicationId;
private final String queue;
private final String user;
private final ReservationId reservationID;
private final boolean isAppRecovering;
public AppAddedSchedulerEvent(ApplicationId applicationId, String queue, String user) {
this(applicationId, queue, user, false, null);
}
public AppAddedSchedulerEvent(ApplicationId applicationId, String queue, String user, ReservationId reservationID) {
this(applicationId, queue, user, false, reservationID);
}
public AppAddedSchedulerEvent(ApplicationId applicationId, String queue, String user, boolean isAppRecovering, ReservationId reservationID) {
super(SchedulerEventType.APP_ADDED);
this.applicationId = applicationId;
this.queue = queue;
this.user = user;
this.reservationID = reservationID;
this.isAppRecovering = isAppRecovering;
}
public ApplicationId getApplicationId() {
return applicationId;
}
public String getQueue() {
return queue;
}
public String getUser() {
return user;
}
public boolean getIsAppRecovering() {
return isAppRecovering;
}
public ReservationId getReservationID() {
return reservationID;
}
}
18
Source : InMemoryReservationAllocation.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
/**
* An in memory implementation of a reservation allocation using the
* {@link RLESparseResourceAllocation}
*/
clreplaced InMemoryReservationAllocation implements ReservationAllocation {
private final String planName;
private final ReservationId reservationID;
private final String user;
private final ReservationDefinition contract;
private final long startTime;
private final long endTime;
private final Map<ReservationInterval, ReservationRequest> allocationRequests;
private boolean hasGang = false;
private long acceptedAt = -1;
private RLESparseResourceAllocation resourcesOverTime;
InMemoryReservationAllocation(ReservationId reservationID, ReservationDefinition contract, String user, String planName, long startTime, long endTime, Map<ReservationInterval, ReservationRequest> allocationRequests, ResourceCalculator calculator, Resource minAlloc) {
this.contract = contract;
this.startTime = startTime;
this.endTime = endTime;
this.reservationID = reservationID;
this.user = user;
this.allocationRequests = allocationRequests;
this.planName = planName;
resourcesOverTime = new RLESparseResourceAllocation(calculator, minAlloc);
for (Map.Entry<ReservationInterval, ReservationRequest> r : allocationRequests.entrySet()) {
resourcesOverTime.addInterval(r.getKey(), r.getValue());
if (r.getValue().getConcurrency() > 1) {
hasGang = true;
}
}
}
@Override
public ReservationId getReservationId() {
return reservationID;
}
@Override
public ReservationDefinition getReservationDefinition() {
return contract;
}
@Override
public long getStartTime() {
return startTime;
}
@Override
public long getEndTime() {
return endTime;
}
@Override
public Map<ReservationInterval, ReservationRequest> getAllocationRequests() {
return Collections.unmodifiableMap(allocationRequests);
}
@Override
public String getPlanName() {
return planName;
}
@Override
public String getUser() {
return user;
}
@Override
public boolean containsGangs() {
return hasGang;
}
@Override
public void setAcceptanceTimestamp(long acceptedAt) {
this.acceptedAt = acceptedAt;
}
@Override
public long getAcceptanceTime() {
return acceptedAt;
}
@Override
public Resource getResourcesAtTime(long tick) {
if (tick < startTime || tick >= endTime) {
return Resource.newInstance(0, 0);
}
return Resources.clone(resourcesOverTime.getCapacityAtTime(tick));
}
@Override
public String toString() {
StringBuilder sBuf = new StringBuilder();
sBuf.append(getReservationId()).append(" user:").append(getUser()).append(" startTime: ").append(getStartTime()).append(" endTime: ").append(getEndTime()).append(" alloc:[").append(resourcesOverTime.toString()).append("] ");
return sBuf.toString();
}
@Override
public int compareTo(ReservationAllocation other) {
// reverse order of acceptance
if (this.getAcceptanceTime() > other.getAcceptanceTime()) {
return -1;
}
if (this.getAcceptanceTime() < other.getAcceptanceTime()) {
return 1;
}
return 0;
}
@Override
public int hashCode() {
return reservationID.hashCode();
}
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClreplaced() != obj.getClreplaced())
return false;
InMemoryReservationAllocation other = (InMemoryReservationAllocation) obj;
return this.reservationID.equals(other.getReservationId());
}
}
18
Source : GreedyReservationAgent.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Override
public boolean deleteReservation(ReservationId reservationId, String user, Plan plan) throws PlanningException {
return plan.deleteReservation(reservationId);
}
18
Source : AbstractReservationSystem.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Override
public ReservationId getNewReservationId() {
writeLock.lock();
try {
ReservationId resId = ReservationId.newInstance(ResourceManager.getClusterTimeStamp(), resCounter.incrementAndGet());
LOG.info("Allocated new reservationId: " + resId);
return resId;
} finally {
writeLock.unlock();
}
}
18
Source : AbstractReservationSystem.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Override
public void setQueueForReservation(ReservationId reservationId, String queueName) {
writeLock.lock();
try {
resQMap.put(reservationId, queueName);
} finally {
writeLock.unlock();
}
}
18
Source : ReservationUpdateRequestPBImpl.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Override
public void setReservationId(ReservationId reservationId) {
maybeInitBuilder();
if (reservationId == null) {
builder.clearReservationId();
return;
}
this.reservationId = reservationId;
}
18
Source : ReservationSubmissionResponsePBImpl.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
public clreplaced ReservationSubmissionResponsePBImpl extends ReservationSubmissionResponse {
ReservationSubmissionResponseProto proto = ReservationSubmissionResponseProto.getDefaultInstance();
ReservationSubmissionResponseProto.Builder builder = null;
boolean viaProto = false;
private ReservationId reservationId;
public ReservationSubmissionResponsePBImpl() {
builder = ReservationSubmissionResponseProto.newBuilder();
}
public ReservationSubmissionResponsePBImpl(ReservationSubmissionResponseProto proto) {
this.proto = proto;
viaProto = true;
}
public ReservationSubmissionResponseProto getProto() {
mergeLocalToProto();
proto = viaProto ? proto : builder.build();
viaProto = true;
return proto;
}
private void mergeLocalToBuilder() {
if (this.reservationId != null) {
builder.setReservationId(convertToProtoFormat(this.reservationId));
}
}
private void mergeLocalToProto() {
if (viaProto)
maybeInitBuilder();
mergeLocalToBuilder();
proto = builder.build();
viaProto = true;
}
private void maybeInitBuilder() {
if (viaProto || builder == null) {
builder = ReservationSubmissionResponseProto.newBuilder(proto);
}
viaProto = false;
}
@Override
public ReservationId getReservationId() {
ReservationSubmissionResponseProtoOrBuilder p = viaProto ? proto : builder;
if (reservationId != null) {
return reservationId;
}
if (!p.hasReservationId()) {
return null;
}
reservationId = convertFromProtoFormat(p.getReservationId());
return reservationId;
}
@Override
public void setReservationId(ReservationId reservationId) {
maybeInitBuilder();
if (reservationId == null) {
builder.clearReservationId();
return;
}
this.reservationId = reservationId;
}
private ReservationIdPBImpl convertFromProtoFormat(ReservationIdProto p) {
return new ReservationIdPBImpl(p);
}
private ReservationIdProto convertToProtoFormat(ReservationId t) {
return ((ReservationIdPBImpl) t).getProto();
}
@Override
public int hashCode() {
return getProto().hashCode();
}
@Override
public boolean equals(Object other) {
if (other == null)
return false;
if (other.getClreplaced().isreplacedignableFrom(this.getClreplaced())) {
return this.getProto().equals(this.getClreplaced().cast(other).getProto());
}
return false;
}
@Override
public String toString() {
return TextFormat.shortDebugString(getProto());
}
}
18
Source : ReservationDeleteRequestPBImpl.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
public clreplaced ReservationDeleteRequestPBImpl extends ReservationDeleteRequest {
ReservationDeleteRequestProto proto = ReservationDeleteRequestProto.getDefaultInstance();
ReservationDeleteRequestProto.Builder builder = null;
boolean viaProto = false;
private ReservationId reservationId;
public ReservationDeleteRequestPBImpl() {
builder = ReservationDeleteRequestProto.newBuilder();
}
public ReservationDeleteRequestPBImpl(ReservationDeleteRequestProto proto) {
this.proto = proto;
viaProto = true;
}
public ReservationDeleteRequestProto getProto() {
mergeLocalToProto();
proto = viaProto ? proto : builder.build();
viaProto = true;
return proto;
}
private void mergeLocalToBuilder() {
if (this.reservationId != null) {
builder.setReservationId(convertToProtoFormat(this.reservationId));
}
}
private void mergeLocalToProto() {
if (viaProto)
maybeInitBuilder();
mergeLocalToBuilder();
proto = builder.build();
viaProto = true;
}
private void maybeInitBuilder() {
if (viaProto || builder == null) {
builder = ReservationDeleteRequestProto.newBuilder(proto);
}
viaProto = false;
}
@Override
public ReservationId getReservationId() {
ReservationDeleteRequestProtoOrBuilder p = viaProto ? proto : builder;
if (reservationId != null) {
return reservationId;
}
if (!p.hasReservationId()) {
return null;
}
reservationId = convertFromProtoFormat(p.getReservationId());
return reservationId;
}
@Override
public void setReservationId(ReservationId reservationId) {
maybeInitBuilder();
if (reservationId == null) {
builder.clearReservationId();
return;
}
this.reservationId = reservationId;
}
private ReservationIdPBImpl convertFromProtoFormat(ReservationIdProto p) {
return new ReservationIdPBImpl(p);
}
private ReservationIdProto convertToProtoFormat(ReservationId t) {
return ((ReservationIdPBImpl) t).getProto();
}
@Override
public int hashCode() {
return getProto().hashCode();
}
@Override
public boolean equals(Object other) {
if (other == null)
return false;
if (other.getClreplaced().isreplacedignableFrom(this.getClreplaced())) {
return this.getProto().equals(this.getClreplaced().cast(other).getProto());
}
return false;
}
@Override
public String toString() {
return TextFormat.shortDebugString(getProto());
}
}
18
Source : InMemoryReservationAllocation.java
with Apache License 2.0
from naver
with Apache License 2.0
from naver
/**
* An in memory implementation of a reservation allocation using the
* {@link RLESparseResourceAllocation}
*/
clreplaced InMemoryReservationAllocation implements ReservationAllocation {
private final String planName;
private final ReservationId reservationID;
private final String user;
private final ReservationDefinition contract;
private final long startTime;
private final long endTime;
private final Map<ReservationInterval, ReservationRequest> allocationRequests;
private boolean hasGang = false;
private long acceptedAt = -1;
private RLESparseResourceAllocation resourcesOverTime;
InMemoryReservationAllocation(ReservationId reservationID, ReservationDefinition contract, String user, String planName, long startTime, long endTime, Map<ReservationInterval, ReservationRequest> allocationRequests, ResourceCalculator calculator, Resource minAlloc) {
this.contract = contract;
this.startTime = startTime;
this.endTime = endTime;
this.reservationID = reservationID;
this.user = user;
this.allocationRequests = allocationRequests;
this.planName = planName;
resourcesOverTime = new RLESparseResourceAllocation(calculator, minAlloc);
for (Map.Entry<ReservationInterval, ReservationRequest> r : allocationRequests.entrySet()) {
resourcesOverTime.addInterval(r.getKey(), r.getValue());
if (r.getValue().getConcurrency() > 1) {
hasGang = true;
}
}
}
@Override
public ReservationId getReservationId() {
return reservationID;
}
@Override
public ReservationDefinition getReservationDefinition() {
return contract;
}
@Override
public long getStartTime() {
return startTime;
}
@Override
public long getEndTime() {
return endTime;
}
@Override
public Map<ReservationInterval, ReservationRequest> getAllocationRequests() {
return Collections.unmodifiableMap(allocationRequests);
}
@Override
public String getPlanName() {
return planName;
}
@Override
public String getUser() {
return user;
}
@Override
public boolean containsGangs() {
return hasGang;
}
@Override
public void setAcceptanceTimestamp(long acceptedAt) {
this.acceptedAt = acceptedAt;
}
@Override
public long getAcceptanceTime() {
return acceptedAt;
}
@Override
public Resource getResourcesAtTime(long tick) {
if (tick < startTime || tick >= endTime) {
return Resource.newInstance(0, 0, 0);
}
return Resources.clone(resourcesOverTime.getCapacityAtTime(tick));
}
@Override
public String toString() {
StringBuilder sBuf = new StringBuilder();
sBuf.append(getReservationId()).append(" user:").append(getUser()).append(" startTime: ").append(getStartTime()).append(" endTime: ").append(getEndTime()).append(" alloc:[").append(resourcesOverTime.toString()).append("] ");
return sBuf.toString();
}
@Override
public int compareTo(ReservationAllocation other) {
// reverse order of acceptance
if (this.getAcceptanceTime() > other.getAcceptanceTime()) {
return -1;
}
if (this.getAcceptanceTime() < other.getAcceptanceTime()) {
return 1;
}
return 0;
}
@Override
public int hashCode() {
return reservationID.hashCode();
}
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClreplaced() != obj.getClreplaced())
return false;
InMemoryReservationAllocation other = (InMemoryReservationAllocation) obj;
return this.reservationID.equals(other.getReservationId());
}
}
18
Source : FairSchedulerPlanFollower.java
with Apache License 2.0
from naver
with Apache License 2.0
from naver
@Override
protected Resource getReservationQueueResourceIfExists(Plan plan, ReservationId reservationId) {
String reservationQueueName = getReservationQueueName(plan.getQueueName(), reservationId.toString());
FSLeafQueue reservationQueue = fs.getQueueManager().getLeafQueue(reservationQueueName, false);
Resource reservationResource = null;
if (reservationQueue != null) {
reservationResource = reservationQueue.getSteadyFairShare();
}
return reservationResource;
}
17
Source : TestSimpleCapacityReplanner.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Test
public void testReplanningPlanCapacityLoss() throws PlanningException {
Resource clusterCapacity = Resource.newInstance(100 * 1024, 10);
Resource minAlloc = Resource.newInstance(1024, 1);
Resource maxAlloc = Resource.newInstance(1024 * 8, 8);
ResourceCalculator res = new DefaultResourceCalculator();
long step = 1L;
Clock clock = mock(Clock.clreplaced);
ReservationAgent agent = mock(ReservationAgent.clreplaced);
SharingPolicy policy = new NoOverCommitPolicy();
policy.init("root.dedicated", null);
QueueMetrics queueMetrics = mock(QueueMetrics.clreplaced);
when(clock.getTime()).thenReturn(0L);
SimpleCapacityReplanner enf = new SimpleCapacityReplanner(clock);
CapacitySchedulerConfiguration conf = mock(CapacitySchedulerConfiguration.clreplaced);
when(conf.getEnforcementWindow(any(String.clreplaced))).thenReturn(6L);
conf.setLong(CapacitySchedulerConfiguration.PREFIX + "blah" + CapacitySchedulerConfiguration.DOT + CapacitySchedulerConfiguration.RESERVATION_ENFORCEMENT_WINDOW, 6);
enf.init("blah", conf);
// Initialize the plan with more resources
InMemoryPlan plan = new InMemoryPlan(queueMetrics, policy, agent, clusterCapacity, step, res, minAlloc, maxAlloc, "dedicated", enf, true, clock);
// add reservation filling the plan (separating them 1ms, so we are sure
// s2 follows s1 on acceptance
long ts = System.currentTimeMillis();
ReservationId r1 = ReservationId.newInstance(ts, 1);
int[] f5 = { 20, 20, 20, 20, 20 };
replacedertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r1, null, "u3", "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res, minAlloc)));
when(clock.getTime()).thenReturn(1L);
ReservationId r2 = ReservationId.newInstance(ts, 2);
replacedertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r2, null, "u4", "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res, minAlloc)));
when(clock.getTime()).thenReturn(2L);
ReservationId r3 = ReservationId.newInstance(ts, 3);
replacedertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r3, null, "u5", "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res, minAlloc)));
when(clock.getTime()).thenReturn(3L);
ReservationId r4 = ReservationId.newInstance(ts, 4);
replacedertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r4, null, "u6", "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res, minAlloc)));
when(clock.getTime()).thenReturn(4L);
ReservationId r5 = ReservationId.newInstance(ts, 5);
replacedertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r5, null, "u7", "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res, minAlloc)));
int[] f6 = { 50, 50, 50, 50, 50 };
ReservationId r6 = ReservationId.newInstance(ts, 6);
replacedertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r6, null, "u3", "dedicated", 10, 10 + f6.length, generateAllocation(10, f6), res, minAlloc)));
when(clock.getTime()).thenReturn(6L);
ReservationId r7 = ReservationId.newInstance(ts, 7);
replacedertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r7, null, "u4", "dedicated", 10, 10 + f6.length, generateAllocation(10, f6), res, minAlloc)));
// remove some of the resources (requires replanning)
plan.setTotalCapacity(Resource.newInstance(70 * 1024, 70));
when(clock.getTime()).thenReturn(0L);
// run the replanner
enf.plan(plan, null);
// check which reservation are still present
replacedertNotNull(plan.getReservationById(r1));
replacedertNotNull(plan.getReservationById(r2));
replacedertNotNull(plan.getReservationById(r3));
replacedertNotNull(plan.getReservationById(r6));
replacedertNotNull(plan.getReservationById(r7));
// and which ones are removed
replacedertNull(plan.getReservationById(r4));
replacedertNull(plan.getReservationById(r5));
// check resources at each moment in time no more exceed capacity
for (int i = 0; i < 20; i++) {
int tot = 0;
for (ReservationAllocation r : plan.getReservationsAtTime(i)) {
tot = r.getResourcesAtTime(i).getMemory();
}
replacedertTrue(tot <= 70 * 1024);
}
}
17
Source : TestInMemoryReservationAllocation.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private void doreplacedertions(ReservationAllocation rAllocation, ReservationId reservationID, ReservationDefinition rDef, Map<ReservationInterval, ReservationRequest> allocations, int start, int[] alloc) {
replacedert.replacedertEquals(reservationID, rAllocation.getReservationId());
replacedert.replacedertEquals(rDef, rAllocation.getReservationDefinition());
replacedert.replacedertEquals(allocations, rAllocation.getAllocationRequests());
replacedert.replacedertEquals(user, rAllocation.getUser());
replacedert.replacedertEquals(planName, rAllocation.getPlanName());
replacedert.replacedertEquals(start, rAllocation.getStartTime());
replacedert.replacedertEquals(start + alloc.length + 1, rAllocation.getEndTime());
}
17
Source : TestInMemoryPlan.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private void doreplacedertions(Plan plan, ReservationAllocation rAllocation) {
ReservationId reservationID = rAllocation.getReservationId();
replacedert.replacedertNotNull(plan.getReservationById(reservationID));
replacedert.replacedertEquals(rAllocation, plan.getReservationById(reservationID));
replacedert.replacedertTrue(((InMemoryPlan) plan).getAllReservations().size() == 1);
replacedert.replacedertEquals(rAllocation.getEndTime(), plan.getLastEndTime());
replacedert.replacedertEquals(totalCapacity, plan.getTotalCapacity());
replacedert.replacedertEquals(minAlloc, plan.getMinimumAllocation());
replacedert.replacedertEquals(maxAlloc, plan.getMaximumAllocation());
replacedert.replacedertEquals(resCalc, plan.getResourceCalculator());
replacedert.replacedertEquals(planName, plan.getQueueName());
replacedert.replacedertTrue(plan.getMoveOnExpiry());
}
17
Source : ReservationInputValidator.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
/**
* Quick validation on the input to check some obvious fail conditions (fail
* fast) the input and returns the appropriate {@link Plan} replacedociated with
* the specified {@link Queue} or throws an exception message illustrating the
* details of any validation check failures
*
* @param reservationSystem the {@link ReservationSystem} to validate against
* @param request the {@link ReservationSubmissionRequest} defining the
* resources required over time for the request
* @param reservationId the {@link ReservationId} replacedociated with the current
* request
* @return the {@link Plan} to submit the request to
* @throws YarnException
*/
public Plan validateReservationSubmissionRequest(ReservationSystem reservationSystem, ReservationSubmissionRequest request, ReservationId reservationId) throws YarnException {
// Check if it is a managed queue
String queueName = request.getQueue();
if (queueName == null || queueName.isEmpty()) {
String errMsg = "The queue to submit is not specified." + " Please try again with a valid reservable queue.";
RMAuditLogger.logFailure("UNKNOWN", AuditConstants.SUBMIT_RESERVATION_REQUEST, "validate reservation input", "ClientRMService", errMsg);
throw RPCUtil.getRemoteException(errMsg);
}
Plan plan = reservationSystem.getPlan(queueName);
if (plan == null) {
String errMsg = "The specified queue: " + queueName + " is not managed by reservation system." + " Please try again with a valid reservable queue.";
RMAuditLogger.logFailure("UNKNOWN", AuditConstants.SUBMIT_RESERVATION_REQUEST, "validate reservation input", "ClientRMService", errMsg);
throw RPCUtil.getRemoteException(errMsg);
}
validateReservationDefinition(reservationId, request.getReservationDefinition(), plan, AuditConstants.SUBMIT_RESERVATION_REQUEST);
return plan;
}
17
Source : ReservationInputValidator.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
private void validateReservationDefinition(ReservationId reservationId, ReservationDefinition contract, Plan plan, String auditConstant) throws YarnException {
String message = "";
// check if deadline is in the past
if (contract == null) {
message = "Missing reservation definition." + " Please try again by specifying a reservation definition.";
RMAuditLogger.logFailure("UNKNOWN", auditConstant, "validate reservation input definition", "ClientRMService", message);
throw RPCUtil.getRemoteException(message);
}
if (contract.getDeadline() <= clock.getTime()) {
message = "The specified deadline: " + contract.getDeadline() + " is the past. Please try again with deadline in the future.";
RMAuditLogger.logFailure("UNKNOWN", auditConstant, "validate reservation input definition", "ClientRMService", message);
throw RPCUtil.getRemoteException(message);
}
// Check if at least one RR has been specified
ReservationRequests resReqs = contract.getReservationRequests();
if (resReqs == null) {
message = "No resources have been specified to reserve." + "Please try again by specifying the resources to reserve.";
RMAuditLogger.logFailure("UNKNOWN", auditConstant, "validate reservation input definition", "ClientRMService", message);
throw RPCUtil.getRemoteException(message);
}
List<ReservationRequest> resReq = resReqs.getReservationResources();
if (resReq == null || resReq.isEmpty()) {
message = "No resources have been specified to reserve." + " Please try again by specifying the resources to reserve.";
RMAuditLogger.logFailure("UNKNOWN", auditConstant, "validate reservation input definition", "ClientRMService", message);
throw RPCUtil.getRemoteException(message);
}
// compute minimum duration and max gang size
long minDuration = 0;
Resource maxGangSize = Resource.newInstance(0, 0);
ReservationRequestInterpreter type = contract.getReservationRequests().getInterpreter();
for (ReservationRequest rr : resReq) {
if (type == ReservationRequestInterpreter.R_ALL || type == ReservationRequestInterpreter.R_ANY) {
minDuration = Math.max(minDuration, rr.getDuration());
} else {
minDuration += rr.getDuration();
}
maxGangSize = Resources.max(plan.getResourceCalculator(), plan.getTotalCapacity(), maxGangSize, Resources.multiply(rr.getCapability(), rr.getConcurrency()));
}
// verify the allocation is possible (skip for ANY)
if (contract.getDeadline() - contract.getArrival() < minDuration && type != ReservationRequestInterpreter.R_ANY) {
message = "The time difference (" + (contract.getDeadline() - contract.getArrival()) + ") between arrival (" + contract.getArrival() + ") " + "and deadline (" + contract.getDeadline() + ") must " + " be greater or equal to the minimum resource duration (" + minDuration + ")";
RMAuditLogger.logFailure("UNKNOWN", auditConstant, "validate reservation input definition", "ClientRMService", message);
throw RPCUtil.getRemoteException(message);
}
// check that the largest gang does not exceed the inventory available
// capacity (skip for ANY)
if (Resources.greaterThan(plan.getResourceCalculator(), plan.getTotalCapacity(), maxGangSize, plan.getTotalCapacity()) && type != ReservationRequestInterpreter.R_ANY) {
message = "The size of the largest gang in the reservation refinition (" + maxGangSize + ") exceed the capacity available (" + plan.getTotalCapacity() + " )";
RMAuditLogger.logFailure("UNKNOWN", auditConstant, "validate reservation input definition", "ClientRMService", message);
throw RPCUtil.getRemoteException(message);
}
}
17
Source : InMemoryPlan.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Override
public boolean deleteReservation(ReservationId reservationID) {
writeLock.lock();
try {
ReservationAllocation reservation = getReservationById(reservationID);
if (reservation == null) {
String errMsg = "The specified Reservation with ID " + reservationID + " does not exist in the plan";
LOG.error(errMsg);
throw new IllegalArgumentException(errMsg);
}
return removeReservation(reservation);
} finally {
writeLock.unlock();
}
}
17
Source : GreedyReservationAgent.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Override
public boolean createReservation(ReservationId reservationId, String user, Plan plan, ReservationDefinition contract) throws PlanningException {
return computeAllocation(reservationId, user, plan, contract, null);
}
17
Source : GreedyReservationAgent.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Override
public boolean updateReservation(ReservationId reservationId, String user, Plan plan, ReservationDefinition contract) throws PlanningException {
return computeAllocation(reservationId, user, plan, contract, plan.getReservationById(reservationId));
}
17
Source : ReservationUpdateRequestPBImpl.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
public clreplaced ReservationUpdateRequestPBImpl extends ReservationUpdateRequest {
ReservationUpdateRequestProto proto = ReservationUpdateRequestProto.getDefaultInstance();
ReservationUpdateRequestProto.Builder builder = null;
boolean viaProto = false;
private ReservationDefinition reservationDefinition;
private ReservationId reservationId;
public ReservationUpdateRequestPBImpl() {
builder = ReservationUpdateRequestProto.newBuilder();
}
public ReservationUpdateRequestPBImpl(ReservationUpdateRequestProto proto) {
this.proto = proto;
viaProto = true;
}
public ReservationUpdateRequestProto getProto() {
mergeLocalToProto();
proto = viaProto ? proto : builder.build();
viaProto = true;
return proto;
}
private void mergeLocalToBuilder() {
if (this.reservationId != null) {
builder.setReservationId(convertToProtoFormat(this.reservationId));
}
if (this.reservationDefinition != null) {
builder.setReservationDefinition(convertToProtoFormat(reservationDefinition));
}
}
private void mergeLocalToProto() {
if (viaProto)
maybeInitBuilder();
mergeLocalToBuilder();
proto = builder.build();
viaProto = true;
}
private void maybeInitBuilder() {
if (viaProto || builder == null) {
builder = ReservationUpdateRequestProto.newBuilder(proto);
}
viaProto = false;
}
@Override
public ReservationDefinition getReservationDefinition() {
ReservationUpdateRequestProtoOrBuilder p = viaProto ? proto : builder;
if (reservationDefinition != null) {
return reservationDefinition;
}
if (!p.hasReservationDefinition()) {
return null;
}
reservationDefinition = convertFromProtoFormat(p.getReservationDefinition());
return reservationDefinition;
}
@Override
public void setReservationDefinition(ReservationDefinition reservationDefinition) {
maybeInitBuilder();
if (reservationDefinition == null) {
builder.clearReservationDefinition();
}
this.reservationDefinition = reservationDefinition;
}
@Override
public ReservationId getReservationId() {
ReservationUpdateRequestProtoOrBuilder p = viaProto ? proto : builder;
if (reservationId != null) {
return reservationId;
}
if (!p.hasReservationId()) {
return null;
}
reservationId = convertFromProtoFormat(p.getReservationId());
return reservationId;
}
@Override
public void setReservationId(ReservationId reservationId) {
maybeInitBuilder();
if (reservationId == null) {
builder.clearReservationId();
return;
}
this.reservationId = reservationId;
}
private ReservationIdPBImpl convertFromProtoFormat(ReservationIdProto p) {
return new ReservationIdPBImpl(p);
}
private ReservationIdProto convertToProtoFormat(ReservationId t) {
return ((ReservationIdPBImpl) t).getProto();
}
private ReservationDefinitionProto convertToProtoFormat(ReservationDefinition r) {
return ((ReservationDefinitionPBImpl) r).getProto();
}
private ReservationDefinitionPBImpl convertFromProtoFormat(ReservationDefinitionProto r) {
return new ReservationDefinitionPBImpl(r);
}
@Override
public int hashCode() {
return getProto().hashCode();
}
@Override
public boolean equals(Object other) {
if (other == null)
return false;
if (other.getClreplaced().isreplacedignableFrom(this.getClreplaced())) {
return this.getProto().equals(this.getClreplaced().cast(other).getProto());
}
return false;
}
@Override
public String toString() {
return TextFormat.shortDebugString(getProto());
}
}
17
Source : TestFairSchedulerPlanFollower.java
with Apache License 2.0
from naver
with Apache License 2.0
from naver
@Override
protected void replacedertReservationQueueDoesNotExist(ReservationId r) {
Queue q = getReservationQueue(r.toString());
replacedertNull(q);
}
17
Source : TestFairSchedulerPlanFollower.java
with Apache License 2.0
from naver
with Apache License 2.0
from naver
@Override
protected void replacedertReservationQueueExists(ReservationId r) {
Queue q = getReservationQueue(r.toString());
replacedertNotNull(q);
}
17
Source : ReservationInputValidator.java
with Apache License 2.0
from naver
with Apache License 2.0
from naver
private void validateReservationDefinition(ReservationId reservationId, ReservationDefinition contract, Plan plan, String auditConstant) throws YarnException {
String message = "";
// check if deadline is in the past
if (contract == null) {
message = "Missing reservation definition." + " Please try again by specifying a reservation definition.";
RMAuditLogger.logFailure("UNKNOWN", auditConstant, "validate reservation input definition", "ClientRMService", message);
throw RPCUtil.getRemoteException(message);
}
if (contract.getDeadline() <= clock.getTime()) {
message = "The specified deadline: " + contract.getDeadline() + " is the past. Please try again with deadline in the future.";
RMAuditLogger.logFailure("UNKNOWN", auditConstant, "validate reservation input definition", "ClientRMService", message);
throw RPCUtil.getRemoteException(message);
}
// Check if at least one RR has been specified
ReservationRequests resReqs = contract.getReservationRequests();
if (resReqs == null) {
message = "No resources have been specified to reserve." + "Please try again by specifying the resources to reserve.";
RMAuditLogger.logFailure("UNKNOWN", auditConstant, "validate reservation input definition", "ClientRMService", message);
throw RPCUtil.getRemoteException(message);
}
List<ReservationRequest> resReq = resReqs.getReservationResources();
if (resReq == null || resReq.isEmpty()) {
message = "No resources have been specified to reserve." + " Please try again by specifying the resources to reserve.";
RMAuditLogger.logFailure("UNKNOWN", auditConstant, "validate reservation input definition", "ClientRMService", message);
throw RPCUtil.getRemoteException(message);
}
// compute minimum duration and max gang size
long minDuration = 0;
Resource maxGangSize = Resource.newInstance(0, 0, 0);
ReservationRequestInterpreter type = contract.getReservationRequests().getInterpreter();
for (ReservationRequest rr : resReq) {
if (type == ReservationRequestInterpreter.R_ALL || type == ReservationRequestInterpreter.R_ANY) {
minDuration = Math.max(minDuration, rr.getDuration());
} else {
minDuration += rr.getDuration();
}
maxGangSize = Resources.max(plan.getResourceCalculator(), plan.getTotalCapacity(), maxGangSize, Resources.multiply(rr.getCapability(), rr.getConcurrency()));
}
// verify the allocation is possible (skip for ANY)
if (contract.getDeadline() - contract.getArrival() < minDuration && type != ReservationRequestInterpreter.R_ANY) {
message = "The time difference (" + (contract.getDeadline() - contract.getArrival()) + ") between arrival (" + contract.getArrival() + ") " + "and deadline (" + contract.getDeadline() + ") must " + " be greater or equal to the minimum resource duration (" + minDuration + ")";
RMAuditLogger.logFailure("UNKNOWN", auditConstant, "validate reservation input definition", "ClientRMService", message);
throw RPCUtil.getRemoteException(message);
}
// check that the largest gang does not exceed the inventory available
// capacity (skip for ANY)
if (Resources.greaterThan(plan.getResourceCalculator(), plan.getTotalCapacity(), maxGangSize, plan.getTotalCapacity()) && type != ReservationRequestInterpreter.R_ANY) {
message = "The size of the largest gang in the reservation refinition (" + maxGangSize + ") exceed the capacity available (" + plan.getTotalCapacity() + " )";
RMAuditLogger.logFailure("UNKNOWN", auditConstant, "validate reservation input definition", "ClientRMService", message);
throw RPCUtil.getRemoteException(message);
}
}
17
Source : CapacitySchedulerPlanFollower.java
with Apache License 2.0
from naver
with Apache License 2.0
from naver
@Override
protected Resource getReservationQueueResourceIfExists(Plan plan, ReservationId reservationId) {
CSQueue resQueue = cs.getQueue(reservationId.toString());
Resource reservationResource = null;
if (resQueue != null) {
reservationResource = Resources.multiply(cs.getClusterResource(), resQueue.getAbsoluteCapacity());
}
return reservationResource;
}
17
Source : Job.java
with Apache License 2.0
from naver
with Apache License 2.0
from naver
/**
* The job submitter's view of the Job.
*
* <p>It allows the user to configure the
* job, submit it, control its execution, and query the state. The set methods
* only work until the job is submitted, afterwards they will throw an
* IllegalStateException. </p>
*
* <p>
* Normally the user creates the application, describes various facets of the
* job via {@link Job} and then submits the job and monitor its progress.</p>
*
* <p>Here is an example on how to submit a job:</p>
* <p><blockquote><pre>
* // Create a new Job
* Job job = Job.getInstance();
* job.setJarByClreplaced(MyJob.clreplaced);
*
* // Specify various job-specific parameters
* job.setJobName("myjob");
*
* job.setInputPath(new Path("in"));
* job.setOutputPath(new Path("out"));
*
* job.setMapperClreplaced(MyJob.MyMapper.clreplaced);
* job.setReducerClreplaced(MyJob.MyReducer.clreplaced);
*
* // Submit the job, then poll for progress until the job is complete
* job.waitForCompletion(true);
* </pre></blockquote>
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
public clreplaced Job extends JobContextImpl implements JobContext {
private static final Log LOG = LogFactory.getLog(Job.clreplaced);
@InterfaceStability.Evolving
public static enum JobState {
DEFINE, RUNNING
}
private static final long MAX_JOBSTATUS_AGE = 1000 * 2;
public static final String OUTPUT_FILTER = "mapreduce.client.output.filter";
/**
* Key in mapred-*.xml that sets completionPollInvervalMillis
*/
public static final String COMPLETION_POLL_INTERVAL_KEY = "mapreduce.client.completion.pollinterval";
/**
* Default completionPollIntervalMillis is 5000 ms.
*/
static final int DEFAULT_COMPLETION_POLL_INTERVAL = 5000;
/**
* Key in mapred-*.xml that sets progMonitorPollIntervalMillis
*/
public static final String PROGRESS_MONITOR_POLL_INTERVAL_KEY = "mapreduce.client.progressmonitor.pollinterval";
/**
* Default progMonitorPollIntervalMillis is 1000 ms.
*/
static final int DEFAULT_MONITOR_POLL_INTERVAL = 1000;
public static final String USED_GENERIC_PARSER = "mapreduce.client.genericoptionsparser.used";
public static final String SUBMIT_REPLICATION = "mapreduce.client.submit.file.replication";
public static final int DEFAULT_SUBMIT_REPLICATION = 10;
@InterfaceStability.Evolving
public static enum TaskStatusFilter {
NONE, KILLED, FAILED, SUCCEEDED, ALL
}
static {
ConfigUtil.loadResources();
}
private JobState state = JobState.DEFINE;
private JobStatus status;
private long statustime;
private Cluster cluster;
private ReservationId reservationId;
/**
* @deprecated Use {@link #getInstance()}
*/
@Deprecated
public Job() throws IOException {
this(new JobConf(new Configuration()));
}
/**
* @deprecated Use {@link #getInstance(Configuration)}
*/
@Deprecated
public Job(Configuration conf) throws IOException {
this(new JobConf(conf));
}
/**
* @deprecated Use {@link #getInstance(Configuration, String)}
*/
@Deprecated
public Job(Configuration conf, String jobName) throws IOException {
this(new JobConf(conf));
setJobName(jobName);
}
Job(JobConf conf) throws IOException {
super(conf, null);
// propagate existing user credentials to job
this.credentials.mergeAll(this.ugi.getCredentials());
this.cluster = null;
}
Job(JobStatus status, JobConf conf) throws IOException {
this(conf);
setJobID(status.getJobID());
this.status = status;
state = JobState.RUNNING;
}
/**
* Creates a new {@link Job} with no particular {@link Cluster} .
* A Cluster will be created with a generic {@link Configuration}.
*
* @return the {@link Job} , with no connection to a cluster yet.
* @throws IOException
*/
public static Job getInstance() throws IOException {
// create with a null Cluster
return getInstance(new Configuration());
}
/**
* Creates a new {@link Job} with no particular {@link Cluster} and a
* given {@link Configuration}.
*
* The <code>Job</code> makes a copy of the <code>Configuration</code> so
* that any necessary internal modifications do not reflect on the incoming
* parameter.
*
* A Cluster will be created from the conf parameter only when it's needed.
*
* @param conf the configuration
* @return the {@link Job} , with no connection to a cluster yet.
* @throws IOException
*/
public static Job getInstance(Configuration conf) throws IOException {
// create with a null Cluster
JobConf jobConf = new JobConf(conf);
return new Job(jobConf);
}
/**
* Creates a new {@link Job} with no particular {@link Cluster} and a given jobName.
* A Cluster will be created from the conf parameter only when it's needed.
*
* The <code>Job</code> makes a copy of the <code>Configuration</code> so
* that any necessary internal modifications do not reflect on the incoming
* parameter.
*
* @param conf the configuration
* @return the {@link Job} , with no connection to a cluster yet.
* @throws IOException
*/
public static Job getInstance(Configuration conf, String jobName) throws IOException {
// create with a null Cluster
Job result = getInstance(conf);
result.setJobName(jobName);
return result;
}
/**
* Creates a new {@link Job} with no particular {@link Cluster} and given
* {@link Configuration} and {@link JobStatus}.
* A Cluster will be created from the conf parameter only when it's needed.
*
* The <code>Job</code> makes a copy of the <code>Configuration</code> so
* that any necessary internal modifications do not reflect on the incoming
* parameter.
*
* @param status job status
* @param conf job configuration
* @return the {@link Job} , with no connection to a cluster yet.
* @throws IOException
*/
public static Job getInstance(JobStatus status, Configuration conf) throws IOException {
return new Job(status, new JobConf(conf));
}
/**
* Creates a new {@link Job} with no particular {@link Cluster}.
* A Cluster will be created from the conf parameter only when it's needed.
*
* The <code>Job</code> makes a copy of the <code>Configuration</code> so
* that any necessary internal modifications do not reflect on the incoming
* parameter.
*
* @param ignored
* @return the {@link Job} , with no connection to a cluster yet.
* @throws IOException
* @deprecated Use {@link #getInstance()}
*/
@Deprecated
public static Job getInstance(Cluster ignored) throws IOException {
return getInstance();
}
/**
* Creates a new {@link Job} with no particular {@link Cluster} and given
* {@link Configuration}.
* A Cluster will be created from the conf parameter only when it's needed.
*
* The <code>Job</code> makes a copy of the <code>Configuration</code> so
* that any necessary internal modifications do not reflect on the incoming
* parameter.
*
* @param ignored
* @param conf job configuration
* @return the {@link Job} , with no connection to a cluster yet.
* @throws IOException
* @deprecated Use {@link #getInstance(Configuration)}
*/
@Deprecated
public static Job getInstance(Cluster ignored, Configuration conf) throws IOException {
return getInstance(conf);
}
/**
* Creates a new {@link Job} with no particular {@link Cluster} and given
* {@link Configuration} and {@link JobStatus}.
* A Cluster will be created from the conf parameter only when it's needed.
*
* The <code>Job</code> makes a copy of the <code>Configuration</code> so
* that any necessary internal modifications do not reflect on the incoming
* parameter.
*
* @param cluster cluster
* @param status job status
* @param conf job configuration
* @return the {@link Job} , with no connection to a cluster yet.
* @throws IOException
*/
@Private
public static Job getInstance(Cluster cluster, JobStatus status, Configuration conf) throws IOException {
Job job = getInstance(status, conf);
job.setCluster(cluster);
return job;
}
private void ensureState(JobState state) throws IllegalStateException {
if (state != this.state) {
throw new IllegalStateException("Job in state " + this.state + " instead of " + state);
}
if (state == JobState.RUNNING && cluster == null) {
throw new IllegalStateException("Job in state " + this.state + ", but it isn't attached to any job tracker!");
}
}
/**
* Some methods rely on having a recent job status object. Refresh
* it, if necessary
*/
synchronized void ensureFreshStatus() throws IOException {
if (System.currentTimeMillis() - statustime > MAX_JOBSTATUS_AGE) {
updateStatus();
}
}
/**
* Some methods need to update status immediately. So, refresh
* immediately
* @throws IOException
*/
synchronized void updateStatus() throws IOException {
try {
this.status = ugi.doAs(new PrivilegedExceptionAction<JobStatus>() {
@Override
public JobStatus run() throws IOException, InterruptedException {
return cluster.getClient().getJobStatus(status.getJobID());
}
});
} catch (InterruptedException ie) {
throw new IOException(ie);
}
if (this.status == null) {
throw new IOException("Job status not available ");
}
this.statustime = System.currentTimeMillis();
}
public JobStatus getStatus() throws IOException, InterruptedException {
ensureState(JobState.RUNNING);
updateStatus();
return status;
}
/**
* Returns the current state of the Job.
*
* @return JobStatus#State
* @throws IOException
* @throws InterruptedException
*/
public JobStatus.State getJobState() throws IOException, InterruptedException {
ensureState(JobState.RUNNING);
updateStatus();
return status.getState();
}
/**
* Get the URL where some job progress information will be displayed.
*
* @return the URL where some job progress information will be displayed.
*/
public String getTrackingURL() {
ensureState(JobState.RUNNING);
return status.getTrackingUrl().toString();
}
/**
* Get the path of the submitted job configuration.
*
* @return the path of the submitted job configuration.
*/
public String getJobFile() {
ensureState(JobState.RUNNING);
return status.getJobFile();
}
/**
* Get start time of the job.
*
* @return the start time of the job
*/
public long getStartTime() {
ensureState(JobState.RUNNING);
return status.getStartTime();
}
/**
* Get finish time of the job.
*
* @return the finish time of the job
*/
public long getFinishTime() throws IOException, InterruptedException {
ensureState(JobState.RUNNING);
updateStatus();
return status.getFinishTime();
}
/**
* Get scheduling info of the job.
*
* @return the scheduling info of the job
*/
public String getSchedulingInfo() {
ensureState(JobState.RUNNING);
return status.getSchedulingInfo();
}
/**
* Get scheduling info of the job.
*
* @return the scheduling info of the job
*/
public JobPriority getPriority() throws IOException, InterruptedException {
ensureState(JobState.RUNNING);
updateStatus();
return status.getPriority();
}
/**
* The user-specified job name.
*/
public String getJobName() {
if (state == JobState.DEFINE) {
return super.getJobName();
}
ensureState(JobState.RUNNING);
return status.getJobName();
}
public String getHistoryUrl() throws IOException, InterruptedException {
ensureState(JobState.RUNNING);
updateStatus();
return status.getHistoryFile();
}
public boolean isRetired() throws IOException, InterruptedException {
ensureState(JobState.RUNNING);
updateStatus();
return status.isRetired();
}
@Private
public Cluster getCluster() {
return cluster;
}
/**
* Only for mocks in unit tests.
*/
@Private
private void setCluster(Cluster cluster) {
this.cluster = cluster;
}
/**
* Dump stats to screen.
*/
@Override
public String toString() {
ensureState(JobState.RUNNING);
String reasonforFailure = " ";
int numMaps = 0;
int numReduces = 0;
try {
updateStatus();
if (status.getState().equals(JobStatus.State.FAILED))
reasonforFailure = getTaskFailureEventString();
numMaps = getTaskReports(TaskType.MAP).length;
numReduces = getTaskReports(TaskType.REDUCE).length;
} catch (IOException e) {
} catch (InterruptedException ie) {
}
StringBuffer sb = new StringBuffer();
sb.append("Job: ").append(status.getJobID()).append("\n");
sb.append("Job File: ").append(status.getJobFile()).append("\n");
sb.append("Job Tracking URL : ").append(status.getTrackingUrl());
sb.append("\n");
sb.append("Uber job : ").append(status.isUber()).append("\n");
sb.append("Number of maps: ").append(numMaps).append("\n");
sb.append("Number of reduces: ").append(numReduces).append("\n");
sb.append("map() completion: ");
sb.append(status.getMapProgress()).append("\n");
sb.append("reduce() completion: ");
sb.append(status.getReduceProgress()).append("\n");
sb.append("Job state: ");
sb.append(status.getState()).append("\n");
sb.append("retired: ").append(status.isRetired()).append("\n");
sb.append("reason for failure: ").append(reasonforFailure);
return sb.toString();
}
/**
* @return taskid which caused job failure
* @throws IOException
* @throws InterruptedException
*/
String getTaskFailureEventString() throws IOException, InterruptedException {
int failCount = 1;
TaskCompletionEvent lastEvent = null;
TaskCompletionEvent[] events = ugi.doAs(new PrivilegedExceptionAction<TaskCompletionEvent[]>() {
@Override
public TaskCompletionEvent[] run() throws IOException, InterruptedException {
return cluster.getClient().getTaskCompletionEvents(status.getJobID(), 0, 10);
}
});
for (TaskCompletionEvent event : events) {
if (event.getStatus().equals(TaskCompletionEvent.Status.FAILED)) {
failCount++;
lastEvent = event;
}
}
if (lastEvent == null) {
return "There are no failed tasks for the job. " + "Job is failed due to some other reason and reason " + "can be found in the logs.";
}
String[] taskAttemptID = lastEvent.getTaskAttemptId().toString().split("_", 2);
String taskID = taskAttemptID[1].substring(0, taskAttemptID[1].length() - 2);
return (" task " + taskID + " failed " + failCount + " times " + "For details check tasktracker at: " + lastEvent.getTaskTrackerHttp());
}
/**
* Get the information of the current state of the tasks of a job.
*
* @param type Type of the task
* @return the list of all of the map tips.
* @throws IOException
*/
public TaskReport[] getTaskReports(TaskType type) throws IOException, InterruptedException {
ensureState(JobState.RUNNING);
final TaskType tmpType = type;
return ugi.doAs(new PrivilegedExceptionAction<TaskReport[]>() {
public TaskReport[] run() throws IOException, InterruptedException {
return cluster.getClient().getTaskReports(getJobID(), tmpType);
}
});
}
/**
* Get the <i>progress</i> of the job's map-tasks, as a float between 0.0
* and 1.0. When all map tasks have completed, the function returns 1.0.
*
* @return the progress of the job's map-tasks.
* @throws IOException
*/
public float mapProgress() throws IOException {
ensureState(JobState.RUNNING);
ensureFreshStatus();
return status.getMapProgress();
}
/**
* Get the <i>progress</i> of the job's reduce-tasks, as a float between 0.0
* and 1.0. When all reduce tasks have completed, the function returns 1.0.
*
* @return the progress of the job's reduce-tasks.
* @throws IOException
*/
public float reduceProgress() throws IOException {
ensureState(JobState.RUNNING);
ensureFreshStatus();
return status.getReduceProgress();
}
/**
* Get the <i>progress</i> of the job's cleanup-tasks, as a float between 0.0
* and 1.0. When all cleanup tasks have completed, the function returns 1.0.
*
* @return the progress of the job's cleanup-tasks.
* @throws IOException
*/
public float cleanupProgress() throws IOException, InterruptedException {
ensureState(JobState.RUNNING);
ensureFreshStatus();
return status.getCleanupProgress();
}
/**
* Get the <i>progress</i> of the job's setup-tasks, as a float between 0.0
* and 1.0. When all setup tasks have completed, the function returns 1.0.
*
* @return the progress of the job's setup-tasks.
* @throws IOException
*/
public float setupProgress() throws IOException {
ensureState(JobState.RUNNING);
ensureFreshStatus();
return status.getSetupProgress();
}
/**
* Check if the job is finished or not.
* This is a non-blocking call.
*
* @return <code>true</code> if the job is complete, else <code>false</code>.
* @throws IOException
*/
public boolean isComplete() throws IOException {
ensureState(JobState.RUNNING);
updateStatus();
return status.isJobComplete();
}
/**
* Check if the job completed successfully.
*
* @return <code>true</code> if the job succeeded, else <code>false</code>.
* @throws IOException
*/
public boolean isSuccessful() throws IOException {
ensureState(JobState.RUNNING);
updateStatus();
return status.getState() == JobStatus.State.SUCCEEDED;
}
/**
* Kill the running job. Blocks until all job tasks have been
* killed as well. If the job is no longer running, it simply returns.
*
* @throws IOException
*/
public void killJob() throws IOException {
ensureState(JobState.RUNNING);
try {
cluster.getClient().killJob(getJobID());
} catch (InterruptedException ie) {
throw new IOException(ie);
}
}
/**
* Set the priority of a running job.
* @param priority the new priority for the job.
* @throws IOException
*/
public void setPriority(JobPriority priority) throws IOException, InterruptedException {
if (state == JobState.DEFINE) {
conf.setJobPriority(org.apache.hadoop.mapred.JobPriority.valueOf(priority.name()));
} else {
ensureState(JobState.RUNNING);
final JobPriority tmpPriority = priority;
ugi.doAs(new PrivilegedExceptionAction<Object>() {
@Override
public Object run() throws IOException, InterruptedException {
cluster.getClient().setJobPriority(getJobID(), tmpPriority.toString());
return null;
}
});
}
}
/**
* Get events indicating completion (success/failure) of component tasks.
*
* @param startFrom index to start fetching events from
* @param numEvents number of events to fetch
* @return an array of {@link TaskCompletionEvent}s
* @throws IOException
*/
public TaskCompletionEvent[] getTaskCompletionEvents(final int startFrom, final int numEvents) throws IOException, InterruptedException {
ensureState(JobState.RUNNING);
return ugi.doAs(new PrivilegedExceptionAction<TaskCompletionEvent[]>() {
@Override
public TaskCompletionEvent[] run() throws IOException, InterruptedException {
return cluster.getClient().getTaskCompletionEvents(getJobID(), startFrom, numEvents);
}
});
}
/**
* Get events indicating completion (success/failure) of component tasks.
*
* @param startFrom index to start fetching events from
* @return an array of {@link org.apache.hadoop.mapred.TaskCompletionEvent}s
* @throws IOException
*/
public org.apache.hadoop.mapred.TaskCompletionEvent[] getTaskCompletionEvents(final int startFrom) throws IOException {
try {
TaskCompletionEvent[] events = getTaskCompletionEvents(startFrom, 10);
org.apache.hadoop.mapred.TaskCompletionEvent[] retEvents = new org.apache.hadoop.mapred.TaskCompletionEvent[events.length];
for (int i = 0; i < events.length; i++) {
retEvents[i] = org.apache.hadoop.mapred.TaskCompletionEvent.downgrade(events[i]);
}
return retEvents;
} catch (InterruptedException ie) {
throw new IOException(ie);
}
}
/**
* Kill indicated task attempt.
* @param taskId the id of the task to kill.
* @param shouldFail if <code>true</code> the task is failed and added
* to failed tasks list, otherwise it is just killed,
* w/o affecting job failure status.
*/
@Private
public boolean killTask(final TaskAttemptID taskId, final boolean shouldFail) throws IOException {
ensureState(JobState.RUNNING);
try {
return ugi.doAs(new PrivilegedExceptionAction<Boolean>() {
public Boolean run() throws IOException, InterruptedException {
return cluster.getClient().killTask(taskId, shouldFail);
}
});
} catch (InterruptedException ie) {
throw new IOException(ie);
}
}
/**
* Kill indicated task attempt.
*
* @param taskId the id of the task to be terminated.
* @throws IOException
*/
public void killTask(final TaskAttemptID taskId) throws IOException {
killTask(taskId, false);
}
/**
* Fail indicated task attempt.
*
* @param taskId the id of the task to be terminated.
* @throws IOException
*/
public void failTask(final TaskAttemptID taskId) throws IOException {
killTask(taskId, true);
}
/**
* Gets the counters for this job. May return null if the job has been
* retired and the job is no longer in the completed job store.
*
* @return the counters for this job.
* @throws IOException
*/
public Counters getCounters() throws IOException {
ensureState(JobState.RUNNING);
try {
return ugi.doAs(new PrivilegedExceptionAction<Counters>() {
@Override
public Counters run() throws IOException, InterruptedException {
return cluster.getClient().getJobCounters(getJobID());
}
});
} catch (InterruptedException ie) {
throw new IOException(ie);
}
}
/**
* Gets the diagnostic messages for a given task attempt.
* @param taskid
* @return the list of diagnostic messages for the task
* @throws IOException
*/
public String[] getTaskDiagnostics(final TaskAttemptID taskid) throws IOException, InterruptedException {
ensureState(JobState.RUNNING);
return ugi.doAs(new PrivilegedExceptionAction<String[]>() {
@Override
public String[] run() throws IOException, InterruptedException {
return cluster.getClient().getTaskDiagnostics(taskid);
}
});
}
/**
* Set the number of reduce tasks for the job.
* @param tasks the number of reduce tasks
* @throws IllegalStateException if the job is submitted
*/
public void setNumReduceTasks(int tasks) throws IllegalStateException {
ensureState(JobState.DEFINE);
conf.setNumReduceTasks(tasks);
}
/**
* Set the current working directory for the default file system.
*
* @param dir the new current working directory.
* @throws IllegalStateException if the job is submitted
*/
public void setWorkingDirectory(Path dir) throws IOException {
ensureState(JobState.DEFINE);
conf.setWorkingDirectory(dir);
}
/**
* Set the {@link InputFormat} for the job.
* @param cls the <code>InputFormat</code> to use
* @throws IllegalStateException if the job is submitted
*/
public void setInputFormatClreplaced(Clreplaced<? extends InputFormat> cls) throws IllegalStateException {
ensureState(JobState.DEFINE);
conf.setClreplaced(INPUT_FORMAT_CLreplaced_ATTR, cls, InputFormat.clreplaced);
}
/**
* Set the {@link OutputFormat} for the job.
* @param cls the <code>OutputFormat</code> to use
* @throws IllegalStateException if the job is submitted
*/
public void setOutputFormatClreplaced(Clreplaced<? extends OutputFormat> cls) throws IllegalStateException {
ensureState(JobState.DEFINE);
conf.setClreplaced(OUTPUT_FORMAT_CLreplaced_ATTR, cls, OutputFormat.clreplaced);
}
/**
* Set the {@link Mapper} for the job.
* @param cls the <code>Mapper</code> to use
* @throws IllegalStateException if the job is submitted
*/
public void setMapperClreplaced(Clreplaced<? extends Mapper> cls) throws IllegalStateException {
ensureState(JobState.DEFINE);
conf.setClreplaced(MAP_CLreplaced_ATTR, cls, Mapper.clreplaced);
}
/**
* Set the Jar by finding where a given clreplaced came from.
* @param cls the example clreplaced
*/
public void setJarByClreplaced(Clreplaced<?> cls) {
ensureState(JobState.DEFINE);
conf.setJarByClreplaced(cls);
}
/**
* Set the job jar
*/
public void setJar(String jar) {
ensureState(JobState.DEFINE);
conf.setJar(jar);
}
/**
* Set the reported username for this job.
*
* @param user the username for this job.
*/
public void setUser(String user) {
ensureState(JobState.DEFINE);
conf.setUser(user);
}
/**
* Set the combiner clreplaced for the job.
* @param cls the combiner to use
* @throws IllegalStateException if the job is submitted
*/
public void setCombinerClreplaced(Clreplaced<? extends Reducer> cls) throws IllegalStateException {
ensureState(JobState.DEFINE);
conf.setClreplaced(COMBINE_CLreplaced_ATTR, cls, Reducer.clreplaced);
}
/**
* Set the {@link Reducer} for the job.
* @param cls the <code>Reducer</code> to use
* @throws IllegalStateException if the job is submitted
*/
public void setReducerClreplaced(Clreplaced<? extends Reducer> cls) throws IllegalStateException {
ensureState(JobState.DEFINE);
conf.setClreplaced(REDUCE_CLreplaced_ATTR, cls, Reducer.clreplaced);
}
/**
* Set the {@link Parreplacedioner} for the job.
* @param cls the <code>Parreplacedioner</code> to use
* @throws IllegalStateException if the job is submitted
*/
public void setParreplacedionerClreplaced(Clreplaced<? extends Parreplacedioner> cls) throws IllegalStateException {
ensureState(JobState.DEFINE);
conf.setClreplaced(PARreplacedIONER_CLreplaced_ATTR, cls, Parreplacedioner.clreplaced);
}
/**
* Set the key clreplaced for the map output data. This allows the user to
* specify the map output key clreplaced to be different than the final output
* value clreplaced.
*
* @param theClreplaced the map output key clreplaced.
* @throws IllegalStateException if the job is submitted
*/
public void setMapOutputKeyClreplaced(Clreplaced<?> theClreplaced) throws IllegalStateException {
ensureState(JobState.DEFINE);
conf.setMapOutputKeyClreplaced(theClreplaced);
}
/**
* Set the value clreplaced for the map output data. This allows the user to
* specify the map output value clreplaced to be different than the final output
* value clreplaced.
*
* @param theClreplaced the map output value clreplaced.
* @throws IllegalStateException if the job is submitted
*/
public void setMapOutputValueClreplaced(Clreplaced<?> theClreplaced) throws IllegalStateException {
ensureState(JobState.DEFINE);
conf.setMapOutputValueClreplaced(theClreplaced);
}
/**
* Set the key clreplaced for the job output data.
*
* @param theClreplaced the key clreplaced for the job output data.
* @throws IllegalStateException if the job is submitted
*/
public void setOutputKeyClreplaced(Clreplaced<?> theClreplaced) throws IllegalStateException {
ensureState(JobState.DEFINE);
conf.setOutputKeyClreplaced(theClreplaced);
}
/**
* Set the value clreplaced for job outputs.
*
* @param theClreplaced the value clreplaced for job outputs.
* @throws IllegalStateException if the job is submitted
*/
public void setOutputValueClreplaced(Clreplaced<?> theClreplaced) throws IllegalStateException {
ensureState(JobState.DEFINE);
conf.setOutputValueClreplaced(theClreplaced);
}
/**
* Define the comparator that controls which keys are grouped together
* for a single call to combiner,
* {@link Reducer#reduce(Object, Iterable,
* org.apache.hadoop.mapreduce.Reducer.Context)}
*
* @param cls the raw comparator to use
* @throws IllegalStateException if the job is submitted
*/
public void setCombinerKeyGroupingComparatorClreplaced(Clreplaced<? extends RawComparator> cls) throws IllegalStateException {
ensureState(JobState.DEFINE);
conf.setCombinerKeyGroupingComparator(cls);
}
/**
* Define the comparator that controls how the keys are sorted before they
* are preplaceded to the {@link Reducer}.
* @param cls the raw comparator
* @throws IllegalStateException if the job is submitted
* @see #setCombinerKeyGroupingComparatorClreplaced(Clreplaced)
*/
public void setSortComparatorClreplaced(Clreplaced<? extends RawComparator> cls) throws IllegalStateException {
ensureState(JobState.DEFINE);
conf.setOutputKeyComparatorClreplaced(cls);
}
/**
* Define the comparator that controls which keys are grouped together
* for a single call to
* {@link Reducer#reduce(Object, Iterable,
* org.apache.hadoop.mapreduce.Reducer.Context)}
* @param cls the raw comparator to use
* @throws IllegalStateException if the job is submitted
* @see #setCombinerKeyGroupingComparatorClreplaced(Clreplaced)
*/
public void setGroupingComparatorClreplaced(Clreplaced<? extends RawComparator> cls) throws IllegalStateException {
ensureState(JobState.DEFINE);
conf.setOutputValueGroupingComparator(cls);
}
/**
* Set the user-specified job name.
*
* @param name the job's new name.
* @throws IllegalStateException if the job is submitted
*/
public void setJobName(String name) throws IllegalStateException {
ensureState(JobState.DEFINE);
conf.setJobName(name);
}
/**
* Turn speculative execution on or off for this job.
*
* @param speculativeExecution <code>true</code> if speculative execution
* should be turned on, else <code>false</code>.
*/
public void setSpeculativeExecution(boolean speculativeExecution) {
ensureState(JobState.DEFINE);
conf.setSpeculativeExecution(speculativeExecution);
}
/**
* Turn speculative execution on or off for this job for map tasks.
*
* @param speculativeExecution <code>true</code> if speculative execution
* should be turned on for map tasks,
* else <code>false</code>.
*/
public void setMapSpeculativeExecution(boolean speculativeExecution) {
ensureState(JobState.DEFINE);
conf.setMapSpeculativeExecution(speculativeExecution);
}
/**
* Turn speculative execution on or off for this job for reduce tasks.
*
* @param speculativeExecution <code>true</code> if speculative execution
* should be turned on for reduce tasks,
* else <code>false</code>.
*/
public void setReduceSpeculativeExecution(boolean speculativeExecution) {
ensureState(JobState.DEFINE);
conf.setReduceSpeculativeExecution(speculativeExecution);
}
/**
* Specify whether job-setup and job-cleanup is needed for the job
*
* @param needed If <code>true</code>, job-setup and job-cleanup will be
* considered from {@link OutputCommitter}
* else ignored.
*/
public void setJobSetupCleanupNeeded(boolean needed) {
ensureState(JobState.DEFINE);
conf.setBoolean(SETUP_CLEANUP_NEEDED, needed);
}
/**
* Set the given set of archives
* @param archives The list of archives that need to be localized
*/
public void setCacheArchives(URI[] archives) {
ensureState(JobState.DEFINE);
DistributedCache.setCacheArchives(archives, conf);
}
/**
* Set the given set of files
* @param files The list of files that need to be localized
*/
public void setCacheFiles(URI[] files) {
ensureState(JobState.DEFINE);
DistributedCache.setCacheFiles(files, conf);
}
/**
* Add a archives to be localized
* @param uri The uri of the cache to be localized
*/
public void addCacheArchive(URI uri) {
ensureState(JobState.DEFINE);
DistributedCache.addCacheArchive(uri, conf);
}
/**
* Add a file to be localized
* @param uri The uri of the cache to be localized
*/
public void addCacheFile(URI uri) {
ensureState(JobState.DEFINE);
DistributedCache.addCacheFile(uri, conf);
}
/**
* Add an file path to the current set of clreplacedpath entries It adds the file
* to cache as well.
*
* Files added with this method will not be unpacked while being added to the
* clreplacedpath.
* To add archives to clreplacedpath, use the {@link #addArchiveToClreplacedPath(Path)}
* method instead.
*
* @param file Path of the file to be added
*/
public void addFileToClreplacedPath(Path file) throws IOException {
ensureState(JobState.DEFINE);
DistributedCache.addFileToClreplacedPath(file, conf, file.getFileSystem(conf));
}
/**
* Add an archive path to the current set of clreplacedpath entries. It adds the
* archive to cache as well.
*
* Archive files will be unpacked and added to the clreplacedpath
* when being distributed.
*
* @param archive Path of the archive to be added
*/
public void addArchiveToClreplacedPath(Path archive) throws IOException {
ensureState(JobState.DEFINE);
DistributedCache.addArchiveToClreplacedPath(archive, conf, archive.getFileSystem(conf));
}
/**
* Originally intended to enable symlinks, but currently symlinks cannot be
* disabled.
*/
@Deprecated
public void createSymlink() {
ensureState(JobState.DEFINE);
DistributedCache.createSymlink(conf);
}
/**
* Expert: Set the number of maximum attempts that will be made to run a
* map task.
*
* @param n the number of attempts per map task.
*/
public void setMaxMapAttempts(int n) {
ensureState(JobState.DEFINE);
conf.setMaxMapAttempts(n);
}
/**
* Expert: Set the number of maximum attempts that will be made to run a
* reduce task.
*
* @param n the number of attempts per reduce task.
*/
public void setMaxReduceAttempts(int n) {
ensureState(JobState.DEFINE);
conf.setMaxReduceAttempts(n);
}
/**
* Set whether the system should collect profiler information for some of
* the tasks in this job? The information is stored in the user log
* directory.
* @param newValue true means it should be gathered
*/
public void setProfileEnabled(boolean newValue) {
ensureState(JobState.DEFINE);
conf.setProfileEnabled(newValue);
}
/**
* Set the profiler configuration arguments. If the string contains a '%s' it
* will be replaced with the name of the profiling output file when the task
* runs.
*
* This value is preplaceded to the task child JVM on the command line.
*
* @param value the configuration string
*/
public void setProfileParams(String value) {
ensureState(JobState.DEFINE);
conf.setProfileParams(value);
}
/**
* Set the ranges of maps or reduces to profile. setProfileEnabled(true)
* must also be called.
* @param newValue a set of integer ranges of the map ids
*/
public void setProfileTaskRange(boolean isMap, String newValue) {
ensureState(JobState.DEFINE);
conf.setProfileTaskRange(isMap, newValue);
}
private void ensureNotSet(String attr, String msg) throws IOException {
if (conf.get(attr) != null) {
throw new IOException(attr + " is incompatible with " + msg + " mode.");
}
}
/**
* Sets the flag that will allow the JobTracker to cancel the HDFS delegation
* tokens upon job completion. Defaults to true.
*/
public void setCancelDelegationTokenUponJobCompletion(boolean value) {
ensureState(JobState.DEFINE);
conf.setBoolean(JOB_CANCEL_DELEGATION_TOKEN, value);
}
/**
* Default to the new APIs unless they are explicitly set or the old mapper or
* reduce attributes are used.
* @throws IOException if the configuration is inconsistant
*/
private void setUseNewAPI() throws IOException {
int numReduces = conf.getNumReduceTasks();
String oldMapperClreplaced = "mapred.mapper.clreplaced";
String oldReduceClreplaced = "mapred.reducer.clreplaced";
conf.setBooleanIfUnset("mapred.mapper.new-api", conf.get(oldMapperClreplaced) == null);
if (conf.getUseNewMapper()) {
String mode = "new map API";
ensureNotSet("mapred.input.format.clreplaced", mode);
ensureNotSet(oldMapperClreplaced, mode);
if (numReduces != 0) {
ensureNotSet("mapred.parreplacedioner.clreplaced", mode);
} else {
ensureNotSet("mapred.output.format.clreplaced", mode);
}
} else {
String mode = "map compatability";
ensureNotSet(INPUT_FORMAT_CLreplaced_ATTR, mode);
ensureNotSet(MAP_CLreplaced_ATTR, mode);
if (numReduces != 0) {
ensureNotSet(PARreplacedIONER_CLreplaced_ATTR, mode);
} else {
ensureNotSet(OUTPUT_FORMAT_CLreplaced_ATTR, mode);
}
}
if (numReduces != 0) {
conf.setBooleanIfUnset("mapred.reducer.new-api", conf.get(oldReduceClreplaced) == null);
if (conf.getUseNewReducer()) {
String mode = "new reduce API";
ensureNotSet("mapred.output.format.clreplaced", mode);
ensureNotSet(oldReduceClreplaced, mode);
} else {
String mode = "reduce compatability";
ensureNotSet(OUTPUT_FORMAT_CLreplaced_ATTR, mode);
ensureNotSet(REDUCE_CLreplaced_ATTR, mode);
}
}
}
private synchronized void connect() throws IOException, InterruptedException, ClreplacedNotFoundException {
if (cluster == null) {
cluster = ugi.doAs(new PrivilegedExceptionAction<Cluster>() {
public Cluster run() throws IOException, InterruptedException, ClreplacedNotFoundException {
return new Cluster(getConfiguration());
}
});
}
}
boolean isConnected() {
return cluster != null;
}
/**
* Only for mocking via unit tests.
*/
@Private
public JobSubmitter getJobSubmitter(FileSystem fs, ClientProtocol submitClient) throws IOException {
return new JobSubmitter(fs, submitClient);
}
/**
* Submit the job to the cluster and return immediately.
* @throws IOException
*/
public void submit() throws IOException, InterruptedException, ClreplacedNotFoundException {
ensureState(JobState.DEFINE);
setUseNewAPI();
connect();
final JobSubmitter submitter = getJobSubmitter(cluster.getFileSystem(), cluster.getClient());
status = ugi.doAs(new PrivilegedExceptionAction<JobStatus>() {
public JobStatus run() throws IOException, InterruptedException, ClreplacedNotFoundException {
return submitter.submitJobInternal(Job.this, cluster);
}
});
state = JobState.RUNNING;
LOG.info("The url to track the job: " + getTrackingURL());
}
/**
* Submit the job to the cluster and wait for it to finish.
* @param verbose print the progress to the user
* @return true if the job succeeded
* @throws IOException thrown if the communication with the
* <code>JobTracker</code> is lost
*/
public boolean waitForCompletion(boolean verbose) throws IOException, InterruptedException, ClreplacedNotFoundException {
if (state == JobState.DEFINE) {
submit();
}
if (verbose) {
monitorAndPrintJob();
} else {
// get the completion poll interval from the client.
int completionPollIntervalMillis = Job.getCompletionPollInterval(cluster.getConf());
while (!isComplete()) {
try {
Thread.sleep(completionPollIntervalMillis);
} catch (InterruptedException ie) {
}
}
}
return isSuccessful();
}
/**
* Monitor a job and print status in real-time as progress is made and tasks
* fail.
* @return true if the job succeeded
* @throws IOException if communication to the JobTracker fails
*/
public boolean monitorAndPrintJob() throws IOException, InterruptedException {
String lastReport = null;
Job.TaskStatusFilter filter;
Configuration clientConf = getConfiguration();
filter = Job.getTaskOutputFilter(clientConf);
JobID jobId = getJobID();
LOG.info("Running job: " + jobId);
int eventCounter = 0;
boolean profiling = getProfileEnabled();
IntegerRanges mapRanges = getProfileTaskRange(true);
IntegerRanges reduceRanges = getProfileTaskRange(false);
int progMonitorPollIntervalMillis = Job.getProgressPollInterval(clientConf);
/* make sure to report full progress after the job is done */
boolean reportedAfterCompletion = false;
boolean reportedUberMode = false;
while (!isComplete() || !reportedAfterCompletion) {
if (isComplete()) {
reportedAfterCompletion = true;
} else {
Thread.sleep(progMonitorPollIntervalMillis);
}
if (status.getState() == JobStatus.State.PREP) {
continue;
}
if (!reportedUberMode) {
reportedUberMode = true;
LOG.info("Job " + jobId + " running in uber mode : " + isUber());
}
String report = (" map " + StringUtils.formatPercent(mapProgress(), 0) + " reduce " + StringUtils.formatPercent(reduceProgress(), 0));
if (!report.equals(lastReport)) {
LOG.info(report);
lastReport = report;
}
TaskCompletionEvent[] events = getTaskCompletionEvents(eventCounter, 10);
eventCounter += events.length;
printTaskEvents(events, filter, profiling, mapRanges, reduceRanges);
}
boolean success = isSuccessful();
if (success) {
LOG.info("Job " + jobId + " completed successfully");
} else {
LOG.info("Job " + jobId + " failed with state " + status.getState() + " due to: " + status.getFailureInfo());
}
Counters counters = getCounters();
if (counters != null) {
LOG.info(counters.toString());
}
return success;
}
private void printTaskEvents(TaskCompletionEvent[] events, Job.TaskStatusFilter filter, boolean profiling, IntegerRanges mapRanges, IntegerRanges reduceRanges) throws IOException, InterruptedException {
for (TaskCompletionEvent event : events) {
switch(filter) {
case NONE:
break;
case SUCCEEDED:
if (event.getStatus() == TaskCompletionEvent.Status.SUCCEEDED) {
LOG.info(event.toString());
}
break;
case FAILED:
if (event.getStatus() == TaskCompletionEvent.Status.FAILED) {
LOG.info(event.toString());
// Displaying the task diagnostic information
TaskAttemptID taskId = event.getTaskAttemptId();
String[] taskDiagnostics = getTaskDiagnostics(taskId);
if (taskDiagnostics != null) {
for (String diagnostics : taskDiagnostics) {
System.err.println(diagnostics);
}
}
}
break;
case KILLED:
if (event.getStatus() == TaskCompletionEvent.Status.KILLED) {
LOG.info(event.toString());
}
break;
case ALL:
LOG.info(event.toString());
break;
}
}
}
/**
* The interval at which monitorAndPrintJob() prints status
*/
public static int getProgressPollInterval(Configuration conf) {
// Read progress monitor poll interval from config. Default is 1 second.
int progMonitorPollIntervalMillis = conf.getInt(PROGRESS_MONITOR_POLL_INTERVAL_KEY, DEFAULT_MONITOR_POLL_INTERVAL);
if (progMonitorPollIntervalMillis < 1) {
LOG.warn(PROGRESS_MONITOR_POLL_INTERVAL_KEY + " has been set to an invalid value; " + " replacing with " + DEFAULT_MONITOR_POLL_INTERVAL);
progMonitorPollIntervalMillis = DEFAULT_MONITOR_POLL_INTERVAL;
}
return progMonitorPollIntervalMillis;
}
/**
* The interval at which waitForCompletion() should check.
*/
public static int getCompletionPollInterval(Configuration conf) {
int completionPollIntervalMillis = conf.getInt(COMPLETION_POLL_INTERVAL_KEY, DEFAULT_COMPLETION_POLL_INTERVAL);
if (completionPollIntervalMillis < 1) {
LOG.warn(COMPLETION_POLL_INTERVAL_KEY + " has been set to an invalid value; " + "replacing with " + DEFAULT_COMPLETION_POLL_INTERVAL);
completionPollIntervalMillis = DEFAULT_COMPLETION_POLL_INTERVAL;
}
return completionPollIntervalMillis;
}
/**
* Get the task output filter.
*
* @param conf the configuration.
* @return the filter level.
*/
public static TaskStatusFilter getTaskOutputFilter(Configuration conf) {
return TaskStatusFilter.valueOf(conf.get(Job.OUTPUT_FILTER, "FAILED"));
}
/**
* Modify the Configuration to set the task output filter.
*
* @param conf the Configuration to modify.
* @param newValue the value to set.
*/
public static void setTaskOutputFilter(Configuration conf, TaskStatusFilter newValue) {
conf.set(Job.OUTPUT_FILTER, newValue.toString());
}
public boolean isUber() throws IOException, InterruptedException {
ensureState(JobState.RUNNING);
updateStatus();
return status.isUber();
}
/**
* Get the reservation to which the job is submitted to, if any
*
* @return the reservationId the identifier of the job's reservation, null if
* the job does not have any reservation replacedociated with it
*/
public ReservationId getReservationId() {
return reservationId;
}
/**
* Set the reservation to which the job is submitted to
*
* @param reservationId the reservationId to set
*/
public void setReservationId(ReservationId reservationId) {
this.reservationId = reservationId;
}
}
16
Source : TestInMemoryPlan.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Test
public void testDeleteNonExistingReservation() {
Plan plan = new InMemoryPlan(queueMetrics, policy, agent, totalCapacity, 1L, resCalc, minAlloc, maxAlloc, planName, replanner, true);
ReservationId reservationID = ReservationSystemTestUtil.getNewReservationId();
// Try to delete a reservation without adding
replacedert.replacedertNull(plan.getReservationById(reservationID));
try {
plan.deleteReservation(reservationID);
replacedert.fail("Delete should fail as it does not exist in the plan");
} catch (IllegalArgumentException e) {
replacedert.replacedertTrue(e.getMessage().endsWith("does not exist in the plan"));
} catch (PlanningException e) {
replacedert.fail(e.getMessage());
}
replacedert.replacedertNull(plan.getReservationById(reservationID));
}
16
Source : InMemoryPlan.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Override
public ReservationAllocation getReservationById(ReservationId reservationID) {
if (reservationID == null) {
return null;
}
readLock.lock();
try {
return reservationTable.get(reservationID);
} finally {
readLock.unlock();
}
}
16
Source : InMemoryPlan.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Override
public boolean updateReservation(ReservationAllocation reservation) throws PlanningException {
writeLock.lock();
boolean result = false;
try {
ReservationId resId = reservation.getReservationId();
ReservationAllocation currReservation = getReservationById(resId);
if (currReservation == null) {
String errMsg = "The specified Reservation with ID " + resId + " does not exist in the plan";
LOG.error(errMsg);
throw new IllegalArgumentException(errMsg);
}
// validate if we can accept this reservation, throws exception if
// validation fails
policy.validate(this, reservation);
if (!removeReservation(currReservation)) {
LOG.error("Unable to replace reservation: {} from plan.", reservation.getReservationId());
return result;
}
try {
result = addReservation(reservation);
} catch (PlanningException e) {
LOG.error("Unable to update reservation: {} from plan due to {}.", reservation.getReservationId(), e.getMessage());
}
if (result) {
LOG.info("Sucessfully updated reservation: {} in plan.", reservation.getReservationId());
return result;
} else {
// rollback delete
addReservation(currReservation);
LOG.info("Rollbacked update reservation: {} from plan.", reservation.getReservationId());
return result;
}
} finally {
writeLock.unlock();
}
}
16
Source : ReservationSubmissionResponse.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Private
@Unstable
public static ReservationSubmissionResponse newInstance(ReservationId reservationId) {
ReservationSubmissionResponse response = Records.newRecord(ReservationSubmissionResponse.clreplaced);
response.setReservationId(reservationId);
return response;
}
16
Source : ReservationDeleteRequest.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Public
@Unstable
public static ReservationDeleteRequest newInstance(ReservationId reservationId) {
ReservationDeleteRequest request = Records.newRecord(ReservationDeleteRequest.clreplaced);
request.setReservationId(reservationId);
return request;
}
16
Source : TestCapacitySchedulerPlanFollower.java
with Apache License 2.0
from naver
with Apache License 2.0
from naver
@Override
protected void replacedertReservationQueueExists(ReservationId r) {
CSQueue q = cs.getQueue(r.toString());
replacedertNotNull(q);
}
16
Source : TestCapacitySchedulerPlanFollower.java
with Apache License 2.0
from naver
with Apache License 2.0
from naver
@Override
protected void replacedertReservationQueueDoesNotExist(ReservationId r2) {
CSQueue q2 = cs.getQueue(r2.toString());
replacedertNull(q2);
}
15
Source : TestReservationInputValidator.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Test
public void testDeleteReservationNormal() {
ReservationDeleteRequest request = new ReservationDeleteRequestPBImpl();
ReservationId reservationID = ReservationSystemTestUtil.getNewReservationId();
request.setReservationId(reservationID);
ReservationAllocation reservation = mock(ReservationAllocation.clreplaced);
when(plan.getReservationById(reservationID)).thenReturn(reservation);
Plan plan = null;
try {
plan = rrValidator.validateReservationDeleteRequest(rSystem, request);
} catch (YarnException e) {
replacedert.fail(e.getMessage());
}
replacedert.replacedertNotNull(plan);
}
15
Source : ReservationUpdateRequest.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Public
@Unstable
public static ReservationUpdateRequest newInstance(ReservationDefinition reservationDefinition, ReservationId reservationId) {
ReservationUpdateRequest request = Records.newRecord(ReservationUpdateRequest.clreplaced);
request.setReservationDefinition(reservationDefinition);
request.setReservationId(reservationId);
return request;
}
15
Source : TestFairSchedulerPlanFollower.java
with Apache License 2.0
from naver
with Apache License 2.0
from naver
@Override
protected void replacedertReservationQueueExists(ReservationId r, double expectedCapacity, double expectedMaxCapacity) {
FSLeafQueue q = fs.getQueueManager().getLeafQueue(plan.getQueueName() + "" + "." + r, false);
replacedertNotNull(q);
// For now we are setting both to same weight
replacedert.replacedertEquals(expectedCapacity, q.getWeights().getWeight(ResourceType.MEMORY), 0.01);
}
14
Source : TestReservationInputValidator.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Test
public void testDeleteReservationInvalidPlan() {
ReservationDeleteRequest request = new ReservationDeleteRequestPBImpl();
ReservationId reservationID = ReservationSystemTestUtil.getNewReservationId();
request.setReservationId(reservationID);
when(rSystem.getPlan(PLAN_NAME)).thenReturn(null);
Plan plan = null;
try {
plan = rrValidator.validateReservationDeleteRequest(rSystem, request);
replacedert.fail();
} catch (YarnException e) {
replacedert.replacedertNull(plan);
String message = e.getMessage();
replacedert.replacedertTrue(message.endsWith(" is not replacedociated with any valid plan. Please try again with a valid reservation."));
LOG.info(message);
}
}
14
Source : TestInMemoryReservationAllocation.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Test
public void testZeroAlloaction() {
ReservationId reservationID = ReservationId.newInstance(rand.nextLong(), rand.nextLong());
int[] alloc = {};
long start = 0;
ReservationDefinition rDef = createSimpleReservationDefinition(start, start + alloc.length + 1, alloc.length);
Map<ReservationInterval, ReservationRequest> allocations = new HashMap<ReservationInterval, ReservationRequest>();
ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, start, start + alloc.length + 1, allocations, resCalc, minAlloc);
doreplacedertions(rAllocation, reservationID, rDef, allocations, (int) start, alloc);
replacedert.replacedertFalse(rAllocation.containsGangs());
}
13
Source : TestReservationInputValidator.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Test
public void testDeleteReservationDoesnotExist() {
ReservationDeleteRequest request = new ReservationDeleteRequestPBImpl();
ReservationId rId = ReservationSystemTestUtil.getNewReservationId();
request.setReservationId(rId);
when(rSystem.getQueueForReservation(rId)).thenReturn(null);
Plan plan = null;
try {
plan = rrValidator.validateReservationDeleteRequest(rSystem, request);
replacedert.fail();
} catch (YarnException e) {
replacedert.replacedertNull(plan);
String message = e.getMessage();
replacedert.replacedertTrue(message.equals(MessageFormat.format("The specified reservation with ID: {0} is unknown. Please try again with a valid reservation.", rId)));
LOG.info(message);
}
}
13
Source : TestInMemoryReservationAllocation.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Test
public void testGangAlloaction() {
ReservationId reservationID = ReservationId.newInstance(rand.nextLong(), rand.nextLong());
int[] alloc = { 10, 10, 10, 10, 10, 10 };
int start = 100;
ReservationDefinition rDef = createSimpleReservationDefinition(start, start + alloc.length + 1, alloc.length);
Map<ReservationInterval, ReservationRequest> allocations = generateAllocation(start, alloc, false, true);
ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, start, start + alloc.length + 1, allocations, resCalc, minAlloc);
doreplacedertions(rAllocation, reservationID, rDef, allocations, start, alloc);
replacedert.replacedertTrue(rAllocation.containsGangs());
for (int i = 0; i < alloc.length; i++) {
replacedert.replacedertEquals(Resource.newInstance(1024 * (alloc[i]), (alloc[i])), rAllocation.getResourcesAtTime(start + i));
}
}
13
Source : TestInMemoryReservationAllocation.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Test
public void testSteps() {
ReservationId reservationID = ReservationId.newInstance(rand.nextLong(), rand.nextLong());
int[] alloc = { 10, 10, 10, 10, 10, 10 };
int start = 100;
ReservationDefinition rDef = createSimpleReservationDefinition(start, start + alloc.length + 1, alloc.length);
Map<ReservationInterval, ReservationRequest> allocations = generateAllocation(start, alloc, true, false);
ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, start, start + alloc.length + 1, allocations, resCalc, minAlloc);
doreplacedertions(rAllocation, reservationID, rDef, allocations, start, alloc);
replacedert.replacedertFalse(rAllocation.containsGangs());
for (int i = 0; i < alloc.length; i++) {
replacedert.replacedertEquals(Resource.newInstance(1024 * (alloc[i] + i), (alloc[i] + i)), rAllocation.getResourcesAtTime(start + i));
}
}
13
Source : TestInMemoryReservationAllocation.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Test
public void testBlocks() {
ReservationId reservationID = ReservationId.newInstance(rand.nextLong(), rand.nextLong());
int[] alloc = { 10, 10, 10, 10, 10, 10 };
int start = 100;
ReservationDefinition rDef = createSimpleReservationDefinition(start, start + alloc.length + 1, alloc.length);
Map<ReservationInterval, ReservationRequest> allocations = generateAllocation(start, alloc, false, false);
ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, start, start + alloc.length + 1, allocations, resCalc, minAlloc);
doreplacedertions(rAllocation, reservationID, rDef, allocations, start, alloc);
replacedert.replacedertFalse(rAllocation.containsGangs());
for (int i = 0; i < alloc.length; i++) {
replacedert.replacedertEquals(Resource.newInstance(1024 * (alloc[i]), (alloc[i])), rAllocation.getResourcesAtTime(start + i));
}
}
13
Source : TestInMemoryReservationAllocation.java
with Apache License 2.0
from NJUJYB
with Apache License 2.0
from NJUJYB
@Test
public void testSkyline() {
ReservationId reservationID = ReservationId.newInstance(rand.nextLong(), rand.nextLong());
int[] alloc = { 0, 5, 10, 10, 5, 0 };
int start = 100;
ReservationDefinition rDef = createSimpleReservationDefinition(start, start + alloc.length + 1, alloc.length);
Map<ReservationInterval, ReservationRequest> allocations = generateAllocation(start, alloc, true, false);
ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, start, start + alloc.length + 1, allocations, resCalc, minAlloc);
doreplacedertions(rAllocation, reservationID, rDef, allocations, start, alloc);
replacedert.replacedertFalse(rAllocation.containsGangs());
for (int i = 0; i < alloc.length; i++) {
replacedert.replacedertEquals(Resource.newInstance(1024 * (alloc[i] + i), (alloc[i] + i)), rAllocation.getResourcesAtTime(start + i));
}
}
13
Source : TestSimpleCapacityReplanner.java
with Apache License 2.0
from naver
with Apache License 2.0
from naver
@Test
public void testReplanningPlanCapacityLoss() throws PlanningException {
Resource clusterCapacity = Resource.newInstance(100 * 1024, 10, 10);
Resource minAlloc = Resource.newInstance(1024, 1, 1);
Resource maxAlloc = Resource.newInstance(1024 * 8, 8, 8);
ResourceCalculator res = new DefaultResourceCalculator();
long step = 1L;
Clock clock = mock(Clock.clreplaced);
ReservationAgent agent = mock(ReservationAgent.clreplaced);
SharingPolicy policy = new NoOverCommitPolicy();
policy.init("root.dedicated", null);
QueueMetrics queueMetrics = mock(QueueMetrics.clreplaced);
when(clock.getTime()).thenReturn(0L);
SimpleCapacityReplanner enf = new SimpleCapacityReplanner(clock);
ReservationSchedulerConfiguration conf = mock(ReservationSchedulerConfiguration.clreplaced);
when(conf.getEnforcementWindow(any(String.clreplaced))).thenReturn(6L);
enf.init("blah", conf);
// Initialize the plan with more resources
InMemoryPlan plan = new InMemoryPlan(queueMetrics, policy, agent, clusterCapacity, step, res, minAlloc, maxAlloc, "dedicated", enf, true, clock);
// add reservation filling the plan (separating them 1ms, so we are sure
// s2 follows s1 on acceptance
long ts = System.currentTimeMillis();
ReservationId r1 = ReservationId.newInstance(ts, 1);
int[] f5 = { 20, 20, 20, 20, 20 };
replacedertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r1, null, "u3", "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res, minAlloc)));
when(clock.getTime()).thenReturn(1L);
ReservationId r2 = ReservationId.newInstance(ts, 2);
replacedertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r2, null, "u4", "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res, minAlloc)));
when(clock.getTime()).thenReturn(2L);
ReservationId r3 = ReservationId.newInstance(ts, 3);
replacedertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r3, null, "u5", "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res, minAlloc)));
when(clock.getTime()).thenReturn(3L);
ReservationId r4 = ReservationId.newInstance(ts, 4);
replacedertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r4, null, "u6", "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res, minAlloc)));
when(clock.getTime()).thenReturn(4L);
ReservationId r5 = ReservationId.newInstance(ts, 5);
replacedertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r5, null, "u7", "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res, minAlloc)));
int[] f6 = { 50, 50, 50, 50, 50 };
ReservationId r6 = ReservationId.newInstance(ts, 6);
replacedertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r6, null, "u3", "dedicated", 10, 10 + f6.length, generateAllocation(10, f6), res, minAlloc)));
when(clock.getTime()).thenReturn(6L);
ReservationId r7 = ReservationId.newInstance(ts, 7);
replacedertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation(r7, null, "u4", "dedicated", 10, 10 + f6.length, generateAllocation(10, f6), res, minAlloc)));
// remove some of the resources (requires replanning)
plan.setTotalCapacity(Resource.newInstance(70 * 1024, 70, 70));
when(clock.getTime()).thenReturn(0L);
// run the replanner
enf.plan(plan, null);
// check which reservation are still present
replacedertNotNull(plan.getReservationById(r1));
replacedertNotNull(plan.getReservationById(r2));
replacedertNotNull(plan.getReservationById(r3));
replacedertNotNull(plan.getReservationById(r6));
replacedertNotNull(plan.getReservationById(r7));
// and which ones are removed
replacedertNull(plan.getReservationById(r4));
replacedertNull(plan.getReservationById(r5));
// check resources at each moment in time no more exceed capacity
for (int i = 0; i < 20; i++) {
int tot = 0;
for (ReservationAllocation r : plan.getReservationsAtTime(i)) {
tot = r.getResourcesAtTime(i).getMemory();
}
replacedertTrue(tot <= 70 * 1024);
}
}
13
Source : TestInMemoryReservationAllocation.java
with Apache License 2.0
from naver
with Apache License 2.0
from naver
@Test
public void testBlocks() {
ReservationId reservationID = ReservationId.newInstance(rand.nextLong(), rand.nextLong());
int[] alloc = { 10, 10, 10, 10, 10, 10 };
int start = 100;
ReservationDefinition rDef = createSimpleReservationDefinition(start, start + alloc.length + 1, alloc.length);
Map<ReservationInterval, ReservationRequest> allocations = generateAllocation(start, alloc, false, false);
ReservationAllocation rAllocation = new InMemoryReservationAllocation(reservationID, rDef, user, planName, start, start + alloc.length + 1, allocations, resCalc, minAlloc);
doreplacedertions(rAllocation, reservationID, rDef, allocations, start, alloc);
replacedert.replacedertFalse(rAllocation.containsGangs());
for (int i = 0; i < alloc.length; i++) {
replacedert.replacedertEquals(Resource.newInstance(1024 * (alloc[i]), (alloc[i]), (alloc[i])), rAllocation.getResourcesAtTime(start + i));
}
}
See More Examples