org.apache.hadoop.yarn.api.records.timeline.TimelineEntity

Here are the examples of the java api org.apache.hadoop.yarn.api.records.timeline.TimelineEntity taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

87 Examples 7

19 Source : SystemMetricsPublisher.java
with Apache License 2.0
from NJUJYB

private static TimelineEnreplacedy createAppAttemptEnreplacedy(ApplicationAttemptId appAttemptId) {
    TimelineEnreplacedy enreplacedy = new TimelineEnreplacedy();
    enreplacedy.setEnreplacedyType(AppAttemptMetricsConstants.ENreplacedY_TYPE);
    enreplacedy.setEnreplacedyId(appAttemptId.toString());
    enreplacedy.addPrimaryFilter(AppAttemptMetricsConstants.PARENT_PRIMARY_FILTER, appAttemptId.getApplicationId().toString());
    return enreplacedy;
}

19 Source : SystemMetricsPublisher.java
with Apache License 2.0
from NJUJYB

private void putEnreplacedy(TimelineEnreplacedy enreplacedy) {
    try {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Publishing the enreplacedy " + enreplacedy.getEnreplacedyId() + ", JSON-style content: " + TimelineUtils.dumpTimelineRecordtoJSON(enreplacedy));
        }
        client.putEnreplacedies(enreplacedy);
    } catch (Exception e) {
        LOG.error("Error when publishing enreplacedy [" + enreplacedy.getEnreplacedyType() + "," + enreplacedy.getEnreplacedyId() + "]", e);
    }
}

19 Source : TimelineStoreTestUtils.java
with Apache License 2.0
from NJUJYB

/**
 * Verify a single enreplacedy and its start time
 */
protected static void verifyEnreplacedyInfo(String enreplacedyId, String enreplacedyType, List<TimelineEvent> events, Map<String, Set<String>> relatedEnreplacedies, Map<String, Set<Object>> primaryFilters, Map<String, Object> otherInfo, Long startTime, TimelineEnreplacedy retrievedEnreplacedyInfo, String domainId) {
    verifyEnreplacedyInfo(enreplacedyId, enreplacedyType, events, relatedEnreplacedies, primaryFilters, otherInfo, retrievedEnreplacedyInfo, domainId);
    replacedertEquals(startTime, retrievedEnreplacedyInfo.getStartTime());
}

19 Source : TestTimelineACLsManager.java
with Apache License 2.0
from NJUJYB

@Test
public void testYarnACLsNotEnabledForEnreplacedy() throws Exception {
    Configuration conf = new YarnConfiguration();
    conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, false);
    TimelineACLsManager timelineACLsManager = new TimelineACLsManager(conf);
    timelineACLsManager.setTimelineStore(new TestTimelineStore());
    TimelineEnreplacedy enreplacedy = new TimelineEnreplacedy();
    enreplacedy.addPrimaryFilter(TimelineStore.SystemFilter.ENreplacedY_OWNER.toString(), "owner");
    enreplacedy.setDomainId("domain_id_1");
    replacedert.replacedertTrue("Always true when ACLs are not enabled", timelineACLsManager.checkAccess(UserGroupInformation.createRemoteUser("user"), ApplicationAccessType.VIEW_APP, enreplacedy));
    replacedert.replacedertTrue("Always true when ACLs are not enabled", timelineACLsManager.checkAccess(UserGroupInformation.createRemoteUser("user"), ApplicationAccessType.MODIFY_APP, enreplacedy));
}

19 Source : TestTimelineACLsManager.java
with Apache License 2.0
from NJUJYB

@Test
public void testYarnACLsEnabledForEnreplacedy() throws Exception {
    Configuration conf = new YarnConfiguration();
    conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
    conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
    TimelineACLsManager timelineACLsManager = new TimelineACLsManager(conf);
    timelineACLsManager.setTimelineStore(new TestTimelineStore());
    TimelineEnreplacedy enreplacedy = new TimelineEnreplacedy();
    enreplacedy.addPrimaryFilter(TimelineStore.SystemFilter.ENreplacedY_OWNER.toString(), "owner");
    enreplacedy.setDomainId("domain_id_1");
    replacedert.replacedertTrue("Owner should be allowed to view", timelineACLsManager.checkAccess(UserGroupInformation.createRemoteUser("owner"), ApplicationAccessType.VIEW_APP, enreplacedy));
    replacedert.replacedertTrue("Reader should be allowed to view", timelineACLsManager.checkAccess(UserGroupInformation.createRemoteUser("reader"), ApplicationAccessType.VIEW_APP, enreplacedy));
    replacedert.replacedertFalse("Other shouldn't be allowed to view", timelineACLsManager.checkAccess(UserGroupInformation.createRemoteUser("other"), ApplicationAccessType.VIEW_APP, enreplacedy));
    replacedert.replacedertTrue("Admin should be allowed to view", timelineACLsManager.checkAccess(UserGroupInformation.createRemoteUser("admin"), ApplicationAccessType.VIEW_APP, enreplacedy));
    replacedert.replacedertTrue("Owner should be allowed to modify", timelineACLsManager.checkAccess(UserGroupInformation.createRemoteUser("owner"), ApplicationAccessType.MODIFY_APP, enreplacedy));
    replacedert.replacedertTrue("Writer should be allowed to modify", timelineACLsManager.checkAccess(UserGroupInformation.createRemoteUser("writer"), ApplicationAccessType.MODIFY_APP, enreplacedy));
    replacedert.replacedertFalse("Other shouldn't be allowed to modify", timelineACLsManager.checkAccess(UserGroupInformation.createRemoteUser("other"), ApplicationAccessType.MODIFY_APP, enreplacedy));
    replacedert.replacedertTrue("Admin should be allowed to modify", timelineACLsManager.checkAccess(UserGroupInformation.createRemoteUser("admin"), ApplicationAccessType.MODIFY_APP, enreplacedy));
}

19 Source : TimelineDataManager.java
with Apache License 2.0
from NJUJYB

private static void addDefaultDomainIdIfAbsent(TimelineEnreplacedy enreplacedy) {
    // be compatible with the timeline data created before 2.6
    if (enreplacedy.getDomainId() == null) {
        enreplacedy.setDomainId(DEFAULT_DOMAIN_ID);
    }
}

18 Source : TestSystemMetricsPublisher.java
with Apache License 2.0
from NJUJYB

@Test(timeout = 10000)
public void testPublishAppAttemptMetrics() throws Exception {
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1);
    RMAppAttempt appAttempt = createRMAppAttempt(appAttemptId);
    metricsPublisher.appAttemptRegistered(appAttempt, Integer.MAX_VALUE + 1L);
    RMApp app = mock(RMApp.clreplaced);
    when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.UNDEFINED);
    metricsPublisher.appAttemptFinished(appAttempt, RMAppAttemptState.FINISHED, app, Integer.MAX_VALUE + 2L);
    TimelineEnreplacedy enreplacedy = null;
    do {
        enreplacedy = store.getEnreplacedy(appAttemptId.toString(), AppAttemptMetricsConstants.ENreplacedY_TYPE, EnumSet.allOf(Field.clreplaced));
    // ensure two events are both published before leaving the loop
    } while (enreplacedy == null || enreplacedy.getEvents().size() < 2);
    // verify all the fields
    replacedert.replacedertEquals(AppAttemptMetricsConstants.ENreplacedY_TYPE, enreplacedy.getEnreplacedyType());
    replacedert.replacedertEquals(appAttemptId.toString(), enreplacedy.getEnreplacedyId());
    replacedert.replacedertEquals(appAttemptId.getApplicationId().toString(), enreplacedy.getPrimaryFilters().get(AppAttemptMetricsConstants.PARENT_PRIMARY_FILTER).iterator().next());
    boolean hasRegisteredEvent = false;
    boolean hasFinishedEvent = false;
    for (TimelineEvent event : enreplacedy.getEvents()) {
        if (event.getEventType().equals(AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE)) {
            hasRegisteredEvent = true;
            replacedert.replacedertEquals(appAttempt.getHost(), event.getEventInfo().get(AppAttemptMetricsConstants.HOST_EVENT_INFO));
            replacedert.replacedertEquals(appAttempt.getRpcPort(), event.getEventInfo().get(AppAttemptMetricsConstants.RPC_PORT_EVENT_INFO));
            replacedert.replacedertEquals(appAttempt.getMasterContainer().getId().toString(), event.getEventInfo().get(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO));
        } else if (event.getEventType().equals(AppAttemptMetricsConstants.FINISHED_EVENT_TYPE)) {
            hasFinishedEvent = true;
            replacedert.replacedertEquals(appAttempt.getDiagnostics(), event.getEventInfo().get(AppAttemptMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO));
            replacedert.replacedertEquals(appAttempt.getTrackingUrl(), event.getEventInfo().get(AppAttemptMetricsConstants.TRACKING_URL_EVENT_INFO));
            replacedert.replacedertEquals(appAttempt.getOriginalTrackingUrl(), event.getEventInfo().get(AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_EVENT_INFO));
            replacedert.replacedertEquals(FinalApplicationStatus.UNDEFINED.toString(), event.getEventInfo().get(AppAttemptMetricsConstants.FINAL_STATUS_EVENT_INFO));
            replacedert.replacedertEquals(YarnApplicationAttemptState.FINISHED.toString(), event.getEventInfo().get(AppAttemptMetricsConstants.STATE_EVENT_INFO));
        }
    }
    replacedert.replacedertTrue(hasRegisteredEvent && hasFinishedEvent);
}

18 Source : SystemMetricsPublisher.java
with Apache License 2.0
from NJUJYB

private static TimelineEnreplacedy createContainerEnreplacedy(ContainerId containerId) {
    TimelineEnreplacedy enreplacedy = new TimelineEnreplacedy();
    enreplacedy.setEnreplacedyType(ContainerMetricsConstants.ENreplacedY_TYPE);
    enreplacedy.setEnreplacedyId(containerId.toString());
    enreplacedy.addPrimaryFilter(ContainerMetricsConstants.PARENT_PRIMARIY_FILTER, containerId.getApplicationAttemptId().toString());
    return enreplacedy;
}

18 Source : TestTimelineACLsManager.java
with Apache License 2.0
from NJUJYB

@Test
public void testCorruptedOwnerInfoForEnreplacedy() throws Exception {
    Configuration conf = new YarnConfiguration();
    conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
    conf.set(YarnConfiguration.YARN_ADMIN_ACL, "owner");
    TimelineACLsManager timelineACLsManager = new TimelineACLsManager(conf);
    timelineACLsManager.setTimelineStore(new TestTimelineStore());
    TimelineEnreplacedy enreplacedy = new TimelineEnreplacedy();
    try {
        timelineACLsManager.checkAccess(UserGroupInformation.createRemoteUser("owner"), ApplicationAccessType.VIEW_APP, enreplacedy);
        replacedert.fail("Exception is expected");
    } catch (YarnException e) {
        replacedert.replacedertTrue("It's not the exact expected exception", e.getMessage().contains("doesn't exist."));
    }
}

18 Source : TimelineDataManager.java
with Apache License 2.0
from NJUJYB

/**
 * Get the single timeline enreplacedy that the given user has access to. The
 * meaning of each argument has been doreplacedented with
 * {@link TimelineReader#getEnreplacedy}.
 *
 * @see TimelineReader#getEnreplacedy
 */
public TimelineEnreplacedy getEnreplacedy(String enreplacedyType, String enreplacedyId, EnumSet<Field> fields, UserGroupInformation callerUGI) throws YarnException, IOException {
    TimelineEnreplacedy enreplacedy = null;
    enreplacedy = store.getEnreplacedy(enreplacedyId, enreplacedyType, fields);
    if (enreplacedy != null) {
        addDefaultDomainIdIfAbsent(enreplacedy);
        // check ACLs
        if (!timelineACLsManager.checkAccess(callerUGI, ApplicationAccessType.VIEW_APP, enreplacedy)) {
            enreplacedy = null;
        }
    }
    return enreplacedy;
}

18 Source : TimelineACLsManager.java
with Apache License 2.0
from NJUJYB

public boolean checkAccess(UserGroupInformation callerUGI, ApplicationAccessType applicationAccessType, TimelineEnreplacedy enreplacedy) throws YarnException, IOException {
    if (LOG.isDebugEnabled()) {
        LOG.debug("Verifying the access of " + (callerUGI == null ? null : callerUGI.getShortUserName()) + " on the timeline enreplacedy " + new EnreplacedyIdentifier(enreplacedy.getEnreplacedyId(), enreplacedy.getEnreplacedyType()));
    }
    if (!adminAclsManager.areACLsEnabled()) {
        return true;
    }
    // find domain owner and acls
    AccessControlListExt aclExt = aclExts.get(enreplacedy.getDomainId());
    if (aclExt == null) {
        aclExt = loadDomainFromTimelineStore(enreplacedy.getDomainId());
    }
    if (aclExt == null) {
        throw new YarnException("Domain information of the timeline enreplacedy " + new EnreplacedyIdentifier(enreplacedy.getEnreplacedyId(), enreplacedy.getEnreplacedyType()) + " doesn't exist.");
    }
    String owner = aclExt.owner;
    AccessControlList domainACL = aclExt.acls.get(applicationAccessType);
    if (domainACL == null) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("ACL not found for access-type " + applicationAccessType + " for domain " + enreplacedy.getDomainId() + " owned by " + owner + ". Using default [" + YarnConfiguration.DEFAULT_YARN_APP_ACL + "]");
        }
        domainACL = new AccessControlList(YarnConfiguration.DEFAULT_YARN_APP_ACL);
    }
    if (callerUGI != null && (adminAclsManager.isAdmin(callerUGI) || callerUGI.getShortUserName().equals(owner) || domainACL.isUserAllowed(callerUGI))) {
        return true;
    }
    return false;
}

18 Source : MemoryTimelineStore.java
with Apache License 2.0
from NJUJYB

@Override
public synchronized TimelineEnreplacedy getEnreplacedy(String enreplacedyId, String enreplacedyType, EnumSet<Field> fieldsToRetrieve) {
    if (fieldsToRetrieve == null) {
        fieldsToRetrieve = EnumSet.allOf(Field.clreplaced);
    }
    TimelineEnreplacedy enreplacedy = enreplacedies.get(new EnreplacedyIdentifier(enreplacedyId, enreplacedyType));
    if (enreplacedy == null) {
        return null;
    } else {
        return maskFields(enreplacedy, fieldsToRetrieve);
    }
}

18 Source : ApplicationHistoryManagerOnTimelineStore.java
with Apache License 2.0
from naver

private ApplicationAttemptReport getApplicationAttempt(ApplicationAttemptId appAttemptId, boolean checkACLs) throws YarnException, IOException {
    if (checkACLs) {
        ApplicationReportExt app = getApplication(appAttemptId.getApplicationId(), ApplicationReportField.USER_AND_ACLS);
        checkAccess(app);
    }
    TimelineEnreplacedy enreplacedy = timelineDataManager.getEnreplacedy(AppAttemptMetricsConstants.ENreplacedY_TYPE, appAttemptId.toString(), EnumSet.allOf(Field.clreplaced), UserGroupInformation.getLoginUser());
    if (enreplacedy == null) {
        throw new ApplicationAttemptNotFoundException("The enreplacedy for application attempt " + appAttemptId + " doesn't exist in the timeline store");
    } else {
        return convertToApplicationAttemptReport(enreplacedy);
    }
}

18 Source : ApplicationHistoryManagerOnTimelineStore.java
with Apache License 2.0
from naver

@Override
public Map<ApplicationId, ApplicationReport> getAllApplications() throws YarnException, IOException {
    TimelineEnreplacedies enreplacedies = timelineDataManager.getEnreplacedies(ApplicationMetricsConstants.ENreplacedY_TYPE, null, null, null, null, null, null, Long.MAX_VALUE, EnumSet.allOf(Field.clreplaced), UserGroupInformation.getLoginUser());
    Map<ApplicationId, ApplicationReport> apps = new LinkedHashMap<ApplicationId, ApplicationReport>();
    if (enreplacedies != null && enreplacedies.getEnreplacedies() != null) {
        for (TimelineEnreplacedy enreplacedy : enreplacedies.getEnreplacedies()) {
            try {
                ApplicationReportExt app = generateApplicationReport(enreplacedy, ApplicationReportField.ALL);
                apps.put(app.appReport.getApplicationId(), app.appReport);
            } catch (Exception e) {
                LOG.error("Error on generating application report for " + enreplacedy.getEnreplacedyId(), e);
            }
        }
    }
    return apps;
}

18 Source : ApplicationMaster.java
with Apache License 2.0
from naver

private static void publishApplicationAttemptEvent(final TimelineClient timelineClient, String appAttemptId, DSEvent appEvent, String domainId, UserGroupInformation ugi) {
    final TimelineEnreplacedy enreplacedy = new TimelineEnreplacedy();
    enreplacedy.setEnreplacedyId(appAttemptId);
    enreplacedy.setEnreplacedyType(DSEnreplacedy.DS_APP_ATTEMPT.toString());
    enreplacedy.setDomainId(domainId);
    enreplacedy.addPrimaryFilter("user", ugi.getShortUserName());
    TimelineEvent event = new TimelineEvent();
    event.setEventType(appEvent.toString());
    event.setTimestamp(System.currentTimeMillis());
    enreplacedy.addEvent(event);
    try {
        timelineClient.putEnreplacedies(enreplacedy);
    } catch (YarnException | IOException e) {
        LOG.error("App Attempt " + (appEvent.equals(DSEvent.DS_APP_ATTEMPT_START) ? "start" : "end") + " event could not be published for " + appAttemptId.toString(), e);
    }
}

18 Source : TestSystemMetricsPublisher.java
with Apache License 2.0
from lsds

@Test(timeout = 10000)
public void testPublishAppAttemptMetricsForUnmanagedAM() throws Exception {
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1);
    RMAppAttempt appAttempt = createRMAppAttempt(appAttemptId, true);
    metricsPublisher.appAttemptRegistered(appAttempt, Integer.MAX_VALUE + 1L);
    RMApp app = mock(RMApp.clreplaced);
    when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.UNDEFINED);
    metricsPublisher.appAttemptFinished(appAttempt, RMAppAttemptState.FINISHED, app, Integer.MAX_VALUE + 2L);
    TimelineEnreplacedy enreplacedy = null;
    do {
        enreplacedy = store.getEnreplacedy(appAttemptId.toString(), AppAttemptMetricsConstants.ENreplacedY_TYPE, EnumSet.allOf(Field.clreplaced));
    // ensure two events are both published before leaving the loop
    } while (enreplacedy == null || enreplacedy.getEvents().size() < 2);
}

18 Source : TestSystemMetricsPublisher.java
with Apache License 2.0
from lsds

@Test(timeout = 10000)
public void testPublishAppAttemptMetrics() throws Exception {
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1);
    RMAppAttempt appAttempt = createRMAppAttempt(appAttemptId, false);
    metricsPublisher.appAttemptRegistered(appAttempt, Integer.MAX_VALUE + 1L);
    RMApp app = mock(RMApp.clreplaced);
    when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.UNDEFINED);
    metricsPublisher.appAttemptFinished(appAttempt, RMAppAttemptState.FINISHED, app, Integer.MAX_VALUE + 2L);
    TimelineEnreplacedy enreplacedy = null;
    do {
        enreplacedy = store.getEnreplacedy(appAttemptId.toString(), AppAttemptMetricsConstants.ENreplacedY_TYPE, EnumSet.allOf(Field.clreplaced));
    // ensure two events are both published before leaving the loop
    } while (enreplacedy == null || enreplacedy.getEvents().size() < 2);
    // verify all the fields
    replacedert.replacedertEquals(AppAttemptMetricsConstants.ENreplacedY_TYPE, enreplacedy.getEnreplacedyType());
    replacedert.replacedertEquals(appAttemptId.toString(), enreplacedy.getEnreplacedyId());
    replacedert.replacedertEquals(appAttemptId.getApplicationId().toString(), enreplacedy.getPrimaryFilters().get(AppAttemptMetricsConstants.PARENT_PRIMARY_FILTER).iterator().next());
    boolean hasRegisteredEvent = false;
    boolean hasFinishedEvent = false;
    for (TimelineEvent event : enreplacedy.getEvents()) {
        if (event.getEventType().equals(AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE)) {
            hasRegisteredEvent = true;
            replacedert.replacedertEquals(appAttempt.getHost(), event.getEventInfo().get(AppAttemptMetricsConstants.HOST_EVENT_INFO));
            replacedert.replacedertEquals(appAttempt.getRpcPort(), event.getEventInfo().get(AppAttemptMetricsConstants.RPC_PORT_EVENT_INFO));
            replacedert.replacedertEquals(appAttempt.getMasterContainer().getId().toString(), event.getEventInfo().get(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO));
        } else if (event.getEventType().equals(AppAttemptMetricsConstants.FINISHED_EVENT_TYPE)) {
            hasFinishedEvent = true;
            replacedert.replacedertEquals(appAttempt.getDiagnostics(), event.getEventInfo().get(AppAttemptMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO));
            replacedert.replacedertEquals(appAttempt.getTrackingUrl(), event.getEventInfo().get(AppAttemptMetricsConstants.TRACKING_URL_EVENT_INFO));
            replacedert.replacedertEquals(appAttempt.getOriginalTrackingUrl(), event.getEventInfo().get(AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_EVENT_INFO));
            replacedert.replacedertEquals(FinalApplicationStatus.UNDEFINED.toString(), event.getEventInfo().get(AppAttemptMetricsConstants.FINAL_STATUS_EVENT_INFO));
            replacedert.replacedertEquals(YarnApplicationAttemptState.FINISHED.toString(), event.getEventInfo().get(AppAttemptMetricsConstants.STATE_EVENT_INFO));
        }
    }
    replacedert.replacedertTrue(hasRegisteredEvent && hasFinishedEvent);
}

17 Source : TestSystemMetricsPublisher.java
with Apache License 2.0
from NJUJYB

@Test(timeout = 10000)
public void testPublishApplicationMetrics() throws Exception {
    for (int i = 1; i <= 2; ++i) {
        ApplicationId appId = ApplicationId.newInstance(0, i);
        RMApp app = createRMApp(appId);
        metricsPublisher.appCreated(app, app.getStartTime());
        metricsPublisher.appFinished(app, RMAppState.FINISHED, app.getFinishTime());
        if (i == 1) {
            metricsPublisher.appACLsUpdated(app, "uers1,user2", 4L);
        } else {
            // in case user doesn't specify the ACLs
            metricsPublisher.appACLsUpdated(app, null, 4L);
        }
        TimelineEnreplacedy enreplacedy = null;
        do {
            enreplacedy = store.getEnreplacedy(appId.toString(), ApplicationMetricsConstants.ENreplacedY_TYPE, EnumSet.allOf(Field.clreplaced));
        // ensure three events are both published before leaving the loop
        } while (enreplacedy == null || enreplacedy.getEvents().size() < 3);
        // verify all the fields
        replacedert.replacedertEquals(ApplicationMetricsConstants.ENreplacedY_TYPE, enreplacedy.getEnreplacedyType());
        replacedert.replacedertEquals(app.getApplicationId().toString(), enreplacedy.getEnreplacedyId());
        replacedert.replacedertEquals(app.getName(), enreplacedy.getOtherInfo().get(ApplicationMetricsConstants.NAME_ENreplacedY_INFO));
        replacedert.replacedertEquals(app.getQueue(), enreplacedy.getOtherInfo().get(ApplicationMetricsConstants.QUEUE_ENreplacedY_INFO));
        replacedert.replacedertEquals(app.getUser(), enreplacedy.getOtherInfo().get(ApplicationMetricsConstants.USER_ENreplacedY_INFO));
        replacedert.replacedertEquals(app.getApplicationType(), enreplacedy.getOtherInfo().get(ApplicationMetricsConstants.TYPE_ENreplacedY_INFO));
        replacedert.replacedertEquals(app.getSubmitTime(), enreplacedy.getOtherInfo().get(ApplicationMetricsConstants.SUBMITTED_TIME_ENreplacedY_INFO));
        if (i == 1) {
            replacedert.replacedertEquals("uers1,user2", enreplacedy.getOtherInfo().get(ApplicationMetricsConstants.APP_VIEW_ACLS_ENreplacedY_INFO));
        } else {
            replacedert.replacedertEquals("", enreplacedy.getOtherInfo().get(ApplicationMetricsConstants.APP_VIEW_ACLS_ENreplacedY_INFO));
        }
        boolean hasCreatedEvent = false;
        boolean hasFinishedEvent = false;
        boolean hasACLsUpdatedEvent = false;
        for (TimelineEvent event : enreplacedy.getEvents()) {
            if (event.getEventType().equals(ApplicationMetricsConstants.CREATED_EVENT_TYPE)) {
                hasCreatedEvent = true;
                replacedert.replacedertEquals(app.getStartTime(), event.getTimestamp());
            } else if (event.getEventType().equals(ApplicationMetricsConstants.FINISHED_EVENT_TYPE)) {
                hasFinishedEvent = true;
                replacedert.replacedertEquals(app.getFinishTime(), event.getTimestamp());
                replacedert.replacedertEquals(app.getDiagnostics().toString(), event.getEventInfo().get(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO));
                replacedert.replacedertEquals(app.getFinalApplicationStatus().toString(), event.getEventInfo().get(ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO));
                replacedert.replacedertEquals(YarnApplicationState.FINISHED.toString(), event.getEventInfo().get(ApplicationMetricsConstants.STATE_EVENT_INFO));
            } else if (event.getEventType().equals(ApplicationMetricsConstants.ACLS_UPDATED_EVENT_TYPE)) {
                hasACLsUpdatedEvent = true;
                replacedert.replacedertEquals(4L, event.getTimestamp());
            }
        }
        replacedert.replacedertTrue(hasCreatedEvent && hasFinishedEvent && hasACLsUpdatedEvent);
    }
}

17 Source : SystemMetricsPublisher.java
with Apache License 2.0
from NJUJYB

private static TimelineEnreplacedy createApplicationEnreplacedy(ApplicationId applicationId) {
    TimelineEnreplacedy enreplacedy = new TimelineEnreplacedy();
    enreplacedy.setEnreplacedyType(ApplicationMetricsConstants.ENreplacedY_TYPE);
    enreplacedy.setEnreplacedyId(applicationId.toString());
    return enreplacedy;
}

17 Source : TestTimelineWebServices.java
with Apache License 2.0
from NJUJYB

private static void verifyEnreplacedies(TimelineEnreplacedies enreplacedies) {
    replacedert.replacedertNotNull(enreplacedies);
    replacedert.replacedertEquals(3, enreplacedies.getEnreplacedies().size());
    TimelineEnreplacedy enreplacedy1 = enreplacedies.getEnreplacedies().get(0);
    replacedert.replacedertNotNull(enreplacedy1);
    replacedert.replacedertEquals("id_1", enreplacedy1.getEnreplacedyId());
    replacedert.replacedertEquals("type_1", enreplacedy1.getEnreplacedyType());
    replacedert.replacedertEquals(123l, enreplacedy1.getStartTime().longValue());
    replacedert.replacedertEquals(2, enreplacedy1.getEvents().size());
    replacedert.replacedertEquals(4, enreplacedy1.getPrimaryFilters().size());
    replacedert.replacedertEquals(4, enreplacedy1.getOtherInfo().size());
    TimelineEnreplacedy enreplacedy2 = enreplacedies.getEnreplacedies().get(1);
    replacedert.replacedertNotNull(enreplacedy2);
    replacedert.replacedertEquals("id_2", enreplacedy2.getEnreplacedyId());
    replacedert.replacedertEquals("type_1", enreplacedy2.getEnreplacedyType());
    replacedert.replacedertEquals(123l, enreplacedy2.getStartTime().longValue());
    replacedert.replacedertEquals(2, enreplacedy2.getEvents().size());
    replacedert.replacedertEquals(4, enreplacedy2.getPrimaryFilters().size());
    replacedert.replacedertEquals(4, enreplacedy2.getOtherInfo().size());
    TimelineEnreplacedy enreplacedy3 = enreplacedies.getEnreplacedies().get(2);
    replacedert.replacedertNotNull(enreplacedy2);
    replacedert.replacedertEquals("id_6", enreplacedy3.getEnreplacedyId());
    replacedert.replacedertEquals("type_1", enreplacedy3.getEnreplacedyType());
    replacedert.replacedertEquals(61l, enreplacedy3.getStartTime().longValue());
    replacedert.replacedertEquals(0, enreplacedy3.getEvents().size());
    replacedert.replacedertEquals(4, enreplacedy3.getPrimaryFilters().size());
    replacedert.replacedertEquals(4, enreplacedy3.getOtherInfo().size());
}

17 Source : MemoryTimelineStore.java
with Apache License 2.0
from NJUJYB

@Override
public synchronized TimelineEvents getEnreplacedyTimelines(String enreplacedyType, SortedSet<String> enreplacedyIds, Long limit, Long windowStart, Long windowEnd, Set<String> eventTypes) {
    TimelineEvents allEvents = new TimelineEvents();
    if (enreplacedyIds == null) {
        return allEvents;
    }
    if (limit == null) {
        limit = DEFAULT_LIMIT;
    }
    if (windowStart == null) {
        windowStart = Long.MIN_VALUE;
    }
    if (windowEnd == null) {
        windowEnd = Long.MAX_VALUE;
    }
    for (String enreplacedyId : enreplacedyIds) {
        EnreplacedyIdentifier enreplacedyID = new EnreplacedyIdentifier(enreplacedyId, enreplacedyType);
        TimelineEnreplacedy enreplacedy = enreplacedies.get(enreplacedyID);
        if (enreplacedy == null) {
            continue;
        }
        EventsOfOneEnreplacedy events = new EventsOfOneEnreplacedy();
        events.setEnreplacedyId(enreplacedyId);
        events.setEnreplacedyType(enreplacedyType);
        for (TimelineEvent event : enreplacedy.getEvents()) {
            if (events.getEvents().size() >= limit) {
                break;
            }
            if (event.getTimestamp() <= windowStart) {
                continue;
            }
            if (event.getTimestamp() > windowEnd) {
                continue;
            }
            if (eventTypes != null && !eventTypes.contains(event.getEventType())) {
                continue;
            }
            events.addEvent(event);
        }
        allEvents.addEvent(events);
    }
    return allEvents;
}

17 Source : ApplicationHistoryManagerOnTimelineStore.java
with Apache License 2.0
from NJUJYB

@Override
public ContainerReport getContainer(ContainerId containerId) throws YarnException, IOException {
    ApplicationReportExt app = getApplication(containerId.getApplicationAttemptId().getApplicationId(), ApplicationReportField.USER_AND_ACLS);
    checkAccess(app);
    TimelineEnreplacedy enreplacedy = timelineDataManager.getEnreplacedy(ContainerMetricsConstants.ENreplacedY_TYPE, containerId.toString(), EnumSet.allOf(Field.clreplaced), UserGroupInformation.getLoginUser());
    if (enreplacedy == null) {
        throw new ContainerNotFoundException("The enreplacedy for container " + containerId + " doesn't exist in the timeline store");
    } else {
        return convertToContainerReport(enreplacedy, serverHttpAddress, app.appReport.getUser());
    }
}

17 Source : ApplicationHistoryManagerOnTimelineStore.java
with Apache License 2.0
from NJUJYB

@Override
public Map<ApplicationAttemptId, ApplicationAttemptReport> getApplicationAttempts(ApplicationId appId) throws YarnException, IOException {
    ApplicationReportExt app = getApplication(appId, ApplicationReportField.USER_AND_ACLS);
    checkAccess(app);
    TimelineEnreplacedies enreplacedies = timelineDataManager.getEnreplacedies(AppAttemptMetricsConstants.ENreplacedY_TYPE, new NameValuePair(AppAttemptMetricsConstants.PARENT_PRIMARY_FILTER, appId.toString()), null, null, null, null, null, Long.MAX_VALUE, EnumSet.allOf(Field.clreplaced), UserGroupInformation.getLoginUser());
    Map<ApplicationAttemptId, ApplicationAttemptReport> appAttempts = new LinkedHashMap<ApplicationAttemptId, ApplicationAttemptReport>();
    for (TimelineEnreplacedy enreplacedy : enreplacedies.getEnreplacedies()) {
        ApplicationAttemptReport appAttempt = convertToApplicationAttemptReport(enreplacedy);
        appAttempts.put(appAttempt.getApplicationAttemptId(), appAttempt);
    }
    return appAttempts;
}

17 Source : ApplicationHistoryManagerOnTimelineStore.java
with Apache License 2.0
from NJUJYB

@Override
public Map<ContainerId, ContainerReport> getContainers(ApplicationAttemptId appAttemptId) throws YarnException, IOException {
    ApplicationReportExt app = getApplication(appAttemptId.getApplicationId(), ApplicationReportField.USER_AND_ACLS);
    checkAccess(app);
    TimelineEnreplacedies enreplacedies = timelineDataManager.getEnreplacedies(ContainerMetricsConstants.ENreplacedY_TYPE, new NameValuePair(ContainerMetricsConstants.PARENT_PRIMARIY_FILTER, appAttemptId.toString()), null, null, null, null, null, Long.MAX_VALUE, EnumSet.allOf(Field.clreplaced), UserGroupInformation.getLoginUser());
    Map<ContainerId, ContainerReport> containers = new LinkedHashMap<ContainerId, ContainerReport>();
    if (enreplacedies != null && enreplacedies.getEnreplacedies() != null) {
        for (TimelineEnreplacedy enreplacedy : enreplacedies.getEnreplacedies()) {
            ContainerReport container = convertToContainerReport(enreplacedy, serverHttpAddress, app.appReport.getUser());
            containers.put(container.getContainerId(), container);
        }
    }
    return containers;
}

17 Source : ApplicationHistoryManagerOnTimelineStore.java
with Apache License 2.0
from NJUJYB

@Override
public ApplicationAttemptReport getApplicationAttempt(ApplicationAttemptId appAttemptId) throws YarnException, IOException {
    ApplicationReportExt app = getApplication(appAttemptId.getApplicationId(), ApplicationReportField.USER_AND_ACLS);
    checkAccess(app);
    TimelineEnreplacedy enreplacedy = timelineDataManager.getEnreplacedy(AppAttemptMetricsConstants.ENreplacedY_TYPE, appAttemptId.toString(), EnumSet.allOf(Field.clreplaced), UserGroupInformation.getLoginUser());
    if (enreplacedy == null) {
        throw new ApplicationAttemptNotFoundException("The enreplacedy for application attempt " + appAttemptId + " doesn't exist in the timeline store");
    } else {
        return convertToApplicationAttemptReport(enreplacedy);
    }
}

17 Source : TimelineClientImpl.java
with Apache License 2.0
from NJUJYB

@Override
public TimelinePutResponse putEnreplacedies(TimelineEnreplacedy... enreplacedies) throws IOException, YarnException {
    TimelineEnreplacedies enreplacediesContainer = new TimelineEnreplacedies();
    enreplacediesContainer.addEnreplacedies(Arrays.asList(enreplacedies));
    ClientResponse resp = doPosting(enreplacediesContainer, null);
    return resp.getEnreplacedy(TimelinePutResponse.clreplaced);
}

17 Source : ApplicationMaster.java
with Apache License 2.0
from NJUJYB

private static void publishApplicationAttemptEvent(final TimelineClient timelineClient, String appAttemptId, DSEvent appEvent, String domainId, UserGroupInformation ugi) {
    final TimelineEnreplacedy enreplacedy = new TimelineEnreplacedy();
    enreplacedy.setEnreplacedyId(appAttemptId);
    enreplacedy.setEnreplacedyType(DSEnreplacedy.DS_APP_ATTEMPT.toString());
    enreplacedy.setDomainId(domainId);
    enreplacedy.addPrimaryFilter("user", ugi.getShortUserName());
    TimelineEvent event = new TimelineEvent();
    event.setEventType(appEvent.toString());
    event.setTimestamp(System.currentTimeMillis());
    enreplacedy.addEvent(event);
    try {
        ugi.doAs(new PrivilegedExceptionAction<TimelinePutResponse>() {

            @Override
            public TimelinePutResponse run() throws Exception {
                return timelineClient.putEnreplacedies(enreplacedy);
            }
        });
    } catch (Exception e) {
        LOG.error("App Attempt " + (appEvent.equals(DSEvent.DS_APP_ATTEMPT_START) ? "start" : "end") + " event could not be published for " + appAttemptId.toString(), e instanceof UndeclaredThrowableException ? e.getCause() : e);
    }
}

17 Source : ApplicationHistoryManagerOnTimelineStore.java
with Apache License 2.0
from naver

private ApplicationReportExt generateApplicationReport(TimelineEnreplacedy enreplacedy, ApplicationReportField field) throws YarnException, IOException {
    ApplicationReportExt app = convertToApplicationReport(enreplacedy, field);
    // If only user and acls are pulled to check attempt(s)/container(s) access
    // control, we can return immediately
    if (field == ApplicationReportField.USER_AND_ACLS) {
        return app;
    }
    try {
        checkAccess(app);
        if (app.appReport.getCurrentApplicationAttemptId() != null) {
            ApplicationAttemptReport appAttempt = getApplicationAttempt(app.appReport.getCurrentApplicationAttemptId(), false);
            app.appReport.setHost(appAttempt.getHost());
            app.appReport.setRpcPort(appAttempt.getRpcPort());
            app.appReport.setTrackingUrl(appAttempt.getTrackingUrl());
            app.appReport.setOriginalTrackingUrl(appAttempt.getOriginalTrackingUrl());
        }
    } catch (AuthorizationException | ApplicationAttemptNotFoundException e) {
        // AuthorizationException is thrown because the user doesn't have access
        // It's possible that the app is finished before the first attempt is created.
        app.appReport.setDiagnostics(null);
        app.appReport.setCurrentApplicationAttemptId(null);
    }
    if (app.appReport.getCurrentApplicationAttemptId() == null) {
        app.appReport.setCurrentApplicationAttemptId(ApplicationAttemptId.newInstance(app.appReport.getApplicationId(), -1));
    }
    if (app.appReport.getHost() == null) {
        app.appReport.setHost(UNAVAILABLE);
    }
    if (app.appReport.getRpcPort() < 0) {
        app.appReport.setRpcPort(-1);
    }
    if (app.appReport.getTrackingUrl() == null) {
        app.appReport.setTrackingUrl(UNAVAILABLE);
    }
    if (app.appReport.getOriginalTrackingUrl() == null) {
        app.appReport.setOriginalTrackingUrl(UNAVAILABLE);
    }
    if (app.appReport.getDiagnostics() == null) {
        app.appReport.setDiagnostics("");
    }
    return app;
}

17 Source : ApplicationHistoryManagerOnTimelineStore.java
with Apache License 2.0
from naver

private static ContainerReport convertToContainerReport(TimelineEnreplacedy enreplacedy, String serverHttpAddress, String user) {
    int allocatedMem = 0;
    int allocatedVcore = 0;
    int allocatedGcore = 0;
    String allocatedHost = null;
    int allocatedPort = -1;
    int allocatedPriority = 0;
    long createdTime = 0;
    long finishedTime = 0;
    String diagnosticsInfo = null;
    int exitStatus = ContainerExitStatus.INVALID;
    ContainerState state = null;
    String nodeHttpAddress = null;
    Map<String, Object> enreplacedyInfo = enreplacedy.getOtherInfo();
    if (enreplacedyInfo != null) {
        if (enreplacedyInfo.containsKey(ContainerMetricsConstants.ALLOCATED_MEMORY_ENreplacedY_INFO)) {
            allocatedMem = (Integer) enreplacedyInfo.get(ContainerMetricsConstants.ALLOCATED_MEMORY_ENreplacedY_INFO);
        }
        if (enreplacedyInfo.containsKey(ContainerMetricsConstants.ALLOCATED_VCORE_ENreplacedY_INFO)) {
            allocatedVcore = (Integer) enreplacedyInfo.get(ContainerMetricsConstants.ALLOCATED_VCORE_ENreplacedY_INFO);
        }
        if (enreplacedyInfo.containsKey(ContainerMetricsConstants.ALLOCATED_GCORE_ENreplacedY_INFO)) {
            allocatedGcore = (Integer) enreplacedyInfo.get(ContainerMetricsConstants.ALLOCATED_GCORE_ENreplacedY_INFO);
        }
        if (enreplacedyInfo.containsKey(ContainerMetricsConstants.ALLOCATED_HOST_ENreplacedY_INFO)) {
            allocatedHost = enreplacedyInfo.get(ContainerMetricsConstants.ALLOCATED_HOST_ENreplacedY_INFO).toString();
        }
        if (enreplacedyInfo.containsKey(ContainerMetricsConstants.ALLOCATED_PORT_ENreplacedY_INFO)) {
            allocatedPort = (Integer) enreplacedyInfo.get(ContainerMetricsConstants.ALLOCATED_PORT_ENreplacedY_INFO);
        }
        if (enreplacedyInfo.containsKey(ContainerMetricsConstants.ALLOCATED_PRIORITY_ENreplacedY_INFO)) {
            allocatedPriority = (Integer) enreplacedyInfo.get(ContainerMetricsConstants.ALLOCATED_PRIORITY_ENreplacedY_INFO);
        }
        if (enreplacedyInfo.containsKey(ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_ENreplacedY_INFO)) {
            nodeHttpAddress = (String) enreplacedyInfo.get(ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_ENreplacedY_INFO);
        }
    }
    List<TimelineEvent> events = enreplacedy.getEvents();
    if (events != null) {
        for (TimelineEvent event : events) {
            if (event.getEventType().equals(ContainerMetricsConstants.CREATED_EVENT_TYPE)) {
                createdTime = event.getTimestamp();
            } else if (event.getEventType().equals(ContainerMetricsConstants.FINISHED_EVENT_TYPE)) {
                finishedTime = event.getTimestamp();
                Map<String, Object> eventInfo = event.getEventInfo();
                if (eventInfo == null) {
                    continue;
                }
                if (eventInfo.containsKey(ContainerMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO)) {
                    diagnosticsInfo = eventInfo.get(ContainerMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO).toString();
                }
                if (eventInfo.containsKey(ContainerMetricsConstants.EXIT_STATUS_EVENT_INFO)) {
                    exitStatus = (Integer) eventInfo.get(ContainerMetricsConstants.EXIT_STATUS_EVENT_INFO);
                }
                if (eventInfo.containsKey(ContainerMetricsConstants.STATE_EVENT_INFO)) {
                    state = ContainerState.valueOf(eventInfo.get(ContainerMetricsConstants.STATE_EVENT_INFO).toString());
                }
            }
        }
    }
    NodeId allocatedNode = NodeId.newInstance(allocatedHost, allocatedPort);
    ContainerId containerId = ConverterUtils.toContainerId(enreplacedy.getEnreplacedyId());
    String logUrl = WebAppUtils.getAggregatedLogURL(serverHttpAddress, allocatedNode.toString(), containerId.toString(), containerId.toString(), user);
    return ContainerReport.newInstance(ConverterUtils.toContainerId(enreplacedy.getEnreplacedyId()), Resource.newInstance(allocatedMem, allocatedVcore, allocatedGcore), NodeId.newInstance(allocatedHost, allocatedPort), Priority.newInstance(allocatedPriority), createdTime, finishedTime, diagnosticsInfo, logUrl, exitStatus, state, nodeHttpAddress);
}

16 Source : TestSystemMetricsPublisher.java
with Apache License 2.0
from NJUJYB

@Test(timeout = 10000)
public void testPublishContainerMetrics() throws Exception {
    ContainerId containerId = ContainerId.newContainerId(ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1), 1);
    RMContainer container = createRMContainer(containerId);
    metricsPublisher.containerCreated(container, container.getCreationTime());
    metricsPublisher.containerFinished(container, container.getFinishTime());
    TimelineEnreplacedy enreplacedy = null;
    do {
        enreplacedy = store.getEnreplacedy(containerId.toString(), ContainerMetricsConstants.ENreplacedY_TYPE, EnumSet.allOf(Field.clreplaced));
    // ensure two events are both published before leaving the loop
    } while (enreplacedy == null || enreplacedy.getEvents().size() < 2);
    // verify all the fields
    replacedert.replacedertEquals(ContainerMetricsConstants.ENreplacedY_TYPE, enreplacedy.getEnreplacedyType());
    replacedert.replacedertEquals(containerId.toString(), enreplacedy.getEnreplacedyId());
    replacedert.replacedertEquals(containerId.getApplicationAttemptId().toString(), enreplacedy.getPrimaryFilters().get(ContainerMetricsConstants.PARENT_PRIMARIY_FILTER).iterator().next());
    replacedert.replacedertEquals(container.getAllocatedNode().getHost(), enreplacedy.getOtherInfo().get(ContainerMetricsConstants.ALLOCATED_HOST_ENreplacedY_INFO));
    replacedert.replacedertEquals(container.getAllocatedNode().getPort(), enreplacedy.getOtherInfo().get(ContainerMetricsConstants.ALLOCATED_PORT_ENreplacedY_INFO));
    replacedert.replacedertEquals(container.getAllocatedResource().getMemory(), enreplacedy.getOtherInfo().get(ContainerMetricsConstants.ALLOCATED_MEMORY_ENreplacedY_INFO));
    replacedert.replacedertEquals(container.getAllocatedResource().getVirtualCores(), enreplacedy.getOtherInfo().get(ContainerMetricsConstants.ALLOCATED_VCORE_ENreplacedY_INFO));
    replacedert.replacedertEquals(container.getAllocatedPriority().getPriority(), enreplacedy.getOtherInfo().get(ContainerMetricsConstants.ALLOCATED_PRIORITY_ENreplacedY_INFO));
    boolean hasCreatedEvent = false;
    boolean hasFinishedEvent = false;
    for (TimelineEvent event : enreplacedy.getEvents()) {
        if (event.getEventType().equals(ContainerMetricsConstants.CREATED_EVENT_TYPE)) {
            hasCreatedEvent = true;
            replacedert.replacedertEquals(container.getCreationTime(), event.getTimestamp());
        } else if (event.getEventType().equals(ContainerMetricsConstants.FINISHED_EVENT_TYPE)) {
            hasFinishedEvent = true;
            replacedert.replacedertEquals(container.getFinishTime(), event.getTimestamp());
            replacedert.replacedertEquals(container.getDiagnosticsInfo(), event.getEventInfo().get(ContainerMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO));
            replacedert.replacedertEquals(container.getContainerExitStatus(), event.getEventInfo().get(ContainerMetricsConstants.EXIT_STATUS_EVENT_INFO));
            replacedert.replacedertEquals(container.getContainerState().toString(), event.getEventInfo().get(ContainerMetricsConstants.STATE_EVENT_INFO));
        }
    }
    replacedert.replacedertTrue(hasCreatedEvent && hasFinishedEvent);
}

16 Source : TimelineStoreTestUtils.java
with Apache License 2.0
from NJUJYB

/**
 * Create a test enreplacedy
 */
protected static TimelineEnreplacedy createEnreplacedy(String enreplacedyId, String enreplacedyType, Long startTime, List<TimelineEvent> events, Map<String, Set<String>> relatedEnreplacedies, Map<String, Set<Object>> primaryFilters, Map<String, Object> otherInfo, String domainId) {
    TimelineEnreplacedy enreplacedy = new TimelineEnreplacedy();
    enreplacedy.setEnreplacedyId(enreplacedyId);
    enreplacedy.setEnreplacedyType(enreplacedyType);
    enreplacedy.setStartTime(startTime);
    enreplacedy.setEvents(events);
    if (relatedEnreplacedies != null) {
        for (Entry<String, Set<String>> e : relatedEnreplacedies.entrySet()) {
            for (String v : e.getValue()) {
                enreplacedy.addRelatedEnreplacedy(e.getKey(), v);
            }
        }
    } else {
        enreplacedy.setRelatedEnreplacedies(null);
    }
    enreplacedy.setPrimaryFilters(primaryFilters);
    enreplacedy.setOtherInfo(otherInfo);
    enreplacedy.setDomainId(domainId);
    return enreplacedy;
}

16 Source : TimelineStoreTestUtils.java
with Apache License 2.0
from NJUJYB

/**
 * Verify a single enreplacedy
 */
protected static void verifyEnreplacedyInfo(String enreplacedyId, String enreplacedyType, List<TimelineEvent> events, Map<String, Set<String>> relatedEnreplacedies, Map<String, Set<Object>> primaryFilters, Map<String, Object> otherInfo, TimelineEnreplacedy retrievedEnreplacedyInfo, String domainId) {
    if (enreplacedyId == null) {
        replacedertNull(retrievedEnreplacedyInfo);
        return;
    }
    replacedertEquals(enreplacedyId, retrievedEnreplacedyInfo.getEnreplacedyId());
    replacedertEquals(enreplacedyType, retrievedEnreplacedyInfo.getEnreplacedyType());
    replacedertEquals(domainId, retrievedEnreplacedyInfo.getDomainId());
    if (events == null) {
        replacedertNull(retrievedEnreplacedyInfo.getEvents());
    } else {
        replacedertEquals(events, retrievedEnreplacedyInfo.getEvents());
    }
    if (relatedEnreplacedies == null) {
        replacedertNull(retrievedEnreplacedyInfo.getRelatedEnreplacedies());
    } else {
        replacedertEquals(relatedEnreplacedies, retrievedEnreplacedyInfo.getRelatedEnreplacedies());
    }
    if (primaryFilters == null) {
        replacedertNull(retrievedEnreplacedyInfo.getPrimaryFilters());
    } else {
        replacedertTrue(primaryFilters.equals(retrievedEnreplacedyInfo.getPrimaryFilters()));
    }
    if (otherInfo == null) {
        replacedertNull(retrievedEnreplacedyInfo.getOtherInfo());
    } else {
        replacedertTrue(otherInfo.equals(retrievedEnreplacedyInfo.getOtherInfo()));
    }
}

16 Source : TestTimelineDataManager.java
with Apache License 2.0
from NJUJYB

@Test
public void testGetOldEnreplacedyWithOutDomainId() throws Exception {
    TimelineEnreplacedy enreplacedy = dataManaer.getEnreplacedy("OLD_ENreplacedY_TYPE_1", "OLD_ENreplacedY_ID_1", null, UserGroupInformation.getCurrentUser());
    replacedert.replacedertNotNull(enreplacedy);
    replacedert.replacedertEquals("OLD_ENreplacedY_ID_1", enreplacedy.getEnreplacedyId());
    replacedert.replacedertEquals("OLD_ENreplacedY_TYPE_1", enreplacedy.getEnreplacedyType());
    replacedert.replacedertEquals(TimelineDataManager.DEFAULT_DOMAIN_ID, enreplacedy.getDomainId());
}

16 Source : TestApplicationHistoryManagerOnTimelineStore.java
with Apache License 2.0
from NJUJYB

private static TimelineEnreplacedy createAppAttemptTimelineEnreplacedy(ApplicationAttemptId appAttemptId) {
    TimelineEnreplacedy enreplacedy = new TimelineEnreplacedy();
    enreplacedy.setEnreplacedyType(AppAttemptMetricsConstants.ENreplacedY_TYPE);
    enreplacedy.setEnreplacedyId(appAttemptId.toString());
    enreplacedy.setDomainId(TimelineDataManager.DEFAULT_DOMAIN_ID);
    enreplacedy.addPrimaryFilter(AppAttemptMetricsConstants.PARENT_PRIMARY_FILTER, appAttemptId.getApplicationId().toString());
    enreplacedy.addPrimaryFilter(TimelineStore.SystemFilter.ENreplacedY_OWNER.toString(), "yarn");
    TimelineEvent tEvent = new TimelineEvent();
    tEvent.setEventType(AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE);
    tEvent.setTimestamp(Integer.MAX_VALUE + 1L);
    Map<String, Object> eventInfo = new HashMap<String, Object>();
    eventInfo.put(AppAttemptMetricsConstants.TRACKING_URL_EVENT_INFO, "test tracking url");
    eventInfo.put(AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_EVENT_INFO, "test original tracking url");
    eventInfo.put(AppAttemptMetricsConstants.HOST_EVENT_INFO, "test host");
    eventInfo.put(AppAttemptMetricsConstants.RPC_PORT_EVENT_INFO, 100);
    eventInfo.put(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO, ContainerId.newContainerId(appAttemptId, 1));
    tEvent.setEventInfo(eventInfo);
    enreplacedy.addEvent(tEvent);
    tEvent = new TimelineEvent();
    tEvent.setEventType(AppAttemptMetricsConstants.FINISHED_EVENT_TYPE);
    tEvent.setTimestamp(Integer.MAX_VALUE + 2L);
    eventInfo = new HashMap<String, Object>();
    eventInfo.put(AppAttemptMetricsConstants.TRACKING_URL_EVENT_INFO, "test tracking url");
    eventInfo.put(AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_EVENT_INFO, "test original tracking url");
    eventInfo.put(AppAttemptMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO, "test diagnostics info");
    eventInfo.put(AppAttemptMetricsConstants.FINAL_STATUS_EVENT_INFO, FinalApplicationStatus.UNDEFINED.toString());
    eventInfo.put(AppAttemptMetricsConstants.STATE_EVENT_INFO, YarnApplicationAttemptState.FINISHED.toString());
    tEvent.setEventInfo(eventInfo);
    enreplacedy.addEvent(tEvent);
    return enreplacedy;
}

16 Source : TimelineWebServices.java
with Apache License 2.0
from NJUJYB

/**
 * Return a single enreplacedy of the given enreplacedy type and Id.
 */
@GET
@Path("/{enreplacedyType}/{enreplacedyId}")
@Produces({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */
})
public TimelineEnreplacedy getEnreplacedy(@Context HttpServletRequest req, @Context HttpServletResponse res, @PathParam("enreplacedyType") String enreplacedyType, @PathParam("enreplacedyId") String enreplacedyId, @QueryParam("fields") String fields) {
    init(res);
    TimelineEnreplacedy enreplacedy = null;
    try {
        enreplacedy = timelineDataManager.getEnreplacedy(parseStr(enreplacedyType), parseStr(enreplacedyId), parseFieldsStr(fields, ","), getUser(req));
    } catch (IllegalArgumentException e) {
        throw new BadRequestException("requested invalid field.");
    } catch (Exception e) {
        LOG.error("Error getting enreplacedy", e);
        throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR);
    }
    if (enreplacedy == null) {
        throw new NotFoundException("Timeline enreplacedy " + new EnreplacedyIdentifier(parseStr(enreplacedyId), parseStr(enreplacedyType)) + " is not found");
    }
    return enreplacedy;
}

16 Source : TimelineDataManager.java
with Apache License 2.0
from NJUJYB

/**
 * Get the events whose enreplacedies the given user has access to. The meaning of
 * each argument has been doreplacedented with
 * {@link TimelineReader#getEnreplacedyTimelines}.
 *
 * @see TimelineReader#getEnreplacedyTimelines
 */
public TimelineEvents getEvents(String enreplacedyType, SortedSet<String> enreplacedyIds, SortedSet<String> eventTypes, Long windowStart, Long windowEnd, Long limit, UserGroupInformation callerUGI) throws YarnException, IOException {
    TimelineEvents events = null;
    events = store.getEnreplacedyTimelines(enreplacedyType, enreplacedyIds, limit, windowStart, windowEnd, eventTypes);
    if (events != null) {
        Iterator<TimelineEvents.EventsOfOneEnreplacedy> eventsItr = events.getAllEvents().iterator();
        while (eventsItr.hasNext()) {
            TimelineEvents.EventsOfOneEnreplacedy eventsOfOneEnreplacedy = eventsItr.next();
            try {
                TimelineEnreplacedy enreplacedy = store.getEnreplacedy(eventsOfOneEnreplacedy.getEnreplacedyId(), eventsOfOneEnreplacedy.getEnreplacedyType(), EnumSet.of(Field.PRIMARY_FILTERS));
                addDefaultDomainIdIfAbsent(enreplacedy);
                // check ACLs
                if (!timelineACLsManager.checkAccess(callerUGI, ApplicationAccessType.VIEW_APP, enreplacedy)) {
                    eventsItr.remove();
                }
            } catch (Exception e) {
                LOG.error("Error when verifying access for user " + callerUGI + " on the events of the timeline enreplacedy " + new EnreplacedyIdentifier(eventsOfOneEnreplacedy.getEnreplacedyId(), eventsOfOneEnreplacedy.getEnreplacedyType()), e);
                eventsItr.remove();
            }
        }
    }
    if (events == null) {
        return new TimelineEvents();
    }
    return events;
}

16 Source : MemoryTimelineStore.java
with Apache License 2.0
from NJUJYB

@Override
public synchronized TimelineEnreplacedies getEnreplacedies(String enreplacedyType, Long limit, Long windowStart, Long windowEnd, String fromId, Long fromTs, NameValuePair primaryFilter, Collection<NameValuePair> secondaryFilters, EnumSet<Field> fields, CheckAcl checkAcl) throws IOException {
    if (limit == null) {
        limit = DEFAULT_LIMIT;
    }
    if (windowStart == null) {
        windowStart = Long.MIN_VALUE;
    }
    if (windowEnd == null) {
        windowEnd = Long.MAX_VALUE;
    }
    if (fields == null) {
        fields = EnumSet.allOf(Field.clreplaced);
    }
    Iterator<TimelineEnreplacedy> enreplacedyIterator = null;
    if (fromId != null) {
        TimelineEnreplacedy firstEnreplacedy = enreplacedies.get(new EnreplacedyIdentifier(fromId, enreplacedyType));
        if (firstEnreplacedy == null) {
            return new TimelineEnreplacedies();
        } else {
            enreplacedyIterator = new TreeSet<TimelineEnreplacedy>(enreplacedies.values()).tailSet(firstEnreplacedy, true).iterator();
        }
    }
    if (enreplacedyIterator == null) {
        enreplacedyIterator = new PriorityQueue<TimelineEnreplacedy>(enreplacedies.values()).iterator();
    }
    List<TimelineEnreplacedy> enreplacediesSelected = new ArrayList<TimelineEnreplacedy>();
    while (enreplacedyIterator.hasNext()) {
        TimelineEnreplacedy enreplacedy = enreplacedyIterator.next();
        if (enreplacediesSelected.size() >= limit) {
            break;
        }
        if (!enreplacedy.getEnreplacedyType().equals(enreplacedyType)) {
            continue;
        }
        if (enreplacedy.getStartTime() <= windowStart) {
            continue;
        }
        if (enreplacedy.getStartTime() > windowEnd) {
            continue;
        }
        if (fromTs != null && enreplacedyInsertTimes.get(new EnreplacedyIdentifier(enreplacedy.getEnreplacedyId(), enreplacedy.getEnreplacedyType())) > fromTs) {
            continue;
        }
        if (primaryFilter != null && !matchPrimaryFilter(enreplacedy.getPrimaryFilters(), primaryFilter)) {
            continue;
        }
        if (secondaryFilters != null) {
            // AND logic
            boolean flag = true;
            for (NameValuePair secondaryFilter : secondaryFilters) {
                if (secondaryFilter != null && !matchPrimaryFilter(enreplacedy.getPrimaryFilters(), secondaryFilter) && !matchFilter(enreplacedy.getOtherInfo(), secondaryFilter)) {
                    flag = false;
                    break;
                }
            }
            if (!flag) {
                continue;
            }
        }
        if (enreplacedy.getDomainId() == null) {
            enreplacedy.setDomainId(DEFAULT_DOMAIN_ID);
        }
        if (checkAcl == null || checkAcl.check(enreplacedy)) {
            enreplacediesSelected.add(enreplacedy);
        }
    }
    List<TimelineEnreplacedy> enreplacediesToReturn = new ArrayList<TimelineEnreplacedy>();
    for (TimelineEnreplacedy enreplacedySelected : enreplacediesSelected) {
        enreplacediesToReturn.add(maskFields(enreplacedySelected, fields));
    }
    Collections.sort(enreplacediesToReturn);
    TimelineEnreplacedies enreplacediesWrapper = new TimelineEnreplacedies();
    enreplacediesWrapper.setEnreplacedies(enreplacediesToReturn);
    return enreplacediesWrapper;
}

16 Source : ApplicationHistoryManagerOnTimelineStore.java
with Apache License 2.0
from NJUJYB

@Override
public Map<ApplicationId, ApplicationReport> getApplications(long appsNum) throws YarnException, IOException {
    TimelineEnreplacedies enreplacedies = timelineDataManager.getEnreplacedies(ApplicationMetricsConstants.ENreplacedY_TYPE, null, null, null, null, null, null, appsNum == Long.MAX_VALUE ? this.maxLoadedApplications : appsNum, EnumSet.allOf(Field.clreplaced), UserGroupInformation.getLoginUser());
    Map<ApplicationId, ApplicationReport> apps = new LinkedHashMap<ApplicationId, ApplicationReport>();
    if (enreplacedies != null && enreplacedies.getEnreplacedies() != null) {
        for (TimelineEnreplacedy enreplacedy : enreplacedies.getEnreplacedies()) {
            ApplicationReportExt app = generateApplicationReport(enreplacedy, ApplicationReportField.ALL);
            apps.put(app.appReport.getApplicationId(), app.appReport);
        }
    }
    return apps;
}

16 Source : ApplicationHistoryManagerOnTimelineStore.java
with Apache License 2.0
from NJUJYB

private ApplicationReportExt generateApplicationReport(TimelineEnreplacedy enreplacedy, ApplicationReportField field) throws YarnException, IOException {
    ApplicationReportExt app = convertToApplicationReport(enreplacedy, field);
    // If only user and acls are pulled to check attempt(s)/container(s) access
    // control, we can return immediately
    if (field == ApplicationReportField.USER_AND_ACLS) {
        return app;
    }
    try {
        checkAccess(app);
        if (app.appReport.getCurrentApplicationAttemptId() != null) {
            ApplicationAttemptReport appAttempt = getApplicationAttempt(app.appReport.getCurrentApplicationAttemptId());
            app.appReport.setHost(appAttempt.getHost());
            app.appReport.setRpcPort(appAttempt.getRpcPort());
            app.appReport.setTrackingUrl(appAttempt.getTrackingUrl());
            app.appReport.setOriginalTrackingUrl(appAttempt.getOriginalTrackingUrl());
        }
    } catch (AuthorizationException e) {
        // AuthorizationException is thrown because the user doesn't have access
        app.appReport.setDiagnostics(null);
        app.appReport.setCurrentApplicationAttemptId(null);
    } catch (ApplicationAttemptNotFoundException e) {
        // It's possible that the app is finished before the first attempt is created.
        app.appReport.setDiagnostics(null);
        app.appReport.setCurrentApplicationAttemptId(null);
    }
    if (app.appReport.getCurrentApplicationAttemptId() == null) {
        app.appReport.setCurrentApplicationAttemptId(ApplicationAttemptId.newInstance(app.appReport.getApplicationId(), -1));
    }
    if (app.appReport.getHost() == null) {
        app.appReport.setHost(UNAVAILABLE);
    }
    if (app.appReport.getRpcPort() < 0) {
        app.appReport.setRpcPort(-1);
    }
    if (app.appReport.getTrackingUrl() == null) {
        app.appReport.setTrackingUrl(UNAVAILABLE);
    }
    if (app.appReport.getOriginalTrackingUrl() == null) {
        app.appReport.setOriginalTrackingUrl(UNAVAILABLE);
    }
    if (app.appReport.getDiagnostics() == null) {
        app.appReport.setDiagnostics("");
    }
    return app;
}

16 Source : ApplicationHistoryManagerOnTimelineStore.java
with Apache License 2.0
from NJUJYB

private ApplicationReportExt getApplication(ApplicationId appId, ApplicationReportField field) throws YarnException, IOException {
    TimelineEnreplacedy enreplacedy = timelineDataManager.getEnreplacedy(ApplicationMetricsConstants.ENreplacedY_TYPE, appId.toString(), EnumSet.allOf(Field.clreplaced), UserGroupInformation.getLoginUser());
    if (enreplacedy == null) {
        throw new ApplicationNotFoundException("The enreplacedy for application " + appId + " doesn't exist in the timeline store");
    } else {
        return generateApplicationReport(enreplacedy, field);
    }
}

16 Source : TestSystemMetricsPublisher.java
with Apache License 2.0
from naver

@Test(timeout = 10000)
public void testPublishApplicationMetrics() throws Exception {
    for (int i = 1; i <= 2; ++i) {
        ApplicationId appId = ApplicationId.newInstance(0, i);
        RMApp app = createRMApp(appId);
        metricsPublisher.appCreated(app, app.getStartTime());
        metricsPublisher.appFinished(app, RMAppState.FINISHED, app.getFinishTime());
        if (i == 1) {
            metricsPublisher.appACLsUpdated(app, "uers1,user2", 4L);
        } else {
            // in case user doesn't specify the ACLs
            metricsPublisher.appACLsUpdated(app, null, 4L);
        }
        TimelineEnreplacedy enreplacedy = null;
        do {
            enreplacedy = store.getEnreplacedy(appId.toString(), ApplicationMetricsConstants.ENreplacedY_TYPE, EnumSet.allOf(Field.clreplaced));
        // ensure three events are both published before leaving the loop
        } while (enreplacedy == null || enreplacedy.getEvents().size() < 3);
        // verify all the fields
        replacedert.replacedertEquals(ApplicationMetricsConstants.ENreplacedY_TYPE, enreplacedy.getEnreplacedyType());
        replacedert.replacedertEquals(app.getApplicationId().toString(), enreplacedy.getEnreplacedyId());
        replacedert.replacedertEquals(app.getName(), enreplacedy.getOtherInfo().get(ApplicationMetricsConstants.NAME_ENreplacedY_INFO));
        replacedert.replacedertEquals(app.getQueue(), enreplacedy.getOtherInfo().get(ApplicationMetricsConstants.QUEUE_ENreplacedY_INFO));
        replacedert.replacedertEquals(app.getUser(), enreplacedy.getOtherInfo().get(ApplicationMetricsConstants.USER_ENreplacedY_INFO));
        replacedert.replacedertEquals(app.getApplicationType(), enreplacedy.getOtherInfo().get(ApplicationMetricsConstants.TYPE_ENreplacedY_INFO));
        replacedert.replacedertEquals(app.getSubmitTime(), enreplacedy.getOtherInfo().get(ApplicationMetricsConstants.SUBMITTED_TIME_ENreplacedY_INFO));
        if (i == 1) {
            replacedert.replacedertEquals("uers1,user2", enreplacedy.getOtherInfo().get(ApplicationMetricsConstants.APP_VIEW_ACLS_ENreplacedY_INFO));
        } else {
            replacedert.replacedertEquals("", enreplacedy.getOtherInfo().get(ApplicationMetricsConstants.APP_VIEW_ACLS_ENreplacedY_INFO));
            replacedert.replacedertEquals(app.getRMAppMetrics().getMemorySeconds(), Long.parseLong(enreplacedy.getOtherInfo().get(ApplicationMetricsConstants.APP_MEM_METRICS).toString()));
            replacedert.replacedertEquals(app.getRMAppMetrics().getVcoreSeconds(), Long.parseLong(enreplacedy.getOtherInfo().get(ApplicationMetricsConstants.APP_CPU_METRICS).toString()));
            replacedert.replacedertEquals(app.getRMAppMetrics().getGcoreSeconds(), Long.parseLong(enreplacedy.getOtherInfo().get(ApplicationMetricsConstants.APP_GPU_METRICS).toString()));
        }
        boolean hasCreatedEvent = false;
        boolean hasFinishedEvent = false;
        boolean hasACLsUpdatedEvent = false;
        for (TimelineEvent event : enreplacedy.getEvents()) {
            if (event.getEventType().equals(ApplicationMetricsConstants.CREATED_EVENT_TYPE)) {
                hasCreatedEvent = true;
                replacedert.replacedertEquals(app.getStartTime(), event.getTimestamp());
            } else if (event.getEventType().equals(ApplicationMetricsConstants.FINISHED_EVENT_TYPE)) {
                hasFinishedEvent = true;
                replacedert.replacedertEquals(app.getFinishTime(), event.getTimestamp());
                replacedert.replacedertEquals(app.getDiagnostics().toString(), event.getEventInfo().get(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO));
                replacedert.replacedertEquals(app.getFinalApplicationStatus().toString(), event.getEventInfo().get(ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO));
                replacedert.replacedertEquals(YarnApplicationState.FINISHED.toString(), event.getEventInfo().get(ApplicationMetricsConstants.STATE_EVENT_INFO));
            } else if (event.getEventType().equals(ApplicationMetricsConstants.ACLS_UPDATED_EVENT_TYPE)) {
                hasACLsUpdatedEvent = true;
                replacedert.replacedertEquals(4L, event.getTimestamp());
            }
        }
        replacedert.replacedertTrue(hasCreatedEvent && hasFinishedEvent && hasACLsUpdatedEvent);
    }
}

16 Source : TestSystemMetricsPublisher.java
with Apache License 2.0
from naver

@Test(timeout = 10000)
public void testPublishContainerMetrics() throws Exception {
    ContainerId containerId = ContainerId.newContainerId(ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1), 1);
    RMContainer container = createRMContainer(containerId);
    metricsPublisher.containerCreated(container, container.getCreationTime());
    metricsPublisher.containerFinished(container, container.getFinishTime());
    TimelineEnreplacedy enreplacedy = null;
    do {
        enreplacedy = store.getEnreplacedy(containerId.toString(), ContainerMetricsConstants.ENreplacedY_TYPE, EnumSet.allOf(Field.clreplaced));
    // ensure two events are both published before leaving the loop
    } while (enreplacedy == null || enreplacedy.getEvents().size() < 2);
    // verify all the fields
    replacedert.replacedertEquals(ContainerMetricsConstants.ENreplacedY_TYPE, enreplacedy.getEnreplacedyType());
    replacedert.replacedertEquals(containerId.toString(), enreplacedy.getEnreplacedyId());
    replacedert.replacedertEquals(containerId.getApplicationAttemptId().toString(), enreplacedy.getPrimaryFilters().get(ContainerMetricsConstants.PARENT_PRIMARIY_FILTER).iterator().next());
    replacedert.replacedertEquals(container.getAllocatedNode().getHost(), enreplacedy.getOtherInfo().get(ContainerMetricsConstants.ALLOCATED_HOST_ENreplacedY_INFO));
    replacedert.replacedertEquals(container.getAllocatedNode().getPort(), enreplacedy.getOtherInfo().get(ContainerMetricsConstants.ALLOCATED_PORT_ENreplacedY_INFO));
    replacedert.replacedertEquals(container.getAllocatedResource().getMemory(), enreplacedy.getOtherInfo().get(ContainerMetricsConstants.ALLOCATED_MEMORY_ENreplacedY_INFO));
    replacedert.replacedertEquals(container.getAllocatedResource().getVirtualCores(), enreplacedy.getOtherInfo().get(ContainerMetricsConstants.ALLOCATED_VCORE_ENreplacedY_INFO));
    replacedert.replacedertEquals(container.getAllocatedResource().getGpuCores(), enreplacedy.getOtherInfo().get(ContainerMetricsConstants.ALLOCATED_GCORE_ENreplacedY_INFO));
    replacedert.replacedertEquals(container.getAllocatedPriority().getPriority(), enreplacedy.getOtherInfo().get(ContainerMetricsConstants.ALLOCATED_PRIORITY_ENreplacedY_INFO));
    boolean hasCreatedEvent = false;
    boolean hasFinishedEvent = false;
    for (TimelineEvent event : enreplacedy.getEvents()) {
        if (event.getEventType().equals(ContainerMetricsConstants.CREATED_EVENT_TYPE)) {
            hasCreatedEvent = true;
            replacedert.replacedertEquals(container.getCreationTime(), event.getTimestamp());
        } else if (event.getEventType().equals(ContainerMetricsConstants.FINISHED_EVENT_TYPE)) {
            hasFinishedEvent = true;
            replacedert.replacedertEquals(container.getFinishTime(), event.getTimestamp());
            replacedert.replacedertEquals(container.getDiagnosticsInfo(), event.getEventInfo().get(ContainerMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO));
            replacedert.replacedertEquals(container.getContainerExitStatus(), event.getEventInfo().get(ContainerMetricsConstants.EXIT_STATUS_EVENT_INFO));
            replacedert.replacedertEquals(container.getContainerState().toString(), event.getEventInfo().get(ContainerMetricsConstants.STATE_EVENT_INFO));
        }
    }
    replacedert.replacedertTrue(hasCreatedEvent && hasFinishedEvent);
}

16 Source : ApplicationHistoryManagerOnTimelineStore.java
with Apache License 2.0
from naver

private static ApplicationReportExt convertToApplicationReport(TimelineEnreplacedy enreplacedy, ApplicationReportField field) {
    String user = null;
    String queue = null;
    String name = null;
    String type = null;
    long createdTime = 0;
    long finishedTime = 0;
    ApplicationAttemptId latestApplicationAttemptId = null;
    String diagnosticsInfo = null;
    FinalApplicationStatus finalStatus = FinalApplicationStatus.UNDEFINED;
    YarnApplicationState state = null;
    ApplicationResourceUsageReport appResources = null;
    Map<ApplicationAccessType, String> appViewACLs = new HashMap<ApplicationAccessType, String>();
    Map<String, Object> enreplacedyInfo = enreplacedy.getOtherInfo();
    if (enreplacedyInfo != null) {
        if (enreplacedyInfo.containsKey(ApplicationMetricsConstants.USER_ENreplacedY_INFO)) {
            user = enreplacedyInfo.get(ApplicationMetricsConstants.USER_ENreplacedY_INFO).toString();
        }
        if (enreplacedyInfo.containsKey(ApplicationMetricsConstants.APP_VIEW_ACLS_ENreplacedY_INFO)) {
            String appViewACLsStr = enreplacedyInfo.get(ApplicationMetricsConstants.APP_VIEW_ACLS_ENreplacedY_INFO).toString();
            if (appViewACLsStr.length() > 0) {
                appViewACLs.put(ApplicationAccessType.VIEW_APP, appViewACLsStr);
            }
        }
        if (field == ApplicationReportField.USER_AND_ACLS) {
            return new ApplicationReportExt(ApplicationReport.newInstance(ConverterUtils.toApplicationId(enreplacedy.getEnreplacedyId()), latestApplicationAttemptId, user, queue, name, null, -1, null, state, diagnosticsInfo, null, createdTime, finishedTime, finalStatus, null, null, 1.0F, type, null), appViewACLs);
        }
        if (enreplacedyInfo.containsKey(ApplicationMetricsConstants.QUEUE_ENreplacedY_INFO)) {
            queue = enreplacedyInfo.get(ApplicationMetricsConstants.QUEUE_ENreplacedY_INFO).toString();
        }
        if (enreplacedyInfo.containsKey(ApplicationMetricsConstants.NAME_ENreplacedY_INFO)) {
            name = enreplacedyInfo.get(ApplicationMetricsConstants.NAME_ENreplacedY_INFO).toString();
        }
        if (enreplacedyInfo.containsKey(ApplicationMetricsConstants.TYPE_ENreplacedY_INFO)) {
            type = enreplacedyInfo.get(ApplicationMetricsConstants.TYPE_ENreplacedY_INFO).toString();
        }
        if (enreplacedyInfo.containsKey(ApplicationMetricsConstants.APP_CPU_METRICS)) {
            long vcoreSeconds = Long.parseLong(enreplacedyInfo.get(ApplicationMetricsConstants.APP_CPU_METRICS).toString());
            long memorySeconds = Long.parseLong(enreplacedyInfo.get(ApplicationMetricsConstants.APP_MEM_METRICS).toString());
            long gcoreSeconds = Long.parseLong(enreplacedyInfo.get(ApplicationMetricsConstants.APP_GPU_METRICS).toString());
            appResources = ApplicationResourceUsageReport.newInstance(0, 0, null, null, null, memorySeconds, vcoreSeconds, gcoreSeconds);
        }
    }
    List<TimelineEvent> events = enreplacedy.getEvents();
    if (events != null) {
        for (TimelineEvent event : events) {
            if (event.getEventType().equals(ApplicationMetricsConstants.CREATED_EVENT_TYPE)) {
                createdTime = event.getTimestamp();
            } else if (event.getEventType().equals(ApplicationMetricsConstants.FINISHED_EVENT_TYPE)) {
                finishedTime = event.getTimestamp();
                Map<String, Object> eventInfo = event.getEventInfo();
                if (eventInfo == null) {
                    continue;
                }
                if (eventInfo.containsKey(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO)) {
                    latestApplicationAttemptId = ConverterUtils.toApplicationAttemptId(eventInfo.get(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO).toString());
                }
                if (eventInfo.containsKey(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO)) {
                    diagnosticsInfo = eventInfo.get(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO).toString();
                }
                if (eventInfo.containsKey(ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO)) {
                    finalStatus = FinalApplicationStatus.valueOf(eventInfo.get(ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO).toString());
                }
                if (eventInfo.containsKey(ApplicationMetricsConstants.STATE_EVENT_INFO)) {
                    state = YarnApplicationState.valueOf(eventInfo.get(ApplicationMetricsConstants.STATE_EVENT_INFO).toString());
                }
            }
        }
    }
    return new ApplicationReportExt(ApplicationReport.newInstance(ConverterUtils.toApplicationId(enreplacedy.getEnreplacedyId()), latestApplicationAttemptId, user, queue, name, null, -1, null, state, diagnosticsInfo, null, createdTime, finishedTime, finalStatus, appResources, null, 1.0F, type, null), appViewACLs);
}

15 Source : SystemMetricsPublisher.java
with Apache License 2.0
from NJUJYB

private void publishAppAttemptRegisteredEvent(AppAttemptRegisteredEvent event) {
    TimelineEnreplacedy enreplacedy = createAppAttemptEnreplacedy(event.getApplicationAttemptId());
    TimelineEvent tEvent = new TimelineEvent();
    tEvent.setEventType(AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE);
    tEvent.setTimestamp(event.getTimestamp());
    Map<String, Object> eventInfo = new HashMap<String, Object>();
    eventInfo.put(AppAttemptMetricsConstants.TRACKING_URL_EVENT_INFO, event.getTrackingUrl());
    eventInfo.put(AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_EVENT_INFO, event.getOriginalTrackingURL());
    eventInfo.put(AppAttemptMetricsConstants.HOST_EVENT_INFO, event.getHost());
    eventInfo.put(AppAttemptMetricsConstants.RPC_PORT_EVENT_INFO, event.getRpcPort());
    eventInfo.put(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO, event.getMasterContainerId().toString());
    tEvent.setEventInfo(eventInfo);
    enreplacedy.addEvent(tEvent);
    putEnreplacedy(enreplacedy);
}

15 Source : SystemMetricsPublisher.java
with Apache License 2.0
from NJUJYB

private void publishContainerCreatedEvent(ContainerCreatedEvent event) {
    TimelineEnreplacedy enreplacedy = createContainerEnreplacedy(event.getContainerId());
    Map<String, Object> enreplacedyInfo = new HashMap<String, Object>();
    enreplacedyInfo.put(ContainerMetricsConstants.ALLOCATED_MEMORY_ENreplacedY_INFO, event.getAllocatedResource().getMemory());
    enreplacedyInfo.put(ContainerMetricsConstants.ALLOCATED_VCORE_ENreplacedY_INFO, event.getAllocatedResource().getVirtualCores());
    enreplacedyInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_ENreplacedY_INFO, event.getAllocatedNode().getHost());
    enreplacedyInfo.put(ContainerMetricsConstants.ALLOCATED_PORT_ENreplacedY_INFO, event.getAllocatedNode().getPort());
    enreplacedyInfo.put(ContainerMetricsConstants.ALLOCATED_PRIORITY_ENreplacedY_INFO, event.getAllocatedPriority().getPriority());
    enreplacedyInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_ENreplacedY_INFO, event.getNodeHttpAddress());
    enreplacedy.setOtherInfo(enreplacedyInfo);
    TimelineEvent tEvent = new TimelineEvent();
    tEvent.setEventType(ContainerMetricsConstants.CREATED_EVENT_TYPE);
    tEvent.setTimestamp(event.getTimestamp());
    enreplacedy.addEvent(tEvent);
    putEnreplacedy(enreplacedy);
}

15 Source : SystemMetricsPublisher.java
with Apache License 2.0
from NJUJYB

private void publishApplicationCreatedEvent(ApplicationCreatedEvent event) {
    TimelineEnreplacedy enreplacedy = createApplicationEnreplacedy(event.getApplicationId());
    Map<String, Object> enreplacedyInfo = new HashMap<String, Object>();
    enreplacedyInfo.put(ApplicationMetricsConstants.NAME_ENreplacedY_INFO, event.getApplicationName());
    enreplacedyInfo.put(ApplicationMetricsConstants.TYPE_ENreplacedY_INFO, event.getApplicationType());
    enreplacedyInfo.put(ApplicationMetricsConstants.USER_ENreplacedY_INFO, event.getUser());
    enreplacedyInfo.put(ApplicationMetricsConstants.QUEUE_ENreplacedY_INFO, event.getQueue());
    enreplacedyInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENreplacedY_INFO, event.getSubmittedTime());
    enreplacedy.setOtherInfo(enreplacedyInfo);
    TimelineEvent tEvent = new TimelineEvent();
    tEvent.setEventType(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
    tEvent.setTimestamp(event.getTimestamp());
    enreplacedy.addEvent(tEvent);
    putEnreplacedy(enreplacedy);
}

15 Source : TestLeveldbTimelineStore.java
with Apache License 2.0
from NJUJYB

@Test
public void testRelatingToNonExistingEnreplacedy() throws IOException {
    TimelineEnreplacedy enreplacedyToStore = new TimelineEnreplacedy();
    enreplacedyToStore.setEnreplacedyType("TEST_ENreplacedY_TYPE_1");
    enreplacedyToStore.setEnreplacedyId("TEST_ENreplacedY_ID_1");
    enreplacedyToStore.setDomainId(TimelineDataManager.DEFAULT_DOMAIN_ID);
    enreplacedyToStore.addRelatedEnreplacedy("TEST_ENreplacedY_TYPE_2", "TEST_ENreplacedY_ID_2");
    TimelineEnreplacedies enreplacedies = new TimelineEnreplacedies();
    enreplacedies.addEnreplacedy(enreplacedyToStore);
    store.put(enreplacedies);
    TimelineEnreplacedy enreplacedyToGet = store.getEnreplacedy("TEST_ENreplacedY_ID_2", "TEST_ENreplacedY_TYPE_2", null);
    replacedert.replacedertNotNull(enreplacedyToGet);
    replacedert.replacedertEquals("DEFAULT", enreplacedyToGet.getDomainId());
    replacedert.replacedertEquals("TEST_ENreplacedY_TYPE_1", enreplacedyToGet.getRelatedEnreplacedies().keySet().iterator().next());
    replacedert.replacedertEquals("TEST_ENreplacedY_ID_1", enreplacedyToGet.getRelatedEnreplacedies().values().iterator().next().iterator().next());
}

15 Source : TestApplicationHistoryManagerOnTimelineStore.java
with Apache License 2.0
from NJUJYB

private static TimelineEnreplacedy createApplicationTimelineEnreplacedy(ApplicationId appId, boolean emptyACLs, boolean noAttempt) {
    TimelineEnreplacedy enreplacedy = new TimelineEnreplacedy();
    enreplacedy.setEnreplacedyType(ApplicationMetricsConstants.ENreplacedY_TYPE);
    enreplacedy.setEnreplacedyId(appId.toString());
    enreplacedy.setDomainId(TimelineDataManager.DEFAULT_DOMAIN_ID);
    enreplacedy.addPrimaryFilter(TimelineStore.SystemFilter.ENreplacedY_OWNER.toString(), "yarn");
    Map<String, Object> enreplacedyInfo = new HashMap<String, Object>();
    enreplacedyInfo.put(ApplicationMetricsConstants.NAME_ENreplacedY_INFO, "test app");
    enreplacedyInfo.put(ApplicationMetricsConstants.TYPE_ENreplacedY_INFO, "test app type");
    enreplacedyInfo.put(ApplicationMetricsConstants.USER_ENreplacedY_INFO, "user1");
    enreplacedyInfo.put(ApplicationMetricsConstants.QUEUE_ENreplacedY_INFO, "test queue");
    enreplacedyInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENreplacedY_INFO, Integer.MAX_VALUE + 1L);
    if (emptyACLs) {
        enreplacedyInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENreplacedY_INFO, "");
    } else {
        enreplacedyInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENreplacedY_INFO, "user2");
    }
    enreplacedy.setOtherInfo(enreplacedyInfo);
    TimelineEvent tEvent = new TimelineEvent();
    tEvent.setEventType(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
    tEvent.setTimestamp(Integer.MAX_VALUE + 2L + appId.getId());
    enreplacedy.addEvent(tEvent);
    tEvent = new TimelineEvent();
    tEvent.setEventType(ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
    tEvent.setTimestamp(Integer.MAX_VALUE + 3L + appId.getId());
    Map<String, Object> eventInfo = new HashMap<String, Object>();
    eventInfo.put(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO, "test diagnostics info");
    eventInfo.put(ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO, FinalApplicationStatus.UNDEFINED.toString());
    eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO, YarnApplicationState.FINISHED.toString());
    if (noAttempt) {
        eventInfo.put(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO, ApplicationAttemptId.newInstance(appId, 1));
    }
    tEvent.setEventInfo(eventInfo);
    enreplacedy.addEvent(tEvent);
    return enreplacedy;
}

15 Source : TestApplicationHistoryManagerOnTimelineStore.java
with Apache License 2.0
from NJUJYB

private static TimelineEnreplacedy createContainerEnreplacedy(ContainerId containerId) {
    TimelineEnreplacedy enreplacedy = new TimelineEnreplacedy();
    enreplacedy.setEnreplacedyType(ContainerMetricsConstants.ENreplacedY_TYPE);
    enreplacedy.setEnreplacedyId(containerId.toString());
    enreplacedy.setDomainId(TimelineDataManager.DEFAULT_DOMAIN_ID);
    enreplacedy.addPrimaryFilter(ContainerMetricsConstants.PARENT_PRIMARIY_FILTER, containerId.getApplicationAttemptId().toString());
    enreplacedy.addPrimaryFilter(TimelineStore.SystemFilter.ENreplacedY_OWNER.toString(), "yarn");
    Map<String, Object> enreplacedyInfo = new HashMap<String, Object>();
    enreplacedyInfo.put(ContainerMetricsConstants.ALLOCATED_MEMORY_ENreplacedY_INFO, -1);
    enreplacedyInfo.put(ContainerMetricsConstants.ALLOCATED_VCORE_ENreplacedY_INFO, -1);
    enreplacedyInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_ENreplacedY_INFO, "test host");
    enreplacedyInfo.put(ContainerMetricsConstants.ALLOCATED_PORT_ENreplacedY_INFO, 100);
    enreplacedyInfo.put(ContainerMetricsConstants.ALLOCATED_PRIORITY_ENreplacedY_INFO, -1);
    enreplacedy.setOtherInfo(enreplacedyInfo);
    TimelineEvent tEvent = new TimelineEvent();
    tEvent.setEventType(ContainerMetricsConstants.CREATED_EVENT_TYPE);
    tEvent.setTimestamp(Integer.MAX_VALUE + 1L);
    enreplacedy.addEvent(tEvent);
    ;
    tEvent = new TimelineEvent();
    tEvent.setEventType(ContainerMetricsConstants.FINISHED_EVENT_TYPE);
    tEvent.setTimestamp(Integer.MAX_VALUE + 2L);
    Map<String, Object> eventInfo = new HashMap<String, Object>();
    eventInfo.put(ContainerMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO, "test diagnostics info");
    eventInfo.put(ContainerMetricsConstants.EXIT_STATUS_EVENT_INFO, -1);
    eventInfo.put(ContainerMetricsConstants.STATE_EVENT_INFO, ContainerState.COMPLETE.toString());
    tEvent.setEventInfo(eventInfo);
    enreplacedy.addEvent(tEvent);
    return enreplacedy;
}

15 Source : ApplicationMaster.java
with Apache License 2.0
from NJUJYB

private static void publishContainerStartEvent(final TimelineClient timelineClient, Container container, String domainId, UserGroupInformation ugi) {
    final TimelineEnreplacedy enreplacedy = new TimelineEnreplacedy();
    enreplacedy.setEnreplacedyId(container.getId().toString());
    enreplacedy.setEnreplacedyType(DSEnreplacedy.DS_CONTAINER.toString());
    enreplacedy.setDomainId(domainId);
    enreplacedy.addPrimaryFilter("user", ugi.getShortUserName());
    TimelineEvent event = new TimelineEvent();
    event.setTimestamp(System.currentTimeMillis());
    event.setEventType(DSEvent.DS_CONTAINER_START.toString());
    event.addEventInfo("Node", container.getNodeId().toString());
    event.addEventInfo("Resources", container.getResource().toString());
    enreplacedy.addEvent(event);
    try {
        ugi.doAs(new PrivilegedExceptionAction<TimelinePutResponse>() {

            @Override
            public TimelinePutResponse run() throws Exception {
                return timelineClient.putEnreplacedies(enreplacedy);
            }
        });
    } catch (Exception e) {
        LOG.error("Container start event could not be published for " + container.getId().toString(), e instanceof UndeclaredThrowableException ? e.getCause() : e);
    }
}

See More Examples